xref: /titanic_51/usr/src/uts/common/io/nxge/nxge_send.c (revision 5a00db9d04809df47502f8002f0295cb0b7966e0)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/nxge/nxge_impl.h>
27 #include <sys/nxge/nxge_hio.h>
28 #include <npi_tx_wr64.h>
29 
30 /* Software LSO required header files */
31 #include <netinet/tcp.h>
32 #include <inet/ip_impl.h>
33 #include <inet/tcp.h>
34 
35 static mblk_t *nxge_lso_eliminate(mblk_t *);
36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss);
37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *);
38 static void nxge_hcksum_retrieve(mblk_t *,
39     uint32_t *, uint32_t *, uint32_t *,
40     uint32_t *, uint32_t *);
41 static uint32_t nxge_csgen(uint16_t *, int);
42 
43 extern uint32_t		nxge_reclaim_pending;
44 extern uint32_t 	nxge_bcopy_thresh;
45 extern uint32_t 	nxge_dvma_thresh;
46 extern uint32_t 	nxge_dma_stream_thresh;
47 extern uint32_t		nxge_tx_minfree;
48 extern uint32_t		nxge_tx_intr_thres;
49 extern uint32_t		nxge_tx_max_gathers;
50 extern uint32_t		nxge_tx_tiny_pack;
51 extern uint32_t		nxge_tx_use_bcopy;
52 extern nxge_tx_mode_t	nxge_tx_scheme;
53 uint32_t		nxge_lso_kick_cnt = 2;
54 
55 
56 void
57 nxge_tx_ring_task(void *arg)
58 {
59 	p_tx_ring_t	ring = (p_tx_ring_t)arg;
60 
61 	MUTEX_ENTER(&ring->lock);
62 	(void) nxge_txdma_reclaim(ring->nxgep, ring, 0);
63 	MUTEX_EXIT(&ring->lock);
64 
65 	if (!isLDOMguest(ring->nxgep) && !ring->tx_ring_offline)
66 		mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle);
67 #if defined(sun4v)
68 	else {
69 		nxge_hio_data_t *nhd =
70 		    (nxge_hio_data_t *)ring->nxgep->nxge_hw_p->hio;
71 		nx_vio_fp_t *vio = &nhd->hio.vio;
72 
73 		/* Call back vnet. */
74 		if (vio->cb.vio_net_tx_update) {
75 			(*vio->cb.vio_net_tx_update)(ring->nxgep->hio_vr->vhp);
76 		}
77 	}
78 #endif
79 }
80 
81 static void
82 nxge_tx_ring_dispatch(p_tx_ring_t ring)
83 {
84 	/*
85 	 * Kick the ring task to reclaim some buffers.
86 	 */
87 	(void) ddi_taskq_dispatch(ring->taskq,
88 	    nxge_tx_ring_task, (void *)ring, DDI_SLEEP);
89 }
90 
91 mblk_t *
92 nxge_tx_ring_send(void *arg, mblk_t *mp)
93 {
94 	p_nxge_ring_handle_t	nrhp = (p_nxge_ring_handle_t)arg;
95 	p_nxge_t		nxgep;
96 	p_tx_ring_t		tx_ring_p;
97 	int			status, channel;
98 
99 	ASSERT(nrhp != NULL);
100 	nxgep = nrhp->nxgep;
101 	channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index;
102 	tx_ring_p = nxgep->tx_rings->rings[channel];
103 
104 	ASSERT(nxgep == tx_ring_p->nxgep);
105 
106 #ifdef DEBUG
107 	if (isLDOMservice(nxgep)) {
108 		ASSERT(!tx_ring_p->tx_ring_offline);
109 	}
110 #endif
111 
112 	status = nxge_start(nxgep, tx_ring_p, mp);
113 	if (status) {
114 		nxge_tx_ring_dispatch(tx_ring_p);
115 		return (mp);
116 	}
117 
118 	return ((mblk_t *)NULL);
119 }
120 
121 #if defined(sun4v)
122 
123 /*
124  * nxge_m_tx() is needed for Hybrid I/O operation of the vnet in
125  *	the guest domain.  See CR 6778758 for long term solution.
126  */
127 
128 mblk_t *
129 nxge_m_tx(void *arg, mblk_t *mp)
130 {
131 	p_nxge_t		nxgep = (p_nxge_t)arg;
132 	mblk_t			*next;
133 	p_tx_ring_t		tx_ring_p;
134 	int			status;
135 
136 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_m_tx"));
137 
138 	/*
139 	 * Get the default ring handle.
140 	 */
141 	tx_ring_p = nxgep->tx_rings->rings[0];
142 
143 	while (mp != NULL) {
144 		next = mp->b_next;
145 		mp->b_next = NULL;
146 
147 		status = nxge_start(nxgep, tx_ring_p, mp);
148 		if (status != 0) {
149 			mp->b_next = next;
150 			nxge_tx_ring_dispatch(tx_ring_p);
151 			return (mp);
152 		}
153 
154 		mp = next;
155 	}
156 
157 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_m_tx"));
158 	return ((mblk_t *)NULL);
159 }
160 
161 #endif
162 
163 int
164 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
165 {
166 	int 			status = 0;
167 	p_tx_desc_t 		tx_desc_ring_vp;
168 	npi_handle_t		npi_desc_handle;
169 	nxge_os_dma_handle_t 	tx_desc_dma_handle;
170 	p_tx_desc_t 		tx_desc_p;
171 	p_tx_msg_t 		tx_msg_ring;
172 	p_tx_msg_t 		tx_msg_p;
173 	tx_desc_t		tx_desc, *tmp_desc_p;
174 	tx_desc_t		sop_tx_desc, *sop_tx_desc_p;
175 	p_tx_pkt_header_t	hdrp;
176 	tx_pkt_header_t		tmp_hdrp;
177 	p_tx_pkt_hdr_all_t	pkthdrp;
178 	uint8_t			npads = 0;
179 	uint64_t 		dma_ioaddr;
180 	uint32_t		dma_flags;
181 	int			last_bidx;
182 	uint8_t 		*b_rptr;
183 	caddr_t 		kaddr;
184 	uint32_t		nmblks;
185 	uint32_t		ngathers;
186 	uint32_t		clen;
187 	int 			len;
188 	uint32_t		pkt_len, pack_len, min_len;
189 	uint32_t		bcopy_thresh;
190 	int 			i, cur_index, sop_index;
191 	uint16_t		tail_index;
192 	boolean_t		tail_wrap = B_FALSE;
193 	nxge_dma_common_t	desc_area;
194 	nxge_os_dma_handle_t 	dma_handle;
195 	ddi_dma_cookie_t 	dma_cookie;
196 	npi_handle_t		npi_handle;
197 	p_mblk_t 		nmp;
198 	p_mblk_t		t_mp;
199 	uint32_t 		ncookies;
200 	boolean_t 		good_packet;
201 	boolean_t 		mark_mode = B_FALSE;
202 	p_nxge_stats_t 		statsp;
203 	p_nxge_tx_ring_stats_t tdc_stats;
204 	t_uscalar_t 		start_offset = 0;
205 	t_uscalar_t 		stuff_offset = 0;
206 	t_uscalar_t 		end_offset = 0;
207 	t_uscalar_t 		value = 0;
208 	t_uscalar_t 		cksum_flags = 0;
209 	boolean_t		cksum_on = B_FALSE;
210 	uint32_t		boff = 0;
211 	uint64_t		tot_xfer_len = 0;
212 	boolean_t		header_set = B_FALSE;
213 #ifdef NXGE_DEBUG
214 	p_tx_desc_t 		tx_desc_ring_pp;
215 	p_tx_desc_t 		tx_desc_pp;
216 	tx_desc_t		*save_desc_p;
217 	int			dump_len;
218 	int			sad_len;
219 	uint64_t		sad;
220 	int			xfer_len;
221 	uint32_t		msgsize;
222 #endif
223 	p_mblk_t 		mp_chain = NULL;
224 	boolean_t		is_lso = B_FALSE;
225 	boolean_t		lso_again;
226 	int			cur_index_lso;
227 	p_mblk_t 		nmp_lso_save;
228 	uint32_t		lso_ngathers;
229 	boolean_t		lso_tail_wrap = B_FALSE;
230 
231 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
232 	    "==> nxge_start: tx dma channel %d", tx_ring_p->tdc));
233 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
234 	    "==> nxge_start: Starting tdc %d desc pending %d",
235 	    tx_ring_p->tdc, tx_ring_p->descs_pending));
236 
237 	statsp = nxgep->statsp;
238 
239 	if (!isLDOMguest(nxgep)) {
240 		switch (nxgep->mac.portmode) {
241 		default:
242 			if (nxgep->statsp->port_stats.lb_mode ==
243 			    nxge_lb_normal) {
244 				if (!statsp->mac_stats.link_up) {
245 					freemsg(mp);
246 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
247 					    "==> nxge_start: "
248 					    "link not up"));
249 					goto nxge_start_fail1;
250 				}
251 			}
252 			break;
253 		case PORT_10G_FIBER:
254 			/*
255 			 * For the following modes, check the link status
256 			 * before sending the packet out:
257 			 * nxge_lb_normal,
258 			 * nxge_lb_ext10g,
259 			 * nxge_lb_ext1000,
260 			 * nxge_lb_ext100,
261 			 * nxge_lb_ext10.
262 			 */
263 			if (nxgep->statsp->port_stats.lb_mode <
264 			    nxge_lb_phy10g) {
265 				if (!statsp->mac_stats.link_up) {
266 					freemsg(mp);
267 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
268 					    "==> nxge_start: "
269 					    "link not up"));
270 					goto nxge_start_fail1;
271 				}
272 			}
273 			break;
274 		}
275 	}
276 
277 	if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
278 	    (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
279 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
280 		    "==> nxge_start: hardware not initialized or stopped"));
281 		freemsg(mp);
282 		goto nxge_start_fail1;
283 	}
284 
285 	if (nxgep->soft_lso_enable) {
286 		mp_chain = nxge_lso_eliminate(mp);
287 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
288 		    "==> nxge_start(0): LSO mp $%p mp_chain $%p",
289 		    mp, mp_chain));
290 		if (mp_chain == NULL) {
291 			NXGE_ERROR_MSG((nxgep, TX_CTL,
292 			    "==> nxge_send(0): NULL mp_chain $%p != mp $%p",
293 			    mp_chain, mp));
294 			goto nxge_start_fail1;
295 		}
296 		if (mp_chain != mp) {
297 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
298 			    "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p",
299 			    mp_chain, mp));
300 			is_lso = B_TRUE;
301 			mp = mp_chain;
302 			mp_chain = mp_chain->b_next;
303 			mp->b_next = NULL;
304 		}
305 	}
306 
307 	hcksum_retrieve(mp, NULL, NULL, &start_offset,
308 	    &stuff_offset, &end_offset, &value, &cksum_flags);
309 	if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) {
310 		start_offset += sizeof (ether_header_t);
311 		stuff_offset += sizeof (ether_header_t);
312 	} else {
313 		start_offset += sizeof (struct ether_vlan_header);
314 		stuff_offset += sizeof (struct ether_vlan_header);
315 	}
316 
317 	if (cksum_flags & HCK_PARTIALCKSUM) {
318 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
319 		    "==> nxge_start: mp $%p len %d "
320 		    "cksum_flags 0x%x (partial checksum) ",
321 		    mp, MBLKL(mp), cksum_flags));
322 		cksum_on = B_TRUE;
323 	}
324 
325 	pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp;
326 	pkthdrp->reserved = 0;
327 	tmp_hdrp.value = 0;
328 	nxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
329 	    0, 0, pkthdrp,
330 	    start_offset, stuff_offset);
331 
332 	lso_again = B_FALSE;
333 	lso_ngathers = 0;
334 
335 	MUTEX_ENTER(&tx_ring_p->lock);
336 
337 	if (isLDOMservice(nxgep)) {
338 		tx_ring_p->tx_ring_busy = B_TRUE;
339 		if (tx_ring_p->tx_ring_offline) {
340 			freemsg(mp);
341 			tx_ring_p->tx_ring_busy = B_FALSE;
342 			(void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
343 			    NXGE_TX_RING_OFFLINED);
344 			MUTEX_EXIT(&tx_ring_p->lock);
345 			return (status);
346 		}
347 	}
348 
349 	cur_index_lso = tx_ring_p->wr_index;
350 	lso_tail_wrap = tx_ring_p->wr_index_wrap;
351 start_again:
352 	ngathers = 0;
353 	sop_index = tx_ring_p->wr_index;
354 #ifdef	NXGE_DEBUG
355 	if (tx_ring_p->descs_pending) {
356 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
357 		    "desc pending %d ", tx_ring_p->descs_pending));
358 	}
359 
360 	dump_len = (int)(MBLKL(mp));
361 	dump_len = (dump_len > 128) ? 128: dump_len;
362 
363 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
364 	    "==> nxge_start: tdc %d: dumping ...: b_rptr $%p "
365 	    "(Before header reserve: ORIGINAL LEN %d)",
366 	    tx_ring_p->tdc,
367 	    mp->b_rptr,
368 	    dump_len));
369 
370 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets "
371 	    "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr,
372 	    nxge_dump_packet((char *)mp->b_rptr, dump_len)));
373 #endif
374 
375 	tdc_stats = tx_ring_p->tdc_stats;
376 	mark_mode = (tx_ring_p->descs_pending &&
377 	    ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending)
378 	    < nxge_tx_minfree));
379 
380 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
381 	    "TX Descriptor ring is channel %d mark mode %d",
382 	    tx_ring_p->tdc, mark_mode));
383 
384 	if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) {
385 		if (!nxge_txdma_reclaim(nxgep, tx_ring_p,
386 		    (nxge_tx_minfree + lso_ngathers))) {
387 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
388 			    "TX Descriptor ring is full: channel %d",
389 			    tx_ring_p->tdc));
390 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
391 			    "TX Descriptor ring is full: channel %d",
392 			    tx_ring_p->tdc));
393 			if (is_lso) {
394 				/*
395 				 * free the current mp and mp_chain if not FULL.
396 				 */
397 				tdc_stats->tx_no_desc++;
398 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
399 				    "LSO packet: TX Descriptor ring is full: "
400 				    "channel %d",
401 				    tx_ring_p->tdc));
402 				goto nxge_start_fail_lso;
403 			} else {
404 				cas32((uint32_t *)&tx_ring_p->queueing, 0, 1);
405 				tdc_stats->tx_no_desc++;
406 
407 				if (isLDOMservice(nxgep)) {
408 					tx_ring_p->tx_ring_busy = B_FALSE;
409 					if (tx_ring_p->tx_ring_offline) {
410 						(void) atomic_swap_32(
411 						    &tx_ring_p->tx_ring_offline,
412 						    NXGE_TX_RING_OFFLINED);
413 					}
414 				}
415 
416 				MUTEX_EXIT(&tx_ring_p->lock);
417 				status = 1;
418 				goto nxge_start_fail1;
419 			}
420 		}
421 	}
422 
423 	nmp = mp;
424 	i = sop_index = tx_ring_p->wr_index;
425 	nmblks = 0;
426 	ngathers = 0;
427 	pkt_len = 0;
428 	pack_len = 0;
429 	clen = 0;
430 	last_bidx = -1;
431 	good_packet = B_TRUE;
432 
433 	desc_area = tx_ring_p->tdc_desc;
434 	npi_handle = desc_area.npi_handle;
435 	npi_desc_handle.regh = (nxge_os_acc_handle_t)
436 	    DMA_COMMON_ACC_HANDLE(desc_area);
437 	tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
438 	tx_desc_dma_handle = (nxge_os_dma_handle_t)
439 	    DMA_COMMON_HANDLE(desc_area);
440 	tx_msg_ring = tx_ring_p->tx_msg_ring;
441 
442 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d",
443 	    sop_index, i));
444 
445 #ifdef	NXGE_DEBUG
446 	msgsize = msgdsize(nmp);
447 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
448 	    "==> nxge_start(1): wr_index %d i %d msgdsize %d",
449 	    sop_index, i, msgsize));
450 #endif
451 	/*
452 	 * The first 16 bytes of the premapped buffer are reserved
453 	 * for header. No padding will be used.
454 	 */
455 	pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
456 	if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) {
457 		bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
458 	} else {
459 		bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
460 	}
461 	while (nmp) {
462 		good_packet = B_TRUE;
463 		b_rptr = nmp->b_rptr;
464 		len = MBLKL(nmp);
465 		if (len <= 0) {
466 			nmp = nmp->b_cont;
467 			continue;
468 		}
469 		nmblks++;
470 
471 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d "
472 		    "len %d pkt_len %d pack_len %d",
473 		    nmblks, len, pkt_len, pack_len));
474 		/*
475 		 * Hardware limits the transfer length to 4K for NIU and
476 		 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just
477 		 * use TX_MAX_TRANSFER_LENGTH as the limit for both.
478 		 * If len is longer than the limit, then we break nmp into
479 		 * two chunks: Make the first chunk equal to the limit and
480 		 * the second chunk for the remaining data. If the second
481 		 * chunk is still larger than the limit, then it will be
482 		 * broken into two in the next pass.
483 		 */
484 		if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) {
485 			if ((t_mp = dupb(nmp)) != NULL) {
486 				nmp->b_wptr = nmp->b_rptr +
487 				    (TX_MAX_TRANSFER_LENGTH
488 				    - TX_PKT_HEADER_SIZE);
489 				t_mp->b_rptr = nmp->b_wptr;
490 				t_mp->b_cont = nmp->b_cont;
491 				nmp->b_cont = t_mp;
492 				len = MBLKL(nmp);
493 			} else {
494 				if (is_lso) {
495 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
496 					    "LSO packet: dupb failed: "
497 					    "channel %d",
498 					    tx_ring_p->tdc));
499 					mp = nmp;
500 					goto nxge_start_fail_lso;
501 				} else {
502 					good_packet = B_FALSE;
503 					goto nxge_start_fail2;
504 				}
505 			}
506 		}
507 		tx_desc.value = 0;
508 		tx_desc_p = &tx_desc_ring_vp[i];
509 #ifdef	NXGE_DEBUG
510 		tx_desc_pp = &tx_desc_ring_pp[i];
511 #endif
512 		tx_msg_p = &tx_msg_ring[i];
513 #if defined(__i386)
514 		npi_desc_handle.regp = (uint32_t)tx_desc_p;
515 #else
516 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
517 #endif
518 		if (!header_set &&
519 		    ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
520 		    (len >= bcopy_thresh))) {
521 			header_set = B_TRUE;
522 			bcopy_thresh += TX_PKT_HEADER_SIZE;
523 			boff = 0;
524 			pack_len = 0;
525 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
526 			hdrp = (p_tx_pkt_header_t)kaddr;
527 			clen = pkt_len;
528 			dma_handle = tx_msg_p->buf_dma_handle;
529 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
530 			(void) ddi_dma_sync(dma_handle,
531 			    i * nxge_bcopy_thresh, nxge_bcopy_thresh,
532 			    DDI_DMA_SYNC_FORDEV);
533 
534 			tx_msg_p->flags.dma_type = USE_BCOPY;
535 			goto nxge_start_control_header_only;
536 		}
537 
538 		pkt_len += len;
539 		pack_len += len;
540 
541 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): "
542 		    "desc entry %d "
543 		    "DESC IOADDR $%p "
544 		    "desc_vp $%p tx_desc_p $%p "
545 		    "desc_pp $%p tx_desc_pp $%p "
546 		    "len %d pkt_len %d pack_len %d",
547 		    i,
548 		    DMA_COMMON_IOADDR(desc_area),
549 		    tx_desc_ring_vp, tx_desc_p,
550 		    tx_desc_ring_pp, tx_desc_pp,
551 		    len, pkt_len, pack_len));
552 
553 		if (len < bcopy_thresh) {
554 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): "
555 			    "USE BCOPY: "));
556 			if (nxge_tx_tiny_pack) {
557 				uint32_t blst =
558 				    TXDMA_DESC_NEXT_INDEX(i, -1,
559 				    tx_ring_p->tx_wrap_mask);
560 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
561 				    "==> nxge_start(5): pack"));
562 				if ((pack_len <= bcopy_thresh) &&
563 				    (last_bidx == blst)) {
564 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
565 					    "==> nxge_start: pack(6) "
566 					    "(pkt_len %d pack_len %d)",
567 					    pkt_len, pack_len));
568 					i = blst;
569 					tx_desc_p = &tx_desc_ring_vp[i];
570 #ifdef	NXGE_DEBUG
571 					tx_desc_pp = &tx_desc_ring_pp[i];
572 #endif
573 					tx_msg_p = &tx_msg_ring[i];
574 					boff = pack_len - len;
575 					ngathers--;
576 				} else if (pack_len > bcopy_thresh &&
577 				    header_set) {
578 					pack_len = len;
579 					boff = 0;
580 					bcopy_thresh = nxge_bcopy_thresh;
581 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
582 					    "==> nxge_start(7): > max NEW "
583 					    "bcopy thresh %d "
584 					    "pkt_len %d pack_len %d(next)",
585 					    bcopy_thresh,
586 					    pkt_len, pack_len));
587 				}
588 				last_bidx = i;
589 			}
590 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
591 			if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
592 				hdrp = (p_tx_pkt_header_t)kaddr;
593 				header_set = B_TRUE;
594 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
595 				    "==> nxge_start(7_x2): "
596 				    "pkt_len %d pack_len %d (new hdrp $%p)",
597 				    pkt_len, pack_len, hdrp));
598 			}
599 			tx_msg_p->flags.dma_type = USE_BCOPY;
600 			kaddr += boff;
601 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): "
602 			    "USE BCOPY: before bcopy "
603 			    "DESC IOADDR $%p entry %d "
604 			    "bcopy packets %d "
605 			    "bcopy kaddr $%p "
606 			    "bcopy ioaddr (SAD) $%p "
607 			    "bcopy clen %d "
608 			    "bcopy boff %d",
609 			    DMA_COMMON_IOADDR(desc_area), i,
610 			    tdc_stats->tx_hdr_pkts,
611 			    kaddr,
612 			    dma_ioaddr,
613 			    clen,
614 			    boff));
615 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
616 			    "1USE BCOPY: "));
617 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
618 			    "2USE BCOPY: "));
619 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
620 			    "last USE BCOPY: copy from b_rptr $%p "
621 			    "to KADDR $%p (len %d offset %d",
622 			    b_rptr, kaddr, len, boff));
623 
624 			bcopy(b_rptr, kaddr, len);
625 
626 #ifdef	NXGE_DEBUG
627 			dump_len = (len > 128) ? 128: len;
628 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
629 			    "==> nxge_start: dump packets "
630 			    "(After BCOPY len %d)"
631 			    "(b_rptr $%p): %s", len, nmp->b_rptr,
632 			    nxge_dump_packet((char *)nmp->b_rptr,
633 			    dump_len)));
634 #endif
635 
636 			dma_handle = tx_msg_p->buf_dma_handle;
637 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
638 			(void) ddi_dma_sync(dma_handle,
639 			    i * nxge_bcopy_thresh, nxge_bcopy_thresh,
640 			    DDI_DMA_SYNC_FORDEV);
641 			clen = len + boff;
642 			tdc_stats->tx_hdr_pkts++;
643 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): "
644 			    "USE BCOPY: "
645 			    "DESC IOADDR $%p entry %d "
646 			    "bcopy packets %d "
647 			    "bcopy kaddr $%p "
648 			    "bcopy ioaddr (SAD) $%p "
649 			    "bcopy clen %d "
650 			    "bcopy boff %d",
651 			    DMA_COMMON_IOADDR(desc_area),
652 			    i,
653 			    tdc_stats->tx_hdr_pkts,
654 			    kaddr,
655 			    dma_ioaddr,
656 			    clen,
657 			    boff));
658 		} else {
659 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): "
660 			    "USE DVMA: len %d", len));
661 			tx_msg_p->flags.dma_type = USE_DMA;
662 			dma_flags = DDI_DMA_WRITE;
663 			if (len < nxge_dma_stream_thresh) {
664 				dma_flags |= DDI_DMA_CONSISTENT;
665 			} else {
666 				dma_flags |= DDI_DMA_STREAMING;
667 			}
668 
669 			dma_handle = tx_msg_p->dma_handle;
670 			status = ddi_dma_addr_bind_handle(dma_handle, NULL,
671 			    (caddr_t)b_rptr, len, dma_flags,
672 			    DDI_DMA_DONTWAIT, NULL,
673 			    &dma_cookie, &ncookies);
674 			if (status == DDI_DMA_MAPPED) {
675 				dma_ioaddr = dma_cookie.dmac_laddress;
676 				len = (int)dma_cookie.dmac_size;
677 				clen = (uint32_t)dma_cookie.dmac_size;
678 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
679 				    "==> nxge_start(12_1): "
680 				    "USE DVMA: len %d clen %d "
681 				    "ngathers %d",
682 				    len, clen,
683 				    ngathers));
684 #if defined(__i386)
685 				npi_desc_handle.regp = (uint32_t)tx_desc_p;
686 #else
687 				npi_desc_handle.regp = (uint64_t)tx_desc_p;
688 #endif
689 				while (ncookies > 1) {
690 					ngathers++;
691 					/*
692 					 * this is the fix for multiple
693 					 * cookies, which are basically
694 					 * a descriptor entry, we don't set
695 					 * SOP bit as well as related fields
696 					 */
697 
698 					(void) npi_txdma_desc_gather_set(
699 					    npi_desc_handle,
700 					    &tx_desc,
701 					    (ngathers -1),
702 					    mark_mode,
703 					    ngathers,
704 					    dma_ioaddr,
705 					    clen);
706 
707 					tx_msg_p->tx_msg_size = clen;
708 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
709 					    "==> nxge_start:  DMA "
710 					    "ncookie %d "
711 					    "ngathers %d "
712 					    "dma_ioaddr $%p len %d"
713 					    "desc $%p descp $%p (%d)",
714 					    ncookies,
715 					    ngathers,
716 					    dma_ioaddr, clen,
717 					    *tx_desc_p, tx_desc_p, i));
718 
719 					ddi_dma_nextcookie(dma_handle,
720 					    &dma_cookie);
721 					dma_ioaddr =
722 					    dma_cookie.dmac_laddress;
723 
724 					len = (int)dma_cookie.dmac_size;
725 					clen = (uint32_t)dma_cookie.dmac_size;
726 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
727 					    "==> nxge_start(12_2): "
728 					    "USE DVMA: len %d clen %d ",
729 					    len, clen));
730 
731 					i = TXDMA_DESC_NEXT_INDEX(i, 1,
732 					    tx_ring_p->tx_wrap_mask);
733 					tx_desc_p = &tx_desc_ring_vp[i];
734 
735 #if defined(__i386)
736 					npi_desc_handle.regp =
737 					    (uint32_t)tx_desc_p;
738 #else
739 					npi_desc_handle.regp =
740 					    (uint64_t)tx_desc_p;
741 #endif
742 					tx_msg_p = &tx_msg_ring[i];
743 					tx_msg_p->flags.dma_type = USE_NONE;
744 					tx_desc.value = 0;
745 
746 					ncookies--;
747 				}
748 				tdc_stats->tx_ddi_pkts++;
749 				NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:"
750 				    "DMA: ddi packets %d",
751 				    tdc_stats->tx_ddi_pkts));
752 			} else {
753 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
754 				    "dma mapping failed for %d "
755 				    "bytes addr $%p flags %x (%d)",
756 				    len, b_rptr, status, status));
757 				good_packet = B_FALSE;
758 				tdc_stats->tx_dma_bind_fail++;
759 				tx_msg_p->flags.dma_type = USE_NONE;
760 				if (is_lso) {
761 					mp = nmp;
762 					goto nxge_start_fail_lso;
763 				} else {
764 					goto nxge_start_fail2;
765 				}
766 			}
767 		} /* ddi dvma */
768 
769 		if (is_lso) {
770 			nmp_lso_save = nmp;
771 		}
772 		nmp = nmp->b_cont;
773 nxge_start_control_header_only:
774 #if defined(__i386)
775 		npi_desc_handle.regp = (uint32_t)tx_desc_p;
776 #else
777 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
778 #endif
779 		ngathers++;
780 
781 		if (ngathers == 1) {
782 #ifdef	NXGE_DEBUG
783 			save_desc_p = &sop_tx_desc;
784 #endif
785 			sop_tx_desc_p = &sop_tx_desc;
786 			sop_tx_desc_p->value = 0;
787 			sop_tx_desc_p->bits.hdw.tr_len = clen;
788 			sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
789 			sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
790 		} else {
791 #ifdef	NXGE_DEBUG
792 			save_desc_p = &tx_desc;
793 #endif
794 			tmp_desc_p = &tx_desc;
795 			tmp_desc_p->value = 0;
796 			tmp_desc_p->bits.hdw.tr_len = clen;
797 			tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
798 			tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
799 
800 			tx_desc_p->value = tmp_desc_p->value;
801 		}
802 
803 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): "
804 		    "Desc_entry %d ngathers %d "
805 		    "desc_vp $%p tx_desc_p $%p "
806 		    "len %d clen %d pkt_len %d pack_len %d nmblks %d "
807 		    "dma_ioaddr (SAD) $%p mark %d",
808 		    i, ngathers,
809 		    tx_desc_ring_vp, tx_desc_p,
810 		    len, clen, pkt_len, pack_len, nmblks,
811 		    dma_ioaddr, mark_mode));
812 
813 #ifdef NXGE_DEBUG
814 		npi_desc_handle.nxgep = nxgep;
815 		npi_desc_handle.function.function = nxgep->function_num;
816 		npi_desc_handle.function.instance = nxgep->instance;
817 		sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK);
818 		xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >>
819 		    TX_PKT_DESC_TR_LEN_SHIFT);
820 
821 
822 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
823 		    "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
824 		    "mark %d sop %d\n",
825 		    save_desc_p->value,
826 		    sad,
827 		    save_desc_p->bits.hdw.tr_len,
828 		    xfer_len,
829 		    save_desc_p->bits.hdw.num_ptr,
830 		    save_desc_p->bits.hdw.mark,
831 		    save_desc_p->bits.hdw.sop));
832 
833 		npi_txdma_dump_desc_one(npi_desc_handle, NULL, i);
834 #endif
835 
836 		tx_msg_p->tx_msg_size = clen;
837 		i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
838 		if (ngathers > nxge_tx_max_gathers) {
839 			good_packet = B_FALSE;
840 			hcksum_retrieve(mp, NULL, NULL, &start_offset,
841 			    &stuff_offset, &end_offset, &value,
842 			    &cksum_flags);
843 
844 			NXGE_DEBUG_MSG((NULL, TX_CTL,
845 			    "==> nxge_start(14): pull msg - "
846 			    "len %d pkt_len %d ngathers %d",
847 			    len, pkt_len, ngathers));
848 			/* Pull all message blocks from b_cont */
849 			if (is_lso) {
850 				mp = nmp_lso_save;
851 				goto nxge_start_fail_lso;
852 			}
853 			if ((msgpullup(mp, -1)) == NULL) {
854 				goto nxge_start_fail2;
855 			}
856 			goto nxge_start_fail2;
857 		}
858 	} /* while (nmp) */
859 
860 	tx_msg_p->tx_message = mp;
861 	tx_desc_p = &tx_desc_ring_vp[sop_index];
862 #if defined(__i386)
863 	npi_desc_handle.regp = (uint32_t)tx_desc_p;
864 #else
865 	npi_desc_handle.regp = (uint64_t)tx_desc_p;
866 #endif
867 
868 	pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
869 	pkthdrp->reserved = 0;
870 	hdrp->value = 0;
871 	bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t));
872 
873 	if (pkt_len > NXGE_MTU_DEFAULT_MAX) {
874 		tdc_stats->tx_jumbo_pkts++;
875 	}
876 
877 	min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2));
878 	if (pkt_len < min_len) {
879 		/* Assume we use bcopy to premapped buffers */
880 		kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
881 		NXGE_DEBUG_MSG((NULL, TX_CTL,
882 		    "==> nxge_start(14-1): < (msg_min + 16)"
883 		    "len %d pkt_len %d min_len %d bzero %d ngathers %d",
884 		    len, pkt_len, min_len, (min_len - pkt_len), ngathers));
885 		bzero((kaddr + pkt_len), (min_len - pkt_len));
886 		pkt_len = tx_msg_p->tx_msg_size = min_len;
887 
888 		sop_tx_desc_p->bits.hdw.tr_len = min_len;
889 
890 		NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
891 		tx_desc_p->value = sop_tx_desc_p->value;
892 
893 		NXGE_DEBUG_MSG((NULL, TX_CTL,
894 		    "==> nxge_start(14-2): < msg_min - "
895 		    "len %d pkt_len %d min_len %d ngathers %d",
896 		    len, pkt_len, min_len, ngathers));
897 	}
898 
899 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ",
900 	    cksum_flags));
901 	{
902 		uint64_t	tmp_len;
903 
904 		/* pkt_len already includes 16 + paddings!! */
905 		/* Update the control header length */
906 		tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
907 		tmp_len = hdrp->value |
908 		    (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
909 
910 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
911 		    "==> nxge_start(15_x1): setting SOP "
912 		    "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
913 		    "0x%llx hdrp->value 0x%llx",
914 		    tot_xfer_len, tot_xfer_len, pkt_len,
915 		    tmp_len, hdrp->value));
916 #if defined(_BIG_ENDIAN)
917 		hdrp->value = ddi_swap64(tmp_len);
918 #else
919 		hdrp->value = tmp_len;
920 #endif
921 		NXGE_DEBUG_MSG((nxgep,
922 		    TX_CTL, "==> nxge_start(15_x2): setting SOP "
923 		    "after SWAP: tot_xfer_len 0x%llx pkt_len %d "
924 		    "tmp_len 0x%llx hdrp->value 0x%llx",
925 		    tot_xfer_len, pkt_len,
926 		    tmp_len, hdrp->value));
927 	}
928 
929 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP "
930 	    "wr_index %d "
931 	    "tot_xfer_len (%d) pkt_len %d npads %d",
932 	    sop_index,
933 	    tot_xfer_len, pkt_len,
934 	    npads));
935 
936 	sop_tx_desc_p->bits.hdw.sop = 1;
937 	sop_tx_desc_p->bits.hdw.mark = mark_mode;
938 	sop_tx_desc_p->bits.hdw.num_ptr = ngathers;
939 
940 	NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
941 
942 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done"));
943 
944 #ifdef NXGE_DEBUG
945 	npi_desc_handle.nxgep = nxgep;
946 	npi_desc_handle.function.function = nxgep->function_num;
947 	npi_desc_handle.function.instance = nxgep->instance;
948 
949 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
950 	    "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
951 	    save_desc_p->value,
952 	    sad,
953 	    save_desc_p->bits.hdw.tr_len,
954 	    xfer_len,
955 	    save_desc_p->bits.hdw.num_ptr,
956 	    save_desc_p->bits.hdw.mark,
957 	    save_desc_p->bits.hdw.sop));
958 	(void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index);
959 
960 	dump_len = (pkt_len > 128) ? 128: pkt_len;
961 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
962 	    "==> nxge_start: dump packets(17) (after sop set, len "
963 	    " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
964 	    "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
965 	    (char *)hdrp,
966 	    nxge_dump_packet((char *)hdrp, dump_len)));
967 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
968 	    "==> nxge_start(18): TX desc sync: sop_index %d",
969 	    sop_index));
970 #endif
971 
972 	if ((ngathers == 1) || tx_ring_p->wr_index < i) {
973 		(void) ddi_dma_sync(tx_desc_dma_handle,
974 		    sop_index * sizeof (tx_desc_t),
975 		    ngathers * sizeof (tx_desc_t),
976 		    DDI_DMA_SYNC_FORDEV);
977 
978 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 "
979 		    "cs_off = 0x%02X cs_s_off = 0x%02X "
980 		    "pkt_len %d ngathers %d sop_index %d\n",
981 		    stuff_offset, start_offset,
982 		    pkt_len, ngathers, sop_index));
983 	} else { /* more than one descriptor and wrap around */
984 		uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
985 		(void) ddi_dma_sync(tx_desc_dma_handle,
986 		    sop_index * sizeof (tx_desc_t),
987 		    nsdescs * sizeof (tx_desc_t),
988 		    DDI_DMA_SYNC_FORDEV);
989 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 "
990 		    "cs_off = 0x%02X cs_s_off = 0x%02X "
991 		    "pkt_len %d ngathers %d sop_index %d\n",
992 		    stuff_offset, start_offset,
993 		    pkt_len, ngathers, sop_index));
994 
995 		(void) ddi_dma_sync(tx_desc_dma_handle,
996 		    0,
997 		    (ngathers - nsdescs) * sizeof (tx_desc_t),
998 		    DDI_DMA_SYNC_FORDEV);
999 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 "
1000 		    "cs_off = 0x%02X cs_s_off = 0x%02X "
1001 		    "pkt_len %d ngathers %d sop_index %d\n",
1002 		    stuff_offset, start_offset,
1003 		    pkt_len, ngathers, sop_index));
1004 	}
1005 
1006 	tail_index = tx_ring_p->wr_index;
1007 	tail_wrap = tx_ring_p->wr_index_wrap;
1008 
1009 	tx_ring_p->wr_index = i;
1010 	if (tx_ring_p->wr_index <= tail_index) {
1011 		tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ?
1012 		    B_FALSE : B_TRUE);
1013 	}
1014 
1015 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: "
1016 	    "channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
1017 	    tx_ring_p->tdc,
1018 	    tx_ring_p->wr_index,
1019 	    tx_ring_p->wr_index_wrap,
1020 	    ngathers,
1021 	    tx_ring_p->descs_pending));
1022 
1023 	if (is_lso) {
1024 		lso_ngathers += ngathers;
1025 		if (mp_chain != NULL) {
1026 			mp = mp_chain;
1027 			mp_chain = mp_chain->b_next;
1028 			mp->b_next = NULL;
1029 			if (nxge_lso_kick_cnt == lso_ngathers) {
1030 				tx_ring_p->descs_pending += lso_ngathers;
1031 				{
1032 					tx_ring_kick_t		kick;
1033 
1034 					kick.value = 0;
1035 					kick.bits.ldw.wrap =
1036 					    tx_ring_p->wr_index_wrap;
1037 					kick.bits.ldw.tail =
1038 					    (uint16_t)tx_ring_p->wr_index;
1039 
1040 					/* Kick the Transmit kick register */
1041 					TXDMA_REG_WRITE64(
1042 					    NXGE_DEV_NPI_HANDLE(nxgep),
1043 					    TX_RING_KICK_REG,
1044 					    (uint8_t)tx_ring_p->tdc,
1045 					    kick.value);
1046 					tdc_stats->tx_starts++;
1047 
1048 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
1049 					    "==> nxge_start: more LSO: "
1050 					    "LSO_CNT %d",
1051 					    lso_ngathers));
1052 				}
1053 				lso_ngathers = 0;
1054 				ngathers = 0;
1055 				cur_index_lso = sop_index = tx_ring_p->wr_index;
1056 				lso_tail_wrap = tx_ring_p->wr_index_wrap;
1057 			}
1058 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
1059 			    "==> nxge_start: lso again: "
1060 			    "lso_gathers %d ngathers %d cur_index_lso %d "
1061 			    "wr_index %d sop_index %d",
1062 			    lso_ngathers, ngathers, cur_index_lso,
1063 			    tx_ring_p->wr_index, sop_index));
1064 
1065 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
1066 			    "==> nxge_start: next : count %d",
1067 			    lso_ngathers));
1068 			lso_again = B_TRUE;
1069 			goto start_again;
1070 		}
1071 		ngathers = lso_ngathers;
1072 	}
1073 
1074 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: "));
1075 
1076 	{
1077 		tx_ring_kick_t		kick;
1078 
1079 		kick.value = 0;
1080 		kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap;
1081 		kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index;
1082 
1083 		/* Kick start the Transmit kick register */
1084 		TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep),
1085 		    TX_RING_KICK_REG,
1086 		    (uint8_t)tx_ring_p->tdc,
1087 		    kick.value);
1088 	}
1089 
1090 	tx_ring_p->descs_pending += ngathers;
1091 	tdc_stats->tx_starts++;
1092 
1093 	if (isLDOMservice(nxgep)) {
1094 		tx_ring_p->tx_ring_busy = B_FALSE;
1095 		if (tx_ring_p->tx_ring_offline) {
1096 			(void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
1097 			    NXGE_TX_RING_OFFLINED);
1098 		}
1099 	}
1100 
1101 	MUTEX_EXIT(&tx_ring_p->lock);
1102 
1103 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
1104 	return (status);
1105 
1106 nxge_start_fail_lso:
1107 	status = 0;
1108 	good_packet = B_FALSE;
1109 	if (mp != NULL) {
1110 		freemsg(mp);
1111 	}
1112 	if (mp_chain != NULL) {
1113 		freemsg(mp_chain);
1114 	}
1115 	if (!lso_again && !ngathers) {
1116 		if (isLDOMservice(nxgep)) {
1117 			tx_ring_p->tx_ring_busy = B_FALSE;
1118 			if (tx_ring_p->tx_ring_offline) {
1119 				(void) atomic_swap_32(
1120 				    &tx_ring_p->tx_ring_offline,
1121 				    NXGE_TX_RING_OFFLINED);
1122 			}
1123 		}
1124 
1125 		MUTEX_EXIT(&tx_ring_p->lock);
1126 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1127 		    "==> nxge_start: lso exit (nothing changed)"));
1128 		goto nxge_start_fail1;
1129 	}
1130 
1131 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
1132 	    "==> nxge_start (channel %d): before lso "
1133 	    "lso_gathers %d ngathers %d cur_index_lso %d "
1134 	    "wr_index %d sop_index %d lso_again %d",
1135 	    tx_ring_p->tdc,
1136 	    lso_ngathers, ngathers, cur_index_lso,
1137 	    tx_ring_p->wr_index, sop_index, lso_again));
1138 
1139 	if (lso_again) {
1140 		lso_ngathers += ngathers;
1141 		ngathers = lso_ngathers;
1142 		sop_index = cur_index_lso;
1143 		tx_ring_p->wr_index = sop_index;
1144 		tx_ring_p->wr_index_wrap = lso_tail_wrap;
1145 	}
1146 
1147 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
1148 	    "==> nxge_start (channel %d): after lso "
1149 	    "lso_gathers %d ngathers %d cur_index_lso %d "
1150 	    "wr_index %d sop_index %d lso_again %d",
1151 	    tx_ring_p->tdc,
1152 	    lso_ngathers, ngathers, cur_index_lso,
1153 	    tx_ring_p->wr_index, sop_index, lso_again));
1154 
1155 nxge_start_fail2:
1156 	if (good_packet == B_FALSE) {
1157 		cur_index = sop_index;
1158 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up"));
1159 		for (i = 0; i < ngathers; i++) {
1160 			tx_desc_p = &tx_desc_ring_vp[cur_index];
1161 #if defined(__i386)
1162 			npi_handle.regp = (uint32_t)tx_desc_p;
1163 #else
1164 			npi_handle.regp = (uint64_t)tx_desc_p;
1165 #endif
1166 			tx_msg_p = &tx_msg_ring[cur_index];
1167 			(void) npi_txdma_desc_set_zero(npi_handle, 1);
1168 			if (tx_msg_p->flags.dma_type == USE_DVMA) {
1169 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
1170 				    "tx_desc_p = %X index = %d",
1171 				    tx_desc_p, tx_ring_p->rd_index));
1172 				(void) dvma_unload(tx_msg_p->dvma_handle,
1173 				    0, -1);
1174 				tx_msg_p->dvma_handle = NULL;
1175 				if (tx_ring_p->dvma_wr_index ==
1176 				    tx_ring_p->dvma_wrap_mask)
1177 					tx_ring_p->dvma_wr_index = 0;
1178 				else
1179 					tx_ring_p->dvma_wr_index++;
1180 				tx_ring_p->dvma_pending--;
1181 			} else if (tx_msg_p->flags.dma_type == USE_DMA) {
1182 				if (ddi_dma_unbind_handle(
1183 				    tx_msg_p->dma_handle)) {
1184 					cmn_err(CE_WARN, "!nxge_start: "
1185 					    "ddi_dma_unbind_handle failed");
1186 				}
1187 			}
1188 			tx_msg_p->flags.dma_type = USE_NONE;
1189 			cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1,
1190 			    tx_ring_p->tx_wrap_mask);
1191 
1192 		}
1193 	}
1194 
1195 	if (isLDOMservice(nxgep)) {
1196 		tx_ring_p->tx_ring_busy = B_FALSE;
1197 		if (tx_ring_p->tx_ring_offline) {
1198 			(void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
1199 			    NXGE_TX_RING_OFFLINED);
1200 		}
1201 	}
1202 
1203 	MUTEX_EXIT(&tx_ring_p->lock);
1204 
1205 nxge_start_fail1:
1206 	/* Add FMA to check the access handle nxge_hregh */
1207 
1208 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
1209 	return (status);
1210 }
1211 
1212 /* Software LSO starts here */
1213 static void
1214 nxge_hcksum_retrieve(mblk_t *mp,
1215     uint32_t *start, uint32_t *stuff, uint32_t *end,
1216     uint32_t *value, uint32_t *flags)
1217 {
1218 	if (mp->b_datap->db_type == M_DATA) {
1219 		if (flags != NULL) {
1220 			*flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM |
1221 			    HCK_PARTIALCKSUM | HCK_FULLCKSUM |
1222 			    HCK_FULLCKSUM_OK);
1223 			if ((*flags & (HCK_PARTIALCKSUM |
1224 			    HCK_FULLCKSUM)) != 0) {
1225 				if (value != NULL)
1226 					*value = (uint32_t)DB_CKSUM16(mp);
1227 				if ((*flags & HCK_PARTIALCKSUM) != 0) {
1228 					if (start != NULL)
1229 						*start =
1230 						    (uint32_t)DB_CKSUMSTART(mp);
1231 					if (stuff != NULL)
1232 						*stuff =
1233 						    (uint32_t)DB_CKSUMSTUFF(mp);
1234 					if (end != NULL)
1235 						*end =
1236 						    (uint32_t)DB_CKSUMEND(mp);
1237 				}
1238 			}
1239 		}
1240 	}
1241 }
1242 
1243 static void
1244 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags)
1245 {
1246 	ASSERT(DB_TYPE(mp) == M_DATA);
1247 
1248 	*mss = 0;
1249 	if (flags != NULL) {
1250 		*flags = DB_CKSUMFLAGS(mp) & HW_LSO;
1251 		if ((*flags != 0) && (mss != NULL)) {
1252 			*mss = (uint32_t)DB_LSOMSS(mp);
1253 		}
1254 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1255 		    "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x",
1256 		    *mss, *flags));
1257 	}
1258 
1259 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1260 	    "<== nxge_lso_info_get: mss %d", *mss));
1261 }
1262 
1263 /*
1264  * Do Soft LSO on the oversized packet.
1265  *
1266  * 1. Create a chain of message for headers.
1267  * 2. Fill up header messages with proper information.
1268  * 3. Copy Eithernet, IP, and TCP headers from the original message to
1269  *    each new message with necessary adjustments.
1270  *    * Unchange the ethernet header for DIX frames. (by default)
1271  *    * IP Total Length field is updated to MSS or less(only for the last one).
1272  *    * IP Identification value is incremented by one for each packet.
1273  *    * TCP sequence Number is recalculated according to the payload length.
1274  *    * Set FIN and/or PSH flags for the *last* packet if applied.
1275  *    * TCP partial Checksum
1276  * 4. Update LSO information in the first message header.
1277  * 5. Release the original message header.
1278  */
1279 static mblk_t *
1280 nxge_do_softlso(mblk_t *mp, uint32_t mss)
1281 {
1282 	uint32_t	hckflags;
1283 	int		pktlen;
1284 	int		hdrlen;
1285 	int		segnum;
1286 	int		i;
1287 	struct ether_vlan_header *evh;
1288 	int		ehlen, iphlen, tcphlen;
1289 	struct ip	*oiph, *niph;
1290 	struct tcphdr *otcph, *ntcph;
1291 	int		available, len, left;
1292 	uint16_t	ip_id;
1293 	uint32_t	tcp_seq;
1294 #ifdef __sparc
1295 	uint32_t	tcp_seq_tmp;
1296 #endif
1297 	mblk_t		*datamp;
1298 	uchar_t		*rptr;
1299 	mblk_t		*nmp;
1300 	mblk_t		*cmp;
1301 	mblk_t		*mp_chain;
1302 	boolean_t do_cleanup = B_FALSE;
1303 	t_uscalar_t start_offset = 0;
1304 	t_uscalar_t stuff_offset = 0;
1305 	t_uscalar_t value = 0;
1306 	uint16_t	l4_len;
1307 	ipaddr_t	src, dst;
1308 	uint32_t	cksum, sum, l4cksum;
1309 
1310 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1311 	    "==> nxge_do_softlso"));
1312 	/*
1313 	 * check the length of LSO packet payload and calculate the number of
1314 	 * segments to be generated.
1315 	 */
1316 	pktlen = msgsize(mp);
1317 	evh = (struct ether_vlan_header *)mp->b_rptr;
1318 
1319 	/* VLAN? */
1320 	if (evh->ether_tpid == htons(ETHERTYPE_VLAN))
1321 		ehlen = sizeof (struct ether_vlan_header);
1322 	else
1323 		ehlen = sizeof (struct ether_header);
1324 	oiph = (struct ip *)(mp->b_rptr + ehlen);
1325 	iphlen = oiph->ip_hl * 4;
1326 	otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1327 	tcphlen = otcph->th_off * 4;
1328 
1329 	l4_len = pktlen - ehlen - iphlen;
1330 
1331 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1332 	    "==> nxge_do_softlso: mss %d oiph $%p "
1333 	    "original ip_sum oiph->ip_sum 0x%x "
1334 	    "original tcp_sum otcph->th_sum 0x%x "
1335 	    "oiph->ip_len %d pktlen %d ehlen %d "
1336 	    "l4_len %d (0x%x) ip_len - iphlen %d ",
1337 	    mss,
1338 	    oiph,
1339 	    oiph->ip_sum,
1340 	    otcph->th_sum,
1341 	    ntohs(oiph->ip_len), pktlen,
1342 	    ehlen,
1343 	    l4_len,
1344 	    l4_len,
1345 	    ntohs(oiph->ip_len) - iphlen));
1346 
1347 	/* IPv4 + TCP */
1348 	if (!(oiph->ip_v == IPV4_VERSION)) {
1349 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1350 		    "<== nxge_do_softlso: not IPV4 "
1351 		    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1352 		    ntohs(oiph->ip_len), pktlen, ehlen,
1353 		    tcphlen));
1354 		freemsg(mp);
1355 		return (NULL);
1356 	}
1357 
1358 	if (!(oiph->ip_p == IPPROTO_TCP)) {
1359 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1360 		    "<== nxge_do_softlso: not TCP "
1361 		    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1362 		    ntohs(oiph->ip_len), pktlen, ehlen,
1363 		    tcphlen));
1364 		freemsg(mp);
1365 		return (NULL);
1366 	}
1367 
1368 	if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) {
1369 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1370 		    "<== nxge_do_softlso: len not matched  "
1371 		    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1372 		    ntohs(oiph->ip_len), pktlen, ehlen,
1373 		    tcphlen));
1374 		freemsg(mp);
1375 		return (NULL);
1376 	}
1377 
1378 	otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1379 	tcphlen = otcph->th_off * 4;
1380 
1381 	/* TCP flags can not include URG, RST, or SYN */
1382 	VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0);
1383 
1384 	hdrlen = ehlen + iphlen + tcphlen;
1385 
1386 	VERIFY(MBLKL(mp) >= hdrlen);
1387 
1388 	if (MBLKL(mp) > hdrlen) {
1389 		datamp = mp;
1390 		rptr = mp->b_rptr + hdrlen;
1391 	} else { /* = */
1392 		datamp = mp->b_cont;
1393 		rptr = datamp->b_rptr;
1394 	}
1395 
1396 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1397 	    "nxge_do_softlso: otcph $%p pktlen: %d, "
1398 	    "hdrlen %d ehlen %d iphlen %d tcphlen %d "
1399 	    "mblkl(mp): %d, mblkl(datamp): %d",
1400 	    otcph,
1401 	    pktlen, hdrlen, ehlen, iphlen, tcphlen,
1402 	    (int)MBLKL(mp), (int)MBLKL(datamp)));
1403 
1404 	hckflags = 0;
1405 	nxge_hcksum_retrieve(mp,
1406 	    &start_offset, &stuff_offset, &value, NULL, &hckflags);
1407 
1408 	dst = oiph->ip_dst.s_addr;
1409 	src = oiph->ip_src.s_addr;
1410 
1411 	cksum = (dst >> 16) + (dst & 0xFFFF) +
1412 	    (src >> 16) + (src & 0xFFFF);
1413 	l4cksum = cksum + IP_TCP_CSUM_COMP;
1414 
1415 	sum = l4_len + l4cksum;
1416 	sum = (sum & 0xFFFF) + (sum >> 16);
1417 
1418 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1419 	    "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x "
1420 	    "hckflags 0x%x start_offset %d stuff_offset %d "
1421 	    "value (original) 0x%x th_sum 0x%x "
1422 	    "pktlen %d l4_len %d (0x%x) "
1423 	    "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s",
1424 	    dst, src,
1425 	    (sum & 0xffff), (~sum & 0xffff),
1426 	    hckflags, start_offset, stuff_offset,
1427 	    value, otcph->th_sum,
1428 	    pktlen,
1429 	    l4_len,
1430 	    l4_len,
1431 	    ntohs(oiph->ip_len) - (int)MBLKL(mp),
1432 	    (int)MBLKL(datamp),
1433 	    nxge_dump_packet((char *)evh, 12)));
1434 
1435 	/*
1436 	 * Start to process.
1437 	 */
1438 	available = pktlen - hdrlen;
1439 	segnum = (available - 1) / mss + 1;
1440 
1441 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1442 	    "==> nxge_do_softlso: pktlen %d "
1443 	    "MBLKL(mp): %d, MBLKL(datamp): %d "
1444 	    "available %d mss %d segnum %d",
1445 	    pktlen, (int)MBLKL(mp), (int)MBLKL(datamp),
1446 	    available,
1447 	    mss,
1448 	    segnum));
1449 
1450 	VERIFY(segnum >= 2);
1451 
1452 	/*
1453 	 * Try to pre-allocate all header messages
1454 	 */
1455 	mp_chain = NULL;
1456 	for (i = 0; i < segnum; i++) {
1457 		if ((nmp = allocb(hdrlen, 0)) == NULL) {
1458 			/* Clean up the mp_chain */
1459 			while (mp_chain != NULL) {
1460 				nmp = mp_chain;
1461 				mp_chain = mp_chain->b_next;
1462 				freemsg(nmp);
1463 			}
1464 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1465 			    "<== nxge_do_softlso: "
1466 			    "Could not allocate enough messages for headers!"));
1467 			freemsg(mp);
1468 			return (NULL);
1469 		}
1470 		nmp->b_next = mp_chain;
1471 		mp_chain = nmp;
1472 
1473 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1474 		    "==> nxge_do_softlso: "
1475 		    "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p",
1476 		    mp, nmp, mp_chain, mp_chain->b_next));
1477 	}
1478 
1479 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1480 	    "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p",
1481 	    mp, nmp, mp_chain));
1482 
1483 	/*
1484 	 * Associate payload with new packets
1485 	 */
1486 	cmp = mp_chain;
1487 	left = available;
1488 	while (cmp != NULL) {
1489 		nmp = dupb(datamp);
1490 		if (nmp == NULL) {
1491 			do_cleanup = B_TRUE;
1492 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1493 			    "==>nxge_do_softlso: "
1494 			    "Can not dupb(datamp), have to do clean up"));
1495 			goto cleanup_allocated_msgs;
1496 		}
1497 
1498 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1499 		    "==> nxge_do_softlso: (loop) before mp $%p cmp $%p "
1500 		    "dupb nmp $%p len %d left %d msd %d ",
1501 		    mp, cmp, nmp, len, left, mss));
1502 
1503 		cmp->b_cont = nmp;
1504 		nmp->b_rptr = rptr;
1505 		len = (left < mss) ? left : mss;
1506 		left -= len;
1507 
1508 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1509 		    "==> nxge_do_softlso: (loop) after mp $%p cmp $%p "
1510 		    "dupb nmp $%p len %d left %d mss %d ",
1511 		    mp, cmp, nmp, len, left, mss));
1512 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1513 		    "nxge_do_softlso: before available: %d, "
1514 		    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1515 		    available, left, len, segnum, (int)MBLKL(nmp)));
1516 
1517 		len -= MBLKL(nmp);
1518 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1519 		    "nxge_do_softlso: after available: %d, "
1520 		    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1521 		    available, left, len, segnum, (int)MBLKL(nmp)));
1522 
1523 		while (len > 0) {
1524 			mblk_t *mmp = NULL;
1525 
1526 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1527 			    "nxge_do_softlso: (4) len > 0 available: %d, "
1528 			    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1529 			    available, left, len, segnum, (int)MBLKL(nmp)));
1530 
1531 			if (datamp->b_cont != NULL) {
1532 				datamp = datamp->b_cont;
1533 				rptr = datamp->b_rptr;
1534 				mmp = dupb(datamp);
1535 				if (mmp == NULL) {
1536 					do_cleanup = B_TRUE;
1537 					NXGE_DEBUG_MSG((NULL, TX_CTL,
1538 					    "==> nxge_do_softlso: "
1539 					    "Can not dupb(datamp) (1), :"
1540 					    "have to do clean up"));
1541 					NXGE_DEBUG_MSG((NULL, TX_CTL,
1542 					    "==> nxge_do_softlso: "
1543 					    "available: %d, left: %d, "
1544 					    "len: %d, MBLKL(nmp): %d",
1545 					    available, left, len,
1546 					    (int)MBLKL(nmp)));
1547 					goto cleanup_allocated_msgs;
1548 				}
1549 			} else {
1550 				NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1551 				    "==> nxge_do_softlso: "
1552 				    "(1)available: %d, left: %d, "
1553 				    "len: %d, MBLKL(nmp): %d",
1554 				    available, left, len,
1555 				    (int)MBLKL(nmp)));
1556 				cmn_err(CE_PANIC,
1557 				    "==> nxge_do_softlso: "
1558 				    "Pointers must have been corrupted!\n"
1559 				    "datamp: $%p, nmp: $%p, rptr: $%p",
1560 				    (void *)datamp,
1561 				    (void *)nmp,
1562 				    (void *)rptr);
1563 			}
1564 			nmp->b_cont = mmp;
1565 			nmp = mmp;
1566 			len -= MBLKL(nmp);
1567 		}
1568 		if (len < 0) {
1569 			nmp->b_wptr += len;
1570 			rptr = nmp->b_wptr;
1571 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1572 			    "(5) len < 0 (less than 0)"
1573 			    "available: %d, left: %d, len: %d, MBLKL(nmp): %d",
1574 			    available, left, len, (int)MBLKL(nmp)));
1575 
1576 		} else if (len == 0) {
1577 			if (datamp->b_cont != NULL) {
1578 				NXGE_DEBUG_MSG((NULL, TX_CTL,
1579 				    "(5) len == 0"
1580 				    "available: %d, left: %d, len: %d, "
1581 				    "MBLKL(nmp): %d",
1582 				    available, left, len, (int)MBLKL(nmp)));
1583 				datamp = datamp->b_cont;
1584 				rptr = datamp->b_rptr;
1585 			} else {
1586 				NXGE_DEBUG_MSG((NULL, TX_CTL,
1587 				    "(6)available b_cont == NULL : %d, "
1588 				    "left: %d, len: %d, MBLKL(nmp): %d",
1589 				    available, left, len, (int)MBLKL(nmp)));
1590 
1591 				VERIFY(cmp->b_next == NULL);
1592 				VERIFY(left == 0);
1593 				break; /* Done! */
1594 			}
1595 		}
1596 		cmp = cmp->b_next;
1597 
1598 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1599 		    "(7) do_softlso: "
1600 		    "next mp in mp_chain available len != 0 : %d, "
1601 		    "left: %d, len: %d, MBLKL(nmp): %d",
1602 		    available, left, len, (int)MBLKL(nmp)));
1603 	}
1604 
1605 	/*
1606 	 * From now, start to fill up all headers for the first message
1607 	 * Hardware checksum flags need to be updated separately for FULLCKSUM
1608 	 * and PARTIALCKSUM cases. For full checksum, copy the original flags
1609 	 * into every new packet is enough. But for HCK_PARTIALCKSUM, all
1610 	 * required fields need to be updated properly.
1611 	 */
1612 	nmp = mp_chain;
1613 	bcopy(mp->b_rptr, nmp->b_rptr, hdrlen);
1614 	nmp->b_wptr = nmp->b_rptr + hdrlen;
1615 	niph = (struct ip *)(nmp->b_rptr + ehlen);
1616 	niph->ip_len = htons(mss + iphlen + tcphlen);
1617 	ip_id = ntohs(niph->ip_id);
1618 	ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1619 #ifdef __sparc
1620 	bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4);
1621 	tcp_seq = ntohl(tcp_seq_tmp);
1622 #else
1623 	tcp_seq = ntohl(ntcph->th_seq);
1624 #endif
1625 
1626 	ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST);
1627 
1628 	DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1629 	DB_CKSUMSTART(nmp) = start_offset;
1630 	DB_CKSUMSTUFF(nmp) = stuff_offset;
1631 
1632 	/* calculate IP checksum and TCP pseudo header checksum */
1633 	niph->ip_sum = 0;
1634 	niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1635 
1636 	l4_len = mss + tcphlen;
1637 	sum = htons(l4_len) + l4cksum;
1638 	sum = (sum & 0xFFFF) + (sum >> 16);
1639 	ntcph->th_sum = (sum & 0xffff);
1640 
1641 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1642 	    "==> nxge_do_softlso: first mp $%p (mp_chain $%p) "
1643 	    "mss %d pktlen %d l4_len %d (0x%x) "
1644 	    "MBLKL(mp): %d, MBLKL(datamp): %d "
1645 	    "ip_sum 0x%x "
1646 	    "th_sum 0x%x sum 0x%x ) "
1647 	    "dump first ip->tcp %s",
1648 	    nmp, mp_chain,
1649 	    mss,
1650 	    pktlen,
1651 	    l4_len,
1652 	    l4_len,
1653 	    (int)MBLKL(mp), (int)MBLKL(datamp),
1654 	    niph->ip_sum,
1655 	    ntcph->th_sum,
1656 	    sum,
1657 	    nxge_dump_packet((char *)niph, 52)));
1658 
1659 	cmp = nmp;
1660 	while ((nmp = nmp->b_next)->b_next != NULL) {
1661 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1662 		    "==>nxge_do_softlso: middle l4_len %d ", l4_len));
1663 		bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1664 		nmp->b_wptr = nmp->b_rptr + hdrlen;
1665 		niph = (struct ip *)(nmp->b_rptr + ehlen);
1666 		niph->ip_id = htons(++ip_id);
1667 		niph->ip_len = htons(mss + iphlen + tcphlen);
1668 		ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1669 		tcp_seq += mss;
1670 
1671 		ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG);
1672 
1673 #ifdef __sparc
1674 		tcp_seq_tmp = htonl(tcp_seq);
1675 		bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1676 #else
1677 		ntcph->th_seq = htonl(tcp_seq);
1678 #endif
1679 		DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1680 		DB_CKSUMSTART(nmp) = start_offset;
1681 		DB_CKSUMSTUFF(nmp) = stuff_offset;
1682 
1683 		/* calculate IP checksum and TCP pseudo header checksum */
1684 		niph->ip_sum = 0;
1685 		niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1686 		ntcph->th_sum = (sum & 0xffff);
1687 
1688 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1689 		    "==> nxge_do_softlso: middle ip_sum 0x%x "
1690 		    "th_sum 0x%x "
1691 		    " mp $%p (mp_chain $%p) pktlen %d "
1692 		    "MBLKL(mp): %d, MBLKL(datamp): %d ",
1693 		    niph->ip_sum,
1694 		    ntcph->th_sum,
1695 		    nmp, mp_chain,
1696 		    pktlen, (int)MBLKL(mp), (int)MBLKL(datamp)));
1697 	}
1698 
1699 	/* Last segment */
1700 	/*
1701 	 * Set FIN and/or PSH flags if present only in the last packet.
1702 	 * The ip_len could be different from prior packets.
1703 	 */
1704 	bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1705 	nmp->b_wptr = nmp->b_rptr + hdrlen;
1706 	niph = (struct ip *)(nmp->b_rptr + ehlen);
1707 	niph->ip_id = htons(++ip_id);
1708 	niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen);
1709 	ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1710 	tcp_seq += mss;
1711 #ifdef __sparc
1712 	tcp_seq_tmp = htonl(tcp_seq);
1713 	bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1714 #else
1715 	ntcph->th_seq = htonl(tcp_seq);
1716 #endif
1717 	ntcph->th_flags = (otcph->th_flags & ~TH_URG);
1718 
1719 	DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1720 	DB_CKSUMSTART(nmp) = start_offset;
1721 	DB_CKSUMSTUFF(nmp) = stuff_offset;
1722 
1723 	/* calculate IP checksum and TCP pseudo header checksum */
1724 	niph->ip_sum = 0;
1725 	niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1726 
1727 	l4_len = ntohs(niph->ip_len) - iphlen;
1728 	sum = htons(l4_len) + l4cksum;
1729 	sum = (sum & 0xFFFF) + (sum >> 16);
1730 	ntcph->th_sum = (sum & 0xffff);
1731 
1732 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1733 	    "==> nxge_do_softlso: last next "
1734 	    "niph->ip_sum 0x%x "
1735 	    "ntcph->th_sum 0x%x sum 0x%x "
1736 	    "dump last ip->tcp %s "
1737 	    "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) "
1738 	    "l4_len %d (0x%x) "
1739 	    "MBLKL(mp): %d, MBLKL(datamp): %d ",
1740 	    niph->ip_sum,
1741 	    ntcph->th_sum, sum,
1742 	    nxge_dump_packet((char *)niph, 52),
1743 	    cmp, nmp, mp_chain,
1744 	    pktlen, pktlen,
1745 	    l4_len,
1746 	    l4_len,
1747 	    (int)MBLKL(mp), (int)MBLKL(datamp)));
1748 
1749 cleanup_allocated_msgs:
1750 	if (do_cleanup) {
1751 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1752 		    "==> nxge_do_softlso: "
1753 		    "Failed allocating messages, "
1754 		    "have to clean up and fail!"));
1755 		while (mp_chain != NULL) {
1756 			nmp = mp_chain;
1757 			mp_chain = mp_chain->b_next;
1758 			freemsg(nmp);
1759 		}
1760 	}
1761 	/*
1762 	 * We're done here, so just free the original message and return the
1763 	 * new message chain, that could be NULL if failed, back to the caller.
1764 	 */
1765 	freemsg(mp);
1766 
1767 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1768 	    "<== nxge_do_softlso:mp_chain $%p", mp_chain));
1769 	return (mp_chain);
1770 }
1771 
1772 /*
1773  * Will be called before NIC driver do further operation on the message.
1774  * The input message may include LSO information, if so, go to softlso logic
1775  * to eliminate the oversized LSO packet for the incapable underlying h/w.
1776  * The return could be the same non-LSO message or a message chain for LSO case.
1777  *
1778  * The driver needs to call this function per packet and process the whole chain
1779  * if applied.
1780  */
1781 static mblk_t *
1782 nxge_lso_eliminate(mblk_t *mp)
1783 {
1784 	uint32_t lsoflags;
1785 	uint32_t mss;
1786 
1787 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1788 	    "==>nxge_lso_eliminate:"));
1789 	nxge_lso_info_get(mp, &mss, &lsoflags);
1790 
1791 	if (lsoflags & HW_LSO) {
1792 		mblk_t *nmp;
1793 
1794 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1795 		    "==>nxge_lso_eliminate:"
1796 		    "HW_LSO:mss %d mp $%p",
1797 		    mss, mp));
1798 		if ((nmp = nxge_do_softlso(mp, mss)) != NULL) {
1799 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1800 			    "<== nxge_lso_eliminate: "
1801 			    "LSO: nmp not NULL nmp $%p mss %d mp $%p",
1802 			    nmp, mss, mp));
1803 			return (nmp);
1804 		} else {
1805 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1806 			    "<== nxge_lso_eliminate_ "
1807 			    "LSO: failed nmp NULL nmp $%p mss %d mp $%p",
1808 			    nmp, mss, mp));
1809 			return (NULL);
1810 		}
1811 	}
1812 
1813 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1814 	    "<== nxge_lso_eliminate"));
1815 	return (mp);
1816 }
1817 
1818 static uint32_t
1819 nxge_csgen(uint16_t *adr, int len)
1820 {
1821 	int		i, odd;
1822 	uint32_t	sum = 0;
1823 	uint32_t	c = 0;
1824 
1825 	odd = len % 2;
1826 	for (i = 0; i < (len / 2); i++) {
1827 		sum += (adr[i] & 0xffff);
1828 	}
1829 	if (odd) {
1830 		sum += adr[len / 2] & 0xff00;
1831 	}
1832 	while ((c = ((sum & 0xffff0000) >> 16)) != 0) {
1833 		sum &= 0xffff;
1834 		sum += c;
1835 	}
1836 	return (~sum & 0xffff);
1837 }
1838