xref: /titanic_50/usr/src/uts/common/io/nxge/nxge_send.c (revision 275c9da86e89f8abf71135cf63d9fc23671b2e60)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/nxge/nxge_impl.h>
29 
30 /* Software LSO required header files */
31 #include <netinet/tcp.h>
32 #include <inet/ip_impl.h>
33 #include <inet/tcp.h>
34 
35 static mblk_t *nxge_lso_eliminate(mblk_t *);
36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss);
37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *);
38 static void nxge_hcksum_retrieve(mblk_t *,
39     uint32_t *, uint32_t *, uint32_t *,
40     uint32_t *, uint32_t *);
41 static uint32_t nxge_csgen(uint16_t *, int);
42 
43 extern uint32_t		nxge_reclaim_pending;
44 extern uint32_t 	nxge_bcopy_thresh;
45 extern uint32_t 	nxge_dvma_thresh;
46 extern uint32_t 	nxge_dma_stream_thresh;
47 extern uint32_t		nxge_tx_minfree;
48 extern uint32_t		nxge_tx_intr_thres;
49 extern uint32_t		nxge_tx_max_gathers;
50 extern uint32_t		nxge_tx_tiny_pack;
51 extern uint32_t		nxge_tx_use_bcopy;
52 extern uint32_t		nxge_tx_lb_policy;
53 extern uint32_t		nxge_no_tx_lb;
54 extern nxge_tx_mode_t	nxge_tx_scheme;
55 uint32_t		nxge_lso_kick_cnt = 2;
56 
57 typedef struct _mac_tx_hint {
58 	uint16_t	sap;
59 	uint16_t	vid;
60 	void		*hash;
61 } mac_tx_hint_t, *p_mac_tx_hint_t;
62 
63 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t);
64 
65 int
66 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
67 {
68 	int 			status = 0;
69 	p_tx_desc_t 		tx_desc_ring_vp;
70 	npi_handle_t		npi_desc_handle;
71 	nxge_os_dma_handle_t 	tx_desc_dma_handle;
72 	p_tx_desc_t 		tx_desc_p;
73 	p_tx_msg_t 		tx_msg_ring;
74 	p_tx_msg_t 		tx_msg_p;
75 	tx_desc_t		tx_desc, *tmp_desc_p;
76 	tx_desc_t		sop_tx_desc, *sop_tx_desc_p;
77 	p_tx_pkt_header_t	hdrp;
78 	p_tx_pkt_hdr_all_t	pkthdrp;
79 	uint8_t			npads = 0;
80 	uint64_t 		dma_ioaddr;
81 	uint32_t		dma_flags;
82 	int			last_bidx;
83 	uint8_t 		*b_rptr;
84 	caddr_t 		kaddr;
85 	uint32_t		nmblks;
86 	uint32_t		ngathers;
87 	uint32_t		clen;
88 	int 			len;
89 	uint32_t		pkt_len, pack_len, min_len;
90 	uint32_t		bcopy_thresh;
91 	int 			i, cur_index, sop_index;
92 	uint16_t		tail_index;
93 	boolean_t		tail_wrap = B_FALSE;
94 	nxge_dma_common_t	desc_area;
95 	nxge_os_dma_handle_t 	dma_handle;
96 	ddi_dma_cookie_t 	dma_cookie;
97 	npi_handle_t		npi_handle;
98 	p_mblk_t 		nmp;
99 	p_mblk_t		t_mp;
100 	uint32_t 		ncookies;
101 	boolean_t 		good_packet;
102 	boolean_t 		mark_mode = B_FALSE;
103 	p_nxge_stats_t 		statsp;
104 	p_nxge_tx_ring_stats_t tdc_stats;
105 	t_uscalar_t 		start_offset = 0;
106 	t_uscalar_t 		stuff_offset = 0;
107 	t_uscalar_t 		end_offset = 0;
108 	t_uscalar_t 		value = 0;
109 	t_uscalar_t 		cksum_flags = 0;
110 	boolean_t		cksum_on = B_FALSE;
111 	uint32_t		boff = 0;
112 	uint64_t		tot_xfer_len = 0, tmp_len = 0;
113 	boolean_t		header_set = B_FALSE;
114 #ifdef NXGE_DEBUG
115 	p_tx_desc_t 		tx_desc_ring_pp;
116 	p_tx_desc_t 		tx_desc_pp;
117 	tx_desc_t		*save_desc_p;
118 	int			dump_len;
119 	int			sad_len;
120 	uint64_t		sad;
121 	int			xfer_len;
122 	uint32_t		msgsize;
123 #endif
124 	p_mblk_t 		mp_chain = NULL;
125 	boolean_t		is_lso = B_FALSE;
126 	boolean_t		lso_again;
127 	int			cur_index_lso;
128 	p_mblk_t 		nmp_lso_save;
129 	uint32_t		lso_ngathers;
130 	boolean_t		lso_tail_wrap = B_FALSE;
131 
132 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
133 		"==> nxge_start: tx dma channel %d", tx_ring_p->tdc));
134 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
135 		"==> nxge_start: Starting tdc %d desc pending %d",
136 		tx_ring_p->tdc, tx_ring_p->descs_pending));
137 
138 	statsp = nxgep->statsp;
139 
140 	switch (nxgep->mac.portmode) {
141 	default:
142 		if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) {
143 			if (!statsp->mac_stats.link_up) {
144 				freemsg(mp);
145 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
146 				    "==> nxge_start: "
147 				    "link not up"));
148 				goto nxge_start_fail1;
149 			}
150 		}
151 		break;
152 	case PORT_10G_FIBER:
153 		/*
154 		 * For the following modes, check the link status
155 		 * before sending the packet out:
156 		 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g
157 		 */
158 		if (nxgep->statsp->port_stats.lb_mode < nxge_lb_serdes10g) {
159 			if (!statsp->mac_stats.link_up) {
160 				freemsg(mp);
161 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
162 				    "==> nxge_start: "
163 				    "link not up"));
164 				goto nxge_start_fail1;
165 			}
166 		}
167 		break;
168 	}
169 
170 	if (nxgep->soft_lso_enable) {
171 		mp_chain = nxge_lso_eliminate(mp);
172 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
173 		    "==> nxge_start(0): LSO mp $%p mp_chain $%p",
174 		    mp, mp_chain));
175 		if (mp_chain == NULL) {
176 			NXGE_ERROR_MSG((nxgep, TX_CTL,
177 			    "==> nxge_send(0): NULL mp_chain $%p != mp $%p",
178 			    mp_chain, mp));
179 			goto nxge_start_fail1;
180 		}
181 		if (mp_chain != mp) {
182 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
183 			    "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p",
184 			    mp_chain, mp));
185 			is_lso = B_TRUE;
186 			mp = mp_chain;
187 			mp_chain = mp_chain->b_next;
188 			mp->b_next = NULL;
189 		}
190 	}
191 
192 	hcksum_retrieve(mp, NULL, NULL, &start_offset,
193 		&stuff_offset, &end_offset, &value, &cksum_flags);
194 	if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) {
195 		start_offset += sizeof (ether_header_t);
196 		stuff_offset += sizeof (ether_header_t);
197 	} else {
198 		start_offset += sizeof (struct ether_vlan_header);
199 		stuff_offset += sizeof (struct ether_vlan_header);
200 	}
201 
202 	if (cksum_flags & HCK_PARTIALCKSUM) {
203 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
204 			"==> nxge_start: mp $%p len %d "
205 			"cksum_flags 0x%x (partial checksum) ",
206 			mp, MBLKL(mp), cksum_flags));
207 		cksum_on = B_TRUE;
208 	}
209 
210 	lso_again = B_FALSE;
211 	lso_ngathers = 0;
212 
213 	MUTEX_ENTER(&tx_ring_p->lock);
214 	cur_index_lso = tx_ring_p->wr_index;
215 	lso_tail_wrap = tx_ring_p->wr_index_wrap;
216 start_again:
217 	ngathers = 0;
218 	sop_index = tx_ring_p->wr_index;
219 #ifdef	NXGE_DEBUG
220 	if (tx_ring_p->descs_pending) {
221 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
222 			"desc pending %d ", tx_ring_p->descs_pending));
223 	}
224 
225 	dump_len = (int)(MBLKL(mp));
226 	dump_len = (dump_len > 128) ? 128: dump_len;
227 
228 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
229 		"==> nxge_start: tdc %d: dumping ...: b_rptr $%p "
230 		"(Before header reserve: ORIGINAL LEN %d)",
231 		tx_ring_p->tdc,
232 		mp->b_rptr,
233 		dump_len));
234 
235 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets "
236 		"(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr,
237 		nxge_dump_packet((char *)mp->b_rptr, dump_len)));
238 #endif
239 
240 	tdc_stats = tx_ring_p->tdc_stats;
241 	mark_mode = (tx_ring_p->descs_pending &&
242 		((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending)
243 		< nxge_tx_minfree));
244 
245 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
246 		"TX Descriptor ring is channel %d mark mode %d",
247 		tx_ring_p->tdc, mark_mode));
248 
249 	if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) {
250 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
251 			"TX Descriptor ring is full: channel %d",
252 			tx_ring_p->tdc));
253 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
254 			"TX Descriptor ring is full: channel %d",
255 			tx_ring_p->tdc));
256 		if (is_lso) {
257 			/* free the current mp and mp_chain if not FULL */
258 			tdc_stats->tx_no_desc++;
259 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
260 			    "LSO packet: TX Descriptor ring is full: "
261 			    "channel %d",
262 			    tx_ring_p->tdc));
263 			goto nxge_start_fail_lso;
264 		} else {
265 			cas32((uint32_t *)&tx_ring_p->queueing, 0, 1);
266 			tdc_stats->tx_no_desc++;
267 			MUTEX_EXIT(&tx_ring_p->lock);
268 			if (nxgep->resched_needed && !nxgep->resched_running) {
269 				nxgep->resched_running = B_TRUE;
270 				ddi_trigger_softintr(nxgep->resched_id);
271 			}
272 			status = 1;
273 			goto nxge_start_fail1;
274 		}
275 	}
276 
277 	nmp = mp;
278 	i = sop_index = tx_ring_p->wr_index;
279 	nmblks = 0;
280 	ngathers = 0;
281 	pkt_len = 0;
282 	pack_len = 0;
283 	clen = 0;
284 	last_bidx = -1;
285 	good_packet = B_TRUE;
286 
287 	desc_area = tx_ring_p->tdc_desc;
288 	npi_handle = desc_area.npi_handle;
289 	npi_desc_handle.regh = (nxge_os_acc_handle_t)
290 			DMA_COMMON_ACC_HANDLE(desc_area);
291 	tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
292 #ifdef	NXGE_DEBUG
293 	tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area);
294 #endif
295 	tx_desc_dma_handle = (nxge_os_dma_handle_t)
296 			DMA_COMMON_HANDLE(desc_area);
297 	tx_msg_ring = tx_ring_p->tx_msg_ring;
298 
299 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d",
300 		sop_index, i));
301 
302 #ifdef	NXGE_DEBUG
303 	msgsize = msgdsize(nmp);
304 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
305 		"==> nxge_start(1): wr_index %d i %d msgdsize %d",
306 		sop_index, i, msgsize));
307 #endif
308 	/*
309 	 * The first 16 bytes of the premapped buffer are reserved
310 	 * for header. No padding will be used.
311 	 */
312 	pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
313 	if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) {
314 		bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
315 	} else {
316 		bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
317 	}
318 	while (nmp) {
319 		good_packet = B_TRUE;
320 		b_rptr = nmp->b_rptr;
321 		len = MBLKL(nmp);
322 		if (len <= 0) {
323 			nmp = nmp->b_cont;
324 			continue;
325 		}
326 		nmblks++;
327 
328 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d "
329 			"len %d pkt_len %d pack_len %d",
330 			nmblks, len, pkt_len, pack_len));
331 		/*
332 		 * Hardware limits the transfer length to 4K for NIU and
333 		 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just
334 		 * use TX_MAX_TRANSFER_LENGTH as the limit for both.
335 		 * If len is longer than the limit, then we break nmp into
336 		 * two chunks: Make the first chunk equal to the limit and
337 		 * the second chunk for the remaining data. If the second
338 		 * chunk is still larger than the limit, then it will be
339 		 * broken into two in the next pass.
340 		 */
341 		if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) {
342 			if ((t_mp = dupb(nmp)) != NULL) {
343 				nmp->b_wptr = nmp->b_rptr +
344 				    (TX_MAX_TRANSFER_LENGTH
345 				    - TX_PKT_HEADER_SIZE);
346 				t_mp->b_rptr = nmp->b_wptr;
347 				t_mp->b_cont = nmp->b_cont;
348 				nmp->b_cont = t_mp;
349 				len = MBLKL(nmp);
350 			} else {
351 				if (is_lso) {
352 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
353 					    "LSO packet: dupb failed: "
354 					    "channel %d",
355 					    tx_ring_p->tdc));
356 					mp = nmp;
357 					goto nxge_start_fail_lso;
358 				} else {
359 					good_packet = B_FALSE;
360 					goto nxge_start_fail2;
361 				}
362 			}
363 		}
364 		tx_desc.value = 0;
365 		tx_desc_p = &tx_desc_ring_vp[i];
366 #ifdef	NXGE_DEBUG
367 		tx_desc_pp = &tx_desc_ring_pp[i];
368 #endif
369 		tx_msg_p = &tx_msg_ring[i];
370 #if defined(__i386)
371 		npi_desc_handle.regp = (uint32_t)tx_desc_p;
372 #else
373 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
374 #endif
375 		if (!header_set &&
376 			((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
377 				(len >= bcopy_thresh))) {
378 			header_set = B_TRUE;
379 			bcopy_thresh += TX_PKT_HEADER_SIZE;
380 			boff = 0;
381 			pack_len = 0;
382 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
383 			hdrp = (p_tx_pkt_header_t)kaddr;
384 			clen = pkt_len;
385 			dma_handle = tx_msg_p->buf_dma_handle;
386 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
387 			(void) ddi_dma_sync(dma_handle,
388 				i * nxge_bcopy_thresh, nxge_bcopy_thresh,
389 				DDI_DMA_SYNC_FORDEV);
390 
391 			tx_msg_p->flags.dma_type = USE_BCOPY;
392 			goto nxge_start_control_header_only;
393 		}
394 
395 		pkt_len += len;
396 		pack_len += len;
397 
398 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): "
399 			"desc entry %d "
400 			"DESC IOADDR $%p "
401 			"desc_vp $%p tx_desc_p $%p "
402 			"desc_pp $%p tx_desc_pp $%p "
403 			"len %d pkt_len %d pack_len %d",
404 			i,
405 			DMA_COMMON_IOADDR(desc_area),
406 			tx_desc_ring_vp, tx_desc_p,
407 			tx_desc_ring_pp, tx_desc_pp,
408 			len, pkt_len, pack_len));
409 
410 		if (len < bcopy_thresh) {
411 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): "
412 				"USE BCOPY: "));
413 			if (nxge_tx_tiny_pack) {
414 				uint32_t blst =
415 					TXDMA_DESC_NEXT_INDEX(i, -1,
416 						tx_ring_p->tx_wrap_mask);
417 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
418 					"==> nxge_start(5): pack"));
419 				if ((pack_len <= bcopy_thresh) &&
420 					(last_bidx == blst)) {
421 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
422 						"==> nxge_start: pack(6) "
423 						"(pkt_len %d pack_len %d)",
424 						pkt_len, pack_len));
425 					i = blst;
426 					tx_desc_p = &tx_desc_ring_vp[i];
427 #ifdef	NXGE_DEBUG
428 					tx_desc_pp = &tx_desc_ring_pp[i];
429 #endif
430 					tx_msg_p = &tx_msg_ring[i];
431 					boff = pack_len - len;
432 					ngathers--;
433 				} else if (pack_len > bcopy_thresh &&
434 					header_set) {
435 					pack_len = len;
436 					boff = 0;
437 					bcopy_thresh = nxge_bcopy_thresh;
438 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
439 						"==> nxge_start(7): > max NEW "
440 						"bcopy thresh %d "
441 						"pkt_len %d pack_len %d(next)",
442 						bcopy_thresh,
443 						pkt_len, pack_len));
444 				}
445 				last_bidx = i;
446 			}
447 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
448 			if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
449 				hdrp = (p_tx_pkt_header_t)kaddr;
450 				header_set = B_TRUE;
451 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
452 					"==> nxge_start(7_x2): "
453 					"pkt_len %d pack_len %d (new hdrp $%p)",
454 					pkt_len, pack_len, hdrp));
455 			}
456 			tx_msg_p->flags.dma_type = USE_BCOPY;
457 			kaddr += boff;
458 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): "
459 				"USE BCOPY: before bcopy "
460 				"DESC IOADDR $%p entry %d "
461 				"bcopy packets %d "
462 				"bcopy kaddr $%p "
463 				"bcopy ioaddr (SAD) $%p "
464 				"bcopy clen %d "
465 				"bcopy boff %d",
466 				DMA_COMMON_IOADDR(desc_area), i,
467 				tdc_stats->tx_hdr_pkts,
468 				kaddr,
469 				dma_ioaddr,
470 				clen,
471 				boff));
472 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
473 				"1USE BCOPY: "));
474 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
475 				"2USE BCOPY: "));
476 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
477 				"last USE BCOPY: copy from b_rptr $%p "
478 				"to KADDR $%p (len %d offset %d",
479 				b_rptr, kaddr, len, boff));
480 
481 			bcopy(b_rptr, kaddr, len);
482 
483 #ifdef	NXGE_DEBUG
484 			dump_len = (len > 128) ? 128: len;
485 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
486 				"==> nxge_start: dump packets "
487 				"(After BCOPY len %d)"
488 				"(b_rptr $%p): %s", len, nmp->b_rptr,
489 				nxge_dump_packet((char *)nmp->b_rptr,
490 				dump_len)));
491 #endif
492 
493 			dma_handle = tx_msg_p->buf_dma_handle;
494 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
495 			(void) ddi_dma_sync(dma_handle,
496 				i * nxge_bcopy_thresh, nxge_bcopy_thresh,
497 					DDI_DMA_SYNC_FORDEV);
498 			clen = len + boff;
499 			tdc_stats->tx_hdr_pkts++;
500 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): "
501 				"USE BCOPY: "
502 				"DESC IOADDR $%p entry %d "
503 				"bcopy packets %d "
504 				"bcopy kaddr $%p "
505 				"bcopy ioaddr (SAD) $%p "
506 				"bcopy clen %d "
507 				"bcopy boff %d",
508 				DMA_COMMON_IOADDR(desc_area),
509 				i,
510 				tdc_stats->tx_hdr_pkts,
511 				kaddr,
512 				dma_ioaddr,
513 				clen,
514 				boff));
515 		} else {
516 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): "
517 				"USE DVMA: len %d", len));
518 			tx_msg_p->flags.dma_type = USE_DMA;
519 			dma_flags = DDI_DMA_WRITE;
520 			if (len < nxge_dma_stream_thresh) {
521 				dma_flags |= DDI_DMA_CONSISTENT;
522 			} else {
523 				dma_flags |= DDI_DMA_STREAMING;
524 			}
525 
526 			dma_handle = tx_msg_p->dma_handle;
527 			status = ddi_dma_addr_bind_handle(dma_handle, NULL,
528 				(caddr_t)b_rptr, len, dma_flags,
529 				DDI_DMA_DONTWAIT, NULL,
530 				&dma_cookie, &ncookies);
531 			if (status == DDI_DMA_MAPPED) {
532 				dma_ioaddr = dma_cookie.dmac_laddress;
533 				len = (int)dma_cookie.dmac_size;
534 				clen = (uint32_t)dma_cookie.dmac_size;
535 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
536 					"==> nxge_start(12_1): "
537 					"USE DVMA: len %d clen %d "
538 					"ngathers %d",
539 					len, clen,
540 					ngathers));
541 #if defined(__i386)
542 				npi_desc_handle.regp = (uint32_t)tx_desc_p;
543 #else
544 				npi_desc_handle.regp = (uint64_t)tx_desc_p;
545 #endif
546 				while (ncookies > 1) {
547 					ngathers++;
548 					/*
549 					 * this is the fix for multiple
550 					 * cookies, which are basically
551 					 * a descriptor entry, we don't set
552 					 * SOP bit as well as related fields
553 					 */
554 
555 					(void) npi_txdma_desc_gather_set(
556 						npi_desc_handle,
557 						&tx_desc,
558 						(ngathers -1),
559 						mark_mode,
560 						ngathers,
561 						dma_ioaddr,
562 						clen);
563 
564 					tx_msg_p->tx_msg_size = clen;
565 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
566 						"==> nxge_start:  DMA "
567 						"ncookie %d "
568 						"ngathers %d "
569 						"dma_ioaddr $%p len %d"
570 						"desc $%p descp $%p (%d)",
571 						ncookies,
572 						ngathers,
573 						dma_ioaddr, clen,
574 						*tx_desc_p, tx_desc_p, i));
575 
576 					ddi_dma_nextcookie(dma_handle,
577 							&dma_cookie);
578 					dma_ioaddr =
579 						dma_cookie.dmac_laddress;
580 
581 					len = (int)dma_cookie.dmac_size;
582 					clen = (uint32_t)dma_cookie.dmac_size;
583 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
584 						"==> nxge_start(12_2): "
585 						"USE DVMA: len %d clen %d ",
586 						len, clen));
587 
588 					i = TXDMA_DESC_NEXT_INDEX(i, 1,
589 						tx_ring_p->tx_wrap_mask);
590 					tx_desc_p = &tx_desc_ring_vp[i];
591 
592 					npi_desc_handle.regp =
593 #if defined(__i386)
594 						(uint32_t)tx_desc_p;
595 #else
596 						(uint64_t)tx_desc_p;
597 #endif
598 					tx_msg_p = &tx_msg_ring[i];
599 					tx_msg_p->flags.dma_type = USE_NONE;
600 					tx_desc.value = 0;
601 
602 					ncookies--;
603 				}
604 				tdc_stats->tx_ddi_pkts++;
605 				NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:"
606 					"DMA: ddi packets %d",
607 					tdc_stats->tx_ddi_pkts));
608 			} else {
609 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
610 				    "dma mapping failed for %d "
611 				    "bytes addr $%p flags %x (%d)",
612 				    len, b_rptr, status, status));
613 				good_packet = B_FALSE;
614 				tdc_stats->tx_dma_bind_fail++;
615 				tx_msg_p->flags.dma_type = USE_NONE;
616 				if (is_lso) {
617 					mp = nmp;
618 					goto nxge_start_fail_lso;
619 				} else {
620 					goto nxge_start_fail2;
621 				}
622 			}
623 		} /* ddi dvma */
624 
625 		if (is_lso) {
626 			nmp_lso_save = nmp;
627 		}
628 		nmp = nmp->b_cont;
629 nxge_start_control_header_only:
630 #if defined(__i386)
631 		npi_desc_handle.regp = (uint32_t)tx_desc_p;
632 #else
633 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
634 #endif
635 		ngathers++;
636 
637 		if (ngathers == 1) {
638 #ifdef	NXGE_DEBUG
639 			save_desc_p = &sop_tx_desc;
640 #endif
641 			sop_tx_desc_p = &sop_tx_desc;
642 			sop_tx_desc_p->value = 0;
643 			sop_tx_desc_p->bits.hdw.tr_len = clen;
644 			sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
645 			sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
646 		} else {
647 #ifdef	NXGE_DEBUG
648 			save_desc_p = &tx_desc;
649 #endif
650 			tmp_desc_p = &tx_desc;
651 			tmp_desc_p->value = 0;
652 			tmp_desc_p->bits.hdw.tr_len = clen;
653 			tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
654 			tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
655 
656 			tx_desc_p->value = tmp_desc_p->value;
657 		}
658 
659 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): "
660 			"Desc_entry %d ngathers %d "
661 			"desc_vp $%p tx_desc_p $%p "
662 			"len %d clen %d pkt_len %d pack_len %d nmblks %d "
663 			"dma_ioaddr (SAD) $%p mark %d",
664 			i, ngathers,
665 			tx_desc_ring_vp, tx_desc_p,
666 			len, clen, pkt_len, pack_len, nmblks,
667 			dma_ioaddr, mark_mode));
668 
669 #ifdef NXGE_DEBUG
670 		npi_desc_handle.nxgep = nxgep;
671 		npi_desc_handle.function.function = nxgep->function_num;
672 		npi_desc_handle.function.instance = nxgep->instance;
673 		sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK);
674 		xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >>
675 			TX_PKT_DESC_TR_LEN_SHIFT);
676 
677 
678 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
679 			"\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
680 			"mark %d sop %d\n",
681 			save_desc_p->value,
682 			sad,
683 			save_desc_p->bits.hdw.tr_len,
684 			xfer_len,
685 			save_desc_p->bits.hdw.num_ptr,
686 			save_desc_p->bits.hdw.mark,
687 			save_desc_p->bits.hdw.sop));
688 
689 		npi_txdma_dump_desc_one(npi_desc_handle, NULL, i);
690 #endif
691 
692 		tx_msg_p->tx_msg_size = clen;
693 		i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
694 		if (ngathers > nxge_tx_max_gathers) {
695 			good_packet = B_FALSE;
696 			hcksum_retrieve(mp, NULL, NULL, &start_offset,
697 				&stuff_offset, &end_offset, &value,
698 				&cksum_flags);
699 
700 			NXGE_DEBUG_MSG((NULL, TX_CTL,
701 				"==> nxge_start(14): pull msg - "
702 				"len %d pkt_len %d ngathers %d",
703 				len, pkt_len, ngathers));
704 			/* Pull all message blocks from b_cont */
705 			if (is_lso) {
706 				mp = nmp_lso_save;
707 				goto nxge_start_fail_lso;
708 			}
709 			if ((msgpullup(mp, -1)) == NULL) {
710 				goto nxge_start_fail2;
711 			}
712 			goto nxge_start_fail2;
713 		}
714 	} /* while (nmp) */
715 
716 	tx_msg_p->tx_message = mp;
717 	tx_desc_p = &tx_desc_ring_vp[sop_index];
718 #if defined(__i386)
719 	npi_desc_handle.regp = (uint32_t)tx_desc_p;
720 #else
721 	npi_desc_handle.regp = (uint64_t)tx_desc_p;
722 #endif
723 
724 	pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
725 	pkthdrp->reserved = 0;
726 	hdrp->value = 0;
727 	(void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
728 		(pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp);
729 
730 	if (pkt_len > NXGE_MTU_DEFAULT_MAX) {
731 		tdc_stats->tx_jumbo_pkts++;
732 	}
733 
734 	min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2));
735 	if (pkt_len < min_len) {
736 		/* Assume we use bcopy to premapped buffers */
737 		kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
738 		NXGE_DEBUG_MSG((NULL, TX_CTL,
739 			"==> nxge_start(14-1): < (msg_min + 16)"
740 			"len %d pkt_len %d min_len %d bzero %d ngathers %d",
741 			len, pkt_len, min_len, (min_len - pkt_len), ngathers));
742 		bzero((kaddr + pkt_len), (min_len - pkt_len));
743 		pkt_len = tx_msg_p->tx_msg_size = min_len;
744 
745 		sop_tx_desc_p->bits.hdw.tr_len = min_len;
746 
747 		NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
748 		tx_desc_p->value = sop_tx_desc_p->value;
749 
750 		NXGE_DEBUG_MSG((NULL, TX_CTL,
751 			"==> nxge_start(14-2): < msg_min - "
752 			"len %d pkt_len %d min_len %d ngathers %d",
753 			len, pkt_len, min_len, ngathers));
754 	}
755 
756 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ",
757 		cksum_flags));
758 	if (cksum_flags & HCK_PARTIALCKSUM) {
759 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
760 			"==> nxge_start: cksum_flags 0x%x (partial checksum) ",
761 			cksum_flags));
762 		cksum_on = B_TRUE;
763 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
764 			"==> nxge_start: from IP cksum_flags 0x%x "
765 			"(partial checksum) "
766 			"start_offset %d stuff_offset %d",
767 			cksum_flags, start_offset, stuff_offset));
768 		tmp_len = (uint64_t)(start_offset >> 1);
769 		hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT);
770 		tmp_len = (uint64_t)(stuff_offset >> 1);
771 		hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT);
772 
773 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
774 			"==> nxge_start: from IP cksum_flags 0x%x "
775 			"(partial checksum) "
776 			"after SHIFT start_offset %d stuff_offset %d",
777 			cksum_flags, start_offset, stuff_offset));
778 	}
779 	{
780 		uint64_t	tmp_len;
781 
782 		/* pkt_len already includes 16 + paddings!! */
783 		/* Update the control header length */
784 		tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
785 		tmp_len = hdrp->value |
786 			(tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
787 
788 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
789 			"==> nxge_start(15_x1): setting SOP "
790 			"tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
791 			"0x%llx hdrp->value 0x%llx",
792 			tot_xfer_len, tot_xfer_len, pkt_len,
793 			tmp_len, hdrp->value));
794 #if defined(_BIG_ENDIAN)
795 		hdrp->value = ddi_swap64(tmp_len);
796 #else
797 		hdrp->value = tmp_len;
798 #endif
799 		NXGE_DEBUG_MSG((nxgep,
800 			TX_CTL, "==> nxge_start(15_x2): setting SOP "
801 			"after SWAP: tot_xfer_len 0x%llx pkt_len %d "
802 			"tmp_len 0x%llx hdrp->value 0x%llx",
803 			tot_xfer_len, pkt_len,
804 			tmp_len, hdrp->value));
805 	}
806 
807 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP "
808 		"wr_index %d "
809 		"tot_xfer_len (%d) pkt_len %d npads %d",
810 		sop_index,
811 		tot_xfer_len, pkt_len,
812 		npads));
813 
814 	sop_tx_desc_p->bits.hdw.sop = 1;
815 	sop_tx_desc_p->bits.hdw.mark = mark_mode;
816 	sop_tx_desc_p->bits.hdw.num_ptr = ngathers;
817 
818 	NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
819 
820 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done"));
821 
822 #ifdef NXGE_DEBUG
823 	npi_desc_handle.nxgep = nxgep;
824 	npi_desc_handle.function.function = nxgep->function_num;
825 	npi_desc_handle.function.instance = nxgep->instance;
826 
827 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
828 		"\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
829 		save_desc_p->value,
830 		sad,
831 		save_desc_p->bits.hdw.tr_len,
832 		xfer_len,
833 		save_desc_p->bits.hdw.num_ptr,
834 		save_desc_p->bits.hdw.mark,
835 		save_desc_p->bits.hdw.sop));
836 	(void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index);
837 
838 	dump_len = (pkt_len > 128) ? 128: pkt_len;
839 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
840 		"==> nxge_start: dump packets(17) (after sop set, len "
841 		" (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
842 		"ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
843 		(char *)hdrp,
844 		nxge_dump_packet((char *)hdrp, dump_len)));
845 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
846 		"==> nxge_start(18): TX desc sync: sop_index %d",
847 			sop_index));
848 #endif
849 
850 	if ((ngathers == 1) || tx_ring_p->wr_index < i) {
851 		(void) ddi_dma_sync(tx_desc_dma_handle,
852 			sop_index * sizeof (tx_desc_t),
853 			ngathers * sizeof (tx_desc_t),
854 			DDI_DMA_SYNC_FORDEV);
855 
856 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 "
857 			"cs_off = 0x%02X cs_s_off = 0x%02X "
858 			"pkt_len %d ngathers %d sop_index %d\n",
859 			stuff_offset, start_offset,
860 			pkt_len, ngathers, sop_index));
861 	} else { /* more than one descriptor and wrap around */
862 		uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
863 		(void) ddi_dma_sync(tx_desc_dma_handle,
864 			sop_index * sizeof (tx_desc_t),
865 			nsdescs * sizeof (tx_desc_t),
866 			DDI_DMA_SYNC_FORDEV);
867 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 "
868 			"cs_off = 0x%02X cs_s_off = 0x%02X "
869 			"pkt_len %d ngathers %d sop_index %d\n",
870 			stuff_offset, start_offset,
871 				pkt_len, ngathers, sop_index));
872 
873 		(void) ddi_dma_sync(tx_desc_dma_handle,
874 			0,
875 			(ngathers - nsdescs) * sizeof (tx_desc_t),
876 			DDI_DMA_SYNC_FORDEV);
877 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 "
878 			"cs_off = 0x%02X cs_s_off = 0x%02X "
879 			"pkt_len %d ngathers %d sop_index %d\n",
880 			stuff_offset, start_offset,
881 			pkt_len, ngathers, sop_index));
882 	}
883 
884 	tail_index = tx_ring_p->wr_index;
885 	tail_wrap = tx_ring_p->wr_index_wrap;
886 
887 	tx_ring_p->wr_index = i;
888 	if (tx_ring_p->wr_index <= tail_index) {
889 		tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ?
890 						B_FALSE : B_TRUE);
891 	}
892 
893 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: "
894 		"channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
895 		tx_ring_p->tdc,
896 		tx_ring_p->wr_index,
897 		tx_ring_p->wr_index_wrap,
898 		ngathers,
899 		tx_ring_p->descs_pending));
900 
901 	if (is_lso) {
902 		lso_ngathers += ngathers;
903 		if (mp_chain != NULL) {
904 			mp = mp_chain;
905 			mp_chain = mp_chain->b_next;
906 			mp->b_next = NULL;
907 			if (nxge_lso_kick_cnt == lso_ngathers) {
908 				tx_ring_p->descs_pending += lso_ngathers;
909 				{
910 					tx_ring_kick_t		kick;
911 
912 					kick.value = 0;
913 					kick.bits.ldw.wrap =
914 					    tx_ring_p->wr_index_wrap;
915 					kick.bits.ldw.tail =
916 					    (uint16_t)tx_ring_p->wr_index;
917 
918 					/* Kick the Transmit kick register */
919 					TXDMA_REG_WRITE64(
920 					    NXGE_DEV_NPI_HANDLE(nxgep),
921 					    TX_RING_KICK_REG,
922 					    (uint8_t)tx_ring_p->tdc,
923 					    kick.value);
924 					tdc_stats->tx_starts++;
925 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
926 					    "==> nxge_start: more LSO: "
927 					    "LSO_CNT %d",
928 					    lso_gathers));
929 				}
930 				lso_ngathers = 0;
931 				ngathers = 0;
932 				cur_index_lso = sop_index = tx_ring_p->wr_index;
933 				lso_tail_wrap = tx_ring_p->wr_index_wrap;
934 			}
935 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
936 			    "==> nxge_start: lso again: "
937 			    "lso_gathers %d ngathers %d cur_index_lso %d "
938 			    "wr_index %d sop_index %d",
939 			    lso_ngathers, ngathers, cur_index_lso,
940 			    tx_ring_p->wr_index, sop_index));
941 
942 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
943 			    "==> nxge_start: next : count %d",
944 			    lso_gathers));
945 			lso_again = B_TRUE;
946 			goto start_again;
947 		}
948 		ngathers = lso_ngathers;
949 	}
950 
951 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: "));
952 
953 	{
954 		tx_ring_kick_t		kick;
955 
956 		kick.value = 0;
957 		kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap;
958 		kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index;
959 
960 		/* Kick start the Transmit kick register */
961 		TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep),
962 			TX_RING_KICK_REG,
963 			(uint8_t)tx_ring_p->tdc,
964 			kick.value);
965 	}
966 
967 	tx_ring_p->descs_pending += ngathers;
968 	tdc_stats->tx_starts++;
969 
970 	MUTEX_EXIT(&tx_ring_p->lock);
971 
972 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
973 
974 	return (status);
975 
976 nxge_start_fail_lso:
977 	status = 0;
978 	good_packet = B_FALSE;
979 	if (mp != NULL) {
980 		freemsg(mp);
981 	}
982 	if (mp_chain != NULL) {
983 		freemsg(mp_chain);
984 	}
985 	if (!lso_again && !ngathers) {
986 		MUTEX_EXIT(&tx_ring_p->lock);
987 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
988 		    "==> nxge_start: lso exit (nothing changed)"));
989 		goto nxge_start_fail1;
990 	}
991 
992 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
993 	    "==> nxge_start (channel %d): before lso "
994 	    "lso_gathers %d ngathers %d cur_index_lso %d "
995 	    "wr_index %d sop_index %d lso_again %d",
996 	    tx_ring_p->tdc,
997 	    lso_ngathers, ngathers, cur_index_lso,
998 	    tx_ring_p->wr_index, sop_index, lso_again));
999 
1000 	if (lso_again) {
1001 		lso_ngathers += ngathers;
1002 		ngathers = lso_ngathers;
1003 		sop_index = cur_index_lso;
1004 		tx_ring_p->wr_index = sop_index;
1005 		tx_ring_p->wr_index_wrap = lso_tail_wrap;
1006 	}
1007 
1008 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
1009 	    "==> nxge_start (channel %d): after lso "
1010 	    "lso_gathers %d ngathers %d cur_index_lso %d "
1011 	    "wr_index %d sop_index %d lso_again %d",
1012 	    tx_ring_p->tdc,
1013 	    lso_ngathers, ngathers, cur_index_lso,
1014 	    tx_ring_p->wr_index, sop_index, lso_again));
1015 
1016 nxge_start_fail2:
1017 	if (good_packet == B_FALSE) {
1018 		cur_index = sop_index;
1019 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up"));
1020 		for (i = 0; i < ngathers; i++) {
1021 			tx_desc_p = &tx_desc_ring_vp[cur_index];
1022 #if defined(__i386)
1023 			npi_handle.regp = (uint32_t)tx_desc_p;
1024 #else
1025 			npi_handle.regp = (uint64_t)tx_desc_p;
1026 #endif
1027 			tx_msg_p = &tx_msg_ring[cur_index];
1028 			(void) npi_txdma_desc_set_zero(npi_handle, 1);
1029 			if (tx_msg_p->flags.dma_type == USE_DVMA) {
1030 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
1031 				    "tx_desc_p = %X index = %d",
1032 				    tx_desc_p, tx_ring_p->rd_index));
1033 				(void) dvma_unload(tx_msg_p->dvma_handle,
1034 				    0, -1);
1035 				tx_msg_p->dvma_handle = NULL;
1036 				if (tx_ring_p->dvma_wr_index ==
1037 				    tx_ring_p->dvma_wrap_mask)
1038 					tx_ring_p->dvma_wr_index = 0;
1039 				else
1040 					tx_ring_p->dvma_wr_index++;
1041 				tx_ring_p->dvma_pending--;
1042 			} else if (tx_msg_p->flags.dma_type == USE_DMA) {
1043 				if (ddi_dma_unbind_handle(
1044 				    tx_msg_p->dma_handle)) {
1045 					cmn_err(CE_WARN, "!nxge_start: "
1046 					    "ddi_dma_unbind_handle failed");
1047 				}
1048 			}
1049 			tx_msg_p->flags.dma_type = USE_NONE;
1050 			cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1,
1051 				tx_ring_p->tx_wrap_mask);
1052 
1053 		}
1054 
1055 		nxgep->resched_needed = B_TRUE;
1056 	}
1057 
1058 	MUTEX_EXIT(&tx_ring_p->lock);
1059 
1060 nxge_start_fail1:
1061 	/* Add FMA to check the access handle nxge_hregh */
1062 
1063 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
1064 
1065 	return (status);
1066 }
1067 
1068 int
1069 nxge_serial_tx(mblk_t *mp, void *arg)
1070 {
1071 	p_tx_ring_t		tx_ring_p = (p_tx_ring_t)arg;
1072 	p_nxge_t		nxgep = tx_ring_p->nxgep;
1073 
1074 	return (nxge_start(nxgep, tx_ring_p, mp));
1075 }
1076 
1077 boolean_t
1078 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp)
1079 {
1080 	p_tx_ring_t 		*tx_rings;
1081 	uint8_t			ring_index;
1082 	p_tx_ring_t		tx_ring_p;
1083 
1084 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send"));
1085 
1086 	ASSERT(mp->b_next == NULL);
1087 
1088 	ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp);
1089 	tx_rings = nxgep->tx_rings->rings;
1090 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p",
1091 		tx_rings));
1092 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d "
1093 		"ring_index %d", nxgep->max_tdcs, ring_index));
1094 
1095 	switch (nxge_tx_scheme) {
1096 	case NXGE_USE_START:
1097 		if (nxge_start(nxgep, tx_rings[ring_index], mp)) {
1098 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed "
1099 				"ring index %d", ring_index));
1100 			return (B_FALSE);
1101 		}
1102 		break;
1103 
1104 	case NXGE_USE_SERIAL:
1105 	default:
1106 		tx_ring_p = tx_rings[ring_index];
1107 		nxge_serialize_enter(tx_ring_p->serial, mp);
1108 		break;
1109 	}
1110 
1111 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d",
1112 		ring_index));
1113 
1114 	return (B_TRUE);
1115 }
1116 
1117 /*
1118  * nxge_m_tx() - send a chain of packets
1119  */
1120 mblk_t *
1121 nxge_m_tx(void *arg, mblk_t *mp)
1122 {
1123 	p_nxge_t 		nxgep = (p_nxge_t)arg;
1124 	mblk_t 			*next;
1125 	mac_tx_hint_t		hint;
1126 
1127 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
1128 		NXGE_DEBUG_MSG((nxgep, DDI_CTL,
1129 			"==> nxge_m_tx: hardware not initialized"));
1130 		NXGE_DEBUG_MSG((nxgep, DDI_CTL,
1131 			"<== nxge_m_tx"));
1132 		return (mp);
1133 	}
1134 
1135 	hint.hash =  NULL;
1136 	hint.vid =  0;
1137 	hint.sap =  0;
1138 
1139 	while (mp != NULL) {
1140 		next = mp->b_next;
1141 		mp->b_next = NULL;
1142 
1143 		/*
1144 		 * Until Nemo tx resource works, the mac driver
1145 		 * does the load balancing based on TCP port,
1146 		 * or CPU. For debugging, we use a system
1147 		 * configurable parameter.
1148 		 */
1149 		if (!nxge_send(nxgep, mp, &hint)) {
1150 			mp->b_next = next;
1151 			break;
1152 		}
1153 
1154 		mp = next;
1155 
1156 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1157 		    "==> nxge_m_tx: (go back to loop) mp $%p next $%p",
1158 		    mp, next));
1159 	}
1160 
1161 	return (mp);
1162 }
1163 
1164 int
1165 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp)
1166 {
1167 	uint8_t 		ring_index = 0;
1168 	uint8_t 		*tcp_port;
1169 	p_mblk_t 		nmp;
1170 	size_t 			mblk_len;
1171 	size_t 			iph_len;
1172 	size_t 			hdrs_size;
1173 	uint8_t			hdrs_buf[sizeof (struct  ether_header) +
1174 					IP_MAX_HDR_LENGTH + sizeof (uint32_t)];
1175 				/*
1176 				 * allocate space big enough to cover
1177 				 * the max ip header length and the first
1178 				 * 4 bytes of the TCP/IP header.
1179 				 */
1180 
1181 	boolean_t		qos = B_FALSE;
1182 
1183 	NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring"));
1184 
1185 	if (hp->vid) {
1186 		qos = B_TRUE;
1187 	}
1188 	switch (nxge_tx_lb_policy) {
1189 	case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */
1190 	default:
1191 		tcp_port = mp->b_rptr;
1192 		if (!nxge_no_tx_lb && !qos &&
1193 			(ntohs(((p_ether_header_t)tcp_port)->ether_type)
1194 				== ETHERTYPE_IP)) {
1195 			nmp = mp;
1196 			mblk_len = MBLKL(nmp);
1197 			tcp_port = NULL;
1198 			if (mblk_len > sizeof (struct ether_header) +
1199 					sizeof (uint8_t)) {
1200 				tcp_port = nmp->b_rptr +
1201 					sizeof (struct ether_header);
1202 				mblk_len -= sizeof (struct ether_header);
1203 				iph_len = ((*tcp_port) & 0x0f) << 2;
1204 				if (mblk_len > (iph_len + sizeof (uint32_t))) {
1205 					tcp_port = nmp->b_rptr;
1206 				} else {
1207 					tcp_port = NULL;
1208 				}
1209 			}
1210 			if (tcp_port == NULL) {
1211 				hdrs_size = 0;
1212 				((p_ether_header_t)hdrs_buf)->ether_type = 0;
1213 				while ((nmp) && (hdrs_size <
1214 						sizeof (hdrs_buf))) {
1215 					mblk_len = MBLKL(nmp);
1216 					if (mblk_len >=
1217 						(sizeof (hdrs_buf) - hdrs_size))
1218 						mblk_len = sizeof (hdrs_buf) -
1219 							hdrs_size;
1220 					bcopy(nmp->b_rptr,
1221 						&hdrs_buf[hdrs_size], mblk_len);
1222 					hdrs_size += mblk_len;
1223 					nmp = nmp->b_cont;
1224 				}
1225 				tcp_port = hdrs_buf;
1226 			}
1227 			tcp_port += sizeof (ether_header_t);
1228 			if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) {
1229 				switch (tcp_port[9]) {
1230 				case IPPROTO_TCP:
1231 				case IPPROTO_UDP:
1232 				case IPPROTO_ESP:
1233 					tcp_port += ((*tcp_port) & 0x0f) << 2;
1234 					ring_index =
1235 					    ((tcp_port[0] ^
1236 					    tcp_port[1] ^
1237 					    tcp_port[2] ^
1238 					    tcp_port[3]) % maxtdcs);
1239 					break;
1240 
1241 				case IPPROTO_AH:
1242 					/* SPI starts at the 4th byte */
1243 					tcp_port += ((*tcp_port) & 0x0f) << 2;
1244 					ring_index =
1245 					    ((tcp_port[4] ^
1246 					    tcp_port[5] ^
1247 					    tcp_port[6] ^
1248 					    tcp_port[7]) % maxtdcs);
1249 					break;
1250 
1251 				default:
1252 					ring_index = tcp_port[19] % maxtdcs;
1253 					break;
1254 				}
1255 			} else { /* fragmented packet */
1256 				ring_index = tcp_port[19] % maxtdcs;
1257 			}
1258 		} else {
1259 			ring_index = mp->b_band % maxtdcs;
1260 		}
1261 		break;
1262 
1263 	case NXGE_TX_LB_HASH:
1264 		if (hp->hash) {
1265 #if defined(__i386)
1266 			ring_index = ((uint32_t)(hp->hash) % maxtdcs);
1267 #else
1268 			ring_index = ((uint64_t)(hp->hash) % maxtdcs);
1269 #endif
1270 		} else {
1271 			ring_index = mp->b_band % maxtdcs;
1272 		}
1273 		break;
1274 
1275 	case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */
1276 		tcp_port = mp->b_rptr;
1277 		ring_index = tcp_port[5] % maxtdcs;
1278 		break;
1279 	}
1280 
1281 	NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring"));
1282 
1283 	return (ring_index);
1284 }
1285 
1286 uint_t
1287 nxge_reschedule(caddr_t arg)
1288 {
1289 	p_nxge_t nxgep;
1290 
1291 	nxgep = (p_nxge_t)arg;
1292 
1293 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule"));
1294 
1295 	if (nxgep->nxge_mac_state == NXGE_MAC_STARTED &&
1296 			nxgep->resched_needed) {
1297 		mac_tx_update(nxgep->mach);
1298 		nxgep->resched_needed = B_FALSE;
1299 		nxgep->resched_running = B_FALSE;
1300 	}
1301 
1302 	NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule"));
1303 	return (DDI_INTR_CLAIMED);
1304 }
1305 
1306 
1307 /* Software LSO starts here */
1308 static void
1309 nxge_hcksum_retrieve(mblk_t *mp,
1310     uint32_t *start, uint32_t *stuff, uint32_t *end,
1311     uint32_t *value, uint32_t *flags)
1312 {
1313 	if (mp->b_datap->db_type == M_DATA) {
1314 		if (flags != NULL) {
1315 			*flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM |
1316 			    HCK_PARTIALCKSUM | HCK_FULLCKSUM |
1317 			    HCK_FULLCKSUM_OK);
1318 			if ((*flags & (HCK_PARTIALCKSUM |
1319 			    HCK_FULLCKSUM)) != 0) {
1320 				if (value != NULL)
1321 					*value = (uint32_t)DB_CKSUM16(mp);
1322 				if ((*flags & HCK_PARTIALCKSUM) != 0) {
1323 					if (start != NULL)
1324 						*start =
1325 						    (uint32_t)DB_CKSUMSTART(mp);
1326 					if (stuff != NULL)
1327 						*stuff =
1328 						    (uint32_t)DB_CKSUMSTUFF(mp);
1329 					if (end != NULL)
1330 						*end =
1331 						    (uint32_t)DB_CKSUMEND(mp);
1332 				}
1333 			}
1334 		}
1335 	}
1336 }
1337 
1338 static void
1339 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags)
1340 {
1341 	ASSERT(DB_TYPE(mp) == M_DATA);
1342 
1343 	*mss = 0;
1344 	if (flags != NULL) {
1345 		*flags = DB_CKSUMFLAGS(mp) & HW_LSO;
1346 		if ((*flags != 0) && (mss != NULL)) {
1347 			*mss = (uint32_t)DB_LSOMSS(mp);
1348 		}
1349 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1350 		    "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x",
1351 		    *mss, *flags));
1352 	}
1353 
1354 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1355 	    "<== nxge_lso_info_get: mss %d", *mss));
1356 }
1357 
1358 /*
1359  * Do Soft LSO on the oversized packet.
1360  *
1361  * 1. Create a chain of message for headers.
1362  * 2. Fill up header messages with proper information.
1363  * 3. Copy Eithernet, IP, and TCP headers from the original message to
1364  *    each new message with necessary adjustments.
1365  *    * Unchange the ethernet header for DIX frames. (by default)
1366  *    * IP Total Length field is updated to MSS or less(only for the last one).
1367  *    * IP Identification value is incremented by one for each packet.
1368  *    * TCP sequence Number is recalculated according to the payload length.
1369  *    * Set FIN and/or PSH flags for the *last* packet if applied.
1370  *    * TCP partial Checksum
1371  * 4. Update LSO information in the first message header.
1372  * 5. Release the original message header.
1373  */
1374 static mblk_t *
1375 nxge_do_softlso(mblk_t *mp, uint32_t mss)
1376 {
1377 	uint32_t	hckflags;
1378 	int		pktlen;
1379 	int		hdrlen;
1380 	int		segnum;
1381 	int		i;
1382 	struct ether_vlan_header *evh;
1383 	int		ehlen, iphlen, tcphlen;
1384 	struct ip	*oiph, *niph;
1385 	struct tcphdr *otcph, *ntcph;
1386 	int		available, len, left;
1387 	uint16_t	ip_id;
1388 	uint32_t	tcp_seq;
1389 #ifdef __sparc
1390 	uint32_t	tcp_seq_tmp;
1391 #endif
1392 	mblk_t		*datamp;
1393 	uchar_t		*rptr;
1394 	mblk_t		*nmp;
1395 	mblk_t		*cmp;
1396 	mblk_t		*mp_chain;
1397 	boolean_t do_cleanup = B_FALSE;
1398 	t_uscalar_t start_offset = 0;
1399 	t_uscalar_t stuff_offset = 0;
1400 	t_uscalar_t value = 0;
1401 	uint16_t	l4_len;
1402 	ipaddr_t	src, dst;
1403 	uint32_t	cksum, sum, l4cksum;
1404 
1405 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1406 	    "==> nxge_do_softlso"));
1407 	/*
1408 	 * check the length of LSO packet payload and calculate the number of
1409 	 * segments to be generated.
1410 	 */
1411 	pktlen = msgsize(mp);
1412 	evh = (struct ether_vlan_header *)mp->b_rptr;
1413 
1414 	/* VLAN? */
1415 	if (evh->ether_tpid == htons(ETHERTYPE_VLAN))
1416 		ehlen = sizeof (struct ether_vlan_header);
1417 	else
1418 		ehlen = sizeof (struct ether_header);
1419 	oiph = (struct ip *)(mp->b_rptr + ehlen);
1420 	iphlen = oiph->ip_hl * 4;
1421 	otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1422 	tcphlen = otcph->th_off * 4;
1423 
1424 	l4_len = pktlen - ehlen - iphlen;
1425 
1426 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1427 	    "==> nxge_do_softlso: mss %d oiph $%p "
1428 	    "original ip_sum oiph->ip_sum 0x%x "
1429 	    "original tcp_sum otcph->th_sum 0x%x "
1430 	    "oiph->ip_len %d pktlen %d ehlen %d "
1431 	    "l4_len %d (0x%x) ip_len - iphlen %d ",
1432 	    mss,
1433 	    oiph,
1434 	    oiph->ip_sum,
1435 	    otcph->th_sum,
1436 	    ntohs(oiph->ip_len), pktlen,
1437 	    ehlen,
1438 	    l4_len,
1439 	    l4_len,
1440 	    ntohs(oiph->ip_len) - iphlen));
1441 
1442 	/* IPv4 + TCP */
1443 	if (!(oiph->ip_v == IPV4_VERSION)) {
1444 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1445 		    "<== nxge_do_softlso: not IPV4 "
1446 		    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1447 		    ntohs(oiph->ip_len), pktlen, ehlen,
1448 		    tcphlen));
1449 		freemsg(mp);
1450 		return (NULL);
1451 	}
1452 
1453 	if (!(oiph->ip_p == IPPROTO_TCP)) {
1454 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1455 		    "<== nxge_do_softlso: not TCP "
1456 		    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1457 		    ntohs(oiph->ip_len), pktlen, ehlen,
1458 		    tcphlen));
1459 		freemsg(mp);
1460 		return (NULL);
1461 	}
1462 
1463 	if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) {
1464 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1465 		    "<== nxge_do_softlso: len not matched  "
1466 		    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1467 		    ntohs(oiph->ip_len), pktlen, ehlen,
1468 		    tcphlen));
1469 		freemsg(mp);
1470 		return (NULL);
1471 	}
1472 
1473 	otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1474 	tcphlen = otcph->th_off * 4;
1475 
1476 	/* TCP flags can not include URG, RST, or SYN */
1477 	VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0);
1478 
1479 	hdrlen = ehlen + iphlen + tcphlen;
1480 
1481 	VERIFY(MBLKL(mp) >= hdrlen);
1482 
1483 	if (MBLKL(mp) > hdrlen) {
1484 		datamp = mp;
1485 		rptr = mp->b_rptr + hdrlen;
1486 	} else { /* = */
1487 		datamp = mp->b_cont;
1488 		rptr = datamp->b_rptr;
1489 	}
1490 
1491 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1492 	    "nxge_do_softlso: otcph $%p pktlen: %d, "
1493 	    "hdrlen %d ehlen %d iphlen %d tcphlen %d "
1494 	    "mblkl(mp): %d, mblkl(datamp): %d",
1495 	    otcph,
1496 	    pktlen, hdrlen, ehlen, iphlen, tcphlen,
1497 	    (int)MBLKL(mp), (int)MBLKL(datamp)));
1498 
1499 	hckflags = 0;
1500 	nxge_hcksum_retrieve(mp,
1501 	    &start_offset, &stuff_offset, &value, NULL, &hckflags);
1502 
1503 	dst = oiph->ip_dst.s_addr;
1504 	src = oiph->ip_src.s_addr;
1505 
1506 	cksum = (dst >> 16) + (dst & 0xFFFF) +
1507 	    (src >> 16) + (src & 0xFFFF);
1508 	l4cksum = cksum + IP_TCP_CSUM_COMP;
1509 
1510 	sum = l4_len + l4cksum;
1511 	sum = (sum & 0xFFFF) + (sum >> 16);
1512 
1513 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1514 	    "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x "
1515 	    "hckflags 0x%x start_offset %d stuff_offset %d "
1516 	    "value (original) 0x%x th_sum 0x%x "
1517 	    "pktlen %d l4_len %d (0x%x) "
1518 	    "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s",
1519 	    dst, src,
1520 	    (sum & 0xffff), (~sum & 0xffff),
1521 	    hckflags, start_offset, stuff_offset,
1522 	    value, otcph->th_sum,
1523 	    pktlen,
1524 	    l4_len,
1525 	    l4_len,
1526 	    ntohs(oiph->ip_len) - (int)MBLKL(mp),
1527 	    (int)MBLKL(datamp),
1528 	    nxge_dump_packet((char *)evh, 12)));
1529 
1530 	/*
1531 	 * Start to process.
1532 	 */
1533 	available = pktlen - hdrlen;
1534 	segnum = (available - 1) / mss + 1;
1535 
1536 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1537 	    "==> nxge_do_softlso: pktlen %d "
1538 	    "MBLKL(mp): %d, MBLKL(datamp): %d "
1539 	    "available %d mss %d segnum %d",
1540 	    pktlen, (int)MBLKL(mp), (int)MBLKL(datamp),
1541 	    available,
1542 	    mss,
1543 	    segnum));
1544 
1545 	VERIFY(segnum >= 2);
1546 
1547 	/*
1548 	 * Try to pre-allocate all header messages
1549 	 */
1550 	mp_chain = NULL;
1551 	for (i = 0; i < segnum; i++) {
1552 		if ((nmp = allocb(hdrlen, 0)) == NULL) {
1553 			/* Clean up the mp_chain */
1554 			while (mp_chain != NULL) {
1555 				nmp = mp_chain;
1556 				mp_chain = mp_chain->b_next;
1557 				freemsg(nmp);
1558 			}
1559 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1560 			    "<== nxge_do_softlso: "
1561 			    "Could not allocate enough messages for headers!"));
1562 			freemsg(mp);
1563 			return (NULL);
1564 		}
1565 		nmp->b_next = mp_chain;
1566 		mp_chain = nmp;
1567 
1568 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1569 		    "==> nxge_do_softlso: "
1570 		    "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p",
1571 		    mp, nmp, mp_chain, mp_chain->b_next));
1572 	}
1573 
1574 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1575 	    "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p",
1576 	    mp, nmp, mp_chain));
1577 
1578 	/*
1579 	 * Associate payload with new packets
1580 	 */
1581 	cmp = mp_chain;
1582 	left = available;
1583 	while (cmp != NULL) {
1584 		nmp = dupb(datamp);
1585 		if (nmp == NULL) {
1586 			do_cleanup = B_TRUE;
1587 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1588 			    "==>nxge_do_softlso: "
1589 			    "Can not dupb(datamp), have to do clean up"));
1590 			goto cleanup_allocated_msgs;
1591 		}
1592 
1593 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1594 		    "==> nxge_do_softlso: (loop) before mp $%p cmp $%p "
1595 		    "dupb nmp $%p len %d left %d msd %d ",
1596 		    mp, cmp, nmp, len, left, mss));
1597 
1598 		cmp->b_cont = nmp;
1599 		nmp->b_rptr = rptr;
1600 		len = (left < mss) ? left : mss;
1601 		left -= len;
1602 
1603 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1604 		    "==> nxge_do_softlso: (loop) after mp $%p cmp $%p "
1605 		    "dupb nmp $%p len %d left %d mss %d ",
1606 		    mp, cmp, nmp, len, left, mss));
1607 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1608 		    "nxge_do_softlso: before available: %d, "
1609 		    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1610 		    available, left, len, segnum, (int)MBLKL(nmp)));
1611 
1612 		len -= MBLKL(nmp);
1613 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1614 		    "nxge_do_softlso: after available: %d, "
1615 		    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1616 		    available, left, len, segnum, (int)MBLKL(nmp)));
1617 
1618 		while (len > 0) {
1619 			mblk_t *mmp = NULL;
1620 
1621 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1622 			    "nxge_do_softlso: (4) len > 0 available: %d, "
1623 			    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1624 			    available, left, len, segnum, (int)MBLKL(nmp)));
1625 
1626 			if (datamp->b_cont != NULL) {
1627 				datamp = datamp->b_cont;
1628 				rptr = datamp->b_rptr;
1629 				mmp = dupb(datamp);
1630 				if (mmp == NULL) {
1631 					do_cleanup = B_TRUE;
1632 					NXGE_DEBUG_MSG((NULL, TX_CTL,
1633 					    "==> nxge_do_softlso: "
1634 					    "Can not dupb(datamp) (1), :
1635 					    "have to do clean up"));
1636 					NXGE_DEBUG_MSG((NULL, TX_CTL,
1637 					    "==> nxge_do_softlso: "
1638 					    "available: %d, left: %d, "
1639 					    "len: %d, MBLKL(nmp): %d",
1640 					    available, left, len,
1641 					    (int)MBLKL(nmp)));
1642 					goto cleanup_allocated_msgs;
1643 				}
1644 			} else {
1645 				NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1646 				    "==> nxge_do_softlso: "
1647 				    "(1)available: %d, left: %d, "
1648 				    "len: %d, MBLKL(nmp): %d",
1649 				    available, left, len,
1650 				    (int)MBLKL(nmp)));
1651 				cmn_err(CE_PANIC,
1652 				    "==> nxge_do_softlso: "
1653 				    "Pointers must have been corrupted!\n"
1654 				    "datamp: $%p, nmp: $%p, rptr: $%p",
1655 				    (void *)datamp,
1656 				    (void *)nmp,
1657 				    (void *)rptr);
1658 			}
1659 			nmp->b_cont = mmp;
1660 			nmp = mmp;
1661 			len -= MBLKL(nmp);
1662 		}
1663 		if (len < 0) {
1664 			nmp->b_wptr += len;
1665 			rptr = nmp->b_wptr;
1666 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1667 			    "(5) len < 0 (less than 0)"
1668 			    "available: %d, left: %d, len: %d, MBLKL(nmp): %d",
1669 			    available, left, len, (int)MBLKL(nmp)));
1670 
1671 		} else if (len == 0) {
1672 			if (datamp->b_cont != NULL) {
1673 				NXGE_DEBUG_MSG((NULL, TX_CTL,
1674 				    "(5) len == 0"
1675 				    "available: %d, left: %d, len: %d, "
1676 				    "MBLKL(nmp): %d",
1677 				    available, left, len, (int)MBLKL(nmp)));
1678 				datamp = datamp->b_cont;
1679 				rptr = datamp->b_rptr;
1680 			} else {
1681 				NXGE_DEBUG_MSG((NULL, TX_CTL,
1682 				    "(6)available b_cont == NULL : %d, "
1683 				    "left: %d, len: %d, MBLKL(nmp): %d",
1684 				    available, left, len, (int)MBLKL(nmp)));
1685 
1686 				VERIFY(cmp->b_next == NULL);
1687 				VERIFY(left == 0);
1688 				break; /* Done! */
1689 			}
1690 		}
1691 		cmp = cmp->b_next;
1692 
1693 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1694 		    "(7) do_softlso: "
1695 		    "next mp in mp_chain available len != 0 : %d, "
1696 		    "left: %d, len: %d, MBLKL(nmp): %d",
1697 		    available, left, len, (int)MBLKL(nmp)));
1698 	}
1699 
1700 	/*
1701 	 * From now, start to fill up all headers for the first message
1702 	 * Hardware checksum flags need to be updated separately for FULLCKSUM
1703 	 * and PARTIALCKSUM cases. For full checksum, copy the original flags
1704 	 * into every new packet is enough. But for HCK_PARTIALCKSUM, all
1705 	 * required fields need to be updated properly.
1706 	 */
1707 	nmp = mp_chain;
1708 	bcopy(mp->b_rptr, nmp->b_rptr, hdrlen);
1709 	nmp->b_wptr = nmp->b_rptr + hdrlen;
1710 	niph = (struct ip *)(nmp->b_rptr + ehlen);
1711 	niph->ip_len = htons(mss + iphlen + tcphlen);
1712 	ip_id = ntohs(niph->ip_id);
1713 	ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1714 #ifdef __sparc
1715 	bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4);
1716 	tcp_seq = ntohl(tcp_seq_tmp);
1717 #else
1718 	tcp_seq = ntohl(ntcph->th_seq);
1719 #endif
1720 
1721 	ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST);
1722 
1723 	DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1724 	DB_CKSUMSTART(nmp) = start_offset;
1725 	DB_CKSUMSTUFF(nmp) = stuff_offset;
1726 
1727 	/* calculate IP checksum and TCP pseudo header checksum */
1728 	niph->ip_sum = 0;
1729 	niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1730 
1731 	l4_len = mss + tcphlen;
1732 	sum = htons(l4_len) + l4cksum;
1733 	sum = (sum & 0xFFFF) + (sum >> 16);
1734 	ntcph->th_sum = (sum & 0xffff);
1735 
1736 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1737 	    "==> nxge_do_softlso: first mp $%p (mp_chain $%p) "
1738 	    "mss %d pktlen %d l4_len %d (0x%x) "
1739 	    "MBLKL(mp): %d, MBLKL(datamp): %d "
1740 	    "ip_sum 0x%x "
1741 	    "th_sum 0x%x sum 0x%x ) "
1742 	    "dump first ip->tcp %s",
1743 	    nmp, mp_chain,
1744 	    mss,
1745 	    pktlen,
1746 	    l4_len,
1747 	    l4_len,
1748 	    (int)MBLKL(mp), (int)MBLKL(datamp),
1749 	    niph->ip_sum,
1750 	    ntcph->th_sum,
1751 	    sum,
1752 	    nxge_dump_packet((char *)niph, 52)));
1753 
1754 	cmp = nmp;
1755 	while ((nmp = nmp->b_next)->b_next != NULL) {
1756 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1757 		    "==>nxge_do_softlso: middle l4_len %d ", l4_len));
1758 		bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1759 		nmp->b_wptr = nmp->b_rptr + hdrlen;
1760 		niph = (struct ip *)(nmp->b_rptr + ehlen);
1761 		niph->ip_id = htons(++ip_id);
1762 		niph->ip_len = htons(mss + iphlen + tcphlen);
1763 		ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1764 		tcp_seq += mss;
1765 
1766 		ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG);
1767 
1768 #ifdef __sparc
1769 		tcp_seq_tmp = htonl(tcp_seq);
1770 		bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1771 #else
1772 		ntcph->th_seq = htonl(tcp_seq);
1773 #endif
1774 		DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1775 		DB_CKSUMSTART(nmp) = start_offset;
1776 		DB_CKSUMSTUFF(nmp) = stuff_offset;
1777 
1778 		/* calculate IP checksum and TCP pseudo header checksum */
1779 		niph->ip_sum = 0;
1780 		niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1781 		ntcph->th_sum = (sum & 0xffff);
1782 
1783 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1784 		    "==> nxge_do_softlso: middle ip_sum 0x%x "
1785 		    "th_sum 0x%x "
1786 		    " mp $%p (mp_chain $%p) pktlen %d "
1787 		    "MBLKL(mp): %d, MBLKL(datamp): %d ",
1788 		    niph->ip_sum,
1789 		    ntcph->th_sum,
1790 		    nmp, mp_chain,
1791 		    pktlen, (int)MBLKL(mp), (int)MBLKL(datamp)));
1792 	}
1793 
1794 	/* Last segment */
1795 	/*
1796 	 * Set FIN and/or PSH flags if present only in the last packet.
1797 	 * The ip_len could be different from prior packets.
1798 	 */
1799 	bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1800 	nmp->b_wptr = nmp->b_rptr + hdrlen;
1801 	niph = (struct ip *)(nmp->b_rptr + ehlen);
1802 	niph->ip_id = htons(++ip_id);
1803 	niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen);
1804 	ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1805 	tcp_seq += mss;
1806 #ifdef __sparc
1807 	tcp_seq_tmp = htonl(tcp_seq);
1808 	bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1809 #else
1810 	ntcph->th_seq = htonl(tcp_seq);
1811 #endif
1812 	ntcph->th_flags = (otcph->th_flags & ~TH_URG);
1813 
1814 	DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1815 	DB_CKSUMSTART(nmp) = start_offset;
1816 	DB_CKSUMSTUFF(nmp) = stuff_offset;
1817 
1818 	/* calculate IP checksum and TCP pseudo header checksum */
1819 	niph->ip_sum = 0;
1820 	niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1821 
1822 	l4_len = ntohs(niph->ip_len) - iphlen;
1823 	sum = htons(l4_len) + l4cksum;
1824 	sum = (sum & 0xFFFF) + (sum >> 16);
1825 	ntcph->th_sum = (sum & 0xffff);
1826 
1827 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1828 	    "==> nxge_do_softlso: last next "
1829 	    "niph->ip_sum 0x%x "
1830 	    "ntcph->th_sum 0x%x sum 0x%x "
1831 	    "dump last ip->tcp %s "
1832 	    "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) "
1833 	    "l4_len %d (0x%x) "
1834 	    "MBLKL(mp): %d, MBLKL(datamp): %d ",
1835 	    niph->ip_sum,
1836 	    ntcph->th_sum, sum,
1837 	    nxge_dump_packet((char *)niph, 52),
1838 	    cmp, nmp, mp_chain,
1839 	    pktlen, pktlen,
1840 	    l4_len,
1841 	    l4_len,
1842 	    (int)MBLKL(mp), (int)MBLKL(datamp)));
1843 
1844 cleanup_allocated_msgs:
1845 	if (do_cleanup) {
1846 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1847 		    "==> nxge_do_softlso: "
1848 		    "Failed allocating messages, "
1849 		    "have to clean up and fail!"));
1850 		while (mp_chain != NULL) {
1851 			nmp = mp_chain;
1852 			mp_chain = mp_chain->b_next;
1853 			freemsg(nmp);
1854 		}
1855 	}
1856 	/*
1857 	 * We're done here, so just free the original message and return the
1858 	 * new message chain, that could be NULL if failed, back to the caller.
1859 	 */
1860 	freemsg(mp);
1861 
1862 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1863 	    "<== nxge_do_softlso:mp_chain $%p", mp_chain));
1864 	return (mp_chain);
1865 }
1866 
1867 /*
1868  * Will be called before NIC driver do further operation on the message.
1869  * The input message may include LSO information, if so, go to softlso logic
1870  * to eliminate the oversized LSO packet for the incapable underlying h/w.
1871  * The return could be the same non-LSO message or a message chain for LSO case.
1872  *
1873  * The driver needs to call this function per packet and process the whole chain
1874  * if applied.
1875  */
1876 static mblk_t *
1877 nxge_lso_eliminate(mblk_t *mp)
1878 {
1879 	uint32_t lsoflags;
1880 	uint32_t mss;
1881 
1882 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1883 	    "==>nxge_lso_eliminate:"));
1884 	nxge_lso_info_get(mp, &mss, &lsoflags);
1885 
1886 	if (lsoflags & HW_LSO) {
1887 		mblk_t *nmp;
1888 
1889 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1890 		    "==>nxge_lso_eliminate:"
1891 		    "HW_LSO:mss %d mp $%p",
1892 		    mss, mp));
1893 		if ((nmp = nxge_do_softlso(mp, mss)) != NULL) {
1894 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1895 			    "<== nxge_lso_eliminate: "
1896 			    "LSO: nmp not NULL nmp $%p mss %d mp $%p",
1897 			    nmp, mss, mp));
1898 			return (nmp);
1899 		} else {
1900 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1901 			    "<== nxge_lso_eliminate_ "
1902 			    "LSO: failed nmp NULL nmp $%p mss %d mp $%p",
1903 			    nmp, mss, mp));
1904 			return (NULL);
1905 		}
1906 	}
1907 
1908 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1909 	    "<== nxge_lso_eliminate"));
1910 	return (mp);
1911 }
1912 
1913 static uint32_t
1914 nxge_csgen(uint16_t *adr, int len)
1915 {
1916 	int		i, odd;
1917 	uint32_t	sum = 0;
1918 	uint32_t	c = 0;
1919 
1920 	odd = len % 2;
1921 	for (i = 0; i < (len / 2); i++) {
1922 		sum += (adr[i] & 0xffff);
1923 	}
1924 	if (odd) {
1925 		sum += adr[len / 2] & 0xff00;
1926 	}
1927 	while ((c = ((sum & 0xffff0000) >> 16)) != 0) {
1928 		sum &= 0xffff;
1929 		sum += c;
1930 	}
1931 	return (~sum & 0xffff);
1932 }
1933