xref: /titanic_44/usr/src/uts/common/io/hxge/hxge_txdma.c (revision cd3e933325e68e23516a196a8fea7f49b1e497c3)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <hxge_impl.h>
27 #include <hxge_txdma.h>
28 #include <sys/llc1.h>
29 
30 uint32_t hxge_reclaim_pending = TXDMA_RECLAIM_PENDING_DEFAULT;
31 uint32_t hxge_tx_minfree = 64;
32 uint32_t hxge_tx_intr_thres = 0;
33 uint32_t hxge_tx_max_gathers = TX_MAX_GATHER_POINTERS;
34 uint32_t hxge_tx_tiny_pack = 1;
35 uint32_t hxge_tx_use_bcopy = 1;
36 
37 extern uint32_t hxge_tx_ring_size;
38 extern uint32_t hxge_bcopy_thresh;
39 extern uint32_t hxge_dvma_thresh;
40 extern uint32_t hxge_dma_stream_thresh;
41 extern dma_method_t hxge_force_dma;
42 
43 /* Device register access attributes for PIO.  */
44 extern ddi_device_acc_attr_t hxge_dev_reg_acc_attr;
45 
46 /* Device descriptor access attributes for DMA.  */
47 extern ddi_device_acc_attr_t hxge_dev_desc_dma_acc_attr;
48 
49 /* Device buffer access attributes for DMA.  */
50 extern ddi_device_acc_attr_t hxge_dev_buf_dma_acc_attr;
51 extern ddi_dma_attr_t hxge_desc_dma_attr;
52 extern ddi_dma_attr_t hxge_tx_dma_attr;
53 
54 static hxge_status_t hxge_map_txdma(p_hxge_t hxgep);
55 static void hxge_unmap_txdma(p_hxge_t hxgep);
56 static hxge_status_t hxge_txdma_hw_start(p_hxge_t hxgep);
57 static void hxge_txdma_hw_stop(p_hxge_t hxgep);
58 
59 static hxge_status_t hxge_map_txdma_channel(p_hxge_t hxgep, uint16_t channel,
60     p_hxge_dma_common_t *dma_buf_p, p_tx_ring_t *tx_desc_p,
61     uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p,
62     p_tx_mbox_t *tx_mbox_p);
63 static void hxge_unmap_txdma_channel(p_hxge_t hxgep, uint16_t channel,
64     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
65 static hxge_status_t hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep, uint16_t,
66     p_hxge_dma_common_t *, p_tx_ring_t *, uint32_t);
67 static void hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep,
68     p_tx_ring_t tx_ring_p);
69 static void hxge_map_txdma_channel_cfg_ring(p_hxge_t, uint16_t,
70     p_hxge_dma_common_t *, p_tx_ring_t, p_tx_mbox_t *);
71 static void hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,
72     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
73 static hxge_status_t hxge_txdma_start_channel(p_hxge_t hxgep, uint16_t channel,
74     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
75 static hxge_status_t hxge_txdma_stop_channel(p_hxge_t hxgep, uint16_t channel,
76     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
77 static p_tx_ring_t hxge_txdma_get_ring(p_hxge_t hxgep, uint16_t channel);
78 static hxge_status_t hxge_tx_err_evnts(p_hxge_t hxgep, uint_t index,
79     p_hxge_ldv_t ldvp, tdc_stat_t cs);
80 static p_tx_mbox_t hxge_txdma_get_mbox(p_hxge_t hxgep, uint16_t channel);
81 static hxge_status_t hxge_txdma_fatal_err_recover(p_hxge_t hxgep,
82     uint16_t channel, p_tx_ring_t tx_ring_p);
83 static hxge_status_t hxge_tx_port_fatal_err_recover(p_hxge_t hxgep);
84 
85 hxge_status_t
86 hxge_init_txdma_channels(p_hxge_t hxgep)
87 {
88 	hxge_status_t	status = HXGE_OK;
89 	block_reset_t	reset_reg;
90 
91 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_init_txdma_channels"));
92 
93 	/*
94 	 * Reset TDC block from PEU to cleanup any unknown configuration.
95 	 * This may be resulted from previous reboot.
96 	 */
97 	reset_reg.value = 0;
98 	reset_reg.bits.tdc_rst = 1;
99 	HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value);
100 
101 	HXGE_DELAY(1000);
102 
103 	status = hxge_map_txdma(hxgep);
104 	if (status != HXGE_OK) {
105 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
106 		    "<== hxge_init_txdma_channels: status 0x%x", status));
107 		return (status);
108 	}
109 
110 	status = hxge_txdma_hw_start(hxgep);
111 	if (status != HXGE_OK) {
112 		hxge_unmap_txdma(hxgep);
113 		return (status);
114 	}
115 
116 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
117 	    "<== hxge_init_txdma_channels: status 0x%x", status));
118 
119 	return (HXGE_OK);
120 }
121 
122 void
123 hxge_uninit_txdma_channels(p_hxge_t hxgep)
124 {
125 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_uninit_txdma_channels"));
126 
127 	hxge_txdma_hw_stop(hxgep);
128 	hxge_unmap_txdma(hxgep);
129 
130 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_uinit_txdma_channels"));
131 }
132 
133 void
134 hxge_setup_dma_common(p_hxge_dma_common_t dest_p, p_hxge_dma_common_t src_p,
135     uint32_t entries, uint32_t size)
136 {
137 	size_t tsize;
138 	*dest_p = *src_p;
139 	tsize = size * entries;
140 	dest_p->alength = tsize;
141 	dest_p->nblocks = entries;
142 	dest_p->block_size = size;
143 	dest_p->offset += tsize;
144 
145 	src_p->kaddrp = (caddr_t)dest_p->kaddrp + tsize;
146 	src_p->alength -= tsize;
147 	src_p->dma_cookie.dmac_laddress += tsize;
148 	src_p->dma_cookie.dmac_size -= tsize;
149 }
150 
151 hxge_status_t
152 hxge_reset_txdma_channel(p_hxge_t hxgep, uint16_t channel, uint64_t reg_data)
153 {
154 	hpi_status_t	rs = HPI_SUCCESS;
155 	hxge_status_t	status = HXGE_OK;
156 	hpi_handle_t	handle;
157 
158 	HXGE_DEBUG_MSG((hxgep, TX_CTL, " ==> hxge_reset_txdma_channel"));
159 
160 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
161 	if ((reg_data & TDC_TDR_RST_MASK) == TDC_TDR_RST_MASK) {
162 		rs = hpi_txdma_channel_reset(handle, channel);
163 	} else {
164 		rs = hpi_txdma_channel_control(handle, TXDMA_RESET, channel);
165 	}
166 
167 	if (rs != HPI_SUCCESS) {
168 		status = HXGE_ERROR | rs;
169 	}
170 
171 	/*
172 	 * Reset the tail (kick) register to 0. (Hardware will not reset it. Tx
173 	 * overflow fatal error if tail is not set to 0 after reset!
174 	 */
175 	TXDMA_REG_WRITE64(handle, TDC_TDR_KICK, channel, 0);
176 
177 	HXGE_DEBUG_MSG((hxgep, TX_CTL, " <== hxge_reset_txdma_channel"));
178 
179 	return (status);
180 }
181 
182 hxge_status_t
183 hxge_init_txdma_channel_event_mask(p_hxge_t hxgep, uint16_t channel,
184     tdc_int_mask_t *mask_p)
185 {
186 	hpi_handle_t	handle;
187 	hpi_status_t	rs = HPI_SUCCESS;
188 	hxge_status_t	status = HXGE_OK;
189 
190 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
191 	    "<== hxge_init_txdma_channel_event_mask"));
192 
193 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
194 
195 	/*
196 	 * Mask off tx_rng_oflow since it is a false alarm. The driver
197 	 * ensures not over flowing the hardware and check the hardware
198 	 * status.
199 	 */
200 	mask_p->bits.tx_rng_oflow = 1;
201 	rs = hpi_txdma_event_mask(handle, OP_SET, channel, mask_p);
202 	if (rs != HPI_SUCCESS) {
203 		status = HXGE_ERROR | rs;
204 	}
205 
206 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
207 	    "==> hxge_init_txdma_channel_event_mask"));
208 	return (status);
209 }
210 
211 hxge_status_t
212 hxge_enable_txdma_channel(p_hxge_t hxgep,
213     uint16_t channel, p_tx_ring_t tx_desc_p, p_tx_mbox_t mbox_p)
214 {
215 	hpi_handle_t	handle;
216 	hpi_status_t	rs = HPI_SUCCESS;
217 	hxge_status_t	status = HXGE_OK;
218 
219 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_enable_txdma_channel"));
220 
221 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
222 	/*
223 	 * Use configuration data composed at init time. Write to hardware the
224 	 * transmit ring configurations.
225 	 */
226 	rs = hpi_txdma_ring_config(handle, OP_SET, channel,
227 	    (uint64_t *)&(tx_desc_p->tx_ring_cfig.value));
228 
229 	if (rs != HPI_SUCCESS) {
230 		return (HXGE_ERROR | rs);
231 	}
232 
233 	/* Write to hardware the mailbox */
234 	rs = hpi_txdma_mbox_config(handle, OP_SET, channel,
235 	    (uint64_t *)&mbox_p->tx_mbox.dma_cookie.dmac_laddress);
236 
237 	if (rs != HPI_SUCCESS) {
238 		return (HXGE_ERROR | rs);
239 	}
240 
241 	/* Start the DMA engine. */
242 	rs = hpi_txdma_channel_init_enable(handle, channel);
243 	if (rs != HPI_SUCCESS) {
244 		return (HXGE_ERROR | rs);
245 	}
246 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_enable_txdma_channel"));
247 	return (status);
248 }
249 
250 void
251 hxge_fill_tx_hdr(p_mblk_t mp, boolean_t fill_len, boolean_t l4_cksum,
252     int pkt_len, uint8_t npads, p_tx_pkt_hdr_all_t pkthdrp)
253 {
254 	p_tx_pkt_header_t	hdrp;
255 	p_mblk_t		nmp;
256 	uint64_t		tmp;
257 	size_t			mblk_len;
258 	size_t			iph_len;
259 	size_t			hdrs_size;
260 	uint8_t			*ip_buf;
261 	uint16_t		eth_type;
262 	uint8_t			ipproto;
263 	boolean_t		is_vlan = B_FALSE;
264 	size_t			eth_hdr_size;
265 	uint8_t hdrs_buf[sizeof (struct ether_header) + 64 + sizeof (uint32_t)];
266 
267 	HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: mp $%p", mp));
268 
269 	/*
270 	 * Caller should zero out the headers first.
271 	 */
272 	hdrp = (p_tx_pkt_header_t)&pkthdrp->pkthdr;
273 
274 	if (fill_len) {
275 		HXGE_DEBUG_MSG((NULL, TX_CTL,
276 		    "==> hxge_fill_tx_hdr: pkt_len %d npads %d",
277 		    pkt_len, npads));
278 		tmp = (uint64_t)pkt_len;
279 		hdrp->value |= (tmp << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
280 
281 		goto fill_tx_header_done;
282 	}
283 	tmp = (uint64_t)npads;
284 	hdrp->value |= (tmp << TX_PKT_HEADER_PAD_SHIFT);
285 
286 	/*
287 	 * mp is the original data packet (does not include the Neptune
288 	 * transmit header).
289 	 */
290 	nmp = mp;
291 	mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
292 	HXGE_DEBUG_MSG((NULL, TX_CTL,
293 	    "==> hxge_fill_tx_hdr: mp $%p b_rptr $%p len %d",
294 	    mp, nmp->b_rptr, mblk_len));
295 	ip_buf = NULL;
296 	bcopy(nmp->b_rptr, &hdrs_buf[0], sizeof (struct ether_vlan_header));
297 	eth_type = ntohs(((p_ether_header_t)hdrs_buf)->ether_type);
298 	HXGE_DEBUG_MSG((NULL, TX_CTL,
299 	    "==> : hxge_fill_tx_hdr: (value 0x%llx) ether type 0x%x",
300 	    eth_type, hdrp->value));
301 
302 	if (eth_type < ETHERMTU) {
303 		tmp = 1ull;
304 		hdrp->value |= (tmp << TX_PKT_HEADER_LLC_SHIFT);
305 		HXGE_DEBUG_MSG((NULL, TX_CTL,
306 		    "==> hxge_tx_pkt_hdr_init: LLC value 0x%llx", hdrp->value));
307 		if (*(hdrs_buf + sizeof (struct ether_header)) ==
308 		    LLC_SNAP_SAP) {
309 			eth_type = ntohs(*((uint16_t *)(hdrs_buf +
310 			    sizeof (struct ether_header) + 6)));
311 			HXGE_DEBUG_MSG((NULL, TX_CTL,
312 			    "==> hxge_tx_pkt_hdr_init: LLC ether type 0x%x",
313 			    eth_type));
314 		} else {
315 			goto fill_tx_header_done;
316 		}
317 	} else if (eth_type == VLAN_ETHERTYPE) {
318 		tmp = 1ull;
319 		hdrp->value |= (tmp << TX_PKT_HEADER_VLAN__SHIFT);
320 
321 		eth_type = ntohs(((struct ether_vlan_header *)
322 		    hdrs_buf)->ether_type);
323 		is_vlan = B_TRUE;
324 		HXGE_DEBUG_MSG((NULL, TX_CTL,
325 		    "==> hxge_tx_pkt_hdr_init: VLAN value 0x%llx",
326 		    hdrp->value));
327 	}
328 	if (!is_vlan) {
329 		eth_hdr_size = sizeof (struct ether_header);
330 	} else {
331 		eth_hdr_size = sizeof (struct ether_vlan_header);
332 	}
333 
334 	switch (eth_type) {
335 	case ETHERTYPE_IP:
336 		if (mblk_len > eth_hdr_size + sizeof (uint8_t)) {
337 			ip_buf = nmp->b_rptr + eth_hdr_size;
338 			mblk_len -= eth_hdr_size;
339 			iph_len = ((*ip_buf) & 0x0f);
340 			if (mblk_len > (iph_len + sizeof (uint32_t))) {
341 				ip_buf = nmp->b_rptr;
342 				ip_buf += eth_hdr_size;
343 			} else {
344 				ip_buf = NULL;
345 			}
346 		}
347 		if (ip_buf == NULL) {
348 			hdrs_size = 0;
349 			((p_ether_header_t)hdrs_buf)->ether_type = 0;
350 			while ((nmp) && (hdrs_size < sizeof (hdrs_buf))) {
351 				mblk_len = (size_t)nmp->b_wptr -
352 				    (size_t)nmp->b_rptr;
353 				if (mblk_len >=
354 				    (sizeof (hdrs_buf) - hdrs_size))
355 					mblk_len = sizeof (hdrs_buf) -
356 					    hdrs_size;
357 				bcopy(nmp->b_rptr,
358 				    &hdrs_buf[hdrs_size], mblk_len);
359 				hdrs_size += mblk_len;
360 				nmp = nmp->b_cont;
361 			}
362 			ip_buf = hdrs_buf;
363 			ip_buf += eth_hdr_size;
364 			iph_len = ((*ip_buf) & 0x0f);
365 		}
366 		ipproto = ip_buf[9];
367 
368 		tmp = (uint64_t)iph_len;
369 		hdrp->value |= (tmp << TX_PKT_HEADER_IHL_SHIFT);
370 		tmp = (uint64_t)(eth_hdr_size >> 1);
371 		hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
372 
373 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: IPv4 "
374 		    " iph_len %d l3start %d eth_hdr_size %d proto 0x%x"
375 		    "tmp 0x%x", iph_len, hdrp->bits.l3start, eth_hdr_size,
376 		    ipproto, tmp));
377 		HXGE_DEBUG_MSG((NULL, TX_CTL,
378 		    "==> hxge_tx_pkt_hdr_init: IP value 0x%llx", hdrp->value));
379 		break;
380 
381 	case ETHERTYPE_IPV6:
382 		hdrs_size = 0;
383 		((p_ether_header_t)hdrs_buf)->ether_type = 0;
384 		while ((nmp) && (hdrs_size < sizeof (hdrs_buf))) {
385 			mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
386 			if (mblk_len >= (sizeof (hdrs_buf) - hdrs_size))
387 				mblk_len = sizeof (hdrs_buf) - hdrs_size;
388 			bcopy(nmp->b_rptr, &hdrs_buf[hdrs_size], mblk_len);
389 			hdrs_size += mblk_len;
390 			nmp = nmp->b_cont;
391 		}
392 		ip_buf = hdrs_buf;
393 		ip_buf += eth_hdr_size;
394 
395 		tmp = 1ull;
396 		hdrp->value |= (tmp << TX_PKT_HEADER_IP_VER_SHIFT);
397 
398 		tmp = (eth_hdr_size >> 1);
399 		hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
400 
401 		/* byte 6 is the next header protocol */
402 		ipproto = ip_buf[6];
403 
404 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: IPv6 "
405 		    " iph_len %d l3start %d eth_hdr_size %d proto 0x%x",
406 		    iph_len, hdrp->bits.l3start, eth_hdr_size, ipproto));
407 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_hdr_init: IPv6 "
408 		    "value 0x%llx", hdrp->value));
409 		break;
410 
411 	default:
412 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: non-IP"));
413 		goto fill_tx_header_done;
414 	}
415 
416 	switch (ipproto) {
417 	case IPPROTO_TCP:
418 		HXGE_DEBUG_MSG((NULL, TX_CTL,
419 		    "==> hxge_fill_tx_hdr: TCP (cksum flag %d)", l4_cksum));
420 		if (l4_cksum) {
421 			tmp = 1ull;
422 			hdrp->value |= (tmp << TX_PKT_HEADER_PKT_TYPE_SHIFT);
423 			HXGE_DEBUG_MSG((NULL, TX_CTL,
424 			    "==> hxge_tx_pkt_hdr_init: TCP CKSUM"
425 			    "value 0x%llx", hdrp->value));
426 		}
427 		HXGE_DEBUG_MSG((NULL, TX_CTL,
428 		    "==> hxge_tx_pkt_hdr_init: TCP value 0x%llx", hdrp->value));
429 		break;
430 
431 	case IPPROTO_UDP:
432 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: UDP"));
433 		if (l4_cksum) {
434 			tmp = 0x2ull;
435 			hdrp->value |= (tmp << TX_PKT_HEADER_PKT_TYPE_SHIFT);
436 		}
437 		HXGE_DEBUG_MSG((NULL, TX_CTL,
438 		    "==> hxge_tx_pkt_hdr_init: UDP value 0x%llx",
439 		    hdrp->value));
440 		break;
441 
442 	default:
443 		goto fill_tx_header_done;
444 	}
445 
446 fill_tx_header_done:
447 	HXGE_DEBUG_MSG((NULL, TX_CTL,
448 	    "==> hxge_fill_tx_hdr: pkt_len %d npads %d value 0x%llx",
449 	    pkt_len, npads, hdrp->value));
450 	HXGE_DEBUG_MSG((NULL, TX_CTL, "<== hxge_fill_tx_hdr"));
451 }
452 
453 /*ARGSUSED*/
454 p_mblk_t
455 hxge_tx_pkt_header_reserve(p_mblk_t mp, uint8_t *npads)
456 {
457 	p_mblk_t newmp = NULL;
458 
459 	if ((newmp = allocb(TX_PKT_HEADER_SIZE, BPRI_MED)) == NULL) {
460 		HXGE_DEBUG_MSG((NULL, TX_CTL,
461 		    "<== hxge_tx_pkt_header_reserve: allocb failed"));
462 		return (NULL);
463 	}
464 	HXGE_DEBUG_MSG((NULL, TX_CTL,
465 	    "==> hxge_tx_pkt_header_reserve: get new mp"));
466 	DB_TYPE(newmp) = M_DATA;
467 	newmp->b_rptr = newmp->b_wptr = DB_LIM(newmp);
468 	linkb(newmp, mp);
469 	newmp->b_rptr -= TX_PKT_HEADER_SIZE;
470 
471 	HXGE_DEBUG_MSG((NULL, TX_CTL,
472 	    "==>hxge_tx_pkt_header_reserve: b_rptr $%p b_wptr $%p",
473 	    newmp->b_rptr, newmp->b_wptr));
474 	HXGE_DEBUG_MSG((NULL, TX_CTL,
475 	    "<== hxge_tx_pkt_header_reserve: use new mp"));
476 	return (newmp);
477 }
478 
479 int
480 hxge_tx_pkt_nmblocks(p_mblk_t mp, int *tot_xfer_len_p)
481 {
482 	uint_t		nmblks;
483 	ssize_t		len;
484 	uint_t		pkt_len;
485 	p_mblk_t	nmp, bmp, tmp;
486 	uint8_t		*b_wptr;
487 
488 	HXGE_DEBUG_MSG((NULL, TX_CTL,
489 	    "==> hxge_tx_pkt_nmblocks: mp $%p rptr $%p wptr $%p len %d",
490 	    mp, mp->b_rptr, mp->b_wptr, MBLKL(mp)));
491 
492 	nmp = mp;
493 	bmp = mp;
494 	nmblks = 0;
495 	pkt_len = 0;
496 	*tot_xfer_len_p = 0;
497 
498 	while (nmp) {
499 		len = MBLKL(nmp);
500 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_nmblocks: "
501 		    "len %d pkt_len %d nmblks %d tot_xfer_len %d",
502 		    len, pkt_len, nmblks, *tot_xfer_len_p));
503 
504 		if (len <= 0) {
505 			bmp = nmp;
506 			nmp = nmp->b_cont;
507 			HXGE_DEBUG_MSG((NULL, TX_CTL,
508 			    "==> hxge_tx_pkt_nmblocks:"
509 			    " len (0) pkt_len %d nmblks %d", pkt_len, nmblks));
510 			continue;
511 		}
512 		*tot_xfer_len_p += len;
513 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_nmblocks: "
514 		    "len %d pkt_len %d nmblks %d tot_xfer_len %d",
515 		    len, pkt_len, nmblks, *tot_xfer_len_p));
516 
517 		if (len < hxge_bcopy_thresh) {
518 			HXGE_DEBUG_MSG((NULL, TX_CTL,
519 			    "==> hxge_tx_pkt_nmblocks: "
520 			    "len %d (< thresh) pkt_len %d nmblks %d",
521 			    len, pkt_len, nmblks));
522 			if (pkt_len == 0)
523 				nmblks++;
524 			pkt_len += len;
525 			if (pkt_len >= hxge_bcopy_thresh) {
526 				pkt_len = 0;
527 				len = 0;
528 				nmp = bmp;
529 			}
530 		} else {
531 			HXGE_DEBUG_MSG((NULL, TX_CTL,
532 			    "==> hxge_tx_pkt_nmblocks: "
533 			    "len %d (> thresh) pkt_len %d nmblks %d",
534 			    len, pkt_len, nmblks));
535 			pkt_len = 0;
536 			nmblks++;
537 			/*
538 			 * Hardware limits the transfer length to 4K. If len is
539 			 * more than 4K, we need to break it up to at most 2
540 			 * more blocks.
541 			 */
542 			if (len > TX_MAX_TRANSFER_LENGTH) {
543 				uint32_t nsegs;
544 
545 				HXGE_DEBUG_MSG((NULL, TX_CTL,
546 				    "==> hxge_tx_pkt_nmblocks: "
547 				    "len %d pkt_len %d nmblks %d nsegs %d",
548 				    len, pkt_len, nmblks, nsegs));
549 				nsegs = 1;
550 				if (len % (TX_MAX_TRANSFER_LENGTH * 2)) {
551 					++nsegs;
552 				}
553 				do {
554 					b_wptr = nmp->b_rptr +
555 					    TX_MAX_TRANSFER_LENGTH;
556 					nmp->b_wptr = b_wptr;
557 					if ((tmp = dupb(nmp)) == NULL) {
558 						return (0);
559 					}
560 					tmp->b_rptr = b_wptr;
561 					tmp->b_wptr = nmp->b_wptr;
562 					tmp->b_cont = nmp->b_cont;
563 					nmp->b_cont = tmp;
564 					nmblks++;
565 					if (--nsegs) {
566 						nmp = tmp;
567 					}
568 				} while (nsegs);
569 				nmp = tmp;
570 			}
571 		}
572 
573 		/*
574 		 * Hardware limits the transmit gather pointers to 15.
575 		 */
576 		if (nmp->b_cont && (nmblks + TX_GATHER_POINTERS_THRESHOLD) >
577 		    TX_MAX_GATHER_POINTERS) {
578 			HXGE_DEBUG_MSG((NULL, TX_CTL,
579 			    "==> hxge_tx_pkt_nmblocks: pull msg - "
580 			    "len %d pkt_len %d nmblks %d",
581 			    len, pkt_len, nmblks));
582 			/* Pull all message blocks from b_cont */
583 			if ((tmp = msgpullup(nmp->b_cont, -1)) == NULL) {
584 				return (0);
585 			}
586 			freemsg(nmp->b_cont);
587 			nmp->b_cont = tmp;
588 			pkt_len = 0;
589 		}
590 		bmp = nmp;
591 		nmp = nmp->b_cont;
592 	}
593 
594 	HXGE_DEBUG_MSG((NULL, TX_CTL,
595 	    "<== hxge_tx_pkt_nmblocks: rptr $%p wptr $%p "
596 	    "nmblks %d len %d tot_xfer_len %d",
597 	    mp->b_rptr, mp->b_wptr, nmblks, MBLKL(mp), *tot_xfer_len_p));
598 	return (nmblks);
599 }
600 
601 boolean_t
602 hxge_txdma_reclaim(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, int nmblks)
603 {
604 	boolean_t		status = B_TRUE;
605 	p_hxge_dma_common_t	tx_desc_dma_p;
606 	hxge_dma_common_t	desc_area;
607 	p_tx_desc_t		tx_desc_ring_vp;
608 	p_tx_desc_t		tx_desc_p;
609 	p_tx_desc_t		tx_desc_pp;
610 	tx_desc_t		r_tx_desc;
611 	p_tx_msg_t		tx_msg_ring;
612 	p_tx_msg_t		tx_msg_p;
613 	hpi_handle_t		handle;
614 	tdc_tdr_head_t		tx_head;
615 	uint32_t		pkt_len;
616 	uint_t			tx_rd_index;
617 	uint16_t		head_index, tail_index;
618 	uint8_t			tdc;
619 	boolean_t		head_wrap, tail_wrap;
620 	p_hxge_tx_ring_stats_t	tdc_stats;
621 	tdc_byte_cnt_t		byte_cnt;
622 	tdc_tdr_qlen_t		qlen;
623 	int			rc;
624 
625 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_reclaim"));
626 
627 	status = ((tx_ring_p->descs_pending < hxge_reclaim_pending) &&
628 	    (nmblks != 0));
629 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
630 	    "==> hxge_txdma_reclaim: pending %d  reclaim %d nmblks %d",
631 	    tx_ring_p->descs_pending, hxge_reclaim_pending, nmblks));
632 
633 	if (!status) {
634 		tx_desc_dma_p = &tx_ring_p->tdc_desc;
635 		desc_area = tx_ring_p->tdc_desc;
636 		tx_desc_ring_vp = tx_desc_dma_p->kaddrp;
637 		tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
638 		tx_rd_index = tx_ring_p->rd_index;
639 		tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
640 		tx_msg_ring = tx_ring_p->tx_msg_ring;
641 		tx_msg_p = &tx_msg_ring[tx_rd_index];
642 		tdc = tx_ring_p->tdc;
643 		tdc_stats = tx_ring_p->tdc_stats;
644 		if (tx_ring_p->descs_pending > tdc_stats->tx_max_pend) {
645 			tdc_stats->tx_max_pend = tx_ring_p->descs_pending;
646 		}
647 		tail_index = tx_ring_p->wr_index;
648 		tail_wrap = tx_ring_p->wr_index_wrap;
649 
650 		/*
651 		 * tdc_byte_cnt reg can be used to get bytes transmitted. It
652 		 * includes padding too in case of runt packets.
653 		 */
654 		handle = HXGE_DEV_HPI_HANDLE(hxgep);
655 		TXDMA_REG_READ64(handle, TDC_BYTE_CNT, tdc, &byte_cnt.value);
656 		tdc_stats->obytes_with_pad += byte_cnt.bits.byte_count;
657 
658 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
659 		    "==> hxge_txdma_reclaim: tdc %d tx_rd_index %d "
660 		    "tail_index %d tail_wrap %d tx_desc_p $%p ($%p) ",
661 		    tdc, tx_rd_index, tail_index, tail_wrap,
662 		    tx_desc_p, (*(uint64_t *)tx_desc_p)));
663 
664 		/*
665 		 * Read the hardware maintained transmit head and wrap around
666 		 * bit.
667 		 */
668 		TXDMA_REG_READ64(handle, TDC_TDR_HEAD, tdc, &tx_head.value);
669 		head_index = tx_head.bits.head;
670 		head_wrap = tx_head.bits.wrap;
671 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
672 		    "==> hxge_txdma_reclaim: "
673 		    "tx_rd_index %d tail %d tail_wrap %d head %d wrap %d",
674 		    tx_rd_index, tail_index, tail_wrap, head_index, head_wrap));
675 
676 		/*
677 		 * For debug only. This can be used to verify the qlen and make
678 		 * sure the hardware is wrapping the Tdr correctly.
679 		 */
680 		TXDMA_REG_READ64(handle, TDC_TDR_QLEN, tdc, &qlen.value);
681 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
682 		    "==> hxge_txdma_reclaim: tdr_qlen %d tdr_pref_qlen %d",
683 		    qlen.bits.tdr_qlen, qlen.bits.tdr_pref_qlen));
684 
685 		if (head_index == tail_index) {
686 			if (TXDMA_RING_EMPTY(head_index, head_wrap, tail_index,
687 			    tail_wrap) && (head_index == tx_rd_index)) {
688 				HXGE_DEBUG_MSG((hxgep, TX_CTL,
689 				    "==> hxge_txdma_reclaim: EMPTY"));
690 				return (B_TRUE);
691 			}
692 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
693 			    "==> hxge_txdma_reclaim: Checking if ring full"));
694 			if (TXDMA_RING_FULL(head_index, head_wrap, tail_index,
695 			    tail_wrap)) {
696 				HXGE_DEBUG_MSG((hxgep, TX_CTL,
697 				    "==> hxge_txdma_reclaim: full"));
698 				return (B_FALSE);
699 			}
700 		}
701 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
702 		    "==> hxge_txdma_reclaim: tx_rd_index and head_index"));
703 
704 		/* XXXX: limit the # of reclaims */
705 		tx_desc_pp = &r_tx_desc;
706 		while ((tx_rd_index != head_index) &&
707 		    (tx_ring_p->descs_pending != 0)) {
708 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
709 			    "==> hxge_txdma_reclaim: Checking if pending"));
710 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
711 			    "==> hxge_txdma_reclaim: descs_pending %d ",
712 			    tx_ring_p->descs_pending));
713 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
714 			    "==> hxge_txdma_reclaim: "
715 			    "(tx_rd_index %d head_index %d (tx_desc_p $%p)",
716 			    tx_rd_index, head_index, tx_desc_p));
717 
718 			tx_desc_pp->value = tx_desc_p->value;
719 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
720 			    "==> hxge_txdma_reclaim: "
721 			    "(tx_rd_index %d head_index %d "
722 			    "tx_desc_p $%p (desc value 0x%llx) ",
723 			    tx_rd_index, head_index,
724 			    tx_desc_pp, (*(uint64_t *)tx_desc_pp)));
725 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
726 			    "==> hxge_txdma_reclaim: dump desc:"));
727 
728 			/*
729 			 * tdc_byte_cnt reg can be used to get bytes
730 			 * transmitted
731 			 */
732 			pkt_len = tx_desc_pp->bits.tr_len;
733 			tdc_stats->obytes += pkt_len;
734 			tdc_stats->opackets += tx_desc_pp->bits.sop;
735 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
736 			    "==> hxge_txdma_reclaim: pkt_len %d "
737 			    "tdc channel %d opackets %d",
738 			    pkt_len, tdc, tdc_stats->opackets));
739 
740 			if (tx_msg_p->flags.dma_type == USE_DVMA) {
741 				HXGE_DEBUG_MSG((hxgep, TX_CTL,
742 				    "tx_desc_p = $%p tx_desc_pp = $%p "
743 				    "index = %d",
744 				    tx_desc_p, tx_desc_pp,
745 				    tx_ring_p->rd_index));
746 				(void) dvma_unload(tx_msg_p->dvma_handle,
747 				    0, -1);
748 				tx_msg_p->dvma_handle = NULL;
749 				if (tx_ring_p->dvma_wr_index ==
750 				    tx_ring_p->dvma_wrap_mask) {
751 					tx_ring_p->dvma_wr_index = 0;
752 				} else {
753 					tx_ring_p->dvma_wr_index++;
754 				}
755 				tx_ring_p->dvma_pending--;
756 			} else if (tx_msg_p->flags.dma_type == USE_DMA) {
757 				HXGE_DEBUG_MSG((hxgep, TX_CTL,
758 				    "==> hxge_txdma_reclaim: USE DMA"));
759 				if (rc = ddi_dma_unbind_handle
760 				    (tx_msg_p->dma_handle)) {
761 					cmn_err(CE_WARN, "hxge_reclaim: "
762 					    "ddi_dma_unbind_handle "
763 					    "failed. status %d", rc);
764 				}
765 			}
766 
767 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
768 			    "==> hxge_txdma_reclaim: count packets"));
769 
770 			/*
771 			 * count a chained packet only once.
772 			 */
773 			if (tx_msg_p->tx_message != NULL) {
774 				freemsg(tx_msg_p->tx_message);
775 				tx_msg_p->tx_message = NULL;
776 			}
777 			tx_msg_p->flags.dma_type = USE_NONE;
778 			tx_rd_index = tx_ring_p->rd_index;
779 			tx_rd_index = (tx_rd_index + 1) &
780 			    tx_ring_p->tx_wrap_mask;
781 			tx_ring_p->rd_index = tx_rd_index;
782 			tx_ring_p->descs_pending--;
783 			tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
784 			tx_msg_p = &tx_msg_ring[tx_rd_index];
785 		}
786 
787 		status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
788 		    (int)tx_ring_p->descs_pending - TX_FULL_MARK));
789 		if (status) {
790 			(void) cas32((uint32_t *)&tx_ring_p->queueing, 1, 0);
791 		}
792 	} else {
793 		status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
794 		    (int)tx_ring_p->descs_pending - TX_FULL_MARK));
795 	}
796 
797 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
798 	    "<== hxge_txdma_reclaim status = 0x%08x", status));
799 	return (status);
800 }
801 
802 uint_t
803 hxge_tx_intr(caddr_t arg1, caddr_t arg2)
804 {
805 	p_hxge_ldv_t	ldvp = (p_hxge_ldv_t)arg1;
806 	p_hxge_t	hxgep = (p_hxge_t)arg2;
807 	p_hxge_ldg_t	ldgp;
808 	uint8_t		channel;
809 	uint32_t	vindex;
810 	hpi_handle_t	handle;
811 	tdc_stat_t	cs;
812 	p_tx_ring_t	*tx_rings;
813 	p_tx_ring_t	tx_ring_p;
814 	hpi_status_t	rs = HPI_SUCCESS;
815 	uint_t		serviced = DDI_INTR_UNCLAIMED;
816 	hxge_status_t	status = HXGE_OK;
817 
818 	if (ldvp == NULL) {
819 		HXGE_DEBUG_MSG((NULL, INT_CTL,
820 		    "<== hxge_tx_intr: hxgep $%p ldvp $%p", hxgep, ldvp));
821 		return (DDI_INTR_UNCLAIMED);
822 	}
823 
824 	if (arg2 == NULL || (void *) ldvp->hxgep != arg2) {
825 		hxgep = ldvp->hxgep;
826 	}
827 
828 	/*
829 	 * If the interface is not started, just swallow the interrupt
830 	 * and don't rearm the logical device.
831 	 */
832 	if (hxgep->hxge_mac_state != HXGE_MAC_STARTED)
833 		return (DDI_INTR_CLAIMED);
834 
835 	HXGE_DEBUG_MSG((hxgep, INT_CTL,
836 	    "==> hxge_tx_intr: hxgep(arg2) $%p ldvp(arg1) $%p", hxgep, ldvp));
837 
838 	/*
839 	 * This interrupt handler is for a specific transmit dma channel.
840 	 */
841 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
842 
843 	/* Get the control and status for this channel. */
844 	channel = ldvp->channel;
845 	ldgp = ldvp->ldgp;
846 	HXGE_DEBUG_MSG((hxgep, INT_CTL,
847 	    "==> hxge_tx_intr: hxgep $%p ldvp (ldvp) $%p channel %d",
848 	    hxgep, ldvp, channel));
849 
850 	rs = hpi_txdma_control_status(handle, OP_GET, channel, &cs);
851 	vindex = ldvp->vdma_index;
852 	HXGE_DEBUG_MSG((hxgep, INT_CTL,
853 	    "==> hxge_tx_intr:channel %d ring index %d status 0x%08x",
854 	    channel, vindex, rs));
855 
856 	if (!rs && cs.bits.marked) {
857 		HXGE_DEBUG_MSG((hxgep, INT_CTL,
858 		    "==> hxge_tx_intr:channel %d ring index %d "
859 		    "status 0x%08x (marked bit set)", channel, vindex, rs));
860 		tx_rings = hxgep->tx_rings->rings;
861 		tx_ring_p = tx_rings[vindex];
862 		HXGE_DEBUG_MSG((hxgep, INT_CTL,
863 		    "==> hxge_tx_intr:channel %d ring index %d "
864 		    "status 0x%08x (marked bit set, calling reclaim)",
865 		    channel, vindex, rs));
866 
867 		MUTEX_ENTER(&tx_ring_p->lock);
868 		(void) hxge_txdma_reclaim(hxgep, tx_rings[vindex], 0);
869 		MUTEX_EXIT(&tx_ring_p->lock);
870 		mac_tx_update(hxgep->mach);
871 	}
872 
873 	/*
874 	 * Process other transmit control and status. Check the ldv state.
875 	 */
876 	status = hxge_tx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs);
877 
878 	/* Clear the error bits */
879 	RXDMA_REG_WRITE64(handle, TDC_STAT, channel, cs.value);
880 
881 	/*
882 	 * Rearm this logical group if this is a single device group.
883 	 */
884 	if (ldgp->nldvs == 1) {
885 		HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_tx_intr: rearm"));
886 		if (status == HXGE_OK) {
887 			(void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg,
888 			    B_TRUE, ldgp->ldg_timer);
889 		}
890 	}
891 	HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_tx_intr"));
892 	serviced = DDI_INTR_CLAIMED;
893 	return (serviced);
894 }
895 
896 void
897 hxge_txdma_stop(p_hxge_t hxgep)
898 {
899 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_stop"));
900 
901 	(void) hxge_tx_vmac_disable(hxgep);
902 	(void) hxge_txdma_hw_mode(hxgep, HXGE_DMA_STOP);
903 
904 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_stop"));
905 }
906 
907 hxge_status_t
908 hxge_txdma_hw_mode(p_hxge_t hxgep, boolean_t enable)
909 {
910 	int		i, ndmas;
911 	uint16_t	channel;
912 	p_tx_rings_t	tx_rings;
913 	p_tx_ring_t	*tx_desc_rings;
914 	hpi_handle_t	handle;
915 	hpi_status_t	rs = HPI_SUCCESS;
916 	hxge_status_t	status = HXGE_OK;
917 
918 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
919 	    "==> hxge_txdma_hw_mode: enable mode %d", enable));
920 
921 	if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) {
922 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
923 		    "<== hxge_txdma_mode: not initialized"));
924 		return (HXGE_ERROR);
925 	}
926 	tx_rings = hxgep->tx_rings;
927 	if (tx_rings == NULL) {
928 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
929 		    "<== hxge_txdma_hw_mode: NULL global ring pointer"));
930 		return (HXGE_ERROR);
931 	}
932 	tx_desc_rings = tx_rings->rings;
933 	if (tx_desc_rings == NULL) {
934 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
935 		    "<== hxge_txdma_hw_mode: NULL rings pointer"));
936 		return (HXGE_ERROR);
937 	}
938 	ndmas = tx_rings->ndmas;
939 	if (!ndmas) {
940 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
941 		    "<== hxge_txdma_hw_mode: no dma channel allocated"));
942 		return (HXGE_ERROR);
943 	}
944 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_mode: "
945 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
946 	    tx_rings, tx_desc_rings, ndmas));
947 
948 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
949 	for (i = 0; i < ndmas; i++) {
950 		if (tx_desc_rings[i] == NULL) {
951 			continue;
952 		}
953 		channel = tx_desc_rings[i]->tdc;
954 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
955 		    "==> hxge_txdma_hw_mode: channel %d", channel));
956 		if (enable) {
957 			rs = hpi_txdma_channel_enable(handle, channel);
958 			HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
959 			    "==> hxge_txdma_hw_mode: channel %d (enable) "
960 			    "rs 0x%x", channel, rs));
961 		} else {
962 			/*
963 			 * Stop the dma channel and waits for the stop done. If
964 			 * the stop done bit is not set, then force an error so
965 			 * TXC will stop. All channels bound to this port need
966 			 * to be stopped and reset after injecting an interrupt
967 			 * error.
968 			 */
969 			rs = hpi_txdma_channel_disable(handle, channel);
970 			HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
971 			    "==> hxge_txdma_hw_mode: channel %d (disable) "
972 			    "rs 0x%x", channel, rs));
973 		}
974 	}
975 
976 	status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
977 
978 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
979 	    "<== hxge_txdma_hw_mode: status 0x%x", status));
980 
981 	return (status);
982 }
983 
984 void
985 hxge_txdma_enable_channel(p_hxge_t hxgep, uint16_t channel)
986 {
987 	hpi_handle_t handle;
988 
989 	HXGE_DEBUG_MSG((hxgep, DMA_CTL,
990 	    "==> hxge_txdma_enable_channel: channel %d", channel));
991 
992 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
993 	/* enable the transmit dma channels */
994 	(void) hpi_txdma_channel_enable(handle, channel);
995 
996 	HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_txdma_enable_channel"));
997 }
998 
999 void
1000 hxge_txdma_disable_channel(p_hxge_t hxgep, uint16_t channel)
1001 {
1002 	hpi_handle_t handle;
1003 
1004 	HXGE_DEBUG_MSG((hxgep, DMA_CTL,
1005 	    "==> hxge_txdma_disable_channel: channel %d", channel));
1006 
1007 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
1008 	/* stop the transmit dma channels */
1009 	(void) hpi_txdma_channel_disable(handle, channel);
1010 
1011 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_disable_channel"));
1012 }
1013 
1014 int
1015 hxge_txdma_stop_inj_err(p_hxge_t hxgep, int channel)
1016 {
1017 	hpi_handle_t	handle;
1018 	int		status;
1019 	hpi_status_t	rs = HPI_SUCCESS;
1020 
1021 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_stop_inj_err"));
1022 
1023 	/*
1024 	 * Stop the dma channel waits for the stop done. If the stop done bit
1025 	 * is not set, then create an error.
1026 	 */
1027 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
1028 	rs = hpi_txdma_channel_disable(handle, channel);
1029 	status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
1030 	if (status == HXGE_OK) {
1031 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1032 		    "<== hxge_txdma_stop_inj_err (channel %d): "
1033 		    "stopped OK", channel));
1034 		return (status);
1035 	}
1036 
1037 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1038 	    "==> hxge_txdma_stop_inj_err (channel): stop failed (0x%x) "
1039 	    " (injected error but still not stopped)", channel, rs));
1040 
1041 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_stop_inj_err"));
1042 
1043 	return (status);
1044 }
1045 
1046 /*ARGSUSED*/
1047 void
1048 hxge_fixup_txdma_rings(p_hxge_t hxgep)
1049 {
1050 	int		index, ndmas;
1051 	uint16_t	channel;
1052 	p_tx_rings_t	tx_rings;
1053 
1054 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_txdma_rings"));
1055 
1056 	/*
1057 	 * For each transmit channel, reclaim each descriptor and free buffers.
1058 	 */
1059 	tx_rings = hxgep->tx_rings;
1060 	if (tx_rings == NULL) {
1061 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1062 		    "<== hxge_fixup_txdma_rings: NULL ring pointer"));
1063 		return;
1064 	}
1065 
1066 	ndmas = tx_rings->ndmas;
1067 	if (!ndmas) {
1068 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1069 		    "<== hxge_fixup_txdma_rings: no channel allocated"));
1070 		return;
1071 	}
1072 
1073 	if (tx_rings->rings == NULL) {
1074 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1075 		    "<== hxge_fixup_txdma_rings: NULL rings pointer"));
1076 		return;
1077 	}
1078 
1079 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_fixup_txdma_rings: "
1080 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1081 	    tx_rings, tx_rings->rings, ndmas));
1082 
1083 	for (index = 0; index < ndmas; index++) {
1084 		channel = tx_rings->rings[index]->tdc;
1085 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1086 		    "==> hxge_fixup_txdma_rings: channel %d", channel));
1087 		hxge_txdma_fixup_channel(hxgep, tx_rings->rings[index],
1088 		    channel);
1089 	}
1090 
1091 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_fixup_txdma_rings"));
1092 }
1093 
1094 /*ARGSUSED*/
1095 void
1096 hxge_txdma_fix_channel(p_hxge_t hxgep, uint16_t channel)
1097 {
1098 	p_tx_ring_t ring_p;
1099 
1100 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fix_channel"));
1101 
1102 	ring_p = hxge_txdma_get_ring(hxgep, channel);
1103 	if (ring_p == NULL) {
1104 		HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_channel"));
1105 		return;
1106 	}
1107 
1108 	if (ring_p->tdc != channel) {
1109 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1110 		    "<== hxge_txdma_fix_channel: channel not matched "
1111 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1112 		return;
1113 	}
1114 
1115 	hxge_txdma_fixup_channel(hxgep, ring_p, channel);
1116 
1117 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_channel"));
1118 }
1119 
1120 /*ARGSUSED*/
1121 void
1122 hxge_txdma_fixup_channel(p_hxge_t hxgep, p_tx_ring_t ring_p, uint16_t channel)
1123 {
1124 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fixup_channel"));
1125 
1126 	if (ring_p == NULL) {
1127 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1128 		    "<== hxge_txdma_fixup_channel: NULL ring pointer"));
1129 		return;
1130 	}
1131 	if (ring_p->tdc != channel) {
1132 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1133 		    "<== hxge_txdma_fixup_channel: channel not matched "
1134 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1135 		return;
1136 	}
1137 	MUTEX_ENTER(&ring_p->lock);
1138 	(void) hxge_txdma_reclaim(hxgep, ring_p, 0);
1139 
1140 	ring_p->rd_index = 0;
1141 	ring_p->wr_index = 0;
1142 	ring_p->ring_head.value = 0;
1143 	ring_p->ring_kick_tail.value = 0;
1144 	ring_p->descs_pending = 0;
1145 	MUTEX_EXIT(&ring_p->lock);
1146 
1147 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fixup_channel"));
1148 }
1149 
1150 /*ARGSUSED*/
1151 void
1152 hxge_txdma_hw_kick(p_hxge_t hxgep)
1153 {
1154 	int		index, ndmas;
1155 	uint16_t	channel;
1156 	p_tx_rings_t	tx_rings;
1157 
1158 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hw_kick"));
1159 
1160 	tx_rings = hxgep->tx_rings;
1161 	if (tx_rings == NULL) {
1162 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1163 		    "<== hxge_txdma_hw_kick: NULL ring pointer"));
1164 		return;
1165 	}
1166 	ndmas = tx_rings->ndmas;
1167 	if (!ndmas) {
1168 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1169 		    "<== hxge_txdma_hw_kick: no channel allocated"));
1170 		return;
1171 	}
1172 	if (tx_rings->rings == NULL) {
1173 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1174 		    "<== hxge_txdma_hw_kick: NULL rings pointer"));
1175 		return;
1176 	}
1177 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_kick: "
1178 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1179 	    tx_rings, tx_rings->rings, ndmas));
1180 
1181 	for (index = 0; index < ndmas; index++) {
1182 		channel = tx_rings->rings[index]->tdc;
1183 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1184 		    "==> hxge_txdma_hw_kick: channel %d", channel));
1185 		hxge_txdma_hw_kick_channel(hxgep, tx_rings->rings[index],
1186 		    channel);
1187 	}
1188 
1189 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hw_kick"));
1190 }
1191 
1192 /*ARGSUSED*/
1193 void
1194 hxge_txdma_kick_channel(p_hxge_t hxgep, uint16_t channel)
1195 {
1196 	p_tx_ring_t ring_p;
1197 
1198 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_kick_channel"));
1199 
1200 	ring_p = hxge_txdma_get_ring(hxgep, channel);
1201 	if (ring_p == NULL) {
1202 		HXGE_DEBUG_MSG((hxgep, TX_CTL, " hxge_txdma_kick_channel"));
1203 		return;
1204 	}
1205 
1206 	if (ring_p->tdc != channel) {
1207 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1208 		    "<== hxge_txdma_kick_channel: channel not matched "
1209 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1210 		return;
1211 	}
1212 
1213 	hxge_txdma_hw_kick_channel(hxgep, ring_p, channel);
1214 
1215 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_kick_channel"));
1216 }
1217 
1218 /*ARGSUSED*/
1219 void
1220 hxge_txdma_hw_kick_channel(p_hxge_t hxgep, p_tx_ring_t ring_p, uint16_t channel)
1221 {
1222 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hw_kick_channel"));
1223 
1224 	if (ring_p == NULL) {
1225 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1226 		    "<== hxge_txdma_hw_kick_channel: NULL ring pointer"));
1227 		return;
1228 	}
1229 
1230 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hw_kick_channel"));
1231 }
1232 
1233 /*ARGSUSED*/
1234 void
1235 hxge_check_tx_hang(p_hxge_t hxgep)
1236 {
1237 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_check_tx_hang"));
1238 
1239 	/*
1240 	 * Needs inputs from hardware for regs: head index had not moved since
1241 	 * last timeout. packets not transmitted or stuffed registers.
1242 	 */
1243 	if (hxge_txdma_hung(hxgep)) {
1244 		hxge_fixup_hung_txdma_rings(hxgep);
1245 	}
1246 
1247 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_check_tx_hang"));
1248 }
1249 
1250 int
1251 hxge_txdma_hung(p_hxge_t hxgep)
1252 {
1253 	int		index, ndmas;
1254 	uint16_t	channel;
1255 	p_tx_rings_t	tx_rings;
1256 	p_tx_ring_t	tx_ring_p;
1257 
1258 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hung"));
1259 
1260 	tx_rings = hxgep->tx_rings;
1261 	if (tx_rings == NULL) {
1262 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1263 		    "<== hxge_txdma_hung: NULL ring pointer"));
1264 		return (B_FALSE);
1265 	}
1266 
1267 	ndmas = tx_rings->ndmas;
1268 	if (!ndmas) {
1269 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1270 		    "<== hxge_txdma_hung: no channel allocated"));
1271 		return (B_FALSE);
1272 	}
1273 
1274 	if (tx_rings->rings == NULL) {
1275 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1276 		    "<== hxge_txdma_hung: NULL rings pointer"));
1277 		return (B_FALSE);
1278 	}
1279 
1280 	for (index = 0; index < ndmas; index++) {
1281 		channel = tx_rings->rings[index]->tdc;
1282 		tx_ring_p = tx_rings->rings[index];
1283 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1284 		    "==> hxge_txdma_hung: channel %d", channel));
1285 		if (hxge_txdma_channel_hung(hxgep, tx_ring_p, channel)) {
1286 			return (B_TRUE);
1287 		}
1288 	}
1289 
1290 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hung"));
1291 
1292 	return (B_FALSE);
1293 }
1294 
1295 int
1296 hxge_txdma_channel_hung(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, uint16_t channel)
1297 {
1298 	uint16_t	head_index, tail_index;
1299 	boolean_t	head_wrap, tail_wrap;
1300 	hpi_handle_t	handle;
1301 	tdc_tdr_head_t	tx_head;
1302 	uint_t		tx_rd_index;
1303 
1304 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_channel_hung"));
1305 
1306 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
1307 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
1308 	    "==> hxge_txdma_channel_hung: channel %d", channel));
1309 	MUTEX_ENTER(&tx_ring_p->lock);
1310 	(void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
1311 
1312 	tail_index = tx_ring_p->wr_index;
1313 	tail_wrap = tx_ring_p->wr_index_wrap;
1314 	tx_rd_index = tx_ring_p->rd_index;
1315 	MUTEX_EXIT(&tx_ring_p->lock);
1316 
1317 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
1318 	    "==> hxge_txdma_channel_hung: tdc %d tx_rd_index %d "
1319 	    "tail_index %d tail_wrap %d ",
1320 	    channel, tx_rd_index, tail_index, tail_wrap));
1321 	/*
1322 	 * Read the hardware maintained transmit head and wrap around bit.
1323 	 */
1324 	(void) hpi_txdma_ring_head_get(handle, channel, &tx_head);
1325 	head_index = tx_head.bits.head;
1326 	head_wrap = tx_head.bits.wrap;
1327 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_channel_hung: "
1328 	    "tx_rd_index %d tail %d tail_wrap %d head %d wrap %d",
1329 	    tx_rd_index, tail_index, tail_wrap, head_index, head_wrap));
1330 
1331 	if (TXDMA_RING_EMPTY(head_index, head_wrap, tail_index, tail_wrap) &&
1332 	    (head_index == tx_rd_index)) {
1333 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1334 		    "==> hxge_txdma_channel_hung: EMPTY"));
1335 		return (B_FALSE);
1336 	}
1337 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
1338 	    "==> hxge_txdma_channel_hung: Checking if ring full"));
1339 	if (TXDMA_RING_FULL(head_index, head_wrap, tail_index, tail_wrap)) {
1340 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1341 		    "==> hxge_txdma_channel_hung: full"));
1342 		return (B_TRUE);
1343 	}
1344 
1345 	/* If not full, check with hardware to see if it is hung */
1346 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_channel_hung"));
1347 
1348 	return (B_FALSE);
1349 }
1350 
1351 /*ARGSUSED*/
1352 void
1353 hxge_fixup_hung_txdma_rings(p_hxge_t hxgep)
1354 {
1355 	int		index, ndmas;
1356 	uint16_t	channel;
1357 	p_tx_rings_t	tx_rings;
1358 
1359 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_hung_txdma_rings"));
1360 	tx_rings = hxgep->tx_rings;
1361 	if (tx_rings == NULL) {
1362 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1363 		    "<== hxge_fixup_hung_txdma_rings: NULL ring pointer"));
1364 		return;
1365 	}
1366 	ndmas = tx_rings->ndmas;
1367 	if (!ndmas) {
1368 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1369 		    "<== hxge_fixup_hung_txdma_rings: no channel allocated"));
1370 		return;
1371 	}
1372 	if (tx_rings->rings == NULL) {
1373 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1374 		    "<== hxge_fixup_hung_txdma_rings: NULL rings pointer"));
1375 		return;
1376 	}
1377 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_hung_txdma_rings: "
1378 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1379 	    tx_rings, tx_rings->rings, ndmas));
1380 
1381 	for (index = 0; index < ndmas; index++) {
1382 		channel = tx_rings->rings[index]->tdc;
1383 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1384 		    "==> hxge_fixup_hung_txdma_rings: channel %d", channel));
1385 		hxge_txdma_fixup_hung_channel(hxgep, tx_rings->rings[index],
1386 		    channel);
1387 	}
1388 
1389 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_fixup_hung_txdma_rings"));
1390 }
1391 
1392 /*ARGSUSED*/
1393 void
1394 hxge_txdma_fix_hung_channel(p_hxge_t hxgep, uint16_t channel)
1395 {
1396 	p_tx_ring_t ring_p;
1397 
1398 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fix_hung_channel"));
1399 	ring_p = hxge_txdma_get_ring(hxgep, channel);
1400 	if (ring_p == NULL) {
1401 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1402 		    "<== hxge_txdma_fix_hung_channel"));
1403 		return;
1404 	}
1405 	if (ring_p->tdc != channel) {
1406 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1407 		    "<== hxge_txdma_fix_hung_channel: channel not matched "
1408 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1409 		return;
1410 	}
1411 	hxge_txdma_fixup_channel(hxgep, ring_p, channel);
1412 
1413 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_hung_channel"));
1414 }
1415 
1416 /*ARGSUSED*/
1417 void
1418 hxge_txdma_fixup_hung_channel(p_hxge_t hxgep, p_tx_ring_t ring_p,
1419     uint16_t channel)
1420 {
1421 	hpi_handle_t	handle;
1422 	int		status = HXGE_OK;
1423 
1424 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fixup_hung_channel"));
1425 
1426 	if (ring_p == NULL) {
1427 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1428 		    "<== hxge_txdma_fixup_hung_channel: NULL ring pointer"));
1429 		return;
1430 	}
1431 	if (ring_p->tdc != channel) {
1432 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1433 		    "<== hxge_txdma_fixup_hung_channel: channel "
1434 		    "not matched ring tdc %d passed channel",
1435 		    ring_p->tdc, channel));
1436 		return;
1437 	}
1438 	/* Reclaim descriptors */
1439 	MUTEX_ENTER(&ring_p->lock);
1440 	(void) hxge_txdma_reclaim(hxgep, ring_p, 0);
1441 	MUTEX_EXIT(&ring_p->lock);
1442 
1443 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
1444 	/*
1445 	 * Stop the dma channel waits for the stop done. If the stop done bit
1446 	 * is not set, then force an error.
1447 	 */
1448 	status = hpi_txdma_channel_disable(handle, channel);
1449 	if (!(status & HPI_TXDMA_STOP_FAILED)) {
1450 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1451 		    "<== hxge_txdma_fixup_hung_channel: stopped OK "
1452 		    "ring tdc %d passed channel %d", ring_p->tdc, channel));
1453 		return;
1454 	}
1455 	/* Stop done bit will be set as a result of error injection */
1456 	status = hpi_txdma_channel_disable(handle, channel);
1457 	if (!(status & HPI_TXDMA_STOP_FAILED)) {
1458 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1459 		    "<== hxge_txdma_fixup_hung_channel: stopped again"
1460 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1461 		return;
1462 	}
1463 
1464 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
1465 	    "<== hxge_txdma_fixup_hung_channel: stop done still not set!! "
1466 	    "ring tdc %d passed channel", ring_p->tdc, channel));
1467 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fixup_hung_channel"));
1468 }
1469 
1470 /*ARGSUSED*/
1471 void
1472 hxge_reclaim_rings(p_hxge_t hxgep)
1473 {
1474 	int		index, ndmas;
1475 	uint16_t	channel;
1476 	p_tx_rings_t	tx_rings;
1477 	p_tx_ring_t	tx_ring_p;
1478 
1479 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_reclaim_ring"));
1480 	tx_rings = hxgep->tx_rings;
1481 	if (tx_rings == NULL) {
1482 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1483 		    "<== hxge_reclain_rimgs: NULL ring pointer"));
1484 		return;
1485 	}
1486 	ndmas = tx_rings->ndmas;
1487 	if (!ndmas) {
1488 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1489 		    "<== hxge_reclain_rimgs: no channel allocated"));
1490 		return;
1491 	}
1492 	if (tx_rings->rings == NULL) {
1493 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1494 		    "<== hxge_reclain_rimgs: NULL rings pointer"));
1495 		return;
1496 	}
1497 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_reclain_rimgs: "
1498 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1499 	    tx_rings, tx_rings->rings, ndmas));
1500 
1501 	for (index = 0; index < ndmas; index++) {
1502 		channel = tx_rings->rings[index]->tdc;
1503 		HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> reclain_rimgs: channel %d",
1504 		    channel));
1505 		tx_ring_p = tx_rings->rings[index];
1506 		MUTEX_ENTER(&tx_ring_p->lock);
1507 		(void) hxge_txdma_reclaim(hxgep, tx_ring_p, channel);
1508 		MUTEX_EXIT(&tx_ring_p->lock);
1509 	}
1510 
1511 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_reclaim_rings"));
1512 }
1513 
1514 /*
1515  * Static functions start here.
1516  */
1517 static hxge_status_t
1518 hxge_map_txdma(p_hxge_t hxgep)
1519 {
1520 	int			i, ndmas;
1521 	uint16_t		channel;
1522 	p_tx_rings_t		tx_rings;
1523 	p_tx_ring_t		*tx_desc_rings;
1524 	p_tx_mbox_areas_t	tx_mbox_areas_p;
1525 	p_tx_mbox_t		*tx_mbox_p;
1526 	p_hxge_dma_pool_t	dma_buf_poolp;
1527 	p_hxge_dma_pool_t	dma_cntl_poolp;
1528 	p_hxge_dma_common_t	*dma_buf_p;
1529 	p_hxge_dma_common_t	*dma_cntl_p;
1530 	hxge_status_t		status = HXGE_OK;
1531 
1532 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma"));
1533 
1534 	dma_buf_poolp = hxgep->tx_buf_pool_p;
1535 	dma_cntl_poolp = hxgep->tx_cntl_pool_p;
1536 
1537 	if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) {
1538 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1539 		    "==> hxge_map_txdma: buf not allocated"));
1540 		return (HXGE_ERROR);
1541 	}
1542 	ndmas = dma_buf_poolp->ndmas;
1543 	if (!ndmas) {
1544 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1545 		    "<== hxge_map_txdma: no dma allocated"));
1546 		return (HXGE_ERROR);
1547 	}
1548 	dma_buf_p = dma_buf_poolp->dma_buf_pool_p;
1549 	dma_cntl_p = dma_cntl_poolp->dma_buf_pool_p;
1550 
1551 	tx_rings = (p_tx_rings_t)KMEM_ZALLOC(sizeof (tx_rings_t), KM_SLEEP);
1552 	tx_desc_rings = (p_tx_ring_t *)KMEM_ZALLOC(
1553 	    sizeof (p_tx_ring_t) * ndmas, KM_SLEEP);
1554 
1555 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1556 	    "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
1557 
1558 	tx_mbox_areas_p = (p_tx_mbox_areas_t)
1559 	    KMEM_ZALLOC(sizeof (tx_mbox_areas_t), KM_SLEEP);
1560 	tx_mbox_p = (p_tx_mbox_t *)KMEM_ZALLOC(
1561 	    sizeof (p_tx_mbox_t) * ndmas, KM_SLEEP);
1562 
1563 	/*
1564 	 * Map descriptors from the buffer pools for each dma channel.
1565 	 */
1566 	for (i = 0; i < ndmas; i++) {
1567 		/*
1568 		 * Set up and prepare buffer blocks, descriptors and mailbox.
1569 		 */
1570 		channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
1571 		status = hxge_map_txdma_channel(hxgep, channel,
1572 		    (p_hxge_dma_common_t *)&dma_buf_p[i],
1573 		    (p_tx_ring_t *)&tx_desc_rings[i],
1574 		    dma_buf_poolp->num_chunks[i],
1575 		    (p_hxge_dma_common_t *)&dma_cntl_p[i],
1576 		    (p_tx_mbox_t *)&tx_mbox_p[i]);
1577 		if (status != HXGE_OK) {
1578 			goto hxge_map_txdma_fail1;
1579 		}
1580 		tx_desc_rings[i]->index = (uint16_t)i;
1581 		tx_desc_rings[i]->tdc_stats = &hxgep->statsp->tdc_stats[i];
1582 	}
1583 
1584 	tx_rings->ndmas = ndmas;
1585 	tx_rings->rings = tx_desc_rings;
1586 	hxgep->tx_rings = tx_rings;
1587 	tx_mbox_areas_p->txmbox_areas_p = tx_mbox_p;
1588 	hxgep->tx_mbox_areas_p = tx_mbox_areas_p;
1589 
1590 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1591 	    "tx_rings $%p rings $%p", hxgep->tx_rings, hxgep->tx_rings->rings));
1592 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1593 	    "tx_rings $%p tx_desc_rings $%p",
1594 	    hxgep->tx_rings, tx_desc_rings));
1595 
1596 	goto hxge_map_txdma_exit;
1597 
1598 hxge_map_txdma_fail1:
1599 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1600 	    "==> hxge_map_txdma: uninit tx desc "
1601 	    "(status 0x%x channel %d i %d)", hxgep, status, channel, i));
1602 	i--;
1603 	for (; i >= 0; i--) {
1604 		channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
1605 		hxge_unmap_txdma_channel(hxgep, channel, tx_desc_rings[i],
1606 		    tx_mbox_p[i]);
1607 	}
1608 
1609 	KMEM_FREE(tx_desc_rings, sizeof (p_tx_ring_t) * ndmas);
1610 	KMEM_FREE(tx_rings, sizeof (tx_rings_t));
1611 	KMEM_FREE(tx_mbox_p, sizeof (p_tx_mbox_t) * ndmas);
1612 	KMEM_FREE(tx_mbox_areas_p, sizeof (tx_mbox_areas_t));
1613 
1614 hxge_map_txdma_exit:
1615 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1616 	    "==> hxge_map_txdma: (status 0x%x channel %d)", status, channel));
1617 
1618 	return (status);
1619 }
1620 
1621 static void
1622 hxge_unmap_txdma(p_hxge_t hxgep)
1623 {
1624 	int			i, ndmas;
1625 	uint8_t			channel;
1626 	p_tx_rings_t		tx_rings;
1627 	p_tx_ring_t		*tx_desc_rings;
1628 	p_tx_mbox_areas_t	tx_mbox_areas_p;
1629 	p_tx_mbox_t		*tx_mbox_p;
1630 	p_hxge_dma_pool_t	dma_buf_poolp;
1631 
1632 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_unmap_txdma"));
1633 
1634 	dma_buf_poolp = hxgep->tx_buf_pool_p;
1635 	if (!dma_buf_poolp->buf_allocated) {
1636 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1637 		    "==> hxge_unmap_txdma: buf not allocated"));
1638 		return;
1639 	}
1640 	ndmas = dma_buf_poolp->ndmas;
1641 	if (!ndmas) {
1642 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1643 		    "<== hxge_unmap_txdma: no dma allocated"));
1644 		return;
1645 	}
1646 	tx_rings = hxgep->tx_rings;
1647 	tx_desc_rings = tx_rings->rings;
1648 	if (tx_rings == NULL) {
1649 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1650 		    "<== hxge_unmap_txdma: NULL ring pointer"));
1651 		return;
1652 	}
1653 	tx_desc_rings = tx_rings->rings;
1654 	if (tx_desc_rings == NULL) {
1655 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1656 		    "<== hxge_unmap_txdma: NULL ring pointers"));
1657 		return;
1658 	}
1659 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_unmap_txdma: "
1660 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1661 	    tx_rings, tx_desc_rings, ndmas));
1662 
1663 	tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
1664 	tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
1665 
1666 	for (i = 0; i < ndmas; i++) {
1667 		channel = tx_desc_rings[i]->tdc;
1668 		(void) hxge_unmap_txdma_channel(hxgep, channel,
1669 		    (p_tx_ring_t)tx_desc_rings[i],
1670 		    (p_tx_mbox_t)tx_mbox_p[i]);
1671 	}
1672 
1673 	KMEM_FREE(tx_desc_rings, sizeof (p_tx_ring_t) * ndmas);
1674 	KMEM_FREE(tx_rings, sizeof (tx_rings_t));
1675 	KMEM_FREE(tx_mbox_p, sizeof (p_tx_mbox_t) * ndmas);
1676 	KMEM_FREE(tx_mbox_areas_p, sizeof (tx_mbox_areas_t));
1677 
1678 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_unmap_txdma"));
1679 }
1680 
1681 static hxge_status_t
1682 hxge_map_txdma_channel(p_hxge_t hxgep, uint16_t channel,
1683     p_hxge_dma_common_t *dma_buf_p, p_tx_ring_t *tx_desc_p,
1684     uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p,
1685     p_tx_mbox_t *tx_mbox_p)
1686 {
1687 	int status = HXGE_OK;
1688 
1689 	/*
1690 	 * Set up and prepare buffer blocks, descriptors and mailbox.
1691 	 */
1692 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1693 	    "==> hxge_map_txdma_channel (channel %d)", channel));
1694 
1695 	/*
1696 	 * Transmit buffer blocks
1697 	 */
1698 	status = hxge_map_txdma_channel_buf_ring(hxgep, channel,
1699 	    dma_buf_p, tx_desc_p, num_chunks);
1700 	if (status != HXGE_OK) {
1701 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1702 		    "==> hxge_map_txdma_channel (channel %d): "
1703 		    "map buffer failed 0x%x", channel, status));
1704 		goto hxge_map_txdma_channel_exit;
1705 	}
1706 	/*
1707 	 * Transmit block ring, and mailbox.
1708 	 */
1709 	hxge_map_txdma_channel_cfg_ring(hxgep, channel, dma_cntl_p, *tx_desc_p,
1710 	    tx_mbox_p);
1711 
1712 	goto hxge_map_txdma_channel_exit;
1713 
1714 hxge_map_txdma_channel_fail1:
1715 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1716 	    "==> hxge_map_txdma_channel: unmap buf"
1717 	    "(status 0x%x channel %d)", status, channel));
1718 	hxge_unmap_txdma_channel_buf_ring(hxgep, *tx_desc_p);
1719 
1720 hxge_map_txdma_channel_exit:
1721 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1722 	    "<== hxge_map_txdma_channel: (status 0x%x channel %d)",
1723 	    status, channel));
1724 
1725 	return (status);
1726 }
1727 
1728 /*ARGSUSED*/
1729 static void
1730 hxge_unmap_txdma_channel(p_hxge_t hxgep, uint16_t channel,
1731     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
1732 {
1733 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1734 	    "==> hxge_unmap_txdma_channel (channel %d)", channel));
1735 
1736 	/* unmap tx block ring, and mailbox.  */
1737 	(void) hxge_unmap_txdma_channel_cfg_ring(hxgep, tx_ring_p, tx_mbox_p);
1738 
1739 	/* unmap buffer blocks */
1740 	(void) hxge_unmap_txdma_channel_buf_ring(hxgep, tx_ring_p);
1741 
1742 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_unmap_txdma_channel"));
1743 }
1744 
1745 /*ARGSUSED*/
1746 static void
1747 hxge_map_txdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel,
1748     p_hxge_dma_common_t *dma_cntl_p, p_tx_ring_t tx_ring_p,
1749     p_tx_mbox_t *tx_mbox_p)
1750 {
1751 	p_tx_mbox_t		mboxp;
1752 	p_hxge_dma_common_t	cntl_dmap;
1753 	p_hxge_dma_common_t	dmap;
1754 	tdc_tdr_cfg_t		*tx_ring_cfig_p;
1755 	tdc_tdr_kick_t		*tx_ring_kick_p;
1756 	tdc_tdr_cfg_t		*tx_cs_p;
1757 	tdc_int_mask_t		*tx_evmask_p;
1758 	tdc_mbh_t		*mboxh_p;
1759 	tdc_mbl_t		*mboxl_p;
1760 	uint64_t		tx_desc_len;
1761 
1762 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1763 	    "==> hxge_map_txdma_channel_cfg_ring"));
1764 
1765 	cntl_dmap = *dma_cntl_p;
1766 
1767 	dmap = (p_hxge_dma_common_t)&tx_ring_p->tdc_desc;
1768 	hxge_setup_dma_common(dmap, cntl_dmap, tx_ring_p->tx_ring_size,
1769 	    sizeof (tx_desc_t));
1770 
1771 	/*
1772 	 * Zero out transmit ring descriptors.
1773 	 */
1774 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
1775 	tx_ring_cfig_p = &(tx_ring_p->tx_ring_cfig);
1776 	tx_ring_kick_p = &(tx_ring_p->tx_ring_kick);
1777 	tx_cs_p = &(tx_ring_p->tx_cs);
1778 	tx_evmask_p = &(tx_ring_p->tx_evmask);
1779 	tx_ring_cfig_p->value = 0;
1780 	tx_ring_kick_p->value = 0;
1781 	tx_cs_p->value = 0;
1782 	tx_evmask_p->value = 0;
1783 
1784 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1785 	    "==> hxge_map_txdma_channel_cfg_ring: channel %d des $%p",
1786 	    dma_channel, dmap->dma_cookie.dmac_laddress));
1787 
1788 	tx_ring_cfig_p->value = 0;
1789 
1790 	/* Hydra len is 11 bits and the lower 5 bits are 0s */
1791 	tx_desc_len = (uint64_t)(tx_ring_p->tx_ring_size >> 5);
1792 	tx_ring_cfig_p->value =
1793 	    (dmap->dma_cookie.dmac_laddress & TDC_TDR_CFG_ADDR_MASK) |
1794 	    (tx_desc_len << TDC_TDR_CFG_LEN_SHIFT);
1795 
1796 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1797 	    "==> hxge_map_txdma_channel_cfg_ring: channel %d cfg 0x%llx",
1798 	    dma_channel, tx_ring_cfig_p->value));
1799 
1800 	tx_cs_p->bits.reset = 1;
1801 
1802 	/* Map in mailbox */
1803 	mboxp = (p_tx_mbox_t)KMEM_ZALLOC(sizeof (tx_mbox_t), KM_SLEEP);
1804 	dmap = (p_hxge_dma_common_t)&mboxp->tx_mbox;
1805 	hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (txdma_mailbox_t));
1806 	mboxh_p = (tdc_mbh_t *)&tx_ring_p->tx_mbox_mbh;
1807 	mboxl_p = (tdc_mbl_t *)&tx_ring_p->tx_mbox_mbl;
1808 	mboxh_p->value = mboxl_p->value = 0;
1809 
1810 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1811 	    "==> hxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
1812 	    dmap->dma_cookie.dmac_laddress));
1813 
1814 	mboxh_p->bits.mbaddr = ((dmap->dma_cookie.dmac_laddress >>
1815 	    TDC_MBH_ADDR_SHIFT) & TDC_MBH_MASK);
1816 	mboxl_p->bits.mbaddr = ((dmap->dma_cookie.dmac_laddress &
1817 	    TDC_MBL_MASK) >> TDC_MBL_SHIFT);
1818 
1819 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1820 	    "==> hxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
1821 	    dmap->dma_cookie.dmac_laddress));
1822 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1823 	    "==> hxge_map_txdma_channel_cfg_ring: hmbox $%p mbox $%p",
1824 	    mboxh_p->bits.mbaddr, mboxl_p->bits.mbaddr));
1825 
1826 	/*
1827 	 * Set page valid and no mask
1828 	 */
1829 	tx_ring_p->page_hdl.value = 0;
1830 
1831 	*tx_mbox_p = mboxp;
1832 
1833 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1834 	    "<== hxge_map_txdma_channel_cfg_ring"));
1835 }
1836 
1837 /*ARGSUSED*/
1838 static void
1839 hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,
1840     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
1841 {
1842 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1843 	    "==> hxge_unmap_txdma_channel_cfg_ring: channel %d",
1844 	    tx_ring_p->tdc));
1845 
1846 	KMEM_FREE(tx_mbox_p, sizeof (tx_mbox_t));
1847 
1848 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1849 	    "<== hxge_unmap_txdma_channel_cfg_ring"));
1850 }
1851 
1852 static hxge_status_t
1853 hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel,
1854     p_hxge_dma_common_t *dma_buf_p,
1855     p_tx_ring_t *tx_desc_p, uint32_t num_chunks)
1856 {
1857 	p_hxge_dma_common_t	dma_bufp, tmp_bufp;
1858 	p_hxge_dma_common_t	dmap;
1859 	hxge_os_dma_handle_t	tx_buf_dma_handle;
1860 	p_tx_ring_t		tx_ring_p;
1861 	p_tx_msg_t		tx_msg_ring;
1862 	hxge_status_t		status = HXGE_OK;
1863 	int			ddi_status = DDI_SUCCESS;
1864 	int			i, j, index;
1865 	uint32_t		size, bsize;
1866 	uint32_t		nblocks, nmsgs;
1867 	char			qname[TASKQ_NAMELEN];
1868 
1869 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1870 	    "==> hxge_map_txdma_channel_buf_ring"));
1871 
1872 	dma_bufp = tmp_bufp = *dma_buf_p;
1873 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1874 	    " hxge_map_txdma_channel_buf_ring: channel %d to map %d "
1875 	    "chunks bufp $%p", channel, num_chunks, dma_bufp));
1876 
1877 	nmsgs = 0;
1878 	for (i = 0; i < num_chunks; i++, tmp_bufp++) {
1879 		nmsgs += tmp_bufp->nblocks;
1880 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1881 		    "==> hxge_map_txdma_channel_buf_ring: channel %d "
1882 		    "bufp $%p nblocks %d nmsgs %d",
1883 		    channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
1884 	}
1885 	if (!nmsgs) {
1886 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1887 		    "<== hxge_map_txdma_channel_buf_ring: channel %d "
1888 		    "no msg blocks", channel));
1889 		status = HXGE_ERROR;
1890 
1891 		goto hxge_map_txdma_channel_buf_ring_exit;
1892 	}
1893 
1894 	tx_ring_p = (p_tx_ring_t)KMEM_ZALLOC(sizeof (tx_ring_t), KM_SLEEP);
1895 	tx_ring_p->hxgep = hxgep;
1896 	(void) snprintf(qname, TASKQ_NAMELEN, "hxge_%d_%d",
1897 	    hxgep->instance, channel);
1898 	tx_ring_p->taskq = ddi_taskq_create(hxgep->dip, qname, 1,
1899 	    TASKQ_DEFAULTPRI, 0);
1900 	if (tx_ring_p->taskq == NULL) {
1901 		goto hxge_map_txdma_channel_buf_ring_fail1;
1902 	}
1903 
1904 	MUTEX_INIT(&tx_ring_p->lock, NULL, MUTEX_DRIVER,
1905 	    (void *) hxgep->interrupt_cookie);
1906 	/*
1907 	 * Allocate transmit message rings and handles for packets not to be
1908 	 * copied to premapped buffers.
1909 	 */
1910 	size = nmsgs * sizeof (tx_msg_t);
1911 	tx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
1912 	for (i = 0; i < nmsgs; i++) {
1913 		ddi_status = ddi_dma_alloc_handle(hxgep->dip, &hxge_tx_dma_attr,
1914 		    DDI_DMA_DONTWAIT, 0, &tx_msg_ring[i].dma_handle);
1915 		if (ddi_status != DDI_SUCCESS) {
1916 			status |= HXGE_DDI_FAILED;
1917 			break;
1918 		}
1919 	}
1920 
1921 	if (i < nmsgs) {
1922 		HXGE_DEBUG_MSG((hxgep, HXGE_ERR_CTL,
1923 		    "Allocate handles failed."));
1924 
1925 		goto hxge_map_txdma_channel_buf_ring_fail1;
1926 	}
1927 	tx_ring_p->tdc = channel;
1928 	tx_ring_p->tx_msg_ring = tx_msg_ring;
1929 	tx_ring_p->tx_ring_size = nmsgs;
1930 	tx_ring_p->num_chunks = num_chunks;
1931 	if (!hxge_tx_intr_thres) {
1932 		hxge_tx_intr_thres = tx_ring_p->tx_ring_size / 4;
1933 	}
1934 	tx_ring_p->tx_wrap_mask = tx_ring_p->tx_ring_size - 1;
1935 	tx_ring_p->rd_index = 0;
1936 	tx_ring_p->wr_index = 0;
1937 	tx_ring_p->ring_head.value = 0;
1938 	tx_ring_p->ring_kick_tail.value = 0;
1939 	tx_ring_p->descs_pending = 0;
1940 
1941 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1942 	    "==> hxge_map_txdma_channel_buf_ring: channel %d "
1943 	    "actual tx desc max %d nmsgs %d (config hxge_tx_ring_size %d)",
1944 	    channel, tx_ring_p->tx_ring_size, nmsgs, hxge_tx_ring_size));
1945 
1946 	/*
1947 	 * Map in buffers from the buffer pool.
1948 	 */
1949 	index = 0;
1950 	bsize = dma_bufp->block_size;
1951 
1952 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma_channel_buf_ring: "
1953 	    "dma_bufp $%p tx_rng_p $%p tx_msg_rng_p $%p bsize %d",
1954 	    dma_bufp, tx_ring_p, tx_msg_ring, bsize));
1955 
1956 	for (i = 0; i < num_chunks; i++, dma_bufp++) {
1957 		bsize = dma_bufp->block_size;
1958 		nblocks = dma_bufp->nblocks;
1959 		tx_buf_dma_handle = dma_bufp->dma_handle;
1960 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1961 		    "==> hxge_map_txdma_channel_buf_ring: dma chunk %d "
1962 		    "size %d dma_bufp $%p",
1963 		    i, sizeof (hxge_dma_common_t), dma_bufp));
1964 
1965 		for (j = 0; j < nblocks; j++) {
1966 			tx_msg_ring[index].buf_dma_handle = tx_buf_dma_handle;
1967 			tx_msg_ring[index].offset_index = j;
1968 			dmap = &tx_msg_ring[index++].buf_dma;
1969 			HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1970 			    "==> hxge_map_txdma_channel_buf_ring: j %d"
1971 			    "dmap $%p", i, dmap));
1972 			hxge_setup_dma_common(dmap, dma_bufp, 1, bsize);
1973 		}
1974 	}
1975 
1976 	if (i < num_chunks) {
1977 		status = HXGE_ERROR;
1978 
1979 		goto hxge_map_txdma_channel_buf_ring_fail1;
1980 	}
1981 
1982 	*tx_desc_p = tx_ring_p;
1983 
1984 	goto hxge_map_txdma_channel_buf_ring_exit;
1985 
1986 hxge_map_txdma_channel_buf_ring_fail1:
1987 	if (tx_ring_p->taskq) {
1988 		ddi_taskq_destroy(tx_ring_p->taskq);
1989 		tx_ring_p->taskq = NULL;
1990 	}
1991 
1992 	index--;
1993 	for (; index >= 0; index--) {
1994 		if (tx_msg_ring[index].dma_handle != NULL) {
1995 			ddi_dma_free_handle(&tx_msg_ring[index].dma_handle);
1996 		}
1997 	}
1998 	MUTEX_DESTROY(&tx_ring_p->lock);
1999 	KMEM_FREE(tx_msg_ring, size);
2000 	KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2001 
2002 	status = HXGE_ERROR;
2003 
2004 hxge_map_txdma_channel_buf_ring_exit:
2005 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2006 	    "<== hxge_map_txdma_channel_buf_ring status 0x%x", status));
2007 
2008 	return (status);
2009 }
2010 
2011 /*ARGSUSED*/
2012 static void
2013 hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep, p_tx_ring_t tx_ring_p)
2014 {
2015 	p_tx_msg_t	tx_msg_ring;
2016 	p_tx_msg_t	tx_msg_p;
2017 	int		i;
2018 
2019 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2020 	    "==> hxge_unmap_txdma_channel_buf_ring"));
2021 	if (tx_ring_p == NULL) {
2022 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2023 		    "<== hxge_unmap_txdma_channel_buf_ring: NULL ringp"));
2024 		return;
2025 	}
2026 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2027 	    "==> hxge_unmap_txdma_channel_buf_ring: channel %d",
2028 	    tx_ring_p->tdc));
2029 
2030 	MUTEX_ENTER(&tx_ring_p->lock);
2031 	tx_msg_ring = tx_ring_p->tx_msg_ring;
2032 	for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2033 		tx_msg_p = &tx_msg_ring[i];
2034 		if (tx_msg_p->flags.dma_type == USE_DVMA) {
2035 			HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "entry = %d", i));
2036 			(void) dvma_unload(tx_msg_p->dvma_handle, 0, -1);
2037 			tx_msg_p->dvma_handle = NULL;
2038 			if (tx_ring_p->dvma_wr_index ==
2039 			    tx_ring_p->dvma_wrap_mask) {
2040 				tx_ring_p->dvma_wr_index = 0;
2041 			} else {
2042 				tx_ring_p->dvma_wr_index++;
2043 			}
2044 			tx_ring_p->dvma_pending--;
2045 		} else if (tx_msg_p->flags.dma_type == USE_DMA) {
2046 			if (ddi_dma_unbind_handle(tx_msg_p->dma_handle)) {
2047 				cmn_err(CE_WARN, "hxge_unmap_tx_bug_ring: "
2048 				    "ddi_dma_unbind_handle failed.");
2049 			}
2050 		}
2051 		if (tx_msg_p->tx_message != NULL) {
2052 			freemsg(tx_msg_p->tx_message);
2053 			tx_msg_p->tx_message = NULL;
2054 		}
2055 	}
2056 
2057 	for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2058 		if (tx_msg_ring[i].dma_handle != NULL) {
2059 			ddi_dma_free_handle(&tx_msg_ring[i].dma_handle);
2060 		}
2061 	}
2062 	MUTEX_EXIT(&tx_ring_p->lock);
2063 
2064 	if (tx_ring_p->taskq) {
2065 		ddi_taskq_destroy(tx_ring_p->taskq);
2066 		tx_ring_p->taskq = NULL;
2067 	}
2068 
2069 	MUTEX_DESTROY(&tx_ring_p->lock);
2070 	KMEM_FREE(tx_msg_ring, sizeof (tx_msg_t) * tx_ring_p->tx_ring_size);
2071 	KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2072 
2073 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2074 	    "<== hxge_unmap_txdma_channel_buf_ring"));
2075 }
2076 
2077 static hxge_status_t
2078 hxge_txdma_hw_start(p_hxge_t hxgep)
2079 {
2080 	int			i, ndmas;
2081 	uint16_t		channel;
2082 	p_tx_rings_t		tx_rings;
2083 	p_tx_ring_t		*tx_desc_rings;
2084 	p_tx_mbox_areas_t	tx_mbox_areas_p;
2085 	p_tx_mbox_t		*tx_mbox_p;
2086 	hxge_status_t		status = HXGE_OK;
2087 	uint64_t		tmp;
2088 
2089 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_start"));
2090 
2091 	/*
2092 	 * Initialize REORD Table 1. Disable VMAC 2. Reset the FIFO Err Stat.
2093 	 * 3. Scrub memory and check for errors.
2094 	 */
2095 	(void) hxge_tx_vmac_disable(hxgep);
2096 
2097 	/*
2098 	 * Clear the error status
2099 	 */
2100 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, 0x7);
2101 
2102 	/*
2103 	 * Scrub the rtab memory for the TDC and reset the TDC.
2104 	 */
2105 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_HI, 0x0ULL);
2106 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_LO, 0x0ULL);
2107 
2108 	for (i = 0; i < 256; i++) {
2109 		HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2110 		    (uint64_t)i);
2111 
2112 		/*
2113 		 * Write the command register with an indirect read instruction
2114 		 */
2115 		tmp = (0x1ULL << 30) | i;
2116 		HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD, tmp);
2117 
2118 		/*
2119 		 * Wait for status done
2120 		 */
2121 		tmp = 0;
2122 		do {
2123 			HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2124 			    &tmp);
2125 		} while (((tmp >> 31) & 0x1ULL) == 0x0);
2126 	}
2127 
2128 	for (i = 0; i < 256; i++) {
2129 		/*
2130 		 * Write the command register with an indirect read instruction
2131 		 */
2132 		tmp = (0x1ULL << 30) | i;
2133 		HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD, tmp);
2134 
2135 		/*
2136 		 * Wait for status done
2137 		 */
2138 		tmp = 0;
2139 		do {
2140 			HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2141 			    &tmp);
2142 		} while (((tmp >> 31) & 0x1ULL) == 0x0);
2143 
2144 		HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_HI, &tmp);
2145 		if (0x1ff00ULL != (0x1ffffULL & tmp)) {
2146 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2147 			    "unexpected data (hi), entry: %x, value: 0x%0llx\n",
2148 			    i, (unsigned long long)tmp));
2149 			status = HXGE_ERROR;
2150 		}
2151 
2152 		HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_LO, &tmp);
2153 		if (tmp != 0) {
2154 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2155 			    "unexpected data (lo), entry: %x\n", i));
2156 			status = HXGE_ERROR;
2157 		}
2158 
2159 		HXGE_REG_RD64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, &tmp);
2160 		if (tmp != 0) {
2161 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2162 			    "parity error, entry: %x, val 0x%llx\n",
2163 			    i, (unsigned long long)tmp));
2164 			status = HXGE_ERROR;
2165 		}
2166 
2167 		HXGE_REG_RD64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, &tmp);
2168 		if (tmp != 0) {
2169 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2170 			    "parity error, entry: %x\n", i));
2171 			status = HXGE_ERROR;
2172 		}
2173 	}
2174 
2175 	if (status != HXGE_OK)
2176 		goto hxge_txdma_hw_start_exit;
2177 
2178 	/*
2179 	 * Reset FIFO Error Status for the TDC and enable FIFO error events.
2180 	 */
2181 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, 0x7);
2182 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_MASK, 0x0);
2183 
2184 	/*
2185 	 * Initialize the Transmit DMAs.
2186 	 */
2187 	tx_rings = hxgep->tx_rings;
2188 	if (tx_rings == NULL) {
2189 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2190 		    "<== hxge_txdma_hw_start: NULL ring pointer"));
2191 		return (HXGE_ERROR);
2192 	}
2193 
2194 	tx_desc_rings = tx_rings->rings;
2195 	if (tx_desc_rings == NULL) {
2196 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2197 		    "<== hxge_txdma_hw_start: NULL ring pointers"));
2198 		return (HXGE_ERROR);
2199 	}
2200 	ndmas = tx_rings->ndmas;
2201 	if (!ndmas) {
2202 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2203 		    "<== hxge_txdma_hw_start: no dma channel allocated"));
2204 		return (HXGE_ERROR);
2205 	}
2206 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_start: "
2207 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
2208 	    tx_rings, tx_desc_rings, ndmas));
2209 
2210 	tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2211 	tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2212 
2213 	/*
2214 	 * Init the DMAs.
2215 	 */
2216 	for (i = 0; i < ndmas; i++) {
2217 		channel = tx_desc_rings[i]->tdc;
2218 		status = hxge_txdma_start_channel(hxgep, channel,
2219 		    (p_tx_ring_t)tx_desc_rings[i],
2220 		    (p_tx_mbox_t)tx_mbox_p[i]);
2221 		if (status != HXGE_OK) {
2222 			goto hxge_txdma_hw_start_fail1;
2223 		}
2224 	}
2225 
2226 	(void) hxge_tx_vmac_enable(hxgep);
2227 
2228 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2229 	    "==> hxge_txdma_hw_start: tx_rings $%p rings $%p",
2230 	    hxgep->tx_rings, hxgep->tx_rings->rings));
2231 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2232 	    "==> hxge_txdma_hw_start: tx_rings $%p tx_desc_rings $%p",
2233 	    hxgep->tx_rings, tx_desc_rings));
2234 
2235 	goto hxge_txdma_hw_start_exit;
2236 
2237 hxge_txdma_hw_start_fail1:
2238 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2239 	    "==> hxge_txdma_hw_start: disable (status 0x%x channel %d i %d)",
2240 	    status, channel, i));
2241 
2242 	for (; i >= 0; i--) {
2243 		channel = tx_desc_rings[i]->tdc,
2244 		    (void) hxge_txdma_stop_channel(hxgep, channel,
2245 		    (p_tx_ring_t)tx_desc_rings[i],
2246 		    (p_tx_mbox_t)tx_mbox_p[i]);
2247 	}
2248 
2249 hxge_txdma_hw_start_exit:
2250 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2251 	    "==> hxge_txdma_hw_start: (status 0x%x)", status));
2252 
2253 	return (status);
2254 }
2255 
2256 static void
2257 hxge_txdma_hw_stop(p_hxge_t hxgep)
2258 {
2259 	int			i, ndmas;
2260 	uint16_t		channel;
2261 	p_tx_rings_t		tx_rings;
2262 	p_tx_ring_t		*tx_desc_rings;
2263 	p_tx_mbox_areas_t	tx_mbox_areas_p;
2264 	p_tx_mbox_t		*tx_mbox_p;
2265 
2266 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop"));
2267 
2268 	tx_rings = hxgep->tx_rings;
2269 	if (tx_rings == NULL) {
2270 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2271 		    "<== hxge_txdma_hw_stop: NULL ring pointer"));
2272 		return;
2273 	}
2274 
2275 	tx_desc_rings = tx_rings->rings;
2276 	if (tx_desc_rings == NULL) {
2277 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2278 		    "<== hxge_txdma_hw_stop: NULL ring pointers"));
2279 		return;
2280 	}
2281 
2282 	ndmas = tx_rings->ndmas;
2283 	if (!ndmas) {
2284 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2285 		    "<== hxge_txdma_hw_stop: no dma channel allocated"));
2286 		return;
2287 	}
2288 
2289 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop: "
2290 	    "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
2291 
2292 	tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2293 	tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2294 
2295 	for (i = 0; i < ndmas; i++) {
2296 		channel = tx_desc_rings[i]->tdc;
2297 		(void) hxge_txdma_stop_channel(hxgep, channel,
2298 		    (p_tx_ring_t)tx_desc_rings[i],
2299 		    (p_tx_mbox_t)tx_mbox_p[i]);
2300 	}
2301 
2302 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop: "
2303 	    "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
2304 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_hw_stop"));
2305 }
2306 
2307 static hxge_status_t
2308 hxge_txdma_start_channel(p_hxge_t hxgep, uint16_t channel,
2309     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2310 {
2311 	hxge_status_t status = HXGE_OK;
2312 
2313 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2314 	    "==> hxge_txdma_start_channel (channel %d)", channel));
2315 	/*
2316 	 * TXDMA/TXC must be in stopped state.
2317 	 */
2318 	(void) hxge_txdma_stop_inj_err(hxgep, channel);
2319 
2320 	/*
2321 	 * Reset TXDMA channel
2322 	 */
2323 	tx_ring_p->tx_cs.value = 0;
2324 	tx_ring_p->tx_cs.bits.reset = 1;
2325 	status = hxge_reset_txdma_channel(hxgep, channel,
2326 	    tx_ring_p->tx_cs.value);
2327 	if (status != HXGE_OK) {
2328 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2329 		    "==> hxge_txdma_start_channel (channel %d)"
2330 		    " reset channel failed 0x%x", channel, status));
2331 
2332 		goto hxge_txdma_start_channel_exit;
2333 	}
2334 
2335 	/*
2336 	 * Initialize the TXDMA channel specific FZC control configurations.
2337 	 * These FZC registers are pertaining to each TX channel (i.e. logical
2338 	 * pages).
2339 	 */
2340 	status = hxge_init_fzc_txdma_channel(hxgep, channel,
2341 	    tx_ring_p, tx_mbox_p);
2342 	if (status != HXGE_OK) {
2343 		goto hxge_txdma_start_channel_exit;
2344 	}
2345 
2346 	/*
2347 	 * Initialize the event masks.
2348 	 */
2349 	tx_ring_p->tx_evmask.value = 0;
2350 	status = hxge_init_txdma_channel_event_mask(hxgep,
2351 	    channel, &tx_ring_p->tx_evmask);
2352 	if (status != HXGE_OK) {
2353 		goto hxge_txdma_start_channel_exit;
2354 	}
2355 
2356 	/*
2357 	 * Load TXDMA descriptors, buffers, mailbox, initialise the DMA
2358 	 * channels and enable each DMA channel.
2359 	 */
2360 	status = hxge_enable_txdma_channel(hxgep, channel,
2361 	    tx_ring_p, tx_mbox_p);
2362 	if (status != HXGE_OK) {
2363 		goto hxge_txdma_start_channel_exit;
2364 	}
2365 
2366 hxge_txdma_start_channel_exit:
2367 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_start_channel"));
2368 
2369 	return (status);
2370 }
2371 
2372 /*ARGSUSED*/
2373 static hxge_status_t
2374 hxge_txdma_stop_channel(p_hxge_t hxgep, uint16_t channel,
2375     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2376 {
2377 	int status = HXGE_OK;
2378 
2379 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2380 	    "==> hxge_txdma_stop_channel: channel %d", channel));
2381 
2382 	/*
2383 	 * Stop (disable) TXDMA and TXC (if stop bit is set and STOP_N_GO bit
2384 	 * not set, the TXDMA reset state will not be set if reset TXDMA.
2385 	 */
2386 	(void) hxge_txdma_stop_inj_err(hxgep, channel);
2387 
2388 	/*
2389 	 * Reset TXDMA channel
2390 	 */
2391 	tx_ring_p->tx_cs.value = 0;
2392 	tx_ring_p->tx_cs.bits.reset = 1;
2393 	status = hxge_reset_txdma_channel(hxgep, channel,
2394 	    tx_ring_p->tx_cs.value);
2395 	if (status != HXGE_OK) {
2396 		goto hxge_txdma_stop_channel_exit;
2397 	}
2398 
2399 hxge_txdma_stop_channel_exit:
2400 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_stop_channel"));
2401 
2402 	return (status);
2403 }
2404 
2405 static p_tx_ring_t
2406 hxge_txdma_get_ring(p_hxge_t hxgep, uint16_t channel)
2407 {
2408 	int		index, ndmas;
2409 	uint16_t	tdc;
2410 	p_tx_rings_t	tx_rings;
2411 
2412 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_get_ring"));
2413 
2414 	tx_rings = hxgep->tx_rings;
2415 	if (tx_rings == NULL) {
2416 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2417 		    "<== hxge_txdma_get_ring: NULL ring pointer"));
2418 		return (NULL);
2419 	}
2420 	ndmas = tx_rings->ndmas;
2421 	if (!ndmas) {
2422 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2423 		    "<== hxge_txdma_get_ring: no channel allocated"));
2424 		return (NULL);
2425 	}
2426 	if (tx_rings->rings == NULL) {
2427 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2428 		    "<== hxge_txdma_get_ring: NULL rings pointer"));
2429 		return (NULL);
2430 	}
2431 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_get_ring: "
2432 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
2433 	    tx_rings, tx_rings, ndmas));
2434 
2435 	for (index = 0; index < ndmas; index++) {
2436 		tdc = tx_rings->rings[index]->tdc;
2437 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2438 		    "==> hxge_fixup_txdma_rings: channel %d", tdc));
2439 		if (channel == tdc) {
2440 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
2441 			    "<== hxge_txdma_get_ring: tdc %d ring $%p",
2442 			    tdc, tx_rings->rings[index]));
2443 			return (p_tx_ring_t)(tx_rings->rings[index]);
2444 		}
2445 	}
2446 
2447 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_get_ring"));
2448 
2449 	return (NULL);
2450 }
2451 
2452 static p_tx_mbox_t
2453 hxge_txdma_get_mbox(p_hxge_t hxgep, uint16_t channel)
2454 {
2455 	int			index, tdc, ndmas;
2456 	p_tx_rings_t		tx_rings;
2457 	p_tx_mbox_areas_t	tx_mbox_areas_p;
2458 	p_tx_mbox_t		*tx_mbox_p;
2459 
2460 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_get_mbox"));
2461 
2462 	tx_rings = hxgep->tx_rings;
2463 	if (tx_rings == NULL) {
2464 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2465 		    "<== hxge_txdma_get_mbox: NULL ring pointer"));
2466 		return (NULL);
2467 	}
2468 	tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2469 	if (tx_mbox_areas_p == NULL) {
2470 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2471 		    "<== hxge_txdma_get_mbox: NULL mbox pointer"));
2472 		return (NULL);
2473 	}
2474 	tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2475 
2476 	ndmas = tx_rings->ndmas;
2477 	if (!ndmas) {
2478 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2479 		    "<== hxge_txdma_get_mbox: no channel allocated"));
2480 		return (NULL);
2481 	}
2482 	if (tx_rings->rings == NULL) {
2483 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2484 		    "<== hxge_txdma_get_mbox: NULL rings pointer"));
2485 		return (NULL);
2486 	}
2487 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_get_mbox: "
2488 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
2489 	    tx_rings, tx_rings, ndmas));
2490 
2491 	for (index = 0; index < ndmas; index++) {
2492 		tdc = tx_rings->rings[index]->tdc;
2493 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2494 		    "==> hxge_txdma_get_mbox: channel %d", tdc));
2495 		if (channel == tdc) {
2496 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
2497 			    "<== hxge_txdma_get_mbox: tdc %d ring $%p",
2498 			    tdc, tx_rings->rings[index]));
2499 			return (p_tx_mbox_t)(tx_mbox_p[index]);
2500 		}
2501 	}
2502 
2503 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_get_mbox"));
2504 
2505 	return (NULL);
2506 }
2507 
2508 /*ARGSUSED*/
2509 static hxge_status_t
2510 hxge_tx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp,
2511     tdc_stat_t cs)
2512 {
2513 	hpi_handle_t		handle;
2514 	uint8_t			channel;
2515 	p_tx_ring_t		*tx_rings;
2516 	p_tx_ring_t		tx_ring_p;
2517 	p_hxge_tx_ring_stats_t	tdc_stats;
2518 	boolean_t		txchan_fatal = B_FALSE;
2519 	hxge_status_t		status = HXGE_OK;
2520 	tdc_drop_cnt_t		drop_cnt;
2521 
2522 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_tx_err_evnts"));
2523 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
2524 	channel = ldvp->channel;
2525 
2526 	tx_rings = hxgep->tx_rings->rings;
2527 	tx_ring_p = tx_rings[index];
2528 	tdc_stats = tx_ring_p->tdc_stats;
2529 
2530 	/* Get the error counts if any */
2531 	TXDMA_REG_READ64(handle, TDC_DROP_CNT, channel, &drop_cnt.value);
2532 	tdc_stats->count_hdr_size_err += drop_cnt.bits.hdr_size_error_count;
2533 	tdc_stats->count_runt += drop_cnt.bits.runt_count;
2534 	tdc_stats->count_abort += drop_cnt.bits.abort_count;
2535 
2536 	if (cs.bits.peu_resp_err) {
2537 		tdc_stats->peu_resp_err++;
2538 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2539 		    HXGE_FM_EREPORT_TDMC_PEU_RESP_ERR);
2540 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2541 		    "==> hxge_tx_err_evnts(channel %d): "
2542 		    "fatal error: peu_resp_err", channel));
2543 		txchan_fatal = B_TRUE;
2544 	}
2545 
2546 	if (cs.bits.pkt_size_hdr_err) {
2547 		tdc_stats->pkt_size_hdr_err++;
2548 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2549 		    HXGE_FM_EREPORT_TDMC_PKT_SIZE_HDR_ERR);
2550 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2551 		    "==> hxge_tx_err_evnts(channel %d): "
2552 		    "fatal error: pkt_size_hdr_err", channel));
2553 		txchan_fatal = B_TRUE;
2554 	}
2555 
2556 	if (cs.bits.runt_pkt_drop_err) {
2557 		tdc_stats->runt_pkt_drop_err++;
2558 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2559 		    HXGE_FM_EREPORT_TDMC_RUNT_PKT_DROP_ERR);
2560 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2561 		    "==> hxge_tx_err_evnts(channel %d): "
2562 		    "fatal error: runt_pkt_drop_err", channel));
2563 		txchan_fatal = B_TRUE;
2564 	}
2565 
2566 	if (cs.bits.pkt_size_err) {
2567 		tdc_stats->pkt_size_err++;
2568 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2569 		    HXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR);
2570 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2571 		    "==> hxge_tx_err_evnts(channel %d): "
2572 		    "fatal error: pkt_size_err", channel));
2573 		txchan_fatal = B_TRUE;
2574 	}
2575 
2576 	if (cs.bits.tx_rng_oflow) {
2577 		tdc_stats->tx_rng_oflow++;
2578 		if (tdc_stats->tx_rng_oflow)
2579 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2580 			    "==> hxge_tx_err_evnts(channel %d): "
2581 			    "fatal error: tx_rng_oflow", channel));
2582 	}
2583 
2584 	if (cs.bits.pref_par_err) {
2585 		tdc_stats->pref_par_err++;
2586 
2587 		/* Get the address of parity error read data */
2588 		TXDMA_REG_READ64(hxgep->hpi_handle, TDC_PREF_PAR_LOG,
2589 		    channel, &tdc_stats->errlog.value);
2590 
2591 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2592 		    HXGE_FM_EREPORT_TDMC_PREF_PAR_ERR);
2593 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2594 		    "==> hxge_tx_err_evnts(channel %d): "
2595 		    "fatal error: pref_par_err", channel));
2596 		txchan_fatal = B_TRUE;
2597 	}
2598 
2599 	if (cs.bits.tdr_pref_cpl_to) {
2600 		tdc_stats->tdr_pref_cpl_to++;
2601 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2602 		    HXGE_FM_EREPORT_TDMC_TDR_PREF_CPL_TO);
2603 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2604 		    "==> hxge_tx_err_evnts(channel %d): "
2605 		    "fatal error: tdr_pref_cpl_to", channel));
2606 		txchan_fatal = B_TRUE;
2607 	}
2608 
2609 	if (cs.bits.pkt_cpl_to) {
2610 		tdc_stats->pkt_cpl_to++;
2611 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2612 		    HXGE_FM_EREPORT_TDMC_PKT_CPL_TO);
2613 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2614 		    "==> hxge_tx_err_evnts(channel %d): "
2615 		    "fatal error: pkt_cpl_to", channel));
2616 		txchan_fatal = B_TRUE;
2617 	}
2618 
2619 	if (cs.bits.invalid_sop) {
2620 		tdc_stats->invalid_sop++;
2621 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2622 		    HXGE_FM_EREPORT_TDMC_INVALID_SOP);
2623 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2624 		    "==> hxge_tx_err_evnts(channel %d): "
2625 		    "fatal error: invalid_sop", channel));
2626 		txchan_fatal = B_TRUE;
2627 	}
2628 
2629 	if (cs.bits.unexpected_sop) {
2630 		tdc_stats->unexpected_sop++;
2631 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2632 		    HXGE_FM_EREPORT_TDMC_UNEXPECTED_SOP);
2633 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2634 		    "==> hxge_tx_err_evnts(channel %d): "
2635 		    "fatal error: unexpected_sop", channel));
2636 		txchan_fatal = B_TRUE;
2637 	}
2638 
2639 	/* Clear error injection source in case this is an injected error */
2640 	TXDMA_REG_WRITE64(hxgep->hpi_handle, TDC_STAT_INT_DBG, channel, 0);
2641 
2642 	if (txchan_fatal) {
2643 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2644 		    " hxge_tx_err_evnts: "
2645 		    " fatal error on channel %d cs 0x%llx\n",
2646 		    channel, cs.value));
2647 		status = hxge_txdma_fatal_err_recover(hxgep, channel,
2648 		    tx_ring_p);
2649 		if (status == HXGE_OK) {
2650 			FM_SERVICE_RESTORED(hxgep);
2651 		}
2652 	}
2653 
2654 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "<== hxge_tx_err_evnts"));
2655 
2656 	return (status);
2657 }
2658 
2659 hxge_status_t
2660 hxge_txdma_handle_sys_errors(p_hxge_t hxgep)
2661 {
2662 	hpi_handle_t		handle;
2663 	hxge_status_t		status = HXGE_OK;
2664 	tdc_fifo_err_stat_t	fifo_stat;
2665 	hxge_tdc_sys_stats_t	*tdc_sys_stats;
2666 
2667 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_handle_sys_errors"));
2668 
2669 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
2670 
2671 	/*
2672 	 * The FIFO is shared by all channels.
2673 	 * Get the status of Reorder Buffer and Reorder Table Buffer Errors
2674 	 */
2675 	HXGE_REG_RD64(handle, TDC_FIFO_ERR_STAT, &fifo_stat.value);
2676 
2677 	/*
2678 	 * Clear the error bits. Note that writing a 1 clears the bit. Writing
2679 	 * a 0 does nothing.
2680 	 */
2681 	HXGE_REG_WR64(handle, TDC_FIFO_ERR_STAT, fifo_stat.value);
2682 
2683 	tdc_sys_stats = &hxgep->statsp->tdc_sys_stats;
2684 	if (fifo_stat.bits.reord_tbl_par_err) {
2685 		tdc_sys_stats->reord_tbl_par_err++;
2686 		HXGE_FM_REPORT_ERROR(hxgep, NULL,
2687 		    HXGE_FM_EREPORT_TDMC_REORD_TBL_PAR);
2688 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2689 		    "==> hxge_txdma_handle_sys_errors: fatal error: "
2690 		    "reord_tbl_par_err"));
2691 	}
2692 
2693 	if (fifo_stat.bits.reord_buf_ded_err) {
2694 		tdc_sys_stats->reord_buf_ded_err++;
2695 		HXGE_FM_REPORT_ERROR(hxgep, NULL,
2696 		    HXGE_FM_EREPORT_TDMC_REORD_BUF_DED);
2697 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2698 		    "==> hxge_txdma_handle_sys_errors: "
2699 		    "fatal error: reord_buf_ded_err"));
2700 	}
2701 
2702 	if (fifo_stat.bits.reord_buf_sec_err) {
2703 		tdc_sys_stats->reord_buf_sec_err++;
2704 		if (tdc_sys_stats->reord_buf_sec_err == 1)
2705 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2706 			    "==> hxge_txdma_handle_sys_errors: "
2707 			    "reord_buf_sec_err"));
2708 	}
2709 
2710 	if (fifo_stat.bits.reord_tbl_par_err ||
2711 	    fifo_stat.bits.reord_buf_ded_err) {
2712 		status = hxge_tx_port_fatal_err_recover(hxgep);
2713 		if (status == HXGE_OK) {
2714 			FM_SERVICE_RESTORED(hxgep);
2715 		}
2716 	}
2717 
2718 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_handle_sys_errors"));
2719 
2720 	return (status);
2721 }
2722 
2723 static hxge_status_t
2724 hxge_txdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel,
2725     p_tx_ring_t tx_ring_p)
2726 {
2727 	hpi_handle_t	handle;
2728 	hpi_status_t	rs = HPI_SUCCESS;
2729 	p_tx_mbox_t	tx_mbox_p;
2730 	hxge_status_t	status = HXGE_OK;
2731 
2732 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_txdma_fatal_err_recover"));
2733 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2734 	    "Recovering from TxDMAChannel#%d error...", channel));
2735 
2736 	/*
2737 	 * Stop the dma channel waits for the stop done. If the stop done bit
2738 	 * is not set, then create an error.
2739 	 */
2740 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
2741 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "stopping txdma channel(%d)",
2742 	    channel));
2743 	MUTEX_ENTER(&tx_ring_p->lock);
2744 	rs = hpi_txdma_channel_control(handle, TXDMA_STOP, channel);
2745 	if (rs != HPI_SUCCESS) {
2746 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2747 		    "==> hxge_txdma_fatal_err_recover (channel %d): "
2748 		    "stop failed ", channel));
2749 
2750 		goto fail;
2751 	}
2752 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "reclaiming txdma channel(%d)",
2753 	    channel));
2754 	(void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
2755 
2756 	/*
2757 	 * Reset TXDMA channel
2758 	 */
2759 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "resetting txdma channel(%d)",
2760 	    channel));
2761 	if ((rs = hpi_txdma_channel_control(handle, TXDMA_RESET, channel)) !=
2762 	    HPI_SUCCESS) {
2763 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2764 		    "==> hxge_txdma_fatal_err_recover (channel %d)"
2765 		    " reset channel failed 0x%x", channel, rs));
2766 
2767 		goto fail;
2768 	}
2769 	/*
2770 	 * Reset the tail (kick) register to 0. (Hardware will not reset it. Tx
2771 	 * overflow fatal error if tail is not set to 0 after reset!
2772 	 */
2773 	TXDMA_REG_WRITE64(handle, TDC_TDR_KICK, channel, 0);
2774 
2775 	/*
2776 	 * Restart TXDMA channel
2777 	 *
2778 	 * Initialize the TXDMA channel specific FZC control configurations.
2779 	 * These FZC registers are pertaining to each TX channel (i.e. logical
2780 	 * pages).
2781 	 */
2782 	tx_mbox_p = hxge_txdma_get_mbox(hxgep, channel);
2783 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "restarting txdma channel(%d)",
2784 	    channel));
2785 	status = hxge_init_fzc_txdma_channel(hxgep, channel,
2786 	    tx_ring_p, tx_mbox_p);
2787 	if (status != HXGE_OK)
2788 		goto fail;
2789 
2790 	/*
2791 	 * Initialize the event masks.
2792 	 */
2793 	tx_ring_p->tx_evmask.value = 0;
2794 	status = hxge_init_txdma_channel_event_mask(hxgep, channel,
2795 	    &tx_ring_p->tx_evmask);
2796 	if (status != HXGE_OK)
2797 		goto fail;
2798 
2799 	tx_ring_p->wr_index_wrap = B_FALSE;
2800 	tx_ring_p->wr_index = 0;
2801 	tx_ring_p->rd_index = 0;
2802 
2803 	/*
2804 	 * Load TXDMA descriptors, buffers, mailbox, initialise the DMA
2805 	 * channels and enable each DMA channel.
2806 	 */
2807 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "enabling txdma channel(%d)",
2808 	    channel));
2809 	status = hxge_enable_txdma_channel(hxgep, channel,
2810 	    tx_ring_p, tx_mbox_p);
2811 	MUTEX_EXIT(&tx_ring_p->lock);
2812 	if (status != HXGE_OK)
2813 		goto fail;
2814 
2815 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2816 	    "Recovery Successful, TxDMAChannel#%d Restored", channel));
2817 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_txdma_fatal_err_recover"));
2818 
2819 	return (HXGE_OK);
2820 
2821 fail:
2822 	MUTEX_EXIT(&tx_ring_p->lock);
2823 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2824 	    "hxge_txdma_fatal_err_recover (channel %d): "
2825 	    "failed to recover this txdma channel", channel));
2826 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed"));
2827 
2828 	return (status);
2829 }
2830 
2831 static hxge_status_t
2832 hxge_tx_port_fatal_err_recover(p_hxge_t hxgep)
2833 {
2834 	hpi_handle_t	handle;
2835 	hpi_status_t	rs = HPI_SUCCESS;
2836 	hxge_status_t	status = HXGE_OK;
2837 	p_tx_ring_t	*tx_desc_rings;
2838 	p_tx_rings_t	tx_rings;
2839 	p_tx_ring_t	tx_ring_p;
2840 	int		i, ndmas;
2841 	uint16_t	channel;
2842 	block_reset_t	reset_reg;
2843 
2844 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2845 	    "==> hxge_tx_port_fatal_err_recover"));
2846 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2847 	    "Recovering from TxPort error..."));
2848 
2849 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
2850 
2851 	/* Reset TDC block from PEU for this fatal error */
2852 	reset_reg.value = 0;
2853 	reset_reg.bits.tdc_rst = 1;
2854 	HXGE_REG_WR32(handle, BLOCK_RESET, reset_reg.value);
2855 
2856 	HXGE_DELAY(1000);
2857 
2858 	/*
2859 	 * Stop the dma channel waits for the stop done. If the stop done bit
2860 	 * is not set, then create an error.
2861 	 */
2862 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "stopping all DMA channels..."));
2863 
2864 	tx_rings = hxgep->tx_rings;
2865 	tx_desc_rings = tx_rings->rings;
2866 	ndmas = tx_rings->ndmas;
2867 
2868 	for (i = 0; i < ndmas; i++) {
2869 		if (tx_desc_rings[i] == NULL) {
2870 			continue;
2871 		}
2872 		tx_ring_p = tx_rings->rings[i];
2873 		MUTEX_ENTER(&tx_ring_p->lock);
2874 	}
2875 
2876 	for (i = 0; i < ndmas; i++) {
2877 		if (tx_desc_rings[i] == NULL) {
2878 			continue;
2879 		}
2880 		channel = tx_desc_rings[i]->tdc;
2881 		tx_ring_p = tx_rings->rings[i];
2882 		rs = hpi_txdma_channel_control(handle, TXDMA_STOP, channel);
2883 		if (rs != HPI_SUCCESS) {
2884 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2885 			    "==> hxge_txdma_fatal_err_recover (channel %d): "
2886 			    "stop failed ", channel));
2887 
2888 			goto fail;
2889 		}
2890 	}
2891 
2892 	/*
2893 	 * Do reclaim on all of th DMAs.
2894 	 */
2895 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "reclaiming all DMA channels..."));
2896 	for (i = 0; i < ndmas; i++) {
2897 		if (tx_desc_rings[i] == NULL) {
2898 			continue;
2899 		}
2900 		tx_ring_p = tx_rings->rings[i];
2901 		(void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
2902 	}
2903 
2904 	/* Restart the TDC */
2905 	if ((status = hxge_txdma_hw_start(hxgep)) != HXGE_OK)
2906 		goto fail;
2907 
2908 	for (i = 0; i < ndmas; i++) {
2909 		if (tx_desc_rings[i] == NULL) {
2910 			continue;
2911 		}
2912 		tx_ring_p = tx_rings->rings[i];
2913 		MUTEX_EXIT(&tx_ring_p->lock);
2914 	}
2915 
2916 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2917 	    "Recovery Successful, TxPort Restored"));
2918 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2919 	    "<== hxge_tx_port_fatal_err_recover"));
2920 	return (HXGE_OK);
2921 
2922 fail:
2923 	for (i = 0; i < ndmas; i++) {
2924 		if (tx_desc_rings[i] == NULL) {
2925 			continue;
2926 		}
2927 		tx_ring_p = tx_rings->rings[i];
2928 		MUTEX_EXIT(&tx_ring_p->lock);
2929 	}
2930 
2931 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed"));
2932 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2933 	    "hxge_txdma_fatal_err_recover (channel %d): "
2934 	    "failed to recover this txdma channel"));
2935 
2936 	return (status);
2937 }
2938