xref: /titanic_51/usr/src/uts/common/io/nxge/nxge_rxdma.c (revision 7380b00ccf9fc5638da98d756925bc6d6c002a42)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <sys/nxge/nxge_impl.h>
28 #include <sys/nxge/nxge_rxdma.h>
29 #include <sys/nxge/nxge_hio.h>
30 
31 #if !defined(_BIG_ENDIAN)
32 #include <npi_rx_rd32.h>
33 #endif
34 #include <npi_rx_rd64.h>
35 #include <npi_rx_wr64.h>
36 
37 #define	NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp)	\
38 	(rdcgrp + nxgep->pt_config.hw_config.def_mac_rxdma_grpid)
39 #define	NXGE_ACTUAL_RDC(nxgep, rdc)	\
40 	(rdc + nxgep->pt_config.hw_config.start_rdc)
41 
42 /*
43  * Globals: tunable parameters (/etc/system or adb)
44  *
45  */
46 extern uint32_t nxge_rbr_size;
47 extern uint32_t nxge_rcr_size;
48 extern uint32_t	nxge_rbr_spare_size;
49 
50 extern uint32_t nxge_mblks_pending;
51 
52 /*
53  * Tunable to reduce the amount of time spent in the
54  * ISR doing Rx Processing.
55  */
56 extern uint32_t nxge_max_rx_pkts;
57 
58 /*
59  * Tunables to manage the receive buffer blocks.
60  *
61  * nxge_rx_threshold_hi: copy all buffers.
62  * nxge_rx_bcopy_size_type: receive buffer block size type.
63  * nxge_rx_threshold_lo: copy only up to tunable block size type.
64  */
65 extern nxge_rxbuf_threshold_t nxge_rx_threshold_hi;
66 extern nxge_rxbuf_type_t nxge_rx_buf_size_type;
67 extern nxge_rxbuf_threshold_t nxge_rx_threshold_lo;
68 
69 extern uint32_t	nxge_cksum_offload;
70 
71 static nxge_status_t nxge_map_rxdma(p_nxge_t, int);
72 static void nxge_unmap_rxdma(p_nxge_t, int);
73 
74 static nxge_status_t nxge_rxdma_hw_start_common(p_nxge_t);
75 
76 static nxge_status_t nxge_rxdma_hw_start(p_nxge_t, int);
77 static void nxge_rxdma_hw_stop(p_nxge_t, int);
78 
79 static nxge_status_t nxge_map_rxdma_channel(p_nxge_t, uint16_t,
80     p_nxge_dma_common_t *,  p_rx_rbr_ring_t *,
81     uint32_t,
82     p_nxge_dma_common_t *, p_rx_rcr_ring_t *,
83     p_rx_mbox_t *);
84 static void nxge_unmap_rxdma_channel(p_nxge_t, uint16_t,
85     p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t);
86 
87 static nxge_status_t nxge_map_rxdma_channel_cfg_ring(p_nxge_t,
88     uint16_t,
89     p_nxge_dma_common_t *, p_rx_rbr_ring_t *,
90     p_rx_rcr_ring_t *, p_rx_mbox_t *);
91 static void nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t,
92     p_rx_rcr_ring_t, p_rx_mbox_t);
93 
94 static nxge_status_t nxge_map_rxdma_channel_buf_ring(p_nxge_t,
95     uint16_t,
96     p_nxge_dma_common_t *,
97     p_rx_rbr_ring_t *, uint32_t);
98 static void nxge_unmap_rxdma_channel_buf_ring(p_nxge_t,
99     p_rx_rbr_ring_t);
100 
101 static nxge_status_t nxge_rxdma_start_channel(p_nxge_t, uint16_t,
102     p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t);
103 static nxge_status_t nxge_rxdma_stop_channel(p_nxge_t, uint16_t);
104 
105 static mblk_t *
106 nxge_rx_pkts(p_nxge_t, p_rx_rcr_ring_t, rx_dma_ctl_stat_t, int);
107 
108 static void nxge_receive_packet(p_nxge_t,
109 	p_rx_rcr_ring_t,
110 	p_rcr_entry_t,
111 	boolean_t *,
112 	mblk_t **, mblk_t **);
113 
114 nxge_status_t nxge_disable_rxdma_channel(p_nxge_t, uint16_t);
115 
116 static p_rx_msg_t nxge_allocb(size_t, uint32_t, p_nxge_dma_common_t);
117 static void nxge_freeb(p_rx_msg_t);
118 static nxge_status_t nxge_rx_err_evnts(p_nxge_t, int, rx_dma_ctl_stat_t);
119 
120 static nxge_status_t nxge_rxdma_handle_port_errors(p_nxge_t,
121 				uint32_t, uint32_t);
122 
123 static nxge_status_t nxge_rxbuf_index_info_init(p_nxge_t,
124     p_rx_rbr_ring_t);
125 
126 
127 static nxge_status_t
128 nxge_rxdma_fatal_err_recover(p_nxge_t, uint16_t);
129 
130 nxge_status_t
131 nxge_rx_port_fatal_err_recover(p_nxge_t);
132 
133 static void nxge_rxdma_databuf_free(p_rx_rbr_ring_t);
134 
135 nxge_status_t
136 nxge_init_rxdma_channels(p_nxge_t nxgep)
137 {
138 	nxge_grp_set_t	*set = &nxgep->rx_set;
139 	int		i, count, channel;
140 	nxge_grp_t	*group;
141 	dc_map_t	map;
142 	int		dev_gindex;
143 
144 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_init_rxdma_channels"));
145 
146 	if (!isLDOMguest(nxgep)) {
147 		if (nxge_rxdma_hw_start_common(nxgep) != NXGE_OK) {
148 			cmn_err(CE_NOTE, "hw_start_common");
149 			return (NXGE_ERROR);
150 		}
151 	}
152 
153 	/*
154 	 * NXGE_LOGICAL_GROUP_MAX > NXGE_MAX_RDC_GROUPS (8)
155 	 * We only have 8 hardware RDC tables, but we may have
156 	 * up to 16 logical (software-defined) groups of RDCS,
157 	 * if we make use of layer 3 & 4 hardware classification.
158 	 */
159 	for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) {
160 		if ((1 << i) & set->lg.map) {
161 			group = set->group[i];
162 			dev_gindex =
163 			    nxgep->pt_config.hw_config.def_mac_rxdma_grpid + i;
164 			map = nxgep->pt_config.rdc_grps[dev_gindex].map;
165 			for (channel = 0; channel < NXGE_MAX_RDCS; channel++) {
166 				if ((1 << channel) & map) {
167 					if ((nxge_grp_dc_add(nxgep,
168 					    group, VP_BOUND_RX, channel)))
169 						goto init_rxdma_channels_exit;
170 				}
171 			}
172 		}
173 		if (++count == set->lg.count)
174 			break;
175 	}
176 
177 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_rxdma_channels"));
178 	return (NXGE_OK);
179 
180 init_rxdma_channels_exit:
181 	for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) {
182 		if ((1 << i) & set->lg.map) {
183 			group = set->group[i];
184 			dev_gindex =
185 			    nxgep->pt_config.hw_config.def_mac_rxdma_grpid + i;
186 			map = nxgep->pt_config.rdc_grps[dev_gindex].map;
187 			for (channel = 0; channel < NXGE_MAX_RDCS; channel++) {
188 				if ((1 << channel) & map) {
189 					nxge_grp_dc_remove(nxgep,
190 					    VP_BOUND_RX, channel);
191 				}
192 			}
193 		}
194 		if (++count == set->lg.count)
195 			break;
196 	}
197 
198 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_rxdma_channels"));
199 	return (NXGE_ERROR);
200 }
201 
202 nxge_status_t
203 nxge_init_rxdma_channel(p_nxge_t nxge, int channel)
204 {
205 	nxge_status_t	status;
206 
207 	NXGE_DEBUG_MSG((nxge, MEM2_CTL, "==> nxge_init_rxdma_channel"));
208 
209 	status = nxge_map_rxdma(nxge, channel);
210 	if (status != NXGE_OK) {
211 		NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
212 		    "<== nxge_init_rxdma: status 0x%x", status));
213 		return (status);
214 	}
215 
216 #if defined(sun4v)
217 	if (isLDOMguest(nxge)) {
218 		/* set rcr_ring */
219 		p_rx_rcr_ring_t ring = nxge->rx_rcr_rings->rcr_rings[channel];
220 
221 		status = nxge_hio_rxdma_bind_intr(nxge, ring, channel);
222 		if (status != NXGE_OK) {
223 			nxge_unmap_rxdma(nxge, channel);
224 			return (status);
225 		}
226 	}
227 #endif
228 
229 	status = nxge_rxdma_hw_start(nxge, channel);
230 	if (status != NXGE_OK) {
231 		nxge_unmap_rxdma(nxge, channel);
232 	}
233 
234 	if (!nxge->statsp->rdc_ksp[channel])
235 		nxge_setup_rdc_kstats(nxge, channel);
236 
237 	NXGE_DEBUG_MSG((nxge, MEM2_CTL,
238 	    "<== nxge_init_rxdma_channel: status 0x%x", status));
239 
240 	return (status);
241 }
242 
243 void
244 nxge_uninit_rxdma_channels(p_nxge_t nxgep)
245 {
246 	nxge_grp_set_t *set = &nxgep->rx_set;
247 	int rdc;
248 
249 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channels"));
250 
251 	if (set->owned.map == 0) {
252 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
253 		    "nxge_uninit_rxdma_channels: no channels"));
254 		return;
255 	}
256 
257 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
258 		if ((1 << rdc) & set->owned.map) {
259 			nxge_grp_dc_remove(nxgep, VP_BOUND_RX, rdc);
260 		}
261 	}
262 
263 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uninit_rxdma_channels"));
264 }
265 
266 void
267 nxge_uninit_rxdma_channel(p_nxge_t nxgep, int channel)
268 {
269 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channel"));
270 
271 	if (nxgep->statsp->rdc_ksp[channel]) {
272 		kstat_delete(nxgep->statsp->rdc_ksp[channel]);
273 		nxgep->statsp->rdc_ksp[channel] = 0;
274 	}
275 
276 	nxge_rxdma_hw_stop(nxgep, channel);
277 	nxge_unmap_rxdma(nxgep, channel);
278 
279 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uinit_rxdma_channel"));
280 }
281 
282 nxge_status_t
283 nxge_reset_rxdma_channel(p_nxge_t nxgep, uint16_t channel)
284 {
285 	npi_handle_t		handle;
286 	npi_status_t		rs = NPI_SUCCESS;
287 	nxge_status_t		status = NXGE_OK;
288 
289 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_reset_rxdma_channel"));
290 
291 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
292 	rs = npi_rxdma_cfg_rdc_reset(handle, channel);
293 
294 	if (rs != NPI_SUCCESS) {
295 		status = NXGE_ERROR | rs;
296 	}
297 
298 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_reset_rxdma_channel"));
299 
300 	return (status);
301 }
302 
303 void
304 nxge_rxdma_regs_dump_channels(p_nxge_t nxgep)
305 {
306 	nxge_grp_set_t *set = &nxgep->rx_set;
307 	int rdc;
308 
309 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_regs_dump_channels"));
310 
311 	if (!isLDOMguest(nxgep)) {
312 		npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep);
313 		(void) npi_rxdma_dump_fzc_regs(handle);
314 	}
315 
316 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
317 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
318 		    "nxge_rxdma_regs_dump_channels: "
319 		    "NULL ring pointer(s)"));
320 		return;
321 	}
322 
323 	if (set->owned.map == 0) {
324 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
325 		    "nxge_rxdma_regs_dump_channels: no channels"));
326 		return;
327 	}
328 
329 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
330 		if ((1 << rdc) & set->owned.map) {
331 			rx_rbr_ring_t *ring =
332 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
333 			if (ring) {
334 				(void) nxge_dump_rxdma_channel(nxgep, rdc);
335 			}
336 		}
337 	}
338 
339 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_regs_dump"));
340 }
341 
342 nxge_status_t
343 nxge_dump_rxdma_channel(p_nxge_t nxgep, uint8_t channel)
344 {
345 	npi_handle_t		handle;
346 	npi_status_t		rs = NPI_SUCCESS;
347 	nxge_status_t		status = NXGE_OK;
348 
349 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_dump_rxdma_channel"));
350 
351 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
352 	rs = npi_rxdma_dump_rdc_regs(handle, channel);
353 
354 	if (rs != NPI_SUCCESS) {
355 		status = NXGE_ERROR | rs;
356 	}
357 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_dump_rxdma_channel"));
358 	return (status);
359 }
360 
361 nxge_status_t
362 nxge_init_rxdma_channel_event_mask(p_nxge_t nxgep, uint16_t channel,
363     p_rx_dma_ent_msk_t mask_p)
364 {
365 	npi_handle_t		handle;
366 	npi_status_t		rs = NPI_SUCCESS;
367 	nxge_status_t		status = NXGE_OK;
368 
369 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
370 	    "<== nxge_init_rxdma_channel_event_mask"));
371 
372 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
373 	rs = npi_rxdma_event_mask(handle, OP_SET, channel, mask_p);
374 	if (rs != NPI_SUCCESS) {
375 		status = NXGE_ERROR | rs;
376 	}
377 
378 	return (status);
379 }
380 
381 nxge_status_t
382 nxge_init_rxdma_channel_cntl_stat(p_nxge_t nxgep, uint16_t channel,
383     p_rx_dma_ctl_stat_t cs_p)
384 {
385 	npi_handle_t		handle;
386 	npi_status_t		rs = NPI_SUCCESS;
387 	nxge_status_t		status = NXGE_OK;
388 
389 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
390 	    "<== nxge_init_rxdma_channel_cntl_stat"));
391 
392 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
393 	rs = npi_rxdma_control_status(handle, OP_SET, channel, cs_p);
394 
395 	if (rs != NPI_SUCCESS) {
396 		status = NXGE_ERROR | rs;
397 	}
398 
399 	return (status);
400 }
401 
402 /*
403  * nxge_rxdma_cfg_rdcgrp_default_rdc
404  *
405  *	Set the default RDC for an RDC Group (Table)
406  *
407  * Arguments:
408  * 	nxgep
409  *	rdcgrp	The group to modify
410  *	rdc	The new default RDC.
411  *
412  * Notes:
413  *
414  * NPI/NXGE function calls:
415  *	npi_rxdma_cfg_rdc_table_default_rdc()
416  *
417  * Registers accessed:
418  *	RDC_TBL_REG: FZC_ZCP + 0x10000
419  *
420  * Context:
421  *	Service domain
422  */
423 nxge_status_t
424 nxge_rxdma_cfg_rdcgrp_default_rdc(
425 	p_nxge_t nxgep,
426 	uint8_t rdcgrp,
427 	uint8_t rdc)
428 {
429 	npi_handle_t		handle;
430 	npi_status_t		rs = NPI_SUCCESS;
431 	p_nxge_dma_pt_cfg_t	p_dma_cfgp;
432 	p_nxge_rdc_grp_t	rdc_grp_p;
433 	uint8_t actual_rdcgrp, actual_rdc;
434 
435 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
436 	    " ==> nxge_rxdma_cfg_rdcgrp_default_rdc"));
437 	p_dma_cfgp = (p_nxge_dma_pt_cfg_t)&nxgep->pt_config;
438 
439 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
440 
441 	/*
442 	 * This has to be rewritten.  Do we even allow this anymore?
443 	 */
444 	rdc_grp_p = &p_dma_cfgp->rdc_grps[rdcgrp];
445 	RDC_MAP_IN(rdc_grp_p->map, rdc);
446 	rdc_grp_p->def_rdc = rdc;
447 
448 	actual_rdcgrp = NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp);
449 	actual_rdc = NXGE_ACTUAL_RDC(nxgep, rdc);
450 
451 	rs = npi_rxdma_cfg_rdc_table_default_rdc(
452 	    handle, actual_rdcgrp, actual_rdc);
453 
454 	if (rs != NPI_SUCCESS) {
455 		return (NXGE_ERROR | rs);
456 	}
457 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
458 	    " <== nxge_rxdma_cfg_rdcgrp_default_rdc"));
459 	return (NXGE_OK);
460 }
461 
462 nxge_status_t
463 nxge_rxdma_cfg_port_default_rdc(p_nxge_t nxgep, uint8_t port, uint8_t rdc)
464 {
465 	npi_handle_t		handle;
466 
467 	uint8_t actual_rdc;
468 	npi_status_t		rs = NPI_SUCCESS;
469 
470 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
471 	    " ==> nxge_rxdma_cfg_port_default_rdc"));
472 
473 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
474 	actual_rdc = rdc;	/* XXX Hack! */
475 	rs = npi_rxdma_cfg_default_port_rdc(handle, port, actual_rdc);
476 
477 
478 	if (rs != NPI_SUCCESS) {
479 		return (NXGE_ERROR | rs);
480 	}
481 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
482 	    " <== nxge_rxdma_cfg_port_default_rdc"));
483 
484 	return (NXGE_OK);
485 }
486 
487 nxge_status_t
488 nxge_rxdma_cfg_rcr_threshold(p_nxge_t nxgep, uint8_t channel,
489 				    uint16_t pkts)
490 {
491 	npi_status_t	rs = NPI_SUCCESS;
492 	npi_handle_t	handle;
493 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
494 	    " ==> nxge_rxdma_cfg_rcr_threshold"));
495 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
496 
497 	rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel, pkts);
498 
499 	if (rs != NPI_SUCCESS) {
500 		return (NXGE_ERROR | rs);
501 	}
502 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_threshold"));
503 	return (NXGE_OK);
504 }
505 
506 nxge_status_t
507 nxge_rxdma_cfg_rcr_timeout(p_nxge_t nxgep, uint8_t channel,
508 			    uint16_t tout, uint8_t enable)
509 {
510 	npi_status_t	rs = NPI_SUCCESS;
511 	npi_handle_t	handle;
512 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, " ==> nxge_rxdma_cfg_rcr_timeout"));
513 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
514 	if (enable == 0) {
515 		rs = npi_rxdma_cfg_rdc_rcr_timeout_disable(handle, channel);
516 	} else {
517 		rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel,
518 		    tout);
519 	}
520 
521 	if (rs != NPI_SUCCESS) {
522 		return (NXGE_ERROR | rs);
523 	}
524 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_timeout"));
525 	return (NXGE_OK);
526 }
527 
528 nxge_status_t
529 nxge_enable_rxdma_channel(p_nxge_t nxgep, uint16_t channel,
530     p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p)
531 {
532 	npi_handle_t		handle;
533 	rdc_desc_cfg_t 		rdc_desc;
534 	p_rcrcfig_b_t		cfgb_p;
535 	npi_status_t		rs = NPI_SUCCESS;
536 
537 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel"));
538 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
539 	/*
540 	 * Use configuration data composed at init time.
541 	 * Write to hardware the receive ring configurations.
542 	 */
543 	rdc_desc.mbox_enable = 1;
544 	rdc_desc.mbox_addr = mbox_p->mbox_addr;
545 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
546 	    "==> nxge_enable_rxdma_channel: mboxp $%p($%p)",
547 	    mbox_p->mbox_addr, rdc_desc.mbox_addr));
548 
549 	rdc_desc.rbr_len = rbr_p->rbb_max;
550 	rdc_desc.rbr_addr = rbr_p->rbr_addr;
551 
552 	switch (nxgep->rx_bksize_code) {
553 	case RBR_BKSIZE_4K:
554 		rdc_desc.page_size = SIZE_4KB;
555 		break;
556 	case RBR_BKSIZE_8K:
557 		rdc_desc.page_size = SIZE_8KB;
558 		break;
559 	case RBR_BKSIZE_16K:
560 		rdc_desc.page_size = SIZE_16KB;
561 		break;
562 	case RBR_BKSIZE_32K:
563 		rdc_desc.page_size = SIZE_32KB;
564 		break;
565 	}
566 
567 	rdc_desc.size0 = rbr_p->npi_pkt_buf_size0;
568 	rdc_desc.valid0 = 1;
569 
570 	rdc_desc.size1 = rbr_p->npi_pkt_buf_size1;
571 	rdc_desc.valid1 = 1;
572 
573 	rdc_desc.size2 = rbr_p->npi_pkt_buf_size2;
574 	rdc_desc.valid2 = 1;
575 
576 	rdc_desc.full_hdr = rcr_p->full_hdr_flag;
577 	rdc_desc.offset = rcr_p->sw_priv_hdr_len;
578 
579 	rdc_desc.rcr_len = rcr_p->comp_size;
580 	rdc_desc.rcr_addr = rcr_p->rcr_addr;
581 
582 	cfgb_p = &(rcr_p->rcr_cfgb);
583 	rdc_desc.rcr_threshold = cfgb_p->bits.ldw.pthres;
584 	/* For now, disable this timeout in a guest domain. */
585 	if (isLDOMguest(nxgep)) {
586 		rdc_desc.rcr_timeout = 0;
587 		rdc_desc.rcr_timeout_enable = 0;
588 	} else {
589 		rdc_desc.rcr_timeout = cfgb_p->bits.ldw.timeout;
590 		rdc_desc.rcr_timeout_enable = cfgb_p->bits.ldw.entout;
591 	}
592 
593 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: "
594 	    "rbr_len qlen %d pagesize code %d rcr_len %d",
595 	    rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len));
596 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: "
597 	    "size 0 %d size 1 %d size 2 %d",
598 	    rbr_p->npi_pkt_buf_size0, rbr_p->npi_pkt_buf_size1,
599 	    rbr_p->npi_pkt_buf_size2));
600 
601 	rs = npi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc);
602 	if (rs != NPI_SUCCESS) {
603 		return (NXGE_ERROR | rs);
604 	}
605 
606 	/*
607 	 * Enable the timeout and threshold.
608 	 */
609 	rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel,
610 	    rdc_desc.rcr_threshold);
611 	if (rs != NPI_SUCCESS) {
612 		return (NXGE_ERROR | rs);
613 	}
614 
615 	rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel,
616 	    rdc_desc.rcr_timeout);
617 	if (rs != NPI_SUCCESS) {
618 		return (NXGE_ERROR | rs);
619 	}
620 
621 	if (!isLDOMguest(nxgep)) {
622 		/* Enable the DMA */
623 		rs = npi_rxdma_cfg_rdc_enable(handle, channel);
624 		if (rs != NPI_SUCCESS) {
625 			return (NXGE_ERROR | rs);
626 		}
627 	}
628 
629 	/* Kick the DMA engine. */
630 	npi_rxdma_rdc_rbr_kick(handle, channel, rbr_p->rbb_max);
631 
632 	if (!isLDOMguest(nxgep)) {
633 		/* Clear the rbr empty bit */
634 		(void) npi_rxdma_channel_rbr_empty_clear(handle, channel);
635 	}
636 
637 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_enable_rxdma_channel"));
638 
639 	return (NXGE_OK);
640 }
641 
642 nxge_status_t
643 nxge_disable_rxdma_channel(p_nxge_t nxgep, uint16_t channel)
644 {
645 	npi_handle_t		handle;
646 	npi_status_t		rs = NPI_SUCCESS;
647 
648 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_disable_rxdma_channel"));
649 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
650 
651 	/* disable the DMA */
652 	rs = npi_rxdma_cfg_rdc_disable(handle, channel);
653 	if (rs != NPI_SUCCESS) {
654 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
655 		    "<== nxge_disable_rxdma_channel:failed (0x%x)",
656 		    rs));
657 		return (NXGE_ERROR | rs);
658 	}
659 
660 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_disable_rxdma_channel"));
661 	return (NXGE_OK);
662 }
663 
664 nxge_status_t
665 nxge_rxdma_channel_rcrflush(p_nxge_t nxgep, uint8_t channel)
666 {
667 	npi_handle_t		handle;
668 	nxge_status_t		status = NXGE_OK;
669 
670 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
671 	    "<== nxge_init_rxdma_channel_rcrflush"));
672 
673 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
674 	npi_rxdma_rdc_rcr_flush(handle, channel);
675 
676 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
677 	    "<== nxge_init_rxdma_channel_rcrflsh"));
678 	return (status);
679 
680 }
681 
682 #define	MID_INDEX(l, r) ((r + l + 1) >> 1)
683 
684 #define	TO_LEFT -1
685 #define	TO_RIGHT 1
686 #define	BOTH_RIGHT (TO_RIGHT + TO_RIGHT)
687 #define	BOTH_LEFT (TO_LEFT + TO_LEFT)
688 #define	IN_MIDDLE (TO_RIGHT + TO_LEFT)
689 #define	NO_HINT 0xffffffff
690 
691 /*ARGSUSED*/
692 nxge_status_t
693 nxge_rxbuf_pp_to_vp(p_nxge_t nxgep, p_rx_rbr_ring_t rbr_p,
694 	uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp,
695 	uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index)
696 {
697 	int			bufsize;
698 	uint64_t		pktbuf_pp;
699 	uint64_t 		dvma_addr;
700 	rxring_info_t 		*ring_info;
701 	int 			base_side, end_side;
702 	int 			r_index, l_index, anchor_index;
703 	int 			found, search_done;
704 	uint32_t offset, chunk_size, block_size, page_size_mask;
705 	uint32_t chunk_index, block_index, total_index;
706 	int 			max_iterations, iteration;
707 	rxbuf_index_info_t 	*bufinfo;
708 
709 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_rxbuf_pp_to_vp"));
710 
711 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
712 	    "==> nxge_rxbuf_pp_to_vp: buf_pp $%p btype %d",
713 	    pkt_buf_addr_pp,
714 	    pktbufsz_type));
715 #if defined(__i386)
716 	pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp;
717 #else
718 	pktbuf_pp = (uint64_t)pkt_buf_addr_pp;
719 #endif
720 
721 	switch (pktbufsz_type) {
722 	case 0:
723 		bufsize = rbr_p->pkt_buf_size0;
724 		break;
725 	case 1:
726 		bufsize = rbr_p->pkt_buf_size1;
727 		break;
728 	case 2:
729 		bufsize = rbr_p->pkt_buf_size2;
730 		break;
731 	case RCR_SINGLE_BLOCK:
732 		bufsize = 0;
733 		anchor_index = 0;
734 		break;
735 	default:
736 		return (NXGE_ERROR);
737 	}
738 
739 	if (rbr_p->num_blocks == 1) {
740 		anchor_index = 0;
741 		ring_info = rbr_p->ring_info;
742 		bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
743 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
744 		    "==> nxge_rxbuf_pp_to_vp: (found, 1 block) "
745 		    "buf_pp $%p btype %d anchor_index %d "
746 		    "bufinfo $%p",
747 		    pkt_buf_addr_pp,
748 		    pktbufsz_type,
749 		    anchor_index,
750 		    bufinfo));
751 
752 		goto found_index;
753 	}
754 
755 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
756 	    "==> nxge_rxbuf_pp_to_vp: "
757 	    "buf_pp $%p btype %d  anchor_index %d",
758 	    pkt_buf_addr_pp,
759 	    pktbufsz_type,
760 	    anchor_index));
761 
762 	ring_info = rbr_p->ring_info;
763 	found = B_FALSE;
764 	bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
765 	iteration = 0;
766 	max_iterations = ring_info->max_iterations;
767 		/*
768 		 * First check if this block has been seen
769 		 * recently. This is indicated by a hint which
770 		 * is initialized when the first buffer of the block
771 		 * is seen. The hint is reset when the last buffer of
772 		 * the block has been processed.
773 		 * As three block sizes are supported, three hints
774 		 * are kept. The idea behind the hints is that once
775 		 * the hardware  uses a block for a buffer  of that
776 		 * size, it will use it exclusively for that size
777 		 * and will use it until it is exhausted. It is assumed
778 		 * that there would a single block being used for the same
779 		 * buffer sizes at any given time.
780 		 */
781 	if (ring_info->hint[pktbufsz_type] != NO_HINT) {
782 		anchor_index = ring_info->hint[pktbufsz_type];
783 		dvma_addr =  bufinfo[anchor_index].dvma_addr;
784 		chunk_size = bufinfo[anchor_index].buf_size;
785 		if ((pktbuf_pp >= dvma_addr) &&
786 		    (pktbuf_pp < (dvma_addr + chunk_size))) {
787 			found = B_TRUE;
788 				/*
789 				 * check if this is the last buffer in the block
790 				 * If so, then reset the hint for the size;
791 				 */
792 
793 			if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size))
794 				ring_info->hint[pktbufsz_type] = NO_HINT;
795 		}
796 	}
797 
798 	if (found == B_FALSE) {
799 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
800 		    "==> nxge_rxbuf_pp_to_vp: (!found)"
801 		    "buf_pp $%p btype %d anchor_index %d",
802 		    pkt_buf_addr_pp,
803 		    pktbufsz_type,
804 		    anchor_index));
805 
806 			/*
807 			 * This is the first buffer of the block of this
808 			 * size. Need to search the whole information
809 			 * array.
810 			 * the search algorithm uses a binary tree search
811 			 * algorithm. It assumes that the information is
812 			 * already sorted with increasing order
813 			 * info[0] < info[1] < info[2]  .... < info[n-1]
814 			 * where n is the size of the information array
815 			 */
816 		r_index = rbr_p->num_blocks - 1;
817 		l_index = 0;
818 		search_done = B_FALSE;
819 		anchor_index = MID_INDEX(r_index, l_index);
820 		while (search_done == B_FALSE) {
821 			if ((r_index == l_index) ||
822 			    (iteration >= max_iterations))
823 				search_done = B_TRUE;
824 			end_side = TO_RIGHT; /* to the right */
825 			base_side = TO_LEFT; /* to the left */
826 			/* read the DVMA address information and sort it */
827 			dvma_addr =  bufinfo[anchor_index].dvma_addr;
828 			chunk_size = bufinfo[anchor_index].buf_size;
829 			NXGE_DEBUG_MSG((nxgep, RX2_CTL,
830 			    "==> nxge_rxbuf_pp_to_vp: (searching)"
831 			    "buf_pp $%p btype %d "
832 			    "anchor_index %d chunk_size %d dvmaaddr $%p",
833 			    pkt_buf_addr_pp,
834 			    pktbufsz_type,
835 			    anchor_index,
836 			    chunk_size,
837 			    dvma_addr));
838 
839 			if (pktbuf_pp >= dvma_addr)
840 				base_side = TO_RIGHT; /* to the right */
841 			if (pktbuf_pp < (dvma_addr + chunk_size))
842 				end_side = TO_LEFT; /* to the left */
843 
844 			switch (base_side + end_side) {
845 			case IN_MIDDLE:
846 				/* found */
847 				found = B_TRUE;
848 				search_done = B_TRUE;
849 				if ((pktbuf_pp + bufsize) <
850 				    (dvma_addr + chunk_size))
851 					ring_info->hint[pktbufsz_type] =
852 					    bufinfo[anchor_index].buf_index;
853 				break;
854 			case BOTH_RIGHT:
855 				/* not found: go to the right */
856 				l_index = anchor_index + 1;
857 				anchor_index = MID_INDEX(r_index, l_index);
858 				break;
859 
860 			case BOTH_LEFT:
861 				/* not found: go to the left */
862 				r_index = anchor_index - 1;
863 				anchor_index = MID_INDEX(r_index, l_index);
864 				break;
865 			default: /* should not come here */
866 				return (NXGE_ERROR);
867 			}
868 			iteration++;
869 		}
870 
871 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
872 		    "==> nxge_rxbuf_pp_to_vp: (search done)"
873 		    "buf_pp $%p btype %d anchor_index %d",
874 		    pkt_buf_addr_pp,
875 		    pktbufsz_type,
876 		    anchor_index));
877 	}
878 
879 	if (found == B_FALSE) {
880 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
881 		    "==> nxge_rxbuf_pp_to_vp: (search failed)"
882 		    "buf_pp $%p btype %d anchor_index %d",
883 		    pkt_buf_addr_pp,
884 		    pktbufsz_type,
885 		    anchor_index));
886 		return (NXGE_ERROR);
887 	}
888 
889 found_index:
890 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
891 	    "==> nxge_rxbuf_pp_to_vp: (FOUND1)"
892 	    "buf_pp $%p btype %d bufsize %d anchor_index %d",
893 	    pkt_buf_addr_pp,
894 	    pktbufsz_type,
895 	    bufsize,
896 	    anchor_index));
897 
898 	/* index of the first block in this chunk */
899 	chunk_index = bufinfo[anchor_index].start_index;
900 	dvma_addr =  bufinfo[anchor_index].dvma_addr;
901 	page_size_mask = ring_info->block_size_mask;
902 
903 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
904 	    "==> nxge_rxbuf_pp_to_vp: (FOUND3), get chunk)"
905 	    "buf_pp $%p btype %d bufsize %d "
906 	    "anchor_index %d chunk_index %d dvma $%p",
907 	    pkt_buf_addr_pp,
908 	    pktbufsz_type,
909 	    bufsize,
910 	    anchor_index,
911 	    chunk_index,
912 	    dvma_addr));
913 
914 	offset = pktbuf_pp - dvma_addr; /* offset within the chunk */
915 	block_size = rbr_p->block_size; /* System  block(page) size */
916 
917 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
918 	    "==> nxge_rxbuf_pp_to_vp: (FOUND4), get chunk)"
919 	    "buf_pp $%p btype %d bufsize %d "
920 	    "anchor_index %d chunk_index %d dvma $%p "
921 	    "offset %d block_size %d",
922 	    pkt_buf_addr_pp,
923 	    pktbufsz_type,
924 	    bufsize,
925 	    anchor_index,
926 	    chunk_index,
927 	    dvma_addr,
928 	    offset,
929 	    block_size));
930 
931 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> getting total index"));
932 
933 	block_index = (offset / block_size); /* index within chunk */
934 	total_index = chunk_index + block_index;
935 
936 
937 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
938 	    "==> nxge_rxbuf_pp_to_vp: "
939 	    "total_index %d dvma_addr $%p "
940 	    "offset %d block_size %d "
941 	    "block_index %d ",
942 	    total_index, dvma_addr,
943 	    offset, block_size,
944 	    block_index));
945 #if defined(__i386)
946 	*pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr +
947 	    (uint32_t)offset);
948 #else
949 	*pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr +
950 	    (uint64_t)offset);
951 #endif
952 
953 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
954 	    "==> nxge_rxbuf_pp_to_vp: "
955 	    "total_index %d dvma_addr $%p "
956 	    "offset %d block_size %d "
957 	    "block_index %d "
958 	    "*pkt_buf_addr_p $%p",
959 	    total_index, dvma_addr,
960 	    offset, block_size,
961 	    block_index,
962 	    *pkt_buf_addr_p));
963 
964 
965 	*msg_index = total_index;
966 	*bufoffset =  (offset & page_size_mask);
967 
968 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
969 	    "==> nxge_rxbuf_pp_to_vp: get msg index: "
970 	    "msg_index %d bufoffset_index %d",
971 	    *msg_index,
972 	    *bufoffset));
973 
974 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rxbuf_pp_to_vp"));
975 
976 	return (NXGE_OK);
977 }
978 
979 /*
980  * used by quick sort (qsort) function
981  * to perform comparison
982  */
983 static int
984 nxge_sort_compare(const void *p1, const void *p2)
985 {
986 
987 	rxbuf_index_info_t *a, *b;
988 
989 	a = (rxbuf_index_info_t *)p1;
990 	b = (rxbuf_index_info_t *)p2;
991 
992 	if (a->dvma_addr > b->dvma_addr)
993 		return (1);
994 	if (a->dvma_addr < b->dvma_addr)
995 		return (-1);
996 	return (0);
997 }
998 
999 
1000 
1001 /*
1002  * grabbed this sort implementation from common/syscall/avl.c
1003  *
1004  */
1005 /*
1006  * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
1007  * v = Ptr to array/vector of objs
1008  * n = # objs in the array
1009  * s = size of each obj (must be multiples of a word size)
1010  * f = ptr to function to compare two objs
1011  *	returns (-1 = less than, 0 = equal, 1 = greater than
1012  */
1013 void
1014 nxge_ksort(caddr_t v, int n, int s, int (*f)())
1015 {
1016 	int g, i, j, ii;
1017 	unsigned int *p1, *p2;
1018 	unsigned int tmp;
1019 
1020 	/* No work to do */
1021 	if (v == NULL || n <= 1)
1022 		return;
1023 	/* Sanity check on arguments */
1024 	ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
1025 	ASSERT(s > 0);
1026 
1027 	for (g = n / 2; g > 0; g /= 2) {
1028 		for (i = g; i < n; i++) {
1029 			for (j = i - g; j >= 0 &&
1030 			    (*f)(v + j * s, v + (j + g) * s) == 1;
1031 			    j -= g) {
1032 				p1 = (unsigned *)(v + j * s);
1033 				p2 = (unsigned *)(v + (j + g) * s);
1034 				for (ii = 0; ii < s / 4; ii++) {
1035 					tmp = *p1;
1036 					*p1++ = *p2;
1037 					*p2++ = tmp;
1038 				}
1039 			}
1040 		}
1041 	}
1042 }
1043 
1044 /*
1045  * Initialize data structures required for rxdma
1046  * buffer dvma->vmem address lookup
1047  */
1048 /*ARGSUSED*/
1049 static nxge_status_t
1050 nxge_rxbuf_index_info_init(p_nxge_t nxgep, p_rx_rbr_ring_t rbrp)
1051 {
1052 
1053 	int index;
1054 	rxring_info_t *ring_info;
1055 	int max_iteration = 0, max_index = 0;
1056 
1057 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_rxbuf_index_info_init"));
1058 
1059 	ring_info = rbrp->ring_info;
1060 	ring_info->hint[0] = NO_HINT;
1061 	ring_info->hint[1] = NO_HINT;
1062 	ring_info->hint[2] = NO_HINT;
1063 	max_index = rbrp->num_blocks;
1064 
1065 		/* read the DVMA address information and sort it */
1066 		/* do init of the information array */
1067 
1068 
1069 	NXGE_DEBUG_MSG((nxgep, DMA2_CTL,
1070 	    " nxge_rxbuf_index_info_init Sort ptrs"));
1071 
1072 		/* sort the array */
1073 	nxge_ksort((void *)ring_info->buffer, max_index,
1074 	    sizeof (rxbuf_index_info_t), nxge_sort_compare);
1075 
1076 
1077 
1078 	for (index = 0; index < max_index; index++) {
1079 		NXGE_DEBUG_MSG((nxgep, DMA2_CTL,
1080 		    " nxge_rxbuf_index_info_init: sorted chunk %d "
1081 		    " ioaddr $%p kaddr $%p size %x",
1082 		    index, ring_info->buffer[index].dvma_addr,
1083 		    ring_info->buffer[index].kaddr,
1084 		    ring_info->buffer[index].buf_size));
1085 	}
1086 
1087 	max_iteration = 0;
1088 	while (max_index >= (1ULL << max_iteration))
1089 		max_iteration++;
1090 	ring_info->max_iterations = max_iteration + 1;
1091 	NXGE_DEBUG_MSG((nxgep, DMA2_CTL,
1092 	    " nxge_rxbuf_index_info_init Find max iter %d",
1093 	    ring_info->max_iterations));
1094 
1095 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxbuf_index_info_init"));
1096 	return (NXGE_OK);
1097 }
1098 
1099 /* ARGSUSED */
1100 void
1101 nxge_dump_rcr_entry(p_nxge_t nxgep, p_rcr_entry_t entry_p)
1102 {
1103 #ifdef	NXGE_DEBUG
1104 
1105 	uint32_t bptr;
1106 	uint64_t pp;
1107 
1108 	bptr = entry_p->bits.hdw.pkt_buf_addr;
1109 
1110 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1111 	    "\trcr entry $%p "
1112 	    "\trcr entry 0x%0llx "
1113 	    "\trcr entry 0x%08x "
1114 	    "\trcr entry 0x%08x "
1115 	    "\tvalue 0x%0llx\n"
1116 	    "\tmulti = %d\n"
1117 	    "\tpkt_type = 0x%x\n"
1118 	    "\tzero_copy = %d\n"
1119 	    "\tnoport = %d\n"
1120 	    "\tpromis = %d\n"
1121 	    "\terror = 0x%04x\n"
1122 	    "\tdcf_err = 0x%01x\n"
1123 	    "\tl2_len = %d\n"
1124 	    "\tpktbufsize = %d\n"
1125 	    "\tpkt_buf_addr = $%p\n"
1126 	    "\tpkt_buf_addr (<< 6) = $%p\n",
1127 	    entry_p,
1128 	    *(int64_t *)entry_p,
1129 	    *(int32_t *)entry_p,
1130 	    *(int32_t *)((char *)entry_p + 32),
1131 	    entry_p->value,
1132 	    entry_p->bits.hdw.multi,
1133 	    entry_p->bits.hdw.pkt_type,
1134 	    entry_p->bits.hdw.zero_copy,
1135 	    entry_p->bits.hdw.noport,
1136 	    entry_p->bits.hdw.promis,
1137 	    entry_p->bits.hdw.error,
1138 	    entry_p->bits.hdw.dcf_err,
1139 	    entry_p->bits.hdw.l2_len,
1140 	    entry_p->bits.hdw.pktbufsz,
1141 	    bptr,
1142 	    entry_p->bits.ldw.pkt_buf_addr));
1143 
1144 	pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) <<
1145 	    RCR_PKT_BUF_ADDR_SHIFT;
1146 
1147 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "rcr pp 0x%llx l2 len %d",
1148 	    pp, (*(int64_t *)entry_p >> 40) & 0x3fff));
1149 #endif
1150 }
1151 
1152 void
1153 nxge_rxdma_regs_dump(p_nxge_t nxgep, int rdc)
1154 {
1155 	npi_handle_t		handle;
1156 	rbr_stat_t 		rbr_stat;
1157 	addr44_t 		hd_addr;
1158 	addr44_t 		tail_addr;
1159 	uint16_t 		qlen;
1160 
1161 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1162 	    "==> nxge_rxdma_regs_dump: rdc channel %d", rdc));
1163 
1164 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
1165 
1166 	/* RBR head */
1167 	hd_addr.addr = 0;
1168 	(void) npi_rxdma_rdc_rbr_head_get(handle, rdc, &hd_addr);
1169 #if defined(__i386)
1170 	printf("nxge_rxdma_regs_dump: got hdptr $%p \n",
1171 	    (void *)(uint32_t)hd_addr.addr);
1172 #else
1173 	printf("nxge_rxdma_regs_dump: got hdptr $%p \n",
1174 	    (void *)hd_addr.addr);
1175 #endif
1176 
1177 	/* RBR stats */
1178 	(void) npi_rxdma_rdc_rbr_stat_get(handle, rdc, &rbr_stat);
1179 	printf("nxge_rxdma_regs_dump: rbr len %d \n", rbr_stat.bits.ldw.qlen);
1180 
1181 	/* RCR tail */
1182 	tail_addr.addr = 0;
1183 	(void) npi_rxdma_rdc_rcr_tail_get(handle, rdc, &tail_addr);
1184 #if defined(__i386)
1185 	printf("nxge_rxdma_regs_dump: got tail ptr $%p \n",
1186 	    (void *)(uint32_t)tail_addr.addr);
1187 #else
1188 	printf("nxge_rxdma_regs_dump: got tail ptr $%p \n",
1189 	    (void *)tail_addr.addr);
1190 #endif
1191 
1192 	/* RCR qlen */
1193 	(void) npi_rxdma_rdc_rcr_qlen_get(handle, rdc, &qlen);
1194 	printf("nxge_rxdma_regs_dump: rcr len %x \n", qlen);
1195 
1196 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1197 	    "<== nxge_rxdma_regs_dump: rdc rdc %d", rdc));
1198 }
1199 
1200 nxge_status_t
1201 nxge_rxdma_hw_mode(p_nxge_t nxgep, boolean_t enable)
1202 {
1203 	nxge_grp_set_t *set = &nxgep->rx_set;
1204 	nxge_status_t status;
1205 	npi_status_t rs;
1206 	int rdc;
1207 
1208 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
1209 	    "==> nxge_rxdma_hw_mode: mode %d", enable));
1210 
1211 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
1212 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1213 		    "<== nxge_rxdma_mode: not initialized"));
1214 		return (NXGE_ERROR);
1215 	}
1216 
1217 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
1218 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1219 		    "<== nxge_tx_port_fatal_err_recover: "
1220 		    "NULL ring pointer(s)"));
1221 		return (NXGE_ERROR);
1222 	}
1223 
1224 	if (set->owned.map == 0) {
1225 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1226 		    "nxge_rxdma_regs_dump_channels: no channels"));
1227 		return (NULL);
1228 	}
1229 
1230 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
1231 		if ((1 << rdc) & set->owned.map) {
1232 			rx_rbr_ring_t *ring =
1233 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
1234 			npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep);
1235 			if (ring) {
1236 				if (enable) {
1237 					NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
1238 					    "==> nxge_rxdma_hw_mode: "
1239 					    "channel %d (enable)", rdc));
1240 					rs = npi_rxdma_cfg_rdc_enable
1241 					    (handle, rdc);
1242 				} else {
1243 					NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
1244 					    "==> nxge_rxdma_hw_mode: "
1245 					    "channel %d disable)", rdc));
1246 					rs = npi_rxdma_cfg_rdc_disable
1247 					    (handle, rdc);
1248 				}
1249 			}
1250 		}
1251 	}
1252 
1253 	status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs);
1254 
1255 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
1256 	    "<== nxge_rxdma_hw_mode: status 0x%x", status));
1257 
1258 	return (status);
1259 }
1260 
1261 void
1262 nxge_rxdma_enable_channel(p_nxge_t nxgep, uint16_t channel)
1263 {
1264 	npi_handle_t		handle;
1265 
1266 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
1267 	    "==> nxge_rxdma_enable_channel: channel %d", channel));
1268 
1269 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
1270 	(void) npi_rxdma_cfg_rdc_enable(handle, channel);
1271 
1272 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_enable_channel"));
1273 }
1274 
1275 void
1276 nxge_rxdma_disable_channel(p_nxge_t nxgep, uint16_t channel)
1277 {
1278 	npi_handle_t		handle;
1279 
1280 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
1281 	    "==> nxge_rxdma_disable_channel: channel %d", channel));
1282 
1283 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
1284 	(void) npi_rxdma_cfg_rdc_disable(handle, channel);
1285 
1286 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_disable_channel"));
1287 }
1288 
1289 void
1290 nxge_hw_start_rx(p_nxge_t nxgep)
1291 {
1292 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_hw_start_rx"));
1293 
1294 	(void) nxge_rxdma_hw_mode(nxgep, NXGE_DMA_START);
1295 	(void) nxge_rx_mac_enable(nxgep);
1296 
1297 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_hw_start_rx"));
1298 }
1299 
1300 /*ARGSUSED*/
1301 void
1302 nxge_fixup_rxdma_rings(p_nxge_t nxgep)
1303 {
1304 	nxge_grp_set_t *set = &nxgep->rx_set;
1305 	int rdc;
1306 
1307 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_fixup_rxdma_rings"));
1308 
1309 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
1310 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1311 		    "<== nxge_tx_port_fatal_err_recover: "
1312 		    "NULL ring pointer(s)"));
1313 		return;
1314 	}
1315 
1316 	if (set->owned.map == 0) {
1317 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1318 		    "nxge_rxdma_regs_dump_channels: no channels"));
1319 		return;
1320 	}
1321 
1322 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
1323 		if ((1 << rdc) & set->owned.map) {
1324 			rx_rbr_ring_t *ring =
1325 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
1326 			if (ring) {
1327 				nxge_rxdma_hw_stop(nxgep, rdc);
1328 				NXGE_DEBUG_MSG((nxgep, RX_CTL,
1329 				    "==> nxge_fixup_rxdma_rings: "
1330 				    "channel %d ring $%px",
1331 				    rdc, ring));
1332 				(void) nxge_rxdma_fix_channel(nxgep, rdc);
1333 			}
1334 		}
1335 	}
1336 
1337 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_fixup_rxdma_rings"));
1338 }
1339 
1340 void
1341 nxge_rxdma_fix_channel(p_nxge_t nxgep, uint16_t channel)
1342 {
1343 	int			ndmas;
1344 	p_rx_rbr_rings_t 	rx_rbr_rings;
1345 	p_rx_rbr_ring_t		*rbr_rings;
1346 	p_rx_rcr_rings_t 	rx_rcr_rings;
1347 	p_rx_rcr_ring_t		*rcr_rings;
1348 	p_rx_mbox_areas_t 	rx_mbox_areas_p;
1349 	p_rx_mbox_t		*rx_mbox_p;
1350 	p_nxge_dma_pool_t	dma_buf_poolp;
1351 	p_nxge_dma_pool_t	dma_cntl_poolp;
1352 	p_rx_rbr_ring_t 	rbrp;
1353 	p_rx_rcr_ring_t 	rcrp;
1354 	p_rx_mbox_t 		mboxp;
1355 	p_nxge_dma_common_t 	dmap;
1356 	nxge_status_t		status = NXGE_OK;
1357 
1358 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fix_channel"));
1359 
1360 	(void) nxge_rxdma_stop_channel(nxgep, channel);
1361 
1362 	dma_buf_poolp = nxgep->rx_buf_pool_p;
1363 	dma_cntl_poolp = nxgep->rx_cntl_pool_p;
1364 
1365 	if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) {
1366 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
1367 		    "<== nxge_rxdma_fix_channel: buf not allocated"));
1368 		return;
1369 	}
1370 
1371 	ndmas = dma_buf_poolp->ndmas;
1372 	if (!ndmas) {
1373 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
1374 		    "<== nxge_rxdma_fix_channel: no dma allocated"));
1375 		return;
1376 	}
1377 
1378 	rx_rbr_rings = nxgep->rx_rbr_rings;
1379 	rx_rcr_rings = nxgep->rx_rcr_rings;
1380 	rbr_rings = rx_rbr_rings->rbr_rings;
1381 	rcr_rings = rx_rcr_rings->rcr_rings;
1382 	rx_mbox_areas_p = nxgep->rx_mbox_areas_p;
1383 	rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
1384 
1385 	/* Reinitialize the receive block and completion rings */
1386 	rbrp = (p_rx_rbr_ring_t)rbr_rings[channel],
1387 	    rcrp = (p_rx_rcr_ring_t)rcr_rings[channel],
1388 	    mboxp = (p_rx_mbox_t)rx_mbox_p[channel];
1389 
1390 	rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
1391 	rbrp->rbr_rd_index = 0;
1392 	rcrp->comp_rd_index = 0;
1393 	rcrp->comp_wt_index = 0;
1394 
1395 	dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc;
1396 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
1397 
1398 	status = nxge_rxdma_start_channel(nxgep, channel,
1399 	    rbrp, rcrp, mboxp);
1400 	if (status != NXGE_OK) {
1401 		goto nxge_rxdma_fix_channel_fail;
1402 	}
1403 
1404 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1405 	    "<== nxge_rxdma_fix_channel: success (0x%08x)", status));
1406 	return;
1407 
1408 nxge_rxdma_fix_channel_fail:
1409 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1410 	    "<== nxge_rxdma_fix_channel: failed (0x%08x)", status));
1411 }
1412 
1413 p_rx_rbr_ring_t
1414 nxge_rxdma_get_rbr_ring(p_nxge_t nxgep, uint16_t channel)
1415 {
1416 	nxge_grp_set_t *set = &nxgep->rx_set;
1417 	nxge_channel_t rdc;
1418 
1419 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1420 	    "==> nxge_rxdma_get_rbr_ring: channel %d", channel));
1421 
1422 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
1423 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1424 		    "<== nxge_rxdma_get_rbr_ring: "
1425 		    "NULL ring pointer(s)"));
1426 		return (NULL);
1427 	}
1428 
1429 	if (set->owned.map == 0) {
1430 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1431 		    "<== nxge_rxdma_get_rbr_ring: no channels"));
1432 		return (NULL);
1433 	}
1434 
1435 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
1436 		if ((1 << rdc) & set->owned.map) {
1437 			rx_rbr_ring_t *ring =
1438 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
1439 			if (ring) {
1440 				if (channel == ring->rdc) {
1441 					NXGE_DEBUG_MSG((nxgep, RX_CTL,
1442 					    "==> nxge_rxdma_get_rbr_ring: "
1443 					    "channel %d ring $%p", rdc, ring));
1444 					return (ring);
1445 				}
1446 			}
1447 		}
1448 	}
1449 
1450 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1451 	    "<== nxge_rxdma_get_rbr_ring: not found"));
1452 
1453 	return (NULL);
1454 }
1455 
1456 p_rx_rcr_ring_t
1457 nxge_rxdma_get_rcr_ring(p_nxge_t nxgep, uint16_t channel)
1458 {
1459 	nxge_grp_set_t *set = &nxgep->rx_set;
1460 	nxge_channel_t rdc;
1461 
1462 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1463 	    "==> nxge_rxdma_get_rcr_ring: channel %d", channel));
1464 
1465 	if (nxgep->rx_rcr_rings == 0 || nxgep->rx_rcr_rings->rcr_rings == 0) {
1466 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1467 		    "<== nxge_rxdma_get_rcr_ring: "
1468 		    "NULL ring pointer(s)"));
1469 		return (NULL);
1470 	}
1471 
1472 	if (set->owned.map == 0) {
1473 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1474 		    "<== nxge_rxdma_get_rbr_ring: no channels"));
1475 		return (NULL);
1476 	}
1477 
1478 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
1479 		if ((1 << rdc) & set->owned.map) {
1480 			rx_rcr_ring_t *ring =
1481 			    nxgep->rx_rcr_rings->rcr_rings[rdc];
1482 			if (ring) {
1483 				if (channel == ring->rdc) {
1484 					NXGE_DEBUG_MSG((nxgep, RX_CTL,
1485 					    "==> nxge_rxdma_get_rcr_ring: "
1486 					    "channel %d ring $%p", rdc, ring));
1487 					return (ring);
1488 				}
1489 			}
1490 		}
1491 	}
1492 
1493 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1494 	    "<== nxge_rxdma_get_rcr_ring: not found"));
1495 
1496 	return (NULL);
1497 }
1498 
1499 /*
1500  * Static functions start here.
1501  */
1502 static p_rx_msg_t
1503 nxge_allocb(size_t size, uint32_t pri, p_nxge_dma_common_t dmabuf_p)
1504 {
1505 	p_rx_msg_t nxge_mp 		= NULL;
1506 	p_nxge_dma_common_t		dmamsg_p;
1507 	uchar_t 			*buffer;
1508 
1509 	nxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP);
1510 	if (nxge_mp == NULL) {
1511 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1512 		    "Allocation of a rx msg failed."));
1513 		goto nxge_allocb_exit;
1514 	}
1515 
1516 	nxge_mp->use_buf_pool = B_FALSE;
1517 	if (dmabuf_p) {
1518 		nxge_mp->use_buf_pool = B_TRUE;
1519 		dmamsg_p = (p_nxge_dma_common_t)&nxge_mp->buf_dma;
1520 		*dmamsg_p = *dmabuf_p;
1521 		dmamsg_p->nblocks = 1;
1522 		dmamsg_p->block_size = size;
1523 		dmamsg_p->alength = size;
1524 		buffer = (uchar_t *)dmabuf_p->kaddrp;
1525 
1526 		dmabuf_p->kaddrp = (void *)
1527 		    ((char *)dmabuf_p->kaddrp + size);
1528 		dmabuf_p->ioaddr_pp = (void *)
1529 		    ((char *)dmabuf_p->ioaddr_pp + size);
1530 		dmabuf_p->alength -= size;
1531 		dmabuf_p->offset += size;
1532 		dmabuf_p->dma_cookie.dmac_laddress += size;
1533 		dmabuf_p->dma_cookie.dmac_size -= size;
1534 
1535 	} else {
1536 		buffer = KMEM_ALLOC(size, KM_NOSLEEP);
1537 		if (buffer == NULL) {
1538 			NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1539 			    "Allocation of a receive page failed."));
1540 			goto nxge_allocb_fail1;
1541 		}
1542 	}
1543 
1544 	nxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &nxge_mp->freeb);
1545 	if (nxge_mp->rx_mblk_p == NULL) {
1546 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, "desballoc failed."));
1547 		goto nxge_allocb_fail2;
1548 	}
1549 
1550 	nxge_mp->buffer = buffer;
1551 	nxge_mp->block_size = size;
1552 	nxge_mp->freeb.free_func = (void (*)())nxge_freeb;
1553 	nxge_mp->freeb.free_arg = (caddr_t)nxge_mp;
1554 	nxge_mp->ref_cnt = 1;
1555 	nxge_mp->free = B_TRUE;
1556 	nxge_mp->rx_use_bcopy = B_FALSE;
1557 
1558 	atomic_inc_32(&nxge_mblks_pending);
1559 
1560 	goto nxge_allocb_exit;
1561 
1562 nxge_allocb_fail2:
1563 	if (!nxge_mp->use_buf_pool) {
1564 		KMEM_FREE(buffer, size);
1565 	}
1566 
1567 nxge_allocb_fail1:
1568 	KMEM_FREE(nxge_mp, sizeof (rx_msg_t));
1569 	nxge_mp = NULL;
1570 
1571 nxge_allocb_exit:
1572 	return (nxge_mp);
1573 }
1574 
1575 p_mblk_t
1576 nxge_dupb(p_rx_msg_t nxge_mp, uint_t offset, size_t size)
1577 {
1578 	p_mblk_t mp;
1579 
1580 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "==> nxge_dupb"));
1581 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "nxge_mp = $%p "
1582 	    "offset = 0x%08X "
1583 	    "size = 0x%08X",
1584 	    nxge_mp, offset, size));
1585 
1586 	mp = desballoc(&nxge_mp->buffer[offset], size,
1587 	    0, &nxge_mp->freeb);
1588 	if (mp == NULL) {
1589 		NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
1590 		goto nxge_dupb_exit;
1591 	}
1592 	atomic_inc_32(&nxge_mp->ref_cnt);
1593 
1594 
1595 nxge_dupb_exit:
1596 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p",
1597 	    nxge_mp));
1598 	return (mp);
1599 }
1600 
1601 p_mblk_t
1602 nxge_dupb_bcopy(p_rx_msg_t nxge_mp, uint_t offset, size_t size)
1603 {
1604 	p_mblk_t mp;
1605 	uchar_t *dp;
1606 
1607 	mp = allocb(size + NXGE_RXBUF_EXTRA, 0);
1608 	if (mp == NULL) {
1609 		NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
1610 		goto nxge_dupb_bcopy_exit;
1611 	}
1612 	dp = mp->b_rptr = mp->b_rptr + NXGE_RXBUF_EXTRA;
1613 	bcopy((void *)&nxge_mp->buffer[offset], dp, size);
1614 	mp->b_wptr = dp + size;
1615 
1616 nxge_dupb_bcopy_exit:
1617 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p",
1618 	    nxge_mp));
1619 	return (mp);
1620 }
1621 
1622 void nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p,
1623 	p_rx_msg_t rx_msg_p);
1624 
1625 void
1626 nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p)
1627 {
1628 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_post_page"));
1629 
1630 	/* Reuse this buffer */
1631 	rx_msg_p->free = B_FALSE;
1632 	rx_msg_p->cur_usage_cnt = 0;
1633 	rx_msg_p->max_usage_cnt = 0;
1634 	rx_msg_p->pkt_buf_size = 0;
1635 
1636 	if (rx_rbr_p->rbr_use_bcopy) {
1637 		rx_msg_p->rx_use_bcopy = B_FALSE;
1638 		atomic_dec_32(&rx_rbr_p->rbr_consumed);
1639 	}
1640 
1641 	/*
1642 	 * Get the rbr header pointer and its offset index.
1643 	 */
1644 	MUTEX_ENTER(&rx_rbr_p->post_lock);
1645 	rx_rbr_p->rbr_wr_index =  ((rx_rbr_p->rbr_wr_index + 1) &
1646 	    rx_rbr_p->rbr_wrap_mask);
1647 	rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr;
1648 	MUTEX_EXIT(&rx_rbr_p->post_lock);
1649 	npi_rxdma_rdc_rbr_kick(NXGE_DEV_NPI_HANDLE(nxgep),
1650 	    rx_rbr_p->rdc, 1);
1651 
1652 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1653 	    "<== nxge_post_page (channel %d post_next_index %d)",
1654 	    rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index));
1655 
1656 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_post_page"));
1657 }
1658 
1659 void
1660 nxge_freeb(p_rx_msg_t rx_msg_p)
1661 {
1662 	size_t size;
1663 	uchar_t *buffer = NULL;
1664 	int ref_cnt;
1665 	boolean_t free_state = B_FALSE;
1666 
1667 	rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p;
1668 
1669 	NXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> nxge_freeb"));
1670 	NXGE_DEBUG_MSG((NULL, MEM2_CTL,
1671 	    "nxge_freeb:rx_msg_p = $%p (block pending %d)",
1672 	    rx_msg_p, nxge_mblks_pending));
1673 
1674 	/*
1675 	 * First we need to get the free state, then
1676 	 * atomic decrement the reference count to prevent
1677 	 * the race condition with the interrupt thread that
1678 	 * is processing a loaned up buffer block.
1679 	 */
1680 	free_state = rx_msg_p->free;
1681 	ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1);
1682 	if (!ref_cnt) {
1683 		atomic_dec_32(&nxge_mblks_pending);
1684 		buffer = rx_msg_p->buffer;
1685 		size = rx_msg_p->block_size;
1686 		NXGE_DEBUG_MSG((NULL, MEM2_CTL, "nxge_freeb: "
1687 		    "will free: rx_msg_p = $%p (block pending %d)",
1688 		    rx_msg_p, nxge_mblks_pending));
1689 
1690 		if (!rx_msg_p->use_buf_pool) {
1691 			KMEM_FREE(buffer, size);
1692 		}
1693 
1694 		KMEM_FREE(rx_msg_p, sizeof (rx_msg_t));
1695 
1696 		if (ring) {
1697 			/*
1698 			 * Decrement the receive buffer ring's reference
1699 			 * count, too.
1700 			 */
1701 			atomic_dec_32(&ring->rbr_ref_cnt);
1702 
1703 			/*
1704 			 * Free the receive buffer ring, if
1705 			 * 1. all the receive buffers have been freed
1706 			 * 2. and we are in the proper state (that is,
1707 			 *    we are not UNMAPPING).
1708 			 */
1709 			if (ring->rbr_ref_cnt == 0 &&
1710 			    ring->rbr_state == RBR_UNMAPPED) {
1711 				/*
1712 				 * Free receive data buffers,
1713 				 * buffer index information
1714 				 * (rxring_info) and
1715 				 * the message block ring.
1716 				 */
1717 				NXGE_DEBUG_MSG((NULL, RX_CTL,
1718 				    "nxge_freeb:rx_msg_p = $%p "
1719 				    "(block pending %d) free buffers",
1720 				    rx_msg_p, nxge_mblks_pending));
1721 				nxge_rxdma_databuf_free(ring);
1722 				if (ring->ring_info) {
1723 					KMEM_FREE(ring->ring_info,
1724 					    sizeof (rxring_info_t));
1725 				}
1726 
1727 				if (ring->rx_msg_ring) {
1728 					KMEM_FREE(ring->rx_msg_ring,
1729 					    ring->tnblocks *
1730 					    sizeof (p_rx_msg_t));
1731 				}
1732 				KMEM_FREE(ring, sizeof (*ring));
1733 			}
1734 		}
1735 		return;
1736 	}
1737 
1738 	/*
1739 	 * Repost buffer.
1740 	 */
1741 	if (free_state && (ref_cnt == 1) && ring) {
1742 		NXGE_DEBUG_MSG((NULL, RX_CTL,
1743 		    "nxge_freeb: post page $%p:", rx_msg_p));
1744 		if (ring->rbr_state == RBR_POSTING)
1745 			nxge_post_page(rx_msg_p->nxgep, ring, rx_msg_p);
1746 	}
1747 
1748 	NXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== nxge_freeb"));
1749 }
1750 
1751 uint_t
1752 nxge_rx_intr(void *arg1, void *arg2)
1753 {
1754 	p_nxge_ldv_t		ldvp = (p_nxge_ldv_t)arg1;
1755 	p_nxge_t		nxgep = (p_nxge_t)arg2;
1756 	p_nxge_ldg_t		ldgp;
1757 	uint8_t			channel;
1758 	npi_handle_t		handle;
1759 	rx_dma_ctl_stat_t	cs;
1760 	p_rx_rcr_ring_t		rcrp;
1761 	mblk_t			*mp = NULL;
1762 
1763 	if (ldvp == NULL) {
1764 		NXGE_DEBUG_MSG((NULL, INT_CTL,
1765 		    "<== nxge_rx_intr: arg2 $%p arg1 $%p",
1766 		    nxgep, ldvp));
1767 		return (DDI_INTR_CLAIMED);
1768 	}
1769 
1770 	if (arg2 == NULL || (void *)ldvp->nxgep != arg2) {
1771 		nxgep = ldvp->nxgep;
1772 	}
1773 
1774 	if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
1775 	    (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
1776 		NXGE_DEBUG_MSG((nxgep, INT_CTL,
1777 		    "<== nxge_rx_intr: interface not started or intialized"));
1778 		return (DDI_INTR_CLAIMED);
1779 	}
1780 
1781 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1782 	    "==> nxge_rx_intr: arg2 $%p arg1 $%p",
1783 	    nxgep, ldvp));
1784 
1785 	/*
1786 	 * Get the PIO handle.
1787 	 */
1788 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
1789 
1790 	/*
1791 	 * Get the ring to enable us to process packets.
1792 	 */
1793 	rcrp = nxgep->rx_rcr_rings->rcr_rings[ldvp->vdma_index];
1794 
1795 	/*
1796 	 * The RCR ring lock must be held when packets
1797 	 * are being processed and the hardware registers are
1798 	 * being read or written to prevent race condition
1799 	 * among the interrupt thread, the polling thread
1800 	 * (will cause fatal errors such as rcrincon bit set)
1801 	 * and the setting of the poll_flag.
1802 	 */
1803 	MUTEX_ENTER(&rcrp->lock);
1804 
1805 	/*
1806 	 * Get the control and status for this channel.
1807 	 */
1808 	channel = ldvp->channel;
1809 	ldgp = ldvp->ldgp;
1810 
1811 	if (!isLDOMguest(nxgep) && (!nxgep->rx_channel_started[channel])) {
1812 		NXGE_DEBUG_MSG((nxgep, INT_CTL,
1813 		    "<== nxge_rx_intr: channel is not started"));
1814 
1815 		/*
1816 		 * We received an interrupt before the ring is started.
1817 		 */
1818 		RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, channel,
1819 		    &cs.value);
1820 		cs.value &= RX_DMA_CTL_STAT_WR1C;
1821 		cs.bits.hdw.mex = 1;
1822 		RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, channel,
1823 		    cs.value);
1824 
1825 		/*
1826 		 * Rearm this logical group if this is a single device
1827 		 * group.
1828 		 */
1829 		if (ldgp->nldvs == 1) {
1830 			if (isLDOMguest(nxgep)) {
1831 				nxge_hio_ldgimgn(nxgep, ldgp);
1832 			} else {
1833 				ldgimgm_t mgm;
1834 
1835 				mgm.value = 0;
1836 				mgm.bits.ldw.arm = 1;
1837 				mgm.bits.ldw.timer = ldgp->ldg_timer;
1838 
1839 				NXGE_REG_WR64(handle,
1840 				    LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg),
1841 				    mgm.value);
1842 			}
1843 		}
1844 		MUTEX_EXIT(&rcrp->lock);
1845 		return (DDI_INTR_CLAIMED);
1846 	}
1847 
1848 	ASSERT(rcrp->ldgp == ldgp);
1849 	ASSERT(rcrp->ldvp == ldvp);
1850 
1851 	RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, channel, &cs.value);
1852 
1853 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_intr:channel %d "
1854 	    "cs 0x%016llx rcrto 0x%x rcrthres %x",
1855 	    channel,
1856 	    cs.value,
1857 	    cs.bits.hdw.rcrto,
1858 	    cs.bits.hdw.rcrthres));
1859 
1860 	if (!rcrp->poll_flag) {
1861 		mp = nxge_rx_pkts(nxgep, rcrp, cs, -1);
1862 	}
1863 
1864 	/* error events. */
1865 	if (cs.value & RX_DMA_CTL_STAT_ERROR) {
1866 		(void) nxge_rx_err_evnts(nxgep, channel, cs);
1867 	}
1868 
1869 	/*
1870 	 * Enable the mailbox update interrupt if we want
1871 	 * to use mailbox. We probably don't need to use
1872 	 * mailbox as it only saves us one pio read.
1873 	 * Also write 1 to rcrthres and rcrto to clear
1874 	 * these two edge triggered bits.
1875 	 */
1876 	cs.value &= RX_DMA_CTL_STAT_WR1C;
1877 	cs.bits.hdw.mex = rcrp->poll_flag ? 0 : 1;
1878 	RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, channel,
1879 	    cs.value);
1880 
1881 	/*
1882 	 * If the polling mode is enabled, disable the interrupt.
1883 	 */
1884 	if (rcrp->poll_flag) {
1885 		NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
1886 		    "==> nxge_rx_intr: rdc %d ldgp $%p ldvp $%p "
1887 		    "(disabling interrupts)", channel, ldgp, ldvp));
1888 
1889 		/*
1890 		 * Disarm this logical group if this is a single device
1891 		 * group.
1892 		 */
1893 		if (ldgp->nldvs == 1) {
1894 			if (isLDOMguest(nxgep)) {
1895 				ldgp->arm = B_FALSE;
1896 				nxge_hio_ldgimgn(nxgep, ldgp);
1897 			} else {
1898 				ldgimgm_t mgm;
1899 				mgm.value = 0;
1900 				mgm.bits.ldw.arm = 0;
1901 				NXGE_REG_WR64(handle,
1902 				    LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg),
1903 				    mgm.value);
1904 			}
1905 		}
1906 	} else {
1907 		/*
1908 		 * Rearm this logical group if this is a single device
1909 		 * group.
1910 		 */
1911 		if (ldgp->nldvs == 1) {
1912 			if (isLDOMguest(nxgep)) {
1913 				nxge_hio_ldgimgn(nxgep, ldgp);
1914 			} else {
1915 				ldgimgm_t mgm;
1916 
1917 				mgm.value = 0;
1918 				mgm.bits.ldw.arm = 1;
1919 				mgm.bits.ldw.timer = ldgp->ldg_timer;
1920 
1921 				NXGE_REG_WR64(handle,
1922 				    LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg),
1923 				    mgm.value);
1924 			}
1925 		}
1926 
1927 		NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
1928 		    "==> nxge_rx_intr: rdc %d ldgp $%p "
1929 		    "exiting ISR (and call mac_rx_ring)", channel, ldgp));
1930 	}
1931 	MUTEX_EXIT(&rcrp->lock);
1932 
1933 	if (mp != NULL) {
1934 		mac_rx_ring(nxgep->mach, rcrp->rcr_mac_handle, mp,
1935 		    rcrp->rcr_gen_num);
1936 	}
1937 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_intr: DDI_INTR_CLAIMED"));
1938 	return (DDI_INTR_CLAIMED);
1939 }
1940 
1941 /*
1942  * This routine is the main packet receive processing function.
1943  * It gets the packet type, error code, and buffer related
1944  * information from the receive completion entry.
1945  * How many completion entries to process is based on the number of packets
1946  * queued by the hardware, a hardware maintained tail pointer
1947  * and a configurable receive packet count.
1948  *
1949  * A chain of message blocks will be created as result of processing
1950  * the completion entries. This chain of message blocks will be returned and
1951  * a hardware control status register will be updated with the number of
1952  * packets were removed from the hardware queue.
1953  *
1954  * The RCR ring lock is held when entering this function.
1955  */
1956 static mblk_t *
1957 nxge_rx_pkts(p_nxge_t nxgep, p_rx_rcr_ring_t rcr_p, rx_dma_ctl_stat_t cs,
1958     int bytes_to_pickup)
1959 {
1960 	npi_handle_t		handle;
1961 	uint8_t			channel;
1962 	uint32_t		comp_rd_index;
1963 	p_rcr_entry_t		rcr_desc_rd_head_p;
1964 	p_rcr_entry_t		rcr_desc_rd_head_pp;
1965 	p_mblk_t		nmp, mp_cont, head_mp, *tail_mp;
1966 	uint16_t		qlen, nrcr_read, npkt_read;
1967 	uint32_t		qlen_hw;
1968 	boolean_t		multi;
1969 	rcrcfig_b_t		rcr_cfg_b;
1970 	int			totallen = 0;
1971 #if defined(_BIG_ENDIAN)
1972 	npi_status_t		rs = NPI_SUCCESS;
1973 #endif
1974 
1975 	NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, "==> nxge_rx_pkts: "
1976 	    "channel %d", rcr_p->rdc));
1977 
1978 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
1979 		return (NULL);
1980 	}
1981 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
1982 	channel = rcr_p->rdc;
1983 
1984 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
1985 	    "==> nxge_rx_pkts: START: rcr channel %d "
1986 	    "head_p $%p head_pp $%p  index %d ",
1987 	    channel, rcr_p->rcr_desc_rd_head_p,
1988 	    rcr_p->rcr_desc_rd_head_pp,
1989 	    rcr_p->comp_rd_index));
1990 
1991 
1992 #if !defined(_BIG_ENDIAN)
1993 	qlen = RXDMA_REG_READ32(handle, RCRSTAT_A_REG, channel) & 0xffff;
1994 #else
1995 	rs = npi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen);
1996 	if (rs != NPI_SUCCESS) {
1997 		NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts: "
1998 		"channel %d, get qlen failed 0x%08x",
1999 		    channel, rs));
2000 		return (NULL);
2001 	}
2002 #endif
2003 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts:rcr channel %d "
2004 	    "qlen %d", channel, qlen));
2005 
2006 
2007 
2008 	if (!qlen) {
2009 		NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
2010 		    "==> nxge_rx_pkts:rcr channel %d "
2011 		    "qlen %d (no pkts)", channel, qlen));
2012 
2013 		return (NULL);
2014 	}
2015 
2016 	comp_rd_index = rcr_p->comp_rd_index;
2017 
2018 	rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p;
2019 	rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp;
2020 	nrcr_read = npkt_read = 0;
2021 
2022 	/*
2023 	 * Number of packets queued
2024 	 * (The jumbo or multi packet will be counted as only one
2025 	 *  packets and it may take up more than one completion entry).
2026 	 */
2027 	qlen_hw = (qlen < nxge_max_rx_pkts) ?
2028 	    qlen : nxge_max_rx_pkts;
2029 	head_mp = NULL;
2030 	tail_mp = &head_mp;
2031 	nmp = mp_cont = NULL;
2032 	multi = B_FALSE;
2033 
2034 	while (qlen_hw) {
2035 
2036 #ifdef NXGE_DEBUG
2037 		nxge_dump_rcr_entry(nxgep, rcr_desc_rd_head_p);
2038 #endif
2039 		/*
2040 		 * Process one completion ring entry.
2041 		 */
2042 		nxge_receive_packet(nxgep,
2043 		    rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont);
2044 
2045 		/*
2046 		 * message chaining modes
2047 		 */
2048 		if (nmp) {
2049 			nmp->b_next = NULL;
2050 			if (!multi && !mp_cont) { /* frame fits a partition */
2051 				*tail_mp = nmp;
2052 				tail_mp = &nmp->b_next;
2053 				totallen += MBLKL(nmp);
2054 				nmp = NULL;
2055 			} else if (multi && !mp_cont) { /* first segment */
2056 				*tail_mp = nmp;
2057 				tail_mp = &nmp->b_cont;
2058 				totallen += MBLKL(nmp);
2059 			} else if (multi && mp_cont) {	/* mid of multi segs */
2060 				*tail_mp = mp_cont;
2061 				tail_mp = &mp_cont->b_cont;
2062 				totallen += MBLKL(mp_cont);
2063 			} else if (!multi && mp_cont) { /* last segment */
2064 				*tail_mp = mp_cont;
2065 				tail_mp = &nmp->b_next;
2066 				totallen += MBLKL(mp_cont);
2067 				nmp = NULL;
2068 			}
2069 		}
2070 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
2071 		    "==> nxge_rx_pkts: loop: rcr channel %d "
2072 		    "before updating: multi %d "
2073 		    "nrcr_read %d "
2074 		    "npk read %d "
2075 		    "head_pp $%p  index %d ",
2076 		    channel,
2077 		    multi,
2078 		    nrcr_read, npkt_read, rcr_desc_rd_head_pp,
2079 		    comp_rd_index));
2080 
2081 		if (!multi) {
2082 			qlen_hw--;
2083 			npkt_read++;
2084 		}
2085 
2086 		/*
2087 		 * Update the next read entry.
2088 		 */
2089 		comp_rd_index = NEXT_ENTRY(comp_rd_index,
2090 		    rcr_p->comp_wrap_mask);
2091 
2092 		rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p,
2093 		    rcr_p->rcr_desc_first_p,
2094 		    rcr_p->rcr_desc_last_p);
2095 
2096 		nrcr_read++;
2097 
2098 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
2099 		    "<== nxge_rx_pkts: (SAM, process one packet) "
2100 		    "nrcr_read %d",
2101 		    nrcr_read));
2102 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
2103 		    "==> nxge_rx_pkts: loop: rcr channel %d "
2104 		    "multi %d "
2105 		    "nrcr_read %d "
2106 		    "npk read %d "
2107 		    "head_pp $%p  index %d ",
2108 		    channel,
2109 		    multi,
2110 		    nrcr_read, npkt_read, rcr_desc_rd_head_pp,
2111 		    comp_rd_index));
2112 
2113 		if ((bytes_to_pickup != -1) &&
2114 		    (totallen >= bytes_to_pickup)) {
2115 			break;
2116 		}
2117 	}
2118 
2119 	rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp;
2120 	rcr_p->comp_rd_index = comp_rd_index;
2121 	rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p;
2122 	if ((nxgep->intr_timeout != rcr_p->intr_timeout) ||
2123 	    (nxgep->intr_threshold != rcr_p->intr_threshold)) {
2124 
2125 		rcr_p->intr_timeout = (nxgep->intr_timeout <
2126 		    NXGE_RDC_RCR_TIMEOUT_MIN) ? NXGE_RDC_RCR_TIMEOUT_MIN :
2127 		    nxgep->intr_timeout;
2128 
2129 		rcr_p->intr_threshold = (nxgep->intr_threshold <
2130 		    NXGE_RDC_RCR_THRESHOLD_MIN) ? NXGE_RDC_RCR_THRESHOLD_MIN :
2131 		    nxgep->intr_threshold;
2132 
2133 		rcr_cfg_b.value = 0x0ULL;
2134 		rcr_cfg_b.bits.ldw.entout = 1;
2135 		rcr_cfg_b.bits.ldw.timeout = rcr_p->intr_timeout;
2136 		rcr_cfg_b.bits.ldw.pthres = rcr_p->intr_threshold;
2137 
2138 		RXDMA_REG_WRITE64(handle, RCRCFIG_B_REG,
2139 		    channel, rcr_cfg_b.value);
2140 	}
2141 
2142 	cs.bits.ldw.pktread = npkt_read;
2143 	cs.bits.ldw.ptrread = nrcr_read;
2144 	RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG,
2145 	    channel, cs.value);
2146 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
2147 	    "==> nxge_rx_pkts: EXIT: rcr channel %d "
2148 	    "head_pp $%p  index %016llx ",
2149 	    channel,
2150 	    rcr_p->rcr_desc_rd_head_pp,
2151 	    rcr_p->comp_rd_index));
2152 	/*
2153 	 * Update RCR buffer pointer read and number of packets
2154 	 * read.
2155 	 */
2156 
2157 	NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, "<== nxge_rx_pkts: return"
2158 	    "channel %d", rcr_p->rdc));
2159 
2160 	return (head_mp);
2161 }
2162 
2163 void
2164 nxge_receive_packet(p_nxge_t nxgep,
2165     p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p,
2166     boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont)
2167 {
2168 	p_mblk_t		nmp = NULL;
2169 	uint64_t		multi;
2170 	uint64_t		dcf_err;
2171 	uint8_t			channel;
2172 
2173 	boolean_t		first_entry = B_TRUE;
2174 	boolean_t		is_tcp_udp = B_FALSE;
2175 	boolean_t		buffer_free = B_FALSE;
2176 	boolean_t		error_send_up = B_FALSE;
2177 	uint8_t			error_type;
2178 	uint16_t		l2_len;
2179 	uint16_t		skip_len;
2180 	uint8_t			pktbufsz_type;
2181 	uint64_t		rcr_entry;
2182 	uint64_t		*pkt_buf_addr_pp;
2183 	uint64_t		*pkt_buf_addr_p;
2184 	uint32_t		buf_offset;
2185 	uint32_t		bsize;
2186 	uint32_t		error_disp_cnt;
2187 	uint32_t		msg_index;
2188 	p_rx_rbr_ring_t		rx_rbr_p;
2189 	p_rx_msg_t 		*rx_msg_ring_p;
2190 	p_rx_msg_t		rx_msg_p;
2191 	uint16_t		sw_offset_bytes = 0, hdr_size = 0;
2192 	nxge_status_t		status = NXGE_OK;
2193 	boolean_t		is_valid = B_FALSE;
2194 	p_nxge_rx_ring_stats_t	rdc_stats;
2195 	uint32_t		bytes_read;
2196 	uint64_t		pkt_type;
2197 	uint64_t		frag;
2198 	boolean_t		pkt_too_long_err = B_FALSE;
2199 #ifdef	NXGE_DEBUG
2200 	int			dump_len;
2201 #endif
2202 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_receive_packet"));
2203 	first_entry = (*mp == NULL) ? B_TRUE : B_FALSE;
2204 
2205 	rcr_entry = *((uint64_t *)rcr_desc_rd_head_p);
2206 
2207 	multi = (rcr_entry & RCR_MULTI_MASK);
2208 	dcf_err = (rcr_entry & RCR_DCF_ERROR_MASK);
2209 	pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK);
2210 
2211 	error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT);
2212 	frag = (rcr_entry & RCR_FRAG_MASK);
2213 
2214 	l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT);
2215 
2216 	pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >>
2217 	    RCR_PKTBUFSZ_SHIFT);
2218 #if defined(__i386)
2219 	pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry &
2220 	    RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT);
2221 #else
2222 	pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) <<
2223 	    RCR_PKT_BUF_ADDR_SHIFT);
2224 #endif
2225 
2226 	channel = rcr_p->rdc;
2227 
2228 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2229 	    "==> nxge_receive_packet: entryp $%p entry 0x%0llx "
2230 	    "pkt_buf_addr_pp $%p l2_len %d multi 0x%llx "
2231 	    "error_type 0x%x pkt_type 0x%x  "
2232 	    "pktbufsz_type %d ",
2233 	    rcr_desc_rd_head_p,
2234 	    rcr_entry, pkt_buf_addr_pp, l2_len,
2235 	    multi,
2236 	    error_type,
2237 	    pkt_type,
2238 	    pktbufsz_type));
2239 
2240 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2241 	    "==> nxge_receive_packet: entryp $%p entry 0x%0llx "
2242 	    "pkt_buf_addr_pp $%p l2_len %d multi 0x%llx "
2243 	    "error_type 0x%x pkt_type 0x%x ", rcr_desc_rd_head_p,
2244 	    rcr_entry, pkt_buf_addr_pp, l2_len,
2245 	    multi,
2246 	    error_type,
2247 	    pkt_type));
2248 
2249 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2250 	    "==> (rbr) nxge_receive_packet: entry 0x%0llx "
2251 	    "full pkt_buf_addr_pp $%p l2_len %d",
2252 	    rcr_entry, pkt_buf_addr_pp, l2_len));
2253 
2254 	/* get the stats ptr */
2255 	rdc_stats = rcr_p->rdc_stats;
2256 
2257 	if (!l2_len) {
2258 
2259 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
2260 		    "<== nxge_receive_packet: failed: l2 length is 0."));
2261 		return;
2262 	}
2263 
2264 	/*
2265 	 * Software workaround for BMAC hardware limitation that allows
2266 	 * maxframe size of 1526, instead of 1522 for non-jumbo and 0x2406
2267 	 * instead of 0x2400 for jumbo.
2268 	 */
2269 	if (l2_len > nxgep->mac.maxframesize) {
2270 		pkt_too_long_err = B_TRUE;
2271 	}
2272 
2273 	/* Hardware sends us 4 bytes of CRC as no stripping is done.  */
2274 	l2_len -= ETHERFCSL;
2275 
2276 	/* shift 6 bits to get the full io address */
2277 #if defined(__i386)
2278 	pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp <<
2279 	    RCR_PKT_BUF_ADDR_SHIFT_FULL);
2280 #else
2281 	pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp <<
2282 	    RCR_PKT_BUF_ADDR_SHIFT_FULL);
2283 #endif
2284 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2285 	    "==> (rbr) nxge_receive_packet: entry 0x%0llx "
2286 	    "full pkt_buf_addr_pp $%p l2_len %d",
2287 	    rcr_entry, pkt_buf_addr_pp, l2_len));
2288 
2289 	rx_rbr_p = rcr_p->rx_rbr_p;
2290 	rx_msg_ring_p = rx_rbr_p->rx_msg_ring;
2291 
2292 	if (first_entry) {
2293 		hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL :
2294 		    RXDMA_HDR_SIZE_DEFAULT);
2295 
2296 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
2297 		    "==> nxge_receive_packet: first entry 0x%016llx "
2298 		    "pkt_buf_addr_pp $%p l2_len %d hdr %d",
2299 		    rcr_entry, pkt_buf_addr_pp, l2_len,
2300 		    hdr_size));
2301 	}
2302 
2303 	MUTEX_ENTER(&rx_rbr_p->lock);
2304 
2305 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
2306 	    "==> (rbr 1) nxge_receive_packet: entry 0x%0llx "
2307 	    "full pkt_buf_addr_pp $%p l2_len %d",
2308 	    rcr_entry, pkt_buf_addr_pp, l2_len));
2309 
2310 	/*
2311 	 * Packet buffer address in the completion entry points
2312 	 * to the starting buffer address (offset 0).
2313 	 * Use the starting buffer address to locate the corresponding
2314 	 * kernel address.
2315 	 */
2316 	status = nxge_rxbuf_pp_to_vp(nxgep, rx_rbr_p,
2317 	    pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p,
2318 	    &buf_offset,
2319 	    &msg_index);
2320 
2321 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
2322 	    "==> (rbr 2) nxge_receive_packet: entry 0x%0llx "
2323 	    "full pkt_buf_addr_pp $%p l2_len %d",
2324 	    rcr_entry, pkt_buf_addr_pp, l2_len));
2325 
2326 	if (status != NXGE_OK) {
2327 		MUTEX_EXIT(&rx_rbr_p->lock);
2328 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
2329 		    "<== nxge_receive_packet: found vaddr failed %d",
2330 		    status));
2331 		return;
2332 	}
2333 
2334 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2335 	    "==> (rbr 3) nxge_receive_packet: entry 0x%0llx "
2336 	    "full pkt_buf_addr_pp $%p l2_len %d",
2337 	    rcr_entry, pkt_buf_addr_pp, l2_len));
2338 
2339 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2340 	    "==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx "
2341 	    "full pkt_buf_addr_pp $%p l2_len %d",
2342 	    msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
2343 
2344 	rx_msg_p = rx_msg_ring_p[msg_index];
2345 
2346 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2347 	    "==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx "
2348 	    "full pkt_buf_addr_pp $%p l2_len %d",
2349 	    msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
2350 
2351 	switch (pktbufsz_type) {
2352 	case RCR_PKTBUFSZ_0:
2353 		bsize = rx_rbr_p->pkt_buf_size0_bytes;
2354 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2355 		    "==> nxge_receive_packet: 0 buf %d", bsize));
2356 		break;
2357 	case RCR_PKTBUFSZ_1:
2358 		bsize = rx_rbr_p->pkt_buf_size1_bytes;
2359 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2360 		    "==> nxge_receive_packet: 1 buf %d", bsize));
2361 		break;
2362 	case RCR_PKTBUFSZ_2:
2363 		bsize = rx_rbr_p->pkt_buf_size2_bytes;
2364 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
2365 		    "==> nxge_receive_packet: 2 buf %d", bsize));
2366 		break;
2367 	case RCR_SINGLE_BLOCK:
2368 		bsize = rx_msg_p->block_size;
2369 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2370 		    "==> nxge_receive_packet: single %d", bsize));
2371 
2372 		break;
2373 	default:
2374 		MUTEX_EXIT(&rx_rbr_p->lock);
2375 		return;
2376 	}
2377 
2378 	DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma,
2379 	    (buf_offset + sw_offset_bytes),
2380 	    (hdr_size + l2_len),
2381 	    DDI_DMA_SYNC_FORCPU);
2382 
2383 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2384 	    "==> nxge_receive_packet: after first dump:usage count"));
2385 
2386 	if (rx_msg_p->cur_usage_cnt == 0) {
2387 		if (rx_rbr_p->rbr_use_bcopy) {
2388 			atomic_inc_32(&rx_rbr_p->rbr_consumed);
2389 			if (rx_rbr_p->rbr_consumed <
2390 			    rx_rbr_p->rbr_threshold_hi) {
2391 				if (rx_rbr_p->rbr_threshold_lo == 0 ||
2392 				    ((rx_rbr_p->rbr_consumed >=
2393 				    rx_rbr_p->rbr_threshold_lo) &&
2394 				    (rx_rbr_p->rbr_bufsize_type >=
2395 				    pktbufsz_type))) {
2396 					rx_msg_p->rx_use_bcopy = B_TRUE;
2397 				}
2398 			} else {
2399 				rx_msg_p->rx_use_bcopy = B_TRUE;
2400 			}
2401 		}
2402 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2403 		    "==> nxge_receive_packet: buf %d (new block) ",
2404 		    bsize));
2405 
2406 		rx_msg_p->pkt_buf_size_code = pktbufsz_type;
2407 		rx_msg_p->pkt_buf_size = bsize;
2408 		rx_msg_p->cur_usage_cnt = 1;
2409 		if (pktbufsz_type == RCR_SINGLE_BLOCK) {
2410 			NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2411 			    "==> nxge_receive_packet: buf %d "
2412 			    "(single block) ",
2413 			    bsize));
2414 			/*
2415 			 * Buffer can be reused once the free function
2416 			 * is called.
2417 			 */
2418 			rx_msg_p->max_usage_cnt = 1;
2419 			buffer_free = B_TRUE;
2420 		} else {
2421 			rx_msg_p->max_usage_cnt = rx_msg_p->block_size/bsize;
2422 			if (rx_msg_p->max_usage_cnt == 1) {
2423 				buffer_free = B_TRUE;
2424 			}
2425 		}
2426 	} else {
2427 		rx_msg_p->cur_usage_cnt++;
2428 		if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) {
2429 			buffer_free = B_TRUE;
2430 		}
2431 	}
2432 
2433 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
2434 	    "msgbuf index = %d l2len %d bytes usage %d max_usage %d ",
2435 	    msg_index, l2_len,
2436 	    rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt));
2437 
2438 	if ((error_type) || (dcf_err) || (pkt_too_long_err)) {
2439 		rdc_stats->ierrors++;
2440 		if (dcf_err) {
2441 			rdc_stats->dcf_err++;
2442 #ifdef	NXGE_DEBUG
2443 			if (!rdc_stats->dcf_err) {
2444 				NXGE_DEBUG_MSG((nxgep, RX_CTL,
2445 				"nxge_receive_packet: channel %d dcf_err rcr"
2446 				" 0x%llx", channel, rcr_entry));
2447 			}
2448 #endif
2449 			NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, NULL,
2450 			    NXGE_FM_EREPORT_RDMC_DCF_ERR);
2451 		} else if (pkt_too_long_err) {
2452 			rdc_stats->pkt_too_long_err++;
2453 			NXGE_DEBUG_MSG((nxgep, RX_CTL, " nxge_receive_packet:"
2454 			    " channel %d packet length [%d] > "
2455 			    "maxframesize [%d]", channel, l2_len + ETHERFCSL,
2456 			    nxgep->mac.maxframesize));
2457 		} else {
2458 				/* Update error stats */
2459 			error_disp_cnt = NXGE_ERROR_SHOW_MAX;
2460 			rdc_stats->errlog.compl_err_type = error_type;
2461 
2462 			switch (error_type) {
2463 			/*
2464 			 * Do not send FMA ereport for RCR_L2_ERROR and
2465 			 * RCR_L4_CSUM_ERROR because most likely they indicate
2466 			 * back pressure rather than HW failures.
2467 			 */
2468 			case RCR_L2_ERROR:
2469 				rdc_stats->l2_err++;
2470 				if (rdc_stats->l2_err <
2471 				    error_disp_cnt) {
2472 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2473 					    " nxge_receive_packet:"
2474 					    " channel %d RCR L2_ERROR",
2475 					    channel));
2476 				}
2477 				break;
2478 			case RCR_L4_CSUM_ERROR:
2479 				error_send_up = B_TRUE;
2480 				rdc_stats->l4_cksum_err++;
2481 				if (rdc_stats->l4_cksum_err <
2482 				    error_disp_cnt) {
2483 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2484 					    " nxge_receive_packet:"
2485 					    " channel %d"
2486 					    " RCR L4_CSUM_ERROR", channel));
2487 				}
2488 				break;
2489 			/*
2490 			 * Do not send FMA ereport for RCR_FFLP_SOFT_ERROR and
2491 			 * RCR_ZCP_SOFT_ERROR because they reflect the same
2492 			 * FFLP and ZCP errors that have been reported by
2493 			 * nxge_fflp.c and nxge_zcp.c.
2494 			 */
2495 			case RCR_FFLP_SOFT_ERROR:
2496 				error_send_up = B_TRUE;
2497 				rdc_stats->fflp_soft_err++;
2498 				if (rdc_stats->fflp_soft_err <
2499 				    error_disp_cnt) {
2500 					NXGE_ERROR_MSG((nxgep,
2501 					    NXGE_ERR_CTL,
2502 					    " nxge_receive_packet:"
2503 					    " channel %d"
2504 					    " RCR FFLP_SOFT_ERROR", channel));
2505 				}
2506 				break;
2507 			case RCR_ZCP_SOFT_ERROR:
2508 				error_send_up = B_TRUE;
2509 				rdc_stats->fflp_soft_err++;
2510 				if (rdc_stats->zcp_soft_err <
2511 				    error_disp_cnt)
2512 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2513 					    " nxge_receive_packet: Channel %d"
2514 					    " RCR ZCP_SOFT_ERROR", channel));
2515 				break;
2516 			default:
2517 				rdc_stats->rcr_unknown_err++;
2518 				if (rdc_stats->rcr_unknown_err
2519 				    < error_disp_cnt) {
2520 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2521 					    " nxge_receive_packet: Channel %d"
2522 					    " RCR entry 0x%llx error 0x%x",
2523 					    rcr_entry, channel, error_type));
2524 				}
2525 				break;
2526 			}
2527 		}
2528 
2529 		/*
2530 		 * Update and repost buffer block if max usage
2531 		 * count is reached.
2532 		 */
2533 		if (error_send_up == B_FALSE) {
2534 			atomic_inc_32(&rx_msg_p->ref_cnt);
2535 			if (buffer_free == B_TRUE) {
2536 				rx_msg_p->free = B_TRUE;
2537 			}
2538 
2539 			MUTEX_EXIT(&rx_rbr_p->lock);
2540 			nxge_freeb(rx_msg_p);
2541 			return;
2542 		}
2543 	}
2544 
2545 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2546 	    "==> nxge_receive_packet: DMA sync second "));
2547 
2548 	bytes_read = rcr_p->rcvd_pkt_bytes;
2549 	skip_len = sw_offset_bytes + hdr_size;
2550 	if (!rx_msg_p->rx_use_bcopy) {
2551 		/*
2552 		 * For loaned up buffers, the driver reference count
2553 		 * will be incremented first and then the free state.
2554 		 */
2555 		if ((nmp = nxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) {
2556 			if (first_entry) {
2557 				nmp->b_rptr = &nmp->b_rptr[skip_len];
2558 				if (l2_len < bsize - skip_len) {
2559 					nmp->b_wptr = &nmp->b_rptr[l2_len];
2560 				} else {
2561 					nmp->b_wptr = &nmp->b_rptr[bsize
2562 					    - skip_len];
2563 				}
2564 			} else {
2565 				if (l2_len - bytes_read < bsize) {
2566 					nmp->b_wptr =
2567 					    &nmp->b_rptr[l2_len - bytes_read];
2568 				} else {
2569 					nmp->b_wptr = &nmp->b_rptr[bsize];
2570 				}
2571 			}
2572 		}
2573 	} else {
2574 		if (first_entry) {
2575 			nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len,
2576 			    l2_len < bsize - skip_len ?
2577 			    l2_len : bsize - skip_len);
2578 		} else {
2579 			nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset,
2580 			    l2_len - bytes_read < bsize ?
2581 			    l2_len - bytes_read : bsize);
2582 		}
2583 	}
2584 	if (nmp != NULL) {
2585 		if (first_entry) {
2586 			/*
2587 			 * Jumbo packets may be received with more than one
2588 			 * buffer, increment ipackets for the first entry only.
2589 			 */
2590 			rdc_stats->ipackets++;
2591 
2592 			/* Update ibytes for kstat. */
2593 			rdc_stats->ibytes += skip_len
2594 			    + l2_len < bsize ? l2_len : bsize;
2595 			/*
2596 			 * Update the number of bytes read so far for the
2597 			 * current frame.
2598 			 */
2599 			bytes_read  = nmp->b_wptr - nmp->b_rptr;
2600 		} else {
2601 			rdc_stats->ibytes += l2_len - bytes_read < bsize ?
2602 			    l2_len - bytes_read : bsize;
2603 			bytes_read += nmp->b_wptr - nmp->b_rptr;
2604 		}
2605 
2606 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
2607 		    "==> nxge_receive_packet after dupb: "
2608 		    "rbr consumed %d "
2609 		    "pktbufsz_type %d "
2610 		    "nmp $%p rptr $%p wptr $%p "
2611 		    "buf_offset %d bzise %d l2_len %d skip_len %d",
2612 		    rx_rbr_p->rbr_consumed,
2613 		    pktbufsz_type,
2614 		    nmp, nmp->b_rptr, nmp->b_wptr,
2615 		    buf_offset, bsize, l2_len, skip_len));
2616 	} else {
2617 		cmn_err(CE_WARN, "!nxge_receive_packet: "
2618 		    "update stats (error)");
2619 		atomic_inc_32(&rx_msg_p->ref_cnt);
2620 		if (buffer_free == B_TRUE) {
2621 			rx_msg_p->free = B_TRUE;
2622 		}
2623 		MUTEX_EXIT(&rx_rbr_p->lock);
2624 		nxge_freeb(rx_msg_p);
2625 		return;
2626 	}
2627 
2628 	if (buffer_free == B_TRUE) {
2629 		rx_msg_p->free = B_TRUE;
2630 	}
2631 
2632 	is_valid = (nmp != NULL);
2633 
2634 	rcr_p->rcvd_pkt_bytes = bytes_read;
2635 
2636 	MUTEX_EXIT(&rx_rbr_p->lock);
2637 
2638 	if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) {
2639 		atomic_inc_32(&rx_msg_p->ref_cnt);
2640 		nxge_freeb(rx_msg_p);
2641 	}
2642 
2643 	if (is_valid) {
2644 		nmp->b_cont = NULL;
2645 		if (first_entry) {
2646 			*mp = nmp;
2647 			*mp_cont = NULL;
2648 		} else {
2649 			*mp_cont = nmp;
2650 		}
2651 	}
2652 
2653 	/*
2654 	 * ERROR, FRAG and PKT_TYPE are only reported in the first entry.
2655 	 * If a packet is not fragmented and no error bit is set, then
2656 	 * L4 checksum is OK.
2657 	 */
2658 
2659 	if (is_valid && !multi) {
2660 		/*
2661 		 * If the checksum flag nxge_chksum_offload
2662 		 * is 1, TCP and UDP packets can be sent
2663 		 * up with good checksum. If the checksum flag
2664 		 * is set to 0, checksum reporting will apply to
2665 		 * TCP packets only (workaround for a hardware bug).
2666 		 * If the checksum flag nxge_cksum_offload is
2667 		 * greater than 1, both TCP and UDP packets
2668 		 * will not be reported its hardware checksum results.
2669 		 */
2670 		if (nxge_cksum_offload == 1) {
2671 			is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP ||
2672 			    pkt_type == RCR_PKT_IS_UDP) ?
2673 			    B_TRUE: B_FALSE);
2674 		} else if (!nxge_cksum_offload) {
2675 			/* TCP checksum only. */
2676 			is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP) ?
2677 			    B_TRUE: B_FALSE);
2678 		}
2679 
2680 		NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_receive_packet: "
2681 		    "is_valid 0x%x multi 0x%llx pkt %d frag %d error %d",
2682 		    is_valid, multi, is_tcp_udp, frag, error_type));
2683 
2684 		if (is_tcp_udp && !frag && !error_type) {
2685 			(void) hcksum_assoc(nmp, NULL, NULL, 0, 0, 0, 0,
2686 			    HCK_FULLCKSUM_OK | HCK_FULLCKSUM, 0);
2687 			NXGE_DEBUG_MSG((nxgep, RX_CTL,
2688 			    "==> nxge_receive_packet: Full tcp/udp cksum "
2689 			    "is_valid 0x%x multi 0x%llx pkt %d frag %d "
2690 			    "error %d",
2691 			    is_valid, multi, is_tcp_udp, frag, error_type));
2692 		}
2693 	}
2694 
2695 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
2696 	    "==> nxge_receive_packet: *mp 0x%016llx", *mp));
2697 
2698 	*multi_p = (multi == RCR_MULTI_MASK);
2699 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_receive_packet: "
2700 	    "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx",
2701 	    *multi_p, nmp, *mp, *mp_cont));
2702 }
2703 
2704 /*
2705  * Enable polling for a ring. Interrupt for the ring is disabled when
2706  * the nxge interrupt comes (see nxge_rx_intr).
2707  */
2708 int
2709 nxge_enable_poll(void *arg)
2710 {
2711 	p_nxge_ring_handle_t	ring_handle = (p_nxge_ring_handle_t)arg;
2712 	p_rx_rcr_ring_t		ringp;
2713 	p_nxge_t		nxgep;
2714 	p_nxge_ldg_t		ldgp;
2715 	uint32_t		channel;
2716 
2717 	if (ring_handle == NULL) {
2718 		ASSERT(ring_handle != NULL);
2719 		return (0);
2720 	}
2721 
2722 	nxgep = ring_handle->nxgep;
2723 	channel = nxgep->pt_config.hw_config.start_rdc + ring_handle->index;
2724 	ringp = nxgep->rx_rcr_rings->rcr_rings[channel];
2725 	NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
2726 	    "==> nxge_enable_poll: rdc %d ", ringp->rdc));
2727 	ldgp = ringp->ldgp;
2728 	if (ldgp == NULL) {
2729 		NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
2730 		    "==> nxge_enable_poll: rdc %d NULL ldgp: no change",
2731 		    ringp->rdc));
2732 		return (0);
2733 	}
2734 
2735 	MUTEX_ENTER(&ringp->lock);
2736 	/* enable polling */
2737 	if (ringp->poll_flag == 0) {
2738 		ringp->poll_flag = 1;
2739 		NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
2740 		    "==> nxge_enable_poll: rdc %d set poll flag to 1",
2741 		    ringp->rdc));
2742 	}
2743 
2744 	MUTEX_EXIT(&ringp->lock);
2745 	return (0);
2746 }
2747 /*
2748  * Disable polling for a ring and enable its interrupt.
2749  */
2750 int
2751 nxge_disable_poll(void *arg)
2752 {
2753 	p_nxge_ring_handle_t	ring_handle = (p_nxge_ring_handle_t)arg;
2754 	p_rx_rcr_ring_t		ringp;
2755 	p_nxge_t		nxgep;
2756 	uint32_t		channel;
2757 
2758 	if (ring_handle == NULL) {
2759 		ASSERT(ring_handle != NULL);
2760 		return (0);
2761 	}
2762 
2763 	nxgep = ring_handle->nxgep;
2764 	channel = nxgep->pt_config.hw_config.start_rdc + ring_handle->index;
2765 	ringp = nxgep->rx_rcr_rings->rcr_rings[channel];
2766 
2767 	NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
2768 	    "==> nxge_disable_poll: rdc %d poll_flag %d", ringp->rdc));
2769 
2770 	MUTEX_ENTER(&ringp->lock);
2771 
2772 	/* disable polling: enable interrupt */
2773 	if (ringp->poll_flag) {
2774 		npi_handle_t		handle;
2775 		rx_dma_ctl_stat_t	cs;
2776 		uint8_t			channel;
2777 		p_nxge_ldg_t		ldgp;
2778 
2779 		/*
2780 		 * Get the control and status for this channel.
2781 		 */
2782 		handle = NXGE_DEV_NPI_HANDLE(nxgep);
2783 		channel = ringp->rdc;
2784 		RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG,
2785 		    channel, &cs.value);
2786 
2787 		/*
2788 		 * Enable mailbox update
2789 		 * Since packets were not read and the hardware uses
2790 		 * bits pktread and ptrread to update the queue
2791 		 * length, we need to set both bits to 0.
2792 		 */
2793 		cs.bits.ldw.pktread = 0;
2794 		cs.bits.ldw.ptrread = 0;
2795 		cs.bits.hdw.mex = 1;
2796 		RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, channel,
2797 		    cs.value);
2798 
2799 		/*
2800 		 * Rearm this logical group if this is a single device
2801 		 * group.
2802 		 */
2803 		ldgp = ringp->ldgp;
2804 		if (ldgp == NULL) {
2805 			ringp->poll_flag = 0;
2806 			MUTEX_EXIT(&ringp->lock);
2807 			NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
2808 			    "==> nxge_disable_poll: no ldgp rdc %d "
2809 			    "(still set poll to 0", ringp->rdc));
2810 			return (0);
2811 		}
2812 		NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
2813 		    "==> nxge_disable_poll: rdc %d ldgp $%p (enable intr)",
2814 		    ringp->rdc, ldgp));
2815 		if (ldgp->nldvs == 1) {
2816 			if (isLDOMguest(nxgep)) {
2817 				ldgp->arm = B_TRUE;
2818 				nxge_hio_ldgimgn(nxgep, ldgp);
2819 			} else {
2820 				ldgimgm_t	mgm;
2821 				mgm.value = 0;
2822 				mgm.bits.ldw.arm = 1;
2823 				mgm.bits.ldw.timer = ldgp->ldg_timer;
2824 				NXGE_REG_WR64(handle,
2825 				    LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg),
2826 				    mgm.value);
2827 			}
2828 		}
2829 		ringp->poll_flag = 0;
2830 	}
2831 
2832 	MUTEX_EXIT(&ringp->lock);
2833 	return (0);
2834 }
2835 
2836 /*
2837  * Poll 'bytes_to_pickup' bytes of message from the rx ring.
2838  */
2839 mblk_t *
2840 nxge_rx_poll(void *arg, int bytes_to_pickup)
2841 {
2842 	p_nxge_ring_handle_t	ring_handle = (p_nxge_ring_handle_t)arg;
2843 	p_rx_rcr_ring_t		rcr_p;
2844 	p_nxge_t		nxgep;
2845 	npi_handle_t		handle;
2846 	rx_dma_ctl_stat_t	cs;
2847 	mblk_t			*mblk;
2848 	p_nxge_ldv_t		ldvp;
2849 	uint32_t		channel;
2850 
2851 	nxgep = ring_handle->nxgep;
2852 
2853 	/*
2854 	 * Get the control and status for this channel.
2855 	 */
2856 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
2857 	channel = nxgep->pt_config.hw_config.start_rdc + ring_handle->index;
2858 	rcr_p = nxgep->rx_rcr_rings->rcr_rings[channel];
2859 	MUTEX_ENTER(&rcr_p->lock);
2860 	ASSERT(rcr_p->poll_flag == 1);
2861 
2862 	RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, rcr_p->rdc, &cs.value);
2863 
2864 	NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
2865 	    "==> nxge_rx_poll: calling nxge_rx_pkts: rdc %d poll_flag %d",
2866 	    rcr_p->rdc, rcr_p->poll_flag));
2867 	mblk = nxge_rx_pkts(nxgep, rcr_p, cs, bytes_to_pickup);
2868 
2869 	ldvp = rcr_p->ldvp;
2870 	/* error events. */
2871 	if (ldvp && (cs.value & RX_DMA_CTL_STAT_ERROR)) {
2872 		(void) nxge_rx_err_evnts(nxgep, ldvp->vdma_index, cs);
2873 	}
2874 
2875 	MUTEX_EXIT(&rcr_p->lock);
2876 
2877 	NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL,
2878 	    "<== nxge_rx_poll: rdc %d mblk $%p", rcr_p->rdc, mblk));
2879 	return (mblk);
2880 }
2881 
2882 
2883 /*ARGSUSED*/
2884 static nxge_status_t
2885 nxge_rx_err_evnts(p_nxge_t nxgep, int channel, rx_dma_ctl_stat_t cs)
2886 {
2887 	p_nxge_rx_ring_stats_t	rdc_stats;
2888 	npi_handle_t		handle;
2889 	npi_status_t		rs;
2890 	boolean_t		rxchan_fatal = B_FALSE;
2891 	boolean_t		rxport_fatal = B_FALSE;
2892 	uint8_t			portn;
2893 	nxge_status_t		status = NXGE_OK;
2894 	uint32_t		error_disp_cnt = NXGE_ERROR_SHOW_MAX;
2895 	NXGE_DEBUG_MSG((nxgep, INT_CTL, "==> nxge_rx_err_evnts"));
2896 
2897 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
2898 	portn = nxgep->mac.portnum;
2899 	rdc_stats = &nxgep->statsp->rdc_stats[channel];
2900 
2901 	if (cs.bits.hdw.rbr_tmout) {
2902 		rdc_stats->rx_rbr_tmout++;
2903 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
2904 		    NXGE_FM_EREPORT_RDMC_RBR_TMOUT);
2905 		rxchan_fatal = B_TRUE;
2906 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2907 		    "==> nxge_rx_err_evnts: rx_rbr_timeout"));
2908 	}
2909 	if (cs.bits.hdw.rsp_cnt_err) {
2910 		rdc_stats->rsp_cnt_err++;
2911 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
2912 		    NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR);
2913 		rxchan_fatal = B_TRUE;
2914 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2915 		    "==> nxge_rx_err_evnts(channel %d): "
2916 		    "rsp_cnt_err", channel));
2917 	}
2918 	if (cs.bits.hdw.byte_en_bus) {
2919 		rdc_stats->byte_en_bus++;
2920 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
2921 		    NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS);
2922 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2923 		    "==> nxge_rx_err_evnts(channel %d): "
2924 		    "fatal error: byte_en_bus", channel));
2925 		rxchan_fatal = B_TRUE;
2926 	}
2927 	if (cs.bits.hdw.rsp_dat_err) {
2928 		rdc_stats->rsp_dat_err++;
2929 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
2930 		    NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR);
2931 		rxchan_fatal = B_TRUE;
2932 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2933 		    "==> nxge_rx_err_evnts(channel %d): "
2934 		    "fatal error: rsp_dat_err", channel));
2935 	}
2936 	if (cs.bits.hdw.rcr_ack_err) {
2937 		rdc_stats->rcr_ack_err++;
2938 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
2939 		    NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR);
2940 		rxchan_fatal = B_TRUE;
2941 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2942 		    "==> nxge_rx_err_evnts(channel %d): "
2943 		    "fatal error: rcr_ack_err", channel));
2944 	}
2945 	if (cs.bits.hdw.dc_fifo_err) {
2946 		rdc_stats->dc_fifo_err++;
2947 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
2948 		    NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR);
2949 		/* This is not a fatal error! */
2950 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2951 		    "==> nxge_rx_err_evnts(channel %d): "
2952 		    "dc_fifo_err", channel));
2953 		rxport_fatal = B_TRUE;
2954 	}
2955 	if ((cs.bits.hdw.rcr_sha_par) || (cs.bits.hdw.rbr_pre_par)) {
2956 		if ((rs = npi_rxdma_ring_perr_stat_get(handle,
2957 		    &rdc_stats->errlog.pre_par,
2958 		    &rdc_stats->errlog.sha_par))
2959 		    != NPI_SUCCESS) {
2960 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2961 			    "==> nxge_rx_err_evnts(channel %d): "
2962 			    "rcr_sha_par: get perr", channel));
2963 			return (NXGE_ERROR | rs);
2964 		}
2965 		if (cs.bits.hdw.rcr_sha_par) {
2966 			rdc_stats->rcr_sha_par++;
2967 			NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
2968 			    NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR);
2969 			rxchan_fatal = B_TRUE;
2970 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2971 			    "==> nxge_rx_err_evnts(channel %d): "
2972 			    "fatal error: rcr_sha_par", channel));
2973 		}
2974 		if (cs.bits.hdw.rbr_pre_par) {
2975 			rdc_stats->rbr_pre_par++;
2976 			NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
2977 			    NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR);
2978 			rxchan_fatal = B_TRUE;
2979 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2980 			    "==> nxge_rx_err_evnts(channel %d): "
2981 			    "fatal error: rbr_pre_par", channel));
2982 		}
2983 	}
2984 	/*
2985 	 * The Following 4 status bits are for information, the system
2986 	 * is running fine. There is no need to send FMA ereports or
2987 	 * log messages.
2988 	 */
2989 	if (cs.bits.hdw.port_drop_pkt) {
2990 		rdc_stats->port_drop_pkt++;
2991 	}
2992 	if (cs.bits.hdw.wred_drop) {
2993 		rdc_stats->wred_drop++;
2994 	}
2995 	if (cs.bits.hdw.rbr_pre_empty) {
2996 		rdc_stats->rbr_pre_empty++;
2997 	}
2998 	if (cs.bits.hdw.rcr_shadow_full) {
2999 		rdc_stats->rcr_shadow_full++;
3000 	}
3001 	if (cs.bits.hdw.config_err) {
3002 		rdc_stats->config_err++;
3003 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
3004 		    NXGE_FM_EREPORT_RDMC_CONFIG_ERR);
3005 		rxchan_fatal = B_TRUE;
3006 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3007 		    "==> nxge_rx_err_evnts(channel %d): "
3008 		    "config error", channel));
3009 	}
3010 	if (cs.bits.hdw.rcrincon) {
3011 		rdc_stats->rcrincon++;
3012 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
3013 		    NXGE_FM_EREPORT_RDMC_RCRINCON);
3014 		rxchan_fatal = B_TRUE;
3015 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3016 		    "==> nxge_rx_err_evnts(channel %d): "
3017 		    "fatal error: rcrincon error", channel));
3018 	}
3019 	if (cs.bits.hdw.rcrfull) {
3020 		rdc_stats->rcrfull++;
3021 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
3022 		    NXGE_FM_EREPORT_RDMC_RCRFULL);
3023 		rxchan_fatal = B_TRUE;
3024 		if (rdc_stats->rcrfull < error_disp_cnt)
3025 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3026 		    "==> nxge_rx_err_evnts(channel %d): "
3027 		    "fatal error: rcrfull error", channel));
3028 	}
3029 	if (cs.bits.hdw.rbr_empty) {
3030 		/*
3031 		 * This bit is for information, there is no need
3032 		 * send FMA ereport or log a message.
3033 		 */
3034 		rdc_stats->rbr_empty++;
3035 	}
3036 	if (cs.bits.hdw.rbrfull) {
3037 		rdc_stats->rbrfull++;
3038 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
3039 		    NXGE_FM_EREPORT_RDMC_RBRFULL);
3040 		rxchan_fatal = B_TRUE;
3041 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3042 		    "==> nxge_rx_err_evnts(channel %d): "
3043 		    "fatal error: rbr_full error", channel));
3044 	}
3045 	if (cs.bits.hdw.rbrlogpage) {
3046 		rdc_stats->rbrlogpage++;
3047 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
3048 		    NXGE_FM_EREPORT_RDMC_RBRLOGPAGE);
3049 		rxchan_fatal = B_TRUE;
3050 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3051 		    "==> nxge_rx_err_evnts(channel %d): "
3052 		    "fatal error: rbr logical page error", channel));
3053 	}
3054 	if (cs.bits.hdw.cfiglogpage) {
3055 		rdc_stats->cfiglogpage++;
3056 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
3057 		    NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE);
3058 		rxchan_fatal = B_TRUE;
3059 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3060 		    "==> nxge_rx_err_evnts(channel %d): "
3061 		    "fatal error: cfig logical page error", channel));
3062 	}
3063 
3064 	if (rxport_fatal)  {
3065 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3066 		    " nxge_rx_err_evnts: fatal error on Port #%d\n",
3067 		    portn));
3068 		if (isLDOMguest(nxgep)) {
3069 			status = NXGE_ERROR;
3070 		} else {
3071 			status = nxge_ipp_fatal_err_recover(nxgep);
3072 			if (status == NXGE_OK) {
3073 				FM_SERVICE_RESTORED(nxgep);
3074 			}
3075 		}
3076 	}
3077 
3078 	if (rxchan_fatal) {
3079 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3080 		    " nxge_rx_err_evnts: fatal error on Channel #%d\n",
3081 		    channel));
3082 		if (isLDOMguest(nxgep)) {
3083 			status = NXGE_ERROR;
3084 		} else {
3085 			status = nxge_rxdma_fatal_err_recover(nxgep, channel);
3086 			if (status == NXGE_OK) {
3087 				FM_SERVICE_RESTORED(nxgep);
3088 			}
3089 		}
3090 	}
3091 
3092 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rx_err_evnts"));
3093 
3094 	return (status);
3095 }
3096 
3097 /*
3098  * nxge_rdc_hvio_setup
3099  *
3100  *	This code appears to setup some Hypervisor variables.
3101  *
3102  * Arguments:
3103  * 	nxgep
3104  * 	channel
3105  *
3106  * Notes:
3107  *	What does NIU_LP_WORKAROUND mean?
3108  *
3109  * NPI/NXGE function calls:
3110  *	na
3111  *
3112  * Context:
3113  *	Any domain
3114  */
3115 #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
3116 static void
3117 nxge_rdc_hvio_setup(
3118 	nxge_t *nxgep, int channel)
3119 {
3120 	nxge_dma_common_t	*dma_common;
3121 	nxge_dma_common_t	*dma_control;
3122 	rx_rbr_ring_t		*ring;
3123 
3124 	ring = nxgep->rx_rbr_rings->rbr_rings[channel];
3125 	dma_common = nxgep->rx_buf_pool_p->dma_buf_pool_p[channel];
3126 
3127 	ring->hv_set = B_FALSE;
3128 
3129 	ring->hv_rx_buf_base_ioaddr_pp = (uint64_t)
3130 	    dma_common->orig_ioaddr_pp;
3131 	ring->hv_rx_buf_ioaddr_size = (uint64_t)
3132 	    dma_common->orig_alength;
3133 
3134 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: "
3135 	    "channel %d data buf base io $%lx ($%p) size 0x%lx (%ld 0x%lx)",
3136 	    channel, ring->hv_rx_buf_base_ioaddr_pp,
3137 	    dma_common->ioaddr_pp, ring->hv_rx_buf_ioaddr_size,
3138 	    dma_common->orig_alength, dma_common->orig_alength));
3139 
3140 	dma_control = nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel];
3141 
3142 	ring->hv_rx_cntl_base_ioaddr_pp =
3143 	    (uint64_t)dma_control->orig_ioaddr_pp;
3144 	ring->hv_rx_cntl_ioaddr_size =
3145 	    (uint64_t)dma_control->orig_alength;
3146 
3147 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: "
3148 	    "channel %d cntl base io $%p ($%p) size 0x%llx (%d 0x%x)",
3149 	    channel, ring->hv_rx_cntl_base_ioaddr_pp,
3150 	    dma_control->ioaddr_pp, ring->hv_rx_cntl_ioaddr_size,
3151 	    dma_control->orig_alength, dma_control->orig_alength));
3152 }
3153 #endif
3154 
3155 /*
3156  * nxge_map_rxdma
3157  *
3158  *	Map an RDC into our kernel space.
3159  *
3160  * Arguments:
3161  * 	nxgep
3162  * 	channel	The channel to map.
3163  *
3164  * Notes:
3165  *	1. Allocate & initialise a memory pool, if necessary.
3166  *	2. Allocate however many receive buffers are required.
3167  *	3. Setup buffers, descriptors, and mailbox.
3168  *
3169  * NPI/NXGE function calls:
3170  *	nxge_alloc_rx_mem_pool()
3171  *	nxge_alloc_rbb()
3172  *	nxge_map_rxdma_channel()
3173  *
3174  * Registers accessed:
3175  *
3176  * Context:
3177  *	Any domain
3178  */
3179 static nxge_status_t
3180 nxge_map_rxdma(p_nxge_t nxgep, int channel)
3181 {
3182 	nxge_dma_common_t	**data;
3183 	nxge_dma_common_t	**control;
3184 	rx_rbr_ring_t		**rbr_ring;
3185 	rx_rcr_ring_t		**rcr_ring;
3186 	rx_mbox_t		**mailbox;
3187 	uint32_t		chunks;
3188 
3189 	nxge_status_t		status;
3190 
3191 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma"));
3192 
3193 	if (!nxgep->rx_buf_pool_p) {
3194 		if (nxge_alloc_rx_mem_pool(nxgep) != NXGE_OK) {
3195 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3196 			    "<== nxge_map_rxdma: buf not allocated"));
3197 			return (NXGE_ERROR);
3198 		}
3199 	}
3200 
3201 	if (nxge_alloc_rxb(nxgep, channel) != NXGE_OK)
3202 		return (NXGE_ERROR);
3203 
3204 	/*
3205 	 * Map descriptors from the buffer polls for each dma channel.
3206 	 */
3207 
3208 	/*
3209 	 * Set up and prepare buffer blocks, descriptors
3210 	 * and mailbox.
3211 	 */
3212 	data = &nxgep->rx_buf_pool_p->dma_buf_pool_p[channel];
3213 	rbr_ring = &nxgep->rx_rbr_rings->rbr_rings[channel];
3214 	chunks = nxgep->rx_buf_pool_p->num_chunks[channel];
3215 
3216 	control = &nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel];
3217 	rcr_ring = &nxgep->rx_rcr_rings->rcr_rings[channel];
3218 
3219 	mailbox = &nxgep->rx_mbox_areas_p->rxmbox_areas[channel];
3220 
3221 	status = nxge_map_rxdma_channel(nxgep, channel, data, rbr_ring,
3222 	    chunks, control, rcr_ring, mailbox);
3223 	if (status != NXGE_OK) {
3224 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3225 		    "==> nxge_map_rxdma: nxge_map_rxdma_channel(%d) "
3226 		    "returned 0x%x",
3227 		    channel, status));
3228 		return (status);
3229 	}
3230 	nxgep->rx_rbr_rings->rbr_rings[channel]->index = (uint16_t)channel;
3231 	nxgep->rx_rcr_rings->rcr_rings[channel]->index = (uint16_t)channel;
3232 	nxgep->rx_rcr_rings->rcr_rings[channel]->rdc_stats =
3233 	    &nxgep->statsp->rdc_stats[channel];
3234 
3235 #if	defined(sun4v) && defined(NIU_LP_WORKAROUND)
3236 	if (!isLDOMguest(nxgep))
3237 		nxge_rdc_hvio_setup(nxgep, channel);
3238 #endif
3239 
3240 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3241 	    "<== nxge_map_rxdma: (status 0x%x channel %d)", status, channel));
3242 
3243 	return (status);
3244 }
3245 
3246 static void
3247 nxge_unmap_rxdma(p_nxge_t nxgep, int channel)
3248 {
3249 	rx_rbr_ring_t	*rbr_ring;
3250 	rx_rcr_ring_t	*rcr_ring;
3251 	rx_mbox_t	*mailbox;
3252 
3253 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_unmap_rxdma(%d)", channel));
3254 
3255 	if (!nxgep->rx_rbr_rings || !nxgep->rx_rcr_rings ||
3256 	    !nxgep->rx_mbox_areas_p)
3257 		return;
3258 
3259 	rbr_ring = nxgep->rx_rbr_rings->rbr_rings[channel];
3260 	rcr_ring = nxgep->rx_rcr_rings->rcr_rings[channel];
3261 	mailbox = nxgep->rx_mbox_areas_p->rxmbox_areas[channel];
3262 
3263 	if (!rbr_ring || !rcr_ring || !mailbox)
3264 		return;
3265 
3266 	(void) nxge_unmap_rxdma_channel(
3267 	    nxgep, channel, rbr_ring, rcr_ring, mailbox);
3268 
3269 	nxge_free_rxb(nxgep, channel);
3270 
3271 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma"));
3272 }
3273 
3274 nxge_status_t
3275 nxge_map_rxdma_channel(p_nxge_t nxgep, uint16_t channel,
3276     p_nxge_dma_common_t *dma_buf_p,  p_rx_rbr_ring_t *rbr_p,
3277     uint32_t num_chunks,
3278     p_nxge_dma_common_t *dma_cntl_p, p_rx_rcr_ring_t *rcr_p,
3279     p_rx_mbox_t *rx_mbox_p)
3280 {
3281 	int	status = NXGE_OK;
3282 
3283 	/*
3284 	 * Set up and prepare buffer blocks, descriptors
3285 	 * and mailbox.
3286 	 */
3287 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3288 	    "==> nxge_map_rxdma_channel (channel %d)", channel));
3289 	/*
3290 	 * Receive buffer blocks
3291 	 */
3292 	status = nxge_map_rxdma_channel_buf_ring(nxgep, channel,
3293 	    dma_buf_p, rbr_p, num_chunks);
3294 	if (status != NXGE_OK) {
3295 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3296 		    "==> nxge_map_rxdma_channel (channel %d): "
3297 		    "map buffer failed 0x%x", channel, status));
3298 		goto nxge_map_rxdma_channel_exit;
3299 	}
3300 
3301 	/*
3302 	 * Receive block ring, completion ring and mailbox.
3303 	 */
3304 	status = nxge_map_rxdma_channel_cfg_ring(nxgep, channel,
3305 	    dma_cntl_p, rbr_p, rcr_p, rx_mbox_p);
3306 	if (status != NXGE_OK) {
3307 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3308 		    "==> nxge_map_rxdma_channel (channel %d): "
3309 		    "map config failed 0x%x", channel, status));
3310 		goto nxge_map_rxdma_channel_fail2;
3311 	}
3312 
3313 	goto nxge_map_rxdma_channel_exit;
3314 
3315 nxge_map_rxdma_channel_fail3:
3316 	/* Free rbr, rcr */
3317 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3318 	    "==> nxge_map_rxdma_channel: free rbr/rcr "
3319 	    "(status 0x%x channel %d)",
3320 	    status, channel));
3321 	nxge_unmap_rxdma_channel_cfg_ring(nxgep,
3322 	    *rcr_p, *rx_mbox_p);
3323 
3324 nxge_map_rxdma_channel_fail2:
3325 	/* Free buffer blocks */
3326 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3327 	    "==> nxge_map_rxdma_channel: free rx buffers"
3328 	    "(nxgep 0x%x status 0x%x channel %d)",
3329 	    nxgep, status, channel));
3330 	nxge_unmap_rxdma_channel_buf_ring(nxgep, *rbr_p);
3331 
3332 	status = NXGE_ERROR;
3333 
3334 nxge_map_rxdma_channel_exit:
3335 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3336 	    "<== nxge_map_rxdma_channel: "
3337 	    "(nxgep 0x%x status 0x%x channel %d)",
3338 	    nxgep, status, channel));
3339 
3340 	return (status);
3341 }
3342 
3343 /*ARGSUSED*/
3344 static void
3345 nxge_unmap_rxdma_channel(p_nxge_t nxgep, uint16_t channel,
3346     p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
3347 {
3348 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3349 	    "==> nxge_unmap_rxdma_channel (channel %d)", channel));
3350 
3351 	/*
3352 	 * unmap receive block ring, completion ring and mailbox.
3353 	 */
3354 	(void) nxge_unmap_rxdma_channel_cfg_ring(nxgep,
3355 	    rcr_p, rx_mbox_p);
3356 
3357 	/* unmap buffer blocks */
3358 	(void) nxge_unmap_rxdma_channel_buf_ring(nxgep, rbr_p);
3359 
3360 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma_channel"));
3361 }
3362 
3363 /*ARGSUSED*/
3364 static nxge_status_t
3365 nxge_map_rxdma_channel_cfg_ring(p_nxge_t nxgep, uint16_t dma_channel,
3366     p_nxge_dma_common_t *dma_cntl_p, p_rx_rbr_ring_t *rbr_p,
3367     p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p)
3368 {
3369 	p_rx_rbr_ring_t 	rbrp;
3370 	p_rx_rcr_ring_t 	rcrp;
3371 	p_rx_mbox_t 		mboxp;
3372 	p_nxge_dma_common_t 	cntl_dmap;
3373 	p_nxge_dma_common_t 	dmap;
3374 	p_rx_msg_t 		*rx_msg_ring;
3375 	p_rx_msg_t 		rx_msg_p;
3376 	p_rbr_cfig_a_t		rcfga_p;
3377 	p_rbr_cfig_b_t		rcfgb_p;
3378 	p_rcrcfig_a_t		cfga_p;
3379 	p_rcrcfig_b_t		cfgb_p;
3380 	p_rxdma_cfig1_t		cfig1_p;
3381 	p_rxdma_cfig2_t		cfig2_p;
3382 	p_rbr_kick_t		kick_p;
3383 	uint32_t		dmaaddrp;
3384 	uint32_t		*rbr_vaddrp;
3385 	uint32_t		bkaddr;
3386 	nxge_status_t		status = NXGE_OK;
3387 	int			i;
3388 	uint32_t 		nxge_port_rcr_size;
3389 
3390 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3391 	    "==> nxge_map_rxdma_channel_cfg_ring"));
3392 
3393 	cntl_dmap = *dma_cntl_p;
3394 
3395 	/* Map in the receive block ring */
3396 	rbrp = *rbr_p;
3397 	dmap = (p_nxge_dma_common_t)&rbrp->rbr_desc;
3398 	nxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4);
3399 	/*
3400 	 * Zero out buffer block ring descriptors.
3401 	 */
3402 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
3403 
3404 	rcfga_p = &(rbrp->rbr_cfga);
3405 	rcfgb_p = &(rbrp->rbr_cfgb);
3406 	kick_p = &(rbrp->rbr_kick);
3407 	rcfga_p->value = 0;
3408 	rcfgb_p->value = 0;
3409 	kick_p->value = 0;
3410 	rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress;
3411 	rcfga_p->value = (rbrp->rbr_addr &
3412 	    (RBR_CFIG_A_STDADDR_MASK |
3413 	    RBR_CFIG_A_STDADDR_BASE_MASK));
3414 	rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT);
3415 
3416 	rcfgb_p->bits.ldw.bufsz0 = rbrp->pkt_buf_size0;
3417 	rcfgb_p->bits.ldw.vld0 = 1;
3418 	rcfgb_p->bits.ldw.bufsz1 = rbrp->pkt_buf_size1;
3419 	rcfgb_p->bits.ldw.vld1 = 1;
3420 	rcfgb_p->bits.ldw.bufsz2 = rbrp->pkt_buf_size2;
3421 	rcfgb_p->bits.ldw.vld2 = 1;
3422 	rcfgb_p->bits.ldw.bksize = nxgep->rx_bksize_code;
3423 
3424 	/*
3425 	 * For each buffer block, enter receive block address to the ring.
3426 	 */
3427 	rbr_vaddrp = (uint32_t *)dmap->kaddrp;
3428 	rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp;
3429 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3430 	    "==> nxge_map_rxdma_channel_cfg_ring: channel %d "
3431 	    "rbr_vaddrp $%p", dma_channel, rbr_vaddrp));
3432 
3433 	rx_msg_ring = rbrp->rx_msg_ring;
3434 	for (i = 0; i < rbrp->tnblocks; i++) {
3435 		rx_msg_p = rx_msg_ring[i];
3436 		rx_msg_p->nxgep = nxgep;
3437 		rx_msg_p->rx_rbr_p = rbrp;
3438 		bkaddr = (uint32_t)
3439 		    ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress
3440 		    >> RBR_BKADDR_SHIFT));
3441 		rx_msg_p->free = B_FALSE;
3442 		rx_msg_p->max_usage_cnt = 0xbaddcafe;
3443 
3444 		*rbr_vaddrp++ = bkaddr;
3445 	}
3446 
3447 	kick_p->bits.ldw.bkadd = rbrp->rbb_max;
3448 	rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
3449 
3450 	rbrp->rbr_rd_index = 0;
3451 
3452 	rbrp->rbr_consumed = 0;
3453 	rbrp->rbr_use_bcopy = B_TRUE;
3454 	rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0;
3455 	/*
3456 	 * Do bcopy on packets greater than bcopy size once
3457 	 * the lo threshold is reached.
3458 	 * This lo threshold should be less than the hi threshold.
3459 	 *
3460 	 * Do bcopy on every packet once the hi threshold is reached.
3461 	 */
3462 	if (nxge_rx_threshold_lo >= nxge_rx_threshold_hi) {
3463 		/* default it to use hi */
3464 		nxge_rx_threshold_lo = nxge_rx_threshold_hi;
3465 	}
3466 
3467 	if (nxge_rx_buf_size_type > NXGE_RBR_TYPE2) {
3468 		nxge_rx_buf_size_type = NXGE_RBR_TYPE2;
3469 	}
3470 	rbrp->rbr_bufsize_type = nxge_rx_buf_size_type;
3471 
3472 	switch (nxge_rx_threshold_hi) {
3473 	default:
3474 	case	NXGE_RX_COPY_NONE:
3475 		/* Do not do bcopy at all */
3476 		rbrp->rbr_use_bcopy = B_FALSE;
3477 		rbrp->rbr_threshold_hi = rbrp->rbb_max;
3478 		break;
3479 
3480 	case NXGE_RX_COPY_1:
3481 	case NXGE_RX_COPY_2:
3482 	case NXGE_RX_COPY_3:
3483 	case NXGE_RX_COPY_4:
3484 	case NXGE_RX_COPY_5:
3485 	case NXGE_RX_COPY_6:
3486 	case NXGE_RX_COPY_7:
3487 		rbrp->rbr_threshold_hi =
3488 		    rbrp->rbb_max *
3489 		    (nxge_rx_threshold_hi)/NXGE_RX_BCOPY_SCALE;
3490 		break;
3491 
3492 	case NXGE_RX_COPY_ALL:
3493 		rbrp->rbr_threshold_hi = 0;
3494 		break;
3495 	}
3496 
3497 	switch (nxge_rx_threshold_lo) {
3498 	default:
3499 	case	NXGE_RX_COPY_NONE:
3500 		/* Do not do bcopy at all */
3501 		if (rbrp->rbr_use_bcopy) {
3502 			rbrp->rbr_use_bcopy = B_FALSE;
3503 		}
3504 		rbrp->rbr_threshold_lo = rbrp->rbb_max;
3505 		break;
3506 
3507 	case NXGE_RX_COPY_1:
3508 	case NXGE_RX_COPY_2:
3509 	case NXGE_RX_COPY_3:
3510 	case NXGE_RX_COPY_4:
3511 	case NXGE_RX_COPY_5:
3512 	case NXGE_RX_COPY_6:
3513 	case NXGE_RX_COPY_7:
3514 		rbrp->rbr_threshold_lo =
3515 		    rbrp->rbb_max *
3516 		    (nxge_rx_threshold_lo)/NXGE_RX_BCOPY_SCALE;
3517 		break;
3518 
3519 	case NXGE_RX_COPY_ALL:
3520 		rbrp->rbr_threshold_lo = 0;
3521 		break;
3522 	}
3523 
3524 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
3525 	    "nxge_map_rxdma_channel_cfg_ring: channel %d "
3526 	    "rbb_max %d "
3527 	    "rbrp->rbr_bufsize_type %d "
3528 	    "rbb_threshold_hi %d "
3529 	    "rbb_threshold_lo %d",
3530 	    dma_channel,
3531 	    rbrp->rbb_max,
3532 	    rbrp->rbr_bufsize_type,
3533 	    rbrp->rbr_threshold_hi,
3534 	    rbrp->rbr_threshold_lo));
3535 
3536 	rbrp->page_valid.value = 0;
3537 	rbrp->page_mask_1.value = rbrp->page_mask_2.value = 0;
3538 	rbrp->page_value_1.value = rbrp->page_value_2.value = 0;
3539 	rbrp->page_reloc_1.value = rbrp->page_reloc_2.value = 0;
3540 	rbrp->page_hdl.value = 0;
3541 
3542 	rbrp->page_valid.bits.ldw.page0 = 1;
3543 	rbrp->page_valid.bits.ldw.page1 = 1;
3544 
3545 	/* Map in the receive completion ring */
3546 	rcrp = (p_rx_rcr_ring_t)
3547 	    KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP);
3548 	rcrp->rdc = dma_channel;
3549 
3550 	nxge_port_rcr_size = nxgep->nxge_port_rcr_size;
3551 	rcrp->comp_size = nxge_port_rcr_size;
3552 	rcrp->comp_wrap_mask = nxge_port_rcr_size - 1;
3553 
3554 	rcrp->max_receive_pkts = nxge_max_rx_pkts;
3555 
3556 	dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc;
3557 	nxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size,
3558 	    sizeof (rcr_entry_t));
3559 	rcrp->comp_rd_index = 0;
3560 	rcrp->comp_wt_index = 0;
3561 	rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
3562 	    (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
3563 #if defined(__i386)
3564 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
3565 	    (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
3566 #else
3567 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
3568 	    (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
3569 #endif
3570 
3571 	rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
3572 	    (nxge_port_rcr_size - 1);
3573 	rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
3574 	    (nxge_port_rcr_size - 1);
3575 
3576 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3577 	    "==> nxge_map_rxdma_channel_cfg_ring: "
3578 	    "channel %d "
3579 	    "rbr_vaddrp $%p "
3580 	    "rcr_desc_rd_head_p $%p "
3581 	    "rcr_desc_rd_head_pp $%p "
3582 	    "rcr_desc_rd_last_p $%p "
3583 	    "rcr_desc_rd_last_pp $%p ",
3584 	    dma_channel,
3585 	    rbr_vaddrp,
3586 	    rcrp->rcr_desc_rd_head_p,
3587 	    rcrp->rcr_desc_rd_head_pp,
3588 	    rcrp->rcr_desc_last_p,
3589 	    rcrp->rcr_desc_last_pp));
3590 
3591 	/*
3592 	 * Zero out buffer block ring descriptors.
3593 	 */
3594 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
3595 
3596 	rcrp->intr_timeout = (nxgep->intr_timeout <
3597 	    NXGE_RDC_RCR_TIMEOUT_MIN) ? NXGE_RDC_RCR_TIMEOUT_MIN :
3598 	    nxgep->intr_timeout;
3599 
3600 	rcrp->intr_threshold = (nxgep->intr_threshold <
3601 	    NXGE_RDC_RCR_THRESHOLD_MIN) ? NXGE_RDC_RCR_THRESHOLD_MIN :
3602 	    nxgep->intr_threshold;
3603 
3604 	rcrp->full_hdr_flag = B_FALSE;
3605 	rcrp->sw_priv_hdr_len = 0;
3606 
3607 	cfga_p = &(rcrp->rcr_cfga);
3608 	cfgb_p = &(rcrp->rcr_cfgb);
3609 	cfga_p->value = 0;
3610 	cfgb_p->value = 0;
3611 	rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress;
3612 	cfga_p->value = (rcrp->rcr_addr &
3613 	    (RCRCFIG_A_STADDR_MASK |
3614 	    RCRCFIG_A_STADDR_BASE_MASK));
3615 
3616 	rcfga_p->value |= ((uint64_t)rcrp->comp_size <<
3617 	    RCRCFIG_A_LEN_SHIF);
3618 
3619 	/*
3620 	 * Timeout should be set based on the system clock divider.
3621 	 * A timeout value of 1 assumes that the
3622 	 * granularity (1000) is 3 microseconds running at 300MHz.
3623 	 */
3624 	cfgb_p->bits.ldw.pthres = rcrp->intr_threshold;
3625 	cfgb_p->bits.ldw.timeout = rcrp->intr_timeout;
3626 	cfgb_p->bits.ldw.entout = 1;
3627 
3628 	/* Map in the mailbox */
3629 	mboxp = (p_rx_mbox_t)
3630 	    KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP);
3631 	dmap = (p_nxge_dma_common_t)&mboxp->rx_mbox;
3632 	nxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t));
3633 	cfig1_p = (p_rxdma_cfig1_t)&mboxp->rx_cfg1;
3634 	cfig2_p = (p_rxdma_cfig2_t)&mboxp->rx_cfg2;
3635 	cfig1_p->value = cfig2_p->value = 0;
3636 
3637 	mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress;
3638 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3639 	    "==> nxge_map_rxdma_channel_cfg_ring: "
3640 	    "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx",
3641 	    dma_channel, cfig1_p->value, cfig2_p->value,
3642 	    mboxp->mbox_addr));
3643 
3644 	dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress >> 32
3645 	    & 0xfff);
3646 	cfig1_p->bits.ldw.mbaddr_h = dmaaddrp;
3647 
3648 
3649 	dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff);
3650 	dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress &
3651 	    RXDMA_CFIG2_MBADDR_L_MASK);
3652 
3653 	cfig2_p->bits.ldw.mbaddr = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT);
3654 
3655 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3656 	    "==> nxge_map_rxdma_channel_cfg_ring: "
3657 	    "channel %d damaddrp $%p "
3658 	    "cfg1 0x%016llx cfig2 0x%016llx",
3659 	    dma_channel, dmaaddrp,
3660 	    cfig1_p->value, cfig2_p->value));
3661 
3662 	cfig2_p->bits.ldw.full_hdr = rcrp->full_hdr_flag;
3663 	cfig2_p->bits.ldw.offset = rcrp->sw_priv_hdr_len;
3664 
3665 	rbrp->rx_rcr_p = rcrp;
3666 	rcrp->rx_rbr_p = rbrp;
3667 	*rcr_p = rcrp;
3668 	*rx_mbox_p = mboxp;
3669 
3670 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3671 	    "<== nxge_map_rxdma_channel_cfg_ring status 0x%08x", status));
3672 
3673 	return (status);
3674 }
3675 
3676 /*ARGSUSED*/
3677 static void
3678 nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t nxgep,
3679     p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
3680 {
3681 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3682 	    "==> nxge_unmap_rxdma_channel_cfg_ring: channel %d",
3683 	    rcr_p->rdc));
3684 
3685 	KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t));
3686 	KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t));
3687 
3688 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3689 	    "<== nxge_unmap_rxdma_channel_cfg_ring"));
3690 }
3691 
3692 static nxge_status_t
3693 nxge_map_rxdma_channel_buf_ring(p_nxge_t nxgep, uint16_t channel,
3694     p_nxge_dma_common_t *dma_buf_p,
3695     p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks)
3696 {
3697 	p_rx_rbr_ring_t 	rbrp;
3698 	p_nxge_dma_common_t 	dma_bufp, tmp_bufp;
3699 	p_rx_msg_t 		*rx_msg_ring;
3700 	p_rx_msg_t 		rx_msg_p;
3701 	p_mblk_t 		mblk_p;
3702 
3703 	rxring_info_t *ring_info;
3704 	nxge_status_t		status = NXGE_OK;
3705 	int			i, j, index;
3706 	uint32_t		size, bsize, nblocks, nmsgs;
3707 
3708 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3709 	    "==> nxge_map_rxdma_channel_buf_ring: channel %d",
3710 	    channel));
3711 
3712 	dma_bufp = tmp_bufp = *dma_buf_p;
3713 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3714 	    " nxge_map_rxdma_channel_buf_ring: channel %d to map %d "
3715 	    "chunks bufp 0x%016llx",
3716 	    channel, num_chunks, dma_bufp));
3717 
3718 	nmsgs = 0;
3719 	for (i = 0; i < num_chunks; i++, tmp_bufp++) {
3720 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3721 		    "==> nxge_map_rxdma_channel_buf_ring: channel %d "
3722 		    "bufp 0x%016llx nblocks %d nmsgs %d",
3723 		    channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
3724 		nmsgs += tmp_bufp->nblocks;
3725 	}
3726 	if (!nmsgs) {
3727 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3728 		    "<== nxge_map_rxdma_channel_buf_ring: channel %d "
3729 		    "no msg blocks",
3730 		    channel));
3731 		status = NXGE_ERROR;
3732 		goto nxge_map_rxdma_channel_buf_ring_exit;
3733 	}
3734 
3735 	rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (*rbrp), KM_SLEEP);
3736 
3737 	size = nmsgs * sizeof (p_rx_msg_t);
3738 	rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
3739 	ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t),
3740 	    KM_SLEEP);
3741 
3742 	MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER,
3743 	    (void *)nxgep->interrupt_cookie);
3744 	MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER,
3745 	    (void *)nxgep->interrupt_cookie);
3746 	rbrp->rdc = channel;
3747 	rbrp->num_blocks = num_chunks;
3748 	rbrp->tnblocks = nmsgs;
3749 	rbrp->rbb_max = nmsgs;
3750 	rbrp->rbr_max_size = nmsgs;
3751 	rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1);
3752 
3753 	/*
3754 	 * Buffer sizes suggested by NIU architect.
3755 	 * 256, 512 and 2K.
3756 	 */
3757 
3758 	rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B;
3759 	rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES;
3760 	rbrp->npi_pkt_buf_size0 = SIZE_256B;
3761 
3762 	rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K;
3763 	rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES;
3764 	rbrp->npi_pkt_buf_size1 = SIZE_1KB;
3765 
3766 	rbrp->block_size = nxgep->rx_default_block_size;
3767 
3768 	if (!nxgep->mac.is_jumbo) {
3769 		rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K;
3770 		rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES;
3771 		rbrp->npi_pkt_buf_size2 = SIZE_2KB;
3772 	} else {
3773 		if (rbrp->block_size >= 0x2000) {
3774 			rbrp->pkt_buf_size2 = RBR_BUFSZ2_8K;
3775 			rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_8K_BYTES;
3776 			rbrp->npi_pkt_buf_size2 = SIZE_8KB;
3777 		} else {
3778 			rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K;
3779 			rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES;
3780 			rbrp->npi_pkt_buf_size2 = SIZE_4KB;
3781 		}
3782 	}
3783 
3784 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3785 	    "==> nxge_map_rxdma_channel_buf_ring: channel %d "
3786 	    "actual rbr max %d rbb_max %d nmsgs %d "
3787 	    "rbrp->block_size %d default_block_size %d "
3788 	    "(config nxge_rbr_size %d nxge_rbr_spare_size %d)",
3789 	    channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs,
3790 	    rbrp->block_size, nxgep->rx_default_block_size,
3791 	    nxge_rbr_size, nxge_rbr_spare_size));
3792 
3793 	/* Map in buffers from the buffer pool.  */
3794 	index = 0;
3795 	for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) {
3796 		bsize = dma_bufp->block_size;
3797 		nblocks = dma_bufp->nblocks;
3798 #if defined(__i386)
3799 		ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp;
3800 #else
3801 		ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp;
3802 #endif
3803 		ring_info->buffer[i].buf_index = i;
3804 		ring_info->buffer[i].buf_size = dma_bufp->alength;
3805 		ring_info->buffer[i].start_index = index;
3806 #if defined(__i386)
3807 		ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp;
3808 #else
3809 		ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp;
3810 #endif
3811 
3812 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3813 		    " nxge_map_rxdma_channel_buf_ring: map channel %d "
3814 		    "chunk %d"
3815 		    " nblocks %d chunk_size %x block_size 0x%x "
3816 		    "dma_bufp $%p", channel, i,
3817 		    dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize,
3818 		    dma_bufp));
3819 
3820 		for (j = 0; j < nblocks; j++) {
3821 			if ((rx_msg_p = nxge_allocb(bsize, BPRI_LO,
3822 			    dma_bufp)) == NULL) {
3823 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3824 				    "allocb failed (index %d i %d j %d)",
3825 				    index, i, j));
3826 				goto nxge_map_rxdma_channel_buf_ring_fail1;
3827 			}
3828 			rx_msg_ring[index] = rx_msg_p;
3829 			rx_msg_p->block_index = index;
3830 			rx_msg_p->shifted_addr = (uint32_t)
3831 			    ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >>
3832 			    RBR_BKADDR_SHIFT));
3833 
3834 			NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3835 			    "index %d j %d rx_msg_p $%p mblk %p",
3836 			    index, j, rx_msg_p, rx_msg_p->rx_mblk_p));
3837 
3838 			mblk_p = rx_msg_p->rx_mblk_p;
3839 			mblk_p->b_wptr = mblk_p->b_rptr + bsize;
3840 
3841 			rbrp->rbr_ref_cnt++;
3842 			index++;
3843 			rx_msg_p->buf_dma.dma_channel = channel;
3844 		}
3845 
3846 		rbrp->rbr_alloc_type = DDI_MEM_ALLOC;
3847 		if (dma_bufp->contig_alloc_type) {
3848 			rbrp->rbr_alloc_type = CONTIG_MEM_ALLOC;
3849 		}
3850 
3851 		if (dma_bufp->kmem_alloc_type) {
3852 			rbrp->rbr_alloc_type = KMEM_ALLOC;
3853 		}
3854 
3855 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3856 		    " nxge_map_rxdma_channel_buf_ring: map channel %d "
3857 		    "chunk %d"
3858 		    " nblocks %d chunk_size %x block_size 0x%x "
3859 		    "dma_bufp $%p",
3860 		    channel, i,
3861 		    dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize,
3862 		    dma_bufp));
3863 	}
3864 	if (i < rbrp->num_blocks) {
3865 		goto nxge_map_rxdma_channel_buf_ring_fail1;
3866 	}
3867 
3868 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3869 	    "nxge_map_rxdma_channel_buf_ring: done buf init "
3870 	    "channel %d msg block entries %d",
3871 	    channel, index));
3872 	ring_info->block_size_mask = bsize - 1;
3873 	rbrp->rx_msg_ring = rx_msg_ring;
3874 	rbrp->dma_bufp = dma_buf_p;
3875 	rbrp->ring_info = ring_info;
3876 
3877 	status = nxge_rxbuf_index_info_init(nxgep, rbrp);
3878 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3879 	    " nxge_map_rxdma_channel_buf_ring: "
3880 	    "channel %d done buf info init", channel));
3881 
3882 	/*
3883 	 * Finally, permit nxge_freeb() to call nxge_post_page().
3884 	 */
3885 	rbrp->rbr_state = RBR_POSTING;
3886 
3887 	*rbr_p = rbrp;
3888 	goto nxge_map_rxdma_channel_buf_ring_exit;
3889 
3890 nxge_map_rxdma_channel_buf_ring_fail1:
3891 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3892 	    " nxge_map_rxdma_channel_buf_ring: failed channel (0x%x)",
3893 	    channel, status));
3894 
3895 	index--;
3896 	for (; index >= 0; index--) {
3897 		rx_msg_p = rx_msg_ring[index];
3898 		if (rx_msg_p != NULL) {
3899 			freeb(rx_msg_p->rx_mblk_p);
3900 			rx_msg_ring[index] = NULL;
3901 		}
3902 	}
3903 nxge_map_rxdma_channel_buf_ring_fail:
3904 	MUTEX_DESTROY(&rbrp->post_lock);
3905 	MUTEX_DESTROY(&rbrp->lock);
3906 	KMEM_FREE(ring_info, sizeof (rxring_info_t));
3907 	KMEM_FREE(rx_msg_ring, size);
3908 	KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t));
3909 
3910 	status = NXGE_ERROR;
3911 
3912 nxge_map_rxdma_channel_buf_ring_exit:
3913 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3914 	    "<== nxge_map_rxdma_channel_buf_ring status 0x%08x", status));
3915 
3916 	return (status);
3917 }
3918 
3919 /*ARGSUSED*/
3920 static void
3921 nxge_unmap_rxdma_channel_buf_ring(p_nxge_t nxgep,
3922     p_rx_rbr_ring_t rbr_p)
3923 {
3924 	p_rx_msg_t 		*rx_msg_ring;
3925 	p_rx_msg_t 		rx_msg_p;
3926 	rxring_info_t 		*ring_info;
3927 	int			i;
3928 	uint32_t		size;
3929 #ifdef	NXGE_DEBUG
3930 	int			num_chunks;
3931 #endif
3932 
3933 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3934 	    "==> nxge_unmap_rxdma_channel_buf_ring"));
3935 	if (rbr_p == NULL) {
3936 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
3937 		    "<== nxge_unmap_rxdma_channel_buf_ring: NULL rbrp"));
3938 		return;
3939 	}
3940 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3941 	    "==> nxge_unmap_rxdma_channel_buf_ring: channel %d",
3942 	    rbr_p->rdc));
3943 
3944 	rx_msg_ring = rbr_p->rx_msg_ring;
3945 	ring_info = rbr_p->ring_info;
3946 
3947 	if (rx_msg_ring == NULL || ring_info == NULL) {
3948 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3949 		    "<== nxge_unmap_rxdma_channel_buf_ring: "
3950 		    "rx_msg_ring $%p ring_info $%p",
3951 		    rx_msg_p, ring_info));
3952 		return;
3953 	}
3954 
3955 #ifdef	NXGE_DEBUG
3956 	num_chunks = rbr_p->num_blocks;
3957 #endif
3958 	size = rbr_p->tnblocks * sizeof (p_rx_msg_t);
3959 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3960 	    " nxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d "
3961 	    "tnblocks %d (max %d) size ptrs %d ",
3962 	    rbr_p->rdc, num_chunks,
3963 	    rbr_p->tnblocks, rbr_p->rbr_max_size, size));
3964 
3965 	for (i = 0; i < rbr_p->tnblocks; i++) {
3966 		rx_msg_p = rx_msg_ring[i];
3967 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3968 		    " nxge_unmap_rxdma_channel_buf_ring: "
3969 		    "rx_msg_p $%p",
3970 		    rx_msg_p));
3971 		if (rx_msg_p != NULL) {
3972 			freeb(rx_msg_p->rx_mblk_p);
3973 			rx_msg_ring[i] = NULL;
3974 		}
3975 	}
3976 
3977 	/*
3978 	 * We no longer may use the mutex <post_lock>. By setting
3979 	 * <rbr_state> to anything but POSTING, we prevent
3980 	 * nxge_post_page() from accessing a dead mutex.
3981 	 */
3982 	rbr_p->rbr_state = RBR_UNMAPPING;
3983 	MUTEX_DESTROY(&rbr_p->post_lock);
3984 
3985 	MUTEX_DESTROY(&rbr_p->lock);
3986 
3987 	if (rbr_p->rbr_ref_cnt == 0) {
3988 		/*
3989 		 * This is the normal state of affairs.
3990 		 * Need to free the following buffers:
3991 		 *  - data buffers
3992 		 *  - rx_msg ring
3993 		 *  - ring_info
3994 		 *  - rbr ring
3995 		 */
3996 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
3997 		    "unmap_rxdma_buf_ring: No outstanding - freeing "));
3998 		nxge_rxdma_databuf_free(rbr_p);
3999 		KMEM_FREE(ring_info, sizeof (rxring_info_t));
4000 		KMEM_FREE(rx_msg_ring, size);
4001 		KMEM_FREE(rbr_p, sizeof (*rbr_p));
4002 	} else {
4003 		/*
4004 		 * Some of our buffers are still being used.
4005 		 * Therefore, tell nxge_freeb() this ring is
4006 		 * unmapped, so it may free <rbr_p> for us.
4007 		 */
4008 		rbr_p->rbr_state = RBR_UNMAPPED;
4009 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4010 		    "unmap_rxdma_buf_ring: %d %s outstanding.",
4011 		    rbr_p->rbr_ref_cnt,
4012 		    rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs"));
4013 	}
4014 
4015 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4016 	    "<== nxge_unmap_rxdma_channel_buf_ring"));
4017 }
4018 
4019 /*
4020  * nxge_rxdma_hw_start_common
4021  *
4022  * Arguments:
4023  * 	nxgep
4024  *
4025  * Notes:
4026  *
4027  * NPI/NXGE function calls:
4028  *	nxge_init_fzc_rx_common();
4029  *	nxge_init_fzc_rxdma_port();
4030  *
4031  * Registers accessed:
4032  *
4033  * Context:
4034  *	Service domain
4035  */
4036 static nxge_status_t
4037 nxge_rxdma_hw_start_common(p_nxge_t nxgep)
4038 {
4039 	nxge_status_t		status = NXGE_OK;
4040 
4041 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common"));
4042 
4043 	/*
4044 	 * Load the sharable parameters by writing to the
4045 	 * function zero control registers. These FZC registers
4046 	 * should be initialized only once for the entire chip.
4047 	 */
4048 	(void) nxge_init_fzc_rx_common(nxgep);
4049 
4050 	/*
4051 	 * Initialize the RXDMA port specific FZC control configurations.
4052 	 * These FZC registers are pertaining to each port.
4053 	 */
4054 	(void) nxge_init_fzc_rxdma_port(nxgep);
4055 
4056 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common"));
4057 
4058 	return (status);
4059 }
4060 
4061 static nxge_status_t
4062 nxge_rxdma_hw_start(p_nxge_t nxgep, int channel)
4063 {
4064 	int			i, ndmas;
4065 	p_rx_rbr_rings_t 	rx_rbr_rings;
4066 	p_rx_rbr_ring_t		*rbr_rings;
4067 	p_rx_rcr_rings_t 	rx_rcr_rings;
4068 	p_rx_rcr_ring_t		*rcr_rings;
4069 	p_rx_mbox_areas_t 	rx_mbox_areas_p;
4070 	p_rx_mbox_t		*rx_mbox_p;
4071 	nxge_status_t		status = NXGE_OK;
4072 
4073 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start"));
4074 
4075 	rx_rbr_rings = nxgep->rx_rbr_rings;
4076 	rx_rcr_rings = nxgep->rx_rcr_rings;
4077 	if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
4078 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
4079 		    "<== nxge_rxdma_hw_start: NULL ring pointers"));
4080 		return (NXGE_ERROR);
4081 	}
4082 	ndmas = rx_rbr_rings->ndmas;
4083 	if (ndmas == 0) {
4084 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
4085 		    "<== nxge_rxdma_hw_start: no dma channel allocated"));
4086 		return (NXGE_ERROR);
4087 	}
4088 
4089 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4090 	    "==> nxge_rxdma_hw_start (ndmas %d)", ndmas));
4091 
4092 	rbr_rings = rx_rbr_rings->rbr_rings;
4093 	rcr_rings = rx_rcr_rings->rcr_rings;
4094 	rx_mbox_areas_p = nxgep->rx_mbox_areas_p;
4095 	if (rx_mbox_areas_p) {
4096 		rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
4097 	}
4098 
4099 	i = channel;
4100 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4101 	    "==> nxge_rxdma_hw_start (ndmas %d) channel %d",
4102 	    ndmas, channel));
4103 	status = nxge_rxdma_start_channel(nxgep, channel,
4104 	    (p_rx_rbr_ring_t)rbr_rings[i],
4105 	    (p_rx_rcr_ring_t)rcr_rings[i],
4106 	    (p_rx_mbox_t)rx_mbox_p[i]);
4107 	if (status != NXGE_OK) {
4108 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4109 		    "==> nxge_rxdma_hw_start: disable "
4110 		    "(status 0x%x channel %d)", status, channel));
4111 		return (status);
4112 	}
4113 
4114 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start: "
4115 	    "rx_rbr_rings 0x%016llx rings 0x%016llx",
4116 	    rx_rbr_rings, rx_rcr_rings));
4117 
4118 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4119 	    "==> nxge_rxdma_hw_start: (status 0x%x)", status));
4120 
4121 	return (status);
4122 }
4123 
4124 static void
4125 nxge_rxdma_hw_stop(p_nxge_t nxgep, int channel)
4126 {
4127 	p_rx_rbr_rings_t 	rx_rbr_rings;
4128 	p_rx_rcr_rings_t 	rx_rcr_rings;
4129 
4130 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop"));
4131 
4132 	rx_rbr_rings = nxgep->rx_rbr_rings;
4133 	rx_rcr_rings = nxgep->rx_rcr_rings;
4134 	if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
4135 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
4136 		    "<== nxge_rxdma_hw_stop: NULL ring pointers"));
4137 		return;
4138 	}
4139 
4140 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4141 	    "==> nxge_rxdma_hw_stop(channel %d)",
4142 	    channel));
4143 	(void) nxge_rxdma_stop_channel(nxgep, channel);
4144 
4145 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop: "
4146 	    "rx_rbr_rings 0x%016llx rings 0x%016llx",
4147 	    rx_rbr_rings, rx_rcr_rings));
4148 
4149 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_hw_stop"));
4150 }
4151 
4152 
4153 static nxge_status_t
4154 nxge_rxdma_start_channel(p_nxge_t nxgep, uint16_t channel,
4155     p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p)
4156 
4157 {
4158 	npi_handle_t		handle;
4159 	npi_status_t		rs = NPI_SUCCESS;
4160 	rx_dma_ctl_stat_t	cs;
4161 	rx_dma_ent_msk_t	ent_mask;
4162 	nxge_status_t		status = NXGE_OK;
4163 
4164 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel"));
4165 
4166 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
4167 
4168 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "nxge_rxdma_start_channel: "
4169 		"npi handle addr $%p acc $%p",
4170 		nxgep->npi_handle.regp, nxgep->npi_handle.regh));
4171 
4172 	/* Reset RXDMA channel, but not if you're a guest. */
4173 	if (!isLDOMguest(nxgep)) {
4174 		rs = npi_rxdma_cfg_rdc_reset(handle, channel);
4175 		if (rs != NPI_SUCCESS) {
4176 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4177 			    "==> nxge_init_fzc_rdc: "
4178 			    "npi_rxdma_cfg_rdc_reset(%d) returned 0x%08x",
4179 			    channel, rs));
4180 			return (NXGE_ERROR | rs);
4181 		}
4182 
4183 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4184 		    "==> nxge_rxdma_start_channel: reset done: channel %d",
4185 		    channel));
4186 	}
4187 
4188 #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
4189 	if (isLDOMguest(nxgep))
4190 		(void) nxge_rdc_lp_conf(nxgep, channel);
4191 #endif
4192 
4193 	/*
4194 	 * Initialize the RXDMA channel specific FZC control
4195 	 * configurations. These FZC registers are pertaining
4196 	 * to each RX channel (logical pages).
4197 	 */
4198 	if (!isLDOMguest(nxgep)) {
4199 		status = nxge_init_fzc_rxdma_channel(nxgep, channel);
4200 		if (status != NXGE_OK) {
4201 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4202 				"==> nxge_rxdma_start_channel: "
4203 				"init fzc rxdma failed (0x%08x channel %d)",
4204 				status, channel));
4205 			return (status);
4206 		}
4207 
4208 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4209 			"==> nxge_rxdma_start_channel: fzc done"));
4210 	}
4211 
4212 	/* Set up the interrupt event masks. */
4213 	ent_mask.value = 0;
4214 	ent_mask.value |= RX_DMA_ENT_MSK_RBREMPTY_MASK;
4215 	rs = npi_rxdma_event_mask(handle, OP_SET, channel,
4216 	    &ent_mask);
4217 	if (rs != NPI_SUCCESS) {
4218 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4219 			"==> nxge_rxdma_start_channel: "
4220 			"init rxdma event masks failed "
4221 			"(0x%08x channel %d)",
4222 			status, channel));
4223 		return (NXGE_ERROR | rs);
4224 	}
4225 
4226 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4227 		"==> nxge_rxdma_start_channel: "
4228 		"event done: channel %d (mask 0x%016llx)",
4229 		channel, ent_mask.value));
4230 
4231 	/* Initialize the receive DMA control and status register */
4232 	cs.value = 0;
4233 	cs.bits.hdw.mex = 1;
4234 	cs.bits.hdw.rcrthres = 1;
4235 	cs.bits.hdw.rcrto = 1;
4236 	cs.bits.hdw.rbr_empty = 1;
4237 	status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel, &cs);
4238 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: "
4239 		"channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value));
4240 	if (status != NXGE_OK) {
4241 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4242 			"==> nxge_rxdma_start_channel: "
4243 			"init rxdma control register failed (0x%08x channel %d",
4244 			status, channel));
4245 		return (status);
4246 	}
4247 
4248 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: "
4249 		"control done - channel %d cs 0x%016llx", channel, cs.value));
4250 
4251 	/*
4252 	 * Load RXDMA descriptors, buffers, mailbox,
4253 	 * initialise the receive DMA channels and
4254 	 * enable each DMA channel.
4255 	 */
4256 	status = nxge_enable_rxdma_channel(nxgep,
4257 	    channel, rbr_p, rcr_p, mbox_p);
4258 
4259 	if (status != NXGE_OK) {
4260 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4261 		    " nxge_rxdma_start_channel: "
4262 		    " enable rxdma failed (0x%08x channel %d)",
4263 		    status, channel));
4264 		return (status);
4265 	}
4266 
4267 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4268 	    "==> nxge_rxdma_start_channel: enabled channel %d"));
4269 
4270 	if (isLDOMguest(nxgep)) {
4271 		/* Add interrupt handler for this channel. */
4272 		status = nxge_hio_intr_add(nxgep, VP_BOUND_RX, channel);
4273 		if (status != NXGE_OK) {
4274 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4275 			    " nxge_rxdma_start_channel: "
4276 			    " nxge_hio_intr_add failed (0x%08x channel %d)",
4277 			    status, channel));
4278 			return (status);
4279 		}
4280 	}
4281 
4282 	ent_mask.value = 0;
4283 	ent_mask.value |= (RX_DMA_ENT_MSK_WRED_DROP_MASK |
4284 				RX_DMA_ENT_MSK_PTDROP_PKT_MASK);
4285 	rs = npi_rxdma_event_mask(handle, OP_SET, channel,
4286 			&ent_mask);
4287 	if (rs != NPI_SUCCESS) {
4288 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4289 			"==> nxge_rxdma_start_channel: "
4290 			"init rxdma event masks failed (0x%08x channel %d)",
4291 			status, channel));
4292 		return (NXGE_ERROR | rs);
4293 	}
4294 
4295 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: "
4296 		"control done - channel %d cs 0x%016llx", channel, cs.value));
4297 
4298 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_start_channel"));
4299 
4300 	return (NXGE_OK);
4301 }
4302 
4303 static nxge_status_t
4304 nxge_rxdma_stop_channel(p_nxge_t nxgep, uint16_t channel)
4305 {
4306 	npi_handle_t		handle;
4307 	npi_status_t		rs = NPI_SUCCESS;
4308 	rx_dma_ctl_stat_t	cs;
4309 	rx_dma_ent_msk_t	ent_mask;
4310 	nxge_status_t		status = NXGE_OK;
4311 
4312 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel"));
4313 
4314 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
4315 
4316 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "nxge_rxdma_stop_channel: "
4317 	    "npi handle addr $%p acc $%p",
4318 	    nxgep->npi_handle.regp, nxgep->npi_handle.regh));
4319 
4320 	if (!isLDOMguest(nxgep)) {
4321 		/*
4322 		 * Stop RxMAC = A.9.2.6
4323 		 */
4324 		if (nxge_rx_mac_disable(nxgep) != NXGE_OK) {
4325 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4326 			    "nxge_rxdma_stop_channel: "
4327 			    "Failed to disable RxMAC"));
4328 		}
4329 
4330 		/*
4331 		 * Drain IPP Port = A.9.3.6
4332 		 */
4333 		(void) nxge_ipp_drain(nxgep);
4334 	}
4335 
4336 	/* Reset RXDMA channel */
4337 	rs = npi_rxdma_cfg_rdc_reset(handle, channel);
4338 	if (rs != NPI_SUCCESS) {
4339 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4340 		    " nxge_rxdma_stop_channel: "
4341 		    " reset rxdma failed (0x%08x channel %d)",
4342 		    rs, channel));
4343 		return (NXGE_ERROR | rs);
4344 	}
4345 
4346 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
4347 	    "==> nxge_rxdma_stop_channel: reset done"));
4348 
4349 	/* Set up the interrupt event masks. */
4350 	ent_mask.value = RX_DMA_ENT_MSK_ALL;
4351 	rs = npi_rxdma_event_mask(handle, OP_SET, channel,
4352 	    &ent_mask);
4353 	if (rs != NPI_SUCCESS) {
4354 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4355 		    "==> nxge_rxdma_stop_channel: "
4356 		    "set rxdma event masks failed (0x%08x channel %d)",
4357 		    rs, channel));
4358 		return (NXGE_ERROR | rs);
4359 	}
4360 
4361 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
4362 	    "==> nxge_rxdma_stop_channel: event done"));
4363 
4364 	/*
4365 	 * Initialize the receive DMA control and status register
4366 	 */
4367 	cs.value = 0;
4368 	status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel, &cs);
4369 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel: control "
4370 	    " to default (all 0s) 0x%08x", cs.value));
4371 	if (status != NXGE_OK) {
4372 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4373 		    " nxge_rxdma_stop_channel: init rxdma"
4374 		    " control register failed (0x%08x channel %d",
4375 		    status, channel));
4376 		return (status);
4377 	}
4378 
4379 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
4380 	    "==> nxge_rxdma_stop_channel: control done"));
4381 
4382 	/*
4383 	 * Make sure channel is disabled.
4384 	 */
4385 	status = nxge_disable_rxdma_channel(nxgep, channel);
4386 
4387 	if (status != NXGE_OK) {
4388 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4389 		    " nxge_rxdma_stop_channel: "
4390 		    " init enable rxdma failed (0x%08x channel %d)",
4391 		    status, channel));
4392 		return (status);
4393 	}
4394 
4395 	if (!isLDOMguest(nxgep)) {
4396 		/*
4397 		 * Enable RxMAC = A.9.2.10
4398 		 */
4399 		if (nxge_rx_mac_enable(nxgep) != NXGE_OK) {
4400 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4401 			    "nxge_rxdma_stop_channel: Rx MAC still disabled"));
4402 		}
4403 	}
4404 
4405 	NXGE_DEBUG_MSG((nxgep,
4406 	    RX_CTL, "==> nxge_rxdma_stop_channel: disable done"));
4407 
4408 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop_channel"));
4409 
4410 	return (NXGE_OK);
4411 }
4412 
4413 nxge_status_t
4414 nxge_rxdma_handle_sys_errors(p_nxge_t nxgep)
4415 {
4416 	npi_handle_t		handle;
4417 	p_nxge_rdc_sys_stats_t	statsp;
4418 	rx_ctl_dat_fifo_stat_t	stat;
4419 	uint32_t		zcp_err_status;
4420 	uint32_t		ipp_err_status;
4421 	nxge_status_t		status = NXGE_OK;
4422 	npi_status_t		rs = NPI_SUCCESS;
4423 	boolean_t		my_err = B_FALSE;
4424 
4425 	handle = nxgep->npi_handle;
4426 	statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats;
4427 
4428 	rs = npi_rxdma_rxctl_fifo_error_intr_get(handle, &stat);
4429 
4430 	if (rs != NPI_SUCCESS)
4431 		return (NXGE_ERROR | rs);
4432 
4433 	if (stat.bits.ldw.id_mismatch) {
4434 		statsp->id_mismatch++;
4435 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, NULL,
4436 		    NXGE_FM_EREPORT_RDMC_ID_MISMATCH);
4437 		/* Global fatal error encountered */
4438 	}
4439 
4440 	if ((stat.bits.ldw.zcp_eop_err) || (stat.bits.ldw.ipp_eop_err)) {
4441 		switch (nxgep->mac.portnum) {
4442 		case 0:
4443 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT0) ||
4444 			    (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT0)) {
4445 				my_err = B_TRUE;
4446 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
4447 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
4448 			}
4449 			break;
4450 		case 1:
4451 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT1) ||
4452 			    (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT1)) {
4453 				my_err = B_TRUE;
4454 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
4455 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
4456 			}
4457 			break;
4458 		case 2:
4459 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT2) ||
4460 			    (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT2)) {
4461 				my_err = B_TRUE;
4462 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
4463 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
4464 			}
4465 			break;
4466 		case 3:
4467 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT3) ||
4468 			    (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT3)) {
4469 				my_err = B_TRUE;
4470 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
4471 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
4472 			}
4473 			break;
4474 		default:
4475 			return (NXGE_ERROR);
4476 		}
4477 	}
4478 
4479 	if (my_err) {
4480 		status = nxge_rxdma_handle_port_errors(nxgep, ipp_err_status,
4481 		    zcp_err_status);
4482 		if (status != NXGE_OK)
4483 			return (status);
4484 	}
4485 
4486 	return (NXGE_OK);
4487 }
4488 
4489 static nxge_status_t
4490 nxge_rxdma_handle_port_errors(p_nxge_t nxgep, uint32_t ipp_status,
4491 							uint32_t zcp_status)
4492 {
4493 	boolean_t		rxport_fatal = B_FALSE;
4494 	p_nxge_rdc_sys_stats_t	statsp;
4495 	nxge_status_t		status = NXGE_OK;
4496 	uint8_t			portn;
4497 
4498 	portn = nxgep->mac.portnum;
4499 	statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats;
4500 
4501 	if (ipp_status & (0x1 << portn)) {
4502 		statsp->ipp_eop_err++;
4503 		NXGE_FM_REPORT_ERROR(nxgep, portn, NULL,
4504 		    NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR);
4505 		rxport_fatal = B_TRUE;
4506 	}
4507 
4508 	if (zcp_status & (0x1 << portn)) {
4509 		statsp->zcp_eop_err++;
4510 		NXGE_FM_REPORT_ERROR(nxgep, portn, NULL,
4511 		    NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR);
4512 		rxport_fatal = B_TRUE;
4513 	}
4514 
4515 	if (rxport_fatal) {
4516 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4517 		    " nxge_rxdma_handle_port_error: "
4518 		    " fatal error on Port #%d\n",
4519 		    portn));
4520 		status = nxge_rx_port_fatal_err_recover(nxgep);
4521 		if (status == NXGE_OK) {
4522 			FM_SERVICE_RESTORED(nxgep);
4523 		}
4524 	}
4525 
4526 	return (status);
4527 }
4528 
4529 static nxge_status_t
4530 nxge_rxdma_fatal_err_recover(p_nxge_t nxgep, uint16_t channel)
4531 {
4532 	npi_handle_t		handle;
4533 	npi_status_t		rs = NPI_SUCCESS;
4534 	nxge_status_t		status = NXGE_OK;
4535 	p_rx_rbr_ring_t		rbrp;
4536 	p_rx_rcr_ring_t		rcrp;
4537 	p_rx_mbox_t		mboxp;
4538 	rx_dma_ent_msk_t	ent_mask;
4539 	p_nxge_dma_common_t	dmap;
4540 	uint32_t		ref_cnt;
4541 	p_rx_msg_t		rx_msg_p;
4542 	int			i;
4543 	uint32_t		nxge_port_rcr_size;
4544 
4545 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fatal_err_recover"));
4546 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4547 	    "Recovering from RxDMAChannel#%d error...", channel));
4548 
4549 	/*
4550 	 * Stop the dma channel waits for the stop done.
4551 	 * If the stop done bit is not set, then create
4552 	 * an error.
4553 	 */
4554 
4555 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
4556 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Rx DMA stop..."));
4557 
4558 	rbrp = (p_rx_rbr_ring_t)nxgep->rx_rbr_rings->rbr_rings[channel];
4559 	rcrp = (p_rx_rcr_ring_t)nxgep->rx_rcr_rings->rcr_rings[channel];
4560 
4561 	MUTEX_ENTER(&rbrp->lock);
4562 	MUTEX_ENTER(&rbrp->post_lock);
4563 
4564 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA channel..."));
4565 
4566 	rs = npi_rxdma_cfg_rdc_disable(handle, channel);
4567 	if (rs != NPI_SUCCESS) {
4568 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4569 		    "nxge_disable_rxdma_channel:failed"));
4570 		goto fail;
4571 	}
4572 
4573 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA interrupt..."));
4574 
4575 	/* Disable interrupt */
4576 	ent_mask.value = RX_DMA_ENT_MSK_ALL;
4577 	rs = npi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask);
4578 	if (rs != NPI_SUCCESS) {
4579 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4580 		    "nxge_rxdma_stop_channel: "
4581 		    "set rxdma event masks failed (channel %d)",
4582 		    channel));
4583 	}
4584 
4585 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel reset..."));
4586 
4587 	/* Reset RXDMA channel */
4588 	rs = npi_rxdma_cfg_rdc_reset(handle, channel);
4589 	if (rs != NPI_SUCCESS) {
4590 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4591 		    "nxge_rxdma_fatal_err_recover: "
4592 		    " reset rxdma failed (channel %d)", channel));
4593 		goto fail;
4594 	}
4595 
4596 	nxge_port_rcr_size = nxgep->nxge_port_rcr_size;
4597 
4598 	mboxp = (p_rx_mbox_t)nxgep->rx_mbox_areas_p->rxmbox_areas[channel];
4599 
4600 	rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
4601 	rbrp->rbr_rd_index = 0;
4602 
4603 	rcrp->comp_rd_index = 0;
4604 	rcrp->comp_wt_index = 0;
4605 	rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
4606 	    (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
4607 #if defined(__i386)
4608 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
4609 	    (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
4610 #else
4611 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
4612 	    (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
4613 #endif
4614 
4615 	rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
4616 	    (nxge_port_rcr_size - 1);
4617 	rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
4618 	    (nxge_port_rcr_size - 1);
4619 
4620 	dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc;
4621 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
4622 
4623 	cmn_err(CE_NOTE, "!rbr entries = %d\n", rbrp->rbr_max_size);
4624 
4625 	for (i = 0; i < rbrp->rbr_max_size; i++) {
4626 		rx_msg_p = rbrp->rx_msg_ring[i];
4627 		ref_cnt = rx_msg_p->ref_cnt;
4628 		if (ref_cnt != 1) {
4629 			if (rx_msg_p->cur_usage_cnt !=
4630 			    rx_msg_p->max_usage_cnt) {
4631 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4632 				    "buf[%d]: cur_usage_cnt = %d "
4633 				    "max_usage_cnt = %d\n", i,
4634 				    rx_msg_p->cur_usage_cnt,
4635 				    rx_msg_p->max_usage_cnt));
4636 			} else {
4637 				/* Buffer can be re-posted */
4638 				rx_msg_p->free = B_TRUE;
4639 				rx_msg_p->cur_usage_cnt = 0;
4640 				rx_msg_p->max_usage_cnt = 0xbaddcafe;
4641 				rx_msg_p->pkt_buf_size = 0;
4642 			}
4643 		}
4644 	}
4645 
4646 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel re-start..."));
4647 
4648 	status = nxge_rxdma_start_channel(nxgep, channel, rbrp, rcrp, mboxp);
4649 	if (status != NXGE_OK) {
4650 		goto fail;
4651 	}
4652 
4653 	MUTEX_EXIT(&rbrp->post_lock);
4654 	MUTEX_EXIT(&rbrp->lock);
4655 
4656 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4657 	    "Recovery Successful, RxDMAChannel#%d Restored",
4658 	    channel));
4659 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fatal_err_recover"));
4660 	return (NXGE_OK);
4661 
4662 fail:
4663 	MUTEX_EXIT(&rbrp->post_lock);
4664 	MUTEX_EXIT(&rbrp->lock);
4665 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed"));
4666 	return (NXGE_ERROR | rs);
4667 }
4668 
4669 nxge_status_t
4670 nxge_rx_port_fatal_err_recover(p_nxge_t nxgep)
4671 {
4672 	nxge_grp_set_t *set = &nxgep->rx_set;
4673 	nxge_status_t status = NXGE_OK;
4674 	p_rx_rcr_ring_t rcrp;
4675 	int rdc;
4676 
4677 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_port_fatal_err_recover"));
4678 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4679 	    "Recovering from RxPort error..."));
4680 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disabling RxMAC...\n"));
4681 
4682 	if (nxge_rx_mac_disable(nxgep) != NXGE_OK)
4683 		goto fail;
4684 
4685 	NXGE_DELAY(1000);
4686 
4687 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Stopping all RxDMA channels..."));
4688 
4689 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
4690 		if ((1 << rdc) & set->owned.map) {
4691 			rcrp = nxgep->rx_rcr_rings->rcr_rings[rdc];
4692 			if (rcrp != NULL) {
4693 				MUTEX_ENTER(&rcrp->lock);
4694 				if (nxge_rxdma_fatal_err_recover(nxgep,
4695 				    rdc) != NXGE_OK) {
4696 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4697 					    "Could not recover "
4698 					    "channel %d", rdc));
4699 				}
4700 				MUTEX_EXIT(&rcrp->lock);
4701 			}
4702 		}
4703 	}
4704 
4705 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Resetting IPP..."));
4706 
4707 	/* Reset IPP */
4708 	if (nxge_ipp_reset(nxgep) != NXGE_OK) {
4709 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4710 		    "nxge_rx_port_fatal_err_recover: "
4711 		    "Failed to reset IPP"));
4712 		goto fail;
4713 	}
4714 
4715 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Reset RxMAC..."));
4716 
4717 	/* Reset RxMAC */
4718 	if (nxge_rx_mac_reset(nxgep) != NXGE_OK) {
4719 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4720 		    "nxge_rx_port_fatal_err_recover: "
4721 		    "Failed to reset RxMAC"));
4722 		goto fail;
4723 	}
4724 
4725 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize IPP..."));
4726 
4727 	/* Re-Initialize IPP */
4728 	if (nxge_ipp_init(nxgep) != NXGE_OK) {
4729 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4730 		    "nxge_rx_port_fatal_err_recover: "
4731 		    "Failed to init IPP"));
4732 		goto fail;
4733 	}
4734 
4735 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize RxMAC..."));
4736 
4737 	/* Re-Initialize RxMAC */
4738 	if ((status = nxge_rx_mac_init(nxgep)) != NXGE_OK) {
4739 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4740 		    "nxge_rx_port_fatal_err_recover: "
4741 		    "Failed to reset RxMAC"));
4742 		goto fail;
4743 	}
4744 
4745 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-enable RxMAC..."));
4746 
4747 	/* Re-enable RxMAC */
4748 	if ((status = nxge_rx_mac_enable(nxgep)) != NXGE_OK) {
4749 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4750 		    "nxge_rx_port_fatal_err_recover: "
4751 		    "Failed to enable RxMAC"));
4752 		goto fail;
4753 	}
4754 
4755 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4756 	    "Recovery Successful, RxPort Restored"));
4757 
4758 	return (NXGE_OK);
4759 fail:
4760 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed"));
4761 	return (status);
4762 }
4763 
4764 void
4765 nxge_rxdma_inject_err(p_nxge_t nxgep, uint32_t err_id, uint8_t chan)
4766 {
4767 	rx_dma_ctl_stat_t	cs;
4768 	rx_ctl_dat_fifo_stat_t	cdfs;
4769 
4770 	switch (err_id) {
4771 	case NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR:
4772 	case NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR:
4773 	case NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR:
4774 	case NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR:
4775 	case NXGE_FM_EREPORT_RDMC_RBR_TMOUT:
4776 	case NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR:
4777 	case NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS:
4778 	case NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR:
4779 	case NXGE_FM_EREPORT_RDMC_RCRINCON:
4780 	case NXGE_FM_EREPORT_RDMC_RCRFULL:
4781 	case NXGE_FM_EREPORT_RDMC_RBRFULL:
4782 	case NXGE_FM_EREPORT_RDMC_RBRLOGPAGE:
4783 	case NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE:
4784 	case NXGE_FM_EREPORT_RDMC_CONFIG_ERR:
4785 		RXDMA_REG_READ64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG,
4786 		    chan, &cs.value);
4787 		if (err_id == NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR)
4788 			cs.bits.hdw.rcr_ack_err = 1;
4789 		else if (err_id == NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR)
4790 			cs.bits.hdw.dc_fifo_err = 1;
4791 		else if (err_id == NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR)
4792 			cs.bits.hdw.rcr_sha_par = 1;
4793 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR)
4794 			cs.bits.hdw.rbr_pre_par = 1;
4795 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_TMOUT)
4796 			cs.bits.hdw.rbr_tmout = 1;
4797 		else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR)
4798 			cs.bits.hdw.rsp_cnt_err = 1;
4799 		else if (err_id == NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS)
4800 			cs.bits.hdw.byte_en_bus = 1;
4801 		else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR)
4802 			cs.bits.hdw.rsp_dat_err = 1;
4803 		else if (err_id == NXGE_FM_EREPORT_RDMC_CONFIG_ERR)
4804 			cs.bits.hdw.config_err = 1;
4805 		else if (err_id == NXGE_FM_EREPORT_RDMC_RCRINCON)
4806 			cs.bits.hdw.rcrincon = 1;
4807 		else if (err_id == NXGE_FM_EREPORT_RDMC_RCRFULL)
4808 			cs.bits.hdw.rcrfull = 1;
4809 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBRFULL)
4810 			cs.bits.hdw.rbrfull = 1;
4811 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBRLOGPAGE)
4812 			cs.bits.hdw.rbrlogpage = 1;
4813 		else if (err_id == NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE)
4814 			cs.bits.hdw.cfiglogpage = 1;
4815 #if defined(__i386)
4816 		cmn_err(CE_NOTE, "!Write 0x%llx to RX_DMA_CTL_STAT_DBG_REG\n",
4817 		    cs.value);
4818 #else
4819 		cmn_err(CE_NOTE, "!Write 0x%lx to RX_DMA_CTL_STAT_DBG_REG\n",
4820 		    cs.value);
4821 #endif
4822 		RXDMA_REG_WRITE64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG,
4823 		    chan, cs.value);
4824 		break;
4825 	case NXGE_FM_EREPORT_RDMC_ID_MISMATCH:
4826 	case NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR:
4827 	case NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR:
4828 		cdfs.value = 0;
4829 		if (err_id ==  NXGE_FM_EREPORT_RDMC_ID_MISMATCH)
4830 			cdfs.bits.ldw.id_mismatch = (1 << nxgep->mac.portnum);
4831 		else if (err_id == NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR)
4832 			cdfs.bits.ldw.zcp_eop_err = (1 << nxgep->mac.portnum);
4833 		else if (err_id == NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR)
4834 			cdfs.bits.ldw.ipp_eop_err = (1 << nxgep->mac.portnum);
4835 #if defined(__i386)
4836 		cmn_err(CE_NOTE,
4837 		    "!Write 0x%llx to RX_CTL_DAT_FIFO_STAT_DBG_REG\n",
4838 		    cdfs.value);
4839 #else
4840 		cmn_err(CE_NOTE,
4841 		    "!Write 0x%lx to RX_CTL_DAT_FIFO_STAT_DBG_REG\n",
4842 		    cdfs.value);
4843 #endif
4844 		NXGE_REG_WR64(nxgep->npi_handle,
4845 		    RX_CTL_DAT_FIFO_STAT_DBG_REG, cdfs.value);
4846 		break;
4847 	case NXGE_FM_EREPORT_RDMC_DCF_ERR:
4848 		break;
4849 	case NXGE_FM_EREPORT_RDMC_RCR_ERR:
4850 		break;
4851 	}
4852 }
4853 
4854 static void
4855 nxge_rxdma_databuf_free(p_rx_rbr_ring_t rbr_p)
4856 {
4857 	rxring_info_t 		*ring_info;
4858 	int			index;
4859 	uint32_t		chunk_size;
4860 	uint64_t		kaddr;
4861 	uint_t			num_blocks;
4862 
4863 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_rxdma_databuf_free"));
4864 
4865 	if (rbr_p == NULL) {
4866 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
4867 		    "==> nxge_rxdma_databuf_free: NULL rbr pointer"));
4868 		return;
4869 	}
4870 
4871 	if (rbr_p->rbr_alloc_type == DDI_MEM_ALLOC) {
4872 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
4873 		    "<== nxge_rxdma_databuf_free: DDI"));
4874 		return;
4875 	}
4876 
4877 	ring_info = rbr_p->ring_info;
4878 	if (ring_info == NULL) {
4879 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
4880 		    "==> nxge_rxdma_databuf_free: NULL ring info"));
4881 		return;
4882 	}
4883 	num_blocks = rbr_p->num_blocks;
4884 	for (index = 0; index < num_blocks; index++) {
4885 		kaddr = ring_info->buffer[index].kaddr;
4886 		chunk_size = ring_info->buffer[index].buf_size;
4887 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
4888 		    "==> nxge_rxdma_databuf_free: free chunk %d "
4889 		    "kaddrp $%p chunk size %d",
4890 		    index, kaddr, chunk_size));
4891 		if (kaddr == NULL) continue;
4892 		nxge_free_buf(rbr_p->rbr_alloc_type, kaddr, chunk_size);
4893 		ring_info->buffer[index].kaddr = NULL;
4894 	}
4895 
4896 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_rxdma_databuf_free"));
4897 }
4898 
4899 #if	defined(sun4v) && defined(NIU_LP_WORKAROUND)
4900 extern void contig_mem_free(void *, size_t);
4901 #endif
4902 
4903 void
4904 nxge_free_buf(buf_alloc_type_t alloc_type, uint64_t kaddr, uint32_t buf_size)
4905 {
4906 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_free_buf"));
4907 
4908 	if (kaddr == NULL || !buf_size) {
4909 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
4910 		    "==> nxge_free_buf: invalid kaddr $%p size to free %d",
4911 		    kaddr, buf_size));
4912 		return;
4913 	}
4914 
4915 	switch (alloc_type) {
4916 	case KMEM_ALLOC:
4917 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
4918 		    "==> nxge_free_buf: freeing kmem $%p size %d",
4919 		    kaddr, buf_size));
4920 #if defined(__i386)
4921 		KMEM_FREE((void *)(uint32_t)kaddr, buf_size);
4922 #else
4923 		KMEM_FREE((void *)kaddr, buf_size);
4924 #endif
4925 		break;
4926 
4927 #if	defined(sun4v) && defined(NIU_LP_WORKAROUND)
4928 	case CONTIG_MEM_ALLOC:
4929 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
4930 		    "==> nxge_free_buf: freeing contig_mem kaddr $%p size %d",
4931 		    kaddr, buf_size));
4932 		contig_mem_free((void *)kaddr, buf_size);
4933 		break;
4934 #endif
4935 
4936 	default:
4937 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
4938 		    "<== nxge_free_buf: unsupported alloc type %d",
4939 		    alloc_type));
4940 		return;
4941 	}
4942 
4943 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_free_buf"));
4944 }
4945