1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <hxge_impl.h>
27 #include <hxge_txdma.h>
28 #include <sys/llc1.h>
29
30 uint32_t hxge_reclaim_pending = TXDMA_RECLAIM_PENDING_DEFAULT;
31 uint32_t hxge_tx_minfree = 64;
32 uint32_t hxge_tx_intr_thres = 0;
33 uint32_t hxge_tx_max_gathers = TX_MAX_GATHER_POINTERS;
34 uint32_t hxge_tx_tiny_pack = 1;
35 uint32_t hxge_tx_use_bcopy = 1;
36
37 extern uint32_t hxge_tx_ring_size;
38 extern uint32_t hxge_bcopy_thresh;
39 extern uint32_t hxge_dvma_thresh;
40 extern uint32_t hxge_dma_stream_thresh;
41 extern dma_method_t hxge_force_dma;
42
43 /* Device register access attributes for PIO. */
44 extern ddi_device_acc_attr_t hxge_dev_reg_acc_attr;
45
46 /* Device descriptor access attributes for DMA. */
47 extern ddi_device_acc_attr_t hxge_dev_desc_dma_acc_attr;
48
49 /* Device buffer access attributes for DMA. */
50 extern ddi_device_acc_attr_t hxge_dev_buf_dma_acc_attr;
51 extern ddi_dma_attr_t hxge_desc_dma_attr;
52 extern ddi_dma_attr_t hxge_tx_dma_attr;
53
54 static hxge_status_t hxge_map_txdma(p_hxge_t hxgep);
55 static void hxge_unmap_txdma(p_hxge_t hxgep);
56 static hxge_status_t hxge_txdma_hw_start(p_hxge_t hxgep);
57 static void hxge_txdma_hw_stop(p_hxge_t hxgep);
58
59 static hxge_status_t hxge_map_txdma_channel(p_hxge_t hxgep, uint16_t channel,
60 p_hxge_dma_common_t *dma_buf_p, p_tx_ring_t *tx_desc_p,
61 uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p,
62 p_tx_mbox_t *tx_mbox_p);
63 static void hxge_unmap_txdma_channel(p_hxge_t hxgep, uint16_t channel,
64 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
65 static hxge_status_t hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep, uint16_t,
66 p_hxge_dma_common_t *, p_tx_ring_t *, uint32_t);
67 static void hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep,
68 p_tx_ring_t tx_ring_p);
69 static void hxge_map_txdma_channel_cfg_ring(p_hxge_t, uint16_t,
70 p_hxge_dma_common_t *, p_tx_ring_t, p_tx_mbox_t *);
71 static void hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,
72 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
73 static hxge_status_t hxge_txdma_start_channel(p_hxge_t hxgep, uint16_t channel,
74 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
75 static hxge_status_t hxge_txdma_stop_channel(p_hxge_t hxgep, uint16_t channel,
76 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
77 static p_tx_ring_t hxge_txdma_get_ring(p_hxge_t hxgep, uint16_t channel);
78 static hxge_status_t hxge_tx_err_evnts(p_hxge_t hxgep, uint_t index,
79 p_hxge_ldv_t ldvp, tdc_stat_t cs);
80 static p_tx_mbox_t hxge_txdma_get_mbox(p_hxge_t hxgep, uint16_t channel);
81 static hxge_status_t hxge_txdma_fatal_err_recover(p_hxge_t hxgep,
82 uint16_t channel, p_tx_ring_t tx_ring_p);
83 static hxge_status_t hxge_tx_port_fatal_err_recover(p_hxge_t hxgep);
84
85 hxge_status_t
hxge_init_txdma_channels(p_hxge_t hxgep)86 hxge_init_txdma_channels(p_hxge_t hxgep)
87 {
88 hxge_status_t status = HXGE_OK;
89 block_reset_t reset_reg;
90
91 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_init_txdma_channels"));
92
93 /*
94 * Reset TDC block from PEU to cleanup any unknown configuration.
95 * This may be resulted from previous reboot.
96 */
97 reset_reg.value = 0;
98 reset_reg.bits.tdc_rst = 1;
99 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value);
100
101 HXGE_DELAY(1000);
102
103 status = hxge_map_txdma(hxgep);
104 if (status != HXGE_OK) {
105 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
106 "<== hxge_init_txdma_channels: status 0x%x", status));
107 return (status);
108 }
109
110 status = hxge_txdma_hw_start(hxgep);
111 if (status != HXGE_OK) {
112 hxge_unmap_txdma(hxgep);
113 return (status);
114 }
115
116 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
117 "<== hxge_init_txdma_channels: status 0x%x", status));
118
119 return (HXGE_OK);
120 }
121
122 void
hxge_uninit_txdma_channels(p_hxge_t hxgep)123 hxge_uninit_txdma_channels(p_hxge_t hxgep)
124 {
125 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_uninit_txdma_channels"));
126
127 hxge_txdma_hw_stop(hxgep);
128 hxge_unmap_txdma(hxgep);
129
130 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_uinit_txdma_channels"));
131 }
132
133 void
hxge_setup_dma_common(p_hxge_dma_common_t dest_p,p_hxge_dma_common_t src_p,uint32_t entries,uint32_t size)134 hxge_setup_dma_common(p_hxge_dma_common_t dest_p, p_hxge_dma_common_t src_p,
135 uint32_t entries, uint32_t size)
136 {
137 size_t tsize;
138 *dest_p = *src_p;
139 tsize = size * entries;
140 dest_p->alength = tsize;
141 dest_p->nblocks = entries;
142 dest_p->block_size = size;
143 dest_p->offset += tsize;
144
145 src_p->kaddrp = (caddr_t)dest_p->kaddrp + tsize;
146 src_p->alength -= tsize;
147 src_p->dma_cookie.dmac_laddress += tsize;
148 src_p->dma_cookie.dmac_size -= tsize;
149 }
150
151 hxge_status_t
hxge_reset_txdma_channel(p_hxge_t hxgep,uint16_t channel,uint64_t reg_data)152 hxge_reset_txdma_channel(p_hxge_t hxgep, uint16_t channel, uint64_t reg_data)
153 {
154 hpi_status_t rs = HPI_SUCCESS;
155 hxge_status_t status = HXGE_OK;
156 hpi_handle_t handle;
157
158 HXGE_DEBUG_MSG((hxgep, TX_CTL, " ==> hxge_reset_txdma_channel"));
159
160 handle = HXGE_DEV_HPI_HANDLE(hxgep);
161 if ((reg_data & TDC_TDR_RST_MASK) == TDC_TDR_RST_MASK) {
162 rs = hpi_txdma_channel_reset(handle, channel);
163 } else {
164 rs = hpi_txdma_channel_control(handle, TXDMA_RESET, channel);
165 }
166
167 if (rs != HPI_SUCCESS) {
168 status = HXGE_ERROR | rs;
169 }
170
171 /*
172 * Reset the tail (kick) register to 0. (Hardware will not reset it. Tx
173 * overflow fatal error if tail is not set to 0 after reset!
174 */
175 TXDMA_REG_WRITE64(handle, TDC_TDR_KICK, channel, 0);
176
177 HXGE_DEBUG_MSG((hxgep, TX_CTL, " <== hxge_reset_txdma_channel"));
178
179 return (status);
180 }
181
182 hxge_status_t
hxge_init_txdma_channel_event_mask(p_hxge_t hxgep,uint16_t channel,tdc_int_mask_t * mask_p)183 hxge_init_txdma_channel_event_mask(p_hxge_t hxgep, uint16_t channel,
184 tdc_int_mask_t *mask_p)
185 {
186 hpi_handle_t handle;
187 hpi_status_t rs = HPI_SUCCESS;
188 hxge_status_t status = HXGE_OK;
189
190 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
191 "<== hxge_init_txdma_channel_event_mask"));
192
193 handle = HXGE_DEV_HPI_HANDLE(hxgep);
194
195 /*
196 * Mask off tx_rng_oflow since it is a false alarm. The driver
197 * ensures not over flowing the hardware and check the hardware
198 * status.
199 */
200 mask_p->bits.tx_rng_oflow = 1;
201 rs = hpi_txdma_event_mask(handle, OP_SET, channel, mask_p);
202 if (rs != HPI_SUCCESS) {
203 status = HXGE_ERROR | rs;
204 }
205
206 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
207 "==> hxge_init_txdma_channel_event_mask"));
208 return (status);
209 }
210
211 hxge_status_t
hxge_enable_txdma_channel(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_desc_p,p_tx_mbox_t mbox_p)212 hxge_enable_txdma_channel(p_hxge_t hxgep,
213 uint16_t channel, p_tx_ring_t tx_desc_p, p_tx_mbox_t mbox_p)
214 {
215 hpi_handle_t handle;
216 hpi_status_t rs = HPI_SUCCESS;
217 hxge_status_t status = HXGE_OK;
218
219 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_enable_txdma_channel"));
220
221 handle = HXGE_DEV_HPI_HANDLE(hxgep);
222 /*
223 * Use configuration data composed at init time. Write to hardware the
224 * transmit ring configurations.
225 */
226 rs = hpi_txdma_ring_config(handle, OP_SET, channel,
227 (uint64_t *)&(tx_desc_p->tx_ring_cfig.value));
228
229 if (rs != HPI_SUCCESS) {
230 return (HXGE_ERROR | rs);
231 }
232
233 /* Write to hardware the mailbox */
234 rs = hpi_txdma_mbox_config(handle, OP_SET, channel,
235 (uint64_t *)&mbox_p->tx_mbox.dma_cookie.dmac_laddress);
236
237 if (rs != HPI_SUCCESS) {
238 return (HXGE_ERROR | rs);
239 }
240
241 /* Start the DMA engine. */
242 rs = hpi_txdma_channel_init_enable(handle, channel);
243 if (rs != HPI_SUCCESS) {
244 return (HXGE_ERROR | rs);
245 }
246 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_enable_txdma_channel"));
247 return (status);
248 }
249
250 void
hxge_fill_tx_hdr(p_mblk_t mp,boolean_t fill_len,boolean_t l4_cksum,int pkt_len,uint8_t npads,p_tx_pkt_hdr_all_t pkthdrp)251 hxge_fill_tx_hdr(p_mblk_t mp, boolean_t fill_len, boolean_t l4_cksum,
252 int pkt_len, uint8_t npads, p_tx_pkt_hdr_all_t pkthdrp)
253 {
254 p_tx_pkt_header_t hdrp;
255 p_mblk_t nmp;
256 uint64_t tmp;
257 size_t mblk_len;
258 size_t iph_len;
259 size_t hdrs_size;
260 uint8_t *ip_buf;
261 uint16_t eth_type;
262 uint8_t ipproto;
263 boolean_t is_vlan = B_FALSE;
264 size_t eth_hdr_size;
265 uint8_t hdrs_buf[sizeof (struct ether_header) + 64 + sizeof (uint32_t)];
266
267 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: mp $%p", mp));
268
269 /*
270 * Caller should zero out the headers first.
271 */
272 hdrp = (p_tx_pkt_header_t)&pkthdrp->pkthdr;
273
274 if (fill_len) {
275 HXGE_DEBUG_MSG((NULL, TX_CTL,
276 "==> hxge_fill_tx_hdr: pkt_len %d npads %d",
277 pkt_len, npads));
278 tmp = (uint64_t)pkt_len;
279 hdrp->value |= (tmp << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
280
281 goto fill_tx_header_done;
282 }
283 tmp = (uint64_t)npads;
284 hdrp->value |= (tmp << TX_PKT_HEADER_PAD_SHIFT);
285
286 /*
287 * mp is the original data packet (does not include the Neptune
288 * transmit header).
289 */
290 nmp = mp;
291 mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
292 HXGE_DEBUG_MSG((NULL, TX_CTL,
293 "==> hxge_fill_tx_hdr: mp $%p b_rptr $%p len %d",
294 mp, nmp->b_rptr, mblk_len));
295 ip_buf = NULL;
296 bcopy(nmp->b_rptr, &hdrs_buf[0], sizeof (struct ether_vlan_header));
297 eth_type = ntohs(((p_ether_header_t)hdrs_buf)->ether_type);
298 HXGE_DEBUG_MSG((NULL, TX_CTL,
299 "==> : hxge_fill_tx_hdr: (value 0x%llx) ether type 0x%x",
300 eth_type, hdrp->value));
301
302 if (eth_type < ETHERMTU) {
303 tmp = 1ull;
304 hdrp->value |= (tmp << TX_PKT_HEADER_LLC_SHIFT);
305 HXGE_DEBUG_MSG((NULL, TX_CTL,
306 "==> hxge_tx_pkt_hdr_init: LLC value 0x%llx", hdrp->value));
307 if (*(hdrs_buf + sizeof (struct ether_header)) ==
308 LLC_SNAP_SAP) {
309 eth_type = ntohs(*((uint16_t *)(hdrs_buf +
310 sizeof (struct ether_header) + 6)));
311 HXGE_DEBUG_MSG((NULL, TX_CTL,
312 "==> hxge_tx_pkt_hdr_init: LLC ether type 0x%x",
313 eth_type));
314 } else {
315 goto fill_tx_header_done;
316 }
317 } else if (eth_type == VLAN_ETHERTYPE) {
318 tmp = 1ull;
319 hdrp->value |= (tmp << TX_PKT_HEADER_VLAN__SHIFT);
320
321 eth_type = ntohs(((struct ether_vlan_header *)
322 hdrs_buf)->ether_type);
323 is_vlan = B_TRUE;
324 HXGE_DEBUG_MSG((NULL, TX_CTL,
325 "==> hxge_tx_pkt_hdr_init: VLAN value 0x%llx",
326 hdrp->value));
327 }
328 if (!is_vlan) {
329 eth_hdr_size = sizeof (struct ether_header);
330 } else {
331 eth_hdr_size = sizeof (struct ether_vlan_header);
332 }
333
334 switch (eth_type) {
335 case ETHERTYPE_IP:
336 if (mblk_len > eth_hdr_size + sizeof (uint8_t)) {
337 ip_buf = nmp->b_rptr + eth_hdr_size;
338 mblk_len -= eth_hdr_size;
339 iph_len = ((*ip_buf) & 0x0f);
340 if (mblk_len > (iph_len + sizeof (uint32_t))) {
341 ip_buf = nmp->b_rptr;
342 ip_buf += eth_hdr_size;
343 } else {
344 ip_buf = NULL;
345 }
346 }
347 if (ip_buf == NULL) {
348 hdrs_size = 0;
349 ((p_ether_header_t)hdrs_buf)->ether_type = 0;
350 while ((nmp) && (hdrs_size < sizeof (hdrs_buf))) {
351 mblk_len = (size_t)nmp->b_wptr -
352 (size_t)nmp->b_rptr;
353 if (mblk_len >=
354 (sizeof (hdrs_buf) - hdrs_size))
355 mblk_len = sizeof (hdrs_buf) -
356 hdrs_size;
357 bcopy(nmp->b_rptr,
358 &hdrs_buf[hdrs_size], mblk_len);
359 hdrs_size += mblk_len;
360 nmp = nmp->b_cont;
361 }
362 ip_buf = hdrs_buf;
363 ip_buf += eth_hdr_size;
364 iph_len = ((*ip_buf) & 0x0f);
365 }
366 ipproto = ip_buf[9];
367
368 tmp = (uint64_t)iph_len;
369 hdrp->value |= (tmp << TX_PKT_HEADER_IHL_SHIFT);
370 tmp = (uint64_t)(eth_hdr_size >> 1);
371 hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
372
373 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: IPv4 "
374 " iph_len %d l3start %d eth_hdr_size %d proto 0x%x"
375 "tmp 0x%x", iph_len, hdrp->bits.l3start, eth_hdr_size,
376 ipproto, tmp));
377 HXGE_DEBUG_MSG((NULL, TX_CTL,
378 "==> hxge_tx_pkt_hdr_init: IP value 0x%llx", hdrp->value));
379 break;
380
381 case ETHERTYPE_IPV6:
382 hdrs_size = 0;
383 ((p_ether_header_t)hdrs_buf)->ether_type = 0;
384 while ((nmp) && (hdrs_size < sizeof (hdrs_buf))) {
385 mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
386 if (mblk_len >= (sizeof (hdrs_buf) - hdrs_size))
387 mblk_len = sizeof (hdrs_buf) - hdrs_size;
388 bcopy(nmp->b_rptr, &hdrs_buf[hdrs_size], mblk_len);
389 hdrs_size += mblk_len;
390 nmp = nmp->b_cont;
391 }
392 ip_buf = hdrs_buf;
393 ip_buf += eth_hdr_size;
394
395 tmp = 1ull;
396 hdrp->value |= (tmp << TX_PKT_HEADER_IP_VER_SHIFT);
397
398 tmp = (eth_hdr_size >> 1);
399 hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
400
401 /* byte 6 is the next header protocol */
402 ipproto = ip_buf[6];
403
404 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: IPv6 "
405 " iph_len %d l3start %d eth_hdr_size %d proto 0x%x",
406 iph_len, hdrp->bits.l3start, eth_hdr_size, ipproto));
407 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_hdr_init: IPv6 "
408 "value 0x%llx", hdrp->value));
409 break;
410
411 default:
412 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: non-IP"));
413 goto fill_tx_header_done;
414 }
415
416 switch (ipproto) {
417 case IPPROTO_TCP:
418 HXGE_DEBUG_MSG((NULL, TX_CTL,
419 "==> hxge_fill_tx_hdr: TCP (cksum flag %d)", l4_cksum));
420 if (l4_cksum) {
421 tmp = 1ull;
422 hdrp->value |= (tmp << TX_PKT_HEADER_PKT_TYPE_SHIFT);
423 HXGE_DEBUG_MSG((NULL, TX_CTL,
424 "==> hxge_tx_pkt_hdr_init: TCP CKSUM"
425 "value 0x%llx", hdrp->value));
426 }
427 HXGE_DEBUG_MSG((NULL, TX_CTL,
428 "==> hxge_tx_pkt_hdr_init: TCP value 0x%llx", hdrp->value));
429 break;
430
431 case IPPROTO_UDP:
432 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: UDP"));
433 if (l4_cksum) {
434 tmp = 0x2ull;
435 hdrp->value |= (tmp << TX_PKT_HEADER_PKT_TYPE_SHIFT);
436 }
437 HXGE_DEBUG_MSG((NULL, TX_CTL,
438 "==> hxge_tx_pkt_hdr_init: UDP value 0x%llx",
439 hdrp->value));
440 break;
441
442 default:
443 goto fill_tx_header_done;
444 }
445
446 fill_tx_header_done:
447 HXGE_DEBUG_MSG((NULL, TX_CTL,
448 "==> hxge_fill_tx_hdr: pkt_len %d npads %d value 0x%llx",
449 pkt_len, npads, hdrp->value));
450 HXGE_DEBUG_MSG((NULL, TX_CTL, "<== hxge_fill_tx_hdr"));
451 }
452
453 /*ARGSUSED*/
454 p_mblk_t
hxge_tx_pkt_header_reserve(p_mblk_t mp,uint8_t * npads)455 hxge_tx_pkt_header_reserve(p_mblk_t mp, uint8_t *npads)
456 {
457 p_mblk_t newmp = NULL;
458
459 if ((newmp = allocb(TX_PKT_HEADER_SIZE, BPRI_MED)) == NULL) {
460 HXGE_DEBUG_MSG((NULL, TX_CTL,
461 "<== hxge_tx_pkt_header_reserve: allocb failed"));
462 return (NULL);
463 }
464 HXGE_DEBUG_MSG((NULL, TX_CTL,
465 "==> hxge_tx_pkt_header_reserve: get new mp"));
466 DB_TYPE(newmp) = M_DATA;
467 newmp->b_rptr = newmp->b_wptr = DB_LIM(newmp);
468 linkb(newmp, mp);
469 newmp->b_rptr -= TX_PKT_HEADER_SIZE;
470
471 HXGE_DEBUG_MSG((NULL, TX_CTL,
472 "==>hxge_tx_pkt_header_reserve: b_rptr $%p b_wptr $%p",
473 newmp->b_rptr, newmp->b_wptr));
474 HXGE_DEBUG_MSG((NULL, TX_CTL,
475 "<== hxge_tx_pkt_header_reserve: use new mp"));
476 return (newmp);
477 }
478
479 int
hxge_tx_pkt_nmblocks(p_mblk_t mp,int * tot_xfer_len_p)480 hxge_tx_pkt_nmblocks(p_mblk_t mp, int *tot_xfer_len_p)
481 {
482 uint_t nmblks;
483 ssize_t len;
484 uint_t pkt_len;
485 p_mblk_t nmp, bmp, tmp;
486 uint8_t *b_wptr;
487
488 HXGE_DEBUG_MSG((NULL, TX_CTL,
489 "==> hxge_tx_pkt_nmblocks: mp $%p rptr $%p wptr $%p len %d",
490 mp, mp->b_rptr, mp->b_wptr, MBLKL(mp)));
491
492 nmp = mp;
493 bmp = mp;
494 nmblks = 0;
495 pkt_len = 0;
496 *tot_xfer_len_p = 0;
497
498 while (nmp) {
499 len = MBLKL(nmp);
500 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_nmblocks: "
501 "len %d pkt_len %d nmblks %d tot_xfer_len %d",
502 len, pkt_len, nmblks, *tot_xfer_len_p));
503
504 if (len <= 0) {
505 bmp = nmp;
506 nmp = nmp->b_cont;
507 HXGE_DEBUG_MSG((NULL, TX_CTL,
508 "==> hxge_tx_pkt_nmblocks:"
509 " len (0) pkt_len %d nmblks %d", pkt_len, nmblks));
510 continue;
511 }
512 *tot_xfer_len_p += len;
513 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_nmblocks: "
514 "len %d pkt_len %d nmblks %d tot_xfer_len %d",
515 len, pkt_len, nmblks, *tot_xfer_len_p));
516
517 if (len < hxge_bcopy_thresh) {
518 HXGE_DEBUG_MSG((NULL, TX_CTL,
519 "==> hxge_tx_pkt_nmblocks: "
520 "len %d (< thresh) pkt_len %d nmblks %d",
521 len, pkt_len, nmblks));
522 if (pkt_len == 0)
523 nmblks++;
524 pkt_len += len;
525 if (pkt_len >= hxge_bcopy_thresh) {
526 pkt_len = 0;
527 len = 0;
528 nmp = bmp;
529 }
530 } else {
531 HXGE_DEBUG_MSG((NULL, TX_CTL,
532 "==> hxge_tx_pkt_nmblocks: "
533 "len %d (> thresh) pkt_len %d nmblks %d",
534 len, pkt_len, nmblks));
535 pkt_len = 0;
536 nmblks++;
537 /*
538 * Hardware limits the transfer length to 4K. If len is
539 * more than 4K, we need to break it up to at most 2
540 * more blocks.
541 */
542 if (len > TX_MAX_TRANSFER_LENGTH) {
543 uint32_t nsegs;
544
545 HXGE_DEBUG_MSG((NULL, TX_CTL,
546 "==> hxge_tx_pkt_nmblocks: "
547 "len %d pkt_len %d nmblks %d nsegs %d",
548 len, pkt_len, nmblks, nsegs));
549 nsegs = 1;
550 if (len % (TX_MAX_TRANSFER_LENGTH * 2)) {
551 ++nsegs;
552 }
553 do {
554 b_wptr = nmp->b_rptr +
555 TX_MAX_TRANSFER_LENGTH;
556 nmp->b_wptr = b_wptr;
557 if ((tmp = dupb(nmp)) == NULL) {
558 return (0);
559 }
560 tmp->b_rptr = b_wptr;
561 tmp->b_wptr = nmp->b_wptr;
562 tmp->b_cont = nmp->b_cont;
563 nmp->b_cont = tmp;
564 nmblks++;
565 if (--nsegs) {
566 nmp = tmp;
567 }
568 } while (nsegs);
569 nmp = tmp;
570 }
571 }
572
573 /*
574 * Hardware limits the transmit gather pointers to 15.
575 */
576 if (nmp->b_cont && (nmblks + TX_GATHER_POINTERS_THRESHOLD) >
577 TX_MAX_GATHER_POINTERS) {
578 HXGE_DEBUG_MSG((NULL, TX_CTL,
579 "==> hxge_tx_pkt_nmblocks: pull msg - "
580 "len %d pkt_len %d nmblks %d",
581 len, pkt_len, nmblks));
582 /* Pull all message blocks from b_cont */
583 if ((tmp = msgpullup(nmp->b_cont, -1)) == NULL) {
584 return (0);
585 }
586 freemsg(nmp->b_cont);
587 nmp->b_cont = tmp;
588 pkt_len = 0;
589 }
590 bmp = nmp;
591 nmp = nmp->b_cont;
592 }
593
594 HXGE_DEBUG_MSG((NULL, TX_CTL,
595 "<== hxge_tx_pkt_nmblocks: rptr $%p wptr $%p "
596 "nmblks %d len %d tot_xfer_len %d",
597 mp->b_rptr, mp->b_wptr, nmblks, MBLKL(mp), *tot_xfer_len_p));
598 return (nmblks);
599 }
600
601 boolean_t
hxge_txdma_reclaim(p_hxge_t hxgep,p_tx_ring_t tx_ring_p,int nmblks)602 hxge_txdma_reclaim(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, int nmblks)
603 {
604 boolean_t status = B_TRUE;
605 p_hxge_dma_common_t tx_desc_dma_p;
606 hxge_dma_common_t desc_area;
607 p_tx_desc_t tx_desc_ring_vp;
608 p_tx_desc_t tx_desc_p;
609 p_tx_desc_t tx_desc_pp;
610 tx_desc_t r_tx_desc;
611 p_tx_msg_t tx_msg_ring;
612 p_tx_msg_t tx_msg_p;
613 hpi_handle_t handle;
614 tdc_tdr_head_t tx_head;
615 uint32_t pkt_len;
616 uint_t tx_rd_index;
617 uint16_t head_index, tail_index;
618 uint8_t tdc;
619 boolean_t head_wrap, tail_wrap;
620 p_hxge_tx_ring_stats_t tdc_stats;
621 tdc_byte_cnt_t byte_cnt;
622 tdc_tdr_qlen_t qlen;
623 int rc;
624
625 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_reclaim"));
626
627 status = ((tx_ring_p->descs_pending < hxge_reclaim_pending) &&
628 (nmblks != 0));
629 HXGE_DEBUG_MSG((hxgep, TX_CTL,
630 "==> hxge_txdma_reclaim: pending %d reclaim %d nmblks %d",
631 tx_ring_p->descs_pending, hxge_reclaim_pending, nmblks));
632
633 if (!status) {
634 tx_desc_dma_p = &tx_ring_p->tdc_desc;
635 desc_area = tx_ring_p->tdc_desc;
636 tx_desc_ring_vp = tx_desc_dma_p->kaddrp;
637 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
638 tx_rd_index = tx_ring_p->rd_index;
639 tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
640 tx_msg_ring = tx_ring_p->tx_msg_ring;
641 tx_msg_p = &tx_msg_ring[tx_rd_index];
642 tdc = tx_ring_p->tdc;
643 tdc_stats = tx_ring_p->tdc_stats;
644 if (tx_ring_p->descs_pending > tdc_stats->tx_max_pend) {
645 tdc_stats->tx_max_pend = tx_ring_p->descs_pending;
646 }
647 tail_index = tx_ring_p->wr_index;
648 tail_wrap = tx_ring_p->wr_index_wrap;
649
650 /*
651 * tdc_byte_cnt reg can be used to get bytes transmitted. It
652 * includes padding too in case of runt packets.
653 */
654 handle = HXGE_DEV_HPI_HANDLE(hxgep);
655 TXDMA_REG_READ64(handle, TDC_BYTE_CNT, tdc, &byte_cnt.value);
656 tdc_stats->obytes_with_pad += byte_cnt.bits.byte_count;
657
658 HXGE_DEBUG_MSG((hxgep, TX_CTL,
659 "==> hxge_txdma_reclaim: tdc %d tx_rd_index %d "
660 "tail_index %d tail_wrap %d tx_desc_p $%p ($%p) ",
661 tdc, tx_rd_index, tail_index, tail_wrap,
662 tx_desc_p, (*(uint64_t *)tx_desc_p)));
663
664 /*
665 * Read the hardware maintained transmit head and wrap around
666 * bit.
667 */
668 TXDMA_REG_READ64(handle, TDC_TDR_HEAD, tdc, &tx_head.value);
669 head_index = tx_head.bits.head;
670 head_wrap = tx_head.bits.wrap;
671 HXGE_DEBUG_MSG((hxgep, TX_CTL,
672 "==> hxge_txdma_reclaim: "
673 "tx_rd_index %d tail %d tail_wrap %d head %d wrap %d",
674 tx_rd_index, tail_index, tail_wrap, head_index, head_wrap));
675
676 /*
677 * For debug only. This can be used to verify the qlen and make
678 * sure the hardware is wrapping the Tdr correctly.
679 */
680 TXDMA_REG_READ64(handle, TDC_TDR_QLEN, tdc, &qlen.value);
681 HXGE_DEBUG_MSG((hxgep, TX_CTL,
682 "==> hxge_txdma_reclaim: tdr_qlen %d tdr_pref_qlen %d",
683 qlen.bits.tdr_qlen, qlen.bits.tdr_pref_qlen));
684
685 if (head_index == tail_index) {
686 if (TXDMA_RING_EMPTY(head_index, head_wrap, tail_index,
687 tail_wrap) && (head_index == tx_rd_index)) {
688 HXGE_DEBUG_MSG((hxgep, TX_CTL,
689 "==> hxge_txdma_reclaim: EMPTY"));
690 return (B_TRUE);
691 }
692 HXGE_DEBUG_MSG((hxgep, TX_CTL,
693 "==> hxge_txdma_reclaim: Checking if ring full"));
694 if (TXDMA_RING_FULL(head_index, head_wrap, tail_index,
695 tail_wrap)) {
696 HXGE_DEBUG_MSG((hxgep, TX_CTL,
697 "==> hxge_txdma_reclaim: full"));
698 return (B_FALSE);
699 }
700 }
701 HXGE_DEBUG_MSG((hxgep, TX_CTL,
702 "==> hxge_txdma_reclaim: tx_rd_index and head_index"));
703
704 /* XXXX: limit the # of reclaims */
705 tx_desc_pp = &r_tx_desc;
706 while ((tx_rd_index != head_index) &&
707 (tx_ring_p->descs_pending != 0)) {
708 HXGE_DEBUG_MSG((hxgep, TX_CTL,
709 "==> hxge_txdma_reclaim: Checking if pending"));
710 HXGE_DEBUG_MSG((hxgep, TX_CTL,
711 "==> hxge_txdma_reclaim: descs_pending %d ",
712 tx_ring_p->descs_pending));
713 HXGE_DEBUG_MSG((hxgep, TX_CTL,
714 "==> hxge_txdma_reclaim: "
715 "(tx_rd_index %d head_index %d (tx_desc_p $%p)",
716 tx_rd_index, head_index, tx_desc_p));
717
718 tx_desc_pp->value = tx_desc_p->value;
719 HXGE_DEBUG_MSG((hxgep, TX_CTL,
720 "==> hxge_txdma_reclaim: "
721 "(tx_rd_index %d head_index %d "
722 "tx_desc_p $%p (desc value 0x%llx) ",
723 tx_rd_index, head_index,
724 tx_desc_pp, (*(uint64_t *)tx_desc_pp)));
725 HXGE_DEBUG_MSG((hxgep, TX_CTL,
726 "==> hxge_txdma_reclaim: dump desc:"));
727
728 /*
729 * tdc_byte_cnt reg can be used to get bytes
730 * transmitted
731 */
732 pkt_len = tx_desc_pp->bits.tr_len;
733 tdc_stats->obytes += pkt_len;
734 tdc_stats->opackets += tx_desc_pp->bits.sop;
735 HXGE_DEBUG_MSG((hxgep, TX_CTL,
736 "==> hxge_txdma_reclaim: pkt_len %d "
737 "tdc channel %d opackets %d",
738 pkt_len, tdc, tdc_stats->opackets));
739
740 if (tx_msg_p->flags.dma_type == USE_DVMA) {
741 HXGE_DEBUG_MSG((hxgep, TX_CTL,
742 "tx_desc_p = $%p tx_desc_pp = $%p "
743 "index = %d",
744 tx_desc_p, tx_desc_pp,
745 tx_ring_p->rd_index));
746 (void) dvma_unload(tx_msg_p->dvma_handle,
747 0, -1);
748 tx_msg_p->dvma_handle = NULL;
749 if (tx_ring_p->dvma_wr_index ==
750 tx_ring_p->dvma_wrap_mask) {
751 tx_ring_p->dvma_wr_index = 0;
752 } else {
753 tx_ring_p->dvma_wr_index++;
754 }
755 tx_ring_p->dvma_pending--;
756 } else if (tx_msg_p->flags.dma_type == USE_DMA) {
757 HXGE_DEBUG_MSG((hxgep, TX_CTL,
758 "==> hxge_txdma_reclaim: USE DMA"));
759 if (rc = ddi_dma_unbind_handle
760 (tx_msg_p->dma_handle)) {
761 cmn_err(CE_WARN, "hxge_reclaim: "
762 "ddi_dma_unbind_handle "
763 "failed. status %d", rc);
764 }
765 }
766
767 HXGE_DEBUG_MSG((hxgep, TX_CTL,
768 "==> hxge_txdma_reclaim: count packets"));
769
770 /*
771 * count a chained packet only once.
772 */
773 if (tx_msg_p->tx_message != NULL) {
774 freemsg(tx_msg_p->tx_message);
775 tx_msg_p->tx_message = NULL;
776 }
777 tx_msg_p->flags.dma_type = USE_NONE;
778 tx_rd_index = tx_ring_p->rd_index;
779 tx_rd_index = (tx_rd_index + 1) &
780 tx_ring_p->tx_wrap_mask;
781 tx_ring_p->rd_index = tx_rd_index;
782 tx_ring_p->descs_pending--;
783 tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
784 tx_msg_p = &tx_msg_ring[tx_rd_index];
785 }
786
787 status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
788 (int)tx_ring_p->descs_pending - TX_FULL_MARK));
789 if (status) {
790 (void) atomic_cas_32((uint32_t *)&tx_ring_p->queueing,
791 1, 0);
792 }
793 } else {
794 status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
795 (int)tx_ring_p->descs_pending - TX_FULL_MARK));
796 }
797
798 HXGE_DEBUG_MSG((hxgep, TX_CTL,
799 "<== hxge_txdma_reclaim status = 0x%08x", status));
800 return (status);
801 }
802
803 uint_t
hxge_tx_intr(caddr_t arg1,caddr_t arg2)804 hxge_tx_intr(caddr_t arg1, caddr_t arg2)
805 {
806 p_hxge_ldv_t ldvp = (p_hxge_ldv_t)arg1;
807 p_hxge_t hxgep = (p_hxge_t)arg2;
808 p_hxge_ldg_t ldgp;
809 uint8_t channel;
810 uint32_t vindex;
811 hpi_handle_t handle;
812 tdc_stat_t cs;
813 p_tx_ring_t *tx_rings;
814 p_tx_ring_t tx_ring_p;
815 hpi_status_t rs = HPI_SUCCESS;
816 uint_t serviced = DDI_INTR_UNCLAIMED;
817 hxge_status_t status = HXGE_OK;
818
819 if (ldvp == NULL) {
820 HXGE_DEBUG_MSG((NULL, INT_CTL,
821 "<== hxge_tx_intr: hxgep $%p ldvp $%p", hxgep, ldvp));
822 return (DDI_INTR_UNCLAIMED);
823 }
824
825 if (arg2 == NULL || (void *) ldvp->hxgep != arg2) {
826 hxgep = ldvp->hxgep;
827 }
828
829 /*
830 * If the interface is not started, just swallow the interrupt
831 * and don't rearm the logical device.
832 */
833 if (hxgep->hxge_mac_state != HXGE_MAC_STARTED)
834 return (DDI_INTR_CLAIMED);
835
836 HXGE_DEBUG_MSG((hxgep, INT_CTL,
837 "==> hxge_tx_intr: hxgep(arg2) $%p ldvp(arg1) $%p", hxgep, ldvp));
838
839 /*
840 * This interrupt handler is for a specific transmit dma channel.
841 */
842 handle = HXGE_DEV_HPI_HANDLE(hxgep);
843
844 /* Get the control and status for this channel. */
845 channel = ldvp->channel;
846 ldgp = ldvp->ldgp;
847 HXGE_DEBUG_MSG((hxgep, INT_CTL,
848 "==> hxge_tx_intr: hxgep $%p ldvp (ldvp) $%p channel %d",
849 hxgep, ldvp, channel));
850
851 rs = hpi_txdma_control_status(handle, OP_GET, channel, &cs);
852 vindex = ldvp->vdma_index;
853 HXGE_DEBUG_MSG((hxgep, INT_CTL,
854 "==> hxge_tx_intr:channel %d ring index %d status 0x%08x",
855 channel, vindex, rs));
856
857 if (!rs && cs.bits.marked) {
858 HXGE_DEBUG_MSG((hxgep, INT_CTL,
859 "==> hxge_tx_intr:channel %d ring index %d "
860 "status 0x%08x (marked bit set)", channel, vindex, rs));
861 tx_rings = hxgep->tx_rings->rings;
862 tx_ring_p = tx_rings[vindex];
863 HXGE_DEBUG_MSG((hxgep, INT_CTL,
864 "==> hxge_tx_intr:channel %d ring index %d "
865 "status 0x%08x (marked bit set, calling reclaim)",
866 channel, vindex, rs));
867
868 MUTEX_ENTER(&tx_ring_p->lock);
869 (void) hxge_txdma_reclaim(hxgep, tx_rings[vindex], 0);
870 MUTEX_EXIT(&tx_ring_p->lock);
871 mac_tx_update(hxgep->mach);
872 }
873
874 /*
875 * Process other transmit control and status. Check the ldv state.
876 */
877 status = hxge_tx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs);
878
879 /* Clear the error bits */
880 RXDMA_REG_WRITE64(handle, TDC_STAT, channel, cs.value);
881
882 /*
883 * Rearm this logical group if this is a single device group.
884 */
885 if (ldgp->nldvs == 1) {
886 HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_tx_intr: rearm"));
887 if (status == HXGE_OK) {
888 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg,
889 B_TRUE, ldgp->ldg_timer);
890 }
891 }
892 HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_tx_intr"));
893 serviced = DDI_INTR_CLAIMED;
894 return (serviced);
895 }
896
897 void
hxge_txdma_stop(p_hxge_t hxgep)898 hxge_txdma_stop(p_hxge_t hxgep)
899 {
900 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_stop"));
901
902 (void) hxge_tx_vmac_disable(hxgep);
903 (void) hxge_txdma_hw_mode(hxgep, HXGE_DMA_STOP);
904
905 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_stop"));
906 }
907
908 hxge_status_t
hxge_txdma_hw_mode(p_hxge_t hxgep,boolean_t enable)909 hxge_txdma_hw_mode(p_hxge_t hxgep, boolean_t enable)
910 {
911 int i, ndmas;
912 uint16_t channel;
913 p_tx_rings_t tx_rings;
914 p_tx_ring_t *tx_desc_rings;
915 hpi_handle_t handle;
916 hpi_status_t rs = HPI_SUCCESS;
917 hxge_status_t status = HXGE_OK;
918
919 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
920 "==> hxge_txdma_hw_mode: enable mode %d", enable));
921
922 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) {
923 HXGE_DEBUG_MSG((hxgep, TX_CTL,
924 "<== hxge_txdma_mode: not initialized"));
925 return (HXGE_ERROR);
926 }
927 tx_rings = hxgep->tx_rings;
928 if (tx_rings == NULL) {
929 HXGE_DEBUG_MSG((hxgep, TX_CTL,
930 "<== hxge_txdma_hw_mode: NULL global ring pointer"));
931 return (HXGE_ERROR);
932 }
933 tx_desc_rings = tx_rings->rings;
934 if (tx_desc_rings == NULL) {
935 HXGE_DEBUG_MSG((hxgep, TX_CTL,
936 "<== hxge_txdma_hw_mode: NULL rings pointer"));
937 return (HXGE_ERROR);
938 }
939 ndmas = tx_rings->ndmas;
940 if (!ndmas) {
941 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
942 "<== hxge_txdma_hw_mode: no dma channel allocated"));
943 return (HXGE_ERROR);
944 }
945 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_mode: "
946 "tx_rings $%p tx_desc_rings $%p ndmas %d",
947 tx_rings, tx_desc_rings, ndmas));
948
949 handle = HXGE_DEV_HPI_HANDLE(hxgep);
950 for (i = 0; i < ndmas; i++) {
951 if (tx_desc_rings[i] == NULL) {
952 continue;
953 }
954 channel = tx_desc_rings[i]->tdc;
955 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
956 "==> hxge_txdma_hw_mode: channel %d", channel));
957 if (enable) {
958 rs = hpi_txdma_channel_enable(handle, channel);
959 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
960 "==> hxge_txdma_hw_mode: channel %d (enable) "
961 "rs 0x%x", channel, rs));
962 } else {
963 /*
964 * Stop the dma channel and waits for the stop done. If
965 * the stop done bit is not set, then force an error so
966 * TXC will stop. All channels bound to this port need
967 * to be stopped and reset after injecting an interrupt
968 * error.
969 */
970 rs = hpi_txdma_channel_disable(handle, channel);
971 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
972 "==> hxge_txdma_hw_mode: channel %d (disable) "
973 "rs 0x%x", channel, rs));
974 }
975 }
976
977 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
978
979 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
980 "<== hxge_txdma_hw_mode: status 0x%x", status));
981
982 return (status);
983 }
984
985 void
hxge_txdma_enable_channel(p_hxge_t hxgep,uint16_t channel)986 hxge_txdma_enable_channel(p_hxge_t hxgep, uint16_t channel)
987 {
988 hpi_handle_t handle;
989
990 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
991 "==> hxge_txdma_enable_channel: channel %d", channel));
992
993 handle = HXGE_DEV_HPI_HANDLE(hxgep);
994 /* enable the transmit dma channels */
995 (void) hpi_txdma_channel_enable(handle, channel);
996
997 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_txdma_enable_channel"));
998 }
999
1000 void
hxge_txdma_disable_channel(p_hxge_t hxgep,uint16_t channel)1001 hxge_txdma_disable_channel(p_hxge_t hxgep, uint16_t channel)
1002 {
1003 hpi_handle_t handle;
1004
1005 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
1006 "==> hxge_txdma_disable_channel: channel %d", channel));
1007
1008 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1009 /* stop the transmit dma channels */
1010 (void) hpi_txdma_channel_disable(handle, channel);
1011
1012 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_disable_channel"));
1013 }
1014
1015 int
hxge_txdma_stop_inj_err(p_hxge_t hxgep,int channel)1016 hxge_txdma_stop_inj_err(p_hxge_t hxgep, int channel)
1017 {
1018 hpi_handle_t handle;
1019 int status;
1020 hpi_status_t rs = HPI_SUCCESS;
1021
1022 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_stop_inj_err"));
1023
1024 /*
1025 * Stop the dma channel waits for the stop done. If the stop done bit
1026 * is not set, then create an error.
1027 */
1028 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1029 rs = hpi_txdma_channel_disable(handle, channel);
1030 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
1031 if (status == HXGE_OK) {
1032 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1033 "<== hxge_txdma_stop_inj_err (channel %d): "
1034 "stopped OK", channel));
1035 return (status);
1036 }
1037
1038 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1039 "==> hxge_txdma_stop_inj_err (channel): stop failed (0x%x) "
1040 " (injected error but still not stopped)", channel, rs));
1041
1042 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_stop_inj_err"));
1043
1044 return (status);
1045 }
1046
1047 /*ARGSUSED*/
1048 void
hxge_fixup_txdma_rings(p_hxge_t hxgep)1049 hxge_fixup_txdma_rings(p_hxge_t hxgep)
1050 {
1051 int index, ndmas;
1052 uint16_t channel;
1053 p_tx_rings_t tx_rings;
1054
1055 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_txdma_rings"));
1056
1057 /*
1058 * For each transmit channel, reclaim each descriptor and free buffers.
1059 */
1060 tx_rings = hxgep->tx_rings;
1061 if (tx_rings == NULL) {
1062 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1063 "<== hxge_fixup_txdma_rings: NULL ring pointer"));
1064 return;
1065 }
1066
1067 ndmas = tx_rings->ndmas;
1068 if (!ndmas) {
1069 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1070 "<== hxge_fixup_txdma_rings: no channel allocated"));
1071 return;
1072 }
1073
1074 if (tx_rings->rings == NULL) {
1075 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1076 "<== hxge_fixup_txdma_rings: NULL rings pointer"));
1077 return;
1078 }
1079
1080 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_fixup_txdma_rings: "
1081 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1082 tx_rings, tx_rings->rings, ndmas));
1083
1084 for (index = 0; index < ndmas; index++) {
1085 channel = tx_rings->rings[index]->tdc;
1086 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1087 "==> hxge_fixup_txdma_rings: channel %d", channel));
1088 hxge_txdma_fixup_channel(hxgep, tx_rings->rings[index],
1089 channel);
1090 }
1091
1092 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_fixup_txdma_rings"));
1093 }
1094
1095 /*ARGSUSED*/
1096 void
hxge_txdma_fix_channel(p_hxge_t hxgep,uint16_t channel)1097 hxge_txdma_fix_channel(p_hxge_t hxgep, uint16_t channel)
1098 {
1099 p_tx_ring_t ring_p;
1100
1101 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fix_channel"));
1102
1103 ring_p = hxge_txdma_get_ring(hxgep, channel);
1104 if (ring_p == NULL) {
1105 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_channel"));
1106 return;
1107 }
1108
1109 if (ring_p->tdc != channel) {
1110 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1111 "<== hxge_txdma_fix_channel: channel not matched "
1112 "ring tdc %d passed channel", ring_p->tdc, channel));
1113 return;
1114 }
1115
1116 hxge_txdma_fixup_channel(hxgep, ring_p, channel);
1117
1118 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_channel"));
1119 }
1120
1121 /*ARGSUSED*/
1122 void
hxge_txdma_fixup_channel(p_hxge_t hxgep,p_tx_ring_t ring_p,uint16_t channel)1123 hxge_txdma_fixup_channel(p_hxge_t hxgep, p_tx_ring_t ring_p, uint16_t channel)
1124 {
1125 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fixup_channel"));
1126
1127 if (ring_p == NULL) {
1128 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1129 "<== hxge_txdma_fixup_channel: NULL ring pointer"));
1130 return;
1131 }
1132 if (ring_p->tdc != channel) {
1133 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1134 "<== hxge_txdma_fixup_channel: channel not matched "
1135 "ring tdc %d passed channel", ring_p->tdc, channel));
1136 return;
1137 }
1138 MUTEX_ENTER(&ring_p->lock);
1139 (void) hxge_txdma_reclaim(hxgep, ring_p, 0);
1140
1141 ring_p->rd_index = 0;
1142 ring_p->wr_index = 0;
1143 ring_p->ring_head.value = 0;
1144 ring_p->ring_kick_tail.value = 0;
1145 ring_p->descs_pending = 0;
1146 MUTEX_EXIT(&ring_p->lock);
1147
1148 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fixup_channel"));
1149 }
1150
1151 /*ARGSUSED*/
1152 void
hxge_txdma_hw_kick(p_hxge_t hxgep)1153 hxge_txdma_hw_kick(p_hxge_t hxgep)
1154 {
1155 int index, ndmas;
1156 uint16_t channel;
1157 p_tx_rings_t tx_rings;
1158
1159 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hw_kick"));
1160
1161 tx_rings = hxgep->tx_rings;
1162 if (tx_rings == NULL) {
1163 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1164 "<== hxge_txdma_hw_kick: NULL ring pointer"));
1165 return;
1166 }
1167 ndmas = tx_rings->ndmas;
1168 if (!ndmas) {
1169 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1170 "<== hxge_txdma_hw_kick: no channel allocated"));
1171 return;
1172 }
1173 if (tx_rings->rings == NULL) {
1174 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1175 "<== hxge_txdma_hw_kick: NULL rings pointer"));
1176 return;
1177 }
1178 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_kick: "
1179 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1180 tx_rings, tx_rings->rings, ndmas));
1181
1182 for (index = 0; index < ndmas; index++) {
1183 channel = tx_rings->rings[index]->tdc;
1184 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1185 "==> hxge_txdma_hw_kick: channel %d", channel));
1186 hxge_txdma_hw_kick_channel(hxgep, tx_rings->rings[index],
1187 channel);
1188 }
1189
1190 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hw_kick"));
1191 }
1192
1193 /*ARGSUSED*/
1194 void
hxge_txdma_kick_channel(p_hxge_t hxgep,uint16_t channel)1195 hxge_txdma_kick_channel(p_hxge_t hxgep, uint16_t channel)
1196 {
1197 p_tx_ring_t ring_p;
1198
1199 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_kick_channel"));
1200
1201 ring_p = hxge_txdma_get_ring(hxgep, channel);
1202 if (ring_p == NULL) {
1203 HXGE_DEBUG_MSG((hxgep, TX_CTL, " hxge_txdma_kick_channel"));
1204 return;
1205 }
1206
1207 if (ring_p->tdc != channel) {
1208 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1209 "<== hxge_txdma_kick_channel: channel not matched "
1210 "ring tdc %d passed channel", ring_p->tdc, channel));
1211 return;
1212 }
1213
1214 hxge_txdma_hw_kick_channel(hxgep, ring_p, channel);
1215
1216 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_kick_channel"));
1217 }
1218
1219 /*ARGSUSED*/
1220 void
hxge_txdma_hw_kick_channel(p_hxge_t hxgep,p_tx_ring_t ring_p,uint16_t channel)1221 hxge_txdma_hw_kick_channel(p_hxge_t hxgep, p_tx_ring_t ring_p, uint16_t channel)
1222 {
1223 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hw_kick_channel"));
1224
1225 if (ring_p == NULL) {
1226 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1227 "<== hxge_txdma_hw_kick_channel: NULL ring pointer"));
1228 return;
1229 }
1230
1231 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hw_kick_channel"));
1232 }
1233
1234 /*ARGSUSED*/
1235 void
hxge_check_tx_hang(p_hxge_t hxgep)1236 hxge_check_tx_hang(p_hxge_t hxgep)
1237 {
1238 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_check_tx_hang"));
1239
1240 /*
1241 * Needs inputs from hardware for regs: head index had not moved since
1242 * last timeout. packets not transmitted or stuffed registers.
1243 */
1244 if (hxge_txdma_hung(hxgep)) {
1245 hxge_fixup_hung_txdma_rings(hxgep);
1246 }
1247
1248 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_check_tx_hang"));
1249 }
1250
1251 int
hxge_txdma_hung(p_hxge_t hxgep)1252 hxge_txdma_hung(p_hxge_t hxgep)
1253 {
1254 int index, ndmas;
1255 uint16_t channel;
1256 p_tx_rings_t tx_rings;
1257 p_tx_ring_t tx_ring_p;
1258
1259 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hung"));
1260
1261 tx_rings = hxgep->tx_rings;
1262 if (tx_rings == NULL) {
1263 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1264 "<== hxge_txdma_hung: NULL ring pointer"));
1265 return (B_FALSE);
1266 }
1267
1268 ndmas = tx_rings->ndmas;
1269 if (!ndmas) {
1270 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1271 "<== hxge_txdma_hung: no channel allocated"));
1272 return (B_FALSE);
1273 }
1274
1275 if (tx_rings->rings == NULL) {
1276 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1277 "<== hxge_txdma_hung: NULL rings pointer"));
1278 return (B_FALSE);
1279 }
1280
1281 for (index = 0; index < ndmas; index++) {
1282 channel = tx_rings->rings[index]->tdc;
1283 tx_ring_p = tx_rings->rings[index];
1284 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1285 "==> hxge_txdma_hung: channel %d", channel));
1286 if (hxge_txdma_channel_hung(hxgep, tx_ring_p, channel)) {
1287 return (B_TRUE);
1288 }
1289 }
1290
1291 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hung"));
1292
1293 return (B_FALSE);
1294 }
1295
1296 int
hxge_txdma_channel_hung(p_hxge_t hxgep,p_tx_ring_t tx_ring_p,uint16_t channel)1297 hxge_txdma_channel_hung(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, uint16_t channel)
1298 {
1299 uint16_t head_index, tail_index;
1300 boolean_t head_wrap, tail_wrap;
1301 hpi_handle_t handle;
1302 tdc_tdr_head_t tx_head;
1303 uint_t tx_rd_index;
1304
1305 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_channel_hung"));
1306
1307 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1308 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1309 "==> hxge_txdma_channel_hung: channel %d", channel));
1310 MUTEX_ENTER(&tx_ring_p->lock);
1311 (void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
1312
1313 tail_index = tx_ring_p->wr_index;
1314 tail_wrap = tx_ring_p->wr_index_wrap;
1315 tx_rd_index = tx_ring_p->rd_index;
1316 MUTEX_EXIT(&tx_ring_p->lock);
1317
1318 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1319 "==> hxge_txdma_channel_hung: tdc %d tx_rd_index %d "
1320 "tail_index %d tail_wrap %d ",
1321 channel, tx_rd_index, tail_index, tail_wrap));
1322 /*
1323 * Read the hardware maintained transmit head and wrap around bit.
1324 */
1325 (void) hpi_txdma_ring_head_get(handle, channel, &tx_head);
1326 head_index = tx_head.bits.head;
1327 head_wrap = tx_head.bits.wrap;
1328 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_channel_hung: "
1329 "tx_rd_index %d tail %d tail_wrap %d head %d wrap %d",
1330 tx_rd_index, tail_index, tail_wrap, head_index, head_wrap));
1331
1332 if (TXDMA_RING_EMPTY(head_index, head_wrap, tail_index, tail_wrap) &&
1333 (head_index == tx_rd_index)) {
1334 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1335 "==> hxge_txdma_channel_hung: EMPTY"));
1336 return (B_FALSE);
1337 }
1338 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1339 "==> hxge_txdma_channel_hung: Checking if ring full"));
1340 if (TXDMA_RING_FULL(head_index, head_wrap, tail_index, tail_wrap)) {
1341 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1342 "==> hxge_txdma_channel_hung: full"));
1343 return (B_TRUE);
1344 }
1345
1346 /* If not full, check with hardware to see if it is hung */
1347 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_channel_hung"));
1348
1349 return (B_FALSE);
1350 }
1351
1352 /*ARGSUSED*/
1353 void
hxge_fixup_hung_txdma_rings(p_hxge_t hxgep)1354 hxge_fixup_hung_txdma_rings(p_hxge_t hxgep)
1355 {
1356 int index, ndmas;
1357 uint16_t channel;
1358 p_tx_rings_t tx_rings;
1359
1360 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_hung_txdma_rings"));
1361 tx_rings = hxgep->tx_rings;
1362 if (tx_rings == NULL) {
1363 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1364 "<== hxge_fixup_hung_txdma_rings: NULL ring pointer"));
1365 return;
1366 }
1367 ndmas = tx_rings->ndmas;
1368 if (!ndmas) {
1369 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1370 "<== hxge_fixup_hung_txdma_rings: no channel allocated"));
1371 return;
1372 }
1373 if (tx_rings->rings == NULL) {
1374 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1375 "<== hxge_fixup_hung_txdma_rings: NULL rings pointer"));
1376 return;
1377 }
1378 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_hung_txdma_rings: "
1379 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1380 tx_rings, tx_rings->rings, ndmas));
1381
1382 for (index = 0; index < ndmas; index++) {
1383 channel = tx_rings->rings[index]->tdc;
1384 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1385 "==> hxge_fixup_hung_txdma_rings: channel %d", channel));
1386 hxge_txdma_fixup_hung_channel(hxgep, tx_rings->rings[index],
1387 channel);
1388 }
1389
1390 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_fixup_hung_txdma_rings"));
1391 }
1392
1393 /*ARGSUSED*/
1394 void
hxge_txdma_fix_hung_channel(p_hxge_t hxgep,uint16_t channel)1395 hxge_txdma_fix_hung_channel(p_hxge_t hxgep, uint16_t channel)
1396 {
1397 p_tx_ring_t ring_p;
1398
1399 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fix_hung_channel"));
1400 ring_p = hxge_txdma_get_ring(hxgep, channel);
1401 if (ring_p == NULL) {
1402 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1403 "<== hxge_txdma_fix_hung_channel"));
1404 return;
1405 }
1406 if (ring_p->tdc != channel) {
1407 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1408 "<== hxge_txdma_fix_hung_channel: channel not matched "
1409 "ring tdc %d passed channel", ring_p->tdc, channel));
1410 return;
1411 }
1412 hxge_txdma_fixup_channel(hxgep, ring_p, channel);
1413
1414 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_hung_channel"));
1415 }
1416
1417 /*ARGSUSED*/
1418 void
hxge_txdma_fixup_hung_channel(p_hxge_t hxgep,p_tx_ring_t ring_p,uint16_t channel)1419 hxge_txdma_fixup_hung_channel(p_hxge_t hxgep, p_tx_ring_t ring_p,
1420 uint16_t channel)
1421 {
1422 hpi_handle_t handle;
1423 int status = HXGE_OK;
1424
1425 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fixup_hung_channel"));
1426
1427 if (ring_p == NULL) {
1428 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1429 "<== hxge_txdma_fixup_hung_channel: NULL ring pointer"));
1430 return;
1431 }
1432 if (ring_p->tdc != channel) {
1433 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1434 "<== hxge_txdma_fixup_hung_channel: channel "
1435 "not matched ring tdc %d passed channel",
1436 ring_p->tdc, channel));
1437 return;
1438 }
1439 /* Reclaim descriptors */
1440 MUTEX_ENTER(&ring_p->lock);
1441 (void) hxge_txdma_reclaim(hxgep, ring_p, 0);
1442 MUTEX_EXIT(&ring_p->lock);
1443
1444 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1445 /*
1446 * Stop the dma channel waits for the stop done. If the stop done bit
1447 * is not set, then force an error.
1448 */
1449 status = hpi_txdma_channel_disable(handle, channel);
1450 if (!(status & HPI_TXDMA_STOP_FAILED)) {
1451 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1452 "<== hxge_txdma_fixup_hung_channel: stopped OK "
1453 "ring tdc %d passed channel %d", ring_p->tdc, channel));
1454 return;
1455 }
1456 /* Stop done bit will be set as a result of error injection */
1457 status = hpi_txdma_channel_disable(handle, channel);
1458 if (!(status & HPI_TXDMA_STOP_FAILED)) {
1459 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1460 "<== hxge_txdma_fixup_hung_channel: stopped again"
1461 "ring tdc %d passed channel", ring_p->tdc, channel));
1462 return;
1463 }
1464
1465 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1466 "<== hxge_txdma_fixup_hung_channel: stop done still not set!! "
1467 "ring tdc %d passed channel", ring_p->tdc, channel));
1468 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fixup_hung_channel"));
1469 }
1470
1471 /*ARGSUSED*/
1472 void
hxge_reclaim_rings(p_hxge_t hxgep)1473 hxge_reclaim_rings(p_hxge_t hxgep)
1474 {
1475 int index, ndmas;
1476 uint16_t channel;
1477 p_tx_rings_t tx_rings;
1478 p_tx_ring_t tx_ring_p;
1479
1480 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_reclaim_ring"));
1481 tx_rings = hxgep->tx_rings;
1482 if (tx_rings == NULL) {
1483 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1484 "<== hxge_reclain_rimgs: NULL ring pointer"));
1485 return;
1486 }
1487 ndmas = tx_rings->ndmas;
1488 if (!ndmas) {
1489 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1490 "<== hxge_reclain_rimgs: no channel allocated"));
1491 return;
1492 }
1493 if (tx_rings->rings == NULL) {
1494 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1495 "<== hxge_reclain_rimgs: NULL rings pointer"));
1496 return;
1497 }
1498 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_reclain_rimgs: "
1499 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1500 tx_rings, tx_rings->rings, ndmas));
1501
1502 for (index = 0; index < ndmas; index++) {
1503 channel = tx_rings->rings[index]->tdc;
1504 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> reclain_rimgs: channel %d",
1505 channel));
1506 tx_ring_p = tx_rings->rings[index];
1507 MUTEX_ENTER(&tx_ring_p->lock);
1508 (void) hxge_txdma_reclaim(hxgep, tx_ring_p, channel);
1509 MUTEX_EXIT(&tx_ring_p->lock);
1510 }
1511
1512 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_reclaim_rings"));
1513 }
1514
1515 /*
1516 * Static functions start here.
1517 */
1518 static hxge_status_t
hxge_map_txdma(p_hxge_t hxgep)1519 hxge_map_txdma(p_hxge_t hxgep)
1520 {
1521 int i, ndmas;
1522 uint16_t channel;
1523 p_tx_rings_t tx_rings;
1524 p_tx_ring_t *tx_desc_rings;
1525 p_tx_mbox_areas_t tx_mbox_areas_p;
1526 p_tx_mbox_t *tx_mbox_p;
1527 p_hxge_dma_pool_t dma_buf_poolp;
1528 p_hxge_dma_pool_t dma_cntl_poolp;
1529 p_hxge_dma_common_t *dma_buf_p;
1530 p_hxge_dma_common_t *dma_cntl_p;
1531 hxge_status_t status = HXGE_OK;
1532
1533 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma"));
1534
1535 dma_buf_poolp = hxgep->tx_buf_pool_p;
1536 dma_cntl_poolp = hxgep->tx_cntl_pool_p;
1537
1538 if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) {
1539 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1540 "==> hxge_map_txdma: buf not allocated"));
1541 return (HXGE_ERROR);
1542 }
1543 ndmas = dma_buf_poolp->ndmas;
1544 if (!ndmas) {
1545 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1546 "<== hxge_map_txdma: no dma allocated"));
1547 return (HXGE_ERROR);
1548 }
1549 dma_buf_p = dma_buf_poolp->dma_buf_pool_p;
1550 dma_cntl_p = dma_cntl_poolp->dma_buf_pool_p;
1551
1552 tx_rings = (p_tx_rings_t)KMEM_ZALLOC(sizeof (tx_rings_t), KM_SLEEP);
1553 tx_desc_rings = (p_tx_ring_t *)KMEM_ZALLOC(
1554 sizeof (p_tx_ring_t) * ndmas, KM_SLEEP);
1555
1556 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1557 "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
1558
1559 tx_mbox_areas_p = (p_tx_mbox_areas_t)
1560 KMEM_ZALLOC(sizeof (tx_mbox_areas_t), KM_SLEEP);
1561 tx_mbox_p = (p_tx_mbox_t *)KMEM_ZALLOC(
1562 sizeof (p_tx_mbox_t) * ndmas, KM_SLEEP);
1563
1564 /*
1565 * Map descriptors from the buffer pools for each dma channel.
1566 */
1567 for (i = 0; i < ndmas; i++) {
1568 /*
1569 * Set up and prepare buffer blocks, descriptors and mailbox.
1570 */
1571 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
1572 status = hxge_map_txdma_channel(hxgep, channel,
1573 (p_hxge_dma_common_t *)&dma_buf_p[i],
1574 (p_tx_ring_t *)&tx_desc_rings[i],
1575 dma_buf_poolp->num_chunks[i],
1576 (p_hxge_dma_common_t *)&dma_cntl_p[i],
1577 (p_tx_mbox_t *)&tx_mbox_p[i]);
1578 if (status != HXGE_OK) {
1579 goto hxge_map_txdma_fail1;
1580 }
1581 tx_desc_rings[i]->index = (uint16_t)i;
1582 tx_desc_rings[i]->tdc_stats = &hxgep->statsp->tdc_stats[i];
1583 }
1584
1585 tx_rings->ndmas = ndmas;
1586 tx_rings->rings = tx_desc_rings;
1587 hxgep->tx_rings = tx_rings;
1588 tx_mbox_areas_p->txmbox_areas_p = tx_mbox_p;
1589 hxgep->tx_mbox_areas_p = tx_mbox_areas_p;
1590
1591 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1592 "tx_rings $%p rings $%p", hxgep->tx_rings, hxgep->tx_rings->rings));
1593 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1594 "tx_rings $%p tx_desc_rings $%p",
1595 hxgep->tx_rings, tx_desc_rings));
1596
1597 goto hxge_map_txdma_exit;
1598
1599 hxge_map_txdma_fail1:
1600 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1601 "==> hxge_map_txdma: uninit tx desc "
1602 "(status 0x%x channel %d i %d)", hxgep, status, channel, i));
1603 i--;
1604 for (; i >= 0; i--) {
1605 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
1606 hxge_unmap_txdma_channel(hxgep, channel, tx_desc_rings[i],
1607 tx_mbox_p[i]);
1608 }
1609
1610 KMEM_FREE(tx_desc_rings, sizeof (p_tx_ring_t) * ndmas);
1611 KMEM_FREE(tx_rings, sizeof (tx_rings_t));
1612 KMEM_FREE(tx_mbox_p, sizeof (p_tx_mbox_t) * ndmas);
1613 KMEM_FREE(tx_mbox_areas_p, sizeof (tx_mbox_areas_t));
1614
1615 hxge_map_txdma_exit:
1616 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1617 "==> hxge_map_txdma: (status 0x%x channel %d)", status, channel));
1618
1619 return (status);
1620 }
1621
1622 static void
hxge_unmap_txdma(p_hxge_t hxgep)1623 hxge_unmap_txdma(p_hxge_t hxgep)
1624 {
1625 int i, ndmas;
1626 uint8_t channel;
1627 p_tx_rings_t tx_rings;
1628 p_tx_ring_t *tx_desc_rings;
1629 p_tx_mbox_areas_t tx_mbox_areas_p;
1630 p_tx_mbox_t *tx_mbox_p;
1631 p_hxge_dma_pool_t dma_buf_poolp;
1632
1633 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_unmap_txdma"));
1634
1635 dma_buf_poolp = hxgep->tx_buf_pool_p;
1636 if (!dma_buf_poolp->buf_allocated) {
1637 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1638 "==> hxge_unmap_txdma: buf not allocated"));
1639 return;
1640 }
1641 ndmas = dma_buf_poolp->ndmas;
1642 if (!ndmas) {
1643 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1644 "<== hxge_unmap_txdma: no dma allocated"));
1645 return;
1646 }
1647 tx_rings = hxgep->tx_rings;
1648 tx_desc_rings = tx_rings->rings;
1649 if (tx_rings == NULL) {
1650 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1651 "<== hxge_unmap_txdma: NULL ring pointer"));
1652 return;
1653 }
1654 tx_desc_rings = tx_rings->rings;
1655 if (tx_desc_rings == NULL) {
1656 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1657 "<== hxge_unmap_txdma: NULL ring pointers"));
1658 return;
1659 }
1660 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_unmap_txdma: "
1661 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1662 tx_rings, tx_desc_rings, ndmas));
1663
1664 tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
1665 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
1666
1667 for (i = 0; i < ndmas; i++) {
1668 channel = tx_desc_rings[i]->tdc;
1669 (void) hxge_unmap_txdma_channel(hxgep, channel,
1670 (p_tx_ring_t)tx_desc_rings[i],
1671 (p_tx_mbox_t)tx_mbox_p[i]);
1672 }
1673
1674 KMEM_FREE(tx_desc_rings, sizeof (p_tx_ring_t) * ndmas);
1675 KMEM_FREE(tx_rings, sizeof (tx_rings_t));
1676 KMEM_FREE(tx_mbox_p, sizeof (p_tx_mbox_t) * ndmas);
1677 KMEM_FREE(tx_mbox_areas_p, sizeof (tx_mbox_areas_t));
1678
1679 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_unmap_txdma"));
1680 }
1681
1682 static hxge_status_t
hxge_map_txdma_channel(p_hxge_t hxgep,uint16_t channel,p_hxge_dma_common_t * dma_buf_p,p_tx_ring_t * tx_desc_p,uint32_t num_chunks,p_hxge_dma_common_t * dma_cntl_p,p_tx_mbox_t * tx_mbox_p)1683 hxge_map_txdma_channel(p_hxge_t hxgep, uint16_t channel,
1684 p_hxge_dma_common_t *dma_buf_p, p_tx_ring_t *tx_desc_p,
1685 uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p,
1686 p_tx_mbox_t *tx_mbox_p)
1687 {
1688 int status = HXGE_OK;
1689
1690 /*
1691 * Set up and prepare buffer blocks, descriptors and mailbox.
1692 */
1693 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1694 "==> hxge_map_txdma_channel (channel %d)", channel));
1695
1696 /*
1697 * Transmit buffer blocks
1698 */
1699 status = hxge_map_txdma_channel_buf_ring(hxgep, channel,
1700 dma_buf_p, tx_desc_p, num_chunks);
1701 if (status != HXGE_OK) {
1702 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1703 "==> hxge_map_txdma_channel (channel %d): "
1704 "map buffer failed 0x%x", channel, status));
1705 goto hxge_map_txdma_channel_exit;
1706 }
1707 /*
1708 * Transmit block ring, and mailbox.
1709 */
1710 hxge_map_txdma_channel_cfg_ring(hxgep, channel, dma_cntl_p, *tx_desc_p,
1711 tx_mbox_p);
1712
1713 goto hxge_map_txdma_channel_exit;
1714
1715 hxge_map_txdma_channel_fail1:
1716 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1717 "==> hxge_map_txdma_channel: unmap buf"
1718 "(status 0x%x channel %d)", status, channel));
1719 hxge_unmap_txdma_channel_buf_ring(hxgep, *tx_desc_p);
1720
1721 hxge_map_txdma_channel_exit:
1722 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1723 "<== hxge_map_txdma_channel: (status 0x%x channel %d)",
1724 status, channel));
1725
1726 return (status);
1727 }
1728
1729 /*ARGSUSED*/
1730 static void
hxge_unmap_txdma_channel(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)1731 hxge_unmap_txdma_channel(p_hxge_t hxgep, uint16_t channel,
1732 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
1733 {
1734 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1735 "==> hxge_unmap_txdma_channel (channel %d)", channel));
1736
1737 /* unmap tx block ring, and mailbox. */
1738 (void) hxge_unmap_txdma_channel_cfg_ring(hxgep, tx_ring_p, tx_mbox_p);
1739
1740 /* unmap buffer blocks */
1741 (void) hxge_unmap_txdma_channel_buf_ring(hxgep, tx_ring_p);
1742
1743 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_unmap_txdma_channel"));
1744 }
1745
1746 /*ARGSUSED*/
1747 static void
hxge_map_txdma_channel_cfg_ring(p_hxge_t hxgep,uint16_t dma_channel,p_hxge_dma_common_t * dma_cntl_p,p_tx_ring_t tx_ring_p,p_tx_mbox_t * tx_mbox_p)1748 hxge_map_txdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel,
1749 p_hxge_dma_common_t *dma_cntl_p, p_tx_ring_t tx_ring_p,
1750 p_tx_mbox_t *tx_mbox_p)
1751 {
1752 p_tx_mbox_t mboxp;
1753 p_hxge_dma_common_t cntl_dmap;
1754 p_hxge_dma_common_t dmap;
1755 tdc_tdr_cfg_t *tx_ring_cfig_p;
1756 tdc_tdr_kick_t *tx_ring_kick_p;
1757 tdc_tdr_cfg_t *tx_cs_p;
1758 tdc_int_mask_t *tx_evmask_p;
1759 tdc_mbh_t *mboxh_p;
1760 tdc_mbl_t *mboxl_p;
1761 uint64_t tx_desc_len;
1762
1763 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1764 "==> hxge_map_txdma_channel_cfg_ring"));
1765
1766 cntl_dmap = *dma_cntl_p;
1767
1768 dmap = (p_hxge_dma_common_t)&tx_ring_p->tdc_desc;
1769 hxge_setup_dma_common(dmap, cntl_dmap, tx_ring_p->tx_ring_size,
1770 sizeof (tx_desc_t));
1771
1772 /*
1773 * Zero out transmit ring descriptors.
1774 */
1775 bzero((caddr_t)dmap->kaddrp, dmap->alength);
1776 tx_ring_cfig_p = &(tx_ring_p->tx_ring_cfig);
1777 tx_ring_kick_p = &(tx_ring_p->tx_ring_kick);
1778 tx_cs_p = &(tx_ring_p->tx_cs);
1779 tx_evmask_p = &(tx_ring_p->tx_evmask);
1780 tx_ring_cfig_p->value = 0;
1781 tx_ring_kick_p->value = 0;
1782 tx_cs_p->value = 0;
1783 tx_evmask_p->value = 0;
1784
1785 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1786 "==> hxge_map_txdma_channel_cfg_ring: channel %d des $%p",
1787 dma_channel, dmap->dma_cookie.dmac_laddress));
1788
1789 tx_ring_cfig_p->value = 0;
1790
1791 /* Hydra len is 11 bits and the lower 5 bits are 0s */
1792 tx_desc_len = (uint64_t)(tx_ring_p->tx_ring_size >> 5);
1793 tx_ring_cfig_p->value =
1794 (dmap->dma_cookie.dmac_laddress & TDC_TDR_CFG_ADDR_MASK) |
1795 (tx_desc_len << TDC_TDR_CFG_LEN_SHIFT);
1796
1797 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1798 "==> hxge_map_txdma_channel_cfg_ring: channel %d cfg 0x%llx",
1799 dma_channel, tx_ring_cfig_p->value));
1800
1801 tx_cs_p->bits.reset = 1;
1802
1803 /* Map in mailbox */
1804 mboxp = (p_tx_mbox_t)KMEM_ZALLOC(sizeof (tx_mbox_t), KM_SLEEP);
1805 dmap = (p_hxge_dma_common_t)&mboxp->tx_mbox;
1806 hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (txdma_mailbox_t));
1807 mboxh_p = (tdc_mbh_t *)&tx_ring_p->tx_mbox_mbh;
1808 mboxl_p = (tdc_mbl_t *)&tx_ring_p->tx_mbox_mbl;
1809 mboxh_p->value = mboxl_p->value = 0;
1810
1811 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1812 "==> hxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
1813 dmap->dma_cookie.dmac_laddress));
1814
1815 mboxh_p->bits.mbaddr = ((dmap->dma_cookie.dmac_laddress >>
1816 TDC_MBH_ADDR_SHIFT) & TDC_MBH_MASK);
1817 mboxl_p->bits.mbaddr = ((dmap->dma_cookie.dmac_laddress &
1818 TDC_MBL_MASK) >> TDC_MBL_SHIFT);
1819
1820 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1821 "==> hxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
1822 dmap->dma_cookie.dmac_laddress));
1823 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1824 "==> hxge_map_txdma_channel_cfg_ring: hmbox $%p mbox $%p",
1825 mboxh_p->bits.mbaddr, mboxl_p->bits.mbaddr));
1826
1827 /*
1828 * Set page valid and no mask
1829 */
1830 tx_ring_p->page_hdl.value = 0;
1831
1832 *tx_mbox_p = mboxp;
1833
1834 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1835 "<== hxge_map_txdma_channel_cfg_ring"));
1836 }
1837
1838 /*ARGSUSED*/
1839 static void
hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)1840 hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,
1841 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
1842 {
1843 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1844 "==> hxge_unmap_txdma_channel_cfg_ring: channel %d",
1845 tx_ring_p->tdc));
1846
1847 KMEM_FREE(tx_mbox_p, sizeof (tx_mbox_t));
1848
1849 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1850 "<== hxge_unmap_txdma_channel_cfg_ring"));
1851 }
1852
1853 static hxge_status_t
hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep,uint16_t channel,p_hxge_dma_common_t * dma_buf_p,p_tx_ring_t * tx_desc_p,uint32_t num_chunks)1854 hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel,
1855 p_hxge_dma_common_t *dma_buf_p,
1856 p_tx_ring_t *tx_desc_p, uint32_t num_chunks)
1857 {
1858 p_hxge_dma_common_t dma_bufp, tmp_bufp;
1859 p_hxge_dma_common_t dmap;
1860 hxge_os_dma_handle_t tx_buf_dma_handle;
1861 p_tx_ring_t tx_ring_p;
1862 p_tx_msg_t tx_msg_ring;
1863 hxge_status_t status = HXGE_OK;
1864 int ddi_status = DDI_SUCCESS;
1865 int i, j, index;
1866 uint32_t size, bsize;
1867 uint32_t nblocks, nmsgs;
1868 char qname[TASKQ_NAMELEN];
1869
1870 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1871 "==> hxge_map_txdma_channel_buf_ring"));
1872
1873 dma_bufp = tmp_bufp = *dma_buf_p;
1874 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1875 " hxge_map_txdma_channel_buf_ring: channel %d to map %d "
1876 "chunks bufp $%p", channel, num_chunks, dma_bufp));
1877
1878 nmsgs = 0;
1879 for (i = 0; i < num_chunks; i++, tmp_bufp++) {
1880 nmsgs += tmp_bufp->nblocks;
1881 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1882 "==> hxge_map_txdma_channel_buf_ring: channel %d "
1883 "bufp $%p nblocks %d nmsgs %d",
1884 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
1885 }
1886 if (!nmsgs) {
1887 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1888 "<== hxge_map_txdma_channel_buf_ring: channel %d "
1889 "no msg blocks", channel));
1890 status = HXGE_ERROR;
1891
1892 goto hxge_map_txdma_channel_buf_ring_exit;
1893 }
1894
1895 tx_ring_p = (p_tx_ring_t)KMEM_ZALLOC(sizeof (tx_ring_t), KM_SLEEP);
1896 tx_ring_p->hxgep = hxgep;
1897 (void) snprintf(qname, TASKQ_NAMELEN, "hxge_%d_%d",
1898 hxgep->instance, channel);
1899 tx_ring_p->taskq = ddi_taskq_create(hxgep->dip, qname, 1,
1900 TASKQ_DEFAULTPRI, 0);
1901 if (tx_ring_p->taskq == NULL) {
1902 goto hxge_map_txdma_channel_buf_ring_fail1;
1903 }
1904
1905 MUTEX_INIT(&tx_ring_p->lock, NULL, MUTEX_DRIVER,
1906 (void *) hxgep->interrupt_cookie);
1907 /*
1908 * Allocate transmit message rings and handles for packets not to be
1909 * copied to premapped buffers.
1910 */
1911 size = nmsgs * sizeof (tx_msg_t);
1912 tx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
1913 for (i = 0; i < nmsgs; i++) {
1914 ddi_status = ddi_dma_alloc_handle(hxgep->dip, &hxge_tx_dma_attr,
1915 DDI_DMA_DONTWAIT, 0, &tx_msg_ring[i].dma_handle);
1916 if (ddi_status != DDI_SUCCESS) {
1917 status |= HXGE_DDI_FAILED;
1918 break;
1919 }
1920 }
1921
1922 if (i < nmsgs) {
1923 HXGE_DEBUG_MSG((hxgep, HXGE_ERR_CTL,
1924 "Allocate handles failed."));
1925
1926 goto hxge_map_txdma_channel_buf_ring_fail1;
1927 }
1928 tx_ring_p->tdc = channel;
1929 tx_ring_p->tx_msg_ring = tx_msg_ring;
1930 tx_ring_p->tx_ring_size = nmsgs;
1931 tx_ring_p->num_chunks = num_chunks;
1932 if (!hxge_tx_intr_thres) {
1933 hxge_tx_intr_thres = tx_ring_p->tx_ring_size / 4;
1934 }
1935 tx_ring_p->tx_wrap_mask = tx_ring_p->tx_ring_size - 1;
1936 tx_ring_p->rd_index = 0;
1937 tx_ring_p->wr_index = 0;
1938 tx_ring_p->ring_head.value = 0;
1939 tx_ring_p->ring_kick_tail.value = 0;
1940 tx_ring_p->descs_pending = 0;
1941
1942 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1943 "==> hxge_map_txdma_channel_buf_ring: channel %d "
1944 "actual tx desc max %d nmsgs %d (config hxge_tx_ring_size %d)",
1945 channel, tx_ring_p->tx_ring_size, nmsgs, hxge_tx_ring_size));
1946
1947 /*
1948 * Map in buffers from the buffer pool.
1949 */
1950 index = 0;
1951 bsize = dma_bufp->block_size;
1952
1953 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma_channel_buf_ring: "
1954 "dma_bufp $%p tx_rng_p $%p tx_msg_rng_p $%p bsize %d",
1955 dma_bufp, tx_ring_p, tx_msg_ring, bsize));
1956
1957 for (i = 0; i < num_chunks; i++, dma_bufp++) {
1958 bsize = dma_bufp->block_size;
1959 nblocks = dma_bufp->nblocks;
1960 tx_buf_dma_handle = dma_bufp->dma_handle;
1961 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1962 "==> hxge_map_txdma_channel_buf_ring: dma chunk %d "
1963 "size %d dma_bufp $%p",
1964 i, sizeof (hxge_dma_common_t), dma_bufp));
1965
1966 for (j = 0; j < nblocks; j++) {
1967 tx_msg_ring[index].buf_dma_handle = tx_buf_dma_handle;
1968 tx_msg_ring[index].offset_index = j;
1969 dmap = &tx_msg_ring[index++].buf_dma;
1970 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1971 "==> hxge_map_txdma_channel_buf_ring: j %d"
1972 "dmap $%p", i, dmap));
1973 hxge_setup_dma_common(dmap, dma_bufp, 1, bsize);
1974 }
1975 }
1976
1977 if (i < num_chunks) {
1978 status = HXGE_ERROR;
1979
1980 goto hxge_map_txdma_channel_buf_ring_fail1;
1981 }
1982
1983 *tx_desc_p = tx_ring_p;
1984
1985 goto hxge_map_txdma_channel_buf_ring_exit;
1986
1987 hxge_map_txdma_channel_buf_ring_fail1:
1988 if (tx_ring_p->taskq) {
1989 ddi_taskq_destroy(tx_ring_p->taskq);
1990 tx_ring_p->taskq = NULL;
1991 }
1992
1993 index--;
1994 for (; index >= 0; index--) {
1995 if (tx_msg_ring[index].dma_handle != NULL) {
1996 ddi_dma_free_handle(&tx_msg_ring[index].dma_handle);
1997 }
1998 }
1999 MUTEX_DESTROY(&tx_ring_p->lock);
2000 KMEM_FREE(tx_msg_ring, size);
2001 KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2002
2003 status = HXGE_ERROR;
2004
2005 hxge_map_txdma_channel_buf_ring_exit:
2006 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2007 "<== hxge_map_txdma_channel_buf_ring status 0x%x", status));
2008
2009 return (status);
2010 }
2011
2012 /*ARGSUSED*/
2013 static void
hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep,p_tx_ring_t tx_ring_p)2014 hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep, p_tx_ring_t tx_ring_p)
2015 {
2016 p_tx_msg_t tx_msg_ring;
2017 p_tx_msg_t tx_msg_p;
2018 int i;
2019
2020 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2021 "==> hxge_unmap_txdma_channel_buf_ring"));
2022 if (tx_ring_p == NULL) {
2023 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2024 "<== hxge_unmap_txdma_channel_buf_ring: NULL ringp"));
2025 return;
2026 }
2027 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2028 "==> hxge_unmap_txdma_channel_buf_ring: channel %d",
2029 tx_ring_p->tdc));
2030
2031 MUTEX_ENTER(&tx_ring_p->lock);
2032 tx_msg_ring = tx_ring_p->tx_msg_ring;
2033 for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2034 tx_msg_p = &tx_msg_ring[i];
2035 if (tx_msg_p->flags.dma_type == USE_DVMA) {
2036 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "entry = %d", i));
2037 (void) dvma_unload(tx_msg_p->dvma_handle, 0, -1);
2038 tx_msg_p->dvma_handle = NULL;
2039 if (tx_ring_p->dvma_wr_index ==
2040 tx_ring_p->dvma_wrap_mask) {
2041 tx_ring_p->dvma_wr_index = 0;
2042 } else {
2043 tx_ring_p->dvma_wr_index++;
2044 }
2045 tx_ring_p->dvma_pending--;
2046 } else if (tx_msg_p->flags.dma_type == USE_DMA) {
2047 if (ddi_dma_unbind_handle(tx_msg_p->dma_handle)) {
2048 cmn_err(CE_WARN, "hxge_unmap_tx_bug_ring: "
2049 "ddi_dma_unbind_handle failed.");
2050 }
2051 }
2052 if (tx_msg_p->tx_message != NULL) {
2053 freemsg(tx_msg_p->tx_message);
2054 tx_msg_p->tx_message = NULL;
2055 }
2056 }
2057
2058 for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2059 if (tx_msg_ring[i].dma_handle != NULL) {
2060 ddi_dma_free_handle(&tx_msg_ring[i].dma_handle);
2061 }
2062 }
2063 MUTEX_EXIT(&tx_ring_p->lock);
2064
2065 if (tx_ring_p->taskq) {
2066 ddi_taskq_destroy(tx_ring_p->taskq);
2067 tx_ring_p->taskq = NULL;
2068 }
2069
2070 MUTEX_DESTROY(&tx_ring_p->lock);
2071 KMEM_FREE(tx_msg_ring, sizeof (tx_msg_t) * tx_ring_p->tx_ring_size);
2072 KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2073
2074 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2075 "<== hxge_unmap_txdma_channel_buf_ring"));
2076 }
2077
2078 static hxge_status_t
hxge_txdma_hw_start(p_hxge_t hxgep)2079 hxge_txdma_hw_start(p_hxge_t hxgep)
2080 {
2081 int i, ndmas;
2082 uint16_t channel;
2083 p_tx_rings_t tx_rings;
2084 p_tx_ring_t *tx_desc_rings;
2085 p_tx_mbox_areas_t tx_mbox_areas_p;
2086 p_tx_mbox_t *tx_mbox_p;
2087 hxge_status_t status = HXGE_OK;
2088 uint64_t tmp;
2089
2090 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_start"));
2091
2092 /*
2093 * Initialize REORD Table 1. Disable VMAC 2. Reset the FIFO Err Stat.
2094 * 3. Scrub memory and check for errors.
2095 */
2096 (void) hxge_tx_vmac_disable(hxgep);
2097
2098 /*
2099 * Clear the error status
2100 */
2101 HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, 0x7);
2102
2103 /*
2104 * Scrub the rtab memory for the TDC and reset the TDC.
2105 */
2106 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_HI, 0x0ULL);
2107 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_LO, 0x0ULL);
2108
2109 for (i = 0; i < 256; i++) {
2110 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2111 (uint64_t)i);
2112
2113 /*
2114 * Write the command register with an indirect read instruction
2115 */
2116 tmp = (0x1ULL << 30) | i;
2117 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD, tmp);
2118
2119 /*
2120 * Wait for status done
2121 */
2122 tmp = 0;
2123 do {
2124 HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2125 &tmp);
2126 } while (((tmp >> 31) & 0x1ULL) == 0x0);
2127 }
2128
2129 for (i = 0; i < 256; i++) {
2130 /*
2131 * Write the command register with an indirect read instruction
2132 */
2133 tmp = (0x1ULL << 30) | i;
2134 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD, tmp);
2135
2136 /*
2137 * Wait for status done
2138 */
2139 tmp = 0;
2140 do {
2141 HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2142 &tmp);
2143 } while (((tmp >> 31) & 0x1ULL) == 0x0);
2144
2145 HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_HI, &tmp);
2146 if (0x1ff00ULL != (0x1ffffULL & tmp)) {
2147 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2148 "unexpected data (hi), entry: %x, value: 0x%0llx\n",
2149 i, (unsigned long long)tmp));
2150 status = HXGE_ERROR;
2151 }
2152
2153 HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_LO, &tmp);
2154 if (tmp != 0) {
2155 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2156 "unexpected data (lo), entry: %x\n", i));
2157 status = HXGE_ERROR;
2158 }
2159
2160 HXGE_REG_RD64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, &tmp);
2161 if (tmp != 0) {
2162 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2163 "parity error, entry: %x, val 0x%llx\n",
2164 i, (unsigned long long)tmp));
2165 status = HXGE_ERROR;
2166 }
2167
2168 HXGE_REG_RD64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, &tmp);
2169 if (tmp != 0) {
2170 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2171 "parity error, entry: %x\n", i));
2172 status = HXGE_ERROR;
2173 }
2174 }
2175
2176 if (status != HXGE_OK)
2177 goto hxge_txdma_hw_start_exit;
2178
2179 /*
2180 * Reset FIFO Error Status for the TDC and enable FIFO error events.
2181 */
2182 HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, 0x7);
2183 HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_MASK, 0x0);
2184
2185 /*
2186 * Initialize the Transmit DMAs.
2187 */
2188 tx_rings = hxgep->tx_rings;
2189 if (tx_rings == NULL) {
2190 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2191 "<== hxge_txdma_hw_start: NULL ring pointer"));
2192 return (HXGE_ERROR);
2193 }
2194
2195 tx_desc_rings = tx_rings->rings;
2196 if (tx_desc_rings == NULL) {
2197 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2198 "<== hxge_txdma_hw_start: NULL ring pointers"));
2199 return (HXGE_ERROR);
2200 }
2201 ndmas = tx_rings->ndmas;
2202 if (!ndmas) {
2203 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2204 "<== hxge_txdma_hw_start: no dma channel allocated"));
2205 return (HXGE_ERROR);
2206 }
2207 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_start: "
2208 "tx_rings $%p tx_desc_rings $%p ndmas %d",
2209 tx_rings, tx_desc_rings, ndmas));
2210
2211 tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2212 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2213
2214 /*
2215 * Init the DMAs.
2216 */
2217 for (i = 0; i < ndmas; i++) {
2218 channel = tx_desc_rings[i]->tdc;
2219 status = hxge_txdma_start_channel(hxgep, channel,
2220 (p_tx_ring_t)tx_desc_rings[i],
2221 (p_tx_mbox_t)tx_mbox_p[i]);
2222 if (status != HXGE_OK) {
2223 goto hxge_txdma_hw_start_fail1;
2224 }
2225 }
2226
2227 (void) hxge_tx_vmac_enable(hxgep);
2228
2229 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2230 "==> hxge_txdma_hw_start: tx_rings $%p rings $%p",
2231 hxgep->tx_rings, hxgep->tx_rings->rings));
2232 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2233 "==> hxge_txdma_hw_start: tx_rings $%p tx_desc_rings $%p",
2234 hxgep->tx_rings, tx_desc_rings));
2235
2236 goto hxge_txdma_hw_start_exit;
2237
2238 hxge_txdma_hw_start_fail1:
2239 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2240 "==> hxge_txdma_hw_start: disable (status 0x%x channel %d i %d)",
2241 status, channel, i));
2242
2243 for (; i >= 0; i--) {
2244 channel = tx_desc_rings[i]->tdc,
2245 (void) hxge_txdma_stop_channel(hxgep, channel,
2246 (p_tx_ring_t)tx_desc_rings[i],
2247 (p_tx_mbox_t)tx_mbox_p[i]);
2248 }
2249
2250 hxge_txdma_hw_start_exit:
2251 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2252 "==> hxge_txdma_hw_start: (status 0x%x)", status));
2253
2254 return (status);
2255 }
2256
2257 static void
hxge_txdma_hw_stop(p_hxge_t hxgep)2258 hxge_txdma_hw_stop(p_hxge_t hxgep)
2259 {
2260 int i, ndmas;
2261 uint16_t channel;
2262 p_tx_rings_t tx_rings;
2263 p_tx_ring_t *tx_desc_rings;
2264 p_tx_mbox_areas_t tx_mbox_areas_p;
2265 p_tx_mbox_t *tx_mbox_p;
2266
2267 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop"));
2268
2269 tx_rings = hxgep->tx_rings;
2270 if (tx_rings == NULL) {
2271 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2272 "<== hxge_txdma_hw_stop: NULL ring pointer"));
2273 return;
2274 }
2275
2276 tx_desc_rings = tx_rings->rings;
2277 if (tx_desc_rings == NULL) {
2278 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2279 "<== hxge_txdma_hw_stop: NULL ring pointers"));
2280 return;
2281 }
2282
2283 ndmas = tx_rings->ndmas;
2284 if (!ndmas) {
2285 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2286 "<== hxge_txdma_hw_stop: no dma channel allocated"));
2287 return;
2288 }
2289
2290 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop: "
2291 "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
2292
2293 tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2294 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2295
2296 for (i = 0; i < ndmas; i++) {
2297 channel = tx_desc_rings[i]->tdc;
2298 (void) hxge_txdma_stop_channel(hxgep, channel,
2299 (p_tx_ring_t)tx_desc_rings[i],
2300 (p_tx_mbox_t)tx_mbox_p[i]);
2301 }
2302
2303 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop: "
2304 "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
2305 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_hw_stop"));
2306 }
2307
2308 static hxge_status_t
hxge_txdma_start_channel(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)2309 hxge_txdma_start_channel(p_hxge_t hxgep, uint16_t channel,
2310 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2311 {
2312 hxge_status_t status = HXGE_OK;
2313
2314 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2315 "==> hxge_txdma_start_channel (channel %d)", channel));
2316 /*
2317 * TXDMA/TXC must be in stopped state.
2318 */
2319 (void) hxge_txdma_stop_inj_err(hxgep, channel);
2320
2321 /*
2322 * Reset TXDMA channel
2323 */
2324 tx_ring_p->tx_cs.value = 0;
2325 tx_ring_p->tx_cs.bits.reset = 1;
2326 status = hxge_reset_txdma_channel(hxgep, channel,
2327 tx_ring_p->tx_cs.value);
2328 if (status != HXGE_OK) {
2329 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2330 "==> hxge_txdma_start_channel (channel %d)"
2331 " reset channel failed 0x%x", channel, status));
2332
2333 goto hxge_txdma_start_channel_exit;
2334 }
2335
2336 /*
2337 * Initialize the TXDMA channel specific FZC control configurations.
2338 * These FZC registers are pertaining to each TX channel (i.e. logical
2339 * pages).
2340 */
2341 status = hxge_init_fzc_txdma_channel(hxgep, channel,
2342 tx_ring_p, tx_mbox_p);
2343 if (status != HXGE_OK) {
2344 goto hxge_txdma_start_channel_exit;
2345 }
2346
2347 /*
2348 * Initialize the event masks.
2349 */
2350 tx_ring_p->tx_evmask.value = 0;
2351 status = hxge_init_txdma_channel_event_mask(hxgep,
2352 channel, &tx_ring_p->tx_evmask);
2353 if (status != HXGE_OK) {
2354 goto hxge_txdma_start_channel_exit;
2355 }
2356
2357 /*
2358 * Load TXDMA descriptors, buffers, mailbox, initialise the DMA
2359 * channels and enable each DMA channel.
2360 */
2361 status = hxge_enable_txdma_channel(hxgep, channel,
2362 tx_ring_p, tx_mbox_p);
2363 if (status != HXGE_OK) {
2364 goto hxge_txdma_start_channel_exit;
2365 }
2366
2367 hxge_txdma_start_channel_exit:
2368 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_start_channel"));
2369
2370 return (status);
2371 }
2372
2373 /*ARGSUSED*/
2374 static hxge_status_t
hxge_txdma_stop_channel(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)2375 hxge_txdma_stop_channel(p_hxge_t hxgep, uint16_t channel,
2376 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2377 {
2378 int status = HXGE_OK;
2379
2380 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2381 "==> hxge_txdma_stop_channel: channel %d", channel));
2382
2383 /*
2384 * Stop (disable) TXDMA and TXC (if stop bit is set and STOP_N_GO bit
2385 * not set, the TXDMA reset state will not be set if reset TXDMA.
2386 */
2387 (void) hxge_txdma_stop_inj_err(hxgep, channel);
2388
2389 /*
2390 * Reset TXDMA channel
2391 */
2392 tx_ring_p->tx_cs.value = 0;
2393 tx_ring_p->tx_cs.bits.reset = 1;
2394 status = hxge_reset_txdma_channel(hxgep, channel,
2395 tx_ring_p->tx_cs.value);
2396 if (status != HXGE_OK) {
2397 goto hxge_txdma_stop_channel_exit;
2398 }
2399
2400 hxge_txdma_stop_channel_exit:
2401 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_stop_channel"));
2402
2403 return (status);
2404 }
2405
2406 static p_tx_ring_t
hxge_txdma_get_ring(p_hxge_t hxgep,uint16_t channel)2407 hxge_txdma_get_ring(p_hxge_t hxgep, uint16_t channel)
2408 {
2409 int index, ndmas;
2410 uint16_t tdc;
2411 p_tx_rings_t tx_rings;
2412
2413 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_get_ring"));
2414
2415 tx_rings = hxgep->tx_rings;
2416 if (tx_rings == NULL) {
2417 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2418 "<== hxge_txdma_get_ring: NULL ring pointer"));
2419 return (NULL);
2420 }
2421 ndmas = tx_rings->ndmas;
2422 if (!ndmas) {
2423 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2424 "<== hxge_txdma_get_ring: no channel allocated"));
2425 return (NULL);
2426 }
2427 if (tx_rings->rings == NULL) {
2428 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2429 "<== hxge_txdma_get_ring: NULL rings pointer"));
2430 return (NULL);
2431 }
2432 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_get_ring: "
2433 "tx_rings $%p tx_desc_rings $%p ndmas %d",
2434 tx_rings, tx_rings, ndmas));
2435
2436 for (index = 0; index < ndmas; index++) {
2437 tdc = tx_rings->rings[index]->tdc;
2438 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2439 "==> hxge_fixup_txdma_rings: channel %d", tdc));
2440 if (channel == tdc) {
2441 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2442 "<== hxge_txdma_get_ring: tdc %d ring $%p",
2443 tdc, tx_rings->rings[index]));
2444 return (p_tx_ring_t)(tx_rings->rings[index]);
2445 }
2446 }
2447
2448 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_get_ring"));
2449
2450 return (NULL);
2451 }
2452
2453 static p_tx_mbox_t
hxge_txdma_get_mbox(p_hxge_t hxgep,uint16_t channel)2454 hxge_txdma_get_mbox(p_hxge_t hxgep, uint16_t channel)
2455 {
2456 int index, tdc, ndmas;
2457 p_tx_rings_t tx_rings;
2458 p_tx_mbox_areas_t tx_mbox_areas_p;
2459 p_tx_mbox_t *tx_mbox_p;
2460
2461 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_get_mbox"));
2462
2463 tx_rings = hxgep->tx_rings;
2464 if (tx_rings == NULL) {
2465 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2466 "<== hxge_txdma_get_mbox: NULL ring pointer"));
2467 return (NULL);
2468 }
2469 tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2470 if (tx_mbox_areas_p == NULL) {
2471 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2472 "<== hxge_txdma_get_mbox: NULL mbox pointer"));
2473 return (NULL);
2474 }
2475 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2476
2477 ndmas = tx_rings->ndmas;
2478 if (!ndmas) {
2479 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2480 "<== hxge_txdma_get_mbox: no channel allocated"));
2481 return (NULL);
2482 }
2483 if (tx_rings->rings == NULL) {
2484 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2485 "<== hxge_txdma_get_mbox: NULL rings pointer"));
2486 return (NULL);
2487 }
2488 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_get_mbox: "
2489 "tx_rings $%p tx_desc_rings $%p ndmas %d",
2490 tx_rings, tx_rings, ndmas));
2491
2492 for (index = 0; index < ndmas; index++) {
2493 tdc = tx_rings->rings[index]->tdc;
2494 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2495 "==> hxge_txdma_get_mbox: channel %d", tdc));
2496 if (channel == tdc) {
2497 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2498 "<== hxge_txdma_get_mbox: tdc %d ring $%p",
2499 tdc, tx_rings->rings[index]));
2500 return (p_tx_mbox_t)(tx_mbox_p[index]);
2501 }
2502 }
2503
2504 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_get_mbox"));
2505
2506 return (NULL);
2507 }
2508
2509 /*ARGSUSED*/
2510 static hxge_status_t
hxge_tx_err_evnts(p_hxge_t hxgep,uint_t index,p_hxge_ldv_t ldvp,tdc_stat_t cs)2511 hxge_tx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp,
2512 tdc_stat_t cs)
2513 {
2514 hpi_handle_t handle;
2515 uint8_t channel;
2516 p_tx_ring_t *tx_rings;
2517 p_tx_ring_t tx_ring_p;
2518 p_hxge_tx_ring_stats_t tdc_stats;
2519 boolean_t txchan_fatal = B_FALSE;
2520 hxge_status_t status = HXGE_OK;
2521 tdc_drop_cnt_t drop_cnt;
2522
2523 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_tx_err_evnts"));
2524 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2525 channel = ldvp->channel;
2526
2527 tx_rings = hxgep->tx_rings->rings;
2528 tx_ring_p = tx_rings[index];
2529 tdc_stats = tx_ring_p->tdc_stats;
2530
2531 /* Get the error counts if any */
2532 TXDMA_REG_READ64(handle, TDC_DROP_CNT, channel, &drop_cnt.value);
2533 tdc_stats->count_hdr_size_err += drop_cnt.bits.hdr_size_error_count;
2534 tdc_stats->count_runt += drop_cnt.bits.runt_count;
2535 tdc_stats->count_abort += drop_cnt.bits.abort_count;
2536
2537 if (cs.bits.peu_resp_err) {
2538 tdc_stats->peu_resp_err++;
2539 HXGE_FM_REPORT_ERROR(hxgep, channel,
2540 HXGE_FM_EREPORT_TDMC_PEU_RESP_ERR);
2541 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2542 "==> hxge_tx_err_evnts(channel %d): "
2543 "fatal error: peu_resp_err", channel));
2544 txchan_fatal = B_TRUE;
2545 }
2546
2547 if (cs.bits.pkt_size_hdr_err) {
2548 tdc_stats->pkt_size_hdr_err++;
2549 HXGE_FM_REPORT_ERROR(hxgep, channel,
2550 HXGE_FM_EREPORT_TDMC_PKT_SIZE_HDR_ERR);
2551 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2552 "==> hxge_tx_err_evnts(channel %d): "
2553 "fatal error: pkt_size_hdr_err", channel));
2554 txchan_fatal = B_TRUE;
2555 }
2556
2557 if (cs.bits.runt_pkt_drop_err) {
2558 tdc_stats->runt_pkt_drop_err++;
2559 HXGE_FM_REPORT_ERROR(hxgep, channel,
2560 HXGE_FM_EREPORT_TDMC_RUNT_PKT_DROP_ERR);
2561 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2562 "==> hxge_tx_err_evnts(channel %d): "
2563 "fatal error: runt_pkt_drop_err", channel));
2564 txchan_fatal = B_TRUE;
2565 }
2566
2567 if (cs.bits.pkt_size_err) {
2568 tdc_stats->pkt_size_err++;
2569 HXGE_FM_REPORT_ERROR(hxgep, channel,
2570 HXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR);
2571 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2572 "==> hxge_tx_err_evnts(channel %d): "
2573 "fatal error: pkt_size_err", channel));
2574 txchan_fatal = B_TRUE;
2575 }
2576
2577 if (cs.bits.tx_rng_oflow) {
2578 tdc_stats->tx_rng_oflow++;
2579 if (tdc_stats->tx_rng_oflow)
2580 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2581 "==> hxge_tx_err_evnts(channel %d): "
2582 "fatal error: tx_rng_oflow", channel));
2583 }
2584
2585 if (cs.bits.pref_par_err) {
2586 tdc_stats->pref_par_err++;
2587
2588 /* Get the address of parity error read data */
2589 TXDMA_REG_READ64(hxgep->hpi_handle, TDC_PREF_PAR_LOG,
2590 channel, &tdc_stats->errlog.value);
2591
2592 HXGE_FM_REPORT_ERROR(hxgep, channel,
2593 HXGE_FM_EREPORT_TDMC_PREF_PAR_ERR);
2594 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2595 "==> hxge_tx_err_evnts(channel %d): "
2596 "fatal error: pref_par_err", channel));
2597 txchan_fatal = B_TRUE;
2598 }
2599
2600 if (cs.bits.tdr_pref_cpl_to) {
2601 tdc_stats->tdr_pref_cpl_to++;
2602 HXGE_FM_REPORT_ERROR(hxgep, channel,
2603 HXGE_FM_EREPORT_TDMC_TDR_PREF_CPL_TO);
2604 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2605 "==> hxge_tx_err_evnts(channel %d): "
2606 "fatal error: tdr_pref_cpl_to", channel));
2607 txchan_fatal = B_TRUE;
2608 }
2609
2610 if (cs.bits.pkt_cpl_to) {
2611 tdc_stats->pkt_cpl_to++;
2612 HXGE_FM_REPORT_ERROR(hxgep, channel,
2613 HXGE_FM_EREPORT_TDMC_PKT_CPL_TO);
2614 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2615 "==> hxge_tx_err_evnts(channel %d): "
2616 "fatal error: pkt_cpl_to", channel));
2617 txchan_fatal = B_TRUE;
2618 }
2619
2620 if (cs.bits.invalid_sop) {
2621 tdc_stats->invalid_sop++;
2622 HXGE_FM_REPORT_ERROR(hxgep, channel,
2623 HXGE_FM_EREPORT_TDMC_INVALID_SOP);
2624 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2625 "==> hxge_tx_err_evnts(channel %d): "
2626 "fatal error: invalid_sop", channel));
2627 txchan_fatal = B_TRUE;
2628 }
2629
2630 if (cs.bits.unexpected_sop) {
2631 tdc_stats->unexpected_sop++;
2632 HXGE_FM_REPORT_ERROR(hxgep, channel,
2633 HXGE_FM_EREPORT_TDMC_UNEXPECTED_SOP);
2634 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2635 "==> hxge_tx_err_evnts(channel %d): "
2636 "fatal error: unexpected_sop", channel));
2637 txchan_fatal = B_TRUE;
2638 }
2639
2640 /* Clear error injection source in case this is an injected error */
2641 TXDMA_REG_WRITE64(hxgep->hpi_handle, TDC_STAT_INT_DBG, channel, 0);
2642
2643 if (txchan_fatal) {
2644 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2645 " hxge_tx_err_evnts: "
2646 " fatal error on channel %d cs 0x%llx\n",
2647 channel, cs.value));
2648 status = hxge_txdma_fatal_err_recover(hxgep, channel,
2649 tx_ring_p);
2650 if (status == HXGE_OK) {
2651 FM_SERVICE_RESTORED(hxgep);
2652 }
2653 }
2654
2655 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "<== hxge_tx_err_evnts"));
2656
2657 return (status);
2658 }
2659
2660 hxge_status_t
hxge_txdma_handle_sys_errors(p_hxge_t hxgep)2661 hxge_txdma_handle_sys_errors(p_hxge_t hxgep)
2662 {
2663 hpi_handle_t handle;
2664 hxge_status_t status = HXGE_OK;
2665 tdc_fifo_err_stat_t fifo_stat;
2666 hxge_tdc_sys_stats_t *tdc_sys_stats;
2667
2668 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_handle_sys_errors"));
2669
2670 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2671
2672 /*
2673 * The FIFO is shared by all channels.
2674 * Get the status of Reorder Buffer and Reorder Table Buffer Errors
2675 */
2676 HXGE_REG_RD64(handle, TDC_FIFO_ERR_STAT, &fifo_stat.value);
2677
2678 /*
2679 * Clear the error bits. Note that writing a 1 clears the bit. Writing
2680 * a 0 does nothing.
2681 */
2682 HXGE_REG_WR64(handle, TDC_FIFO_ERR_STAT, fifo_stat.value);
2683
2684 tdc_sys_stats = &hxgep->statsp->tdc_sys_stats;
2685 if (fifo_stat.bits.reord_tbl_par_err) {
2686 tdc_sys_stats->reord_tbl_par_err++;
2687 HXGE_FM_REPORT_ERROR(hxgep, 0,
2688 HXGE_FM_EREPORT_TDMC_REORD_TBL_PAR);
2689 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2690 "==> hxge_txdma_handle_sys_errors: fatal error: "
2691 "reord_tbl_par_err"));
2692 }
2693
2694 if (fifo_stat.bits.reord_buf_ded_err) {
2695 tdc_sys_stats->reord_buf_ded_err++;
2696 HXGE_FM_REPORT_ERROR(hxgep, 0,
2697 HXGE_FM_EREPORT_TDMC_REORD_BUF_DED);
2698 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2699 "==> hxge_txdma_handle_sys_errors: "
2700 "fatal error: reord_buf_ded_err"));
2701 }
2702
2703 if (fifo_stat.bits.reord_buf_sec_err) {
2704 tdc_sys_stats->reord_buf_sec_err++;
2705 if (tdc_sys_stats->reord_buf_sec_err == 1)
2706 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2707 "==> hxge_txdma_handle_sys_errors: "
2708 "reord_buf_sec_err"));
2709 }
2710
2711 if (fifo_stat.bits.reord_tbl_par_err ||
2712 fifo_stat.bits.reord_buf_ded_err) {
2713 status = hxge_tx_port_fatal_err_recover(hxgep);
2714 if (status == HXGE_OK) {
2715 FM_SERVICE_RESTORED(hxgep);
2716 }
2717 }
2718
2719 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_handle_sys_errors"));
2720
2721 return (status);
2722 }
2723
2724 static hxge_status_t
hxge_txdma_fatal_err_recover(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_ring_p)2725 hxge_txdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel,
2726 p_tx_ring_t tx_ring_p)
2727 {
2728 hpi_handle_t handle;
2729 hpi_status_t rs = HPI_SUCCESS;
2730 p_tx_mbox_t tx_mbox_p;
2731 hxge_status_t status = HXGE_OK;
2732
2733 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_txdma_fatal_err_recover"));
2734 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2735 "Recovering from TxDMAChannel#%d error...", channel));
2736
2737 /*
2738 * Stop the dma channel waits for the stop done. If the stop done bit
2739 * is not set, then create an error.
2740 */
2741 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2742 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "stopping txdma channel(%d)",
2743 channel));
2744 MUTEX_ENTER(&tx_ring_p->lock);
2745 rs = hpi_txdma_channel_control(handle, TXDMA_STOP, channel);
2746 if (rs != HPI_SUCCESS) {
2747 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2748 "==> hxge_txdma_fatal_err_recover (channel %d): "
2749 "stop failed ", channel));
2750
2751 goto fail;
2752 }
2753 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "reclaiming txdma channel(%d)",
2754 channel));
2755 (void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
2756
2757 /*
2758 * Reset TXDMA channel
2759 */
2760 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "resetting txdma channel(%d)",
2761 channel));
2762 if ((rs = hpi_txdma_channel_control(handle, TXDMA_RESET, channel)) !=
2763 HPI_SUCCESS) {
2764 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2765 "==> hxge_txdma_fatal_err_recover (channel %d)"
2766 " reset channel failed 0x%x", channel, rs));
2767
2768 goto fail;
2769 }
2770 /*
2771 * Reset the tail (kick) register to 0. (Hardware will not reset it. Tx
2772 * overflow fatal error if tail is not set to 0 after reset!
2773 */
2774 TXDMA_REG_WRITE64(handle, TDC_TDR_KICK, channel, 0);
2775
2776 /*
2777 * Restart TXDMA channel
2778 *
2779 * Initialize the TXDMA channel specific FZC control configurations.
2780 * These FZC registers are pertaining to each TX channel (i.e. logical
2781 * pages).
2782 */
2783 tx_mbox_p = hxge_txdma_get_mbox(hxgep, channel);
2784 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "restarting txdma channel(%d)",
2785 channel));
2786 status = hxge_init_fzc_txdma_channel(hxgep, channel,
2787 tx_ring_p, tx_mbox_p);
2788 if (status != HXGE_OK)
2789 goto fail;
2790
2791 /*
2792 * Initialize the event masks.
2793 */
2794 tx_ring_p->tx_evmask.value = 0;
2795 status = hxge_init_txdma_channel_event_mask(hxgep, channel,
2796 &tx_ring_p->tx_evmask);
2797 if (status != HXGE_OK)
2798 goto fail;
2799
2800 tx_ring_p->wr_index_wrap = B_FALSE;
2801 tx_ring_p->wr_index = 0;
2802 tx_ring_p->rd_index = 0;
2803
2804 /*
2805 * Load TXDMA descriptors, buffers, mailbox, initialise the DMA
2806 * channels and enable each DMA channel.
2807 */
2808 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "enabling txdma channel(%d)",
2809 channel));
2810 status = hxge_enable_txdma_channel(hxgep, channel,
2811 tx_ring_p, tx_mbox_p);
2812 MUTEX_EXIT(&tx_ring_p->lock);
2813 if (status != HXGE_OK)
2814 goto fail;
2815
2816 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2817 "Recovery Successful, TxDMAChannel#%d Restored", channel));
2818 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_txdma_fatal_err_recover"));
2819
2820 return (HXGE_OK);
2821
2822 fail:
2823 MUTEX_EXIT(&tx_ring_p->lock);
2824 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2825 "hxge_txdma_fatal_err_recover (channel %d): "
2826 "failed to recover this txdma channel", channel));
2827 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed"));
2828
2829 return (status);
2830 }
2831
2832 static hxge_status_t
hxge_tx_port_fatal_err_recover(p_hxge_t hxgep)2833 hxge_tx_port_fatal_err_recover(p_hxge_t hxgep)
2834 {
2835 hpi_handle_t handle;
2836 hpi_status_t rs = HPI_SUCCESS;
2837 hxge_status_t status = HXGE_OK;
2838 p_tx_ring_t *tx_desc_rings;
2839 p_tx_rings_t tx_rings;
2840 p_tx_ring_t tx_ring_p;
2841 int i, ndmas;
2842 uint16_t channel;
2843 block_reset_t reset_reg;
2844
2845 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2846 "==> hxge_tx_port_fatal_err_recover"));
2847 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2848 "Recovering from TxPort error..."));
2849
2850 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2851
2852 /* Reset TDC block from PEU for this fatal error */
2853 reset_reg.value = 0;
2854 reset_reg.bits.tdc_rst = 1;
2855 HXGE_REG_WR32(handle, BLOCK_RESET, reset_reg.value);
2856
2857 HXGE_DELAY(1000);
2858
2859 /*
2860 * Stop the dma channel waits for the stop done. If the stop done bit
2861 * is not set, then create an error.
2862 */
2863 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "stopping all DMA channels..."));
2864
2865 tx_rings = hxgep->tx_rings;
2866 tx_desc_rings = tx_rings->rings;
2867 ndmas = tx_rings->ndmas;
2868
2869 for (i = 0; i < ndmas; i++) {
2870 if (tx_desc_rings[i] == NULL) {
2871 continue;
2872 }
2873 tx_ring_p = tx_rings->rings[i];
2874 MUTEX_ENTER(&tx_ring_p->lock);
2875 }
2876
2877 for (i = 0; i < ndmas; i++) {
2878 if (tx_desc_rings[i] == NULL) {
2879 continue;
2880 }
2881 channel = tx_desc_rings[i]->tdc;
2882 tx_ring_p = tx_rings->rings[i];
2883 rs = hpi_txdma_channel_control(handle, TXDMA_STOP, channel);
2884 if (rs != HPI_SUCCESS) {
2885 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2886 "==> hxge_txdma_fatal_err_recover (channel %d): "
2887 "stop failed ", channel));
2888
2889 goto fail;
2890 }
2891 }
2892
2893 /*
2894 * Do reclaim on all of th DMAs.
2895 */
2896 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "reclaiming all DMA channels..."));
2897 for (i = 0; i < ndmas; i++) {
2898 if (tx_desc_rings[i] == NULL) {
2899 continue;
2900 }
2901 tx_ring_p = tx_rings->rings[i];
2902 (void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
2903 }
2904
2905 /* Restart the TDC */
2906 if ((status = hxge_txdma_hw_start(hxgep)) != HXGE_OK)
2907 goto fail;
2908
2909 for (i = 0; i < ndmas; i++) {
2910 if (tx_desc_rings[i] == NULL) {
2911 continue;
2912 }
2913 tx_ring_p = tx_rings->rings[i];
2914 MUTEX_EXIT(&tx_ring_p->lock);
2915 }
2916
2917 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2918 "Recovery Successful, TxPort Restored"));
2919 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2920 "<== hxge_tx_port_fatal_err_recover"));
2921 return (HXGE_OK);
2922
2923 fail:
2924 for (i = 0; i < ndmas; i++) {
2925 if (tx_desc_rings[i] == NULL) {
2926 continue;
2927 }
2928 tx_ring_p = tx_rings->rings[i];
2929 MUTEX_EXIT(&tx_ring_p->lock);
2930 }
2931
2932 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed"));
2933 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2934 "hxge_txdma_fatal_err_recover (channel %d): "
2935 "failed to recover this txdma channel"));
2936
2937 return (status);
2938 }
2939