1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/nxge/nxge_impl.h> 28 #include <sys/nxge/nxge_txdma.h> 29 #include <sys/nxge/nxge_hio.h> 30 #include <npi_tx_rd64.h> 31 #include <npi_tx_wr64.h> 32 #include <sys/llc1.h> 33 34 uint32_t nxge_reclaim_pending = TXDMA_RECLAIM_PENDING_DEFAULT; 35 uint32_t nxge_tx_minfree = 64; 36 uint32_t nxge_tx_intr_thres = 0; 37 uint32_t nxge_tx_max_gathers = TX_MAX_GATHER_POINTERS; 38 uint32_t nxge_tx_tiny_pack = 1; 39 uint32_t nxge_tx_use_bcopy = 1; 40 41 extern uint32_t nxge_tx_ring_size; 42 extern uint32_t nxge_bcopy_thresh; 43 extern uint32_t nxge_dvma_thresh; 44 extern uint32_t nxge_dma_stream_thresh; 45 extern dma_method_t nxge_force_dma; 46 extern uint32_t nxge_cksum_offload; 47 48 /* Device register access attributes for PIO. */ 49 extern ddi_device_acc_attr_t nxge_dev_reg_acc_attr; 50 /* Device descriptor access attributes for DMA. */ 51 extern ddi_device_acc_attr_t nxge_dev_desc_dma_acc_attr; 52 /* Device buffer access attributes for DMA. */ 53 extern ddi_device_acc_attr_t nxge_dev_buf_dma_acc_attr; 54 extern ddi_dma_attr_t nxge_desc_dma_attr; 55 extern ddi_dma_attr_t nxge_tx_dma_attr; 56 57 extern void nxge_tx_ring_task(void *arg); 58 59 static nxge_status_t nxge_map_txdma(p_nxge_t, int); 60 61 static nxge_status_t nxge_txdma_hw_start(p_nxge_t, int); 62 63 static nxge_status_t nxge_map_txdma_channel(p_nxge_t, uint16_t, 64 p_nxge_dma_common_t *, p_tx_ring_t *, 65 uint32_t, p_nxge_dma_common_t *, 66 p_tx_mbox_t *); 67 static void nxge_unmap_txdma_channel(p_nxge_t, uint16_t); 68 69 static nxge_status_t nxge_map_txdma_channel_buf_ring(p_nxge_t, uint16_t, 70 p_nxge_dma_common_t *, p_tx_ring_t *, uint32_t); 71 static void nxge_unmap_txdma_channel_buf_ring(p_nxge_t, p_tx_ring_t); 72 73 static void nxge_map_txdma_channel_cfg_ring(p_nxge_t, uint16_t, 74 p_nxge_dma_common_t *, p_tx_ring_t, 75 p_tx_mbox_t *); 76 static void nxge_unmap_txdma_channel_cfg_ring(p_nxge_t, 77 p_tx_ring_t, p_tx_mbox_t); 78 79 static nxge_status_t nxge_txdma_start_channel(p_nxge_t, uint16_t, 80 p_tx_ring_t, p_tx_mbox_t); 81 static nxge_status_t nxge_txdma_stop_channel(p_nxge_t, uint16_t); 82 83 static p_tx_ring_t nxge_txdma_get_ring(p_nxge_t, uint16_t); 84 static nxge_status_t nxge_tx_err_evnts(p_nxge_t, uint_t, 85 p_nxge_ldv_t, tx_cs_t); 86 static p_tx_mbox_t nxge_txdma_get_mbox(p_nxge_t, uint16_t); 87 static nxge_status_t nxge_txdma_fatal_err_recover(p_nxge_t, 88 uint16_t, p_tx_ring_t); 89 90 static void nxge_txdma_fixup_hung_channel(p_nxge_t nxgep, 91 p_tx_ring_t ring_p, uint16_t channel); 92 93 nxge_status_t 94 nxge_init_txdma_channels(p_nxge_t nxgep) 95 { 96 nxge_grp_set_t *set = &nxgep->tx_set; 97 int i, tdc, count; 98 nxge_grp_t *group; 99 dc_map_t map; 100 int dev_gindex; 101 102 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_init_txdma_channels")); 103 104 for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) { 105 if ((1 << i) & set->lg.map) { 106 group = set->group[i]; 107 dev_gindex = 108 nxgep->pt_config.hw_config.def_mac_txdma_grpid + i; 109 map = nxgep->pt_config.tdc_grps[dev_gindex].map; 110 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 111 if ((1 << tdc) & map) { 112 if ((nxge_grp_dc_add(nxgep, 113 group, VP_BOUND_TX, tdc))) 114 goto init_txdma_channels_exit; 115 } 116 } 117 } 118 if (++count == set->lg.count) 119 break; 120 } 121 122 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_txdma_channels")); 123 return (NXGE_OK); 124 125 init_txdma_channels_exit: 126 for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) { 127 if ((1 << i) & set->lg.map) { 128 group = set->group[i]; 129 dev_gindex = 130 nxgep->pt_config.hw_config.def_mac_txdma_grpid + i; 131 map = nxgep->pt_config.tdc_grps[dev_gindex].map; 132 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 133 if ((1 << tdc) & map) { 134 nxge_grp_dc_remove(nxgep, 135 VP_BOUND_TX, tdc); 136 } 137 } 138 } 139 if (++count == set->lg.count) 140 break; 141 } 142 143 return (NXGE_ERROR); 144 145 } 146 147 nxge_status_t 148 nxge_init_txdma_channel( 149 p_nxge_t nxge, 150 int channel) 151 { 152 nxge_status_t status; 153 154 NXGE_DEBUG_MSG((nxge, MEM2_CTL, "==> nxge_init_txdma_channel")); 155 156 status = nxge_map_txdma(nxge, channel); 157 if (status != NXGE_OK) { 158 NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, 159 "<== nxge_init_txdma_channel: status 0x%x", status)); 160 (void) npi_txdma_dump_tdc_regs(nxge->npi_handle, channel); 161 return (status); 162 } 163 164 status = nxge_txdma_hw_start(nxge, channel); 165 if (status != NXGE_OK) { 166 (void) nxge_unmap_txdma_channel(nxge, channel); 167 (void) npi_txdma_dump_tdc_regs(nxge->npi_handle, channel); 168 return (status); 169 } 170 171 if (!nxge->statsp->tdc_ksp[channel]) 172 nxge_setup_tdc_kstats(nxge, channel); 173 174 NXGE_DEBUG_MSG((nxge, MEM2_CTL, "<== nxge_init_txdma_channel")); 175 176 return (status); 177 } 178 179 void 180 nxge_uninit_txdma_channels(p_nxge_t nxgep) 181 { 182 nxge_grp_set_t *set = &nxgep->tx_set; 183 int tdc; 184 185 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_txdma_channels")); 186 187 if (set->owned.map == 0) { 188 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 189 "nxge_uninit_txdma_channels: no channels")); 190 return; 191 } 192 193 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 194 if ((1 << tdc) & set->owned.map) { 195 nxge_grp_dc_remove(nxgep, VP_BOUND_TX, tdc); 196 } 197 } 198 199 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uninit_txdma_channels")); 200 } 201 202 void 203 nxge_uninit_txdma_channel(p_nxge_t nxgep, int channel) 204 { 205 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_uninit_txdma_channel")); 206 207 if (nxgep->statsp->tdc_ksp[channel]) { 208 kstat_delete(nxgep->statsp->tdc_ksp[channel]); 209 nxgep->statsp->tdc_ksp[channel] = 0; 210 } 211 212 if (nxge_txdma_stop_channel(nxgep, channel) != NXGE_OK) 213 goto nxge_uninit_txdma_channel_exit; 214 215 nxge_unmap_txdma_channel(nxgep, channel); 216 217 nxge_uninit_txdma_channel_exit: 218 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_uninit_txdma_channel")); 219 } 220 221 void 222 nxge_setup_dma_common(p_nxge_dma_common_t dest_p, p_nxge_dma_common_t src_p, 223 uint32_t entries, uint32_t size) 224 { 225 size_t tsize; 226 *dest_p = *src_p; 227 tsize = size * entries; 228 dest_p->alength = tsize; 229 dest_p->nblocks = entries; 230 dest_p->block_size = size; 231 dest_p->offset += tsize; 232 233 src_p->kaddrp = (caddr_t)dest_p->kaddrp + tsize; 234 src_p->alength -= tsize; 235 src_p->dma_cookie.dmac_laddress += tsize; 236 src_p->dma_cookie.dmac_size -= tsize; 237 } 238 239 /* 240 * nxge_reset_txdma_channel 241 * 242 * Reset a TDC. 243 * 244 * Arguments: 245 * nxgep 246 * channel The channel to reset. 247 * reg_data The current TX_CS. 248 * 249 * Notes: 250 * 251 * NPI/NXGE function calls: 252 * npi_txdma_channel_reset() 253 * npi_txdma_channel_control() 254 * 255 * Registers accessed: 256 * TX_CS DMC+0x40028 Transmit Control And Status 257 * TX_RING_KICK DMC+0x40018 Transmit Ring Kick 258 * 259 * Context: 260 * Any domain 261 */ 262 nxge_status_t 263 nxge_reset_txdma_channel(p_nxge_t nxgep, uint16_t channel, uint64_t reg_data) 264 { 265 npi_status_t rs = NPI_SUCCESS; 266 nxge_status_t status = NXGE_OK; 267 npi_handle_t handle; 268 269 NXGE_DEBUG_MSG((nxgep, TX_CTL, " ==> nxge_reset_txdma_channel")); 270 271 handle = NXGE_DEV_NPI_HANDLE(nxgep); 272 if ((reg_data & TX_CS_RST_MASK) == TX_CS_RST_MASK) { 273 rs = npi_txdma_channel_reset(handle, channel); 274 } else { 275 rs = npi_txdma_channel_control(handle, TXDMA_RESET, 276 channel); 277 } 278 279 if (rs != NPI_SUCCESS) { 280 status = NXGE_ERROR | rs; 281 } 282 283 /* 284 * Reset the tail (kick) register to 0. 285 * (Hardware will not reset it. Tx overflow fatal 286 * error if tail is not set to 0 after reset! 287 */ 288 TXDMA_REG_WRITE64(handle, TX_RING_KICK_REG, channel, 0); 289 290 NXGE_DEBUG_MSG((nxgep, TX_CTL, " <== nxge_reset_txdma_channel")); 291 return (status); 292 } 293 294 /* 295 * nxge_init_txdma_channel_event_mask 296 * 297 * Enable interrupts for a set of events. 298 * 299 * Arguments: 300 * nxgep 301 * channel The channel to map. 302 * mask_p The events to enable. 303 * 304 * Notes: 305 * 306 * NPI/NXGE function calls: 307 * npi_txdma_event_mask() 308 * 309 * Registers accessed: 310 * TX_ENT_MSK DMC+0x40020 Transmit Event Mask 311 * 312 * Context: 313 * Any domain 314 */ 315 nxge_status_t 316 nxge_init_txdma_channel_event_mask(p_nxge_t nxgep, uint16_t channel, 317 p_tx_dma_ent_msk_t mask_p) 318 { 319 npi_handle_t handle; 320 npi_status_t rs = NPI_SUCCESS; 321 nxge_status_t status = NXGE_OK; 322 323 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 324 "<== nxge_init_txdma_channel_event_mask")); 325 326 handle = NXGE_DEV_NPI_HANDLE(nxgep); 327 rs = npi_txdma_event_mask(handle, OP_SET, channel, mask_p); 328 if (rs != NPI_SUCCESS) { 329 status = NXGE_ERROR | rs; 330 } 331 332 return (status); 333 } 334 335 /* 336 * nxge_init_txdma_channel_cntl_stat 337 * 338 * Stop a TDC. If at first we don't succeed, inject an error. 339 * 340 * Arguments: 341 * nxgep 342 * channel The channel to stop. 343 * 344 * Notes: 345 * 346 * NPI/NXGE function calls: 347 * npi_txdma_control_status() 348 * 349 * Registers accessed: 350 * TX_CS DMC+0x40028 Transmit Control And Status 351 * 352 * Context: 353 * Any domain 354 */ 355 nxge_status_t 356 nxge_init_txdma_channel_cntl_stat(p_nxge_t nxgep, uint16_t channel, 357 uint64_t reg_data) 358 { 359 npi_handle_t handle; 360 npi_status_t rs = NPI_SUCCESS; 361 nxge_status_t status = NXGE_OK; 362 363 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 364 "<== nxge_init_txdma_channel_cntl_stat")); 365 366 handle = NXGE_DEV_NPI_HANDLE(nxgep); 367 rs = npi_txdma_control_status(handle, OP_SET, channel, 368 (p_tx_cs_t)®_data); 369 370 if (rs != NPI_SUCCESS) { 371 status = NXGE_ERROR | rs; 372 } 373 374 return (status); 375 } 376 377 /* 378 * nxge_enable_txdma_channel 379 * 380 * Enable a TDC. 381 * 382 * Arguments: 383 * nxgep 384 * channel The channel to enable. 385 * tx_desc_p channel's transmit descriptor ring. 386 * mbox_p channel's mailbox, 387 * 388 * Notes: 389 * 390 * NPI/NXGE function calls: 391 * npi_txdma_ring_config() 392 * npi_txdma_mbox_config() 393 * npi_txdma_channel_init_enable() 394 * 395 * Registers accessed: 396 * TX_RNG_CFIG DMC+0x40000 Transmit Ring Configuration 397 * TXDMA_MBH DMC+0x40030 TXDMA Mailbox High 398 * TXDMA_MBL DMC+0x40038 TXDMA Mailbox Low 399 * TX_CS DMC+0x40028 Transmit Control And Status 400 * 401 * Context: 402 * Any domain 403 */ 404 nxge_status_t 405 nxge_enable_txdma_channel(p_nxge_t nxgep, 406 uint16_t channel, p_tx_ring_t tx_desc_p, p_tx_mbox_t mbox_p) 407 { 408 npi_handle_t handle; 409 npi_status_t rs = NPI_SUCCESS; 410 nxge_status_t status = NXGE_OK; 411 412 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_enable_txdma_channel")); 413 414 handle = NXGE_DEV_NPI_HANDLE(nxgep); 415 /* 416 * Use configuration data composed at init time. 417 * Write to hardware the transmit ring configurations. 418 */ 419 rs = npi_txdma_ring_config(handle, OP_SET, channel, 420 (uint64_t *)&(tx_desc_p->tx_ring_cfig.value)); 421 422 if (rs != NPI_SUCCESS) { 423 return (NXGE_ERROR | rs); 424 } 425 426 if (isLDOMguest(nxgep)) { 427 /* Add interrupt handler for this channel. */ 428 if (nxge_hio_intr_add(nxgep, VP_BOUND_TX, channel) != NXGE_OK) 429 return (NXGE_ERROR); 430 } 431 432 /* Write to hardware the mailbox */ 433 rs = npi_txdma_mbox_config(handle, OP_SET, channel, 434 (uint64_t *)&mbox_p->tx_mbox.dma_cookie.dmac_laddress); 435 436 if (rs != NPI_SUCCESS) { 437 return (NXGE_ERROR | rs); 438 } 439 440 /* Start the DMA engine. */ 441 rs = npi_txdma_channel_init_enable(handle, channel); 442 443 if (rs != NPI_SUCCESS) { 444 return (NXGE_ERROR | rs); 445 } 446 447 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_enable_txdma_channel")); 448 449 return (status); 450 } 451 452 void 453 nxge_fill_tx_hdr(p_mblk_t mp, boolean_t fill_len, 454 boolean_t l4_cksum, int pkt_len, uint8_t npads, 455 p_tx_pkt_hdr_all_t pkthdrp, 456 t_uscalar_t start_offset, 457 t_uscalar_t stuff_offset) 458 { 459 p_tx_pkt_header_t hdrp; 460 p_mblk_t nmp; 461 uint64_t tmp; 462 size_t mblk_len; 463 size_t iph_len; 464 size_t hdrs_size; 465 uint8_t hdrs_buf[sizeof (struct ether_header) + 466 64 + sizeof (uint32_t)]; 467 uint8_t *cursor; 468 uint8_t *ip_buf; 469 uint16_t eth_type; 470 uint8_t ipproto; 471 boolean_t is_vlan = B_FALSE; 472 size_t eth_hdr_size; 473 474 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: mp $%p", mp)); 475 476 /* 477 * Caller should zero out the headers first. 478 */ 479 hdrp = (p_tx_pkt_header_t)&pkthdrp->pkthdr; 480 481 if (fill_len) { 482 NXGE_DEBUG_MSG((NULL, TX_CTL, 483 "==> nxge_fill_tx_hdr: pkt_len %d " 484 "npads %d", pkt_len, npads)); 485 tmp = (uint64_t)pkt_len; 486 hdrp->value |= (tmp << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 487 goto fill_tx_header_done; 488 } 489 490 hdrp->value |= (((uint64_t)npads) << TX_PKT_HEADER_PAD_SHIFT); 491 492 /* 493 * mp is the original data packet (does not include the 494 * Neptune transmit header). 495 */ 496 nmp = mp; 497 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: " 498 "mp $%p b_rptr $%p len %d", 499 mp, nmp->b_rptr, MBLKL(nmp))); 500 /* copy ether_header from mblk to hdrs_buf */ 501 cursor = &hdrs_buf[0]; 502 tmp = sizeof (struct ether_vlan_header); 503 while ((nmp != NULL) && (tmp > 0)) { 504 size_t buflen; 505 mblk_len = MBLKL(nmp); 506 buflen = min((size_t)tmp, mblk_len); 507 bcopy(nmp->b_rptr, cursor, buflen); 508 cursor += buflen; 509 tmp -= buflen; 510 nmp = nmp->b_cont; 511 } 512 513 nmp = mp; 514 mblk_len = MBLKL(nmp); 515 ip_buf = NULL; 516 eth_type = ntohs(((p_ether_header_t)hdrs_buf)->ether_type); 517 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> : nxge_fill_tx_hdr: (value 0x%llx) " 518 "ether type 0x%x", eth_type, hdrp->value)); 519 520 if (eth_type < ETHERMTU) { 521 tmp = 1ull; 522 hdrp->value |= (tmp << TX_PKT_HEADER_LLC_SHIFT); 523 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: LLC " 524 "value 0x%llx", hdrp->value)); 525 if (*(hdrs_buf + sizeof (struct ether_header)) 526 == LLC_SNAP_SAP) { 527 eth_type = ntohs(*((uint16_t *)(hdrs_buf + 528 sizeof (struct ether_header) + 6))); 529 NXGE_DEBUG_MSG((NULL, TX_CTL, 530 "==> nxge_tx_pkt_hdr_init: LLC ether type 0x%x", 531 eth_type)); 532 } else { 533 goto fill_tx_header_done; 534 } 535 } else if (eth_type == VLAN_ETHERTYPE) { 536 tmp = 1ull; 537 hdrp->value |= (tmp << TX_PKT_HEADER_VLAN__SHIFT); 538 539 eth_type = ntohs(((struct ether_vlan_header *) 540 hdrs_buf)->ether_type); 541 is_vlan = B_TRUE; 542 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: VLAN " 543 "value 0x%llx", hdrp->value)); 544 } 545 546 if (!is_vlan) { 547 eth_hdr_size = sizeof (struct ether_header); 548 } else { 549 eth_hdr_size = sizeof (struct ether_vlan_header); 550 } 551 552 switch (eth_type) { 553 case ETHERTYPE_IP: 554 if (mblk_len > eth_hdr_size + sizeof (uint8_t)) { 555 ip_buf = nmp->b_rptr + eth_hdr_size; 556 mblk_len -= eth_hdr_size; 557 iph_len = ((*ip_buf) & 0x0f); 558 if (mblk_len > (iph_len + sizeof (uint32_t))) { 559 ip_buf = nmp->b_rptr; 560 ip_buf += eth_hdr_size; 561 } else { 562 ip_buf = NULL; 563 } 564 565 } 566 if (ip_buf == NULL) { 567 hdrs_size = 0; 568 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 569 while ((nmp) && (hdrs_size < 570 sizeof (hdrs_buf))) { 571 mblk_len = (size_t)nmp->b_wptr - 572 (size_t)nmp->b_rptr; 573 if (mblk_len >= 574 (sizeof (hdrs_buf) - hdrs_size)) 575 mblk_len = sizeof (hdrs_buf) - 576 hdrs_size; 577 bcopy(nmp->b_rptr, 578 &hdrs_buf[hdrs_size], mblk_len); 579 hdrs_size += mblk_len; 580 nmp = nmp->b_cont; 581 } 582 ip_buf = hdrs_buf; 583 ip_buf += eth_hdr_size; 584 iph_len = ((*ip_buf) & 0x0f); 585 } 586 587 ipproto = ip_buf[9]; 588 589 tmp = (uint64_t)iph_len; 590 hdrp->value |= (tmp << TX_PKT_HEADER_IHL_SHIFT); 591 tmp = (uint64_t)(eth_hdr_size >> 1); 592 hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT); 593 594 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: IPv4 " 595 " iph_len %d l3start %d eth_hdr_size %d proto 0x%x" 596 "tmp 0x%x", 597 iph_len, hdrp->bits.hdw.l3start, eth_hdr_size, 598 ipproto, tmp)); 599 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: IP " 600 "value 0x%llx", hdrp->value)); 601 602 break; 603 604 case ETHERTYPE_IPV6: 605 hdrs_size = 0; 606 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 607 while ((nmp) && (hdrs_size < 608 sizeof (hdrs_buf))) { 609 mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr; 610 if (mblk_len >= 611 (sizeof (hdrs_buf) - hdrs_size)) 612 mblk_len = sizeof (hdrs_buf) - 613 hdrs_size; 614 bcopy(nmp->b_rptr, 615 &hdrs_buf[hdrs_size], mblk_len); 616 hdrs_size += mblk_len; 617 nmp = nmp->b_cont; 618 } 619 ip_buf = hdrs_buf; 620 ip_buf += eth_hdr_size; 621 622 tmp = 1ull; 623 hdrp->value |= (tmp << TX_PKT_HEADER_IP_VER_SHIFT); 624 625 tmp = (eth_hdr_size >> 1); 626 hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT); 627 628 /* byte 6 is the next header protocol */ 629 ipproto = ip_buf[6]; 630 631 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: IPv6 " 632 " iph_len %d l3start %d eth_hdr_size %d proto 0x%x", 633 iph_len, hdrp->bits.hdw.l3start, eth_hdr_size, 634 ipproto)); 635 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: IPv6 " 636 "value 0x%llx", hdrp->value)); 637 638 break; 639 640 default: 641 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: non-IP")); 642 goto fill_tx_header_done; 643 } 644 645 switch (ipproto) { 646 case IPPROTO_TCP: 647 NXGE_DEBUG_MSG((NULL, TX_CTL, 648 "==> nxge_fill_tx_hdr: TCP (cksum flag %d)", l4_cksum)); 649 if (l4_cksum) { 650 hdrp->value |= TX_CKSUM_EN_PKT_TYPE_TCP; 651 hdrp->value |= 652 (((uint64_t)(start_offset >> 1)) << 653 TX_PKT_HEADER_L4START_SHIFT); 654 hdrp->value |= 655 (((uint64_t)(stuff_offset >> 1)) << 656 TX_PKT_HEADER_L4STUFF_SHIFT); 657 658 NXGE_DEBUG_MSG((NULL, TX_CTL, 659 "==> nxge_tx_pkt_hdr_init: TCP CKSUM " 660 "value 0x%llx", hdrp->value)); 661 } 662 663 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: TCP " 664 "value 0x%llx", hdrp->value)); 665 break; 666 667 case IPPROTO_UDP: 668 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: UDP")); 669 if (l4_cksum) { 670 if (!nxge_cksum_offload) { 671 uint16_t *up; 672 uint16_t cksum; 673 t_uscalar_t stuff_len; 674 675 /* 676 * The checksum field has the 677 * partial checksum. 678 * IP_CSUM() macro calls ip_cksum() which 679 * can add in the partial checksum. 680 */ 681 cksum = IP_CSUM(mp, start_offset, 0); 682 stuff_len = stuff_offset; 683 nmp = mp; 684 mblk_len = MBLKL(nmp); 685 while ((nmp != NULL) && 686 (mblk_len < stuff_len)) { 687 stuff_len -= mblk_len; 688 nmp = nmp->b_cont; 689 if (nmp) 690 mblk_len = MBLKL(nmp); 691 } 692 ASSERT(nmp); 693 up = (uint16_t *)(nmp->b_rptr + stuff_len); 694 695 *up = cksum; 696 hdrp->value &= ~TX_CKSUM_EN_PKT_TYPE_UDP; 697 NXGE_DEBUG_MSG((NULL, TX_CTL, 698 "==> nxge_tx_pkt_hdr_init: UDP offset %d " 699 "use sw cksum " 700 "write to $%p cksum 0x%x content up 0x%x", 701 stuff_len, 702 up, 703 cksum, 704 *up)); 705 } else { 706 /* Hardware will compute the full checksum */ 707 hdrp->value |= TX_CKSUM_EN_PKT_TYPE_UDP; 708 hdrp->value |= 709 (((uint64_t)(start_offset >> 1)) << 710 TX_PKT_HEADER_L4START_SHIFT); 711 hdrp->value |= 712 (((uint64_t)(stuff_offset >> 1)) << 713 TX_PKT_HEADER_L4STUFF_SHIFT); 714 715 NXGE_DEBUG_MSG((NULL, TX_CTL, 716 "==> nxge_tx_pkt_hdr_init: UDP offset %d " 717 " use partial checksum " 718 "cksum 0x%x ", 719 "value 0x%llx", 720 stuff_offset, 721 IP_CSUM(mp, start_offset, 0), 722 hdrp->value)); 723 } 724 } 725 726 NXGE_DEBUG_MSG((NULL, TX_CTL, 727 "==> nxge_tx_pkt_hdr_init: UDP" 728 "value 0x%llx", hdrp->value)); 729 break; 730 731 default: 732 goto fill_tx_header_done; 733 } 734 735 fill_tx_header_done: 736 NXGE_DEBUG_MSG((NULL, TX_CTL, 737 "==> nxge_fill_tx_hdr: pkt_len %d " 738 "npads %d value 0x%llx", pkt_len, npads, hdrp->value)); 739 740 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_fill_tx_hdr")); 741 } 742 743 /*ARGSUSED*/ 744 p_mblk_t 745 nxge_tx_pkt_header_reserve(p_mblk_t mp, uint8_t *npads) 746 { 747 p_mblk_t newmp = NULL; 748 749 if ((newmp = allocb(TX_PKT_HEADER_SIZE, BPRI_MED)) == NULL) { 750 NXGE_DEBUG_MSG((NULL, TX_CTL, 751 "<== nxge_tx_pkt_header_reserve: allocb failed")); 752 return (NULL); 753 } 754 755 NXGE_DEBUG_MSG((NULL, TX_CTL, 756 "==> nxge_tx_pkt_header_reserve: get new mp")); 757 DB_TYPE(newmp) = M_DATA; 758 newmp->b_rptr = newmp->b_wptr = DB_LIM(newmp); 759 linkb(newmp, mp); 760 newmp->b_rptr -= TX_PKT_HEADER_SIZE; 761 762 NXGE_DEBUG_MSG((NULL, TX_CTL, "==>nxge_tx_pkt_header_reserve: " 763 "b_rptr $%p b_wptr $%p", 764 newmp->b_rptr, newmp->b_wptr)); 765 766 NXGE_DEBUG_MSG((NULL, TX_CTL, 767 "<== nxge_tx_pkt_header_reserve: use new mp")); 768 769 return (newmp); 770 } 771 772 int 773 nxge_tx_pkt_nmblocks(p_mblk_t mp, int *tot_xfer_len_p) 774 { 775 uint_t nmblks; 776 ssize_t len; 777 uint_t pkt_len; 778 p_mblk_t nmp, bmp, tmp; 779 uint8_t *b_wptr; 780 781 NXGE_DEBUG_MSG((NULL, TX_CTL, 782 "==> nxge_tx_pkt_nmblocks: mp $%p rptr $%p wptr $%p " 783 "len %d", mp, mp->b_rptr, mp->b_wptr, MBLKL(mp))); 784 785 nmp = mp; 786 bmp = mp; 787 nmblks = 0; 788 pkt_len = 0; 789 *tot_xfer_len_p = 0; 790 791 while (nmp) { 792 len = MBLKL(nmp); 793 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_nmblocks: " 794 "len %d pkt_len %d nmblks %d tot_xfer_len %d", 795 len, pkt_len, nmblks, 796 *tot_xfer_len_p)); 797 798 if (len <= 0) { 799 bmp = nmp; 800 nmp = nmp->b_cont; 801 NXGE_DEBUG_MSG((NULL, TX_CTL, 802 "==> nxge_tx_pkt_nmblocks: " 803 "len (0) pkt_len %d nmblks %d", 804 pkt_len, nmblks)); 805 continue; 806 } 807 808 *tot_xfer_len_p += len; 809 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_nmblocks: " 810 "len %d pkt_len %d nmblks %d tot_xfer_len %d", 811 len, pkt_len, nmblks, 812 *tot_xfer_len_p)); 813 814 if (len < nxge_bcopy_thresh) { 815 NXGE_DEBUG_MSG((NULL, TX_CTL, 816 "==> nxge_tx_pkt_nmblocks: " 817 "len %d (< thresh) pkt_len %d nmblks %d", 818 len, pkt_len, nmblks)); 819 if (pkt_len == 0) 820 nmblks++; 821 pkt_len += len; 822 if (pkt_len >= nxge_bcopy_thresh) { 823 pkt_len = 0; 824 len = 0; 825 nmp = bmp; 826 } 827 } else { 828 NXGE_DEBUG_MSG((NULL, TX_CTL, 829 "==> nxge_tx_pkt_nmblocks: " 830 "len %d (> thresh) pkt_len %d nmblks %d", 831 len, pkt_len, nmblks)); 832 pkt_len = 0; 833 nmblks++; 834 /* 835 * Hardware limits the transfer length to 4K. 836 * If len is more than 4K, we need to break 837 * it up to at most 2 more blocks. 838 */ 839 if (len > TX_MAX_TRANSFER_LENGTH) { 840 uint32_t nsegs; 841 842 nsegs = 1; 843 NXGE_DEBUG_MSG((NULL, TX_CTL, 844 "==> nxge_tx_pkt_nmblocks: " 845 "len %d pkt_len %d nmblks %d nsegs %d", 846 len, pkt_len, nmblks, nsegs)); 847 if (len % (TX_MAX_TRANSFER_LENGTH * 2)) { 848 ++nsegs; 849 } 850 do { 851 b_wptr = nmp->b_rptr + 852 TX_MAX_TRANSFER_LENGTH; 853 nmp->b_wptr = b_wptr; 854 if ((tmp = dupb(nmp)) == NULL) { 855 return (0); 856 } 857 tmp->b_rptr = b_wptr; 858 tmp->b_wptr = nmp->b_wptr; 859 tmp->b_cont = nmp->b_cont; 860 nmp->b_cont = tmp; 861 nmblks++; 862 if (--nsegs) { 863 nmp = tmp; 864 } 865 } while (nsegs); 866 nmp = tmp; 867 } 868 } 869 870 /* 871 * Hardware limits the transmit gather pointers to 15. 872 */ 873 if (nmp->b_cont && (nmblks + TX_GATHER_POINTERS_THRESHOLD) > 874 TX_MAX_GATHER_POINTERS) { 875 NXGE_DEBUG_MSG((NULL, TX_CTL, 876 "==> nxge_tx_pkt_nmblocks: pull msg - " 877 "len %d pkt_len %d nmblks %d", 878 len, pkt_len, nmblks)); 879 /* Pull all message blocks from b_cont */ 880 if ((tmp = msgpullup(nmp->b_cont, -1)) == NULL) { 881 return (0); 882 } 883 freemsg(nmp->b_cont); 884 nmp->b_cont = tmp; 885 pkt_len = 0; 886 } 887 bmp = nmp; 888 nmp = nmp->b_cont; 889 } 890 891 NXGE_DEBUG_MSG((NULL, TX_CTL, 892 "<== nxge_tx_pkt_nmblocks: rptr $%p wptr $%p " 893 "nmblks %d len %d tot_xfer_len %d", 894 mp->b_rptr, mp->b_wptr, nmblks, 895 MBLKL(mp), *tot_xfer_len_p)); 896 897 return (nmblks); 898 } 899 900 boolean_t 901 nxge_txdma_reclaim(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, int nmblks) 902 { 903 boolean_t status = B_TRUE; 904 p_nxge_dma_common_t tx_desc_dma_p; 905 nxge_dma_common_t desc_area; 906 p_tx_desc_t tx_desc_ring_vp; 907 p_tx_desc_t tx_desc_p; 908 p_tx_desc_t tx_desc_pp; 909 tx_desc_t r_tx_desc; 910 p_tx_msg_t tx_msg_ring; 911 p_tx_msg_t tx_msg_p; 912 npi_handle_t handle; 913 tx_ring_hdl_t tx_head; 914 uint32_t pkt_len; 915 uint_t tx_rd_index; 916 uint16_t head_index, tail_index; 917 uint8_t tdc; 918 boolean_t head_wrap, tail_wrap; 919 p_nxge_tx_ring_stats_t tdc_stats; 920 int rc; 921 922 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_reclaim")); 923 924 status = ((tx_ring_p->descs_pending < nxge_reclaim_pending) && 925 (nmblks != 0)); 926 NXGE_DEBUG_MSG((nxgep, TX_CTL, 927 "==> nxge_txdma_reclaim: pending %d reclaim %d nmblks %d", 928 tx_ring_p->descs_pending, nxge_reclaim_pending, 929 nmblks)); 930 if (!status) { 931 tx_desc_dma_p = &tx_ring_p->tdc_desc; 932 desc_area = tx_ring_p->tdc_desc; 933 handle = NXGE_DEV_NPI_HANDLE(nxgep); 934 tx_desc_ring_vp = tx_desc_dma_p->kaddrp; 935 tx_desc_ring_vp = 936 (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 937 tx_rd_index = tx_ring_p->rd_index; 938 tx_desc_p = &tx_desc_ring_vp[tx_rd_index]; 939 tx_msg_ring = tx_ring_p->tx_msg_ring; 940 tx_msg_p = &tx_msg_ring[tx_rd_index]; 941 tdc = tx_ring_p->tdc; 942 tdc_stats = tx_ring_p->tdc_stats; 943 if (tx_ring_p->descs_pending > tdc_stats->tx_max_pend) { 944 tdc_stats->tx_max_pend = tx_ring_p->descs_pending; 945 } 946 947 tail_index = tx_ring_p->wr_index; 948 tail_wrap = tx_ring_p->wr_index_wrap; 949 950 NXGE_DEBUG_MSG((nxgep, TX_CTL, 951 "==> nxge_txdma_reclaim: tdc %d tx_rd_index %d " 952 "tail_index %d tail_wrap %d " 953 "tx_desc_p $%p ($%p) ", 954 tdc, tx_rd_index, tail_index, tail_wrap, 955 tx_desc_p, (*(uint64_t *)tx_desc_p))); 956 /* 957 * Read the hardware maintained transmit head 958 * and wrap around bit. 959 */ 960 TXDMA_REG_READ64(handle, TX_RING_HDL_REG, tdc, &tx_head.value); 961 head_index = tx_head.bits.ldw.head; 962 head_wrap = tx_head.bits.ldw.wrap; 963 NXGE_DEBUG_MSG((nxgep, TX_CTL, 964 "==> nxge_txdma_reclaim: " 965 "tx_rd_index %d tail %d tail_wrap %d " 966 "head %d wrap %d", 967 tx_rd_index, tail_index, tail_wrap, 968 head_index, head_wrap)); 969 970 if (head_index == tail_index) { 971 if (TXDMA_RING_EMPTY(head_index, head_wrap, 972 tail_index, tail_wrap) && 973 (head_index == tx_rd_index)) { 974 NXGE_DEBUG_MSG((nxgep, TX_CTL, 975 "==> nxge_txdma_reclaim: EMPTY")); 976 return (B_TRUE); 977 } 978 979 NXGE_DEBUG_MSG((nxgep, TX_CTL, 980 "==> nxge_txdma_reclaim: Checking " 981 "if ring full")); 982 if (TXDMA_RING_FULL(head_index, head_wrap, tail_index, 983 tail_wrap)) { 984 NXGE_DEBUG_MSG((nxgep, TX_CTL, 985 "==> nxge_txdma_reclaim: full")); 986 return (B_FALSE); 987 } 988 } 989 990 NXGE_DEBUG_MSG((nxgep, TX_CTL, 991 "==> nxge_txdma_reclaim: tx_rd_index and head_index")); 992 993 tx_desc_pp = &r_tx_desc; 994 while ((tx_rd_index != head_index) && 995 (tx_ring_p->descs_pending != 0)) { 996 997 NXGE_DEBUG_MSG((nxgep, TX_CTL, 998 "==> nxge_txdma_reclaim: Checking if pending")); 999 1000 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1001 "==> nxge_txdma_reclaim: " 1002 "descs_pending %d ", 1003 tx_ring_p->descs_pending)); 1004 1005 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1006 "==> nxge_txdma_reclaim: " 1007 "(tx_rd_index %d head_index %d " 1008 "(tx_desc_p $%p)", 1009 tx_rd_index, head_index, 1010 tx_desc_p)); 1011 1012 tx_desc_pp->value = tx_desc_p->value; 1013 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1014 "==> nxge_txdma_reclaim: " 1015 "(tx_rd_index %d head_index %d " 1016 "tx_desc_p $%p (desc value 0x%llx) ", 1017 tx_rd_index, head_index, 1018 tx_desc_pp, (*(uint64_t *)tx_desc_pp))); 1019 1020 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1021 "==> nxge_txdma_reclaim: dump desc:")); 1022 1023 pkt_len = tx_desc_pp->bits.hdw.tr_len; 1024 tdc_stats->obytes += (pkt_len - TX_PKT_HEADER_SIZE); 1025 tdc_stats->opackets += tx_desc_pp->bits.hdw.sop; 1026 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1027 "==> nxge_txdma_reclaim: pkt_len %d " 1028 "tdc channel %d opackets %d", 1029 pkt_len, 1030 tdc, 1031 tdc_stats->opackets)); 1032 1033 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1034 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1035 "tx_desc_p = $%p " 1036 "tx_desc_pp = $%p " 1037 "index = %d", 1038 tx_desc_p, 1039 tx_desc_pp, 1040 tx_ring_p->rd_index)); 1041 (void) dvma_unload(tx_msg_p->dvma_handle, 1042 0, -1); 1043 tx_msg_p->dvma_handle = NULL; 1044 if (tx_ring_p->dvma_wr_index == 1045 tx_ring_p->dvma_wrap_mask) { 1046 tx_ring_p->dvma_wr_index = 0; 1047 } else { 1048 tx_ring_p->dvma_wr_index++; 1049 } 1050 tx_ring_p->dvma_pending--; 1051 } else if (tx_msg_p->flags.dma_type == 1052 USE_DMA) { 1053 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1054 "==> nxge_txdma_reclaim: " 1055 "USE DMA")); 1056 if (rc = ddi_dma_unbind_handle 1057 (tx_msg_p->dma_handle)) { 1058 cmn_err(CE_WARN, "!nxge_reclaim: " 1059 "ddi_dma_unbind_handle " 1060 "failed. status %d", rc); 1061 } 1062 } 1063 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1064 "==> nxge_txdma_reclaim: count packets")); 1065 /* 1066 * count a chained packet only once. 1067 */ 1068 if (tx_msg_p->tx_message != NULL) { 1069 freemsg(tx_msg_p->tx_message); 1070 tx_msg_p->tx_message = NULL; 1071 } 1072 1073 tx_msg_p->flags.dma_type = USE_NONE; 1074 tx_rd_index = tx_ring_p->rd_index; 1075 tx_rd_index = (tx_rd_index + 1) & 1076 tx_ring_p->tx_wrap_mask; 1077 tx_ring_p->rd_index = tx_rd_index; 1078 tx_ring_p->descs_pending--; 1079 tx_desc_p = &tx_desc_ring_vp[tx_rd_index]; 1080 tx_msg_p = &tx_msg_ring[tx_rd_index]; 1081 } 1082 1083 status = (nmblks <= ((int)tx_ring_p->tx_ring_size - 1084 (int)tx_ring_p->descs_pending - TX_FULL_MARK)); 1085 if (status) { 1086 (void) atomic_cas_32((uint32_t *)&tx_ring_p->queueing, 1087 1, 0); 1088 } 1089 } else { 1090 status = (nmblks <= ((int)tx_ring_p->tx_ring_size - 1091 (int)tx_ring_p->descs_pending - TX_FULL_MARK)); 1092 } 1093 1094 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1095 "<== nxge_txdma_reclaim status = 0x%08x", status)); 1096 1097 return (status); 1098 } 1099 1100 /* 1101 * nxge_tx_intr 1102 * 1103 * Process a TDC interrupt 1104 * 1105 * Arguments: 1106 * arg1 A Logical Device state Vector (LSV) data structure. 1107 * arg2 nxge_t * 1108 * 1109 * Notes: 1110 * 1111 * NPI/NXGE function calls: 1112 * npi_txdma_control_status() 1113 * npi_intr_ldg_mgmt_set() 1114 * 1115 * nxge_tx_err_evnts() 1116 * nxge_txdma_reclaim() 1117 * 1118 * Registers accessed: 1119 * TX_CS DMC+0x40028 Transmit Control And Status 1120 * PIO_LDSV 1121 * 1122 * Context: 1123 * Any domain 1124 */ 1125 uint_t 1126 nxge_tx_intr(void *arg1, void *arg2) 1127 { 1128 p_nxge_ldv_t ldvp = (p_nxge_ldv_t)arg1; 1129 p_nxge_t nxgep = (p_nxge_t)arg2; 1130 p_nxge_ldg_t ldgp; 1131 uint8_t channel; 1132 uint32_t vindex; 1133 npi_handle_t handle; 1134 tx_cs_t cs; 1135 p_tx_ring_t *tx_rings; 1136 p_tx_ring_t tx_ring_p; 1137 npi_status_t rs = NPI_SUCCESS; 1138 uint_t serviced = DDI_INTR_UNCLAIMED; 1139 nxge_status_t status = NXGE_OK; 1140 1141 if (ldvp == NULL) { 1142 NXGE_DEBUG_MSG((NULL, INT_CTL, 1143 "<== nxge_tx_intr: nxgep $%p ldvp $%p", 1144 nxgep, ldvp)); 1145 return (DDI_INTR_UNCLAIMED); 1146 } 1147 1148 if (arg2 == NULL || (void *)ldvp->nxgep != arg2) { 1149 nxgep = ldvp->nxgep; 1150 } 1151 NXGE_DEBUG_MSG((nxgep, INT_CTL, 1152 "==> nxge_tx_intr: nxgep(arg2) $%p ldvp(arg1) $%p", 1153 nxgep, ldvp)); 1154 1155 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1156 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1157 NXGE_DEBUG_MSG((nxgep, INT_CTL, 1158 "<== nxge_tx_intr: interface not started or intialized")); 1159 return (DDI_INTR_CLAIMED); 1160 } 1161 1162 /* 1163 * This interrupt handler is for a specific 1164 * transmit dma channel. 1165 */ 1166 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1167 /* Get the control and status for this channel. */ 1168 channel = ldvp->channel; 1169 ldgp = ldvp->ldgp; 1170 NXGE_DEBUG_MSG((nxgep, INT_CTL, 1171 "==> nxge_tx_intr: nxgep $%p ldvp (ldvp) $%p " 1172 "channel %d", 1173 nxgep, ldvp, channel)); 1174 1175 rs = npi_txdma_control_status(handle, OP_GET, channel, &cs); 1176 vindex = ldvp->vdma_index; 1177 NXGE_DEBUG_MSG((nxgep, INT_CTL, 1178 "==> nxge_tx_intr:channel %d ring index %d status 0x%08x", 1179 channel, vindex, rs)); 1180 if (!rs && cs.bits.ldw.mk) { 1181 NXGE_DEBUG_MSG((nxgep, INT_CTL, 1182 "==> nxge_tx_intr:channel %d ring index %d " 1183 "status 0x%08x (mk bit set)", 1184 channel, vindex, rs)); 1185 tx_rings = nxgep->tx_rings->rings; 1186 tx_ring_p = tx_rings[vindex]; 1187 NXGE_DEBUG_MSG((nxgep, INT_CTL, 1188 "==> nxge_tx_intr:channel %d ring index %d " 1189 "status 0x%08x (mk bit set, calling reclaim)", 1190 channel, vindex, rs)); 1191 1192 nxge_tx_ring_task((void *)tx_ring_p); 1193 } 1194 1195 /* 1196 * Process other transmit control and status. 1197 * Check the ldv state. 1198 */ 1199 status = nxge_tx_err_evnts(nxgep, ldvp->vdma_index, ldvp, cs); 1200 /* 1201 * Rearm this logical group if this is a single device 1202 * group. 1203 */ 1204 if (ldgp->nldvs == 1) { 1205 NXGE_DEBUG_MSG((nxgep, INT_CTL, 1206 "==> nxge_tx_intr: rearm")); 1207 if (status == NXGE_OK) { 1208 if (isLDOMguest(nxgep)) { 1209 nxge_hio_ldgimgn(nxgep, ldgp); 1210 } else { 1211 (void) npi_intr_ldg_mgmt_set(handle, ldgp->ldg, 1212 B_TRUE, ldgp->ldg_timer); 1213 } 1214 } 1215 } 1216 1217 NXGE_DEBUG_MSG((nxgep, INT_CTL, "<== nxge_tx_intr")); 1218 serviced = DDI_INTR_CLAIMED; 1219 return (serviced); 1220 } 1221 1222 void 1223 nxge_txdma_stop(p_nxge_t nxgep) /* Dead */ 1224 { 1225 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_stop")); 1226 1227 (void) nxge_link_monitor(nxgep, LINK_MONITOR_STOP); 1228 1229 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_stop")); 1230 } 1231 1232 void 1233 nxge_txdma_stop_start(p_nxge_t nxgep) /* Dead */ 1234 { 1235 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_stop_start")); 1236 1237 (void) nxge_txdma_stop(nxgep); 1238 1239 (void) nxge_fixup_txdma_rings(nxgep); 1240 (void) nxge_txdma_hw_mode(nxgep, NXGE_DMA_START); 1241 (void) nxge_tx_mac_enable(nxgep); 1242 (void) nxge_txdma_hw_kick(nxgep); 1243 1244 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_stop_start")); 1245 } 1246 1247 npi_status_t 1248 nxge_txdma_channel_disable( 1249 nxge_t *nxge, 1250 int channel) 1251 { 1252 npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxge); 1253 npi_status_t rs; 1254 tdmc_intr_dbg_t intr_dbg; 1255 1256 /* 1257 * Stop the dma channel and wait for the stop-done. 1258 * If the stop-done bit is not present, then force 1259 * an error so TXC will stop. 1260 * All channels bound to this port need to be stopped 1261 * and reset after injecting an interrupt error. 1262 */ 1263 rs = npi_txdma_channel_disable(handle, channel); 1264 NXGE_DEBUG_MSG((nxge, MEM3_CTL, 1265 "==> nxge_txdma_channel_disable(%d) " 1266 "rs 0x%x", channel, rs)); 1267 if (rs != NPI_SUCCESS) { 1268 /* Inject any error */ 1269 intr_dbg.value = 0; 1270 intr_dbg.bits.ldw.nack_pref = 1; 1271 NXGE_DEBUG_MSG((nxge, MEM3_CTL, 1272 "==> nxge_txdma_hw_mode: " 1273 "channel %d (stop failed 0x%x) " 1274 "(inject err)", rs, channel)); 1275 (void) npi_txdma_inj_int_error_set( 1276 handle, channel, &intr_dbg); 1277 rs = npi_txdma_channel_disable(handle, channel); 1278 NXGE_DEBUG_MSG((nxge, MEM3_CTL, 1279 "==> nxge_txdma_hw_mode: " 1280 "channel %d (stop again 0x%x) " 1281 "(after inject err)", 1282 rs, channel)); 1283 } 1284 1285 return (rs); 1286 } 1287 1288 /* 1289 * nxge_txdma_hw_mode 1290 * 1291 * Toggle all TDCs on (enable) or off (disable). 1292 * 1293 * Arguments: 1294 * nxgep 1295 * enable Enable or disable a TDC. 1296 * 1297 * Notes: 1298 * 1299 * NPI/NXGE function calls: 1300 * npi_txdma_channel_enable(TX_CS) 1301 * npi_txdma_channel_disable(TX_CS) 1302 * npi_txdma_inj_int_error_set(TDMC_INTR_DBG) 1303 * 1304 * Registers accessed: 1305 * TX_CS DMC+0x40028 Transmit Control And Status 1306 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug 1307 * 1308 * Context: 1309 * Any domain 1310 */ 1311 nxge_status_t 1312 nxge_txdma_hw_mode(p_nxge_t nxgep, boolean_t enable) 1313 { 1314 nxge_grp_set_t *set = &nxgep->tx_set; 1315 1316 npi_handle_t handle; 1317 nxge_status_t status; 1318 npi_status_t rs; 1319 int tdc; 1320 1321 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 1322 "==> nxge_txdma_hw_mode: enable mode %d", enable)); 1323 1324 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 1325 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1326 "<== nxge_txdma_mode: not initialized")); 1327 return (NXGE_ERROR); 1328 } 1329 1330 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 1331 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1332 "<== nxge_txdma_hw_mode: NULL ring pointer(s)")); 1333 return (NXGE_ERROR); 1334 } 1335 1336 /* Enable or disable all of the TDCs owned by us. */ 1337 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1338 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 1339 if ((1 << tdc) & set->owned.map) { 1340 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 1341 if (ring) { 1342 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 1343 "==> nxge_txdma_hw_mode: channel %d", tdc)); 1344 if (enable) { 1345 rs = npi_txdma_channel_enable 1346 (handle, tdc); 1347 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 1348 "==> nxge_txdma_hw_mode: " 1349 "channel %d (enable) rs 0x%x", 1350 tdc, rs)); 1351 } else { 1352 rs = nxge_txdma_channel_disable 1353 (nxgep, tdc); 1354 } 1355 } 1356 } 1357 } 1358 1359 status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs); 1360 1361 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 1362 "<== nxge_txdma_hw_mode: status 0x%x", status)); 1363 1364 return (status); 1365 } 1366 1367 void 1368 nxge_txdma_enable_channel(p_nxge_t nxgep, uint16_t channel) 1369 { 1370 npi_handle_t handle; 1371 1372 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 1373 "==> nxge_txdma_enable_channel: channel %d", channel)); 1374 1375 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1376 /* enable the transmit dma channels */ 1377 (void) npi_txdma_channel_enable(handle, channel); 1378 1379 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_txdma_enable_channel")); 1380 } 1381 1382 void 1383 nxge_txdma_disable_channel(p_nxge_t nxgep, uint16_t channel) 1384 { 1385 npi_handle_t handle; 1386 1387 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 1388 "==> nxge_txdma_disable_channel: channel %d", channel)); 1389 1390 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1391 /* stop the transmit dma channels */ 1392 (void) npi_txdma_channel_disable(handle, channel); 1393 1394 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_disable_channel")); 1395 } 1396 1397 /* 1398 * nxge_txdma_stop_inj_err 1399 * 1400 * Stop a TDC. If at first we don't succeed, inject an error. 1401 * 1402 * Arguments: 1403 * nxgep 1404 * channel The channel to stop. 1405 * 1406 * Notes: 1407 * 1408 * NPI/NXGE function calls: 1409 * npi_txdma_channel_disable() 1410 * npi_txdma_inj_int_error_set() 1411 * #if defined(NXGE_DEBUG) 1412 * nxge_txdma_regs_dump_channels(nxgep); 1413 * #endif 1414 * 1415 * Registers accessed: 1416 * TX_CS DMC+0x40028 Transmit Control And Status 1417 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug 1418 * 1419 * Context: 1420 * Any domain 1421 */ 1422 int 1423 nxge_txdma_stop_inj_err(p_nxge_t nxgep, int channel) 1424 { 1425 npi_handle_t handle; 1426 tdmc_intr_dbg_t intr_dbg; 1427 int status; 1428 npi_status_t rs = NPI_SUCCESS; 1429 1430 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_stop_inj_err")); 1431 /* 1432 * Stop the dma channel waits for the stop done. 1433 * If the stop done bit is not set, then create 1434 * an error. 1435 */ 1436 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1437 rs = npi_txdma_channel_disable(handle, channel); 1438 status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs); 1439 if (status == NXGE_OK) { 1440 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1441 "<== nxge_txdma_stop_inj_err (channel %d): " 1442 "stopped OK", channel)); 1443 return (status); 1444 } 1445 1446 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 1447 "==> nxge_txdma_stop_inj_err (channel %d): stop failed (0x%x) " 1448 "injecting error", channel, rs)); 1449 /* Inject any error */ 1450 intr_dbg.value = 0; 1451 intr_dbg.bits.ldw.nack_pref = 1; 1452 (void) npi_txdma_inj_int_error_set(handle, channel, &intr_dbg); 1453 1454 /* Stop done bit will be set as a result of error injection */ 1455 rs = npi_txdma_channel_disable(handle, channel); 1456 status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs); 1457 if (!(rs & NPI_TXDMA_STOP_FAILED)) { 1458 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1459 "<== nxge_txdma_stop_inj_err (channel %d): " 1460 "stopped OK ", channel)); 1461 return (status); 1462 } 1463 1464 #if defined(NXGE_DEBUG) 1465 nxge_txdma_regs_dump_channels(nxgep); 1466 #endif 1467 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 1468 "==> nxge_txdma_stop_inj_err (channel): stop failed (0x%x) " 1469 " (injected error but still not stopped)", channel, rs)); 1470 1471 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_stop_inj_err")); 1472 return (status); 1473 } 1474 1475 /*ARGSUSED*/ 1476 void 1477 nxge_fixup_txdma_rings(p_nxge_t nxgep) 1478 { 1479 nxge_grp_set_t *set = &nxgep->tx_set; 1480 int tdc; 1481 1482 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_fixup_txdma_rings")); 1483 1484 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 1485 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1486 "<== nxge_fixup_txdma_rings: NULL ring pointer(s)")); 1487 return; 1488 } 1489 1490 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 1491 if ((1 << tdc) & set->owned.map) { 1492 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 1493 if (ring) { 1494 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 1495 "==> nxge_fixup_txdma_rings: channel %d", 1496 tdc)); 1497 nxge_txdma_fixup_channel(nxgep, ring, tdc); 1498 } 1499 } 1500 } 1501 1502 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_fixup_txdma_rings")); 1503 } 1504 1505 /*ARGSUSED*/ 1506 void 1507 nxge_txdma_fix_channel(p_nxge_t nxgep, uint16_t channel) 1508 { 1509 p_tx_ring_t ring_p; 1510 1511 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fix_channel")); 1512 ring_p = nxge_txdma_get_ring(nxgep, channel); 1513 if (ring_p == NULL) { 1514 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fix_channel")); 1515 return; 1516 } 1517 1518 if (ring_p->tdc != channel) { 1519 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1520 "<== nxge_txdma_fix_channel: channel not matched " 1521 "ring tdc %d passed channel", 1522 ring_p->tdc, channel)); 1523 return; 1524 } 1525 1526 nxge_txdma_fixup_channel(nxgep, ring_p, channel); 1527 1528 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fix_channel")); 1529 } 1530 1531 /*ARGSUSED*/ 1532 void 1533 nxge_txdma_fixup_channel(p_nxge_t nxgep, p_tx_ring_t ring_p, uint16_t channel) 1534 { 1535 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fixup_channel")); 1536 1537 if (ring_p == NULL) { 1538 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1539 "<== nxge_txdma_fixup_channel: NULL ring pointer")); 1540 return; 1541 } 1542 1543 if (ring_p->tdc != channel) { 1544 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1545 "<== nxge_txdma_fixup_channel: channel not matched " 1546 "ring tdc %d passed channel", 1547 ring_p->tdc, channel)); 1548 return; 1549 } 1550 1551 MUTEX_ENTER(&ring_p->lock); 1552 (void) nxge_txdma_reclaim(nxgep, ring_p, 0); 1553 ring_p->rd_index = 0; 1554 ring_p->wr_index = 0; 1555 ring_p->ring_head.value = 0; 1556 ring_p->ring_kick_tail.value = 0; 1557 ring_p->descs_pending = 0; 1558 MUTEX_EXIT(&ring_p->lock); 1559 1560 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fixup_channel")); 1561 } 1562 1563 /*ARGSUSED*/ 1564 void 1565 nxge_txdma_hw_kick(p_nxge_t nxgep) 1566 { 1567 nxge_grp_set_t *set = &nxgep->tx_set; 1568 int tdc; 1569 1570 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_hw_kick")); 1571 1572 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 1573 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1574 "<== nxge_txdma_hw_kick: NULL ring pointer(s)")); 1575 return; 1576 } 1577 1578 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 1579 if ((1 << tdc) & set->owned.map) { 1580 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 1581 if (ring) { 1582 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 1583 "==> nxge_txdma_hw_kick: channel %d", tdc)); 1584 nxge_txdma_hw_kick_channel(nxgep, ring, tdc); 1585 } 1586 } 1587 } 1588 1589 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_hw_kick")); 1590 } 1591 1592 /*ARGSUSED*/ 1593 void 1594 nxge_txdma_kick_channel(p_nxge_t nxgep, uint16_t channel) 1595 { 1596 p_tx_ring_t ring_p; 1597 1598 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_kick_channel")); 1599 1600 ring_p = nxge_txdma_get_ring(nxgep, channel); 1601 if (ring_p == NULL) { 1602 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1603 " nxge_txdma_kick_channel")); 1604 return; 1605 } 1606 1607 if (ring_p->tdc != channel) { 1608 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1609 "<== nxge_txdma_kick_channel: channel not matched " 1610 "ring tdc %d passed channel", 1611 ring_p->tdc, channel)); 1612 return; 1613 } 1614 1615 nxge_txdma_hw_kick_channel(nxgep, ring_p, channel); 1616 1617 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_kick_channel")); 1618 } 1619 1620 /*ARGSUSED*/ 1621 void 1622 nxge_txdma_hw_kick_channel(p_nxge_t nxgep, p_tx_ring_t ring_p, uint16_t channel) 1623 { 1624 1625 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_hw_kick_channel")); 1626 1627 if (ring_p == NULL) { 1628 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1629 "<== nxge_txdma_hw_kick_channel: NULL ring pointer")); 1630 return; 1631 } 1632 1633 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_hw_kick_channel")); 1634 } 1635 1636 /* 1637 * nxge_check_tx_hang 1638 * 1639 * Check the state of all TDCs belonging to nxgep. 1640 * 1641 * Arguments: 1642 * nxgep 1643 * 1644 * Notes: 1645 * Called by nxge_hw.c:nxge_check_hw_state(). 1646 * 1647 * NPI/NXGE function calls: 1648 * 1649 * Registers accessed: 1650 * 1651 * Context: 1652 * Any domain 1653 */ 1654 /*ARGSUSED*/ 1655 void 1656 nxge_check_tx_hang(p_nxge_t nxgep) 1657 { 1658 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_check_tx_hang")); 1659 1660 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1661 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1662 goto nxge_check_tx_hang_exit; 1663 } 1664 1665 /* 1666 * Needs inputs from hardware for regs: 1667 * head index had not moved since last timeout. 1668 * packets not transmitted or stuffed registers. 1669 */ 1670 if (nxge_txdma_hung(nxgep)) { 1671 nxge_fixup_hung_txdma_rings(nxgep); 1672 } 1673 1674 nxge_check_tx_hang_exit: 1675 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_check_tx_hang")); 1676 } 1677 1678 /* 1679 * nxge_txdma_hung 1680 * 1681 * Reset a TDC. 1682 * 1683 * Arguments: 1684 * nxgep 1685 * channel The channel to reset. 1686 * reg_data The current TX_CS. 1687 * 1688 * Notes: 1689 * Called by nxge_check_tx_hang() 1690 * 1691 * NPI/NXGE function calls: 1692 * nxge_txdma_channel_hung() 1693 * 1694 * Registers accessed: 1695 * 1696 * Context: 1697 * Any domain 1698 */ 1699 int 1700 nxge_txdma_hung(p_nxge_t nxgep) 1701 { 1702 nxge_grp_set_t *set = &nxgep->tx_set; 1703 int tdc; 1704 boolean_t shared; 1705 1706 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_hung")); 1707 1708 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 1709 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1710 "<== nxge_txdma_hung: NULL ring pointer(s)")); 1711 return (B_FALSE); 1712 } 1713 1714 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 1715 /* 1716 * Grab the shared state of the TDC. 1717 */ 1718 if (isLDOMservice(nxgep)) { 1719 nxge_hio_data_t *nhd = 1720 (nxge_hio_data_t *)nxgep->nxge_hw_p->hio; 1721 1722 MUTEX_ENTER(&nhd->lock); 1723 shared = nxgep->tdc_is_shared[tdc]; 1724 MUTEX_EXIT(&nhd->lock); 1725 } else { 1726 shared = B_FALSE; 1727 } 1728 1729 /* 1730 * Now, process continue to process. 1731 */ 1732 if (((1 << tdc) & set->owned.map) && !shared) { 1733 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 1734 if (ring) { 1735 if (nxge_txdma_channel_hung(nxgep, ring, tdc)) { 1736 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1737 "==> nxge_txdma_hung: TDC %d hung", 1738 tdc)); 1739 return (B_TRUE); 1740 } 1741 } 1742 } 1743 } 1744 1745 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_hung")); 1746 1747 return (B_FALSE); 1748 } 1749 1750 /* 1751 * nxge_txdma_channel_hung 1752 * 1753 * Reset a TDC. 1754 * 1755 * Arguments: 1756 * nxgep 1757 * ring <channel>'s ring. 1758 * channel The channel to reset. 1759 * 1760 * Notes: 1761 * Called by nxge_txdma.c:nxge_txdma_hung() 1762 * 1763 * NPI/NXGE function calls: 1764 * npi_txdma_ring_head_get() 1765 * 1766 * Registers accessed: 1767 * TX_RING_HDL DMC+0x40010 Transmit Ring Head Low 1768 * 1769 * Context: 1770 * Any domain 1771 */ 1772 int 1773 nxge_txdma_channel_hung(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, uint16_t channel) 1774 { 1775 uint16_t head_index, tail_index; 1776 boolean_t head_wrap, tail_wrap; 1777 npi_handle_t handle; 1778 tx_ring_hdl_t tx_head; 1779 uint_t tx_rd_index; 1780 1781 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_channel_hung")); 1782 1783 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1784 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1785 "==> nxge_txdma_channel_hung: channel %d", channel)); 1786 MUTEX_ENTER(&tx_ring_p->lock); 1787 (void) nxge_txdma_reclaim(nxgep, tx_ring_p, 0); 1788 1789 tail_index = tx_ring_p->wr_index; 1790 tail_wrap = tx_ring_p->wr_index_wrap; 1791 tx_rd_index = tx_ring_p->rd_index; 1792 MUTEX_EXIT(&tx_ring_p->lock); 1793 1794 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1795 "==> nxge_txdma_channel_hung: tdc %d tx_rd_index %d " 1796 "tail_index %d tail_wrap %d ", 1797 channel, tx_rd_index, tail_index, tail_wrap)); 1798 /* 1799 * Read the hardware maintained transmit head 1800 * and wrap around bit. 1801 */ 1802 (void) npi_txdma_ring_head_get(handle, channel, &tx_head); 1803 head_index = tx_head.bits.ldw.head; 1804 head_wrap = tx_head.bits.ldw.wrap; 1805 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1806 "==> nxge_txdma_channel_hung: " 1807 "tx_rd_index %d tail %d tail_wrap %d " 1808 "head %d wrap %d", 1809 tx_rd_index, tail_index, tail_wrap, 1810 head_index, head_wrap)); 1811 1812 if (TXDMA_RING_EMPTY(head_index, head_wrap, 1813 tail_index, tail_wrap) && 1814 (head_index == tx_rd_index)) { 1815 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1816 "==> nxge_txdma_channel_hung: EMPTY")); 1817 return (B_FALSE); 1818 } 1819 1820 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1821 "==> nxge_txdma_channel_hung: Checking if ring full")); 1822 if (TXDMA_RING_FULL(head_index, head_wrap, tail_index, 1823 tail_wrap)) { 1824 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1825 "==> nxge_txdma_channel_hung: full")); 1826 return (B_TRUE); 1827 } 1828 1829 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_channel_hung")); 1830 1831 return (B_FALSE); 1832 } 1833 1834 /* 1835 * nxge_fixup_hung_txdma_rings 1836 * 1837 * Disable a TDC. 1838 * 1839 * Arguments: 1840 * nxgep 1841 * channel The channel to reset. 1842 * reg_data The current TX_CS. 1843 * 1844 * Notes: 1845 * Called by nxge_check_tx_hang() 1846 * 1847 * NPI/NXGE function calls: 1848 * npi_txdma_ring_head_get() 1849 * 1850 * Registers accessed: 1851 * TX_RING_HDL DMC+0x40010 Transmit Ring Head Low 1852 * 1853 * Context: 1854 * Any domain 1855 */ 1856 /*ARGSUSED*/ 1857 void 1858 nxge_fixup_hung_txdma_rings(p_nxge_t nxgep) 1859 { 1860 nxge_grp_set_t *set = &nxgep->tx_set; 1861 int tdc; 1862 1863 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_fixup_hung_txdma_rings")); 1864 1865 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 1866 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1867 "<== nxge_fixup_hung_txdma_rings: NULL ring pointer(s)")); 1868 return; 1869 } 1870 1871 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 1872 if ((1 << tdc) & set->owned.map) { 1873 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 1874 if (ring) { 1875 nxge_txdma_fixup_hung_channel(nxgep, ring, tdc); 1876 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1877 "==> nxge_fixup_hung_txdma_rings: TDC %d", 1878 tdc)); 1879 } 1880 } 1881 } 1882 1883 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_fixup_hung_txdma_rings")); 1884 } 1885 1886 /* 1887 * nxge_txdma_fixup_hung_channel 1888 * 1889 * 'Fix' a hung TDC. 1890 * 1891 * Arguments: 1892 * nxgep 1893 * channel The channel to fix. 1894 * 1895 * Notes: 1896 * Called by nxge_fixup_hung_txdma_rings() 1897 * 1898 * 1. Reclaim the TDC. 1899 * 2. Disable the TDC. 1900 * 1901 * NPI/NXGE function calls: 1902 * nxge_txdma_reclaim() 1903 * npi_txdma_channel_disable(TX_CS) 1904 * npi_txdma_inj_int_error_set(TDMC_INTR_DBG) 1905 * 1906 * Registers accessed: 1907 * TX_CS DMC+0x40028 Transmit Control And Status 1908 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug 1909 * 1910 * Context: 1911 * Any domain 1912 */ 1913 /*ARGSUSED*/ 1914 void 1915 nxge_txdma_fix_hung_channel(p_nxge_t nxgep, uint16_t channel) 1916 { 1917 p_tx_ring_t ring_p; 1918 1919 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fix_hung_channel")); 1920 ring_p = nxge_txdma_get_ring(nxgep, channel); 1921 if (ring_p == NULL) { 1922 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1923 "<== nxge_txdma_fix_hung_channel")); 1924 return; 1925 } 1926 1927 if (ring_p->tdc != channel) { 1928 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1929 "<== nxge_txdma_fix_hung_channel: channel not matched " 1930 "ring tdc %d passed channel", 1931 ring_p->tdc, channel)); 1932 return; 1933 } 1934 1935 nxge_txdma_fixup_channel(nxgep, ring_p, channel); 1936 1937 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fix_hung_channel")); 1938 } 1939 1940 /*ARGSUSED*/ 1941 void 1942 nxge_txdma_fixup_hung_channel(p_nxge_t nxgep, p_tx_ring_t ring_p, 1943 uint16_t channel) 1944 { 1945 npi_handle_t handle; 1946 tdmc_intr_dbg_t intr_dbg; 1947 int status = NXGE_OK; 1948 1949 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fixup_hung_channel")); 1950 1951 if (ring_p == NULL) { 1952 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1953 "<== nxge_txdma_fixup_channel: NULL ring pointer")); 1954 return; 1955 } 1956 1957 if (ring_p->tdc != channel) { 1958 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1959 "<== nxge_txdma_fixup_hung_channel: channel " 1960 "not matched " 1961 "ring tdc %d passed channel", 1962 ring_p->tdc, channel)); 1963 return; 1964 } 1965 1966 /* Reclaim descriptors */ 1967 MUTEX_ENTER(&ring_p->lock); 1968 (void) nxge_txdma_reclaim(nxgep, ring_p, 0); 1969 MUTEX_EXIT(&ring_p->lock); 1970 1971 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1972 /* 1973 * Stop the dma channel waits for the stop done. 1974 * If the stop done bit is not set, then force 1975 * an error. 1976 */ 1977 status = npi_txdma_channel_disable(handle, channel); 1978 if (!(status & NPI_TXDMA_STOP_FAILED)) { 1979 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1980 "<== nxge_txdma_fixup_hung_channel: stopped OK " 1981 "ring tdc %d passed channel %d", 1982 ring_p->tdc, channel)); 1983 return; 1984 } 1985 1986 /* Inject any error */ 1987 intr_dbg.value = 0; 1988 intr_dbg.bits.ldw.nack_pref = 1; 1989 (void) npi_txdma_inj_int_error_set(handle, channel, &intr_dbg); 1990 1991 /* Stop done bit will be set as a result of error injection */ 1992 status = npi_txdma_channel_disable(handle, channel); 1993 if (!(status & NPI_TXDMA_STOP_FAILED)) { 1994 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1995 "<== nxge_txdma_fixup_hung_channel: stopped again" 1996 "ring tdc %d passed channel", 1997 ring_p->tdc, channel)); 1998 return; 1999 } 2000 2001 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2002 "<== nxge_txdma_fixup_hung_channel: stop done still not set!! " 2003 "ring tdc %d passed channel", 2004 ring_p->tdc, channel)); 2005 2006 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fixup_hung_channel")); 2007 } 2008 2009 /*ARGSUSED*/ 2010 void 2011 nxge_reclaim_rings(p_nxge_t nxgep) 2012 { 2013 nxge_grp_set_t *set = &nxgep->tx_set; 2014 int tdc; 2015 2016 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reclaim_rings")); 2017 2018 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 2019 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2020 "<== nxge_fixup_hung_txdma_rings: NULL ring pointer(s)")); 2021 return; 2022 } 2023 2024 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 2025 if ((1 << tdc) & set->owned.map) { 2026 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 2027 if (ring) { 2028 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2029 "==> nxge_reclaim_rings: TDC %d", tdc)); 2030 MUTEX_ENTER(&ring->lock); 2031 (void) nxge_txdma_reclaim(nxgep, ring, 0); 2032 MUTEX_EXIT(&ring->lock); 2033 } 2034 } 2035 } 2036 2037 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_reclaim_rings")); 2038 } 2039 2040 void 2041 nxge_txdma_regs_dump_channels(p_nxge_t nxgep) 2042 { 2043 nxge_grp_set_t *set = &nxgep->tx_set; 2044 npi_handle_t handle; 2045 int tdc; 2046 2047 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_regs_dump_channels")); 2048 2049 handle = NXGE_DEV_NPI_HANDLE(nxgep); 2050 2051 if (!isLDOMguest(nxgep)) { 2052 (void) npi_txdma_dump_fzc_regs(handle); 2053 2054 /* Dump TXC registers. */ 2055 (void) npi_txc_dump_fzc_regs(handle); 2056 (void) npi_txc_dump_port_fzc_regs(handle, nxgep->function_num); 2057 } 2058 2059 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 2060 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2061 "<== nxge_fixup_hung_txdma_rings: NULL ring pointer(s)")); 2062 return; 2063 } 2064 2065 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 2066 if ((1 << tdc) & set->owned.map) { 2067 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 2068 if (ring) { 2069 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2070 "==> nxge_txdma_regs_dump_channels: " 2071 "TDC %d", tdc)); 2072 (void) npi_txdma_dump_tdc_regs(handle, tdc); 2073 2074 /* Dump TXC registers, if able to. */ 2075 if (!isLDOMguest(nxgep)) { 2076 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2077 "==> nxge_txdma_regs_dump_channels:" 2078 " FZC TDC %d", tdc)); 2079 (void) npi_txc_dump_tdc_fzc_regs 2080 (handle, tdc); 2081 } 2082 nxge_txdma_regs_dump(nxgep, tdc); 2083 } 2084 } 2085 } 2086 2087 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_regs_dump")); 2088 } 2089 2090 void 2091 nxge_txdma_regs_dump(p_nxge_t nxgep, int channel) 2092 { 2093 npi_handle_t handle; 2094 tx_ring_hdl_t hdl; 2095 tx_ring_kick_t kick; 2096 tx_cs_t cs; 2097 txc_control_t control; 2098 uint32_t bitmap = 0; 2099 uint32_t burst = 0; 2100 uint32_t bytes = 0; 2101 dma_log_page_t cfg; 2102 2103 printf("\n\tfunc # %d tdc %d ", 2104 nxgep->function_num, channel); 2105 cfg.page_num = 0; 2106 handle = NXGE_DEV_NPI_HANDLE(nxgep); 2107 (void) npi_txdma_log_page_get(handle, channel, &cfg); 2108 printf("\n\tlog page func %d valid page 0 %d", 2109 cfg.func_num, cfg.valid); 2110 cfg.page_num = 1; 2111 (void) npi_txdma_log_page_get(handle, channel, &cfg); 2112 printf("\n\tlog page func %d valid page 1 %d", 2113 cfg.func_num, cfg.valid); 2114 2115 (void) npi_txdma_ring_head_get(handle, channel, &hdl); 2116 (void) npi_txdma_desc_kick_reg_get(handle, channel, &kick); 2117 printf("\n\thead value is 0x%0llx", 2118 (long long)hdl.value); 2119 printf("\n\thead index %d", hdl.bits.ldw.head); 2120 printf("\n\tkick value is 0x%0llx", 2121 (long long)kick.value); 2122 printf("\n\ttail index %d\n", kick.bits.ldw.tail); 2123 2124 (void) npi_txdma_control_status(handle, OP_GET, channel, &cs); 2125 printf("\n\tControl statue is 0x%0llx", (long long)cs.value); 2126 printf("\n\tControl status RST state %d", cs.bits.ldw.rst); 2127 2128 (void) npi_txc_control(handle, OP_GET, &control); 2129 (void) npi_txc_port_dma_list_get(handle, nxgep->function_num, &bitmap); 2130 (void) npi_txc_dma_max_burst(handle, OP_GET, channel, &burst); 2131 (void) npi_txc_dma_bytes_transmitted(handle, channel, &bytes); 2132 2133 printf("\n\tTXC port control 0x%0llx", 2134 (long long)control.value); 2135 printf("\n\tTXC port bitmap 0x%x", bitmap); 2136 printf("\n\tTXC max burst %d", burst); 2137 printf("\n\tTXC bytes xmt %d\n", bytes); 2138 2139 { 2140 ipp_status_t status; 2141 2142 (void) npi_ipp_get_status(handle, nxgep->function_num, &status); 2143 #if defined(__i386) 2144 printf("\n\tIPP status 0x%llux\n", (uint64_t)status.value); 2145 #else 2146 printf("\n\tIPP status 0x%lux\n", (uint64_t)status.value); 2147 #endif 2148 } 2149 } 2150 2151 /* 2152 * nxge_tdc_hvio_setup 2153 * 2154 * I'm not exactly sure what this code does. 2155 * 2156 * Arguments: 2157 * nxgep 2158 * channel The channel to map. 2159 * 2160 * Notes: 2161 * 2162 * NPI/NXGE function calls: 2163 * na 2164 * 2165 * Context: 2166 * Service domain? 2167 */ 2168 #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 2169 static void 2170 nxge_tdc_hvio_setup( 2171 nxge_t *nxgep, int channel) 2172 { 2173 nxge_dma_common_t *data; 2174 nxge_dma_common_t *control; 2175 tx_ring_t *ring; 2176 2177 ring = nxgep->tx_rings->rings[channel]; 2178 data = nxgep->tx_buf_pool_p->dma_buf_pool_p[channel]; 2179 2180 ring->hv_set = B_FALSE; 2181 2182 ring->hv_tx_buf_base_ioaddr_pp = 2183 (uint64_t)data->orig_ioaddr_pp; 2184 ring->hv_tx_buf_ioaddr_size = 2185 (uint64_t)data->orig_alength; 2186 2187 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma_channel: " 2188 "hv data buf base io $%p size 0x%llx (%d) buf base io $%p " 2189 "orig vatopa base io $%p orig_len 0x%llx (%d)", 2190 ring->hv_tx_buf_base_ioaddr_pp, 2191 ring->hv_tx_buf_ioaddr_size, ring->hv_tx_buf_ioaddr_size, 2192 data->ioaddr_pp, data->orig_vatopa, 2193 data->orig_alength, data->orig_alength)); 2194 2195 control = nxgep->tx_cntl_pool_p->dma_buf_pool_p[channel]; 2196 2197 ring->hv_tx_cntl_base_ioaddr_pp = 2198 (uint64_t)control->orig_ioaddr_pp; 2199 ring->hv_tx_cntl_ioaddr_size = 2200 (uint64_t)control->orig_alength; 2201 2202 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma_channel: " 2203 "hv cntl base io $%p orig ioaddr_pp ($%p) " 2204 "orig vatopa ($%p) size 0x%llx (%d 0x%x)", 2205 ring->hv_tx_cntl_base_ioaddr_pp, 2206 control->orig_ioaddr_pp, control->orig_vatopa, 2207 ring->hv_tx_cntl_ioaddr_size, 2208 control->orig_alength, control->orig_alength)); 2209 } 2210 #endif 2211 2212 static nxge_status_t 2213 nxge_map_txdma(p_nxge_t nxgep, int channel) 2214 { 2215 nxge_dma_common_t **pData; 2216 nxge_dma_common_t **pControl; 2217 tx_ring_t **pRing, *ring; 2218 tx_mbox_t **mailbox; 2219 uint32_t num_chunks; 2220 2221 nxge_status_t status = NXGE_OK; 2222 2223 NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma")); 2224 2225 if (!nxgep->tx_cntl_pool_p->buf_allocated) { 2226 if (nxge_alloc_tx_mem_pool(nxgep) != NXGE_OK) { 2227 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2228 "<== nxge_map_txdma: buf not allocated")); 2229 return (NXGE_ERROR); 2230 } 2231 } 2232 2233 if (nxge_alloc_txb(nxgep, channel) != NXGE_OK) 2234 return (NXGE_ERROR); 2235 2236 num_chunks = nxgep->tx_buf_pool_p->num_chunks[channel]; 2237 pData = &nxgep->tx_buf_pool_p->dma_buf_pool_p[channel]; 2238 pControl = &nxgep->tx_cntl_pool_p->dma_buf_pool_p[channel]; 2239 pRing = &nxgep->tx_rings->rings[channel]; 2240 mailbox = &nxgep->tx_mbox_areas_p->txmbox_areas_p[channel]; 2241 2242 NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma: " 2243 "tx_rings $%p tx_desc_rings $%p", 2244 nxgep->tx_rings, nxgep->tx_rings->rings)); 2245 2246 /* 2247 * Map descriptors from the buffer pools for <channel>. 2248 */ 2249 2250 /* 2251 * Set up and prepare buffer blocks, descriptors 2252 * and mailbox. 2253 */ 2254 status = nxge_map_txdma_channel(nxgep, channel, 2255 pData, pRing, num_chunks, pControl, mailbox); 2256 if (status != NXGE_OK) { 2257 NXGE_ERROR_MSG((nxgep, MEM3_CTL, 2258 "==> nxge_map_txdma(%d): nxge_map_txdma_channel() " 2259 "returned 0x%x", 2260 nxgep, channel, status)); 2261 return (status); 2262 } 2263 2264 ring = *pRing; 2265 2266 ring->index = (uint16_t)channel; 2267 ring->tdc_stats = &nxgep->statsp->tdc_stats[channel]; 2268 2269 #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 2270 if (isLDOMguest(nxgep)) { 2271 (void) nxge_tdc_lp_conf(nxgep, channel); 2272 } else { 2273 nxge_tdc_hvio_setup(nxgep, channel); 2274 } 2275 #endif 2276 2277 NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma: " 2278 "(status 0x%x channel %d)", status, channel)); 2279 2280 return (status); 2281 } 2282 2283 static nxge_status_t 2284 nxge_map_txdma_channel(p_nxge_t nxgep, uint16_t channel, 2285 p_nxge_dma_common_t *dma_buf_p, 2286 p_tx_ring_t *tx_desc_p, 2287 uint32_t num_chunks, 2288 p_nxge_dma_common_t *dma_cntl_p, 2289 p_tx_mbox_t *tx_mbox_p) 2290 { 2291 int status = NXGE_OK; 2292 2293 /* 2294 * Set up and prepare buffer blocks, descriptors 2295 * and mailbox. 2296 */ 2297 NXGE_ERROR_MSG((nxgep, MEM3_CTL, 2298 "==> nxge_map_txdma_channel (channel %d)", channel)); 2299 /* 2300 * Transmit buffer blocks 2301 */ 2302 status = nxge_map_txdma_channel_buf_ring(nxgep, channel, 2303 dma_buf_p, tx_desc_p, num_chunks); 2304 if (status != NXGE_OK) { 2305 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2306 "==> nxge_map_txdma_channel (channel %d): " 2307 "map buffer failed 0x%x", channel, status)); 2308 goto nxge_map_txdma_channel_exit; 2309 } 2310 2311 /* 2312 * Transmit block ring, and mailbox. 2313 */ 2314 nxge_map_txdma_channel_cfg_ring(nxgep, channel, dma_cntl_p, *tx_desc_p, 2315 tx_mbox_p); 2316 2317 goto nxge_map_txdma_channel_exit; 2318 2319 nxge_map_txdma_channel_fail1: 2320 NXGE_ERROR_MSG((nxgep, MEM3_CTL, 2321 "==> nxge_map_txdma_channel: unmap buf" 2322 "(status 0x%x channel %d)", 2323 status, channel)); 2324 nxge_unmap_txdma_channel_buf_ring(nxgep, *tx_desc_p); 2325 2326 nxge_map_txdma_channel_exit: 2327 NXGE_ERROR_MSG((nxgep, MEM3_CTL, 2328 "<== nxge_map_txdma_channel: " 2329 "(status 0x%x channel %d)", 2330 status, channel)); 2331 2332 return (status); 2333 } 2334 2335 /*ARGSUSED*/ 2336 static void 2337 nxge_unmap_txdma_channel(p_nxge_t nxgep, uint16_t channel) 2338 { 2339 tx_ring_t *ring; 2340 tx_mbox_t *mailbox; 2341 2342 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2343 "==> nxge_unmap_txdma_channel (channel %d)", channel)); 2344 /* 2345 * unmap tx block ring, and mailbox. 2346 */ 2347 ring = nxgep->tx_rings->rings[channel]; 2348 mailbox = nxgep->tx_mbox_areas_p->txmbox_areas_p[channel]; 2349 2350 (void) nxge_unmap_txdma_channel_cfg_ring(nxgep, ring, mailbox); 2351 2352 /* unmap buffer blocks */ 2353 (void) nxge_unmap_txdma_channel_buf_ring(nxgep, ring); 2354 2355 nxge_free_txb(nxgep, channel); 2356 2357 /* 2358 * Cleanup the reference to the ring now that it does not exist. 2359 */ 2360 nxgep->tx_rings->rings[channel] = NULL; 2361 2362 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_unmap_txdma_channel")); 2363 } 2364 2365 /* 2366 * nxge_map_txdma_channel_cfg_ring 2367 * 2368 * Map a TDC into our kernel space. 2369 * This function allocates all of the per-channel data structures. 2370 * 2371 * Arguments: 2372 * nxgep 2373 * dma_channel The channel to map. 2374 * dma_cntl_p 2375 * tx_ring_p dma_channel's transmit ring 2376 * tx_mbox_p dma_channel's mailbox 2377 * 2378 * Notes: 2379 * 2380 * NPI/NXGE function calls: 2381 * nxge_setup_dma_common() 2382 * 2383 * Registers accessed: 2384 * none. 2385 * 2386 * Context: 2387 * Any domain 2388 */ 2389 /*ARGSUSED*/ 2390 static void 2391 nxge_map_txdma_channel_cfg_ring(p_nxge_t nxgep, uint16_t dma_channel, 2392 p_nxge_dma_common_t *dma_cntl_p, 2393 p_tx_ring_t tx_ring_p, 2394 p_tx_mbox_t *tx_mbox_p) 2395 { 2396 p_tx_mbox_t mboxp; 2397 p_nxge_dma_common_t cntl_dmap; 2398 p_nxge_dma_common_t dmap; 2399 p_tx_rng_cfig_t tx_ring_cfig_p; 2400 p_tx_ring_kick_t tx_ring_kick_p; 2401 p_tx_cs_t tx_cs_p; 2402 p_tx_dma_ent_msk_t tx_evmask_p; 2403 p_txdma_mbh_t mboxh_p; 2404 p_txdma_mbl_t mboxl_p; 2405 uint64_t tx_desc_len; 2406 2407 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2408 "==> nxge_map_txdma_channel_cfg_ring")); 2409 2410 cntl_dmap = *dma_cntl_p; 2411 2412 dmap = (p_nxge_dma_common_t)&tx_ring_p->tdc_desc; 2413 nxge_setup_dma_common(dmap, cntl_dmap, tx_ring_p->tx_ring_size, 2414 sizeof (tx_desc_t)); 2415 /* 2416 * Zero out transmit ring descriptors. 2417 */ 2418 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2419 tx_ring_cfig_p = &(tx_ring_p->tx_ring_cfig); 2420 tx_ring_kick_p = &(tx_ring_p->tx_ring_kick); 2421 tx_cs_p = &(tx_ring_p->tx_cs); 2422 tx_evmask_p = &(tx_ring_p->tx_evmask); 2423 tx_ring_cfig_p->value = 0; 2424 tx_ring_kick_p->value = 0; 2425 tx_cs_p->value = 0; 2426 tx_evmask_p->value = 0; 2427 2428 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2429 "==> nxge_map_txdma_channel_cfg_ring: channel %d des $%p", 2430 dma_channel, 2431 dmap->dma_cookie.dmac_laddress)); 2432 2433 tx_ring_cfig_p->value = 0; 2434 tx_desc_len = (uint64_t)(tx_ring_p->tx_ring_size >> 3); 2435 tx_ring_cfig_p->value = 2436 (dmap->dma_cookie.dmac_laddress & TX_RNG_CFIG_ADDR_MASK) | 2437 (tx_desc_len << TX_RNG_CFIG_LEN_SHIFT); 2438 2439 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2440 "==> nxge_map_txdma_channel_cfg_ring: channel %d cfg 0x%llx", 2441 dma_channel, 2442 tx_ring_cfig_p->value)); 2443 2444 tx_cs_p->bits.ldw.rst = 1; 2445 2446 /* Map in mailbox */ 2447 mboxp = (p_tx_mbox_t) 2448 KMEM_ZALLOC(sizeof (tx_mbox_t), KM_SLEEP); 2449 dmap = (p_nxge_dma_common_t)&mboxp->tx_mbox; 2450 nxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (txdma_mailbox_t)); 2451 mboxh_p = (p_txdma_mbh_t)&tx_ring_p->tx_mbox_mbh; 2452 mboxl_p = (p_txdma_mbl_t)&tx_ring_p->tx_mbox_mbl; 2453 mboxh_p->value = mboxl_p->value = 0; 2454 2455 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2456 "==> nxge_map_txdma_channel_cfg_ring: mbox 0x%lx", 2457 dmap->dma_cookie.dmac_laddress)); 2458 2459 mboxh_p->bits.ldw.mbaddr = ((dmap->dma_cookie.dmac_laddress >> 2460 TXDMA_MBH_ADDR_SHIFT) & TXDMA_MBH_MASK); 2461 2462 mboxl_p->bits.ldw.mbaddr = ((dmap->dma_cookie.dmac_laddress & 2463 TXDMA_MBL_MASK) >> TXDMA_MBL_SHIFT); 2464 2465 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2466 "==> nxge_map_txdma_channel_cfg_ring: mbox 0x%lx", 2467 dmap->dma_cookie.dmac_laddress)); 2468 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2469 "==> nxge_map_txdma_channel_cfg_ring: hmbox $%p " 2470 "mbox $%p", 2471 mboxh_p->bits.ldw.mbaddr, mboxl_p->bits.ldw.mbaddr)); 2472 tx_ring_p->page_valid.value = 0; 2473 tx_ring_p->page_mask_1.value = tx_ring_p->page_mask_2.value = 0; 2474 tx_ring_p->page_value_1.value = tx_ring_p->page_value_2.value = 0; 2475 tx_ring_p->page_reloc_1.value = tx_ring_p->page_reloc_2.value = 0; 2476 tx_ring_p->page_hdl.value = 0; 2477 2478 tx_ring_p->page_valid.bits.ldw.page0 = 1; 2479 tx_ring_p->page_valid.bits.ldw.page1 = 1; 2480 2481 tx_ring_p->max_burst.value = 0; 2482 tx_ring_p->max_burst.bits.ldw.dma_max_burst = TXC_DMA_MAX_BURST_DEFAULT; 2483 2484 *tx_mbox_p = mboxp; 2485 2486 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2487 "<== nxge_map_txdma_channel_cfg_ring")); 2488 } 2489 2490 /*ARGSUSED*/ 2491 static void 2492 nxge_unmap_txdma_channel_cfg_ring(p_nxge_t nxgep, 2493 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p) 2494 { 2495 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2496 "==> nxge_unmap_txdma_channel_cfg_ring: channel %d", 2497 tx_ring_p->tdc)); 2498 2499 KMEM_FREE(tx_mbox_p, sizeof (tx_mbox_t)); 2500 2501 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2502 "<== nxge_unmap_txdma_channel_cfg_ring")); 2503 } 2504 2505 /* 2506 * nxge_map_txdma_channel_buf_ring 2507 * 2508 * 2509 * Arguments: 2510 * nxgep 2511 * channel The channel to map. 2512 * dma_buf_p 2513 * tx_desc_p channel's descriptor ring 2514 * num_chunks 2515 * 2516 * Notes: 2517 * 2518 * NPI/NXGE function calls: 2519 * nxge_setup_dma_common() 2520 * 2521 * Registers accessed: 2522 * none. 2523 * 2524 * Context: 2525 * Any domain 2526 */ 2527 static nxge_status_t 2528 nxge_map_txdma_channel_buf_ring(p_nxge_t nxgep, uint16_t channel, 2529 p_nxge_dma_common_t *dma_buf_p, 2530 p_tx_ring_t *tx_desc_p, uint32_t num_chunks) 2531 { 2532 p_nxge_dma_common_t dma_bufp, tmp_bufp; 2533 p_nxge_dma_common_t dmap; 2534 nxge_os_dma_handle_t tx_buf_dma_handle; 2535 p_tx_ring_t tx_ring_p; 2536 p_tx_msg_t tx_msg_ring; 2537 nxge_status_t status = NXGE_OK; 2538 int ddi_status = DDI_SUCCESS; 2539 int i, j, index; 2540 uint32_t size, bsize; 2541 uint32_t nblocks, nmsgs; 2542 char qname[TASKQ_NAMELEN]; 2543 2544 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2545 "==> nxge_map_txdma_channel_buf_ring")); 2546 2547 dma_bufp = tmp_bufp = *dma_buf_p; 2548 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2549 " nxge_map_txdma_channel_buf_ring: channel %d to map %d " 2550 "chunks bufp $%p", 2551 channel, num_chunks, dma_bufp)); 2552 2553 nmsgs = 0; 2554 for (i = 0; i < num_chunks; i++, tmp_bufp++) { 2555 nmsgs += tmp_bufp->nblocks; 2556 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2557 "==> nxge_map_txdma_channel_buf_ring: channel %d " 2558 "bufp $%p nblocks %d nmsgs %d", 2559 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs)); 2560 } 2561 if (!nmsgs) { 2562 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2563 "<== nxge_map_txdma_channel_buf_ring: channel %d " 2564 "no msg blocks", 2565 channel)); 2566 status = NXGE_ERROR; 2567 goto nxge_map_txdma_channel_buf_ring_exit; 2568 } 2569 2570 tx_ring_p = (p_tx_ring_t) 2571 KMEM_ZALLOC(sizeof (tx_ring_t), KM_SLEEP); 2572 MUTEX_INIT(&tx_ring_p->lock, NULL, MUTEX_DRIVER, 2573 (void *)nxgep->interrupt_cookie); 2574 2575 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, NXGE_TX_RING_ONLINE); 2576 tx_ring_p->tx_ring_busy = B_FALSE; 2577 tx_ring_p->nxgep = nxgep; 2578 tx_ring_p->tx_ring_handle = (mac_ring_handle_t)NULL; 2579 (void) snprintf(qname, TASKQ_NAMELEN, "tx_%d_%d", 2580 nxgep->instance, channel); 2581 tx_ring_p->taskq = ddi_taskq_create(nxgep->dip, qname, 1, 2582 TASKQ_DEFAULTPRI, 0); 2583 if (tx_ring_p->taskq == NULL) { 2584 goto nxge_map_txdma_channel_buf_ring_fail1; 2585 } 2586 2587 /* 2588 * Allocate transmit message rings and handles for packets 2589 * not to be copied to premapped buffers. 2590 */ 2591 size = nmsgs * sizeof (tx_msg_t); 2592 tx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP); 2593 for (i = 0; i < nmsgs; i++) { 2594 ddi_status = ddi_dma_alloc_handle(nxgep->dip, &nxge_tx_dma_attr, 2595 DDI_DMA_DONTWAIT, 0, 2596 &tx_msg_ring[i].dma_handle); 2597 if (ddi_status != DDI_SUCCESS) { 2598 status |= NXGE_DDI_FAILED; 2599 break; 2600 } 2601 } 2602 if (i < nmsgs) { 2603 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2604 "Allocate handles failed.")); 2605 goto nxge_map_txdma_channel_buf_ring_fail1; 2606 } 2607 2608 tx_ring_p->tdc = channel; 2609 tx_ring_p->tx_msg_ring = tx_msg_ring; 2610 tx_ring_p->tx_ring_size = nmsgs; 2611 tx_ring_p->num_chunks = num_chunks; 2612 if (!nxge_tx_intr_thres) { 2613 nxge_tx_intr_thres = tx_ring_p->tx_ring_size/4; 2614 } 2615 tx_ring_p->tx_wrap_mask = tx_ring_p->tx_ring_size - 1; 2616 tx_ring_p->rd_index = 0; 2617 tx_ring_p->wr_index = 0; 2618 tx_ring_p->ring_head.value = 0; 2619 tx_ring_p->ring_kick_tail.value = 0; 2620 tx_ring_p->descs_pending = 0; 2621 2622 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2623 "==> nxge_map_txdma_channel_buf_ring: channel %d " 2624 "actual tx desc max %d nmsgs %d " 2625 "(config nxge_tx_ring_size %d)", 2626 channel, tx_ring_p->tx_ring_size, nmsgs, 2627 nxge_tx_ring_size)); 2628 2629 /* 2630 * Map in buffers from the buffer pool. 2631 */ 2632 index = 0; 2633 bsize = dma_bufp->block_size; 2634 2635 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma_channel_buf_ring: " 2636 "dma_bufp $%p tx_rng_p $%p " 2637 "tx_msg_rng_p $%p bsize %d", 2638 dma_bufp, tx_ring_p, tx_msg_ring, bsize)); 2639 2640 tx_buf_dma_handle = dma_bufp->dma_handle; 2641 for (i = 0; i < num_chunks; i++, dma_bufp++) { 2642 bsize = dma_bufp->block_size; 2643 nblocks = dma_bufp->nblocks; 2644 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2645 "==> nxge_map_txdma_channel_buf_ring: dma chunk %d " 2646 "size %d dma_bufp $%p", 2647 i, sizeof (nxge_dma_common_t), dma_bufp)); 2648 2649 for (j = 0; j < nblocks; j++) { 2650 tx_msg_ring[index].buf_dma_handle = tx_buf_dma_handle; 2651 dmap = &tx_msg_ring[index++].buf_dma; 2652 #ifdef TX_MEM_DEBUG 2653 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2654 "==> nxge_map_txdma_channel_buf_ring: j %d" 2655 "dmap $%p", i, dmap)); 2656 #endif 2657 nxge_setup_dma_common(dmap, dma_bufp, 1, 2658 bsize); 2659 } 2660 } 2661 2662 if (i < num_chunks) { 2663 status = NXGE_ERROR; 2664 goto nxge_map_txdma_channel_buf_ring_fail1; 2665 } 2666 2667 *tx_desc_p = tx_ring_p; 2668 2669 goto nxge_map_txdma_channel_buf_ring_exit; 2670 2671 nxge_map_txdma_channel_buf_ring_fail1: 2672 if (tx_ring_p->taskq) { 2673 ddi_taskq_destroy(tx_ring_p->taskq); 2674 tx_ring_p->taskq = NULL; 2675 } 2676 2677 index--; 2678 for (; index >= 0; index--) { 2679 if (tx_msg_ring[index].dma_handle != NULL) { 2680 ddi_dma_free_handle(&tx_msg_ring[index].dma_handle); 2681 } 2682 } 2683 MUTEX_DESTROY(&tx_ring_p->lock); 2684 KMEM_FREE(tx_msg_ring, size); 2685 KMEM_FREE(tx_ring_p, sizeof (tx_ring_t)); 2686 2687 status = NXGE_ERROR; 2688 2689 nxge_map_txdma_channel_buf_ring_exit: 2690 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2691 "<== nxge_map_txdma_channel_buf_ring status 0x%x", status)); 2692 2693 return (status); 2694 } 2695 2696 /*ARGSUSED*/ 2697 static void 2698 nxge_unmap_txdma_channel_buf_ring(p_nxge_t nxgep, p_tx_ring_t tx_ring_p) 2699 { 2700 p_tx_msg_t tx_msg_ring; 2701 p_tx_msg_t tx_msg_p; 2702 int i; 2703 2704 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2705 "==> nxge_unmap_txdma_channel_buf_ring")); 2706 if (tx_ring_p == NULL) { 2707 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2708 "<== nxge_unmap_txdma_channel_buf_ring: NULL ringp")); 2709 return; 2710 } 2711 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2712 "==> nxge_unmap_txdma_channel_buf_ring: channel %d", 2713 tx_ring_p->tdc)); 2714 2715 tx_msg_ring = tx_ring_p->tx_msg_ring; 2716 2717 /* 2718 * Since the serialization thread, timer thread and 2719 * interrupt thread can all call the transmit reclaim, 2720 * the unmapping function needs to acquire the lock 2721 * to free those buffers which were transmitted 2722 * by the hardware already. 2723 */ 2724 MUTEX_ENTER(&tx_ring_p->lock); 2725 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2726 "==> nxge_unmap_txdma_channel_buf_ring (reclaim): " 2727 "channel %d", 2728 tx_ring_p->tdc)); 2729 (void) nxge_txdma_reclaim(nxgep, tx_ring_p, 0); 2730 2731 for (i = 0; i < tx_ring_p->tx_ring_size; i++) { 2732 tx_msg_p = &tx_msg_ring[i]; 2733 if (tx_msg_p->tx_message != NULL) { 2734 freemsg(tx_msg_p->tx_message); 2735 tx_msg_p->tx_message = NULL; 2736 } 2737 } 2738 2739 for (i = 0; i < tx_ring_p->tx_ring_size; i++) { 2740 if (tx_msg_ring[i].dma_handle != NULL) { 2741 ddi_dma_free_handle(&tx_msg_ring[i].dma_handle); 2742 } 2743 tx_msg_ring[i].dma_handle = NULL; 2744 } 2745 2746 MUTEX_EXIT(&tx_ring_p->lock); 2747 2748 if (tx_ring_p->taskq) { 2749 ddi_taskq_destroy(tx_ring_p->taskq); 2750 tx_ring_p->taskq = NULL; 2751 } 2752 2753 MUTEX_DESTROY(&tx_ring_p->lock); 2754 KMEM_FREE(tx_msg_ring, sizeof (tx_msg_t) * tx_ring_p->tx_ring_size); 2755 KMEM_FREE(tx_ring_p, sizeof (tx_ring_t)); 2756 2757 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2758 "<== nxge_unmap_txdma_channel_buf_ring")); 2759 } 2760 2761 static nxge_status_t 2762 nxge_txdma_hw_start(p_nxge_t nxgep, int channel) 2763 { 2764 p_tx_rings_t tx_rings; 2765 p_tx_ring_t *tx_desc_rings; 2766 p_tx_mbox_areas_t tx_mbox_areas_p; 2767 p_tx_mbox_t *tx_mbox_p; 2768 nxge_status_t status = NXGE_OK; 2769 2770 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start")); 2771 2772 tx_rings = nxgep->tx_rings; 2773 if (tx_rings == NULL) { 2774 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2775 "<== nxge_txdma_hw_start: NULL ring pointer")); 2776 return (NXGE_ERROR); 2777 } 2778 tx_desc_rings = tx_rings->rings; 2779 if (tx_desc_rings == NULL) { 2780 NXGE_DEBUG_MSG((nxgep, TX_CTL, 2781 "<== nxge_txdma_hw_start: NULL ring pointers")); 2782 return (NXGE_ERROR); 2783 } 2784 2785 NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start: " 2786 "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings)); 2787 2788 tx_mbox_areas_p = nxgep->tx_mbox_areas_p; 2789 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p; 2790 2791 status = nxge_txdma_start_channel(nxgep, channel, 2792 (p_tx_ring_t)tx_desc_rings[channel], 2793 (p_tx_mbox_t)tx_mbox_p[channel]); 2794 if (status != NXGE_OK) { 2795 goto nxge_txdma_hw_start_fail1; 2796 } 2797 2798 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start: " 2799 "tx_rings $%p rings $%p", 2800 nxgep->tx_rings, nxgep->tx_rings->rings)); 2801 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start: " 2802 "tx_rings $%p tx_desc_rings $%p", 2803 nxgep->tx_rings, tx_desc_rings)); 2804 2805 goto nxge_txdma_hw_start_exit; 2806 2807 nxge_txdma_hw_start_fail1: 2808 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2809 "==> nxge_txdma_hw_start: disable " 2810 "(status 0x%x channel %d)", status, channel)); 2811 2812 nxge_txdma_hw_start_exit: 2813 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2814 "==> nxge_txdma_hw_start: (status 0x%x)", status)); 2815 2816 return (status); 2817 } 2818 2819 /* 2820 * nxge_txdma_start_channel 2821 * 2822 * Start a TDC. 2823 * 2824 * Arguments: 2825 * nxgep 2826 * channel The channel to start. 2827 * tx_ring_p channel's transmit descriptor ring. 2828 * tx_mbox_p channel' smailbox. 2829 * 2830 * Notes: 2831 * 2832 * NPI/NXGE function calls: 2833 * nxge_reset_txdma_channel() 2834 * nxge_init_txdma_channel_event_mask() 2835 * nxge_enable_txdma_channel() 2836 * 2837 * Registers accessed: 2838 * none directly (see functions above). 2839 * 2840 * Context: 2841 * Any domain 2842 */ 2843 static nxge_status_t 2844 nxge_txdma_start_channel(p_nxge_t nxgep, uint16_t channel, 2845 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p) 2846 2847 { 2848 nxge_status_t status = NXGE_OK; 2849 2850 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2851 "==> nxge_txdma_start_channel (channel %d)", channel)); 2852 /* 2853 * TXDMA/TXC must be in stopped state. 2854 */ 2855 (void) nxge_txdma_stop_inj_err(nxgep, channel); 2856 2857 /* 2858 * Reset TXDMA channel 2859 */ 2860 tx_ring_p->tx_cs.value = 0; 2861 tx_ring_p->tx_cs.bits.ldw.rst = 1; 2862 status = nxge_reset_txdma_channel(nxgep, channel, 2863 tx_ring_p->tx_cs.value); 2864 if (status != NXGE_OK) { 2865 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2866 "==> nxge_txdma_start_channel (channel %d)" 2867 " reset channel failed 0x%x", channel, status)); 2868 goto nxge_txdma_start_channel_exit; 2869 } 2870 2871 /* 2872 * Initialize the TXDMA channel specific FZC control 2873 * configurations. These FZC registers are pertaining 2874 * to each TX channel (i.e. logical pages). 2875 */ 2876 if (!isLDOMguest(nxgep)) { 2877 status = nxge_init_fzc_txdma_channel(nxgep, channel, 2878 tx_ring_p, tx_mbox_p); 2879 if (status != NXGE_OK) { 2880 goto nxge_txdma_start_channel_exit; 2881 } 2882 } 2883 2884 /* 2885 * Initialize the event masks. 2886 */ 2887 tx_ring_p->tx_evmask.value = 0; 2888 status = nxge_init_txdma_channel_event_mask(nxgep, 2889 channel, &tx_ring_p->tx_evmask); 2890 if (status != NXGE_OK) { 2891 goto nxge_txdma_start_channel_exit; 2892 } 2893 2894 /* 2895 * Load TXDMA descriptors, buffers, mailbox, 2896 * initialise the DMA channels and 2897 * enable each DMA channel. 2898 */ 2899 status = nxge_enable_txdma_channel(nxgep, channel, 2900 tx_ring_p, tx_mbox_p); 2901 if (status != NXGE_OK) { 2902 goto nxge_txdma_start_channel_exit; 2903 } 2904 2905 nxge_txdma_start_channel_exit: 2906 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_txdma_start_channel")); 2907 2908 return (status); 2909 } 2910 2911 /* 2912 * nxge_txdma_stop_channel 2913 * 2914 * Stop a TDC. 2915 * 2916 * Arguments: 2917 * nxgep 2918 * channel The channel to stop. 2919 * tx_ring_p channel's transmit descriptor ring. 2920 * tx_mbox_p channel' smailbox. 2921 * 2922 * Notes: 2923 * 2924 * NPI/NXGE function calls: 2925 * nxge_txdma_stop_inj_err() 2926 * nxge_reset_txdma_channel() 2927 * nxge_init_txdma_channel_event_mask() 2928 * nxge_init_txdma_channel_cntl_stat() 2929 * nxge_disable_txdma_channel() 2930 * 2931 * Registers accessed: 2932 * none directly (see functions above). 2933 * 2934 * Context: 2935 * Any domain 2936 */ 2937 /*ARGSUSED*/ 2938 static nxge_status_t 2939 nxge_txdma_stop_channel(p_nxge_t nxgep, uint16_t channel) 2940 { 2941 p_tx_ring_t tx_ring_p; 2942 int status = NXGE_OK; 2943 2944 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 2945 "==> nxge_txdma_stop_channel: channel %d", channel)); 2946 2947 /* 2948 * Stop (disable) TXDMA and TXC (if stop bit is set 2949 * and STOP_N_GO bit not set, the TXDMA reset state will 2950 * not be set if reset TXDMA. 2951 */ 2952 (void) nxge_txdma_stop_inj_err(nxgep, channel); 2953 2954 if (nxgep->tx_rings == NULL) { 2955 status = NXGE_ERROR; 2956 goto nxge_txdma_stop_channel_exit; 2957 } 2958 2959 tx_ring_p = nxgep->tx_rings->rings[channel]; 2960 if (tx_ring_p == NULL) { 2961 status = NXGE_ERROR; 2962 goto nxge_txdma_stop_channel_exit; 2963 } 2964 2965 /* 2966 * Reset TXDMA channel 2967 */ 2968 tx_ring_p->tx_cs.value = 0; 2969 tx_ring_p->tx_cs.bits.ldw.rst = 1; 2970 status = nxge_reset_txdma_channel(nxgep, channel, 2971 tx_ring_p->tx_cs.value); 2972 if (status != NXGE_OK) { 2973 goto nxge_txdma_stop_channel_exit; 2974 } 2975 2976 #ifdef HARDWARE_REQUIRED 2977 /* Set up the interrupt event masks. */ 2978 tx_ring_p->tx_evmask.value = 0; 2979 status = nxge_init_txdma_channel_event_mask(nxgep, 2980 channel, &tx_ring_p->tx_evmask); 2981 if (status != NXGE_OK) { 2982 goto nxge_txdma_stop_channel_exit; 2983 } 2984 2985 /* Initialize the DMA control and status register */ 2986 tx_ring_p->tx_cs.value = TX_ENT_MSK_MK_ALL; 2987 status = nxge_init_txdma_channel_cntl_stat(nxgep, channel, 2988 tx_ring_p->tx_cs.value); 2989 if (status != NXGE_OK) { 2990 goto nxge_txdma_stop_channel_exit; 2991 } 2992 2993 tx_mbox_p = nxgep->tx_mbox_areas_p->txmbox_areas_p[channel]; 2994 2995 /* Disable channel */ 2996 status = nxge_disable_txdma_channel(nxgep, channel, 2997 tx_ring_p, tx_mbox_p); 2998 if (status != NXGE_OK) { 2999 goto nxge_txdma_start_channel_exit; 3000 } 3001 3002 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, 3003 "==> nxge_txdma_stop_channel: event done")); 3004 3005 #endif 3006 3007 nxge_txdma_stop_channel_exit: 3008 NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_txdma_stop_channel")); 3009 return (status); 3010 } 3011 3012 /* 3013 * nxge_txdma_get_ring 3014 * 3015 * Get the ring for a TDC. 3016 * 3017 * Arguments: 3018 * nxgep 3019 * channel 3020 * 3021 * Notes: 3022 * 3023 * NPI/NXGE function calls: 3024 * 3025 * Registers accessed: 3026 * 3027 * Context: 3028 * Any domain 3029 */ 3030 static p_tx_ring_t 3031 nxge_txdma_get_ring(p_nxge_t nxgep, uint16_t channel) 3032 { 3033 nxge_grp_set_t *set = &nxgep->tx_set; 3034 int tdc; 3035 3036 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_get_ring")); 3037 3038 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 3039 NXGE_DEBUG_MSG((nxgep, TX_CTL, 3040 "<== nxge_txdma_get_ring: NULL ring pointer(s)")); 3041 goto return_null; 3042 } 3043 3044 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3045 if ((1 << tdc) & set->owned.map) { 3046 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 3047 if (ring) { 3048 if (channel == ring->tdc) { 3049 NXGE_DEBUG_MSG((nxgep, TX_CTL, 3050 "<== nxge_txdma_get_ring: " 3051 "tdc %d ring $%p", tdc, ring)); 3052 return (ring); 3053 } 3054 } 3055 } 3056 } 3057 3058 return_null: 3059 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_get_ring: " 3060 "ring not found")); 3061 3062 return (NULL); 3063 } 3064 3065 /* 3066 * nxge_txdma_get_mbox 3067 * 3068 * Get the mailbox for a TDC. 3069 * 3070 * Arguments: 3071 * nxgep 3072 * channel 3073 * 3074 * Notes: 3075 * 3076 * NPI/NXGE function calls: 3077 * 3078 * Registers accessed: 3079 * 3080 * Context: 3081 * Any domain 3082 */ 3083 static p_tx_mbox_t 3084 nxge_txdma_get_mbox(p_nxge_t nxgep, uint16_t channel) 3085 { 3086 nxge_grp_set_t *set = &nxgep->tx_set; 3087 int tdc; 3088 3089 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_get_mbox")); 3090 3091 if (nxgep->tx_mbox_areas_p == 0 || 3092 nxgep->tx_mbox_areas_p->txmbox_areas_p == 0) { 3093 NXGE_DEBUG_MSG((nxgep, TX_CTL, 3094 "<== nxge_txdma_get_mbox: NULL mailbox pointer(s)")); 3095 goto return_null; 3096 } 3097 3098 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 3099 NXGE_DEBUG_MSG((nxgep, TX_CTL, 3100 "<== nxge_txdma_get_mbox: NULL ring pointer(s)")); 3101 goto return_null; 3102 } 3103 3104 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3105 if ((1 << tdc) & set->owned.map) { 3106 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 3107 if (ring) { 3108 if (channel == ring->tdc) { 3109 tx_mbox_t *mailbox = nxgep-> 3110 tx_mbox_areas_p-> 3111 txmbox_areas_p[tdc]; 3112 NXGE_DEBUG_MSG((nxgep, TX_CTL, 3113 "<== nxge_txdma_get_mbox: tdc %d " 3114 "ring $%p", tdc, mailbox)); 3115 return (mailbox); 3116 } 3117 } 3118 } 3119 } 3120 3121 return_null: 3122 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_get_mbox: " 3123 "mailbox not found")); 3124 3125 return (NULL); 3126 } 3127 3128 /* 3129 * nxge_tx_err_evnts 3130 * 3131 * Recover a TDC. 3132 * 3133 * Arguments: 3134 * nxgep 3135 * index The index to the TDC ring. 3136 * ldvp Used to get the channel number ONLY. 3137 * cs A copy of the bits from TX_CS. 3138 * 3139 * Notes: 3140 * Calling tree: 3141 * nxge_tx_intr() 3142 * 3143 * NPI/NXGE function calls: 3144 * npi_txdma_ring_error_get() 3145 * npi_txdma_inj_par_error_get() 3146 * nxge_txdma_fatal_err_recover() 3147 * 3148 * Registers accessed: 3149 * TX_RNG_ERR_LOGH DMC+0x40048 Transmit Ring Error Log High 3150 * TX_RNG_ERR_LOGL DMC+0x40050 Transmit Ring Error Log Low 3151 * TDMC_INJ_PAR_ERR (FZC_DMC + 0x45040) TDMC Inject Parity Error 3152 * 3153 * Context: 3154 * Any domain XXX Remove code which accesses TDMC_INJ_PAR_ERR. 3155 */ 3156 /*ARGSUSED*/ 3157 static nxge_status_t 3158 nxge_tx_err_evnts(p_nxge_t nxgep, uint_t index, p_nxge_ldv_t ldvp, tx_cs_t cs) 3159 { 3160 npi_handle_t handle; 3161 npi_status_t rs; 3162 uint8_t channel; 3163 p_tx_ring_t *tx_rings; 3164 p_tx_ring_t tx_ring_p; 3165 p_nxge_tx_ring_stats_t tdc_stats; 3166 boolean_t txchan_fatal = B_FALSE; 3167 nxge_status_t status = NXGE_OK; 3168 tdmc_inj_par_err_t par_err; 3169 uint32_t value; 3170 3171 NXGE_DEBUG_MSG((nxgep, TX2_CTL, "==> nxge_tx_err_evnts")); 3172 handle = NXGE_DEV_NPI_HANDLE(nxgep); 3173 channel = ldvp->channel; 3174 3175 tx_rings = nxgep->tx_rings->rings; 3176 tx_ring_p = tx_rings[index]; 3177 tdc_stats = tx_ring_p->tdc_stats; 3178 if ((cs.bits.ldw.pkt_size_err) || (cs.bits.ldw.pref_buf_par_err) || 3179 (cs.bits.ldw.nack_pref) || (cs.bits.ldw.nack_pkt_rd) || 3180 (cs.bits.ldw.conf_part_err) || (cs.bits.ldw.pkt_prt_err)) { 3181 if ((rs = npi_txdma_ring_error_get(handle, channel, 3182 &tdc_stats->errlog)) != NPI_SUCCESS) 3183 return (NXGE_ERROR | rs); 3184 } 3185 3186 if (cs.bits.ldw.mbox_err) { 3187 tdc_stats->mbox_err++; 3188 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel, 3189 NXGE_FM_EREPORT_TDMC_MBOX_ERR); 3190 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3191 "==> nxge_tx_err_evnts(channel %d): " 3192 "fatal error: mailbox", channel)); 3193 txchan_fatal = B_TRUE; 3194 } 3195 if (cs.bits.ldw.pkt_size_err) { 3196 tdc_stats->pkt_size_err++; 3197 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel, 3198 NXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR); 3199 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3200 "==> nxge_tx_err_evnts(channel %d): " 3201 "fatal error: pkt_size_err", channel)); 3202 txchan_fatal = B_TRUE; 3203 } 3204 if (cs.bits.ldw.tx_ring_oflow) { 3205 tdc_stats->tx_ring_oflow++; 3206 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel, 3207 NXGE_FM_EREPORT_TDMC_TX_RING_OFLOW); 3208 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3209 "==> nxge_tx_err_evnts(channel %d): " 3210 "fatal error: tx_ring_oflow", channel)); 3211 txchan_fatal = B_TRUE; 3212 } 3213 if (cs.bits.ldw.pref_buf_par_err) { 3214 tdc_stats->pre_buf_par_err++; 3215 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel, 3216 NXGE_FM_EREPORT_TDMC_PREF_BUF_PAR_ERR); 3217 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3218 "==> nxge_tx_err_evnts(channel %d): " 3219 "fatal error: pre_buf_par_err", channel)); 3220 /* Clear error injection source for parity error */ 3221 (void) npi_txdma_inj_par_error_get(handle, &value); 3222 par_err.value = value; 3223 par_err.bits.ldw.inject_parity_error &= ~(1 << channel); 3224 (void) npi_txdma_inj_par_error_set(handle, par_err.value); 3225 txchan_fatal = B_TRUE; 3226 } 3227 if (cs.bits.ldw.nack_pref) { 3228 tdc_stats->nack_pref++; 3229 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel, 3230 NXGE_FM_EREPORT_TDMC_NACK_PREF); 3231 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3232 "==> nxge_tx_err_evnts(channel %d): " 3233 "fatal error: nack_pref", channel)); 3234 txchan_fatal = B_TRUE; 3235 } 3236 if (cs.bits.ldw.nack_pkt_rd) { 3237 tdc_stats->nack_pkt_rd++; 3238 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel, 3239 NXGE_FM_EREPORT_TDMC_NACK_PKT_RD); 3240 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3241 "==> nxge_tx_err_evnts(channel %d): " 3242 "fatal error: nack_pkt_rd", channel)); 3243 txchan_fatal = B_TRUE; 3244 } 3245 if (cs.bits.ldw.conf_part_err) { 3246 tdc_stats->conf_part_err++; 3247 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel, 3248 NXGE_FM_EREPORT_TDMC_CONF_PART_ERR); 3249 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3250 "==> nxge_tx_err_evnts(channel %d): " 3251 "fatal error: config_partition_err", channel)); 3252 txchan_fatal = B_TRUE; 3253 } 3254 if (cs.bits.ldw.pkt_prt_err) { 3255 tdc_stats->pkt_part_err++; 3256 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel, 3257 NXGE_FM_EREPORT_TDMC_PKT_PRT_ERR); 3258 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3259 "==> nxge_tx_err_evnts(channel %d): " 3260 "fatal error: pkt_prt_err", channel)); 3261 txchan_fatal = B_TRUE; 3262 } 3263 3264 /* Clear error injection source in case this is an injected error */ 3265 TXDMA_REG_WRITE64(nxgep->npi_handle, TDMC_INTR_DBG_REG, channel, 0); 3266 3267 if (txchan_fatal) { 3268 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3269 " nxge_tx_err_evnts: " 3270 " fatal error on channel %d cs 0x%llx\n", 3271 channel, cs.value)); 3272 status = nxge_txdma_fatal_err_recover(nxgep, channel, 3273 tx_ring_p); 3274 if (status == NXGE_OK) { 3275 FM_SERVICE_RESTORED(nxgep); 3276 } 3277 } 3278 3279 NXGE_DEBUG_MSG((nxgep, TX2_CTL, "<== nxge_tx_err_evnts")); 3280 3281 return (status); 3282 } 3283 3284 static nxge_status_t 3285 nxge_txdma_fatal_err_recover( 3286 p_nxge_t nxgep, 3287 uint16_t channel, 3288 p_tx_ring_t tx_ring_p) 3289 { 3290 npi_handle_t handle; 3291 npi_status_t rs = NPI_SUCCESS; 3292 p_tx_mbox_t tx_mbox_p; 3293 nxge_status_t status = NXGE_OK; 3294 3295 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fatal_err_recover")); 3296 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3297 "Recovering from TxDMAChannel#%d error...", channel)); 3298 3299 /* 3300 * Stop the dma channel waits for the stop done. 3301 * If the stop done bit is not set, then create 3302 * an error. 3303 */ 3304 3305 handle = NXGE_DEV_NPI_HANDLE(nxgep); 3306 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel stop...")); 3307 MUTEX_ENTER(&tx_ring_p->lock); 3308 rs = npi_txdma_channel_control(handle, TXDMA_STOP, channel); 3309 if (rs != NPI_SUCCESS) { 3310 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3311 "==> nxge_txdma_fatal_err_recover (channel %d): " 3312 "stop failed ", channel)); 3313 goto fail; 3314 } 3315 3316 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel reclaim...")); 3317 (void) nxge_txdma_reclaim(nxgep, tx_ring_p, 0); 3318 3319 /* 3320 * Reset TXDMA channel 3321 */ 3322 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel reset...")); 3323 if ((rs = npi_txdma_channel_control(handle, TXDMA_RESET, channel)) != 3324 NPI_SUCCESS) { 3325 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3326 "==> nxge_txdma_fatal_err_recover (channel %d)" 3327 " reset channel failed 0x%x", channel, rs)); 3328 goto fail; 3329 } 3330 3331 /* 3332 * Reset the tail (kick) register to 0. 3333 * (Hardware will not reset it. Tx overflow fatal 3334 * error if tail is not set to 0 after reset! 3335 */ 3336 TXDMA_REG_WRITE64(handle, TX_RING_KICK_REG, channel, 0); 3337 3338 /* Restart TXDMA channel */ 3339 3340 if (!isLDOMguest(nxgep)) { 3341 tx_mbox_p = nxge_txdma_get_mbox(nxgep, channel); 3342 3343 // XXX This is a problem in HIO! 3344 /* 3345 * Initialize the TXDMA channel specific FZC control 3346 * configurations. These FZC registers are pertaining 3347 * to each TX channel (i.e. logical pages). 3348 */ 3349 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel restart...")); 3350 status = nxge_init_fzc_txdma_channel(nxgep, channel, 3351 tx_ring_p, tx_mbox_p); 3352 if (status != NXGE_OK) 3353 goto fail; 3354 } 3355 3356 /* 3357 * Initialize the event masks. 3358 */ 3359 tx_ring_p->tx_evmask.value = 0; 3360 status = nxge_init_txdma_channel_event_mask(nxgep, channel, 3361 &tx_ring_p->tx_evmask); 3362 if (status != NXGE_OK) 3363 goto fail; 3364 3365 tx_ring_p->wr_index_wrap = B_FALSE; 3366 tx_ring_p->wr_index = 0; 3367 tx_ring_p->rd_index = 0; 3368 3369 /* 3370 * Load TXDMA descriptors, buffers, mailbox, 3371 * initialise the DMA channels and 3372 * enable each DMA channel. 3373 */ 3374 NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel enable...")); 3375 status = nxge_enable_txdma_channel(nxgep, channel, 3376 tx_ring_p, tx_mbox_p); 3377 MUTEX_EXIT(&tx_ring_p->lock); 3378 if (status != NXGE_OK) 3379 goto fail; 3380 3381 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3382 "Recovery Successful, TxDMAChannel#%d Restored", 3383 channel)); 3384 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fatal_err_recover")); 3385 3386 return (NXGE_OK); 3387 3388 fail: 3389 MUTEX_EXIT(&tx_ring_p->lock); 3390 3391 NXGE_DEBUG_MSG((nxgep, TX_CTL, 3392 "nxge_txdma_fatal_err_recover (channel %d): " 3393 "failed to recover this txdma channel", channel)); 3394 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed")); 3395 3396 return (status); 3397 } 3398 3399 /* 3400 * nxge_tx_port_fatal_err_recover 3401 * 3402 * Attempt to recover from a fatal port error. 3403 * 3404 * Arguments: 3405 * nxgep 3406 * 3407 * Notes: 3408 * How would a guest do this? 3409 * 3410 * NPI/NXGE function calls: 3411 * 3412 * Registers accessed: 3413 * 3414 * Context: 3415 * Service domain 3416 */ 3417 nxge_status_t 3418 nxge_tx_port_fatal_err_recover(p_nxge_t nxgep) 3419 { 3420 nxge_grp_set_t *set = &nxgep->tx_set; 3421 nxge_channel_t tdc; 3422 3423 tx_ring_t *ring; 3424 tx_mbox_t *mailbox; 3425 3426 npi_handle_t handle; 3427 nxge_status_t status; 3428 npi_status_t rs; 3429 3430 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_tx_port_fatal_err_recover")); 3431 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3432 "Recovering from TxPort error...")); 3433 3434 if (isLDOMguest(nxgep)) { 3435 return (NXGE_OK); 3436 } 3437 3438 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 3439 NXGE_DEBUG_MSG((nxgep, TX_CTL, 3440 "<== nxge_tx_port_fatal_err_recover: not initialized")); 3441 return (NXGE_ERROR); 3442 } 3443 3444 if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) { 3445 NXGE_DEBUG_MSG((nxgep, TX_CTL, 3446 "<== nxge_tx_port_fatal_err_recover: " 3447 "NULL ring pointer(s)")); 3448 return (NXGE_ERROR); 3449 } 3450 3451 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3452 if ((1 << tdc) & set->owned.map) { 3453 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 3454 if (ring) 3455 MUTEX_ENTER(&ring->lock); 3456 } 3457 } 3458 3459 handle = NXGE_DEV_NPI_HANDLE(nxgep); 3460 3461 /* 3462 * Stop all the TDCs owned by us. 3463 * (The shared TDCs will have been stopped by their owners.) 3464 */ 3465 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3466 if ((1 << tdc) & set->owned.map) { 3467 ring = nxgep->tx_rings->rings[tdc]; 3468 if (ring) { 3469 rs = npi_txdma_channel_control 3470 (handle, TXDMA_STOP, tdc); 3471 if (rs != NPI_SUCCESS) { 3472 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3473 "nxge_tx_port_fatal_err_recover " 3474 "(channel %d): stop failed ", tdc)); 3475 goto fail; 3476 } 3477 } 3478 } 3479 } 3480 3481 NXGE_DEBUG_MSG((nxgep, TX_CTL, "Reclaiming all TDCs...")); 3482 3483 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3484 if ((1 << tdc) & set->owned.map) { 3485 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 3486 if (ring) { 3487 (void) nxge_txdma_reclaim(nxgep, ring, 0); 3488 } 3489 } 3490 } 3491 3492 /* 3493 * Reset all the TDCs. 3494 */ 3495 NXGE_DEBUG_MSG((nxgep, TX_CTL, "Resetting all TDCs...")); 3496 3497 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3498 if ((1 << tdc) & set->owned.map) { 3499 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 3500 if (ring) { 3501 if ((rs = npi_txdma_channel_control 3502 (handle, TXDMA_RESET, tdc)) 3503 != NPI_SUCCESS) { 3504 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3505 "nxge_tx_port_fatal_err_recover " 3506 "(channel %d) reset channel " 3507 "failed 0x%x", tdc, rs)); 3508 goto fail; 3509 } 3510 } 3511 /* 3512 * Reset the tail (kick) register to 0. 3513 * (Hardware will not reset it. Tx overflow fatal 3514 * error if tail is not set to 0 after reset! 3515 */ 3516 TXDMA_REG_WRITE64(handle, TX_RING_KICK_REG, tdc, 0); 3517 } 3518 } 3519 3520 NXGE_DEBUG_MSG((nxgep, TX_CTL, "Restarting all TDCs...")); 3521 3522 /* Restart all the TDCs */ 3523 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3524 if ((1 << tdc) & set->owned.map) { 3525 ring = nxgep->tx_rings->rings[tdc]; 3526 if (ring) { 3527 mailbox = nxge_txdma_get_mbox(nxgep, tdc); 3528 status = nxge_init_fzc_txdma_channel(nxgep, tdc, 3529 ring, mailbox); 3530 ring->tx_evmask.value = 0; 3531 /* 3532 * Initialize the event masks. 3533 */ 3534 status = nxge_init_txdma_channel_event_mask 3535 (nxgep, tdc, &ring->tx_evmask); 3536 3537 ring->wr_index_wrap = B_FALSE; 3538 ring->wr_index = 0; 3539 ring->rd_index = 0; 3540 3541 if (status != NXGE_OK) 3542 goto fail; 3543 if (status != NXGE_OK) 3544 goto fail; 3545 } 3546 } 3547 } 3548 3549 NXGE_DEBUG_MSG((nxgep, TX_CTL, "Re-enabling all TDCs...")); 3550 3551 /* Re-enable all the TDCs */ 3552 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3553 if ((1 << tdc) & set->owned.map) { 3554 ring = nxgep->tx_rings->rings[tdc]; 3555 if (ring) { 3556 mailbox = nxge_txdma_get_mbox(nxgep, tdc); 3557 status = nxge_enable_txdma_channel(nxgep, tdc, 3558 ring, mailbox); 3559 if (status != NXGE_OK) 3560 goto fail; 3561 } 3562 } 3563 } 3564 3565 /* 3566 * Unlock all the TDCs. 3567 */ 3568 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3569 if ((1 << tdc) & set->owned.map) { 3570 tx_ring_t *ring = nxgep->tx_rings->rings[tdc]; 3571 if (ring) 3572 MUTEX_EXIT(&ring->lock); 3573 } 3574 } 3575 3576 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Tx port recovery succeeded")); 3577 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_port_fatal_err_recover")); 3578 3579 return (NXGE_OK); 3580 3581 fail: 3582 for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) { 3583 if ((1 << tdc) & set->owned.map) { 3584 ring = nxgep->tx_rings->rings[tdc]; 3585 if (ring) 3586 MUTEX_EXIT(&ring->lock); 3587 } 3588 } 3589 3590 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Tx port recovery failed")); 3591 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_port_fatal_err_recover")); 3592 3593 return (status); 3594 } 3595 3596 /* 3597 * nxge_txdma_inject_err 3598 * 3599 * Inject an error into a TDC. 3600 * 3601 * Arguments: 3602 * nxgep 3603 * err_id The error to inject. 3604 * chan The channel to inject into. 3605 * 3606 * Notes: 3607 * This is called from nxge_main.c:nxge_err_inject() 3608 * Has this ioctl ever been used? 3609 * 3610 * NPI/NXGE function calls: 3611 * npi_txdma_inj_par_error_get() 3612 * npi_txdma_inj_par_error_set() 3613 * 3614 * Registers accessed: 3615 * TDMC_INJ_PAR_ERR (FZC_DMC + 0x45040) TDMC Inject Parity Error 3616 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug 3617 * TDMC_INTR_DBG DMC + 0x40060 Transmit DMA Interrupt Debug 3618 * 3619 * Context: 3620 * Service domain 3621 */ 3622 void 3623 nxge_txdma_inject_err(p_nxge_t nxgep, uint32_t err_id, uint8_t chan) 3624 { 3625 tdmc_intr_dbg_t tdi; 3626 tdmc_inj_par_err_t par_err; 3627 uint32_t value; 3628 npi_handle_t handle; 3629 3630 switch (err_id) { 3631 3632 case NXGE_FM_EREPORT_TDMC_PREF_BUF_PAR_ERR: 3633 handle = NXGE_DEV_NPI_HANDLE(nxgep); 3634 /* Clear error injection source for parity error */ 3635 (void) npi_txdma_inj_par_error_get(handle, &value); 3636 par_err.value = value; 3637 par_err.bits.ldw.inject_parity_error &= ~(1 << chan); 3638 (void) npi_txdma_inj_par_error_set(handle, par_err.value); 3639 3640 par_err.bits.ldw.inject_parity_error = (1 << chan); 3641 (void) npi_txdma_inj_par_error_get(handle, &value); 3642 par_err.value = value; 3643 par_err.bits.ldw.inject_parity_error |= (1 << chan); 3644 cmn_err(CE_NOTE, "!Write 0x%llx to TDMC_INJ_PAR_ERR_REG\n", 3645 (unsigned long long)par_err.value); 3646 (void) npi_txdma_inj_par_error_set(handle, par_err.value); 3647 break; 3648 3649 case NXGE_FM_EREPORT_TDMC_MBOX_ERR: 3650 case NXGE_FM_EREPORT_TDMC_NACK_PREF: 3651 case NXGE_FM_EREPORT_TDMC_NACK_PKT_RD: 3652 case NXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR: 3653 case NXGE_FM_EREPORT_TDMC_TX_RING_OFLOW: 3654 case NXGE_FM_EREPORT_TDMC_CONF_PART_ERR: 3655 case NXGE_FM_EREPORT_TDMC_PKT_PRT_ERR: 3656 TXDMA_REG_READ64(nxgep->npi_handle, TDMC_INTR_DBG_REG, 3657 chan, &tdi.value); 3658 if (err_id == NXGE_FM_EREPORT_TDMC_PREF_BUF_PAR_ERR) 3659 tdi.bits.ldw.pref_buf_par_err = 1; 3660 else if (err_id == NXGE_FM_EREPORT_TDMC_MBOX_ERR) 3661 tdi.bits.ldw.mbox_err = 1; 3662 else if (err_id == NXGE_FM_EREPORT_TDMC_NACK_PREF) 3663 tdi.bits.ldw.nack_pref = 1; 3664 else if (err_id == NXGE_FM_EREPORT_TDMC_NACK_PKT_RD) 3665 tdi.bits.ldw.nack_pkt_rd = 1; 3666 else if (err_id == NXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR) 3667 tdi.bits.ldw.pkt_size_err = 1; 3668 else if (err_id == NXGE_FM_EREPORT_TDMC_TX_RING_OFLOW) 3669 tdi.bits.ldw.tx_ring_oflow = 1; 3670 else if (err_id == NXGE_FM_EREPORT_TDMC_CONF_PART_ERR) 3671 tdi.bits.ldw.conf_part_err = 1; 3672 else if (err_id == NXGE_FM_EREPORT_TDMC_PKT_PRT_ERR) 3673 tdi.bits.ldw.pkt_part_err = 1; 3674 #if defined(__i386) 3675 cmn_err(CE_NOTE, "!Write 0x%llx to TDMC_INTR_DBG_REG\n", 3676 tdi.value); 3677 #else 3678 cmn_err(CE_NOTE, "!Write 0x%lx to TDMC_INTR_DBG_REG\n", 3679 tdi.value); 3680 #endif 3681 TXDMA_REG_WRITE64(nxgep->npi_handle, TDMC_INTR_DBG_REG, 3682 chan, tdi.value); 3683 3684 break; 3685 } 3686 } 3687