1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/nxge/nxge_impl.h> 28 #include <sys/nxge/nxge_rxdma.h> 29 #include <sys/nxge/nxge_hio.h> 30 31 #if !defined(_BIG_ENDIAN) 32 #include <npi_rx_rd32.h> 33 #endif 34 #include <npi_rx_rd64.h> 35 #include <npi_rx_wr64.h> 36 37 #define NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp) \ 38 (rdcgrp + nxgep->pt_config.hw_config.def_mac_rxdma_grpid) 39 #define NXGE_ACTUAL_RDC(nxgep, rdc) \ 40 (rdc + nxgep->pt_config.hw_config.start_rdc) 41 42 /* 43 * Globals: tunable parameters (/etc/system or adb) 44 * 45 */ 46 extern uint32_t nxge_rbr_size; 47 extern uint32_t nxge_rcr_size; 48 extern uint32_t nxge_rbr_spare_size; 49 extern uint16_t nxge_rdc_buf_offset; 50 51 extern uint32_t nxge_mblks_pending; 52 53 /* 54 * Tunable to reduce the amount of time spent in the 55 * ISR doing Rx Processing. 56 */ 57 extern uint32_t nxge_max_rx_pkts; 58 59 /* 60 * Tunables to manage the receive buffer blocks. 61 * 62 * nxge_rx_threshold_hi: copy all buffers. 63 * nxge_rx_bcopy_size_type: receive buffer block size type. 64 * nxge_rx_threshold_lo: copy only up to tunable block size type. 65 */ 66 extern nxge_rxbuf_threshold_t nxge_rx_threshold_hi; 67 extern nxge_rxbuf_type_t nxge_rx_buf_size_type; 68 extern nxge_rxbuf_threshold_t nxge_rx_threshold_lo; 69 70 extern uint32_t nxge_cksum_offload; 71 72 static nxge_status_t nxge_map_rxdma(p_nxge_t, int); 73 static void nxge_unmap_rxdma(p_nxge_t, int); 74 75 static nxge_status_t nxge_rxdma_hw_start_common(p_nxge_t); 76 77 static nxge_status_t nxge_rxdma_hw_start(p_nxge_t, int); 78 static void nxge_rxdma_hw_stop(p_nxge_t, int); 79 80 static nxge_status_t nxge_map_rxdma_channel(p_nxge_t, uint16_t, 81 p_nxge_dma_common_t *, p_rx_rbr_ring_t *, 82 uint32_t, 83 p_nxge_dma_common_t *, p_rx_rcr_ring_t *, 84 p_rx_mbox_t *); 85 static void nxge_unmap_rxdma_channel(p_nxge_t, uint16_t, 86 p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t); 87 88 static nxge_status_t nxge_map_rxdma_channel_cfg_ring(p_nxge_t, 89 uint16_t, 90 p_nxge_dma_common_t *, p_rx_rbr_ring_t *, 91 p_rx_rcr_ring_t *, p_rx_mbox_t *); 92 static void nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t, 93 p_rx_rcr_ring_t, p_rx_mbox_t); 94 95 static nxge_status_t nxge_map_rxdma_channel_buf_ring(p_nxge_t, 96 uint16_t, 97 p_nxge_dma_common_t *, 98 p_rx_rbr_ring_t *, uint32_t); 99 static void nxge_unmap_rxdma_channel_buf_ring(p_nxge_t, 100 p_rx_rbr_ring_t); 101 102 static nxge_status_t nxge_rxdma_start_channel(p_nxge_t, uint16_t, 103 p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t); 104 static nxge_status_t nxge_rxdma_stop_channel(p_nxge_t, uint16_t); 105 106 static mblk_t * 107 nxge_rx_pkts(p_nxge_t, p_rx_rcr_ring_t, rx_dma_ctl_stat_t, int); 108 109 static void nxge_receive_packet(p_nxge_t, 110 p_rx_rcr_ring_t, 111 p_rcr_entry_t, 112 boolean_t *, 113 mblk_t **, mblk_t **); 114 115 nxge_status_t nxge_disable_rxdma_channel(p_nxge_t, uint16_t); 116 117 static p_rx_msg_t nxge_allocb(size_t, uint32_t, p_nxge_dma_common_t); 118 static void nxge_freeb(p_rx_msg_t); 119 static nxge_status_t nxge_rx_err_evnts(p_nxge_t, int, rx_dma_ctl_stat_t); 120 121 static nxge_status_t nxge_rxdma_handle_port_errors(p_nxge_t, 122 uint32_t, uint32_t); 123 124 static nxge_status_t nxge_rxbuf_index_info_init(p_nxge_t, 125 p_rx_rbr_ring_t); 126 127 128 static nxge_status_t 129 nxge_rxdma_fatal_err_recover(p_nxge_t, uint16_t); 130 131 nxge_status_t 132 nxge_rx_port_fatal_err_recover(p_nxge_t); 133 134 static void nxge_rxdma_databuf_free(p_rx_rbr_ring_t); 135 136 nxge_status_t 137 nxge_init_rxdma_channels(p_nxge_t nxgep) 138 { 139 nxge_grp_set_t *set = &nxgep->rx_set; 140 int i, count, channel; 141 nxge_grp_t *group; 142 dc_map_t map; 143 int dev_gindex; 144 145 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_init_rxdma_channels")); 146 147 if (!isLDOMguest(nxgep)) { 148 if (nxge_rxdma_hw_start_common(nxgep) != NXGE_OK) { 149 cmn_err(CE_NOTE, "hw_start_common"); 150 return (NXGE_ERROR); 151 } 152 } 153 154 /* 155 * NXGE_LOGICAL_GROUP_MAX > NXGE_MAX_RDC_GROUPS (8) 156 * We only have 8 hardware RDC tables, but we may have 157 * up to 16 logical (software-defined) groups of RDCS, 158 * if we make use of layer 3 & 4 hardware classification. 159 */ 160 for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) { 161 if ((1 << i) & set->lg.map) { 162 group = set->group[i]; 163 dev_gindex = 164 nxgep->pt_config.hw_config.def_mac_rxdma_grpid + i; 165 map = nxgep->pt_config.rdc_grps[dev_gindex].map; 166 for (channel = 0; channel < NXGE_MAX_RDCS; channel++) { 167 if ((1 << channel) & map) { 168 if ((nxge_grp_dc_add(nxgep, 169 group, VP_BOUND_RX, channel))) 170 goto init_rxdma_channels_exit; 171 } 172 } 173 } 174 if (++count == set->lg.count) 175 break; 176 } 177 178 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_rxdma_channels")); 179 return (NXGE_OK); 180 181 init_rxdma_channels_exit: 182 for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) { 183 if ((1 << i) & set->lg.map) { 184 group = set->group[i]; 185 dev_gindex = 186 nxgep->pt_config.hw_config.def_mac_rxdma_grpid + i; 187 map = nxgep->pt_config.rdc_grps[dev_gindex].map; 188 for (channel = 0; channel < NXGE_MAX_RDCS; channel++) { 189 if ((1 << channel) & map) { 190 nxge_grp_dc_remove(nxgep, 191 VP_BOUND_RX, channel); 192 } 193 } 194 } 195 if (++count == set->lg.count) 196 break; 197 } 198 199 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_rxdma_channels")); 200 return (NXGE_ERROR); 201 } 202 203 nxge_status_t 204 nxge_init_rxdma_channel(p_nxge_t nxge, int channel) 205 { 206 nxge_status_t status; 207 208 NXGE_DEBUG_MSG((nxge, MEM2_CTL, "==> nxge_init_rxdma_channel")); 209 210 status = nxge_map_rxdma(nxge, channel); 211 if (status != NXGE_OK) { 212 NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, 213 "<== nxge_init_rxdma: status 0x%x", status)); 214 return (status); 215 } 216 217 #if defined(sun4v) 218 if (isLDOMguest(nxge)) { 219 /* set rcr_ring */ 220 p_rx_rcr_ring_t ring = nxge->rx_rcr_rings->rcr_rings[channel]; 221 222 status = nxge_hio_rxdma_bind_intr(nxge, ring, channel); 223 if (status != NXGE_OK) { 224 nxge_unmap_rxdma(nxge, channel); 225 return (status); 226 } 227 } 228 #endif 229 230 status = nxge_rxdma_hw_start(nxge, channel); 231 if (status != NXGE_OK) { 232 nxge_unmap_rxdma(nxge, channel); 233 } 234 235 if (!nxge->statsp->rdc_ksp[channel]) 236 nxge_setup_rdc_kstats(nxge, channel); 237 238 NXGE_DEBUG_MSG((nxge, MEM2_CTL, 239 "<== nxge_init_rxdma_channel: status 0x%x", status)); 240 241 return (status); 242 } 243 244 void 245 nxge_uninit_rxdma_channels(p_nxge_t nxgep) 246 { 247 nxge_grp_set_t *set = &nxgep->rx_set; 248 int rdc; 249 250 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channels")); 251 252 if (set->owned.map == 0) { 253 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 254 "nxge_uninit_rxdma_channels: no channels")); 255 return; 256 } 257 258 for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 259 if ((1 << rdc) & set->owned.map) { 260 nxge_grp_dc_remove(nxgep, VP_BOUND_RX, rdc); 261 } 262 } 263 264 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uninit_rxdma_channels")); 265 } 266 267 void 268 nxge_uninit_rxdma_channel(p_nxge_t nxgep, int channel) 269 { 270 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channel")); 271 272 if (nxgep->statsp->rdc_ksp[channel]) { 273 kstat_delete(nxgep->statsp->rdc_ksp[channel]); 274 nxgep->statsp->rdc_ksp[channel] = 0; 275 } 276 277 nxge_rxdma_hw_stop(nxgep, channel); 278 nxge_unmap_rxdma(nxgep, channel); 279 280 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uinit_rxdma_channel")); 281 } 282 283 nxge_status_t 284 nxge_reset_rxdma_channel(p_nxge_t nxgep, uint16_t channel) 285 { 286 npi_handle_t handle; 287 npi_status_t rs = NPI_SUCCESS; 288 nxge_status_t status = NXGE_OK; 289 290 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_reset_rxdma_channel")); 291 292 handle = NXGE_DEV_NPI_HANDLE(nxgep); 293 rs = npi_rxdma_cfg_rdc_reset(handle, channel); 294 295 if (rs != NPI_SUCCESS) { 296 status = NXGE_ERROR | rs; 297 } 298 299 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_reset_rxdma_channel")); 300 301 return (status); 302 } 303 304 void 305 nxge_rxdma_regs_dump_channels(p_nxge_t nxgep) 306 { 307 nxge_grp_set_t *set = &nxgep->rx_set; 308 int rdc; 309 310 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_regs_dump_channels")); 311 312 if (!isLDOMguest(nxgep)) { 313 npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep); 314 (void) npi_rxdma_dump_fzc_regs(handle); 315 } 316 317 if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) { 318 NXGE_DEBUG_MSG((nxgep, TX_CTL, 319 "nxge_rxdma_regs_dump_channels: " 320 "NULL ring pointer(s)")); 321 return; 322 } 323 324 if (set->owned.map == 0) { 325 NXGE_DEBUG_MSG((nxgep, RX_CTL, 326 "nxge_rxdma_regs_dump_channels: no channels")); 327 return; 328 } 329 330 for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 331 if ((1 << rdc) & set->owned.map) { 332 rx_rbr_ring_t *ring = 333 nxgep->rx_rbr_rings->rbr_rings[rdc]; 334 if (ring) { 335 (void) nxge_dump_rxdma_channel(nxgep, rdc); 336 } 337 } 338 } 339 340 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_regs_dump")); 341 } 342 343 nxge_status_t 344 nxge_dump_rxdma_channel(p_nxge_t nxgep, uint8_t channel) 345 { 346 npi_handle_t handle; 347 npi_status_t rs = NPI_SUCCESS; 348 nxge_status_t status = NXGE_OK; 349 350 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_dump_rxdma_channel")); 351 352 handle = NXGE_DEV_NPI_HANDLE(nxgep); 353 rs = npi_rxdma_dump_rdc_regs(handle, channel); 354 355 if (rs != NPI_SUCCESS) { 356 status = NXGE_ERROR | rs; 357 } 358 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_dump_rxdma_channel")); 359 return (status); 360 } 361 362 nxge_status_t 363 nxge_init_rxdma_channel_event_mask(p_nxge_t nxgep, uint16_t channel, 364 p_rx_dma_ent_msk_t mask_p) 365 { 366 npi_handle_t handle; 367 npi_status_t rs = NPI_SUCCESS; 368 nxge_status_t status = NXGE_OK; 369 370 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 371 "<== nxge_init_rxdma_channel_event_mask")); 372 373 handle = NXGE_DEV_NPI_HANDLE(nxgep); 374 rs = npi_rxdma_event_mask(handle, OP_SET, channel, mask_p); 375 if (rs != NPI_SUCCESS) { 376 status = NXGE_ERROR | rs; 377 } 378 379 return (status); 380 } 381 382 nxge_status_t 383 nxge_init_rxdma_channel_cntl_stat(p_nxge_t nxgep, uint16_t channel, 384 p_rx_dma_ctl_stat_t cs_p) 385 { 386 npi_handle_t handle; 387 npi_status_t rs = NPI_SUCCESS; 388 nxge_status_t status = NXGE_OK; 389 390 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 391 "<== nxge_init_rxdma_channel_cntl_stat")); 392 393 handle = NXGE_DEV_NPI_HANDLE(nxgep); 394 rs = npi_rxdma_control_status(handle, OP_SET, channel, cs_p); 395 396 if (rs != NPI_SUCCESS) { 397 status = NXGE_ERROR | rs; 398 } 399 400 return (status); 401 } 402 403 /* 404 * nxge_rxdma_cfg_rdcgrp_default_rdc 405 * 406 * Set the default RDC for an RDC Group (Table) 407 * 408 * Arguments: 409 * nxgep 410 * rdcgrp The group to modify 411 * rdc The new default RDC. 412 * 413 * Notes: 414 * 415 * NPI/NXGE function calls: 416 * npi_rxdma_cfg_rdc_table_default_rdc() 417 * 418 * Registers accessed: 419 * RDC_TBL_REG: FZC_ZCP + 0x10000 420 * 421 * Context: 422 * Service domain 423 */ 424 nxge_status_t 425 nxge_rxdma_cfg_rdcgrp_default_rdc( 426 p_nxge_t nxgep, 427 uint8_t rdcgrp, 428 uint8_t rdc) 429 { 430 npi_handle_t handle; 431 npi_status_t rs = NPI_SUCCESS; 432 p_nxge_dma_pt_cfg_t p_dma_cfgp; 433 p_nxge_rdc_grp_t rdc_grp_p; 434 uint8_t actual_rdcgrp, actual_rdc; 435 436 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 437 " ==> nxge_rxdma_cfg_rdcgrp_default_rdc")); 438 p_dma_cfgp = (p_nxge_dma_pt_cfg_t)&nxgep->pt_config; 439 440 handle = NXGE_DEV_NPI_HANDLE(nxgep); 441 442 /* 443 * This has to be rewritten. Do we even allow this anymore? 444 */ 445 rdc_grp_p = &p_dma_cfgp->rdc_grps[rdcgrp]; 446 RDC_MAP_IN(rdc_grp_p->map, rdc); 447 rdc_grp_p->def_rdc = rdc; 448 449 actual_rdcgrp = NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp); 450 actual_rdc = NXGE_ACTUAL_RDC(nxgep, rdc); 451 452 rs = npi_rxdma_cfg_rdc_table_default_rdc( 453 handle, actual_rdcgrp, actual_rdc); 454 455 if (rs != NPI_SUCCESS) { 456 return (NXGE_ERROR | rs); 457 } 458 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 459 " <== nxge_rxdma_cfg_rdcgrp_default_rdc")); 460 return (NXGE_OK); 461 } 462 463 nxge_status_t 464 nxge_rxdma_cfg_port_default_rdc(p_nxge_t nxgep, uint8_t port, uint8_t rdc) 465 { 466 npi_handle_t handle; 467 468 uint8_t actual_rdc; 469 npi_status_t rs = NPI_SUCCESS; 470 471 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 472 " ==> nxge_rxdma_cfg_port_default_rdc")); 473 474 handle = NXGE_DEV_NPI_HANDLE(nxgep); 475 actual_rdc = rdc; /* XXX Hack! */ 476 rs = npi_rxdma_cfg_default_port_rdc(handle, port, actual_rdc); 477 478 479 if (rs != NPI_SUCCESS) { 480 return (NXGE_ERROR | rs); 481 } 482 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 483 " <== nxge_rxdma_cfg_port_default_rdc")); 484 485 return (NXGE_OK); 486 } 487 488 nxge_status_t 489 nxge_rxdma_cfg_rcr_threshold(p_nxge_t nxgep, uint8_t channel, 490 uint16_t pkts) 491 { 492 npi_status_t rs = NPI_SUCCESS; 493 npi_handle_t handle; 494 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 495 " ==> nxge_rxdma_cfg_rcr_threshold")); 496 handle = NXGE_DEV_NPI_HANDLE(nxgep); 497 498 rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel, pkts); 499 500 if (rs != NPI_SUCCESS) { 501 return (NXGE_ERROR | rs); 502 } 503 NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_threshold")); 504 return (NXGE_OK); 505 } 506 507 nxge_status_t 508 nxge_rxdma_cfg_rcr_timeout(p_nxge_t nxgep, uint8_t channel, 509 uint16_t tout, uint8_t enable) 510 { 511 npi_status_t rs = NPI_SUCCESS; 512 npi_handle_t handle; 513 NXGE_DEBUG_MSG((nxgep, RX2_CTL, " ==> nxge_rxdma_cfg_rcr_timeout")); 514 handle = NXGE_DEV_NPI_HANDLE(nxgep); 515 if (enable == 0) { 516 rs = npi_rxdma_cfg_rdc_rcr_timeout_disable(handle, channel); 517 } else { 518 rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel, 519 tout); 520 } 521 522 if (rs != NPI_SUCCESS) { 523 return (NXGE_ERROR | rs); 524 } 525 NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_timeout")); 526 return (NXGE_OK); 527 } 528 529 nxge_status_t 530 nxge_enable_rxdma_channel(p_nxge_t nxgep, uint16_t channel, 531 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p) 532 { 533 npi_handle_t handle; 534 rdc_desc_cfg_t rdc_desc; 535 p_rcrcfig_b_t cfgb_p; 536 npi_status_t rs = NPI_SUCCESS; 537 538 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel")); 539 handle = NXGE_DEV_NPI_HANDLE(nxgep); 540 /* 541 * Use configuration data composed at init time. 542 * Write to hardware the receive ring configurations. 543 */ 544 rdc_desc.mbox_enable = 1; 545 rdc_desc.mbox_addr = mbox_p->mbox_addr; 546 NXGE_DEBUG_MSG((nxgep, RX_CTL, 547 "==> nxge_enable_rxdma_channel: mboxp $%p($%p)", 548 mbox_p->mbox_addr, rdc_desc.mbox_addr)); 549 550 rdc_desc.rbr_len = rbr_p->rbb_max; 551 rdc_desc.rbr_addr = rbr_p->rbr_addr; 552 553 switch (nxgep->rx_bksize_code) { 554 case RBR_BKSIZE_4K: 555 rdc_desc.page_size = SIZE_4KB; 556 break; 557 case RBR_BKSIZE_8K: 558 rdc_desc.page_size = SIZE_8KB; 559 break; 560 case RBR_BKSIZE_16K: 561 rdc_desc.page_size = SIZE_16KB; 562 break; 563 case RBR_BKSIZE_32K: 564 rdc_desc.page_size = SIZE_32KB; 565 break; 566 } 567 568 rdc_desc.size0 = rbr_p->npi_pkt_buf_size0; 569 rdc_desc.valid0 = 1; 570 571 rdc_desc.size1 = rbr_p->npi_pkt_buf_size1; 572 rdc_desc.valid1 = 1; 573 574 rdc_desc.size2 = rbr_p->npi_pkt_buf_size2; 575 rdc_desc.valid2 = 1; 576 577 rdc_desc.full_hdr = rcr_p->full_hdr_flag; 578 rdc_desc.offset = rcr_p->sw_priv_hdr_len; 579 580 rdc_desc.rcr_len = rcr_p->comp_size; 581 rdc_desc.rcr_addr = rcr_p->rcr_addr; 582 583 cfgb_p = &(rcr_p->rcr_cfgb); 584 rdc_desc.rcr_threshold = cfgb_p->bits.ldw.pthres; 585 /* For now, disable this timeout in a guest domain. */ 586 if (isLDOMguest(nxgep)) { 587 rdc_desc.rcr_timeout = 0; 588 rdc_desc.rcr_timeout_enable = 0; 589 } else { 590 rdc_desc.rcr_timeout = cfgb_p->bits.ldw.timeout; 591 rdc_desc.rcr_timeout_enable = cfgb_p->bits.ldw.entout; 592 } 593 594 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: " 595 "rbr_len qlen %d pagesize code %d rcr_len %d", 596 rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len)); 597 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: " 598 "size 0 %d size 1 %d size 2 %d", 599 rbr_p->npi_pkt_buf_size0, rbr_p->npi_pkt_buf_size1, 600 rbr_p->npi_pkt_buf_size2)); 601 602 if (nxgep->niu_hw_type == NIU_HW_TYPE_RF) 603 rs = npi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, 604 &rdc_desc, B_TRUE); 605 else 606 rs = npi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, 607 &rdc_desc, B_FALSE); 608 if (rs != NPI_SUCCESS) { 609 return (NXGE_ERROR | rs); 610 } 611 612 /* 613 * Enable the timeout and threshold. 614 */ 615 rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel, 616 rdc_desc.rcr_threshold); 617 if (rs != NPI_SUCCESS) { 618 return (NXGE_ERROR | rs); 619 } 620 621 rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel, 622 rdc_desc.rcr_timeout); 623 if (rs != NPI_SUCCESS) { 624 return (NXGE_ERROR | rs); 625 } 626 627 if (!isLDOMguest(nxgep)) { 628 /* Enable the DMA */ 629 rs = npi_rxdma_cfg_rdc_enable(handle, channel); 630 if (rs != NPI_SUCCESS) { 631 return (NXGE_ERROR | rs); 632 } 633 } 634 635 /* Kick the DMA engine. */ 636 npi_rxdma_rdc_rbr_kick(handle, channel, rbr_p->rbb_max); 637 638 if (!isLDOMguest(nxgep)) { 639 /* Clear the rbr empty bit */ 640 (void) npi_rxdma_channel_rbr_empty_clear(handle, channel); 641 } 642 643 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_enable_rxdma_channel")); 644 645 return (NXGE_OK); 646 } 647 648 nxge_status_t 649 nxge_disable_rxdma_channel(p_nxge_t nxgep, uint16_t channel) 650 { 651 npi_handle_t handle; 652 npi_status_t rs = NPI_SUCCESS; 653 654 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_disable_rxdma_channel")); 655 handle = NXGE_DEV_NPI_HANDLE(nxgep); 656 657 /* disable the DMA */ 658 rs = npi_rxdma_cfg_rdc_disable(handle, channel); 659 if (rs != NPI_SUCCESS) { 660 NXGE_DEBUG_MSG((nxgep, RX_CTL, 661 "<== nxge_disable_rxdma_channel:failed (0x%x)", 662 rs)); 663 return (NXGE_ERROR | rs); 664 } 665 666 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_disable_rxdma_channel")); 667 return (NXGE_OK); 668 } 669 670 nxge_status_t 671 nxge_rxdma_channel_rcrflush(p_nxge_t nxgep, uint8_t channel) 672 { 673 npi_handle_t handle; 674 nxge_status_t status = NXGE_OK; 675 676 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 677 "<== nxge_init_rxdma_channel_rcrflush")); 678 679 handle = NXGE_DEV_NPI_HANDLE(nxgep); 680 npi_rxdma_rdc_rcr_flush(handle, channel); 681 682 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 683 "<== nxge_init_rxdma_channel_rcrflsh")); 684 return (status); 685 686 } 687 688 #define MID_INDEX(l, r) ((r + l + 1) >> 1) 689 690 #define TO_LEFT -1 691 #define TO_RIGHT 1 692 #define BOTH_RIGHT (TO_RIGHT + TO_RIGHT) 693 #define BOTH_LEFT (TO_LEFT + TO_LEFT) 694 #define IN_MIDDLE (TO_RIGHT + TO_LEFT) 695 #define NO_HINT 0xffffffff 696 697 /*ARGSUSED*/ 698 nxge_status_t 699 nxge_rxbuf_pp_to_vp(p_nxge_t nxgep, p_rx_rbr_ring_t rbr_p, 700 uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp, 701 uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index) 702 { 703 int bufsize; 704 uint64_t pktbuf_pp; 705 uint64_t dvma_addr; 706 rxring_info_t *ring_info; 707 int base_side, end_side; 708 int r_index, l_index, anchor_index; 709 int found, search_done; 710 uint32_t offset, chunk_size, block_size, page_size_mask; 711 uint32_t chunk_index, block_index, total_index; 712 int max_iterations, iteration; 713 rxbuf_index_info_t *bufinfo; 714 715 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_rxbuf_pp_to_vp")); 716 717 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 718 "==> nxge_rxbuf_pp_to_vp: buf_pp $%p btype %d", 719 pkt_buf_addr_pp, 720 pktbufsz_type)); 721 pktbuf_pp = (uint64_t)pkt_buf_addr_pp; 722 723 switch (pktbufsz_type) { 724 case 0: 725 bufsize = rbr_p->pkt_buf_size0; 726 break; 727 case 1: 728 bufsize = rbr_p->pkt_buf_size1; 729 break; 730 case 2: 731 bufsize = rbr_p->pkt_buf_size2; 732 break; 733 case RCR_SINGLE_BLOCK: 734 bufsize = 0; 735 anchor_index = 0; 736 break; 737 default: 738 return (NXGE_ERROR); 739 } 740 741 if (rbr_p->num_blocks == 1) { 742 anchor_index = 0; 743 ring_info = rbr_p->ring_info; 744 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 745 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 746 "==> nxge_rxbuf_pp_to_vp: (found, 1 block) " 747 "buf_pp $%p btype %d anchor_index %d " 748 "bufinfo $%p", 749 pkt_buf_addr_pp, 750 pktbufsz_type, 751 anchor_index, 752 bufinfo)); 753 754 goto found_index; 755 } 756 757 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 758 "==> nxge_rxbuf_pp_to_vp: " 759 "buf_pp $%p btype %d anchor_index %d", 760 pkt_buf_addr_pp, 761 pktbufsz_type, 762 anchor_index)); 763 764 ring_info = rbr_p->ring_info; 765 found = B_FALSE; 766 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 767 iteration = 0; 768 max_iterations = ring_info->max_iterations; 769 /* 770 * First check if this block has been seen 771 * recently. This is indicated by a hint which 772 * is initialized when the first buffer of the block 773 * is seen. The hint is reset when the last buffer of 774 * the block has been processed. 775 * As three block sizes are supported, three hints 776 * are kept. The idea behind the hints is that once 777 * the hardware uses a block for a buffer of that 778 * size, it will use it exclusively for that size 779 * and will use it until it is exhausted. It is assumed 780 * that there would a single block being used for the same 781 * buffer sizes at any given time. 782 */ 783 if (ring_info->hint[pktbufsz_type] != NO_HINT) { 784 anchor_index = ring_info->hint[pktbufsz_type]; 785 dvma_addr = bufinfo[anchor_index].dvma_addr; 786 chunk_size = bufinfo[anchor_index].buf_size; 787 if ((pktbuf_pp >= dvma_addr) && 788 (pktbuf_pp < (dvma_addr + chunk_size))) { 789 found = B_TRUE; 790 /* 791 * check if this is the last buffer in the block 792 * If so, then reset the hint for the size; 793 */ 794 795 if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size)) 796 ring_info->hint[pktbufsz_type] = NO_HINT; 797 } 798 } 799 800 if (found == B_FALSE) { 801 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 802 "==> nxge_rxbuf_pp_to_vp: (!found)" 803 "buf_pp $%p btype %d anchor_index %d", 804 pkt_buf_addr_pp, 805 pktbufsz_type, 806 anchor_index)); 807 808 /* 809 * This is the first buffer of the block of this 810 * size. Need to search the whole information 811 * array. 812 * the search algorithm uses a binary tree search 813 * algorithm. It assumes that the information is 814 * already sorted with increasing order 815 * info[0] < info[1] < info[2] .... < info[n-1] 816 * where n is the size of the information array 817 */ 818 r_index = rbr_p->num_blocks - 1; 819 l_index = 0; 820 search_done = B_FALSE; 821 anchor_index = MID_INDEX(r_index, l_index); 822 while (search_done == B_FALSE) { 823 if ((r_index == l_index) || 824 (iteration >= max_iterations)) 825 search_done = B_TRUE; 826 end_side = TO_RIGHT; /* to the right */ 827 base_side = TO_LEFT; /* to the left */ 828 /* read the DVMA address information and sort it */ 829 dvma_addr = bufinfo[anchor_index].dvma_addr; 830 chunk_size = bufinfo[anchor_index].buf_size; 831 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 832 "==> nxge_rxbuf_pp_to_vp: (searching)" 833 "buf_pp $%p btype %d " 834 "anchor_index %d chunk_size %d dvmaaddr $%p", 835 pkt_buf_addr_pp, 836 pktbufsz_type, 837 anchor_index, 838 chunk_size, 839 dvma_addr)); 840 841 if (pktbuf_pp >= dvma_addr) 842 base_side = TO_RIGHT; /* to the right */ 843 if (pktbuf_pp < (dvma_addr + chunk_size)) 844 end_side = TO_LEFT; /* to the left */ 845 846 switch (base_side + end_side) { 847 case IN_MIDDLE: 848 /* found */ 849 found = B_TRUE; 850 search_done = B_TRUE; 851 if ((pktbuf_pp + bufsize) < 852 (dvma_addr + chunk_size)) 853 ring_info->hint[pktbufsz_type] = 854 bufinfo[anchor_index].buf_index; 855 break; 856 case BOTH_RIGHT: 857 /* not found: go to the right */ 858 l_index = anchor_index + 1; 859 anchor_index = MID_INDEX(r_index, l_index); 860 break; 861 862 case BOTH_LEFT: 863 /* not found: go to the left */ 864 r_index = anchor_index - 1; 865 anchor_index = MID_INDEX(r_index, l_index); 866 break; 867 default: /* should not come here */ 868 return (NXGE_ERROR); 869 } 870 iteration++; 871 } 872 873 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 874 "==> nxge_rxbuf_pp_to_vp: (search done)" 875 "buf_pp $%p btype %d anchor_index %d", 876 pkt_buf_addr_pp, 877 pktbufsz_type, 878 anchor_index)); 879 } 880 881 if (found == B_FALSE) { 882 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 883 "==> nxge_rxbuf_pp_to_vp: (search failed)" 884 "buf_pp $%p btype %d anchor_index %d", 885 pkt_buf_addr_pp, 886 pktbufsz_type, 887 anchor_index)); 888 return (NXGE_ERROR); 889 } 890 891 found_index: 892 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 893 "==> nxge_rxbuf_pp_to_vp: (FOUND1)" 894 "buf_pp $%p btype %d bufsize %d anchor_index %d", 895 pkt_buf_addr_pp, 896 pktbufsz_type, 897 bufsize, 898 anchor_index)); 899 900 /* index of the first block in this chunk */ 901 chunk_index = bufinfo[anchor_index].start_index; 902 dvma_addr = bufinfo[anchor_index].dvma_addr; 903 page_size_mask = ring_info->block_size_mask; 904 905 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 906 "==> nxge_rxbuf_pp_to_vp: (FOUND3), get chunk)" 907 "buf_pp $%p btype %d bufsize %d " 908 "anchor_index %d chunk_index %d dvma $%p", 909 pkt_buf_addr_pp, 910 pktbufsz_type, 911 bufsize, 912 anchor_index, 913 chunk_index, 914 dvma_addr)); 915 916 offset = pktbuf_pp - dvma_addr; /* offset within the chunk */ 917 block_size = rbr_p->block_size; /* System block(page) size */ 918 919 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 920 "==> nxge_rxbuf_pp_to_vp: (FOUND4), get chunk)" 921 "buf_pp $%p btype %d bufsize %d " 922 "anchor_index %d chunk_index %d dvma $%p " 923 "offset %d block_size %d", 924 pkt_buf_addr_pp, 925 pktbufsz_type, 926 bufsize, 927 anchor_index, 928 chunk_index, 929 dvma_addr, 930 offset, 931 block_size)); 932 933 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> getting total index")); 934 935 block_index = (offset / block_size); /* index within chunk */ 936 total_index = chunk_index + block_index; 937 938 939 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 940 "==> nxge_rxbuf_pp_to_vp: " 941 "total_index %d dvma_addr $%p " 942 "offset %d block_size %d " 943 "block_index %d ", 944 total_index, dvma_addr, 945 offset, block_size, 946 block_index)); 947 *pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr + 948 (uint64_t)offset); 949 950 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 951 "==> nxge_rxbuf_pp_to_vp: " 952 "total_index %d dvma_addr $%p " 953 "offset %d block_size %d " 954 "block_index %d " 955 "*pkt_buf_addr_p $%p", 956 total_index, dvma_addr, 957 offset, block_size, 958 block_index, 959 *pkt_buf_addr_p)); 960 961 962 *msg_index = total_index; 963 *bufoffset = (offset & page_size_mask); 964 965 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 966 "==> nxge_rxbuf_pp_to_vp: get msg index: " 967 "msg_index %d bufoffset_index %d", 968 *msg_index, 969 *bufoffset)); 970 971 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rxbuf_pp_to_vp")); 972 973 return (NXGE_OK); 974 } 975 976 /* 977 * used by quick sort (qsort) function 978 * to perform comparison 979 */ 980 static int 981 nxge_sort_compare(const void *p1, const void *p2) 982 { 983 984 rxbuf_index_info_t *a, *b; 985 986 a = (rxbuf_index_info_t *)p1; 987 b = (rxbuf_index_info_t *)p2; 988 989 if (a->dvma_addr > b->dvma_addr) 990 return (1); 991 if (a->dvma_addr < b->dvma_addr) 992 return (-1); 993 return (0); 994 } 995 996 997 998 /* 999 * grabbed this sort implementation from common/syscall/avl.c 1000 * 1001 */ 1002 /* 1003 * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified. 1004 * v = Ptr to array/vector of objs 1005 * n = # objs in the array 1006 * s = size of each obj (must be multiples of a word size) 1007 * f = ptr to function to compare two objs 1008 * returns (-1 = less than, 0 = equal, 1 = greater than 1009 */ 1010 void 1011 nxge_ksort(caddr_t v, int n, int s, int (*f)()) 1012 { 1013 int g, i, j, ii; 1014 unsigned int *p1, *p2; 1015 unsigned int tmp; 1016 1017 /* No work to do */ 1018 if (v == NULL || n <= 1) 1019 return; 1020 /* Sanity check on arguments */ 1021 ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0); 1022 ASSERT(s > 0); 1023 1024 for (g = n / 2; g > 0; g /= 2) { 1025 for (i = g; i < n; i++) { 1026 for (j = i - g; j >= 0 && 1027 (*f)(v + j * s, v + (j + g) * s) == 1; 1028 j -= g) { 1029 p1 = (unsigned *)(v + j * s); 1030 p2 = (unsigned *)(v + (j + g) * s); 1031 for (ii = 0; ii < s / 4; ii++) { 1032 tmp = *p1; 1033 *p1++ = *p2; 1034 *p2++ = tmp; 1035 } 1036 } 1037 } 1038 } 1039 } 1040 1041 /* 1042 * Initialize data structures required for rxdma 1043 * buffer dvma->vmem address lookup 1044 */ 1045 /*ARGSUSED*/ 1046 static nxge_status_t 1047 nxge_rxbuf_index_info_init(p_nxge_t nxgep, p_rx_rbr_ring_t rbrp) 1048 { 1049 1050 int index; 1051 rxring_info_t *ring_info; 1052 int max_iteration = 0, max_index = 0; 1053 1054 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_rxbuf_index_info_init")); 1055 1056 ring_info = rbrp->ring_info; 1057 ring_info->hint[0] = NO_HINT; 1058 ring_info->hint[1] = NO_HINT; 1059 ring_info->hint[2] = NO_HINT; 1060 max_index = rbrp->num_blocks; 1061 1062 /* read the DVMA address information and sort it */ 1063 /* do init of the information array */ 1064 1065 1066 NXGE_DEBUG_MSG((nxgep, DMA2_CTL, 1067 " nxge_rxbuf_index_info_init Sort ptrs")); 1068 1069 /* sort the array */ 1070 nxge_ksort((void *)ring_info->buffer, max_index, 1071 sizeof (rxbuf_index_info_t), nxge_sort_compare); 1072 1073 1074 1075 for (index = 0; index < max_index; index++) { 1076 NXGE_DEBUG_MSG((nxgep, DMA2_CTL, 1077 " nxge_rxbuf_index_info_init: sorted chunk %d " 1078 " ioaddr $%p kaddr $%p size %x", 1079 index, ring_info->buffer[index].dvma_addr, 1080 ring_info->buffer[index].kaddr, 1081 ring_info->buffer[index].buf_size)); 1082 } 1083 1084 max_iteration = 0; 1085 while (max_index >= (1ULL << max_iteration)) 1086 max_iteration++; 1087 ring_info->max_iterations = max_iteration + 1; 1088 NXGE_DEBUG_MSG((nxgep, DMA2_CTL, 1089 " nxge_rxbuf_index_info_init Find max iter %d", 1090 ring_info->max_iterations)); 1091 1092 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxbuf_index_info_init")); 1093 return (NXGE_OK); 1094 } 1095 1096 /* ARGSUSED */ 1097 void 1098 nxge_dump_rcr_entry(p_nxge_t nxgep, p_rcr_entry_t entry_p) 1099 { 1100 #ifdef NXGE_DEBUG 1101 1102 uint32_t bptr; 1103 uint64_t pp; 1104 1105 bptr = entry_p->bits.hdw.pkt_buf_addr; 1106 1107 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1108 "\trcr entry $%p " 1109 "\trcr entry 0x%0llx " 1110 "\trcr entry 0x%08x " 1111 "\trcr entry 0x%08x " 1112 "\tvalue 0x%0llx\n" 1113 "\tmulti = %d\n" 1114 "\tpkt_type = 0x%x\n" 1115 "\tzero_copy = %d\n" 1116 "\tnoport = %d\n" 1117 "\tpromis = %d\n" 1118 "\terror = 0x%04x\n" 1119 "\tdcf_err = 0x%01x\n" 1120 "\tl2_len = %d\n" 1121 "\tpktbufsize = %d\n" 1122 "\tpkt_buf_addr = $%p\n" 1123 "\tpkt_buf_addr (<< 6) = $%p\n", 1124 entry_p, 1125 *(int64_t *)entry_p, 1126 *(int32_t *)entry_p, 1127 *(int32_t *)((char *)entry_p + 32), 1128 entry_p->value, 1129 entry_p->bits.hdw.multi, 1130 entry_p->bits.hdw.pkt_type, 1131 entry_p->bits.hdw.zero_copy, 1132 entry_p->bits.hdw.noport, 1133 entry_p->bits.hdw.promis, 1134 entry_p->bits.hdw.error, 1135 entry_p->bits.hdw.dcf_err, 1136 entry_p->bits.hdw.l2_len, 1137 entry_p->bits.hdw.pktbufsz, 1138 bptr, 1139 entry_p->bits.ldw.pkt_buf_addr)); 1140 1141 pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) << 1142 RCR_PKT_BUF_ADDR_SHIFT; 1143 1144 NXGE_DEBUG_MSG((nxgep, RX_CTL, "rcr pp 0x%llx l2 len %d", 1145 pp, (*(int64_t *)entry_p >> 40) & 0x3fff)); 1146 #endif 1147 } 1148 1149 void 1150 nxge_rxdma_regs_dump(p_nxge_t nxgep, int rdc) 1151 { 1152 npi_handle_t handle; 1153 rbr_stat_t rbr_stat; 1154 addr44_t hd_addr; 1155 addr44_t tail_addr; 1156 uint16_t qlen; 1157 1158 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1159 "==> nxge_rxdma_regs_dump: rdc channel %d", rdc)); 1160 1161 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1162 1163 /* RBR head */ 1164 hd_addr.addr = 0; 1165 (void) npi_rxdma_rdc_rbr_head_get(handle, rdc, &hd_addr); 1166 printf("nxge_rxdma_regs_dump: got hdptr $%p \n", 1167 (void *)hd_addr.addr); 1168 1169 /* RBR stats */ 1170 (void) npi_rxdma_rdc_rbr_stat_get(handle, rdc, &rbr_stat); 1171 printf("nxge_rxdma_regs_dump: rbr len %d \n", rbr_stat.bits.ldw.qlen); 1172 1173 /* RCR tail */ 1174 tail_addr.addr = 0; 1175 (void) npi_rxdma_rdc_rcr_tail_get(handle, rdc, &tail_addr); 1176 printf("nxge_rxdma_regs_dump: got tail ptr $%p \n", 1177 (void *)tail_addr.addr); 1178 1179 /* RCR qlen */ 1180 (void) npi_rxdma_rdc_rcr_qlen_get(handle, rdc, &qlen); 1181 printf("nxge_rxdma_regs_dump: rcr len %x \n", qlen); 1182 1183 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1184 "<== nxge_rxdma_regs_dump: rdc rdc %d", rdc)); 1185 } 1186 1187 nxge_status_t 1188 nxge_rxdma_hw_mode(p_nxge_t nxgep, boolean_t enable) 1189 { 1190 nxge_grp_set_t *set = &nxgep->rx_set; 1191 nxge_status_t status; 1192 npi_status_t rs; 1193 int rdc; 1194 1195 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 1196 "==> nxge_rxdma_hw_mode: mode %d", enable)); 1197 1198 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 1199 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1200 "<== nxge_rxdma_mode: not initialized")); 1201 return (NXGE_ERROR); 1202 } 1203 1204 if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) { 1205 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1206 "<== nxge_tx_port_fatal_err_recover: " 1207 "NULL ring pointer(s)")); 1208 return (NXGE_ERROR); 1209 } 1210 1211 if (set->owned.map == 0) { 1212 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1213 "nxge_rxdma_regs_dump_channels: no channels")); 1214 return (0); 1215 } 1216 1217 rs = 0; 1218 for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 1219 if ((1 << rdc) & set->owned.map) { 1220 rx_rbr_ring_t *ring = 1221 nxgep->rx_rbr_rings->rbr_rings[rdc]; 1222 npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep); 1223 if (ring) { 1224 if (enable) { 1225 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 1226 "==> nxge_rxdma_hw_mode: " 1227 "channel %d (enable)", rdc)); 1228 rs = npi_rxdma_cfg_rdc_enable 1229 (handle, rdc); 1230 } else { 1231 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 1232 "==> nxge_rxdma_hw_mode: " 1233 "channel %d disable)", rdc)); 1234 rs = npi_rxdma_cfg_rdc_disable 1235 (handle, rdc); 1236 } 1237 } 1238 } 1239 } 1240 1241 status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs); 1242 1243 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 1244 "<== nxge_rxdma_hw_mode: status 0x%x", status)); 1245 1246 return (status); 1247 } 1248 1249 void 1250 nxge_rxdma_enable_channel(p_nxge_t nxgep, uint16_t channel) 1251 { 1252 npi_handle_t handle; 1253 1254 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 1255 "==> nxge_rxdma_enable_channel: channel %d", channel)); 1256 1257 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1258 (void) npi_rxdma_cfg_rdc_enable(handle, channel); 1259 1260 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_enable_channel")); 1261 } 1262 1263 void 1264 nxge_rxdma_disable_channel(p_nxge_t nxgep, uint16_t channel) 1265 { 1266 npi_handle_t handle; 1267 1268 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 1269 "==> nxge_rxdma_disable_channel: channel %d", channel)); 1270 1271 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1272 (void) npi_rxdma_cfg_rdc_disable(handle, channel); 1273 1274 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_disable_channel")); 1275 } 1276 1277 void 1278 nxge_hw_start_rx(p_nxge_t nxgep) 1279 { 1280 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_hw_start_rx")); 1281 1282 (void) nxge_rxdma_hw_mode(nxgep, NXGE_DMA_START); 1283 (void) nxge_rx_mac_enable(nxgep); 1284 1285 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_hw_start_rx")); 1286 } 1287 1288 /*ARGSUSED*/ 1289 void 1290 nxge_fixup_rxdma_rings(p_nxge_t nxgep) 1291 { 1292 nxge_grp_set_t *set = &nxgep->rx_set; 1293 int rdc; 1294 1295 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_fixup_rxdma_rings")); 1296 1297 if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) { 1298 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1299 "<== nxge_tx_port_fatal_err_recover: " 1300 "NULL ring pointer(s)")); 1301 return; 1302 } 1303 1304 if (set->owned.map == 0) { 1305 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1306 "nxge_rxdma_regs_dump_channels: no channels")); 1307 return; 1308 } 1309 1310 for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 1311 if ((1 << rdc) & set->owned.map) { 1312 rx_rbr_ring_t *ring = 1313 nxgep->rx_rbr_rings->rbr_rings[rdc]; 1314 if (ring) { 1315 nxge_rxdma_hw_stop(nxgep, rdc); 1316 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1317 "==> nxge_fixup_rxdma_rings: " 1318 "channel %d ring $%px", 1319 rdc, ring)); 1320 (void) nxge_rxdma_fix_channel(nxgep, rdc); 1321 } 1322 } 1323 } 1324 1325 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_fixup_rxdma_rings")); 1326 } 1327 1328 void 1329 nxge_rxdma_fix_channel(p_nxge_t nxgep, uint16_t channel) 1330 { 1331 int ndmas; 1332 p_rx_rbr_rings_t rx_rbr_rings; 1333 p_rx_rbr_ring_t *rbr_rings; 1334 p_rx_rcr_rings_t rx_rcr_rings; 1335 p_rx_rcr_ring_t *rcr_rings; 1336 p_rx_mbox_areas_t rx_mbox_areas_p; 1337 p_rx_mbox_t *rx_mbox_p; 1338 p_nxge_dma_pool_t dma_buf_poolp; 1339 p_nxge_dma_pool_t dma_cntl_poolp; 1340 p_rx_rbr_ring_t rbrp; 1341 p_rx_rcr_ring_t rcrp; 1342 p_rx_mbox_t mboxp; 1343 p_nxge_dma_common_t dmap; 1344 nxge_status_t status = NXGE_OK; 1345 1346 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fix_channel")); 1347 1348 (void) nxge_rxdma_stop_channel(nxgep, channel); 1349 1350 dma_buf_poolp = nxgep->rx_buf_pool_p; 1351 dma_cntl_poolp = nxgep->rx_cntl_pool_p; 1352 1353 if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) { 1354 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 1355 "<== nxge_rxdma_fix_channel: buf not allocated")); 1356 return; 1357 } 1358 1359 ndmas = dma_buf_poolp->ndmas; 1360 if (!ndmas) { 1361 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 1362 "<== nxge_rxdma_fix_channel: no dma allocated")); 1363 return; 1364 } 1365 1366 rx_rbr_rings = nxgep->rx_rbr_rings; 1367 rx_rcr_rings = nxgep->rx_rcr_rings; 1368 rbr_rings = rx_rbr_rings->rbr_rings; 1369 rcr_rings = rx_rcr_rings->rcr_rings; 1370 rx_mbox_areas_p = nxgep->rx_mbox_areas_p; 1371 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 1372 1373 /* Reinitialize the receive block and completion rings */ 1374 rbrp = (p_rx_rbr_ring_t)rbr_rings[channel], 1375 rcrp = (p_rx_rcr_ring_t)rcr_rings[channel], 1376 mboxp = (p_rx_mbox_t)rx_mbox_p[channel]; 1377 1378 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 1379 rbrp->rbr_rd_index = 0; 1380 rcrp->comp_rd_index = 0; 1381 rcrp->comp_wt_index = 0; 1382 1383 dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc; 1384 bzero((caddr_t)dmap->kaddrp, dmap->alength); 1385 1386 status = nxge_rxdma_start_channel(nxgep, channel, 1387 rbrp, rcrp, mboxp); 1388 if (status != NXGE_OK) { 1389 goto nxge_rxdma_fix_channel_fail; 1390 } 1391 1392 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1393 "<== nxge_rxdma_fix_channel: success (0x%08x)", status)); 1394 return; 1395 1396 nxge_rxdma_fix_channel_fail: 1397 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1398 "<== nxge_rxdma_fix_channel: failed (0x%08x)", status)); 1399 } 1400 1401 p_rx_rbr_ring_t 1402 nxge_rxdma_get_rbr_ring(p_nxge_t nxgep, uint16_t channel) 1403 { 1404 nxge_grp_set_t *set = &nxgep->rx_set; 1405 nxge_channel_t rdc; 1406 1407 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1408 "==> nxge_rxdma_get_rbr_ring: channel %d", channel)); 1409 1410 if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) { 1411 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1412 "<== nxge_rxdma_get_rbr_ring: " 1413 "NULL ring pointer(s)")); 1414 return (NULL); 1415 } 1416 1417 if (set->owned.map == 0) { 1418 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1419 "<== nxge_rxdma_get_rbr_ring: no channels")); 1420 return (NULL); 1421 } 1422 1423 for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 1424 if ((1 << rdc) & set->owned.map) { 1425 rx_rbr_ring_t *ring = 1426 nxgep->rx_rbr_rings->rbr_rings[rdc]; 1427 if (ring) { 1428 if (channel == ring->rdc) { 1429 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1430 "==> nxge_rxdma_get_rbr_ring: " 1431 "channel %d ring $%p", rdc, ring)); 1432 return (ring); 1433 } 1434 } 1435 } 1436 } 1437 1438 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1439 "<== nxge_rxdma_get_rbr_ring: not found")); 1440 1441 return (NULL); 1442 } 1443 1444 p_rx_rcr_ring_t 1445 nxge_rxdma_get_rcr_ring(p_nxge_t nxgep, uint16_t channel) 1446 { 1447 nxge_grp_set_t *set = &nxgep->rx_set; 1448 nxge_channel_t rdc; 1449 1450 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1451 "==> nxge_rxdma_get_rcr_ring: channel %d", channel)); 1452 1453 if (nxgep->rx_rcr_rings == 0 || nxgep->rx_rcr_rings->rcr_rings == 0) { 1454 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1455 "<== nxge_rxdma_get_rcr_ring: " 1456 "NULL ring pointer(s)")); 1457 return (NULL); 1458 } 1459 1460 if (set->owned.map == 0) { 1461 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1462 "<== nxge_rxdma_get_rbr_ring: no channels")); 1463 return (NULL); 1464 } 1465 1466 for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 1467 if ((1 << rdc) & set->owned.map) { 1468 rx_rcr_ring_t *ring = 1469 nxgep->rx_rcr_rings->rcr_rings[rdc]; 1470 if (ring) { 1471 if (channel == ring->rdc) { 1472 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1473 "==> nxge_rxdma_get_rcr_ring: " 1474 "channel %d ring $%p", rdc, ring)); 1475 return (ring); 1476 } 1477 } 1478 } 1479 } 1480 1481 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1482 "<== nxge_rxdma_get_rcr_ring: not found")); 1483 1484 return (NULL); 1485 } 1486 1487 /* 1488 * Static functions start here. 1489 */ 1490 static p_rx_msg_t 1491 nxge_allocb(size_t size, uint32_t pri, p_nxge_dma_common_t dmabuf_p) 1492 { 1493 p_rx_msg_t nxge_mp = NULL; 1494 p_nxge_dma_common_t dmamsg_p; 1495 uchar_t *buffer; 1496 1497 nxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP); 1498 if (nxge_mp == NULL) { 1499 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1500 "Allocation of a rx msg failed.")); 1501 goto nxge_allocb_exit; 1502 } 1503 1504 nxge_mp->use_buf_pool = B_FALSE; 1505 if (dmabuf_p) { 1506 nxge_mp->use_buf_pool = B_TRUE; 1507 dmamsg_p = (p_nxge_dma_common_t)&nxge_mp->buf_dma; 1508 *dmamsg_p = *dmabuf_p; 1509 dmamsg_p->nblocks = 1; 1510 dmamsg_p->block_size = size; 1511 dmamsg_p->alength = size; 1512 buffer = (uchar_t *)dmabuf_p->kaddrp; 1513 1514 dmabuf_p->kaddrp = (void *) 1515 ((char *)dmabuf_p->kaddrp + size); 1516 dmabuf_p->ioaddr_pp = (void *) 1517 ((char *)dmabuf_p->ioaddr_pp + size); 1518 dmabuf_p->alength -= size; 1519 dmabuf_p->offset += size; 1520 dmabuf_p->dma_cookie.dmac_laddress += size; 1521 dmabuf_p->dma_cookie.dmac_size -= size; 1522 1523 } else { 1524 buffer = KMEM_ALLOC(size, KM_NOSLEEP); 1525 if (buffer == NULL) { 1526 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1527 "Allocation of a receive page failed.")); 1528 goto nxge_allocb_fail1; 1529 } 1530 } 1531 1532 nxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &nxge_mp->freeb); 1533 if (nxge_mp->rx_mblk_p == NULL) { 1534 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, "desballoc failed.")); 1535 goto nxge_allocb_fail2; 1536 } 1537 1538 nxge_mp->buffer = buffer; 1539 nxge_mp->block_size = size; 1540 nxge_mp->freeb.free_func = (void (*)())nxge_freeb; 1541 nxge_mp->freeb.free_arg = (caddr_t)nxge_mp; 1542 nxge_mp->ref_cnt = 1; 1543 nxge_mp->free = B_TRUE; 1544 nxge_mp->rx_use_bcopy = B_FALSE; 1545 1546 atomic_inc_32(&nxge_mblks_pending); 1547 1548 goto nxge_allocb_exit; 1549 1550 nxge_allocb_fail2: 1551 if (!nxge_mp->use_buf_pool) { 1552 KMEM_FREE(buffer, size); 1553 } 1554 1555 nxge_allocb_fail1: 1556 KMEM_FREE(nxge_mp, sizeof (rx_msg_t)); 1557 nxge_mp = NULL; 1558 1559 nxge_allocb_exit: 1560 return (nxge_mp); 1561 } 1562 1563 p_mblk_t 1564 nxge_dupb(p_rx_msg_t nxge_mp, uint_t offset, size_t size) 1565 { 1566 p_mblk_t mp; 1567 1568 NXGE_DEBUG_MSG((NULL, MEM_CTL, "==> nxge_dupb")); 1569 NXGE_DEBUG_MSG((NULL, MEM_CTL, "nxge_mp = $%p " 1570 "offset = 0x%08X " 1571 "size = 0x%08X", 1572 nxge_mp, offset, size)); 1573 1574 mp = desballoc(&nxge_mp->buffer[offset], size, 1575 0, &nxge_mp->freeb); 1576 if (mp == NULL) { 1577 NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 1578 goto nxge_dupb_exit; 1579 } 1580 atomic_inc_32(&nxge_mp->ref_cnt); 1581 1582 1583 nxge_dupb_exit: 1584 NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p", 1585 nxge_mp)); 1586 return (mp); 1587 } 1588 1589 p_mblk_t 1590 nxge_dupb_bcopy(p_rx_msg_t nxge_mp, uint_t offset, size_t size) 1591 { 1592 p_mblk_t mp; 1593 uchar_t *dp; 1594 1595 mp = allocb(size + NXGE_RXBUF_EXTRA, 0); 1596 if (mp == NULL) { 1597 NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 1598 goto nxge_dupb_bcopy_exit; 1599 } 1600 dp = mp->b_rptr = mp->b_rptr + NXGE_RXBUF_EXTRA; 1601 bcopy((void *)&nxge_mp->buffer[offset], dp, size); 1602 mp->b_wptr = dp + size; 1603 1604 nxge_dupb_bcopy_exit: 1605 NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p", 1606 nxge_mp)); 1607 return (mp); 1608 } 1609 1610 void nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p, 1611 p_rx_msg_t rx_msg_p); 1612 1613 void 1614 nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p) 1615 { 1616 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_post_page")); 1617 1618 /* Reuse this buffer */ 1619 rx_msg_p->free = B_FALSE; 1620 rx_msg_p->cur_usage_cnt = 0; 1621 rx_msg_p->max_usage_cnt = 0; 1622 rx_msg_p->pkt_buf_size = 0; 1623 1624 if (rx_rbr_p->rbr_use_bcopy) { 1625 rx_msg_p->rx_use_bcopy = B_FALSE; 1626 atomic_dec_32(&rx_rbr_p->rbr_consumed); 1627 } 1628 1629 /* 1630 * Get the rbr header pointer and its offset index. 1631 */ 1632 MUTEX_ENTER(&rx_rbr_p->post_lock); 1633 rx_rbr_p->rbr_wr_index = ((rx_rbr_p->rbr_wr_index + 1) & 1634 rx_rbr_p->rbr_wrap_mask); 1635 rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr; 1636 MUTEX_EXIT(&rx_rbr_p->post_lock); 1637 npi_rxdma_rdc_rbr_kick(NXGE_DEV_NPI_HANDLE(nxgep), 1638 rx_rbr_p->rdc, 1); 1639 1640 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1641 "<== nxge_post_page (channel %d post_next_index %d)", 1642 rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index)); 1643 1644 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_post_page")); 1645 } 1646 1647 void 1648 nxge_freeb(p_rx_msg_t rx_msg_p) 1649 { 1650 size_t size; 1651 uchar_t *buffer = NULL; 1652 int ref_cnt; 1653 boolean_t free_state = B_FALSE; 1654 1655 rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p; 1656 1657 NXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> nxge_freeb")); 1658 NXGE_DEBUG_MSG((NULL, MEM2_CTL, 1659 "nxge_freeb:rx_msg_p = $%p (block pending %d)", 1660 rx_msg_p, nxge_mblks_pending)); 1661 1662 /* 1663 * First we need to get the free state, then 1664 * atomic decrement the reference count to prevent 1665 * the race condition with the interrupt thread that 1666 * is processing a loaned up buffer block. 1667 */ 1668 free_state = rx_msg_p->free; 1669 ref_cnt = atomic_dec_32_nv(&rx_msg_p->ref_cnt); 1670 if (!ref_cnt) { 1671 atomic_dec_32(&nxge_mblks_pending); 1672 buffer = rx_msg_p->buffer; 1673 size = rx_msg_p->block_size; 1674 NXGE_DEBUG_MSG((NULL, MEM2_CTL, "nxge_freeb: " 1675 "will free: rx_msg_p = $%p (block pending %d)", 1676 rx_msg_p, nxge_mblks_pending)); 1677 1678 if (!rx_msg_p->use_buf_pool) { 1679 KMEM_FREE(buffer, size); 1680 } 1681 1682 KMEM_FREE(rx_msg_p, sizeof (rx_msg_t)); 1683 1684 if (ring) { 1685 /* 1686 * Decrement the receive buffer ring's reference 1687 * count, too. 1688 */ 1689 atomic_dec_32(&ring->rbr_ref_cnt); 1690 1691 /* 1692 * Free the receive buffer ring, if 1693 * 1. all the receive buffers have been freed 1694 * 2. and we are in the proper state (that is, 1695 * we are not UNMAPPING). 1696 */ 1697 if (ring->rbr_ref_cnt == 0 && 1698 ring->rbr_state == RBR_UNMAPPED) { 1699 /* 1700 * Free receive data buffers, 1701 * buffer index information 1702 * (rxring_info) and 1703 * the message block ring. 1704 */ 1705 NXGE_DEBUG_MSG((NULL, RX_CTL, 1706 "nxge_freeb:rx_msg_p = $%p " 1707 "(block pending %d) free buffers", 1708 rx_msg_p, nxge_mblks_pending)); 1709 nxge_rxdma_databuf_free(ring); 1710 if (ring->ring_info) { 1711 KMEM_FREE(ring->ring_info, 1712 sizeof (rxring_info_t)); 1713 } 1714 1715 if (ring->rx_msg_ring) { 1716 KMEM_FREE(ring->rx_msg_ring, 1717 ring->tnblocks * 1718 sizeof (p_rx_msg_t)); 1719 } 1720 KMEM_FREE(ring, sizeof (*ring)); 1721 } 1722 } 1723 return; 1724 } 1725 1726 /* 1727 * Repost buffer. 1728 */ 1729 if (free_state && (ref_cnt == 1) && ring) { 1730 NXGE_DEBUG_MSG((NULL, RX_CTL, 1731 "nxge_freeb: post page $%p:", rx_msg_p)); 1732 if (ring->rbr_state == RBR_POSTING) 1733 nxge_post_page(rx_msg_p->nxgep, ring, rx_msg_p); 1734 } 1735 1736 NXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== nxge_freeb")); 1737 } 1738 1739 uint_t 1740 nxge_rx_intr(char *arg1, char *arg2) 1741 { 1742 p_nxge_ldv_t ldvp = (p_nxge_ldv_t)arg1; 1743 p_nxge_t nxgep = (p_nxge_t)arg2; 1744 p_nxge_ldg_t ldgp; 1745 uint8_t channel; 1746 npi_handle_t handle; 1747 rx_dma_ctl_stat_t cs; 1748 p_rx_rcr_ring_t rcrp; 1749 mblk_t *mp = NULL; 1750 1751 if (ldvp == NULL) { 1752 NXGE_DEBUG_MSG((NULL, INT_CTL, 1753 "<== nxge_rx_intr: arg2 $%p arg1 $%p", 1754 nxgep, ldvp)); 1755 return (DDI_INTR_CLAIMED); 1756 } 1757 1758 if (arg2 == NULL || (void *)ldvp->nxgep != arg2) { 1759 nxgep = ldvp->nxgep; 1760 } 1761 1762 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1763 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1764 NXGE_DEBUG_MSG((nxgep, INT_CTL, 1765 "<== nxge_rx_intr: interface not started or intialized")); 1766 return (DDI_INTR_CLAIMED); 1767 } 1768 1769 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1770 "==> nxge_rx_intr: arg2 $%p arg1 $%p", 1771 nxgep, ldvp)); 1772 1773 /* 1774 * Get the PIO handle. 1775 */ 1776 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1777 1778 /* 1779 * Get the ring to enable us to process packets. 1780 */ 1781 rcrp = nxgep->rx_rcr_rings->rcr_rings[ldvp->vdma_index]; 1782 1783 /* 1784 * The RCR ring lock must be held when packets 1785 * are being processed and the hardware registers are 1786 * being read or written to prevent race condition 1787 * among the interrupt thread, the polling thread 1788 * (will cause fatal errors such as rcrincon bit set) 1789 * and the setting of the poll_flag. 1790 */ 1791 MUTEX_ENTER(&rcrp->lock); 1792 1793 /* 1794 * Get the control and status for this channel. 1795 */ 1796 channel = ldvp->channel; 1797 ldgp = ldvp->ldgp; 1798 1799 if (!isLDOMguest(nxgep) && (!rcrp->started)) { 1800 NXGE_DEBUG_MSG((nxgep, INT_CTL, 1801 "<== nxge_rx_intr: channel is not started")); 1802 1803 /* 1804 * We received an interrupt before the ring is started. 1805 */ 1806 RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, channel, 1807 &cs.value); 1808 cs.value &= RX_DMA_CTL_STAT_WR1C; 1809 cs.bits.hdw.mex = 1; 1810 RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, channel, 1811 cs.value); 1812 1813 /* 1814 * Rearm this logical group if this is a single device 1815 * group. 1816 */ 1817 if (ldgp->nldvs == 1) { 1818 if (isLDOMguest(nxgep)) { 1819 nxge_hio_ldgimgn(nxgep, ldgp); 1820 } else { 1821 ldgimgm_t mgm; 1822 1823 mgm.value = 0; 1824 mgm.bits.ldw.arm = 1; 1825 mgm.bits.ldw.timer = ldgp->ldg_timer; 1826 1827 NXGE_REG_WR64(handle, 1828 LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg), 1829 mgm.value); 1830 } 1831 } 1832 MUTEX_EXIT(&rcrp->lock); 1833 return (DDI_INTR_CLAIMED); 1834 } 1835 1836 ASSERT(rcrp->ldgp == ldgp); 1837 ASSERT(rcrp->ldvp == ldvp); 1838 1839 RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, channel, &cs.value); 1840 1841 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_intr:channel %d " 1842 "cs 0x%016llx rcrto 0x%x rcrthres %x", 1843 channel, 1844 cs.value, 1845 cs.bits.hdw.rcrto, 1846 cs.bits.hdw.rcrthres)); 1847 1848 if (!rcrp->poll_flag) { 1849 mp = nxge_rx_pkts(nxgep, rcrp, cs, -1); 1850 } 1851 1852 /* error events. */ 1853 if (cs.value & RX_DMA_CTL_STAT_ERROR) { 1854 (void) nxge_rx_err_evnts(nxgep, channel, cs); 1855 } 1856 1857 /* 1858 * Enable the mailbox update interrupt if we want 1859 * to use mailbox. We probably don't need to use 1860 * mailbox as it only saves us one pio read. 1861 * Also write 1 to rcrthres and rcrto to clear 1862 * these two edge triggered bits. 1863 */ 1864 cs.value &= RX_DMA_CTL_STAT_WR1C; 1865 cs.bits.hdw.mex = rcrp->poll_flag ? 0 : 1; 1866 RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, channel, 1867 cs.value); 1868 1869 /* 1870 * If the polling mode is enabled, disable the interrupt. 1871 */ 1872 if (rcrp->poll_flag) { 1873 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 1874 "==> nxge_rx_intr: rdc %d ldgp $%p ldvp $%p " 1875 "(disabling interrupts)", channel, ldgp, ldvp)); 1876 1877 /* 1878 * Disarm this logical group if this is a single device 1879 * group. 1880 */ 1881 if (ldgp->nldvs == 1) { 1882 if (isLDOMguest(nxgep)) { 1883 ldgp->arm = B_FALSE; 1884 nxge_hio_ldgimgn(nxgep, ldgp); 1885 } else { 1886 ldgimgm_t mgm; 1887 mgm.value = 0; 1888 mgm.bits.ldw.arm = 0; 1889 NXGE_REG_WR64(handle, 1890 LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg), 1891 mgm.value); 1892 } 1893 } 1894 } else { 1895 /* 1896 * Rearm this logical group if this is a single device 1897 * group. 1898 */ 1899 if (ldgp->nldvs == 1) { 1900 if (isLDOMguest(nxgep)) { 1901 nxge_hio_ldgimgn(nxgep, ldgp); 1902 } else { 1903 ldgimgm_t mgm; 1904 1905 mgm.value = 0; 1906 mgm.bits.ldw.arm = 1; 1907 mgm.bits.ldw.timer = ldgp->ldg_timer; 1908 1909 NXGE_REG_WR64(handle, 1910 LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg), 1911 mgm.value); 1912 } 1913 } 1914 1915 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 1916 "==> nxge_rx_intr: rdc %d ldgp $%p " 1917 "exiting ISR (and call mac_rx_ring)", channel, ldgp)); 1918 } 1919 MUTEX_EXIT(&rcrp->lock); 1920 1921 if (mp != NULL) { 1922 mac_rx_ring(nxgep->mach, rcrp->rcr_mac_handle, mp, 1923 rcrp->rcr_gen_num); 1924 } 1925 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_intr: DDI_INTR_CLAIMED")); 1926 return (DDI_INTR_CLAIMED); 1927 } 1928 1929 /* 1930 * This routine is the main packet receive processing function. 1931 * It gets the packet type, error code, and buffer related 1932 * information from the receive completion entry. 1933 * How many completion entries to process is based on the number of packets 1934 * queued by the hardware, a hardware maintained tail pointer 1935 * and a configurable receive packet count. 1936 * 1937 * A chain of message blocks will be created as result of processing 1938 * the completion entries. This chain of message blocks will be returned and 1939 * a hardware control status register will be updated with the number of 1940 * packets were removed from the hardware queue. 1941 * 1942 * The RCR ring lock is held when entering this function. 1943 */ 1944 static mblk_t * 1945 nxge_rx_pkts(p_nxge_t nxgep, p_rx_rcr_ring_t rcr_p, rx_dma_ctl_stat_t cs, 1946 int bytes_to_pickup) 1947 { 1948 npi_handle_t handle; 1949 uint8_t channel; 1950 uint32_t comp_rd_index; 1951 p_rcr_entry_t rcr_desc_rd_head_p; 1952 p_rcr_entry_t rcr_desc_rd_head_pp; 1953 p_mblk_t nmp, mp_cont, head_mp, *tail_mp; 1954 uint16_t qlen, nrcr_read, npkt_read; 1955 uint32_t qlen_hw; 1956 boolean_t multi; 1957 rcrcfig_b_t rcr_cfg_b; 1958 int totallen = 0; 1959 #if defined(_BIG_ENDIAN) 1960 npi_status_t rs = NPI_SUCCESS; 1961 #endif 1962 1963 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, "==> nxge_rx_pkts: " 1964 "channel %d", rcr_p->rdc)); 1965 1966 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 1967 return (NULL); 1968 } 1969 handle = NXGE_DEV_NPI_HANDLE(nxgep); 1970 channel = rcr_p->rdc; 1971 1972 NXGE_DEBUG_MSG((nxgep, RX_CTL, 1973 "==> nxge_rx_pkts: START: rcr channel %d " 1974 "head_p $%p head_pp $%p index %d ", 1975 channel, rcr_p->rcr_desc_rd_head_p, 1976 rcr_p->rcr_desc_rd_head_pp, 1977 rcr_p->comp_rd_index)); 1978 1979 1980 #if !defined(_BIG_ENDIAN) 1981 qlen = RXDMA_REG_READ32(handle, RCRSTAT_A_REG, channel) & 0xffff; 1982 #else 1983 rs = npi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen); 1984 if (rs != NPI_SUCCESS) { 1985 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts: " 1986 "channel %d, get qlen failed 0x%08x", 1987 channel, rs)); 1988 return (NULL); 1989 } 1990 #endif 1991 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts:rcr channel %d " 1992 "qlen %d", channel, qlen)); 1993 1994 1995 1996 if (!qlen) { 1997 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 1998 "==> nxge_rx_pkts:rcr channel %d " 1999 "qlen %d (no pkts)", channel, qlen)); 2000 2001 return (NULL); 2002 } 2003 2004 comp_rd_index = rcr_p->comp_rd_index; 2005 2006 rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p; 2007 rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp; 2008 nrcr_read = npkt_read = 0; 2009 2010 /* 2011 * Number of packets queued 2012 * (The jumbo or multi packet will be counted as only one 2013 * packets and it may take up more than one completion entry). 2014 */ 2015 qlen_hw = (qlen < nxge_max_rx_pkts) ? 2016 qlen : nxge_max_rx_pkts; 2017 head_mp = NULL; 2018 tail_mp = &head_mp; 2019 nmp = mp_cont = NULL; 2020 multi = B_FALSE; 2021 2022 while (qlen_hw) { 2023 2024 #ifdef NXGE_DEBUG 2025 nxge_dump_rcr_entry(nxgep, rcr_desc_rd_head_p); 2026 #endif 2027 /* 2028 * Process one completion ring entry. 2029 */ 2030 nxge_receive_packet(nxgep, 2031 rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont); 2032 2033 /* 2034 * message chaining modes 2035 */ 2036 if (nmp) { 2037 nmp->b_next = NULL; 2038 if (!multi && !mp_cont) { /* frame fits a partition */ 2039 *tail_mp = nmp; 2040 tail_mp = &nmp->b_next; 2041 totallen += MBLKL(nmp); 2042 nmp = NULL; 2043 } else if (multi && !mp_cont) { /* first segment */ 2044 *tail_mp = nmp; 2045 tail_mp = &nmp->b_cont; 2046 totallen += MBLKL(nmp); 2047 } else if (multi && mp_cont) { /* mid of multi segs */ 2048 *tail_mp = mp_cont; 2049 tail_mp = &mp_cont->b_cont; 2050 totallen += MBLKL(mp_cont); 2051 } else if (!multi && mp_cont) { /* last segment */ 2052 *tail_mp = mp_cont; 2053 tail_mp = &nmp->b_next; 2054 totallen += MBLKL(mp_cont); 2055 nmp = NULL; 2056 } 2057 } 2058 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2059 "==> nxge_rx_pkts: loop: rcr channel %d " 2060 "before updating: multi %d " 2061 "nrcr_read %d " 2062 "npk read %d " 2063 "head_pp $%p index %d ", 2064 channel, 2065 multi, 2066 nrcr_read, npkt_read, rcr_desc_rd_head_pp, 2067 comp_rd_index)); 2068 2069 if (!multi) { 2070 qlen_hw--; 2071 npkt_read++; 2072 } 2073 2074 /* 2075 * Update the next read entry. 2076 */ 2077 comp_rd_index = NEXT_ENTRY(comp_rd_index, 2078 rcr_p->comp_wrap_mask); 2079 2080 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p, 2081 rcr_p->rcr_desc_first_p, 2082 rcr_p->rcr_desc_last_p); 2083 2084 nrcr_read++; 2085 2086 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2087 "<== nxge_rx_pkts: (SAM, process one packet) " 2088 "nrcr_read %d", 2089 nrcr_read)); 2090 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2091 "==> nxge_rx_pkts: loop: rcr channel %d " 2092 "multi %d " 2093 "nrcr_read %d " 2094 "npk read %d " 2095 "head_pp $%p index %d ", 2096 channel, 2097 multi, 2098 nrcr_read, npkt_read, rcr_desc_rd_head_pp, 2099 comp_rd_index)); 2100 2101 if ((bytes_to_pickup != -1) && 2102 (totallen >= bytes_to_pickup)) { 2103 break; 2104 } 2105 } 2106 2107 rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp; 2108 rcr_p->comp_rd_index = comp_rd_index; 2109 rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p; 2110 if ((nxgep->intr_timeout != rcr_p->intr_timeout) || 2111 (nxgep->intr_threshold != rcr_p->intr_threshold)) { 2112 2113 rcr_p->intr_timeout = (nxgep->intr_timeout < 2114 NXGE_RDC_RCR_TIMEOUT_MIN) ? NXGE_RDC_RCR_TIMEOUT_MIN : 2115 nxgep->intr_timeout; 2116 2117 rcr_p->intr_threshold = (nxgep->intr_threshold < 2118 NXGE_RDC_RCR_THRESHOLD_MIN) ? NXGE_RDC_RCR_THRESHOLD_MIN : 2119 nxgep->intr_threshold; 2120 2121 rcr_cfg_b.value = 0x0ULL; 2122 rcr_cfg_b.bits.ldw.entout = 1; 2123 rcr_cfg_b.bits.ldw.timeout = rcr_p->intr_timeout; 2124 rcr_cfg_b.bits.ldw.pthres = rcr_p->intr_threshold; 2125 2126 RXDMA_REG_WRITE64(handle, RCRCFIG_B_REG, 2127 channel, rcr_cfg_b.value); 2128 } 2129 2130 cs.bits.ldw.pktread = npkt_read; 2131 cs.bits.ldw.ptrread = nrcr_read; 2132 RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, 2133 channel, cs.value); 2134 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2135 "==> nxge_rx_pkts: EXIT: rcr channel %d " 2136 "head_pp $%p index %016llx ", 2137 channel, 2138 rcr_p->rcr_desc_rd_head_pp, 2139 rcr_p->comp_rd_index)); 2140 /* 2141 * Update RCR buffer pointer read and number of packets 2142 * read. 2143 */ 2144 2145 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, "<== nxge_rx_pkts: return" 2146 "channel %d", rcr_p->rdc)); 2147 2148 return (head_mp); 2149 } 2150 2151 void 2152 nxge_receive_packet(p_nxge_t nxgep, 2153 p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p, 2154 boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont) 2155 { 2156 p_mblk_t nmp = NULL; 2157 uint64_t multi; 2158 uint64_t dcf_err; 2159 uint8_t channel; 2160 2161 boolean_t first_entry = B_TRUE; 2162 boolean_t is_tcp_udp = B_FALSE; 2163 boolean_t buffer_free = B_FALSE; 2164 boolean_t error_send_up = B_FALSE; 2165 uint8_t error_type; 2166 uint16_t l2_len; 2167 uint16_t skip_len; 2168 uint8_t pktbufsz_type; 2169 uint64_t rcr_entry; 2170 uint64_t *pkt_buf_addr_pp; 2171 uint64_t *pkt_buf_addr_p; 2172 uint32_t buf_offset; 2173 uint32_t bsize; 2174 uint32_t error_disp_cnt; 2175 uint32_t msg_index; 2176 p_rx_rbr_ring_t rx_rbr_p; 2177 p_rx_msg_t *rx_msg_ring_p; 2178 p_rx_msg_t rx_msg_p; 2179 uint16_t sw_offset_bytes = 0, hdr_size = 0; 2180 nxge_status_t status = NXGE_OK; 2181 boolean_t is_valid = B_FALSE; 2182 p_nxge_rx_ring_stats_t rdc_stats; 2183 uint32_t bytes_read; 2184 uint64_t pkt_type; 2185 uint64_t frag; 2186 boolean_t pkt_too_long_err = B_FALSE; 2187 #ifdef NXGE_DEBUG 2188 int dump_len; 2189 #endif 2190 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_receive_packet")); 2191 first_entry = (*mp == NULL) ? B_TRUE : B_FALSE; 2192 2193 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p); 2194 2195 multi = (rcr_entry & RCR_MULTI_MASK); 2196 dcf_err = (rcr_entry & RCR_DCF_ERROR_MASK); 2197 pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK); 2198 2199 error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT); 2200 frag = (rcr_entry & RCR_FRAG_MASK); 2201 2202 l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT); 2203 2204 pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >> 2205 RCR_PKTBUFSZ_SHIFT); 2206 pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) << 2207 RCR_PKT_BUF_ADDR_SHIFT); 2208 2209 channel = rcr_p->rdc; 2210 2211 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2212 "==> nxge_receive_packet: entryp $%p entry 0x%0llx " 2213 "pkt_buf_addr_pp $%p l2_len %d multi 0x%llx " 2214 "error_type 0x%x pkt_type 0x%x " 2215 "pktbufsz_type %d ", 2216 rcr_desc_rd_head_p, 2217 rcr_entry, pkt_buf_addr_pp, l2_len, 2218 multi, 2219 error_type, 2220 pkt_type, 2221 pktbufsz_type)); 2222 2223 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2224 "==> nxge_receive_packet: entryp $%p entry 0x%0llx " 2225 "pkt_buf_addr_pp $%p l2_len %d multi 0x%llx " 2226 "error_type 0x%x pkt_type 0x%x ", rcr_desc_rd_head_p, 2227 rcr_entry, pkt_buf_addr_pp, l2_len, 2228 multi, 2229 error_type, 2230 pkt_type)); 2231 2232 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2233 "==> (rbr) nxge_receive_packet: entry 0x%0llx " 2234 "full pkt_buf_addr_pp $%p l2_len %d", 2235 rcr_entry, pkt_buf_addr_pp, l2_len)); 2236 2237 /* get the stats ptr */ 2238 rdc_stats = rcr_p->rdc_stats; 2239 2240 if (!l2_len) { 2241 2242 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2243 "<== nxge_receive_packet: failed: l2 length is 0.")); 2244 return; 2245 } 2246 2247 /* 2248 * Software workaround for BMAC hardware limitation that allows 2249 * maxframe size of 1526, instead of 1522 for non-jumbo and 0x2406 2250 * instead of 0x2400 for jumbo. 2251 */ 2252 if (l2_len > nxgep->mac.maxframesize) { 2253 pkt_too_long_err = B_TRUE; 2254 } 2255 2256 /* Hardware sends us 4 bytes of CRC as no stripping is done. */ 2257 l2_len -= ETHERFCSL; 2258 2259 /* shift 6 bits to get the full io address */ 2260 pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp << 2261 RCR_PKT_BUF_ADDR_SHIFT_FULL); 2262 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2263 "==> (rbr) nxge_receive_packet: entry 0x%0llx " 2264 "full pkt_buf_addr_pp $%p l2_len %d", 2265 rcr_entry, pkt_buf_addr_pp, l2_len)); 2266 2267 rx_rbr_p = rcr_p->rx_rbr_p; 2268 rx_msg_ring_p = rx_rbr_p->rx_msg_ring; 2269 2270 if (first_entry) { 2271 hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL : 2272 RXDMA_HDR_SIZE_DEFAULT); 2273 2274 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2275 "==> nxge_receive_packet: first entry 0x%016llx " 2276 "pkt_buf_addr_pp $%p l2_len %d hdr %d", 2277 rcr_entry, pkt_buf_addr_pp, l2_len, 2278 hdr_size)); 2279 } 2280 2281 MUTEX_ENTER(&rx_rbr_p->lock); 2282 2283 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2284 "==> (rbr 1) nxge_receive_packet: entry 0x%0llx " 2285 "full pkt_buf_addr_pp $%p l2_len %d", 2286 rcr_entry, pkt_buf_addr_pp, l2_len)); 2287 2288 /* 2289 * Packet buffer address in the completion entry points 2290 * to the starting buffer address (offset 0). 2291 * Use the starting buffer address to locate the corresponding 2292 * kernel address. 2293 */ 2294 status = nxge_rxbuf_pp_to_vp(nxgep, rx_rbr_p, 2295 pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p, 2296 &buf_offset, 2297 &msg_index); 2298 2299 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2300 "==> (rbr 2) nxge_receive_packet: entry 0x%0llx " 2301 "full pkt_buf_addr_pp $%p l2_len %d", 2302 rcr_entry, pkt_buf_addr_pp, l2_len)); 2303 2304 if (status != NXGE_OK) { 2305 MUTEX_EXIT(&rx_rbr_p->lock); 2306 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2307 "<== nxge_receive_packet: found vaddr failed %d", 2308 status)); 2309 return; 2310 } 2311 2312 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2313 "==> (rbr 3) nxge_receive_packet: entry 0x%0llx " 2314 "full pkt_buf_addr_pp $%p l2_len %d", 2315 rcr_entry, pkt_buf_addr_pp, l2_len)); 2316 2317 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2318 "==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx " 2319 "full pkt_buf_addr_pp $%p l2_len %d", 2320 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 2321 2322 rx_msg_p = rx_msg_ring_p[msg_index]; 2323 2324 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2325 "==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx " 2326 "full pkt_buf_addr_pp $%p l2_len %d", 2327 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 2328 2329 switch (pktbufsz_type) { 2330 case RCR_PKTBUFSZ_0: 2331 bsize = rx_rbr_p->pkt_buf_size0_bytes; 2332 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2333 "==> nxge_receive_packet: 0 buf %d", bsize)); 2334 break; 2335 case RCR_PKTBUFSZ_1: 2336 bsize = rx_rbr_p->pkt_buf_size1_bytes; 2337 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2338 "==> nxge_receive_packet: 1 buf %d", bsize)); 2339 break; 2340 case RCR_PKTBUFSZ_2: 2341 bsize = rx_rbr_p->pkt_buf_size2_bytes; 2342 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2343 "==> nxge_receive_packet: 2 buf %d", bsize)); 2344 break; 2345 case RCR_SINGLE_BLOCK: 2346 bsize = rx_msg_p->block_size; 2347 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2348 "==> nxge_receive_packet: single %d", bsize)); 2349 2350 break; 2351 default: 2352 MUTEX_EXIT(&rx_rbr_p->lock); 2353 return; 2354 } 2355 2356 switch (nxge_rdc_buf_offset) { 2357 case SW_OFFSET_NO_OFFSET: 2358 sw_offset_bytes = 0; 2359 break; 2360 case SW_OFFSET_64: 2361 sw_offset_bytes = 64; 2362 break; 2363 case SW_OFFSET_128: 2364 sw_offset_bytes = 128; 2365 break; 2366 case SW_OFFSET_192: 2367 sw_offset_bytes = 192; 2368 break; 2369 case SW_OFFSET_256: 2370 sw_offset_bytes = 256; 2371 break; 2372 case SW_OFFSET_320: 2373 sw_offset_bytes = 320; 2374 break; 2375 case SW_OFFSET_384: 2376 sw_offset_bytes = 384; 2377 break; 2378 case SW_OFFSET_448: 2379 sw_offset_bytes = 448; 2380 break; 2381 default: 2382 sw_offset_bytes = 0; 2383 break; 2384 } 2385 2386 DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma, 2387 (buf_offset + sw_offset_bytes), 2388 (hdr_size + l2_len), 2389 DDI_DMA_SYNC_FORCPU); 2390 2391 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2392 "==> nxge_receive_packet: after first dump:usage count")); 2393 2394 if (rx_msg_p->cur_usage_cnt == 0) { 2395 if (rx_rbr_p->rbr_use_bcopy) { 2396 atomic_inc_32(&rx_rbr_p->rbr_consumed); 2397 if (rx_rbr_p->rbr_consumed < 2398 rx_rbr_p->rbr_threshold_hi) { 2399 if (rx_rbr_p->rbr_threshold_lo == 0 || 2400 ((rx_rbr_p->rbr_consumed >= 2401 rx_rbr_p->rbr_threshold_lo) && 2402 (rx_rbr_p->rbr_bufsize_type >= 2403 pktbufsz_type))) { 2404 rx_msg_p->rx_use_bcopy = B_TRUE; 2405 } 2406 } else { 2407 rx_msg_p->rx_use_bcopy = B_TRUE; 2408 } 2409 } 2410 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2411 "==> nxge_receive_packet: buf %d (new block) ", 2412 bsize)); 2413 2414 rx_msg_p->pkt_buf_size_code = pktbufsz_type; 2415 rx_msg_p->pkt_buf_size = bsize; 2416 rx_msg_p->cur_usage_cnt = 1; 2417 if (pktbufsz_type == RCR_SINGLE_BLOCK) { 2418 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2419 "==> nxge_receive_packet: buf %d " 2420 "(single block) ", 2421 bsize)); 2422 /* 2423 * Buffer can be reused once the free function 2424 * is called. 2425 */ 2426 rx_msg_p->max_usage_cnt = 1; 2427 buffer_free = B_TRUE; 2428 } else { 2429 rx_msg_p->max_usage_cnt = rx_msg_p->block_size/bsize; 2430 if (rx_msg_p->max_usage_cnt == 1) { 2431 buffer_free = B_TRUE; 2432 } 2433 } 2434 } else { 2435 rx_msg_p->cur_usage_cnt++; 2436 if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) { 2437 buffer_free = B_TRUE; 2438 } 2439 } 2440 2441 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2442 "msgbuf index = %d l2len %d bytes usage %d max_usage %d ", 2443 msg_index, l2_len, 2444 rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt)); 2445 2446 if ((error_type) || (dcf_err) || (pkt_too_long_err)) { 2447 rdc_stats->ierrors++; 2448 if (dcf_err) { 2449 rdc_stats->dcf_err++; 2450 #ifdef NXGE_DEBUG 2451 if (!rdc_stats->dcf_err) { 2452 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2453 "nxge_receive_packet: channel %d dcf_err rcr" 2454 " 0x%llx", channel, rcr_entry)); 2455 } 2456 #endif 2457 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, 0, 2458 NXGE_FM_EREPORT_RDMC_DCF_ERR); 2459 } else if (pkt_too_long_err) { 2460 rdc_stats->pkt_too_long_err++; 2461 NXGE_DEBUG_MSG((nxgep, RX_CTL, " nxge_receive_packet:" 2462 " channel %d packet length [%d] > " 2463 "maxframesize [%d]", channel, l2_len + ETHERFCSL, 2464 nxgep->mac.maxframesize)); 2465 } else { 2466 /* Update error stats */ 2467 error_disp_cnt = NXGE_ERROR_SHOW_MAX; 2468 rdc_stats->errlog.compl_err_type = error_type; 2469 2470 switch (error_type) { 2471 /* 2472 * Do not send FMA ereport for RCR_L2_ERROR and 2473 * RCR_L4_CSUM_ERROR because most likely they indicate 2474 * back pressure rather than HW failures. 2475 */ 2476 case RCR_L2_ERROR: 2477 rdc_stats->l2_err++; 2478 if (rdc_stats->l2_err < 2479 error_disp_cnt) { 2480 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2481 " nxge_receive_packet:" 2482 " channel %d RCR L2_ERROR", 2483 channel)); 2484 } 2485 break; 2486 case RCR_L4_CSUM_ERROR: 2487 error_send_up = B_TRUE; 2488 rdc_stats->l4_cksum_err++; 2489 if (rdc_stats->l4_cksum_err < 2490 error_disp_cnt) { 2491 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2492 " nxge_receive_packet:" 2493 " channel %d" 2494 " RCR L4_CSUM_ERROR", channel)); 2495 } 2496 break; 2497 /* 2498 * Do not send FMA ereport for RCR_FFLP_SOFT_ERROR and 2499 * RCR_ZCP_SOFT_ERROR because they reflect the same 2500 * FFLP and ZCP errors that have been reported by 2501 * nxge_fflp.c and nxge_zcp.c. 2502 */ 2503 case RCR_FFLP_SOFT_ERROR: 2504 error_send_up = B_TRUE; 2505 rdc_stats->fflp_soft_err++; 2506 if (rdc_stats->fflp_soft_err < 2507 error_disp_cnt) { 2508 NXGE_ERROR_MSG((nxgep, 2509 NXGE_ERR_CTL, 2510 " nxge_receive_packet:" 2511 " channel %d" 2512 " RCR FFLP_SOFT_ERROR", channel)); 2513 } 2514 break; 2515 case RCR_ZCP_SOFT_ERROR: 2516 error_send_up = B_TRUE; 2517 rdc_stats->fflp_soft_err++; 2518 if (rdc_stats->zcp_soft_err < 2519 error_disp_cnt) 2520 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2521 " nxge_receive_packet: Channel %d" 2522 " RCR ZCP_SOFT_ERROR", channel)); 2523 break; 2524 default: 2525 rdc_stats->rcr_unknown_err++; 2526 if (rdc_stats->rcr_unknown_err 2527 < error_disp_cnt) { 2528 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2529 " nxge_receive_packet: Channel %d" 2530 " RCR entry 0x%llx error 0x%x", 2531 rcr_entry, channel, error_type)); 2532 } 2533 break; 2534 } 2535 } 2536 2537 /* 2538 * Update and repost buffer block if max usage 2539 * count is reached. 2540 */ 2541 if (error_send_up == B_FALSE) { 2542 atomic_inc_32(&rx_msg_p->ref_cnt); 2543 if (buffer_free == B_TRUE) { 2544 rx_msg_p->free = B_TRUE; 2545 } 2546 2547 MUTEX_EXIT(&rx_rbr_p->lock); 2548 nxge_freeb(rx_msg_p); 2549 return; 2550 } 2551 } 2552 2553 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2554 "==> nxge_receive_packet: DMA sync second ")); 2555 2556 bytes_read = rcr_p->rcvd_pkt_bytes; 2557 skip_len = sw_offset_bytes + hdr_size; 2558 if (!rx_msg_p->rx_use_bcopy) { 2559 /* 2560 * For loaned up buffers, the driver reference count 2561 * will be incremented first and then the free state. 2562 */ 2563 if ((nmp = nxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) { 2564 if (first_entry) { 2565 nmp->b_rptr = &nmp->b_rptr[skip_len]; 2566 if (l2_len < bsize - skip_len) { 2567 nmp->b_wptr = &nmp->b_rptr[l2_len]; 2568 } else { 2569 nmp->b_wptr = &nmp->b_rptr[bsize 2570 - skip_len]; 2571 } 2572 } else { 2573 if (l2_len - bytes_read < bsize) { 2574 nmp->b_wptr = 2575 &nmp->b_rptr[l2_len - bytes_read]; 2576 } else { 2577 nmp->b_wptr = &nmp->b_rptr[bsize]; 2578 } 2579 } 2580 } 2581 } else { 2582 if (first_entry) { 2583 nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len, 2584 l2_len < bsize - skip_len ? 2585 l2_len : bsize - skip_len); 2586 } else { 2587 nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset, 2588 l2_len - bytes_read < bsize ? 2589 l2_len - bytes_read : bsize); 2590 } 2591 } 2592 if (nmp != NULL) { 2593 if (first_entry) { 2594 /* 2595 * Jumbo packets may be received with more than one 2596 * buffer, increment ipackets for the first entry only. 2597 */ 2598 rdc_stats->ipackets++; 2599 2600 /* Update ibytes for kstat. */ 2601 rdc_stats->ibytes += skip_len 2602 + l2_len < bsize ? l2_len : bsize; 2603 /* 2604 * Update the number of bytes read so far for the 2605 * current frame. 2606 */ 2607 bytes_read = nmp->b_wptr - nmp->b_rptr; 2608 } else { 2609 rdc_stats->ibytes += l2_len - bytes_read < bsize ? 2610 l2_len - bytes_read : bsize; 2611 bytes_read += nmp->b_wptr - nmp->b_rptr; 2612 } 2613 2614 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2615 "==> nxge_receive_packet after dupb: " 2616 "rbr consumed %d " 2617 "pktbufsz_type %d " 2618 "nmp $%p rptr $%p wptr $%p " 2619 "buf_offset %d bzise %d l2_len %d skip_len %d", 2620 rx_rbr_p->rbr_consumed, 2621 pktbufsz_type, 2622 nmp, nmp->b_rptr, nmp->b_wptr, 2623 buf_offset, bsize, l2_len, skip_len)); 2624 } else { 2625 cmn_err(CE_WARN, "!nxge_receive_packet: " 2626 "update stats (error)"); 2627 atomic_inc_32(&rx_msg_p->ref_cnt); 2628 if (buffer_free == B_TRUE) { 2629 rx_msg_p->free = B_TRUE; 2630 } 2631 MUTEX_EXIT(&rx_rbr_p->lock); 2632 nxge_freeb(rx_msg_p); 2633 return; 2634 } 2635 2636 if (buffer_free == B_TRUE) { 2637 rx_msg_p->free = B_TRUE; 2638 } 2639 2640 is_valid = (nmp != NULL); 2641 2642 rcr_p->rcvd_pkt_bytes = bytes_read; 2643 2644 MUTEX_EXIT(&rx_rbr_p->lock); 2645 2646 if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) { 2647 atomic_inc_32(&rx_msg_p->ref_cnt); 2648 nxge_freeb(rx_msg_p); 2649 } 2650 2651 if (is_valid) { 2652 nmp->b_cont = NULL; 2653 if (first_entry) { 2654 *mp = nmp; 2655 *mp_cont = NULL; 2656 } else { 2657 *mp_cont = nmp; 2658 } 2659 } 2660 2661 /* 2662 * ERROR, FRAG and PKT_TYPE are only reported in the first entry. 2663 * If a packet is not fragmented and no error bit is set, then 2664 * L4 checksum is OK. 2665 */ 2666 2667 if (is_valid && !multi) { 2668 /* 2669 * If the checksum flag nxge_chksum_offload 2670 * is 1, TCP and UDP packets can be sent 2671 * up with good checksum. If the checksum flag 2672 * is set to 0, checksum reporting will apply to 2673 * TCP packets only (workaround for a hardware bug). 2674 * If the checksum flag nxge_cksum_offload is 2675 * greater than 1, both TCP and UDP packets 2676 * will not be reported its hardware checksum results. 2677 */ 2678 if (nxge_cksum_offload == 1) { 2679 is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP || 2680 pkt_type == RCR_PKT_IS_UDP) ? 2681 B_TRUE: B_FALSE); 2682 } else if (!nxge_cksum_offload) { 2683 /* TCP checksum only. */ 2684 is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP) ? 2685 B_TRUE: B_FALSE); 2686 } 2687 2688 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_receive_packet: " 2689 "is_valid 0x%x multi 0x%llx pkt %d frag %d error %d", 2690 is_valid, multi, is_tcp_udp, frag, error_type)); 2691 2692 if (is_tcp_udp && !frag && !error_type) { 2693 mac_hcksum_set(nmp, 0, 0, 0, 0, HCK_FULLCKSUM_OK); 2694 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2695 "==> nxge_receive_packet: Full tcp/udp cksum " 2696 "is_valid 0x%x multi 0x%llx pkt %d frag %d " 2697 "error %d", 2698 is_valid, multi, is_tcp_udp, frag, error_type)); 2699 } 2700 } 2701 2702 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 2703 "==> nxge_receive_packet: *mp 0x%016llx", *mp)); 2704 2705 *multi_p = (multi == RCR_MULTI_MASK); 2706 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_receive_packet: " 2707 "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx", 2708 *multi_p, nmp, *mp, *mp_cont)); 2709 } 2710 2711 /* 2712 * Enable polling for a ring. Interrupt for the ring is disabled when 2713 * the nxge interrupt comes (see nxge_rx_intr). 2714 */ 2715 int 2716 nxge_enable_poll(void *arg) 2717 { 2718 p_nxge_ring_handle_t ring_handle = (p_nxge_ring_handle_t)arg; 2719 p_rx_rcr_ring_t ringp; 2720 p_nxge_t nxgep; 2721 p_nxge_ldg_t ldgp; 2722 uint32_t channel; 2723 2724 if (ring_handle == NULL) { 2725 ASSERT(ring_handle != NULL); 2726 return (0); 2727 } 2728 2729 nxgep = ring_handle->nxgep; 2730 channel = nxgep->pt_config.hw_config.start_rdc + ring_handle->index; 2731 ringp = nxgep->rx_rcr_rings->rcr_rings[channel]; 2732 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 2733 "==> nxge_enable_poll: rdc %d ", ringp->rdc)); 2734 ldgp = ringp->ldgp; 2735 if (ldgp == NULL) { 2736 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 2737 "==> nxge_enable_poll: rdc %d NULL ldgp: no change", 2738 ringp->rdc)); 2739 return (0); 2740 } 2741 2742 MUTEX_ENTER(&ringp->lock); 2743 /* enable polling */ 2744 if (ringp->poll_flag == 0) { 2745 ringp->poll_flag = 1; 2746 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 2747 "==> nxge_enable_poll: rdc %d set poll flag to 1", 2748 ringp->rdc)); 2749 } 2750 2751 MUTEX_EXIT(&ringp->lock); 2752 return (0); 2753 } 2754 /* 2755 * Disable polling for a ring and enable its interrupt. 2756 */ 2757 int 2758 nxge_disable_poll(void *arg) 2759 { 2760 p_nxge_ring_handle_t ring_handle = (p_nxge_ring_handle_t)arg; 2761 p_rx_rcr_ring_t ringp; 2762 p_nxge_t nxgep; 2763 uint32_t channel; 2764 2765 if (ring_handle == NULL) { 2766 ASSERT(ring_handle != NULL); 2767 return (0); 2768 } 2769 2770 nxgep = ring_handle->nxgep; 2771 channel = nxgep->pt_config.hw_config.start_rdc + ring_handle->index; 2772 ringp = nxgep->rx_rcr_rings->rcr_rings[channel]; 2773 2774 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 2775 "==> nxge_disable_poll: rdc %d poll_flag %d", ringp->rdc)); 2776 2777 MUTEX_ENTER(&ringp->lock); 2778 2779 /* disable polling: enable interrupt */ 2780 if (ringp->poll_flag) { 2781 npi_handle_t handle; 2782 rx_dma_ctl_stat_t cs; 2783 uint8_t channel; 2784 p_nxge_ldg_t ldgp; 2785 2786 /* 2787 * Get the control and status for this channel. 2788 */ 2789 handle = NXGE_DEV_NPI_HANDLE(nxgep); 2790 channel = ringp->rdc; 2791 RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, 2792 channel, &cs.value); 2793 2794 /* 2795 * Enable mailbox update 2796 * Since packets were not read and the hardware uses 2797 * bits pktread and ptrread to update the queue 2798 * length, we need to set both bits to 0. 2799 */ 2800 cs.bits.ldw.pktread = 0; 2801 cs.bits.ldw.ptrread = 0; 2802 cs.bits.hdw.mex = 1; 2803 RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, channel, 2804 cs.value); 2805 2806 /* 2807 * Rearm this logical group if this is a single device 2808 * group. 2809 */ 2810 ldgp = ringp->ldgp; 2811 if (ldgp == NULL) { 2812 ringp->poll_flag = 0; 2813 MUTEX_EXIT(&ringp->lock); 2814 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 2815 "==> nxge_disable_poll: no ldgp rdc %d " 2816 "(still set poll to 0", ringp->rdc)); 2817 return (0); 2818 } 2819 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 2820 "==> nxge_disable_poll: rdc %d ldgp $%p (enable intr)", 2821 ringp->rdc, ldgp)); 2822 if (ldgp->nldvs == 1) { 2823 if (isLDOMguest(nxgep)) { 2824 ldgp->arm = B_TRUE; 2825 nxge_hio_ldgimgn(nxgep, ldgp); 2826 } else { 2827 ldgimgm_t mgm; 2828 mgm.value = 0; 2829 mgm.bits.ldw.arm = 1; 2830 mgm.bits.ldw.timer = ldgp->ldg_timer; 2831 NXGE_REG_WR64(handle, 2832 LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg), 2833 mgm.value); 2834 } 2835 } 2836 ringp->poll_flag = 0; 2837 } 2838 2839 MUTEX_EXIT(&ringp->lock); 2840 return (0); 2841 } 2842 2843 /* 2844 * Poll 'bytes_to_pickup' bytes of message from the rx ring. 2845 */ 2846 mblk_t * 2847 nxge_rx_poll(void *arg, int bytes_to_pickup) 2848 { 2849 p_nxge_ring_handle_t ring_handle = (p_nxge_ring_handle_t)arg; 2850 p_rx_rcr_ring_t rcr_p; 2851 p_nxge_t nxgep; 2852 npi_handle_t handle; 2853 rx_dma_ctl_stat_t cs; 2854 mblk_t *mblk; 2855 p_nxge_ldv_t ldvp; 2856 uint32_t channel; 2857 2858 nxgep = ring_handle->nxgep; 2859 2860 /* 2861 * Get the control and status for this channel. 2862 */ 2863 handle = NXGE_DEV_NPI_HANDLE(nxgep); 2864 channel = nxgep->pt_config.hw_config.start_rdc + ring_handle->index; 2865 rcr_p = nxgep->rx_rcr_rings->rcr_rings[channel]; 2866 MUTEX_ENTER(&rcr_p->lock); 2867 ASSERT(rcr_p->poll_flag == 1); 2868 2869 RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, rcr_p->rdc, &cs.value); 2870 2871 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 2872 "==> nxge_rx_poll: calling nxge_rx_pkts: rdc %d poll_flag %d", 2873 rcr_p->rdc, rcr_p->poll_flag)); 2874 mblk = nxge_rx_pkts(nxgep, rcr_p, cs, bytes_to_pickup); 2875 2876 ldvp = rcr_p->ldvp; 2877 /* error events. */ 2878 if (ldvp && (cs.value & RX_DMA_CTL_STAT_ERROR)) { 2879 (void) nxge_rx_err_evnts(nxgep, ldvp->vdma_index, cs); 2880 } 2881 2882 MUTEX_EXIT(&rcr_p->lock); 2883 2884 NXGE_DEBUG_MSG((nxgep, NXGE_ERR_CTL, 2885 "<== nxge_rx_poll: rdc %d mblk $%p", rcr_p->rdc, mblk)); 2886 return (mblk); 2887 } 2888 2889 2890 /*ARGSUSED*/ 2891 static nxge_status_t 2892 nxge_rx_err_evnts(p_nxge_t nxgep, int channel, rx_dma_ctl_stat_t cs) 2893 { 2894 p_nxge_rx_ring_stats_t rdc_stats; 2895 npi_handle_t handle; 2896 npi_status_t rs; 2897 boolean_t rxchan_fatal = B_FALSE; 2898 boolean_t rxport_fatal = B_FALSE; 2899 uint8_t portn; 2900 nxge_status_t status = NXGE_OK; 2901 uint32_t error_disp_cnt = NXGE_ERROR_SHOW_MAX; 2902 NXGE_DEBUG_MSG((nxgep, INT_CTL, "==> nxge_rx_err_evnts")); 2903 2904 handle = NXGE_DEV_NPI_HANDLE(nxgep); 2905 portn = nxgep->mac.portnum; 2906 rdc_stats = &nxgep->statsp->rdc_stats[channel]; 2907 2908 if (cs.bits.hdw.rbr_tmout) { 2909 rdc_stats->rx_rbr_tmout++; 2910 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 2911 NXGE_FM_EREPORT_RDMC_RBR_TMOUT); 2912 rxchan_fatal = B_TRUE; 2913 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2914 "==> nxge_rx_err_evnts: rx_rbr_timeout")); 2915 } 2916 if (cs.bits.hdw.rsp_cnt_err) { 2917 rdc_stats->rsp_cnt_err++; 2918 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 2919 NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR); 2920 rxchan_fatal = B_TRUE; 2921 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2922 "==> nxge_rx_err_evnts(channel %d): " 2923 "rsp_cnt_err", channel)); 2924 } 2925 if (cs.bits.hdw.byte_en_bus) { 2926 rdc_stats->byte_en_bus++; 2927 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 2928 NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS); 2929 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2930 "==> nxge_rx_err_evnts(channel %d): " 2931 "fatal error: byte_en_bus", channel)); 2932 rxchan_fatal = B_TRUE; 2933 } 2934 if (cs.bits.hdw.rsp_dat_err) { 2935 rdc_stats->rsp_dat_err++; 2936 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 2937 NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR); 2938 rxchan_fatal = B_TRUE; 2939 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2940 "==> nxge_rx_err_evnts(channel %d): " 2941 "fatal error: rsp_dat_err", channel)); 2942 } 2943 if (cs.bits.hdw.rcr_ack_err) { 2944 rdc_stats->rcr_ack_err++; 2945 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 2946 NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR); 2947 rxchan_fatal = B_TRUE; 2948 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2949 "==> nxge_rx_err_evnts(channel %d): " 2950 "fatal error: rcr_ack_err", channel)); 2951 } 2952 if (cs.bits.hdw.dc_fifo_err) { 2953 rdc_stats->dc_fifo_err++; 2954 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 2955 NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR); 2956 /* This is not a fatal error! */ 2957 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2958 "==> nxge_rx_err_evnts(channel %d): " 2959 "dc_fifo_err", channel)); 2960 rxport_fatal = B_TRUE; 2961 } 2962 if ((cs.bits.hdw.rcr_sha_par) || (cs.bits.hdw.rbr_pre_par)) { 2963 if ((rs = npi_rxdma_ring_perr_stat_get(handle, 2964 &rdc_stats->errlog.pre_par, 2965 &rdc_stats->errlog.sha_par)) 2966 != NPI_SUCCESS) { 2967 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2968 "==> nxge_rx_err_evnts(channel %d): " 2969 "rcr_sha_par: get perr", channel)); 2970 return (NXGE_ERROR | rs); 2971 } 2972 if (cs.bits.hdw.rcr_sha_par) { 2973 rdc_stats->rcr_sha_par++; 2974 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 2975 NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR); 2976 rxchan_fatal = B_TRUE; 2977 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2978 "==> nxge_rx_err_evnts(channel %d): " 2979 "fatal error: rcr_sha_par", channel)); 2980 } 2981 if (cs.bits.hdw.rbr_pre_par) { 2982 rdc_stats->rbr_pre_par++; 2983 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 2984 NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR); 2985 rxchan_fatal = B_TRUE; 2986 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 2987 "==> nxge_rx_err_evnts(channel %d): " 2988 "fatal error: rbr_pre_par", channel)); 2989 } 2990 } 2991 /* 2992 * The Following 4 status bits are for information, the system 2993 * is running fine. There is no need to send FMA ereports or 2994 * log messages. 2995 */ 2996 if (cs.bits.hdw.port_drop_pkt) { 2997 rdc_stats->port_drop_pkt++; 2998 } 2999 if (cs.bits.hdw.wred_drop) { 3000 rdc_stats->wred_drop++; 3001 } 3002 if (cs.bits.hdw.rbr_pre_empty) { 3003 rdc_stats->rbr_pre_empty++; 3004 } 3005 if (cs.bits.hdw.rcr_shadow_full) { 3006 rdc_stats->rcr_shadow_full++; 3007 } 3008 if (cs.bits.hdw.config_err) { 3009 rdc_stats->config_err++; 3010 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 3011 NXGE_FM_EREPORT_RDMC_CONFIG_ERR); 3012 rxchan_fatal = B_TRUE; 3013 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3014 "==> nxge_rx_err_evnts(channel %d): " 3015 "config error", channel)); 3016 } 3017 if (cs.bits.hdw.rcrincon) { 3018 rdc_stats->rcrincon++; 3019 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 3020 NXGE_FM_EREPORT_RDMC_RCRINCON); 3021 rxchan_fatal = B_TRUE; 3022 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3023 "==> nxge_rx_err_evnts(channel %d): " 3024 "fatal error: rcrincon error", channel)); 3025 } 3026 if (cs.bits.hdw.rcrfull) { 3027 rdc_stats->rcrfull++; 3028 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 3029 NXGE_FM_EREPORT_RDMC_RCRFULL); 3030 rxchan_fatal = B_TRUE; 3031 if (rdc_stats->rcrfull < error_disp_cnt) { 3032 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3033 "==> nxge_rx_err_evnts(channel %d): " 3034 "fatal error: rcrfull error", channel)); 3035 } 3036 } 3037 if (cs.bits.hdw.rbr_empty) { 3038 /* 3039 * This bit is for information, there is no need 3040 * send FMA ereport or log a message. 3041 */ 3042 rdc_stats->rbr_empty++; 3043 } 3044 if (cs.bits.hdw.rbrfull) { 3045 rdc_stats->rbrfull++; 3046 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 3047 NXGE_FM_EREPORT_RDMC_RBRFULL); 3048 rxchan_fatal = B_TRUE; 3049 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3050 "==> nxge_rx_err_evnts(channel %d): " 3051 "fatal error: rbr_full error", channel)); 3052 } 3053 if (cs.bits.hdw.rbrlogpage) { 3054 rdc_stats->rbrlogpage++; 3055 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 3056 NXGE_FM_EREPORT_RDMC_RBRLOGPAGE); 3057 rxchan_fatal = B_TRUE; 3058 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3059 "==> nxge_rx_err_evnts(channel %d): " 3060 "fatal error: rbr logical page error", channel)); 3061 } 3062 if (cs.bits.hdw.cfiglogpage) { 3063 rdc_stats->cfiglogpage++; 3064 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 3065 NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE); 3066 rxchan_fatal = B_TRUE; 3067 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3068 "==> nxge_rx_err_evnts(channel %d): " 3069 "fatal error: cfig logical page error", channel)); 3070 } 3071 3072 if (rxport_fatal) { 3073 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3074 " nxge_rx_err_evnts: fatal error on Port #%d\n", 3075 portn)); 3076 if (isLDOMguest(nxgep)) { 3077 status = NXGE_ERROR; 3078 } else { 3079 status = nxge_ipp_fatal_err_recover(nxgep); 3080 if (status == NXGE_OK) { 3081 FM_SERVICE_RESTORED(nxgep); 3082 } 3083 } 3084 } 3085 3086 if (rxchan_fatal) { 3087 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3088 " nxge_rx_err_evnts: fatal error on Channel #%d\n", 3089 channel)); 3090 if (isLDOMguest(nxgep)) { 3091 status = NXGE_ERROR; 3092 } else { 3093 status = nxge_rxdma_fatal_err_recover(nxgep, channel); 3094 if (status == NXGE_OK) { 3095 FM_SERVICE_RESTORED(nxgep); 3096 } 3097 } 3098 } 3099 3100 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rx_err_evnts")); 3101 3102 return (status); 3103 } 3104 3105 /* 3106 * nxge_rdc_hvio_setup 3107 * 3108 * This code appears to setup some Hypervisor variables. 3109 * 3110 * Arguments: 3111 * nxgep 3112 * channel 3113 * 3114 * Notes: 3115 * What does NIU_LP_WORKAROUND mean? 3116 * 3117 * NPI/NXGE function calls: 3118 * na 3119 * 3120 * Context: 3121 * Any domain 3122 */ 3123 #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 3124 static void 3125 nxge_rdc_hvio_setup( 3126 nxge_t *nxgep, int channel) 3127 { 3128 nxge_dma_common_t *dma_common; 3129 nxge_dma_common_t *dma_control; 3130 rx_rbr_ring_t *ring; 3131 3132 ring = nxgep->rx_rbr_rings->rbr_rings[channel]; 3133 dma_common = nxgep->rx_buf_pool_p->dma_buf_pool_p[channel]; 3134 3135 ring->hv_set = B_FALSE; 3136 3137 ring->hv_rx_buf_base_ioaddr_pp = (uint64_t) 3138 dma_common->orig_ioaddr_pp; 3139 ring->hv_rx_buf_ioaddr_size = (uint64_t) 3140 dma_common->orig_alength; 3141 3142 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: " 3143 "channel %d data buf base io $%lx ($%p) size 0x%lx (%ld 0x%lx)", 3144 channel, ring->hv_rx_buf_base_ioaddr_pp, 3145 dma_common->ioaddr_pp, ring->hv_rx_buf_ioaddr_size, 3146 dma_common->orig_alength, dma_common->orig_alength)); 3147 3148 dma_control = nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel]; 3149 3150 ring->hv_rx_cntl_base_ioaddr_pp = 3151 (uint64_t)dma_control->orig_ioaddr_pp; 3152 ring->hv_rx_cntl_ioaddr_size = 3153 (uint64_t)dma_control->orig_alength; 3154 3155 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: " 3156 "channel %d cntl base io $%p ($%p) size 0x%llx (%d 0x%x)", 3157 channel, ring->hv_rx_cntl_base_ioaddr_pp, 3158 dma_control->ioaddr_pp, ring->hv_rx_cntl_ioaddr_size, 3159 dma_control->orig_alength, dma_control->orig_alength)); 3160 } 3161 #endif 3162 3163 /* 3164 * nxge_map_rxdma 3165 * 3166 * Map an RDC into our kernel space. 3167 * 3168 * Arguments: 3169 * nxgep 3170 * channel The channel to map. 3171 * 3172 * Notes: 3173 * 1. Allocate & initialise a memory pool, if necessary. 3174 * 2. Allocate however many receive buffers are required. 3175 * 3. Setup buffers, descriptors, and mailbox. 3176 * 3177 * NPI/NXGE function calls: 3178 * nxge_alloc_rx_mem_pool() 3179 * nxge_alloc_rbb() 3180 * nxge_map_rxdma_channel() 3181 * 3182 * Registers accessed: 3183 * 3184 * Context: 3185 * Any domain 3186 */ 3187 static nxge_status_t 3188 nxge_map_rxdma(p_nxge_t nxgep, int channel) 3189 { 3190 nxge_dma_common_t **data; 3191 nxge_dma_common_t **control; 3192 rx_rbr_ring_t **rbr_ring; 3193 rx_rcr_ring_t **rcr_ring; 3194 rx_mbox_t **mailbox; 3195 uint32_t chunks; 3196 3197 nxge_status_t status; 3198 3199 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma")); 3200 3201 if (!nxgep->rx_buf_pool_p) { 3202 if (nxge_alloc_rx_mem_pool(nxgep) != NXGE_OK) { 3203 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3204 "<== nxge_map_rxdma: buf not allocated")); 3205 return (NXGE_ERROR); 3206 } 3207 } 3208 3209 if (nxge_alloc_rxb(nxgep, channel) != NXGE_OK) 3210 return (NXGE_ERROR); 3211 3212 /* 3213 * Map descriptors from the buffer polls for each dma channel. 3214 */ 3215 3216 /* 3217 * Set up and prepare buffer blocks, descriptors 3218 * and mailbox. 3219 */ 3220 data = &nxgep->rx_buf_pool_p->dma_buf_pool_p[channel]; 3221 rbr_ring = &nxgep->rx_rbr_rings->rbr_rings[channel]; 3222 chunks = nxgep->rx_buf_pool_p->num_chunks[channel]; 3223 3224 control = &nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel]; 3225 rcr_ring = &nxgep->rx_rcr_rings->rcr_rings[channel]; 3226 3227 mailbox = &nxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 3228 3229 status = nxge_map_rxdma_channel(nxgep, channel, data, rbr_ring, 3230 chunks, control, rcr_ring, mailbox); 3231 if (status != NXGE_OK) { 3232 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3233 "==> nxge_map_rxdma: nxge_map_rxdma_channel(%d) " 3234 "returned 0x%x", 3235 channel, status)); 3236 return (status); 3237 } 3238 nxgep->rx_rbr_rings->rbr_rings[channel]->index = (uint16_t)channel; 3239 nxgep->rx_rcr_rings->rcr_rings[channel]->index = (uint16_t)channel; 3240 nxgep->rx_rcr_rings->rcr_rings[channel]->rdc_stats = 3241 &nxgep->statsp->rdc_stats[channel]; 3242 3243 #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 3244 if (!isLDOMguest(nxgep)) 3245 nxge_rdc_hvio_setup(nxgep, channel); 3246 #endif 3247 3248 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3249 "<== nxge_map_rxdma: (status 0x%x channel %d)", status, channel)); 3250 3251 return (status); 3252 } 3253 3254 static void 3255 nxge_unmap_rxdma(p_nxge_t nxgep, int channel) 3256 { 3257 rx_rbr_ring_t *rbr_ring; 3258 rx_rcr_ring_t *rcr_ring; 3259 rx_mbox_t *mailbox; 3260 3261 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_unmap_rxdma(%d)", channel)); 3262 3263 if (!nxgep->rx_rbr_rings || !nxgep->rx_rcr_rings || 3264 !nxgep->rx_mbox_areas_p) 3265 return; 3266 3267 rbr_ring = nxgep->rx_rbr_rings->rbr_rings[channel]; 3268 rcr_ring = nxgep->rx_rcr_rings->rcr_rings[channel]; 3269 mailbox = nxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 3270 3271 if (!rbr_ring || !rcr_ring || !mailbox) 3272 return; 3273 3274 (void) nxge_unmap_rxdma_channel( 3275 nxgep, channel, rbr_ring, rcr_ring, mailbox); 3276 3277 nxge_free_rxb(nxgep, channel); 3278 3279 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma")); 3280 } 3281 3282 nxge_status_t 3283 nxge_map_rxdma_channel(p_nxge_t nxgep, uint16_t channel, 3284 p_nxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 3285 uint32_t num_chunks, 3286 p_nxge_dma_common_t *dma_cntl_p, p_rx_rcr_ring_t *rcr_p, 3287 p_rx_mbox_t *rx_mbox_p) 3288 { 3289 int status = NXGE_OK; 3290 3291 /* 3292 * Set up and prepare buffer blocks, descriptors 3293 * and mailbox. 3294 */ 3295 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3296 "==> nxge_map_rxdma_channel (channel %d)", channel)); 3297 /* 3298 * Receive buffer blocks 3299 */ 3300 status = nxge_map_rxdma_channel_buf_ring(nxgep, channel, 3301 dma_buf_p, rbr_p, num_chunks); 3302 if (status != NXGE_OK) { 3303 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3304 "==> nxge_map_rxdma_channel (channel %d): " 3305 "map buffer failed 0x%x", channel, status)); 3306 goto nxge_map_rxdma_channel_exit; 3307 } 3308 3309 /* 3310 * Receive block ring, completion ring and mailbox. 3311 */ 3312 status = nxge_map_rxdma_channel_cfg_ring(nxgep, channel, 3313 dma_cntl_p, rbr_p, rcr_p, rx_mbox_p); 3314 if (status != NXGE_OK) { 3315 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3316 "==> nxge_map_rxdma_channel (channel %d): " 3317 "map config failed 0x%x", channel, status)); 3318 goto nxge_map_rxdma_channel_fail2; 3319 } 3320 3321 goto nxge_map_rxdma_channel_exit; 3322 3323 nxge_map_rxdma_channel_fail3: 3324 /* Free rbr, rcr */ 3325 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3326 "==> nxge_map_rxdma_channel: free rbr/rcr " 3327 "(status 0x%x channel %d)", 3328 status, channel)); 3329 nxge_unmap_rxdma_channel_cfg_ring(nxgep, 3330 *rcr_p, *rx_mbox_p); 3331 3332 nxge_map_rxdma_channel_fail2: 3333 /* Free buffer blocks */ 3334 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3335 "==> nxge_map_rxdma_channel: free rx buffers" 3336 "(nxgep 0x%x status 0x%x channel %d)", 3337 nxgep, status, channel)); 3338 nxge_unmap_rxdma_channel_buf_ring(nxgep, *rbr_p); 3339 3340 status = NXGE_ERROR; 3341 3342 nxge_map_rxdma_channel_exit: 3343 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3344 "<== nxge_map_rxdma_channel: " 3345 "(nxgep 0x%x status 0x%x channel %d)", 3346 nxgep, status, channel)); 3347 3348 return (status); 3349 } 3350 3351 /*ARGSUSED*/ 3352 static void 3353 nxge_unmap_rxdma_channel(p_nxge_t nxgep, uint16_t channel, 3354 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 3355 { 3356 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3357 "==> nxge_unmap_rxdma_channel (channel %d)", channel)); 3358 3359 /* 3360 * unmap receive block ring, completion ring and mailbox. 3361 */ 3362 (void) nxge_unmap_rxdma_channel_cfg_ring(nxgep, 3363 rcr_p, rx_mbox_p); 3364 3365 /* unmap buffer blocks */ 3366 (void) nxge_unmap_rxdma_channel_buf_ring(nxgep, rbr_p); 3367 3368 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma_channel")); 3369 } 3370 3371 /*ARGSUSED*/ 3372 static nxge_status_t 3373 nxge_map_rxdma_channel_cfg_ring(p_nxge_t nxgep, uint16_t dma_channel, 3374 p_nxge_dma_common_t *dma_cntl_p, p_rx_rbr_ring_t *rbr_p, 3375 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 3376 { 3377 p_rx_rbr_ring_t rbrp; 3378 p_rx_rcr_ring_t rcrp; 3379 p_rx_mbox_t mboxp; 3380 p_nxge_dma_common_t cntl_dmap; 3381 p_nxge_dma_common_t dmap; 3382 p_rx_msg_t *rx_msg_ring; 3383 p_rx_msg_t rx_msg_p; 3384 p_rbr_cfig_a_t rcfga_p; 3385 p_rbr_cfig_b_t rcfgb_p; 3386 p_rcrcfig_a_t cfga_p; 3387 p_rcrcfig_b_t cfgb_p; 3388 p_rxdma_cfig1_t cfig1_p; 3389 p_rxdma_cfig2_t cfig2_p; 3390 p_rbr_kick_t kick_p; 3391 uint32_t dmaaddrp; 3392 uint32_t *rbr_vaddrp; 3393 uint32_t bkaddr; 3394 nxge_status_t status = NXGE_OK; 3395 int i; 3396 uint32_t nxge_port_rcr_size; 3397 3398 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3399 "==> nxge_map_rxdma_channel_cfg_ring")); 3400 3401 cntl_dmap = *dma_cntl_p; 3402 3403 /* Map in the receive block ring */ 3404 rbrp = *rbr_p; 3405 dmap = (p_nxge_dma_common_t)&rbrp->rbr_desc; 3406 nxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4); 3407 /* 3408 * Zero out buffer block ring descriptors. 3409 */ 3410 bzero((caddr_t)dmap->kaddrp, dmap->alength); 3411 3412 rcfga_p = &(rbrp->rbr_cfga); 3413 rcfgb_p = &(rbrp->rbr_cfgb); 3414 kick_p = &(rbrp->rbr_kick); 3415 rcfga_p->value = 0; 3416 rcfgb_p->value = 0; 3417 kick_p->value = 0; 3418 rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress; 3419 rcfga_p->value = (rbrp->rbr_addr & 3420 (RBR_CFIG_A_STDADDR_MASK | 3421 RBR_CFIG_A_STDADDR_BASE_MASK)); 3422 rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT); 3423 3424 rcfgb_p->bits.ldw.bufsz0 = rbrp->pkt_buf_size0; 3425 rcfgb_p->bits.ldw.vld0 = 1; 3426 rcfgb_p->bits.ldw.bufsz1 = rbrp->pkt_buf_size1; 3427 rcfgb_p->bits.ldw.vld1 = 1; 3428 rcfgb_p->bits.ldw.bufsz2 = rbrp->pkt_buf_size2; 3429 rcfgb_p->bits.ldw.vld2 = 1; 3430 rcfgb_p->bits.ldw.bksize = nxgep->rx_bksize_code; 3431 3432 /* 3433 * For each buffer block, enter receive block address to the ring. 3434 */ 3435 rbr_vaddrp = (uint32_t *)dmap->kaddrp; 3436 rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp; 3437 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3438 "==> nxge_map_rxdma_channel_cfg_ring: channel %d " 3439 "rbr_vaddrp $%p", dma_channel, rbr_vaddrp)); 3440 3441 rx_msg_ring = rbrp->rx_msg_ring; 3442 for (i = 0; i < rbrp->tnblocks; i++) { 3443 rx_msg_p = rx_msg_ring[i]; 3444 rx_msg_p->nxgep = nxgep; 3445 rx_msg_p->rx_rbr_p = rbrp; 3446 bkaddr = (uint32_t) 3447 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress 3448 >> RBR_BKADDR_SHIFT)); 3449 rx_msg_p->free = B_FALSE; 3450 rx_msg_p->max_usage_cnt = 0xbaddcafe; 3451 3452 *rbr_vaddrp++ = bkaddr; 3453 } 3454 3455 kick_p->bits.ldw.bkadd = rbrp->rbb_max; 3456 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 3457 3458 rbrp->rbr_rd_index = 0; 3459 3460 rbrp->rbr_consumed = 0; 3461 rbrp->rbr_use_bcopy = B_TRUE; 3462 rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0; 3463 /* 3464 * Do bcopy on packets greater than bcopy size once 3465 * the lo threshold is reached. 3466 * This lo threshold should be less than the hi threshold. 3467 * 3468 * Do bcopy on every packet once the hi threshold is reached. 3469 */ 3470 if (nxge_rx_threshold_lo >= nxge_rx_threshold_hi) { 3471 /* default it to use hi */ 3472 nxge_rx_threshold_lo = nxge_rx_threshold_hi; 3473 } 3474 3475 if (nxge_rx_buf_size_type > NXGE_RBR_TYPE2) { 3476 nxge_rx_buf_size_type = NXGE_RBR_TYPE2; 3477 } 3478 rbrp->rbr_bufsize_type = nxge_rx_buf_size_type; 3479 3480 switch (nxge_rx_threshold_hi) { 3481 default: 3482 case NXGE_RX_COPY_NONE: 3483 /* Do not do bcopy at all */ 3484 rbrp->rbr_use_bcopy = B_FALSE; 3485 rbrp->rbr_threshold_hi = rbrp->rbb_max; 3486 break; 3487 3488 case NXGE_RX_COPY_1: 3489 case NXGE_RX_COPY_2: 3490 case NXGE_RX_COPY_3: 3491 case NXGE_RX_COPY_4: 3492 case NXGE_RX_COPY_5: 3493 case NXGE_RX_COPY_6: 3494 case NXGE_RX_COPY_7: 3495 rbrp->rbr_threshold_hi = 3496 rbrp->rbb_max * 3497 (nxge_rx_threshold_hi)/NXGE_RX_BCOPY_SCALE; 3498 break; 3499 3500 case NXGE_RX_COPY_ALL: 3501 rbrp->rbr_threshold_hi = 0; 3502 break; 3503 } 3504 3505 switch (nxge_rx_threshold_lo) { 3506 default: 3507 case NXGE_RX_COPY_NONE: 3508 /* Do not do bcopy at all */ 3509 if (rbrp->rbr_use_bcopy) { 3510 rbrp->rbr_use_bcopy = B_FALSE; 3511 } 3512 rbrp->rbr_threshold_lo = rbrp->rbb_max; 3513 break; 3514 3515 case NXGE_RX_COPY_1: 3516 case NXGE_RX_COPY_2: 3517 case NXGE_RX_COPY_3: 3518 case NXGE_RX_COPY_4: 3519 case NXGE_RX_COPY_5: 3520 case NXGE_RX_COPY_6: 3521 case NXGE_RX_COPY_7: 3522 rbrp->rbr_threshold_lo = 3523 rbrp->rbb_max * 3524 (nxge_rx_threshold_lo)/NXGE_RX_BCOPY_SCALE; 3525 break; 3526 3527 case NXGE_RX_COPY_ALL: 3528 rbrp->rbr_threshold_lo = 0; 3529 break; 3530 } 3531 3532 NXGE_DEBUG_MSG((nxgep, RX_CTL, 3533 "nxge_map_rxdma_channel_cfg_ring: channel %d " 3534 "rbb_max %d " 3535 "rbrp->rbr_bufsize_type %d " 3536 "rbb_threshold_hi %d " 3537 "rbb_threshold_lo %d", 3538 dma_channel, 3539 rbrp->rbb_max, 3540 rbrp->rbr_bufsize_type, 3541 rbrp->rbr_threshold_hi, 3542 rbrp->rbr_threshold_lo)); 3543 3544 rbrp->page_valid.value = 0; 3545 rbrp->page_mask_1.value = rbrp->page_mask_2.value = 0; 3546 rbrp->page_value_1.value = rbrp->page_value_2.value = 0; 3547 rbrp->page_reloc_1.value = rbrp->page_reloc_2.value = 0; 3548 rbrp->page_hdl.value = 0; 3549 3550 rbrp->page_valid.bits.ldw.page0 = 1; 3551 rbrp->page_valid.bits.ldw.page1 = 1; 3552 3553 /* Map in the receive completion ring */ 3554 rcrp = (p_rx_rcr_ring_t) 3555 KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP); 3556 rcrp->rdc = dma_channel; 3557 3558 nxge_port_rcr_size = nxgep->nxge_port_rcr_size; 3559 rcrp->comp_size = nxge_port_rcr_size; 3560 rcrp->comp_wrap_mask = nxge_port_rcr_size - 1; 3561 3562 rcrp->max_receive_pkts = nxge_max_rx_pkts; 3563 3564 dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc; 3565 nxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size, 3566 sizeof (rcr_entry_t)); 3567 rcrp->comp_rd_index = 0; 3568 rcrp->comp_wt_index = 0; 3569 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 3570 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 3571 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3572 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3573 3574 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 3575 (nxge_port_rcr_size - 1); 3576 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 3577 (nxge_port_rcr_size - 1); 3578 3579 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3580 "==> nxge_map_rxdma_channel_cfg_ring: " 3581 "channel %d " 3582 "rbr_vaddrp $%p " 3583 "rcr_desc_rd_head_p $%p " 3584 "rcr_desc_rd_head_pp $%p " 3585 "rcr_desc_rd_last_p $%p " 3586 "rcr_desc_rd_last_pp $%p ", 3587 dma_channel, 3588 rbr_vaddrp, 3589 rcrp->rcr_desc_rd_head_p, 3590 rcrp->rcr_desc_rd_head_pp, 3591 rcrp->rcr_desc_last_p, 3592 rcrp->rcr_desc_last_pp)); 3593 3594 /* 3595 * Zero out buffer block ring descriptors. 3596 */ 3597 bzero((caddr_t)dmap->kaddrp, dmap->alength); 3598 3599 rcrp->intr_timeout = (nxgep->intr_timeout < 3600 NXGE_RDC_RCR_TIMEOUT_MIN) ? NXGE_RDC_RCR_TIMEOUT_MIN : 3601 nxgep->intr_timeout; 3602 3603 rcrp->intr_threshold = (nxgep->intr_threshold < 3604 NXGE_RDC_RCR_THRESHOLD_MIN) ? NXGE_RDC_RCR_THRESHOLD_MIN : 3605 nxgep->intr_threshold; 3606 3607 rcrp->full_hdr_flag = B_FALSE; 3608 3609 rcrp->sw_priv_hdr_len = nxge_rdc_buf_offset; 3610 3611 3612 cfga_p = &(rcrp->rcr_cfga); 3613 cfgb_p = &(rcrp->rcr_cfgb); 3614 cfga_p->value = 0; 3615 cfgb_p->value = 0; 3616 rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress; 3617 cfga_p->value = (rcrp->rcr_addr & 3618 (RCRCFIG_A_STADDR_MASK | 3619 RCRCFIG_A_STADDR_BASE_MASK)); 3620 3621 rcfga_p->value |= ((uint64_t)rcrp->comp_size << 3622 RCRCFIG_A_LEN_SHIF); 3623 3624 /* 3625 * Timeout should be set based on the system clock divider. 3626 * A timeout value of 1 assumes that the 3627 * granularity (1000) is 3 microseconds running at 300MHz. 3628 */ 3629 cfgb_p->bits.ldw.pthres = rcrp->intr_threshold; 3630 cfgb_p->bits.ldw.timeout = rcrp->intr_timeout; 3631 cfgb_p->bits.ldw.entout = 1; 3632 3633 /* Map in the mailbox */ 3634 mboxp = (p_rx_mbox_t) 3635 KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP); 3636 dmap = (p_nxge_dma_common_t)&mboxp->rx_mbox; 3637 nxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t)); 3638 cfig1_p = (p_rxdma_cfig1_t)&mboxp->rx_cfg1; 3639 cfig2_p = (p_rxdma_cfig2_t)&mboxp->rx_cfg2; 3640 cfig1_p->value = cfig2_p->value = 0; 3641 3642 mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress; 3643 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3644 "==> nxge_map_rxdma_channel_cfg_ring: " 3645 "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx", 3646 dma_channel, cfig1_p->value, cfig2_p->value, 3647 mboxp->mbox_addr)); 3648 3649 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress >> 32 3650 & 0xfff); 3651 cfig1_p->bits.ldw.mbaddr_h = dmaaddrp; 3652 3653 3654 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff); 3655 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 3656 RXDMA_CFIG2_MBADDR_L_MASK); 3657 3658 cfig2_p->bits.ldw.mbaddr = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT); 3659 3660 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3661 "==> nxge_map_rxdma_channel_cfg_ring: " 3662 "channel %d damaddrp $%p " 3663 "cfg1 0x%016llx cfig2 0x%016llx", 3664 dma_channel, dmaaddrp, 3665 cfig1_p->value, cfig2_p->value)); 3666 3667 cfig2_p->bits.ldw.full_hdr = rcrp->full_hdr_flag; 3668 if (nxgep->niu_hw_type == NIU_HW_TYPE_RF) { 3669 switch (rcrp->sw_priv_hdr_len) { 3670 case SW_OFFSET_NO_OFFSET: 3671 case SW_OFFSET_64: 3672 case SW_OFFSET_128: 3673 case SW_OFFSET_192: 3674 cfig2_p->bits.ldw.offset = 3675 rcrp->sw_priv_hdr_len; 3676 cfig2_p->bits.ldw.offset256 = 0; 3677 break; 3678 case SW_OFFSET_256: 3679 case SW_OFFSET_320: 3680 case SW_OFFSET_384: 3681 case SW_OFFSET_448: 3682 cfig2_p->bits.ldw.offset = 3683 rcrp->sw_priv_hdr_len & 0x3; 3684 cfig2_p->bits.ldw.offset256 = 1; 3685 break; 3686 default: 3687 cfig2_p->bits.ldw.offset = SW_OFFSET_NO_OFFSET; 3688 cfig2_p->bits.ldw.offset256 = 0; 3689 } 3690 } else { 3691 cfig2_p->bits.ldw.offset = rcrp->sw_priv_hdr_len; 3692 } 3693 3694 rbrp->rx_rcr_p = rcrp; 3695 rcrp->rx_rbr_p = rbrp; 3696 *rcr_p = rcrp; 3697 *rx_mbox_p = mboxp; 3698 3699 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3700 "<== nxge_map_rxdma_channel_cfg_ring status 0x%08x", status)); 3701 3702 return (status); 3703 } 3704 3705 /*ARGSUSED*/ 3706 static void 3707 nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t nxgep, 3708 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 3709 { 3710 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3711 "==> nxge_unmap_rxdma_channel_cfg_ring: channel %d", 3712 rcr_p->rdc)); 3713 3714 KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t)); 3715 KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t)); 3716 3717 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3718 "<== nxge_unmap_rxdma_channel_cfg_ring")); 3719 } 3720 3721 static nxge_status_t 3722 nxge_map_rxdma_channel_buf_ring(p_nxge_t nxgep, uint16_t channel, 3723 p_nxge_dma_common_t *dma_buf_p, 3724 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks) 3725 { 3726 p_rx_rbr_ring_t rbrp; 3727 p_nxge_dma_common_t dma_bufp, tmp_bufp; 3728 p_rx_msg_t *rx_msg_ring; 3729 p_rx_msg_t rx_msg_p; 3730 p_mblk_t mblk_p; 3731 3732 rxring_info_t *ring_info; 3733 nxge_status_t status = NXGE_OK; 3734 int i, j, index; 3735 uint32_t size, bsize, nblocks, nmsgs; 3736 3737 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3738 "==> nxge_map_rxdma_channel_buf_ring: channel %d", 3739 channel)); 3740 3741 dma_bufp = tmp_bufp = *dma_buf_p; 3742 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3743 " nxge_map_rxdma_channel_buf_ring: channel %d to map %d " 3744 "chunks bufp 0x%016llx", 3745 channel, num_chunks, dma_bufp)); 3746 3747 nmsgs = 0; 3748 for (i = 0; i < num_chunks; i++, tmp_bufp++) { 3749 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3750 "==> nxge_map_rxdma_channel_buf_ring: channel %d " 3751 "bufp 0x%016llx nblocks %d nmsgs %d", 3752 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs)); 3753 nmsgs += tmp_bufp->nblocks; 3754 } 3755 if (!nmsgs) { 3756 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3757 "<== nxge_map_rxdma_channel_buf_ring: channel %d " 3758 "no msg blocks", 3759 channel)); 3760 status = NXGE_ERROR; 3761 goto nxge_map_rxdma_channel_buf_ring_exit; 3762 } 3763 3764 rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (*rbrp), KM_SLEEP); 3765 3766 size = nmsgs * sizeof (p_rx_msg_t); 3767 rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP); 3768 ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t), 3769 KM_SLEEP); 3770 3771 MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER, 3772 (void *)nxgep->interrupt_cookie); 3773 MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER, 3774 (void *)nxgep->interrupt_cookie); 3775 rbrp->rdc = channel; 3776 rbrp->num_blocks = num_chunks; 3777 rbrp->tnblocks = nmsgs; 3778 rbrp->rbb_max = nmsgs; 3779 rbrp->rbr_max_size = nmsgs; 3780 rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1); 3781 3782 /* 3783 * Buffer sizes suggested by NIU architect. 3784 * 256, 512 and 2K. 3785 */ 3786 3787 rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B; 3788 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES; 3789 rbrp->npi_pkt_buf_size0 = SIZE_256B; 3790 3791 rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K; 3792 rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES; 3793 rbrp->npi_pkt_buf_size1 = SIZE_1KB; 3794 3795 rbrp->block_size = nxgep->rx_default_block_size; 3796 3797 if (!nxgep->mac.is_jumbo) { 3798 rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K; 3799 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES; 3800 rbrp->npi_pkt_buf_size2 = SIZE_2KB; 3801 } else { 3802 if (rbrp->block_size >= 0x2000) { 3803 rbrp->pkt_buf_size2 = RBR_BUFSZ2_8K; 3804 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_8K_BYTES; 3805 rbrp->npi_pkt_buf_size2 = SIZE_8KB; 3806 } else { 3807 rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K; 3808 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES; 3809 rbrp->npi_pkt_buf_size2 = SIZE_4KB; 3810 } 3811 } 3812 3813 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3814 "==> nxge_map_rxdma_channel_buf_ring: channel %d " 3815 "actual rbr max %d rbb_max %d nmsgs %d " 3816 "rbrp->block_size %d default_block_size %d " 3817 "(config nxge_rbr_size %d nxge_rbr_spare_size %d)", 3818 channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs, 3819 rbrp->block_size, nxgep->rx_default_block_size, 3820 nxge_rbr_size, nxge_rbr_spare_size)); 3821 3822 /* Map in buffers from the buffer pool. */ 3823 index = 0; 3824 for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) { 3825 bsize = dma_bufp->block_size; 3826 nblocks = dma_bufp->nblocks; 3827 ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp; 3828 ring_info->buffer[i].buf_index = i; 3829 ring_info->buffer[i].buf_size = dma_bufp->alength; 3830 ring_info->buffer[i].start_index = index; 3831 ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp; 3832 3833 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3834 " nxge_map_rxdma_channel_buf_ring: map channel %d " 3835 "chunk %d" 3836 " nblocks %d chunk_size %x block_size 0x%x " 3837 "dma_bufp $%p", channel, i, 3838 dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize, 3839 dma_bufp)); 3840 3841 for (j = 0; j < nblocks; j++) { 3842 if ((rx_msg_p = nxge_allocb(bsize, BPRI_LO, 3843 dma_bufp)) == NULL) { 3844 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 3845 "allocb failed (index %d i %d j %d)", 3846 index, i, j)); 3847 goto nxge_map_rxdma_channel_buf_ring_fail1; 3848 } 3849 rx_msg_ring[index] = rx_msg_p; 3850 rx_msg_p->block_index = index; 3851 rx_msg_p->shifted_addr = (uint32_t) 3852 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 3853 RBR_BKADDR_SHIFT)); 3854 3855 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3856 "index %d j %d rx_msg_p $%p mblk %p", 3857 index, j, rx_msg_p, rx_msg_p->rx_mblk_p)); 3858 3859 mblk_p = rx_msg_p->rx_mblk_p; 3860 mblk_p->b_wptr = mblk_p->b_rptr + bsize; 3861 3862 rbrp->rbr_ref_cnt++; 3863 index++; 3864 rx_msg_p->buf_dma.dma_channel = channel; 3865 } 3866 3867 rbrp->rbr_alloc_type = DDI_MEM_ALLOC; 3868 if (dma_bufp->contig_alloc_type) { 3869 rbrp->rbr_alloc_type = CONTIG_MEM_ALLOC; 3870 } 3871 3872 if (dma_bufp->kmem_alloc_type) { 3873 rbrp->rbr_alloc_type = KMEM_ALLOC; 3874 } 3875 3876 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3877 " nxge_map_rxdma_channel_buf_ring: map channel %d " 3878 "chunk %d" 3879 " nblocks %d chunk_size %x block_size 0x%x " 3880 "dma_bufp $%p", 3881 channel, i, 3882 dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize, 3883 dma_bufp)); 3884 } 3885 if (i < rbrp->num_blocks) { 3886 goto nxge_map_rxdma_channel_buf_ring_fail1; 3887 } 3888 3889 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3890 "nxge_map_rxdma_channel_buf_ring: done buf init " 3891 "channel %d msg block entries %d", 3892 channel, index)); 3893 ring_info->block_size_mask = bsize - 1; 3894 rbrp->rx_msg_ring = rx_msg_ring; 3895 rbrp->dma_bufp = dma_buf_p; 3896 rbrp->ring_info = ring_info; 3897 3898 status = nxge_rxbuf_index_info_init(nxgep, rbrp); 3899 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3900 " nxge_map_rxdma_channel_buf_ring: " 3901 "channel %d done buf info init", channel)); 3902 3903 /* 3904 * Finally, permit nxge_freeb() to call nxge_post_page(). 3905 */ 3906 rbrp->rbr_state = RBR_POSTING; 3907 3908 *rbr_p = rbrp; 3909 goto nxge_map_rxdma_channel_buf_ring_exit; 3910 3911 nxge_map_rxdma_channel_buf_ring_fail1: 3912 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3913 " nxge_map_rxdma_channel_buf_ring: failed channel (0x%x)", 3914 channel, status)); 3915 3916 index--; 3917 for (; index >= 0; index--) { 3918 rx_msg_p = rx_msg_ring[index]; 3919 if (rx_msg_p != NULL) { 3920 freeb(rx_msg_p->rx_mblk_p); 3921 rx_msg_ring[index] = NULL; 3922 } 3923 } 3924 nxge_map_rxdma_channel_buf_ring_fail: 3925 MUTEX_DESTROY(&rbrp->post_lock); 3926 MUTEX_DESTROY(&rbrp->lock); 3927 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 3928 KMEM_FREE(rx_msg_ring, size); 3929 KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t)); 3930 3931 status = NXGE_ERROR; 3932 3933 nxge_map_rxdma_channel_buf_ring_exit: 3934 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3935 "<== nxge_map_rxdma_channel_buf_ring status 0x%08x", status)); 3936 3937 return (status); 3938 } 3939 3940 /*ARGSUSED*/ 3941 static void 3942 nxge_unmap_rxdma_channel_buf_ring(p_nxge_t nxgep, 3943 p_rx_rbr_ring_t rbr_p) 3944 { 3945 p_rx_msg_t *rx_msg_ring; 3946 p_rx_msg_t rx_msg_p; 3947 rxring_info_t *ring_info; 3948 int i; 3949 uint32_t size; 3950 #ifdef NXGE_DEBUG 3951 int num_chunks; 3952 #endif 3953 3954 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3955 "==> nxge_unmap_rxdma_channel_buf_ring")); 3956 if (rbr_p == NULL) { 3957 NXGE_DEBUG_MSG((nxgep, RX_CTL, 3958 "<== nxge_unmap_rxdma_channel_buf_ring: NULL rbrp")); 3959 return; 3960 } 3961 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3962 "==> nxge_unmap_rxdma_channel_buf_ring: channel %d", 3963 rbr_p->rdc)); 3964 3965 rx_msg_ring = rbr_p->rx_msg_ring; 3966 ring_info = rbr_p->ring_info; 3967 3968 if (rx_msg_ring == NULL || ring_info == NULL) { 3969 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3970 "<== nxge_unmap_rxdma_channel_buf_ring: " 3971 "rx_msg_ring $%p ring_info $%p", 3972 rx_msg_p, ring_info)); 3973 return; 3974 } 3975 3976 #ifdef NXGE_DEBUG 3977 num_chunks = rbr_p->num_blocks; 3978 #endif 3979 size = rbr_p->tnblocks * sizeof (p_rx_msg_t); 3980 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3981 " nxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d " 3982 "tnblocks %d (max %d) size ptrs %d ", 3983 rbr_p->rdc, num_chunks, 3984 rbr_p->tnblocks, rbr_p->rbr_max_size, size)); 3985 3986 for (i = 0; i < rbr_p->tnblocks; i++) { 3987 rx_msg_p = rx_msg_ring[i]; 3988 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 3989 " nxge_unmap_rxdma_channel_buf_ring: " 3990 "rx_msg_p $%p", 3991 rx_msg_p)); 3992 if (rx_msg_p != NULL) { 3993 freeb(rx_msg_p->rx_mblk_p); 3994 rx_msg_ring[i] = NULL; 3995 } 3996 } 3997 3998 /* 3999 * We no longer may use the mutex <post_lock>. By setting 4000 * <rbr_state> to anything but POSTING, we prevent 4001 * nxge_post_page() from accessing a dead mutex. 4002 */ 4003 rbr_p->rbr_state = RBR_UNMAPPING; 4004 MUTEX_DESTROY(&rbr_p->post_lock); 4005 4006 MUTEX_DESTROY(&rbr_p->lock); 4007 4008 if (rbr_p->rbr_ref_cnt == 0) { 4009 /* 4010 * This is the normal state of affairs. 4011 * Need to free the following buffers: 4012 * - data buffers 4013 * - rx_msg ring 4014 * - ring_info 4015 * - rbr ring 4016 */ 4017 NXGE_DEBUG_MSG((nxgep, RX_CTL, 4018 "unmap_rxdma_buf_ring: No outstanding - freeing ")); 4019 nxge_rxdma_databuf_free(rbr_p); 4020 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 4021 KMEM_FREE(rx_msg_ring, size); 4022 KMEM_FREE(rbr_p, sizeof (*rbr_p)); 4023 } else { 4024 /* 4025 * Some of our buffers are still being used. 4026 * Therefore, tell nxge_freeb() this ring is 4027 * unmapped, so it may free <rbr_p> for us. 4028 */ 4029 rbr_p->rbr_state = RBR_UNMAPPED; 4030 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4031 "unmap_rxdma_buf_ring: %d %s outstanding.", 4032 rbr_p->rbr_ref_cnt, 4033 rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs")); 4034 } 4035 4036 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4037 "<== nxge_unmap_rxdma_channel_buf_ring")); 4038 } 4039 4040 /* 4041 * nxge_rxdma_hw_start_common 4042 * 4043 * Arguments: 4044 * nxgep 4045 * 4046 * Notes: 4047 * 4048 * NPI/NXGE function calls: 4049 * nxge_init_fzc_rx_common(); 4050 * nxge_init_fzc_rxdma_port(); 4051 * 4052 * Registers accessed: 4053 * 4054 * Context: 4055 * Service domain 4056 */ 4057 static nxge_status_t 4058 nxge_rxdma_hw_start_common(p_nxge_t nxgep) 4059 { 4060 nxge_status_t status = NXGE_OK; 4061 4062 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common")); 4063 4064 /* 4065 * Load the sharable parameters by writing to the 4066 * function zero control registers. These FZC registers 4067 * should be initialized only once for the entire chip. 4068 */ 4069 (void) nxge_init_fzc_rx_common(nxgep); 4070 4071 /* 4072 * Initialize the RXDMA port specific FZC control configurations. 4073 * These FZC registers are pertaining to each port. 4074 */ 4075 (void) nxge_init_fzc_rxdma_port(nxgep); 4076 4077 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common")); 4078 4079 return (status); 4080 } 4081 4082 static nxge_status_t 4083 nxge_rxdma_hw_start(p_nxge_t nxgep, int channel) 4084 { 4085 int i, ndmas; 4086 p_rx_rbr_rings_t rx_rbr_rings; 4087 p_rx_rbr_ring_t *rbr_rings; 4088 p_rx_rcr_rings_t rx_rcr_rings; 4089 p_rx_rcr_ring_t *rcr_rings; 4090 p_rx_mbox_areas_t rx_mbox_areas_p; 4091 p_rx_mbox_t *rx_mbox_p; 4092 nxge_status_t status = NXGE_OK; 4093 4094 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start")); 4095 4096 rx_rbr_rings = nxgep->rx_rbr_rings; 4097 rx_rcr_rings = nxgep->rx_rcr_rings; 4098 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 4099 NXGE_DEBUG_MSG((nxgep, RX_CTL, 4100 "<== nxge_rxdma_hw_start: NULL ring pointers")); 4101 return (NXGE_ERROR); 4102 } 4103 ndmas = rx_rbr_rings->ndmas; 4104 if (ndmas == 0) { 4105 NXGE_DEBUG_MSG((nxgep, RX_CTL, 4106 "<== nxge_rxdma_hw_start: no dma channel allocated")); 4107 return (NXGE_ERROR); 4108 } 4109 4110 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4111 "==> nxge_rxdma_hw_start (ndmas %d)", ndmas)); 4112 4113 rbr_rings = rx_rbr_rings->rbr_rings; 4114 rcr_rings = rx_rcr_rings->rcr_rings; 4115 rx_mbox_areas_p = nxgep->rx_mbox_areas_p; 4116 if (rx_mbox_areas_p) { 4117 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 4118 } else { 4119 rx_mbox_p = NULL; 4120 } 4121 4122 i = channel; 4123 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4124 "==> nxge_rxdma_hw_start (ndmas %d) channel %d", 4125 ndmas, channel)); 4126 status = nxge_rxdma_start_channel(nxgep, channel, 4127 (p_rx_rbr_ring_t)rbr_rings[i], 4128 (p_rx_rcr_ring_t)rcr_rings[i], 4129 (p_rx_mbox_t)rx_mbox_p[i]); 4130 if (status != NXGE_OK) { 4131 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4132 "==> nxge_rxdma_hw_start: disable " 4133 "(status 0x%x channel %d)", status, channel)); 4134 return (status); 4135 } 4136 4137 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start: " 4138 "rx_rbr_rings 0x%016llx rings 0x%016llx", 4139 rx_rbr_rings, rx_rcr_rings)); 4140 4141 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4142 "==> nxge_rxdma_hw_start: (status 0x%x)", status)); 4143 4144 return (status); 4145 } 4146 4147 static void 4148 nxge_rxdma_hw_stop(p_nxge_t nxgep, int channel) 4149 { 4150 p_rx_rbr_rings_t rx_rbr_rings; 4151 p_rx_rcr_rings_t rx_rcr_rings; 4152 4153 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop")); 4154 4155 rx_rbr_rings = nxgep->rx_rbr_rings; 4156 rx_rcr_rings = nxgep->rx_rcr_rings; 4157 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 4158 NXGE_DEBUG_MSG((nxgep, RX_CTL, 4159 "<== nxge_rxdma_hw_stop: NULL ring pointers")); 4160 return; 4161 } 4162 4163 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4164 "==> nxge_rxdma_hw_stop(channel %d)", 4165 channel)); 4166 (void) nxge_rxdma_stop_channel(nxgep, channel); 4167 4168 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop: " 4169 "rx_rbr_rings 0x%016llx rings 0x%016llx", 4170 rx_rbr_rings, rx_rcr_rings)); 4171 4172 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_hw_stop")); 4173 } 4174 4175 4176 static nxge_status_t 4177 nxge_rxdma_start_channel(p_nxge_t nxgep, uint16_t channel, 4178 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p) 4179 { 4180 npi_handle_t handle; 4181 npi_status_t rs = NPI_SUCCESS; 4182 rx_dma_ctl_stat_t cs; 4183 rx_dma_ent_msk_t ent_mask; 4184 nxge_status_t status = NXGE_OK; 4185 4186 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel")); 4187 4188 handle = NXGE_DEV_NPI_HANDLE(nxgep); 4189 4190 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "nxge_rxdma_start_channel: " 4191 "npi handle addr $%p acc $%p", 4192 nxgep->npi_handle.regp, nxgep->npi_handle.regh)); 4193 4194 /* Reset RXDMA channel, but not if you're a guest. */ 4195 if (!isLDOMguest(nxgep)) { 4196 rs = npi_rxdma_cfg_rdc_reset(handle, channel); 4197 if (rs != NPI_SUCCESS) { 4198 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4199 "==> nxge_init_fzc_rdc: " 4200 "npi_rxdma_cfg_rdc_reset(%d) returned 0x%08x", 4201 channel, rs)); 4202 return (NXGE_ERROR | rs); 4203 } 4204 4205 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4206 "==> nxge_rxdma_start_channel: reset done: channel %d", 4207 channel)); 4208 } 4209 4210 #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 4211 if (isLDOMguest(nxgep)) 4212 (void) nxge_rdc_lp_conf(nxgep, channel); 4213 #endif 4214 4215 /* 4216 * Initialize the RXDMA channel specific FZC control 4217 * configurations. These FZC registers are pertaining 4218 * to each RX channel (logical pages). 4219 */ 4220 if (!isLDOMguest(nxgep)) { 4221 status = nxge_init_fzc_rxdma_channel(nxgep, channel); 4222 if (status != NXGE_OK) { 4223 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4224 "==> nxge_rxdma_start_channel: " 4225 "init fzc rxdma failed (0x%08x channel %d)", 4226 status, channel)); 4227 return (status); 4228 } 4229 4230 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4231 "==> nxge_rxdma_start_channel: fzc done")); 4232 } 4233 4234 /* Set up the interrupt event masks. */ 4235 ent_mask.value = 0; 4236 ent_mask.value |= RX_DMA_ENT_MSK_RBREMPTY_MASK; 4237 rs = npi_rxdma_event_mask(handle, OP_SET, channel, 4238 &ent_mask); 4239 if (rs != NPI_SUCCESS) { 4240 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4241 "==> nxge_rxdma_start_channel: " 4242 "init rxdma event masks failed " 4243 "(0x%08x channel %d)", 4244 status, channel)); 4245 return (NXGE_ERROR | rs); 4246 } 4247 4248 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4249 "==> nxge_rxdma_start_channel: " 4250 "event done: channel %d (mask 0x%016llx)", 4251 channel, ent_mask.value)); 4252 4253 /* Initialize the receive DMA control and status register */ 4254 cs.value = 0; 4255 cs.bits.hdw.mex = 1; 4256 cs.bits.hdw.rcrthres = 1; 4257 cs.bits.hdw.rcrto = 1; 4258 cs.bits.hdw.rbr_empty = 1; 4259 status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel, &cs); 4260 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: " 4261 "channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value)); 4262 if (status != NXGE_OK) { 4263 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4264 "==> nxge_rxdma_start_channel: " 4265 "init rxdma control register failed (0x%08x channel %d", 4266 status, channel)); 4267 return (status); 4268 } 4269 4270 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: " 4271 "control done - channel %d cs 0x%016llx", channel, cs.value)); 4272 4273 /* 4274 * Load RXDMA descriptors, buffers, mailbox, 4275 * initialise the receive DMA channels and 4276 * enable each DMA channel. 4277 */ 4278 status = nxge_enable_rxdma_channel(nxgep, 4279 channel, rbr_p, rcr_p, mbox_p); 4280 4281 if (status != NXGE_OK) { 4282 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4283 " nxge_rxdma_start_channel: " 4284 " enable rxdma failed (0x%08x channel %d)", 4285 status, channel)); 4286 return (status); 4287 } 4288 4289 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4290 "==> nxge_rxdma_start_channel: enabled channel %d")); 4291 4292 if (isLDOMguest(nxgep)) { 4293 /* Add interrupt handler for this channel. */ 4294 status = nxge_hio_intr_add(nxgep, VP_BOUND_RX, channel); 4295 if (status != NXGE_OK) { 4296 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4297 " nxge_rxdma_start_channel: " 4298 " nxge_hio_intr_add failed (0x%08x channel %d)", 4299 status, channel)); 4300 return (status); 4301 } 4302 } 4303 4304 ent_mask.value = 0; 4305 ent_mask.value |= (RX_DMA_ENT_MSK_WRED_DROP_MASK | 4306 RX_DMA_ENT_MSK_PTDROP_PKT_MASK); 4307 rs = npi_rxdma_event_mask(handle, OP_SET, channel, 4308 &ent_mask); 4309 if (rs != NPI_SUCCESS) { 4310 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 4311 "==> nxge_rxdma_start_channel: " 4312 "init rxdma event masks failed (0x%08x channel %d)", 4313 status, channel)); 4314 return (NXGE_ERROR | rs); 4315 } 4316 4317 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: " 4318 "control done - channel %d cs 0x%016llx", channel, cs.value)); 4319 4320 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_start_channel")); 4321 4322 return (NXGE_OK); 4323 } 4324 4325 static nxge_status_t 4326 nxge_rxdma_stop_channel(p_nxge_t nxgep, uint16_t channel) 4327 { 4328 npi_handle_t handle; 4329 npi_status_t rs = NPI_SUCCESS; 4330 rx_dma_ctl_stat_t cs; 4331 rx_dma_ent_msk_t ent_mask; 4332 nxge_status_t status = NXGE_OK; 4333 4334 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel")); 4335 4336 handle = NXGE_DEV_NPI_HANDLE(nxgep); 4337 4338 NXGE_DEBUG_MSG((nxgep, RX_CTL, "nxge_rxdma_stop_channel: " 4339 "npi handle addr $%p acc $%p", 4340 nxgep->npi_handle.regp, nxgep->npi_handle.regh)); 4341 4342 if (!isLDOMguest(nxgep)) { 4343 /* 4344 * Stop RxMAC = A.9.2.6 4345 */ 4346 if (nxge_rx_mac_disable(nxgep) != NXGE_OK) { 4347 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4348 "nxge_rxdma_stop_channel: " 4349 "Failed to disable RxMAC")); 4350 } 4351 4352 /* 4353 * Drain IPP Port = A.9.3.6 4354 */ 4355 (void) nxge_ipp_drain(nxgep); 4356 } 4357 4358 /* Reset RXDMA channel */ 4359 rs = npi_rxdma_cfg_rdc_reset(handle, channel); 4360 if (rs != NPI_SUCCESS) { 4361 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4362 " nxge_rxdma_stop_channel: " 4363 " reset rxdma failed (0x%08x channel %d)", 4364 rs, channel)); 4365 return (NXGE_ERROR | rs); 4366 } 4367 4368 NXGE_DEBUG_MSG((nxgep, RX_CTL, 4369 "==> nxge_rxdma_stop_channel: reset done")); 4370 4371 /* Set up the interrupt event masks. */ 4372 ent_mask.value = RX_DMA_ENT_MSK_ALL; 4373 rs = npi_rxdma_event_mask(handle, OP_SET, channel, 4374 &ent_mask); 4375 if (rs != NPI_SUCCESS) { 4376 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4377 "==> nxge_rxdma_stop_channel: " 4378 "set rxdma event masks failed (0x%08x channel %d)", 4379 rs, channel)); 4380 return (NXGE_ERROR | rs); 4381 } 4382 4383 NXGE_DEBUG_MSG((nxgep, RX_CTL, 4384 "==> nxge_rxdma_stop_channel: event done")); 4385 4386 /* 4387 * Initialize the receive DMA control and status register 4388 */ 4389 cs.value = 0; 4390 status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel, &cs); 4391 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel: control " 4392 " to default (all 0s) 0x%08x", cs.value)); 4393 if (status != NXGE_OK) { 4394 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4395 " nxge_rxdma_stop_channel: init rxdma" 4396 " control register failed (0x%08x channel %d", 4397 status, channel)); 4398 return (status); 4399 } 4400 4401 NXGE_DEBUG_MSG((nxgep, RX_CTL, 4402 "==> nxge_rxdma_stop_channel: control done")); 4403 4404 /* 4405 * Make sure channel is disabled. 4406 */ 4407 status = nxge_disable_rxdma_channel(nxgep, channel); 4408 4409 if (status != NXGE_OK) { 4410 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4411 " nxge_rxdma_stop_channel: " 4412 " init enable rxdma failed (0x%08x channel %d)", 4413 status, channel)); 4414 return (status); 4415 } 4416 4417 if (!isLDOMguest(nxgep)) { 4418 /* 4419 * Enable RxMAC = A.9.2.10 4420 */ 4421 if (nxge_rx_mac_enable(nxgep) != NXGE_OK) { 4422 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4423 "nxge_rxdma_stop_channel: Rx MAC still disabled")); 4424 } 4425 } 4426 4427 NXGE_DEBUG_MSG((nxgep, 4428 RX_CTL, "==> nxge_rxdma_stop_channel: disable done")); 4429 4430 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop_channel")); 4431 4432 return (NXGE_OK); 4433 } 4434 4435 nxge_status_t 4436 nxge_rxdma_handle_sys_errors(p_nxge_t nxgep) 4437 { 4438 npi_handle_t handle; 4439 p_nxge_rdc_sys_stats_t statsp; 4440 rx_ctl_dat_fifo_stat_t stat; 4441 uint32_t zcp_err_status; 4442 uint32_t ipp_err_status; 4443 nxge_status_t status = NXGE_OK; 4444 npi_status_t rs = NPI_SUCCESS; 4445 boolean_t my_err = B_FALSE; 4446 4447 handle = nxgep->npi_handle; 4448 statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats; 4449 4450 rs = npi_rxdma_rxctl_fifo_error_intr_get(handle, &stat); 4451 4452 if (rs != NPI_SUCCESS) 4453 return (NXGE_ERROR | rs); 4454 4455 if (stat.bits.ldw.id_mismatch) { 4456 statsp->id_mismatch++; 4457 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, 0, 4458 NXGE_FM_EREPORT_RDMC_ID_MISMATCH); 4459 /* Global fatal error encountered */ 4460 } 4461 4462 if ((stat.bits.ldw.zcp_eop_err) || (stat.bits.ldw.ipp_eop_err)) { 4463 switch (nxgep->mac.portnum) { 4464 case 0: 4465 if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT0) || 4466 (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT0)) { 4467 my_err = B_TRUE; 4468 zcp_err_status = stat.bits.ldw.zcp_eop_err; 4469 ipp_err_status = stat.bits.ldw.ipp_eop_err; 4470 } 4471 break; 4472 case 1: 4473 if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT1) || 4474 (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT1)) { 4475 my_err = B_TRUE; 4476 zcp_err_status = stat.bits.ldw.zcp_eop_err; 4477 ipp_err_status = stat.bits.ldw.ipp_eop_err; 4478 } 4479 break; 4480 case 2: 4481 if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT2) || 4482 (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT2)) { 4483 my_err = B_TRUE; 4484 zcp_err_status = stat.bits.ldw.zcp_eop_err; 4485 ipp_err_status = stat.bits.ldw.ipp_eop_err; 4486 } 4487 break; 4488 case 3: 4489 if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT3) || 4490 (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT3)) { 4491 my_err = B_TRUE; 4492 zcp_err_status = stat.bits.ldw.zcp_eop_err; 4493 ipp_err_status = stat.bits.ldw.ipp_eop_err; 4494 } 4495 break; 4496 default: 4497 return (NXGE_ERROR); 4498 } 4499 } 4500 4501 if (my_err) { 4502 status = nxge_rxdma_handle_port_errors(nxgep, ipp_err_status, 4503 zcp_err_status); 4504 if (status != NXGE_OK) 4505 return (status); 4506 } 4507 4508 return (NXGE_OK); 4509 } 4510 4511 static nxge_status_t 4512 nxge_rxdma_handle_port_errors(p_nxge_t nxgep, uint32_t ipp_status, 4513 uint32_t zcp_status) 4514 { 4515 boolean_t rxport_fatal = B_FALSE; 4516 p_nxge_rdc_sys_stats_t statsp; 4517 nxge_status_t status = NXGE_OK; 4518 uint8_t portn; 4519 4520 portn = nxgep->mac.portnum; 4521 statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats; 4522 4523 if (ipp_status & (0x1 << portn)) { 4524 statsp->ipp_eop_err++; 4525 NXGE_FM_REPORT_ERROR(nxgep, portn, 0, 4526 NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR); 4527 rxport_fatal = B_TRUE; 4528 } 4529 4530 if (zcp_status & (0x1 << portn)) { 4531 statsp->zcp_eop_err++; 4532 NXGE_FM_REPORT_ERROR(nxgep, portn, 0, 4533 NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR); 4534 rxport_fatal = B_TRUE; 4535 } 4536 4537 if (rxport_fatal) { 4538 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4539 " nxge_rxdma_handle_port_error: " 4540 " fatal error on Port #%d\n", 4541 portn)); 4542 status = nxge_rx_port_fatal_err_recover(nxgep); 4543 if (status == NXGE_OK) { 4544 FM_SERVICE_RESTORED(nxgep); 4545 } 4546 } 4547 4548 return (status); 4549 } 4550 4551 static nxge_status_t 4552 nxge_rxdma_fatal_err_recover(p_nxge_t nxgep, uint16_t channel) 4553 { 4554 npi_handle_t handle; 4555 npi_status_t rs = NPI_SUCCESS; 4556 nxge_status_t status = NXGE_OK; 4557 p_rx_rbr_ring_t rbrp; 4558 p_rx_rcr_ring_t rcrp; 4559 p_rx_mbox_t mboxp; 4560 rx_dma_ent_msk_t ent_mask; 4561 p_nxge_dma_common_t dmap; 4562 uint32_t ref_cnt; 4563 p_rx_msg_t rx_msg_p; 4564 int i; 4565 uint32_t nxge_port_rcr_size; 4566 4567 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fatal_err_recover")); 4568 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4569 "Recovering from RxDMAChannel#%d error...", channel)); 4570 4571 /* 4572 * Stop the dma channel waits for the stop done. 4573 * If the stop done bit is not set, then create 4574 * an error. 4575 */ 4576 4577 handle = NXGE_DEV_NPI_HANDLE(nxgep); 4578 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Rx DMA stop...")); 4579 4580 rbrp = (p_rx_rbr_ring_t)nxgep->rx_rbr_rings->rbr_rings[channel]; 4581 rcrp = (p_rx_rcr_ring_t)nxgep->rx_rcr_rings->rcr_rings[channel]; 4582 4583 MUTEX_ENTER(&rbrp->lock); 4584 MUTEX_ENTER(&rbrp->post_lock); 4585 4586 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA channel...")); 4587 4588 rs = npi_rxdma_cfg_rdc_disable(handle, channel); 4589 if (rs != NPI_SUCCESS) { 4590 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4591 "nxge_disable_rxdma_channel:failed")); 4592 goto fail; 4593 } 4594 4595 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA interrupt...")); 4596 4597 /* Disable interrupt */ 4598 ent_mask.value = RX_DMA_ENT_MSK_ALL; 4599 rs = npi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 4600 if (rs != NPI_SUCCESS) { 4601 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4602 "nxge_rxdma_stop_channel: " 4603 "set rxdma event masks failed (channel %d)", 4604 channel)); 4605 } 4606 4607 NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel reset...")); 4608 4609 /* Reset RXDMA channel */ 4610 rs = npi_rxdma_cfg_rdc_reset(handle, channel); 4611 if (rs != NPI_SUCCESS) { 4612 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4613 "nxge_rxdma_fatal_err_recover: " 4614 " reset rxdma failed (channel %d)", channel)); 4615 goto fail; 4616 } 4617 4618 nxge_port_rcr_size = nxgep->nxge_port_rcr_size; 4619 4620 mboxp = (p_rx_mbox_t)nxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 4621 4622 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 4623 rbrp->rbr_rd_index = 0; 4624 4625 rcrp->comp_rd_index = 0; 4626 rcrp->comp_wt_index = 0; 4627 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 4628 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 4629 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 4630 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 4631 4632 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 4633 (nxge_port_rcr_size - 1); 4634 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 4635 (nxge_port_rcr_size - 1); 4636 4637 dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc; 4638 bzero((caddr_t)dmap->kaddrp, dmap->alength); 4639 4640 cmn_err(CE_NOTE, "!rbr entries = %d\n", rbrp->rbr_max_size); 4641 4642 for (i = 0; i < rbrp->rbr_max_size; i++) { 4643 rx_msg_p = rbrp->rx_msg_ring[i]; 4644 ref_cnt = rx_msg_p->ref_cnt; 4645 if (ref_cnt != 1) { 4646 if (rx_msg_p->cur_usage_cnt != 4647 rx_msg_p->max_usage_cnt) { 4648 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4649 "buf[%d]: cur_usage_cnt = %d " 4650 "max_usage_cnt = %d\n", i, 4651 rx_msg_p->cur_usage_cnt, 4652 rx_msg_p->max_usage_cnt)); 4653 } else { 4654 /* Buffer can be re-posted */ 4655 rx_msg_p->free = B_TRUE; 4656 rx_msg_p->cur_usage_cnt = 0; 4657 rx_msg_p->max_usage_cnt = 0xbaddcafe; 4658 rx_msg_p->pkt_buf_size = 0; 4659 } 4660 } 4661 } 4662 4663 NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel re-start...")); 4664 4665 status = nxge_rxdma_start_channel(nxgep, channel, rbrp, rcrp, mboxp); 4666 if (status != NXGE_OK) { 4667 goto fail; 4668 } 4669 4670 MUTEX_EXIT(&rbrp->post_lock); 4671 MUTEX_EXIT(&rbrp->lock); 4672 4673 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4674 "Recovery Successful, RxDMAChannel#%d Restored", 4675 channel)); 4676 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fatal_err_recover")); 4677 return (NXGE_OK); 4678 4679 fail: 4680 MUTEX_EXIT(&rbrp->post_lock); 4681 MUTEX_EXIT(&rbrp->lock); 4682 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed")); 4683 return (NXGE_ERROR | rs); 4684 } 4685 4686 nxge_status_t 4687 nxge_rx_port_fatal_err_recover(p_nxge_t nxgep) 4688 { 4689 nxge_grp_set_t *set = &nxgep->rx_set; 4690 nxge_status_t status = NXGE_OK; 4691 p_rx_rcr_ring_t rcrp; 4692 int rdc; 4693 4694 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_port_fatal_err_recover")); 4695 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4696 "Recovering from RxPort error...")); 4697 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disabling RxMAC...\n")); 4698 4699 if (nxge_rx_mac_disable(nxgep) != NXGE_OK) 4700 goto fail; 4701 4702 NXGE_DELAY(1000); 4703 4704 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Stopping all RxDMA channels...")); 4705 4706 for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 4707 if ((1 << rdc) & set->owned.map) { 4708 rcrp = nxgep->rx_rcr_rings->rcr_rings[rdc]; 4709 if (rcrp != NULL) { 4710 MUTEX_ENTER(&rcrp->lock); 4711 if (nxge_rxdma_fatal_err_recover(nxgep, 4712 rdc) != NXGE_OK) { 4713 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4714 "Could not recover " 4715 "channel %d", rdc)); 4716 } 4717 MUTEX_EXIT(&rcrp->lock); 4718 } 4719 } 4720 } 4721 4722 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Resetting IPP...")); 4723 4724 /* Reset IPP */ 4725 if (nxge_ipp_reset(nxgep) != NXGE_OK) { 4726 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4727 "nxge_rx_port_fatal_err_recover: " 4728 "Failed to reset IPP")); 4729 goto fail; 4730 } 4731 4732 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Reset RxMAC...")); 4733 4734 /* Reset RxMAC */ 4735 if (nxge_rx_mac_reset(nxgep) != NXGE_OK) { 4736 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4737 "nxge_rx_port_fatal_err_recover: " 4738 "Failed to reset RxMAC")); 4739 goto fail; 4740 } 4741 4742 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize IPP...")); 4743 4744 /* Re-Initialize IPP */ 4745 if (nxge_ipp_init(nxgep) != NXGE_OK) { 4746 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4747 "nxge_rx_port_fatal_err_recover: " 4748 "Failed to init IPP")); 4749 goto fail; 4750 } 4751 4752 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize RxMAC...")); 4753 4754 /* Re-Initialize RxMAC */ 4755 if ((status = nxge_rx_mac_init(nxgep)) != NXGE_OK) { 4756 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4757 "nxge_rx_port_fatal_err_recover: " 4758 "Failed to reset RxMAC")); 4759 goto fail; 4760 } 4761 4762 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-enable RxMAC...")); 4763 4764 /* Re-enable RxMAC */ 4765 if ((status = nxge_rx_mac_enable(nxgep)) != NXGE_OK) { 4766 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4767 "nxge_rx_port_fatal_err_recover: " 4768 "Failed to enable RxMAC")); 4769 goto fail; 4770 } 4771 4772 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4773 "Recovery Successful, RxPort Restored")); 4774 4775 return (NXGE_OK); 4776 fail: 4777 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed")); 4778 return (status); 4779 } 4780 4781 void 4782 nxge_rxdma_inject_err(p_nxge_t nxgep, uint32_t err_id, uint8_t chan) 4783 { 4784 rx_dma_ctl_stat_t cs; 4785 rx_ctl_dat_fifo_stat_t cdfs; 4786 4787 switch (err_id) { 4788 case NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR: 4789 case NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR: 4790 case NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR: 4791 case NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR: 4792 case NXGE_FM_EREPORT_RDMC_RBR_TMOUT: 4793 case NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR: 4794 case NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS: 4795 case NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR: 4796 case NXGE_FM_EREPORT_RDMC_RCRINCON: 4797 case NXGE_FM_EREPORT_RDMC_RCRFULL: 4798 case NXGE_FM_EREPORT_RDMC_RBRFULL: 4799 case NXGE_FM_EREPORT_RDMC_RBRLOGPAGE: 4800 case NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE: 4801 case NXGE_FM_EREPORT_RDMC_CONFIG_ERR: 4802 RXDMA_REG_READ64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG, 4803 chan, &cs.value); 4804 if (err_id == NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR) 4805 cs.bits.hdw.rcr_ack_err = 1; 4806 else if (err_id == NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR) 4807 cs.bits.hdw.dc_fifo_err = 1; 4808 else if (err_id == NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR) 4809 cs.bits.hdw.rcr_sha_par = 1; 4810 else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR) 4811 cs.bits.hdw.rbr_pre_par = 1; 4812 else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_TMOUT) 4813 cs.bits.hdw.rbr_tmout = 1; 4814 else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR) 4815 cs.bits.hdw.rsp_cnt_err = 1; 4816 else if (err_id == NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS) 4817 cs.bits.hdw.byte_en_bus = 1; 4818 else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR) 4819 cs.bits.hdw.rsp_dat_err = 1; 4820 else if (err_id == NXGE_FM_EREPORT_RDMC_CONFIG_ERR) 4821 cs.bits.hdw.config_err = 1; 4822 else if (err_id == NXGE_FM_EREPORT_RDMC_RCRINCON) 4823 cs.bits.hdw.rcrincon = 1; 4824 else if (err_id == NXGE_FM_EREPORT_RDMC_RCRFULL) 4825 cs.bits.hdw.rcrfull = 1; 4826 else if (err_id == NXGE_FM_EREPORT_RDMC_RBRFULL) 4827 cs.bits.hdw.rbrfull = 1; 4828 else if (err_id == NXGE_FM_EREPORT_RDMC_RBRLOGPAGE) 4829 cs.bits.hdw.rbrlogpage = 1; 4830 else if (err_id == NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE) 4831 cs.bits.hdw.cfiglogpage = 1; 4832 cmn_err(CE_NOTE, "!Write 0x%lx to RX_DMA_CTL_STAT_DBG_REG\n", 4833 cs.value); 4834 RXDMA_REG_WRITE64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG, 4835 chan, cs.value); 4836 break; 4837 case NXGE_FM_EREPORT_RDMC_ID_MISMATCH: 4838 case NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR: 4839 case NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR: 4840 cdfs.value = 0; 4841 if (err_id == NXGE_FM_EREPORT_RDMC_ID_MISMATCH) 4842 cdfs.bits.ldw.id_mismatch = (1 << nxgep->mac.portnum); 4843 else if (err_id == NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR) 4844 cdfs.bits.ldw.zcp_eop_err = (1 << nxgep->mac.portnum); 4845 else if (err_id == NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR) 4846 cdfs.bits.ldw.ipp_eop_err = (1 << nxgep->mac.portnum); 4847 cmn_err(CE_NOTE, 4848 "!Write 0x%lx to RX_CTL_DAT_FIFO_STAT_DBG_REG\n", 4849 cdfs.value); 4850 NXGE_REG_WR64(nxgep->npi_handle, 4851 RX_CTL_DAT_FIFO_STAT_DBG_REG, cdfs.value); 4852 break; 4853 case NXGE_FM_EREPORT_RDMC_DCF_ERR: 4854 break; 4855 case NXGE_FM_EREPORT_RDMC_RCR_ERR: 4856 break; 4857 } 4858 } 4859 4860 static void 4861 nxge_rxdma_databuf_free(p_rx_rbr_ring_t rbr_p) 4862 { 4863 rxring_info_t *ring_info; 4864 int index; 4865 uint32_t chunk_size; 4866 uint64_t kaddr; 4867 uint_t num_blocks; 4868 4869 NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_rxdma_databuf_free")); 4870 4871 if (rbr_p == NULL) { 4872 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 4873 "==> nxge_rxdma_databuf_free: NULL rbr pointer")); 4874 return; 4875 } 4876 4877 if (rbr_p->rbr_alloc_type == DDI_MEM_ALLOC) { 4878 NXGE_DEBUG_MSG((NULL, DMA_CTL, 4879 "<== nxge_rxdma_databuf_free: DDI")); 4880 return; 4881 } 4882 4883 ring_info = rbr_p->ring_info; 4884 if (ring_info == NULL) { 4885 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 4886 "==> nxge_rxdma_databuf_free: NULL ring info")); 4887 return; 4888 } 4889 num_blocks = rbr_p->num_blocks; 4890 for (index = 0; index < num_blocks; index++) { 4891 kaddr = ring_info->buffer[index].kaddr; 4892 chunk_size = ring_info->buffer[index].buf_size; 4893 NXGE_DEBUG_MSG((NULL, DMA_CTL, 4894 "==> nxge_rxdma_databuf_free: free chunk %d " 4895 "kaddrp $%p chunk size %d", 4896 index, kaddr, chunk_size)); 4897 if (kaddr == 0) 4898 continue; 4899 nxge_free_buf(rbr_p->rbr_alloc_type, kaddr, chunk_size); 4900 ring_info->buffer[index].kaddr = 0; 4901 } 4902 4903 NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_rxdma_databuf_free")); 4904 } 4905 4906 #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 4907 extern void contig_mem_free(void *, size_t); 4908 #endif 4909 4910 void 4911 nxge_free_buf(buf_alloc_type_t alloc_type, uint64_t kaddr, uint32_t buf_size) 4912 { 4913 NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_free_buf")); 4914 4915 if (kaddr == 0 || !buf_size) { 4916 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 4917 "==> nxge_free_buf: invalid kaddr $%p size to free %d", 4918 kaddr, buf_size)); 4919 return; 4920 } 4921 4922 switch (alloc_type) { 4923 case KMEM_ALLOC: 4924 NXGE_DEBUG_MSG((NULL, DMA_CTL, 4925 "==> nxge_free_buf: freeing kmem $%p size %d", 4926 kaddr, buf_size)); 4927 KMEM_FREE((void *)kaddr, buf_size); 4928 break; 4929 4930 #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 4931 case CONTIG_MEM_ALLOC: 4932 NXGE_DEBUG_MSG((NULL, DMA_CTL, 4933 "==> nxge_free_buf: freeing contig_mem kaddr $%p size %d", 4934 kaddr, buf_size)); 4935 contig_mem_free((void *)kaddr, buf_size); 4936 break; 4937 #endif 4938 4939 default: 4940 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 4941 "<== nxge_free_buf: unsupported alloc type %d", 4942 alloc_type)); 4943 return; 4944 } 4945 4946 NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_free_buf")); 4947 } 4948