1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <hxge_impl.h> 27 #include <hxge_rxdma.h> 28 29 /* 30 * Number of blocks to accumulate before re-enabling DMA 31 * when we get RBR empty. 32 */ 33 #define HXGE_RBR_EMPTY_THRESHOLD 64 34 35 /* 36 * Globals: tunable parameters (/etc/system or adb) 37 * 38 */ 39 extern uint32_t hxge_rbr_size; 40 extern uint32_t hxge_rcr_size; 41 extern uint32_t hxge_rbr_spare_size; 42 extern uint32_t hxge_mblks_pending; 43 44 /* 45 * Tunable to reduce the amount of time spent in the 46 * ISR doing Rx Processing. 47 */ 48 extern uint32_t hxge_max_rx_pkts; 49 50 /* 51 * Tunables to manage the receive buffer blocks. 52 * 53 * hxge_rx_threshold_hi: copy all buffers. 54 * hxge_rx_bcopy_size_type: receive buffer block size type. 55 * hxge_rx_threshold_lo: copy only up to tunable block size type. 56 */ 57 extern hxge_rxbuf_threshold_t hxge_rx_threshold_hi; 58 extern hxge_rxbuf_type_t hxge_rx_buf_size_type; 59 extern hxge_rxbuf_threshold_t hxge_rx_threshold_lo; 60 61 /* 62 * Static local functions. 63 */ 64 static hxge_status_t hxge_map_rxdma(p_hxge_t hxgep); 65 static void hxge_unmap_rxdma(p_hxge_t hxgep); 66 static hxge_status_t hxge_rxdma_hw_start_common(p_hxge_t hxgep); 67 static hxge_status_t hxge_rxdma_hw_start(p_hxge_t hxgep); 68 static void hxge_rxdma_hw_stop(p_hxge_t hxgep); 69 static hxge_status_t hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 70 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 71 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p, 72 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 73 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 74 static void hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 75 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 76 static hxge_status_t hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, 77 uint16_t dma_channel, p_hxge_dma_common_t *dma_rbr_cntl_p, 78 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 79 p_rx_rbr_ring_t *rbr_p, p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 80 static void hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 81 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 82 static hxge_status_t hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, 83 uint16_t channel, p_hxge_dma_common_t *dma_buf_p, 84 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks); 85 static void hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 86 p_rx_rbr_ring_t rbr_p); 87 static hxge_status_t hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 88 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 89 int n_init_kick); 90 static hxge_status_t hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel); 91 static mblk_t *hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 92 p_rx_rcr_ring_t *rcr_p, rdc_stat_t cs, 93 uint16_t *nptrs, uint16_t *npkts); 94 static void hxge_receive_packet(p_hxge_t hxgep, p_rx_rcr_ring_t rcr_p, 95 p_rcr_entry_t rcr_desc_rd_head_p, boolean_t *multi_p, 96 mblk_t ** mp, mblk_t ** mp_cont, uint32_t *invalid_rcr_entry); 97 static hxge_status_t hxge_disable_rxdma_channel(p_hxge_t hxgep, 98 uint16_t channel); 99 static p_rx_msg_t hxge_allocb(size_t, uint32_t, p_hxge_dma_common_t); 100 static void hxge_freeb(p_rx_msg_t); 101 static void hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, 102 p_hxge_ldv_t ldvp, rdc_stat_t cs, 103 uint16_t *nptrs, uint16_t *npkts); 104 static hxge_status_t hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, 105 p_hxge_ldv_t ldvp, rdc_stat_t cs); 106 static hxge_status_t hxge_rxbuf_index_info_init(p_hxge_t hxgep, 107 p_rx_rbr_ring_t rx_dmap); 108 static hxge_status_t hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, 109 uint16_t channel); 110 static hxge_status_t hxge_rx_port_fatal_err_recover(p_hxge_t hxgep); 111 static void hxge_rbr_empty_restore(p_hxge_t hxgep, 112 p_rx_rbr_ring_t rx_rbr_p); 113 114 hxge_status_t 115 hxge_init_rxdma_channels(p_hxge_t hxgep) 116 { 117 hxge_status_t status = HXGE_OK; 118 block_reset_t reset_reg; 119 int i; 120 121 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_init_rxdma_channels")); 122 123 for (i = 0; i < HXGE_MAX_RDCS; i++) 124 hxgep->rdc_first_intr[i] = B_TRUE; 125 126 /* Reset RDC block from PEU to clear any previous state */ 127 reset_reg.value = 0; 128 reset_reg.bits.rdc_rst = 1; 129 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 130 HXGE_DELAY(1000); 131 132 status = hxge_map_rxdma(hxgep); 133 if (status != HXGE_OK) { 134 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 135 "<== hxge_init_rxdma: status 0x%x", status)); 136 return (status); 137 } 138 139 status = hxge_rxdma_hw_start_common(hxgep); 140 if (status != HXGE_OK) { 141 hxge_unmap_rxdma(hxgep); 142 } 143 144 status = hxge_rxdma_hw_start(hxgep); 145 if (status != HXGE_OK) { 146 hxge_unmap_rxdma(hxgep); 147 } 148 149 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 150 "<== hxge_init_rxdma_channels: status 0x%x", status)); 151 return (status); 152 } 153 154 void 155 hxge_uninit_rxdma_channels(p_hxge_t hxgep) 156 { 157 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_uninit_rxdma_channels")); 158 159 hxge_rxdma_hw_stop(hxgep); 160 hxge_unmap_rxdma(hxgep); 161 162 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_uinit_rxdma_channels")); 163 } 164 165 hxge_status_t 166 hxge_init_rxdma_channel_cntl_stat(p_hxge_t hxgep, uint16_t channel, 167 rdc_stat_t *cs_p) 168 { 169 hpi_handle_t handle; 170 hpi_status_t rs = HPI_SUCCESS; 171 hxge_status_t status = HXGE_OK; 172 173 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 174 "<== hxge_init_rxdma_channel_cntl_stat")); 175 176 handle = HXGE_DEV_HPI_HANDLE(hxgep); 177 rs = hpi_rxdma_control_status(handle, OP_SET, channel, cs_p); 178 179 if (rs != HPI_SUCCESS) { 180 status = HXGE_ERROR | rs; 181 } 182 return (status); 183 } 184 185 186 hxge_status_t 187 hxge_enable_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 188 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 189 int n_init_kick) 190 { 191 hpi_handle_t handle; 192 rdc_desc_cfg_t rdc_desc; 193 rdc_rcr_cfg_b_t *cfgb_p; 194 hpi_status_t rs = HPI_SUCCESS; 195 196 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel")); 197 handle = HXGE_DEV_HPI_HANDLE(hxgep); 198 199 /* 200 * Use configuration data composed at init time. Write to hardware the 201 * receive ring configurations. 202 */ 203 rdc_desc.mbox_enable = 1; 204 rdc_desc.mbox_addr = mbox_p->mbox_addr; 205 HXGE_DEBUG_MSG((hxgep, RX_CTL, 206 "==> hxge_enable_rxdma_channel: mboxp $%p($%p)", 207 mbox_p->mbox_addr, rdc_desc.mbox_addr)); 208 209 rdc_desc.rbr_len = rbr_p->rbb_max; 210 rdc_desc.rbr_addr = rbr_p->rbr_addr; 211 212 switch (hxgep->rx_bksize_code) { 213 case RBR_BKSIZE_4K: 214 rdc_desc.page_size = SIZE_4KB; 215 break; 216 case RBR_BKSIZE_8K: 217 rdc_desc.page_size = SIZE_8KB; 218 break; 219 } 220 221 rdc_desc.size0 = rbr_p->hpi_pkt_buf_size0; 222 rdc_desc.valid0 = 1; 223 224 rdc_desc.size1 = rbr_p->hpi_pkt_buf_size1; 225 rdc_desc.valid1 = 1; 226 227 rdc_desc.size2 = rbr_p->hpi_pkt_buf_size2; 228 rdc_desc.valid2 = 1; 229 230 rdc_desc.full_hdr = rcr_p->full_hdr_flag; 231 rdc_desc.offset = rcr_p->sw_priv_hdr_len; 232 233 rdc_desc.rcr_len = rcr_p->comp_size; 234 rdc_desc.rcr_addr = rcr_p->rcr_addr; 235 236 cfgb_p = &(rcr_p->rcr_cfgb); 237 rdc_desc.rcr_threshold = cfgb_p->bits.pthres; 238 rdc_desc.rcr_timeout = cfgb_p->bits.timeout; 239 rdc_desc.rcr_timeout_enable = cfgb_p->bits.entout; 240 241 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 242 "rbr_len qlen %d pagesize code %d rcr_len %d", 243 rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len)); 244 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 245 "size 0 %d size 1 %d size 2 %d", 246 rbr_p->hpi_pkt_buf_size0, rbr_p->hpi_pkt_buf_size1, 247 rbr_p->hpi_pkt_buf_size2)); 248 249 rs = hpi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc); 250 if (rs != HPI_SUCCESS) { 251 return (HXGE_ERROR | rs); 252 } 253 254 /* 255 * Enable the timeout and threshold. 256 */ 257 rs = hpi_rxdma_cfg_rdc_rcr_threshold(handle, channel, 258 rdc_desc.rcr_threshold); 259 if (rs != HPI_SUCCESS) { 260 return (HXGE_ERROR | rs); 261 } 262 263 rs = hpi_rxdma_cfg_rdc_rcr_timeout(handle, channel, 264 rdc_desc.rcr_timeout); 265 if (rs != HPI_SUCCESS) { 266 return (HXGE_ERROR | rs); 267 } 268 269 /* Enable the DMA */ 270 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 271 if (rs != HPI_SUCCESS) { 272 return (HXGE_ERROR | rs); 273 } 274 275 /* Kick the DMA engine */ 276 hpi_rxdma_rdc_rbr_kick(handle, channel, n_init_kick); 277 278 /* Clear the rbr empty bit */ 279 (void) hpi_rxdma_channel_rbr_empty_clear(handle, channel); 280 281 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_enable_rxdma_channel")); 282 283 return (HXGE_OK); 284 } 285 286 static hxge_status_t 287 hxge_disable_rxdma_channel(p_hxge_t hxgep, uint16_t channel) 288 { 289 hpi_handle_t handle; 290 hpi_status_t rs = HPI_SUCCESS; 291 292 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_disable_rxdma_channel")); 293 294 handle = HXGE_DEV_HPI_HANDLE(hxgep); 295 296 /* disable the DMA */ 297 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 298 if (rs != HPI_SUCCESS) { 299 HXGE_DEBUG_MSG((hxgep, RX_CTL, 300 "<== hxge_disable_rxdma_channel:failed (0x%x)", rs)); 301 return (HXGE_ERROR | rs); 302 } 303 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_disable_rxdma_channel")); 304 return (HXGE_OK); 305 } 306 307 hxge_status_t 308 hxge_rxdma_channel_rcrflush(p_hxge_t hxgep, uint8_t channel) 309 { 310 hpi_handle_t handle; 311 hxge_status_t status = HXGE_OK; 312 313 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 314 "==> hxge_rxdma_channel_rcrflush")); 315 316 handle = HXGE_DEV_HPI_HANDLE(hxgep); 317 hpi_rxdma_rdc_rcr_flush(handle, channel); 318 319 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 320 "<== hxge_rxdma_channel_rcrflush")); 321 return (status); 322 323 } 324 325 #define MID_INDEX(l, r) ((r + l + 1) >> 1) 326 327 #define TO_LEFT -1 328 #define TO_RIGHT 1 329 #define BOTH_RIGHT (TO_RIGHT + TO_RIGHT) 330 #define BOTH_LEFT (TO_LEFT + TO_LEFT) 331 #define IN_MIDDLE (TO_RIGHT + TO_LEFT) 332 #define NO_HINT 0xffffffff 333 334 /*ARGSUSED*/ 335 hxge_status_t 336 hxge_rxbuf_pp_to_vp(p_hxge_t hxgep, p_rx_rbr_ring_t rbr_p, 337 uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp, 338 uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index) 339 { 340 int bufsize; 341 uint64_t pktbuf_pp; 342 uint64_t dvma_addr; 343 rxring_info_t *ring_info; 344 int base_side, end_side; 345 int r_index, l_index, anchor_index; 346 int found, search_done; 347 uint32_t offset, chunk_size, block_size, page_size_mask; 348 uint32_t chunk_index, block_index, total_index; 349 int max_iterations, iteration; 350 rxbuf_index_info_t *bufinfo; 351 352 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_rxbuf_pp_to_vp")); 353 354 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 355 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d", 356 pkt_buf_addr_pp, pktbufsz_type)); 357 358 #if defined(__i386) 359 pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp; 360 #else 361 pktbuf_pp = (uint64_t)pkt_buf_addr_pp; 362 #endif 363 364 switch (pktbufsz_type) { 365 case 0: 366 bufsize = rbr_p->pkt_buf_size0; 367 break; 368 case 1: 369 bufsize = rbr_p->pkt_buf_size1; 370 break; 371 case 2: 372 bufsize = rbr_p->pkt_buf_size2; 373 break; 374 case RCR_SINGLE_BLOCK: 375 bufsize = 0; 376 anchor_index = 0; 377 break; 378 default: 379 return (HXGE_ERROR); 380 } 381 382 if (rbr_p->num_blocks == 1) { 383 anchor_index = 0; 384 ring_info = rbr_p->ring_info; 385 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 386 387 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 388 "==> hxge_rxbuf_pp_to_vp: (found, 1 block) " 389 "buf_pp $%p btype %d anchor_index %d bufinfo $%p", 390 pkt_buf_addr_pp, pktbufsz_type, anchor_index, bufinfo)); 391 392 goto found_index; 393 } 394 395 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 396 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d anchor_index %d", 397 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 398 399 ring_info = rbr_p->ring_info; 400 found = B_FALSE; 401 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 402 iteration = 0; 403 max_iterations = ring_info->max_iterations; 404 405 /* 406 * First check if this block have been seen recently. This is indicated 407 * by a hint which is initialized when the first buffer of the block is 408 * seen. The hint is reset when the last buffer of the block has been 409 * processed. As three block sizes are supported, three hints are kept. 410 * The idea behind the hints is that once the hardware uses a block 411 * for a buffer of that size, it will use it exclusively for that size 412 * and will use it until it is exhausted. It is assumed that there 413 * would a single block being used for the same buffer sizes at any 414 * given time. 415 */ 416 if (ring_info->hint[pktbufsz_type] != NO_HINT) { 417 anchor_index = ring_info->hint[pktbufsz_type]; 418 dvma_addr = bufinfo[anchor_index].dvma_addr; 419 chunk_size = bufinfo[anchor_index].buf_size; 420 if ((pktbuf_pp >= dvma_addr) && 421 (pktbuf_pp < (dvma_addr + chunk_size))) { 422 found = B_TRUE; 423 /* 424 * check if this is the last buffer in the block If so, 425 * then reset the hint for the size; 426 */ 427 428 if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size)) 429 ring_info->hint[pktbufsz_type] = NO_HINT; 430 } 431 } 432 433 if (found == B_FALSE) { 434 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 435 "==> hxge_rxbuf_pp_to_vp: (!found)" 436 "buf_pp $%p btype %d anchor_index %d", 437 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 438 439 /* 440 * This is the first buffer of the block of this size. Need to 441 * search the whole information array. the search algorithm 442 * uses a binary tree search algorithm. It assumes that the 443 * information is already sorted with increasing order info[0] 444 * < info[1] < info[2] .... < info[n-1] where n is the size of 445 * the information array 446 */ 447 r_index = rbr_p->num_blocks - 1; 448 l_index = 0; 449 search_done = B_FALSE; 450 anchor_index = MID_INDEX(r_index, l_index); 451 while (search_done == B_FALSE) { 452 if ((r_index == l_index) || 453 (iteration >= max_iterations)) 454 search_done = B_TRUE; 455 456 end_side = TO_RIGHT; /* to the right */ 457 base_side = TO_LEFT; /* to the left */ 458 /* read the DVMA address information and sort it */ 459 dvma_addr = bufinfo[anchor_index].dvma_addr; 460 chunk_size = bufinfo[anchor_index].buf_size; 461 462 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 463 "==> hxge_rxbuf_pp_to_vp: (searching)" 464 "buf_pp $%p btype %d " 465 "anchor_index %d chunk_size %d dvmaaddr $%p", 466 pkt_buf_addr_pp, pktbufsz_type, anchor_index, 467 chunk_size, dvma_addr)); 468 469 if (pktbuf_pp >= dvma_addr) 470 base_side = TO_RIGHT; /* to the right */ 471 if (pktbuf_pp < (dvma_addr + chunk_size)) 472 end_side = TO_LEFT; /* to the left */ 473 474 switch (base_side + end_side) { 475 case IN_MIDDLE: 476 /* found */ 477 found = B_TRUE; 478 search_done = B_TRUE; 479 if ((pktbuf_pp + bufsize) < 480 (dvma_addr + chunk_size)) 481 ring_info->hint[pktbufsz_type] = 482 bufinfo[anchor_index].buf_index; 483 break; 484 case BOTH_RIGHT: 485 /* not found: go to the right */ 486 l_index = anchor_index + 1; 487 anchor_index = MID_INDEX(r_index, l_index); 488 break; 489 490 case BOTH_LEFT: 491 /* not found: go to the left */ 492 r_index = anchor_index - 1; 493 anchor_index = MID_INDEX(r_index, l_index); 494 break; 495 default: /* should not come here */ 496 return (HXGE_ERROR); 497 } 498 iteration++; 499 } 500 501 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 502 "==> hxge_rxbuf_pp_to_vp: (search done)" 503 "buf_pp $%p btype %d anchor_index %d", 504 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 505 } 506 507 if (found == B_FALSE) { 508 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 509 "==> hxge_rxbuf_pp_to_vp: (search failed)" 510 "buf_pp $%p btype %d anchor_index %d", 511 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 512 return (HXGE_ERROR); 513 } 514 515 found_index: 516 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 517 "==> hxge_rxbuf_pp_to_vp: (FOUND1)" 518 "buf_pp $%p btype %d bufsize %d anchor_index %d", 519 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index)); 520 521 /* index of the first block in this chunk */ 522 chunk_index = bufinfo[anchor_index].start_index; 523 dvma_addr = bufinfo[anchor_index].dvma_addr; 524 page_size_mask = ring_info->block_size_mask; 525 526 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 527 "==> hxge_rxbuf_pp_to_vp: (FOUND3), get chunk)" 528 "buf_pp $%p btype %d bufsize %d " 529 "anchor_index %d chunk_index %d dvma $%p", 530 pkt_buf_addr_pp, pktbufsz_type, bufsize, 531 anchor_index, chunk_index, dvma_addr)); 532 533 offset = pktbuf_pp - dvma_addr; /* offset within the chunk */ 534 block_size = rbr_p->block_size; /* System block(page) size */ 535 536 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 537 "==> hxge_rxbuf_pp_to_vp: (FOUND4), get chunk)" 538 "buf_pp $%p btype %d bufsize %d " 539 "anchor_index %d chunk_index %d dvma $%p " 540 "offset %d block_size %d", 541 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index, 542 chunk_index, dvma_addr, offset, block_size)); 543 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> getting total index")); 544 545 block_index = (offset / block_size); /* index within chunk */ 546 total_index = chunk_index + block_index; 547 548 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 549 "==> hxge_rxbuf_pp_to_vp: " 550 "total_index %d dvma_addr $%p " 551 "offset %d block_size %d " 552 "block_index %d ", 553 total_index, dvma_addr, offset, block_size, block_index)); 554 555 #if defined(__i386) 556 *pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr + 557 (uint32_t)offset); 558 #else 559 *pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr + 560 offset); 561 #endif 562 563 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 564 "==> hxge_rxbuf_pp_to_vp: " 565 "total_index %d dvma_addr $%p " 566 "offset %d block_size %d " 567 "block_index %d " 568 "*pkt_buf_addr_p $%p", 569 total_index, dvma_addr, offset, block_size, 570 block_index, *pkt_buf_addr_p)); 571 572 *msg_index = total_index; 573 *bufoffset = (offset & page_size_mask); 574 575 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 576 "==> hxge_rxbuf_pp_to_vp: get msg index: " 577 "msg_index %d bufoffset_index %d", 578 *msg_index, *bufoffset)); 579 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "<== hxge_rxbuf_pp_to_vp")); 580 581 return (HXGE_OK); 582 } 583 584 585 /* 586 * used by quick sort (qsort) function 587 * to perform comparison 588 */ 589 static int 590 hxge_sort_compare(const void *p1, const void *p2) 591 { 592 593 rxbuf_index_info_t *a, *b; 594 595 a = (rxbuf_index_info_t *)p1; 596 b = (rxbuf_index_info_t *)p2; 597 598 if (a->dvma_addr > b->dvma_addr) 599 return (1); 600 if (a->dvma_addr < b->dvma_addr) 601 return (-1); 602 return (0); 603 } 604 605 /* 606 * Grabbed this sort implementation from common/syscall/avl.c 607 * 608 * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified. 609 * v = Ptr to array/vector of objs 610 * n = # objs in the array 611 * s = size of each obj (must be multiples of a word size) 612 * f = ptr to function to compare two objs 613 * returns (-1 = less than, 0 = equal, 1 = greater than 614 */ 615 void 616 hxge_ksort(caddr_t v, int n, int s, int (*f) ()) 617 { 618 int g, i, j, ii; 619 unsigned int *p1, *p2; 620 unsigned int tmp; 621 622 /* No work to do */ 623 if (v == NULL || n <= 1) 624 return; 625 /* Sanity check on arguments */ 626 ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0); 627 ASSERT(s > 0); 628 629 for (g = n / 2; g > 0; g /= 2) { 630 for (i = g; i < n; i++) { 631 for (j = i - g; j >= 0 && 632 (*f) (v + j * s, v + (j + g) * s) == 1; j -= g) { 633 p1 = (unsigned *)(v + j * s); 634 p2 = (unsigned *)(v + (j + g) * s); 635 for (ii = 0; ii < s / 4; ii++) { 636 tmp = *p1; 637 *p1++ = *p2; 638 *p2++ = tmp; 639 } 640 } 641 } 642 } 643 } 644 645 /* 646 * Initialize data structures required for rxdma 647 * buffer dvma->vmem address lookup 648 */ 649 /*ARGSUSED*/ 650 static hxge_status_t 651 hxge_rxbuf_index_info_init(p_hxge_t hxgep, p_rx_rbr_ring_t rbrp) 652 { 653 int index; 654 rxring_info_t *ring_info; 655 int max_iteration = 0, max_index = 0; 656 657 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_rxbuf_index_info_init")); 658 659 ring_info = rbrp->ring_info; 660 ring_info->hint[0] = NO_HINT; 661 ring_info->hint[1] = NO_HINT; 662 ring_info->hint[2] = NO_HINT; 663 max_index = rbrp->num_blocks; 664 665 /* read the DVMA address information and sort it */ 666 /* do init of the information array */ 667 668 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 669 " hxge_rxbuf_index_info_init Sort ptrs")); 670 671 /* sort the array */ 672 hxge_ksort((void *) ring_info->buffer, max_index, 673 sizeof (rxbuf_index_info_t), hxge_sort_compare); 674 675 for (index = 0; index < max_index; index++) { 676 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 677 " hxge_rxbuf_index_info_init: sorted chunk %d " 678 " ioaddr $%p kaddr $%p size %x", 679 index, ring_info->buffer[index].dvma_addr, 680 ring_info->buffer[index].kaddr, 681 ring_info->buffer[index].buf_size)); 682 } 683 684 max_iteration = 0; 685 while (max_index >= (1ULL << max_iteration)) 686 max_iteration++; 687 ring_info->max_iterations = max_iteration + 1; 688 689 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 690 " hxge_rxbuf_index_info_init Find max iter %d", 691 ring_info->max_iterations)); 692 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_rxbuf_index_info_init")); 693 694 return (HXGE_OK); 695 } 696 697 /*ARGSUSED*/ 698 void 699 hxge_dump_rcr_entry(p_hxge_t hxgep, p_rcr_entry_t entry_p) 700 { 701 #ifdef HXGE_DEBUG 702 703 uint32_t bptr; 704 uint64_t pp; 705 706 bptr = entry_p->bits.pkt_buf_addr; 707 708 HXGE_DEBUG_MSG((hxgep, RX_CTL, 709 "\trcr entry $%p " 710 "\trcr entry 0x%0llx " 711 "\trcr entry 0x%08x " 712 "\trcr entry 0x%08x " 713 "\tvalue 0x%0llx\n" 714 "\tmulti = %d\n" 715 "\tpkt_type = 0x%x\n" 716 "\terror = 0x%04x\n" 717 "\tl2_len = %d\n" 718 "\tpktbufsize = %d\n" 719 "\tpkt_buf_addr = $%p\n" 720 "\tpkt_buf_addr (<< 6) = $%p\n", 721 entry_p, 722 *(int64_t *)entry_p, 723 *(int32_t *)entry_p, 724 *(int32_t *)((char *)entry_p + 32), 725 entry_p->value, 726 entry_p->bits.multi, 727 entry_p->bits.pkt_type, 728 entry_p->bits.error, 729 entry_p->bits.l2_len, 730 entry_p->bits.pktbufsz, 731 bptr, 732 entry_p->bits.pkt_buf_addr_l)); 733 734 pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) << 735 RCR_PKT_BUF_ADDR_SHIFT; 736 737 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rcr pp 0x%llx l2 len %d", 738 pp, (*(int64_t *)entry_p >> 40) & 0x3fff)); 739 #endif 740 } 741 742 /*ARGSUSED*/ 743 void 744 hxge_rxdma_stop(p_hxge_t hxgep) 745 { 746 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop")); 747 748 (void) hxge_rx_vmac_disable(hxgep); 749 (void) hxge_rxdma_hw_mode(hxgep, HXGE_DMA_STOP); 750 751 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop")); 752 } 753 754 void 755 hxge_rxdma_stop_reinit(p_hxge_t hxgep) 756 { 757 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_reinit")); 758 759 (void) hxge_rxdma_stop(hxgep); 760 (void) hxge_uninit_rxdma_channels(hxgep); 761 (void) hxge_init_rxdma_channels(hxgep); 762 763 (void) hxge_rx_vmac_enable(hxgep); 764 765 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_reinit")); 766 } 767 768 hxge_status_t 769 hxge_rxdma_hw_mode(p_hxge_t hxgep, boolean_t enable) 770 { 771 int i, ndmas; 772 uint16_t channel; 773 p_rx_rbr_rings_t rx_rbr_rings; 774 p_rx_rbr_ring_t *rbr_rings; 775 hpi_handle_t handle; 776 hpi_status_t rs = HPI_SUCCESS; 777 hxge_status_t status = HXGE_OK; 778 779 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 780 "==> hxge_rxdma_hw_mode: mode %d", enable)); 781 782 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 783 HXGE_DEBUG_MSG((hxgep, RX_CTL, 784 "<== hxge_rxdma_mode: not initialized")); 785 return (HXGE_ERROR); 786 } 787 788 rx_rbr_rings = hxgep->rx_rbr_rings; 789 if (rx_rbr_rings == NULL) { 790 HXGE_DEBUG_MSG((hxgep, RX_CTL, 791 "<== hxge_rxdma_mode: NULL ring pointer")); 792 return (HXGE_ERROR); 793 } 794 795 if (rx_rbr_rings->rbr_rings == NULL) { 796 HXGE_DEBUG_MSG((hxgep, RX_CTL, 797 "<== hxge_rxdma_mode: NULL rbr rings pointer")); 798 return (HXGE_ERROR); 799 } 800 801 ndmas = rx_rbr_rings->ndmas; 802 if (!ndmas) { 803 HXGE_DEBUG_MSG((hxgep, RX_CTL, 804 "<== hxge_rxdma_mode: no channel")); 805 return (HXGE_ERROR); 806 } 807 808 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 809 "==> hxge_rxdma_mode (ndmas %d)", ndmas)); 810 811 rbr_rings = rx_rbr_rings->rbr_rings; 812 813 handle = HXGE_DEV_HPI_HANDLE(hxgep); 814 815 for (i = 0; i < ndmas; i++) { 816 if (rbr_rings == NULL || rbr_rings[i] == NULL) { 817 continue; 818 } 819 channel = rbr_rings[i]->rdc; 820 if (enable) { 821 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 822 "==> hxge_rxdma_hw_mode: channel %d (enable)", 823 channel)); 824 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 825 } else { 826 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 827 "==> hxge_rxdma_hw_mode: channel %d (disable)", 828 channel)); 829 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 830 } 831 } 832 833 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs); 834 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 835 "<== hxge_rxdma_hw_mode: status 0x%x", status)); 836 837 return (status); 838 } 839 840 int 841 hxge_rxdma_get_ring_index(p_hxge_t hxgep, uint16_t channel) 842 { 843 int i, ndmas; 844 uint16_t rdc; 845 p_rx_rbr_rings_t rx_rbr_rings; 846 p_rx_rbr_ring_t *rbr_rings; 847 848 HXGE_DEBUG_MSG((hxgep, RX_CTL, 849 "==> hxge_rxdma_get_ring_index: channel %d", channel)); 850 851 rx_rbr_rings = hxgep->rx_rbr_rings; 852 if (rx_rbr_rings == NULL) { 853 HXGE_DEBUG_MSG((hxgep, RX_CTL, 854 "<== hxge_rxdma_get_ring_index: NULL ring pointer")); 855 return (-1); 856 } 857 858 ndmas = rx_rbr_rings->ndmas; 859 if (!ndmas) { 860 HXGE_DEBUG_MSG((hxgep, RX_CTL, 861 "<== hxge_rxdma_get_ring_index: no channel")); 862 return (-1); 863 } 864 865 HXGE_DEBUG_MSG((hxgep, RX_CTL, 866 "==> hxge_rxdma_get_ring_index (ndmas %d)", ndmas)); 867 868 rbr_rings = rx_rbr_rings->rbr_rings; 869 for (i = 0; i < ndmas; i++) { 870 rdc = rbr_rings[i]->rdc; 871 if (channel == rdc) { 872 HXGE_DEBUG_MSG((hxgep, RX_CTL, 873 "==> hxge_rxdma_get_rbr_ring: " 874 "channel %d (index %d) " 875 "ring %d", channel, i, rbr_rings[i])); 876 877 return (i); 878 } 879 } 880 881 HXGE_DEBUG_MSG((hxgep, RX_CTL, 882 "<== hxge_rxdma_get_rbr_ring_index: not found")); 883 884 return (-1); 885 } 886 887 /* 888 * Static functions start here. 889 */ 890 static p_rx_msg_t 891 hxge_allocb(size_t size, uint32_t pri, p_hxge_dma_common_t dmabuf_p) 892 { 893 p_rx_msg_t hxge_mp = NULL; 894 p_hxge_dma_common_t dmamsg_p; 895 uchar_t *buffer; 896 897 hxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP); 898 if (hxge_mp == NULL) { 899 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 900 "Allocation of a rx msg failed.")); 901 goto hxge_allocb_exit; 902 } 903 904 hxge_mp->use_buf_pool = B_FALSE; 905 if (dmabuf_p) { 906 hxge_mp->use_buf_pool = B_TRUE; 907 908 dmamsg_p = (p_hxge_dma_common_t)&hxge_mp->buf_dma; 909 *dmamsg_p = *dmabuf_p; 910 dmamsg_p->nblocks = 1; 911 dmamsg_p->block_size = size; 912 dmamsg_p->alength = size; 913 buffer = (uchar_t *)dmabuf_p->kaddrp; 914 915 dmabuf_p->kaddrp = (void *)((char *)dmabuf_p->kaddrp + size); 916 dmabuf_p->ioaddr_pp = (void *) 917 ((char *)dmabuf_p->ioaddr_pp + size); 918 919 dmabuf_p->alength -= size; 920 dmabuf_p->offset += size; 921 dmabuf_p->dma_cookie.dmac_laddress += size; 922 dmabuf_p->dma_cookie.dmac_size -= size; 923 } else { 924 buffer = KMEM_ALLOC(size, KM_NOSLEEP); 925 if (buffer == NULL) { 926 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 927 "Allocation of a receive page failed.")); 928 goto hxge_allocb_fail1; 929 } 930 } 931 932 hxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &hxge_mp->freeb); 933 if (hxge_mp->rx_mblk_p == NULL) { 934 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, "desballoc failed.")); 935 goto hxge_allocb_fail2; 936 } 937 hxge_mp->buffer = buffer; 938 hxge_mp->block_size = size; 939 hxge_mp->freeb.free_func = (void (*) ()) hxge_freeb; 940 hxge_mp->freeb.free_arg = (caddr_t)hxge_mp; 941 hxge_mp->ref_cnt = 1; 942 hxge_mp->free = B_TRUE; 943 hxge_mp->rx_use_bcopy = B_FALSE; 944 945 atomic_inc_32(&hxge_mblks_pending); 946 947 goto hxge_allocb_exit; 948 949 hxge_allocb_fail2: 950 if (!hxge_mp->use_buf_pool) { 951 KMEM_FREE(buffer, size); 952 } 953 hxge_allocb_fail1: 954 KMEM_FREE(hxge_mp, sizeof (rx_msg_t)); 955 hxge_mp = NULL; 956 957 hxge_allocb_exit: 958 return (hxge_mp); 959 } 960 961 p_mblk_t 962 hxge_dupb(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 963 { 964 p_mblk_t mp; 965 966 HXGE_DEBUG_MSG((NULL, MEM_CTL, "==> hxge_dupb")); 967 HXGE_DEBUG_MSG((NULL, MEM_CTL, "hxge_mp = $%p " 968 "offset = 0x%08X " "size = 0x%08X", hxge_mp, offset, size)); 969 970 mp = desballoc(&hxge_mp->buffer[offset], size, 0, &hxge_mp->freeb); 971 if (mp == NULL) { 972 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 973 goto hxge_dupb_exit; 974 } 975 976 atomic_inc_32(&hxge_mp->ref_cnt); 977 978 hxge_dupb_exit: 979 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 980 return (mp); 981 } 982 983 p_mblk_t 984 hxge_dupb_bcopy(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 985 { 986 p_mblk_t mp; 987 uchar_t *dp; 988 989 mp = allocb(size + HXGE_RXBUF_EXTRA, 0); 990 if (mp == NULL) { 991 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 992 goto hxge_dupb_bcopy_exit; 993 } 994 dp = mp->b_rptr = mp->b_rptr + HXGE_RXBUF_EXTRA; 995 bcopy((void *) &hxge_mp->buffer[offset], dp, size); 996 mp->b_wptr = dp + size; 997 998 hxge_dupb_bcopy_exit: 999 1000 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 1001 1002 return (mp); 1003 } 1004 1005 void hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, 1006 p_rx_msg_t rx_msg_p); 1007 1008 void 1009 hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p) 1010 { 1011 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_post_page")); 1012 1013 /* Reuse this buffer */ 1014 rx_msg_p->free = B_FALSE; 1015 rx_msg_p->cur_usage_cnt = 0; 1016 rx_msg_p->max_usage_cnt = 0; 1017 rx_msg_p->pkt_buf_size = 0; 1018 1019 if (rx_rbr_p->rbr_use_bcopy) { 1020 rx_msg_p->rx_use_bcopy = B_FALSE; 1021 atomic_dec_32(&rx_rbr_p->rbr_consumed); 1022 } 1023 atomic_dec_32(&rx_rbr_p->rbr_used); 1024 1025 /* 1026 * Get the rbr header pointer and its offset index. 1027 */ 1028 rx_rbr_p->rbr_wr_index = ((rx_rbr_p->rbr_wr_index + 1) & 1029 rx_rbr_p->rbr_wrap_mask); 1030 rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr; 1031 1032 /* 1033 * Accumulate some buffers in the ring before re-enabling the 1034 * DMA channel, if rbr empty was signaled. 1035 */ 1036 hpi_rxdma_rdc_rbr_kick(HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc, 1); 1037 if (rx_rbr_p->rbr_is_empty && 1038 (rx_rbr_p->rbb_max - rx_rbr_p->rbr_used) >= 1039 HXGE_RBR_EMPTY_THRESHOLD) { 1040 hxge_rbr_empty_restore(hxgep, rx_rbr_p); 1041 } 1042 1043 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1044 "<== hxge_post_page (channel %d post_next_index %d)", 1045 rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index)); 1046 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_post_page")); 1047 } 1048 1049 void 1050 hxge_freeb(p_rx_msg_t rx_msg_p) 1051 { 1052 size_t size; 1053 uchar_t *buffer = NULL; 1054 int ref_cnt; 1055 boolean_t free_state = B_FALSE; 1056 rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p; 1057 1058 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> hxge_freeb")); 1059 HXGE_DEBUG_MSG((NULL, MEM2_CTL, 1060 "hxge_freeb:rx_msg_p = $%p (block pending %d)", 1061 rx_msg_p, hxge_mblks_pending)); 1062 1063 if (ring == NULL) 1064 return; 1065 1066 /* 1067 * This is to prevent posting activities while we are recovering 1068 * from fatal errors. This should not be a performance drag since 1069 * ref_cnt != 0 most times. 1070 */ 1071 if (ring->rbr_state == RBR_POSTING) 1072 MUTEX_ENTER(&ring->post_lock); 1073 1074 /* 1075 * First we need to get the free state, then 1076 * atomic decrement the reference count to prevent 1077 * the race condition with the interrupt thread that 1078 * is processing a loaned up buffer block. 1079 */ 1080 free_state = rx_msg_p->free; 1081 ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1); 1082 if (!ref_cnt) { 1083 atomic_dec_32(&hxge_mblks_pending); 1084 1085 buffer = rx_msg_p->buffer; 1086 size = rx_msg_p->block_size; 1087 1088 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "hxge_freeb: " 1089 "will free: rx_msg_p = $%p (block pending %d)", 1090 rx_msg_p, hxge_mblks_pending)); 1091 1092 if (!rx_msg_p->use_buf_pool) { 1093 KMEM_FREE(buffer, size); 1094 } 1095 1096 KMEM_FREE(rx_msg_p, sizeof (rx_msg_t)); 1097 /* 1098 * Decrement the receive buffer ring's reference 1099 * count, too. 1100 */ 1101 atomic_dec_32(&ring->rbr_ref_cnt); 1102 1103 /* 1104 * Free the receive buffer ring, iff 1105 * 1. all the receive buffers have been freed 1106 * 2. and we are in the proper state (that is, 1107 * we are not UNMAPPING). 1108 */ 1109 if (ring->rbr_ref_cnt == 0 && 1110 ring->rbr_state == RBR_UNMAPPED) { 1111 KMEM_FREE(ring, sizeof (*ring)); 1112 /* post_lock has been destroyed already */ 1113 return; 1114 } 1115 } 1116 1117 /* 1118 * Repost buffer. 1119 */ 1120 if (free_state && (ref_cnt == 1)) { 1121 HXGE_DEBUG_MSG((NULL, RX_CTL, 1122 "hxge_freeb: post page $%p:", rx_msg_p)); 1123 if (ring->rbr_state == RBR_POSTING) 1124 hxge_post_page(rx_msg_p->hxgep, ring, rx_msg_p); 1125 } 1126 1127 if (ring->rbr_state == RBR_POSTING) 1128 MUTEX_EXIT(&ring->post_lock); 1129 1130 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== hxge_freeb")); 1131 } 1132 1133 uint_t 1134 hxge_rx_intr(caddr_t arg1, caddr_t arg2) 1135 { 1136 p_hxge_ldv_t ldvp = (p_hxge_ldv_t)arg1; 1137 p_hxge_t hxgep = (p_hxge_t)arg2; 1138 p_hxge_ldg_t ldgp; 1139 uint8_t channel; 1140 hpi_handle_t handle; 1141 rdc_stat_t cs, arm_cs; 1142 uint16_t npkts = 0, nptrs = 0; 1143 uint_t serviced = DDI_INTR_UNCLAIMED; 1144 1145 if (ldvp == NULL) { 1146 HXGE_DEBUG_MSG((NULL, RX_INT_CTL, 1147 "<== hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1148 return (DDI_INTR_UNCLAIMED); 1149 } 1150 1151 if (arg2 == NULL || (void *) ldvp->hxgep != arg2) { 1152 hxgep = ldvp->hxgep; 1153 } 1154 1155 /* 1156 * If the interface is not started, just swallow the interrupt 1157 * for the logical device and don't rearm it. 1158 */ 1159 if (hxgep->hxge_mac_state != HXGE_MAC_STARTED) 1160 return (DDI_INTR_CLAIMED); 1161 1162 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1163 "==> hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1164 1165 /* 1166 * This interrupt handler is for a specific receive dma channel. 1167 */ 1168 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1169 1170 /* 1171 * Get the control and status for this channel. 1172 */ 1173 channel = ldvp->channel; 1174 ldgp = ldvp->ldgp; 1175 RXDMA_REG_READ64(handle, RDC_STAT, channel, &cs.value); 1176 1177 /* 1178 * Clear the sources of the interrupts by writing back 1179 * the Control Status registers. 1180 */ 1181 cs.bits.ptrread = 0; 1182 cs.bits.pktread = 0; 1183 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1184 1185 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_intr:channel %d " 1186 "cs 0x%016llx rcrto 0x%x rcrthres %x", 1187 channel, cs.value, cs.bits.rcr_to, cs.bits.rcr_thres)); 1188 1189 /* 1190 * Make sure that either the mex bit is clear or rbr empty 1191 * has occurred before processing packets. 1192 */ 1193 if ((cs.bits.mex == 0) || (cs.bits.rbr_empty)) { 1194 hxge_rx_pkts_vring(hxgep, ldvp->vdma_index, ldvp, cs, 1195 &nptrs, &npkts); 1196 } 1197 serviced = DDI_INTR_CLAIMED; 1198 1199 /* error events. */ 1200 if (cs.value & RDC_STAT_ERROR) { 1201 (void) hxge_rx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs); 1202 } 1203 1204 hxge_intr_exit: 1205 /* 1206 * Enable the mailbox update interrupt if we want to use mailbox. We 1207 * probably don't need to use mailbox as it only saves us one pio read. 1208 * Also write 1 to rcrthres and rcrto to clear these two edge triggered 1209 * bits. 1210 */ 1211 arm_cs.value = 0; 1212 arm_cs.bits.mex = 1; 1213 arm_cs.bits.ptrread = nptrs; 1214 arm_cs.bits.pktread = npkts; 1215 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, arm_cs.value); 1216 1217 /* 1218 * Rearm this logical group if this is a single device group. 1219 */ 1220 if (ldgp->nldvs == 1) { 1221 ld_intr_mgmt_t mgm; 1222 1223 mgm.value = 0; 1224 mgm.bits.arm = 1; 1225 mgm.bits.timer = ldgp->ldg_timer; 1226 HXGE_REG_WR32(handle, 1227 LD_INTR_MGMT + LDSV_OFFSET(ldgp->ldg), mgm.value); 1228 } 1229 1230 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1231 "<== hxge_rx_intr: serviced %d", serviced)); 1232 return (serviced); 1233 } 1234 1235 static void 1236 hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1237 rdc_stat_t cs, uint16_t *nptrs, uint16_t *npkts) 1238 { 1239 p_mblk_t mp; 1240 p_rx_rcr_ring_t rcrp; 1241 1242 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts_vring")); 1243 if ((mp = hxge_rx_pkts(hxgep, vindex, ldvp, &rcrp, cs, 1244 nptrs, npkts)) == NULL) { 1245 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1246 "<== hxge_rx_pkts_vring: no mp")); 1247 return; 1248 } 1249 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts_vring: $%p", mp)); 1250 1251 #ifdef HXGE_DEBUG 1252 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1253 "==> hxge_rx_pkts_vring:calling mac_rx (NEMO) " 1254 "LEN %d mp $%p mp->b_next $%p rcrp $%p", 1255 (mp->b_wptr - mp->b_rptr), mp, mp->b_next, rcrp)); 1256 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1257 "==> hxge_rx_pkts_vring: dump packets " 1258 "(mp $%p b_rptr $%p b_wptr $%p):\n %s", 1259 mp, mp->b_rptr, mp->b_wptr, 1260 hxge_dump_packet((char *)mp->b_rptr, 64))); 1261 1262 if (mp->b_cont) { 1263 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1264 "==> hxge_rx_pkts_vring: dump b_cont packets " 1265 "(mp->b_cont $%p b_rptr $%p b_wptr $%p):\n %s", 1266 mp->b_cont, mp->b_cont->b_rptr, mp->b_cont->b_wptr, 1267 hxge_dump_packet((char *)mp->b_cont->b_rptr, 1268 mp->b_cont->b_wptr - mp->b_cont->b_rptr))); 1269 } 1270 if (mp->b_next) { 1271 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1272 "==> hxge_rx_pkts_vring: dump next packets " 1273 "(b_rptr $%p): %s", 1274 mp->b_next->b_rptr, 1275 hxge_dump_packet((char *)mp->b_next->b_rptr, 64))); 1276 } 1277 #endif 1278 1279 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1280 "==> hxge_rx_pkts_vring: send packet to stack")); 1281 mac_rx(hxgep->mach, NULL, mp); 1282 1283 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_pkts_vring")); 1284 } 1285 1286 /*ARGSUSED*/ 1287 mblk_t * 1288 hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1289 p_rx_rcr_ring_t *rcrp, rdc_stat_t cs, uint16_t *nptrs, uint16_t *npkts) 1290 { 1291 hpi_handle_t handle; 1292 uint8_t channel; 1293 p_rx_rcr_rings_t rx_rcr_rings; 1294 p_rx_rcr_ring_t rcr_p; 1295 uint32_t comp_rd_index; 1296 p_rcr_entry_t rcr_desc_rd_head_p; 1297 p_rcr_entry_t rcr_desc_rd_head_pp; 1298 p_mblk_t nmp, mp_cont, head_mp, *tail_mp; 1299 uint16_t qlen, nrcr_read, npkt_read; 1300 uint32_t qlen_hw, qlen_sw; 1301 uint32_t invalid_rcr_entry; 1302 boolean_t multi; 1303 rdc_rcr_cfg_b_t rcr_cfg_b; 1304 p_rx_mbox_t rx_mboxp; 1305 p_rxdma_mailbox_t mboxp; 1306 uint64_t rcr_head_index, rcr_tail_index; 1307 uint64_t rcr_tail; 1308 rdc_rcr_tail_t rcr_tail_reg; 1309 p_hxge_rx_ring_stats_t rdc_stats; 1310 1311 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:vindex %d " 1312 "channel %d", vindex, ldvp->channel)); 1313 1314 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 1315 return (NULL); 1316 } 1317 1318 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1319 rx_rcr_rings = hxgep->rx_rcr_rings; 1320 rcr_p = rx_rcr_rings->rcr_rings[vindex]; 1321 channel = rcr_p->rdc; 1322 if (channel != ldvp->channel) { 1323 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:index %d " 1324 "channel %d, and rcr channel %d not matched.", 1325 vindex, ldvp->channel, channel)); 1326 return (NULL); 1327 } 1328 1329 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1330 "==> hxge_rx_pkts: START: rcr channel %d " 1331 "head_p $%p head_pp $%p index %d ", 1332 channel, rcr_p->rcr_desc_rd_head_p, 1333 rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1334 1335 if (!cs.bits.mex && !cs.bits.rbr_empty) { 1336 rdc_rcr_qlen_t qlen_mb; 1337 uint64_t value; 1338 1339 /* 1340 * Get the pointer to the mbox. 1341 */ 1342 rx_mboxp = hxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 1343 mboxp = (p_rxdma_mailbox_t)rx_mboxp->rx_mbox.kaddrp; 1344 1345 qlen_mb.value = mboxp->rcrstat_a.value; 1346 value = mboxp->rcrstat_c.value; 1347 1348 /* 1349 * Get the rcr qlen. 1350 */ 1351 qlen = qlen_mb.bits.qlen; 1352 1353 /* 1354 * Get the rcr tail. 1355 */ 1356 rcr_tail = (uint64_t)value >> 3; 1357 } else { 1358 (void) hpi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen); 1359 RXDMA_REG_READ64(handle, RDC_RCR_TAIL, channel, 1360 &rcr_tail_reg.value); 1361 rcr_tail = rcr_tail_reg.bits.tail; 1362 } 1363 1364 if (!qlen) { 1365 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1366 "<== hxge_rx_pkts:rcr channel %d qlen %d (no pkts)", 1367 channel, qlen)); 1368 return (NULL); 1369 } 1370 1371 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts:rcr channel %d " 1372 "qlen %d", channel, qlen)); 1373 1374 comp_rd_index = rcr_p->comp_rd_index; 1375 1376 rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p; 1377 rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp; 1378 nrcr_read = npkt_read = 0; 1379 1380 /* 1381 * Number of packets queued (The jumbo or multi packet will be counted 1382 * as only one paccket and it may take up more than one completion 1383 * entry). 1384 */ 1385 if (cs.bits.rbr_empty) 1386 qlen_hw = qlen; 1387 else 1388 qlen_hw = (qlen < hxge_max_rx_pkts) ? qlen : hxge_max_rx_pkts; 1389 1390 head_mp = NULL; 1391 tail_mp = &head_mp; 1392 nmp = mp_cont = NULL; 1393 multi = B_FALSE; 1394 1395 rcr_head_index = rcr_p->rcr_desc_rd_head_p - rcr_p->rcr_desc_first_p; 1396 rcr_tail_index = rcr_tail - rcr_p->rcr_tail_begin; 1397 1398 if (rcr_tail_index >= rcr_head_index) { 1399 qlen_sw = rcr_tail_index - rcr_head_index; 1400 } else { 1401 /* rcr_tail has wrapped around */ 1402 qlen_sw = (rcr_p->comp_size - rcr_head_index) + rcr_tail_index; 1403 } 1404 1405 if (qlen_hw > qlen_sw) { 1406 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1407 "Channel %d, rcr_qlen from reg %d and from rcr_tail %d\n", 1408 channel, qlen_hw, qlen_sw)); 1409 qlen_hw = qlen_sw; 1410 } 1411 1412 while (qlen_hw) { 1413 #ifdef HXGE_DEBUG 1414 hxge_dump_rcr_entry(hxgep, rcr_desc_rd_head_p); 1415 #endif 1416 /* 1417 * Process one completion ring entry. 1418 */ 1419 invalid_rcr_entry = 0; 1420 hxge_receive_packet(hxgep, 1421 rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont, 1422 &invalid_rcr_entry); 1423 if (invalid_rcr_entry != 0) { 1424 rdc_stats = rcr_p->rdc_stats; 1425 rdc_stats->rcr_invalids++; 1426 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1427 "Channel %d could only read 0x%x packets, " 1428 "but 0x%x pending\n", channel, npkt_read, qlen_hw)); 1429 break; 1430 } 1431 1432 /* 1433 * message chaining modes (nemo msg chaining) 1434 */ 1435 if (nmp) { 1436 nmp->b_next = NULL; 1437 if (!multi && !mp_cont) { /* frame fits a partition */ 1438 *tail_mp = nmp; 1439 tail_mp = &nmp->b_next; 1440 nmp = NULL; 1441 } else if (multi && !mp_cont) { /* first segment */ 1442 *tail_mp = nmp; 1443 tail_mp = &nmp->b_cont; 1444 } else if (multi && mp_cont) { /* mid of multi segs */ 1445 *tail_mp = mp_cont; 1446 tail_mp = &mp_cont->b_cont; 1447 } else if (!multi && mp_cont) { /* last segment */ 1448 *tail_mp = mp_cont; 1449 tail_mp = &nmp->b_next; 1450 nmp = NULL; 1451 } 1452 } 1453 1454 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1455 "==> hxge_rx_pkts: loop: rcr channel %d " 1456 "before updating: multi %d " 1457 "nrcr_read %d " 1458 "npk read %d " 1459 "head_pp $%p index %d ", 1460 channel, multi, 1461 nrcr_read, npkt_read, rcr_desc_rd_head_pp, comp_rd_index)); 1462 1463 if (!multi) { 1464 qlen_hw--; 1465 npkt_read++; 1466 } 1467 1468 /* 1469 * Update the next read entry. 1470 */ 1471 comp_rd_index = NEXT_ENTRY(comp_rd_index, 1472 rcr_p->comp_wrap_mask); 1473 1474 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p, 1475 rcr_p->rcr_desc_first_p, rcr_p->rcr_desc_last_p); 1476 1477 nrcr_read++; 1478 1479 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1480 "<== hxge_rx_pkts: (SAM, process one packet) " 1481 "nrcr_read %d", nrcr_read)); 1482 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1483 "==> hxge_rx_pkts: loop: rcr channel %d " 1484 "multi %d nrcr_read %d npk read %d head_pp $%p index %d ", 1485 channel, multi, nrcr_read, npkt_read, rcr_desc_rd_head_pp, 1486 comp_rd_index)); 1487 } 1488 1489 rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp; 1490 rcr_p->comp_rd_index = comp_rd_index; 1491 rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p; 1492 1493 if ((hxgep->intr_timeout != rcr_p->intr_timeout) || 1494 (hxgep->intr_threshold != rcr_p->intr_threshold)) { 1495 rcr_p->intr_timeout = hxgep->intr_timeout; 1496 rcr_p->intr_threshold = hxgep->intr_threshold; 1497 rcr_cfg_b.value = 0x0ULL; 1498 if (rcr_p->intr_timeout) 1499 rcr_cfg_b.bits.entout = 1; 1500 rcr_cfg_b.bits.timeout = rcr_p->intr_timeout; 1501 rcr_cfg_b.bits.pthres = rcr_p->intr_threshold; 1502 RXDMA_REG_WRITE64(handle, RDC_RCR_CFG_B, 1503 channel, rcr_cfg_b.value); 1504 } 1505 1506 if (hxgep->rdc_first_intr[channel] && (npkt_read > 0)) { 1507 hxgep->rdc_first_intr[channel] = B_FALSE; 1508 *npkts = npkt_read - 1; 1509 } else 1510 *npkts = npkt_read; 1511 *nptrs = nrcr_read; 1512 1513 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1514 "==> hxge_rx_pkts: EXIT: rcr channel %d " 1515 "head_pp $%p index %016llx ", 1516 channel, rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1517 1518 /* 1519 * Update RCR buffer pointer read and number of packets read. 1520 */ 1521 *rcrp = rcr_p; 1522 1523 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "<== hxge_rx_pkts")); 1524 1525 return (head_mp); 1526 } 1527 1528 #define RCR_ENTRY_PATTERN 0x5a5a6b6b7c7c8d8dULL 1529 #define NO_PORT_BIT 0x20 1530 #define L4_CS_EQ_BIT 0x40 1531 1532 /*ARGSUSED*/ 1533 void 1534 hxge_receive_packet(p_hxge_t hxgep, 1535 p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p, 1536 boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont, 1537 uint32_t *invalid_rcr_entry) 1538 { 1539 p_mblk_t nmp = NULL; 1540 uint64_t multi; 1541 uint8_t channel; 1542 1543 boolean_t first_entry = B_TRUE; 1544 boolean_t is_tcp_udp = B_FALSE; 1545 boolean_t buffer_free = B_FALSE; 1546 boolean_t error_send_up = B_FALSE; 1547 uint8_t error_type; 1548 uint16_t l2_len; 1549 uint16_t skip_len; 1550 uint8_t pktbufsz_type; 1551 uint64_t rcr_entry; 1552 uint64_t *pkt_buf_addr_pp; 1553 uint64_t *pkt_buf_addr_p; 1554 uint32_t buf_offset; 1555 uint32_t bsize; 1556 uint32_t msg_index; 1557 p_rx_rbr_ring_t rx_rbr_p; 1558 p_rx_msg_t *rx_msg_ring_p; 1559 p_rx_msg_t rx_msg_p; 1560 1561 uint16_t sw_offset_bytes = 0, hdr_size = 0; 1562 hxge_status_t status = HXGE_OK; 1563 boolean_t is_valid = B_FALSE; 1564 p_hxge_rx_ring_stats_t rdc_stats; 1565 uint32_t bytes_read; 1566 uint8_t header0 = 0; 1567 uint8_t header1 = 0; 1568 uint64_t pkt_type; 1569 uint8_t no_port_bit = 0; 1570 uint8_t l4_cs_eq_bit = 0; 1571 1572 channel = rcr_p->rdc; 1573 1574 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_receive_packet")); 1575 1576 first_entry = (*mp == NULL) ? B_TRUE : B_FALSE; 1577 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p); 1578 1579 /* Verify the content of the rcr_entry for a hardware bug workaround */ 1580 if ((rcr_entry == 0x0) || (rcr_entry == RCR_ENTRY_PATTERN)) { 1581 *invalid_rcr_entry = 1; 1582 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "hxge_receive_packet " 1583 "Channel %d invalid RCR entry 0x%llx found, returning\n", 1584 channel, (long long) rcr_entry)); 1585 return; 1586 } 1587 *((uint64_t *)rcr_desc_rd_head_p) = RCR_ENTRY_PATTERN; 1588 1589 multi = (rcr_entry & RCR_MULTI_MASK); 1590 pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK); 1591 1592 error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT); 1593 l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT); 1594 1595 /* 1596 * Hardware does not strip the CRC due bug ID 11451 where 1597 * the hardware mis handles minimum size packets. 1598 */ 1599 l2_len -= ETHERFCSL; 1600 1601 pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >> 1602 RCR_PKTBUFSZ_SHIFT); 1603 #if defined(__i386) 1604 pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry & 1605 RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT); 1606 #else 1607 pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) << 1608 RCR_PKT_BUF_ADDR_SHIFT); 1609 #endif 1610 1611 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1612 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1613 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1614 "error_type 0x%x pktbufsz_type %d ", 1615 rcr_desc_rd_head_p, rcr_entry, pkt_buf_addr_pp, l2_len, 1616 multi, error_type, pktbufsz_type)); 1617 1618 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1619 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1620 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1621 "error_type 0x%x ", rcr_desc_rd_head_p, 1622 rcr_entry, pkt_buf_addr_pp, l2_len, multi, error_type)); 1623 1624 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1625 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1626 "full pkt_buf_addr_pp $%p l2_len %d", 1627 rcr_entry, pkt_buf_addr_pp, l2_len)); 1628 1629 /* get the stats ptr */ 1630 rdc_stats = rcr_p->rdc_stats; 1631 1632 if (!l2_len) { 1633 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1634 "<== hxge_receive_packet: failed: l2 length is 0.")); 1635 return; 1636 } 1637 1638 /* shift 6 bits to get the full io address */ 1639 #if defined(__i386) 1640 pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp << 1641 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1642 #else 1643 pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp << 1644 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1645 #endif 1646 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1647 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1648 "full pkt_buf_addr_pp $%p l2_len %d", 1649 rcr_entry, pkt_buf_addr_pp, l2_len)); 1650 1651 rx_rbr_p = rcr_p->rx_rbr_p; 1652 rx_msg_ring_p = rx_rbr_p->rx_msg_ring; 1653 1654 if (first_entry) { 1655 hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL : 1656 RXDMA_HDR_SIZE_DEFAULT); 1657 1658 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1659 "==> hxge_receive_packet: first entry 0x%016llx " 1660 "pkt_buf_addr_pp $%p l2_len %d hdr %d", 1661 rcr_entry, pkt_buf_addr_pp, l2_len, hdr_size)); 1662 } 1663 1664 MUTEX_ENTER(&rcr_p->lock); 1665 MUTEX_ENTER(&rx_rbr_p->lock); 1666 1667 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1668 "==> (rbr 1) hxge_receive_packet: entry 0x%0llx " 1669 "full pkt_buf_addr_pp $%p l2_len %d", 1670 rcr_entry, pkt_buf_addr_pp, l2_len)); 1671 1672 /* 1673 * Packet buffer address in the completion entry points to the starting 1674 * buffer address (offset 0). Use the starting buffer address to locate 1675 * the corresponding kernel address. 1676 */ 1677 status = hxge_rxbuf_pp_to_vp(hxgep, rx_rbr_p, 1678 pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p, 1679 &buf_offset, &msg_index); 1680 1681 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1682 "==> (rbr 2) hxge_receive_packet: entry 0x%0llx " 1683 "full pkt_buf_addr_pp $%p l2_len %d", 1684 rcr_entry, pkt_buf_addr_pp, l2_len)); 1685 1686 if (status != HXGE_OK) { 1687 MUTEX_EXIT(&rx_rbr_p->lock); 1688 MUTEX_EXIT(&rcr_p->lock); 1689 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1690 "<== hxge_receive_packet: found vaddr failed %d", status)); 1691 return; 1692 } 1693 1694 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1695 "==> (rbr 3) hxge_receive_packet: entry 0x%0llx " 1696 "full pkt_buf_addr_pp $%p l2_len %d", 1697 rcr_entry, pkt_buf_addr_pp, l2_len)); 1698 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1699 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1700 "full pkt_buf_addr_pp $%p l2_len %d", 1701 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1702 1703 if (msg_index >= rx_rbr_p->tnblocks) { 1704 MUTEX_EXIT(&rx_rbr_p->lock); 1705 MUTEX_EXIT(&rcr_p->lock); 1706 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1707 "==> hxge_receive_packet: FATAL msg_index (%d) " 1708 "should be smaller than tnblocks (%d)\n", 1709 msg_index, rx_rbr_p->tnblocks)); 1710 return; 1711 } 1712 1713 rx_msg_p = rx_msg_ring_p[msg_index]; 1714 1715 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1716 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1717 "full pkt_buf_addr_pp $%p l2_len %d", 1718 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1719 1720 switch (pktbufsz_type) { 1721 case RCR_PKTBUFSZ_0: 1722 bsize = rx_rbr_p->pkt_buf_size0_bytes; 1723 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1724 "==> hxge_receive_packet: 0 buf %d", bsize)); 1725 break; 1726 case RCR_PKTBUFSZ_1: 1727 bsize = rx_rbr_p->pkt_buf_size1_bytes; 1728 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1729 "==> hxge_receive_packet: 1 buf %d", bsize)); 1730 break; 1731 case RCR_PKTBUFSZ_2: 1732 bsize = rx_rbr_p->pkt_buf_size2_bytes; 1733 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1734 "==> hxge_receive_packet: 2 buf %d", bsize)); 1735 break; 1736 case RCR_SINGLE_BLOCK: 1737 bsize = rx_msg_p->block_size; 1738 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1739 "==> hxge_receive_packet: single %d", bsize)); 1740 1741 break; 1742 default: 1743 MUTEX_EXIT(&rx_rbr_p->lock); 1744 MUTEX_EXIT(&rcr_p->lock); 1745 return; 1746 } 1747 1748 DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma, 1749 (buf_offset + sw_offset_bytes), (hdr_size + l2_len), 1750 DDI_DMA_SYNC_FORCPU); 1751 1752 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1753 "==> hxge_receive_packet: after first dump:usage count")); 1754 1755 if (rx_msg_p->cur_usage_cnt == 0) { 1756 atomic_inc_32(&rx_rbr_p->rbr_used); 1757 if (rx_rbr_p->rbr_use_bcopy) { 1758 atomic_inc_32(&rx_rbr_p->rbr_consumed); 1759 if (rx_rbr_p->rbr_consumed < 1760 rx_rbr_p->rbr_threshold_hi) { 1761 if (rx_rbr_p->rbr_threshold_lo == 0 || 1762 ((rx_rbr_p->rbr_consumed >= 1763 rx_rbr_p->rbr_threshold_lo) && 1764 (rx_rbr_p->rbr_bufsize_type >= 1765 pktbufsz_type))) { 1766 rx_msg_p->rx_use_bcopy = B_TRUE; 1767 } 1768 } else { 1769 rx_msg_p->rx_use_bcopy = B_TRUE; 1770 } 1771 } 1772 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1773 "==> hxge_receive_packet: buf %d (new block) ", bsize)); 1774 1775 rx_msg_p->pkt_buf_size_code = pktbufsz_type; 1776 rx_msg_p->pkt_buf_size = bsize; 1777 rx_msg_p->cur_usage_cnt = 1; 1778 if (pktbufsz_type == RCR_SINGLE_BLOCK) { 1779 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1780 "==> hxge_receive_packet: buf %d (single block) ", 1781 bsize)); 1782 /* 1783 * Buffer can be reused once the free function is 1784 * called. 1785 */ 1786 rx_msg_p->max_usage_cnt = 1; 1787 buffer_free = B_TRUE; 1788 } else { 1789 rx_msg_p->max_usage_cnt = rx_msg_p->block_size / bsize; 1790 if (rx_msg_p->max_usage_cnt == 1) { 1791 buffer_free = B_TRUE; 1792 } 1793 } 1794 } else { 1795 rx_msg_p->cur_usage_cnt++; 1796 if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) { 1797 buffer_free = B_TRUE; 1798 } 1799 } 1800 1801 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1802 "msgbuf index = %d l2len %d bytes usage %d max_usage %d ", 1803 msg_index, l2_len, 1804 rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt)); 1805 1806 if (error_type) { 1807 rdc_stats->ierrors++; 1808 /* Update error stats */ 1809 rdc_stats->errlog.compl_err_type = error_type; 1810 HXGE_FM_REPORT_ERROR(hxgep, NULL, HXGE_FM_EREPORT_RDMC_RCR_ERR); 1811 1812 if (error_type & RCR_CTRL_FIFO_DED) { 1813 rdc_stats->ctrl_fifo_ecc_err++; 1814 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1815 " hxge_receive_packet: " 1816 " channel %d RCR ctrl_fifo_ded error", channel)); 1817 } else if (error_type & RCR_DATA_FIFO_DED) { 1818 rdc_stats->data_fifo_ecc_err++; 1819 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1820 " hxge_receive_packet: channel %d" 1821 " RCR data_fifo_ded error", channel)); 1822 } 1823 1824 /* 1825 * Update and repost buffer block if max usage count is 1826 * reached. 1827 */ 1828 if (error_send_up == B_FALSE) { 1829 atomic_inc_32(&rx_msg_p->ref_cnt); 1830 if (buffer_free == B_TRUE) { 1831 rx_msg_p->free = B_TRUE; 1832 } 1833 1834 MUTEX_EXIT(&rx_rbr_p->lock); 1835 MUTEX_EXIT(&rcr_p->lock); 1836 hxge_freeb(rx_msg_p); 1837 return; 1838 } 1839 } 1840 1841 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1842 "==> hxge_receive_packet: DMA sync second ")); 1843 1844 bytes_read = rcr_p->rcvd_pkt_bytes; 1845 skip_len = sw_offset_bytes + hdr_size; 1846 1847 if (first_entry) { 1848 header0 = rx_msg_p->buffer[buf_offset]; 1849 no_port_bit = header0 & NO_PORT_BIT; 1850 1851 header1 = rx_msg_p->buffer[buf_offset + 1]; 1852 l4_cs_eq_bit = header1 & L4_CS_EQ_BIT; 1853 } 1854 1855 if (!rx_msg_p->rx_use_bcopy) { 1856 /* 1857 * For loaned up buffers, the driver reference count 1858 * will be incremented first and then the free state. 1859 */ 1860 if ((nmp = hxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) { 1861 if (first_entry) { 1862 nmp->b_rptr = &nmp->b_rptr[skip_len]; 1863 if (l2_len < bsize - skip_len) { 1864 nmp->b_wptr = &nmp->b_rptr[l2_len]; 1865 } else { 1866 nmp->b_wptr = &nmp->b_rptr[bsize 1867 - skip_len]; 1868 } 1869 } else { 1870 if (l2_len - bytes_read < bsize) { 1871 nmp->b_wptr = 1872 &nmp->b_rptr[l2_len - bytes_read]; 1873 } else { 1874 nmp->b_wptr = &nmp->b_rptr[bsize]; 1875 } 1876 } 1877 } 1878 } else { 1879 if (first_entry) { 1880 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len, 1881 l2_len < bsize - skip_len ? 1882 l2_len : bsize - skip_len); 1883 } else { 1884 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset, 1885 l2_len - bytes_read < bsize ? 1886 l2_len - bytes_read : bsize); 1887 } 1888 } 1889 1890 if (nmp != NULL) { 1891 if (first_entry) 1892 bytes_read = nmp->b_wptr - nmp->b_rptr; 1893 else 1894 bytes_read += nmp->b_wptr - nmp->b_rptr; 1895 1896 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1897 "==> hxge_receive_packet after dupb: " 1898 "rbr consumed %d " 1899 "pktbufsz_type %d " 1900 "nmp $%p rptr $%p wptr $%p " 1901 "buf_offset %d bzise %d l2_len %d skip_len %d", 1902 rx_rbr_p->rbr_consumed, 1903 pktbufsz_type, 1904 nmp, nmp->b_rptr, nmp->b_wptr, 1905 buf_offset, bsize, l2_len, skip_len)); 1906 } else { 1907 cmn_err(CE_WARN, "!hxge_receive_packet: update stats (error)"); 1908 1909 atomic_inc_32(&rx_msg_p->ref_cnt); 1910 if (buffer_free == B_TRUE) { 1911 rx_msg_p->free = B_TRUE; 1912 } 1913 1914 MUTEX_EXIT(&rx_rbr_p->lock); 1915 MUTEX_EXIT(&rcr_p->lock); 1916 hxge_freeb(rx_msg_p); 1917 return; 1918 } 1919 1920 if (buffer_free == B_TRUE) { 1921 rx_msg_p->free = B_TRUE; 1922 } 1923 1924 /* 1925 * ERROR, FRAG and PKT_TYPE are only reported in the first entry. If a 1926 * packet is not fragmented and no error bit is set, then L4 checksum 1927 * is OK. 1928 */ 1929 is_valid = (nmp != NULL); 1930 if (first_entry) { 1931 rdc_stats->ipackets++; /* count only 1st seg for jumbo */ 1932 if (l2_len > (STD_FRAME_SIZE - ETHERFCSL)) 1933 rdc_stats->jumbo_pkts++; 1934 rdc_stats->ibytes += skip_len + l2_len < bsize ? 1935 l2_len : bsize; 1936 } else { 1937 /* 1938 * Add the current portion of the packet to the kstats. 1939 * The current portion of the packet is calculated by using 1940 * length of the packet and the previously received portion. 1941 */ 1942 rdc_stats->ibytes += l2_len - rcr_p->rcvd_pkt_bytes < bsize ? 1943 l2_len - rcr_p->rcvd_pkt_bytes : bsize; 1944 } 1945 1946 rcr_p->rcvd_pkt_bytes = bytes_read; 1947 1948 if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) { 1949 atomic_inc_32(&rx_msg_p->ref_cnt); 1950 MUTEX_EXIT(&rx_rbr_p->lock); 1951 MUTEX_EXIT(&rcr_p->lock); 1952 hxge_freeb(rx_msg_p); 1953 } else { 1954 MUTEX_EXIT(&rx_rbr_p->lock); 1955 MUTEX_EXIT(&rcr_p->lock); 1956 } 1957 1958 if (is_valid) { 1959 nmp->b_cont = NULL; 1960 if (first_entry) { 1961 *mp = nmp; 1962 *mp_cont = NULL; 1963 } else { 1964 *mp_cont = nmp; 1965 } 1966 } 1967 1968 /* 1969 * Update stats and hardware checksuming. 1970 */ 1971 if (is_valid && !multi) { 1972 is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP || 1973 pkt_type == RCR_PKT_IS_UDP) ? B_TRUE : B_FALSE); 1974 1975 if (!no_port_bit && l4_cs_eq_bit && is_tcp_udp && !error_type) { 1976 (void) hcksum_assoc(nmp, NULL, NULL, 0, 0, 0, 0, 1977 HCK_FULLCKSUM_OK | HCK_FULLCKSUM, 0); 1978 1979 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1980 "==> hxge_receive_packet: Full tcp/udp cksum " 1981 "is_valid 0x%x multi %d error %d", 1982 is_valid, multi, error_type)); 1983 } 1984 } 1985 1986 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1987 "==> hxge_receive_packet: *mp 0x%016llx", *mp)); 1988 1989 *multi_p = (multi == RCR_MULTI_MASK); 1990 1991 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_receive_packet: " 1992 "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx", 1993 *multi_p, nmp, *mp, *mp_cont)); 1994 } 1995 1996 static void 1997 hxge_rx_rbr_empty_recover(p_hxge_t hxgep, uint8_t channel) 1998 { 1999 hpi_handle_t handle; 2000 p_rx_rcr_ring_t rcrp; 2001 p_rx_rbr_ring_t rbrp; 2002 2003 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 2004 rbrp = rcrp->rx_rbr_p; 2005 handle = HXGE_DEV_HPI_HANDLE(hxgep); 2006 2007 /* 2008 * Wait for the channel to be quiet 2009 */ 2010 (void) hpi_rxdma_cfg_rdc_wait_for_qst(handle, channel); 2011 2012 /* 2013 * Post page will accumulate some buffers before re-enabling 2014 * the DMA channel. 2015 */ 2016 2017 MUTEX_ENTER(&rbrp->post_lock); 2018 if ((rbrp->rbb_max - rbrp->rbr_used) >= HXGE_RBR_EMPTY_THRESHOLD) { 2019 hxge_rbr_empty_restore(hxgep, rbrp); 2020 } else { 2021 rbrp->rbr_is_empty = B_TRUE; 2022 } 2023 MUTEX_EXIT(&rbrp->post_lock); 2024 } 2025 2026 /*ARGSUSED*/ 2027 static hxge_status_t 2028 hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp, 2029 rdc_stat_t cs) 2030 { 2031 p_hxge_rx_ring_stats_t rdc_stats; 2032 hpi_handle_t handle; 2033 boolean_t rxchan_fatal = B_FALSE; 2034 uint8_t channel; 2035 hxge_status_t status = HXGE_OK; 2036 2037 HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_rx_err_evnts")); 2038 2039 handle = HXGE_DEV_HPI_HANDLE(hxgep); 2040 channel = ldvp->channel; 2041 2042 rdc_stats = &hxgep->statsp->rdc_stats[ldvp->vdma_index]; 2043 2044 if (cs.bits.rbr_cpl_to) { 2045 rdc_stats->rbr_tmout++; 2046 HXGE_FM_REPORT_ERROR(hxgep, channel, 2047 HXGE_FM_EREPORT_RDMC_RBR_CPL_TO); 2048 rxchan_fatal = B_TRUE; 2049 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2050 "==> hxge_rx_err_evnts(channel %d): " 2051 "fatal error: rx_rbr_timeout", channel)); 2052 } 2053 2054 if ((cs.bits.rcr_shadow_par_err) || (cs.bits.rbr_prefetch_par_err)) { 2055 (void) hpi_rxdma_ring_perr_stat_get(handle, 2056 &rdc_stats->errlog.pre_par, &rdc_stats->errlog.sha_par); 2057 } 2058 2059 if (cs.bits.rcr_shadow_par_err) { 2060 rdc_stats->rcr_sha_par++; 2061 HXGE_FM_REPORT_ERROR(hxgep, channel, 2062 HXGE_FM_EREPORT_RDMC_RCR_SHA_PAR); 2063 rxchan_fatal = B_TRUE; 2064 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2065 "==> hxge_rx_err_evnts(channel %d): " 2066 "fatal error: rcr_shadow_par_err", channel)); 2067 } 2068 2069 if (cs.bits.rbr_prefetch_par_err) { 2070 rdc_stats->rbr_pre_par++; 2071 HXGE_FM_REPORT_ERROR(hxgep, channel, 2072 HXGE_FM_EREPORT_RDMC_RBR_PRE_PAR); 2073 rxchan_fatal = B_TRUE; 2074 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2075 "==> hxge_rx_err_evnts(channel %d): " 2076 "fatal error: rbr_prefetch_par_err", channel)); 2077 } 2078 2079 if (cs.bits.rbr_pre_empty) { 2080 rdc_stats->rbr_pre_empty++; 2081 HXGE_FM_REPORT_ERROR(hxgep, channel, 2082 HXGE_FM_EREPORT_RDMC_RBR_PRE_EMPTY); 2083 rxchan_fatal = B_TRUE; 2084 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2085 "==> hxge_rx_err_evnts(channel %d): " 2086 "fatal error: rbr_pre_empty", channel)); 2087 } 2088 2089 if (cs.bits.peu_resp_err) { 2090 rdc_stats->peu_resp_err++; 2091 HXGE_FM_REPORT_ERROR(hxgep, channel, 2092 HXGE_FM_EREPORT_RDMC_PEU_RESP_ERR); 2093 rxchan_fatal = B_TRUE; 2094 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2095 "==> hxge_rx_err_evnts(channel %d): " 2096 "fatal error: peu_resp_err", channel)); 2097 } 2098 2099 if (cs.bits.rcr_thres) { 2100 rdc_stats->rcr_thres++; 2101 } 2102 2103 if (cs.bits.rcr_to) { 2104 rdc_stats->rcr_to++; 2105 } 2106 2107 if (cs.bits.rcr_shadow_full) { 2108 rdc_stats->rcr_shadow_full++; 2109 HXGE_FM_REPORT_ERROR(hxgep, channel, 2110 HXGE_FM_EREPORT_RDMC_RCR_SHA_FULL); 2111 rxchan_fatal = B_TRUE; 2112 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2113 "==> hxge_rx_err_evnts(channel %d): " 2114 "fatal error: rcr_shadow_full", channel)); 2115 } 2116 2117 if (cs.bits.rcr_full) { 2118 rdc_stats->rcrfull++; 2119 HXGE_FM_REPORT_ERROR(hxgep, channel, 2120 HXGE_FM_EREPORT_RDMC_RCRFULL); 2121 rxchan_fatal = B_TRUE; 2122 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2123 "==> hxge_rx_err_evnts(channel %d): " 2124 "fatal error: rcrfull error", channel)); 2125 } 2126 2127 if (cs.bits.rbr_empty) { 2128 rdc_stats->rbr_empty++; 2129 hxge_rx_rbr_empty_recover(hxgep, channel); 2130 } 2131 2132 if (cs.bits.rbr_full) { 2133 rdc_stats->rbrfull++; 2134 HXGE_FM_REPORT_ERROR(hxgep, channel, 2135 HXGE_FM_EREPORT_RDMC_RBRFULL); 2136 rxchan_fatal = B_TRUE; 2137 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2138 "==> hxge_rx_err_evnts(channel %d): " 2139 "fatal error: rbr_full error", channel)); 2140 } 2141 2142 if (rxchan_fatal) { 2143 p_rx_rcr_ring_t rcrp; 2144 p_rx_rbr_ring_t rbrp; 2145 2146 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 2147 rbrp = rcrp->rx_rbr_p; 2148 2149 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2150 " hxge_rx_err_evnts: fatal error on Channel #%d\n", 2151 channel)); 2152 MUTEX_ENTER(&rbrp->post_lock); 2153 /* This function needs to be inside the post_lock */ 2154 status = hxge_rxdma_fatal_err_recover(hxgep, channel); 2155 MUTEX_EXIT(&rbrp->post_lock); 2156 if (status == HXGE_OK) { 2157 FM_SERVICE_RESTORED(hxgep); 2158 } 2159 } 2160 HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_rx_err_evnts")); 2161 2162 return (status); 2163 } 2164 2165 static hxge_status_t 2166 hxge_map_rxdma(p_hxge_t hxgep) 2167 { 2168 int i, ndmas; 2169 uint16_t channel; 2170 p_rx_rbr_rings_t rx_rbr_rings; 2171 p_rx_rbr_ring_t *rbr_rings; 2172 p_rx_rcr_rings_t rx_rcr_rings; 2173 p_rx_rcr_ring_t *rcr_rings; 2174 p_rx_mbox_areas_t rx_mbox_areas_p; 2175 p_rx_mbox_t *rx_mbox_p; 2176 p_hxge_dma_pool_t dma_buf_poolp; 2177 p_hxge_dma_common_t *dma_buf_p; 2178 p_hxge_dma_pool_t dma_rbr_cntl_poolp; 2179 p_hxge_dma_common_t *dma_rbr_cntl_p; 2180 p_hxge_dma_pool_t dma_rcr_cntl_poolp; 2181 p_hxge_dma_common_t *dma_rcr_cntl_p; 2182 p_hxge_dma_pool_t dma_mbox_cntl_poolp; 2183 p_hxge_dma_common_t *dma_mbox_cntl_p; 2184 uint32_t *num_chunks; 2185 hxge_status_t status = HXGE_OK; 2186 2187 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_map_rxdma")); 2188 2189 dma_buf_poolp = hxgep->rx_buf_pool_p; 2190 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p; 2191 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p; 2192 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p; 2193 2194 if (!dma_buf_poolp->buf_allocated || 2195 !dma_rbr_cntl_poolp->buf_allocated || 2196 !dma_rcr_cntl_poolp->buf_allocated || 2197 !dma_mbox_cntl_poolp->buf_allocated) { 2198 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2199 "<== hxge_map_rxdma: buf not allocated")); 2200 return (HXGE_ERROR); 2201 } 2202 2203 ndmas = dma_buf_poolp->ndmas; 2204 if (!ndmas) { 2205 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2206 "<== hxge_map_rxdma: no dma allocated")); 2207 return (HXGE_ERROR); 2208 } 2209 2210 num_chunks = dma_buf_poolp->num_chunks; 2211 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2212 dma_rbr_cntl_p = dma_rbr_cntl_poolp->dma_buf_pool_p; 2213 dma_rcr_cntl_p = dma_rcr_cntl_poolp->dma_buf_pool_p; 2214 dma_mbox_cntl_p = dma_mbox_cntl_poolp->dma_buf_pool_p; 2215 2216 rx_rbr_rings = (p_rx_rbr_rings_t) 2217 KMEM_ZALLOC(sizeof (rx_rbr_rings_t), KM_SLEEP); 2218 rbr_rings = (p_rx_rbr_ring_t *)KMEM_ZALLOC( 2219 sizeof (p_rx_rbr_ring_t) * ndmas, KM_SLEEP); 2220 2221 rx_rcr_rings = (p_rx_rcr_rings_t) 2222 KMEM_ZALLOC(sizeof (rx_rcr_rings_t), KM_SLEEP); 2223 rcr_rings = (p_rx_rcr_ring_t *)KMEM_ZALLOC( 2224 sizeof (p_rx_rcr_ring_t) * ndmas, KM_SLEEP); 2225 2226 rx_mbox_areas_p = (p_rx_mbox_areas_t) 2227 KMEM_ZALLOC(sizeof (rx_mbox_areas_t), KM_SLEEP); 2228 rx_mbox_p = (p_rx_mbox_t *)KMEM_ZALLOC( 2229 sizeof (p_rx_mbox_t) * ndmas, KM_SLEEP); 2230 2231 /* 2232 * Timeout should be set based on the system clock divider. 2233 * The following timeout value of 1 assumes that the 2234 * granularity (1000) is 3 microseconds running at 300MHz. 2235 */ 2236 2237 hxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT; 2238 hxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT; 2239 2240 /* 2241 * Map descriptors from the buffer polls for each dam channel. 2242 */ 2243 for (i = 0; i < ndmas; i++) { 2244 /* 2245 * Set up and prepare buffer blocks, descriptors and mailbox. 2246 */ 2247 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2248 status = hxge_map_rxdma_channel(hxgep, channel, 2249 (p_hxge_dma_common_t *)&dma_buf_p[i], 2250 (p_rx_rbr_ring_t *)&rbr_rings[i], 2251 num_chunks[i], 2252 (p_hxge_dma_common_t *)&dma_rbr_cntl_p[i], 2253 (p_hxge_dma_common_t *)&dma_rcr_cntl_p[i], 2254 (p_hxge_dma_common_t *)&dma_mbox_cntl_p[i], 2255 (p_rx_rcr_ring_t *)&rcr_rings[i], 2256 (p_rx_mbox_t *)&rx_mbox_p[i]); 2257 if (status != HXGE_OK) { 2258 goto hxge_map_rxdma_fail1; 2259 } 2260 rbr_rings[i]->index = (uint16_t)i; 2261 rcr_rings[i]->index = (uint16_t)i; 2262 rcr_rings[i]->rdc_stats = &hxgep->statsp->rdc_stats[i]; 2263 } 2264 2265 rx_rbr_rings->ndmas = rx_rcr_rings->ndmas = ndmas; 2266 rx_rbr_rings->rbr_rings = rbr_rings; 2267 hxgep->rx_rbr_rings = rx_rbr_rings; 2268 rx_rcr_rings->rcr_rings = rcr_rings; 2269 hxgep->rx_rcr_rings = rx_rcr_rings; 2270 2271 rx_mbox_areas_p->rxmbox_areas = rx_mbox_p; 2272 hxgep->rx_mbox_areas_p = rx_mbox_areas_p; 2273 2274 goto hxge_map_rxdma_exit; 2275 2276 hxge_map_rxdma_fail1: 2277 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2278 "==> hxge_map_rxdma: unmap rbr,rcr (status 0x%x channel %d i %d)", 2279 status, channel, i)); 2280 i--; 2281 for (; i >= 0; i--) { 2282 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2283 hxge_unmap_rxdma_channel(hxgep, channel, 2284 rbr_rings[i], rcr_rings[i], rx_mbox_p[i]); 2285 } 2286 2287 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2288 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2289 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2290 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2291 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2292 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2293 2294 hxge_map_rxdma_exit: 2295 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2296 "<== hxge_map_rxdma: (status 0x%x channel %d)", status, channel)); 2297 2298 return (status); 2299 } 2300 2301 static void 2302 hxge_unmap_rxdma(p_hxge_t hxgep) 2303 { 2304 int i, ndmas; 2305 uint16_t channel; 2306 p_rx_rbr_rings_t rx_rbr_rings; 2307 p_rx_rbr_ring_t *rbr_rings; 2308 p_rx_rcr_rings_t rx_rcr_rings; 2309 p_rx_rcr_ring_t *rcr_rings; 2310 p_rx_mbox_areas_t rx_mbox_areas_p; 2311 p_rx_mbox_t *rx_mbox_p; 2312 p_hxge_dma_pool_t dma_buf_poolp; 2313 p_hxge_dma_pool_t dma_rbr_cntl_poolp; 2314 p_hxge_dma_pool_t dma_rcr_cntl_poolp; 2315 p_hxge_dma_pool_t dma_mbox_cntl_poolp; 2316 p_hxge_dma_common_t *dma_buf_p; 2317 2318 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_unmap_rxdma")); 2319 2320 dma_buf_poolp = hxgep->rx_buf_pool_p; 2321 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p; 2322 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p; 2323 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p; 2324 2325 if (!dma_buf_poolp->buf_allocated || 2326 !dma_rbr_cntl_poolp->buf_allocated || 2327 !dma_rcr_cntl_poolp->buf_allocated || 2328 !dma_mbox_cntl_poolp->buf_allocated) { 2329 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2330 "<== hxge_unmap_rxdma: NULL buf pointers")); 2331 return; 2332 } 2333 2334 rx_rbr_rings = hxgep->rx_rbr_rings; 2335 rx_rcr_rings = hxgep->rx_rcr_rings; 2336 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 2337 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2338 "<== hxge_unmap_rxdma: NULL pointers")); 2339 return; 2340 } 2341 2342 ndmas = rx_rbr_rings->ndmas; 2343 if (!ndmas) { 2344 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2345 "<== hxge_unmap_rxdma: no channel")); 2346 return; 2347 } 2348 2349 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2350 "==> hxge_unmap_rxdma (ndmas %d)", ndmas)); 2351 2352 rbr_rings = rx_rbr_rings->rbr_rings; 2353 rcr_rings = rx_rcr_rings->rcr_rings; 2354 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 2355 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 2356 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2357 2358 for (i = 0; i < ndmas; i++) { 2359 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2360 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2361 "==> hxge_unmap_rxdma (ndmas %d) channel %d", 2362 ndmas, channel)); 2363 (void) hxge_unmap_rxdma_channel(hxgep, channel, 2364 (p_rx_rbr_ring_t)rbr_rings[i], 2365 (p_rx_rcr_ring_t)rcr_rings[i], 2366 (p_rx_mbox_t)rx_mbox_p[i]); 2367 } 2368 2369 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2370 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2371 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2372 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2373 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2374 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2375 2376 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma")); 2377 } 2378 2379 hxge_status_t 2380 hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2381 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 2382 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p, 2383 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 2384 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2385 { 2386 int status = HXGE_OK; 2387 2388 /* 2389 * Set up and prepare buffer blocks, descriptors and mailbox. 2390 */ 2391 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2392 "==> hxge_map_rxdma_channel (channel %d)", channel)); 2393 2394 /* 2395 * Receive buffer blocks 2396 */ 2397 status = hxge_map_rxdma_channel_buf_ring(hxgep, channel, 2398 dma_buf_p, rbr_p, num_chunks); 2399 if (status != HXGE_OK) { 2400 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2401 "==> hxge_map_rxdma_channel (channel %d): " 2402 "map buffer failed 0x%x", channel, status)); 2403 goto hxge_map_rxdma_channel_exit; 2404 } 2405 2406 /* 2407 * Receive block ring, completion ring and mailbox. 2408 */ 2409 status = hxge_map_rxdma_channel_cfg_ring(hxgep, channel, 2410 dma_rbr_cntl_p, dma_rcr_cntl_p, dma_mbox_cntl_p, 2411 rbr_p, rcr_p, rx_mbox_p); 2412 if (status != HXGE_OK) { 2413 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2414 "==> hxge_map_rxdma_channel (channel %d): " 2415 "map config failed 0x%x", channel, status)); 2416 goto hxge_map_rxdma_channel_fail2; 2417 } 2418 goto hxge_map_rxdma_channel_exit; 2419 2420 hxge_map_rxdma_channel_fail3: 2421 /* Free rbr, rcr */ 2422 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2423 "==> hxge_map_rxdma_channel: free rbr/rcr (status 0x%x channel %d)", 2424 status, channel)); 2425 hxge_unmap_rxdma_channel_cfg_ring(hxgep, *rcr_p, *rx_mbox_p); 2426 2427 hxge_map_rxdma_channel_fail2: 2428 /* Free buffer blocks */ 2429 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2430 "==> hxge_map_rxdma_channel: free rx buffers" 2431 "(hxgep 0x%x status 0x%x channel %d)", 2432 hxgep, status, channel)); 2433 hxge_unmap_rxdma_channel_buf_ring(hxgep, *rbr_p); 2434 2435 status = HXGE_ERROR; 2436 2437 hxge_map_rxdma_channel_exit: 2438 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2439 "<== hxge_map_rxdma_channel: (hxgep 0x%x status 0x%x channel %d)", 2440 hxgep, status, channel)); 2441 2442 return (status); 2443 } 2444 2445 /*ARGSUSED*/ 2446 static void 2447 hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2448 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2449 { 2450 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2451 "==> hxge_unmap_rxdma_channel (channel %d)", channel)); 2452 2453 /* 2454 * unmap receive block ring, completion ring and mailbox. 2455 */ 2456 (void) hxge_unmap_rxdma_channel_cfg_ring(hxgep, rcr_p, rx_mbox_p); 2457 2458 /* unmap buffer blocks */ 2459 (void) hxge_unmap_rxdma_channel_buf_ring(hxgep, rbr_p); 2460 2461 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma_channel")); 2462 } 2463 2464 /*ARGSUSED*/ 2465 static hxge_status_t 2466 hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel, 2467 p_hxge_dma_common_t *dma_rbr_cntl_p, p_hxge_dma_common_t *dma_rcr_cntl_p, 2468 p_hxge_dma_common_t *dma_mbox_cntl_p, p_rx_rbr_ring_t *rbr_p, 2469 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2470 { 2471 p_rx_rbr_ring_t rbrp; 2472 p_rx_rcr_ring_t rcrp; 2473 p_rx_mbox_t mboxp; 2474 p_hxge_dma_common_t cntl_dmap; 2475 p_hxge_dma_common_t dmap; 2476 p_rx_msg_t *rx_msg_ring; 2477 p_rx_msg_t rx_msg_p; 2478 rdc_rbr_cfg_a_t *rcfga_p; 2479 rdc_rbr_cfg_b_t *rcfgb_p; 2480 rdc_rcr_cfg_a_t *cfga_p; 2481 rdc_rcr_cfg_b_t *cfgb_p; 2482 rdc_rx_cfg1_t *cfig1_p; 2483 rdc_rx_cfg2_t *cfig2_p; 2484 rdc_rbr_kick_t *kick_p; 2485 uint32_t dmaaddrp; 2486 uint32_t *rbr_vaddrp; 2487 uint32_t bkaddr; 2488 hxge_status_t status = HXGE_OK; 2489 int i; 2490 uint32_t hxge_port_rcr_size; 2491 2492 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2493 "==> hxge_map_rxdma_channel_cfg_ring")); 2494 2495 cntl_dmap = *dma_rbr_cntl_p; 2496 2497 /* 2498 * Map in the receive block ring 2499 */ 2500 rbrp = *rbr_p; 2501 dmap = (p_hxge_dma_common_t)&rbrp->rbr_desc; 2502 hxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4); 2503 2504 /* 2505 * Zero out buffer block ring descriptors. 2506 */ 2507 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2508 2509 rcfga_p = &(rbrp->rbr_cfga); 2510 rcfgb_p = &(rbrp->rbr_cfgb); 2511 kick_p = &(rbrp->rbr_kick); 2512 rcfga_p->value = 0; 2513 rcfgb_p->value = 0; 2514 kick_p->value = 0; 2515 rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress; 2516 rcfga_p->value = (rbrp->rbr_addr & 2517 (RBR_CFIG_A_STDADDR_MASK | RBR_CFIG_A_STDADDR_BASE_MASK)); 2518 rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT); 2519 2520 /* XXXX: how to choose packet buffer sizes */ 2521 rcfgb_p->bits.bufsz0 = rbrp->pkt_buf_size0; 2522 rcfgb_p->bits.vld0 = 1; 2523 rcfgb_p->bits.bufsz1 = rbrp->pkt_buf_size1; 2524 rcfgb_p->bits.vld1 = 1; 2525 rcfgb_p->bits.bufsz2 = rbrp->pkt_buf_size2; 2526 rcfgb_p->bits.vld2 = 1; 2527 rcfgb_p->bits.bksize = hxgep->rx_bksize_code; 2528 2529 /* 2530 * For each buffer block, enter receive block address to the ring. 2531 */ 2532 rbr_vaddrp = (uint32_t *)dmap->kaddrp; 2533 rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp; 2534 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2535 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2536 "rbr_vaddrp $%p", dma_channel, rbr_vaddrp)); 2537 2538 rx_msg_ring = rbrp->rx_msg_ring; 2539 for (i = 0; i < rbrp->tnblocks; i++) { 2540 rx_msg_p = rx_msg_ring[i]; 2541 rx_msg_p->hxgep = hxgep; 2542 rx_msg_p->rx_rbr_p = rbrp; 2543 bkaddr = (uint32_t) 2544 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2545 RBR_BKADDR_SHIFT)); 2546 rx_msg_p->free = B_FALSE; 2547 rx_msg_p->max_usage_cnt = 0xbaddcafe; 2548 2549 *rbr_vaddrp++ = bkaddr; 2550 } 2551 2552 kick_p->bits.bkadd = rbrp->rbb_max; 2553 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 2554 2555 rbrp->rbr_rd_index = 0; 2556 2557 rbrp->rbr_consumed = 0; 2558 rbrp->rbr_used = 0; 2559 rbrp->rbr_use_bcopy = B_TRUE; 2560 rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0; 2561 2562 /* 2563 * Do bcopy on packets greater than bcopy size once the lo threshold is 2564 * reached. This lo threshold should be less than the hi threshold. 2565 * 2566 * Do bcopy on every packet once the hi threshold is reached. 2567 */ 2568 if (hxge_rx_threshold_lo >= hxge_rx_threshold_hi) { 2569 /* default it to use hi */ 2570 hxge_rx_threshold_lo = hxge_rx_threshold_hi; 2571 } 2572 if (hxge_rx_buf_size_type > HXGE_RBR_TYPE2) { 2573 hxge_rx_buf_size_type = HXGE_RBR_TYPE2; 2574 } 2575 rbrp->rbr_bufsize_type = hxge_rx_buf_size_type; 2576 2577 switch (hxge_rx_threshold_hi) { 2578 default: 2579 case HXGE_RX_COPY_NONE: 2580 /* Do not do bcopy at all */ 2581 rbrp->rbr_use_bcopy = B_FALSE; 2582 rbrp->rbr_threshold_hi = rbrp->rbb_max; 2583 break; 2584 2585 case HXGE_RX_COPY_1: 2586 case HXGE_RX_COPY_2: 2587 case HXGE_RX_COPY_3: 2588 case HXGE_RX_COPY_4: 2589 case HXGE_RX_COPY_5: 2590 case HXGE_RX_COPY_6: 2591 case HXGE_RX_COPY_7: 2592 rbrp->rbr_threshold_hi = 2593 rbrp->rbb_max * (hxge_rx_threshold_hi) / 2594 HXGE_RX_BCOPY_SCALE; 2595 break; 2596 2597 case HXGE_RX_COPY_ALL: 2598 rbrp->rbr_threshold_hi = 0; 2599 break; 2600 } 2601 2602 switch (hxge_rx_threshold_lo) { 2603 default: 2604 case HXGE_RX_COPY_NONE: 2605 /* Do not do bcopy at all */ 2606 if (rbrp->rbr_use_bcopy) { 2607 rbrp->rbr_use_bcopy = B_FALSE; 2608 } 2609 rbrp->rbr_threshold_lo = rbrp->rbb_max; 2610 break; 2611 2612 case HXGE_RX_COPY_1: 2613 case HXGE_RX_COPY_2: 2614 case HXGE_RX_COPY_3: 2615 case HXGE_RX_COPY_4: 2616 case HXGE_RX_COPY_5: 2617 case HXGE_RX_COPY_6: 2618 case HXGE_RX_COPY_7: 2619 rbrp->rbr_threshold_lo = 2620 rbrp->rbb_max * (hxge_rx_threshold_lo) / 2621 HXGE_RX_BCOPY_SCALE; 2622 break; 2623 2624 case HXGE_RX_COPY_ALL: 2625 rbrp->rbr_threshold_lo = 0; 2626 break; 2627 } 2628 2629 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2630 "hxge_map_rxdma_channel_cfg_ring: channel %d rbb_max %d " 2631 "rbrp->rbr_bufsize_type %d rbb_threshold_hi %d " 2632 "rbb_threshold_lo %d", 2633 dma_channel, rbrp->rbb_max, rbrp->rbr_bufsize_type, 2634 rbrp->rbr_threshold_hi, rbrp->rbr_threshold_lo)); 2635 2636 /* Map in the receive completion ring */ 2637 rcrp = (p_rx_rcr_ring_t)KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP); 2638 rcrp->rdc = dma_channel; 2639 rcrp->hxgep = hxgep; 2640 2641 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 2642 rcrp->comp_size = hxge_port_rcr_size; 2643 rcrp->comp_wrap_mask = hxge_port_rcr_size - 1; 2644 2645 rcrp->max_receive_pkts = hxge_max_rx_pkts; 2646 2647 cntl_dmap = *dma_rcr_cntl_p; 2648 2649 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 2650 hxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size, 2651 sizeof (rcr_entry_t)); 2652 rcrp->comp_rd_index = 0; 2653 rcrp->comp_wt_index = 0; 2654 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 2655 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 2656 #if defined(__i386) 2657 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2658 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2659 #else 2660 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2661 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2662 #endif 2663 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 2664 (hxge_port_rcr_size - 1); 2665 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 2666 (hxge_port_rcr_size - 1); 2667 2668 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc); 2669 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3; 2670 2671 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2672 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2673 "rbr_vaddrp $%p rcr_desc_rd_head_p $%p " 2674 "rcr_desc_rd_head_pp $%p rcr_desc_rd_last_p $%p " 2675 "rcr_desc_rd_last_pp $%p ", 2676 dma_channel, rbr_vaddrp, rcrp->rcr_desc_rd_head_p, 2677 rcrp->rcr_desc_rd_head_pp, rcrp->rcr_desc_last_p, 2678 rcrp->rcr_desc_last_pp)); 2679 2680 /* 2681 * Zero out buffer block ring descriptors. 2682 */ 2683 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2684 rcrp->intr_timeout = hxgep->intr_timeout; 2685 rcrp->intr_threshold = hxgep->intr_threshold; 2686 rcrp->full_hdr_flag = B_FALSE; 2687 rcrp->sw_priv_hdr_len = 0; 2688 2689 cfga_p = &(rcrp->rcr_cfga); 2690 cfgb_p = &(rcrp->rcr_cfgb); 2691 cfga_p->value = 0; 2692 cfgb_p->value = 0; 2693 rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress; 2694 2695 cfga_p->value = (rcrp->rcr_addr & 2696 (RCRCFIG_A_STADDR_MASK | RCRCFIG_A_STADDR_BASE_MASK)); 2697 2698 cfga_p->value |= ((uint64_t)rcrp->comp_size << RCRCFIG_A_LEN_SHIF); 2699 2700 /* 2701 * Timeout should be set based on the system clock divider. The 2702 * following timeout value of 1 assumes that the granularity (1000) is 2703 * 3 microseconds running at 300MHz. 2704 */ 2705 cfgb_p->bits.pthres = rcrp->intr_threshold; 2706 cfgb_p->bits.timeout = rcrp->intr_timeout; 2707 cfgb_p->bits.entout = 1; 2708 2709 /* Map in the mailbox */ 2710 cntl_dmap = *dma_mbox_cntl_p; 2711 mboxp = (p_rx_mbox_t)KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP); 2712 dmap = (p_hxge_dma_common_t)&mboxp->rx_mbox; 2713 hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t)); 2714 cfig1_p = (rdc_rx_cfg1_t *)&mboxp->rx_cfg1; 2715 cfig2_p = (rdc_rx_cfg2_t *)&mboxp->rx_cfg2; 2716 cfig1_p->value = cfig2_p->value = 0; 2717 2718 mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress; 2719 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2720 "==> hxge_map_rxdma_channel_cfg_ring: " 2721 "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx", 2722 dma_channel, cfig1_p->value, cfig2_p->value, 2723 mboxp->mbox_addr)); 2724 2725 dmaaddrp = (uint32_t)((dmap->dma_cookie.dmac_laddress >> 32) & 0xfff); 2726 cfig1_p->bits.mbaddr_h = dmaaddrp; 2727 2728 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff); 2729 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 2730 RXDMA_CFIG2_MBADDR_L_MASK); 2731 2732 cfig2_p->bits.mbaddr_l = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT); 2733 2734 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2735 "==> hxge_map_rxdma_channel_cfg_ring: channel %d damaddrp $%p " 2736 "cfg1 0x%016llx cfig2 0x%016llx", 2737 dma_channel, dmaaddrp, cfig1_p->value, cfig2_p->value)); 2738 2739 cfig2_p->bits.full_hdr = rcrp->full_hdr_flag; 2740 cfig2_p->bits.offset = rcrp->sw_priv_hdr_len; 2741 2742 rbrp->rx_rcr_p = rcrp; 2743 rcrp->rx_rbr_p = rbrp; 2744 *rcr_p = rcrp; 2745 *rx_mbox_p = mboxp; 2746 2747 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2748 "<== hxge_map_rxdma_channel_cfg_ring status 0x%08x", status)); 2749 return (status); 2750 } 2751 2752 /*ARGSUSED*/ 2753 static void 2754 hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 2755 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2756 { 2757 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2758 "==> hxge_unmap_rxdma_channel_cfg_ring: channel %d", rcr_p->rdc)); 2759 2760 KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t)); 2761 KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t)); 2762 2763 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2764 "<== hxge_unmap_rxdma_channel_cfg_ring")); 2765 } 2766 2767 static hxge_status_t 2768 hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel, 2769 p_hxge_dma_common_t *dma_buf_p, 2770 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks) 2771 { 2772 p_rx_rbr_ring_t rbrp; 2773 p_hxge_dma_common_t dma_bufp, tmp_bufp; 2774 p_rx_msg_t *rx_msg_ring; 2775 p_rx_msg_t rx_msg_p; 2776 p_mblk_t mblk_p; 2777 2778 rxring_info_t *ring_info; 2779 hxge_status_t status = HXGE_OK; 2780 int i, j, index; 2781 uint32_t size, bsize, nblocks, nmsgs; 2782 2783 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2784 "==> hxge_map_rxdma_channel_buf_ring: channel %d", channel)); 2785 2786 dma_bufp = tmp_bufp = *dma_buf_p; 2787 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2788 " hxge_map_rxdma_channel_buf_ring: channel %d to map %d " 2789 "chunks bufp 0x%016llx", channel, num_chunks, dma_bufp)); 2790 2791 nmsgs = 0; 2792 for (i = 0; i < num_chunks; i++, tmp_bufp++) { 2793 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2794 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2795 "bufp 0x%016llx nblocks %d nmsgs %d", 2796 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs)); 2797 nmsgs += tmp_bufp->nblocks; 2798 } 2799 if (!nmsgs) { 2800 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2801 "<== hxge_map_rxdma_channel_buf_ring: channel %d " 2802 "no msg blocks", channel)); 2803 status = HXGE_ERROR; 2804 goto hxge_map_rxdma_channel_buf_ring_exit; 2805 } 2806 rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (rx_rbr_ring_t), KM_SLEEP); 2807 2808 size = nmsgs * sizeof (p_rx_msg_t); 2809 rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP); 2810 ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t), 2811 KM_SLEEP); 2812 2813 MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER, 2814 (void *) hxgep->interrupt_cookie); 2815 MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER, 2816 (void *) hxgep->interrupt_cookie); 2817 2818 rbrp->rdc = channel; 2819 rbrp->num_blocks = num_chunks; 2820 rbrp->tnblocks = nmsgs; 2821 rbrp->rbb_max = nmsgs; 2822 rbrp->rbr_max_size = nmsgs; 2823 rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1); 2824 2825 /* 2826 * Buffer sizes suggested by NIU architect. 256, 512 and 2K. 2827 */ 2828 2829 switch (hxgep->rx_bksize_code) { 2830 case RBR_BKSIZE_4K: 2831 rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B; 2832 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES; 2833 rbrp->hpi_pkt_buf_size0 = SIZE_256B; 2834 break; 2835 case RBR_BKSIZE_8K: 2836 /* Use 512 to avoid possible rcr_full condition */ 2837 rbrp->pkt_buf_size0 = RBR_BUFSZ0_512B; 2838 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_512_BYTES; 2839 rbrp->hpi_pkt_buf_size0 = SIZE_512B; 2840 break; 2841 } 2842 2843 rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K; 2844 rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES; 2845 rbrp->hpi_pkt_buf_size1 = SIZE_1KB; 2846 2847 rbrp->block_size = hxgep->rx_default_block_size; 2848 2849 if (!hxgep->param_arr[param_accept_jumbo].value) { 2850 rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K; 2851 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES; 2852 rbrp->hpi_pkt_buf_size2 = SIZE_2KB; 2853 } else { 2854 rbrp->hpi_pkt_buf_size2 = SIZE_4KB; 2855 rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K; 2856 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES; 2857 } 2858 2859 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2860 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2861 "actual rbr max %d rbb_max %d nmsgs %d " 2862 "rbrp->block_size %d default_block_size %d " 2863 "(config hxge_rbr_size %d hxge_rbr_spare_size %d)", 2864 channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs, 2865 rbrp->block_size, hxgep->rx_default_block_size, 2866 hxge_rbr_size, hxge_rbr_spare_size)); 2867 2868 /* 2869 * Map in buffers from the buffer pool. 2870 * Note that num_blocks is the num_chunks. For Sparc, there is likely 2871 * only one chunk. For x86, there will be many chunks. 2872 * Loop over chunks. 2873 */ 2874 index = 0; 2875 for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) { 2876 bsize = dma_bufp->block_size; 2877 nblocks = dma_bufp->nblocks; 2878 #if defined(__i386) 2879 ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp; 2880 #else 2881 ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp; 2882 #endif 2883 ring_info->buffer[i].buf_index = i; 2884 ring_info->buffer[i].buf_size = dma_bufp->alength; 2885 ring_info->buffer[i].start_index = index; 2886 #if defined(__i386) 2887 ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp; 2888 #else 2889 ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp; 2890 #endif 2891 2892 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2893 " hxge_map_rxdma_channel_buf_ring: map channel %d " 2894 "chunk %d nblocks %d chunk_size %x block_size 0x%x " 2895 "dma_bufp $%p dvma_addr $%p", channel, i, 2896 dma_bufp->nblocks, 2897 ring_info->buffer[i].buf_size, bsize, dma_bufp, 2898 ring_info->buffer[i].dvma_addr)); 2899 2900 /* loop over blocks within a chunk */ 2901 for (j = 0; j < nblocks; j++) { 2902 if ((rx_msg_p = hxge_allocb(bsize, BPRI_LO, 2903 dma_bufp)) == NULL) { 2904 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2905 "allocb failed (index %d i %d j %d)", 2906 index, i, j)); 2907 goto hxge_map_rxdma_channel_buf_ring_fail1; 2908 } 2909 rx_msg_ring[index] = rx_msg_p; 2910 rx_msg_p->block_index = index; 2911 rx_msg_p->shifted_addr = (uint32_t) 2912 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2913 RBR_BKADDR_SHIFT)); 2914 /* 2915 * Too much output 2916 * HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2917 * "index %d j %d rx_msg_p $%p mblk %p", 2918 * index, j, rx_msg_p, rx_msg_p->rx_mblk_p)); 2919 */ 2920 mblk_p = rx_msg_p->rx_mblk_p; 2921 mblk_p->b_wptr = mblk_p->b_rptr + bsize; 2922 2923 rbrp->rbr_ref_cnt++; 2924 index++; 2925 rx_msg_p->buf_dma.dma_channel = channel; 2926 } 2927 } 2928 if (i < rbrp->num_blocks) { 2929 goto hxge_map_rxdma_channel_buf_ring_fail1; 2930 } 2931 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2932 "hxge_map_rxdma_channel_buf_ring: done buf init " 2933 "channel %d msg block entries %d", channel, index)); 2934 ring_info->block_size_mask = bsize - 1; 2935 rbrp->rx_msg_ring = rx_msg_ring; 2936 rbrp->dma_bufp = dma_buf_p; 2937 rbrp->ring_info = ring_info; 2938 2939 status = hxge_rxbuf_index_info_init(hxgep, rbrp); 2940 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, " hxge_map_rxdma_channel_buf_ring: " 2941 "channel %d done buf info init", channel)); 2942 2943 /* 2944 * Finally, permit hxge_freeb() to call hxge_post_page(). 2945 */ 2946 rbrp->rbr_state = RBR_POSTING; 2947 2948 *rbr_p = rbrp; 2949 2950 goto hxge_map_rxdma_channel_buf_ring_exit; 2951 2952 hxge_map_rxdma_channel_buf_ring_fail1: 2953 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2954 " hxge_map_rxdma_channel_buf_ring: failed channel (0x%x)", 2955 channel, status)); 2956 2957 index--; 2958 for (; index >= 0; index--) { 2959 rx_msg_p = rx_msg_ring[index]; 2960 if (rx_msg_p != NULL) { 2961 freeb(rx_msg_p->rx_mblk_p); 2962 rx_msg_ring[index] = NULL; 2963 } 2964 } 2965 2966 hxge_map_rxdma_channel_buf_ring_fail: 2967 MUTEX_DESTROY(&rbrp->post_lock); 2968 MUTEX_DESTROY(&rbrp->lock); 2969 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 2970 KMEM_FREE(rx_msg_ring, size); 2971 KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t)); 2972 2973 status = HXGE_ERROR; 2974 2975 hxge_map_rxdma_channel_buf_ring_exit: 2976 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2977 "<== hxge_map_rxdma_channel_buf_ring status 0x%08x", status)); 2978 2979 return (status); 2980 } 2981 2982 /*ARGSUSED*/ 2983 static void 2984 hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 2985 p_rx_rbr_ring_t rbr_p) 2986 { 2987 p_rx_msg_t *rx_msg_ring; 2988 p_rx_msg_t rx_msg_p; 2989 rxring_info_t *ring_info; 2990 int i; 2991 uint32_t size; 2992 2993 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2994 "==> hxge_unmap_rxdma_channel_buf_ring")); 2995 if (rbr_p == NULL) { 2996 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2997 "<== hxge_unmap_rxdma_channel_buf_ring: NULL rbrp")); 2998 return; 2999 } 3000 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3001 "==> hxge_unmap_rxdma_channel_buf_ring: channel %d", rbr_p->rdc)); 3002 3003 rx_msg_ring = rbr_p->rx_msg_ring; 3004 ring_info = rbr_p->ring_info; 3005 3006 if (rx_msg_ring == NULL || ring_info == NULL) { 3007 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3008 "<== hxge_unmap_rxdma_channel_buf_ring: " 3009 "rx_msg_ring $%p ring_info $%p", rx_msg_p, ring_info)); 3010 return; 3011 } 3012 3013 size = rbr_p->tnblocks * sizeof (p_rx_msg_t); 3014 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3015 " hxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d " 3016 "tnblocks %d (max %d) size ptrs %d ", rbr_p->rdc, rbr_p->num_blocks, 3017 rbr_p->tnblocks, rbr_p->rbr_max_size, size)); 3018 3019 for (i = 0; i < rbr_p->tnblocks; i++) { 3020 rx_msg_p = rx_msg_ring[i]; 3021 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3022 " hxge_unmap_rxdma_channel_buf_ring: " 3023 "rx_msg_p $%p", rx_msg_p)); 3024 if (rx_msg_p != NULL) { 3025 freeb(rx_msg_p->rx_mblk_p); 3026 rx_msg_ring[i] = NULL; 3027 } 3028 } 3029 3030 /* 3031 * We no longer may use the mutex <post_lock>. By setting 3032 * <rbr_state> to anything but POSTING, we prevent 3033 * hxge_post_page() from accessing a dead mutex. 3034 */ 3035 rbr_p->rbr_state = RBR_UNMAPPING; 3036 MUTEX_DESTROY(&rbr_p->post_lock); 3037 3038 MUTEX_DESTROY(&rbr_p->lock); 3039 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 3040 KMEM_FREE(rx_msg_ring, size); 3041 3042 if (rbr_p->rbr_ref_cnt == 0) { 3043 /* This is the normal state of affairs. */ 3044 KMEM_FREE(rbr_p, sizeof (*rbr_p)); 3045 } else { 3046 /* 3047 * Some of our buffers are still being used. 3048 * Therefore, tell hxge_freeb() this ring is 3049 * unmapped, so it may free <rbr_p> for us. 3050 */ 3051 rbr_p->rbr_state = RBR_UNMAPPED; 3052 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3053 "unmap_rxdma_buf_ring: %d %s outstanding.", 3054 rbr_p->rbr_ref_cnt, 3055 rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs")); 3056 } 3057 3058 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3059 "<== hxge_unmap_rxdma_channel_buf_ring")); 3060 } 3061 3062 static hxge_status_t 3063 hxge_rxdma_hw_start_common(p_hxge_t hxgep) 3064 { 3065 hxge_status_t status = HXGE_OK; 3066 3067 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 3068 3069 /* 3070 * Load the sharable parameters by writing to the function zero control 3071 * registers. These FZC registers should be initialized only once for 3072 * the entire chip. 3073 */ 3074 (void) hxge_init_fzc_rx_common(hxgep); 3075 3076 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 3077 3078 return (status); 3079 } 3080 3081 static hxge_status_t 3082 hxge_rxdma_hw_start(p_hxge_t hxgep) 3083 { 3084 int i, ndmas; 3085 uint16_t channel; 3086 p_rx_rbr_rings_t rx_rbr_rings; 3087 p_rx_rbr_ring_t *rbr_rings; 3088 p_rx_rcr_rings_t rx_rcr_rings; 3089 p_rx_rcr_ring_t *rcr_rings; 3090 p_rx_mbox_areas_t rx_mbox_areas_p; 3091 p_rx_mbox_t *rx_mbox_p; 3092 hxge_status_t status = HXGE_OK; 3093 3094 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start")); 3095 3096 rx_rbr_rings = hxgep->rx_rbr_rings; 3097 rx_rcr_rings = hxgep->rx_rcr_rings; 3098 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 3099 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3100 "<== hxge_rxdma_hw_start: NULL ring pointers")); 3101 return (HXGE_ERROR); 3102 } 3103 3104 ndmas = rx_rbr_rings->ndmas; 3105 if (ndmas == 0) { 3106 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3107 "<== hxge_rxdma_hw_start: no dma channel allocated")); 3108 return (HXGE_ERROR); 3109 } 3110 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3111 "==> hxge_rxdma_hw_start (ndmas %d)", ndmas)); 3112 3113 /* 3114 * Scrub the RDC Rx DMA Prefetch Buffer Command. 3115 */ 3116 for (i = 0; i < 128; i++) { 3117 HXGE_REG_WR64(hxgep->hpi_handle, RDC_PREF_CMD, i); 3118 } 3119 3120 /* 3121 * Scrub Rx DMA Shadow Tail Command. 3122 */ 3123 for (i = 0; i < 64; i++) { 3124 HXGE_REG_WR64(hxgep->hpi_handle, RDC_SHADOW_CMD, i); 3125 } 3126 3127 /* 3128 * Scrub Rx DMA Control Fifo Command. 3129 */ 3130 for (i = 0; i < 512; i++) { 3131 HXGE_REG_WR64(hxgep->hpi_handle, RDC_CTRL_FIFO_CMD, i); 3132 } 3133 3134 /* 3135 * Scrub Rx DMA Data Fifo Command. 3136 */ 3137 for (i = 0; i < 1536; i++) { 3138 HXGE_REG_WR64(hxgep->hpi_handle, RDC_DATA_FIFO_CMD, i); 3139 } 3140 3141 /* 3142 * Reset the FIFO Error Stat. 3143 */ 3144 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_STAT, 0xFF); 3145 3146 /* Set the error mask to receive interrupts */ 3147 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 3148 3149 rbr_rings = rx_rbr_rings->rbr_rings; 3150 rcr_rings = rx_rcr_rings->rcr_rings; 3151 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 3152 if (rx_mbox_areas_p) { 3153 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 3154 } 3155 3156 for (i = 0; i < ndmas; i++) { 3157 channel = rbr_rings[i]->rdc; 3158 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3159 "==> hxge_rxdma_hw_start (ndmas %d) channel %d", 3160 ndmas, channel)); 3161 status = hxge_rxdma_start_channel(hxgep, channel, 3162 (p_rx_rbr_ring_t)rbr_rings[i], 3163 (p_rx_rcr_ring_t)rcr_rings[i], 3164 (p_rx_mbox_t)rx_mbox_p[i], rbr_rings[i]->rbb_max); 3165 if (status != HXGE_OK) { 3166 goto hxge_rxdma_hw_start_fail1; 3167 } 3168 } 3169 3170 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start: " 3171 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3172 rx_rbr_rings, rx_rcr_rings)); 3173 goto hxge_rxdma_hw_start_exit; 3174 3175 hxge_rxdma_hw_start_fail1: 3176 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3177 "==> hxge_rxdma_hw_start: disable " 3178 "(status 0x%x channel %d i %d)", status, channel, i)); 3179 for (; i >= 0; i--) { 3180 channel = rbr_rings[i]->rdc; 3181 (void) hxge_rxdma_stop_channel(hxgep, channel); 3182 } 3183 3184 hxge_rxdma_hw_start_exit: 3185 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3186 "==> hxge_rxdma_hw_start: (status 0x%x)", status)); 3187 return (status); 3188 } 3189 3190 static void 3191 hxge_rxdma_hw_stop(p_hxge_t hxgep) 3192 { 3193 int i, ndmas; 3194 uint16_t channel; 3195 p_rx_rbr_rings_t rx_rbr_rings; 3196 p_rx_rbr_ring_t *rbr_rings; 3197 p_rx_rcr_rings_t rx_rcr_rings; 3198 3199 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop")); 3200 3201 rx_rbr_rings = hxgep->rx_rbr_rings; 3202 rx_rcr_rings = hxgep->rx_rcr_rings; 3203 3204 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 3205 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3206 "<== hxge_rxdma_hw_stop: NULL ring pointers")); 3207 return; 3208 } 3209 3210 ndmas = rx_rbr_rings->ndmas; 3211 if (!ndmas) { 3212 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3213 "<== hxge_rxdma_hw_stop: no dma channel allocated")); 3214 return; 3215 } 3216 3217 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3218 "==> hxge_rxdma_hw_stop (ndmas %d)", ndmas)); 3219 3220 rbr_rings = rx_rbr_rings->rbr_rings; 3221 for (i = 0; i < ndmas; i++) { 3222 channel = rbr_rings[i]->rdc; 3223 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3224 "==> hxge_rxdma_hw_stop (ndmas %d) channel %d", 3225 ndmas, channel)); 3226 (void) hxge_rxdma_stop_channel(hxgep, channel); 3227 } 3228 3229 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop: " 3230 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3231 rx_rbr_rings, rx_rcr_rings)); 3232 3233 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_hw_stop")); 3234 } 3235 3236 static hxge_status_t 3237 hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 3238 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 3239 int n_init_kick) 3240 { 3241 hpi_handle_t handle; 3242 hpi_status_t rs = HPI_SUCCESS; 3243 rdc_stat_t cs; 3244 rdc_int_mask_t ent_mask; 3245 hxge_status_t status = HXGE_OK; 3246 3247 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel")); 3248 3249 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3250 3251 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "hxge_rxdma_start_channel: " 3252 "hpi handle addr $%p acc $%p", 3253 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3254 3255 /* Reset RXDMA channel */ 3256 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3257 if (rs != HPI_SUCCESS) { 3258 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3259 "==> hxge_rxdma_start_channel: " 3260 "reset rxdma failed (0x%08x channel %d)", 3261 status, channel)); 3262 return (HXGE_ERROR | rs); 3263 } 3264 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3265 "==> hxge_rxdma_start_channel: reset done: channel %d", channel)); 3266 3267 /* 3268 * Initialize the RXDMA channel specific FZC control configurations. 3269 * These FZC registers are pertaining to each RX channel (logical 3270 * pages). 3271 */ 3272 status = hxge_init_fzc_rxdma_channel(hxgep, 3273 channel, rbr_p, rcr_p, mbox_p); 3274 if (status != HXGE_OK) { 3275 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3276 "==> hxge_rxdma_start_channel: " 3277 "init fzc rxdma failed (0x%08x channel %d)", 3278 status, channel)); 3279 return (status); 3280 } 3281 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3282 "==> hxge_rxdma_start_channel: fzc done")); 3283 3284 /* 3285 * Zero out the shadow and prefetch ram. 3286 */ 3287 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3288 "==> hxge_rxdma_start_channel: ram done")); 3289 3290 /* Set up the interrupt event masks. */ 3291 ent_mask.value = 0; 3292 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3293 if (rs != HPI_SUCCESS) { 3294 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3295 "==> hxge_rxdma_start_channel: " 3296 "init rxdma event masks failed (0x%08x channel %d)", 3297 status, channel)); 3298 return (HXGE_ERROR | rs); 3299 } 3300 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3301 "event done: channel %d (mask 0x%016llx)", 3302 channel, ent_mask.value)); 3303 3304 /* 3305 * Load RXDMA descriptors, buffers, mailbox, initialise the receive DMA 3306 * channels and enable each DMA channel. 3307 */ 3308 status = hxge_enable_rxdma_channel(hxgep, 3309 channel, rbr_p, rcr_p, mbox_p, n_init_kick); 3310 if (status != HXGE_OK) { 3311 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3312 " hxge_rxdma_start_channel: " 3313 " init enable rxdma failed (0x%08x channel %d)", 3314 status, channel)); 3315 return (status); 3316 } 3317 3318 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3319 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3320 3321 /* 3322 * Initialize the receive DMA control and status register 3323 * Note that rdc_stat HAS to be set after RBR and RCR rings are set 3324 */ 3325 cs.value = 0; 3326 cs.bits.mex = 1; 3327 cs.bits.rcr_thres = 1; 3328 cs.bits.rcr_to = 1; 3329 cs.bits.rbr_empty = 1; 3330 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3331 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3332 "channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value)); 3333 if (status != HXGE_OK) { 3334 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3335 "==> hxge_rxdma_start_channel: " 3336 "init rxdma control register failed (0x%08x channel %d", 3337 status, channel)); 3338 return (status); 3339 } 3340 3341 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3342 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3343 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3344 "==> hxge_rxdma_start_channel: enable done")); 3345 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_start_channel")); 3346 3347 return (HXGE_OK); 3348 } 3349 3350 static hxge_status_t 3351 hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel) 3352 { 3353 hpi_handle_t handle; 3354 hpi_status_t rs = HPI_SUCCESS; 3355 rdc_stat_t cs; 3356 rdc_int_mask_t ent_mask; 3357 hxge_status_t status = HXGE_OK; 3358 3359 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel")); 3360 3361 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3362 3363 HXGE_DEBUG_MSG((hxgep, RX_CTL, "hxge_rxdma_stop_channel: " 3364 "hpi handle addr $%p acc $%p", 3365 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3366 3367 /* Reset RXDMA channel */ 3368 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3369 if (rs != HPI_SUCCESS) { 3370 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3371 " hxge_rxdma_stop_channel: " 3372 " reset rxdma failed (0x%08x channel %d)", 3373 rs, channel)); 3374 return (HXGE_ERROR | rs); 3375 } 3376 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3377 "==> hxge_rxdma_stop_channel: reset done")); 3378 3379 /* Set up the interrupt event masks. */ 3380 ent_mask.value = RDC_INT_MASK_ALL; 3381 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3382 if (rs != HPI_SUCCESS) { 3383 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3384 "==> hxge_rxdma_stop_channel: " 3385 "set rxdma event masks failed (0x%08x channel %d)", 3386 rs, channel)); 3387 return (HXGE_ERROR | rs); 3388 } 3389 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3390 "==> hxge_rxdma_stop_channel: event done")); 3391 3392 /* Initialize the receive DMA control and status register */ 3393 cs.value = 0; 3394 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3395 3396 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel: control " 3397 " to default (all 0s) 0x%08x", cs.value)); 3398 3399 if (status != HXGE_OK) { 3400 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3401 " hxge_rxdma_stop_channel: init rxdma" 3402 " control register failed (0x%08x channel %d", 3403 status, channel)); 3404 return (status); 3405 } 3406 3407 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3408 "==> hxge_rxdma_stop_channel: control done")); 3409 3410 /* disable dma channel */ 3411 status = hxge_disable_rxdma_channel(hxgep, channel); 3412 3413 if (status != HXGE_OK) { 3414 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3415 " hxge_rxdma_stop_channel: " 3416 " init enable rxdma failed (0x%08x channel %d)", 3417 status, channel)); 3418 return (status); 3419 } 3420 3421 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3422 "==> hxge_rxdma_stop_channel: disable done")); 3423 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_channel")); 3424 3425 return (HXGE_OK); 3426 } 3427 3428 hxge_status_t 3429 hxge_rxdma_handle_sys_errors(p_hxge_t hxgep) 3430 { 3431 hpi_handle_t handle; 3432 p_hxge_rdc_sys_stats_t statsp; 3433 rdc_fifo_err_stat_t stat; 3434 hxge_status_t status = HXGE_OK; 3435 3436 handle = hxgep->hpi_handle; 3437 statsp = (p_hxge_rdc_sys_stats_t)&hxgep->statsp->rdc_sys_stats; 3438 3439 /* Clear the int_dbg register in case it is an injected err */ 3440 HXGE_REG_WR64(handle, RDC_FIFO_ERR_INT_DBG, 0x0); 3441 3442 /* Get the error status and clear the register */ 3443 HXGE_REG_RD64(handle, RDC_FIFO_ERR_STAT, &stat.value); 3444 HXGE_REG_WR64(handle, RDC_FIFO_ERR_STAT, stat.value); 3445 3446 if (stat.bits.rx_ctrl_fifo_sec) { 3447 statsp->ctrl_fifo_sec++; 3448 if (statsp->ctrl_fifo_sec == 1) 3449 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3450 "==> hxge_rxdma_handle_sys_errors: " 3451 "rx_ctrl_fifo_sec")); 3452 } 3453 3454 if (stat.bits.rx_ctrl_fifo_ded) { 3455 /* Global fatal error encountered */ 3456 statsp->ctrl_fifo_ded++; 3457 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3458 HXGE_FM_EREPORT_RDMC_CTRL_FIFO_DED); 3459 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3460 "==> hxge_rxdma_handle_sys_errors: " 3461 "fatal error: rx_ctrl_fifo_ded error")); 3462 } 3463 3464 if (stat.bits.rx_data_fifo_sec) { 3465 statsp->data_fifo_sec++; 3466 if (statsp->data_fifo_sec == 1) 3467 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3468 "==> hxge_rxdma_handle_sys_errors: " 3469 "rx_data_fifo_sec")); 3470 } 3471 3472 if (stat.bits.rx_data_fifo_ded) { 3473 /* Global fatal error encountered */ 3474 statsp->data_fifo_ded++; 3475 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3476 HXGE_FM_EREPORT_RDMC_DATA_FIFO_DED); 3477 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3478 "==> hxge_rxdma_handle_sys_errors: " 3479 "fatal error: rx_data_fifo_ded error")); 3480 } 3481 3482 if (stat.bits.rx_ctrl_fifo_ded || stat.bits.rx_data_fifo_ded) { 3483 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3484 " hxge_rxdma_handle_sys_errors: fatal error\n")); 3485 status = hxge_rx_port_fatal_err_recover(hxgep); 3486 if (status == HXGE_OK) { 3487 FM_SERVICE_RESTORED(hxgep); 3488 } 3489 } 3490 3491 return (HXGE_OK); 3492 } 3493 3494 static hxge_status_t 3495 hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel) 3496 { 3497 hpi_handle_t handle; 3498 hpi_status_t rs = HPI_SUCCESS; 3499 hxge_status_t status = HXGE_OK; 3500 p_rx_rbr_ring_t rbrp; 3501 p_rx_rcr_ring_t rcrp; 3502 p_rx_mbox_t mboxp; 3503 rdc_int_mask_t ent_mask; 3504 p_hxge_dma_common_t dmap; 3505 int ring_idx; 3506 p_rx_msg_t rx_msg_p; 3507 int i; 3508 uint32_t hxge_port_rcr_size; 3509 uint64_t tmp; 3510 int n_init_kick = 0; 3511 3512 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_fatal_err_recover")); 3513 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3514 "Recovering from RxDMAChannel#%d error...", channel)); 3515 3516 /* 3517 * Stop the dma channel waits for the stop done. If the stop done bit 3518 * is not set, then create an error. 3519 */ 3520 3521 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3522 3523 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Rx DMA stop...")); 3524 3525 ring_idx = hxge_rxdma_get_ring_index(hxgep, channel); 3526 rbrp = (p_rx_rbr_ring_t)hxgep->rx_rbr_rings->rbr_rings[ring_idx]; 3527 rcrp = (p_rx_rcr_ring_t)hxgep->rx_rcr_rings->rcr_rings[ring_idx]; 3528 3529 MUTEX_ENTER(&rcrp->lock); 3530 MUTEX_ENTER(&rbrp->lock); 3531 3532 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA channel...")); 3533 3534 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 3535 if (rs != HPI_SUCCESS) { 3536 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3537 "hxge_disable_rxdma_channel:failed")); 3538 goto fail; 3539 } 3540 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA interrupt...")); 3541 3542 /* Disable interrupt */ 3543 ent_mask.value = RDC_INT_MASK_ALL; 3544 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3545 if (rs != HPI_SUCCESS) { 3546 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3547 "Set rxdma event masks failed (channel %d)", channel)); 3548 } 3549 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel reset...")); 3550 3551 /* Reset RXDMA channel */ 3552 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3553 if (rs != HPI_SUCCESS) { 3554 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3555 "Reset rxdma failed (channel %d)", channel)); 3556 goto fail; 3557 } 3558 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 3559 mboxp = (p_rx_mbox_t)hxgep->rx_mbox_areas_p->rxmbox_areas[ring_idx]; 3560 3561 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 3562 rbrp->rbr_rd_index = 0; 3563 3564 rcrp->comp_rd_index = 0; 3565 rcrp->comp_wt_index = 0; 3566 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 3567 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 3568 #if defined(__i386) 3569 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3570 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3571 #else 3572 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3573 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3574 #endif 3575 3576 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 3577 (hxge_port_rcr_size - 1); 3578 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 3579 (hxge_port_rcr_size - 1); 3580 3581 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc); 3582 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3; 3583 3584 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 3585 bzero((caddr_t)dmap->kaddrp, dmap->alength); 3586 3587 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rbr entries = %d\n", 3588 rbrp->rbr_max_size)); 3589 3590 /* Count the number of buffers owned by the hardware at this moment */ 3591 for (i = 0; i < rbrp->rbr_max_size; i++) { 3592 rx_msg_p = rbrp->rx_msg_ring[i]; 3593 if (rx_msg_p->ref_cnt == 1) { 3594 n_init_kick++; 3595 } 3596 } 3597 3598 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel re-start...")); 3599 3600 /* 3601 * This is error recover! Some buffers are owned by the hardware and 3602 * the rest are owned by the apps. We should only kick in those 3603 * owned by the hardware initially. The apps will post theirs 3604 * eventually. 3605 */ 3606 status = hxge_rxdma_start_channel(hxgep, channel, rbrp, rcrp, mboxp, 3607 n_init_kick); 3608 if (status != HXGE_OK) { 3609 goto fail; 3610 } 3611 3612 /* 3613 * The DMA channel may disable itself automatically. 3614 * The following is a work-around. 3615 */ 3616 HXGE_REG_RD64(handle, RDC_RX_CFG1, &tmp); 3617 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 3618 if (rs != HPI_SUCCESS) { 3619 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3620 "hpi_rxdma_cfg_rdc_enable (channel %d)", channel)); 3621 } 3622 3623 MUTEX_EXIT(&rbrp->lock); 3624 MUTEX_EXIT(&rcrp->lock); 3625 3626 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3627 "Recovery Successful, RxDMAChannel#%d Restored", channel)); 3628 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_fatal_err_recover")); 3629 3630 return (HXGE_OK); 3631 3632 fail: 3633 MUTEX_EXIT(&rbrp->lock); 3634 MUTEX_EXIT(&rcrp->lock); 3635 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3636 3637 return (HXGE_ERROR | rs); 3638 } 3639 3640 static hxge_status_t 3641 hxge_rx_port_fatal_err_recover(p_hxge_t hxgep) 3642 { 3643 hxge_status_t status = HXGE_OK; 3644 p_hxge_dma_common_t *dma_buf_p; 3645 uint16_t channel; 3646 int ndmas; 3647 int i; 3648 block_reset_t reset_reg; 3649 p_rx_rcr_ring_t rcrp; 3650 p_rx_rbr_ring_t rbrp; 3651 3652 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_port_fatal_err_recover")); 3653 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovering from RDC error ...")); 3654 3655 /* Reset RDC block from PEU for this fatal error */ 3656 reset_reg.value = 0; 3657 reset_reg.bits.rdc_rst = 1; 3658 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 3659 3660 /* Disable RxMAC */ 3661 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxMAC...\n")); 3662 if (hxge_rx_vmac_disable(hxgep) != HXGE_OK) 3663 goto fail; 3664 3665 HXGE_DELAY(1000); 3666 3667 /* Restore any common settings after PEU reset */ 3668 if (hxge_rxdma_hw_start_common(hxgep) != HXGE_OK) 3669 goto fail; 3670 3671 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Stop all RxDMA channels...")); 3672 3673 ndmas = hxgep->rx_buf_pool_p->ndmas; 3674 dma_buf_p = hxgep->rx_buf_pool_p->dma_buf_pool_p; 3675 3676 for (i = 0; i < ndmas; i++) { 3677 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 3678 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 3679 rbrp = rcrp->rx_rbr_p; 3680 3681 MUTEX_ENTER(&rbrp->post_lock); 3682 /* This function needs to be inside the post_lock */ 3683 if (hxge_rxdma_fatal_err_recover(hxgep, channel) != HXGE_OK) { 3684 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3685 "Could not recover channel %d", channel)); 3686 } 3687 MUTEX_EXIT(&rbrp->post_lock); 3688 } 3689 3690 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Reset RxMAC...")); 3691 3692 /* Reset RxMAC */ 3693 if (hxge_rx_vmac_reset(hxgep) != HXGE_OK) { 3694 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3695 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3696 goto fail; 3697 } 3698 3699 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-initialize RxMAC...")); 3700 3701 /* Re-Initialize RxMAC */ 3702 if ((status = hxge_rx_vmac_init(hxgep)) != HXGE_OK) { 3703 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3704 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3705 goto fail; 3706 } 3707 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-enable RxMAC...")); 3708 3709 /* Re-enable RxMAC */ 3710 if ((status = hxge_rx_vmac_enable(hxgep)) != HXGE_OK) { 3711 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3712 "hxge_rx_port_fatal_err_recover: Failed to enable RxMAC")); 3713 goto fail; 3714 } 3715 3716 /* Reset the error mask since PEU reset cleared it */ 3717 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 3718 3719 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3720 "Recovery Successful, RxPort Restored")); 3721 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_port_fatal_err_recover")); 3722 3723 return (HXGE_OK); 3724 fail: 3725 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3726 return (status); 3727 } 3728 3729 static void 3730 hxge_rbr_empty_restore(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p) 3731 { 3732 hpi_status_t hpi_status; 3733 hxge_status_t status; 3734 int i; 3735 p_hxge_rx_ring_stats_t rdc_stats; 3736 3737 rdc_stats = &hxgep->statsp->rdc_stats[rx_rbr_p->rdc]; 3738 rdc_stats->rbr_empty_restore++; 3739 rx_rbr_p->rbr_is_empty = B_FALSE; 3740 3741 /* 3742 * Complete the processing for the RBR Empty by: 3743 * 0) kicking back HXGE_RBR_EMPTY_THRESHOLD 3744 * packets. 3745 * 1) Disable the RX vmac. 3746 * 2) Re-enable the affected DMA channel. 3747 * 3) Re-enable the RX vmac. 3748 */ 3749 3750 /* 3751 * Disable the RX VMAC, but setting the framelength 3752 * to 0, since there is a hardware bug when disabling 3753 * the vmac. 3754 */ 3755 MUTEX_ENTER(hxgep->genlock); 3756 (void) hpi_vmac_rx_set_framesize( 3757 HXGE_DEV_HPI_HANDLE(hxgep), (uint16_t)0); 3758 3759 hpi_status = hpi_rxdma_cfg_rdc_enable( 3760 HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc); 3761 if (hpi_status != HPI_SUCCESS) { 3762 rdc_stats->rbr_empty_fail++; 3763 3764 /* Assume we are already inside the post_lock */ 3765 status = hxge_rxdma_fatal_err_recover(hxgep, rx_rbr_p->rdc); 3766 if (status != HXGE_OK) { 3767 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3768 "hxge(%d): channel(%d) is empty.", 3769 hxgep->instance, rx_rbr_p->rdc)); 3770 } 3771 } 3772 3773 for (i = 0; i < 1024; i++) { 3774 uint64_t value; 3775 RXDMA_REG_READ64(HXGE_DEV_HPI_HANDLE(hxgep), 3776 RDC_STAT, i & 3, &value); 3777 } 3778 3779 /* 3780 * Re-enable the RX VMAC. 3781 */ 3782 (void) hpi_vmac_rx_set_framesize(HXGE_DEV_HPI_HANDLE(hxgep), 3783 (uint16_t)hxgep->vmac.maxframesize); 3784 MUTEX_EXIT(hxgep->genlock); 3785 } 3786