1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <hxge_impl.h> 27 #include <hxge_rxdma.h> 28 29 /* 30 * Number of blocks to accumulate before re-enabling DMA 31 * when we get RBR empty. 32 */ 33 #define HXGE_RBR_EMPTY_THRESHOLD 64 34 35 /* 36 * Globals: tunable parameters (/etc/system or adb) 37 * 38 */ 39 extern uint32_t hxge_rbr_size; 40 extern uint32_t hxge_rcr_size; 41 extern uint32_t hxge_rbr_spare_size; 42 extern uint32_t hxge_mblks_pending; 43 44 /* 45 * Tunable to reduce the amount of time spent in the 46 * ISR doing Rx Processing. 47 */ 48 extern uint32_t hxge_max_rx_pkts; 49 50 /* 51 * Tunables to manage the receive buffer blocks. 52 * 53 * hxge_rx_threshold_hi: copy all buffers. 54 * hxge_rx_bcopy_size_type: receive buffer block size type. 55 * hxge_rx_threshold_lo: copy only up to tunable block size type. 56 */ 57 extern hxge_rxbuf_threshold_t hxge_rx_threshold_hi; 58 extern hxge_rxbuf_type_t hxge_rx_buf_size_type; 59 extern hxge_rxbuf_threshold_t hxge_rx_threshold_lo; 60 61 /* 62 * Static local functions. 63 */ 64 static hxge_status_t hxge_map_rxdma(p_hxge_t hxgep); 65 static void hxge_unmap_rxdma(p_hxge_t hxgep); 66 static hxge_status_t hxge_rxdma_hw_start_common(p_hxge_t hxgep); 67 static hxge_status_t hxge_rxdma_hw_start(p_hxge_t hxgep); 68 static void hxge_rxdma_hw_stop(p_hxge_t hxgep); 69 static hxge_status_t hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 70 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 71 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p, 72 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 73 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 74 static void hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 75 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 76 static hxge_status_t hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, 77 uint16_t dma_channel, p_hxge_dma_common_t *dma_rbr_cntl_p, 78 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 79 p_rx_rbr_ring_t *rbr_p, p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 80 static void hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 81 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 82 static hxge_status_t hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, 83 uint16_t channel, p_hxge_dma_common_t *dma_buf_p, 84 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks); 85 static void hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 86 p_rx_rbr_ring_t rbr_p); 87 static hxge_status_t hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 88 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 89 int n_init_kick); 90 static hxge_status_t hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel); 91 static mblk_t *hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 92 p_rx_rcr_ring_t *rcr_p, rdc_stat_t cs); 93 static void hxge_receive_packet(p_hxge_t hxgep, p_rx_rcr_ring_t rcr_p, 94 p_rcr_entry_t rcr_desc_rd_head_p, boolean_t *multi_p, 95 mblk_t ** mp, mblk_t ** mp_cont, uint32_t *invalid_rcr_entry); 96 static hxge_status_t hxge_disable_rxdma_channel(p_hxge_t hxgep, 97 uint16_t channel); 98 static p_rx_msg_t hxge_allocb(size_t, uint32_t, p_hxge_dma_common_t); 99 static void hxge_freeb(p_rx_msg_t); 100 static void hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, 101 p_hxge_ldv_t ldvp, rdc_stat_t cs); 102 static hxge_status_t hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, 103 p_hxge_ldv_t ldvp, rdc_stat_t cs); 104 static hxge_status_t hxge_rxbuf_index_info_init(p_hxge_t hxgep, 105 p_rx_rbr_ring_t rx_dmap); 106 static hxge_status_t hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, 107 uint16_t channel); 108 static hxge_status_t hxge_rx_port_fatal_err_recover(p_hxge_t hxgep); 109 static void hxge_rbr_empty_restore(p_hxge_t hxgep, 110 p_rx_rbr_ring_t rx_rbr_p); 111 112 hxge_status_t 113 hxge_init_rxdma_channels(p_hxge_t hxgep) 114 { 115 hxge_status_t status = HXGE_OK; 116 block_reset_t reset_reg; 117 int i; 118 119 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_init_rxdma_channels")); 120 121 for (i = 0; i < HXGE_MAX_RDCS; i++) 122 hxgep->rdc_first_intr[i] = B_TRUE; 123 124 /* Reset RDC block from PEU to clear any previous state */ 125 reset_reg.value = 0; 126 reset_reg.bits.rdc_rst = 1; 127 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 128 HXGE_DELAY(1000); 129 130 status = hxge_map_rxdma(hxgep); 131 if (status != HXGE_OK) { 132 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 133 "<== hxge_init_rxdma: status 0x%x", status)); 134 return (status); 135 } 136 137 status = hxge_rxdma_hw_start_common(hxgep); 138 if (status != HXGE_OK) { 139 hxge_unmap_rxdma(hxgep); 140 } 141 142 status = hxge_rxdma_hw_start(hxgep); 143 if (status != HXGE_OK) { 144 hxge_unmap_rxdma(hxgep); 145 } 146 147 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 148 "<== hxge_init_rxdma_channels: status 0x%x", status)); 149 return (status); 150 } 151 152 void 153 hxge_uninit_rxdma_channels(p_hxge_t hxgep) 154 { 155 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_uninit_rxdma_channels")); 156 157 hxge_rxdma_hw_stop(hxgep); 158 hxge_unmap_rxdma(hxgep); 159 160 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_uinit_rxdma_channels")); 161 } 162 163 hxge_status_t 164 hxge_init_rxdma_channel_cntl_stat(p_hxge_t hxgep, uint16_t channel, 165 rdc_stat_t *cs_p) 166 { 167 hpi_handle_t handle; 168 hpi_status_t rs = HPI_SUCCESS; 169 hxge_status_t status = HXGE_OK; 170 171 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 172 "<== hxge_init_rxdma_channel_cntl_stat")); 173 174 handle = HXGE_DEV_HPI_HANDLE(hxgep); 175 rs = hpi_rxdma_control_status(handle, OP_SET, channel, cs_p); 176 177 if (rs != HPI_SUCCESS) { 178 status = HXGE_ERROR | rs; 179 } 180 return (status); 181 } 182 183 184 hxge_status_t 185 hxge_enable_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 186 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 187 int n_init_kick) 188 { 189 hpi_handle_t handle; 190 rdc_desc_cfg_t rdc_desc; 191 rdc_rcr_cfg_b_t *cfgb_p; 192 hpi_status_t rs = HPI_SUCCESS; 193 194 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel")); 195 handle = HXGE_DEV_HPI_HANDLE(hxgep); 196 197 /* 198 * Use configuration data composed at init time. Write to hardware the 199 * receive ring configurations. 200 */ 201 rdc_desc.mbox_enable = 1; 202 rdc_desc.mbox_addr = mbox_p->mbox_addr; 203 HXGE_DEBUG_MSG((hxgep, RX_CTL, 204 "==> hxge_enable_rxdma_channel: mboxp $%p($%p)", 205 mbox_p->mbox_addr, rdc_desc.mbox_addr)); 206 207 rdc_desc.rbr_len = rbr_p->rbb_max; 208 rdc_desc.rbr_addr = rbr_p->rbr_addr; 209 210 switch (hxgep->rx_bksize_code) { 211 case RBR_BKSIZE_4K: 212 rdc_desc.page_size = SIZE_4KB; 213 break; 214 case RBR_BKSIZE_8K: 215 rdc_desc.page_size = SIZE_8KB; 216 break; 217 } 218 219 rdc_desc.size0 = rbr_p->hpi_pkt_buf_size0; 220 rdc_desc.valid0 = 1; 221 222 rdc_desc.size1 = rbr_p->hpi_pkt_buf_size1; 223 rdc_desc.valid1 = 1; 224 225 rdc_desc.size2 = rbr_p->hpi_pkt_buf_size2; 226 rdc_desc.valid2 = 1; 227 228 rdc_desc.full_hdr = rcr_p->full_hdr_flag; 229 rdc_desc.offset = rcr_p->sw_priv_hdr_len; 230 231 rdc_desc.rcr_len = rcr_p->comp_size; 232 rdc_desc.rcr_addr = rcr_p->rcr_addr; 233 234 cfgb_p = &(rcr_p->rcr_cfgb); 235 rdc_desc.rcr_threshold = cfgb_p->bits.pthres; 236 rdc_desc.rcr_timeout = cfgb_p->bits.timeout; 237 rdc_desc.rcr_timeout_enable = cfgb_p->bits.entout; 238 239 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 240 "rbr_len qlen %d pagesize code %d rcr_len %d", 241 rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len)); 242 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 243 "size 0 %d size 1 %d size 2 %d", 244 rbr_p->hpi_pkt_buf_size0, rbr_p->hpi_pkt_buf_size1, 245 rbr_p->hpi_pkt_buf_size2)); 246 247 rs = hpi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc); 248 if (rs != HPI_SUCCESS) { 249 return (HXGE_ERROR | rs); 250 } 251 252 /* 253 * Enable the timeout and threshold. 254 */ 255 rs = hpi_rxdma_cfg_rdc_rcr_threshold(handle, channel, 256 rdc_desc.rcr_threshold); 257 if (rs != HPI_SUCCESS) { 258 return (HXGE_ERROR | rs); 259 } 260 261 rs = hpi_rxdma_cfg_rdc_rcr_timeout(handle, channel, 262 rdc_desc.rcr_timeout); 263 if (rs != HPI_SUCCESS) { 264 return (HXGE_ERROR | rs); 265 } 266 267 /* Enable the DMA */ 268 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 269 if (rs != HPI_SUCCESS) { 270 return (HXGE_ERROR | rs); 271 } 272 273 /* Kick the DMA engine */ 274 hpi_rxdma_rdc_rbr_kick(handle, channel, n_init_kick); 275 276 /* Clear the rbr empty bit */ 277 (void) hpi_rxdma_channel_rbr_empty_clear(handle, channel); 278 279 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_enable_rxdma_channel")); 280 281 return (HXGE_OK); 282 } 283 284 static hxge_status_t 285 hxge_disable_rxdma_channel(p_hxge_t hxgep, uint16_t channel) 286 { 287 hpi_handle_t handle; 288 hpi_status_t rs = HPI_SUCCESS; 289 290 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_disable_rxdma_channel")); 291 292 handle = HXGE_DEV_HPI_HANDLE(hxgep); 293 294 /* disable the DMA */ 295 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 296 if (rs != HPI_SUCCESS) { 297 HXGE_DEBUG_MSG((hxgep, RX_CTL, 298 "<== hxge_disable_rxdma_channel:failed (0x%x)", rs)); 299 return (HXGE_ERROR | rs); 300 } 301 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_disable_rxdma_channel")); 302 return (HXGE_OK); 303 } 304 305 hxge_status_t 306 hxge_rxdma_channel_rcrflush(p_hxge_t hxgep, uint8_t channel) 307 { 308 hpi_handle_t handle; 309 hxge_status_t status = HXGE_OK; 310 311 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 312 "==> hxge_rxdma_channel_rcrflush")); 313 314 handle = HXGE_DEV_HPI_HANDLE(hxgep); 315 hpi_rxdma_rdc_rcr_flush(handle, channel); 316 317 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 318 "<== hxge_rxdma_channel_rcrflush")); 319 return (status); 320 321 } 322 323 #define MID_INDEX(l, r) ((r + l + 1) >> 1) 324 325 #define TO_LEFT -1 326 #define TO_RIGHT 1 327 #define BOTH_RIGHT (TO_RIGHT + TO_RIGHT) 328 #define BOTH_LEFT (TO_LEFT + TO_LEFT) 329 #define IN_MIDDLE (TO_RIGHT + TO_LEFT) 330 #define NO_HINT 0xffffffff 331 332 /*ARGSUSED*/ 333 hxge_status_t 334 hxge_rxbuf_pp_to_vp(p_hxge_t hxgep, p_rx_rbr_ring_t rbr_p, 335 uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp, 336 uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index) 337 { 338 int bufsize; 339 uint64_t pktbuf_pp; 340 uint64_t dvma_addr; 341 rxring_info_t *ring_info; 342 int base_side, end_side; 343 int r_index, l_index, anchor_index; 344 int found, search_done; 345 uint32_t offset, chunk_size, block_size, page_size_mask; 346 uint32_t chunk_index, block_index, total_index; 347 int max_iterations, iteration; 348 rxbuf_index_info_t *bufinfo; 349 350 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_rxbuf_pp_to_vp")); 351 352 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 353 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d", 354 pkt_buf_addr_pp, pktbufsz_type)); 355 356 #if defined(__i386) 357 pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp; 358 #else 359 pktbuf_pp = (uint64_t)pkt_buf_addr_pp; 360 #endif 361 362 switch (pktbufsz_type) { 363 case 0: 364 bufsize = rbr_p->pkt_buf_size0; 365 break; 366 case 1: 367 bufsize = rbr_p->pkt_buf_size1; 368 break; 369 case 2: 370 bufsize = rbr_p->pkt_buf_size2; 371 break; 372 case RCR_SINGLE_BLOCK: 373 bufsize = 0; 374 anchor_index = 0; 375 break; 376 default: 377 return (HXGE_ERROR); 378 } 379 380 if (rbr_p->num_blocks == 1) { 381 anchor_index = 0; 382 ring_info = rbr_p->ring_info; 383 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 384 385 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 386 "==> hxge_rxbuf_pp_to_vp: (found, 1 block) " 387 "buf_pp $%p btype %d anchor_index %d bufinfo $%p", 388 pkt_buf_addr_pp, pktbufsz_type, anchor_index, bufinfo)); 389 390 goto found_index; 391 } 392 393 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 394 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d anchor_index %d", 395 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 396 397 ring_info = rbr_p->ring_info; 398 found = B_FALSE; 399 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 400 iteration = 0; 401 max_iterations = ring_info->max_iterations; 402 403 /* 404 * First check if this block have been seen recently. This is indicated 405 * by a hint which is initialized when the first buffer of the block is 406 * seen. The hint is reset when the last buffer of the block has been 407 * processed. As three block sizes are supported, three hints are kept. 408 * The idea behind the hints is that once the hardware uses a block 409 * for a buffer of that size, it will use it exclusively for that size 410 * and will use it until it is exhausted. It is assumed that there 411 * would a single block being used for the same buffer sizes at any 412 * given time. 413 */ 414 if (ring_info->hint[pktbufsz_type] != NO_HINT) { 415 anchor_index = ring_info->hint[pktbufsz_type]; 416 dvma_addr = bufinfo[anchor_index].dvma_addr; 417 chunk_size = bufinfo[anchor_index].buf_size; 418 if ((pktbuf_pp >= dvma_addr) && 419 (pktbuf_pp < (dvma_addr + chunk_size))) { 420 found = B_TRUE; 421 /* 422 * check if this is the last buffer in the block If so, 423 * then reset the hint for the size; 424 */ 425 426 if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size)) 427 ring_info->hint[pktbufsz_type] = NO_HINT; 428 } 429 } 430 431 if (found == B_FALSE) { 432 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 433 "==> hxge_rxbuf_pp_to_vp: (!found)" 434 "buf_pp $%p btype %d anchor_index %d", 435 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 436 437 /* 438 * This is the first buffer of the block of this size. Need to 439 * search the whole information array. the search algorithm 440 * uses a binary tree search algorithm. It assumes that the 441 * information is already sorted with increasing order info[0] 442 * < info[1] < info[2] .... < info[n-1] where n is the size of 443 * the information array 444 */ 445 r_index = rbr_p->num_blocks - 1; 446 l_index = 0; 447 search_done = B_FALSE; 448 anchor_index = MID_INDEX(r_index, l_index); 449 while (search_done == B_FALSE) { 450 if ((r_index == l_index) || 451 (iteration >= max_iterations)) 452 search_done = B_TRUE; 453 454 end_side = TO_RIGHT; /* to the right */ 455 base_side = TO_LEFT; /* to the left */ 456 /* read the DVMA address information and sort it */ 457 dvma_addr = bufinfo[anchor_index].dvma_addr; 458 chunk_size = bufinfo[anchor_index].buf_size; 459 460 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 461 "==> hxge_rxbuf_pp_to_vp: (searching)" 462 "buf_pp $%p btype %d " 463 "anchor_index %d chunk_size %d dvmaaddr $%p", 464 pkt_buf_addr_pp, pktbufsz_type, anchor_index, 465 chunk_size, dvma_addr)); 466 467 if (pktbuf_pp >= dvma_addr) 468 base_side = TO_RIGHT; /* to the right */ 469 if (pktbuf_pp < (dvma_addr + chunk_size)) 470 end_side = TO_LEFT; /* to the left */ 471 472 switch (base_side + end_side) { 473 case IN_MIDDLE: 474 /* found */ 475 found = B_TRUE; 476 search_done = B_TRUE; 477 if ((pktbuf_pp + bufsize) < 478 (dvma_addr + chunk_size)) 479 ring_info->hint[pktbufsz_type] = 480 bufinfo[anchor_index].buf_index; 481 break; 482 case BOTH_RIGHT: 483 /* not found: go to the right */ 484 l_index = anchor_index + 1; 485 anchor_index = MID_INDEX(r_index, l_index); 486 break; 487 488 case BOTH_LEFT: 489 /* not found: go to the left */ 490 r_index = anchor_index - 1; 491 anchor_index = MID_INDEX(r_index, l_index); 492 break; 493 default: /* should not come here */ 494 return (HXGE_ERROR); 495 } 496 iteration++; 497 } 498 499 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 500 "==> hxge_rxbuf_pp_to_vp: (search done)" 501 "buf_pp $%p btype %d anchor_index %d", 502 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 503 } 504 505 if (found == B_FALSE) { 506 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 507 "==> hxge_rxbuf_pp_to_vp: (search failed)" 508 "buf_pp $%p btype %d anchor_index %d", 509 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 510 return (HXGE_ERROR); 511 } 512 513 found_index: 514 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 515 "==> hxge_rxbuf_pp_to_vp: (FOUND1)" 516 "buf_pp $%p btype %d bufsize %d anchor_index %d", 517 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index)); 518 519 /* index of the first block in this chunk */ 520 chunk_index = bufinfo[anchor_index].start_index; 521 dvma_addr = bufinfo[anchor_index].dvma_addr; 522 page_size_mask = ring_info->block_size_mask; 523 524 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 525 "==> hxge_rxbuf_pp_to_vp: (FOUND3), get chunk)" 526 "buf_pp $%p btype %d bufsize %d " 527 "anchor_index %d chunk_index %d dvma $%p", 528 pkt_buf_addr_pp, pktbufsz_type, bufsize, 529 anchor_index, chunk_index, dvma_addr)); 530 531 offset = pktbuf_pp - dvma_addr; /* offset within the chunk */ 532 block_size = rbr_p->block_size; /* System block(page) size */ 533 534 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 535 "==> hxge_rxbuf_pp_to_vp: (FOUND4), get chunk)" 536 "buf_pp $%p btype %d bufsize %d " 537 "anchor_index %d chunk_index %d dvma $%p " 538 "offset %d block_size %d", 539 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index, 540 chunk_index, dvma_addr, offset, block_size)); 541 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> getting total index")); 542 543 block_index = (offset / block_size); /* index within chunk */ 544 total_index = chunk_index + block_index; 545 546 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 547 "==> hxge_rxbuf_pp_to_vp: " 548 "total_index %d dvma_addr $%p " 549 "offset %d block_size %d " 550 "block_index %d ", 551 total_index, dvma_addr, offset, block_size, block_index)); 552 553 #if defined(__i386) 554 *pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr + 555 (uint32_t)offset); 556 #else 557 *pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr + 558 offset); 559 #endif 560 561 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 562 "==> hxge_rxbuf_pp_to_vp: " 563 "total_index %d dvma_addr $%p " 564 "offset %d block_size %d " 565 "block_index %d " 566 "*pkt_buf_addr_p $%p", 567 total_index, dvma_addr, offset, block_size, 568 block_index, *pkt_buf_addr_p)); 569 570 *msg_index = total_index; 571 *bufoffset = (offset & page_size_mask); 572 573 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 574 "==> hxge_rxbuf_pp_to_vp: get msg index: " 575 "msg_index %d bufoffset_index %d", 576 *msg_index, *bufoffset)); 577 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "<== hxge_rxbuf_pp_to_vp")); 578 579 return (HXGE_OK); 580 } 581 582 583 /* 584 * used by quick sort (qsort) function 585 * to perform comparison 586 */ 587 static int 588 hxge_sort_compare(const void *p1, const void *p2) 589 { 590 591 rxbuf_index_info_t *a, *b; 592 593 a = (rxbuf_index_info_t *)p1; 594 b = (rxbuf_index_info_t *)p2; 595 596 if (a->dvma_addr > b->dvma_addr) 597 return (1); 598 if (a->dvma_addr < b->dvma_addr) 599 return (-1); 600 return (0); 601 } 602 603 /* 604 * Grabbed this sort implementation from common/syscall/avl.c 605 * 606 * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified. 607 * v = Ptr to array/vector of objs 608 * n = # objs in the array 609 * s = size of each obj (must be multiples of a word size) 610 * f = ptr to function to compare two objs 611 * returns (-1 = less than, 0 = equal, 1 = greater than 612 */ 613 void 614 hxge_ksort(caddr_t v, int n, int s, int (*f) ()) 615 { 616 int g, i, j, ii; 617 unsigned int *p1, *p2; 618 unsigned int tmp; 619 620 /* No work to do */ 621 if (v == NULL || n <= 1) 622 return; 623 /* Sanity check on arguments */ 624 ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0); 625 ASSERT(s > 0); 626 627 for (g = n / 2; g > 0; g /= 2) { 628 for (i = g; i < n; i++) { 629 for (j = i - g; j >= 0 && 630 (*f) (v + j * s, v + (j + g) * s) == 1; j -= g) { 631 p1 = (unsigned *)(v + j * s); 632 p2 = (unsigned *)(v + (j + g) * s); 633 for (ii = 0; ii < s / 4; ii++) { 634 tmp = *p1; 635 *p1++ = *p2; 636 *p2++ = tmp; 637 } 638 } 639 } 640 } 641 } 642 643 /* 644 * Initialize data structures required for rxdma 645 * buffer dvma->vmem address lookup 646 */ 647 /*ARGSUSED*/ 648 static hxge_status_t 649 hxge_rxbuf_index_info_init(p_hxge_t hxgep, p_rx_rbr_ring_t rbrp) 650 { 651 int index; 652 rxring_info_t *ring_info; 653 int max_iteration = 0, max_index = 0; 654 655 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_rxbuf_index_info_init")); 656 657 ring_info = rbrp->ring_info; 658 ring_info->hint[0] = NO_HINT; 659 ring_info->hint[1] = NO_HINT; 660 ring_info->hint[2] = NO_HINT; 661 max_index = rbrp->num_blocks; 662 663 /* read the DVMA address information and sort it */ 664 /* do init of the information array */ 665 666 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 667 " hxge_rxbuf_index_info_init Sort ptrs")); 668 669 /* sort the array */ 670 hxge_ksort((void *) ring_info->buffer, max_index, 671 sizeof (rxbuf_index_info_t), hxge_sort_compare); 672 673 for (index = 0; index < max_index; index++) { 674 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 675 " hxge_rxbuf_index_info_init: sorted chunk %d " 676 " ioaddr $%p kaddr $%p size %x", 677 index, ring_info->buffer[index].dvma_addr, 678 ring_info->buffer[index].kaddr, 679 ring_info->buffer[index].buf_size)); 680 } 681 682 max_iteration = 0; 683 while (max_index >= (1ULL << max_iteration)) 684 max_iteration++; 685 ring_info->max_iterations = max_iteration + 1; 686 687 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 688 " hxge_rxbuf_index_info_init Find max iter %d", 689 ring_info->max_iterations)); 690 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_rxbuf_index_info_init")); 691 692 return (HXGE_OK); 693 } 694 695 /*ARGSUSED*/ 696 void 697 hxge_dump_rcr_entry(p_hxge_t hxgep, p_rcr_entry_t entry_p) 698 { 699 #ifdef HXGE_DEBUG 700 701 uint32_t bptr; 702 uint64_t pp; 703 704 bptr = entry_p->bits.pkt_buf_addr; 705 706 HXGE_DEBUG_MSG((hxgep, RX_CTL, 707 "\trcr entry $%p " 708 "\trcr entry 0x%0llx " 709 "\trcr entry 0x%08x " 710 "\trcr entry 0x%08x " 711 "\tvalue 0x%0llx\n" 712 "\tmulti = %d\n" 713 "\tpkt_type = 0x%x\n" 714 "\terror = 0x%04x\n" 715 "\tl2_len = %d\n" 716 "\tpktbufsize = %d\n" 717 "\tpkt_buf_addr = $%p\n" 718 "\tpkt_buf_addr (<< 6) = $%p\n", 719 entry_p, 720 *(int64_t *)entry_p, 721 *(int32_t *)entry_p, 722 *(int32_t *)((char *)entry_p + 32), 723 entry_p->value, 724 entry_p->bits.multi, 725 entry_p->bits.pkt_type, 726 entry_p->bits.error, 727 entry_p->bits.l2_len, 728 entry_p->bits.pktbufsz, 729 bptr, 730 entry_p->bits.pkt_buf_addr_l)); 731 732 pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) << 733 RCR_PKT_BUF_ADDR_SHIFT; 734 735 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rcr pp 0x%llx l2 len %d", 736 pp, (*(int64_t *)entry_p >> 40) & 0x3fff)); 737 #endif 738 } 739 740 /*ARGSUSED*/ 741 void 742 hxge_rxdma_stop(p_hxge_t hxgep) 743 { 744 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop")); 745 746 (void) hxge_rx_vmac_disable(hxgep); 747 (void) hxge_rxdma_hw_mode(hxgep, HXGE_DMA_STOP); 748 749 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop")); 750 } 751 752 void 753 hxge_rxdma_stop_reinit(p_hxge_t hxgep) 754 { 755 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_reinit")); 756 757 (void) hxge_rxdma_stop(hxgep); 758 (void) hxge_uninit_rxdma_channels(hxgep); 759 (void) hxge_init_rxdma_channels(hxgep); 760 761 (void) hxge_rx_vmac_enable(hxgep); 762 763 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_reinit")); 764 } 765 766 hxge_status_t 767 hxge_rxdma_hw_mode(p_hxge_t hxgep, boolean_t enable) 768 { 769 int i, ndmas; 770 uint16_t channel; 771 p_rx_rbr_rings_t rx_rbr_rings; 772 p_rx_rbr_ring_t *rbr_rings; 773 hpi_handle_t handle; 774 hpi_status_t rs = HPI_SUCCESS; 775 hxge_status_t status = HXGE_OK; 776 777 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 778 "==> hxge_rxdma_hw_mode: mode %d", enable)); 779 780 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 781 HXGE_DEBUG_MSG((hxgep, RX_CTL, 782 "<== hxge_rxdma_mode: not initialized")); 783 return (HXGE_ERROR); 784 } 785 786 rx_rbr_rings = hxgep->rx_rbr_rings; 787 if (rx_rbr_rings == NULL) { 788 HXGE_DEBUG_MSG((hxgep, RX_CTL, 789 "<== hxge_rxdma_mode: NULL ring pointer")); 790 return (HXGE_ERROR); 791 } 792 793 if (rx_rbr_rings->rbr_rings == NULL) { 794 HXGE_DEBUG_MSG((hxgep, RX_CTL, 795 "<== hxge_rxdma_mode: NULL rbr rings pointer")); 796 return (HXGE_ERROR); 797 } 798 799 ndmas = rx_rbr_rings->ndmas; 800 if (!ndmas) { 801 HXGE_DEBUG_MSG((hxgep, RX_CTL, 802 "<== hxge_rxdma_mode: no channel")); 803 return (HXGE_ERROR); 804 } 805 806 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 807 "==> hxge_rxdma_mode (ndmas %d)", ndmas)); 808 809 rbr_rings = rx_rbr_rings->rbr_rings; 810 811 handle = HXGE_DEV_HPI_HANDLE(hxgep); 812 813 for (i = 0; i < ndmas; i++) { 814 if (rbr_rings == NULL || rbr_rings[i] == NULL) { 815 continue; 816 } 817 channel = rbr_rings[i]->rdc; 818 if (enable) { 819 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 820 "==> hxge_rxdma_hw_mode: channel %d (enable)", 821 channel)); 822 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 823 } else { 824 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 825 "==> hxge_rxdma_hw_mode: channel %d (disable)", 826 channel)); 827 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 828 } 829 } 830 831 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs); 832 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 833 "<== hxge_rxdma_hw_mode: status 0x%x", status)); 834 835 return (status); 836 } 837 838 int 839 hxge_rxdma_get_ring_index(p_hxge_t hxgep, uint16_t channel) 840 { 841 int i, ndmas; 842 uint16_t rdc; 843 p_rx_rbr_rings_t rx_rbr_rings; 844 p_rx_rbr_ring_t *rbr_rings; 845 846 HXGE_DEBUG_MSG((hxgep, RX_CTL, 847 "==> hxge_rxdma_get_ring_index: channel %d", channel)); 848 849 rx_rbr_rings = hxgep->rx_rbr_rings; 850 if (rx_rbr_rings == NULL) { 851 HXGE_DEBUG_MSG((hxgep, RX_CTL, 852 "<== hxge_rxdma_get_ring_index: NULL ring pointer")); 853 return (-1); 854 } 855 856 ndmas = rx_rbr_rings->ndmas; 857 if (!ndmas) { 858 HXGE_DEBUG_MSG((hxgep, RX_CTL, 859 "<== hxge_rxdma_get_ring_index: no channel")); 860 return (-1); 861 } 862 863 HXGE_DEBUG_MSG((hxgep, RX_CTL, 864 "==> hxge_rxdma_get_ring_index (ndmas %d)", ndmas)); 865 866 rbr_rings = rx_rbr_rings->rbr_rings; 867 for (i = 0; i < ndmas; i++) { 868 rdc = rbr_rings[i]->rdc; 869 if (channel == rdc) { 870 HXGE_DEBUG_MSG((hxgep, RX_CTL, 871 "==> hxge_rxdma_get_rbr_ring: " 872 "channel %d (index %d) " 873 "ring %d", channel, i, rbr_rings[i])); 874 875 return (i); 876 } 877 } 878 879 HXGE_DEBUG_MSG((hxgep, RX_CTL, 880 "<== hxge_rxdma_get_rbr_ring_index: not found")); 881 882 return (-1); 883 } 884 885 /* 886 * Static functions start here. 887 */ 888 static p_rx_msg_t 889 hxge_allocb(size_t size, uint32_t pri, p_hxge_dma_common_t dmabuf_p) 890 { 891 p_rx_msg_t hxge_mp = NULL; 892 p_hxge_dma_common_t dmamsg_p; 893 uchar_t *buffer; 894 895 hxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP); 896 if (hxge_mp == NULL) { 897 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 898 "Allocation of a rx msg failed.")); 899 goto hxge_allocb_exit; 900 } 901 902 hxge_mp->use_buf_pool = B_FALSE; 903 if (dmabuf_p) { 904 hxge_mp->use_buf_pool = B_TRUE; 905 906 dmamsg_p = (p_hxge_dma_common_t)&hxge_mp->buf_dma; 907 *dmamsg_p = *dmabuf_p; 908 dmamsg_p->nblocks = 1; 909 dmamsg_p->block_size = size; 910 dmamsg_p->alength = size; 911 buffer = (uchar_t *)dmabuf_p->kaddrp; 912 913 dmabuf_p->kaddrp = (void *)((char *)dmabuf_p->kaddrp + size); 914 dmabuf_p->ioaddr_pp = (void *) 915 ((char *)dmabuf_p->ioaddr_pp + size); 916 917 dmabuf_p->alength -= size; 918 dmabuf_p->offset += size; 919 dmabuf_p->dma_cookie.dmac_laddress += size; 920 dmabuf_p->dma_cookie.dmac_size -= size; 921 } else { 922 buffer = KMEM_ALLOC(size, KM_NOSLEEP); 923 if (buffer == NULL) { 924 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 925 "Allocation of a receive page failed.")); 926 goto hxge_allocb_fail1; 927 } 928 } 929 930 hxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &hxge_mp->freeb); 931 if (hxge_mp->rx_mblk_p == NULL) { 932 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, "desballoc failed.")); 933 goto hxge_allocb_fail2; 934 } 935 hxge_mp->buffer = buffer; 936 hxge_mp->block_size = size; 937 hxge_mp->freeb.free_func = (void (*) ()) hxge_freeb; 938 hxge_mp->freeb.free_arg = (caddr_t)hxge_mp; 939 hxge_mp->ref_cnt = 1; 940 hxge_mp->free = B_TRUE; 941 hxge_mp->rx_use_bcopy = B_FALSE; 942 943 atomic_inc_32(&hxge_mblks_pending); 944 945 goto hxge_allocb_exit; 946 947 hxge_allocb_fail2: 948 if (!hxge_mp->use_buf_pool) { 949 KMEM_FREE(buffer, size); 950 } 951 hxge_allocb_fail1: 952 KMEM_FREE(hxge_mp, sizeof (rx_msg_t)); 953 hxge_mp = NULL; 954 955 hxge_allocb_exit: 956 return (hxge_mp); 957 } 958 959 p_mblk_t 960 hxge_dupb(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 961 { 962 p_mblk_t mp; 963 964 HXGE_DEBUG_MSG((NULL, MEM_CTL, "==> hxge_dupb")); 965 HXGE_DEBUG_MSG((NULL, MEM_CTL, "hxge_mp = $%p " 966 "offset = 0x%08X " "size = 0x%08X", hxge_mp, offset, size)); 967 968 mp = desballoc(&hxge_mp->buffer[offset], size, 0, &hxge_mp->freeb); 969 if (mp == NULL) { 970 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 971 goto hxge_dupb_exit; 972 } 973 974 atomic_inc_32(&hxge_mp->ref_cnt); 975 976 hxge_dupb_exit: 977 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 978 return (mp); 979 } 980 981 p_mblk_t 982 hxge_dupb_bcopy(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 983 { 984 p_mblk_t mp; 985 uchar_t *dp; 986 987 mp = allocb(size + HXGE_RXBUF_EXTRA, 0); 988 if (mp == NULL) { 989 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 990 goto hxge_dupb_bcopy_exit; 991 } 992 dp = mp->b_rptr = mp->b_rptr + HXGE_RXBUF_EXTRA; 993 bcopy((void *) &hxge_mp->buffer[offset], dp, size); 994 mp->b_wptr = dp + size; 995 996 hxge_dupb_bcopy_exit: 997 998 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 999 1000 return (mp); 1001 } 1002 1003 void hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, 1004 p_rx_msg_t rx_msg_p); 1005 1006 void 1007 hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p) 1008 { 1009 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_post_page")); 1010 1011 /* Reuse this buffer */ 1012 rx_msg_p->free = B_FALSE; 1013 rx_msg_p->cur_usage_cnt = 0; 1014 rx_msg_p->max_usage_cnt = 0; 1015 rx_msg_p->pkt_buf_size = 0; 1016 1017 if (rx_rbr_p->rbr_use_bcopy) { 1018 rx_msg_p->rx_use_bcopy = B_FALSE; 1019 atomic_dec_32(&rx_rbr_p->rbr_consumed); 1020 } 1021 atomic_dec_32(&rx_rbr_p->rbr_used); 1022 1023 /* 1024 * Get the rbr header pointer and its offset index. 1025 */ 1026 rx_rbr_p->rbr_wr_index = ((rx_rbr_p->rbr_wr_index + 1) & 1027 rx_rbr_p->rbr_wrap_mask); 1028 rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr; 1029 1030 /* 1031 * Accumulate some buffers in the ring before re-enabling the 1032 * DMA channel, if rbr empty was signaled. 1033 */ 1034 hpi_rxdma_rdc_rbr_kick(HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc, 1); 1035 if (rx_rbr_p->rbr_is_empty && 1036 (rx_rbr_p->rbb_max - rx_rbr_p->rbr_used) >= 1037 HXGE_RBR_EMPTY_THRESHOLD) { 1038 hxge_rbr_empty_restore(hxgep, rx_rbr_p); 1039 } 1040 1041 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1042 "<== hxge_post_page (channel %d post_next_index %d)", 1043 rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index)); 1044 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_post_page")); 1045 } 1046 1047 void 1048 hxge_freeb(p_rx_msg_t rx_msg_p) 1049 { 1050 size_t size; 1051 uchar_t *buffer = NULL; 1052 int ref_cnt; 1053 boolean_t free_state = B_FALSE; 1054 rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p; 1055 1056 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> hxge_freeb")); 1057 HXGE_DEBUG_MSG((NULL, MEM2_CTL, 1058 "hxge_freeb:rx_msg_p = $%p (block pending %d)", 1059 rx_msg_p, hxge_mblks_pending)); 1060 1061 if (ring == NULL) 1062 return; 1063 1064 /* 1065 * This is to prevent posting activities while we are recovering 1066 * from fatal errors. This should not be a performance drag since 1067 * ref_cnt != 0 most times. 1068 */ 1069 if (ring->rbr_state == RBR_POSTING) 1070 MUTEX_ENTER(&ring->post_lock); 1071 1072 /* 1073 * First we need to get the free state, then 1074 * atomic decrement the reference count to prevent 1075 * the race condition with the interrupt thread that 1076 * is processing a loaned up buffer block. 1077 */ 1078 free_state = rx_msg_p->free; 1079 ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1); 1080 if (!ref_cnt) { 1081 atomic_dec_32(&hxge_mblks_pending); 1082 1083 buffer = rx_msg_p->buffer; 1084 size = rx_msg_p->block_size; 1085 1086 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "hxge_freeb: " 1087 "will free: rx_msg_p = $%p (block pending %d)", 1088 rx_msg_p, hxge_mblks_pending)); 1089 1090 if (!rx_msg_p->use_buf_pool) { 1091 KMEM_FREE(buffer, size); 1092 } 1093 1094 KMEM_FREE(rx_msg_p, sizeof (rx_msg_t)); 1095 /* 1096 * Decrement the receive buffer ring's reference 1097 * count, too. 1098 */ 1099 atomic_dec_32(&ring->rbr_ref_cnt); 1100 1101 /* 1102 * Free the receive buffer ring, iff 1103 * 1. all the receive buffers have been freed 1104 * 2. and we are in the proper state (that is, 1105 * we are not UNMAPPING). 1106 */ 1107 if (ring->rbr_ref_cnt == 0 && 1108 ring->rbr_state == RBR_UNMAPPED) { 1109 KMEM_FREE(ring, sizeof (*ring)); 1110 /* post_lock has been destroyed already */ 1111 return; 1112 } 1113 } 1114 1115 /* 1116 * Repost buffer. 1117 */ 1118 if (free_state && (ref_cnt == 1)) { 1119 HXGE_DEBUG_MSG((NULL, RX_CTL, 1120 "hxge_freeb: post page $%p:", rx_msg_p)); 1121 if (ring->rbr_state == RBR_POSTING) 1122 hxge_post_page(rx_msg_p->hxgep, ring, rx_msg_p); 1123 } 1124 1125 if (ring->rbr_state == RBR_POSTING) 1126 MUTEX_EXIT(&ring->post_lock); 1127 1128 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== hxge_freeb")); 1129 } 1130 1131 uint_t 1132 hxge_rx_intr(caddr_t arg1, caddr_t arg2) 1133 { 1134 p_hxge_ldv_t ldvp = (p_hxge_ldv_t)arg1; 1135 p_hxge_t hxgep = (p_hxge_t)arg2; 1136 p_hxge_ldg_t ldgp; 1137 uint8_t channel; 1138 hpi_handle_t handle; 1139 rdc_stat_t cs; 1140 uint_t serviced = DDI_INTR_UNCLAIMED; 1141 1142 if (ldvp == NULL) { 1143 HXGE_DEBUG_MSG((NULL, RX_INT_CTL, 1144 "<== hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1145 return (DDI_INTR_UNCLAIMED); 1146 } 1147 1148 if (arg2 == NULL || (void *) ldvp->hxgep != arg2) { 1149 hxgep = ldvp->hxgep; 1150 } 1151 1152 /* 1153 * If the interface is not started, just swallow the interrupt 1154 * for the logical device and don't rearm it. 1155 */ 1156 if (hxgep->hxge_mac_state != HXGE_MAC_STARTED) 1157 return (DDI_INTR_CLAIMED); 1158 1159 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1160 "==> hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1161 1162 /* 1163 * This interrupt handler is for a specific receive dma channel. 1164 */ 1165 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1166 1167 /* 1168 * Get the control and status for this channel. 1169 */ 1170 channel = ldvp->channel; 1171 ldgp = ldvp->ldgp; 1172 RXDMA_REG_READ64(handle, RDC_STAT, channel, &cs.value); 1173 cs.bits.ptrread = 0; 1174 cs.bits.pktread = 0; 1175 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1176 1177 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_intr:channel %d " 1178 "cs 0x%016llx rcrto 0x%x rcrthres %x", 1179 channel, cs.value, cs.bits.rcr_to, cs.bits.rcr_thres)); 1180 1181 hxge_rx_pkts_vring(hxgep, ldvp->vdma_index, ldvp, cs); 1182 serviced = DDI_INTR_CLAIMED; 1183 1184 /* error events. */ 1185 if (cs.value & RDC_STAT_ERROR) { 1186 (void) hxge_rx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs); 1187 } 1188 1189 hxge_intr_exit: 1190 /* 1191 * Enable the mailbox update interrupt if we want to use mailbox. We 1192 * probably don't need to use mailbox as it only saves us one pio read. 1193 * Also write 1 to rcrthres and rcrto to clear these two edge triggered 1194 * bits. 1195 */ 1196 cs.value &= RDC_STAT_WR1C; 1197 cs.bits.mex = 1; 1198 cs.bits.ptrread = 0; 1199 cs.bits.pktread = 0; 1200 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1201 1202 /* 1203 * Rearm this logical group if this is a single device group. 1204 */ 1205 if (ldgp->nldvs == 1) { 1206 ld_intr_mgmt_t mgm; 1207 1208 mgm.value = 0; 1209 mgm.bits.arm = 1; 1210 mgm.bits.timer = ldgp->ldg_timer; 1211 HXGE_REG_WR32(handle, 1212 LD_INTR_MGMT + LDSV_OFFSET(ldgp->ldg), mgm.value); 1213 } 1214 1215 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1216 "<== hxge_rx_intr: serviced %d", serviced)); 1217 return (serviced); 1218 } 1219 1220 static void 1221 hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1222 rdc_stat_t cs) 1223 { 1224 p_mblk_t mp; 1225 p_rx_rcr_ring_t rcrp; 1226 1227 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts_vring")); 1228 if ((mp = hxge_rx_pkts(hxgep, vindex, ldvp, &rcrp, cs)) == NULL) { 1229 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1230 "<== hxge_rx_pkts_vring: no mp")); 1231 return; 1232 } 1233 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts_vring: $%p", mp)); 1234 1235 #ifdef HXGE_DEBUG 1236 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1237 "==> hxge_rx_pkts_vring:calling mac_rx (NEMO) " 1238 "LEN %d mp $%p mp->b_next $%p rcrp $%p", 1239 (mp->b_wptr - mp->b_rptr), mp, mp->b_next, rcrp)); 1240 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1241 "==> hxge_rx_pkts_vring: dump packets " 1242 "(mp $%p b_rptr $%p b_wptr $%p):\n %s", 1243 mp, mp->b_rptr, mp->b_wptr, 1244 hxge_dump_packet((char *)mp->b_rptr, 64))); 1245 1246 if (mp->b_cont) { 1247 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1248 "==> hxge_rx_pkts_vring: dump b_cont packets " 1249 "(mp->b_cont $%p b_rptr $%p b_wptr $%p):\n %s", 1250 mp->b_cont, mp->b_cont->b_rptr, mp->b_cont->b_wptr, 1251 hxge_dump_packet((char *)mp->b_cont->b_rptr, 1252 mp->b_cont->b_wptr - mp->b_cont->b_rptr))); 1253 } 1254 if (mp->b_next) { 1255 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1256 "==> hxge_rx_pkts_vring: dump next packets " 1257 "(b_rptr $%p): %s", 1258 mp->b_next->b_rptr, 1259 hxge_dump_packet((char *)mp->b_next->b_rptr, 64))); 1260 } 1261 #endif 1262 1263 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1264 "==> hxge_rx_pkts_vring: send packet to stack")); 1265 mac_rx(hxgep->mach, NULL, mp); 1266 1267 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_pkts_vring")); 1268 } 1269 1270 /*ARGSUSED*/ 1271 mblk_t * 1272 hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1273 p_rx_rcr_ring_t *rcrp, rdc_stat_t cs) 1274 { 1275 hpi_handle_t handle; 1276 uint8_t channel; 1277 p_rx_rcr_rings_t rx_rcr_rings; 1278 p_rx_rcr_ring_t rcr_p; 1279 uint32_t comp_rd_index; 1280 p_rcr_entry_t rcr_desc_rd_head_p; 1281 p_rcr_entry_t rcr_desc_rd_head_pp; 1282 p_mblk_t nmp, mp_cont, head_mp, *tail_mp; 1283 uint16_t qlen, nrcr_read, npkt_read; 1284 uint32_t qlen_hw, qlen_sw; 1285 uint32_t invalid_rcr_entry; 1286 boolean_t multi; 1287 rdc_rcr_cfg_b_t rcr_cfg_b; 1288 p_rx_mbox_t rx_mboxp; 1289 p_rxdma_mailbox_t mboxp; 1290 uint64_t rcr_head_index, rcr_tail_index; 1291 uint64_t rcr_tail; 1292 rdc_rcr_tail_t rcr_tail_reg; 1293 p_hxge_rx_ring_stats_t rdc_stats; 1294 1295 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:vindex %d " 1296 "channel %d", vindex, ldvp->channel)); 1297 1298 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 1299 return (NULL); 1300 } 1301 1302 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1303 rx_rcr_rings = hxgep->rx_rcr_rings; 1304 rcr_p = rx_rcr_rings->rcr_rings[vindex]; 1305 channel = rcr_p->rdc; 1306 if (channel != ldvp->channel) { 1307 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:index %d " 1308 "channel %d, and rcr channel %d not matched.", 1309 vindex, ldvp->channel, channel)); 1310 return (NULL); 1311 } 1312 1313 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1314 "==> hxge_rx_pkts: START: rcr channel %d " 1315 "head_p $%p head_pp $%p index %d ", 1316 channel, rcr_p->rcr_desc_rd_head_p, 1317 rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1318 1319 rx_mboxp = hxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 1320 mboxp = (p_rxdma_mailbox_t)rx_mboxp->rx_mbox.kaddrp; 1321 (void) hpi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen); 1322 RXDMA_REG_READ64(handle, RDC_RCR_TAIL, channel, &rcr_tail_reg.value); 1323 rcr_tail = rcr_tail_reg.bits.tail; 1324 1325 if (!qlen) { 1326 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1327 "<== hxge_rx_pkts:rcr channel %d qlen %d (no pkts)", 1328 channel, qlen)); 1329 return (NULL); 1330 } 1331 1332 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts:rcr channel %d " 1333 "qlen %d", channel, qlen)); 1334 1335 comp_rd_index = rcr_p->comp_rd_index; 1336 1337 rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p; 1338 rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp; 1339 nrcr_read = npkt_read = 0; 1340 1341 /* 1342 * Number of packets queued (The jumbo or multi packet will be counted 1343 * as only one paccket and it may take up more than one completion 1344 * entry). 1345 */ 1346 if (cs.bits.rbr_empty) 1347 qlen_hw = qlen; 1348 else 1349 qlen_hw = (qlen < hxge_max_rx_pkts) ? qlen : hxge_max_rx_pkts; 1350 1351 head_mp = NULL; 1352 tail_mp = &head_mp; 1353 nmp = mp_cont = NULL; 1354 multi = B_FALSE; 1355 1356 rcr_head_index = rcr_p->rcr_desc_rd_head_p - rcr_p->rcr_desc_first_p; 1357 rcr_tail_index = rcr_tail - rcr_p->rcr_tail_begin; 1358 1359 if (rcr_tail_index >= rcr_head_index) { 1360 qlen_sw = rcr_tail_index - rcr_head_index; 1361 } else { 1362 /* rcr_tail has wrapped around */ 1363 qlen_sw = (rcr_p->comp_size - rcr_head_index) + rcr_tail_index; 1364 } 1365 1366 if (qlen_hw > qlen_sw) { 1367 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1368 "Channel %d, rcr_qlen from reg %d and from rcr_tail %d\n", 1369 channel, qlen_hw, qlen_sw)); 1370 qlen_hw = qlen_sw; 1371 } 1372 1373 while (qlen_hw) { 1374 #ifdef HXGE_DEBUG 1375 hxge_dump_rcr_entry(hxgep, rcr_desc_rd_head_p); 1376 #endif 1377 /* 1378 * Process one completion ring entry. 1379 */ 1380 invalid_rcr_entry = 0; 1381 hxge_receive_packet(hxgep, 1382 rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont, 1383 &invalid_rcr_entry); 1384 if (invalid_rcr_entry != 0) { 1385 rdc_stats = rcr_p->rdc_stats; 1386 rdc_stats->rcr_invalids++; 1387 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1388 "Channel %d could only read 0x%x packets, " 1389 "but 0x%x pending\n", channel, npkt_read, qlen_hw)); 1390 break; 1391 } 1392 1393 /* 1394 * message chaining modes (nemo msg chaining) 1395 */ 1396 if (nmp) { 1397 nmp->b_next = NULL; 1398 if (!multi && !mp_cont) { /* frame fits a partition */ 1399 *tail_mp = nmp; 1400 tail_mp = &nmp->b_next; 1401 nmp = NULL; 1402 } else if (multi && !mp_cont) { /* first segment */ 1403 *tail_mp = nmp; 1404 tail_mp = &nmp->b_cont; 1405 } else if (multi && mp_cont) { /* mid of multi segs */ 1406 *tail_mp = mp_cont; 1407 tail_mp = &mp_cont->b_cont; 1408 } else if (!multi && mp_cont) { /* last segment */ 1409 *tail_mp = mp_cont; 1410 tail_mp = &nmp->b_next; 1411 nmp = NULL; 1412 } 1413 } 1414 1415 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1416 "==> hxge_rx_pkts: loop: rcr channel %d " 1417 "before updating: multi %d " 1418 "nrcr_read %d " 1419 "npk read %d " 1420 "head_pp $%p index %d ", 1421 channel, multi, 1422 nrcr_read, npkt_read, rcr_desc_rd_head_pp, comp_rd_index)); 1423 1424 if (!multi) { 1425 qlen_hw--; 1426 npkt_read++; 1427 } 1428 1429 /* 1430 * Update the next read entry. 1431 */ 1432 comp_rd_index = NEXT_ENTRY(comp_rd_index, 1433 rcr_p->comp_wrap_mask); 1434 1435 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p, 1436 rcr_p->rcr_desc_first_p, rcr_p->rcr_desc_last_p); 1437 1438 nrcr_read++; 1439 1440 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1441 "<== hxge_rx_pkts: (SAM, process one packet) " 1442 "nrcr_read %d", nrcr_read)); 1443 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1444 "==> hxge_rx_pkts: loop: rcr channel %d " 1445 "multi %d nrcr_read %d npk read %d head_pp $%p index %d ", 1446 channel, multi, nrcr_read, npkt_read, rcr_desc_rd_head_pp, 1447 comp_rd_index)); 1448 } 1449 1450 rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp; 1451 rcr_p->comp_rd_index = comp_rd_index; 1452 rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p; 1453 1454 /* Adjust the mailbox queue length for a hardware bug workaround */ 1455 mboxp->rcrstat_a.bits.qlen -= npkt_read; 1456 1457 if ((hxgep->intr_timeout != rcr_p->intr_timeout) || 1458 (hxgep->intr_threshold != rcr_p->intr_threshold)) { 1459 rcr_p->intr_timeout = hxgep->intr_timeout; 1460 rcr_p->intr_threshold = hxgep->intr_threshold; 1461 rcr_cfg_b.value = 0x0ULL; 1462 if (rcr_p->intr_timeout) 1463 rcr_cfg_b.bits.entout = 1; 1464 rcr_cfg_b.bits.timeout = rcr_p->intr_timeout; 1465 rcr_cfg_b.bits.pthres = rcr_p->intr_threshold; 1466 RXDMA_REG_WRITE64(handle, RDC_RCR_CFG_B, 1467 channel, rcr_cfg_b.value); 1468 } 1469 1470 if (hxgep->rdc_first_intr[channel] && (npkt_read > 0)) { 1471 hxgep->rdc_first_intr[channel] = B_FALSE; 1472 cs.bits.pktread = npkt_read - 1; 1473 } else 1474 cs.bits.pktread = npkt_read; 1475 cs.bits.ptrread = nrcr_read; 1476 cs.value &= 0xffffffffULL; 1477 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1478 1479 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1480 "==> hxge_rx_pkts: EXIT: rcr channel %d " 1481 "head_pp $%p index %016llx ", 1482 channel, rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1483 1484 /* 1485 * Update RCR buffer pointer read and number of packets read. 1486 */ 1487 *rcrp = rcr_p; 1488 1489 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "<== hxge_rx_pkts")); 1490 1491 return (head_mp); 1492 } 1493 1494 #define RCR_ENTRY_PATTERN 0x5a5a6b6b7c7c8d8dULL 1495 #define NO_PORT_BIT 0x20 1496 #define L4_CS_EQ_BIT 0x40 1497 1498 /*ARGSUSED*/ 1499 void 1500 hxge_receive_packet(p_hxge_t hxgep, 1501 p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p, 1502 boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont, 1503 uint32_t *invalid_rcr_entry) 1504 { 1505 p_mblk_t nmp = NULL; 1506 uint64_t multi; 1507 uint8_t channel; 1508 1509 boolean_t first_entry = B_TRUE; 1510 boolean_t is_tcp_udp = B_FALSE; 1511 boolean_t buffer_free = B_FALSE; 1512 boolean_t error_send_up = B_FALSE; 1513 uint8_t error_type; 1514 uint16_t l2_len; 1515 uint16_t skip_len; 1516 uint8_t pktbufsz_type; 1517 uint64_t rcr_entry; 1518 uint64_t *pkt_buf_addr_pp; 1519 uint64_t *pkt_buf_addr_p; 1520 uint32_t buf_offset; 1521 uint32_t bsize; 1522 uint32_t msg_index; 1523 p_rx_rbr_ring_t rx_rbr_p; 1524 p_rx_msg_t *rx_msg_ring_p; 1525 p_rx_msg_t rx_msg_p; 1526 1527 uint16_t sw_offset_bytes = 0, hdr_size = 0; 1528 hxge_status_t status = HXGE_OK; 1529 boolean_t is_valid = B_FALSE; 1530 p_hxge_rx_ring_stats_t rdc_stats; 1531 uint32_t bytes_read; 1532 uint8_t header0 = 0; 1533 uint8_t header1 = 0; 1534 uint64_t pkt_type; 1535 uint8_t no_port_bit = 0; 1536 uint8_t l4_cs_eq_bit = 0; 1537 1538 channel = rcr_p->rdc; 1539 1540 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_receive_packet")); 1541 1542 first_entry = (*mp == NULL) ? B_TRUE : B_FALSE; 1543 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p); 1544 1545 /* Verify the content of the rcr_entry for a hardware bug workaround */ 1546 if ((rcr_entry == 0x0) || (rcr_entry == RCR_ENTRY_PATTERN)) { 1547 *invalid_rcr_entry = 1; 1548 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "hxge_receive_packet " 1549 "Channel %d invalid RCR entry 0x%llx found, returning\n", 1550 channel, (long long) rcr_entry)); 1551 return; 1552 } 1553 *((uint64_t *)rcr_desc_rd_head_p) = RCR_ENTRY_PATTERN; 1554 1555 multi = (rcr_entry & RCR_MULTI_MASK); 1556 pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK); 1557 1558 error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT); 1559 l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT); 1560 1561 /* 1562 * Hardware does not strip the CRC due bug ID 11451 where 1563 * the hardware mis handles minimum size packets. 1564 */ 1565 l2_len -= ETHERFCSL; 1566 1567 pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >> 1568 RCR_PKTBUFSZ_SHIFT); 1569 #if defined(__i386) 1570 pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry & 1571 RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT); 1572 #else 1573 pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) << 1574 RCR_PKT_BUF_ADDR_SHIFT); 1575 #endif 1576 1577 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1578 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1579 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1580 "error_type 0x%x pktbufsz_type %d ", 1581 rcr_desc_rd_head_p, rcr_entry, pkt_buf_addr_pp, l2_len, 1582 multi, error_type, pktbufsz_type)); 1583 1584 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1585 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1586 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1587 "error_type 0x%x ", rcr_desc_rd_head_p, 1588 rcr_entry, pkt_buf_addr_pp, l2_len, multi, error_type)); 1589 1590 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1591 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1592 "full pkt_buf_addr_pp $%p l2_len %d", 1593 rcr_entry, pkt_buf_addr_pp, l2_len)); 1594 1595 /* get the stats ptr */ 1596 rdc_stats = rcr_p->rdc_stats; 1597 1598 if (!l2_len) { 1599 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1600 "<== hxge_receive_packet: failed: l2 length is 0.")); 1601 return; 1602 } 1603 1604 /* shift 6 bits to get the full io address */ 1605 #if defined(__i386) 1606 pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp << 1607 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1608 #else 1609 pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp << 1610 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1611 #endif 1612 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1613 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1614 "full pkt_buf_addr_pp $%p l2_len %d", 1615 rcr_entry, pkt_buf_addr_pp, l2_len)); 1616 1617 rx_rbr_p = rcr_p->rx_rbr_p; 1618 rx_msg_ring_p = rx_rbr_p->rx_msg_ring; 1619 1620 if (first_entry) { 1621 hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL : 1622 RXDMA_HDR_SIZE_DEFAULT); 1623 1624 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1625 "==> hxge_receive_packet: first entry 0x%016llx " 1626 "pkt_buf_addr_pp $%p l2_len %d hdr %d", 1627 rcr_entry, pkt_buf_addr_pp, l2_len, hdr_size)); 1628 } 1629 1630 MUTEX_ENTER(&rcr_p->lock); 1631 MUTEX_ENTER(&rx_rbr_p->lock); 1632 1633 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1634 "==> (rbr 1) hxge_receive_packet: entry 0x%0llx " 1635 "full pkt_buf_addr_pp $%p l2_len %d", 1636 rcr_entry, pkt_buf_addr_pp, l2_len)); 1637 1638 /* 1639 * Packet buffer address in the completion entry points to the starting 1640 * buffer address (offset 0). Use the starting buffer address to locate 1641 * the corresponding kernel address. 1642 */ 1643 status = hxge_rxbuf_pp_to_vp(hxgep, rx_rbr_p, 1644 pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p, 1645 &buf_offset, &msg_index); 1646 1647 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1648 "==> (rbr 2) hxge_receive_packet: entry 0x%0llx " 1649 "full pkt_buf_addr_pp $%p l2_len %d", 1650 rcr_entry, pkt_buf_addr_pp, l2_len)); 1651 1652 if (status != HXGE_OK) { 1653 MUTEX_EXIT(&rx_rbr_p->lock); 1654 MUTEX_EXIT(&rcr_p->lock); 1655 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1656 "<== hxge_receive_packet: found vaddr failed %d", status)); 1657 return; 1658 } 1659 1660 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1661 "==> (rbr 3) hxge_receive_packet: entry 0x%0llx " 1662 "full pkt_buf_addr_pp $%p l2_len %d", 1663 rcr_entry, pkt_buf_addr_pp, l2_len)); 1664 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1665 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1666 "full pkt_buf_addr_pp $%p l2_len %d", 1667 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1668 1669 if (msg_index >= rx_rbr_p->tnblocks) { 1670 MUTEX_EXIT(&rx_rbr_p->lock); 1671 MUTEX_EXIT(&rcr_p->lock); 1672 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1673 "==> hxge_receive_packet: FATAL msg_index (%d) " 1674 "should be smaller than tnblocks (%d)\n", 1675 msg_index, rx_rbr_p->tnblocks)); 1676 return; 1677 } 1678 1679 rx_msg_p = rx_msg_ring_p[msg_index]; 1680 1681 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1682 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1683 "full pkt_buf_addr_pp $%p l2_len %d", 1684 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1685 1686 switch (pktbufsz_type) { 1687 case RCR_PKTBUFSZ_0: 1688 bsize = rx_rbr_p->pkt_buf_size0_bytes; 1689 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1690 "==> hxge_receive_packet: 0 buf %d", bsize)); 1691 break; 1692 case RCR_PKTBUFSZ_1: 1693 bsize = rx_rbr_p->pkt_buf_size1_bytes; 1694 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1695 "==> hxge_receive_packet: 1 buf %d", bsize)); 1696 break; 1697 case RCR_PKTBUFSZ_2: 1698 bsize = rx_rbr_p->pkt_buf_size2_bytes; 1699 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1700 "==> hxge_receive_packet: 2 buf %d", bsize)); 1701 break; 1702 case RCR_SINGLE_BLOCK: 1703 bsize = rx_msg_p->block_size; 1704 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1705 "==> hxge_receive_packet: single %d", bsize)); 1706 1707 break; 1708 default: 1709 MUTEX_EXIT(&rx_rbr_p->lock); 1710 MUTEX_EXIT(&rcr_p->lock); 1711 return; 1712 } 1713 1714 DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma, 1715 (buf_offset + sw_offset_bytes), (hdr_size + l2_len), 1716 DDI_DMA_SYNC_FORCPU); 1717 1718 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1719 "==> hxge_receive_packet: after first dump:usage count")); 1720 1721 if (rx_msg_p->cur_usage_cnt == 0) { 1722 atomic_inc_32(&rx_rbr_p->rbr_used); 1723 if (rx_rbr_p->rbr_use_bcopy) { 1724 atomic_inc_32(&rx_rbr_p->rbr_consumed); 1725 if (rx_rbr_p->rbr_consumed < 1726 rx_rbr_p->rbr_threshold_hi) { 1727 if (rx_rbr_p->rbr_threshold_lo == 0 || 1728 ((rx_rbr_p->rbr_consumed >= 1729 rx_rbr_p->rbr_threshold_lo) && 1730 (rx_rbr_p->rbr_bufsize_type >= 1731 pktbufsz_type))) { 1732 rx_msg_p->rx_use_bcopy = B_TRUE; 1733 } 1734 } else { 1735 rx_msg_p->rx_use_bcopy = B_TRUE; 1736 } 1737 } 1738 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1739 "==> hxge_receive_packet: buf %d (new block) ", bsize)); 1740 1741 rx_msg_p->pkt_buf_size_code = pktbufsz_type; 1742 rx_msg_p->pkt_buf_size = bsize; 1743 rx_msg_p->cur_usage_cnt = 1; 1744 if (pktbufsz_type == RCR_SINGLE_BLOCK) { 1745 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1746 "==> hxge_receive_packet: buf %d (single block) ", 1747 bsize)); 1748 /* 1749 * Buffer can be reused once the free function is 1750 * called. 1751 */ 1752 rx_msg_p->max_usage_cnt = 1; 1753 buffer_free = B_TRUE; 1754 } else { 1755 rx_msg_p->max_usage_cnt = rx_msg_p->block_size / bsize; 1756 if (rx_msg_p->max_usage_cnt == 1) { 1757 buffer_free = B_TRUE; 1758 } 1759 } 1760 } else { 1761 rx_msg_p->cur_usage_cnt++; 1762 if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) { 1763 buffer_free = B_TRUE; 1764 } 1765 } 1766 1767 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1768 "msgbuf index = %d l2len %d bytes usage %d max_usage %d ", 1769 msg_index, l2_len, 1770 rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt)); 1771 1772 if (error_type) { 1773 rdc_stats->ierrors++; 1774 /* Update error stats */ 1775 rdc_stats->errlog.compl_err_type = error_type; 1776 HXGE_FM_REPORT_ERROR(hxgep, NULL, HXGE_FM_EREPORT_RDMC_RCR_ERR); 1777 1778 if (error_type & RCR_CTRL_FIFO_DED) { 1779 rdc_stats->ctrl_fifo_ecc_err++; 1780 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1781 " hxge_receive_packet: " 1782 " channel %d RCR ctrl_fifo_ded error", channel)); 1783 } else if (error_type & RCR_DATA_FIFO_DED) { 1784 rdc_stats->data_fifo_ecc_err++; 1785 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1786 " hxge_receive_packet: channel %d" 1787 " RCR data_fifo_ded error", channel)); 1788 } 1789 1790 /* 1791 * Update and repost buffer block if max usage count is 1792 * reached. 1793 */ 1794 if (error_send_up == B_FALSE) { 1795 atomic_inc_32(&rx_msg_p->ref_cnt); 1796 if (buffer_free == B_TRUE) { 1797 rx_msg_p->free = B_TRUE; 1798 } 1799 1800 MUTEX_EXIT(&rx_rbr_p->lock); 1801 MUTEX_EXIT(&rcr_p->lock); 1802 hxge_freeb(rx_msg_p); 1803 return; 1804 } 1805 } 1806 1807 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1808 "==> hxge_receive_packet: DMA sync second ")); 1809 1810 bytes_read = rcr_p->rcvd_pkt_bytes; 1811 skip_len = sw_offset_bytes + hdr_size; 1812 1813 if (first_entry) { 1814 header0 = rx_msg_p->buffer[buf_offset]; 1815 no_port_bit = header0 & NO_PORT_BIT; 1816 1817 header1 = rx_msg_p->buffer[buf_offset + 1]; 1818 l4_cs_eq_bit = header1 & L4_CS_EQ_BIT; 1819 } 1820 1821 if (!rx_msg_p->rx_use_bcopy) { 1822 /* 1823 * For loaned up buffers, the driver reference count 1824 * will be incremented first and then the free state. 1825 */ 1826 if ((nmp = hxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) { 1827 if (first_entry) { 1828 nmp->b_rptr = &nmp->b_rptr[skip_len]; 1829 if (l2_len < bsize - skip_len) { 1830 nmp->b_wptr = &nmp->b_rptr[l2_len]; 1831 } else { 1832 nmp->b_wptr = &nmp->b_rptr[bsize 1833 - skip_len]; 1834 } 1835 } else { 1836 if (l2_len - bytes_read < bsize) { 1837 nmp->b_wptr = 1838 &nmp->b_rptr[l2_len - bytes_read]; 1839 } else { 1840 nmp->b_wptr = &nmp->b_rptr[bsize]; 1841 } 1842 } 1843 } 1844 } else { 1845 if (first_entry) { 1846 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len, 1847 l2_len < bsize - skip_len ? 1848 l2_len : bsize - skip_len); 1849 } else { 1850 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset, 1851 l2_len - bytes_read < bsize ? 1852 l2_len - bytes_read : bsize); 1853 } 1854 } 1855 1856 if (nmp != NULL) { 1857 if (first_entry) 1858 bytes_read = nmp->b_wptr - nmp->b_rptr; 1859 else 1860 bytes_read += nmp->b_wptr - nmp->b_rptr; 1861 1862 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1863 "==> hxge_receive_packet after dupb: " 1864 "rbr consumed %d " 1865 "pktbufsz_type %d " 1866 "nmp $%p rptr $%p wptr $%p " 1867 "buf_offset %d bzise %d l2_len %d skip_len %d", 1868 rx_rbr_p->rbr_consumed, 1869 pktbufsz_type, 1870 nmp, nmp->b_rptr, nmp->b_wptr, 1871 buf_offset, bsize, l2_len, skip_len)); 1872 } else { 1873 cmn_err(CE_WARN, "!hxge_receive_packet: update stats (error)"); 1874 1875 atomic_inc_32(&rx_msg_p->ref_cnt); 1876 if (buffer_free == B_TRUE) { 1877 rx_msg_p->free = B_TRUE; 1878 } 1879 1880 MUTEX_EXIT(&rx_rbr_p->lock); 1881 MUTEX_EXIT(&rcr_p->lock); 1882 hxge_freeb(rx_msg_p); 1883 return; 1884 } 1885 1886 if (buffer_free == B_TRUE) { 1887 rx_msg_p->free = B_TRUE; 1888 } 1889 1890 /* 1891 * ERROR, FRAG and PKT_TYPE are only reported in the first entry. If a 1892 * packet is not fragmented and no error bit is set, then L4 checksum 1893 * is OK. 1894 */ 1895 is_valid = (nmp != NULL); 1896 if (first_entry) { 1897 rdc_stats->ipackets++; /* count only 1st seg for jumbo */ 1898 if (l2_len > (STD_FRAME_SIZE - ETHERFCSL)) 1899 rdc_stats->jumbo_pkts++; 1900 rdc_stats->ibytes += skip_len + l2_len < bsize ? 1901 l2_len : bsize; 1902 } else { 1903 /* 1904 * Add the current portion of the packet to the kstats. 1905 * The current portion of the packet is calculated by using 1906 * length of the packet and the previously received portion. 1907 */ 1908 rdc_stats->ibytes += l2_len - rcr_p->rcvd_pkt_bytes < bsize ? 1909 l2_len - rcr_p->rcvd_pkt_bytes : bsize; 1910 } 1911 1912 rcr_p->rcvd_pkt_bytes = bytes_read; 1913 1914 if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) { 1915 atomic_inc_32(&rx_msg_p->ref_cnt); 1916 MUTEX_EXIT(&rx_rbr_p->lock); 1917 MUTEX_EXIT(&rcr_p->lock); 1918 hxge_freeb(rx_msg_p); 1919 } else { 1920 MUTEX_EXIT(&rx_rbr_p->lock); 1921 MUTEX_EXIT(&rcr_p->lock); 1922 } 1923 1924 if (is_valid) { 1925 nmp->b_cont = NULL; 1926 if (first_entry) { 1927 *mp = nmp; 1928 *mp_cont = NULL; 1929 } else { 1930 *mp_cont = nmp; 1931 } 1932 } 1933 1934 /* 1935 * Update stats and hardware checksuming. 1936 */ 1937 if (is_valid && !multi) { 1938 is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP || 1939 pkt_type == RCR_PKT_IS_UDP) ? B_TRUE : B_FALSE); 1940 1941 if (!no_port_bit && l4_cs_eq_bit && is_tcp_udp && !error_type) { 1942 (void) hcksum_assoc(nmp, NULL, NULL, 0, 0, 0, 0, 1943 HCK_FULLCKSUM_OK | HCK_FULLCKSUM, 0); 1944 1945 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1946 "==> hxge_receive_packet: Full tcp/udp cksum " 1947 "is_valid 0x%x multi %d error %d", 1948 is_valid, multi, error_type)); 1949 } 1950 } 1951 1952 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1953 "==> hxge_receive_packet: *mp 0x%016llx", *mp)); 1954 1955 *multi_p = (multi == RCR_MULTI_MASK); 1956 1957 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_receive_packet: " 1958 "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx", 1959 *multi_p, nmp, *mp, *mp_cont)); 1960 } 1961 1962 static void 1963 hxge_rx_rbr_empty_recover(p_hxge_t hxgep, uint8_t channel) 1964 { 1965 hpi_handle_t handle; 1966 p_rx_rcr_ring_t rcrp; 1967 p_rx_rbr_ring_t rbrp; 1968 1969 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 1970 rbrp = rcrp->rx_rbr_p; 1971 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1972 1973 /* 1974 * Wait for the channel to be quiet 1975 */ 1976 (void) hpi_rxdma_cfg_rdc_wait_for_qst(handle, channel); 1977 1978 /* 1979 * Post page will accumulate some buffers before re-enabling 1980 * the DMA channel. 1981 */ 1982 1983 MUTEX_ENTER(&rbrp->post_lock); 1984 if ((rbrp->rbb_max - rbrp->rbr_used) >= HXGE_RBR_EMPTY_THRESHOLD) { 1985 hxge_rbr_empty_restore(hxgep, rbrp); 1986 } else { 1987 rbrp->rbr_is_empty = B_TRUE; 1988 } 1989 MUTEX_EXIT(&rbrp->post_lock); 1990 } 1991 1992 /*ARGSUSED*/ 1993 static hxge_status_t 1994 hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp, 1995 rdc_stat_t cs) 1996 { 1997 p_hxge_rx_ring_stats_t rdc_stats; 1998 hpi_handle_t handle; 1999 boolean_t rxchan_fatal = B_FALSE; 2000 uint8_t channel; 2001 hxge_status_t status = HXGE_OK; 2002 uint64_t cs_val; 2003 2004 HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_rx_err_evnts")); 2005 2006 handle = HXGE_DEV_HPI_HANDLE(hxgep); 2007 channel = ldvp->channel; 2008 2009 /* Clear the interrupts */ 2010 cs.bits.pktread = 0; 2011 cs.bits.ptrread = 0; 2012 cs_val = cs.value & RDC_STAT_WR1C; 2013 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs_val); 2014 2015 rdc_stats = &hxgep->statsp->rdc_stats[ldvp->vdma_index]; 2016 2017 if (cs.bits.rbr_cpl_to) { 2018 rdc_stats->rbr_tmout++; 2019 HXGE_FM_REPORT_ERROR(hxgep, channel, 2020 HXGE_FM_EREPORT_RDMC_RBR_CPL_TO); 2021 rxchan_fatal = B_TRUE; 2022 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2023 "==> hxge_rx_err_evnts(channel %d): " 2024 "fatal error: rx_rbr_timeout", channel)); 2025 } 2026 2027 if ((cs.bits.rcr_shadow_par_err) || (cs.bits.rbr_prefetch_par_err)) { 2028 (void) hpi_rxdma_ring_perr_stat_get(handle, 2029 &rdc_stats->errlog.pre_par, &rdc_stats->errlog.sha_par); 2030 } 2031 2032 if (cs.bits.rcr_shadow_par_err) { 2033 rdc_stats->rcr_sha_par++; 2034 HXGE_FM_REPORT_ERROR(hxgep, channel, 2035 HXGE_FM_EREPORT_RDMC_RCR_SHA_PAR); 2036 rxchan_fatal = B_TRUE; 2037 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2038 "==> hxge_rx_err_evnts(channel %d): " 2039 "fatal error: rcr_shadow_par_err", channel)); 2040 } 2041 2042 if (cs.bits.rbr_prefetch_par_err) { 2043 rdc_stats->rbr_pre_par++; 2044 HXGE_FM_REPORT_ERROR(hxgep, channel, 2045 HXGE_FM_EREPORT_RDMC_RBR_PRE_PAR); 2046 rxchan_fatal = B_TRUE; 2047 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2048 "==> hxge_rx_err_evnts(channel %d): " 2049 "fatal error: rbr_prefetch_par_err", channel)); 2050 } 2051 2052 if (cs.bits.rbr_pre_empty) { 2053 rdc_stats->rbr_pre_empty++; 2054 HXGE_FM_REPORT_ERROR(hxgep, channel, 2055 HXGE_FM_EREPORT_RDMC_RBR_PRE_EMPTY); 2056 rxchan_fatal = B_TRUE; 2057 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2058 "==> hxge_rx_err_evnts(channel %d): " 2059 "fatal error: rbr_pre_empty", channel)); 2060 } 2061 2062 if (cs.bits.peu_resp_err) { 2063 rdc_stats->peu_resp_err++; 2064 HXGE_FM_REPORT_ERROR(hxgep, channel, 2065 HXGE_FM_EREPORT_RDMC_PEU_RESP_ERR); 2066 rxchan_fatal = B_TRUE; 2067 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2068 "==> hxge_rx_err_evnts(channel %d): " 2069 "fatal error: peu_resp_err", channel)); 2070 } 2071 2072 if (cs.bits.rcr_thres) { 2073 rdc_stats->rcr_thres++; 2074 } 2075 2076 if (cs.bits.rcr_to) { 2077 rdc_stats->rcr_to++; 2078 } 2079 2080 if (cs.bits.rcr_shadow_full) { 2081 rdc_stats->rcr_shadow_full++; 2082 HXGE_FM_REPORT_ERROR(hxgep, channel, 2083 HXGE_FM_EREPORT_RDMC_RCR_SHA_FULL); 2084 rxchan_fatal = B_TRUE; 2085 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2086 "==> hxge_rx_err_evnts(channel %d): " 2087 "fatal error: rcr_shadow_full", channel)); 2088 } 2089 2090 if (cs.bits.rcr_full) { 2091 rdc_stats->rcrfull++; 2092 HXGE_FM_REPORT_ERROR(hxgep, channel, 2093 HXGE_FM_EREPORT_RDMC_RCRFULL); 2094 rxchan_fatal = B_TRUE; 2095 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2096 "==> hxge_rx_err_evnts(channel %d): " 2097 "fatal error: rcrfull error", channel)); 2098 } 2099 2100 if (cs.bits.rbr_empty) { 2101 rdc_stats->rbr_empty++; 2102 hxge_rx_rbr_empty_recover(hxgep, channel); 2103 } 2104 2105 if (cs.bits.rbr_full) { 2106 rdc_stats->rbrfull++; 2107 HXGE_FM_REPORT_ERROR(hxgep, channel, 2108 HXGE_FM_EREPORT_RDMC_RBRFULL); 2109 rxchan_fatal = B_TRUE; 2110 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2111 "==> hxge_rx_err_evnts(channel %d): " 2112 "fatal error: rbr_full error", channel)); 2113 } 2114 2115 if (rxchan_fatal) { 2116 p_rx_rcr_ring_t rcrp; 2117 p_rx_rbr_ring_t rbrp; 2118 2119 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 2120 rbrp = rcrp->rx_rbr_p; 2121 2122 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2123 " hxge_rx_err_evnts: fatal error on Channel #%d\n", 2124 channel)); 2125 MUTEX_ENTER(&rbrp->post_lock); 2126 /* This function needs to be inside the post_lock */ 2127 status = hxge_rxdma_fatal_err_recover(hxgep, channel); 2128 MUTEX_EXIT(&rbrp->post_lock); 2129 if (status == HXGE_OK) { 2130 FM_SERVICE_RESTORED(hxgep); 2131 } 2132 } 2133 HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_rx_err_evnts")); 2134 2135 return (status); 2136 } 2137 2138 static hxge_status_t 2139 hxge_map_rxdma(p_hxge_t hxgep) 2140 { 2141 int i, ndmas; 2142 uint16_t channel; 2143 p_rx_rbr_rings_t rx_rbr_rings; 2144 p_rx_rbr_ring_t *rbr_rings; 2145 p_rx_rcr_rings_t rx_rcr_rings; 2146 p_rx_rcr_ring_t *rcr_rings; 2147 p_rx_mbox_areas_t rx_mbox_areas_p; 2148 p_rx_mbox_t *rx_mbox_p; 2149 p_hxge_dma_pool_t dma_buf_poolp; 2150 p_hxge_dma_common_t *dma_buf_p; 2151 p_hxge_dma_pool_t dma_rbr_cntl_poolp; 2152 p_hxge_dma_common_t *dma_rbr_cntl_p; 2153 p_hxge_dma_pool_t dma_rcr_cntl_poolp; 2154 p_hxge_dma_common_t *dma_rcr_cntl_p; 2155 p_hxge_dma_pool_t dma_mbox_cntl_poolp; 2156 p_hxge_dma_common_t *dma_mbox_cntl_p; 2157 uint32_t *num_chunks; 2158 hxge_status_t status = HXGE_OK; 2159 2160 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_map_rxdma")); 2161 2162 dma_buf_poolp = hxgep->rx_buf_pool_p; 2163 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p; 2164 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p; 2165 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p; 2166 2167 if (!dma_buf_poolp->buf_allocated || 2168 !dma_rbr_cntl_poolp->buf_allocated || 2169 !dma_rcr_cntl_poolp->buf_allocated || 2170 !dma_mbox_cntl_poolp->buf_allocated) { 2171 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2172 "<== hxge_map_rxdma: buf not allocated")); 2173 return (HXGE_ERROR); 2174 } 2175 2176 ndmas = dma_buf_poolp->ndmas; 2177 if (!ndmas) { 2178 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2179 "<== hxge_map_rxdma: no dma allocated")); 2180 return (HXGE_ERROR); 2181 } 2182 2183 num_chunks = dma_buf_poolp->num_chunks; 2184 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2185 dma_rbr_cntl_p = dma_rbr_cntl_poolp->dma_buf_pool_p; 2186 dma_rcr_cntl_p = dma_rcr_cntl_poolp->dma_buf_pool_p; 2187 dma_mbox_cntl_p = dma_mbox_cntl_poolp->dma_buf_pool_p; 2188 2189 rx_rbr_rings = (p_rx_rbr_rings_t) 2190 KMEM_ZALLOC(sizeof (rx_rbr_rings_t), KM_SLEEP); 2191 rbr_rings = (p_rx_rbr_ring_t *)KMEM_ZALLOC( 2192 sizeof (p_rx_rbr_ring_t) * ndmas, KM_SLEEP); 2193 2194 rx_rcr_rings = (p_rx_rcr_rings_t) 2195 KMEM_ZALLOC(sizeof (rx_rcr_rings_t), KM_SLEEP); 2196 rcr_rings = (p_rx_rcr_ring_t *)KMEM_ZALLOC( 2197 sizeof (p_rx_rcr_ring_t) * ndmas, KM_SLEEP); 2198 2199 rx_mbox_areas_p = (p_rx_mbox_areas_t) 2200 KMEM_ZALLOC(sizeof (rx_mbox_areas_t), KM_SLEEP); 2201 rx_mbox_p = (p_rx_mbox_t *)KMEM_ZALLOC( 2202 sizeof (p_rx_mbox_t) * ndmas, KM_SLEEP); 2203 2204 /* 2205 * Timeout should be set based on the system clock divider. 2206 * The following timeout value of 1 assumes that the 2207 * granularity (1000) is 3 microseconds running at 300MHz. 2208 */ 2209 2210 hxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT; 2211 hxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT; 2212 2213 /* 2214 * Map descriptors from the buffer polls for each dam channel. 2215 */ 2216 for (i = 0; i < ndmas; i++) { 2217 /* 2218 * Set up and prepare buffer blocks, descriptors and mailbox. 2219 */ 2220 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2221 status = hxge_map_rxdma_channel(hxgep, channel, 2222 (p_hxge_dma_common_t *)&dma_buf_p[i], 2223 (p_rx_rbr_ring_t *)&rbr_rings[i], 2224 num_chunks[i], 2225 (p_hxge_dma_common_t *)&dma_rbr_cntl_p[i], 2226 (p_hxge_dma_common_t *)&dma_rcr_cntl_p[i], 2227 (p_hxge_dma_common_t *)&dma_mbox_cntl_p[i], 2228 (p_rx_rcr_ring_t *)&rcr_rings[i], 2229 (p_rx_mbox_t *)&rx_mbox_p[i]); 2230 if (status != HXGE_OK) { 2231 goto hxge_map_rxdma_fail1; 2232 } 2233 rbr_rings[i]->index = (uint16_t)i; 2234 rcr_rings[i]->index = (uint16_t)i; 2235 rcr_rings[i]->rdc_stats = &hxgep->statsp->rdc_stats[i]; 2236 } 2237 2238 rx_rbr_rings->ndmas = rx_rcr_rings->ndmas = ndmas; 2239 rx_rbr_rings->rbr_rings = rbr_rings; 2240 hxgep->rx_rbr_rings = rx_rbr_rings; 2241 rx_rcr_rings->rcr_rings = rcr_rings; 2242 hxgep->rx_rcr_rings = rx_rcr_rings; 2243 2244 rx_mbox_areas_p->rxmbox_areas = rx_mbox_p; 2245 hxgep->rx_mbox_areas_p = rx_mbox_areas_p; 2246 2247 goto hxge_map_rxdma_exit; 2248 2249 hxge_map_rxdma_fail1: 2250 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2251 "==> hxge_map_rxdma: unmap rbr,rcr (status 0x%x channel %d i %d)", 2252 status, channel, i)); 2253 i--; 2254 for (; i >= 0; i--) { 2255 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2256 hxge_unmap_rxdma_channel(hxgep, channel, 2257 rbr_rings[i], rcr_rings[i], rx_mbox_p[i]); 2258 } 2259 2260 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2261 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2262 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2263 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2264 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2265 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2266 2267 hxge_map_rxdma_exit: 2268 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2269 "<== hxge_map_rxdma: (status 0x%x channel %d)", status, channel)); 2270 2271 return (status); 2272 } 2273 2274 static void 2275 hxge_unmap_rxdma(p_hxge_t hxgep) 2276 { 2277 int i, ndmas; 2278 uint16_t channel; 2279 p_rx_rbr_rings_t rx_rbr_rings; 2280 p_rx_rbr_ring_t *rbr_rings; 2281 p_rx_rcr_rings_t rx_rcr_rings; 2282 p_rx_rcr_ring_t *rcr_rings; 2283 p_rx_mbox_areas_t rx_mbox_areas_p; 2284 p_rx_mbox_t *rx_mbox_p; 2285 p_hxge_dma_pool_t dma_buf_poolp; 2286 p_hxge_dma_pool_t dma_rbr_cntl_poolp; 2287 p_hxge_dma_pool_t dma_rcr_cntl_poolp; 2288 p_hxge_dma_pool_t dma_mbox_cntl_poolp; 2289 p_hxge_dma_common_t *dma_buf_p; 2290 2291 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_unmap_rxdma")); 2292 2293 dma_buf_poolp = hxgep->rx_buf_pool_p; 2294 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p; 2295 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p; 2296 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p; 2297 2298 if (!dma_buf_poolp->buf_allocated || 2299 !dma_rbr_cntl_poolp->buf_allocated || 2300 !dma_rcr_cntl_poolp->buf_allocated || 2301 !dma_mbox_cntl_poolp->buf_allocated) { 2302 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2303 "<== hxge_unmap_rxdma: NULL buf pointers")); 2304 return; 2305 } 2306 2307 rx_rbr_rings = hxgep->rx_rbr_rings; 2308 rx_rcr_rings = hxgep->rx_rcr_rings; 2309 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 2310 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2311 "<== hxge_unmap_rxdma: NULL pointers")); 2312 return; 2313 } 2314 2315 ndmas = rx_rbr_rings->ndmas; 2316 if (!ndmas) { 2317 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2318 "<== hxge_unmap_rxdma: no channel")); 2319 return; 2320 } 2321 2322 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2323 "==> hxge_unmap_rxdma (ndmas %d)", ndmas)); 2324 2325 rbr_rings = rx_rbr_rings->rbr_rings; 2326 rcr_rings = rx_rcr_rings->rcr_rings; 2327 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 2328 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 2329 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2330 2331 for (i = 0; i < ndmas; i++) { 2332 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2333 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2334 "==> hxge_unmap_rxdma (ndmas %d) channel %d", 2335 ndmas, channel)); 2336 (void) hxge_unmap_rxdma_channel(hxgep, channel, 2337 (p_rx_rbr_ring_t)rbr_rings[i], 2338 (p_rx_rcr_ring_t)rcr_rings[i], 2339 (p_rx_mbox_t)rx_mbox_p[i]); 2340 } 2341 2342 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2343 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2344 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2345 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2346 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2347 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2348 2349 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma")); 2350 } 2351 2352 hxge_status_t 2353 hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2354 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 2355 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p, 2356 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 2357 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2358 { 2359 int status = HXGE_OK; 2360 2361 /* 2362 * Set up and prepare buffer blocks, descriptors and mailbox. 2363 */ 2364 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2365 "==> hxge_map_rxdma_channel (channel %d)", channel)); 2366 2367 /* 2368 * Receive buffer blocks 2369 */ 2370 status = hxge_map_rxdma_channel_buf_ring(hxgep, channel, 2371 dma_buf_p, rbr_p, num_chunks); 2372 if (status != HXGE_OK) { 2373 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2374 "==> hxge_map_rxdma_channel (channel %d): " 2375 "map buffer failed 0x%x", channel, status)); 2376 goto hxge_map_rxdma_channel_exit; 2377 } 2378 2379 /* 2380 * Receive block ring, completion ring and mailbox. 2381 */ 2382 status = hxge_map_rxdma_channel_cfg_ring(hxgep, channel, 2383 dma_rbr_cntl_p, dma_rcr_cntl_p, dma_mbox_cntl_p, 2384 rbr_p, rcr_p, rx_mbox_p); 2385 if (status != HXGE_OK) { 2386 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2387 "==> hxge_map_rxdma_channel (channel %d): " 2388 "map config failed 0x%x", channel, status)); 2389 goto hxge_map_rxdma_channel_fail2; 2390 } 2391 goto hxge_map_rxdma_channel_exit; 2392 2393 hxge_map_rxdma_channel_fail3: 2394 /* Free rbr, rcr */ 2395 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2396 "==> hxge_map_rxdma_channel: free rbr/rcr (status 0x%x channel %d)", 2397 status, channel)); 2398 hxge_unmap_rxdma_channel_cfg_ring(hxgep, *rcr_p, *rx_mbox_p); 2399 2400 hxge_map_rxdma_channel_fail2: 2401 /* Free buffer blocks */ 2402 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2403 "==> hxge_map_rxdma_channel: free rx buffers" 2404 "(hxgep 0x%x status 0x%x channel %d)", 2405 hxgep, status, channel)); 2406 hxge_unmap_rxdma_channel_buf_ring(hxgep, *rbr_p); 2407 2408 status = HXGE_ERROR; 2409 2410 hxge_map_rxdma_channel_exit: 2411 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2412 "<== hxge_map_rxdma_channel: (hxgep 0x%x status 0x%x channel %d)", 2413 hxgep, status, channel)); 2414 2415 return (status); 2416 } 2417 2418 /*ARGSUSED*/ 2419 static void 2420 hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2421 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2422 { 2423 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2424 "==> hxge_unmap_rxdma_channel (channel %d)", channel)); 2425 2426 /* 2427 * unmap receive block ring, completion ring and mailbox. 2428 */ 2429 (void) hxge_unmap_rxdma_channel_cfg_ring(hxgep, rcr_p, rx_mbox_p); 2430 2431 /* unmap buffer blocks */ 2432 (void) hxge_unmap_rxdma_channel_buf_ring(hxgep, rbr_p); 2433 2434 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma_channel")); 2435 } 2436 2437 /*ARGSUSED*/ 2438 static hxge_status_t 2439 hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel, 2440 p_hxge_dma_common_t *dma_rbr_cntl_p, p_hxge_dma_common_t *dma_rcr_cntl_p, 2441 p_hxge_dma_common_t *dma_mbox_cntl_p, p_rx_rbr_ring_t *rbr_p, 2442 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2443 { 2444 p_rx_rbr_ring_t rbrp; 2445 p_rx_rcr_ring_t rcrp; 2446 p_rx_mbox_t mboxp; 2447 p_hxge_dma_common_t cntl_dmap; 2448 p_hxge_dma_common_t dmap; 2449 p_rx_msg_t *rx_msg_ring; 2450 p_rx_msg_t rx_msg_p; 2451 rdc_rbr_cfg_a_t *rcfga_p; 2452 rdc_rbr_cfg_b_t *rcfgb_p; 2453 rdc_rcr_cfg_a_t *cfga_p; 2454 rdc_rcr_cfg_b_t *cfgb_p; 2455 rdc_rx_cfg1_t *cfig1_p; 2456 rdc_rx_cfg2_t *cfig2_p; 2457 rdc_rbr_kick_t *kick_p; 2458 uint32_t dmaaddrp; 2459 uint32_t *rbr_vaddrp; 2460 uint32_t bkaddr; 2461 hxge_status_t status = HXGE_OK; 2462 int i; 2463 uint32_t hxge_port_rcr_size; 2464 2465 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2466 "==> hxge_map_rxdma_channel_cfg_ring")); 2467 2468 cntl_dmap = *dma_rbr_cntl_p; 2469 2470 /* 2471 * Map in the receive block ring 2472 */ 2473 rbrp = *rbr_p; 2474 dmap = (p_hxge_dma_common_t)&rbrp->rbr_desc; 2475 hxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4); 2476 2477 /* 2478 * Zero out buffer block ring descriptors. 2479 */ 2480 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2481 2482 rcfga_p = &(rbrp->rbr_cfga); 2483 rcfgb_p = &(rbrp->rbr_cfgb); 2484 kick_p = &(rbrp->rbr_kick); 2485 rcfga_p->value = 0; 2486 rcfgb_p->value = 0; 2487 kick_p->value = 0; 2488 rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress; 2489 rcfga_p->value = (rbrp->rbr_addr & 2490 (RBR_CFIG_A_STDADDR_MASK | RBR_CFIG_A_STDADDR_BASE_MASK)); 2491 rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT); 2492 2493 /* XXXX: how to choose packet buffer sizes */ 2494 rcfgb_p->bits.bufsz0 = rbrp->pkt_buf_size0; 2495 rcfgb_p->bits.vld0 = 1; 2496 rcfgb_p->bits.bufsz1 = rbrp->pkt_buf_size1; 2497 rcfgb_p->bits.vld1 = 1; 2498 rcfgb_p->bits.bufsz2 = rbrp->pkt_buf_size2; 2499 rcfgb_p->bits.vld2 = 1; 2500 rcfgb_p->bits.bksize = hxgep->rx_bksize_code; 2501 2502 /* 2503 * For each buffer block, enter receive block address to the ring. 2504 */ 2505 rbr_vaddrp = (uint32_t *)dmap->kaddrp; 2506 rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp; 2507 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2508 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2509 "rbr_vaddrp $%p", dma_channel, rbr_vaddrp)); 2510 2511 rx_msg_ring = rbrp->rx_msg_ring; 2512 for (i = 0; i < rbrp->tnblocks; i++) { 2513 rx_msg_p = rx_msg_ring[i]; 2514 rx_msg_p->hxgep = hxgep; 2515 rx_msg_p->rx_rbr_p = rbrp; 2516 bkaddr = (uint32_t) 2517 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2518 RBR_BKADDR_SHIFT)); 2519 rx_msg_p->free = B_FALSE; 2520 rx_msg_p->max_usage_cnt = 0xbaddcafe; 2521 2522 *rbr_vaddrp++ = bkaddr; 2523 } 2524 2525 kick_p->bits.bkadd = rbrp->rbb_max; 2526 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 2527 2528 rbrp->rbr_rd_index = 0; 2529 2530 rbrp->rbr_consumed = 0; 2531 rbrp->rbr_used = 0; 2532 rbrp->rbr_use_bcopy = B_TRUE; 2533 rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0; 2534 2535 /* 2536 * Do bcopy on packets greater than bcopy size once the lo threshold is 2537 * reached. This lo threshold should be less than the hi threshold. 2538 * 2539 * Do bcopy on every packet once the hi threshold is reached. 2540 */ 2541 if (hxge_rx_threshold_lo >= hxge_rx_threshold_hi) { 2542 /* default it to use hi */ 2543 hxge_rx_threshold_lo = hxge_rx_threshold_hi; 2544 } 2545 if (hxge_rx_buf_size_type > HXGE_RBR_TYPE2) { 2546 hxge_rx_buf_size_type = HXGE_RBR_TYPE2; 2547 } 2548 rbrp->rbr_bufsize_type = hxge_rx_buf_size_type; 2549 2550 switch (hxge_rx_threshold_hi) { 2551 default: 2552 case HXGE_RX_COPY_NONE: 2553 /* Do not do bcopy at all */ 2554 rbrp->rbr_use_bcopy = B_FALSE; 2555 rbrp->rbr_threshold_hi = rbrp->rbb_max; 2556 break; 2557 2558 case HXGE_RX_COPY_1: 2559 case HXGE_RX_COPY_2: 2560 case HXGE_RX_COPY_3: 2561 case HXGE_RX_COPY_4: 2562 case HXGE_RX_COPY_5: 2563 case HXGE_RX_COPY_6: 2564 case HXGE_RX_COPY_7: 2565 rbrp->rbr_threshold_hi = 2566 rbrp->rbb_max * (hxge_rx_threshold_hi) / 2567 HXGE_RX_BCOPY_SCALE; 2568 break; 2569 2570 case HXGE_RX_COPY_ALL: 2571 rbrp->rbr_threshold_hi = 0; 2572 break; 2573 } 2574 2575 switch (hxge_rx_threshold_lo) { 2576 default: 2577 case HXGE_RX_COPY_NONE: 2578 /* Do not do bcopy at all */ 2579 if (rbrp->rbr_use_bcopy) { 2580 rbrp->rbr_use_bcopy = B_FALSE; 2581 } 2582 rbrp->rbr_threshold_lo = rbrp->rbb_max; 2583 break; 2584 2585 case HXGE_RX_COPY_1: 2586 case HXGE_RX_COPY_2: 2587 case HXGE_RX_COPY_3: 2588 case HXGE_RX_COPY_4: 2589 case HXGE_RX_COPY_5: 2590 case HXGE_RX_COPY_6: 2591 case HXGE_RX_COPY_7: 2592 rbrp->rbr_threshold_lo = 2593 rbrp->rbb_max * (hxge_rx_threshold_lo) / 2594 HXGE_RX_BCOPY_SCALE; 2595 break; 2596 2597 case HXGE_RX_COPY_ALL: 2598 rbrp->rbr_threshold_lo = 0; 2599 break; 2600 } 2601 2602 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2603 "hxge_map_rxdma_channel_cfg_ring: channel %d rbb_max %d " 2604 "rbrp->rbr_bufsize_type %d rbb_threshold_hi %d " 2605 "rbb_threshold_lo %d", 2606 dma_channel, rbrp->rbb_max, rbrp->rbr_bufsize_type, 2607 rbrp->rbr_threshold_hi, rbrp->rbr_threshold_lo)); 2608 2609 /* Map in the receive completion ring */ 2610 rcrp = (p_rx_rcr_ring_t)KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP); 2611 rcrp->rdc = dma_channel; 2612 rcrp->hxgep = hxgep; 2613 2614 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 2615 rcrp->comp_size = hxge_port_rcr_size; 2616 rcrp->comp_wrap_mask = hxge_port_rcr_size - 1; 2617 2618 rcrp->max_receive_pkts = hxge_max_rx_pkts; 2619 2620 cntl_dmap = *dma_rcr_cntl_p; 2621 2622 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 2623 hxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size, 2624 sizeof (rcr_entry_t)); 2625 rcrp->comp_rd_index = 0; 2626 rcrp->comp_wt_index = 0; 2627 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 2628 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 2629 #if defined(__i386) 2630 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2631 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2632 #else 2633 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2634 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2635 #endif 2636 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 2637 (hxge_port_rcr_size - 1); 2638 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 2639 (hxge_port_rcr_size - 1); 2640 2641 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc); 2642 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3; 2643 2644 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2645 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2646 "rbr_vaddrp $%p rcr_desc_rd_head_p $%p " 2647 "rcr_desc_rd_head_pp $%p rcr_desc_rd_last_p $%p " 2648 "rcr_desc_rd_last_pp $%p ", 2649 dma_channel, rbr_vaddrp, rcrp->rcr_desc_rd_head_p, 2650 rcrp->rcr_desc_rd_head_pp, rcrp->rcr_desc_last_p, 2651 rcrp->rcr_desc_last_pp)); 2652 2653 /* 2654 * Zero out buffer block ring descriptors. 2655 */ 2656 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2657 rcrp->intr_timeout = hxgep->intr_timeout; 2658 rcrp->intr_threshold = hxgep->intr_threshold; 2659 rcrp->full_hdr_flag = B_FALSE; 2660 rcrp->sw_priv_hdr_len = 0; 2661 2662 cfga_p = &(rcrp->rcr_cfga); 2663 cfgb_p = &(rcrp->rcr_cfgb); 2664 cfga_p->value = 0; 2665 cfgb_p->value = 0; 2666 rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress; 2667 2668 cfga_p->value = (rcrp->rcr_addr & 2669 (RCRCFIG_A_STADDR_MASK | RCRCFIG_A_STADDR_BASE_MASK)); 2670 2671 cfga_p->value |= ((uint64_t)rcrp->comp_size << RCRCFIG_A_LEN_SHIF); 2672 2673 /* 2674 * Timeout should be set based on the system clock divider. The 2675 * following timeout value of 1 assumes that the granularity (1000) is 2676 * 3 microseconds running at 300MHz. 2677 */ 2678 cfgb_p->bits.pthres = rcrp->intr_threshold; 2679 cfgb_p->bits.timeout = rcrp->intr_timeout; 2680 cfgb_p->bits.entout = 1; 2681 2682 /* Map in the mailbox */ 2683 cntl_dmap = *dma_mbox_cntl_p; 2684 mboxp = (p_rx_mbox_t)KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP); 2685 dmap = (p_hxge_dma_common_t)&mboxp->rx_mbox; 2686 hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t)); 2687 cfig1_p = (rdc_rx_cfg1_t *)&mboxp->rx_cfg1; 2688 cfig2_p = (rdc_rx_cfg2_t *)&mboxp->rx_cfg2; 2689 cfig1_p->value = cfig2_p->value = 0; 2690 2691 mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress; 2692 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2693 "==> hxge_map_rxdma_channel_cfg_ring: " 2694 "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx", 2695 dma_channel, cfig1_p->value, cfig2_p->value, 2696 mboxp->mbox_addr)); 2697 2698 dmaaddrp = (uint32_t)((dmap->dma_cookie.dmac_laddress >> 32) & 0xfff); 2699 cfig1_p->bits.mbaddr_h = dmaaddrp; 2700 2701 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff); 2702 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 2703 RXDMA_CFIG2_MBADDR_L_MASK); 2704 2705 cfig2_p->bits.mbaddr_l = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT); 2706 2707 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2708 "==> hxge_map_rxdma_channel_cfg_ring: channel %d damaddrp $%p " 2709 "cfg1 0x%016llx cfig2 0x%016llx", 2710 dma_channel, dmaaddrp, cfig1_p->value, cfig2_p->value)); 2711 2712 cfig2_p->bits.full_hdr = rcrp->full_hdr_flag; 2713 cfig2_p->bits.offset = rcrp->sw_priv_hdr_len; 2714 2715 rbrp->rx_rcr_p = rcrp; 2716 rcrp->rx_rbr_p = rbrp; 2717 *rcr_p = rcrp; 2718 *rx_mbox_p = mboxp; 2719 2720 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2721 "<== hxge_map_rxdma_channel_cfg_ring status 0x%08x", status)); 2722 return (status); 2723 } 2724 2725 /*ARGSUSED*/ 2726 static void 2727 hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 2728 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2729 { 2730 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2731 "==> hxge_unmap_rxdma_channel_cfg_ring: channel %d", rcr_p->rdc)); 2732 2733 KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t)); 2734 KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t)); 2735 2736 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2737 "<== hxge_unmap_rxdma_channel_cfg_ring")); 2738 } 2739 2740 static hxge_status_t 2741 hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel, 2742 p_hxge_dma_common_t *dma_buf_p, 2743 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks) 2744 { 2745 p_rx_rbr_ring_t rbrp; 2746 p_hxge_dma_common_t dma_bufp, tmp_bufp; 2747 p_rx_msg_t *rx_msg_ring; 2748 p_rx_msg_t rx_msg_p; 2749 p_mblk_t mblk_p; 2750 2751 rxring_info_t *ring_info; 2752 hxge_status_t status = HXGE_OK; 2753 int i, j, index; 2754 uint32_t size, bsize, nblocks, nmsgs; 2755 2756 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2757 "==> hxge_map_rxdma_channel_buf_ring: channel %d", channel)); 2758 2759 dma_bufp = tmp_bufp = *dma_buf_p; 2760 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2761 " hxge_map_rxdma_channel_buf_ring: channel %d to map %d " 2762 "chunks bufp 0x%016llx", channel, num_chunks, dma_bufp)); 2763 2764 nmsgs = 0; 2765 for (i = 0; i < num_chunks; i++, tmp_bufp++) { 2766 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2767 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2768 "bufp 0x%016llx nblocks %d nmsgs %d", 2769 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs)); 2770 nmsgs += tmp_bufp->nblocks; 2771 } 2772 if (!nmsgs) { 2773 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2774 "<== hxge_map_rxdma_channel_buf_ring: channel %d " 2775 "no msg blocks", channel)); 2776 status = HXGE_ERROR; 2777 goto hxge_map_rxdma_channel_buf_ring_exit; 2778 } 2779 rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (rx_rbr_ring_t), KM_SLEEP); 2780 2781 size = nmsgs * sizeof (p_rx_msg_t); 2782 rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP); 2783 ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t), 2784 KM_SLEEP); 2785 2786 MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER, 2787 (void *) hxgep->interrupt_cookie); 2788 MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER, 2789 (void *) hxgep->interrupt_cookie); 2790 2791 rbrp->rdc = channel; 2792 rbrp->num_blocks = num_chunks; 2793 rbrp->tnblocks = nmsgs; 2794 rbrp->rbb_max = nmsgs; 2795 rbrp->rbr_max_size = nmsgs; 2796 rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1); 2797 2798 /* 2799 * Buffer sizes suggested by NIU architect. 256, 512 and 2K. 2800 */ 2801 2802 switch (hxgep->rx_bksize_code) { 2803 case RBR_BKSIZE_4K: 2804 rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B; 2805 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES; 2806 rbrp->hpi_pkt_buf_size0 = SIZE_256B; 2807 break; 2808 case RBR_BKSIZE_8K: 2809 /* Use 512 to avoid possible rcr_full condition */ 2810 rbrp->pkt_buf_size0 = RBR_BUFSZ0_512B; 2811 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_512_BYTES; 2812 rbrp->hpi_pkt_buf_size0 = SIZE_512B; 2813 break; 2814 } 2815 2816 rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K; 2817 rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES; 2818 rbrp->hpi_pkt_buf_size1 = SIZE_1KB; 2819 2820 rbrp->block_size = hxgep->rx_default_block_size; 2821 2822 if (!hxgep->param_arr[param_accept_jumbo].value) { 2823 rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K; 2824 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES; 2825 rbrp->hpi_pkt_buf_size2 = SIZE_2KB; 2826 } else { 2827 rbrp->hpi_pkt_buf_size2 = SIZE_4KB; 2828 rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K; 2829 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES; 2830 } 2831 2832 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2833 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2834 "actual rbr max %d rbb_max %d nmsgs %d " 2835 "rbrp->block_size %d default_block_size %d " 2836 "(config hxge_rbr_size %d hxge_rbr_spare_size %d)", 2837 channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs, 2838 rbrp->block_size, hxgep->rx_default_block_size, 2839 hxge_rbr_size, hxge_rbr_spare_size)); 2840 2841 /* 2842 * Map in buffers from the buffer pool. 2843 * Note that num_blocks is the num_chunks. For Sparc, there is likely 2844 * only one chunk. For x86, there will be many chunks. 2845 * Loop over chunks. 2846 */ 2847 index = 0; 2848 for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) { 2849 bsize = dma_bufp->block_size; 2850 nblocks = dma_bufp->nblocks; 2851 #if defined(__i386) 2852 ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp; 2853 #else 2854 ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp; 2855 #endif 2856 ring_info->buffer[i].buf_index = i; 2857 ring_info->buffer[i].buf_size = dma_bufp->alength; 2858 ring_info->buffer[i].start_index = index; 2859 #if defined(__i386) 2860 ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp; 2861 #else 2862 ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp; 2863 #endif 2864 2865 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2866 " hxge_map_rxdma_channel_buf_ring: map channel %d " 2867 "chunk %d nblocks %d chunk_size %x block_size 0x%x " 2868 "dma_bufp $%p dvma_addr $%p", channel, i, 2869 dma_bufp->nblocks, 2870 ring_info->buffer[i].buf_size, bsize, dma_bufp, 2871 ring_info->buffer[i].dvma_addr)); 2872 2873 /* loop over blocks within a chunk */ 2874 for (j = 0; j < nblocks; j++) { 2875 if ((rx_msg_p = hxge_allocb(bsize, BPRI_LO, 2876 dma_bufp)) == NULL) { 2877 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2878 "allocb failed (index %d i %d j %d)", 2879 index, i, j)); 2880 goto hxge_map_rxdma_channel_buf_ring_fail1; 2881 } 2882 rx_msg_ring[index] = rx_msg_p; 2883 rx_msg_p->block_index = index; 2884 rx_msg_p->shifted_addr = (uint32_t) 2885 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2886 RBR_BKADDR_SHIFT)); 2887 /* 2888 * Too much output 2889 * HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2890 * "index %d j %d rx_msg_p $%p mblk %p", 2891 * index, j, rx_msg_p, rx_msg_p->rx_mblk_p)); 2892 */ 2893 mblk_p = rx_msg_p->rx_mblk_p; 2894 mblk_p->b_wptr = mblk_p->b_rptr + bsize; 2895 2896 rbrp->rbr_ref_cnt++; 2897 index++; 2898 rx_msg_p->buf_dma.dma_channel = channel; 2899 } 2900 } 2901 if (i < rbrp->num_blocks) { 2902 goto hxge_map_rxdma_channel_buf_ring_fail1; 2903 } 2904 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2905 "hxge_map_rxdma_channel_buf_ring: done buf init " 2906 "channel %d msg block entries %d", channel, index)); 2907 ring_info->block_size_mask = bsize - 1; 2908 rbrp->rx_msg_ring = rx_msg_ring; 2909 rbrp->dma_bufp = dma_buf_p; 2910 rbrp->ring_info = ring_info; 2911 2912 status = hxge_rxbuf_index_info_init(hxgep, rbrp); 2913 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, " hxge_map_rxdma_channel_buf_ring: " 2914 "channel %d done buf info init", channel)); 2915 2916 /* 2917 * Finally, permit hxge_freeb() to call hxge_post_page(). 2918 */ 2919 rbrp->rbr_state = RBR_POSTING; 2920 2921 *rbr_p = rbrp; 2922 2923 goto hxge_map_rxdma_channel_buf_ring_exit; 2924 2925 hxge_map_rxdma_channel_buf_ring_fail1: 2926 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2927 " hxge_map_rxdma_channel_buf_ring: failed channel (0x%x)", 2928 channel, status)); 2929 2930 index--; 2931 for (; index >= 0; index--) { 2932 rx_msg_p = rx_msg_ring[index]; 2933 if (rx_msg_p != NULL) { 2934 freeb(rx_msg_p->rx_mblk_p); 2935 rx_msg_ring[index] = NULL; 2936 } 2937 } 2938 2939 hxge_map_rxdma_channel_buf_ring_fail: 2940 MUTEX_DESTROY(&rbrp->post_lock); 2941 MUTEX_DESTROY(&rbrp->lock); 2942 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 2943 KMEM_FREE(rx_msg_ring, size); 2944 KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t)); 2945 2946 status = HXGE_ERROR; 2947 2948 hxge_map_rxdma_channel_buf_ring_exit: 2949 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2950 "<== hxge_map_rxdma_channel_buf_ring status 0x%08x", status)); 2951 2952 return (status); 2953 } 2954 2955 /*ARGSUSED*/ 2956 static void 2957 hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 2958 p_rx_rbr_ring_t rbr_p) 2959 { 2960 p_rx_msg_t *rx_msg_ring; 2961 p_rx_msg_t rx_msg_p; 2962 rxring_info_t *ring_info; 2963 int i; 2964 uint32_t size; 2965 2966 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2967 "==> hxge_unmap_rxdma_channel_buf_ring")); 2968 if (rbr_p == NULL) { 2969 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2970 "<== hxge_unmap_rxdma_channel_buf_ring: NULL rbrp")); 2971 return; 2972 } 2973 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2974 "==> hxge_unmap_rxdma_channel_buf_ring: channel %d", rbr_p->rdc)); 2975 2976 rx_msg_ring = rbr_p->rx_msg_ring; 2977 ring_info = rbr_p->ring_info; 2978 2979 if (rx_msg_ring == NULL || ring_info == NULL) { 2980 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2981 "<== hxge_unmap_rxdma_channel_buf_ring: " 2982 "rx_msg_ring $%p ring_info $%p", rx_msg_p, ring_info)); 2983 return; 2984 } 2985 2986 size = rbr_p->tnblocks * sizeof (p_rx_msg_t); 2987 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2988 " hxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d " 2989 "tnblocks %d (max %d) size ptrs %d ", rbr_p->rdc, rbr_p->num_blocks, 2990 rbr_p->tnblocks, rbr_p->rbr_max_size, size)); 2991 2992 for (i = 0; i < rbr_p->tnblocks; i++) { 2993 rx_msg_p = rx_msg_ring[i]; 2994 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2995 " hxge_unmap_rxdma_channel_buf_ring: " 2996 "rx_msg_p $%p", rx_msg_p)); 2997 if (rx_msg_p != NULL) { 2998 freeb(rx_msg_p->rx_mblk_p); 2999 rx_msg_ring[i] = NULL; 3000 } 3001 } 3002 3003 /* 3004 * We no longer may use the mutex <post_lock>. By setting 3005 * <rbr_state> to anything but POSTING, we prevent 3006 * hxge_post_page() from accessing a dead mutex. 3007 */ 3008 rbr_p->rbr_state = RBR_UNMAPPING; 3009 MUTEX_DESTROY(&rbr_p->post_lock); 3010 3011 MUTEX_DESTROY(&rbr_p->lock); 3012 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 3013 KMEM_FREE(rx_msg_ring, size); 3014 3015 if (rbr_p->rbr_ref_cnt == 0) { 3016 /* This is the normal state of affairs. */ 3017 KMEM_FREE(rbr_p, sizeof (*rbr_p)); 3018 } else { 3019 /* 3020 * Some of our buffers are still being used. 3021 * Therefore, tell hxge_freeb() this ring is 3022 * unmapped, so it may free <rbr_p> for us. 3023 */ 3024 rbr_p->rbr_state = RBR_UNMAPPED; 3025 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3026 "unmap_rxdma_buf_ring: %d %s outstanding.", 3027 rbr_p->rbr_ref_cnt, 3028 rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs")); 3029 } 3030 3031 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3032 "<== hxge_unmap_rxdma_channel_buf_ring")); 3033 } 3034 3035 static hxge_status_t 3036 hxge_rxdma_hw_start_common(p_hxge_t hxgep) 3037 { 3038 hxge_status_t status = HXGE_OK; 3039 3040 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 3041 3042 /* 3043 * Load the sharable parameters by writing to the function zero control 3044 * registers. These FZC registers should be initialized only once for 3045 * the entire chip. 3046 */ 3047 (void) hxge_init_fzc_rx_common(hxgep); 3048 3049 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 3050 3051 return (status); 3052 } 3053 3054 static hxge_status_t 3055 hxge_rxdma_hw_start(p_hxge_t hxgep) 3056 { 3057 int i, ndmas; 3058 uint16_t channel; 3059 p_rx_rbr_rings_t rx_rbr_rings; 3060 p_rx_rbr_ring_t *rbr_rings; 3061 p_rx_rcr_rings_t rx_rcr_rings; 3062 p_rx_rcr_ring_t *rcr_rings; 3063 p_rx_mbox_areas_t rx_mbox_areas_p; 3064 p_rx_mbox_t *rx_mbox_p; 3065 hxge_status_t status = HXGE_OK; 3066 3067 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start")); 3068 3069 rx_rbr_rings = hxgep->rx_rbr_rings; 3070 rx_rcr_rings = hxgep->rx_rcr_rings; 3071 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 3072 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3073 "<== hxge_rxdma_hw_start: NULL ring pointers")); 3074 return (HXGE_ERROR); 3075 } 3076 3077 ndmas = rx_rbr_rings->ndmas; 3078 if (ndmas == 0) { 3079 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3080 "<== hxge_rxdma_hw_start: no dma channel allocated")); 3081 return (HXGE_ERROR); 3082 } 3083 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3084 "==> hxge_rxdma_hw_start (ndmas %d)", ndmas)); 3085 3086 /* 3087 * Scrub the RDC Rx DMA Prefetch Buffer Command. 3088 */ 3089 for (i = 0; i < 128; i++) { 3090 HXGE_REG_WR64(hxgep->hpi_handle, RDC_PREF_CMD, i); 3091 } 3092 3093 /* 3094 * Scrub Rx DMA Shadow Tail Command. 3095 */ 3096 for (i = 0; i < 64; i++) { 3097 HXGE_REG_WR64(hxgep->hpi_handle, RDC_SHADOW_CMD, i); 3098 } 3099 3100 /* 3101 * Scrub Rx DMA Control Fifo Command. 3102 */ 3103 for (i = 0; i < 512; i++) { 3104 HXGE_REG_WR64(hxgep->hpi_handle, RDC_CTRL_FIFO_CMD, i); 3105 } 3106 3107 /* 3108 * Scrub Rx DMA Data Fifo Command. 3109 */ 3110 for (i = 0; i < 1536; i++) { 3111 HXGE_REG_WR64(hxgep->hpi_handle, RDC_DATA_FIFO_CMD, i); 3112 } 3113 3114 /* 3115 * Reset the FIFO Error Stat. 3116 */ 3117 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_STAT, 0xFF); 3118 3119 /* Set the error mask to receive interrupts */ 3120 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 3121 3122 rbr_rings = rx_rbr_rings->rbr_rings; 3123 rcr_rings = rx_rcr_rings->rcr_rings; 3124 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 3125 if (rx_mbox_areas_p) { 3126 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 3127 } 3128 3129 for (i = 0; i < ndmas; i++) { 3130 channel = rbr_rings[i]->rdc; 3131 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3132 "==> hxge_rxdma_hw_start (ndmas %d) channel %d", 3133 ndmas, channel)); 3134 status = hxge_rxdma_start_channel(hxgep, channel, 3135 (p_rx_rbr_ring_t)rbr_rings[i], 3136 (p_rx_rcr_ring_t)rcr_rings[i], 3137 (p_rx_mbox_t)rx_mbox_p[i], rbr_rings[i]->rbb_max); 3138 if (status != HXGE_OK) { 3139 goto hxge_rxdma_hw_start_fail1; 3140 } 3141 } 3142 3143 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start: " 3144 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3145 rx_rbr_rings, rx_rcr_rings)); 3146 goto hxge_rxdma_hw_start_exit; 3147 3148 hxge_rxdma_hw_start_fail1: 3149 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3150 "==> hxge_rxdma_hw_start: disable " 3151 "(status 0x%x channel %d i %d)", status, channel, i)); 3152 for (; i >= 0; i--) { 3153 channel = rbr_rings[i]->rdc; 3154 (void) hxge_rxdma_stop_channel(hxgep, channel); 3155 } 3156 3157 hxge_rxdma_hw_start_exit: 3158 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3159 "==> hxge_rxdma_hw_start: (status 0x%x)", status)); 3160 return (status); 3161 } 3162 3163 static void 3164 hxge_rxdma_hw_stop(p_hxge_t hxgep) 3165 { 3166 int i, ndmas; 3167 uint16_t channel; 3168 p_rx_rbr_rings_t rx_rbr_rings; 3169 p_rx_rbr_ring_t *rbr_rings; 3170 p_rx_rcr_rings_t rx_rcr_rings; 3171 3172 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop")); 3173 3174 rx_rbr_rings = hxgep->rx_rbr_rings; 3175 rx_rcr_rings = hxgep->rx_rcr_rings; 3176 3177 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 3178 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3179 "<== hxge_rxdma_hw_stop: NULL ring pointers")); 3180 return; 3181 } 3182 3183 ndmas = rx_rbr_rings->ndmas; 3184 if (!ndmas) { 3185 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3186 "<== hxge_rxdma_hw_stop: no dma channel allocated")); 3187 return; 3188 } 3189 3190 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3191 "==> hxge_rxdma_hw_stop (ndmas %d)", ndmas)); 3192 3193 rbr_rings = rx_rbr_rings->rbr_rings; 3194 for (i = 0; i < ndmas; i++) { 3195 channel = rbr_rings[i]->rdc; 3196 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3197 "==> hxge_rxdma_hw_stop (ndmas %d) channel %d", 3198 ndmas, channel)); 3199 (void) hxge_rxdma_stop_channel(hxgep, channel); 3200 } 3201 3202 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop: " 3203 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3204 rx_rbr_rings, rx_rcr_rings)); 3205 3206 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_hw_stop")); 3207 } 3208 3209 static hxge_status_t 3210 hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 3211 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 3212 int n_init_kick) 3213 { 3214 hpi_handle_t handle; 3215 hpi_status_t rs = HPI_SUCCESS; 3216 rdc_stat_t cs; 3217 rdc_int_mask_t ent_mask; 3218 hxge_status_t status = HXGE_OK; 3219 3220 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel")); 3221 3222 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3223 3224 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "hxge_rxdma_start_channel: " 3225 "hpi handle addr $%p acc $%p", 3226 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3227 3228 /* Reset RXDMA channel */ 3229 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3230 if (rs != HPI_SUCCESS) { 3231 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3232 "==> hxge_rxdma_start_channel: " 3233 "reset rxdma failed (0x%08x channel %d)", 3234 status, channel)); 3235 return (HXGE_ERROR | rs); 3236 } 3237 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3238 "==> hxge_rxdma_start_channel: reset done: channel %d", channel)); 3239 3240 /* 3241 * Initialize the RXDMA channel specific FZC control configurations. 3242 * These FZC registers are pertaining to each RX channel (logical 3243 * pages). 3244 */ 3245 status = hxge_init_fzc_rxdma_channel(hxgep, 3246 channel, rbr_p, rcr_p, mbox_p); 3247 if (status != HXGE_OK) { 3248 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3249 "==> hxge_rxdma_start_channel: " 3250 "init fzc rxdma failed (0x%08x channel %d)", 3251 status, channel)); 3252 return (status); 3253 } 3254 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3255 "==> hxge_rxdma_start_channel: fzc done")); 3256 3257 /* 3258 * Zero out the shadow and prefetch ram. 3259 */ 3260 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3261 "==> hxge_rxdma_start_channel: ram done")); 3262 3263 /* Set up the interrupt event masks. */ 3264 ent_mask.value = 0; 3265 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3266 if (rs != HPI_SUCCESS) { 3267 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3268 "==> hxge_rxdma_start_channel: " 3269 "init rxdma event masks failed (0x%08x channel %d)", 3270 status, channel)); 3271 return (HXGE_ERROR | rs); 3272 } 3273 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3274 "event done: channel %d (mask 0x%016llx)", 3275 channel, ent_mask.value)); 3276 3277 /* 3278 * Load RXDMA descriptors, buffers, mailbox, initialise the receive DMA 3279 * channels and enable each DMA channel. 3280 */ 3281 status = hxge_enable_rxdma_channel(hxgep, 3282 channel, rbr_p, rcr_p, mbox_p, n_init_kick); 3283 if (status != HXGE_OK) { 3284 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3285 " hxge_rxdma_start_channel: " 3286 " init enable rxdma failed (0x%08x channel %d)", 3287 status, channel)); 3288 return (status); 3289 } 3290 3291 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3292 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3293 3294 /* 3295 * Initialize the receive DMA control and status register 3296 * Note that rdc_stat HAS to be set after RBR and RCR rings are set 3297 */ 3298 cs.value = 0; 3299 cs.bits.mex = 1; 3300 cs.bits.rcr_thres = 1; 3301 cs.bits.rcr_to = 1; 3302 cs.bits.rbr_empty = 1; 3303 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3304 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3305 "channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value)); 3306 if (status != HXGE_OK) { 3307 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3308 "==> hxge_rxdma_start_channel: " 3309 "init rxdma control register failed (0x%08x channel %d", 3310 status, channel)); 3311 return (status); 3312 } 3313 3314 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3315 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3316 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3317 "==> hxge_rxdma_start_channel: enable done")); 3318 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_start_channel")); 3319 3320 return (HXGE_OK); 3321 } 3322 3323 static hxge_status_t 3324 hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel) 3325 { 3326 hpi_handle_t handle; 3327 hpi_status_t rs = HPI_SUCCESS; 3328 rdc_stat_t cs; 3329 rdc_int_mask_t ent_mask; 3330 hxge_status_t status = HXGE_OK; 3331 3332 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel")); 3333 3334 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3335 3336 HXGE_DEBUG_MSG((hxgep, RX_CTL, "hxge_rxdma_stop_channel: " 3337 "hpi handle addr $%p acc $%p", 3338 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3339 3340 /* Reset RXDMA channel */ 3341 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3342 if (rs != HPI_SUCCESS) { 3343 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3344 " hxge_rxdma_stop_channel: " 3345 " reset rxdma failed (0x%08x channel %d)", 3346 rs, channel)); 3347 return (HXGE_ERROR | rs); 3348 } 3349 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3350 "==> hxge_rxdma_stop_channel: reset done")); 3351 3352 /* Set up the interrupt event masks. */ 3353 ent_mask.value = RDC_INT_MASK_ALL; 3354 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3355 if (rs != HPI_SUCCESS) { 3356 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3357 "==> hxge_rxdma_stop_channel: " 3358 "set rxdma event masks failed (0x%08x channel %d)", 3359 rs, channel)); 3360 return (HXGE_ERROR | rs); 3361 } 3362 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3363 "==> hxge_rxdma_stop_channel: event done")); 3364 3365 /* Initialize the receive DMA control and status register */ 3366 cs.value = 0; 3367 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3368 3369 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel: control " 3370 " to default (all 0s) 0x%08x", cs.value)); 3371 3372 if (status != HXGE_OK) { 3373 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3374 " hxge_rxdma_stop_channel: init rxdma" 3375 " control register failed (0x%08x channel %d", 3376 status, channel)); 3377 return (status); 3378 } 3379 3380 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3381 "==> hxge_rxdma_stop_channel: control done")); 3382 3383 /* disable dma channel */ 3384 status = hxge_disable_rxdma_channel(hxgep, channel); 3385 3386 if (status != HXGE_OK) { 3387 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3388 " hxge_rxdma_stop_channel: " 3389 " init enable rxdma failed (0x%08x channel %d)", 3390 status, channel)); 3391 return (status); 3392 } 3393 3394 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3395 "==> hxge_rxdma_stop_channel: disable done")); 3396 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_channel")); 3397 3398 return (HXGE_OK); 3399 } 3400 3401 hxge_status_t 3402 hxge_rxdma_handle_sys_errors(p_hxge_t hxgep) 3403 { 3404 hpi_handle_t handle; 3405 p_hxge_rdc_sys_stats_t statsp; 3406 rdc_fifo_err_stat_t stat; 3407 hxge_status_t status = HXGE_OK; 3408 3409 handle = hxgep->hpi_handle; 3410 statsp = (p_hxge_rdc_sys_stats_t)&hxgep->statsp->rdc_sys_stats; 3411 3412 /* Clear the int_dbg register in case it is an injected err */ 3413 HXGE_REG_WR64(handle, RDC_FIFO_ERR_INT_DBG, 0x0); 3414 3415 /* Get the error status and clear the register */ 3416 HXGE_REG_RD64(handle, RDC_FIFO_ERR_STAT, &stat.value); 3417 HXGE_REG_WR64(handle, RDC_FIFO_ERR_STAT, stat.value); 3418 3419 if (stat.bits.rx_ctrl_fifo_sec) { 3420 statsp->ctrl_fifo_sec++; 3421 if (statsp->ctrl_fifo_sec == 1) 3422 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3423 "==> hxge_rxdma_handle_sys_errors: " 3424 "rx_ctrl_fifo_sec")); 3425 } 3426 3427 if (stat.bits.rx_ctrl_fifo_ded) { 3428 /* Global fatal error encountered */ 3429 statsp->ctrl_fifo_ded++; 3430 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3431 HXGE_FM_EREPORT_RDMC_CTRL_FIFO_DED); 3432 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3433 "==> hxge_rxdma_handle_sys_errors: " 3434 "fatal error: rx_ctrl_fifo_ded error")); 3435 } 3436 3437 if (stat.bits.rx_data_fifo_sec) { 3438 statsp->data_fifo_sec++; 3439 if (statsp->data_fifo_sec == 1) 3440 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3441 "==> hxge_rxdma_handle_sys_errors: " 3442 "rx_data_fifo_sec")); 3443 } 3444 3445 if (stat.bits.rx_data_fifo_ded) { 3446 /* Global fatal error encountered */ 3447 statsp->data_fifo_ded++; 3448 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3449 HXGE_FM_EREPORT_RDMC_DATA_FIFO_DED); 3450 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3451 "==> hxge_rxdma_handle_sys_errors: " 3452 "fatal error: rx_data_fifo_ded error")); 3453 } 3454 3455 if (stat.bits.rx_ctrl_fifo_ded || stat.bits.rx_data_fifo_ded) { 3456 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3457 " hxge_rxdma_handle_sys_errors: fatal error\n")); 3458 status = hxge_rx_port_fatal_err_recover(hxgep); 3459 if (status == HXGE_OK) { 3460 FM_SERVICE_RESTORED(hxgep); 3461 } 3462 } 3463 3464 return (HXGE_OK); 3465 } 3466 3467 static hxge_status_t 3468 hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel) 3469 { 3470 hpi_handle_t handle; 3471 hpi_status_t rs = HPI_SUCCESS; 3472 hxge_status_t status = HXGE_OK; 3473 p_rx_rbr_ring_t rbrp; 3474 p_rx_rcr_ring_t rcrp; 3475 p_rx_mbox_t mboxp; 3476 rdc_int_mask_t ent_mask; 3477 p_hxge_dma_common_t dmap; 3478 int ring_idx; 3479 p_rx_msg_t rx_msg_p; 3480 int i; 3481 uint32_t hxge_port_rcr_size; 3482 uint64_t tmp; 3483 int n_init_kick = 0; 3484 3485 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_fatal_err_recover")); 3486 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3487 "Recovering from RxDMAChannel#%d error...", channel)); 3488 3489 /* 3490 * Stop the dma channel waits for the stop done. If the stop done bit 3491 * is not set, then create an error. 3492 */ 3493 3494 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3495 3496 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Rx DMA stop...")); 3497 3498 ring_idx = hxge_rxdma_get_ring_index(hxgep, channel); 3499 rbrp = (p_rx_rbr_ring_t)hxgep->rx_rbr_rings->rbr_rings[ring_idx]; 3500 rcrp = (p_rx_rcr_ring_t)hxgep->rx_rcr_rings->rcr_rings[ring_idx]; 3501 3502 MUTEX_ENTER(&rcrp->lock); 3503 MUTEX_ENTER(&rbrp->lock); 3504 3505 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA channel...")); 3506 3507 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 3508 if (rs != HPI_SUCCESS) { 3509 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3510 "hxge_disable_rxdma_channel:failed")); 3511 goto fail; 3512 } 3513 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA interrupt...")); 3514 3515 /* Disable interrupt */ 3516 ent_mask.value = RDC_INT_MASK_ALL; 3517 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3518 if (rs != HPI_SUCCESS) { 3519 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3520 "Set rxdma event masks failed (channel %d)", channel)); 3521 } 3522 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel reset...")); 3523 3524 /* Reset RXDMA channel */ 3525 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3526 if (rs != HPI_SUCCESS) { 3527 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3528 "Reset rxdma failed (channel %d)", channel)); 3529 goto fail; 3530 } 3531 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 3532 mboxp = (p_rx_mbox_t)hxgep->rx_mbox_areas_p->rxmbox_areas[ring_idx]; 3533 3534 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 3535 rbrp->rbr_rd_index = 0; 3536 3537 rcrp->comp_rd_index = 0; 3538 rcrp->comp_wt_index = 0; 3539 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 3540 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 3541 #if defined(__i386) 3542 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3543 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3544 #else 3545 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3546 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3547 #endif 3548 3549 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 3550 (hxge_port_rcr_size - 1); 3551 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 3552 (hxge_port_rcr_size - 1); 3553 3554 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc); 3555 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3; 3556 3557 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 3558 bzero((caddr_t)dmap->kaddrp, dmap->alength); 3559 3560 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rbr entries = %d\n", 3561 rbrp->rbr_max_size)); 3562 3563 /* Count the number of buffers owned by the hardware at this moment */ 3564 for (i = 0; i < rbrp->rbr_max_size; i++) { 3565 rx_msg_p = rbrp->rx_msg_ring[i]; 3566 if (rx_msg_p->ref_cnt == 1) { 3567 n_init_kick++; 3568 } 3569 } 3570 3571 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel re-start...")); 3572 3573 /* 3574 * This is error recover! Some buffers are owned by the hardware and 3575 * the rest are owned by the apps. We should only kick in those 3576 * owned by the hardware initially. The apps will post theirs 3577 * eventually. 3578 */ 3579 status = hxge_rxdma_start_channel(hxgep, channel, rbrp, rcrp, mboxp, 3580 n_init_kick); 3581 if (status != HXGE_OK) { 3582 goto fail; 3583 } 3584 3585 /* 3586 * The DMA channel may disable itself automatically. 3587 * The following is a work-around. 3588 */ 3589 HXGE_REG_RD64(handle, RDC_RX_CFG1, &tmp); 3590 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 3591 if (rs != HPI_SUCCESS) { 3592 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3593 "hpi_rxdma_cfg_rdc_enable (channel %d)", channel)); 3594 } 3595 3596 MUTEX_EXIT(&rbrp->lock); 3597 MUTEX_EXIT(&rcrp->lock); 3598 3599 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3600 "Recovery Successful, RxDMAChannel#%d Restored", channel)); 3601 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_fatal_err_recover")); 3602 3603 return (HXGE_OK); 3604 3605 fail: 3606 MUTEX_EXIT(&rbrp->lock); 3607 MUTEX_EXIT(&rcrp->lock); 3608 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3609 3610 return (HXGE_ERROR | rs); 3611 } 3612 3613 static hxge_status_t 3614 hxge_rx_port_fatal_err_recover(p_hxge_t hxgep) 3615 { 3616 hxge_status_t status = HXGE_OK; 3617 p_hxge_dma_common_t *dma_buf_p; 3618 uint16_t channel; 3619 int ndmas; 3620 int i; 3621 block_reset_t reset_reg; 3622 p_rx_rcr_ring_t rcrp; 3623 p_rx_rbr_ring_t rbrp; 3624 3625 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_port_fatal_err_recover")); 3626 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovering from RDC error ...")); 3627 3628 /* Reset RDC block from PEU for this fatal error */ 3629 reset_reg.value = 0; 3630 reset_reg.bits.rdc_rst = 1; 3631 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 3632 3633 /* Disable RxMAC */ 3634 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxMAC...\n")); 3635 if (hxge_rx_vmac_disable(hxgep) != HXGE_OK) 3636 goto fail; 3637 3638 HXGE_DELAY(1000); 3639 3640 /* Restore any common settings after PEU reset */ 3641 if (hxge_rxdma_hw_start_common(hxgep) != HXGE_OK) 3642 goto fail; 3643 3644 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Stop all RxDMA channels...")); 3645 3646 ndmas = hxgep->rx_buf_pool_p->ndmas; 3647 dma_buf_p = hxgep->rx_buf_pool_p->dma_buf_pool_p; 3648 3649 for (i = 0; i < ndmas; i++) { 3650 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 3651 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 3652 rbrp = rcrp->rx_rbr_p; 3653 3654 MUTEX_ENTER(&rbrp->post_lock); 3655 /* This function needs to be inside the post_lock */ 3656 if (hxge_rxdma_fatal_err_recover(hxgep, channel) != HXGE_OK) { 3657 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3658 "Could not recover channel %d", channel)); 3659 } 3660 MUTEX_EXIT(&rbrp->post_lock); 3661 } 3662 3663 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Reset RxMAC...")); 3664 3665 /* Reset RxMAC */ 3666 if (hxge_rx_vmac_reset(hxgep) != HXGE_OK) { 3667 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3668 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3669 goto fail; 3670 } 3671 3672 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-initialize RxMAC...")); 3673 3674 /* Re-Initialize RxMAC */ 3675 if ((status = hxge_rx_vmac_init(hxgep)) != HXGE_OK) { 3676 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3677 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3678 goto fail; 3679 } 3680 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-enable RxMAC...")); 3681 3682 /* Re-enable RxMAC */ 3683 if ((status = hxge_rx_vmac_enable(hxgep)) != HXGE_OK) { 3684 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3685 "hxge_rx_port_fatal_err_recover: Failed to enable RxMAC")); 3686 goto fail; 3687 } 3688 3689 /* Reset the error mask since PEU reset cleared it */ 3690 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 3691 3692 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3693 "Recovery Successful, RxPort Restored")); 3694 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_port_fatal_err_recover")); 3695 3696 return (HXGE_OK); 3697 fail: 3698 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3699 return (status); 3700 } 3701 3702 static void 3703 hxge_rbr_empty_restore(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p) 3704 { 3705 hpi_status_t hpi_status; 3706 hxge_status_t status; 3707 int i; 3708 p_hxge_rx_ring_stats_t rdc_stats; 3709 3710 rdc_stats = &hxgep->statsp->rdc_stats[rx_rbr_p->rdc]; 3711 rdc_stats->rbr_empty_restore++; 3712 rx_rbr_p->rbr_is_empty = B_FALSE; 3713 3714 /* 3715 * Complete the processing for the RBR Empty by: 3716 * 0) kicking back HXGE_RBR_EMPTY_THRESHOLD 3717 * packets. 3718 * 1) Disable the RX vmac. 3719 * 2) Re-enable the affected DMA channel. 3720 * 3) Re-enable the RX vmac. 3721 */ 3722 3723 /* 3724 * Disable the RX VMAC, but setting the framelength 3725 * to 0, since there is a hardware bug when disabling 3726 * the vmac. 3727 */ 3728 MUTEX_ENTER(hxgep->genlock); 3729 (void) hpi_vmac_rx_set_framesize( 3730 HXGE_DEV_HPI_HANDLE(hxgep), (uint16_t)0); 3731 3732 hpi_status = hpi_rxdma_cfg_rdc_enable( 3733 HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc); 3734 if (hpi_status != HPI_SUCCESS) { 3735 rdc_stats->rbr_empty_fail++; 3736 3737 /* Assume we are already inside the post_lock */ 3738 status = hxge_rxdma_fatal_err_recover(hxgep, rx_rbr_p->rdc); 3739 if (status != HXGE_OK) { 3740 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3741 "hxge(%d): channel(%d) is empty.", 3742 hxgep->instance, rx_rbr_p->rdc)); 3743 } 3744 } 3745 3746 for (i = 0; i < 1024; i++) { 3747 uint64_t value; 3748 RXDMA_REG_READ64(HXGE_DEV_HPI_HANDLE(hxgep), 3749 RDC_STAT, i & 3, &value); 3750 } 3751 3752 /* 3753 * Re-enable the RX VMAC. 3754 */ 3755 (void) hpi_vmac_rx_set_framesize(HXGE_DEV_HPI_HANDLE(hxgep), 3756 (uint16_t)hxgep->vmac.maxframesize); 3757 MUTEX_EXIT(hxgep->genlock); 3758 } 3759