1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <hxge_impl.h> 27 #include <hxge_rxdma.h> 28 29 /* 30 * Number of blocks to accumulate before re-enabling DMA 31 * when we get RBR empty. 32 */ 33 #define HXGE_RBR_EMPTY_THRESHOLD 128 34 35 /* 36 * Globals: tunable parameters (/etc/system or adb) 37 * 38 */ 39 extern uint32_t hxge_rbr_size; 40 extern uint32_t hxge_rcr_size; 41 extern uint32_t hxge_rbr_spare_size; 42 extern uint32_t hxge_mblks_pending; 43 44 /* 45 * Tunable to reduce the amount of time spent in the 46 * ISR doing Rx Processing. 47 */ 48 extern uint32_t hxge_max_rx_pkts; 49 50 /* 51 * Tunables to manage the receive buffer blocks. 52 * 53 * hxge_rx_threshold_hi: copy all buffers. 54 * hxge_rx_bcopy_size_type: receive buffer block size type. 55 * hxge_rx_threshold_lo: copy only up to tunable block size type. 56 */ 57 extern hxge_rxbuf_threshold_t hxge_rx_threshold_hi; 58 extern hxge_rxbuf_type_t hxge_rx_buf_size_type; 59 extern hxge_rxbuf_threshold_t hxge_rx_threshold_lo; 60 61 /* 62 * Static local functions. 63 */ 64 static hxge_status_t hxge_map_rxdma(p_hxge_t hxgep); 65 static void hxge_unmap_rxdma(p_hxge_t hxgep); 66 static hxge_status_t hxge_rxdma_hw_start_common(p_hxge_t hxgep); 67 static hxge_status_t hxge_rxdma_hw_start(p_hxge_t hxgep); 68 static void hxge_rxdma_hw_stop(p_hxge_t hxgep); 69 static hxge_status_t hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 70 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 71 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p, 72 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 73 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 74 static void hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 75 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 76 static hxge_status_t hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, 77 uint16_t dma_channel, p_hxge_dma_common_t *dma_rbr_cntl_p, 78 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 79 p_rx_rbr_ring_t *rbr_p, p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 80 static void hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 81 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 82 static hxge_status_t hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, 83 uint16_t channel, p_hxge_dma_common_t *dma_buf_p, 84 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks); 85 static void hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 86 p_rx_rbr_ring_t rbr_p); 87 static hxge_status_t hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 88 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 89 int n_init_kick); 90 static hxge_status_t hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel); 91 static mblk_t *hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 92 p_rx_rcr_ring_t *rcr_p, rdc_stat_t cs); 93 static void hxge_receive_packet(p_hxge_t hxgep, p_rx_rcr_ring_t rcr_p, 94 p_rcr_entry_t rcr_desc_rd_head_p, boolean_t *multi_p, 95 mblk_t ** mp, mblk_t ** mp_cont, uint32_t *invalid_rcr_entry); 96 static hxge_status_t hxge_disable_rxdma_channel(p_hxge_t hxgep, 97 uint16_t channel); 98 static p_rx_msg_t hxge_allocb(size_t, uint32_t, p_hxge_dma_common_t); 99 static void hxge_freeb(p_rx_msg_t); 100 static void hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, 101 p_hxge_ldv_t ldvp, rdc_stat_t cs); 102 static hxge_status_t hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, 103 p_hxge_ldv_t ldvp, rdc_stat_t cs); 104 static hxge_status_t hxge_rxbuf_index_info_init(p_hxge_t hxgep, 105 p_rx_rbr_ring_t rx_dmap); 106 static hxge_status_t hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, 107 uint16_t channel); 108 static hxge_status_t hxge_rx_port_fatal_err_recover(p_hxge_t hxgep); 109 static void hxge_rbr_empty_restore(p_hxge_t hxgep, 110 p_rx_rbr_ring_t rx_rbr_p); 111 112 hxge_status_t 113 hxge_init_rxdma_channels(p_hxge_t hxgep) 114 { 115 hxge_status_t status = HXGE_OK; 116 block_reset_t reset_reg; 117 118 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_init_rxdma_channels")); 119 120 /* Reset RDC block from PEU to clear any previous state */ 121 reset_reg.value = 0; 122 reset_reg.bits.rdc_rst = 1; 123 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 124 HXGE_DELAY(1000); 125 126 status = hxge_map_rxdma(hxgep); 127 if (status != HXGE_OK) { 128 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 129 "<== hxge_init_rxdma: status 0x%x", status)); 130 return (status); 131 } 132 133 status = hxge_rxdma_hw_start_common(hxgep); 134 if (status != HXGE_OK) { 135 hxge_unmap_rxdma(hxgep); 136 } 137 138 status = hxge_rxdma_hw_start(hxgep); 139 if (status != HXGE_OK) { 140 hxge_unmap_rxdma(hxgep); 141 } 142 143 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 144 "<== hxge_init_rxdma_channels: status 0x%x", status)); 145 return (status); 146 } 147 148 void 149 hxge_uninit_rxdma_channels(p_hxge_t hxgep) 150 { 151 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_uninit_rxdma_channels")); 152 153 hxge_rxdma_hw_stop(hxgep); 154 hxge_unmap_rxdma(hxgep); 155 156 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_uinit_rxdma_channels")); 157 } 158 159 hxge_status_t 160 hxge_init_rxdma_channel_cntl_stat(p_hxge_t hxgep, uint16_t channel, 161 rdc_stat_t *cs_p) 162 { 163 hpi_handle_t handle; 164 hpi_status_t rs = HPI_SUCCESS; 165 hxge_status_t status = HXGE_OK; 166 167 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 168 "<== hxge_init_rxdma_channel_cntl_stat")); 169 170 handle = HXGE_DEV_HPI_HANDLE(hxgep); 171 rs = hpi_rxdma_control_status(handle, OP_SET, channel, cs_p); 172 173 if (rs != HPI_SUCCESS) { 174 status = HXGE_ERROR | rs; 175 } 176 return (status); 177 } 178 179 180 hxge_status_t 181 hxge_enable_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 182 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 183 int n_init_kick) 184 { 185 hpi_handle_t handle; 186 rdc_desc_cfg_t rdc_desc; 187 rdc_rcr_cfg_b_t *cfgb_p; 188 hpi_status_t rs = HPI_SUCCESS; 189 190 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel")); 191 handle = HXGE_DEV_HPI_HANDLE(hxgep); 192 193 /* 194 * Use configuration data composed at init time. Write to hardware the 195 * receive ring configurations. 196 */ 197 rdc_desc.mbox_enable = 1; 198 rdc_desc.mbox_addr = mbox_p->mbox_addr; 199 HXGE_DEBUG_MSG((hxgep, RX_CTL, 200 "==> hxge_enable_rxdma_channel: mboxp $%p($%p)", 201 mbox_p->mbox_addr, rdc_desc.mbox_addr)); 202 203 rdc_desc.rbr_len = rbr_p->rbb_max; 204 rdc_desc.rbr_addr = rbr_p->rbr_addr; 205 206 switch (hxgep->rx_bksize_code) { 207 case RBR_BKSIZE_4K: 208 rdc_desc.page_size = SIZE_4KB; 209 break; 210 case RBR_BKSIZE_8K: 211 rdc_desc.page_size = SIZE_8KB; 212 break; 213 } 214 215 rdc_desc.size0 = rbr_p->hpi_pkt_buf_size0; 216 rdc_desc.valid0 = 1; 217 218 rdc_desc.size1 = rbr_p->hpi_pkt_buf_size1; 219 rdc_desc.valid1 = 1; 220 221 rdc_desc.size2 = rbr_p->hpi_pkt_buf_size2; 222 rdc_desc.valid2 = 1; 223 224 rdc_desc.full_hdr = rcr_p->full_hdr_flag; 225 rdc_desc.offset = rcr_p->sw_priv_hdr_len; 226 227 rdc_desc.rcr_len = rcr_p->comp_size; 228 rdc_desc.rcr_addr = rcr_p->rcr_addr; 229 230 cfgb_p = &(rcr_p->rcr_cfgb); 231 rdc_desc.rcr_threshold = cfgb_p->bits.pthres; 232 rdc_desc.rcr_timeout = cfgb_p->bits.timeout; 233 rdc_desc.rcr_timeout_enable = cfgb_p->bits.entout; 234 235 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 236 "rbr_len qlen %d pagesize code %d rcr_len %d", 237 rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len)); 238 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 239 "size 0 %d size 1 %d size 2 %d", 240 rbr_p->hpi_pkt_buf_size0, rbr_p->hpi_pkt_buf_size1, 241 rbr_p->hpi_pkt_buf_size2)); 242 243 rs = hpi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc); 244 if (rs != HPI_SUCCESS) { 245 return (HXGE_ERROR | rs); 246 } 247 248 /* 249 * Enable the timeout and threshold. 250 */ 251 rs = hpi_rxdma_cfg_rdc_rcr_threshold(handle, channel, 252 rdc_desc.rcr_threshold); 253 if (rs != HPI_SUCCESS) { 254 return (HXGE_ERROR | rs); 255 } 256 257 rs = hpi_rxdma_cfg_rdc_rcr_timeout(handle, channel, 258 rdc_desc.rcr_timeout); 259 if (rs != HPI_SUCCESS) { 260 return (HXGE_ERROR | rs); 261 } 262 263 /* Enable the DMA */ 264 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 265 if (rs != HPI_SUCCESS) { 266 return (HXGE_ERROR | rs); 267 } 268 269 /* Kick the DMA engine */ 270 hpi_rxdma_rdc_rbr_kick(handle, channel, n_init_kick); 271 272 /* Clear the rbr empty bit */ 273 (void) hpi_rxdma_channel_rbr_empty_clear(handle, channel); 274 275 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_enable_rxdma_channel")); 276 277 return (HXGE_OK); 278 } 279 280 static hxge_status_t 281 hxge_disable_rxdma_channel(p_hxge_t hxgep, uint16_t channel) 282 { 283 hpi_handle_t handle; 284 hpi_status_t rs = HPI_SUCCESS; 285 286 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_disable_rxdma_channel")); 287 288 handle = HXGE_DEV_HPI_HANDLE(hxgep); 289 290 /* disable the DMA */ 291 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 292 if (rs != HPI_SUCCESS) { 293 HXGE_DEBUG_MSG((hxgep, RX_CTL, 294 "<== hxge_disable_rxdma_channel:failed (0x%x)", rs)); 295 return (HXGE_ERROR | rs); 296 } 297 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_disable_rxdma_channel")); 298 return (HXGE_OK); 299 } 300 301 hxge_status_t 302 hxge_rxdma_channel_rcrflush(p_hxge_t hxgep, uint8_t channel) 303 { 304 hpi_handle_t handle; 305 hxge_status_t status = HXGE_OK; 306 307 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 308 "==> hxge_rxdma_channel_rcrflush")); 309 310 handle = HXGE_DEV_HPI_HANDLE(hxgep); 311 hpi_rxdma_rdc_rcr_flush(handle, channel); 312 313 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 314 "<== hxge_rxdma_channel_rcrflush")); 315 return (status); 316 317 } 318 319 #define MID_INDEX(l, r) ((r + l + 1) >> 1) 320 321 #define TO_LEFT -1 322 #define TO_RIGHT 1 323 #define BOTH_RIGHT (TO_RIGHT + TO_RIGHT) 324 #define BOTH_LEFT (TO_LEFT + TO_LEFT) 325 #define IN_MIDDLE (TO_RIGHT + TO_LEFT) 326 #define NO_HINT 0xffffffff 327 328 /*ARGSUSED*/ 329 hxge_status_t 330 hxge_rxbuf_pp_to_vp(p_hxge_t hxgep, p_rx_rbr_ring_t rbr_p, 331 uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp, 332 uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index) 333 { 334 int bufsize; 335 uint64_t pktbuf_pp; 336 uint64_t dvma_addr; 337 rxring_info_t *ring_info; 338 int base_side, end_side; 339 int r_index, l_index, anchor_index; 340 int found, search_done; 341 uint32_t offset, chunk_size, block_size, page_size_mask; 342 uint32_t chunk_index, block_index, total_index; 343 int max_iterations, iteration; 344 rxbuf_index_info_t *bufinfo; 345 346 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_rxbuf_pp_to_vp")); 347 348 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 349 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d", 350 pkt_buf_addr_pp, pktbufsz_type)); 351 352 #if defined(__i386) 353 pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp; 354 #else 355 pktbuf_pp = (uint64_t)pkt_buf_addr_pp; 356 #endif 357 358 switch (pktbufsz_type) { 359 case 0: 360 bufsize = rbr_p->pkt_buf_size0; 361 break; 362 case 1: 363 bufsize = rbr_p->pkt_buf_size1; 364 break; 365 case 2: 366 bufsize = rbr_p->pkt_buf_size2; 367 break; 368 case RCR_SINGLE_BLOCK: 369 bufsize = 0; 370 anchor_index = 0; 371 break; 372 default: 373 return (HXGE_ERROR); 374 } 375 376 if (rbr_p->num_blocks == 1) { 377 anchor_index = 0; 378 ring_info = rbr_p->ring_info; 379 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 380 381 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 382 "==> hxge_rxbuf_pp_to_vp: (found, 1 block) " 383 "buf_pp $%p btype %d anchor_index %d bufinfo $%p", 384 pkt_buf_addr_pp, pktbufsz_type, anchor_index, bufinfo)); 385 386 goto found_index; 387 } 388 389 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 390 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d anchor_index %d", 391 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 392 393 ring_info = rbr_p->ring_info; 394 found = B_FALSE; 395 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 396 iteration = 0; 397 max_iterations = ring_info->max_iterations; 398 399 /* 400 * First check if this block have been seen recently. This is indicated 401 * by a hint which is initialized when the first buffer of the block is 402 * seen. The hint is reset when the last buffer of the block has been 403 * processed. As three block sizes are supported, three hints are kept. 404 * The idea behind the hints is that once the hardware uses a block 405 * for a buffer of that size, it will use it exclusively for that size 406 * and will use it until it is exhausted. It is assumed that there 407 * would a single block being used for the same buffer sizes at any 408 * given time. 409 */ 410 if (ring_info->hint[pktbufsz_type] != NO_HINT) { 411 anchor_index = ring_info->hint[pktbufsz_type]; 412 dvma_addr = bufinfo[anchor_index].dvma_addr; 413 chunk_size = bufinfo[anchor_index].buf_size; 414 if ((pktbuf_pp >= dvma_addr) && 415 (pktbuf_pp < (dvma_addr + chunk_size))) { 416 found = B_TRUE; 417 /* 418 * check if this is the last buffer in the block If so, 419 * then reset the hint for the size; 420 */ 421 422 if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size)) 423 ring_info->hint[pktbufsz_type] = NO_HINT; 424 } 425 } 426 427 if (found == B_FALSE) { 428 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 429 "==> hxge_rxbuf_pp_to_vp: (!found)" 430 "buf_pp $%p btype %d anchor_index %d", 431 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 432 433 /* 434 * This is the first buffer of the block of this size. Need to 435 * search the whole information array. the search algorithm 436 * uses a binary tree search algorithm. It assumes that the 437 * information is already sorted with increasing order info[0] 438 * < info[1] < info[2] .... < info[n-1] where n is the size of 439 * the information array 440 */ 441 r_index = rbr_p->num_blocks - 1; 442 l_index = 0; 443 search_done = B_FALSE; 444 anchor_index = MID_INDEX(r_index, l_index); 445 while (search_done == B_FALSE) { 446 if ((r_index == l_index) || 447 (iteration >= max_iterations)) 448 search_done = B_TRUE; 449 450 end_side = TO_RIGHT; /* to the right */ 451 base_side = TO_LEFT; /* to the left */ 452 /* read the DVMA address information and sort it */ 453 dvma_addr = bufinfo[anchor_index].dvma_addr; 454 chunk_size = bufinfo[anchor_index].buf_size; 455 456 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 457 "==> hxge_rxbuf_pp_to_vp: (searching)" 458 "buf_pp $%p btype %d " 459 "anchor_index %d chunk_size %d dvmaaddr $%p", 460 pkt_buf_addr_pp, pktbufsz_type, anchor_index, 461 chunk_size, dvma_addr)); 462 463 if (pktbuf_pp >= dvma_addr) 464 base_side = TO_RIGHT; /* to the right */ 465 if (pktbuf_pp < (dvma_addr + chunk_size)) 466 end_side = TO_LEFT; /* to the left */ 467 468 switch (base_side + end_side) { 469 case IN_MIDDLE: 470 /* found */ 471 found = B_TRUE; 472 search_done = B_TRUE; 473 if ((pktbuf_pp + bufsize) < 474 (dvma_addr + chunk_size)) 475 ring_info->hint[pktbufsz_type] = 476 bufinfo[anchor_index].buf_index; 477 break; 478 case BOTH_RIGHT: 479 /* not found: go to the right */ 480 l_index = anchor_index + 1; 481 anchor_index = MID_INDEX(r_index, l_index); 482 break; 483 484 case BOTH_LEFT: 485 /* not found: go to the left */ 486 r_index = anchor_index - 1; 487 anchor_index = MID_INDEX(r_index, l_index); 488 break; 489 default: /* should not come here */ 490 return (HXGE_ERROR); 491 } 492 iteration++; 493 } 494 495 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 496 "==> hxge_rxbuf_pp_to_vp: (search done)" 497 "buf_pp $%p btype %d anchor_index %d", 498 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 499 } 500 501 if (found == B_FALSE) { 502 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 503 "==> hxge_rxbuf_pp_to_vp: (search failed)" 504 "buf_pp $%p btype %d anchor_index %d", 505 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 506 return (HXGE_ERROR); 507 } 508 509 found_index: 510 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 511 "==> hxge_rxbuf_pp_to_vp: (FOUND1)" 512 "buf_pp $%p btype %d bufsize %d anchor_index %d", 513 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index)); 514 515 /* index of the first block in this chunk */ 516 chunk_index = bufinfo[anchor_index].start_index; 517 dvma_addr = bufinfo[anchor_index].dvma_addr; 518 page_size_mask = ring_info->block_size_mask; 519 520 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 521 "==> hxge_rxbuf_pp_to_vp: (FOUND3), get chunk)" 522 "buf_pp $%p btype %d bufsize %d " 523 "anchor_index %d chunk_index %d dvma $%p", 524 pkt_buf_addr_pp, pktbufsz_type, bufsize, 525 anchor_index, chunk_index, dvma_addr)); 526 527 offset = pktbuf_pp - dvma_addr; /* offset within the chunk */ 528 block_size = rbr_p->block_size; /* System block(page) size */ 529 530 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 531 "==> hxge_rxbuf_pp_to_vp: (FOUND4), get chunk)" 532 "buf_pp $%p btype %d bufsize %d " 533 "anchor_index %d chunk_index %d dvma $%p " 534 "offset %d block_size %d", 535 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index, 536 chunk_index, dvma_addr, offset, block_size)); 537 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> getting total index")); 538 539 block_index = (offset / block_size); /* index within chunk */ 540 total_index = chunk_index + block_index; 541 542 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 543 "==> hxge_rxbuf_pp_to_vp: " 544 "total_index %d dvma_addr $%p " 545 "offset %d block_size %d " 546 "block_index %d ", 547 total_index, dvma_addr, offset, block_size, block_index)); 548 549 #if defined(__i386) 550 *pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr + 551 (uint32_t)offset); 552 #else 553 *pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr + 554 offset); 555 #endif 556 557 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 558 "==> hxge_rxbuf_pp_to_vp: " 559 "total_index %d dvma_addr $%p " 560 "offset %d block_size %d " 561 "block_index %d " 562 "*pkt_buf_addr_p $%p", 563 total_index, dvma_addr, offset, block_size, 564 block_index, *pkt_buf_addr_p)); 565 566 *msg_index = total_index; 567 *bufoffset = (offset & page_size_mask); 568 569 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 570 "==> hxge_rxbuf_pp_to_vp: get msg index: " 571 "msg_index %d bufoffset_index %d", 572 *msg_index, *bufoffset)); 573 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "<== hxge_rxbuf_pp_to_vp")); 574 575 return (HXGE_OK); 576 } 577 578 579 /* 580 * used by quick sort (qsort) function 581 * to perform comparison 582 */ 583 static int 584 hxge_sort_compare(const void *p1, const void *p2) 585 { 586 587 rxbuf_index_info_t *a, *b; 588 589 a = (rxbuf_index_info_t *)p1; 590 b = (rxbuf_index_info_t *)p2; 591 592 if (a->dvma_addr > b->dvma_addr) 593 return (1); 594 if (a->dvma_addr < b->dvma_addr) 595 return (-1); 596 return (0); 597 } 598 599 /* 600 * Grabbed this sort implementation from common/syscall/avl.c 601 * 602 * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified. 603 * v = Ptr to array/vector of objs 604 * n = # objs in the array 605 * s = size of each obj (must be multiples of a word size) 606 * f = ptr to function to compare two objs 607 * returns (-1 = less than, 0 = equal, 1 = greater than 608 */ 609 void 610 hxge_ksort(caddr_t v, int n, int s, int (*f) ()) 611 { 612 int g, i, j, ii; 613 unsigned int *p1, *p2; 614 unsigned int tmp; 615 616 /* No work to do */ 617 if (v == NULL || n <= 1) 618 return; 619 /* Sanity check on arguments */ 620 ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0); 621 ASSERT(s > 0); 622 623 for (g = n / 2; g > 0; g /= 2) { 624 for (i = g; i < n; i++) { 625 for (j = i - g; j >= 0 && 626 (*f) (v + j * s, v + (j + g) * s) == 1; j -= g) { 627 p1 = (unsigned *)(v + j * s); 628 p2 = (unsigned *)(v + (j + g) * s); 629 for (ii = 0; ii < s / 4; ii++) { 630 tmp = *p1; 631 *p1++ = *p2; 632 *p2++ = tmp; 633 } 634 } 635 } 636 } 637 } 638 639 /* 640 * Initialize data structures required for rxdma 641 * buffer dvma->vmem address lookup 642 */ 643 /*ARGSUSED*/ 644 static hxge_status_t 645 hxge_rxbuf_index_info_init(p_hxge_t hxgep, p_rx_rbr_ring_t rbrp) 646 { 647 int index; 648 rxring_info_t *ring_info; 649 int max_iteration = 0, max_index = 0; 650 651 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_rxbuf_index_info_init")); 652 653 ring_info = rbrp->ring_info; 654 ring_info->hint[0] = NO_HINT; 655 ring_info->hint[1] = NO_HINT; 656 ring_info->hint[2] = NO_HINT; 657 max_index = rbrp->num_blocks; 658 659 /* read the DVMA address information and sort it */ 660 /* do init of the information array */ 661 662 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 663 " hxge_rxbuf_index_info_init Sort ptrs")); 664 665 /* sort the array */ 666 hxge_ksort((void *) ring_info->buffer, max_index, 667 sizeof (rxbuf_index_info_t), hxge_sort_compare); 668 669 for (index = 0; index < max_index; index++) { 670 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 671 " hxge_rxbuf_index_info_init: sorted chunk %d " 672 " ioaddr $%p kaddr $%p size %x", 673 index, ring_info->buffer[index].dvma_addr, 674 ring_info->buffer[index].kaddr, 675 ring_info->buffer[index].buf_size)); 676 } 677 678 max_iteration = 0; 679 while (max_index >= (1ULL << max_iteration)) 680 max_iteration++; 681 ring_info->max_iterations = max_iteration + 1; 682 683 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 684 " hxge_rxbuf_index_info_init Find max iter %d", 685 ring_info->max_iterations)); 686 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_rxbuf_index_info_init")); 687 688 return (HXGE_OK); 689 } 690 691 /*ARGSUSED*/ 692 void 693 hxge_dump_rcr_entry(p_hxge_t hxgep, p_rcr_entry_t entry_p) 694 { 695 #ifdef HXGE_DEBUG 696 697 uint32_t bptr; 698 uint64_t pp; 699 700 bptr = entry_p->bits.pkt_buf_addr; 701 702 HXGE_DEBUG_MSG((hxgep, RX_CTL, 703 "\trcr entry $%p " 704 "\trcr entry 0x%0llx " 705 "\trcr entry 0x%08x " 706 "\trcr entry 0x%08x " 707 "\tvalue 0x%0llx\n" 708 "\tmulti = %d\n" 709 "\tpkt_type = 0x%x\n" 710 "\terror = 0x%04x\n" 711 "\tl2_len = %d\n" 712 "\tpktbufsize = %d\n" 713 "\tpkt_buf_addr = $%p\n" 714 "\tpkt_buf_addr (<< 6) = $%p\n", 715 entry_p, 716 *(int64_t *)entry_p, 717 *(int32_t *)entry_p, 718 *(int32_t *)((char *)entry_p + 32), 719 entry_p->value, 720 entry_p->bits.multi, 721 entry_p->bits.pkt_type, 722 entry_p->bits.error, 723 entry_p->bits.l2_len, 724 entry_p->bits.pktbufsz, 725 bptr, 726 entry_p->bits.pkt_buf_addr_l)); 727 728 pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) << 729 RCR_PKT_BUF_ADDR_SHIFT; 730 731 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rcr pp 0x%llx l2 len %d", 732 pp, (*(int64_t *)entry_p >> 40) & 0x3fff)); 733 #endif 734 } 735 736 /*ARGSUSED*/ 737 void 738 hxge_rxdma_stop(p_hxge_t hxgep) 739 { 740 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop")); 741 742 (void) hxge_rx_vmac_disable(hxgep); 743 (void) hxge_rxdma_hw_mode(hxgep, HXGE_DMA_STOP); 744 745 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop")); 746 } 747 748 void 749 hxge_rxdma_stop_reinit(p_hxge_t hxgep) 750 { 751 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_reinit")); 752 753 (void) hxge_rxdma_stop(hxgep); 754 (void) hxge_uninit_rxdma_channels(hxgep); 755 (void) hxge_init_rxdma_channels(hxgep); 756 757 (void) hxge_rx_vmac_enable(hxgep); 758 759 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_reinit")); 760 } 761 762 hxge_status_t 763 hxge_rxdma_hw_mode(p_hxge_t hxgep, boolean_t enable) 764 { 765 int i, ndmas; 766 uint16_t channel; 767 p_rx_rbr_rings_t rx_rbr_rings; 768 p_rx_rbr_ring_t *rbr_rings; 769 hpi_handle_t handle; 770 hpi_status_t rs = HPI_SUCCESS; 771 hxge_status_t status = HXGE_OK; 772 773 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 774 "==> hxge_rxdma_hw_mode: mode %d", enable)); 775 776 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 777 HXGE_DEBUG_MSG((hxgep, RX_CTL, 778 "<== hxge_rxdma_mode: not initialized")); 779 return (HXGE_ERROR); 780 } 781 782 rx_rbr_rings = hxgep->rx_rbr_rings; 783 if (rx_rbr_rings == NULL) { 784 HXGE_DEBUG_MSG((hxgep, RX_CTL, 785 "<== hxge_rxdma_mode: NULL ring pointer")); 786 return (HXGE_ERROR); 787 } 788 789 if (rx_rbr_rings->rbr_rings == NULL) { 790 HXGE_DEBUG_MSG((hxgep, RX_CTL, 791 "<== hxge_rxdma_mode: NULL rbr rings pointer")); 792 return (HXGE_ERROR); 793 } 794 795 ndmas = rx_rbr_rings->ndmas; 796 if (!ndmas) { 797 HXGE_DEBUG_MSG((hxgep, RX_CTL, 798 "<== hxge_rxdma_mode: no channel")); 799 return (HXGE_ERROR); 800 } 801 802 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 803 "==> hxge_rxdma_mode (ndmas %d)", ndmas)); 804 805 rbr_rings = rx_rbr_rings->rbr_rings; 806 807 handle = HXGE_DEV_HPI_HANDLE(hxgep); 808 809 for (i = 0; i < ndmas; i++) { 810 if (rbr_rings == NULL || rbr_rings[i] == NULL) { 811 continue; 812 } 813 channel = rbr_rings[i]->rdc; 814 if (enable) { 815 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 816 "==> hxge_rxdma_hw_mode: channel %d (enable)", 817 channel)); 818 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 819 } else { 820 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 821 "==> hxge_rxdma_hw_mode: channel %d (disable)", 822 channel)); 823 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 824 } 825 } 826 827 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs); 828 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 829 "<== hxge_rxdma_hw_mode: status 0x%x", status)); 830 831 return (status); 832 } 833 834 int 835 hxge_rxdma_get_ring_index(p_hxge_t hxgep, uint16_t channel) 836 { 837 int i, ndmas; 838 uint16_t rdc; 839 p_rx_rbr_rings_t rx_rbr_rings; 840 p_rx_rbr_ring_t *rbr_rings; 841 842 HXGE_DEBUG_MSG((hxgep, RX_CTL, 843 "==> hxge_rxdma_get_ring_index: channel %d", channel)); 844 845 rx_rbr_rings = hxgep->rx_rbr_rings; 846 if (rx_rbr_rings == NULL) { 847 HXGE_DEBUG_MSG((hxgep, RX_CTL, 848 "<== hxge_rxdma_get_ring_index: NULL ring pointer")); 849 return (-1); 850 } 851 852 ndmas = rx_rbr_rings->ndmas; 853 if (!ndmas) { 854 HXGE_DEBUG_MSG((hxgep, RX_CTL, 855 "<== hxge_rxdma_get_ring_index: no channel")); 856 return (-1); 857 } 858 859 HXGE_DEBUG_MSG((hxgep, RX_CTL, 860 "==> hxge_rxdma_get_ring_index (ndmas %d)", ndmas)); 861 862 rbr_rings = rx_rbr_rings->rbr_rings; 863 for (i = 0; i < ndmas; i++) { 864 rdc = rbr_rings[i]->rdc; 865 if (channel == rdc) { 866 HXGE_DEBUG_MSG((hxgep, RX_CTL, 867 "==> hxge_rxdma_get_rbr_ring: " 868 "channel %d (index %d) " 869 "ring %d", channel, i, rbr_rings[i])); 870 871 return (i); 872 } 873 } 874 875 HXGE_DEBUG_MSG((hxgep, RX_CTL, 876 "<== hxge_rxdma_get_rbr_ring_index: not found")); 877 878 return (-1); 879 } 880 881 /* 882 * Static functions start here. 883 */ 884 static p_rx_msg_t 885 hxge_allocb(size_t size, uint32_t pri, p_hxge_dma_common_t dmabuf_p) 886 { 887 p_rx_msg_t hxge_mp = NULL; 888 p_hxge_dma_common_t dmamsg_p; 889 uchar_t *buffer; 890 891 hxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP); 892 if (hxge_mp == NULL) { 893 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 894 "Allocation of a rx msg failed.")); 895 goto hxge_allocb_exit; 896 } 897 898 hxge_mp->use_buf_pool = B_FALSE; 899 if (dmabuf_p) { 900 hxge_mp->use_buf_pool = B_TRUE; 901 902 dmamsg_p = (p_hxge_dma_common_t)&hxge_mp->buf_dma; 903 *dmamsg_p = *dmabuf_p; 904 dmamsg_p->nblocks = 1; 905 dmamsg_p->block_size = size; 906 dmamsg_p->alength = size; 907 buffer = (uchar_t *)dmabuf_p->kaddrp; 908 909 dmabuf_p->kaddrp = (void *)((char *)dmabuf_p->kaddrp + size); 910 dmabuf_p->ioaddr_pp = (void *) 911 ((char *)dmabuf_p->ioaddr_pp + size); 912 913 dmabuf_p->alength -= size; 914 dmabuf_p->offset += size; 915 dmabuf_p->dma_cookie.dmac_laddress += size; 916 dmabuf_p->dma_cookie.dmac_size -= size; 917 } else { 918 buffer = KMEM_ALLOC(size, KM_NOSLEEP); 919 if (buffer == NULL) { 920 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 921 "Allocation of a receive page failed.")); 922 goto hxge_allocb_fail1; 923 } 924 } 925 926 hxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &hxge_mp->freeb); 927 if (hxge_mp->rx_mblk_p == NULL) { 928 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, "desballoc failed.")); 929 goto hxge_allocb_fail2; 930 } 931 hxge_mp->buffer = buffer; 932 hxge_mp->block_size = size; 933 hxge_mp->freeb.free_func = (void (*) ()) hxge_freeb; 934 hxge_mp->freeb.free_arg = (caddr_t)hxge_mp; 935 hxge_mp->ref_cnt = 1; 936 hxge_mp->free = B_TRUE; 937 hxge_mp->rx_use_bcopy = B_FALSE; 938 939 atomic_inc_32(&hxge_mblks_pending); 940 941 goto hxge_allocb_exit; 942 943 hxge_allocb_fail2: 944 if (!hxge_mp->use_buf_pool) { 945 KMEM_FREE(buffer, size); 946 } 947 hxge_allocb_fail1: 948 KMEM_FREE(hxge_mp, sizeof (rx_msg_t)); 949 hxge_mp = NULL; 950 951 hxge_allocb_exit: 952 return (hxge_mp); 953 } 954 955 p_mblk_t 956 hxge_dupb(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 957 { 958 p_mblk_t mp; 959 960 HXGE_DEBUG_MSG((NULL, MEM_CTL, "==> hxge_dupb")); 961 HXGE_DEBUG_MSG((NULL, MEM_CTL, "hxge_mp = $%p " 962 "offset = 0x%08X " "size = 0x%08X", hxge_mp, offset, size)); 963 964 mp = desballoc(&hxge_mp->buffer[offset], size, 0, &hxge_mp->freeb); 965 if (mp == NULL) { 966 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 967 goto hxge_dupb_exit; 968 } 969 970 atomic_inc_32(&hxge_mp->ref_cnt); 971 972 hxge_dupb_exit: 973 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 974 return (mp); 975 } 976 977 p_mblk_t 978 hxge_dupb_bcopy(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 979 { 980 p_mblk_t mp; 981 uchar_t *dp; 982 983 mp = allocb(size + HXGE_RXBUF_EXTRA, 0); 984 if (mp == NULL) { 985 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 986 goto hxge_dupb_bcopy_exit; 987 } 988 dp = mp->b_rptr = mp->b_rptr + HXGE_RXBUF_EXTRA; 989 bcopy((void *) &hxge_mp->buffer[offset], dp, size); 990 mp->b_wptr = dp + size; 991 992 hxge_dupb_bcopy_exit: 993 994 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 995 996 return (mp); 997 } 998 999 void hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, 1000 p_rx_msg_t rx_msg_p); 1001 1002 void 1003 hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p) 1004 { 1005 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_post_page")); 1006 1007 /* Reuse this buffer */ 1008 rx_msg_p->free = B_FALSE; 1009 rx_msg_p->cur_usage_cnt = 0; 1010 rx_msg_p->max_usage_cnt = 0; 1011 rx_msg_p->pkt_buf_size = 0; 1012 1013 if (rx_rbr_p->rbr_use_bcopy) { 1014 rx_msg_p->rx_use_bcopy = B_FALSE; 1015 atomic_dec_32(&rx_rbr_p->rbr_consumed); 1016 } 1017 atomic_dec_32(&rx_rbr_p->rbr_used); 1018 1019 /* 1020 * Get the rbr header pointer and its offset index. 1021 */ 1022 rx_rbr_p->rbr_wr_index = ((rx_rbr_p->rbr_wr_index + 1) & 1023 rx_rbr_p->rbr_wrap_mask); 1024 rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr; 1025 1026 /* 1027 * Accumulate some buffers in the ring before re-enabling the 1028 * DMA channel, if rbr empty was signaled. 1029 */ 1030 hpi_rxdma_rdc_rbr_kick(HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc, 1); 1031 if (rx_rbr_p->rbr_is_empty && 1032 (rx_rbr_p->rbb_max - rx_rbr_p->rbr_used) >= 1033 HXGE_RBR_EMPTY_THRESHOLD) { 1034 hxge_rbr_empty_restore(hxgep, rx_rbr_p); 1035 } 1036 1037 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1038 "<== hxge_post_page (channel %d post_next_index %d)", 1039 rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index)); 1040 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_post_page")); 1041 } 1042 1043 void 1044 hxge_freeb(p_rx_msg_t rx_msg_p) 1045 { 1046 size_t size; 1047 uchar_t *buffer = NULL; 1048 int ref_cnt; 1049 boolean_t free_state = B_FALSE; 1050 rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p; 1051 1052 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> hxge_freeb")); 1053 HXGE_DEBUG_MSG((NULL, MEM2_CTL, 1054 "hxge_freeb:rx_msg_p = $%p (block pending %d)", 1055 rx_msg_p, hxge_mblks_pending)); 1056 1057 if (ring == NULL) 1058 return; 1059 1060 /* 1061 * This is to prevent posting activities while we are recovering 1062 * from fatal errors. This should not be a performance drag since 1063 * ref_cnt != 0 most times. 1064 */ 1065 if (ring->rbr_state == RBR_POSTING) 1066 MUTEX_ENTER(&ring->post_lock); 1067 1068 /* 1069 * First we need to get the free state, then 1070 * atomic decrement the reference count to prevent 1071 * the race condition with the interrupt thread that 1072 * is processing a loaned up buffer block. 1073 */ 1074 free_state = rx_msg_p->free; 1075 ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1); 1076 if (!ref_cnt) { 1077 atomic_dec_32(&hxge_mblks_pending); 1078 1079 buffer = rx_msg_p->buffer; 1080 size = rx_msg_p->block_size; 1081 1082 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "hxge_freeb: " 1083 "will free: rx_msg_p = $%p (block pending %d)", 1084 rx_msg_p, hxge_mblks_pending)); 1085 1086 if (!rx_msg_p->use_buf_pool) { 1087 KMEM_FREE(buffer, size); 1088 } 1089 1090 KMEM_FREE(rx_msg_p, sizeof (rx_msg_t)); 1091 /* 1092 * Decrement the receive buffer ring's reference 1093 * count, too. 1094 */ 1095 atomic_dec_32(&ring->rbr_ref_cnt); 1096 1097 /* 1098 * Free the receive buffer ring, iff 1099 * 1. all the receive buffers have been freed 1100 * 2. and we are in the proper state (that is, 1101 * we are not UNMAPPING). 1102 */ 1103 if (ring->rbr_ref_cnt == 0 && 1104 ring->rbr_state == RBR_UNMAPPED) { 1105 KMEM_FREE(ring, sizeof (*ring)); 1106 /* post_lock has been destroyed already */ 1107 return; 1108 } 1109 } 1110 1111 /* 1112 * Repost buffer. 1113 */ 1114 if (free_state && (ref_cnt == 1)) { 1115 HXGE_DEBUG_MSG((NULL, RX_CTL, 1116 "hxge_freeb: post page $%p:", rx_msg_p)); 1117 if (ring->rbr_state == RBR_POSTING) 1118 hxge_post_page(rx_msg_p->hxgep, ring, rx_msg_p); 1119 } 1120 1121 if (ring->rbr_state == RBR_POSTING) 1122 MUTEX_EXIT(&ring->post_lock); 1123 1124 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== hxge_freeb")); 1125 } 1126 1127 uint_t 1128 hxge_rx_intr(caddr_t arg1, caddr_t arg2) 1129 { 1130 p_hxge_ldv_t ldvp = (p_hxge_ldv_t)arg1; 1131 p_hxge_t hxgep = (p_hxge_t)arg2; 1132 p_hxge_ldg_t ldgp; 1133 uint8_t channel; 1134 hpi_handle_t handle; 1135 rdc_stat_t cs; 1136 uint_t serviced = DDI_INTR_UNCLAIMED; 1137 1138 if (ldvp == NULL) { 1139 HXGE_DEBUG_MSG((NULL, RX_INT_CTL, 1140 "<== hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1141 return (DDI_INTR_UNCLAIMED); 1142 } 1143 1144 if (arg2 == NULL || (void *) ldvp->hxgep != arg2) { 1145 hxgep = ldvp->hxgep; 1146 } 1147 1148 /* 1149 * If the interface is not started, just swallow the interrupt 1150 * for the logical device and don't rearm it. 1151 */ 1152 if (hxgep->hxge_mac_state != HXGE_MAC_STARTED) 1153 return (DDI_INTR_CLAIMED); 1154 1155 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1156 "==> hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1157 1158 /* 1159 * This interrupt handler is for a specific receive dma channel. 1160 */ 1161 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1162 1163 /* 1164 * Get the control and status for this channel. 1165 */ 1166 channel = ldvp->channel; 1167 ldgp = ldvp->ldgp; 1168 RXDMA_REG_READ64(handle, RDC_STAT, channel, &cs.value); 1169 1170 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_intr:channel %d " 1171 "cs 0x%016llx rcrto 0x%x rcrthres %x", 1172 channel, cs.value, cs.bits.rcr_to, cs.bits.rcr_thres)); 1173 1174 hxge_rx_pkts_vring(hxgep, ldvp->vdma_index, ldvp, cs); 1175 serviced = DDI_INTR_CLAIMED; 1176 1177 /* error events. */ 1178 if (cs.value & RDC_STAT_ERROR) { 1179 (void) hxge_rx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs); 1180 } 1181 1182 hxge_intr_exit: 1183 /* 1184 * Enable the mailbox update interrupt if we want to use mailbox. We 1185 * probably don't need to use mailbox as it only saves us one pio read. 1186 * Also write 1 to rcrthres and rcrto to clear these two edge triggered 1187 * bits. 1188 */ 1189 cs.value &= RDC_STAT_WR1C; 1190 cs.bits.mex = 1; 1191 cs.bits.ptrread = 0; 1192 cs.bits.pktread = 0; 1193 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1194 1195 /* 1196 * Rearm this logical group if this is a single device group. 1197 */ 1198 if (ldgp->nldvs == 1) { 1199 ld_intr_mgmt_t mgm; 1200 1201 mgm.value = 0; 1202 mgm.bits.arm = 1; 1203 mgm.bits.timer = ldgp->ldg_timer; 1204 HXGE_REG_WR32(handle, 1205 LD_INTR_MGMT + LDSV_OFFSET(ldgp->ldg), mgm.value); 1206 } 1207 1208 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1209 "<== hxge_rx_intr: serviced %d", serviced)); 1210 1211 return (serviced); 1212 } 1213 1214 static void 1215 hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1216 rdc_stat_t cs) 1217 { 1218 p_mblk_t mp; 1219 p_rx_rcr_ring_t rcrp; 1220 1221 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts_vring")); 1222 if ((mp = hxge_rx_pkts(hxgep, vindex, ldvp, &rcrp, cs)) == NULL) { 1223 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1224 "<== hxge_rx_pkts_vring: no mp")); 1225 return; 1226 } 1227 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts_vring: $%p", mp)); 1228 1229 #ifdef HXGE_DEBUG 1230 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1231 "==> hxge_rx_pkts_vring:calling mac_rx (NEMO) " 1232 "LEN %d mp $%p mp->b_next $%p rcrp $%p", 1233 (mp->b_wptr - mp->b_rptr), mp, mp->b_next, rcrp)); 1234 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1235 "==> hxge_rx_pkts_vring: dump packets " 1236 "(mp $%p b_rptr $%p b_wptr $%p):\n %s", 1237 mp, mp->b_rptr, mp->b_wptr, 1238 hxge_dump_packet((char *)mp->b_rptr, 64))); 1239 1240 if (mp->b_cont) { 1241 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1242 "==> hxge_rx_pkts_vring: dump b_cont packets " 1243 "(mp->b_cont $%p b_rptr $%p b_wptr $%p):\n %s", 1244 mp->b_cont, mp->b_cont->b_rptr, mp->b_cont->b_wptr, 1245 hxge_dump_packet((char *)mp->b_cont->b_rptr, 1246 mp->b_cont->b_wptr - mp->b_cont->b_rptr))); 1247 } 1248 if (mp->b_next) { 1249 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1250 "==> hxge_rx_pkts_vring: dump next packets " 1251 "(b_rptr $%p): %s", 1252 mp->b_next->b_rptr, 1253 hxge_dump_packet((char *)mp->b_next->b_rptr, 64))); 1254 } 1255 #endif 1256 1257 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1258 "==> hxge_rx_pkts_vring: send packet to stack")); 1259 mac_rx(hxgep->mach, NULL, mp); 1260 1261 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_pkts_vring")); 1262 } 1263 1264 /*ARGSUSED*/ 1265 mblk_t * 1266 hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1267 p_rx_rcr_ring_t *rcrp, rdc_stat_t cs) 1268 { 1269 hpi_handle_t handle; 1270 uint8_t channel; 1271 p_rx_rcr_rings_t rx_rcr_rings; 1272 p_rx_rcr_ring_t rcr_p; 1273 uint32_t comp_rd_index; 1274 p_rcr_entry_t rcr_desc_rd_head_p; 1275 p_rcr_entry_t rcr_desc_rd_head_pp; 1276 p_mblk_t nmp, mp_cont, head_mp, *tail_mp; 1277 uint16_t qlen, nrcr_read, npkt_read; 1278 uint32_t qlen_hw, qlen_sw; 1279 uint32_t invalid_rcr_entry; 1280 boolean_t multi; 1281 rdc_rcr_cfg_b_t rcr_cfg_b; 1282 p_rx_mbox_t rx_mboxp; 1283 p_rxdma_mailbox_t mboxp; 1284 uint64_t rcr_head_index, rcr_tail_index; 1285 uint64_t rcr_tail; 1286 uint64_t value; 1287 rdc_rcr_tail_t rcr_tail_reg; 1288 p_hxge_rx_ring_stats_t rdc_stats; 1289 1290 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:vindex %d " 1291 "channel %d", vindex, ldvp->channel)); 1292 1293 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 1294 return (NULL); 1295 } 1296 1297 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1298 rx_rcr_rings = hxgep->rx_rcr_rings; 1299 rcr_p = rx_rcr_rings->rcr_rings[vindex]; 1300 channel = rcr_p->rdc; 1301 if (channel != ldvp->channel) { 1302 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:index %d " 1303 "channel %d, and rcr channel %d not matched.", 1304 vindex, ldvp->channel, channel)); 1305 return (NULL); 1306 } 1307 1308 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1309 "==> hxge_rx_pkts: START: rcr channel %d " 1310 "head_p $%p head_pp $%p index %d ", 1311 channel, rcr_p->rcr_desc_rd_head_p, 1312 rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1313 1314 rx_mboxp = hxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 1315 mboxp = (p_rxdma_mailbox_t)rx_mboxp->rx_mbox.kaddrp; 1316 1317 (void) hpi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen); 1318 RXDMA_REG_READ64(handle, RDC_RCR_TAIL, channel, &rcr_tail_reg.value); 1319 rcr_tail = rcr_tail_reg.bits.tail; 1320 1321 if (!qlen) { 1322 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1323 "<== hxge_rx_pkts:rcr channel %d qlen %d (no pkts)", 1324 channel, qlen)); 1325 return (NULL); 1326 } 1327 1328 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts:rcr channel %d " 1329 "qlen %d", channel, qlen)); 1330 1331 comp_rd_index = rcr_p->comp_rd_index; 1332 1333 rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p; 1334 rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp; 1335 nrcr_read = npkt_read = 0; 1336 1337 /* 1338 * Number of packets queued (The jumbo or multi packet will be counted 1339 * as only one paccket and it may take up more than one completion 1340 * entry). 1341 */ 1342 if (cs.bits.rbr_empty) 1343 qlen_hw = qlen; 1344 else 1345 qlen_hw = (qlen < hxge_max_rx_pkts) ? qlen : hxge_max_rx_pkts; 1346 1347 head_mp = NULL; 1348 tail_mp = &head_mp; 1349 nmp = mp_cont = NULL; 1350 multi = B_FALSE; 1351 1352 rcr_head_index = rcr_p->rcr_desc_rd_head_p - rcr_p->rcr_desc_first_p; 1353 rcr_tail_index = rcr_tail - rcr_p->rcr_tail_begin; 1354 1355 if (rcr_tail_index >= rcr_head_index) { 1356 qlen_sw = rcr_tail_index - rcr_head_index; 1357 } else { 1358 /* rcr_tail has wrapped around */ 1359 qlen_sw = (rcr_p->comp_size - rcr_head_index) + rcr_tail_index; 1360 } 1361 1362 if (qlen_hw > qlen_sw) { 1363 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1364 "Channel %d, rcr_qlen from reg %d and from rcr_tail %d\n", 1365 channel, qlen_hw, qlen_sw)); 1366 qlen_hw = qlen_sw; 1367 } 1368 1369 while (qlen_hw) { 1370 #ifdef HXGE_DEBUG 1371 hxge_dump_rcr_entry(hxgep, rcr_desc_rd_head_p); 1372 #endif 1373 /* 1374 * Process one completion ring entry. 1375 */ 1376 invalid_rcr_entry = 0; 1377 hxge_receive_packet(hxgep, 1378 rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont, 1379 &invalid_rcr_entry); 1380 if (invalid_rcr_entry != 0) { 1381 rdc_stats = rcr_p->rdc_stats; 1382 rdc_stats->rcr_invalids++; 1383 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1384 "Channel %d could only read 0x%x packets, " 1385 "but 0x%x pending\n", channel, npkt_read, qlen_hw)); 1386 break; 1387 } 1388 1389 /* 1390 * message chaining modes (nemo msg chaining) 1391 */ 1392 if (nmp) { 1393 nmp->b_next = NULL; 1394 if (!multi && !mp_cont) { /* frame fits a partition */ 1395 *tail_mp = nmp; 1396 tail_mp = &nmp->b_next; 1397 nmp = NULL; 1398 } else if (multi && !mp_cont) { /* first segment */ 1399 *tail_mp = nmp; 1400 tail_mp = &nmp->b_cont; 1401 } else if (multi && mp_cont) { /* mid of multi segs */ 1402 *tail_mp = mp_cont; 1403 tail_mp = &mp_cont->b_cont; 1404 } else if (!multi && mp_cont) { /* last segment */ 1405 *tail_mp = mp_cont; 1406 tail_mp = &nmp->b_next; 1407 nmp = NULL; 1408 } 1409 } 1410 1411 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1412 "==> hxge_rx_pkts: loop: rcr channel %d " 1413 "before updating: multi %d " 1414 "nrcr_read %d " 1415 "npk read %d " 1416 "head_pp $%p index %d ", 1417 channel, multi, 1418 nrcr_read, npkt_read, rcr_desc_rd_head_pp, comp_rd_index)); 1419 1420 if (!multi) { 1421 qlen_hw--; 1422 npkt_read++; 1423 } 1424 1425 /* 1426 * Update the next read entry. 1427 */ 1428 comp_rd_index = NEXT_ENTRY(comp_rd_index, 1429 rcr_p->comp_wrap_mask); 1430 1431 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p, 1432 rcr_p->rcr_desc_first_p, rcr_p->rcr_desc_last_p); 1433 1434 nrcr_read++; 1435 1436 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1437 "<== hxge_rx_pkts: (SAM, process one packet) " 1438 "nrcr_read %d", nrcr_read)); 1439 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1440 "==> hxge_rx_pkts: loop: rcr channel %d " 1441 "multi %d nrcr_read %d npk read %d head_pp $%p index %d ", 1442 channel, multi, nrcr_read, npkt_read, rcr_desc_rd_head_pp, 1443 comp_rd_index)); 1444 } 1445 1446 rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp; 1447 rcr_p->comp_rd_index = comp_rd_index; 1448 rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p; 1449 1450 /* Adjust the mailbox queue length for a hardware bug workaround */ 1451 mboxp->rcrstat_a.bits.qlen -= npkt_read; 1452 1453 if ((hxgep->intr_timeout != rcr_p->intr_timeout) || 1454 (hxgep->intr_threshold != rcr_p->intr_threshold)) { 1455 rcr_p->intr_timeout = hxgep->intr_timeout; 1456 rcr_p->intr_threshold = hxgep->intr_threshold; 1457 rcr_cfg_b.value = 0x0ULL; 1458 if (rcr_p->intr_timeout) 1459 rcr_cfg_b.bits.entout = 1; 1460 rcr_cfg_b.bits.timeout = rcr_p->intr_timeout; 1461 rcr_cfg_b.bits.pthres = rcr_p->intr_threshold; 1462 RXDMA_REG_WRITE64(handle, RDC_RCR_CFG_B, 1463 channel, rcr_cfg_b.value); 1464 } 1465 1466 cs.bits.pktread = npkt_read; 1467 cs.bits.ptrread = nrcr_read; 1468 value = cs.value; 1469 cs.value &= 0xffffffffULL; 1470 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1471 1472 cs.value = value & ~0xffffffffULL; 1473 cs.bits.pktread = 0; 1474 cs.bits.ptrread = 0; 1475 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1476 1477 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1478 "==> hxge_rx_pkts: EXIT: rcr channel %d " 1479 "head_pp $%p index %016llx ", 1480 channel, rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1481 1482 /* 1483 * Update RCR buffer pointer read and number of packets read. 1484 */ 1485 1486 *rcrp = rcr_p; 1487 1488 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "<== hxge_rx_pkts")); 1489 1490 return (head_mp); 1491 } 1492 1493 #define RCR_ENTRY_PATTERN 0x5a5a6b6b7c7c8d8dULL 1494 #define NO_PORT_BIT 0x20 1495 #define L4_CS_EQ_BIT 0x40 1496 1497 /*ARGSUSED*/ 1498 void 1499 hxge_receive_packet(p_hxge_t hxgep, 1500 p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p, 1501 boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont, 1502 uint32_t *invalid_rcr_entry) 1503 { 1504 p_mblk_t nmp = NULL; 1505 uint64_t multi; 1506 uint8_t channel; 1507 1508 boolean_t first_entry = B_TRUE; 1509 boolean_t is_tcp_udp = B_FALSE; 1510 boolean_t buffer_free = B_FALSE; 1511 boolean_t error_send_up = B_FALSE; 1512 uint8_t error_type; 1513 uint16_t l2_len; 1514 uint16_t skip_len; 1515 uint8_t pktbufsz_type; 1516 uint64_t rcr_entry; 1517 uint64_t *pkt_buf_addr_pp; 1518 uint64_t *pkt_buf_addr_p; 1519 uint32_t buf_offset; 1520 uint32_t bsize; 1521 uint32_t msg_index; 1522 p_rx_rbr_ring_t rx_rbr_p; 1523 p_rx_msg_t *rx_msg_ring_p; 1524 p_rx_msg_t rx_msg_p; 1525 1526 uint16_t sw_offset_bytes = 0, hdr_size = 0; 1527 hxge_status_t status = HXGE_OK; 1528 boolean_t is_valid = B_FALSE; 1529 p_hxge_rx_ring_stats_t rdc_stats; 1530 uint32_t bytes_read; 1531 uint8_t header0 = 0; 1532 uint8_t header1 = 0; 1533 uint64_t pkt_type; 1534 uint8_t no_port_bit = 0; 1535 uint8_t l4_cs_eq_bit = 0; 1536 1537 channel = rcr_p->rdc; 1538 1539 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_receive_packet")); 1540 1541 first_entry = (*mp == NULL) ? B_TRUE : B_FALSE; 1542 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p); 1543 1544 /* Verify the content of the rcr_entry for a hardware bug workaround */ 1545 if ((rcr_entry == 0x0) || (rcr_entry == RCR_ENTRY_PATTERN)) { 1546 *invalid_rcr_entry = 1; 1547 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "hxge_receive_packet " 1548 "Channel %d invalid RCR entry 0x%llx found, returning\n", 1549 channel, (long long) rcr_entry)); 1550 return; 1551 } 1552 *((uint64_t *)rcr_desc_rd_head_p) = RCR_ENTRY_PATTERN; 1553 1554 multi = (rcr_entry & RCR_MULTI_MASK); 1555 pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK); 1556 1557 error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT); 1558 l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT); 1559 1560 /* 1561 * Hardware does not strip the CRC due bug ID 11451 where 1562 * the hardware mis handles minimum size packets. 1563 */ 1564 l2_len -= ETHERFCSL; 1565 1566 pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >> 1567 RCR_PKTBUFSZ_SHIFT); 1568 #if defined(__i386) 1569 pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry & 1570 RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT); 1571 #else 1572 pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) << 1573 RCR_PKT_BUF_ADDR_SHIFT); 1574 #endif 1575 1576 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1577 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1578 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1579 "error_type 0x%x pktbufsz_type %d ", 1580 rcr_desc_rd_head_p, rcr_entry, pkt_buf_addr_pp, l2_len, 1581 multi, error_type, pktbufsz_type)); 1582 1583 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1584 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1585 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1586 "error_type 0x%x ", rcr_desc_rd_head_p, 1587 rcr_entry, pkt_buf_addr_pp, l2_len, multi, error_type)); 1588 1589 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1590 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1591 "full pkt_buf_addr_pp $%p l2_len %d", 1592 rcr_entry, pkt_buf_addr_pp, l2_len)); 1593 1594 /* get the stats ptr */ 1595 rdc_stats = rcr_p->rdc_stats; 1596 1597 if (!l2_len) { 1598 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1599 "<== hxge_receive_packet: failed: l2 length is 0.")); 1600 return; 1601 } 1602 1603 /* shift 6 bits to get the full io address */ 1604 #if defined(__i386) 1605 pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp << 1606 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1607 #else 1608 pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp << 1609 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1610 #endif 1611 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1612 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1613 "full pkt_buf_addr_pp $%p l2_len %d", 1614 rcr_entry, pkt_buf_addr_pp, l2_len)); 1615 1616 rx_rbr_p = rcr_p->rx_rbr_p; 1617 rx_msg_ring_p = rx_rbr_p->rx_msg_ring; 1618 1619 if (first_entry) { 1620 hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL : 1621 RXDMA_HDR_SIZE_DEFAULT); 1622 1623 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1624 "==> hxge_receive_packet: first entry 0x%016llx " 1625 "pkt_buf_addr_pp $%p l2_len %d hdr %d", 1626 rcr_entry, pkt_buf_addr_pp, l2_len, hdr_size)); 1627 } 1628 1629 MUTEX_ENTER(&rcr_p->lock); 1630 MUTEX_ENTER(&rx_rbr_p->lock); 1631 1632 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1633 "==> (rbr 1) hxge_receive_packet: entry 0x%0llx " 1634 "full pkt_buf_addr_pp $%p l2_len %d", 1635 rcr_entry, pkt_buf_addr_pp, l2_len)); 1636 1637 /* 1638 * Packet buffer address in the completion entry points to the starting 1639 * buffer address (offset 0). Use the starting buffer address to locate 1640 * the corresponding kernel address. 1641 */ 1642 status = hxge_rxbuf_pp_to_vp(hxgep, rx_rbr_p, 1643 pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p, 1644 &buf_offset, &msg_index); 1645 1646 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1647 "==> (rbr 2) hxge_receive_packet: entry 0x%0llx " 1648 "full pkt_buf_addr_pp $%p l2_len %d", 1649 rcr_entry, pkt_buf_addr_pp, l2_len)); 1650 1651 if (status != HXGE_OK) { 1652 MUTEX_EXIT(&rx_rbr_p->lock); 1653 MUTEX_EXIT(&rcr_p->lock); 1654 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1655 "<== hxge_receive_packet: found vaddr failed %d", status)); 1656 return; 1657 } 1658 1659 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1660 "==> (rbr 3) hxge_receive_packet: entry 0x%0llx " 1661 "full pkt_buf_addr_pp $%p l2_len %d", 1662 rcr_entry, pkt_buf_addr_pp, l2_len)); 1663 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1664 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1665 "full pkt_buf_addr_pp $%p l2_len %d", 1666 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1667 1668 if (msg_index >= rx_rbr_p->tnblocks) { 1669 MUTEX_EXIT(&rx_rbr_p->lock); 1670 MUTEX_EXIT(&rcr_p->lock); 1671 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1672 "==> hxge_receive_packet: FATAL msg_index (%d) " 1673 "should be smaller than tnblocks (%d)\n", 1674 msg_index, rx_rbr_p->tnblocks)); 1675 return; 1676 } 1677 1678 rx_msg_p = rx_msg_ring_p[msg_index]; 1679 1680 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1681 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1682 "full pkt_buf_addr_pp $%p l2_len %d", 1683 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1684 1685 switch (pktbufsz_type) { 1686 case RCR_PKTBUFSZ_0: 1687 bsize = rx_rbr_p->pkt_buf_size0_bytes; 1688 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1689 "==> hxge_receive_packet: 0 buf %d", bsize)); 1690 break; 1691 case RCR_PKTBUFSZ_1: 1692 bsize = rx_rbr_p->pkt_buf_size1_bytes; 1693 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1694 "==> hxge_receive_packet: 1 buf %d", bsize)); 1695 break; 1696 case RCR_PKTBUFSZ_2: 1697 bsize = rx_rbr_p->pkt_buf_size2_bytes; 1698 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1699 "==> hxge_receive_packet: 2 buf %d", bsize)); 1700 break; 1701 case RCR_SINGLE_BLOCK: 1702 bsize = rx_msg_p->block_size; 1703 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1704 "==> hxge_receive_packet: single %d", bsize)); 1705 1706 break; 1707 default: 1708 MUTEX_EXIT(&rx_rbr_p->lock); 1709 MUTEX_EXIT(&rcr_p->lock); 1710 return; 1711 } 1712 1713 DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma, 1714 (buf_offset + sw_offset_bytes), (hdr_size + l2_len), 1715 DDI_DMA_SYNC_FORCPU); 1716 1717 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1718 "==> hxge_receive_packet: after first dump:usage count")); 1719 1720 if (rx_msg_p->cur_usage_cnt == 0) { 1721 atomic_inc_32(&rx_rbr_p->rbr_used); 1722 if (rx_rbr_p->rbr_use_bcopy) { 1723 atomic_inc_32(&rx_rbr_p->rbr_consumed); 1724 if (rx_rbr_p->rbr_consumed < 1725 rx_rbr_p->rbr_threshold_hi) { 1726 if (rx_rbr_p->rbr_threshold_lo == 0 || 1727 ((rx_rbr_p->rbr_consumed >= 1728 rx_rbr_p->rbr_threshold_lo) && 1729 (rx_rbr_p->rbr_bufsize_type >= 1730 pktbufsz_type))) { 1731 rx_msg_p->rx_use_bcopy = B_TRUE; 1732 } 1733 } else { 1734 rx_msg_p->rx_use_bcopy = B_TRUE; 1735 } 1736 } 1737 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1738 "==> hxge_receive_packet: buf %d (new block) ", bsize)); 1739 1740 rx_msg_p->pkt_buf_size_code = pktbufsz_type; 1741 rx_msg_p->pkt_buf_size = bsize; 1742 rx_msg_p->cur_usage_cnt = 1; 1743 if (pktbufsz_type == RCR_SINGLE_BLOCK) { 1744 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1745 "==> hxge_receive_packet: buf %d (single block) ", 1746 bsize)); 1747 /* 1748 * Buffer can be reused once the free function is 1749 * called. 1750 */ 1751 rx_msg_p->max_usage_cnt = 1; 1752 buffer_free = B_TRUE; 1753 } else { 1754 rx_msg_p->max_usage_cnt = rx_msg_p->block_size / bsize; 1755 if (rx_msg_p->max_usage_cnt == 1) { 1756 buffer_free = B_TRUE; 1757 } 1758 } 1759 } else { 1760 rx_msg_p->cur_usage_cnt++; 1761 if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) { 1762 buffer_free = B_TRUE; 1763 } 1764 } 1765 1766 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1767 "msgbuf index = %d l2len %d bytes usage %d max_usage %d ", 1768 msg_index, l2_len, 1769 rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt)); 1770 1771 if (error_type) { 1772 rdc_stats->ierrors++; 1773 /* Update error stats */ 1774 rdc_stats->errlog.compl_err_type = error_type; 1775 HXGE_FM_REPORT_ERROR(hxgep, NULL, HXGE_FM_EREPORT_RDMC_RCR_ERR); 1776 1777 if (error_type & RCR_CTRL_FIFO_DED) { 1778 rdc_stats->ctrl_fifo_ecc_err++; 1779 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1780 " hxge_receive_packet: " 1781 " channel %d RCR ctrl_fifo_ded error", channel)); 1782 } else if (error_type & RCR_DATA_FIFO_DED) { 1783 rdc_stats->data_fifo_ecc_err++; 1784 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1785 " hxge_receive_packet: channel %d" 1786 " RCR data_fifo_ded error", channel)); 1787 } 1788 1789 /* 1790 * Update and repost buffer block if max usage count is 1791 * reached. 1792 */ 1793 if (error_send_up == B_FALSE) { 1794 atomic_inc_32(&rx_msg_p->ref_cnt); 1795 if (buffer_free == B_TRUE) { 1796 rx_msg_p->free = B_TRUE; 1797 } 1798 1799 MUTEX_EXIT(&rx_rbr_p->lock); 1800 MUTEX_EXIT(&rcr_p->lock); 1801 hxge_freeb(rx_msg_p); 1802 return; 1803 } 1804 } 1805 1806 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1807 "==> hxge_receive_packet: DMA sync second ")); 1808 1809 bytes_read = rcr_p->rcvd_pkt_bytes; 1810 skip_len = sw_offset_bytes + hdr_size; 1811 1812 if (first_entry) { 1813 header0 = rx_msg_p->buffer[buf_offset]; 1814 no_port_bit = header0 & NO_PORT_BIT; 1815 1816 header1 = rx_msg_p->buffer[buf_offset + 1]; 1817 l4_cs_eq_bit = header1 & L4_CS_EQ_BIT; 1818 } 1819 1820 if (!rx_msg_p->rx_use_bcopy) { 1821 /* 1822 * For loaned up buffers, the driver reference count 1823 * will be incremented first and then the free state. 1824 */ 1825 if ((nmp = hxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) { 1826 if (first_entry) { 1827 nmp->b_rptr = &nmp->b_rptr[skip_len]; 1828 if (l2_len < bsize - skip_len) { 1829 nmp->b_wptr = &nmp->b_rptr[l2_len]; 1830 } else { 1831 nmp->b_wptr = &nmp->b_rptr[bsize 1832 - skip_len]; 1833 } 1834 } else { 1835 if (l2_len - bytes_read < bsize) { 1836 nmp->b_wptr = 1837 &nmp->b_rptr[l2_len - bytes_read]; 1838 } else { 1839 nmp->b_wptr = &nmp->b_rptr[bsize]; 1840 } 1841 } 1842 } 1843 } else { 1844 if (first_entry) { 1845 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len, 1846 l2_len < bsize - skip_len ? 1847 l2_len : bsize - skip_len); 1848 } else { 1849 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset, 1850 l2_len - bytes_read < bsize ? 1851 l2_len - bytes_read : bsize); 1852 } 1853 } 1854 1855 if (nmp != NULL) { 1856 if (first_entry) 1857 bytes_read = nmp->b_wptr - nmp->b_rptr; 1858 else 1859 bytes_read += nmp->b_wptr - nmp->b_rptr; 1860 1861 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1862 "==> hxge_receive_packet after dupb: " 1863 "rbr consumed %d " 1864 "pktbufsz_type %d " 1865 "nmp $%p rptr $%p wptr $%p " 1866 "buf_offset %d bzise %d l2_len %d skip_len %d", 1867 rx_rbr_p->rbr_consumed, 1868 pktbufsz_type, 1869 nmp, nmp->b_rptr, nmp->b_wptr, 1870 buf_offset, bsize, l2_len, skip_len)); 1871 } else { 1872 cmn_err(CE_WARN, "!hxge_receive_packet: update stats (error)"); 1873 1874 atomic_inc_32(&rx_msg_p->ref_cnt); 1875 if (buffer_free == B_TRUE) { 1876 rx_msg_p->free = B_TRUE; 1877 } 1878 1879 MUTEX_EXIT(&rx_rbr_p->lock); 1880 MUTEX_EXIT(&rcr_p->lock); 1881 hxge_freeb(rx_msg_p); 1882 return; 1883 } 1884 1885 if (buffer_free == B_TRUE) { 1886 rx_msg_p->free = B_TRUE; 1887 } 1888 1889 /* 1890 * ERROR, FRAG and PKT_TYPE are only reported in the first entry. If a 1891 * packet is not fragmented and no error bit is set, then L4 checksum 1892 * is OK. 1893 */ 1894 is_valid = (nmp != NULL); 1895 if (first_entry) { 1896 rdc_stats->ipackets++; /* count only 1st seg for jumbo */ 1897 if (l2_len > (STD_FRAME_SIZE - ETHERFCSL)) 1898 rdc_stats->jumbo_pkts++; 1899 rdc_stats->ibytes += skip_len + l2_len < bsize ? 1900 l2_len : bsize; 1901 } else { 1902 /* 1903 * Add the current portion of the packet to the kstats. 1904 * The current portion of the packet is calculated by using 1905 * length of the packet and the previously received portion. 1906 */ 1907 rdc_stats->ibytes += l2_len - rcr_p->rcvd_pkt_bytes < bsize ? 1908 l2_len - rcr_p->rcvd_pkt_bytes : bsize; 1909 } 1910 1911 rcr_p->rcvd_pkt_bytes = bytes_read; 1912 1913 if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) { 1914 atomic_inc_32(&rx_msg_p->ref_cnt); 1915 MUTEX_EXIT(&rx_rbr_p->lock); 1916 MUTEX_EXIT(&rcr_p->lock); 1917 hxge_freeb(rx_msg_p); 1918 } else { 1919 MUTEX_EXIT(&rx_rbr_p->lock); 1920 MUTEX_EXIT(&rcr_p->lock); 1921 } 1922 1923 if (is_valid) { 1924 nmp->b_cont = NULL; 1925 if (first_entry) { 1926 *mp = nmp; 1927 *mp_cont = NULL; 1928 } else { 1929 *mp_cont = nmp; 1930 } 1931 } 1932 1933 /* 1934 * Update stats and hardware checksuming. 1935 */ 1936 if (is_valid && !multi) { 1937 is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP || 1938 pkt_type == RCR_PKT_IS_UDP) ? B_TRUE : B_FALSE); 1939 1940 if (!no_port_bit && l4_cs_eq_bit && is_tcp_udp && !error_type) { 1941 (void) hcksum_assoc(nmp, NULL, NULL, 0, 0, 0, 0, 1942 HCK_FULLCKSUM_OK | HCK_FULLCKSUM, 0); 1943 1944 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1945 "==> hxge_receive_packet: Full tcp/udp cksum " 1946 "is_valid 0x%x multi %d error %d", 1947 is_valid, multi, error_type)); 1948 } 1949 } 1950 1951 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1952 "==> hxge_receive_packet: *mp 0x%016llx", *mp)); 1953 1954 *multi_p = (multi == RCR_MULTI_MASK); 1955 1956 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_receive_packet: " 1957 "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx", 1958 *multi_p, nmp, *mp, *mp_cont)); 1959 } 1960 1961 static void 1962 hxge_rx_rbr_empty_recover(p_hxge_t hxgep, uint8_t channel) 1963 { 1964 hpi_handle_t handle; 1965 p_rx_rcr_ring_t rcrp; 1966 p_rx_rbr_ring_t rbrp; 1967 1968 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 1969 rbrp = rcrp->rx_rbr_p; 1970 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1971 1972 /* 1973 * Wait for the channel to be quiet 1974 */ 1975 (void) hpi_rxdma_cfg_rdc_wait_for_qst(handle, channel); 1976 1977 /* 1978 * Post page will accumulate some buffers before re-enabling 1979 * the DMA channel. 1980 */ 1981 1982 MUTEX_ENTER(&rbrp->post_lock); 1983 if ((rbrp->rbb_max - rbrp->rbr_used) >= HXGE_RBR_EMPTY_THRESHOLD) { 1984 hxge_rbr_empty_restore(hxgep, rbrp); 1985 } else { 1986 rbrp->rbr_is_empty = B_TRUE; 1987 } 1988 MUTEX_EXIT(&rbrp->post_lock); 1989 } 1990 1991 /*ARGSUSED*/ 1992 static hxge_status_t 1993 hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp, 1994 rdc_stat_t cs) 1995 { 1996 p_hxge_rx_ring_stats_t rdc_stats; 1997 hpi_handle_t handle; 1998 boolean_t rxchan_fatal = B_FALSE; 1999 uint8_t channel; 2000 hxge_status_t status = HXGE_OK; 2001 uint64_t cs_val; 2002 2003 HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_rx_err_evnts")); 2004 2005 handle = HXGE_DEV_HPI_HANDLE(hxgep); 2006 channel = ldvp->channel; 2007 2008 /* Clear the interrupts */ 2009 cs.bits.pktread = 0; 2010 cs.bits.ptrread = 0; 2011 cs_val = cs.value & RDC_STAT_WR1C; 2012 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs_val); 2013 2014 rdc_stats = &hxgep->statsp->rdc_stats[ldvp->vdma_index]; 2015 2016 if (cs.bits.rbr_cpl_to) { 2017 rdc_stats->rbr_tmout++; 2018 HXGE_FM_REPORT_ERROR(hxgep, channel, 2019 HXGE_FM_EREPORT_RDMC_RBR_CPL_TO); 2020 rxchan_fatal = B_TRUE; 2021 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2022 "==> hxge_rx_err_evnts(channel %d): " 2023 "fatal error: rx_rbr_timeout", channel)); 2024 } 2025 2026 if ((cs.bits.rcr_shadow_par_err) || (cs.bits.rbr_prefetch_par_err)) { 2027 (void) hpi_rxdma_ring_perr_stat_get(handle, 2028 &rdc_stats->errlog.pre_par, &rdc_stats->errlog.sha_par); 2029 } 2030 2031 if (cs.bits.rcr_shadow_par_err) { 2032 rdc_stats->rcr_sha_par++; 2033 HXGE_FM_REPORT_ERROR(hxgep, channel, 2034 HXGE_FM_EREPORT_RDMC_RCR_SHA_PAR); 2035 rxchan_fatal = B_TRUE; 2036 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2037 "==> hxge_rx_err_evnts(channel %d): " 2038 "fatal error: rcr_shadow_par_err", channel)); 2039 } 2040 2041 if (cs.bits.rbr_prefetch_par_err) { 2042 rdc_stats->rbr_pre_par++; 2043 HXGE_FM_REPORT_ERROR(hxgep, channel, 2044 HXGE_FM_EREPORT_RDMC_RBR_PRE_PAR); 2045 rxchan_fatal = B_TRUE; 2046 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2047 "==> hxge_rx_err_evnts(channel %d): " 2048 "fatal error: rbr_prefetch_par_err", channel)); 2049 } 2050 2051 if (cs.bits.rbr_pre_empty) { 2052 rdc_stats->rbr_pre_empty++; 2053 HXGE_FM_REPORT_ERROR(hxgep, channel, 2054 HXGE_FM_EREPORT_RDMC_RBR_PRE_EMPTY); 2055 rxchan_fatal = B_TRUE; 2056 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2057 "==> hxge_rx_err_evnts(channel %d): " 2058 "fatal error: rbr_pre_empty", channel)); 2059 } 2060 2061 if (cs.bits.peu_resp_err) { 2062 rdc_stats->peu_resp_err++; 2063 HXGE_FM_REPORT_ERROR(hxgep, channel, 2064 HXGE_FM_EREPORT_RDMC_PEU_RESP_ERR); 2065 rxchan_fatal = B_TRUE; 2066 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2067 "==> hxge_rx_err_evnts(channel %d): " 2068 "fatal error: peu_resp_err", channel)); 2069 } 2070 2071 if (cs.bits.rcr_thres) { 2072 rdc_stats->rcr_thres++; 2073 } 2074 2075 if (cs.bits.rcr_to) { 2076 rdc_stats->rcr_to++; 2077 } 2078 2079 if (cs.bits.rcr_shadow_full) { 2080 rdc_stats->rcr_shadow_full++; 2081 HXGE_FM_REPORT_ERROR(hxgep, channel, 2082 HXGE_FM_EREPORT_RDMC_RCR_SHA_FULL); 2083 rxchan_fatal = B_TRUE; 2084 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2085 "==> hxge_rx_err_evnts(channel %d): " 2086 "fatal error: rcr_shadow_full", channel)); 2087 } 2088 2089 if (cs.bits.rcr_full) { 2090 rdc_stats->rcrfull++; 2091 HXGE_FM_REPORT_ERROR(hxgep, channel, 2092 HXGE_FM_EREPORT_RDMC_RCRFULL); 2093 rxchan_fatal = B_TRUE; 2094 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2095 "==> hxge_rx_err_evnts(channel %d): " 2096 "fatal error: rcrfull error", channel)); 2097 } 2098 2099 if (cs.bits.rbr_empty) { 2100 rdc_stats->rbr_empty++; 2101 hxge_rx_rbr_empty_recover(hxgep, channel); 2102 } 2103 2104 if (cs.bits.rbr_full) { 2105 rdc_stats->rbrfull++; 2106 HXGE_FM_REPORT_ERROR(hxgep, channel, 2107 HXGE_FM_EREPORT_RDMC_RBRFULL); 2108 rxchan_fatal = B_TRUE; 2109 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2110 "==> hxge_rx_err_evnts(channel %d): " 2111 "fatal error: rbr_full error", channel)); 2112 } 2113 2114 if (rxchan_fatal) { 2115 p_rx_rcr_ring_t rcrp; 2116 p_rx_rbr_ring_t rbrp; 2117 2118 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 2119 rbrp = rcrp->rx_rbr_p; 2120 2121 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2122 " hxge_rx_err_evnts: fatal error on Channel #%d\n", 2123 channel)); 2124 MUTEX_ENTER(&rbrp->post_lock); 2125 /* This function needs to be inside the post_lock */ 2126 status = hxge_rxdma_fatal_err_recover(hxgep, channel); 2127 MUTEX_EXIT(&rbrp->post_lock); 2128 if (status == HXGE_OK) { 2129 FM_SERVICE_RESTORED(hxgep); 2130 } 2131 } 2132 HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_rx_err_evnts")); 2133 2134 return (status); 2135 } 2136 2137 static hxge_status_t 2138 hxge_map_rxdma(p_hxge_t hxgep) 2139 { 2140 int i, ndmas; 2141 uint16_t channel; 2142 p_rx_rbr_rings_t rx_rbr_rings; 2143 p_rx_rbr_ring_t *rbr_rings; 2144 p_rx_rcr_rings_t rx_rcr_rings; 2145 p_rx_rcr_ring_t *rcr_rings; 2146 p_rx_mbox_areas_t rx_mbox_areas_p; 2147 p_rx_mbox_t *rx_mbox_p; 2148 p_hxge_dma_pool_t dma_buf_poolp; 2149 p_hxge_dma_common_t *dma_buf_p; 2150 p_hxge_dma_pool_t dma_rbr_cntl_poolp; 2151 p_hxge_dma_common_t *dma_rbr_cntl_p; 2152 p_hxge_dma_pool_t dma_rcr_cntl_poolp; 2153 p_hxge_dma_common_t *dma_rcr_cntl_p; 2154 p_hxge_dma_pool_t dma_mbox_cntl_poolp; 2155 p_hxge_dma_common_t *dma_mbox_cntl_p; 2156 uint32_t *num_chunks; 2157 hxge_status_t status = HXGE_OK; 2158 2159 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_map_rxdma")); 2160 2161 dma_buf_poolp = hxgep->rx_buf_pool_p; 2162 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p; 2163 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p; 2164 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p; 2165 2166 if (!dma_buf_poolp->buf_allocated || 2167 !dma_rbr_cntl_poolp->buf_allocated || 2168 !dma_rcr_cntl_poolp->buf_allocated || 2169 !dma_mbox_cntl_poolp->buf_allocated) { 2170 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2171 "<== hxge_map_rxdma: buf not allocated")); 2172 return (HXGE_ERROR); 2173 } 2174 2175 ndmas = dma_buf_poolp->ndmas; 2176 if (!ndmas) { 2177 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2178 "<== hxge_map_rxdma: no dma allocated")); 2179 return (HXGE_ERROR); 2180 } 2181 2182 num_chunks = dma_buf_poolp->num_chunks; 2183 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2184 dma_rbr_cntl_p = dma_rbr_cntl_poolp->dma_buf_pool_p; 2185 dma_rcr_cntl_p = dma_rcr_cntl_poolp->dma_buf_pool_p; 2186 dma_mbox_cntl_p = dma_mbox_cntl_poolp->dma_buf_pool_p; 2187 2188 rx_rbr_rings = (p_rx_rbr_rings_t) 2189 KMEM_ZALLOC(sizeof (rx_rbr_rings_t), KM_SLEEP); 2190 rbr_rings = (p_rx_rbr_ring_t *)KMEM_ZALLOC( 2191 sizeof (p_rx_rbr_ring_t) * ndmas, KM_SLEEP); 2192 2193 rx_rcr_rings = (p_rx_rcr_rings_t) 2194 KMEM_ZALLOC(sizeof (rx_rcr_rings_t), KM_SLEEP); 2195 rcr_rings = (p_rx_rcr_ring_t *)KMEM_ZALLOC( 2196 sizeof (p_rx_rcr_ring_t) * ndmas, KM_SLEEP); 2197 2198 rx_mbox_areas_p = (p_rx_mbox_areas_t) 2199 KMEM_ZALLOC(sizeof (rx_mbox_areas_t), KM_SLEEP); 2200 rx_mbox_p = (p_rx_mbox_t *)KMEM_ZALLOC( 2201 sizeof (p_rx_mbox_t) * ndmas, KM_SLEEP); 2202 2203 /* 2204 * Timeout should be set based on the system clock divider. 2205 * The following timeout value of 1 assumes that the 2206 * granularity (1000) is 3 microseconds running at 300MHz. 2207 */ 2208 2209 hxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT; 2210 hxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT; 2211 2212 /* 2213 * Map descriptors from the buffer polls for each dam channel. 2214 */ 2215 for (i = 0; i < ndmas; i++) { 2216 /* 2217 * Set up and prepare buffer blocks, descriptors and mailbox. 2218 */ 2219 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2220 status = hxge_map_rxdma_channel(hxgep, channel, 2221 (p_hxge_dma_common_t *)&dma_buf_p[i], 2222 (p_rx_rbr_ring_t *)&rbr_rings[i], 2223 num_chunks[i], 2224 (p_hxge_dma_common_t *)&dma_rbr_cntl_p[i], 2225 (p_hxge_dma_common_t *)&dma_rcr_cntl_p[i], 2226 (p_hxge_dma_common_t *)&dma_mbox_cntl_p[i], 2227 (p_rx_rcr_ring_t *)&rcr_rings[i], 2228 (p_rx_mbox_t *)&rx_mbox_p[i]); 2229 if (status != HXGE_OK) { 2230 goto hxge_map_rxdma_fail1; 2231 } 2232 rbr_rings[i]->index = (uint16_t)i; 2233 rcr_rings[i]->index = (uint16_t)i; 2234 rcr_rings[i]->rdc_stats = &hxgep->statsp->rdc_stats[i]; 2235 } 2236 2237 rx_rbr_rings->ndmas = rx_rcr_rings->ndmas = ndmas; 2238 rx_rbr_rings->rbr_rings = rbr_rings; 2239 hxgep->rx_rbr_rings = rx_rbr_rings; 2240 rx_rcr_rings->rcr_rings = rcr_rings; 2241 hxgep->rx_rcr_rings = rx_rcr_rings; 2242 2243 rx_mbox_areas_p->rxmbox_areas = rx_mbox_p; 2244 hxgep->rx_mbox_areas_p = rx_mbox_areas_p; 2245 2246 goto hxge_map_rxdma_exit; 2247 2248 hxge_map_rxdma_fail1: 2249 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2250 "==> hxge_map_rxdma: unmap rbr,rcr (status 0x%x channel %d i %d)", 2251 status, channel, i)); 2252 i--; 2253 for (; i >= 0; i--) { 2254 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2255 hxge_unmap_rxdma_channel(hxgep, channel, 2256 rbr_rings[i], rcr_rings[i], rx_mbox_p[i]); 2257 } 2258 2259 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2260 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2261 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2262 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2263 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2264 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2265 2266 hxge_map_rxdma_exit: 2267 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2268 "<== hxge_map_rxdma: (status 0x%x channel %d)", status, channel)); 2269 2270 return (status); 2271 } 2272 2273 static void 2274 hxge_unmap_rxdma(p_hxge_t hxgep) 2275 { 2276 int i, ndmas; 2277 uint16_t channel; 2278 p_rx_rbr_rings_t rx_rbr_rings; 2279 p_rx_rbr_ring_t *rbr_rings; 2280 p_rx_rcr_rings_t rx_rcr_rings; 2281 p_rx_rcr_ring_t *rcr_rings; 2282 p_rx_mbox_areas_t rx_mbox_areas_p; 2283 p_rx_mbox_t *rx_mbox_p; 2284 p_hxge_dma_pool_t dma_buf_poolp; 2285 p_hxge_dma_pool_t dma_rbr_cntl_poolp; 2286 p_hxge_dma_pool_t dma_rcr_cntl_poolp; 2287 p_hxge_dma_pool_t dma_mbox_cntl_poolp; 2288 p_hxge_dma_common_t *dma_buf_p; 2289 2290 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_unmap_rxdma")); 2291 2292 dma_buf_poolp = hxgep->rx_buf_pool_p; 2293 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p; 2294 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p; 2295 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p; 2296 2297 if (!dma_buf_poolp->buf_allocated || 2298 !dma_rbr_cntl_poolp->buf_allocated || 2299 !dma_rcr_cntl_poolp->buf_allocated || 2300 !dma_mbox_cntl_poolp->buf_allocated) { 2301 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2302 "<== hxge_unmap_rxdma: NULL buf pointers")); 2303 return; 2304 } 2305 2306 rx_rbr_rings = hxgep->rx_rbr_rings; 2307 rx_rcr_rings = hxgep->rx_rcr_rings; 2308 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 2309 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2310 "<== hxge_unmap_rxdma: NULL pointers")); 2311 return; 2312 } 2313 2314 ndmas = rx_rbr_rings->ndmas; 2315 if (!ndmas) { 2316 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2317 "<== hxge_unmap_rxdma: no channel")); 2318 return; 2319 } 2320 2321 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2322 "==> hxge_unmap_rxdma (ndmas %d)", ndmas)); 2323 2324 rbr_rings = rx_rbr_rings->rbr_rings; 2325 rcr_rings = rx_rcr_rings->rcr_rings; 2326 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 2327 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 2328 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2329 2330 for (i = 0; i < ndmas; i++) { 2331 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2332 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2333 "==> hxge_unmap_rxdma (ndmas %d) channel %d", 2334 ndmas, channel)); 2335 (void) hxge_unmap_rxdma_channel(hxgep, channel, 2336 (p_rx_rbr_ring_t)rbr_rings[i], 2337 (p_rx_rcr_ring_t)rcr_rings[i], 2338 (p_rx_mbox_t)rx_mbox_p[i]); 2339 } 2340 2341 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2342 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2343 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2344 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2345 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2346 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2347 2348 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma")); 2349 } 2350 2351 hxge_status_t 2352 hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2353 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 2354 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p, 2355 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 2356 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2357 { 2358 int status = HXGE_OK; 2359 2360 /* 2361 * Set up and prepare buffer blocks, descriptors and mailbox. 2362 */ 2363 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2364 "==> hxge_map_rxdma_channel (channel %d)", channel)); 2365 2366 /* 2367 * Receive buffer blocks 2368 */ 2369 status = hxge_map_rxdma_channel_buf_ring(hxgep, channel, 2370 dma_buf_p, rbr_p, num_chunks); 2371 if (status != HXGE_OK) { 2372 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2373 "==> hxge_map_rxdma_channel (channel %d): " 2374 "map buffer failed 0x%x", channel, status)); 2375 goto hxge_map_rxdma_channel_exit; 2376 } 2377 2378 /* 2379 * Receive block ring, completion ring and mailbox. 2380 */ 2381 status = hxge_map_rxdma_channel_cfg_ring(hxgep, channel, 2382 dma_rbr_cntl_p, dma_rcr_cntl_p, dma_mbox_cntl_p, 2383 rbr_p, rcr_p, rx_mbox_p); 2384 if (status != HXGE_OK) { 2385 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2386 "==> hxge_map_rxdma_channel (channel %d): " 2387 "map config failed 0x%x", channel, status)); 2388 goto hxge_map_rxdma_channel_fail2; 2389 } 2390 goto hxge_map_rxdma_channel_exit; 2391 2392 hxge_map_rxdma_channel_fail3: 2393 /* Free rbr, rcr */ 2394 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2395 "==> hxge_map_rxdma_channel: free rbr/rcr (status 0x%x channel %d)", 2396 status, channel)); 2397 hxge_unmap_rxdma_channel_cfg_ring(hxgep, *rcr_p, *rx_mbox_p); 2398 2399 hxge_map_rxdma_channel_fail2: 2400 /* Free buffer blocks */ 2401 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2402 "==> hxge_map_rxdma_channel: free rx buffers" 2403 "(hxgep 0x%x status 0x%x channel %d)", 2404 hxgep, status, channel)); 2405 hxge_unmap_rxdma_channel_buf_ring(hxgep, *rbr_p); 2406 2407 status = HXGE_ERROR; 2408 2409 hxge_map_rxdma_channel_exit: 2410 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2411 "<== hxge_map_rxdma_channel: (hxgep 0x%x status 0x%x channel %d)", 2412 hxgep, status, channel)); 2413 2414 return (status); 2415 } 2416 2417 /*ARGSUSED*/ 2418 static void 2419 hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2420 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2421 { 2422 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2423 "==> hxge_unmap_rxdma_channel (channel %d)", channel)); 2424 2425 /* 2426 * unmap receive block ring, completion ring and mailbox. 2427 */ 2428 (void) hxge_unmap_rxdma_channel_cfg_ring(hxgep, rcr_p, rx_mbox_p); 2429 2430 /* unmap buffer blocks */ 2431 (void) hxge_unmap_rxdma_channel_buf_ring(hxgep, rbr_p); 2432 2433 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma_channel")); 2434 } 2435 2436 /*ARGSUSED*/ 2437 static hxge_status_t 2438 hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel, 2439 p_hxge_dma_common_t *dma_rbr_cntl_p, p_hxge_dma_common_t *dma_rcr_cntl_p, 2440 p_hxge_dma_common_t *dma_mbox_cntl_p, p_rx_rbr_ring_t *rbr_p, 2441 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2442 { 2443 p_rx_rbr_ring_t rbrp; 2444 p_rx_rcr_ring_t rcrp; 2445 p_rx_mbox_t mboxp; 2446 p_hxge_dma_common_t cntl_dmap; 2447 p_hxge_dma_common_t dmap; 2448 p_rx_msg_t *rx_msg_ring; 2449 p_rx_msg_t rx_msg_p; 2450 rdc_rbr_cfg_a_t *rcfga_p; 2451 rdc_rbr_cfg_b_t *rcfgb_p; 2452 rdc_rcr_cfg_a_t *cfga_p; 2453 rdc_rcr_cfg_b_t *cfgb_p; 2454 rdc_rx_cfg1_t *cfig1_p; 2455 rdc_rx_cfg2_t *cfig2_p; 2456 rdc_rbr_kick_t *kick_p; 2457 uint32_t dmaaddrp; 2458 uint32_t *rbr_vaddrp; 2459 uint32_t bkaddr; 2460 hxge_status_t status = HXGE_OK; 2461 int i; 2462 uint32_t hxge_port_rcr_size; 2463 2464 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2465 "==> hxge_map_rxdma_channel_cfg_ring")); 2466 2467 cntl_dmap = *dma_rbr_cntl_p; 2468 2469 /* 2470 * Map in the receive block ring 2471 */ 2472 rbrp = *rbr_p; 2473 dmap = (p_hxge_dma_common_t)&rbrp->rbr_desc; 2474 hxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4); 2475 2476 /* 2477 * Zero out buffer block ring descriptors. 2478 */ 2479 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2480 2481 rcfga_p = &(rbrp->rbr_cfga); 2482 rcfgb_p = &(rbrp->rbr_cfgb); 2483 kick_p = &(rbrp->rbr_kick); 2484 rcfga_p->value = 0; 2485 rcfgb_p->value = 0; 2486 kick_p->value = 0; 2487 rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress; 2488 rcfga_p->value = (rbrp->rbr_addr & 2489 (RBR_CFIG_A_STDADDR_MASK | RBR_CFIG_A_STDADDR_BASE_MASK)); 2490 rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT); 2491 2492 /* XXXX: how to choose packet buffer sizes */ 2493 rcfgb_p->bits.bufsz0 = rbrp->pkt_buf_size0; 2494 rcfgb_p->bits.vld0 = 1; 2495 rcfgb_p->bits.bufsz1 = rbrp->pkt_buf_size1; 2496 rcfgb_p->bits.vld1 = 1; 2497 rcfgb_p->bits.bufsz2 = rbrp->pkt_buf_size2; 2498 rcfgb_p->bits.vld2 = 1; 2499 rcfgb_p->bits.bksize = hxgep->rx_bksize_code; 2500 2501 /* 2502 * For each buffer block, enter receive block address to the ring. 2503 */ 2504 rbr_vaddrp = (uint32_t *)dmap->kaddrp; 2505 rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp; 2506 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2507 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2508 "rbr_vaddrp $%p", dma_channel, rbr_vaddrp)); 2509 2510 rx_msg_ring = rbrp->rx_msg_ring; 2511 for (i = 0; i < rbrp->tnblocks; i++) { 2512 rx_msg_p = rx_msg_ring[i]; 2513 rx_msg_p->hxgep = hxgep; 2514 rx_msg_p->rx_rbr_p = rbrp; 2515 bkaddr = (uint32_t) 2516 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2517 RBR_BKADDR_SHIFT)); 2518 rx_msg_p->free = B_FALSE; 2519 rx_msg_p->max_usage_cnt = 0xbaddcafe; 2520 2521 *rbr_vaddrp++ = bkaddr; 2522 } 2523 2524 kick_p->bits.bkadd = rbrp->rbb_max; 2525 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 2526 2527 rbrp->rbr_rd_index = 0; 2528 2529 rbrp->rbr_consumed = 0; 2530 rbrp->rbr_used = 0; 2531 rbrp->rbr_use_bcopy = B_TRUE; 2532 rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0; 2533 2534 /* 2535 * Do bcopy on packets greater than bcopy size once the lo threshold is 2536 * reached. This lo threshold should be less than the hi threshold. 2537 * 2538 * Do bcopy on every packet once the hi threshold is reached. 2539 */ 2540 if (hxge_rx_threshold_lo >= hxge_rx_threshold_hi) { 2541 /* default it to use hi */ 2542 hxge_rx_threshold_lo = hxge_rx_threshold_hi; 2543 } 2544 if (hxge_rx_buf_size_type > HXGE_RBR_TYPE2) { 2545 hxge_rx_buf_size_type = HXGE_RBR_TYPE2; 2546 } 2547 rbrp->rbr_bufsize_type = hxge_rx_buf_size_type; 2548 2549 switch (hxge_rx_threshold_hi) { 2550 default: 2551 case HXGE_RX_COPY_NONE: 2552 /* Do not do bcopy at all */ 2553 rbrp->rbr_use_bcopy = B_FALSE; 2554 rbrp->rbr_threshold_hi = rbrp->rbb_max; 2555 break; 2556 2557 case HXGE_RX_COPY_1: 2558 case HXGE_RX_COPY_2: 2559 case HXGE_RX_COPY_3: 2560 case HXGE_RX_COPY_4: 2561 case HXGE_RX_COPY_5: 2562 case HXGE_RX_COPY_6: 2563 case HXGE_RX_COPY_7: 2564 rbrp->rbr_threshold_hi = 2565 rbrp->rbb_max * (hxge_rx_threshold_hi) / 2566 HXGE_RX_BCOPY_SCALE; 2567 break; 2568 2569 case HXGE_RX_COPY_ALL: 2570 rbrp->rbr_threshold_hi = 0; 2571 break; 2572 } 2573 2574 switch (hxge_rx_threshold_lo) { 2575 default: 2576 case HXGE_RX_COPY_NONE: 2577 /* Do not do bcopy at all */ 2578 if (rbrp->rbr_use_bcopy) { 2579 rbrp->rbr_use_bcopy = B_FALSE; 2580 } 2581 rbrp->rbr_threshold_lo = rbrp->rbb_max; 2582 break; 2583 2584 case HXGE_RX_COPY_1: 2585 case HXGE_RX_COPY_2: 2586 case HXGE_RX_COPY_3: 2587 case HXGE_RX_COPY_4: 2588 case HXGE_RX_COPY_5: 2589 case HXGE_RX_COPY_6: 2590 case HXGE_RX_COPY_7: 2591 rbrp->rbr_threshold_lo = 2592 rbrp->rbb_max * (hxge_rx_threshold_lo) / 2593 HXGE_RX_BCOPY_SCALE; 2594 break; 2595 2596 case HXGE_RX_COPY_ALL: 2597 rbrp->rbr_threshold_lo = 0; 2598 break; 2599 } 2600 2601 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2602 "hxge_map_rxdma_channel_cfg_ring: channel %d rbb_max %d " 2603 "rbrp->rbr_bufsize_type %d rbb_threshold_hi %d " 2604 "rbb_threshold_lo %d", 2605 dma_channel, rbrp->rbb_max, rbrp->rbr_bufsize_type, 2606 rbrp->rbr_threshold_hi, rbrp->rbr_threshold_lo)); 2607 2608 /* Map in the receive completion ring */ 2609 rcrp = (p_rx_rcr_ring_t)KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP); 2610 rcrp->rdc = dma_channel; 2611 rcrp->hxgep = hxgep; 2612 2613 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 2614 rcrp->comp_size = hxge_port_rcr_size; 2615 rcrp->comp_wrap_mask = hxge_port_rcr_size - 1; 2616 2617 rcrp->max_receive_pkts = hxge_max_rx_pkts; 2618 2619 cntl_dmap = *dma_rcr_cntl_p; 2620 2621 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 2622 hxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size, 2623 sizeof (rcr_entry_t)); 2624 rcrp->comp_rd_index = 0; 2625 rcrp->comp_wt_index = 0; 2626 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 2627 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 2628 #if defined(__i386) 2629 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2630 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2631 #else 2632 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2633 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2634 #endif 2635 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 2636 (hxge_port_rcr_size - 1); 2637 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 2638 (hxge_port_rcr_size - 1); 2639 2640 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc); 2641 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3; 2642 2643 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2644 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2645 "rbr_vaddrp $%p rcr_desc_rd_head_p $%p " 2646 "rcr_desc_rd_head_pp $%p rcr_desc_rd_last_p $%p " 2647 "rcr_desc_rd_last_pp $%p ", 2648 dma_channel, rbr_vaddrp, rcrp->rcr_desc_rd_head_p, 2649 rcrp->rcr_desc_rd_head_pp, rcrp->rcr_desc_last_p, 2650 rcrp->rcr_desc_last_pp)); 2651 2652 /* 2653 * Zero out buffer block ring descriptors. 2654 */ 2655 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2656 rcrp->intr_timeout = hxgep->intr_timeout; 2657 rcrp->intr_threshold = hxgep->intr_threshold; 2658 rcrp->full_hdr_flag = B_FALSE; 2659 rcrp->sw_priv_hdr_len = 0; 2660 2661 cfga_p = &(rcrp->rcr_cfga); 2662 cfgb_p = &(rcrp->rcr_cfgb); 2663 cfga_p->value = 0; 2664 cfgb_p->value = 0; 2665 rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress; 2666 2667 cfga_p->value = (rcrp->rcr_addr & 2668 (RCRCFIG_A_STADDR_MASK | RCRCFIG_A_STADDR_BASE_MASK)); 2669 2670 cfga_p->value |= ((uint64_t)rcrp->comp_size << RCRCFIG_A_LEN_SHIF); 2671 2672 /* 2673 * Timeout should be set based on the system clock divider. The 2674 * following timeout value of 1 assumes that the granularity (1000) is 2675 * 3 microseconds running at 300MHz. 2676 */ 2677 cfgb_p->bits.pthres = rcrp->intr_threshold; 2678 cfgb_p->bits.timeout = rcrp->intr_timeout; 2679 cfgb_p->bits.entout = 1; 2680 2681 /* Map in the mailbox */ 2682 cntl_dmap = *dma_mbox_cntl_p; 2683 mboxp = (p_rx_mbox_t)KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP); 2684 dmap = (p_hxge_dma_common_t)&mboxp->rx_mbox; 2685 hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t)); 2686 cfig1_p = (rdc_rx_cfg1_t *)&mboxp->rx_cfg1; 2687 cfig2_p = (rdc_rx_cfg2_t *)&mboxp->rx_cfg2; 2688 cfig1_p->value = cfig2_p->value = 0; 2689 2690 mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress; 2691 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2692 "==> hxge_map_rxdma_channel_cfg_ring: " 2693 "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx", 2694 dma_channel, cfig1_p->value, cfig2_p->value, 2695 mboxp->mbox_addr)); 2696 2697 dmaaddrp = (uint32_t)((dmap->dma_cookie.dmac_laddress >> 32) & 0xfff); 2698 cfig1_p->bits.mbaddr_h = dmaaddrp; 2699 2700 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff); 2701 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 2702 RXDMA_CFIG2_MBADDR_L_MASK); 2703 2704 cfig2_p->bits.mbaddr_l = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT); 2705 2706 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2707 "==> hxge_map_rxdma_channel_cfg_ring: channel %d damaddrp $%p " 2708 "cfg1 0x%016llx cfig2 0x%016llx", 2709 dma_channel, dmaaddrp, cfig1_p->value, cfig2_p->value)); 2710 2711 cfig2_p->bits.full_hdr = rcrp->full_hdr_flag; 2712 cfig2_p->bits.offset = rcrp->sw_priv_hdr_len; 2713 2714 rbrp->rx_rcr_p = rcrp; 2715 rcrp->rx_rbr_p = rbrp; 2716 *rcr_p = rcrp; 2717 *rx_mbox_p = mboxp; 2718 2719 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2720 "<== hxge_map_rxdma_channel_cfg_ring status 0x%08x", status)); 2721 return (status); 2722 } 2723 2724 /*ARGSUSED*/ 2725 static void 2726 hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 2727 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2728 { 2729 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2730 "==> hxge_unmap_rxdma_channel_cfg_ring: channel %d", rcr_p->rdc)); 2731 2732 KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t)); 2733 KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t)); 2734 2735 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2736 "<== hxge_unmap_rxdma_channel_cfg_ring")); 2737 } 2738 2739 static hxge_status_t 2740 hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel, 2741 p_hxge_dma_common_t *dma_buf_p, 2742 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks) 2743 { 2744 p_rx_rbr_ring_t rbrp; 2745 p_hxge_dma_common_t dma_bufp, tmp_bufp; 2746 p_rx_msg_t *rx_msg_ring; 2747 p_rx_msg_t rx_msg_p; 2748 p_mblk_t mblk_p; 2749 2750 rxring_info_t *ring_info; 2751 hxge_status_t status = HXGE_OK; 2752 int i, j, index; 2753 uint32_t size, bsize, nblocks, nmsgs; 2754 2755 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2756 "==> hxge_map_rxdma_channel_buf_ring: channel %d", channel)); 2757 2758 dma_bufp = tmp_bufp = *dma_buf_p; 2759 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2760 " hxge_map_rxdma_channel_buf_ring: channel %d to map %d " 2761 "chunks bufp 0x%016llx", channel, num_chunks, dma_bufp)); 2762 2763 nmsgs = 0; 2764 for (i = 0; i < num_chunks; i++, tmp_bufp++) { 2765 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2766 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2767 "bufp 0x%016llx nblocks %d nmsgs %d", 2768 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs)); 2769 nmsgs += tmp_bufp->nblocks; 2770 } 2771 if (!nmsgs) { 2772 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2773 "<== hxge_map_rxdma_channel_buf_ring: channel %d " 2774 "no msg blocks", channel)); 2775 status = HXGE_ERROR; 2776 goto hxge_map_rxdma_channel_buf_ring_exit; 2777 } 2778 rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (rx_rbr_ring_t), KM_SLEEP); 2779 2780 size = nmsgs * sizeof (p_rx_msg_t); 2781 rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP); 2782 ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t), 2783 KM_SLEEP); 2784 2785 MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER, 2786 (void *) hxgep->interrupt_cookie); 2787 MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER, 2788 (void *) hxgep->interrupt_cookie); 2789 2790 rbrp->rdc = channel; 2791 rbrp->num_blocks = num_chunks; 2792 rbrp->tnblocks = nmsgs; 2793 rbrp->rbb_max = nmsgs; 2794 rbrp->rbr_max_size = nmsgs; 2795 rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1); 2796 2797 /* 2798 * Buffer sizes suggested by NIU architect. 256, 512 and 2K. 2799 */ 2800 2801 switch (hxgep->rx_bksize_code) { 2802 case RBR_BKSIZE_4K: 2803 rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B; 2804 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES; 2805 rbrp->hpi_pkt_buf_size0 = SIZE_256B; 2806 break; 2807 case RBR_BKSIZE_8K: 2808 /* Use 512 to avoid possible rcr_full condition */ 2809 rbrp->pkt_buf_size0 = RBR_BUFSZ0_512B; 2810 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_512_BYTES; 2811 rbrp->hpi_pkt_buf_size0 = SIZE_512B; 2812 break; 2813 } 2814 2815 rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K; 2816 rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES; 2817 rbrp->hpi_pkt_buf_size1 = SIZE_1KB; 2818 2819 rbrp->block_size = hxgep->rx_default_block_size; 2820 2821 if (!hxgep->param_arr[param_accept_jumbo].value) { 2822 rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K; 2823 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES; 2824 rbrp->hpi_pkt_buf_size2 = SIZE_2KB; 2825 } else { 2826 rbrp->hpi_pkt_buf_size2 = SIZE_4KB; 2827 rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K; 2828 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES; 2829 } 2830 2831 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2832 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2833 "actual rbr max %d rbb_max %d nmsgs %d " 2834 "rbrp->block_size %d default_block_size %d " 2835 "(config hxge_rbr_size %d hxge_rbr_spare_size %d)", 2836 channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs, 2837 rbrp->block_size, hxgep->rx_default_block_size, 2838 hxge_rbr_size, hxge_rbr_spare_size)); 2839 2840 /* 2841 * Map in buffers from the buffer pool. 2842 * Note that num_blocks is the num_chunks. For Sparc, there is likely 2843 * only one chunk. For x86, there will be many chunks. 2844 * Loop over chunks. 2845 */ 2846 index = 0; 2847 for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) { 2848 bsize = dma_bufp->block_size; 2849 nblocks = dma_bufp->nblocks; 2850 #if defined(__i386) 2851 ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp; 2852 #else 2853 ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp; 2854 #endif 2855 ring_info->buffer[i].buf_index = i; 2856 ring_info->buffer[i].buf_size = dma_bufp->alength; 2857 ring_info->buffer[i].start_index = index; 2858 #if defined(__i386) 2859 ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp; 2860 #else 2861 ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp; 2862 #endif 2863 2864 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2865 " hxge_map_rxdma_channel_buf_ring: map channel %d " 2866 "chunk %d nblocks %d chunk_size %x block_size 0x%x " 2867 "dma_bufp $%p dvma_addr $%p", channel, i, 2868 dma_bufp->nblocks, 2869 ring_info->buffer[i].buf_size, bsize, dma_bufp, 2870 ring_info->buffer[i].dvma_addr)); 2871 2872 /* loop over blocks within a chunk */ 2873 for (j = 0; j < nblocks; j++) { 2874 if ((rx_msg_p = hxge_allocb(bsize, BPRI_LO, 2875 dma_bufp)) == NULL) { 2876 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2877 "allocb failed (index %d i %d j %d)", 2878 index, i, j)); 2879 goto hxge_map_rxdma_channel_buf_ring_fail1; 2880 } 2881 rx_msg_ring[index] = rx_msg_p; 2882 rx_msg_p->block_index = index; 2883 rx_msg_p->shifted_addr = (uint32_t) 2884 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2885 RBR_BKADDR_SHIFT)); 2886 /* 2887 * Too much output 2888 * HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2889 * "index %d j %d rx_msg_p $%p mblk %p", 2890 * index, j, rx_msg_p, rx_msg_p->rx_mblk_p)); 2891 */ 2892 mblk_p = rx_msg_p->rx_mblk_p; 2893 mblk_p->b_wptr = mblk_p->b_rptr + bsize; 2894 2895 rbrp->rbr_ref_cnt++; 2896 index++; 2897 rx_msg_p->buf_dma.dma_channel = channel; 2898 } 2899 } 2900 if (i < rbrp->num_blocks) { 2901 goto hxge_map_rxdma_channel_buf_ring_fail1; 2902 } 2903 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2904 "hxge_map_rxdma_channel_buf_ring: done buf init " 2905 "channel %d msg block entries %d", channel, index)); 2906 ring_info->block_size_mask = bsize - 1; 2907 rbrp->rx_msg_ring = rx_msg_ring; 2908 rbrp->dma_bufp = dma_buf_p; 2909 rbrp->ring_info = ring_info; 2910 2911 status = hxge_rxbuf_index_info_init(hxgep, rbrp); 2912 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, " hxge_map_rxdma_channel_buf_ring: " 2913 "channel %d done buf info init", channel)); 2914 2915 /* 2916 * Finally, permit hxge_freeb() to call hxge_post_page(). 2917 */ 2918 rbrp->rbr_state = RBR_POSTING; 2919 2920 *rbr_p = rbrp; 2921 2922 goto hxge_map_rxdma_channel_buf_ring_exit; 2923 2924 hxge_map_rxdma_channel_buf_ring_fail1: 2925 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2926 " hxge_map_rxdma_channel_buf_ring: failed channel (0x%x)", 2927 channel, status)); 2928 2929 index--; 2930 for (; index >= 0; index--) { 2931 rx_msg_p = rx_msg_ring[index]; 2932 if (rx_msg_p != NULL) { 2933 freeb(rx_msg_p->rx_mblk_p); 2934 rx_msg_ring[index] = NULL; 2935 } 2936 } 2937 2938 hxge_map_rxdma_channel_buf_ring_fail: 2939 MUTEX_DESTROY(&rbrp->post_lock); 2940 MUTEX_DESTROY(&rbrp->lock); 2941 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 2942 KMEM_FREE(rx_msg_ring, size); 2943 KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t)); 2944 2945 status = HXGE_ERROR; 2946 2947 hxge_map_rxdma_channel_buf_ring_exit: 2948 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2949 "<== hxge_map_rxdma_channel_buf_ring status 0x%08x", status)); 2950 2951 return (status); 2952 } 2953 2954 /*ARGSUSED*/ 2955 static void 2956 hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 2957 p_rx_rbr_ring_t rbr_p) 2958 { 2959 p_rx_msg_t *rx_msg_ring; 2960 p_rx_msg_t rx_msg_p; 2961 rxring_info_t *ring_info; 2962 int i; 2963 uint32_t size; 2964 2965 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2966 "==> hxge_unmap_rxdma_channel_buf_ring")); 2967 if (rbr_p == NULL) { 2968 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2969 "<== hxge_unmap_rxdma_channel_buf_ring: NULL rbrp")); 2970 return; 2971 } 2972 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2973 "==> hxge_unmap_rxdma_channel_buf_ring: channel %d", rbr_p->rdc)); 2974 2975 rx_msg_ring = rbr_p->rx_msg_ring; 2976 ring_info = rbr_p->ring_info; 2977 2978 if (rx_msg_ring == NULL || ring_info == NULL) { 2979 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2980 "<== hxge_unmap_rxdma_channel_buf_ring: " 2981 "rx_msg_ring $%p ring_info $%p", rx_msg_p, ring_info)); 2982 return; 2983 } 2984 2985 size = rbr_p->tnblocks * sizeof (p_rx_msg_t); 2986 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2987 " hxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d " 2988 "tnblocks %d (max %d) size ptrs %d ", rbr_p->rdc, rbr_p->num_blocks, 2989 rbr_p->tnblocks, rbr_p->rbr_max_size, size)); 2990 2991 for (i = 0; i < rbr_p->tnblocks; i++) { 2992 rx_msg_p = rx_msg_ring[i]; 2993 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2994 " hxge_unmap_rxdma_channel_buf_ring: " 2995 "rx_msg_p $%p", rx_msg_p)); 2996 if (rx_msg_p != NULL) { 2997 freeb(rx_msg_p->rx_mblk_p); 2998 rx_msg_ring[i] = NULL; 2999 } 3000 } 3001 3002 /* 3003 * We no longer may use the mutex <post_lock>. By setting 3004 * <rbr_state> to anything but POSTING, we prevent 3005 * hxge_post_page() from accessing a dead mutex. 3006 */ 3007 rbr_p->rbr_state = RBR_UNMAPPING; 3008 MUTEX_DESTROY(&rbr_p->post_lock); 3009 3010 MUTEX_DESTROY(&rbr_p->lock); 3011 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 3012 KMEM_FREE(rx_msg_ring, size); 3013 3014 if (rbr_p->rbr_ref_cnt == 0) { 3015 /* This is the normal state of affairs. */ 3016 KMEM_FREE(rbr_p, sizeof (*rbr_p)); 3017 } else { 3018 /* 3019 * Some of our buffers are still being used. 3020 * Therefore, tell hxge_freeb() this ring is 3021 * unmapped, so it may free <rbr_p> for us. 3022 */ 3023 rbr_p->rbr_state = RBR_UNMAPPED; 3024 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3025 "unmap_rxdma_buf_ring: %d %s outstanding.", 3026 rbr_p->rbr_ref_cnt, 3027 rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs")); 3028 } 3029 3030 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3031 "<== hxge_unmap_rxdma_channel_buf_ring")); 3032 } 3033 3034 static hxge_status_t 3035 hxge_rxdma_hw_start_common(p_hxge_t hxgep) 3036 { 3037 hxge_status_t status = HXGE_OK; 3038 3039 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 3040 3041 /* 3042 * Load the sharable parameters by writing to the function zero control 3043 * registers. These FZC registers should be initialized only once for 3044 * the entire chip. 3045 */ 3046 (void) hxge_init_fzc_rx_common(hxgep); 3047 3048 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 3049 3050 return (status); 3051 } 3052 3053 static hxge_status_t 3054 hxge_rxdma_hw_start(p_hxge_t hxgep) 3055 { 3056 int i, ndmas; 3057 uint16_t channel; 3058 p_rx_rbr_rings_t rx_rbr_rings; 3059 p_rx_rbr_ring_t *rbr_rings; 3060 p_rx_rcr_rings_t rx_rcr_rings; 3061 p_rx_rcr_ring_t *rcr_rings; 3062 p_rx_mbox_areas_t rx_mbox_areas_p; 3063 p_rx_mbox_t *rx_mbox_p; 3064 hxge_status_t status = HXGE_OK; 3065 3066 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start")); 3067 3068 rx_rbr_rings = hxgep->rx_rbr_rings; 3069 rx_rcr_rings = hxgep->rx_rcr_rings; 3070 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 3071 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3072 "<== hxge_rxdma_hw_start: NULL ring pointers")); 3073 return (HXGE_ERROR); 3074 } 3075 3076 ndmas = rx_rbr_rings->ndmas; 3077 if (ndmas == 0) { 3078 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3079 "<== hxge_rxdma_hw_start: no dma channel allocated")); 3080 return (HXGE_ERROR); 3081 } 3082 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3083 "==> hxge_rxdma_hw_start (ndmas %d)", ndmas)); 3084 3085 /* 3086 * Scrub the RDC Rx DMA Prefetch Buffer Command. 3087 */ 3088 for (i = 0; i < 128; i++) { 3089 HXGE_REG_WR64(hxgep->hpi_handle, RDC_PREF_CMD, i); 3090 } 3091 3092 /* 3093 * Scrub Rx DMA Shadow Tail Command. 3094 */ 3095 for (i = 0; i < 64; i++) { 3096 HXGE_REG_WR64(hxgep->hpi_handle, RDC_SHADOW_CMD, i); 3097 } 3098 3099 /* 3100 * Scrub Rx DMA Control Fifo Command. 3101 */ 3102 for (i = 0; i < 512; i++) { 3103 HXGE_REG_WR64(hxgep->hpi_handle, RDC_CTRL_FIFO_CMD, i); 3104 } 3105 3106 /* 3107 * Scrub Rx DMA Data Fifo Command. 3108 */ 3109 for (i = 0; i < 1536; i++) { 3110 HXGE_REG_WR64(hxgep->hpi_handle, RDC_DATA_FIFO_CMD, i); 3111 } 3112 3113 /* 3114 * Reset the FIFO Error Stat. 3115 */ 3116 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_STAT, 0xFF); 3117 3118 /* Set the error mask to receive interrupts */ 3119 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 3120 3121 rbr_rings = rx_rbr_rings->rbr_rings; 3122 rcr_rings = rx_rcr_rings->rcr_rings; 3123 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 3124 if (rx_mbox_areas_p) { 3125 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 3126 } 3127 3128 for (i = 0; i < ndmas; i++) { 3129 channel = rbr_rings[i]->rdc; 3130 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3131 "==> hxge_rxdma_hw_start (ndmas %d) channel %d", 3132 ndmas, channel)); 3133 status = hxge_rxdma_start_channel(hxgep, channel, 3134 (p_rx_rbr_ring_t)rbr_rings[i], 3135 (p_rx_rcr_ring_t)rcr_rings[i], 3136 (p_rx_mbox_t)rx_mbox_p[i], rbr_rings[i]->rbb_max); 3137 if (status != HXGE_OK) { 3138 goto hxge_rxdma_hw_start_fail1; 3139 } 3140 } 3141 3142 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start: " 3143 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3144 rx_rbr_rings, rx_rcr_rings)); 3145 goto hxge_rxdma_hw_start_exit; 3146 3147 hxge_rxdma_hw_start_fail1: 3148 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3149 "==> hxge_rxdma_hw_start: disable " 3150 "(status 0x%x channel %d i %d)", status, channel, i)); 3151 for (; i >= 0; i--) { 3152 channel = rbr_rings[i]->rdc; 3153 (void) hxge_rxdma_stop_channel(hxgep, channel); 3154 } 3155 3156 hxge_rxdma_hw_start_exit: 3157 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3158 "==> hxge_rxdma_hw_start: (status 0x%x)", status)); 3159 return (status); 3160 } 3161 3162 static void 3163 hxge_rxdma_hw_stop(p_hxge_t hxgep) 3164 { 3165 int i, ndmas; 3166 uint16_t channel; 3167 p_rx_rbr_rings_t rx_rbr_rings; 3168 p_rx_rbr_ring_t *rbr_rings; 3169 p_rx_rcr_rings_t rx_rcr_rings; 3170 3171 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop")); 3172 3173 rx_rbr_rings = hxgep->rx_rbr_rings; 3174 rx_rcr_rings = hxgep->rx_rcr_rings; 3175 3176 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 3177 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3178 "<== hxge_rxdma_hw_stop: NULL ring pointers")); 3179 return; 3180 } 3181 3182 ndmas = rx_rbr_rings->ndmas; 3183 if (!ndmas) { 3184 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3185 "<== hxge_rxdma_hw_stop: no dma channel allocated")); 3186 return; 3187 } 3188 3189 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3190 "==> hxge_rxdma_hw_stop (ndmas %d)", ndmas)); 3191 3192 rbr_rings = rx_rbr_rings->rbr_rings; 3193 for (i = 0; i < ndmas; i++) { 3194 channel = rbr_rings[i]->rdc; 3195 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3196 "==> hxge_rxdma_hw_stop (ndmas %d) channel %d", 3197 ndmas, channel)); 3198 (void) hxge_rxdma_stop_channel(hxgep, channel); 3199 } 3200 3201 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop: " 3202 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3203 rx_rbr_rings, rx_rcr_rings)); 3204 3205 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_hw_stop")); 3206 } 3207 3208 static hxge_status_t 3209 hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 3210 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 3211 int n_init_kick) 3212 { 3213 hpi_handle_t handle; 3214 hpi_status_t rs = HPI_SUCCESS; 3215 rdc_stat_t cs; 3216 rdc_int_mask_t ent_mask; 3217 hxge_status_t status = HXGE_OK; 3218 3219 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel")); 3220 3221 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3222 3223 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "hxge_rxdma_start_channel: " 3224 "hpi handle addr $%p acc $%p", 3225 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3226 3227 /* Reset RXDMA channel */ 3228 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3229 if (rs != HPI_SUCCESS) { 3230 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3231 "==> hxge_rxdma_start_channel: " 3232 "reset rxdma failed (0x%08x channel %d)", 3233 status, channel)); 3234 return (HXGE_ERROR | rs); 3235 } 3236 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3237 "==> hxge_rxdma_start_channel: reset done: channel %d", channel)); 3238 3239 /* 3240 * Initialize the RXDMA channel specific FZC control configurations. 3241 * These FZC registers are pertaining to each RX channel (logical 3242 * pages). 3243 */ 3244 status = hxge_init_fzc_rxdma_channel(hxgep, 3245 channel, rbr_p, rcr_p, mbox_p); 3246 if (status != HXGE_OK) { 3247 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3248 "==> hxge_rxdma_start_channel: " 3249 "init fzc rxdma failed (0x%08x channel %d)", 3250 status, channel)); 3251 return (status); 3252 } 3253 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3254 "==> hxge_rxdma_start_channel: fzc done")); 3255 3256 /* 3257 * Zero out the shadow and prefetch ram. 3258 */ 3259 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3260 "==> hxge_rxdma_start_channel: ram done")); 3261 3262 /* Set up the interrupt event masks. */ 3263 ent_mask.value = 0; 3264 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3265 if (rs != HPI_SUCCESS) { 3266 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3267 "==> hxge_rxdma_start_channel: " 3268 "init rxdma event masks failed (0x%08x channel %d)", 3269 status, channel)); 3270 return (HXGE_ERROR | rs); 3271 } 3272 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3273 "event done: channel %d (mask 0x%016llx)", 3274 channel, ent_mask.value)); 3275 3276 /* 3277 * Load RXDMA descriptors, buffers, mailbox, initialise the receive DMA 3278 * channels and enable each DMA channel. 3279 */ 3280 status = hxge_enable_rxdma_channel(hxgep, 3281 channel, rbr_p, rcr_p, mbox_p, n_init_kick); 3282 if (status != HXGE_OK) { 3283 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3284 " hxge_rxdma_start_channel: " 3285 " init enable rxdma failed (0x%08x channel %d)", 3286 status, channel)); 3287 return (status); 3288 } 3289 3290 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3291 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3292 3293 /* 3294 * Initialize the receive DMA control and status register 3295 * Note that rdc_stat HAS to be set after RBR and RCR rings are set 3296 */ 3297 cs.value = 0; 3298 cs.bits.mex = 1; 3299 cs.bits.rcr_thres = 1; 3300 cs.bits.rcr_to = 1; 3301 cs.bits.rbr_empty = 1; 3302 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3303 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3304 "channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value)); 3305 if (status != HXGE_OK) { 3306 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3307 "==> hxge_rxdma_start_channel: " 3308 "init rxdma control register failed (0x%08x channel %d", 3309 status, channel)); 3310 return (status); 3311 } 3312 3313 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3314 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3315 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3316 "==> hxge_rxdma_start_channel: enable done")); 3317 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_start_channel")); 3318 3319 return (HXGE_OK); 3320 } 3321 3322 static hxge_status_t 3323 hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel) 3324 { 3325 hpi_handle_t handle; 3326 hpi_status_t rs = HPI_SUCCESS; 3327 rdc_stat_t cs; 3328 rdc_int_mask_t ent_mask; 3329 hxge_status_t status = HXGE_OK; 3330 3331 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel")); 3332 3333 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3334 3335 HXGE_DEBUG_MSG((hxgep, RX_CTL, "hxge_rxdma_stop_channel: " 3336 "hpi handle addr $%p acc $%p", 3337 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3338 3339 /* Reset RXDMA channel */ 3340 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3341 if (rs != HPI_SUCCESS) { 3342 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3343 " hxge_rxdma_stop_channel: " 3344 " reset rxdma failed (0x%08x channel %d)", 3345 rs, channel)); 3346 return (HXGE_ERROR | rs); 3347 } 3348 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3349 "==> hxge_rxdma_stop_channel: reset done")); 3350 3351 /* Set up the interrupt event masks. */ 3352 ent_mask.value = RDC_INT_MASK_ALL; 3353 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3354 if (rs != HPI_SUCCESS) { 3355 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3356 "==> hxge_rxdma_stop_channel: " 3357 "set rxdma event masks failed (0x%08x channel %d)", 3358 rs, channel)); 3359 return (HXGE_ERROR | rs); 3360 } 3361 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3362 "==> hxge_rxdma_stop_channel: event done")); 3363 3364 /* Initialize the receive DMA control and status register */ 3365 cs.value = 0; 3366 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3367 3368 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel: control " 3369 " to default (all 0s) 0x%08x", cs.value)); 3370 3371 if (status != HXGE_OK) { 3372 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3373 " hxge_rxdma_stop_channel: init rxdma" 3374 " control register failed (0x%08x channel %d", 3375 status, channel)); 3376 return (status); 3377 } 3378 3379 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3380 "==> hxge_rxdma_stop_channel: control done")); 3381 3382 /* disable dma channel */ 3383 status = hxge_disable_rxdma_channel(hxgep, channel); 3384 3385 if (status != HXGE_OK) { 3386 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3387 " hxge_rxdma_stop_channel: " 3388 " init enable rxdma failed (0x%08x channel %d)", 3389 status, channel)); 3390 return (status); 3391 } 3392 3393 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3394 "==> hxge_rxdma_stop_channel: disable done")); 3395 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_channel")); 3396 3397 return (HXGE_OK); 3398 } 3399 3400 hxge_status_t 3401 hxge_rxdma_handle_sys_errors(p_hxge_t hxgep) 3402 { 3403 hpi_handle_t handle; 3404 p_hxge_rdc_sys_stats_t statsp; 3405 rdc_fifo_err_stat_t stat; 3406 hxge_status_t status = HXGE_OK; 3407 3408 handle = hxgep->hpi_handle; 3409 statsp = (p_hxge_rdc_sys_stats_t)&hxgep->statsp->rdc_sys_stats; 3410 3411 /* Clear the int_dbg register in case it is an injected err */ 3412 HXGE_REG_WR64(handle, RDC_FIFO_ERR_INT_DBG, 0x0); 3413 3414 /* Get the error status and clear the register */ 3415 HXGE_REG_RD64(handle, RDC_FIFO_ERR_STAT, &stat.value); 3416 HXGE_REG_WR64(handle, RDC_FIFO_ERR_STAT, stat.value); 3417 3418 if (stat.bits.rx_ctrl_fifo_sec) { 3419 statsp->ctrl_fifo_sec++; 3420 if (statsp->ctrl_fifo_sec == 1) 3421 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3422 "==> hxge_rxdma_handle_sys_errors: " 3423 "rx_ctrl_fifo_sec")); 3424 } 3425 3426 if (stat.bits.rx_ctrl_fifo_ded) { 3427 /* Global fatal error encountered */ 3428 statsp->ctrl_fifo_ded++; 3429 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3430 HXGE_FM_EREPORT_RDMC_CTRL_FIFO_DED); 3431 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3432 "==> hxge_rxdma_handle_sys_errors: " 3433 "fatal error: rx_ctrl_fifo_ded error")); 3434 } 3435 3436 if (stat.bits.rx_data_fifo_sec) { 3437 statsp->data_fifo_sec++; 3438 if (statsp->data_fifo_sec == 1) 3439 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3440 "==> hxge_rxdma_handle_sys_errors: " 3441 "rx_data_fifo_sec")); 3442 } 3443 3444 if (stat.bits.rx_data_fifo_ded) { 3445 /* Global fatal error encountered */ 3446 statsp->data_fifo_ded++; 3447 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3448 HXGE_FM_EREPORT_RDMC_DATA_FIFO_DED); 3449 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3450 "==> hxge_rxdma_handle_sys_errors: " 3451 "fatal error: rx_data_fifo_ded error")); 3452 } 3453 3454 if (stat.bits.rx_ctrl_fifo_ded || stat.bits.rx_data_fifo_ded) { 3455 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3456 " hxge_rxdma_handle_sys_errors: fatal error\n")); 3457 status = hxge_rx_port_fatal_err_recover(hxgep); 3458 if (status == HXGE_OK) { 3459 FM_SERVICE_RESTORED(hxgep); 3460 } 3461 } 3462 3463 return (HXGE_OK); 3464 } 3465 3466 static hxge_status_t 3467 hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel) 3468 { 3469 hpi_handle_t handle; 3470 hpi_status_t rs = HPI_SUCCESS; 3471 hxge_status_t status = HXGE_OK; 3472 p_rx_rbr_ring_t rbrp; 3473 p_rx_rcr_ring_t rcrp; 3474 p_rx_mbox_t mboxp; 3475 rdc_int_mask_t ent_mask; 3476 p_hxge_dma_common_t dmap; 3477 int ring_idx; 3478 p_rx_msg_t rx_msg_p; 3479 int i; 3480 uint32_t hxge_port_rcr_size; 3481 uint64_t tmp; 3482 int n_init_kick = 0; 3483 3484 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_fatal_err_recover")); 3485 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3486 "Recovering from RxDMAChannel#%d error...", channel)); 3487 3488 /* 3489 * Stop the dma channel waits for the stop done. If the stop done bit 3490 * is not set, then create an error. 3491 */ 3492 3493 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3494 3495 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Rx DMA stop...")); 3496 3497 ring_idx = hxge_rxdma_get_ring_index(hxgep, channel); 3498 rbrp = (p_rx_rbr_ring_t)hxgep->rx_rbr_rings->rbr_rings[ring_idx]; 3499 rcrp = (p_rx_rcr_ring_t)hxgep->rx_rcr_rings->rcr_rings[ring_idx]; 3500 3501 MUTEX_ENTER(&rcrp->lock); 3502 MUTEX_ENTER(&rbrp->lock); 3503 3504 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA channel...")); 3505 3506 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 3507 if (rs != HPI_SUCCESS) { 3508 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3509 "hxge_disable_rxdma_channel:failed")); 3510 goto fail; 3511 } 3512 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA interrupt...")); 3513 3514 /* Disable interrupt */ 3515 ent_mask.value = RDC_INT_MASK_ALL; 3516 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3517 if (rs != HPI_SUCCESS) { 3518 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3519 "Set rxdma event masks failed (channel %d)", channel)); 3520 } 3521 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel reset...")); 3522 3523 /* Reset RXDMA channel */ 3524 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3525 if (rs != HPI_SUCCESS) { 3526 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3527 "Reset rxdma failed (channel %d)", channel)); 3528 goto fail; 3529 } 3530 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 3531 mboxp = (p_rx_mbox_t)hxgep->rx_mbox_areas_p->rxmbox_areas[ring_idx]; 3532 3533 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 3534 rbrp->rbr_rd_index = 0; 3535 3536 rcrp->comp_rd_index = 0; 3537 rcrp->comp_wt_index = 0; 3538 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 3539 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 3540 #if defined(__i386) 3541 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3542 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3543 #else 3544 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3545 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3546 #endif 3547 3548 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 3549 (hxge_port_rcr_size - 1); 3550 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 3551 (hxge_port_rcr_size - 1); 3552 3553 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc); 3554 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3; 3555 3556 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 3557 bzero((caddr_t)dmap->kaddrp, dmap->alength); 3558 3559 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rbr entries = %d\n", 3560 rbrp->rbr_max_size)); 3561 3562 /* Count the number of buffers owned by the hardware at this moment */ 3563 for (i = 0; i < rbrp->rbr_max_size; i++) { 3564 rx_msg_p = rbrp->rx_msg_ring[i]; 3565 if (rx_msg_p->ref_cnt == 1) { 3566 n_init_kick++; 3567 } 3568 } 3569 3570 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel re-start...")); 3571 3572 /* 3573 * This is error recover! Some buffers are owned by the hardware and 3574 * the rest are owned by the apps. We should only kick in those 3575 * owned by the hardware initially. The apps will post theirs 3576 * eventually. 3577 */ 3578 status = hxge_rxdma_start_channel(hxgep, channel, rbrp, rcrp, mboxp, 3579 n_init_kick); 3580 if (status != HXGE_OK) { 3581 goto fail; 3582 } 3583 3584 /* 3585 * The DMA channel may disable itself automatically. 3586 * The following is a work-around. 3587 */ 3588 HXGE_REG_RD64(handle, RDC_RX_CFG1, &tmp); 3589 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 3590 if (rs != HPI_SUCCESS) { 3591 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3592 "hpi_rxdma_cfg_rdc_enable (channel %d)", channel)); 3593 } 3594 3595 MUTEX_EXIT(&rbrp->lock); 3596 MUTEX_EXIT(&rcrp->lock); 3597 3598 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3599 "Recovery Successful, RxDMAChannel#%d Restored", channel)); 3600 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_fatal_err_recover")); 3601 3602 return (HXGE_OK); 3603 3604 fail: 3605 MUTEX_EXIT(&rbrp->lock); 3606 MUTEX_EXIT(&rcrp->lock); 3607 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3608 3609 return (HXGE_ERROR | rs); 3610 } 3611 3612 static hxge_status_t 3613 hxge_rx_port_fatal_err_recover(p_hxge_t hxgep) 3614 { 3615 hxge_status_t status = HXGE_OK; 3616 p_hxge_dma_common_t *dma_buf_p; 3617 uint16_t channel; 3618 int ndmas; 3619 int i; 3620 block_reset_t reset_reg; 3621 p_rx_rcr_ring_t rcrp; 3622 p_rx_rbr_ring_t rbrp; 3623 3624 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_port_fatal_err_recover")); 3625 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovering from RDC error ...")); 3626 3627 /* Reset RDC block from PEU for this fatal error */ 3628 reset_reg.value = 0; 3629 reset_reg.bits.rdc_rst = 1; 3630 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 3631 3632 /* Disable RxMAC */ 3633 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxMAC...\n")); 3634 if (hxge_rx_vmac_disable(hxgep) != HXGE_OK) 3635 goto fail; 3636 3637 HXGE_DELAY(1000); 3638 3639 /* Restore any common settings after PEU reset */ 3640 if (hxge_rxdma_hw_start_common(hxgep) != HXGE_OK) 3641 goto fail; 3642 3643 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Stop all RxDMA channels...")); 3644 3645 ndmas = hxgep->rx_buf_pool_p->ndmas; 3646 dma_buf_p = hxgep->rx_buf_pool_p->dma_buf_pool_p; 3647 3648 for (i = 0; i < ndmas; i++) { 3649 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 3650 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 3651 rbrp = rcrp->rx_rbr_p; 3652 3653 MUTEX_ENTER(&rbrp->post_lock); 3654 /* This function needs to be inside the post_lock */ 3655 if (hxge_rxdma_fatal_err_recover(hxgep, channel) != HXGE_OK) { 3656 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3657 "Could not recover channel %d", channel)); 3658 } 3659 MUTEX_EXIT(&rbrp->post_lock); 3660 } 3661 3662 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Reset RxMAC...")); 3663 3664 /* Reset RxMAC */ 3665 if (hxge_rx_vmac_reset(hxgep) != HXGE_OK) { 3666 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3667 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3668 goto fail; 3669 } 3670 3671 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-initialize RxMAC...")); 3672 3673 /* Re-Initialize RxMAC */ 3674 if ((status = hxge_rx_vmac_init(hxgep)) != HXGE_OK) { 3675 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3676 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3677 goto fail; 3678 } 3679 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-enable RxMAC...")); 3680 3681 /* Re-enable RxMAC */ 3682 if ((status = hxge_rx_vmac_enable(hxgep)) != HXGE_OK) { 3683 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3684 "hxge_rx_port_fatal_err_recover: Failed to enable RxMAC")); 3685 goto fail; 3686 } 3687 3688 /* Reset the error mask since PEU reset cleared it */ 3689 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 3690 3691 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3692 "Recovery Successful, RxPort Restored")); 3693 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_port_fatal_err_recover")); 3694 3695 return (HXGE_OK); 3696 fail: 3697 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3698 return (status); 3699 } 3700 3701 static void 3702 hxge_rbr_empty_restore(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p) 3703 { 3704 hpi_status_t hpi_status; 3705 hxge_status_t status; 3706 int i; 3707 p_hxge_rx_ring_stats_t rdc_stats; 3708 3709 rdc_stats = &hxgep->statsp->rdc_stats[rx_rbr_p->rdc]; 3710 rdc_stats->rbr_empty_restore++; 3711 rx_rbr_p->rbr_is_empty = B_FALSE; 3712 3713 /* 3714 * Complete the processing for the RBR Empty by: 3715 * 0) kicking back HXGE_RBR_EMPTY_THRESHOLD 3716 * packets. 3717 * 1) Disable the RX vmac. 3718 * 2) Re-enable the affected DMA channel. 3719 * 3) Re-enable the RX vmac. 3720 */ 3721 3722 /* 3723 * Disable the RX VMAC, but setting the framelength 3724 * to 0, since there is a hardware bug when disabling 3725 * the vmac. 3726 */ 3727 MUTEX_ENTER(hxgep->genlock); 3728 (void) hpi_vmac_rx_set_framesize( 3729 HXGE_DEV_HPI_HANDLE(hxgep), (uint16_t)0); 3730 3731 hpi_status = hpi_rxdma_cfg_rdc_enable( 3732 HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc); 3733 if (hpi_status != HPI_SUCCESS) { 3734 rdc_stats->rbr_empty_fail++; 3735 3736 /* Assume we are already inside the post_lock */ 3737 status = hxge_rxdma_fatal_err_recover(hxgep, rx_rbr_p->rdc); 3738 if (status != HXGE_OK) { 3739 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3740 "hxge(%d): channel(%d) is empty.", 3741 hxgep->instance, rx_rbr_p->rdc)); 3742 } 3743 } 3744 3745 for (i = 0; i < 1024; i++) { 3746 uint64_t value; 3747 RXDMA_REG_READ64(HXGE_DEV_HPI_HANDLE(hxgep), 3748 RDC_STAT, i & 3, &value); 3749 } 3750 3751 /* 3752 * Re-enable the RX VMAC. 3753 */ 3754 (void) hpi_vmac_rx_set_framesize(HXGE_DEV_HPI_HANDLE(hxgep), 3755 (uint16_t)hxgep->vmac.maxframesize); 3756 MUTEX_EXIT(hxgep->genlock); 3757 } 3758