1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <hxge_impl.h> 29 #include <hxge_rxdma.h> 30 31 /* 32 * Globals: tunable parameters (/etc/system or adb) 33 * 34 */ 35 extern uint32_t hxge_rbr_size; 36 extern uint32_t hxge_rcr_size; 37 extern uint32_t hxge_rbr_spare_size; 38 39 extern uint32_t hxge_mblks_pending; 40 41 /* 42 * Tunable to reduce the amount of time spent in the 43 * ISR doing Rx Processing. 44 */ 45 extern uint32_t hxge_max_rx_pkts; 46 boolean_t hxge_jumbo_enable; 47 48 /* 49 * Tunables to manage the receive buffer blocks. 50 * 51 * hxge_rx_threshold_hi: copy all buffers. 52 * hxge_rx_bcopy_size_type: receive buffer block size type. 53 * hxge_rx_threshold_lo: copy only up to tunable block size type. 54 */ 55 extern hxge_rxbuf_threshold_t hxge_rx_threshold_hi; 56 extern hxge_rxbuf_type_t hxge_rx_buf_size_type; 57 extern hxge_rxbuf_threshold_t hxge_rx_threshold_lo; 58 59 static hxge_status_t hxge_map_rxdma(p_hxge_t hxgep); 60 static void hxge_unmap_rxdma(p_hxge_t hxgep); 61 static hxge_status_t hxge_rxdma_hw_start_common(p_hxge_t hxgep); 62 static hxge_status_t hxge_rxdma_hw_start(p_hxge_t hxgep); 63 static void hxge_rxdma_hw_stop(p_hxge_t hxgep); 64 static hxge_status_t hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 65 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 66 uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p, 67 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 68 static void hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 69 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 70 static hxge_status_t hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, 71 uint16_t dma_channel, p_hxge_dma_common_t *dma_cntl_p, 72 p_rx_rbr_ring_t *rbr_p, p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 73 static void hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 74 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 75 static hxge_status_t hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, 76 uint16_t channel, p_hxge_dma_common_t *dma_buf_p, 77 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks); 78 static void hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 79 p_rx_rbr_ring_t rbr_p); 80 static hxge_status_t hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 81 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p); 82 static hxge_status_t hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel); 83 static mblk_t *hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 84 p_rx_rcr_ring_t *rcr_p, rdc_stat_t cs); 85 static void hxge_receive_packet(p_hxge_t hxgep, p_rx_rcr_ring_t rcr_p, 86 p_rcr_entry_t rcr_desc_rd_head_p, boolean_t *multi_p, 87 mblk_t ** mp, mblk_t ** mp_cont, uint32_t *invalid_rcr_entry); 88 static hxge_status_t hxge_disable_rxdma_channel(p_hxge_t hxgep, 89 uint16_t channel); 90 static p_rx_msg_t hxge_allocb(size_t, uint32_t, p_hxge_dma_common_t); 91 static void hxge_freeb(p_rx_msg_t); 92 static void hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, 93 p_hxge_ldv_t ldvp, rdc_stat_t cs); 94 static hxge_status_t hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, 95 p_hxge_ldv_t ldvp, rdc_stat_t cs); 96 static hxge_status_t hxge_rxbuf_index_info_init(p_hxge_t hxgep, 97 p_rx_rbr_ring_t rx_dmap); 98 static hxge_status_t hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, 99 uint16_t channel); 100 static hxge_status_t hxge_rx_port_fatal_err_recover(p_hxge_t hxgep); 101 102 hxge_status_t 103 hxge_init_rxdma_channels(p_hxge_t hxgep) 104 { 105 hxge_status_t status = HXGE_OK; 106 block_reset_t reset_reg; 107 108 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_init_rxdma_channels")); 109 110 /* Reset RDC block from PEU to clear any previous state */ 111 reset_reg.value = 0; 112 reset_reg.bits.rdc_rst = 1; 113 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 114 HXGE_DELAY(1000); 115 116 status = hxge_map_rxdma(hxgep); 117 if (status != HXGE_OK) { 118 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 119 "<== hxge_init_rxdma: status 0x%x", status)); 120 return (status); 121 } 122 123 status = hxge_rxdma_hw_start_common(hxgep); 124 if (status != HXGE_OK) { 125 hxge_unmap_rxdma(hxgep); 126 } 127 128 status = hxge_rxdma_hw_start(hxgep); 129 if (status != HXGE_OK) { 130 hxge_unmap_rxdma(hxgep); 131 } 132 133 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 134 "<== hxge_init_rxdma_channels: status 0x%x", status)); 135 return (status); 136 } 137 138 void 139 hxge_uninit_rxdma_channels(p_hxge_t hxgep) 140 { 141 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_uninit_rxdma_channels")); 142 143 hxge_rxdma_hw_stop(hxgep); 144 hxge_unmap_rxdma(hxgep); 145 146 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_uinit_rxdma_channels")); 147 } 148 149 hxge_status_t 150 hxge_init_rxdma_channel_cntl_stat(p_hxge_t hxgep, uint16_t channel, 151 rdc_stat_t *cs_p) 152 { 153 hpi_handle_t handle; 154 hpi_status_t rs = HPI_SUCCESS; 155 hxge_status_t status = HXGE_OK; 156 157 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 158 "<== hxge_init_rxdma_channel_cntl_stat")); 159 160 handle = HXGE_DEV_HPI_HANDLE(hxgep); 161 rs = hpi_rxdma_control_status(handle, OP_SET, channel, cs_p); 162 163 if (rs != HPI_SUCCESS) { 164 status = HXGE_ERROR | rs; 165 } 166 return (status); 167 } 168 169 170 hxge_status_t 171 hxge_enable_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 172 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p) 173 { 174 hpi_handle_t handle; 175 rdc_desc_cfg_t rdc_desc; 176 rdc_rcr_cfg_b_t *cfgb_p; 177 hpi_status_t rs = HPI_SUCCESS; 178 179 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel")); 180 handle = HXGE_DEV_HPI_HANDLE(hxgep); 181 182 /* 183 * Use configuration data composed at init time. Write to hardware the 184 * receive ring configurations. 185 */ 186 rdc_desc.mbox_enable = 1; 187 rdc_desc.mbox_addr = mbox_p->mbox_addr; 188 HXGE_DEBUG_MSG((hxgep, RX_CTL, 189 "==> hxge_enable_rxdma_channel: mboxp $%p($%p)", 190 mbox_p->mbox_addr, rdc_desc.mbox_addr)); 191 192 rdc_desc.rbr_len = rbr_p->rbb_max; 193 rdc_desc.rbr_addr = rbr_p->rbr_addr; 194 195 switch (hxgep->rx_bksize_code) { 196 case RBR_BKSIZE_4K: 197 rdc_desc.page_size = SIZE_4KB; 198 break; 199 case RBR_BKSIZE_8K: 200 rdc_desc.page_size = SIZE_8KB; 201 break; 202 } 203 204 rdc_desc.size0 = rbr_p->hpi_pkt_buf_size0; 205 rdc_desc.valid0 = 1; 206 207 rdc_desc.size1 = rbr_p->hpi_pkt_buf_size1; 208 rdc_desc.valid1 = 1; 209 210 rdc_desc.size2 = rbr_p->hpi_pkt_buf_size2; 211 rdc_desc.valid2 = 1; 212 213 rdc_desc.full_hdr = rcr_p->full_hdr_flag; 214 rdc_desc.offset = rcr_p->sw_priv_hdr_len; 215 216 rdc_desc.rcr_len = rcr_p->comp_size; 217 rdc_desc.rcr_addr = rcr_p->rcr_addr; 218 219 cfgb_p = &(rcr_p->rcr_cfgb); 220 rdc_desc.rcr_threshold = cfgb_p->bits.pthres; 221 rdc_desc.rcr_timeout = cfgb_p->bits.timeout; 222 rdc_desc.rcr_timeout_enable = cfgb_p->bits.entout; 223 224 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 225 "rbr_len qlen %d pagesize code %d rcr_len %d", 226 rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len)); 227 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 228 "size 0 %d size 1 %d size 2 %d", 229 rbr_p->hpi_pkt_buf_size0, rbr_p->hpi_pkt_buf_size1, 230 rbr_p->hpi_pkt_buf_size2)); 231 232 rs = hpi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc); 233 if (rs != HPI_SUCCESS) { 234 return (HXGE_ERROR | rs); 235 } 236 237 /* 238 * Enable the timeout and threshold. 239 */ 240 rs = hpi_rxdma_cfg_rdc_rcr_threshold(handle, channel, 241 rdc_desc.rcr_threshold); 242 if (rs != HPI_SUCCESS) { 243 return (HXGE_ERROR | rs); 244 } 245 246 rs = hpi_rxdma_cfg_rdc_rcr_timeout(handle, channel, 247 rdc_desc.rcr_timeout); 248 if (rs != HPI_SUCCESS) { 249 return (HXGE_ERROR | rs); 250 } 251 252 /* Enable the DMA */ 253 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 254 if (rs != HPI_SUCCESS) { 255 return (HXGE_ERROR | rs); 256 } 257 258 /* Kick the DMA engine. */ 259 hpi_rxdma_rdc_rbr_kick(handle, channel, rbr_p->rbb_max); 260 /* Clear the rbr empty bit */ 261 (void) hpi_rxdma_channel_rbr_empty_clear(handle, channel); 262 263 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_enable_rxdma_channel")); 264 265 return (HXGE_OK); 266 } 267 268 static hxge_status_t 269 hxge_disable_rxdma_channel(p_hxge_t hxgep, uint16_t channel) 270 { 271 hpi_handle_t handle; 272 hpi_status_t rs = HPI_SUCCESS; 273 274 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_disable_rxdma_channel")); 275 276 handle = HXGE_DEV_HPI_HANDLE(hxgep); 277 278 /* disable the DMA */ 279 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 280 if (rs != HPI_SUCCESS) { 281 HXGE_DEBUG_MSG((hxgep, RX_CTL, 282 "<== hxge_disable_rxdma_channel:failed (0x%x)", rs)); 283 return (HXGE_ERROR | rs); 284 } 285 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_disable_rxdma_channel")); 286 return (HXGE_OK); 287 } 288 289 hxge_status_t 290 hxge_rxdma_channel_rcrflush(p_hxge_t hxgep, uint8_t channel) 291 { 292 hpi_handle_t handle; 293 hxge_status_t status = HXGE_OK; 294 295 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 296 "==> hxge_rxdma_channel_rcrflush")); 297 298 handle = HXGE_DEV_HPI_HANDLE(hxgep); 299 hpi_rxdma_rdc_rcr_flush(handle, channel); 300 301 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 302 "<== hxge_rxdma_channel_rcrflush")); 303 return (status); 304 305 } 306 307 #define MID_INDEX(l, r) ((r + l + 1) >> 1) 308 309 #define TO_LEFT -1 310 #define TO_RIGHT 1 311 #define BOTH_RIGHT (TO_RIGHT + TO_RIGHT) 312 #define BOTH_LEFT (TO_LEFT + TO_LEFT) 313 #define IN_MIDDLE (TO_RIGHT + TO_LEFT) 314 #define NO_HINT 0xffffffff 315 316 /*ARGSUSED*/ 317 hxge_status_t 318 hxge_rxbuf_pp_to_vp(p_hxge_t hxgep, p_rx_rbr_ring_t rbr_p, 319 uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp, 320 uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index) 321 { 322 int bufsize; 323 uint64_t pktbuf_pp; 324 uint64_t dvma_addr; 325 rxring_info_t *ring_info; 326 int base_side, end_side; 327 int r_index, l_index, anchor_index; 328 int found, search_done; 329 uint32_t offset, chunk_size, block_size, page_size_mask; 330 uint32_t chunk_index, block_index, total_index; 331 int max_iterations, iteration; 332 rxbuf_index_info_t *bufinfo; 333 334 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_rxbuf_pp_to_vp")); 335 336 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 337 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d", 338 pkt_buf_addr_pp, pktbufsz_type)); 339 340 #if defined(__i386) 341 pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp; 342 #else 343 pktbuf_pp = (uint64_t)pkt_buf_addr_pp; 344 #endif 345 346 switch (pktbufsz_type) { 347 case 0: 348 bufsize = rbr_p->pkt_buf_size0; 349 break; 350 case 1: 351 bufsize = rbr_p->pkt_buf_size1; 352 break; 353 case 2: 354 bufsize = rbr_p->pkt_buf_size2; 355 break; 356 case RCR_SINGLE_BLOCK: 357 bufsize = 0; 358 anchor_index = 0; 359 break; 360 default: 361 return (HXGE_ERROR); 362 } 363 364 if (rbr_p->num_blocks == 1) { 365 anchor_index = 0; 366 ring_info = rbr_p->ring_info; 367 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 368 369 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 370 "==> hxge_rxbuf_pp_to_vp: (found, 1 block) " 371 "buf_pp $%p btype %d anchor_index %d bufinfo $%p", 372 pkt_buf_addr_pp, pktbufsz_type, anchor_index, bufinfo)); 373 374 goto found_index; 375 } 376 377 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 378 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d anchor_index %d", 379 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 380 381 ring_info = rbr_p->ring_info; 382 found = B_FALSE; 383 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 384 iteration = 0; 385 max_iterations = ring_info->max_iterations; 386 387 /* 388 * First check if this block have been seen recently. This is indicated 389 * by a hint which is initialized when the first buffer of the block is 390 * seen. The hint is reset when the last buffer of the block has been 391 * processed. As three block sizes are supported, three hints are kept. 392 * The idea behind the hints is that once the hardware uses a block 393 * for a buffer of that size, it will use it exclusively for that size 394 * and will use it until it is exhausted. It is assumed that there 395 * would a single block being used for the same buffer sizes at any 396 * given time. 397 */ 398 if (ring_info->hint[pktbufsz_type] != NO_HINT) { 399 anchor_index = ring_info->hint[pktbufsz_type]; 400 dvma_addr = bufinfo[anchor_index].dvma_addr; 401 chunk_size = bufinfo[anchor_index].buf_size; 402 if ((pktbuf_pp >= dvma_addr) && 403 (pktbuf_pp < (dvma_addr + chunk_size))) { 404 found = B_TRUE; 405 /* 406 * check if this is the last buffer in the block If so, 407 * then reset the hint for the size; 408 */ 409 410 if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size)) 411 ring_info->hint[pktbufsz_type] = NO_HINT; 412 } 413 } 414 415 if (found == B_FALSE) { 416 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 417 "==> hxge_rxbuf_pp_to_vp: (!found)" 418 "buf_pp $%p btype %d anchor_index %d", 419 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 420 421 /* 422 * This is the first buffer of the block of this size. Need to 423 * search the whole information array. the search algorithm 424 * uses a binary tree search algorithm. It assumes that the 425 * information is already sorted with increasing order info[0] 426 * < info[1] < info[2] .... < info[n-1] where n is the size of 427 * the information array 428 */ 429 r_index = rbr_p->num_blocks - 1; 430 l_index = 0; 431 search_done = B_FALSE; 432 anchor_index = MID_INDEX(r_index, l_index); 433 while (search_done == B_FALSE) { 434 if ((r_index == l_index) || 435 (iteration >= max_iterations)) 436 search_done = B_TRUE; 437 438 end_side = TO_RIGHT; /* to the right */ 439 base_side = TO_LEFT; /* to the left */ 440 /* read the DVMA address information and sort it */ 441 dvma_addr = bufinfo[anchor_index].dvma_addr; 442 chunk_size = bufinfo[anchor_index].buf_size; 443 444 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 445 "==> hxge_rxbuf_pp_to_vp: (searching)" 446 "buf_pp $%p btype %d " 447 "anchor_index %d chunk_size %d dvmaaddr $%p", 448 pkt_buf_addr_pp, pktbufsz_type, anchor_index, 449 chunk_size, dvma_addr)); 450 451 if (pktbuf_pp >= dvma_addr) 452 base_side = TO_RIGHT; /* to the right */ 453 if (pktbuf_pp < (dvma_addr + chunk_size)) 454 end_side = TO_LEFT; /* to the left */ 455 456 switch (base_side + end_side) { 457 case IN_MIDDLE: 458 /* found */ 459 found = B_TRUE; 460 search_done = B_TRUE; 461 if ((pktbuf_pp + bufsize) < 462 (dvma_addr + chunk_size)) 463 ring_info->hint[pktbufsz_type] = 464 bufinfo[anchor_index].buf_index; 465 break; 466 case BOTH_RIGHT: 467 /* not found: go to the right */ 468 l_index = anchor_index + 1; 469 anchor_index = MID_INDEX(r_index, l_index); 470 break; 471 472 case BOTH_LEFT: 473 /* not found: go to the left */ 474 r_index = anchor_index - 1; 475 anchor_index = MID_INDEX(r_index, l_index); 476 break; 477 default: /* should not come here */ 478 return (HXGE_ERROR); 479 } 480 iteration++; 481 } 482 483 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 484 "==> hxge_rxbuf_pp_to_vp: (search done)" 485 "buf_pp $%p btype %d anchor_index %d", 486 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 487 } 488 489 if (found == B_FALSE) { 490 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 491 "==> hxge_rxbuf_pp_to_vp: (search failed)" 492 "buf_pp $%p btype %d anchor_index %d", 493 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 494 return (HXGE_ERROR); 495 } 496 497 found_index: 498 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 499 "==> hxge_rxbuf_pp_to_vp: (FOUND1)" 500 "buf_pp $%p btype %d bufsize %d anchor_index %d", 501 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index)); 502 503 /* index of the first block in this chunk */ 504 chunk_index = bufinfo[anchor_index].start_index; 505 dvma_addr = bufinfo[anchor_index].dvma_addr; 506 page_size_mask = ring_info->block_size_mask; 507 508 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 509 "==> hxge_rxbuf_pp_to_vp: (FOUND3), get chunk)" 510 "buf_pp $%p btype %d bufsize %d " 511 "anchor_index %d chunk_index %d dvma $%p", 512 pkt_buf_addr_pp, pktbufsz_type, bufsize, 513 anchor_index, chunk_index, dvma_addr)); 514 515 offset = pktbuf_pp - dvma_addr; /* offset within the chunk */ 516 block_size = rbr_p->block_size; /* System block(page) size */ 517 518 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 519 "==> hxge_rxbuf_pp_to_vp: (FOUND4), get chunk)" 520 "buf_pp $%p btype %d bufsize %d " 521 "anchor_index %d chunk_index %d dvma $%p " 522 "offset %d block_size %d", 523 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index, 524 chunk_index, dvma_addr, offset, block_size)); 525 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> getting total index")); 526 527 block_index = (offset / block_size); /* index within chunk */ 528 total_index = chunk_index + block_index; 529 530 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 531 "==> hxge_rxbuf_pp_to_vp: " 532 "total_index %d dvma_addr $%p " 533 "offset %d block_size %d " 534 "block_index %d ", 535 total_index, dvma_addr, offset, block_size, block_index)); 536 537 #if defined(__i386) 538 *pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr + 539 (uint32_t)offset); 540 #else 541 *pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr + 542 offset); 543 #endif 544 545 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 546 "==> hxge_rxbuf_pp_to_vp: " 547 "total_index %d dvma_addr $%p " 548 "offset %d block_size %d " 549 "block_index %d " 550 "*pkt_buf_addr_p $%p", 551 total_index, dvma_addr, offset, block_size, 552 block_index, *pkt_buf_addr_p)); 553 554 *msg_index = total_index; 555 *bufoffset = (offset & page_size_mask); 556 557 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 558 "==> hxge_rxbuf_pp_to_vp: get msg index: " 559 "msg_index %d bufoffset_index %d", 560 *msg_index, *bufoffset)); 561 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "<== hxge_rxbuf_pp_to_vp")); 562 563 return (HXGE_OK); 564 } 565 566 567 /* 568 * used by quick sort (qsort) function 569 * to perform comparison 570 */ 571 static int 572 hxge_sort_compare(const void *p1, const void *p2) 573 { 574 575 rxbuf_index_info_t *a, *b; 576 577 a = (rxbuf_index_info_t *)p1; 578 b = (rxbuf_index_info_t *)p2; 579 580 if (a->dvma_addr > b->dvma_addr) 581 return (1); 582 if (a->dvma_addr < b->dvma_addr) 583 return (-1); 584 return (0); 585 } 586 587 /* 588 * Grabbed this sort implementation from common/syscall/avl.c 589 * 590 * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified. 591 * v = Ptr to array/vector of objs 592 * n = # objs in the array 593 * s = size of each obj (must be multiples of a word size) 594 * f = ptr to function to compare two objs 595 * returns (-1 = less than, 0 = equal, 1 = greater than 596 */ 597 void 598 hxge_ksort(caddr_t v, int n, int s, int (*f) ()) 599 { 600 int g, i, j, ii; 601 unsigned int *p1, *p2; 602 unsigned int tmp; 603 604 /* No work to do */ 605 if (v == NULL || n <= 1) 606 return; 607 /* Sanity check on arguments */ 608 ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0); 609 ASSERT(s > 0); 610 611 for (g = n / 2; g > 0; g /= 2) { 612 for (i = g; i < n; i++) { 613 for (j = i - g; j >= 0 && 614 (*f) (v + j * s, v + (j + g) * s) == 1; j -= g) { 615 p1 = (unsigned *)(v + j * s); 616 p2 = (unsigned *)(v + (j + g) * s); 617 for (ii = 0; ii < s / 4; ii++) { 618 tmp = *p1; 619 *p1++ = *p2; 620 *p2++ = tmp; 621 } 622 } 623 } 624 } 625 } 626 627 /* 628 * Initialize data structures required for rxdma 629 * buffer dvma->vmem address lookup 630 */ 631 /*ARGSUSED*/ 632 static hxge_status_t 633 hxge_rxbuf_index_info_init(p_hxge_t hxgep, p_rx_rbr_ring_t rbrp) 634 { 635 int index; 636 rxring_info_t *ring_info; 637 int max_iteration = 0, max_index = 0; 638 639 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_rxbuf_index_info_init")); 640 641 ring_info = rbrp->ring_info; 642 ring_info->hint[0] = NO_HINT; 643 ring_info->hint[1] = NO_HINT; 644 ring_info->hint[2] = NO_HINT; 645 max_index = rbrp->num_blocks; 646 647 /* read the DVMA address information and sort it */ 648 /* do init of the information array */ 649 650 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 651 " hxge_rxbuf_index_info_init Sort ptrs")); 652 653 /* sort the array */ 654 hxge_ksort((void *) ring_info->buffer, max_index, 655 sizeof (rxbuf_index_info_t), hxge_sort_compare); 656 657 for (index = 0; index < max_index; index++) { 658 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 659 " hxge_rxbuf_index_info_init: sorted chunk %d " 660 " ioaddr $%p kaddr $%p size %x", 661 index, ring_info->buffer[index].dvma_addr, 662 ring_info->buffer[index].kaddr, 663 ring_info->buffer[index].buf_size)); 664 } 665 666 max_iteration = 0; 667 while (max_index >= (1ULL << max_iteration)) 668 max_iteration++; 669 ring_info->max_iterations = max_iteration + 1; 670 671 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 672 " hxge_rxbuf_index_info_init Find max iter %d", 673 ring_info->max_iterations)); 674 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_rxbuf_index_info_init")); 675 676 return (HXGE_OK); 677 } 678 679 /*ARGSUSED*/ 680 void 681 hxge_dump_rcr_entry(p_hxge_t hxgep, p_rcr_entry_t entry_p) 682 { 683 #ifdef HXGE_DEBUG 684 685 uint32_t bptr; 686 uint64_t pp; 687 688 bptr = entry_p->bits.pkt_buf_addr; 689 690 HXGE_DEBUG_MSG((hxgep, RX_CTL, 691 "\trcr entry $%p " 692 "\trcr entry 0x%0llx " 693 "\trcr entry 0x%08x " 694 "\trcr entry 0x%08x " 695 "\tvalue 0x%0llx\n" 696 "\tmulti = %d\n" 697 "\tpkt_type = 0x%x\n" 698 "\terror = 0x%04x\n" 699 "\tl2_len = %d\n" 700 "\tpktbufsize = %d\n" 701 "\tpkt_buf_addr = $%p\n" 702 "\tpkt_buf_addr (<< 6) = $%p\n", 703 entry_p, 704 *(int64_t *)entry_p, 705 *(int32_t *)entry_p, 706 *(int32_t *)((char *)entry_p + 32), 707 entry_p->value, 708 entry_p->bits.multi, 709 entry_p->bits.pkt_type, 710 entry_p->bits.error, 711 entry_p->bits.l2_len, 712 entry_p->bits.pktbufsz, 713 bptr, 714 entry_p->bits.pkt_buf_addr_l)); 715 716 pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) << 717 RCR_PKT_BUF_ADDR_SHIFT; 718 719 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rcr pp 0x%llx l2 len %d", 720 pp, (*(int64_t *)entry_p >> 40) & 0x3fff)); 721 #endif 722 } 723 724 /*ARGSUSED*/ 725 void 726 hxge_rxdma_stop(p_hxge_t hxgep) 727 { 728 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop")); 729 730 (void) hxge_rx_vmac_disable(hxgep); 731 (void) hxge_rxdma_hw_mode(hxgep, HXGE_DMA_STOP); 732 733 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop")); 734 } 735 736 void 737 hxge_rxdma_stop_reinit(p_hxge_t hxgep) 738 { 739 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_reinit")); 740 741 (void) hxge_rxdma_stop(hxgep); 742 (void) hxge_uninit_rxdma_channels(hxgep); 743 (void) hxge_init_rxdma_channels(hxgep); 744 745 (void) hxge_rx_vmac_enable(hxgep); 746 747 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_reinit")); 748 } 749 750 hxge_status_t 751 hxge_rxdma_hw_mode(p_hxge_t hxgep, boolean_t enable) 752 { 753 int i, ndmas; 754 uint16_t channel; 755 p_rx_rbr_rings_t rx_rbr_rings; 756 p_rx_rbr_ring_t *rbr_rings; 757 hpi_handle_t handle; 758 hpi_status_t rs = HPI_SUCCESS; 759 hxge_status_t status = HXGE_OK; 760 761 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 762 "==> hxge_rxdma_hw_mode: mode %d", enable)); 763 764 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 765 HXGE_DEBUG_MSG((hxgep, RX_CTL, 766 "<== hxge_rxdma_mode: not initialized")); 767 return (HXGE_ERROR); 768 } 769 770 rx_rbr_rings = hxgep->rx_rbr_rings; 771 if (rx_rbr_rings == NULL) { 772 HXGE_DEBUG_MSG((hxgep, RX_CTL, 773 "<== hxge_rxdma_mode: NULL ring pointer")); 774 return (HXGE_ERROR); 775 } 776 777 if (rx_rbr_rings->rbr_rings == NULL) { 778 HXGE_DEBUG_MSG((hxgep, RX_CTL, 779 "<== hxge_rxdma_mode: NULL rbr rings pointer")); 780 return (HXGE_ERROR); 781 } 782 783 ndmas = rx_rbr_rings->ndmas; 784 if (!ndmas) { 785 HXGE_DEBUG_MSG((hxgep, RX_CTL, 786 "<== hxge_rxdma_mode: no channel")); 787 return (HXGE_ERROR); 788 } 789 790 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 791 "==> hxge_rxdma_mode (ndmas %d)", ndmas)); 792 793 rbr_rings = rx_rbr_rings->rbr_rings; 794 795 handle = HXGE_DEV_HPI_HANDLE(hxgep); 796 797 for (i = 0; i < ndmas; i++) { 798 if (rbr_rings == NULL || rbr_rings[i] == NULL) { 799 continue; 800 } 801 channel = rbr_rings[i]->rdc; 802 if (enable) { 803 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 804 "==> hxge_rxdma_hw_mode: channel %d (enable)", 805 channel)); 806 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 807 } else { 808 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 809 "==> hxge_rxdma_hw_mode: channel %d (disable)", 810 channel)); 811 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 812 } 813 } 814 815 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs); 816 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 817 "<== hxge_rxdma_hw_mode: status 0x%x", status)); 818 819 return (status); 820 } 821 822 int 823 hxge_rxdma_get_ring_index(p_hxge_t hxgep, uint16_t channel) 824 { 825 int i, ndmas; 826 uint16_t rdc; 827 p_rx_rbr_rings_t rx_rbr_rings; 828 p_rx_rbr_ring_t *rbr_rings; 829 830 HXGE_DEBUG_MSG((hxgep, RX_CTL, 831 "==> hxge_rxdma_get_ring_index: channel %d", channel)); 832 833 rx_rbr_rings = hxgep->rx_rbr_rings; 834 if (rx_rbr_rings == NULL) { 835 HXGE_DEBUG_MSG((hxgep, RX_CTL, 836 "<== hxge_rxdma_get_ring_index: NULL ring pointer")); 837 return (-1); 838 } 839 840 ndmas = rx_rbr_rings->ndmas; 841 if (!ndmas) { 842 HXGE_DEBUG_MSG((hxgep, RX_CTL, 843 "<== hxge_rxdma_get_ring_index: no channel")); 844 return (-1); 845 } 846 847 HXGE_DEBUG_MSG((hxgep, RX_CTL, 848 "==> hxge_rxdma_get_ring_index (ndmas %d)", ndmas)); 849 850 rbr_rings = rx_rbr_rings->rbr_rings; 851 for (i = 0; i < ndmas; i++) { 852 rdc = rbr_rings[i]->rdc; 853 if (channel == rdc) { 854 HXGE_DEBUG_MSG((hxgep, RX_CTL, 855 "==> hxge_rxdma_get_rbr_ring: " 856 "channel %d (index %d) " 857 "ring %d", channel, i, rbr_rings[i])); 858 859 return (i); 860 } 861 } 862 863 HXGE_DEBUG_MSG((hxgep, RX_CTL, 864 "<== hxge_rxdma_get_rbr_ring_index: not found")); 865 866 return (-1); 867 } 868 869 /* 870 * Static functions start here. 871 */ 872 static p_rx_msg_t 873 hxge_allocb(size_t size, uint32_t pri, p_hxge_dma_common_t dmabuf_p) 874 { 875 p_rx_msg_t hxge_mp = NULL; 876 p_hxge_dma_common_t dmamsg_p; 877 uchar_t *buffer; 878 879 hxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP); 880 if (hxge_mp == NULL) { 881 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 882 "Allocation of a rx msg failed.")); 883 goto hxge_allocb_exit; 884 } 885 886 hxge_mp->use_buf_pool = B_FALSE; 887 if (dmabuf_p) { 888 hxge_mp->use_buf_pool = B_TRUE; 889 890 dmamsg_p = (p_hxge_dma_common_t)&hxge_mp->buf_dma; 891 *dmamsg_p = *dmabuf_p; 892 dmamsg_p->nblocks = 1; 893 dmamsg_p->block_size = size; 894 dmamsg_p->alength = size; 895 buffer = (uchar_t *)dmabuf_p->kaddrp; 896 897 dmabuf_p->kaddrp = (void *)((char *)dmabuf_p->kaddrp + size); 898 dmabuf_p->ioaddr_pp = (void *) 899 ((char *)dmabuf_p->ioaddr_pp + size); 900 901 dmabuf_p->alength -= size; 902 dmabuf_p->offset += size; 903 dmabuf_p->dma_cookie.dmac_laddress += size; 904 dmabuf_p->dma_cookie.dmac_size -= size; 905 } else { 906 buffer = KMEM_ALLOC(size, KM_NOSLEEP); 907 if (buffer == NULL) { 908 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 909 "Allocation of a receive page failed.")); 910 goto hxge_allocb_fail1; 911 } 912 } 913 914 hxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &hxge_mp->freeb); 915 if (hxge_mp->rx_mblk_p == NULL) { 916 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, "desballoc failed.")); 917 goto hxge_allocb_fail2; 918 } 919 hxge_mp->buffer = buffer; 920 hxge_mp->block_size = size; 921 hxge_mp->freeb.free_func = (void (*) ()) hxge_freeb; 922 hxge_mp->freeb.free_arg = (caddr_t)hxge_mp; 923 hxge_mp->ref_cnt = 1; 924 hxge_mp->free = B_TRUE; 925 hxge_mp->rx_use_bcopy = B_FALSE; 926 927 atomic_add_32(&hxge_mblks_pending, 1); 928 929 goto hxge_allocb_exit; 930 931 hxge_allocb_fail2: 932 if (!hxge_mp->use_buf_pool) { 933 KMEM_FREE(buffer, size); 934 } 935 hxge_allocb_fail1: 936 KMEM_FREE(hxge_mp, sizeof (rx_msg_t)); 937 hxge_mp = NULL; 938 939 hxge_allocb_exit: 940 return (hxge_mp); 941 } 942 943 p_mblk_t 944 hxge_dupb(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 945 { 946 p_mblk_t mp; 947 948 HXGE_DEBUG_MSG((NULL, MEM_CTL, "==> hxge_dupb")); 949 HXGE_DEBUG_MSG((NULL, MEM_CTL, "hxge_mp = $%p " 950 "offset = 0x%08X " "size = 0x%08X", hxge_mp, offset, size)); 951 952 mp = desballoc(&hxge_mp->buffer[offset], size, 0, &hxge_mp->freeb); 953 if (mp == NULL) { 954 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 955 goto hxge_dupb_exit; 956 } 957 958 atomic_inc_32(&hxge_mp->ref_cnt); 959 atomic_inc_32(&hxge_mblks_pending); 960 961 hxge_dupb_exit: 962 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 963 return (mp); 964 } 965 966 p_mblk_t 967 hxge_dupb_bcopy(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 968 { 969 p_mblk_t mp; 970 uchar_t *dp; 971 972 mp = allocb(size + HXGE_RXBUF_EXTRA, 0); 973 if (mp == NULL) { 974 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 975 goto hxge_dupb_bcopy_exit; 976 } 977 dp = mp->b_rptr = mp->b_rptr + HXGE_RXBUF_EXTRA; 978 bcopy((void *) &hxge_mp->buffer[offset], dp, size); 979 mp->b_wptr = dp + size; 980 981 hxge_dupb_bcopy_exit: 982 983 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 984 985 return (mp); 986 } 987 988 void hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, 989 p_rx_msg_t rx_msg_p); 990 991 void 992 hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p) 993 { 994 hpi_handle_t handle; 995 996 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_post_page")); 997 998 /* Reuse this buffer */ 999 rx_msg_p->free = B_FALSE; 1000 rx_msg_p->cur_usage_cnt = 0; 1001 rx_msg_p->max_usage_cnt = 0; 1002 rx_msg_p->pkt_buf_size = 0; 1003 1004 if (rx_rbr_p->rbr_use_bcopy) { 1005 rx_msg_p->rx_use_bcopy = B_FALSE; 1006 atomic_dec_32(&rx_rbr_p->rbr_consumed); 1007 } 1008 1009 /* 1010 * Get the rbr header pointer and its offset index. 1011 */ 1012 MUTEX_ENTER(&rx_rbr_p->post_lock); 1013 1014 rx_rbr_p->rbr_wr_index = ((rx_rbr_p->rbr_wr_index + 1) & 1015 rx_rbr_p->rbr_wrap_mask); 1016 rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr; 1017 1018 /* 1019 * Don't post when index is close to 0 or near the max to reduce the 1020 * number rbr_emepty errors 1021 */ 1022 rx_rbr_p->pages_to_post++; 1023 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1024 if (rx_rbr_p->rbr_wr_index > (rx_rbr_p->pages_to_skip / 2) && 1025 rx_rbr_p->rbr_wr_index < rx_rbr_p->pages_to_post_threshold) { 1026 hpi_rxdma_rdc_rbr_kick(handle, rx_rbr_p->rdc, 1027 rx_rbr_p->pages_to_post); 1028 rx_rbr_p->pages_to_post = 0; 1029 } 1030 1031 MUTEX_EXIT(&rx_rbr_p->post_lock); 1032 1033 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1034 "<== hxge_post_page (channel %d post_next_index %d)", 1035 rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index)); 1036 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_post_page")); 1037 } 1038 1039 void 1040 hxge_freeb(p_rx_msg_t rx_msg_p) 1041 { 1042 size_t size; 1043 uchar_t *buffer = NULL; 1044 int ref_cnt; 1045 boolean_t free_state = B_FALSE; 1046 rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p; 1047 1048 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> hxge_freeb")); 1049 HXGE_DEBUG_MSG((NULL, MEM2_CTL, 1050 "hxge_freeb:rx_msg_p = $%p (block pending %d)", 1051 rx_msg_p, hxge_mblks_pending)); 1052 1053 atomic_dec_32(&hxge_mblks_pending); 1054 1055 /* 1056 * First we need to get the free state, then 1057 * atomic decrement the reference count to prevent 1058 * the race condition with the interrupt thread that 1059 * is processing a loaned up buffer block. 1060 */ 1061 free_state = rx_msg_p->free; 1062 1063 ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1); 1064 if (!ref_cnt) { 1065 buffer = rx_msg_p->buffer; 1066 size = rx_msg_p->block_size; 1067 1068 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "hxge_freeb: " 1069 "will free: rx_msg_p = $%p (block pending %d)", 1070 rx_msg_p, hxge_mblks_pending)); 1071 1072 if (!rx_msg_p->use_buf_pool) { 1073 KMEM_FREE(buffer, size); 1074 } 1075 1076 KMEM_FREE(rx_msg_p, sizeof (rx_msg_t)); 1077 /* Decrement the receive buffer ring's reference count, too. */ 1078 atomic_dec_32(&ring->rbr_ref_cnt); 1079 1080 /* 1081 * Free the receive buffer ring, iff 1082 * 1. all the receive buffers have been freed 1083 * 2. and we are in the proper state (that is, 1084 * we are not UNMAPPING). 1085 */ 1086 if (ring->rbr_ref_cnt == 0 && ring->rbr_state == RBR_UNMAPPED) { 1087 KMEM_FREE(ring, sizeof (*ring)); 1088 } 1089 return; 1090 } 1091 1092 /* 1093 * Repost buffer. 1094 */ 1095 if (free_state && (ref_cnt == 1)) { 1096 HXGE_DEBUG_MSG((NULL, RX_CTL, 1097 "hxge_freeb: post page $%p:", rx_msg_p)); 1098 if (ring->rbr_state == RBR_POSTING) 1099 hxge_post_page(rx_msg_p->hxgep, ring, rx_msg_p); 1100 } 1101 1102 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== hxge_freeb")); 1103 } 1104 1105 uint_t 1106 hxge_rx_intr(caddr_t arg1, caddr_t arg2) 1107 { 1108 p_hxge_ldv_t ldvp = (p_hxge_ldv_t)arg1; 1109 p_hxge_t hxgep = (p_hxge_t)arg2; 1110 p_hxge_ldg_t ldgp; 1111 uint8_t channel; 1112 hpi_handle_t handle; 1113 rdc_stat_t cs; 1114 uint_t serviced = DDI_INTR_UNCLAIMED; 1115 1116 if (ldvp == NULL) { 1117 HXGE_DEBUG_MSG((NULL, RX_INT_CTL, 1118 "<== hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1119 return (DDI_INTR_CLAIMED); 1120 } 1121 1122 if (arg2 == NULL || (void *) ldvp->hxgep != arg2) { 1123 hxgep = ldvp->hxgep; 1124 } 1125 1126 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1127 "==> hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1128 1129 /* 1130 * This interrupt handler is for a specific receive dma channel. 1131 */ 1132 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1133 1134 /* 1135 * Get the control and status for this channel. 1136 */ 1137 channel = ldvp->channel; 1138 ldgp = ldvp->ldgp; 1139 RXDMA_REG_READ64(handle, RDC_STAT, channel, &cs.value); 1140 1141 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_intr:channel %d " 1142 "cs 0x%016llx rcrto 0x%x rcrthres %x", 1143 channel, cs.value, cs.bits.rcr_to, cs.bits.rcr_thres)); 1144 1145 hxge_rx_pkts_vring(hxgep, ldvp->vdma_index, ldvp, cs); 1146 serviced = DDI_INTR_CLAIMED; 1147 1148 /* error events. */ 1149 if (cs.value & RDC_STAT_ERROR) { 1150 (void) hxge_rx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs); 1151 } 1152 1153 hxge_intr_exit: 1154 /* 1155 * Enable the mailbox update interrupt if we want to use mailbox. We 1156 * probably don't need to use mailbox as it only saves us one pio read. 1157 * Also write 1 to rcrthres and rcrto to clear these two edge triggered 1158 * bits. 1159 */ 1160 cs.value &= RDC_STAT_WR1C; 1161 cs.bits.mex = 1; 1162 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1163 1164 /* 1165 * Rearm this logical group if this is a single device group. 1166 */ 1167 if (ldgp->nldvs == 1) { 1168 ld_intr_mgmt_t mgm; 1169 1170 mgm.value = 0; 1171 mgm.bits.arm = 1; 1172 mgm.bits.timer = ldgp->ldg_timer; 1173 HXGE_REG_WR32(handle, 1174 LD_INTR_MGMT + LDSV_OFFSET(ldgp->ldg), mgm.value); 1175 } 1176 1177 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1178 "<== hxge_rx_intr: serviced %d", serviced)); 1179 1180 return (serviced); 1181 } 1182 1183 static void 1184 hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1185 rdc_stat_t cs) 1186 { 1187 p_mblk_t mp; 1188 p_rx_rcr_ring_t rcrp; 1189 1190 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts_vring")); 1191 if ((mp = hxge_rx_pkts(hxgep, vindex, ldvp, &rcrp, cs)) == NULL) { 1192 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1193 "<== hxge_rx_pkts_vring: no mp")); 1194 return; 1195 } 1196 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts_vring: $%p", mp)); 1197 1198 #ifdef HXGE_DEBUG 1199 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1200 "==> hxge_rx_pkts_vring:calling mac_rx (NEMO) " 1201 "LEN %d mp $%p mp->b_next $%p rcrp $%p " 1202 "mac_handle $%p", 1203 (mp->b_wptr - mp->b_rptr), mp, mp->b_next, 1204 rcrp, rcrp->rcr_mac_handle)); 1205 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1206 "==> hxge_rx_pkts_vring: dump packets " 1207 "(mp $%p b_rptr $%p b_wptr $%p):\n %s", 1208 mp, mp->b_rptr, mp->b_wptr, 1209 hxge_dump_packet((char *)mp->b_rptr, 64))); 1210 1211 if (mp->b_cont) { 1212 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1213 "==> hxge_rx_pkts_vring: dump b_cont packets " 1214 "(mp->b_cont $%p b_rptr $%p b_wptr $%p):\n %s", 1215 mp->b_cont, mp->b_cont->b_rptr, mp->b_cont->b_wptr, 1216 hxge_dump_packet((char *)mp->b_cont->b_rptr, 1217 mp->b_cont->b_wptr - mp->b_cont->b_rptr))); 1218 } 1219 if (mp->b_next) { 1220 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1221 "==> hxge_rx_pkts_vring: dump next packets " 1222 "(b_rptr $%p): %s", 1223 mp->b_next->b_rptr, 1224 hxge_dump_packet((char *)mp->b_next->b_rptr, 64))); 1225 } 1226 #endif 1227 1228 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1229 "==> hxge_rx_pkts_vring: send packet to stack")); 1230 mac_rx(hxgep->mach, rcrp->rcr_mac_handle, mp); 1231 1232 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_pkts_vring")); 1233 } 1234 1235 /*ARGSUSED*/ 1236 mblk_t * 1237 hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1238 p_rx_rcr_ring_t *rcrp, rdc_stat_t cs) 1239 { 1240 hpi_handle_t handle; 1241 uint8_t channel; 1242 p_rx_rcr_rings_t rx_rcr_rings; 1243 p_rx_rcr_ring_t rcr_p; 1244 uint32_t comp_rd_index; 1245 p_rcr_entry_t rcr_desc_rd_head_p; 1246 p_rcr_entry_t rcr_desc_rd_head_pp; 1247 p_mblk_t nmp, mp_cont, head_mp, *tail_mp; 1248 uint16_t qlen, nrcr_read, npkt_read; 1249 uint32_t qlen_hw; 1250 uint32_t invalid_rcr_entry; 1251 boolean_t multi; 1252 rdc_rcr_cfg_b_t rcr_cfg_b; 1253 p_rx_mbox_t rx_mboxp; 1254 p_rxdma_mailbox_t mboxp; 1255 #if defined(_BIG_ENDIAN) 1256 hpi_status_t rs = HPI_SUCCESS; 1257 #endif 1258 1259 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:vindex %d " 1260 "channel %d", vindex, ldvp->channel)); 1261 1262 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 1263 return (NULL); 1264 } 1265 1266 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1267 rx_rcr_rings = hxgep->rx_rcr_rings; 1268 rcr_p = rx_rcr_rings->rcr_rings[vindex]; 1269 channel = rcr_p->rdc; 1270 if (channel != ldvp->channel) { 1271 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:index %d " 1272 "channel %d, and rcr channel %d not matched.", 1273 vindex, ldvp->channel, channel)); 1274 return (NULL); 1275 } 1276 1277 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1278 "==> hxge_rx_pkts: START: rcr channel %d " 1279 "head_p $%p head_pp $%p index %d ", 1280 channel, rcr_p->rcr_desc_rd_head_p, 1281 rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1282 1283 rx_mboxp = hxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 1284 mboxp = (p_rxdma_mailbox_t)rx_mboxp->rx_mbox.kaddrp; 1285 qlen = mboxp->rcrstat_a.bits.qlen; 1286 1287 if (!qlen) { 1288 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1289 "<== hxge_rx_pkts:rcr channel %d qlen %d (no pkts)", 1290 channel, qlen)); 1291 return (NULL); 1292 } 1293 1294 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts:rcr channel %d " 1295 "qlen %d", channel, qlen)); 1296 1297 comp_rd_index = rcr_p->comp_rd_index; 1298 1299 rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p; 1300 rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp; 1301 nrcr_read = npkt_read = 0; 1302 1303 /* 1304 * Number of packets queued (The jumbo or multi packet will be counted 1305 * as only one paccket and it may take up more than one completion 1306 * entry). 1307 */ 1308 qlen_hw = (qlen < hxge_max_rx_pkts) ? qlen : hxge_max_rx_pkts; 1309 head_mp = NULL; 1310 tail_mp = &head_mp; 1311 nmp = mp_cont = NULL; 1312 multi = B_FALSE; 1313 1314 while (qlen_hw) { 1315 #ifdef HXGE_DEBUG 1316 hxge_dump_rcr_entry(hxgep, rcr_desc_rd_head_p); 1317 #endif 1318 /* 1319 * Process one completion ring entry. 1320 */ 1321 invalid_rcr_entry = 0; 1322 hxge_receive_packet(hxgep, 1323 rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont, 1324 &invalid_rcr_entry); 1325 if (invalid_rcr_entry != 0) { 1326 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1327 "Channel %d could only read 0x%x packets, " 1328 "but 0x%x pending\n", channel, npkt_read, qlen_hw)); 1329 break; 1330 } 1331 1332 /* 1333 * message chaining modes (nemo msg chaining) 1334 */ 1335 if (nmp) { 1336 nmp->b_next = NULL; 1337 if (!multi && !mp_cont) { /* frame fits a partition */ 1338 *tail_mp = nmp; 1339 tail_mp = &nmp->b_next; 1340 nmp = NULL; 1341 } else if (multi && !mp_cont) { /* first segment */ 1342 *tail_mp = nmp; 1343 tail_mp = &nmp->b_cont; 1344 } else if (multi && mp_cont) { /* mid of multi segs */ 1345 *tail_mp = mp_cont; 1346 tail_mp = &mp_cont->b_cont; 1347 } else if (!multi && mp_cont) { /* last segment */ 1348 *tail_mp = mp_cont; 1349 tail_mp = &nmp->b_next; 1350 nmp = NULL; 1351 } 1352 } 1353 1354 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1355 "==> hxge_rx_pkts: loop: rcr channel %d " 1356 "before updating: multi %d " 1357 "nrcr_read %d " 1358 "npk read %d " 1359 "head_pp $%p index %d ", 1360 channel, multi, 1361 nrcr_read, npkt_read, rcr_desc_rd_head_pp, comp_rd_index)); 1362 1363 if (!multi) { 1364 qlen_hw--; 1365 npkt_read++; 1366 } 1367 1368 /* 1369 * Update the next read entry. 1370 */ 1371 comp_rd_index = NEXT_ENTRY(comp_rd_index, 1372 rcr_p->comp_wrap_mask); 1373 1374 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p, 1375 rcr_p->rcr_desc_first_p, rcr_p->rcr_desc_last_p); 1376 1377 nrcr_read++; 1378 1379 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1380 "<== hxge_rx_pkts: (SAM, process one packet) " 1381 "nrcr_read %d", nrcr_read)); 1382 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1383 "==> hxge_rx_pkts: loop: rcr channel %d " 1384 "multi %d nrcr_read %d npk read %d head_pp $%p index %d ", 1385 channel, multi, nrcr_read, npkt_read, rcr_desc_rd_head_pp, 1386 comp_rd_index)); 1387 } 1388 1389 rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp; 1390 rcr_p->comp_rd_index = comp_rd_index; 1391 rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p; 1392 1393 /* Adjust the mailbox queue length for a hardware bug workaround */ 1394 mboxp->rcrstat_a.bits.qlen -= npkt_read; 1395 1396 if ((hxgep->intr_timeout != rcr_p->intr_timeout) || 1397 (hxgep->intr_threshold != rcr_p->intr_threshold)) { 1398 rcr_p->intr_timeout = hxgep->intr_timeout; 1399 rcr_p->intr_threshold = hxgep->intr_threshold; 1400 rcr_cfg_b.value = 0x0ULL; 1401 if (rcr_p->intr_timeout) 1402 rcr_cfg_b.bits.entout = 1; 1403 rcr_cfg_b.bits.timeout = rcr_p->intr_timeout; 1404 rcr_cfg_b.bits.pthres = rcr_p->intr_threshold; 1405 RXDMA_REG_WRITE64(handle, RDC_RCR_CFG_B, 1406 channel, rcr_cfg_b.value); 1407 } 1408 1409 cs.bits.pktread = npkt_read; 1410 cs.bits.ptrread = nrcr_read; 1411 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1412 1413 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1414 "==> hxge_rx_pkts: EXIT: rcr channel %d " 1415 "head_pp $%p index %016llx ", 1416 channel, rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1417 1418 /* 1419 * Update RCR buffer pointer read and number of packets read. 1420 */ 1421 1422 *rcrp = rcr_p; 1423 1424 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "<== hxge_rx_pkts")); 1425 1426 return (head_mp); 1427 } 1428 1429 #define RCR_ENTRY_PATTERN 0x5a5a6b6b7c7c8d8dULL 1430 1431 /*ARGSUSED*/ 1432 void 1433 hxge_receive_packet(p_hxge_t hxgep, 1434 p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p, 1435 boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont, 1436 uint32_t *invalid_rcr_entry) 1437 { 1438 p_mblk_t nmp = NULL; 1439 uint64_t multi; 1440 uint8_t channel; 1441 1442 boolean_t first_entry = B_TRUE; 1443 boolean_t is_tcp_udp = B_FALSE; 1444 boolean_t buffer_free = B_FALSE; 1445 boolean_t error_send_up = B_FALSE; 1446 uint8_t error_type; 1447 uint16_t l2_len; 1448 uint16_t skip_len; 1449 uint8_t pktbufsz_type; 1450 uint64_t rcr_entry; 1451 uint64_t *pkt_buf_addr_pp; 1452 uint64_t *pkt_buf_addr_p; 1453 uint32_t buf_offset; 1454 uint32_t bsize; 1455 uint32_t msg_index; 1456 p_rx_rbr_ring_t rx_rbr_p; 1457 p_rx_msg_t *rx_msg_ring_p; 1458 p_rx_msg_t rx_msg_p; 1459 1460 uint16_t sw_offset_bytes = 0, hdr_size = 0; 1461 hxge_status_t status = HXGE_OK; 1462 boolean_t is_valid = B_FALSE; 1463 p_hxge_rx_ring_stats_t rdc_stats; 1464 uint32_t bytes_read; 1465 1466 uint64_t pkt_type; 1467 1468 channel = rcr_p->rdc; 1469 1470 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_receive_packet")); 1471 1472 first_entry = (*mp == NULL) ? B_TRUE : B_FALSE; 1473 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p); 1474 1475 /* Verify the content of the rcr_entry for a hardware bug workaround */ 1476 if ((rcr_entry == 0x0) || (rcr_entry == RCR_ENTRY_PATTERN)) { 1477 *invalid_rcr_entry = 1; 1478 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "hxge_receive_packet " 1479 "Channel %d invalid RCR entry 0x%llx found, returning\n", 1480 channel, (long long) rcr_entry)); 1481 return; 1482 } 1483 *((uint64_t *)rcr_desc_rd_head_p) = RCR_ENTRY_PATTERN; 1484 1485 multi = (rcr_entry & RCR_MULTI_MASK); 1486 pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK); 1487 1488 error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT); 1489 l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT); 1490 1491 /* 1492 * Hardware does not strip the CRC due bug ID 11451 where 1493 * the hardware mis handles minimum size packets. 1494 */ 1495 l2_len -= ETHERFCSL; 1496 1497 pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >> 1498 RCR_PKTBUFSZ_SHIFT); 1499 #if defined(__i386) 1500 pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry & 1501 RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT); 1502 #else 1503 pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) << 1504 RCR_PKT_BUF_ADDR_SHIFT); 1505 #endif 1506 1507 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1508 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1509 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1510 "error_type 0x%x pkt_type 0x%x " 1511 "pktbufsz_type %d ", 1512 rcr_desc_rd_head_p, rcr_entry, pkt_buf_addr_pp, l2_len, 1513 multi, error_type, pkt_type, pktbufsz_type)); 1514 1515 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1516 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1517 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1518 "error_type 0x%x pkt_type 0x%x ", rcr_desc_rd_head_p, 1519 rcr_entry, pkt_buf_addr_pp, l2_len, multi, error_type, pkt_type)); 1520 1521 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1522 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1523 "full pkt_buf_addr_pp $%p l2_len %d", 1524 rcr_entry, pkt_buf_addr_pp, l2_len)); 1525 1526 /* get the stats ptr */ 1527 rdc_stats = rcr_p->rdc_stats; 1528 if (l2_len > (STD_FRAME_SIZE - ETHERFCSL)) 1529 rdc_stats->jumbo_pkts++; 1530 1531 if (!l2_len) { 1532 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1533 "<== hxge_receive_packet: failed: l2 length is 0.")); 1534 return; 1535 } 1536 1537 /* shift 6 bits to get the full io address */ 1538 #if defined(__i386) 1539 pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp << 1540 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1541 #else 1542 pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp << 1543 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1544 #endif 1545 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1546 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1547 "full pkt_buf_addr_pp $%p l2_len %d", 1548 rcr_entry, pkt_buf_addr_pp, l2_len)); 1549 1550 rx_rbr_p = rcr_p->rx_rbr_p; 1551 rx_msg_ring_p = rx_rbr_p->rx_msg_ring; 1552 1553 if (first_entry) { 1554 hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL : 1555 RXDMA_HDR_SIZE_DEFAULT); 1556 1557 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1558 "==> hxge_receive_packet: first entry 0x%016llx " 1559 "pkt_buf_addr_pp $%p l2_len %d hdr %d", 1560 rcr_entry, pkt_buf_addr_pp, l2_len, hdr_size)); 1561 } 1562 1563 MUTEX_ENTER(&rcr_p->lock); 1564 MUTEX_ENTER(&rx_rbr_p->lock); 1565 1566 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1567 "==> (rbr 1) hxge_receive_packet: entry 0x%0llx " 1568 "full pkt_buf_addr_pp $%p l2_len %d", 1569 rcr_entry, pkt_buf_addr_pp, l2_len)); 1570 1571 /* 1572 * Packet buffer address in the completion entry points to the starting 1573 * buffer address (offset 0). Use the starting buffer address to locate 1574 * the corresponding kernel address. 1575 */ 1576 status = hxge_rxbuf_pp_to_vp(hxgep, rx_rbr_p, 1577 pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p, 1578 &buf_offset, &msg_index); 1579 1580 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1581 "==> (rbr 2) hxge_receive_packet: entry 0x%0llx " 1582 "full pkt_buf_addr_pp $%p l2_len %d", 1583 rcr_entry, pkt_buf_addr_pp, l2_len)); 1584 1585 if (status != HXGE_OK) { 1586 MUTEX_EXIT(&rx_rbr_p->lock); 1587 MUTEX_EXIT(&rcr_p->lock); 1588 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1589 "<== hxge_receive_packet: found vaddr failed %d", status)); 1590 return; 1591 } 1592 1593 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1594 "==> (rbr 3) hxge_receive_packet: entry 0x%0llx " 1595 "full pkt_buf_addr_pp $%p l2_len %d", 1596 rcr_entry, pkt_buf_addr_pp, l2_len)); 1597 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1598 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1599 "full pkt_buf_addr_pp $%p l2_len %d", 1600 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1601 1602 if (msg_index >= rx_rbr_p->tnblocks) { 1603 MUTEX_EXIT(&rx_rbr_p->lock); 1604 MUTEX_EXIT(&rcr_p->lock); 1605 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1606 "==> hxge_receive_packet: FATAL msg_index (%d) " 1607 "should be smaller than tnblocks (%d)\n", 1608 msg_index, rx_rbr_p->tnblocks)); 1609 return; 1610 } 1611 1612 rx_msg_p = rx_msg_ring_p[msg_index]; 1613 1614 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1615 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1616 "full pkt_buf_addr_pp $%p l2_len %d", 1617 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1618 1619 switch (pktbufsz_type) { 1620 case RCR_PKTBUFSZ_0: 1621 bsize = rx_rbr_p->pkt_buf_size0_bytes; 1622 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1623 "==> hxge_receive_packet: 0 buf %d", bsize)); 1624 break; 1625 case RCR_PKTBUFSZ_1: 1626 bsize = rx_rbr_p->pkt_buf_size1_bytes; 1627 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1628 "==> hxge_receive_packet: 1 buf %d", bsize)); 1629 break; 1630 case RCR_PKTBUFSZ_2: 1631 bsize = rx_rbr_p->pkt_buf_size2_bytes; 1632 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1633 "==> hxge_receive_packet: 2 buf %d", bsize)); 1634 break; 1635 case RCR_SINGLE_BLOCK: 1636 bsize = rx_msg_p->block_size; 1637 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1638 "==> hxge_receive_packet: single %d", bsize)); 1639 1640 break; 1641 default: 1642 MUTEX_EXIT(&rx_rbr_p->lock); 1643 MUTEX_EXIT(&rcr_p->lock); 1644 return; 1645 } 1646 1647 DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma, 1648 (buf_offset + sw_offset_bytes), (hdr_size + l2_len), 1649 DDI_DMA_SYNC_FORCPU); 1650 1651 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1652 "==> hxge_receive_packet: after first dump:usage count")); 1653 1654 if (rx_msg_p->cur_usage_cnt == 0) { 1655 if (rx_rbr_p->rbr_use_bcopy) { 1656 atomic_inc_32(&rx_rbr_p->rbr_consumed); 1657 if (rx_rbr_p->rbr_consumed < 1658 rx_rbr_p->rbr_threshold_hi) { 1659 if (rx_rbr_p->rbr_threshold_lo == 0 || 1660 ((rx_rbr_p->rbr_consumed >= 1661 rx_rbr_p->rbr_threshold_lo) && 1662 (rx_rbr_p->rbr_bufsize_type >= 1663 pktbufsz_type))) { 1664 rx_msg_p->rx_use_bcopy = B_TRUE; 1665 } 1666 } else { 1667 rx_msg_p->rx_use_bcopy = B_TRUE; 1668 } 1669 } 1670 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1671 "==> hxge_receive_packet: buf %d (new block) ", bsize)); 1672 1673 rx_msg_p->pkt_buf_size_code = pktbufsz_type; 1674 rx_msg_p->pkt_buf_size = bsize; 1675 rx_msg_p->cur_usage_cnt = 1; 1676 if (pktbufsz_type == RCR_SINGLE_BLOCK) { 1677 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1678 "==> hxge_receive_packet: buf %d (single block) ", 1679 bsize)); 1680 /* 1681 * Buffer can be reused once the free function is 1682 * called. 1683 */ 1684 rx_msg_p->max_usage_cnt = 1; 1685 buffer_free = B_TRUE; 1686 } else { 1687 rx_msg_p->max_usage_cnt = rx_msg_p->block_size / bsize; 1688 if (rx_msg_p->max_usage_cnt == 1) { 1689 buffer_free = B_TRUE; 1690 } 1691 } 1692 } else { 1693 rx_msg_p->cur_usage_cnt++; 1694 if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) { 1695 buffer_free = B_TRUE; 1696 } 1697 } 1698 1699 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1700 "msgbuf index = %d l2len %d bytes usage %d max_usage %d ", 1701 msg_index, l2_len, 1702 rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt)); 1703 1704 if (error_type) { 1705 rdc_stats->ierrors++; 1706 /* Update error stats */ 1707 rdc_stats->errlog.compl_err_type = error_type; 1708 HXGE_FM_REPORT_ERROR(hxgep, NULL, HXGE_FM_EREPORT_RDMC_RCR_ERR); 1709 1710 if (error_type & RCR_CTRL_FIFO_DED) { 1711 rdc_stats->ctrl_fifo_ecc_err++; 1712 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1713 " hxge_receive_packet: " 1714 " channel %d RCR ctrl_fifo_ded error", channel)); 1715 } else if (error_type & RCR_DATA_FIFO_DED) { 1716 rdc_stats->data_fifo_ecc_err++; 1717 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1718 " hxge_receive_packet: channel %d" 1719 " RCR data_fifo_ded error", channel)); 1720 } 1721 1722 /* 1723 * Update and repost buffer block if max usage count is 1724 * reached. 1725 */ 1726 if (error_send_up == B_FALSE) { 1727 atomic_inc_32(&rx_msg_p->ref_cnt); 1728 atomic_inc_32(&hxge_mblks_pending); 1729 if (buffer_free == B_TRUE) { 1730 rx_msg_p->free = B_TRUE; 1731 } 1732 1733 MUTEX_EXIT(&rx_rbr_p->lock); 1734 MUTEX_EXIT(&rcr_p->lock); 1735 hxge_freeb(rx_msg_p); 1736 return; 1737 } 1738 } 1739 1740 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1741 "==> hxge_receive_packet: DMA sync second ")); 1742 1743 bytes_read = rcr_p->rcvd_pkt_bytes; 1744 skip_len = sw_offset_bytes + hdr_size; 1745 if (!rx_msg_p->rx_use_bcopy) { 1746 /* 1747 * For loaned up buffers, the driver reference count 1748 * will be incremented first and then the free state. 1749 */ 1750 if ((nmp = hxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) { 1751 if (first_entry) { 1752 nmp->b_rptr = &nmp->b_rptr[skip_len]; 1753 if (l2_len < bsize - skip_len) { 1754 nmp->b_wptr = &nmp->b_rptr[l2_len]; 1755 } else { 1756 nmp->b_wptr = &nmp->b_rptr[bsize 1757 - skip_len]; 1758 } 1759 } else { 1760 if (l2_len - bytes_read < bsize) { 1761 nmp->b_wptr = 1762 &nmp->b_rptr[l2_len - bytes_read]; 1763 } else { 1764 nmp->b_wptr = &nmp->b_rptr[bsize]; 1765 } 1766 } 1767 } 1768 } else { 1769 if (first_entry) { 1770 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len, 1771 l2_len < bsize - skip_len ? 1772 l2_len : bsize - skip_len); 1773 } else { 1774 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset, 1775 l2_len - bytes_read < bsize ? 1776 l2_len - bytes_read : bsize); 1777 } 1778 } 1779 1780 if (nmp != NULL) { 1781 if (first_entry) 1782 bytes_read = nmp->b_wptr - nmp->b_rptr; 1783 else 1784 bytes_read += nmp->b_wptr - nmp->b_rptr; 1785 1786 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1787 "==> hxge_receive_packet after dupb: " 1788 "rbr consumed %d " 1789 "pktbufsz_type %d " 1790 "nmp $%p rptr $%p wptr $%p " 1791 "buf_offset %d bzise %d l2_len %d skip_len %d", 1792 rx_rbr_p->rbr_consumed, 1793 pktbufsz_type, 1794 nmp, nmp->b_rptr, nmp->b_wptr, 1795 buf_offset, bsize, l2_len, skip_len)); 1796 } else { 1797 cmn_err(CE_WARN, "!hxge_receive_packet: update stats (error)"); 1798 1799 atomic_inc_32(&rx_msg_p->ref_cnt); 1800 atomic_inc_32(&hxge_mblks_pending); 1801 if (buffer_free == B_TRUE) { 1802 rx_msg_p->free = B_TRUE; 1803 } 1804 1805 MUTEX_EXIT(&rx_rbr_p->lock); 1806 MUTEX_EXIT(&rcr_p->lock); 1807 hxge_freeb(rx_msg_p); 1808 return; 1809 } 1810 1811 if (buffer_free == B_TRUE) { 1812 rx_msg_p->free = B_TRUE; 1813 } 1814 1815 /* 1816 * ERROR, FRAG and PKT_TYPE are only reported in the first entry. If a 1817 * packet is not fragmented and no error bit is set, then L4 checksum 1818 * is OK. 1819 */ 1820 is_valid = (nmp != NULL); 1821 if (first_entry) { 1822 rdc_stats->ipackets++; /* count only 1st seg for jumbo */ 1823 rdc_stats->ibytes += skip_len + l2_len < bsize ? 1824 l2_len : bsize; 1825 } else { 1826 rdc_stats->ibytes += l2_len - bytes_read < bsize ? 1827 l2_len - bytes_read : bsize; 1828 } 1829 1830 rcr_p->rcvd_pkt_bytes = bytes_read; 1831 1832 MUTEX_EXIT(&rx_rbr_p->lock); 1833 MUTEX_EXIT(&rcr_p->lock); 1834 1835 if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) { 1836 atomic_inc_32(&rx_msg_p->ref_cnt); 1837 atomic_inc_32(&hxge_mblks_pending); 1838 hxge_freeb(rx_msg_p); 1839 } 1840 1841 if (is_valid) { 1842 nmp->b_cont = NULL; 1843 if (first_entry) { 1844 *mp = nmp; 1845 *mp_cont = NULL; 1846 } else { 1847 *mp_cont = nmp; 1848 } 1849 } 1850 1851 /* 1852 * Update stats and hardware checksuming. 1853 */ 1854 if (is_valid && !multi) { 1855 1856 is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP || 1857 pkt_type == RCR_PKT_IS_UDP) ? B_TRUE : B_FALSE); 1858 1859 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_receive_packet: " 1860 "is_valid 0x%x multi %d pkt %d d error %d", 1861 is_valid, multi, is_tcp_udp, error_type)); 1862 1863 if (is_tcp_udp && !error_type) { 1864 (void) hcksum_assoc(nmp, NULL, NULL, 0, 0, 0, 0, 1865 HCK_FULLCKSUM_OK | HCK_FULLCKSUM, 0); 1866 1867 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1868 "==> hxge_receive_packet: Full tcp/udp cksum " 1869 "is_valid 0x%x multi %d pkt %d " 1870 "error %d", 1871 is_valid, multi, is_tcp_udp, error_type)); 1872 } 1873 } 1874 1875 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1876 "==> hxge_receive_packet: *mp 0x%016llx", *mp)); 1877 1878 *multi_p = (multi == RCR_MULTI_MASK); 1879 1880 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_receive_packet: " 1881 "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx", 1882 *multi_p, nmp, *mp, *mp_cont)); 1883 } 1884 1885 /*ARGSUSED*/ 1886 static hxge_status_t 1887 hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp, 1888 rdc_stat_t cs) 1889 { 1890 p_hxge_rx_ring_stats_t rdc_stats; 1891 hpi_handle_t handle; 1892 boolean_t rxchan_fatal = B_FALSE; 1893 uint8_t channel; 1894 hxge_status_t status = HXGE_OK; 1895 uint64_t cs_val; 1896 1897 HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_rx_err_evnts")); 1898 1899 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1900 channel = ldvp->channel; 1901 1902 /* Clear the interrupts */ 1903 cs_val = cs.value & RDC_STAT_WR1C; 1904 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs_val); 1905 1906 rdc_stats = &hxgep->statsp->rdc_stats[ldvp->vdma_index]; 1907 1908 if (cs.bits.rbr_cpl_to) { 1909 rdc_stats->rbr_tmout++; 1910 HXGE_FM_REPORT_ERROR(hxgep, channel, 1911 HXGE_FM_EREPORT_RDMC_RBR_CPL_TO); 1912 rxchan_fatal = B_TRUE; 1913 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1914 "==> hxge_rx_err_evnts(channel %d): " 1915 "fatal error: rx_rbr_timeout", channel)); 1916 } 1917 1918 if ((cs.bits.rcr_shadow_par_err) || (cs.bits.rbr_prefetch_par_err)) { 1919 (void) hpi_rxdma_ring_perr_stat_get(handle, 1920 &rdc_stats->errlog.pre_par, &rdc_stats->errlog.sha_par); 1921 } 1922 1923 if (cs.bits.rcr_shadow_par_err) { 1924 rdc_stats->rcr_sha_par++; 1925 HXGE_FM_REPORT_ERROR(hxgep, channel, 1926 HXGE_FM_EREPORT_RDMC_RCR_SHA_PAR); 1927 rxchan_fatal = B_TRUE; 1928 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1929 "==> hxge_rx_err_evnts(channel %d): " 1930 "fatal error: rcr_shadow_par_err", channel)); 1931 } 1932 1933 if (cs.bits.rbr_prefetch_par_err) { 1934 rdc_stats->rbr_pre_par++; 1935 HXGE_FM_REPORT_ERROR(hxgep, channel, 1936 HXGE_FM_EREPORT_RDMC_RBR_PRE_PAR); 1937 rxchan_fatal = B_TRUE; 1938 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1939 "==> hxge_rx_err_evnts(channel %d): " 1940 "fatal error: rbr_prefetch_par_err", channel)); 1941 } 1942 1943 if (cs.bits.rbr_pre_empty) { 1944 rdc_stats->rbr_pre_empty++; 1945 HXGE_FM_REPORT_ERROR(hxgep, channel, 1946 HXGE_FM_EREPORT_RDMC_RBR_PRE_EMPTY); 1947 rxchan_fatal = B_TRUE; 1948 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1949 "==> hxge_rx_err_evnts(channel %d): " 1950 "fatal error: rbr_pre_empty", channel)); 1951 } 1952 1953 if (cs.bits.peu_resp_err) { 1954 rdc_stats->peu_resp_err++; 1955 HXGE_FM_REPORT_ERROR(hxgep, channel, 1956 HXGE_FM_EREPORT_RDMC_PEU_RESP_ERR); 1957 rxchan_fatal = B_TRUE; 1958 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1959 "==> hxge_rx_err_evnts(channel %d): " 1960 "fatal error: peu_resp_err", channel)); 1961 } 1962 1963 if (cs.bits.rcr_thres) { 1964 rdc_stats->rcr_thres++; 1965 } 1966 1967 if (cs.bits.rcr_to) { 1968 rdc_stats->rcr_to++; 1969 } 1970 1971 if (cs.bits.rcr_shadow_full) { 1972 rdc_stats->rcr_shadow_full++; 1973 HXGE_FM_REPORT_ERROR(hxgep, channel, 1974 HXGE_FM_EREPORT_RDMC_RCR_SHA_FULL); 1975 rxchan_fatal = B_TRUE; 1976 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1977 "==> hxge_rx_err_evnts(channel %d): " 1978 "fatal error: rcr_shadow_full", channel)); 1979 } 1980 1981 if (cs.bits.rcr_full) { 1982 rdc_stats->rcrfull++; 1983 HXGE_FM_REPORT_ERROR(hxgep, channel, 1984 HXGE_FM_EREPORT_RDMC_RCRFULL); 1985 rxchan_fatal = B_TRUE; 1986 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1987 "==> hxge_rx_err_evnts(channel %d): " 1988 "fatal error: rcrfull error", channel)); 1989 } 1990 1991 if (cs.bits.rbr_empty) { 1992 rdc_stats->rbr_empty++; 1993 if (rdc_stats->rbr_empty == 1) 1994 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1995 "==> hxge_rx_err_evnts(channel %d): " 1996 "rbr empty error", channel)); 1997 /* 1998 * DMA channel is disabled due to rbr_empty bit is set 1999 * although it is not fatal. Enable the DMA channel here 2000 * to work-around the hardware bug. 2001 */ 2002 (void) hpi_rxdma_cfg_rdc_enable(handle, channel); 2003 } 2004 2005 if (cs.bits.rbr_full) { 2006 rdc_stats->rbrfull++; 2007 HXGE_FM_REPORT_ERROR(hxgep, channel, 2008 HXGE_FM_EREPORT_RDMC_RBRFULL); 2009 rxchan_fatal = B_TRUE; 2010 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2011 "==> hxge_rx_err_evnts(channel %d): " 2012 "fatal error: rbr_full error", channel)); 2013 } 2014 2015 if (rxchan_fatal) { 2016 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2017 " hxge_rx_err_evnts: fatal error on Channel #%d\n", 2018 channel)); 2019 status = hxge_rxdma_fatal_err_recover(hxgep, channel); 2020 if (status == HXGE_OK) { 2021 FM_SERVICE_RESTORED(hxgep); 2022 } 2023 } 2024 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "<== hxge_rx_err_evnts")); 2025 2026 return (status); 2027 } 2028 2029 static hxge_status_t 2030 hxge_map_rxdma(p_hxge_t hxgep) 2031 { 2032 int i, ndmas; 2033 uint16_t channel; 2034 p_rx_rbr_rings_t rx_rbr_rings; 2035 p_rx_rbr_ring_t *rbr_rings; 2036 p_rx_rcr_rings_t rx_rcr_rings; 2037 p_rx_rcr_ring_t *rcr_rings; 2038 p_rx_mbox_areas_t rx_mbox_areas_p; 2039 p_rx_mbox_t *rx_mbox_p; 2040 p_hxge_dma_pool_t dma_buf_poolp; 2041 p_hxge_dma_pool_t dma_cntl_poolp; 2042 p_hxge_dma_common_t *dma_buf_p; 2043 p_hxge_dma_common_t *dma_cntl_p; 2044 uint32_t *num_chunks; 2045 hxge_status_t status = HXGE_OK; 2046 2047 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_map_rxdma")); 2048 2049 dma_buf_poolp = hxgep->rx_buf_pool_p; 2050 dma_cntl_poolp = hxgep->rx_cntl_pool_p; 2051 2052 if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) { 2053 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2054 "<== hxge_map_rxdma: buf not allocated")); 2055 return (HXGE_ERROR); 2056 } 2057 2058 ndmas = dma_buf_poolp->ndmas; 2059 if (!ndmas) { 2060 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2061 "<== hxge_map_rxdma: no dma allocated")); 2062 return (HXGE_ERROR); 2063 } 2064 2065 num_chunks = dma_buf_poolp->num_chunks; 2066 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2067 dma_cntl_p = dma_cntl_poolp->dma_buf_pool_p; 2068 rx_rbr_rings = (p_rx_rbr_rings_t) 2069 KMEM_ZALLOC(sizeof (rx_rbr_rings_t), KM_SLEEP); 2070 rbr_rings = (p_rx_rbr_ring_t *)KMEM_ZALLOC( 2071 sizeof (p_rx_rbr_ring_t) * ndmas, KM_SLEEP); 2072 2073 rx_rcr_rings = (p_rx_rcr_rings_t) 2074 KMEM_ZALLOC(sizeof (rx_rcr_rings_t), KM_SLEEP); 2075 rcr_rings = (p_rx_rcr_ring_t *)KMEM_ZALLOC( 2076 sizeof (p_rx_rcr_ring_t) * ndmas, KM_SLEEP); 2077 2078 rx_mbox_areas_p = (p_rx_mbox_areas_t) 2079 KMEM_ZALLOC(sizeof (rx_mbox_areas_t), KM_SLEEP); 2080 rx_mbox_p = (p_rx_mbox_t *)KMEM_ZALLOC( 2081 sizeof (p_rx_mbox_t) * ndmas, KM_SLEEP); 2082 2083 /* 2084 * Timeout should be set based on the system clock divider. 2085 * The following timeout value of 1 assumes that the 2086 * granularity (1000) is 3 microseconds running at 300MHz. 2087 */ 2088 2089 hxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT; 2090 hxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT; 2091 2092 /* 2093 * Map descriptors from the buffer polls for each dam channel. 2094 */ 2095 for (i = 0; i < ndmas; i++) { 2096 /* 2097 * Set up and prepare buffer blocks, descriptors and mailbox. 2098 */ 2099 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2100 status = hxge_map_rxdma_channel(hxgep, channel, 2101 (p_hxge_dma_common_t *)&dma_buf_p[i], 2102 (p_rx_rbr_ring_t *)&rbr_rings[i], 2103 num_chunks[i], (p_hxge_dma_common_t *)&dma_cntl_p[i], 2104 (p_rx_rcr_ring_t *)&rcr_rings[i], 2105 (p_rx_mbox_t *)&rx_mbox_p[i]); 2106 if (status != HXGE_OK) { 2107 goto hxge_map_rxdma_fail1; 2108 } 2109 rbr_rings[i]->index = (uint16_t)i; 2110 rcr_rings[i]->index = (uint16_t)i; 2111 rcr_rings[i]->rdc_stats = &hxgep->statsp->rdc_stats[i]; 2112 } 2113 2114 rx_rbr_rings->ndmas = rx_rcr_rings->ndmas = ndmas; 2115 rx_rbr_rings->rbr_rings = rbr_rings; 2116 hxgep->rx_rbr_rings = rx_rbr_rings; 2117 rx_rcr_rings->rcr_rings = rcr_rings; 2118 hxgep->rx_rcr_rings = rx_rcr_rings; 2119 2120 rx_mbox_areas_p->rxmbox_areas = rx_mbox_p; 2121 hxgep->rx_mbox_areas_p = rx_mbox_areas_p; 2122 2123 goto hxge_map_rxdma_exit; 2124 2125 hxge_map_rxdma_fail1: 2126 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2127 "==> hxge_map_rxdma: unmap rbr,rcr (status 0x%x channel %d i %d)", 2128 status, channel, i)); 2129 i--; 2130 for (; i >= 0; i--) { 2131 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2132 hxge_unmap_rxdma_channel(hxgep, channel, 2133 rbr_rings[i], rcr_rings[i], rx_mbox_p[i]); 2134 } 2135 2136 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2137 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2138 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2139 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2140 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2141 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2142 2143 hxge_map_rxdma_exit: 2144 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2145 "<== hxge_map_rxdma: (status 0x%x channel %d)", status, channel)); 2146 2147 return (status); 2148 } 2149 2150 static void 2151 hxge_unmap_rxdma(p_hxge_t hxgep) 2152 { 2153 int i, ndmas; 2154 uint16_t channel; 2155 p_rx_rbr_rings_t rx_rbr_rings; 2156 p_rx_rbr_ring_t *rbr_rings; 2157 p_rx_rcr_rings_t rx_rcr_rings; 2158 p_rx_rcr_ring_t *rcr_rings; 2159 p_rx_mbox_areas_t rx_mbox_areas_p; 2160 p_rx_mbox_t *rx_mbox_p; 2161 p_hxge_dma_pool_t dma_buf_poolp; 2162 p_hxge_dma_pool_t dma_cntl_poolp; 2163 p_hxge_dma_common_t *dma_buf_p; 2164 2165 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_unmap_rxdma")); 2166 2167 dma_buf_poolp = hxgep->rx_buf_pool_p; 2168 dma_cntl_poolp = hxgep->rx_cntl_pool_p; 2169 2170 if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) { 2171 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2172 "<== hxge_unmap_rxdma: NULL buf pointers")); 2173 return; 2174 } 2175 2176 rx_rbr_rings = hxgep->rx_rbr_rings; 2177 rx_rcr_rings = hxgep->rx_rcr_rings; 2178 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 2179 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2180 "<== hxge_unmap_rxdma: NULL ring pointers")); 2181 return; 2182 } 2183 2184 ndmas = rx_rbr_rings->ndmas; 2185 if (!ndmas) { 2186 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2187 "<== hxge_unmap_rxdma: no channel")); 2188 return; 2189 } 2190 2191 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2192 "==> hxge_unmap_rxdma (ndmas %d)", ndmas)); 2193 2194 rbr_rings = rx_rbr_rings->rbr_rings; 2195 rcr_rings = rx_rcr_rings->rcr_rings; 2196 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 2197 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 2198 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2199 2200 for (i = 0; i < ndmas; i++) { 2201 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2202 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2203 "==> hxge_unmap_rxdma (ndmas %d) channel %d", 2204 ndmas, channel)); 2205 (void) hxge_unmap_rxdma_channel(hxgep, channel, 2206 (p_rx_rbr_ring_t)rbr_rings[i], 2207 (p_rx_rcr_ring_t)rcr_rings[i], 2208 (p_rx_mbox_t)rx_mbox_p[i]); 2209 } 2210 2211 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2212 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2213 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2214 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2215 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2216 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2217 2218 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma")); 2219 } 2220 2221 hxge_status_t 2222 hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2223 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 2224 uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p, 2225 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2226 { 2227 int status = HXGE_OK; 2228 2229 /* 2230 * Set up and prepare buffer blocks, descriptors and mailbox. 2231 */ 2232 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2233 "==> hxge_map_rxdma_channel (channel %d)", channel)); 2234 2235 /* 2236 * Receive buffer blocks 2237 */ 2238 status = hxge_map_rxdma_channel_buf_ring(hxgep, channel, 2239 dma_buf_p, rbr_p, num_chunks); 2240 if (status != HXGE_OK) { 2241 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2242 "==> hxge_map_rxdma_channel (channel %d): " 2243 "map buffer failed 0x%x", channel, status)); 2244 goto hxge_map_rxdma_channel_exit; 2245 } 2246 2247 /* 2248 * Receive block ring, completion ring and mailbox. 2249 */ 2250 status = hxge_map_rxdma_channel_cfg_ring(hxgep, channel, 2251 dma_cntl_p, rbr_p, rcr_p, rx_mbox_p); 2252 if (status != HXGE_OK) { 2253 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2254 "==> hxge_map_rxdma_channel (channel %d): " 2255 "map config failed 0x%x", channel, status)); 2256 goto hxge_map_rxdma_channel_fail2; 2257 } 2258 goto hxge_map_rxdma_channel_exit; 2259 2260 hxge_map_rxdma_channel_fail3: 2261 /* Free rbr, rcr */ 2262 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2263 "==> hxge_map_rxdma_channel: free rbr/rcr (status 0x%x channel %d)", 2264 status, channel)); 2265 hxge_unmap_rxdma_channel_cfg_ring(hxgep, *rcr_p, *rx_mbox_p); 2266 2267 hxge_map_rxdma_channel_fail2: 2268 /* Free buffer blocks */ 2269 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2270 "==> hxge_map_rxdma_channel: free rx buffers" 2271 "(hxgep 0x%x status 0x%x channel %d)", 2272 hxgep, status, channel)); 2273 hxge_unmap_rxdma_channel_buf_ring(hxgep, *rbr_p); 2274 2275 status = HXGE_ERROR; 2276 2277 hxge_map_rxdma_channel_exit: 2278 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2279 "<== hxge_map_rxdma_channel: (hxgep 0x%x status 0x%x channel %d)", 2280 hxgep, status, channel)); 2281 2282 return (status); 2283 } 2284 2285 /*ARGSUSED*/ 2286 static void 2287 hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2288 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2289 { 2290 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2291 "==> hxge_unmap_rxdma_channel (channel %d)", channel)); 2292 2293 /* 2294 * unmap receive block ring, completion ring and mailbox. 2295 */ 2296 (void) hxge_unmap_rxdma_channel_cfg_ring(hxgep, rcr_p, rx_mbox_p); 2297 2298 /* unmap buffer blocks */ 2299 (void) hxge_unmap_rxdma_channel_buf_ring(hxgep, rbr_p); 2300 2301 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma_channel")); 2302 } 2303 2304 /*ARGSUSED*/ 2305 static hxge_status_t 2306 hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel, 2307 p_hxge_dma_common_t *dma_cntl_p, p_rx_rbr_ring_t *rbr_p, 2308 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2309 { 2310 p_rx_rbr_ring_t rbrp; 2311 p_rx_rcr_ring_t rcrp; 2312 p_rx_mbox_t mboxp; 2313 p_hxge_dma_common_t cntl_dmap; 2314 p_hxge_dma_common_t dmap; 2315 p_rx_msg_t *rx_msg_ring; 2316 p_rx_msg_t rx_msg_p; 2317 rdc_rbr_cfg_a_t *rcfga_p; 2318 rdc_rbr_cfg_b_t *rcfgb_p; 2319 rdc_rcr_cfg_a_t *cfga_p; 2320 rdc_rcr_cfg_b_t *cfgb_p; 2321 rdc_rx_cfg1_t *cfig1_p; 2322 rdc_rx_cfg2_t *cfig2_p; 2323 rdc_rbr_kick_t *kick_p; 2324 uint32_t dmaaddrp; 2325 uint32_t *rbr_vaddrp; 2326 uint32_t bkaddr; 2327 hxge_status_t status = HXGE_OK; 2328 int i; 2329 uint32_t hxge_port_rcr_size; 2330 2331 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2332 "==> hxge_map_rxdma_channel_cfg_ring")); 2333 2334 cntl_dmap = *dma_cntl_p; 2335 2336 /* Map in the receive block ring */ 2337 rbrp = *rbr_p; 2338 dmap = (p_hxge_dma_common_t)&rbrp->rbr_desc; 2339 hxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4); 2340 2341 /* 2342 * Zero out buffer block ring descriptors. 2343 */ 2344 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2345 2346 rcfga_p = &(rbrp->rbr_cfga); 2347 rcfgb_p = &(rbrp->rbr_cfgb); 2348 kick_p = &(rbrp->rbr_kick); 2349 rcfga_p->value = 0; 2350 rcfgb_p->value = 0; 2351 kick_p->value = 0; 2352 rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress; 2353 rcfga_p->value = (rbrp->rbr_addr & 2354 (RBR_CFIG_A_STDADDR_MASK | RBR_CFIG_A_STDADDR_BASE_MASK)); 2355 rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT); 2356 2357 /* XXXX: how to choose packet buffer sizes */ 2358 rcfgb_p->bits.bufsz0 = rbrp->pkt_buf_size0; 2359 rcfgb_p->bits.vld0 = 1; 2360 rcfgb_p->bits.bufsz1 = rbrp->pkt_buf_size1; 2361 rcfgb_p->bits.vld1 = 1; 2362 rcfgb_p->bits.bufsz2 = rbrp->pkt_buf_size2; 2363 rcfgb_p->bits.vld2 = 1; 2364 rcfgb_p->bits.bksize = hxgep->rx_bksize_code; 2365 2366 /* 2367 * For each buffer block, enter receive block address to the ring. 2368 */ 2369 rbr_vaddrp = (uint32_t *)dmap->kaddrp; 2370 rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp; 2371 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2372 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2373 "rbr_vaddrp $%p", dma_channel, rbr_vaddrp)); 2374 2375 rx_msg_ring = rbrp->rx_msg_ring; 2376 for (i = 0; i < rbrp->tnblocks; i++) { 2377 rx_msg_p = rx_msg_ring[i]; 2378 rx_msg_p->hxgep = hxgep; 2379 rx_msg_p->rx_rbr_p = rbrp; 2380 bkaddr = (uint32_t) 2381 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2382 RBR_BKADDR_SHIFT)); 2383 rx_msg_p->free = B_FALSE; 2384 rx_msg_p->max_usage_cnt = 0xbaddcafe; 2385 2386 *rbr_vaddrp++ = bkaddr; 2387 } 2388 2389 kick_p->bits.bkadd = rbrp->rbb_max; 2390 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 2391 2392 rbrp->rbr_rd_index = 0; 2393 2394 rbrp->rbr_consumed = 0; 2395 rbrp->rbr_use_bcopy = B_TRUE; 2396 rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0; 2397 2398 /* 2399 * Do bcopy on packets greater than bcopy size once the lo threshold is 2400 * reached. This lo threshold should be less than the hi threshold. 2401 * 2402 * Do bcopy on every packet once the hi threshold is reached. 2403 */ 2404 if (hxge_rx_threshold_lo >= hxge_rx_threshold_hi) { 2405 /* default it to use hi */ 2406 hxge_rx_threshold_lo = hxge_rx_threshold_hi; 2407 } 2408 if (hxge_rx_buf_size_type > HXGE_RBR_TYPE2) { 2409 hxge_rx_buf_size_type = HXGE_RBR_TYPE2; 2410 } 2411 rbrp->rbr_bufsize_type = hxge_rx_buf_size_type; 2412 2413 switch (hxge_rx_threshold_hi) { 2414 default: 2415 case HXGE_RX_COPY_NONE: 2416 /* Do not do bcopy at all */ 2417 rbrp->rbr_use_bcopy = B_FALSE; 2418 rbrp->rbr_threshold_hi = rbrp->rbb_max; 2419 break; 2420 2421 case HXGE_RX_COPY_1: 2422 case HXGE_RX_COPY_2: 2423 case HXGE_RX_COPY_3: 2424 case HXGE_RX_COPY_4: 2425 case HXGE_RX_COPY_5: 2426 case HXGE_RX_COPY_6: 2427 case HXGE_RX_COPY_7: 2428 rbrp->rbr_threshold_hi = 2429 rbrp->rbb_max * (hxge_rx_threshold_hi) / 2430 HXGE_RX_BCOPY_SCALE; 2431 break; 2432 2433 case HXGE_RX_COPY_ALL: 2434 rbrp->rbr_threshold_hi = 0; 2435 break; 2436 } 2437 2438 switch (hxge_rx_threshold_lo) { 2439 default: 2440 case HXGE_RX_COPY_NONE: 2441 /* Do not do bcopy at all */ 2442 if (rbrp->rbr_use_bcopy) { 2443 rbrp->rbr_use_bcopy = B_FALSE; 2444 } 2445 rbrp->rbr_threshold_lo = rbrp->rbb_max; 2446 break; 2447 2448 case HXGE_RX_COPY_1: 2449 case HXGE_RX_COPY_2: 2450 case HXGE_RX_COPY_3: 2451 case HXGE_RX_COPY_4: 2452 case HXGE_RX_COPY_5: 2453 case HXGE_RX_COPY_6: 2454 case HXGE_RX_COPY_7: 2455 rbrp->rbr_threshold_lo = 2456 rbrp->rbb_max * (hxge_rx_threshold_lo) / 2457 HXGE_RX_BCOPY_SCALE; 2458 break; 2459 2460 case HXGE_RX_COPY_ALL: 2461 rbrp->rbr_threshold_lo = 0; 2462 break; 2463 } 2464 2465 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2466 "hxge_map_rxdma_channel_cfg_ring: channel %d rbb_max %d " 2467 "rbrp->rbr_bufsize_type %d rbb_threshold_hi %d " 2468 "rbb_threshold_lo %d", 2469 dma_channel, rbrp->rbb_max, rbrp->rbr_bufsize_type, 2470 rbrp->rbr_threshold_hi, rbrp->rbr_threshold_lo)); 2471 2472 /* Map in the receive completion ring */ 2473 rcrp = (p_rx_rcr_ring_t)KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP); 2474 rcrp->rdc = dma_channel; 2475 2476 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 2477 rcrp->comp_size = hxge_port_rcr_size; 2478 rcrp->comp_wrap_mask = hxge_port_rcr_size - 1; 2479 2480 rcrp->max_receive_pkts = hxge_max_rx_pkts; 2481 2482 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 2483 hxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size, 2484 sizeof (rcr_entry_t)); 2485 rcrp->comp_rd_index = 0; 2486 rcrp->comp_wt_index = 0; 2487 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 2488 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 2489 #if defined(__i386) 2490 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2491 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2492 #else 2493 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2494 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2495 #endif 2496 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 2497 (hxge_port_rcr_size - 1); 2498 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 2499 (hxge_port_rcr_size - 1); 2500 2501 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2502 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2503 "rbr_vaddrp $%p rcr_desc_rd_head_p $%p " 2504 "rcr_desc_rd_head_pp $%p rcr_desc_rd_last_p $%p " 2505 "rcr_desc_rd_last_pp $%p ", 2506 dma_channel, rbr_vaddrp, rcrp->rcr_desc_rd_head_p, 2507 rcrp->rcr_desc_rd_head_pp, rcrp->rcr_desc_last_p, 2508 rcrp->rcr_desc_last_pp)); 2509 2510 /* 2511 * Zero out buffer block ring descriptors. 2512 */ 2513 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2514 rcrp->intr_timeout = hxgep->intr_timeout; 2515 rcrp->intr_threshold = hxgep->intr_threshold; 2516 rcrp->full_hdr_flag = B_FALSE; 2517 rcrp->sw_priv_hdr_len = 0; 2518 2519 cfga_p = &(rcrp->rcr_cfga); 2520 cfgb_p = &(rcrp->rcr_cfgb); 2521 cfga_p->value = 0; 2522 cfgb_p->value = 0; 2523 rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress; 2524 2525 cfga_p->value = (rcrp->rcr_addr & 2526 (RCRCFIG_A_STADDR_MASK | RCRCFIG_A_STADDR_BASE_MASK)); 2527 2528 cfga_p->value |= ((uint64_t)rcrp->comp_size << RCRCFIG_A_LEN_SHIF); 2529 2530 /* 2531 * Timeout should be set based on the system clock divider. The 2532 * following timeout value of 1 assumes that the granularity (1000) is 2533 * 3 microseconds running at 300MHz. 2534 */ 2535 cfgb_p->bits.pthres = rcrp->intr_threshold; 2536 cfgb_p->bits.timeout = rcrp->intr_timeout; 2537 cfgb_p->bits.entout = 1; 2538 2539 /* Map in the mailbox */ 2540 mboxp = (p_rx_mbox_t)KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP); 2541 dmap = (p_hxge_dma_common_t)&mboxp->rx_mbox; 2542 hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t)); 2543 cfig1_p = (rdc_rx_cfg1_t *)&mboxp->rx_cfg1; 2544 cfig2_p = (rdc_rx_cfg2_t *)&mboxp->rx_cfg2; 2545 cfig1_p->value = cfig2_p->value = 0; 2546 2547 mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress; 2548 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2549 "==> hxge_map_rxdma_channel_cfg_ring: " 2550 "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx", 2551 dma_channel, cfig1_p->value, cfig2_p->value, 2552 mboxp->mbox_addr)); 2553 2554 dmaaddrp = (uint32_t)((dmap->dma_cookie.dmac_laddress >> 32) & 0xfff); 2555 cfig1_p->bits.mbaddr_h = dmaaddrp; 2556 2557 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff); 2558 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 2559 RXDMA_CFIG2_MBADDR_L_MASK); 2560 2561 cfig2_p->bits.mbaddr_l = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT); 2562 2563 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2564 "==> hxge_map_rxdma_channel_cfg_ring: channel %d damaddrp $%p " 2565 "cfg1 0x%016llx cfig2 0x%016llx", 2566 dma_channel, dmaaddrp, cfig1_p->value, cfig2_p->value)); 2567 2568 cfig2_p->bits.full_hdr = rcrp->full_hdr_flag; 2569 cfig2_p->bits.offset = rcrp->sw_priv_hdr_len; 2570 2571 rbrp->rx_rcr_p = rcrp; 2572 rcrp->rx_rbr_p = rbrp; 2573 *rcr_p = rcrp; 2574 *rx_mbox_p = mboxp; 2575 2576 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2577 "<== hxge_map_rxdma_channel_cfg_ring status 0x%08x", status)); 2578 return (status); 2579 } 2580 2581 /*ARGSUSED*/ 2582 static void 2583 hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 2584 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2585 { 2586 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2587 "==> hxge_unmap_rxdma_channel_cfg_ring: channel %d", rcr_p->rdc)); 2588 2589 KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t)); 2590 KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t)); 2591 2592 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2593 "<== hxge_unmap_rxdma_channel_cfg_ring")); 2594 } 2595 2596 static hxge_status_t 2597 hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel, 2598 p_hxge_dma_common_t *dma_buf_p, 2599 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks) 2600 { 2601 p_rx_rbr_ring_t rbrp; 2602 p_hxge_dma_common_t dma_bufp, tmp_bufp; 2603 p_rx_msg_t *rx_msg_ring; 2604 p_rx_msg_t rx_msg_p; 2605 p_mblk_t mblk_p; 2606 2607 rxring_info_t *ring_info; 2608 hxge_status_t status = HXGE_OK; 2609 int i, j, index; 2610 uint32_t size, bsize, nblocks, nmsgs; 2611 2612 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2613 "==> hxge_map_rxdma_channel_buf_ring: channel %d", channel)); 2614 2615 dma_bufp = tmp_bufp = *dma_buf_p; 2616 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2617 " hxge_map_rxdma_channel_buf_ring: channel %d to map %d " 2618 "chunks bufp 0x%016llx", channel, num_chunks, dma_bufp)); 2619 2620 nmsgs = 0; 2621 for (i = 0; i < num_chunks; i++, tmp_bufp++) { 2622 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2623 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2624 "bufp 0x%016llx nblocks %d nmsgs %d", 2625 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs)); 2626 nmsgs += tmp_bufp->nblocks; 2627 } 2628 if (!nmsgs) { 2629 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2630 "<== hxge_map_rxdma_channel_buf_ring: channel %d " 2631 "no msg blocks", channel)); 2632 status = HXGE_ERROR; 2633 goto hxge_map_rxdma_channel_buf_ring_exit; 2634 } 2635 rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (rx_rbr_ring_t), KM_SLEEP); 2636 2637 size = nmsgs * sizeof (p_rx_msg_t); 2638 rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP); 2639 ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t), 2640 KM_SLEEP); 2641 2642 MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER, 2643 (void *) hxgep->interrupt_cookie); 2644 MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER, 2645 (void *) hxgep->interrupt_cookie); 2646 rbrp->rdc = channel; 2647 rbrp->num_blocks = num_chunks; 2648 rbrp->tnblocks = nmsgs; 2649 rbrp->rbb_max = nmsgs; 2650 rbrp->rbr_max_size = nmsgs; 2651 rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1); 2652 2653 rbrp->pages_to_post = 0; 2654 rbrp->pages_to_skip = 20; 2655 rbrp->pages_to_post_threshold = rbrp->rbb_max - rbrp->pages_to_skip / 2; 2656 2657 /* 2658 * Buffer sizes suggested by NIU architect. 256, 512 and 2K. 2659 */ 2660 2661 rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B; 2662 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES; 2663 rbrp->hpi_pkt_buf_size0 = SIZE_256B; 2664 2665 rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K; 2666 rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES; 2667 rbrp->hpi_pkt_buf_size1 = SIZE_1KB; 2668 2669 rbrp->block_size = hxgep->rx_default_block_size; 2670 2671 if (!hxge_jumbo_enable && !hxgep->param_arr[param_accept_jumbo].value) { 2672 rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K; 2673 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES; 2674 rbrp->hpi_pkt_buf_size2 = SIZE_2KB; 2675 } else { 2676 if (rbrp->block_size >= 0x2000) { 2677 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2678 "<== hxge_map_rxdma_channel_buf_ring: channel %d " 2679 "no msg blocks", channel)); 2680 status = HXGE_ERROR; 2681 goto hxge_map_rxdma_channel_buf_ring_fail1; 2682 } else { 2683 rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K; 2684 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES; 2685 rbrp->hpi_pkt_buf_size2 = SIZE_4KB; 2686 } 2687 } 2688 2689 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2690 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2691 "actual rbr max %d rbb_max %d nmsgs %d " 2692 "rbrp->block_size %d default_block_size %d " 2693 "(config hxge_rbr_size %d hxge_rbr_spare_size %d)", 2694 channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs, 2695 rbrp->block_size, hxgep->rx_default_block_size, 2696 hxge_rbr_size, hxge_rbr_spare_size)); 2697 2698 /* 2699 * Map in buffers from the buffer pool. 2700 * Note that num_blocks is the num_chunks. For Sparc, there is likely 2701 * only one chunk. For x86, there will be many chunks. 2702 * Loop over chunks. 2703 */ 2704 index = 0; 2705 for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) { 2706 bsize = dma_bufp->block_size; 2707 nblocks = dma_bufp->nblocks; 2708 #if defined(__i386) 2709 ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp; 2710 #else 2711 ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp; 2712 #endif 2713 ring_info->buffer[i].buf_index = i; 2714 ring_info->buffer[i].buf_size = dma_bufp->alength; 2715 ring_info->buffer[i].start_index = index; 2716 #if defined(__i386) 2717 ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp; 2718 #else 2719 ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp; 2720 #endif 2721 2722 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2723 " hxge_map_rxdma_channel_buf_ring: map channel %d " 2724 "chunk %d nblocks %d chunk_size %x block_size 0x%x " 2725 "dma_bufp $%p dvma_addr $%p", channel, i, 2726 dma_bufp->nblocks, 2727 ring_info->buffer[i].buf_size, bsize, dma_bufp, 2728 ring_info->buffer[i].dvma_addr)); 2729 2730 /* loop over blocks within a chunk */ 2731 for (j = 0; j < nblocks; j++) { 2732 if ((rx_msg_p = hxge_allocb(bsize, BPRI_LO, 2733 dma_bufp)) == NULL) { 2734 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2735 "allocb failed (index %d i %d j %d)", 2736 index, i, j)); 2737 goto hxge_map_rxdma_channel_buf_ring_fail1; 2738 } 2739 rx_msg_ring[index] = rx_msg_p; 2740 rx_msg_p->block_index = index; 2741 rx_msg_p->shifted_addr = (uint32_t) 2742 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2743 RBR_BKADDR_SHIFT)); 2744 /* 2745 * Too much output 2746 * HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2747 * "index %d j %d rx_msg_p $%p mblk %p", 2748 * index, j, rx_msg_p, rx_msg_p->rx_mblk_p)); 2749 */ 2750 mblk_p = rx_msg_p->rx_mblk_p; 2751 mblk_p->b_wptr = mblk_p->b_rptr + bsize; 2752 2753 rbrp->rbr_ref_cnt++; 2754 index++; 2755 rx_msg_p->buf_dma.dma_channel = channel; 2756 } 2757 } 2758 if (i < rbrp->num_blocks) { 2759 goto hxge_map_rxdma_channel_buf_ring_fail1; 2760 } 2761 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2762 "hxge_map_rxdma_channel_buf_ring: done buf init " 2763 "channel %d msg block entries %d", channel, index)); 2764 ring_info->block_size_mask = bsize - 1; 2765 rbrp->rx_msg_ring = rx_msg_ring; 2766 rbrp->dma_bufp = dma_buf_p; 2767 rbrp->ring_info = ring_info; 2768 2769 status = hxge_rxbuf_index_info_init(hxgep, rbrp); 2770 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, " hxge_map_rxdma_channel_buf_ring: " 2771 "channel %d done buf info init", channel)); 2772 2773 /* 2774 * Finally, permit hxge_freeb() to call hxge_post_page(). 2775 */ 2776 rbrp->rbr_state = RBR_POSTING; 2777 2778 *rbr_p = rbrp; 2779 2780 goto hxge_map_rxdma_channel_buf_ring_exit; 2781 2782 hxge_map_rxdma_channel_buf_ring_fail1: 2783 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2784 " hxge_map_rxdma_channel_buf_ring: failed channel (0x%x)", 2785 channel, status)); 2786 2787 index--; 2788 for (; index >= 0; index--) { 2789 rx_msg_p = rx_msg_ring[index]; 2790 if (rx_msg_p != NULL) { 2791 hxge_freeb(rx_msg_p); 2792 rx_msg_ring[index] = NULL; 2793 } 2794 } 2795 2796 hxge_map_rxdma_channel_buf_ring_fail: 2797 MUTEX_DESTROY(&rbrp->post_lock); 2798 MUTEX_DESTROY(&rbrp->lock); 2799 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 2800 KMEM_FREE(rx_msg_ring, size); 2801 KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t)); 2802 2803 status = HXGE_ERROR; 2804 2805 hxge_map_rxdma_channel_buf_ring_exit: 2806 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2807 "<== hxge_map_rxdma_channel_buf_ring status 0x%08x", status)); 2808 2809 return (status); 2810 } 2811 2812 /*ARGSUSED*/ 2813 static void 2814 hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 2815 p_rx_rbr_ring_t rbr_p) 2816 { 2817 p_rx_msg_t *rx_msg_ring; 2818 p_rx_msg_t rx_msg_p; 2819 rxring_info_t *ring_info; 2820 int i; 2821 uint32_t size; 2822 2823 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2824 "==> hxge_unmap_rxdma_channel_buf_ring")); 2825 if (rbr_p == NULL) { 2826 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2827 "<== hxge_unmap_rxdma_channel_buf_ring: NULL rbrp")); 2828 return; 2829 } 2830 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2831 "==> hxge_unmap_rxdma_channel_buf_ring: channel %d", rbr_p->rdc)); 2832 2833 rx_msg_ring = rbr_p->rx_msg_ring; 2834 ring_info = rbr_p->ring_info; 2835 2836 if (rx_msg_ring == NULL || ring_info == NULL) { 2837 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2838 "<== hxge_unmap_rxdma_channel_buf_ring: " 2839 "rx_msg_ring $%p ring_info $%p", rx_msg_p, ring_info)); 2840 return; 2841 } 2842 2843 size = rbr_p->tnblocks * sizeof (p_rx_msg_t); 2844 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2845 " hxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d " 2846 "tnblocks %d (max %d) size ptrs %d ", rbr_p->rdc, rbr_p->num_blocks, 2847 rbr_p->tnblocks, rbr_p->rbr_max_size, size)); 2848 2849 for (i = 0; i < rbr_p->tnblocks; i++) { 2850 rx_msg_p = rx_msg_ring[i]; 2851 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2852 " hxge_unmap_rxdma_channel_buf_ring: " 2853 "rx_msg_p $%p", rx_msg_p)); 2854 if (rx_msg_p != NULL) { 2855 hxge_freeb(rx_msg_p); 2856 rx_msg_ring[i] = NULL; 2857 } 2858 } 2859 2860 /* 2861 * We no longer may use the mutex <post_lock>. By setting 2862 * <rbr_state> to anything but POSTING, we prevent 2863 * hxge_post_page() from accessing a dead mutex. 2864 */ 2865 rbr_p->rbr_state = RBR_UNMAPPING; 2866 MUTEX_DESTROY(&rbr_p->post_lock); 2867 2868 MUTEX_DESTROY(&rbr_p->lock); 2869 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 2870 KMEM_FREE(rx_msg_ring, size); 2871 2872 if (rbr_p->rbr_ref_cnt == 0) { 2873 /* This is the normal state of affairs. */ 2874 KMEM_FREE(rbr_p, sizeof (*rbr_p)); 2875 } else { 2876 /* 2877 * Some of our buffers are still being used. 2878 * Therefore, tell hxge_freeb() this ring is 2879 * unmapped, so it may free <rbr_p> for us. 2880 */ 2881 rbr_p->rbr_state = RBR_UNMAPPED; 2882 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2883 "unmap_rxdma_buf_ring: %d %s outstanding.", 2884 rbr_p->rbr_ref_cnt, 2885 rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs")); 2886 } 2887 2888 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2889 "<== hxge_unmap_rxdma_channel_buf_ring")); 2890 } 2891 2892 static hxge_status_t 2893 hxge_rxdma_hw_start_common(p_hxge_t hxgep) 2894 { 2895 hxge_status_t status = HXGE_OK; 2896 2897 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 2898 2899 /* 2900 * Load the sharable parameters by writing to the function zero control 2901 * registers. These FZC registers should be initialized only once for 2902 * the entire chip. 2903 */ 2904 (void) hxge_init_fzc_rx_common(hxgep); 2905 2906 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 2907 2908 return (status); 2909 } 2910 2911 static hxge_status_t 2912 hxge_rxdma_hw_start(p_hxge_t hxgep) 2913 { 2914 int i, ndmas; 2915 uint16_t channel; 2916 p_rx_rbr_rings_t rx_rbr_rings; 2917 p_rx_rbr_ring_t *rbr_rings; 2918 p_rx_rcr_rings_t rx_rcr_rings; 2919 p_rx_rcr_ring_t *rcr_rings; 2920 p_rx_mbox_areas_t rx_mbox_areas_p; 2921 p_rx_mbox_t *rx_mbox_p; 2922 hxge_status_t status = HXGE_OK; 2923 2924 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start")); 2925 2926 rx_rbr_rings = hxgep->rx_rbr_rings; 2927 rx_rcr_rings = hxgep->rx_rcr_rings; 2928 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 2929 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2930 "<== hxge_rxdma_hw_start: NULL ring pointers")); 2931 return (HXGE_ERROR); 2932 } 2933 2934 ndmas = rx_rbr_rings->ndmas; 2935 if (ndmas == 0) { 2936 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2937 "<== hxge_rxdma_hw_start: no dma channel allocated")); 2938 return (HXGE_ERROR); 2939 } 2940 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2941 "==> hxge_rxdma_hw_start (ndmas %d)", ndmas)); 2942 2943 /* 2944 * Scrub the RDC Rx DMA Prefetch Buffer Command. 2945 */ 2946 for (i = 0; i < 128; i++) { 2947 HXGE_REG_WR64(hxgep->hpi_handle, RDC_PREF_CMD, i); 2948 } 2949 2950 /* 2951 * Scrub Rx DMA Shadow Tail Command. 2952 */ 2953 for (i = 0; i < 64; i++) { 2954 HXGE_REG_WR64(hxgep->hpi_handle, RDC_SHADOW_CMD, i); 2955 } 2956 2957 /* 2958 * Scrub Rx DMA Control Fifo Command. 2959 */ 2960 for (i = 0; i < 512; i++) { 2961 HXGE_REG_WR64(hxgep->hpi_handle, RDC_CTRL_FIFO_CMD, i); 2962 } 2963 2964 /* 2965 * Scrub Rx DMA Data Fifo Command. 2966 */ 2967 for (i = 0; i < 1536; i++) { 2968 HXGE_REG_WR64(hxgep->hpi_handle, RDC_DATA_FIFO_CMD, i); 2969 } 2970 2971 /* 2972 * Reset the FIFO Error Stat. 2973 */ 2974 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_STAT, 0xFF); 2975 2976 /* Set the error mask to receive interrupts */ 2977 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 2978 2979 rbr_rings = rx_rbr_rings->rbr_rings; 2980 rcr_rings = rx_rcr_rings->rcr_rings; 2981 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 2982 if (rx_mbox_areas_p) { 2983 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 2984 } 2985 2986 for (i = 0; i < ndmas; i++) { 2987 channel = rbr_rings[i]->rdc; 2988 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2989 "==> hxge_rxdma_hw_start (ndmas %d) channel %d", 2990 ndmas, channel)); 2991 status = hxge_rxdma_start_channel(hxgep, channel, 2992 (p_rx_rbr_ring_t)rbr_rings[i], 2993 (p_rx_rcr_ring_t)rcr_rings[i], 2994 (p_rx_mbox_t)rx_mbox_p[i]); 2995 if (status != HXGE_OK) { 2996 goto hxge_rxdma_hw_start_fail1; 2997 } 2998 } 2999 3000 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start: " 3001 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3002 rx_rbr_rings, rx_rcr_rings)); 3003 goto hxge_rxdma_hw_start_exit; 3004 3005 hxge_rxdma_hw_start_fail1: 3006 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3007 "==> hxge_rxdma_hw_start: disable " 3008 "(status 0x%x channel %d i %d)", status, channel, i)); 3009 for (; i >= 0; i--) { 3010 channel = rbr_rings[i]->rdc; 3011 (void) hxge_rxdma_stop_channel(hxgep, channel); 3012 } 3013 3014 hxge_rxdma_hw_start_exit: 3015 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3016 "==> hxge_rxdma_hw_start: (status 0x%x)", status)); 3017 return (status); 3018 } 3019 3020 static void 3021 hxge_rxdma_hw_stop(p_hxge_t hxgep) 3022 { 3023 int i, ndmas; 3024 uint16_t channel; 3025 p_rx_rbr_rings_t rx_rbr_rings; 3026 p_rx_rbr_ring_t *rbr_rings; 3027 p_rx_rcr_rings_t rx_rcr_rings; 3028 3029 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop")); 3030 3031 rx_rbr_rings = hxgep->rx_rbr_rings; 3032 rx_rcr_rings = hxgep->rx_rcr_rings; 3033 3034 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 3035 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3036 "<== hxge_rxdma_hw_stop: NULL ring pointers")); 3037 return; 3038 } 3039 3040 ndmas = rx_rbr_rings->ndmas; 3041 if (!ndmas) { 3042 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3043 "<== hxge_rxdma_hw_stop: no dma channel allocated")); 3044 return; 3045 } 3046 3047 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3048 "==> hxge_rxdma_hw_stop (ndmas %d)", ndmas)); 3049 3050 rbr_rings = rx_rbr_rings->rbr_rings; 3051 for (i = 0; i < ndmas; i++) { 3052 channel = rbr_rings[i]->rdc; 3053 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3054 "==> hxge_rxdma_hw_stop (ndmas %d) channel %d", 3055 ndmas, channel)); 3056 (void) hxge_rxdma_stop_channel(hxgep, channel); 3057 } 3058 3059 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop: " 3060 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3061 rx_rbr_rings, rx_rcr_rings)); 3062 3063 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_hw_stop")); 3064 } 3065 3066 static hxge_status_t 3067 hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 3068 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p) 3069 { 3070 hpi_handle_t handle; 3071 hpi_status_t rs = HPI_SUCCESS; 3072 rdc_stat_t cs; 3073 rdc_int_mask_t ent_mask; 3074 hxge_status_t status = HXGE_OK; 3075 3076 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel")); 3077 3078 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3079 3080 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "hxge_rxdma_start_channel: " 3081 "hpi handle addr $%p acc $%p", 3082 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3083 3084 /* Reset RXDMA channel */ 3085 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3086 if (rs != HPI_SUCCESS) { 3087 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3088 "==> hxge_rxdma_start_channel: " 3089 "reset rxdma failed (0x%08x channel %d)", 3090 status, channel)); 3091 return (HXGE_ERROR | rs); 3092 } 3093 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3094 "==> hxge_rxdma_start_channel: reset done: channel %d", channel)); 3095 3096 /* 3097 * Initialize the RXDMA channel specific FZC control configurations. 3098 * These FZC registers are pertaining to each RX channel (logical 3099 * pages). 3100 */ 3101 status = hxge_init_fzc_rxdma_channel(hxgep, 3102 channel, rbr_p, rcr_p, mbox_p); 3103 if (status != HXGE_OK) { 3104 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3105 "==> hxge_rxdma_start_channel: " 3106 "init fzc rxdma failed (0x%08x channel %d)", 3107 status, channel)); 3108 return (status); 3109 } 3110 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3111 "==> hxge_rxdma_start_channel: fzc done")); 3112 3113 /* 3114 * Zero out the shadow and prefetch ram. 3115 */ 3116 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3117 "==> hxge_rxdma_start_channel: ram done")); 3118 3119 /* Set up the interrupt event masks. */ 3120 ent_mask.value = 0; 3121 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3122 if (rs != HPI_SUCCESS) { 3123 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3124 "==> hxge_rxdma_start_channel: " 3125 "init rxdma event masks failed (0x%08x channel %d)", 3126 status, channel)); 3127 return (HXGE_ERROR | rs); 3128 } 3129 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3130 "event done: channel %d (mask 0x%016llx)", 3131 channel, ent_mask.value)); 3132 3133 /* 3134 * Load RXDMA descriptors, buffers, mailbox, initialise the receive DMA 3135 * channels and enable each DMA channel. 3136 */ 3137 status = hxge_enable_rxdma_channel(hxgep, 3138 channel, rbr_p, rcr_p, mbox_p); 3139 if (status != HXGE_OK) { 3140 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3141 " hxge_rxdma_start_channel: " 3142 " init enable rxdma failed (0x%08x channel %d)", 3143 status, channel)); 3144 return (status); 3145 } 3146 3147 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3148 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3149 3150 /* 3151 * Initialize the receive DMA control and status register 3152 * Note that rdc_stat HAS to be set after RBR and RCR rings are set 3153 */ 3154 cs.value = 0; 3155 cs.bits.mex = 1; 3156 cs.bits.rcr_thres = 1; 3157 cs.bits.rcr_to = 1; 3158 cs.bits.rbr_empty = 1; 3159 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3160 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3161 "channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value)); 3162 if (status != HXGE_OK) { 3163 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3164 "==> hxge_rxdma_start_channel: " 3165 "init rxdma control register failed (0x%08x channel %d", 3166 status, channel)); 3167 return (status); 3168 } 3169 3170 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3171 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3172 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3173 "==> hxge_rxdma_start_channel: enable done")); 3174 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_start_channel")); 3175 3176 return (HXGE_OK); 3177 } 3178 3179 static hxge_status_t 3180 hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel) 3181 { 3182 hpi_handle_t handle; 3183 hpi_status_t rs = HPI_SUCCESS; 3184 rdc_stat_t cs; 3185 rdc_int_mask_t ent_mask; 3186 hxge_status_t status = HXGE_OK; 3187 3188 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel")); 3189 3190 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3191 3192 HXGE_DEBUG_MSG((hxgep, RX_CTL, "hxge_rxdma_stop_channel: " 3193 "hpi handle addr $%p acc $%p", 3194 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3195 3196 /* Reset RXDMA channel */ 3197 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3198 if (rs != HPI_SUCCESS) { 3199 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3200 " hxge_rxdma_stop_channel: " 3201 " reset rxdma failed (0x%08x channel %d)", 3202 rs, channel)); 3203 return (HXGE_ERROR | rs); 3204 } 3205 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3206 "==> hxge_rxdma_stop_channel: reset done")); 3207 3208 /* Set up the interrupt event masks. */ 3209 ent_mask.value = RDC_INT_MASK_ALL; 3210 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3211 if (rs != HPI_SUCCESS) { 3212 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3213 "==> hxge_rxdma_stop_channel: " 3214 "set rxdma event masks failed (0x%08x channel %d)", 3215 rs, channel)); 3216 return (HXGE_ERROR | rs); 3217 } 3218 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3219 "==> hxge_rxdma_stop_channel: event done")); 3220 3221 /* Initialize the receive DMA control and status register */ 3222 cs.value = 0; 3223 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3224 3225 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel: control " 3226 " to default (all 0s) 0x%08x", cs.value)); 3227 3228 if (status != HXGE_OK) { 3229 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3230 " hxge_rxdma_stop_channel: init rxdma" 3231 " control register failed (0x%08x channel %d", 3232 status, channel)); 3233 return (status); 3234 } 3235 3236 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3237 "==> hxge_rxdma_stop_channel: control done")); 3238 3239 /* disable dma channel */ 3240 status = hxge_disable_rxdma_channel(hxgep, channel); 3241 3242 if (status != HXGE_OK) { 3243 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3244 " hxge_rxdma_stop_channel: " 3245 " init enable rxdma failed (0x%08x channel %d)", 3246 status, channel)); 3247 return (status); 3248 } 3249 3250 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3251 "==> hxge_rxdma_stop_channel: disable done")); 3252 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_channel")); 3253 3254 return (HXGE_OK); 3255 } 3256 3257 hxge_status_t 3258 hxge_rxdma_handle_sys_errors(p_hxge_t hxgep) 3259 { 3260 hpi_handle_t handle; 3261 p_hxge_rdc_sys_stats_t statsp; 3262 rdc_fifo_err_stat_t stat; 3263 hxge_status_t status = HXGE_OK; 3264 3265 handle = hxgep->hpi_handle; 3266 statsp = (p_hxge_rdc_sys_stats_t)&hxgep->statsp->rdc_sys_stats; 3267 3268 /* Clear the int_dbg register in case it is an injected err */ 3269 HXGE_REG_WR64(handle, RDC_FIFO_ERR_INT_DBG, 0x0); 3270 3271 /* Get the error status and clear the register */ 3272 HXGE_REG_RD64(handle, RDC_FIFO_ERR_STAT, &stat.value); 3273 HXGE_REG_WR64(handle, RDC_FIFO_ERR_STAT, stat.value); 3274 3275 if (stat.bits.rx_ctrl_fifo_sec) { 3276 statsp->ctrl_fifo_sec++; 3277 if (statsp->ctrl_fifo_sec == 1) 3278 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3279 "==> hxge_rxdma_handle_sys_errors: " 3280 "rx_ctrl_fifo_sec")); 3281 } 3282 3283 if (stat.bits.rx_ctrl_fifo_ded) { 3284 /* Global fatal error encountered */ 3285 statsp->ctrl_fifo_ded++; 3286 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3287 HXGE_FM_EREPORT_RDMC_CTRL_FIFO_DED); 3288 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3289 "==> hxge_rxdma_handle_sys_errors: " 3290 "fatal error: rx_ctrl_fifo_ded error")); 3291 } 3292 3293 if (stat.bits.rx_data_fifo_sec) { 3294 statsp->data_fifo_sec++; 3295 if (statsp->data_fifo_sec == 1) 3296 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3297 "==> hxge_rxdma_handle_sys_errors: " 3298 "rx_data_fifo_sec")); 3299 } 3300 3301 if (stat.bits.rx_data_fifo_ded) { 3302 /* Global fatal error encountered */ 3303 statsp->data_fifo_ded++; 3304 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3305 HXGE_FM_EREPORT_RDMC_DATA_FIFO_DED); 3306 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3307 "==> hxge_rxdma_handle_sys_errors: " 3308 "fatal error: rx_data_fifo_ded error")); 3309 } 3310 3311 if (stat.bits.rx_ctrl_fifo_ded || stat.bits.rx_data_fifo_ded) { 3312 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3313 " hxge_rxdma_handle_sys_errors: fatal error\n")); 3314 status = hxge_rx_port_fatal_err_recover(hxgep); 3315 if (status == HXGE_OK) { 3316 FM_SERVICE_RESTORED(hxgep); 3317 } 3318 } 3319 3320 return (HXGE_OK); 3321 } 3322 3323 static hxge_status_t 3324 hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel) 3325 { 3326 hpi_handle_t handle; 3327 hpi_status_t rs = HPI_SUCCESS; 3328 hxge_status_t status = HXGE_OK; 3329 p_rx_rbr_ring_t rbrp; 3330 p_rx_rcr_ring_t rcrp; 3331 p_rx_mbox_t mboxp; 3332 rdc_int_mask_t ent_mask; 3333 p_hxge_dma_common_t dmap; 3334 int ring_idx; 3335 uint32_t ref_cnt; 3336 p_rx_msg_t rx_msg_p; 3337 int i; 3338 uint32_t hxge_port_rcr_size; 3339 uint64_t tmp; 3340 3341 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_fatal_err_recover")); 3342 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3343 "Recovering from RxDMAChannel#%d error...", channel)); 3344 3345 /* 3346 * Stop the dma channel waits for the stop done. If the stop done bit 3347 * is not set, then create an error. 3348 */ 3349 3350 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3351 3352 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Rx DMA stop...")); 3353 3354 ring_idx = hxge_rxdma_get_ring_index(hxgep, channel); 3355 rbrp = (p_rx_rbr_ring_t)hxgep->rx_rbr_rings->rbr_rings[ring_idx]; 3356 rcrp = (p_rx_rcr_ring_t)hxgep->rx_rcr_rings->rcr_rings[ring_idx]; 3357 3358 MUTEX_ENTER(&rcrp->lock); 3359 MUTEX_ENTER(&rbrp->lock); 3360 MUTEX_ENTER(&rbrp->post_lock); 3361 3362 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA channel...")); 3363 3364 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 3365 if (rs != HPI_SUCCESS) { 3366 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3367 "hxge_disable_rxdma_channel:failed")); 3368 goto fail; 3369 } 3370 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA interrupt...")); 3371 3372 /* Disable interrupt */ 3373 ent_mask.value = RDC_INT_MASK_ALL; 3374 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3375 if (rs != HPI_SUCCESS) { 3376 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3377 "Set rxdma event masks failed (channel %d)", channel)); 3378 } 3379 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel reset...")); 3380 3381 /* Reset RXDMA channel */ 3382 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3383 if (rs != HPI_SUCCESS) { 3384 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3385 "Reset rxdma failed (channel %d)", channel)); 3386 goto fail; 3387 } 3388 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 3389 mboxp = (p_rx_mbox_t)hxgep->rx_mbox_areas_p->rxmbox_areas[ring_idx]; 3390 3391 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 3392 rbrp->rbr_rd_index = 0; 3393 rbrp->pages_to_post = 0; 3394 3395 rcrp->comp_rd_index = 0; 3396 rcrp->comp_wt_index = 0; 3397 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 3398 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 3399 #if defined(__i386) 3400 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3401 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3402 #else 3403 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3404 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3405 #endif 3406 3407 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 3408 (hxge_port_rcr_size - 1); 3409 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 3410 (hxge_port_rcr_size - 1); 3411 3412 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 3413 bzero((caddr_t)dmap->kaddrp, dmap->alength); 3414 3415 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rbr entries = %d\n", 3416 rbrp->rbr_max_size)); 3417 3418 for (i = 0; i < rbrp->rbr_max_size; i++) { 3419 /* Reset all the buffers */ 3420 rx_msg_p = rbrp->rx_msg_ring[i]; 3421 ref_cnt = rx_msg_p->ref_cnt; 3422 3423 rx_msg_p->ref_cnt = 1; 3424 rx_msg_p->free = B_TRUE; 3425 rx_msg_p->cur_usage_cnt = 0; 3426 rx_msg_p->max_usage_cnt = 0; 3427 rx_msg_p->pkt_buf_size = 0; 3428 3429 if (ref_cnt > 1) 3430 atomic_add_32(&hxge_mblks_pending, 1 - ref_cnt); 3431 } 3432 3433 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel re-start...")); 3434 3435 status = hxge_rxdma_start_channel(hxgep, channel, rbrp, rcrp, mboxp); 3436 if (status != HXGE_OK) { 3437 goto fail; 3438 } 3439 3440 /* 3441 * The DMA channel may disable itself automatically. 3442 * The following is a work-around. 3443 */ 3444 HXGE_REG_RD64(handle, RDC_RX_CFG1, &tmp); 3445 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 3446 if (rs != HPI_SUCCESS) { 3447 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3448 "hpi_rxdma_cfg_rdc_enable (channel %d)", channel)); 3449 } 3450 3451 MUTEX_EXIT(&rbrp->post_lock); 3452 MUTEX_EXIT(&rbrp->lock); 3453 MUTEX_EXIT(&rcrp->lock); 3454 3455 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3456 "Recovery Successful, RxDMAChannel#%d Restored", channel)); 3457 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_fatal_err_recover")); 3458 3459 return (HXGE_OK); 3460 3461 fail: 3462 MUTEX_EXIT(&rbrp->post_lock); 3463 MUTEX_EXIT(&rbrp->lock); 3464 MUTEX_EXIT(&rcrp->lock); 3465 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3466 3467 return (HXGE_ERROR | rs); 3468 } 3469 3470 static hxge_status_t 3471 hxge_rx_port_fatal_err_recover(p_hxge_t hxgep) 3472 { 3473 hxge_status_t status = HXGE_OK; 3474 p_hxge_dma_common_t *dma_buf_p; 3475 uint16_t channel; 3476 int ndmas; 3477 int i; 3478 block_reset_t reset_reg; 3479 3480 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_port_fatal_err_recover")); 3481 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovering from RDC error ...")); 3482 3483 /* Reset RDC block from PEU for this fatal error */ 3484 reset_reg.value = 0; 3485 reset_reg.bits.rdc_rst = 1; 3486 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 3487 3488 /* Disable RxMAC */ 3489 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxMAC...\n")); 3490 if (hxge_rx_vmac_disable(hxgep) != HXGE_OK) 3491 goto fail; 3492 3493 HXGE_DELAY(1000); 3494 3495 /* Restore any common settings after PEU reset */ 3496 if (hxge_rxdma_hw_start_common(hxgep) != HXGE_OK) 3497 goto fail; 3498 3499 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Stop all RxDMA channels...")); 3500 3501 ndmas = hxgep->rx_buf_pool_p->ndmas; 3502 dma_buf_p = hxgep->rx_buf_pool_p->dma_buf_pool_p; 3503 3504 for (i = 0; i < ndmas; i++) { 3505 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 3506 if (hxge_rxdma_fatal_err_recover(hxgep, channel) != HXGE_OK) { 3507 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3508 "Could not recover channel %d", channel)); 3509 } 3510 } 3511 3512 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Reset RxMAC...")); 3513 3514 /* Reset RxMAC */ 3515 if (hxge_rx_vmac_reset(hxgep) != HXGE_OK) { 3516 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3517 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3518 goto fail; 3519 } 3520 3521 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-initialize RxMAC...")); 3522 3523 /* Re-Initialize RxMAC */ 3524 if ((status = hxge_rx_vmac_init(hxgep)) != HXGE_OK) { 3525 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3526 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3527 goto fail; 3528 } 3529 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-enable RxMAC...")); 3530 3531 /* Re-enable RxMAC */ 3532 if ((status = hxge_rx_vmac_enable(hxgep)) != HXGE_OK) { 3533 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3534 "hxge_rx_port_fatal_err_recover: Failed to enable RxMAC")); 3535 goto fail; 3536 } 3537 3538 /* Reset the error mask since PEU reset cleared it */ 3539 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 3540 3541 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3542 "Recovery Successful, RxPort Restored")); 3543 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_port_fatal_err_recover")); 3544 3545 return (HXGE_OK); 3546 fail: 3547 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3548 return (status); 3549 } 3550