1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* Copyright © 2003-2011 Emulex. All rights reserved. */ 23 24 /* 25 * Source file containing the implementation of the Transmit 26 * Path 27 */ 28 29 #include <oce_impl.h> 30 31 static void oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed); 32 static int oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, 33 mblk_t *mp, uint32_t pkt_len); 34 static int oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 35 uint32_t pkt_len); 36 static void oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 37 static int oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, 38 size_t size, int flags); 39 static inline oce_wq_bdesc_t *oce_wqb_alloc(struct oce_wq *wq); 40 static void oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 41 42 static void oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 43 static void oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 44 static oce_wq_mdesc_t *oce_wqm_alloc(struct oce_wq *wq); 45 static int oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq); 46 static void oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 47 static void oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed); 48 static void oce_remove_vtag(mblk_t *mp); 49 static void oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag); 50 static inline int oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm); 51 52 53 static ddi_dma_attr_t tx_map_dma_attr = { 54 DMA_ATTR_V0, /* version number */ 55 0x0000000000000000ull, /* low address */ 56 0xFFFFFFFFFFFFFFFFull, /* high address */ 57 0x0000000000010000ull, /* dma counter max */ 58 OCE_TXMAP_ALIGN, /* alignment */ 59 0x7FF, /* burst sizes */ 60 0x00000001, /* minimum transfer size */ 61 0x00000000FFFFFFFFull, /* maximum transfer size */ 62 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */ 63 OCE_MAX_TXDMA_COOKIES, /* scatter/gather list length */ 64 0x00000001, /* granularity */ 65 DDI_DMA_FLAGERR /* dma_attr_flags */ 66 }; 67 68 69 ddi_dma_attr_t oce_tx_dma_buf_attr = { 70 DMA_ATTR_V0, /* version number */ 71 0x0000000000000000ull, /* low address */ 72 0xFFFFFFFFFFFFFFFFull, /* high address */ 73 0x00000000FFFFFFFFull, /* dma counter max */ 74 OCE_DMA_ALIGNMENT, /* alignment */ 75 0x000007FF, /* burst sizes */ 76 0x00000001, /* minimum transfer size */ 77 0x00000000FFFFFFFFull, /* maximum transfer size */ 78 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */ 79 1, /* scatter/gather list length */ 80 0x00000001, /* granularity */ 81 DDI_DMA_FLAGERR /* dma_attr_flags */ 82 }; 83 84 /* 85 * WQ map handle destructor 86 * 87 * wq - Pointer to WQ structure 88 * wqmd - pointer to WQE mapping handle descriptor 89 * 90 * return none 91 */ 92 93 static void 94 oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 95 { 96 _NOTE(ARGUNUSED(wq)); 97 /* Free the DMA handle */ 98 if (wqmd->dma_handle != NULL) 99 (void) ddi_dma_free_handle(&(wqmd->dma_handle)); 100 wqmd->dma_handle = NULL; 101 } /* oce_wqm_dtor */ 102 103 /* 104 * WQ map handles contructor 105 * 106 * wqmd - pointer to WQE mapping handle descriptor 107 * wq - Pointer to WQ structure 108 * 109 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 110 */ 111 static int 112 oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq) 113 { 114 struct oce_dev *dev; 115 int ret; 116 117 dev = wq->parent; 118 /* Allocate DMA handle */ 119 ret = ddi_dma_alloc_handle(dev->dip, &tx_map_dma_attr, 120 DDI_DMA_DONTWAIT, NULL, &wqmd->dma_handle); 121 122 return (ret); 123 } /* oce_wqm_ctor */ 124 125 /* 126 * function to create WQ mapping handles cache 127 * 128 * wq - pointer to WQ structure 129 * 130 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 131 */ 132 int 133 oce_wqm_cache_create(struct oce_wq *wq) 134 { 135 struct oce_dev *dev = wq->parent; 136 int size; 137 int cnt; 138 int ret; 139 140 size = wq->cfg.nhdl * sizeof (oce_wq_mdesc_t); 141 wq->wq_mdesc_array = kmem_zalloc(size, KM_NOSLEEP); 142 if (wq->wq_mdesc_array == NULL) { 143 return (DDI_FAILURE); 144 } 145 146 /* Create the free buffer list */ 147 OCE_LIST_CREATE(&wq->wq_mdesc_list, DDI_INTR_PRI(dev->intr_pri)); 148 149 for (cnt = 0; cnt < wq->cfg.nhdl; cnt++) { 150 ret = oce_wqm_ctor(&wq->wq_mdesc_array[cnt], wq); 151 if (ret != DDI_SUCCESS) { 152 goto wqm_fail; 153 } 154 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, 155 &wq->wq_mdesc_array[cnt]); 156 } 157 return (DDI_SUCCESS); 158 159 wqm_fail: 160 oce_wqm_cache_destroy(wq); 161 return (DDI_FAILURE); 162 } 163 164 /* 165 * function to destroy WQ mapping handles cache 166 * 167 * wq - pointer to WQ structure 168 * 169 * return none 170 */ 171 void 172 oce_wqm_cache_destroy(struct oce_wq *wq) 173 { 174 oce_wq_mdesc_t *wqmd; 175 176 while ((wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list)) != NULL) { 177 oce_wqm_dtor(wq, wqmd); 178 } 179 180 kmem_free(wq->wq_mdesc_array, 181 wq->cfg.nhdl * sizeof (oce_wq_mdesc_t)); 182 183 OCE_LIST_DESTROY(&wq->wq_mdesc_list); 184 } 185 186 /* 187 * function to create WQ buffer cache 188 * 189 * wq - pointer to WQ structure 190 * buf_size - size of the buffer 191 * 192 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 193 */ 194 int 195 oce_wqb_cache_create(struct oce_wq *wq, size_t buf_size) 196 { 197 struct oce_dev *dev = wq->parent; 198 int size; 199 int cnt; 200 int ret; 201 202 size = wq->cfg.nbufs * sizeof (oce_wq_bdesc_t); 203 wq->wq_bdesc_array = kmem_zalloc(size, KM_NOSLEEP); 204 if (wq->wq_bdesc_array == NULL) { 205 return (DDI_FAILURE); 206 } 207 208 /* Create the free buffer list */ 209 OCE_LIST_CREATE(&wq->wq_buf_list, DDI_INTR_PRI(dev->intr_pri)); 210 211 for (cnt = 0; cnt < wq->cfg.nbufs; cnt++) { 212 ret = oce_wqb_ctor(&wq->wq_bdesc_array[cnt], 213 wq, buf_size, DDI_DMA_STREAMING); 214 if (ret != DDI_SUCCESS) { 215 goto wqb_fail; 216 } 217 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, 218 &wq->wq_bdesc_array[cnt]); 219 } 220 return (DDI_SUCCESS); 221 222 wqb_fail: 223 oce_wqb_cache_destroy(wq); 224 return (DDI_FAILURE); 225 } 226 227 /* 228 * function to destroy WQ buffer cache 229 * 230 * wq - pointer to WQ structure 231 * 232 * return none 233 */ 234 void 235 oce_wqb_cache_destroy(struct oce_wq *wq) 236 { 237 oce_wq_bdesc_t *wqbd; 238 while ((wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list)) != NULL) { 239 oce_wqb_dtor(wq, wqbd); 240 } 241 kmem_free(wq->wq_bdesc_array, 242 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t)); 243 OCE_LIST_DESTROY(&wq->wq_buf_list); 244 } 245 246 /* 247 * WQ buffer constructor 248 * 249 * wqbd - pointer to WQ buffer descriptor 250 * wq - pointer to WQ structure 251 * size - size of the buffer 252 * flags - KM_SLEEP or KM_NOSLEEP 253 * 254 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 255 */ 256 static int 257 oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, size_t size, int flags) 258 { 259 struct oce_dev *dev; 260 dev = wq->parent; 261 262 wqbd->wqb = oce_alloc_dma_buffer(dev, size, &oce_tx_dma_buf_attr, 263 flags); 264 if (wqbd->wqb == NULL) { 265 return (DDI_FAILURE); 266 } 267 wqbd->frag_addr.dw.addr_lo = ADDR_LO(wqbd->wqb->addr); 268 wqbd->frag_addr.dw.addr_hi = ADDR_HI(wqbd->wqb->addr); 269 return (DDI_SUCCESS); 270 } 271 272 /* 273 * WQ buffer destructor 274 * 275 * wq - pointer to WQ structure 276 * wqbd - pointer to WQ buffer descriptor 277 * 278 * return none 279 */ 280 static void 281 oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 282 { 283 oce_free_dma_buffer(wq->parent, wqbd->wqb); 284 } 285 286 /* 287 * function to alloc WQE buffer descriptor 288 * 289 * wq - pointer to WQ structure 290 * 291 * return pointer to WQE buffer descriptor 292 */ 293 static inline oce_wq_bdesc_t * 294 oce_wqb_alloc(struct oce_wq *wq) 295 { 296 return (OCE_LIST_REM_HEAD(&wq->wq_buf_list)); 297 } 298 299 /* 300 * function to free WQE buffer descriptor 301 * 302 * wq - pointer to WQ structure 303 * wqbd - pointer to WQ buffer descriptor 304 * 305 * return none 306 */ 307 static inline void 308 oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 309 { 310 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, wqbd); 311 } /* oce_wqb_free */ 312 313 /* 314 * function to allocate WQE mapping descriptor 315 * 316 * wq - pointer to WQ structure 317 * 318 * return pointer to WQE mapping descriptor 319 */ 320 static inline oce_wq_mdesc_t * 321 oce_wqm_alloc(struct oce_wq *wq) 322 { 323 return (OCE_LIST_REM_HEAD(&wq->wq_mdesc_list)); 324 } /* oce_wqm_alloc */ 325 326 /* 327 * function to insert WQE mapping descriptor to the list 328 * 329 * wq - pointer to WQ structure 330 * wqmd - Pointer to WQ mapping descriptor 331 * 332 * return none 333 */ 334 static inline void 335 oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 336 { 337 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, wqmd); 338 } 339 340 /* 341 * function to free WQE mapping descriptor 342 * 343 * wq - pointer to WQ structure 344 * wqmd - Pointer to WQ mapping descriptor 345 * 346 * return none 347 */ 348 static void 349 oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 350 { 351 if (wqmd == NULL) { 352 return; 353 } 354 (void) ddi_dma_unbind_handle(wqmd->dma_handle); 355 oce_wqm_free(wq, wqmd); 356 } 357 358 /* 359 * WQED kmem_cache constructor 360 * 361 * buf - pointer to WQE descriptor 362 * 363 * return DDI_SUCCESS 364 */ 365 int 366 oce_wqe_desc_ctor(void *buf, void *arg, int kmflags) 367 { 368 _NOTE(ARGUNUSED(buf)); 369 _NOTE(ARGUNUSED(arg)); 370 _NOTE(ARGUNUSED(kmflags)); 371 372 return (DDI_SUCCESS); 373 } 374 375 /* 376 * WQED kmem_cache destructor 377 * 378 * buf - pointer to WQE descriptor 379 * 380 * return none 381 */ 382 void 383 oce_wqe_desc_dtor(void *buf, void *arg) 384 { 385 _NOTE(ARGUNUSED(buf)); 386 _NOTE(ARGUNUSED(arg)); 387 } 388 389 /* 390 * function to choose a WQ given a mblk depending on priority, flowID etc. 391 * 392 * dev - software handle to device 393 * mp - the mblk to send 394 * 395 * return pointer to the WQ selected 396 */ 397 static uint8_t oce_tx_hash_policy = 0x4; 398 struct oce_wq * 399 oce_get_wq(struct oce_dev *dev, mblk_t *mp) 400 { 401 struct oce_wq *wq; 402 int qidx = 0; 403 if (dev->nwqs > 1) { 404 qidx = mac_pkt_hash(DL_ETHER, mp, oce_tx_hash_policy, B_TRUE); 405 qidx = qidx % dev->nwqs; 406 407 } else { 408 qidx = 0; 409 } 410 wq = dev->wq[qidx]; 411 /* for the time being hardcode */ 412 return (wq); 413 } /* oce_get_wq */ 414 415 /* 416 * function to populate the single WQE 417 * 418 * wq - pointer to wq 419 * wqed - pointer to WQ entry descriptor 420 * 421 * return none 422 */ 423 #pragma inline(oce_fill_ring_descs) 424 static void 425 oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed) 426 { 427 428 struct oce_nic_frag_wqe *wqe; 429 int i; 430 /* Copy the precreate WQE descs to the ring desc */ 431 for (i = 0; i < wqed->wqe_cnt; i++) { 432 wqe = RING_GET_PRODUCER_ITEM_VA(wq->ring, 433 struct oce_nic_frag_wqe); 434 435 bcopy(&wqed->frag[i], wqe, NIC_WQE_SIZE); 436 RING_PUT(wq->ring, 1); 437 } 438 } /* oce_fill_ring_descs */ 439 440 /* 441 * function to copy the packet to preallocated Tx buffer 442 * 443 * wq - pointer to WQ 444 * wqed - Pointer to WQE descriptor 445 * mp - Pointer to packet chain 446 * pktlen - Size of the packet 447 * 448 * return 0=>success, error code otherwise 449 */ 450 static int 451 oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 452 uint32_t pkt_len) 453 { 454 oce_wq_bdesc_t *wqbd; 455 caddr_t buf_va; 456 struct oce_dev *dev = wq->parent; 457 int len = 0; 458 459 wqbd = oce_wqb_alloc(wq); 460 if (wqbd == NULL) { 461 atomic_inc_32(&dev->tx_noxmtbuf); 462 oce_log(dev, CE_WARN, MOD_TX, "%s", 463 "wqb pool empty"); 464 return (ENOMEM); 465 } 466 467 /* create a fragment wqe for the packet */ 468 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi = wqbd->frag_addr.dw.addr_hi; 469 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo = wqbd->frag_addr.dw.addr_lo; 470 buf_va = DBUF_VA(wqbd->wqb); 471 472 /* copy pkt into buffer */ 473 for (len = 0; mp != NULL && len < pkt_len; mp = mp->b_cont) { 474 bcopy(mp->b_rptr, buf_va, MBLKL(mp)); 475 buf_va += MBLKL(mp); 476 len += MBLKL(mp); 477 } 478 479 (void) ddi_dma_sync(DBUF_DHDL(wqbd->wqb), 0, pkt_len, 480 DDI_DMA_SYNC_FORDEV); 481 482 if (oce_fm_check_dma_handle(dev, DBUF_DHDL(wqbd->wqb))) { 483 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED); 484 /* Free the buffer */ 485 oce_wqb_free(wq, wqbd); 486 return (EIO); 487 } 488 wqed->frag[wqed->frag_idx].u0.s.frag_len = pkt_len; 489 wqed->hdesc[wqed->nhdl].hdl = (void *)(wqbd); 490 wqed->hdesc[wqed->nhdl].type = COPY_WQE; 491 wqed->frag_cnt++; 492 wqed->frag_idx++; 493 wqed->nhdl++; 494 return (0); 495 } /* oce_bcopy_wqe */ 496 497 /* 498 * function to copy the packet or dma map on the fly depending on size 499 * 500 * wq - pointer to WQ 501 * wqed - Pointer to WQE descriptor 502 * mp - Pointer to packet chain 503 * 504 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 505 */ 506 static int 507 oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 508 uint32_t pkt_len) 509 { 510 ddi_dma_cookie_t cookie; 511 oce_wq_mdesc_t *wqmd; 512 uint32_t ncookies; 513 int ret; 514 struct oce_dev *dev = wq->parent; 515 516 wqmd = oce_wqm_alloc(wq); 517 if (wqmd == NULL) { 518 oce_log(dev, CE_WARN, MOD_TX, "%s", 519 "wqm pool empty"); 520 return (ENOMEM); 521 } 522 523 ret = ddi_dma_addr_bind_handle(wqmd->dma_handle, 524 (struct as *)0, (caddr_t)mp->b_rptr, 525 pkt_len, DDI_DMA_WRITE | DDI_DMA_STREAMING, 526 DDI_DMA_DONTWAIT, NULL, &cookie, &ncookies); 527 if (ret != DDI_DMA_MAPPED) { 528 oce_log(dev, CE_WARN, MOD_TX, "MAP FAILED %d", 529 ret); 530 /* free the last one */ 531 oce_wqm_free(wq, wqmd); 532 return (ENOMEM); 533 } 534 do { 535 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi = 536 ADDR_HI(cookie.dmac_laddress); 537 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo = 538 ADDR_LO(cookie.dmac_laddress); 539 wqed->frag[wqed->frag_idx].u0.s.frag_len = 540 (uint32_t)cookie.dmac_size; 541 wqed->frag_cnt++; 542 wqed->frag_idx++; 543 if (--ncookies > 0) 544 ddi_dma_nextcookie(wqmd->dma_handle, 545 &cookie); 546 else break; 547 } while (ncookies > 0); 548 549 wqed->hdesc[wqed->nhdl].hdl = (void *)wqmd; 550 wqed->hdesc[wqed->nhdl].type = MAPPED_WQE; 551 wqed->nhdl++; 552 return (0); 553 } /* oce_map_wqe */ 554 555 static inline int 556 oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm) 557 { 558 struct oce_nic_tx_cqe *cqe; 559 uint16_t num_cqe = 0; 560 struct oce_cq *cq; 561 oce_wqe_desc_t *wqed; 562 int wqe_freed = 0; 563 struct oce_dev *dev; 564 565 cq = wq->cq; 566 dev = wq->parent; 567 (void) ddi_dma_sync(cq->ring->dbuf->dma_handle, 0, 0, 568 DDI_DMA_SYNC_FORKERNEL); 569 570 mutex_enter(&wq->txc_lock); 571 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 572 while (WQ_CQE_VALID(cqe)) { 573 574 DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_tx_cqe)); 575 576 /* update stats */ 577 if (cqe->u0.s.status != 0) { 578 atomic_inc_32(&dev->tx_errors); 579 } 580 581 /* complete the WQEs */ 582 wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list); 583 584 wqe_freed = wqed->wqe_cnt; 585 oce_free_wqed(wq, wqed); 586 RING_GET(wq->ring, wqe_freed); 587 atomic_add_32(&wq->wq_free, wqe_freed); 588 /* clear the valid bit and progress cqe */ 589 WQ_CQE_INVALIDATE(cqe); 590 RING_GET(cq->ring, 1); 591 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, 592 struct oce_nic_tx_cqe); 593 num_cqe++; 594 } /* for all valid CQE */ 595 mutex_exit(&wq->txc_lock); 596 if (num_cqe) 597 oce_arm_cq(wq->parent, cq->cq_id, num_cqe, rearm); 598 return (num_cqe); 599 } /* oce_process_tx_completion */ 600 601 /* 602 * function to drain a TxCQ and process its CQEs 603 * 604 * dev - software handle to the device 605 * cq - pointer to the cq to drain 606 * 607 * return the number of CQEs processed 608 */ 609 uint16_t 610 oce_drain_wq_cq(void *arg) 611 { 612 uint16_t num_cqe = 0; 613 struct oce_dev *dev; 614 struct oce_wq *wq; 615 616 wq = (struct oce_wq *)arg; 617 dev = wq->parent; 618 619 /* do while we do not reach a cqe that is not valid */ 620 num_cqe = oce_process_tx_compl(wq, B_FALSE); 621 622 /* check if we need to restart Tx */ 623 if (wq->resched && num_cqe) { 624 wq->resched = B_FALSE; 625 mac_tx_update(dev->mac_handle); 626 } 627 628 return (num_cqe); 629 } /* oce_process_wq_cqe */ 630 631 /* 632 * function to insert vtag to packet 633 * 634 * mp - mblk pointer 635 * vlan_tag - tag to be inserted 636 * 637 * return none 638 */ 639 static inline void 640 oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag) 641 { 642 struct ether_vlan_header *evh; 643 (void) memmove(mp->b_rptr - VTAG_SIZE, 644 mp->b_rptr, 2 * ETHERADDRL); 645 mp->b_rptr -= VTAG_SIZE; 646 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 647 evh->ether_tpid = htons(VLAN_TPID); 648 evh->ether_tci = htons(vlan_tag); 649 } 650 651 /* 652 * function to strip vtag from packet 653 * 654 * mp - mblk pointer 655 * 656 * return none 657 */ 658 659 static inline void 660 oce_remove_vtag(mblk_t *mp) 661 { 662 (void) memmove(mp->b_rptr + VTAG_SIZE, mp->b_rptr, 663 ETHERADDRL * 2); 664 mp->b_rptr += VTAG_SIZE; 665 } 666 667 /* 668 * function to xmit Single packet over the wire 669 * 670 * wq - pointer to WQ 671 * mp - Pointer to packet chain 672 * 673 * return pointer to the packet 674 */ 675 mblk_t * 676 oce_send_packet(struct oce_wq *wq, mblk_t *mp) 677 { 678 struct oce_nic_hdr_wqe *wqeh; 679 struct oce_dev *dev; 680 struct ether_header *eh; 681 struct ether_vlan_header *evh; 682 int32_t num_wqes; 683 uint16_t etype; 684 uint32_t ip_offset; 685 uint32_t csum_flags = 0; 686 boolean_t use_copy = B_FALSE; 687 boolean_t tagged = B_FALSE; 688 uint16_t vlan_tag; 689 uint32_t reg_value = 0; 690 oce_wqe_desc_t *wqed = NULL; 691 mblk_t *nmp = NULL; 692 mblk_t *tmp = NULL; 693 uint32_t pkt_len = 0; 694 int num_mblks = 0; 695 int ret = 0; 696 uint32_t mss = 0; 697 uint32_t flags = 0; 698 int len = 0; 699 700 /* retrieve the adap priv struct ptr */ 701 dev = wq->parent; 702 703 /* check if we have enough free slots */ 704 if (wq->wq_free < dev->tx_reclaim_threshold) { 705 (void) oce_process_tx_compl(wq, B_FALSE); 706 } 707 if (wq->wq_free < OCE_MAX_TX_HDL) { 708 return (mp); 709 } 710 711 /* check if we should copy */ 712 for (tmp = mp; tmp != NULL; tmp = tmp->b_cont) { 713 pkt_len += MBLKL(tmp); 714 num_mblks++; 715 } 716 717 if (pkt_len == 0 || num_mblks == 0) { 718 freemsg(mp); 719 return (NULL); 720 } 721 722 /* retrieve LSO information */ 723 mac_lso_get(mp, &mss, &flags); 724 725 /* get the offload flags */ 726 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &csum_flags); 727 728 /* restrict the mapped segment to wat we support */ 729 if (num_mblks > OCE_MAX_TX_HDL) { 730 nmp = msgpullup(mp, -1); 731 if (nmp == NULL) { 732 atomic_inc_32(&wq->pkt_drops); 733 freemsg(mp); 734 return (NULL); 735 } 736 /* Reset it to new collapsed mp */ 737 freemsg(mp); 738 mp = nmp; 739 } 740 741 /* Get the packet descriptor for Tx */ 742 wqed = kmem_cache_alloc(wq->wqed_cache, KM_NOSLEEP); 743 if (wqed == NULL) { 744 atomic_inc_32(&wq->pkt_drops); 745 freemsg(mp); 746 return (NULL); 747 } 748 eh = (struct ether_header *)(void *)mp->b_rptr; 749 if (ntohs(eh->ether_type) == VLAN_TPID) { 750 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 751 tagged = B_TRUE; 752 etype = ntohs(evh->ether_type); 753 ip_offset = sizeof (struct ether_vlan_header); 754 pkt_len -= VTAG_SIZE; 755 vlan_tag = ntohs(evh->ether_tci); 756 oce_remove_vtag(mp); 757 } else { 758 etype = ntohs(eh->ether_type); 759 ip_offset = sizeof (struct ether_header); 760 } 761 762 /* Save the WQ pointer */ 763 wqed->wq = wq; 764 wqed->frag_idx = 1; /* index zero is always header */ 765 wqed->frag_cnt = 0; 766 wqed->nhdl = 0; 767 wqed->mp = NULL; 768 OCE_LIST_LINK_INIT(&wqed->link); 769 770 /* If entire packet is less than the copy limit just do copy */ 771 if (pkt_len < dev->tx_bcopy_limit) { 772 use_copy = B_TRUE; 773 ret = oce_bcopy_wqe(wq, wqed, mp, pkt_len); 774 } else { 775 /* copy or dma map the individual fragments */ 776 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) { 777 len = MBLKL(nmp); 778 if (len == 0) { 779 continue; 780 } 781 if (len < dev->tx_bcopy_limit) { 782 ret = oce_bcopy_wqe(wq, wqed, nmp, len); 783 } else { 784 ret = oce_map_wqe(wq, wqed, nmp, len); 785 } 786 if (ret != 0) 787 break; 788 } 789 } 790 791 /* 792 * Any failure other than insufficient Q entries 793 * drop the packet 794 */ 795 if (ret != 0) { 796 oce_free_wqed(wq, wqed); 797 atomic_inc_32(&wq->pkt_drops); 798 freemsg(mp); 799 return (NULL); 800 } 801 802 wqeh = (struct oce_nic_hdr_wqe *)&wqed->frag[0]; 803 bzero(wqeh, sizeof (struct oce_nic_hdr_wqe)); 804 805 /* fill rest of wqe header fields based on packet */ 806 if (flags & HW_LSO) { 807 wqeh->u0.s.lso = B_TRUE; 808 wqeh->u0.s.lso_mss = mss; 809 } 810 if (csum_flags & HCK_FULLCKSUM) { 811 uint8_t *proto; 812 if (etype == ETHERTYPE_IP) { 813 proto = (uint8_t *)(void *) 814 (mp->b_rptr + ip_offset); 815 if (proto[9] == 6) 816 /* IPPROTO_TCP */ 817 wqeh->u0.s.tcpcs = B_TRUE; 818 else if (proto[9] == 17) 819 /* IPPROTO_UDP */ 820 wqeh->u0.s.udpcs = B_TRUE; 821 } 822 } 823 824 if (csum_flags & HCK_IPV4_HDRCKSUM) 825 wqeh->u0.s.ipcs = B_TRUE; 826 if (tagged) { 827 wqeh->u0.s.vlan = B_TRUE; 828 wqeh->u0.s.vlan_tag = vlan_tag; 829 } 830 831 wqeh->u0.s.complete = B_TRUE; 832 wqeh->u0.s.event = B_TRUE; 833 wqeh->u0.s.crc = B_TRUE; 834 wqeh->u0.s.total_length = pkt_len; 835 836 num_wqes = wqed->frag_cnt + 1; 837 838 /* h/w expects even no. of WQEs */ 839 if (num_wqes & 0x1) { 840 bzero(&wqed->frag[num_wqes], sizeof (struct oce_nic_frag_wqe)); 841 num_wqes++; 842 } 843 wqed->wqe_cnt = (uint16_t)num_wqes; 844 wqeh->u0.s.num_wqe = num_wqes; 845 DW_SWAP(u32ptr(&wqed->frag[0]), (wqed->wqe_cnt * NIC_WQE_SIZE)); 846 847 mutex_enter(&wq->tx_lock); 848 if (num_wqes > wq->wq_free) { 849 atomic_inc_32(&wq->tx_deferd); 850 mutex_exit(&wq->tx_lock); 851 goto wqe_fail; 852 } 853 atomic_add_32(&wq->wq_free, -num_wqes); 854 855 /* fill the wq for adapter */ 856 oce_fill_ring_descs(wq, wqed); 857 858 /* Set the mp pointer in the wqe descriptor */ 859 if (use_copy == B_FALSE) { 860 wqed->mp = mp; 861 } 862 /* Add the packet desc to list to be retrieved during cmpl */ 863 OCE_LIST_INSERT_TAIL(&wq->wqe_desc_list, wqed); 864 (void) ddi_dma_sync(wq->ring->dbuf->dma_handle, 0, 0, 865 DDI_DMA_SYNC_FORDEV); 866 867 /* ring tx doorbell */ 868 reg_value = (num_wqes << 16) | wq->wq_id; 869 /* Ring the door bell */ 870 OCE_DB_WRITE32(dev, PD_TXULP_DB, reg_value); 871 mutex_exit(&wq->tx_lock); 872 if (oce_fm_check_acc_handle(dev, dev->db_handle) != DDI_FM_OK) { 873 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED); 874 } 875 876 /* free mp if copied or packet chain collapsed */ 877 if (use_copy == B_TRUE) { 878 freemsg(mp); 879 } 880 return (NULL); 881 882 wqe_fail: 883 884 if (tagged) { 885 oce_insert_vtag(mp, vlan_tag); 886 } 887 oce_free_wqed(wq, wqed); 888 return (mp); 889 } /* oce_send_packet */ 890 891 /* 892 * function to free the WQE descriptor 893 * 894 * wq - pointer to WQ 895 * wqed - Pointer to WQE descriptor 896 * 897 * return none 898 */ 899 #pragma inline(oce_free_wqed) 900 static void 901 oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed) 902 { 903 int i = 0; 904 if (wqed == NULL) { 905 return; 906 } 907 908 for (i = 0; i < wqed->nhdl; i++) { 909 if (wqed->hdesc[i].type == COPY_WQE) { 910 oce_wqb_free(wq, wqed->hdesc[i].hdl); 911 } else if (wqed->hdesc[i].type == MAPPED_WQE) { 912 oce_wqmd_free(wq, wqed->hdesc[i].hdl); 913 } 914 } 915 if (wqed->mp) 916 freemsg(wqed->mp); 917 kmem_cache_free(wq->wqed_cache, wqed); 918 } /* oce_free_wqed */ 919 920 /* 921 * function to start the WQ 922 * 923 * wq - pointer to WQ 924 * 925 * return DDI_SUCCESS 926 */ 927 928 int 929 oce_start_wq(struct oce_wq *wq) 930 { 931 _NOTE(ARGUNUSED(wq)); 932 return (DDI_SUCCESS); 933 } /* oce_start_wq */ 934 935 /* 936 * function to stop the WQ 937 * 938 * wq - pointer to WQ 939 * 940 * return none 941 */ 942 void 943 oce_clean_wq(struct oce_wq *wq) 944 { 945 oce_wqe_desc_t *wqed; 946 int ti; 947 948 /* Wait for already posted Tx to complete */ 949 950 for (ti = 0; ti < DEFAULT_DRAIN_TIME; ti++) { 951 (void) oce_process_tx_compl(wq, B_FALSE); 952 OCE_MSDELAY(1); 953 } 954 955 /* Free the remaining descriptors */ 956 while ((wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list)) != NULL) { 957 atomic_add_32(&wq->wq_free, wqed->wqe_cnt); 958 oce_free_wqed(wq, wqed); 959 } 960 oce_drain_eq(wq->cq->eq); 961 } /* oce_stop_wq */ 962 963 /* 964 * function to set the tx mapping handle fma attr 965 * 966 * fm_caps - capability flags 967 * 968 * return none 969 */ 970 971 void 972 oce_set_tx_map_dma_fma_flags(int fm_caps) 973 { 974 if (fm_caps == DDI_FM_NOT_CAPABLE) { 975 return; 976 } 977 978 if (DDI_FM_DMA_ERR_CAP(fm_caps)) { 979 tx_map_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 980 } else { 981 tx_map_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 982 } 983 } /* oce_set_tx_map_dma_fma_flags */ 984