1 /* 2 * Copyright (c) 2008-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/types.h> 32 #include <sys/sysmacros.h> 33 #include <sys/ddi.h> 34 #include <sys/sunddi.h> 35 #include <sys/atomic.h> 36 #include <sys/stream.h> 37 #include <sys/strsun.h> 38 #include <sys/strsubr.h> 39 #include <sys/strft.h> 40 #include <sys/ksynch.h> 41 #include <sys/ethernet.h> 42 #include <sys/crc32.h> 43 #include <sys/pattr.h> 44 #include <sys/cpu.h> 45 46 #include <sys/ethernet.h> 47 #include <inet/ip.h> 48 49 #include <netinet/in.h> 50 #include <netinet/ip.h> 51 #include <netinet/tcp.h> 52 53 #include "sfxge.h" 54 55 #include "efx.h" 56 57 /* RXQ flush response timeout (in microseconds) */ 58 #define SFXGE_RX_QFLUSH_USEC (2000000) 59 60 /* RXQ flush tries in the case of failure */ 61 #define SFXGE_RX_QFLUSH_TRIES (5) 62 63 /* RXQ default packet buffer preallocation (number of packet buffers) */ 64 #define SFXGE_RX_QPREALLOC (0) 65 66 /* Receive packet DMA attributes */ 67 static ddi_device_acc_attr_t sfxge_rx_packet_devacc = { 68 69 DDI_DEVICE_ATTR_V0, /* devacc_attr_version */ 70 DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */ 71 DDI_STRICTORDER_ACC /* devacc_attr_dataorder */ 72 }; 73 74 static ddi_dma_attr_t sfxge_rx_packet_dma_attr = { 75 DMA_ATTR_V0, /* dma_attr_version */ 76 0, /* dma_attr_addr_lo */ 77 0xffffffffffffffffull, /* dma_attr_addr_hi */ 78 0xffffffffffffffffull, /* dma_attr_count_max */ 79 SFXGE_CPU_CACHE_SIZE, /* dma_attr_align */ 80 0xffffffff, /* dma_attr_burstsizes */ 81 1, /* dma_attr_minxfer */ 82 0xffffffffffffffffull, /* dma_attr_maxxfer */ 83 0xffffffffffffffffull, /* dma_attr_seg */ 84 1, /* dma_attr_sgllen */ 85 1, /* dma_attr_granular */ 86 0 /* dma_attr_flags */ 87 }; 88 89 /* Receive queue DMA attributes */ 90 static ddi_device_acc_attr_t sfxge_rxq_devacc = { 91 92 DDI_DEVICE_ATTR_V0, /* devacc_attr_version */ 93 DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */ 94 DDI_STRICTORDER_ACC /* devacc_attr_dataorder */ 95 }; 96 97 static ddi_dma_attr_t sfxge_rxq_dma_attr = { 98 DMA_ATTR_V0, /* dma_attr_version */ 99 0, /* dma_attr_addr_lo */ 100 0xffffffffffffffffull, /* dma_attr_addr_hi */ 101 0xffffffffffffffffull, /* dma_attr_count_max */ 102 EFX_BUF_SIZE, /* dma_attr_align */ 103 0xffffffff, /* dma_attr_burstsizes */ 104 1, /* dma_attr_minxfer */ 105 0xffffffffffffffffull, /* dma_attr_maxxfer */ 106 0xffffffffffffffffull, /* dma_attr_seg */ 107 1, /* dma_attr_sgllen */ 108 1, /* dma_attr_granular */ 109 0 /* dma_attr_flags */ 110 }; 111 112 /* Forward declaration */ 113 static void sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc); 114 115 static int 116 sfxge_rx_packet_ctor(void *buf, void *arg, int kmflags) 117 { 118 sfxge_rx_packet_t *srpp = buf; 119 sfxge_t *sp = arg; 120 dev_info_t *dip = sp->s_dip; 121 int err; 122 123 ASSERT3U(sizeof (srpp->__srp_u1.__srp_s1), <=, 124 sizeof (srpp->__srp_u1.__srp_pad)); 125 ASSERT3U(sizeof (srpp->__srp_u2.__srp_s2), <=, 126 sizeof (srpp->__srp_u2.__srp_pad)); 127 128 bzero(buf, sizeof (sfxge_rx_packet_t)); 129 130 /* Allocate a DMA handle */ 131 err = ddi_dma_alloc_handle(dip, &sfxge_rx_packet_dma_attr, 132 (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, 133 NULL, &(srpp->srp_dma_handle)); 134 if (err != DDI_SUCCESS) 135 goto fail1; 136 137 return (0); 138 139 fail1: 140 DTRACE_PROBE1(fail1, int, err); 141 142 SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t); 143 144 return (-1); 145 } 146 147 static void 148 sfxge_rx_packet_dtor(void *buf, void *arg) 149 { 150 sfxge_rx_packet_t *srpp = buf; 151 152 _NOTE(ARGUNUSED(arg)) 153 154 /* Free the DMA handle */ 155 ddi_dma_free_handle(&(srpp->srp_dma_handle)); 156 srpp->srp_dma_handle = NULL; 157 158 SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t); 159 } 160 161 static int 162 sfxge_rx_qctor(void *buf, void *arg, int kmflags) 163 { 164 sfxge_rxq_t *srp = buf; 165 efsys_mem_t *esmp = &(srp->sr_mem); 166 sfxge_t *sp = arg; 167 sfxge_dma_buffer_attr_t dma_attr; 168 sfxge_rx_fpp_t *srfppp; 169 int nprealloc; 170 unsigned int id; 171 int rc; 172 173 /* Compile-time structure layout checks */ 174 EFX_STATIC_ASSERT(sizeof (srp->__sr_u1.__sr_s1) <= 175 sizeof (srp->__sr_u1.__sr_pad)); 176 EFX_STATIC_ASSERT(sizeof (srp->__sr_u2.__sr_s2) <= 177 sizeof (srp->__sr_u2.__sr_pad)); 178 EFX_STATIC_ASSERT(sizeof (srp->__sr_u3.__sr_s3) <= 179 sizeof (srp->__sr_u3.__sr_pad)); 180 181 bzero(buf, sizeof (sfxge_rxq_t)); 182 183 srp->sr_sp = sp; 184 185 dma_attr.sdba_dip = sp->s_dip; 186 dma_attr.sdba_dattrp = &sfxge_rxq_dma_attr; 187 dma_attr.sdba_callback = DDI_DMA_SLEEP; 188 dma_attr.sdba_length = EFX_RXQ_SIZE(sp->s_rxq_size); 189 dma_attr.sdba_memflags = DDI_DMA_CONSISTENT; 190 dma_attr.sdba_devaccp = &sfxge_rxq_devacc; 191 dma_attr.sdba_bindflags = DDI_DMA_READ | DDI_DMA_CONSISTENT; 192 dma_attr.sdba_maxcookies = 1; 193 dma_attr.sdba_zeroinit = B_FALSE; 194 195 if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0) 196 goto fail1; 197 198 /* Allocate some buffer table entries */ 199 if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_RXQ_NBUFS(sp->s_rxq_size), 200 &(srp->sr_id))) != 0) 201 goto fail2; 202 203 /* Allocate the context array */ 204 if ((srp->sr_srpp = kmem_zalloc(sizeof (sfxge_rx_packet_t *) * 205 sp->s_rxq_size, kmflags)) == NULL) { 206 rc = ENOMEM; 207 goto fail3; 208 } 209 210 /* Allocate the flow table */ 211 if ((srp->sr_flow = kmem_zalloc(sizeof (sfxge_rx_flow_t) * 212 SFXGE_MAX_FLOW, kmflags)) == NULL) { 213 rc = ENOMEM; 214 goto fail4; 215 } 216 217 srp->sr_srfpp = &(srp->sr_srfp); 218 srp->sr_rto = drv_usectohz(200000); 219 220 srp->sr_mpp = &(srp->sr_mp); 221 222 /* Initialize the free packet pool */ 223 srfppp = &(srp->sr_fpp); 224 if ((srfppp->srfpp_putp = kmem_zalloc(SFXGE_CPU_CACHE_SIZE * 225 SFXGE_RX_FPP_NSLOTS, kmflags)) == NULL) { 226 rc = ENOMEM; 227 goto fail5; 228 } 229 for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) { 230 sfxge_rx_fpp_putlist_t *putp; 231 size_t off; 232 233 off = id * SFXGE_CPU_CACHE_SIZE; 234 putp = (void *)(srfppp->srfpp_putp + off); 235 236 putp->srfpl_putp = NULL; 237 putp->srfpl_putpp = &(putp->srfpl_putp); 238 mutex_init(&(putp->srfpl_lock), NULL, MUTEX_DRIVER, 239 DDI_INTR_PRI(sp->s_intr.si_intr_pri)); 240 } 241 242 cv_init(&(srp->sr_flush_kv), NULL, CV_DRIVER, NULL); 243 244 /* Preallocate some packets on the free packet pool */ 245 nprealloc = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip, 246 DDI_PROP_DONTPASS, "rx_prealloc_pkt_buffers", SFXGE_RX_QPREALLOC); 247 sfxge_rx_qpreallocate(srp, nprealloc); 248 249 250 return (0); 251 252 fail5: 253 DTRACE_PROBE(fail5); 254 255 srp->sr_mpp = NULL; 256 257 srp->sr_rto = 0; 258 srp->sr_srfpp = NULL; 259 260 /* Free the flow table */ 261 kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) * 262 SFXGE_MAX_FLOW); 263 srp->sr_flow = NULL; 264 265 fail4: 266 DTRACE_PROBE(fail4); 267 268 /* Free the context array */ 269 kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) * 270 sp->s_rxq_size); 271 srp->sr_srpp = NULL; 272 273 fail3: 274 DTRACE_PROBE(fail3); 275 276 /* Free the buffer table entries */ 277 sfxge_sram_buf_tbl_free(sp, srp->sr_id, 278 EFX_RXQ_NBUFS(sp->s_rxq_size)); 279 srp->sr_id = 0; 280 281 fail2: 282 DTRACE_PROBE(fail2); 283 /* Remove dma setup */ 284 sfxge_dma_buffer_destroy(esmp); 285 286 fail1: 287 DTRACE_PROBE1(fail1, int, rc); 288 289 srp->sr_sp = NULL; 290 291 SFXGE_OBJ_CHECK(srp, sfxge_rxq_t); 292 293 return (-1); 294 } 295 296 static void 297 sfxge_rx_qdtor(void *buf, void *arg) 298 { 299 sfxge_rxq_t *srp = buf; 300 efsys_mem_t *esmp = &(srp->sr_mem); 301 sfxge_t *sp = srp->sr_sp; 302 sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp); 303 unsigned int id; 304 305 _NOTE(ARGUNUSED(arg)) 306 307 cv_destroy(&(srp->sr_flush_kv)); 308 309 /* Tear down the free packet pool */ 310 for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) { 311 sfxge_rx_fpp_putlist_t *putp; 312 size_t off; 313 314 off = id * SFXGE_CPU_CACHE_SIZE; 315 putp = (void *)(srfppp->srfpp_putp + off); 316 317 putp->srfpl_putpp = NULL; 318 mutex_destroy(&(putp->srfpl_lock)); 319 320 SFXGE_OBJ_CHECK(putp, sfxge_rx_fpp_putlist_t); 321 } 322 kmem_free(srfppp->srfpp_putp, SFXGE_CPU_CACHE_SIZE * 323 SFXGE_RX_FPP_NSLOTS); 324 srfppp->srfpp_putp = NULL; 325 326 srp->sr_mpp = NULL; 327 328 srp->sr_rto = 0; 329 srp->sr_srfpp = NULL; 330 331 /* Free the flow table */ 332 kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) * 333 SFXGE_MAX_FLOW); 334 srp->sr_flow = NULL; 335 336 /* Free the context array */ 337 kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) * 338 sp->s_rxq_size); 339 srp->sr_srpp = NULL; 340 341 /* Free the buffer table entries */ 342 sfxge_sram_buf_tbl_free(sp, srp->sr_id, 343 EFX_RXQ_NBUFS(sp->s_rxq_size)); 344 srp->sr_id = 0; 345 346 /* Tear down dma setup */ 347 sfxge_dma_buffer_destroy(esmp); 348 349 SFXGE_OBJ_CHECK(srp, sfxge_rxq_t); 350 } 351 352 /* Note: This function takes ownership of *srpp. */ 353 static inline void 354 sfxge_rx_qfpp_put(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp) 355 { 356 sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp); 357 mblk_t *mp = srpp->srp_mp; 358 unsigned int id; 359 size_t off; 360 sfxge_rx_fpp_putlist_t *putp; 361 362 ASSERT3P(mp->b_next, ==, NULL); 363 ASSERT3P(mp->b_prev, ==, NULL); 364 365 id = CPU->cpu_seqid & SFXGE_RX_FPP_MASK; 366 off = id * SFXGE_CPU_CACHE_SIZE; 367 368 ASSERT3P(srpp->srp_putp, ==, srfppp->srfpp_putp); 369 putp = (void *)(srpp->srp_putp + off); 370 371 mutex_enter(&(putp->srfpl_lock)); 372 putp->srfpl_count++; 373 *putp->srfpl_putpp = mp; 374 putp->srfpl_putpp = &(mp->b_next); 375 mutex_exit(&(putp->srfpl_lock)); 376 } 377 378 static unsigned int 379 sfxge_rx_qfpp_swizzle(sfxge_rxq_t *srp) 380 { 381 sfxge_t *sp = srp->sr_sp; 382 unsigned int index = srp->sr_index; 383 sfxge_evq_t *sep = sp->s_sep[index]; 384 sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp); 385 unsigned int start; 386 unsigned int id; 387 mblk_t *p; 388 mblk_t **pp; 389 unsigned int count; 390 unsigned int loaned; 391 392 ASSERT(mutex_owned(&(sep->se_lock))); 393 394 /* We want to access the put list for the current CPU last */ 395 id = start = (CPU->cpu_seqid + 1) & SFXGE_RX_FPP_MASK; 396 397 do { 398 sfxge_rx_fpp_putlist_t *putp; 399 size_t off; 400 401 off = id * SFXGE_CPU_CACHE_SIZE; 402 id = (id + 1) & SFXGE_RX_FPP_MASK; 403 404 putp = (void *)(srfppp->srfpp_putp + off); 405 406 /* Acquire the put list */ 407 mutex_enter(&(putp->srfpl_lock)); 408 409 p = putp->srfpl_putp; 410 pp = putp->srfpl_putpp; 411 count = putp->srfpl_count; 412 413 putp->srfpl_putp = NULL; 414 putp->srfpl_putpp = &(putp->srfpl_putp); 415 putp->srfpl_count = 0; 416 417 mutex_exit(&(putp->srfpl_lock)); 418 419 if (p == NULL) 420 continue; 421 422 /* Add the list to the head of the get list */ 423 *pp = srfppp->srfpp_get; 424 srfppp->srfpp_get = p; 425 426 /* Adjust the counters */ 427 ASSERT3U(srfppp->srfpp_loaned, >=, count); 428 srfppp->srfpp_loaned -= count; 429 srfppp->srfpp_count += count; 430 431 #if 0 432 /* NOTE: this probe is disabled because it is expensive!! */ 433 DTRACE_PROBE2(count, 434 unsigned int, (id - 1) & SFXGE_RX_FPP_MASK, 435 unsigned int, count); 436 #endif 437 438 } while (id != start); 439 440 /* Return the number of packets yet to appear in the put list */ 441 loaned = srfppp->srfpp_loaned; 442 443 444 return (loaned); 445 } 446 447 448 #define DB_FRTNP(mp) ((mp)->b_datap->db_frtnp) 449 450 static void 451 sfxge_rx_qfpp_empty(sfxge_rxq_t *srp) 452 { 453 sfxge_t *sp = srp->sr_sp; 454 unsigned int index = srp->sr_index; 455 sfxge_evq_t *sep = sp->s_sep[index]; 456 sfxge_rx_fpp_t *srfppp; 457 mblk_t *mp; 458 459 mutex_enter(&(sep->se_lock)); 460 srfppp = &(srp->sr_fpp); 461 462 /* Swizzle put list to get list */ 463 (void) sfxge_rx_qfpp_swizzle(srp); 464 ASSERT3U(srfppp->srfpp_loaned, ==, 0); 465 466 mp = srfppp->srfpp_get; 467 srfppp->srfpp_get = NULL; 468 469 /* Free the remainder */ 470 while (mp != NULL) { 471 mblk_t *next; 472 frtn_t *freep; 473 sfxge_rx_packet_t *srpp; 474 475 next = mp->b_next; 476 mp->b_next = NULL; 477 478 ASSERT3U(srfppp->srfpp_count, >, 0); 479 srfppp->srfpp_count--; 480 481 freep = DB_FRTNP(mp); 482 /* 483 * ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free); 484 * is implied by srpp test below 485 */ 486 /*LINTED*/ 487 srpp = (sfxge_rx_packet_t *)(freep->free_arg); 488 ASSERT3P(srpp->srp_mp, ==, mp); 489 ASSERT3P(mp->b_cont, ==, NULL); 490 srpp->srp_recycle = B_FALSE; 491 492 freeb(mp); 493 494 mp = next; 495 } 496 ASSERT3U(srfppp->srfpp_count, ==, 0); 497 498 srfppp->srfpp_min = 0; 499 500 mutex_exit(&(sep->se_lock)); 501 } 502 503 /* 504 * This is an estimate of all memory consumed per RX packet 505 * it can be inaccurate but but sp->s_rx_pkt_mem_alloc mustn't drift 506 */ 507 static uint64_t 508 sfxge_rx_pkt_mem_approx(const sfxge_rx_packet_t *srpp) 509 { 510 return (srpp->srp_mblksize + sizeof (mblk_t) + sizeof (dblk_t) + 511 sizeof (sfxge_rx_packet_t)); 512 } 513 514 static void 515 sfxge_rx_qpacket_destroy(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp) 516 { 517 sfxge_t *sp = srp->sr_sp; 518 int64_t delta = sfxge_rx_pkt_mem_approx(srpp); 519 520 ASSERT(!(srpp->srp_recycle)); 521 ASSERT3P(srpp->srp_mp, ==, NULL); 522 523 srpp->srp_off = 0; 524 srpp->srp_thp = NULL; 525 srpp->srp_iphp = NULL; 526 srpp->srp_etherhp = NULL; 527 srpp->srp_size = 0; 528 srpp->srp_flags = 0; 529 530 bzero(&(srpp->srp_free), sizeof (frtn_t)); 531 532 srpp->srp_mblksize = 0; 533 srpp->srp_base = NULL; 534 535 /* Unbind the DMA memory from the DMA handle */ 536 srpp->srp_addr = 0; 537 (void) ddi_dma_unbind_handle(srpp->srp_dma_handle); 538 539 /* Free the DMA memory */ 540 srpp->srp_base = NULL; 541 ddi_dma_mem_free(&(srpp->srp_acc_handle)); 542 srpp->srp_acc_handle = NULL; 543 544 srpp->srp_putp = NULL; 545 srpp->srp_srp = NULL; 546 547 kmem_cache_free(sp->s_rpc, srpp); 548 if (sp->s_rx_pkt_mem_max) 549 atomic_add_64(&sp->s_rx_pkt_mem_alloc, -delta); 550 } 551 552 static void 553 sfxge_rx_qpacket_free(void *arg) 554 { 555 sfxge_rx_packet_t *srpp = arg; 556 sfxge_rxq_t *srp = srpp->srp_srp; 557 558 /* 559 * WARNING "man -s 9f esballoc" states: 560 * => runs sync from the thread calling freeb() 561 * => must not sleep, or access data structures that could be freed 562 */ 563 564 /* Check whether we want to recycle the receive packets */ 565 if (srpp->srp_recycle) { 566 frtn_t *freep; 567 mblk_t *mp; 568 size_t size; 569 570 freep = &(srpp->srp_free); 571 ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free); 572 ASSERT3P(freep->free_arg, ==, (caddr_t)srpp); 573 574 /* 575 * Allocate a matching mblk_t before the current one is 576 * freed. 577 */ 578 size = srpp->srp_mblksize; 579 580 if ((mp = desballoc(srpp->srp_base, size, BPRI_HI, 581 freep)) != NULL) { 582 srpp->srp_mp = mp; 583 584 /* NORMAL recycled case */ 585 sfxge_rx_qfpp_put(srp, srpp); 586 return; 587 } 588 } 589 590 srpp->srp_mp = NULL; 591 592 sfxge_rx_qpacket_destroy(srp, srpp); 593 } 594 595 static sfxge_rx_packet_t * 596 sfxge_rx_qpacket_create(sfxge_rxq_t *srp) 597 { 598 sfxge_t *sp = srp->sr_sp; 599 sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp); 600 sfxge_rx_packet_t *srpp; 601 size_t size; 602 caddr_t base; 603 size_t unit; 604 ddi_dma_cookie_t dmac; 605 unsigned int ncookies; 606 frtn_t *freep; 607 mblk_t *mp; 608 int err; 609 int rc; 610 611 size = sp->s_rx_buffer_size; 612 613 if (sp->s_rx_pkt_mem_max && 614 (sp->s_rx_pkt_mem_alloc + size >= sp->s_rx_pkt_mem_max)) { 615 DTRACE_PROBE(rx_pkt_mem_max); 616 srp->sr_kstat.srk_rx_pkt_mem_limit++; 617 return (NULL); 618 } 619 620 /* Allocate a new packet */ 621 if ((srpp = kmem_cache_alloc(sp->s_rpc, KM_NOSLEEP)) == NULL) { 622 srp->sr_kstat.srk_kcache_alloc_nomem++; 623 rc = ENOMEM; 624 goto fail1; 625 } 626 627 srpp->srp_srp = srp; 628 srpp->srp_putp = srfppp->srfpp_putp; 629 630 /* Allocate some DMA memory */ 631 err = ddi_dma_mem_alloc(srpp->srp_dma_handle, size, 632 &sfxge_rx_packet_devacc, DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 633 NULL, &base, &unit, &(srpp->srp_acc_handle)); 634 switch (err) { 635 case DDI_SUCCESS: 636 break; 637 638 case DDI_FAILURE: 639 srp->sr_kstat.srk_dma_alloc_nomem++; 640 rc = ENOMEM; 641 goto fail2; 642 643 default: 644 srp->sr_kstat.srk_dma_alloc_fail++; 645 rc = EFAULT; 646 goto fail2; 647 } 648 649 /* Adjust the buffer to align the start of the DMA area correctly */ 650 base += sp->s_rx_buffer_align; 651 size -= sp->s_rx_buffer_align; 652 653 /* Bind the DMA memory to the DMA handle */ 654 err = ddi_dma_addr_bind_handle(srpp->srp_dma_handle, NULL, 655 base, size, DDI_DMA_READ | DDI_DMA_STREAMING, 656 DDI_DMA_DONTWAIT, NULL, &dmac, &ncookies); 657 switch (err) { 658 case DDI_DMA_MAPPED: 659 break; 660 661 case DDI_DMA_INUSE: 662 srp->sr_kstat.srk_dma_bind_fail++; 663 rc = EEXIST; 664 goto fail3; 665 666 case DDI_DMA_NORESOURCES: 667 srp->sr_kstat.srk_dma_bind_nomem++; 668 rc = ENOMEM; 669 goto fail3; 670 671 case DDI_DMA_NOMAPPING: 672 srp->sr_kstat.srk_dma_bind_fail++; 673 rc = ENOTSUP; 674 goto fail3; 675 676 case DDI_DMA_TOOBIG: 677 srp->sr_kstat.srk_dma_bind_fail++; 678 rc = EFBIG; 679 goto fail3; 680 681 default: 682 srp->sr_kstat.srk_dma_bind_fail++; 683 rc = EFAULT; 684 goto fail3; 685 } 686 ASSERT3U(ncookies, ==, 1); 687 688 srpp->srp_addr = dmac.dmac_laddress; 689 690 srpp->srp_base = (unsigned char *)base; 691 srpp->srp_mblksize = size; 692 693 /* 694 * Allocate a STREAMS block: We use size 1 so that the allocator will 695 * use the first (and smallest) dblk cache. 696 */ 697 freep = &(srpp->srp_free); 698 freep->free_func = sfxge_rx_qpacket_free; 699 freep->free_arg = (caddr_t)srpp; 700 701 if ((mp = desballoc(srpp->srp_base, size, BPRI_HI, freep)) == NULL) { 702 srp->sr_kstat.srk_desballoc_fail++; 703 rc = ENOMEM; 704 goto fail4; 705 } 706 707 srpp->srp_mp = mp; 708 srpp->srp_recycle = B_TRUE; 709 710 if (sp->s_rx_pkt_mem_max) { 711 int64_t delta = sfxge_rx_pkt_mem_approx(srpp); 712 atomic_add_64(&sp->s_rx_pkt_mem_alloc, delta); 713 } 714 715 return (srpp); 716 717 fail4: 718 DTRACE_PROBE(fail4); 719 720 bzero(&(srpp->srp_free), sizeof (frtn_t)); 721 722 srpp->srp_mblksize = 0; 723 srpp->srp_base = NULL; 724 725 /* Unbind the DMA memory from the DMA handle */ 726 srpp->srp_addr = 0; 727 (void) ddi_dma_unbind_handle(srpp->srp_dma_handle); 728 729 fail3: 730 DTRACE_PROBE(fail3); 731 732 /* Free the DMA memory */ 733 ddi_dma_mem_free(&(srpp->srp_acc_handle)); 734 srpp->srp_acc_handle = NULL; 735 736 fail2: 737 DTRACE_PROBE(fail2); 738 739 srpp->srp_putp = NULL; 740 srpp->srp_srp = NULL; 741 742 kmem_cache_free(sp->s_rpc, srpp); 743 744 fail1: 745 DTRACE_PROBE1(fail1, int, rc); 746 747 return (NULL); 748 } 749 750 #define SFXGE_REFILL_BATCH 64 751 752 /* Try to refill the RX descriptor ring from the associated free pkt pool */ 753 static void 754 sfxge_rx_qrefill(sfxge_rxq_t *srp, unsigned int target) 755 { 756 sfxge_t *sp = srp->sr_sp; 757 sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp); 758 unsigned int index = srp->sr_index; 759 sfxge_evq_t *sep = sp->s_sep[index]; 760 efsys_dma_addr_t addr[SFXGE_REFILL_BATCH]; 761 mblk_t *mp; 762 int ntodo; 763 unsigned int count; 764 unsigned int batch; 765 unsigned int rxfill; 766 unsigned int mblksize; 767 768 prefetch_read_many(sp->s_enp); 769 prefetch_read_many(srp->sr_erp); 770 771 ASSERT(mutex_owned(&(sep->se_lock))); 772 773 if (srp->sr_state != SFXGE_RXQ_STARTED) 774 return; 775 776 rxfill = srp->sr_added - srp->sr_completed; 777 ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size)); 778 ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target); 779 ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size)); 780 781 if (ntodo == 0) 782 goto out; 783 784 (void) sfxge_rx_qfpp_swizzle(srp); 785 786 mp = srfppp->srfpp_get; 787 count = srfppp->srfpp_count; 788 mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align; 789 790 batch = 0; 791 while (ntodo-- > 0) { 792 mblk_t *next; 793 frtn_t *freep; 794 sfxge_rx_packet_t *srpp; 795 unsigned int id; 796 797 if (mp == NULL) 798 break; 799 800 next = mp->b_next; 801 mp->b_next = NULL; 802 803 if (next != NULL) 804 prefetch_read_many(next); 805 806 freep = DB_FRTNP(mp); 807 /*LINTED*/ 808 srpp = (sfxge_rx_packet_t *)(freep->free_arg); 809 ASSERT3P(srpp->srp_mp, ==, mp); 810 811 /* The MTU may have changed since the packet was allocated */ 812 if (MBLKSIZE(mp) != mblksize) { 813 srpp->srp_recycle = B_FALSE; 814 815 freeb(mp); 816 817 --count; 818 mp = next; 819 continue; 820 } 821 822 srpp->srp_off = 0; 823 srpp->srp_thp = NULL; 824 srpp->srp_iphp = NULL; 825 srpp->srp_etherhp = NULL; 826 srpp->srp_size = 0; 827 srpp->srp_flags = EFX_DISCARD; 828 829 id = (srp->sr_added + batch) & (sp->s_rxq_size - 1); 830 ASSERT(srp->sr_srpp[id] == NULL); 831 srp->sr_srpp[id] = srpp; 832 833 addr[batch++] = srpp->srp_addr; 834 if (batch == SFXGE_REFILL_BATCH) { 835 efx_rx_qpost(srp->sr_erp, addr, mblksize, batch, 836 srp->sr_completed, srp->sr_added); 837 srp->sr_added += batch; 838 batch = 0; 839 } 840 841 --count; 842 mp = next; 843 } 844 845 srfppp->srfpp_get = mp; 846 srfppp->srfpp_count = count; 847 848 if (batch != 0) { 849 efx_rx_qpost(srp->sr_erp, addr, mblksize, batch, 850 srp->sr_completed, srp->sr_added); 851 srp->sr_added += batch; 852 } 853 854 efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed); 855 856 out: 857 if (srfppp->srfpp_count < srfppp->srfpp_min) 858 srfppp->srfpp_min = srfppp->srfpp_count; 859 } 860 861 /* Preallocate packets and put them in the free packet pool */ 862 static void 863 sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc) 864 { 865 sfxge_rx_fpp_t *srfppp = &((srp)->sr_fpp); 866 srfppp->srfpp_lowat = nprealloc; 867 while (nprealloc-- > 0) { 868 sfxge_rx_packet_t *srpp; 869 870 if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL) 871 break; 872 sfxge_rx_qfpp_put(srp, srpp); 873 } 874 } 875 876 /* Try to refill the RX descriptor ring by allocating new packets */ 877 static void 878 sfxge_rx_qfill(sfxge_rxq_t *srp, unsigned int target) 879 { 880 sfxge_t *sp = srp->sr_sp; 881 unsigned int index = srp->sr_index; 882 sfxge_evq_t *sep = sp->s_sep[index]; 883 unsigned int batch; 884 unsigned int rxfill; 885 unsigned int mblksize; 886 int ntodo; 887 efsys_dma_addr_t addr[SFXGE_REFILL_BATCH]; 888 mblk_t *mp = NULL; 889 890 prefetch_read_many(sp->s_enp); 891 prefetch_read_many(srp->sr_erp); 892 893 ASSERT(mutex_owned(&(sep->se_lock))); 894 895 if (srp->sr_state != SFXGE_RXQ_STARTED) 896 return; 897 898 rxfill = srp->sr_added - srp->sr_completed; 899 ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size)); 900 ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target); 901 ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size)); 902 903 if (ntodo == 0) 904 return; 905 906 mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align; 907 908 batch = 0; 909 while (ntodo-- > 0) { 910 sfxge_rx_packet_t *srpp; 911 unsigned int id; 912 913 if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL) 914 break; 915 916 mp = srpp->srp_mp; 917 918 ASSERT3U(MBLKSIZE(mp), ==, mblksize); 919 920 ASSERT3U(srpp->srp_off, ==, 0); 921 ASSERT3P(srpp->srp_thp, ==, NULL); 922 ASSERT3P(srpp->srp_iphp, ==, NULL); 923 ASSERT3P(srpp->srp_etherhp, ==, NULL); 924 ASSERT3U(srpp->srp_size, ==, 0); 925 926 srpp->srp_flags = EFX_DISCARD; 927 928 id = (srp->sr_added + batch) & (sp->s_rxq_size - 1); 929 ASSERT(srp->sr_srpp[id] == NULL); 930 srp->sr_srpp[id] = srpp; 931 932 addr[batch++] = srpp->srp_addr; 933 if (batch == SFXGE_REFILL_BATCH) { 934 efx_rx_qpost(srp->sr_erp, addr, mblksize, batch, 935 srp->sr_completed, srp->sr_added); 936 srp->sr_added += batch; 937 batch = 0; 938 } 939 } 940 941 if (batch != 0) { 942 efx_rx_qpost(srp->sr_erp, addr, mblksize, batch, 943 srp->sr_completed, srp->sr_added); 944 srp->sr_added += batch; 945 } 946 947 efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed); 948 } 949 950 void 951 sfxge_rx_qfpp_trim(sfxge_rxq_t *srp) 952 { 953 sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp); 954 sfxge_t *sp = srp->sr_sp; 955 unsigned int index = srp->sr_index; 956 sfxge_evq_t *sep = sp->s_sep[index]; 957 mblk_t *p; 958 mblk_t **pp; 959 int count; 960 961 ASSERT(mutex_owned(&(sep->se_lock))); 962 963 if (srp->sr_state != SFXGE_RXQ_STARTED) 964 goto done; 965 966 /* Make sure the queue is full */ 967 sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size)); 968 969 /* The refill may have emptied the pool */ 970 if (srfppp->srfpp_min == 0) 971 goto done; 972 973 /* Don't trim below the pool's low water mark */ 974 if (srfppp->srfpp_count <= srfppp->srfpp_lowat) 975 goto done; 976 977 ASSERT(srfppp->srfpp_min <= srfppp->srfpp_count); 978 979 /* Trim to the largest of srfppp->srfpp_min and srfpp->srfpp_lowat */ 980 if (srfppp->srfpp_lowat > srfppp->srfpp_min) 981 count = srfppp->srfpp_count - srfppp->srfpp_lowat; 982 else 983 count = srfppp->srfpp_count - srfppp->srfpp_min; 984 985 /* Walk the get list */ 986 pp = &(srfppp->srfpp_get); 987 while (--count >= 0) { 988 ASSERT(pp); 989 p = *pp; 990 ASSERT(p != NULL); 991 992 pp = &(p->b_next); 993 } 994 ASSERT(pp); 995 p = *pp; 996 997 /* Truncate the get list */ 998 *pp = NULL; 999 1000 /* Free the remainder */ 1001 while (p != NULL) { 1002 mblk_t *next; 1003 frtn_t *freep; 1004 sfxge_rx_packet_t *srpp; 1005 1006 next = p->b_next; 1007 p->b_next = NULL; 1008 1009 ASSERT3U(srfppp->srfpp_min, >, 0); 1010 srfppp->srfpp_min--; 1011 srfppp->srfpp_count--; 1012 1013 freep = DB_FRTNP(p); 1014 /*LINTED*/ 1015 srpp = (sfxge_rx_packet_t *)(freep->free_arg); 1016 ASSERT3P(srpp->srp_mp, ==, p); 1017 1018 srpp->srp_recycle = B_FALSE; 1019 1020 freeb(p); 1021 1022 p = next; 1023 } 1024 1025 done: 1026 srfppp->srfpp_min = srfppp->srfpp_count; 1027 } 1028 1029 static void 1030 sfxge_rx_qpoll(void *arg) 1031 { 1032 sfxge_rxq_t *srp = arg; 1033 sfxge_t *sp = srp->sr_sp; 1034 unsigned int index = srp->sr_index; 1035 sfxge_evq_t *sep = sp->s_sep[index]; 1036 uint16_t magic; 1037 1038 /* 1039 * man timeout(9f) states that this code should adhere to the 1040 * same requirements as a softirq handler - DO NOT BLOCK 1041 */ 1042 1043 /* 1044 * Post an event to the event queue to cause the free packet pool to be 1045 * trimmed if it is oversize. 1046 */ 1047 magic = SFXGE_MAGIC_RX_QFPP_TRIM | index; 1048 1049 #if defined(DEBUG) 1050 /* This is guaranteed due to the start/stop order of rx and ev */ 1051 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); 1052 ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED); 1053 #else 1054 /* 1055 * Bug22691 WORKAROUND: 1056 * This handler has been observed in the field to be invoked for a 1057 * queue in the INITIALIZED state, which should never happen. 1058 * Until the mechanism for this is properly understood, add defensive 1059 * checks. 1060 */ 1061 if ((sep->se_state != SFXGE_EVQ_STARTED) || 1062 (srp->sr_state != SFXGE_RXQ_STARTED) || 1063 (!sep->se_eep)) { 1064 dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR 1065 "RXQ[%d] bad state in sfxge_rx_qpoll %d %d %p", 1066 index, sep->se_state, srp->sr_state, sep->se_eep); 1067 return; 1068 } 1069 #endif 1070 efx_ev_qpost(sep->se_eep, magic); 1071 1072 srp->sr_tid = timeout(sfxge_rx_qpoll, srp, 1073 drv_usectohz(sp->s_rxq_poll_usec)); 1074 } 1075 1076 static void 1077 sfxge_rx_qpoll_start(sfxge_rxq_t *srp) 1078 { 1079 sfxge_t *sp = srp->sr_sp; 1080 unsigned int index = srp->sr_index; 1081 sfxge_evq_t *sep = sp->s_sep[index]; 1082 1083 ASSERT(mutex_owned(&(sep->se_lock))); 1084 ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED); 1085 1086 /* Schedule a poll */ 1087 ASSERT3P(srp->sr_tid, ==, 0); 1088 srp->sr_tid = timeout(sfxge_rx_qpoll, srp, 0); 1089 } 1090 1091 static void 1092 sfxge_rx_qpoll_stop(sfxge_rxq_t *srp) 1093 { 1094 sfxge_t *sp = srp->sr_sp; 1095 unsigned int index = srp->sr_index; 1096 sfxge_evq_t *sep = sp->s_sep[index]; 1097 timeout_id_t tid; 1098 1099 ASSERT(mutex_owned(&(sep->se_lock))); 1100 ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED); 1101 1102 /* 1103 * Cancel the qpoll timer. Care is needed as this function 1104 * can race with sfxge_rx_qpoll() for timeout id updates. 1105 * 1106 * Do not hold locks used by any timeout(9f) handlers across 1107 * calls to untimeout(9f) as this will deadlock. 1108 */ 1109 tid = 0; 1110 while ((srp->sr_tid != 0) && (srp->sr_tid != tid)) { 1111 tid = srp->sr_tid; 1112 (void) untimeout(tid); 1113 } 1114 srp->sr_tid = 0; 1115 } 1116 1117 static int 1118 sfxge_rx_kstat_update(kstat_t *ksp, int rw) 1119 { 1120 sfxge_rxq_t *srp = ksp->ks_private; 1121 sfxge_t *sp = srp->sr_sp; 1122 unsigned int index = srp->sr_index; 1123 sfxge_evq_t *sep = sp->s_sep[index]; 1124 kstat_named_t *knp; 1125 int rc; 1126 1127 if (rw != KSTAT_READ) { 1128 rc = EACCES; 1129 goto fail1; 1130 } 1131 1132 ASSERT(mutex_owned(&(sep->se_lock))); 1133 if (srp->sr_state != SFXGE_RXQ_STARTED) 1134 goto done; 1135 1136 knp = ksp->ks_data; 1137 /* NB pointer post-increment below */ 1138 knp++->value.ui32 = srp->sr_kstat.srk_rx_pkt_mem_limit; 1139 knp++->value.ui32 = srp->sr_kstat.srk_kcache_alloc_nomem; 1140 knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_nomem; 1141 knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_fail; 1142 knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_nomem; 1143 knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_fail; 1144 knp++->value.ui32 = srp->sr_kstat.srk_desballoc_fail; 1145 knp++->value.ui32 = srp->sr_kstat.srk_rxq_empty_discard; 1146 1147 done: 1148 return (0); 1149 1150 fail1: 1151 DTRACE_PROBE1(fail1, int, rc); 1152 1153 return (rc); 1154 } 1155 1156 static int 1157 sfxge_rx_kstat_init(sfxge_rxq_t *srp) 1158 { 1159 sfxge_t *sp = srp->sr_sp; 1160 unsigned int index = srp->sr_index; 1161 sfxge_evq_t *sep = sp->s_sep[index]; 1162 dev_info_t *dip = sp->s_dip; 1163 char name[MAXNAMELEN]; 1164 kstat_t *ksp; 1165 kstat_named_t *knp; 1166 int rc; 1167 1168 /* Create the set */ 1169 (void) snprintf(name, MAXNAMELEN - 1, "%s_rxq%04d", 1170 ddi_driver_name(dip), index); 1171 1172 if ((ksp = kstat_create((char *)ddi_driver_name(dip), 1173 ddi_get_instance(dip), name, "rxq", KSTAT_TYPE_NAMED, 1174 SFXGE_RX_NSTATS, 0)) == NULL) { 1175 rc = ENOMEM; 1176 goto fail1; 1177 } 1178 1179 srp->sr_ksp = ksp; 1180 1181 ksp->ks_update = sfxge_rx_kstat_update; 1182 ksp->ks_private = srp; 1183 ksp->ks_lock = &(sep->se_lock); 1184 1185 /* Initialise the named stats */ 1186 knp = ksp->ks_data; 1187 kstat_named_init(knp, "rx_pkt_mem_limit", KSTAT_DATA_UINT32); 1188 knp++; 1189 kstat_named_init(knp, "kcache_alloc_nomem", KSTAT_DATA_UINT32); 1190 knp++; 1191 kstat_named_init(knp, "dma_alloc_nomem", KSTAT_DATA_UINT32); 1192 knp++; 1193 kstat_named_init(knp, "dma_alloc_fail", KSTAT_DATA_UINT32); 1194 knp++; 1195 kstat_named_init(knp, "dma_bind_nomem", KSTAT_DATA_UINT32); 1196 knp++; 1197 kstat_named_init(knp, "dma_bind_fail", KSTAT_DATA_UINT32); 1198 knp++; 1199 kstat_named_init(knp, "desballoc_fail", KSTAT_DATA_UINT32); 1200 knp++; 1201 kstat_named_init(knp, "rxq_empty_discard", KSTAT_DATA_UINT32); 1202 1203 kstat_install(ksp); 1204 return (0); 1205 1206 fail1: 1207 DTRACE_PROBE1(fail1, int, rc); 1208 1209 return (rc); 1210 } 1211 1212 static int 1213 sfxge_rx_qinit(sfxge_t *sp, unsigned int index) 1214 { 1215 sfxge_rxq_t *srp; 1216 int rc; 1217 1218 ASSERT3U(index, <, SFXGE_RX_SCALE_MAX); 1219 1220 if ((srp = kmem_cache_alloc(sp->s_rqc, KM_SLEEP)) == NULL) { 1221 rc = ENOMEM; 1222 goto fail1; 1223 } 1224 ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_UNINITIALIZED); 1225 1226 srp->sr_index = index; 1227 sp->s_srp[index] = srp; 1228 1229 if ((rc = sfxge_rx_kstat_init(srp)) != 0) 1230 goto fail2; 1231 1232 srp->sr_state = SFXGE_RXQ_INITIALIZED; 1233 1234 return (0); 1235 1236 fail2: 1237 DTRACE_PROBE(fail2); 1238 kmem_cache_free(sp->s_rqc, srp); 1239 1240 fail1: 1241 DTRACE_PROBE1(fail1, int, rc); 1242 1243 return (rc); 1244 } 1245 1246 static int 1247 sfxge_rx_qstart(sfxge_t *sp, unsigned int index) 1248 { 1249 sfxge_evq_t *sep = sp->s_sep[index]; 1250 sfxge_rxq_t *srp; 1251 efsys_mem_t *esmp; 1252 efx_nic_t *enp; 1253 unsigned int level; 1254 int rc; 1255 1256 mutex_enter(&(sep->se_lock)); 1257 srp = sp->s_srp[index]; 1258 enp = sp->s_enp; 1259 esmp = &(srp->sr_mem); 1260 1261 ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED); 1262 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); 1263 1264 /* Zero the memory */ 1265 bzero(esmp->esm_base, EFX_RXQ_SIZE(sp->s_rxq_size)); 1266 1267 /* Program the buffer table */ 1268 if ((rc = sfxge_sram_buf_tbl_set(sp, srp->sr_id, esmp, 1269 EFX_RXQ_NBUFS(sp->s_rxq_size))) != 0) 1270 goto fail1; 1271 1272 /* Create the receive queue */ 1273 if ((rc = efx_rx_qcreate(enp, index, index, EFX_RXQ_TYPE_DEFAULT, 1274 esmp, sp->s_rxq_size, srp->sr_id, sep->se_eep, &(srp->sr_erp))) 1275 != 0) 1276 goto fail2; 1277 1278 /* Enable the receive queue */ 1279 efx_rx_qenable(srp->sr_erp); 1280 1281 /* Set the water marks */ 1282 srp->sr_hiwat = EFX_RXQ_LIMIT(sp->s_rxq_size) * 9 / 10; 1283 srp->sr_lowat = srp->sr_hiwat / 2; 1284 1285 srp->sr_state = SFXGE_RXQ_STARTED; 1286 srp->sr_flush = SFXGE_FLUSH_INACTIVE; 1287 1288 sfxge_rx_qpoll_start(srp); 1289 1290 /* Try to fill the queue from the pool */ 1291 sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size)); 1292 1293 /* 1294 * If there were insufficient buffers in the pool to reach the at 1295 * least a batch then allocate some. 1296 */ 1297 level = srp->sr_added - srp->sr_completed; 1298 if (level < SFXGE_RX_BATCH) 1299 sfxge_rx_qfill(srp, SFXGE_RX_BATCH); 1300 1301 mutex_exit(&(sep->se_lock)); 1302 1303 return (0); 1304 1305 fail2: 1306 DTRACE_PROBE(fail2); 1307 1308 /* Clear entries from the buffer table */ 1309 sfxge_sram_buf_tbl_clear(sp, srp->sr_id, 1310 EFX_RXQ_NBUFS(sp->s_rxq_size)); 1311 1312 fail1: 1313 DTRACE_PROBE1(fail1, int, rc); 1314 1315 mutex_exit(&(sep->se_lock)); 1316 1317 return (rc); 1318 } 1319 1320 static void 1321 sfxge_rx_qflow_complete(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp) 1322 { 1323 mblk_t *mp; 1324 struct ether_header *etherhp; 1325 struct ip *iphp; 1326 struct tcphdr *thp; 1327 1328 if (srfp->srf_mp == NULL) 1329 return; 1330 1331 mp = srfp->srf_mp; 1332 etherhp = srfp->srf_etherhp; 1333 iphp = srfp->srf_iphp; 1334 thp = srfp->srf_last_thp; 1335 1336 ASSERT3U(((etherhp->ether_type == htons(ETHERTYPE_VLAN)) ? 1337 sizeof (struct ether_vlan_header) : 1338 sizeof (struct ether_header)) + 1339 srfp->srf_len, ==, msgdsize(mp)); 1340 1341 ASSERT3U(srfp->srf_len & 0xffff, ==, srfp->srf_len); 1342 iphp->ip_len = htons(srfp->srf_len); 1343 1344 srfp->srf_first_thp->th_ack = thp->th_ack; 1345 srfp->srf_first_thp->th_win = thp->th_win; 1346 srfp->srf_first_thp->th_flags = thp->th_flags; 1347 1348 DTRACE_PROBE2(flow_complete, uint32_t, srfp->srf_tag, 1349 size_t, srfp->srf_len); 1350 1351 srfp->srf_mp = NULL; 1352 srfp->srf_len = 0; 1353 1354 ASSERT(mp->b_next == NULL); 1355 *(srp->sr_mpp) = mp; 1356 srp->sr_mpp = &(mp->b_next); 1357 } 1358 1359 static boolean_t 1360 sfxge_rx_qflow_add(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp, 1361 sfxge_rx_packet_t *srpp, clock_t now) 1362 { 1363 sfxge_t *sp = srp->sr_sp; 1364 struct ether_header *etherhp = srpp->srp_etherhp; 1365 struct ip *iphp = srpp->srp_iphp; 1366 struct tcphdr *thp = srpp->srp_thp; 1367 size_t off = srpp->srp_off; 1368 size_t size = (size_t)(srpp->srp_size); 1369 mblk_t *mp = srpp->srp_mp; 1370 uint32_t seq; 1371 unsigned int shift; 1372 1373 ASSERT3U(MBLKL(mp), ==, off + size); 1374 ASSERT3U(DB_CKSUMFLAGS(mp), ==, 1375 HCK_FULLCKSUM | HCK_FULLCKSUM_OK | HCK_IPV4_HDRCKSUM); 1376 1377 seq = htonl(thp->th_seq); 1378 1379 /* 1380 * If the time between this segment and the last is greater than RTO 1381 * then consider this a new flow. 1382 */ 1383 if (now - srfp->srf_lbolt > srp->sr_rto) { 1384 srfp->srf_count = 1; 1385 srfp->srf_seq = seq + size; 1386 1387 goto fail1; 1388 } 1389 1390 if (seq != srfp->srf_seq) { 1391 if (srfp->srf_count > SFXGE_SLOW_START) 1392 srfp->srf_count = SFXGE_SLOW_START; 1393 1394 srfp->srf_count >>= 1; 1395 1396 srfp->srf_count++; 1397 srfp->srf_seq = seq + size; 1398 1399 goto fail2; 1400 } 1401 1402 /* Update the in-order segment count and sequence number */ 1403 srfp->srf_count++; 1404 srfp->srf_seq = seq + size; 1405 1406 /* Don't merge across pure ACK, URG, SYN or RST segments */ 1407 if (size == 0 || thp->th_flags & (TH_URG | TH_SYN | TH_RST) || 1408 thp->th_urp != 0) 1409 goto fail3; 1410 1411 /* 1412 * If the in-order segment count has not yet reached the slow-start 1413 * threshold then we cannot coalesce. 1414 */ 1415 if (srfp->srf_count < SFXGE_SLOW_START) 1416 goto fail4; 1417 1418 /* Scale up the packet size from 4k (the maximum being 64k) */ 1419 ASSERT3U(srfp->srf_count, >=, SFXGE_SLOW_START); 1420 shift = MIN(srfp->srf_count - SFXGE_SLOW_START + 12, 16); 1421 if (srfp->srf_len + size >= (1 << shift)) 1422 sfxge_rx_qflow_complete(srp, srfp); 1423 1424 ASSERT(mp->b_cont == NULL); 1425 1426 if (srfp->srf_mp == NULL) { 1427 /* First packet in this flow */ 1428 srfp->srf_etherhp = etherhp; 1429 srfp->srf_iphp = iphp; 1430 srfp->srf_first_thp = srfp->srf_last_thp = thp; 1431 1432 ASSERT3P(mp->b_cont, ==, NULL); 1433 srfp->srf_mp = mp; 1434 srfp->srf_mpp = &(mp->b_cont); 1435 1436 srfp->srf_len = ntohs(iphp->ip_len); 1437 1438 /* 1439 * If the flow is not already in the list of occupied flows then 1440 * add it. 1441 */ 1442 if (srfp->srf_next == NULL && 1443 srp->sr_srfpp != &(srfp->srf_next)) { 1444 *(srp->sr_srfpp) = srfp; 1445 srp->sr_srfpp = &(srfp->srf_next); 1446 } 1447 } else { 1448 /* Later packet in this flow - skip TCP header */ 1449 srfp->srf_last_thp = thp; 1450 1451 mp->b_rptr += off; 1452 ASSERT3U(MBLKL(mp), ==, size); 1453 1454 ASSERT3P(mp->b_cont, ==, NULL); 1455 *(srfp->srf_mpp) = mp; 1456 srfp->srf_mpp = &(mp->b_cont); 1457 1458 srfp->srf_len += size; 1459 1460 ASSERT(srfp->srf_next != NULL || 1461 srp->sr_srfpp == &(srfp->srf_next)); 1462 } 1463 1464 DTRACE_PROBE2(flow_add, uint32_t, srfp->srf_tag, size_t, size); 1465 1466 /* 1467 * Try to align coalesced segments on push boundaries, unless they 1468 * are too frequent. 1469 */ 1470 if (sp->s_rx_coalesce_mode == SFXGE_RX_COALESCE_ALLOW_PUSH && 1471 thp->th_flags & TH_PUSH) 1472 sfxge_rx_qflow_complete(srp, srfp); 1473 1474 srfp->srf_lbolt = now; 1475 return (B_TRUE); 1476 1477 fail4: 1478 fail3: 1479 fail2: 1480 fail1: 1481 sfxge_rx_qflow_complete(srp, srfp); 1482 1483 srfp->srf_lbolt = now; 1484 return (B_FALSE); 1485 } 1486 1487 void 1488 sfxge_rx_qpacket_coalesce(sfxge_rxq_t *srp) 1489 { 1490 sfxge_t *sp = srp->sr_sp; 1491 clock_t now; 1492 mblk_t *mp; 1493 sfxge_rx_flow_t *srfp; 1494 1495 ASSERT(sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF); 1496 1497 now = ddi_get_lbolt(); 1498 1499 mp = srp->sr_mp; 1500 1501 srp->sr_mp = NULL; 1502 srp->sr_mpp = &(srp->sr_mp); 1503 1504 /* Start with the last flow to be appended to */ 1505 srfp = *(srp->sr_srfpp); 1506 1507 while (mp != NULL) { 1508 frtn_t *freep; 1509 sfxge_rx_packet_t *srpp; 1510 struct ether_header *etherhp; 1511 struct ip *iphp; 1512 struct tcphdr *thp; 1513 size_t off; 1514 size_t size; 1515 uint16_t ether_tci; 1516 uint32_t hash; 1517 uint32_t tag; 1518 mblk_t *next; 1519 sfxge_packet_type_t pkt_type; 1520 uint16_t sport, dport; 1521 1522 next = mp->b_next; 1523 mp->b_next = NULL; 1524 1525 if (next != NULL) 1526 prefetch_read_many(next); 1527 1528 freep = DB_FRTNP(mp); 1529 /*LINTED*/ 1530 srpp = (sfxge_rx_packet_t *)(freep->free_arg); 1531 ASSERT3P(srpp->srp_mp, ==, mp); 1532 1533 /* If the packet is not TCP then we cannot coalesce it */ 1534 if (~(srpp->srp_flags) & EFX_PKT_TCP) 1535 goto reject; 1536 1537 /* 1538 * If the packet is not fully checksummed then we cannot 1539 * coalesce it. 1540 */ 1541 if (~(srpp->srp_flags) & (EFX_CKSUM_TCPUDP | EFX_CKSUM_IPV4)) 1542 goto reject; 1543 1544 /* Parse the TCP header */ 1545 pkt_type = sfxge_pkthdr_parse(mp, ðerhp, &iphp, &thp, &off, 1546 &size, &sport, &dport); 1547 ASSERT(pkt_type == SFXGE_PACKET_TYPE_IPV4_TCP); 1548 ASSERT(etherhp != NULL); 1549 ASSERT(iphp != NULL); 1550 ASSERT(thp != NULL); 1551 ASSERT(off != 0); 1552 1553 if ((iphp->ip_off & ~htons(IP_DF)) != 0) 1554 goto reject; 1555 1556 if (etherhp->ether_type == htons(ETHERTYPE_VLAN)) { 1557 struct ether_vlan_header *ethervhp; 1558 1559 ethervhp = (struct ether_vlan_header *)etherhp; 1560 ether_tci = ethervhp->ether_tci; 1561 } else { 1562 ether_tci = 0; 1563 } 1564 1565 /* 1566 * Make sure any minimum length padding is stripped 1567 * before we try to add the packet to a flow. 1568 */ 1569 ASSERT3U(sp->s_rx_prefix_size + MBLKL(mp), ==, 1570 (size_t)(srpp->srp_size)); 1571 ASSERT3U(sp->s_rx_prefix_size + off + size, <=, 1572 (size_t)(srpp->srp_size)); 1573 1574 if (sp->s_rx_prefix_size + off + size < 1575 (size_t)(srpp->srp_size)) 1576 mp->b_wptr = mp->b_rptr + off + size; 1577 1578 /* 1579 * If there is no current flow, or the segment does not match 1580 * the current flow then we must attempt to look up the 1581 * correct flow in the table. 1582 */ 1583 if (srfp == NULL) 1584 goto lookup; 1585 1586 if (srfp->srf_saddr != iphp->ip_src.s_addr || 1587 srfp->srf_daddr != iphp->ip_dst.s_addr) 1588 goto lookup; 1589 1590 if (srfp->srf_sport != thp->th_sport || 1591 srfp->srf_dport != thp->th_dport) 1592 goto lookup; 1593 1594 if (srfp->srf_tci != ether_tci) 1595 goto lookup; 1596 1597 add: 1598 ASSERT(srfp != NULL); 1599 1600 srpp->srp_etherhp = etherhp; 1601 srpp->srp_iphp = iphp; 1602 srpp->srp_thp = thp; 1603 srpp->srp_off = off; 1604 1605 ASSERT3U(size, <, (1 << 16)); 1606 srpp->srp_size = (uint16_t)size; 1607 1608 /* Try to append the packet to the flow */ 1609 if (!sfxge_rx_qflow_add(srp, srfp, srpp, now)) 1610 goto reject; 1611 1612 mp = next; 1613 continue; 1614 1615 lookup: 1616 /* 1617 * If there is a prefix area then read the hash from that, 1618 * otherwise calculate it. 1619 */ 1620 if (sp->s_rx_prefix_size != 0) { 1621 hash = efx_psuedo_hdr_hash_get(sp->s_enp, 1622 EFX_RX_HASHALG_TOEPLITZ, 1623 DB_BASE(mp)); 1624 } else { 1625 SFXGE_TCP_HASH(sp, 1626 &iphp->ip_src.s_addr, 1627 thp->th_sport, 1628 &iphp->ip_dst.s_addr, 1629 thp->th_dport, 1630 hash); 1631 } 1632 1633 srfp = &(srp->sr_flow[(hash >> 6) % SFXGE_MAX_FLOW]); 1634 tag = hash + 1; /* Make sure it's not zero */ 1635 1636 /* 1637 * If the flow we have found does not match the hash then 1638 * it may be an unused flow, or it may be stale. 1639 */ 1640 if (tag != srfp->srf_tag) { 1641 if (srfp->srf_count != 0) { 1642 if (now - srfp->srf_lbolt <= srp->sr_rto) 1643 goto reject; 1644 } 1645 1646 if (srfp->srf_mp != NULL) 1647 goto reject; 1648 1649 /* Start a new flow */ 1650 ASSERT(srfp->srf_next == NULL); 1651 1652 srfp->srf_tag = tag; 1653 1654 srfp->srf_saddr = iphp->ip_src.s_addr; 1655 srfp->srf_daddr = iphp->ip_dst.s_addr; 1656 srfp->srf_sport = thp->th_sport; 1657 srfp->srf_dport = thp->th_dport; 1658 srfp->srf_tci = ether_tci; 1659 1660 srfp->srf_count = 0; 1661 srfp->srf_seq = ntohl(thp->th_seq); 1662 1663 srfp->srf_lbolt = now; 1664 goto add; 1665 } 1666 1667 /* 1668 * If the flow we have found does match the hash then it could 1669 * still be an alias. 1670 */ 1671 if (srfp->srf_saddr != iphp->ip_src.s_addr || 1672 srfp->srf_daddr != iphp->ip_dst.s_addr) 1673 goto reject; 1674 1675 if (srfp->srf_sport != thp->th_sport || 1676 srfp->srf_dport != thp->th_dport) 1677 goto reject; 1678 1679 if (srfp->srf_tci != ether_tci) 1680 goto reject; 1681 1682 goto add; 1683 1684 reject: 1685 *(srp->sr_mpp) = mp; 1686 srp->sr_mpp = &(mp->b_next); 1687 1688 mp = next; 1689 } 1690 } 1691 1692 void 1693 sfxge_rx_qcomplete(sfxge_rxq_t *srp, boolean_t eop) 1694 { 1695 sfxge_t *sp = srp->sr_sp; 1696 unsigned int index = srp->sr_index; 1697 sfxge_evq_t *sep = sp->s_sep[index]; 1698 unsigned int completed; 1699 sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp); 1700 unsigned int level; 1701 1702 ASSERT(mutex_owned(&(sep->se_lock))); 1703 1704 ASSERT(srp->sr_mp == NULL); 1705 ASSERT(srp->sr_mpp == &(srp->sr_mp)); 1706 1707 completed = srp->sr_completed; 1708 while (completed != srp->sr_pending) { 1709 unsigned int id; 1710 sfxge_rx_packet_t *srpp; 1711 mblk_t *mp; 1712 size_t size; 1713 uint16_t flags; 1714 int rc; 1715 1716 id = completed++ & (sp->s_rxq_size - 1); 1717 1718 if (srp->sr_pending - completed >= 4) { 1719 unsigned int prefetch; 1720 1721 prefetch = (id + 4) & (sp->s_rxq_size - 1); 1722 1723 srpp = srp->sr_srpp[prefetch]; 1724 ASSERT(srpp != NULL); 1725 1726 mp = srpp->srp_mp; 1727 prefetch_read_many(mp->b_datap); 1728 } else if (completed == srp->sr_pending) { 1729 prefetch_read_many(srp->sr_mp); 1730 } 1731 1732 srpp = srp->sr_srpp[id]; 1733 ASSERT(srpp != NULL); 1734 1735 srp->sr_srpp[id] = NULL; 1736 1737 mp = srpp->srp_mp; 1738 ASSERT(mp->b_cont == NULL); 1739 1740 /* when called from sfxge_rx_qstop() */ 1741 if (srp->sr_state != SFXGE_RXQ_STARTED) 1742 goto discard; 1743 1744 if (srpp->srp_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD)) 1745 goto discard; 1746 1747 /* Make the data visible to the kernel */ 1748 rc = ddi_dma_sync(srpp->srp_dma_handle, 0, 1749 sp->s_rx_buffer_size, DDI_DMA_SYNC_FORKERNEL); 1750 ASSERT3P(rc, ==, DDI_SUCCESS); 1751 1752 /* Read the length from the psuedo header if required */ 1753 if (srpp->srp_flags & EFX_PKT_PREFIX_LEN) { 1754 rc = efx_psuedo_hdr_pkt_length_get(sp->s_enp, 1755 mp->b_rptr, 1756 &srpp->srp_size); 1757 ASSERT3P(rc, ==, 0); 1758 srpp->srp_size += sp->s_rx_prefix_size; 1759 } 1760 1761 /* Set up the packet length */ 1762 ASSERT3P(mp->b_rptr, ==, DB_BASE(mp)); 1763 mp->b_rptr += sp->s_rx_prefix_size; 1764 1765 prefetch_read_many(mp->b_rptr); 1766 1767 ASSERT3P(mp->b_wptr, ==, DB_BASE(mp)); 1768 mp->b_wptr += (size_t)(srpp->srp_size); 1769 ASSERT3P(mp->b_wptr, <=, DB_LIM(mp)); 1770 1771 /* Calculate the maximum packet size */ 1772 size = sp->s_mtu; 1773 size += (srpp->srp_flags & EFX_PKT_VLAN_TAGGED) ? 1774 sizeof (struct ether_vlan_header) : 1775 sizeof (struct ether_header); 1776 1777 if (MBLKL(mp) > size) 1778 goto discard; 1779 1780 /* Check for loopback packets */ 1781 if (!(srpp->srp_flags & EFX_PKT_IPV4) && 1782 !(srpp->srp_flags & EFX_PKT_IPV6)) { 1783 struct ether_header *etherhp; 1784 1785 /*LINTED*/ 1786 etherhp = (struct ether_header *)(mp->b_rptr); 1787 1788 if (etherhp->ether_type == 1789 htons(SFXGE_ETHERTYPE_LOOPBACK)) { 1790 DTRACE_PROBE(loopback); 1791 1792 srp->sr_loopback++; 1793 goto discard; 1794 } 1795 } 1796 1797 /* Set up the checksum information */ 1798 flags = 0; 1799 1800 if (srpp->srp_flags & EFX_CKSUM_IPV4) { 1801 ASSERT(srpp->srp_flags & EFX_PKT_IPV4); 1802 flags |= HCK_IPV4_HDRCKSUM; 1803 } 1804 1805 if (srpp->srp_flags & EFX_CKSUM_TCPUDP) { 1806 ASSERT(srpp->srp_flags & EFX_PKT_TCP || 1807 srpp->srp_flags & EFX_PKT_UDP); 1808 flags |= HCK_FULLCKSUM | HCK_FULLCKSUM_OK; 1809 } 1810 1811 DB_CKSUMSTART(mp) = 0; 1812 DB_CKSUMSTUFF(mp) = 0; 1813 DB_CKSUMEND(mp) = 0; 1814 DB_CKSUMFLAGS(mp) = flags; 1815 DB_CKSUM16(mp) = 0; 1816 1817 /* Add the packet to the tail of the chain */ 1818 srfppp->srfpp_loaned++; 1819 1820 ASSERT(mp->b_next == NULL); 1821 *(srp->sr_mpp) = mp; 1822 srp->sr_mpp = &(mp->b_next); 1823 1824 continue; 1825 1826 discard: 1827 /* Return the packet to the pool */ 1828 srfppp->srfpp_loaned++; 1829 freeb(mp); /* Equivalent to freemsg() as b_cont==0 */ 1830 } 1831 srp->sr_completed = completed; 1832 1833 /* Attempt to coalesce any TCP packets */ 1834 if (sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF) 1835 sfxge_rx_qpacket_coalesce(srp); 1836 1837 /* 1838 * If there are any pending flows and this is the end of the 1839 * poll then they must be completed. 1840 */ 1841 if (srp->sr_srfp != NULL && eop) { 1842 sfxge_rx_flow_t *srfp; 1843 1844 srfp = srp->sr_srfp; 1845 1846 srp->sr_srfp = NULL; 1847 srp->sr_srfpp = &(srp->sr_srfp); 1848 1849 do { 1850 sfxge_rx_flow_t *next; 1851 1852 next = srfp->srf_next; 1853 srfp->srf_next = NULL; 1854 1855 sfxge_rx_qflow_complete(srp, srfp); 1856 1857 srfp = next; 1858 } while (srfp != NULL); 1859 } 1860 1861 level = srp->sr_pushed - srp->sr_completed; 1862 1863 /* If there are any packets then pass them up the stack */ 1864 if (srp->sr_mp != NULL) { 1865 mblk_t *mp; 1866 1867 mp = srp->sr_mp; 1868 1869 srp->sr_mp = NULL; 1870 srp->sr_mpp = &(srp->sr_mp); 1871 1872 if (level == 0) { 1873 /* Try to refill ASAP */ 1874 sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size)); 1875 level = srp->sr_pushed - srp->sr_completed; 1876 } 1877 1878 /* 1879 * If the RXQ is still empty, discard and recycle the 1880 * current entry to ensure that the ring always 1881 * contains at least one descriptor. This ensures that 1882 * the next hardware RX will trigger an event 1883 * (possibly delayed by interrupt moderation) and 1884 * trigger another refill/fill attempt. 1885 * 1886 * Note this drops a complete LRO fragment from the 1887 * start of the batch. 1888 * 1889 * Note also that copymsgchain() does not help with 1890 * resource starvation here, unless we are short of DMA 1891 * mappings. 1892 */ 1893 if (level == 0) { 1894 mblk_t *nmp; 1895 1896 srp->sr_kstat.srk_rxq_empty_discard++; 1897 DTRACE_PROBE1(rxq_empty_discard, int, index); 1898 nmp = mp->b_next; 1899 if (nmp) 1900 sfxge_gld_rx_post(sp, index, nmp); 1901 /* as level==0 will swizzle,rxpost below */ 1902 freemsg(mp); 1903 } else { 1904 sfxge_gld_rx_post(sp, index, mp); 1905 } 1906 } 1907 1908 /* Top up the queue if necessary */ 1909 if (level < srp->sr_hiwat) { 1910 sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size)); 1911 1912 level = srp->sr_added - srp->sr_completed; 1913 if (level < srp->sr_lowat) 1914 sfxge_rx_qfill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size)); 1915 } 1916 } 1917 1918 void 1919 sfxge_rx_qflush_done(sfxge_rxq_t *srp) 1920 { 1921 sfxge_t *sp = srp->sr_sp; 1922 unsigned int index = srp->sr_index; 1923 sfxge_evq_t *sep = sp->s_sep[index]; 1924 boolean_t flush_pending; 1925 1926 ASSERT(mutex_owned(&(sep->se_lock))); 1927 1928 /* 1929 * Flush successful: wakeup sfxge_rx_qstop() if flush is pending. 1930 * 1931 * A delayed flush event received after RxQ stop has timed out 1932 * will be ignored, as then the flush state will not be PENDING 1933 * (see SFCbug22989). 1934 */ 1935 flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING); 1936 srp->sr_flush = SFXGE_FLUSH_DONE; 1937 if (flush_pending) 1938 cv_broadcast(&(srp->sr_flush_kv)); 1939 } 1940 1941 void 1942 sfxge_rx_qflush_failed(sfxge_rxq_t *srp) 1943 { 1944 sfxge_t *sp = srp->sr_sp; 1945 unsigned int index = srp->sr_index; 1946 sfxge_evq_t *sep = sp->s_sep[index]; 1947 boolean_t flush_pending; 1948 1949 ASSERT(mutex_owned(&(sep->se_lock))); 1950 1951 /* 1952 * Flush failed: wakeup sfxge_rx_qstop() if flush is pending. 1953 * 1954 * A delayed flush event received after RxQ stop has timed out 1955 * will be ignored, as then the flush state will not be PENDING 1956 * (see SFCbug22989). 1957 */ 1958 flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING); 1959 srp->sr_flush = SFXGE_FLUSH_FAILED; 1960 if (flush_pending) 1961 cv_broadcast(&(srp->sr_flush_kv)); 1962 } 1963 1964 static void 1965 sfxge_rx_qstop(sfxge_t *sp, unsigned int index) 1966 { 1967 dev_info_t *dip = sp->s_dip; 1968 sfxge_evq_t *sep = sp->s_sep[index]; 1969 sfxge_rxq_t *srp; 1970 clock_t timeout; 1971 unsigned int flush_tries = SFXGE_RX_QFLUSH_TRIES; 1972 int rc; 1973 1974 ASSERT(mutex_owned(&(sp->s_state_lock))); 1975 1976 mutex_enter(&(sep->se_lock)); 1977 1978 srp = sp->s_srp[index]; 1979 ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED); 1980 1981 sfxge_rx_qpoll_stop(srp); 1982 1983 /* Further packets are discarded by sfxge_rx_qcomplete() */ 1984 srp->sr_state = SFXGE_RXQ_INITIALIZED; 1985 1986 if (sp->s_hw_err != SFXGE_HW_OK) { 1987 /* 1988 * Flag indicates possible hardware failure. 1989 * Attempt flush but do not wait for it to complete. 1990 */ 1991 srp->sr_flush = SFXGE_FLUSH_DONE; 1992 (void) efx_rx_qflush(srp->sr_erp); 1993 } 1994 1995 /* Wait upto 2sec for queue flushing to complete */ 1996 timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_RX_QFLUSH_USEC); 1997 1998 while (srp->sr_flush != SFXGE_FLUSH_DONE && flush_tries-- > 0) { 1999 if ((rc = efx_rx_qflush(srp->sr_erp)) != 0) { 2000 if (rc == EALREADY) 2001 srp->sr_flush = SFXGE_FLUSH_DONE; 2002 else 2003 srp->sr_flush = SFXGE_FLUSH_FAILED; 2004 break; 2005 } 2006 srp->sr_flush = SFXGE_FLUSH_PENDING; 2007 if (cv_timedwait(&(srp->sr_flush_kv), &(sep->se_lock), 2008 timeout) < 0) { 2009 /* Timeout waiting for successful or failed flush */ 2010 dev_err(dip, CE_NOTE, 2011 SFXGE_CMN_ERR "rxq[%d] flush timeout", index); 2012 break; 2013 } 2014 } 2015 2016 if (srp->sr_flush == SFXGE_FLUSH_FAILED) 2017 dev_err(dip, CE_NOTE, 2018 SFXGE_CMN_ERR "rxq[%d] flush failed", index); 2019 2020 DTRACE_PROBE1(flush, sfxge_flush_state_t, srp->sr_flush); 2021 srp->sr_flush = SFXGE_FLUSH_DONE; 2022 2023 /* Destroy the receive queue */ 2024 efx_rx_qdestroy(srp->sr_erp); 2025 srp->sr_erp = NULL; 2026 2027 /* Clear entries from the buffer table */ 2028 sfxge_sram_buf_tbl_clear(sp, srp->sr_id, 2029 EFX_RXQ_NBUFS(sp->s_rxq_size)); 2030 2031 /* 2032 * Free any unused RX packets which had descriptors on the RXQ 2033 * Packets will be discard as state != STARTED 2034 */ 2035 srp->sr_pending = srp->sr_added; 2036 sfxge_rx_qcomplete(srp, B_TRUE); 2037 2038 ASSERT3U(srp->sr_completed, ==, srp->sr_pending); 2039 2040 srp->sr_added = 0; 2041 srp->sr_pushed = 0; 2042 srp->sr_pending = 0; 2043 srp->sr_completed = 0; 2044 srp->sr_loopback = 0; 2045 2046 srp->sr_lowat = 0; 2047 srp->sr_hiwat = 0; 2048 2049 mutex_exit(&(sep->se_lock)); 2050 } 2051 2052 static void 2053 sfxge_rx_kstat_fini(sfxge_rxq_t *srp) 2054 { 2055 kstat_delete(srp->sr_ksp); 2056 srp->sr_ksp = NULL; 2057 } 2058 2059 static void 2060 sfxge_rx_qfini(sfxge_t *sp, unsigned int index) 2061 { 2062 sfxge_rxq_t *srp = sp->s_srp[index]; 2063 2064 ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED); 2065 2066 sp->s_srp[index] = NULL; 2067 srp->sr_state = SFXGE_RXQ_UNINITIALIZED; 2068 2069 sfxge_rx_kstat_fini(srp); 2070 2071 /* Empty the pool */ 2072 sfxge_rx_qfpp_empty(srp); 2073 2074 srp->sr_index = 0; 2075 2076 kmem_cache_free(sp->s_rqc, srp); 2077 } 2078 2079 static int 2080 sfxge_rx_scale_kstat_update(kstat_t *ksp, int rw) 2081 { 2082 sfxge_t *sp = ksp->ks_private; 2083 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2084 sfxge_intr_t *sip = &(sp->s_intr); 2085 kstat_named_t *knp; 2086 unsigned int index; 2087 unsigned int entry; 2088 unsigned int *freq; 2089 int rc; 2090 2091 ASSERT(mutex_owned(&(srsp->srs_lock))); 2092 2093 if (rw != KSTAT_READ) { 2094 rc = EACCES; 2095 goto fail1; 2096 } 2097 2098 if ((freq = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc, 2099 KM_NOSLEEP)) == NULL) { 2100 rc = ENOMEM; 2101 goto fail2; 2102 } 2103 2104 for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) { 2105 index = srsp->srs_tbl[entry]; 2106 2107 freq[index]++; 2108 } 2109 2110 knp = ksp->ks_data; 2111 for (index = 0; index < sip->si_nalloc; index++) { 2112 knp->value.ui64 = freq[index]; 2113 knp++; 2114 } 2115 2116 knp->value.ui64 = srsp->srs_count; 2117 2118 kmem_free(freq, sizeof (unsigned int) * sip->si_nalloc); 2119 2120 return (0); 2121 2122 fail2: 2123 DTRACE_PROBE(fail2); 2124 fail1: 2125 DTRACE_PROBE1(fail1, int, rc); 2126 return (rc); 2127 } 2128 2129 static int 2130 sfxge_rx_scale_kstat_init(sfxge_t *sp) 2131 { 2132 dev_info_t *dip = sp->s_dip; 2133 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2134 sfxge_intr_t *sip = &(sp->s_intr); 2135 char name[MAXNAMELEN]; 2136 kstat_t *ksp; 2137 kstat_named_t *knp; 2138 unsigned int index; 2139 int rc; 2140 2141 /* Create the set */ 2142 (void) snprintf(name, MAXNAMELEN - 1, "%s_rss", ddi_driver_name(dip)); 2143 2144 if ((ksp = kstat_create((char *)ddi_driver_name(dip), 2145 ddi_get_instance(dip), name, "rss", KSTAT_TYPE_NAMED, 2146 sip->si_nalloc + 1, 0)) == NULL) { 2147 rc = ENOMEM; 2148 goto fail1; 2149 } 2150 2151 srsp->srs_ksp = ksp; 2152 2153 ksp->ks_update = sfxge_rx_scale_kstat_update; 2154 ksp->ks_private = sp; 2155 ksp->ks_lock = &(srsp->srs_lock); 2156 2157 /* Initialise the named stats */ 2158 knp = ksp->ks_data; 2159 for (index = 0; index < sip->si_nalloc; index++) { 2160 char name[MAXNAMELEN]; 2161 2162 (void) snprintf(name, MAXNAMELEN - 1, "evq%04d_count", index); 2163 kstat_named_init(knp, name, KSTAT_DATA_UINT64); 2164 knp++; 2165 } 2166 2167 kstat_named_init(knp, "scale", KSTAT_DATA_UINT64); 2168 2169 kstat_install(ksp); 2170 return (0); 2171 2172 fail1: 2173 DTRACE_PROBE1(fail1, int, rc); 2174 2175 return (rc); 2176 } 2177 2178 static void 2179 sfxge_rx_scale_kstat_fini(sfxge_t *sp) 2180 { 2181 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2182 2183 /* Destroy the set */ 2184 kstat_delete(srsp->srs_ksp); 2185 srsp->srs_ksp = NULL; 2186 } 2187 2188 2189 unsigned int 2190 sfxge_rx_scale_prop_get(sfxge_t *sp) 2191 { 2192 int rx_scale; 2193 2194 rx_scale = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip, 2195 DDI_PROP_DONTPASS, "rx_scale_count", SFXGE_RX_SCALE_MAX); 2196 /* 0 and all -ve numbers sets to number of logical CPUs */ 2197 if (rx_scale <= 0) 2198 rx_scale = ncpus; 2199 2200 return (rx_scale); 2201 } 2202 2203 2204 static int 2205 sfxge_rx_scale_init(sfxge_t *sp) 2206 { 2207 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2208 sfxge_intr_t *sip = &(sp->s_intr); 2209 int rc; 2210 2211 ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_UNINITIALIZED); 2212 2213 /* Create tables for CPU, core, cache and chip counts */ 2214 srsp->srs_cpu = kmem_zalloc(sizeof (unsigned int) * NCPU, KM_SLEEP); 2215 2216 mutex_init(&(srsp->srs_lock), NULL, MUTEX_DRIVER, NULL); 2217 2218 /* We need at least one event queue */ 2219 srsp->srs_count = sfxge_rx_scale_prop_get(sp); 2220 if (srsp->srs_count > sip->si_nalloc) 2221 srsp->srs_count = sip->si_nalloc; 2222 if (srsp->srs_count < 1) 2223 srsp->srs_count = 1; 2224 2225 /* Set up the kstats */ 2226 if ((rc = sfxge_rx_scale_kstat_init(sp)) != 0) 2227 goto fail1; 2228 2229 srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED; 2230 2231 return (0); 2232 2233 fail1: 2234 DTRACE_PROBE1(fail1, int, rc); 2235 mutex_destroy(&(srsp->srs_lock)); 2236 2237 return (rc); 2238 } 2239 2240 void 2241 sfxge_rx_scale_update(void *arg) 2242 { 2243 sfxge_t *sp = arg; 2244 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2245 sfxge_intr_t *sip; 2246 processorid_t id; 2247 unsigned int count; 2248 unsigned int *tbl; 2249 unsigned int *rating; 2250 unsigned int entry; 2251 int rc; 2252 2253 mutex_enter(&(srsp->srs_lock)); 2254 2255 if (srsp->srs_state != SFXGE_RX_SCALE_STARTED) { 2256 rc = EFAULT; 2257 goto fail1; 2258 } 2259 2260 if ((tbl = kmem_zalloc(sizeof (unsigned int) * SFXGE_RX_SCALE_MAX, 2261 KM_NOSLEEP)) == NULL) { 2262 rc = ENOMEM; 2263 goto fail2; 2264 } 2265 2266 sip = &(sp->s_intr); 2267 if ((rating = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc, 2268 KM_NOSLEEP)) == NULL) { 2269 rc = ENOMEM; 2270 goto fail3; 2271 } 2272 2273 mutex_enter(&cpu_lock); 2274 2275 /* 2276 * Substract any current CPU, core, cache and chip usage from the 2277 * global contention tables. 2278 */ 2279 for (id = 0; id < NCPU; id++) { 2280 ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]); 2281 sfxge_cpu[id] -= srsp->srs_cpu[id]; 2282 srsp->srs_cpu[id] = 0; 2283 } 2284 2285 ASSERT(srsp->srs_count != 0); 2286 2287 /* Choose as many event queues as we need */ 2288 for (count = 0; count < srsp->srs_count; count++) { 2289 unsigned int index; 2290 sfxge_evq_t *sep; 2291 unsigned int choice; 2292 unsigned int choice_rating; 2293 2294 bzero(rating, sizeof (unsigned int) * sip->si_nalloc); 2295 2296 /* 2297 * Rate each event queue on its global level of CPU 2298 * contention. 2299 */ 2300 for (index = 0; index < sip->si_nalloc; index++) { 2301 sep = sp->s_sep[index]; 2302 2303 id = sep->se_cpu_id; 2304 rating[index] += sfxge_cpu[id]; 2305 } 2306 2307 /* Choose the queue with the lowest CPU contention */ 2308 choice = 0; 2309 choice_rating = rating[0]; 2310 2311 for (index = 1; index < sip->si_nalloc; index++) { 2312 if (rating[index] < choice_rating) { 2313 choice = index; 2314 choice_rating = rating[index]; 2315 } 2316 } 2317 2318 /* Add our choice to the condensed RSS table */ 2319 tbl[count] = choice; 2320 2321 /* Add information to the global contention tables */ 2322 sep = sp->s_sep[choice]; 2323 2324 id = sep->se_cpu_id; 2325 srsp->srs_cpu[id]++; 2326 sfxge_cpu[id]++; 2327 } 2328 2329 mutex_exit(&cpu_lock); 2330 2331 /* Build the expanded RSS table */ 2332 count = 0; 2333 for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) { 2334 unsigned int index; 2335 2336 index = tbl[count]; 2337 count = (count + 1) % srsp->srs_count; 2338 2339 srsp->srs_tbl[entry] = index; 2340 } 2341 2342 /* Program the expanded RSS table into the hardware */ 2343 (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl, 2344 SFXGE_RX_SCALE_MAX); 2345 2346 mutex_exit(&(srsp->srs_lock)); 2347 kmem_free(rating, sizeof (unsigned int) * sip->si_nalloc); 2348 kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX); 2349 return; 2350 2351 fail3: 2352 DTRACE_PROBE(fail3); 2353 kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX); 2354 fail2: 2355 DTRACE_PROBE(fail2); 2356 fail1: 2357 DTRACE_PROBE1(fail1, int, rc); 2358 2359 mutex_exit(&(srsp->srs_lock)); 2360 } 2361 2362 static int 2363 sfxge_rx_scale_start(sfxge_t *sp) 2364 { 2365 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2366 int rc; 2367 2368 mutex_enter(&(srsp->srs_lock)); 2369 2370 ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED); 2371 2372 /* Clear down the RSS table */ 2373 bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX); 2374 2375 (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl, 2376 SFXGE_RX_SCALE_MAX); 2377 2378 if ((rc = sfxge_toeplitz_hash_init(sp)) != 0) 2379 goto fail1; 2380 2381 srsp->srs_state = SFXGE_RX_SCALE_STARTED; 2382 2383 mutex_exit(&(srsp->srs_lock)); 2384 2385 /* sfxge_t->s_state_lock held */ 2386 (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp, 2387 DDI_SLEEP); 2388 2389 return (0); 2390 2391 fail1: 2392 DTRACE_PROBE1(fail1, int, rc); 2393 2394 mutex_exit(&(srsp->srs_lock)); 2395 2396 return (rc); 2397 } 2398 2399 int 2400 sfxge_rx_scale_count_get(sfxge_t *sp, unsigned int *countp) 2401 { 2402 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2403 int rc; 2404 2405 mutex_enter(&(srsp->srs_lock)); 2406 2407 if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED && 2408 srsp->srs_state != SFXGE_RX_SCALE_STARTED) { 2409 rc = ENOTSUP; 2410 goto fail1; 2411 } 2412 2413 *countp = srsp->srs_count; 2414 2415 mutex_exit(&(srsp->srs_lock)); 2416 2417 return (0); 2418 2419 fail1: 2420 DTRACE_PROBE1(fail1, int, rc); 2421 2422 mutex_exit(&(srsp->srs_lock)); 2423 2424 return (rc); 2425 } 2426 2427 int 2428 sfxge_rx_scale_count_set(sfxge_t *sp, unsigned int count) 2429 { 2430 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2431 sfxge_intr_t *sip = &(sp->s_intr); 2432 int dispatch = 1; 2433 int rc; 2434 2435 if (count < 1 || count > sip->si_nalloc) { 2436 rc = EINVAL; 2437 goto fail1; 2438 } 2439 2440 mutex_enter(&(srsp->srs_lock)); 2441 2442 if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED && 2443 srsp->srs_state != SFXGE_RX_SCALE_STARTED) { 2444 rc = ENOTSUP; 2445 goto fail2; 2446 } 2447 2448 srsp->srs_count = count; 2449 2450 if (srsp->srs_state != SFXGE_RX_SCALE_STARTED) 2451 dispatch = 0; 2452 2453 mutex_exit(&(srsp->srs_lock)); 2454 2455 if (dispatch) 2456 /* no locks held */ 2457 (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp, 2458 DDI_SLEEP); 2459 2460 return (0); 2461 2462 fail2: 2463 DTRACE_PROBE(fail2); 2464 2465 mutex_exit(&(srsp->srs_lock)); 2466 2467 fail1: 2468 DTRACE_PROBE1(fail1, int, rc); 2469 2470 return (rc); 2471 } 2472 2473 static void 2474 sfxge_rx_scale_stop(sfxge_t *sp) 2475 { 2476 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2477 processorid_t id; 2478 2479 mutex_enter(&(srsp->srs_lock)); 2480 2481 ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_STARTED); 2482 2483 srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED; 2484 2485 mutex_enter(&cpu_lock); 2486 2487 /* 2488 * Substract any current CPU, core, cache and chip usage from the 2489 * global contention tables. 2490 */ 2491 for (id = 0; id < NCPU; id++) { 2492 ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]); 2493 sfxge_cpu[id] -= srsp->srs_cpu[id]; 2494 srsp->srs_cpu[id] = 0; 2495 } 2496 2497 mutex_exit(&cpu_lock); 2498 2499 /* Clear down the RSS table */ 2500 bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX); 2501 2502 (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl, 2503 SFXGE_RX_SCALE_MAX); 2504 2505 mutex_exit(&(srsp->srs_lock)); 2506 } 2507 2508 static void 2509 sfxge_rx_scale_fini(sfxge_t *sp) 2510 { 2511 sfxge_rx_scale_t *srsp = &(sp->s_rx_scale); 2512 2513 ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED); 2514 2515 srsp->srs_state = SFXGE_RX_SCALE_UNINITIALIZED; 2516 2517 /* Tear down the kstats */ 2518 sfxge_rx_scale_kstat_fini(sp); 2519 2520 srsp->srs_count = 0; 2521 2522 mutex_destroy(&(srsp->srs_lock)); 2523 2524 /* Destroy tables */ 2525 kmem_free(srsp->srs_cpu, sizeof (unsigned int) * NCPU); 2526 srsp->srs_cpu = NULL; 2527 2528 sfxge_toeplitz_hash_fini(sp); 2529 } 2530 2531 int 2532 sfxge_rx_init(sfxge_t *sp) 2533 { 2534 sfxge_intr_t *sip = &(sp->s_intr); 2535 char name[MAXNAMELEN]; 2536 int index; 2537 int rc; 2538 2539 if (sip->si_state == SFXGE_INTR_UNINITIALIZED) { 2540 rc = EINVAL; 2541 goto fail1; 2542 } 2543 2544 if ((rc = sfxge_rx_scale_init(sp)) != 0) 2545 goto fail2; 2546 2547 (void) snprintf(name, MAXNAMELEN - 1, "%s%d_rx_packet_cache", 2548 ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip)); 2549 2550 sp->s_rpc = kmem_cache_create(name, sizeof (sfxge_rx_packet_t), 2551 SFXGE_CPU_CACHE_SIZE, sfxge_rx_packet_ctor, sfxge_rx_packet_dtor, 2552 NULL, sp, NULL, 0); 2553 ASSERT(sp->s_rpc != NULL); 2554 2555 (void) snprintf(name, MAXNAMELEN - 1, "%s%d_rxq_cache", 2556 ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip)); 2557 2558 sp->s_rqc = kmem_cache_create(name, sizeof (sfxge_rxq_t), 2559 SFXGE_CPU_CACHE_SIZE, sfxge_rx_qctor, sfxge_rx_qdtor, NULL, sp, 2560 NULL, 0); 2561 ASSERT(sp->s_rqc != NULL); 2562 2563 sp->s_rx_pkt_mem_max = ddi_prop_get_int64(DDI_DEV_T_ANY, sp->s_dip, 2564 DDI_PROP_DONTPASS, "rx_pkt_mem_max", 0); /* disabled */ 2565 2566 /* Initialize the receive queue(s) */ 2567 for (index = 0; index < sip->si_nalloc; index++) { 2568 if ((rc = sfxge_rx_qinit(sp, index)) != 0) 2569 goto fail3; 2570 } 2571 2572 sp->s_rx_coalesce_mode = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip, 2573 DDI_PROP_DONTPASS, "rx_coalesce_mode", SFXGE_RX_COALESCE_OFF); 2574 2575 return (0); 2576 2577 fail3: 2578 DTRACE_PROBE(fail3); 2579 2580 /* Tear down the receive queue(s) */ 2581 while (--index >= 0) 2582 sfxge_rx_qfini(sp, index); 2583 2584 kmem_cache_destroy(sp->s_rqc); 2585 sp->s_rqc = NULL; 2586 2587 kmem_cache_destroy(sp->s_rpc); 2588 sp->s_rpc = NULL; 2589 2590 sfxge_rx_scale_fini(sp); 2591 2592 fail2: 2593 DTRACE_PROBE(fail2); 2594 fail1: 2595 DTRACE_PROBE1(fail1, int, rc); 2596 2597 return (rc); 2598 } 2599 2600 int 2601 sfxge_rx_start(sfxge_t *sp) 2602 { 2603 sfxge_mac_t *smp = &(sp->s_mac); 2604 sfxge_intr_t *sip; 2605 const efx_nic_cfg_t *encp; 2606 size_t hdrlen, align; 2607 int index; 2608 int rc; 2609 2610 mutex_enter(&(smp->sm_lock)); 2611 2612 /* Calculate the receive packet buffer size and alignment */ 2613 sp->s_rx_buffer_size = EFX_MAC_PDU(sp->s_mtu); 2614 2615 encp = efx_nic_cfg_get(sp->s_enp); 2616 2617 /* Packet buffer allocations are cache line aligned */ 2618 EFSYS_ASSERT3U(encp->enc_rx_buf_align_start, <=, SFXGE_CPU_CACHE_SIZE); 2619 2620 if (sp->s_family == EFX_FAMILY_HUNTINGTON) { 2621 sp->s_rx_prefix_size = encp->enc_rx_prefix_size; 2622 2623 hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header); 2624 2625 /* Ensure IP headers are 32bit aligned */ 2626 sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen; 2627 sp->s_rx_buffer_size += sp->s_rx_buffer_align; 2628 2629 } else if (encp->enc_features & EFX_FEATURE_LFSR_HASH_INSERT) { 2630 sp->s_rx_prefix_size = encp->enc_rx_prefix_size; 2631 2632 /* 2633 * Place the start of the buffer a prefix length minus 2 2634 * before the start of a cache line. This ensures that the 2635 * last two bytes of the prefix (which is where the LFSR hash 2636 * is located) are in the same cache line as the headers, and 2637 * the IP header is 32-bit aligned. 2638 */ 2639 sp->s_rx_buffer_align = 2640 SFXGE_CPU_CACHE_SIZE - (encp->enc_rx_prefix_size - 2); 2641 sp->s_rx_buffer_size += sp->s_rx_buffer_align; 2642 } else { 2643 sp->s_rx_prefix_size = 0; 2644 2645 /* 2646 * Place the start of the buffer 2 bytes after a cache line 2647 * boundary so that the headers fit into the cache line and 2648 * the IP header is 32-bit aligned. 2649 */ 2650 hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header); 2651 2652 sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen; 2653 sp->s_rx_buffer_size += sp->s_rx_buffer_align; 2654 } 2655 2656 /* Align end of packet buffer for RX DMA end padding */ 2657 align = MAX(1, encp->enc_rx_buf_align_end); 2658 EFSYS_ASSERT(ISP2(align)); 2659 sp->s_rx_buffer_size = P2ROUNDUP(sp->s_rx_buffer_size, align); 2660 2661 /* Initialize the receive module */ 2662 if ((rc = efx_rx_init(sp->s_enp)) != 0) 2663 goto fail1; 2664 2665 mutex_exit(&(smp->sm_lock)); 2666 2667 if ((rc = sfxge_rx_scale_start(sp)) != 0) 2668 goto fail2; 2669 2670 /* Start the receive queue(s) */ 2671 sip = &(sp->s_intr); 2672 for (index = 0; index < sip->si_nalloc; index++) { 2673 if ((rc = sfxge_rx_qstart(sp, index)) != 0) 2674 goto fail3; 2675 } 2676 2677 ASSERT3U(sp->s_srp[0]->sr_state, ==, SFXGE_RXQ_STARTED); 2678 /* It is sufficient to have Rx scale initialized */ 2679 ASSERT3U(sp->s_rx_scale.srs_state, ==, SFXGE_RX_SCALE_STARTED); 2680 rc = efx_mac_filter_default_rxq_set(sp->s_enp, sp->s_srp[0]->sr_erp, 2681 sp->s_rx_scale.srs_count > 1); 2682 if (rc != 0) 2683 goto fail4; 2684 2685 return (0); 2686 2687 fail4: 2688 DTRACE_PROBE(fail4); 2689 2690 fail3: 2691 DTRACE_PROBE(fail3); 2692 2693 /* Stop the receive queue(s) */ 2694 while (--index >= 0) 2695 sfxge_rx_qstop(sp, index); 2696 2697 sfxge_rx_scale_stop(sp); 2698 2699 fail2: 2700 DTRACE_PROBE(fail2); 2701 2702 mutex_enter(&(smp->sm_lock)); 2703 2704 /* Tear down the receive module */ 2705 efx_rx_fini(sp->s_enp); 2706 2707 fail1: 2708 DTRACE_PROBE1(fail1, int, rc); 2709 2710 mutex_exit(&(smp->sm_lock)); 2711 2712 return (rc); 2713 } 2714 2715 void 2716 sfxge_rx_coalesce_mode_get(sfxge_t *sp, sfxge_rx_coalesce_mode_t *modep) 2717 { 2718 *modep = sp->s_rx_coalesce_mode; 2719 } 2720 2721 int 2722 sfxge_rx_coalesce_mode_set(sfxge_t *sp, sfxge_rx_coalesce_mode_t mode) 2723 { 2724 int rc; 2725 2726 switch (mode) { 2727 case SFXGE_RX_COALESCE_OFF: 2728 case SFXGE_RX_COALESCE_DISALLOW_PUSH: 2729 case SFXGE_RX_COALESCE_ALLOW_PUSH: 2730 break; 2731 2732 default: 2733 rc = EINVAL; 2734 goto fail1; 2735 } 2736 2737 sp->s_rx_coalesce_mode = mode; 2738 2739 return (0); 2740 2741 fail1: 2742 DTRACE_PROBE1(fail1, int, rc); 2743 2744 return (rc); 2745 } 2746 2747 void 2748 sfxge_rx_stop(sfxge_t *sp) 2749 { 2750 sfxge_mac_t *smp = &(sp->s_mac); 2751 sfxge_intr_t *sip = &(sp->s_intr); 2752 efx_nic_t *enp = sp->s_enp; 2753 int index; 2754 2755 ASSERT(mutex_owned(&(sp->s_state_lock))); 2756 2757 efx_mac_filter_default_rxq_clear(enp); 2758 2759 /* Stop the receive queue(s) */ 2760 index = sip->si_nalloc; 2761 while (--index >= 0) { 2762 /* TBD: Flush RXQs in parallel; HW has limit + may need retry */ 2763 sfxge_rx_qstop(sp, index); 2764 } 2765 2766 sfxge_rx_scale_stop(sp); 2767 2768 mutex_enter(&(smp->sm_lock)); 2769 2770 /* Tear down the receive module */ 2771 efx_rx_fini(enp); 2772 2773 sp->s_rx_buffer_align = 0; 2774 sp->s_rx_prefix_size = 0; 2775 sp->s_rx_buffer_size = 0; 2776 2777 mutex_exit(&(smp->sm_lock)); 2778 } 2779 2780 unsigned int 2781 sfxge_rx_loaned(sfxge_t *sp) 2782 { 2783 sfxge_intr_t *sip = &(sp->s_intr); 2784 int index; 2785 unsigned int loaned; 2786 2787 ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED); 2788 2789 loaned = 0; 2790 for (index = 0; index < sip->si_nalloc; index++) { 2791 sfxge_rxq_t *srp = sp->s_srp[index]; 2792 sfxge_evq_t *sep = sp->s_sep[srp->sr_index]; 2793 2794 mutex_enter(&(sep->se_lock)); 2795 2796 loaned += sfxge_rx_qfpp_swizzle(srp); 2797 2798 mutex_exit(&(sep->se_lock)); 2799 } 2800 2801 return (loaned); 2802 } 2803 2804 void 2805 sfxge_rx_fini(sfxge_t *sp) 2806 { 2807 sfxge_intr_t *sip = &(sp->s_intr); 2808 int index; 2809 2810 ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED); 2811 2812 sp->s_rx_coalesce_mode = SFXGE_RX_COALESCE_OFF; 2813 2814 /* Tear down the receive queue(s) */ 2815 index = sip->si_nalloc; 2816 while (--index >= 0) 2817 sfxge_rx_qfini(sp, index); 2818 2819 ASSERT3U(sp->s_rx_pkt_mem_alloc, ==, 0); 2820 2821 kmem_cache_destroy(sp->s_rqc); 2822 sp->s_rqc = NULL; 2823 2824 kmem_cache_destroy(sp->s_rpc); 2825 sp->s_rpc = NULL; 2826 2827 sfxge_rx_scale_fini(sp); 2828 } 2829