1 /* 2 * Copyright (c) 2007-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include "efx.h" 32 #include "efx_impl.h" 33 34 #if EFSYS_OPT_QSTATS 35 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 36 do { \ 37 (_etp)->et_stat[_stat]++; \ 38 _NOTE(CONSTANTCONDITION) \ 39 } while (B_FALSE) 40 #else 41 #define EFX_TX_QSTAT_INCR(_etp, _stat) 42 #endif 43 44 #if EFSYS_OPT_SIENA 45 46 static __checkReturn efx_rc_t 47 siena_tx_init( 48 __in efx_nic_t *enp); 49 50 static void 51 siena_tx_fini( 52 __in efx_nic_t *enp); 53 54 static __checkReturn efx_rc_t 55 siena_tx_qcreate( 56 __in efx_nic_t *enp, 57 __in unsigned int index, 58 __in unsigned int label, 59 __in efsys_mem_t *esmp, 60 __in size_t n, 61 __in uint32_t id, 62 __in uint16_t flags, 63 __in efx_evq_t *eep, 64 __in efx_txq_t *etp, 65 __out unsigned int *addedp); 66 67 static void 68 siena_tx_qdestroy( 69 __in efx_txq_t *etp); 70 71 static __checkReturn efx_rc_t 72 siena_tx_qpost( 73 __in efx_txq_t *etp, 74 __in_ecount(n) efx_buffer_t *eb, 75 __in unsigned int n, 76 __in unsigned int completed, 77 __inout unsigned int *addedp); 78 79 static void 80 siena_tx_qpush( 81 __in efx_txq_t *etp, 82 __in unsigned int added, 83 __in unsigned int pushed); 84 85 static __checkReturn efx_rc_t 86 siena_tx_qpace( 87 __in efx_txq_t *etp, 88 __in unsigned int ns); 89 90 static __checkReturn efx_rc_t 91 siena_tx_qflush( 92 __in efx_txq_t *etp); 93 94 static void 95 siena_tx_qenable( 96 __in efx_txq_t *etp); 97 98 __checkReturn efx_rc_t 99 siena_tx_qdesc_post( 100 __in efx_txq_t *etp, 101 __in_ecount(n) efx_desc_t *ed, 102 __in unsigned int n, 103 __in unsigned int completed, 104 __inout unsigned int *addedp); 105 106 void 107 siena_tx_qdesc_dma_create( 108 __in efx_txq_t *etp, 109 __in efsys_dma_addr_t addr, 110 __in size_t size, 111 __in boolean_t eop, 112 __out efx_desc_t *edp); 113 114 #if EFSYS_OPT_QSTATS 115 static void 116 siena_tx_qstats_update( 117 __in efx_txq_t *etp, 118 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 119 #endif 120 121 #endif /* EFSYS_OPT_SIENA */ 122 123 124 #if EFSYS_OPT_SIENA 125 static const efx_tx_ops_t __efx_tx_siena_ops = { 126 siena_tx_init, /* etxo_init */ 127 siena_tx_fini, /* etxo_fini */ 128 siena_tx_qcreate, /* etxo_qcreate */ 129 siena_tx_qdestroy, /* etxo_qdestroy */ 130 siena_tx_qpost, /* etxo_qpost */ 131 siena_tx_qpush, /* etxo_qpush */ 132 siena_tx_qpace, /* etxo_qpace */ 133 siena_tx_qflush, /* etxo_qflush */ 134 siena_tx_qenable, /* etxo_qenable */ 135 NULL, /* etxo_qpio_enable */ 136 NULL, /* etxo_qpio_disable */ 137 NULL, /* etxo_qpio_write */ 138 NULL, /* etxo_qpio_post */ 139 siena_tx_qdesc_post, /* etxo_qdesc_post */ 140 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 141 NULL, /* etxo_qdesc_tso_create */ 142 NULL, /* etxo_qdesc_tso2_create */ 143 NULL, /* etxo_qdesc_vlantci_create */ 144 #if EFSYS_OPT_QSTATS 145 siena_tx_qstats_update, /* etxo_qstats_update */ 146 #endif 147 }; 148 #endif /* EFSYS_OPT_SIENA */ 149 150 #if EFSYS_OPT_HUNTINGTON 151 static const efx_tx_ops_t __efx_tx_hunt_ops = { 152 ef10_tx_init, /* etxo_init */ 153 ef10_tx_fini, /* etxo_fini */ 154 ef10_tx_qcreate, /* etxo_qcreate */ 155 ef10_tx_qdestroy, /* etxo_qdestroy */ 156 ef10_tx_qpost, /* etxo_qpost */ 157 ef10_tx_qpush, /* etxo_qpush */ 158 ef10_tx_qpace, /* etxo_qpace */ 159 ef10_tx_qflush, /* etxo_qflush */ 160 ef10_tx_qenable, /* etxo_qenable */ 161 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 162 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 163 ef10_tx_qpio_write, /* etxo_qpio_write */ 164 ef10_tx_qpio_post, /* etxo_qpio_post */ 165 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 166 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 167 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 168 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 169 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 170 #if EFSYS_OPT_QSTATS 171 ef10_tx_qstats_update, /* etxo_qstats_update */ 172 #endif 173 }; 174 #endif /* EFSYS_OPT_HUNTINGTON */ 175 176 #if EFSYS_OPT_MEDFORD 177 static const efx_tx_ops_t __efx_tx_medford_ops = { 178 ef10_tx_init, /* etxo_init */ 179 ef10_tx_fini, /* etxo_fini */ 180 ef10_tx_qcreate, /* etxo_qcreate */ 181 ef10_tx_qdestroy, /* etxo_qdestroy */ 182 ef10_tx_qpost, /* etxo_qpost */ 183 ef10_tx_qpush, /* etxo_qpush */ 184 ef10_tx_qpace, /* etxo_qpace */ 185 ef10_tx_qflush, /* etxo_qflush */ 186 ef10_tx_qenable, /* etxo_qenable */ 187 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 188 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 189 ef10_tx_qpio_write, /* etxo_qpio_write */ 190 ef10_tx_qpio_post, /* etxo_qpio_post */ 191 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 192 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 193 NULL, /* etxo_qdesc_tso_create */ 194 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 195 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 196 #if EFSYS_OPT_QSTATS 197 ef10_tx_qstats_update, /* etxo_qstats_update */ 198 #endif 199 }; 200 #endif /* EFSYS_OPT_MEDFORD */ 201 202 __checkReturn efx_rc_t 203 efx_tx_init( 204 __in efx_nic_t *enp) 205 { 206 const efx_tx_ops_t *etxop; 207 efx_rc_t rc; 208 209 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 210 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 211 212 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 213 rc = EINVAL; 214 goto fail1; 215 } 216 217 if (enp->en_mod_flags & EFX_MOD_TX) { 218 rc = EINVAL; 219 goto fail2; 220 } 221 222 switch (enp->en_family) { 223 #if EFSYS_OPT_SIENA 224 case EFX_FAMILY_SIENA: 225 etxop = &__efx_tx_siena_ops; 226 break; 227 #endif /* EFSYS_OPT_SIENA */ 228 229 #if EFSYS_OPT_HUNTINGTON 230 case EFX_FAMILY_HUNTINGTON: 231 etxop = &__efx_tx_hunt_ops; 232 break; 233 #endif /* EFSYS_OPT_HUNTINGTON */ 234 235 #if EFSYS_OPT_MEDFORD 236 case EFX_FAMILY_MEDFORD: 237 etxop = &__efx_tx_medford_ops; 238 break; 239 #endif /* EFSYS_OPT_MEDFORD */ 240 241 default: 242 EFSYS_ASSERT(0); 243 rc = ENOTSUP; 244 goto fail3; 245 } 246 247 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 248 249 if ((rc = etxop->etxo_init(enp)) != 0) 250 goto fail4; 251 252 enp->en_etxop = etxop; 253 enp->en_mod_flags |= EFX_MOD_TX; 254 return (0); 255 256 fail4: 257 EFSYS_PROBE(fail4); 258 fail3: 259 EFSYS_PROBE(fail3); 260 fail2: 261 EFSYS_PROBE(fail2); 262 fail1: 263 EFSYS_PROBE1(fail1, efx_rc_t, rc); 264 265 enp->en_etxop = NULL; 266 enp->en_mod_flags &= ~EFX_MOD_TX; 267 return (rc); 268 } 269 270 void 271 efx_tx_fini( 272 __in efx_nic_t *enp) 273 { 274 const efx_tx_ops_t *etxop = enp->en_etxop; 275 276 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 277 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 278 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 279 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 280 281 etxop->etxo_fini(enp); 282 283 enp->en_etxop = NULL; 284 enp->en_mod_flags &= ~EFX_MOD_TX; 285 } 286 287 __checkReturn efx_rc_t 288 efx_tx_qcreate( 289 __in efx_nic_t *enp, 290 __in unsigned int index, 291 __in unsigned int label, 292 __in efsys_mem_t *esmp, 293 __in size_t n, 294 __in uint32_t id, 295 __in uint16_t flags, 296 __in efx_evq_t *eep, 297 __deref_out efx_txq_t **etpp, 298 __out unsigned int *addedp) 299 { 300 const efx_tx_ops_t *etxop = enp->en_etxop; 301 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 302 efx_txq_t *etp; 303 efx_rc_t rc; 304 305 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 306 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 307 308 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 309 310 /* Allocate an TXQ object */ 311 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 312 313 if (etp == NULL) { 314 rc = ENOMEM; 315 goto fail1; 316 } 317 318 etp->et_magic = EFX_TXQ_MAGIC; 319 etp->et_enp = enp; 320 etp->et_index = index; 321 etp->et_mask = n - 1; 322 etp->et_esmp = esmp; 323 324 /* Initial descriptor index may be modified by etxo_qcreate */ 325 *addedp = 0; 326 327 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 328 n, id, flags, eep, etp, addedp)) != 0) 329 goto fail2; 330 331 enp->en_tx_qcount++; 332 *etpp = etp; 333 334 return (0); 335 336 fail2: 337 EFSYS_PROBE(fail2); 338 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 339 fail1: 340 EFSYS_PROBE1(fail1, efx_rc_t, rc); 341 return (rc); 342 } 343 344 void 345 efx_tx_qdestroy( 346 __in efx_txq_t *etp) 347 { 348 efx_nic_t *enp = etp->et_enp; 349 const efx_tx_ops_t *etxop = enp->en_etxop; 350 351 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 352 353 EFSYS_ASSERT(enp->en_tx_qcount != 0); 354 --enp->en_tx_qcount; 355 356 etxop->etxo_qdestroy(etp); 357 358 /* Free the TXQ object */ 359 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 360 } 361 362 __checkReturn efx_rc_t 363 efx_tx_qpost( 364 __in efx_txq_t *etp, 365 __in_ecount(n) efx_buffer_t *eb, 366 __in unsigned int n, 367 __in unsigned int completed, 368 __inout unsigned int *addedp) 369 { 370 efx_nic_t *enp = etp->et_enp; 371 const efx_tx_ops_t *etxop = enp->en_etxop; 372 efx_rc_t rc; 373 374 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 375 376 if ((rc = etxop->etxo_qpost(etp, eb, 377 n, completed, addedp)) != 0) 378 goto fail1; 379 380 return (0); 381 382 fail1: 383 EFSYS_PROBE1(fail1, efx_rc_t, rc); 384 return (rc); 385 } 386 387 void 388 efx_tx_qpush( 389 __in efx_txq_t *etp, 390 __in unsigned int added, 391 __in unsigned int pushed) 392 { 393 efx_nic_t *enp = etp->et_enp; 394 const efx_tx_ops_t *etxop = enp->en_etxop; 395 396 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 397 398 etxop->etxo_qpush(etp, added, pushed); 399 } 400 401 __checkReturn efx_rc_t 402 efx_tx_qpace( 403 __in efx_txq_t *etp, 404 __in unsigned int ns) 405 { 406 efx_nic_t *enp = etp->et_enp; 407 const efx_tx_ops_t *etxop = enp->en_etxop; 408 efx_rc_t rc; 409 410 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 411 412 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 413 goto fail1; 414 415 return (0); 416 417 fail1: 418 EFSYS_PROBE1(fail1, efx_rc_t, rc); 419 return (rc); 420 } 421 422 __checkReturn efx_rc_t 423 efx_tx_qflush( 424 __in efx_txq_t *etp) 425 { 426 efx_nic_t *enp = etp->et_enp; 427 const efx_tx_ops_t *etxop = enp->en_etxop; 428 efx_rc_t rc; 429 430 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 431 432 if ((rc = etxop->etxo_qflush(etp)) != 0) 433 goto fail1; 434 435 return (0); 436 437 fail1: 438 EFSYS_PROBE1(fail1, efx_rc_t, rc); 439 return (rc); 440 } 441 442 void 443 efx_tx_qenable( 444 __in efx_txq_t *etp) 445 { 446 efx_nic_t *enp = etp->et_enp; 447 const efx_tx_ops_t *etxop = enp->en_etxop; 448 449 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 450 451 etxop->etxo_qenable(etp); 452 } 453 454 __checkReturn efx_rc_t 455 efx_tx_qpio_enable( 456 __in efx_txq_t *etp) 457 { 458 efx_nic_t *enp = etp->et_enp; 459 const efx_tx_ops_t *etxop = enp->en_etxop; 460 efx_rc_t rc; 461 462 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 463 464 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 465 rc = ENOTSUP; 466 goto fail1; 467 } 468 if (etxop->etxo_qpio_enable == NULL) { 469 rc = ENOTSUP; 470 goto fail2; 471 } 472 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 473 goto fail3; 474 475 return (0); 476 477 fail3: 478 EFSYS_PROBE(fail3); 479 fail2: 480 EFSYS_PROBE(fail2); 481 fail1: 482 EFSYS_PROBE1(fail1, efx_rc_t, rc); 483 return (rc); 484 } 485 486 void 487 efx_tx_qpio_disable( 488 __in efx_txq_t *etp) 489 { 490 efx_nic_t *enp = etp->et_enp; 491 const efx_tx_ops_t *etxop = enp->en_etxop; 492 493 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 494 495 if (etxop->etxo_qpio_disable != NULL) 496 etxop->etxo_qpio_disable(etp); 497 } 498 499 __checkReturn efx_rc_t 500 efx_tx_qpio_write( 501 __in efx_txq_t *etp, 502 __in_ecount(buf_length) uint8_t *buffer, 503 __in size_t buf_length, 504 __in size_t pio_buf_offset) 505 { 506 efx_nic_t *enp = etp->et_enp; 507 const efx_tx_ops_t *etxop = enp->en_etxop; 508 efx_rc_t rc; 509 510 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 511 512 if (etxop->etxo_qpio_write != NULL) { 513 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 514 pio_buf_offset)) != 0) 515 goto fail1; 516 return (0); 517 } 518 519 return (ENOTSUP); 520 521 fail1: 522 EFSYS_PROBE1(fail1, efx_rc_t, rc); 523 return (rc); 524 } 525 526 __checkReturn efx_rc_t 527 efx_tx_qpio_post( 528 __in efx_txq_t *etp, 529 __in size_t pkt_length, 530 __in unsigned int completed, 531 __inout unsigned int *addedp) 532 { 533 efx_nic_t *enp = etp->et_enp; 534 const efx_tx_ops_t *etxop = enp->en_etxop; 535 efx_rc_t rc; 536 537 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 538 539 if (etxop->etxo_qpio_post != NULL) { 540 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 541 addedp)) != 0) 542 goto fail1; 543 return (0); 544 } 545 546 return (ENOTSUP); 547 548 fail1: 549 EFSYS_PROBE1(fail1, efx_rc_t, rc); 550 return (rc); 551 } 552 553 __checkReturn efx_rc_t 554 efx_tx_qdesc_post( 555 __in efx_txq_t *etp, 556 __in_ecount(n) efx_desc_t *ed, 557 __in unsigned int n, 558 __in unsigned int completed, 559 __inout unsigned int *addedp) 560 { 561 efx_nic_t *enp = etp->et_enp; 562 const efx_tx_ops_t *etxop = enp->en_etxop; 563 efx_rc_t rc; 564 565 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 566 567 if ((rc = etxop->etxo_qdesc_post(etp, ed, 568 n, completed, addedp)) != 0) 569 goto fail1; 570 571 return (0); 572 573 fail1: 574 EFSYS_PROBE1(fail1, efx_rc_t, rc); 575 return (rc); 576 } 577 578 void 579 efx_tx_qdesc_dma_create( 580 __in efx_txq_t *etp, 581 __in efsys_dma_addr_t addr, 582 __in size_t size, 583 __in boolean_t eop, 584 __out efx_desc_t *edp) 585 { 586 efx_nic_t *enp = etp->et_enp; 587 const efx_tx_ops_t *etxop = enp->en_etxop; 588 589 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 590 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 591 592 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 593 } 594 595 void 596 efx_tx_qdesc_tso_create( 597 __in efx_txq_t *etp, 598 __in uint16_t ipv4_id, 599 __in uint32_t tcp_seq, 600 __in uint8_t tcp_flags, 601 __out efx_desc_t *edp) 602 { 603 efx_nic_t *enp = etp->et_enp; 604 const efx_tx_ops_t *etxop = enp->en_etxop; 605 606 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 607 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 608 609 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 610 } 611 612 void 613 efx_tx_qdesc_tso2_create( 614 __in efx_txq_t *etp, 615 __in uint16_t ipv4_id, 616 __in uint32_t tcp_seq, 617 __in uint16_t mss, 618 __out_ecount(count) efx_desc_t *edp, 619 __in int count) 620 { 621 efx_nic_t *enp = etp->et_enp; 622 const efx_tx_ops_t *etxop = enp->en_etxop; 623 624 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 625 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 626 627 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count); 628 } 629 630 void 631 efx_tx_qdesc_vlantci_create( 632 __in efx_txq_t *etp, 633 __in uint16_t tci, 634 __out efx_desc_t *edp) 635 { 636 efx_nic_t *enp = etp->et_enp; 637 const efx_tx_ops_t *etxop = enp->en_etxop; 638 639 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 640 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 641 642 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 643 } 644 645 646 #if EFSYS_OPT_QSTATS 647 void 648 efx_tx_qstats_update( 649 __in efx_txq_t *etp, 650 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 651 { 652 efx_nic_t *enp = etp->et_enp; 653 const efx_tx_ops_t *etxop = enp->en_etxop; 654 655 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 656 657 etxop->etxo_qstats_update(etp, stat); 658 } 659 #endif 660 661 662 #if EFSYS_OPT_SIENA 663 664 static __checkReturn efx_rc_t 665 siena_tx_init( 666 __in efx_nic_t *enp) 667 { 668 efx_oword_t oword; 669 670 /* 671 * Disable the timer-based TX DMA backoff and allow TX DMA to be 672 * controlled by the RX FIFO fill level (although always allow a 673 * minimal trickle). 674 */ 675 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 676 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 677 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 678 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 679 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 680 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 682 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 683 684 /* 685 * Filter all packets less than 14 bytes to avoid parsing 686 * errors. 687 */ 688 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 689 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 690 691 /* 692 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 693 * descriptors (which is bad). 694 */ 695 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 696 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 697 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 698 699 return (0); 700 } 701 702 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 703 do { \ 704 unsigned int id; \ 705 size_t offset; \ 706 efx_qword_t qword; \ 707 \ 708 id = (_added)++ & (_etp)->et_mask; \ 709 offset = id * sizeof (efx_qword_t); \ 710 \ 711 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 712 unsigned int, id, efsys_dma_addr_t, (_addr), \ 713 size_t, (_size), boolean_t, (_eop)); \ 714 \ 715 EFX_POPULATE_QWORD_4(qword, \ 716 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 717 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 718 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 719 (uint32_t)((_addr) & 0xffffffff), \ 720 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 721 (uint32_t)((_addr) >> 32)); \ 722 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 723 \ 724 _NOTE(CONSTANTCONDITION) \ 725 } while (B_FALSE) 726 727 static __checkReturn efx_rc_t 728 siena_tx_qpost( 729 __in efx_txq_t *etp, 730 __in_ecount(n) efx_buffer_t *eb, 731 __in unsigned int n, 732 __in unsigned int completed, 733 __inout unsigned int *addedp) 734 { 735 unsigned int added = *addedp; 736 unsigned int i; 737 int rc = ENOSPC; 738 739 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 740 goto fail1; 741 742 for (i = 0; i < n; i++) { 743 efx_buffer_t *ebp = &eb[i]; 744 efsys_dma_addr_t start = ebp->eb_addr; 745 size_t size = ebp->eb_size; 746 efsys_dma_addr_t end = start + size; 747 748 /* Fragments must not span 4k boundaries. */ 749 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 750 751 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 752 } 753 754 EFX_TX_QSTAT_INCR(etp, TX_POST); 755 756 *addedp = added; 757 return (0); 758 759 fail1: 760 EFSYS_PROBE1(fail1, efx_rc_t, rc); 761 762 return (rc); 763 } 764 765 static void 766 siena_tx_qpush( 767 __in efx_txq_t *etp, 768 __in unsigned int added, 769 __in unsigned int pushed) 770 { 771 efx_nic_t *enp = etp->et_enp; 772 uint32_t wptr; 773 efx_dword_t dword; 774 efx_oword_t oword; 775 776 /* Push the populated descriptors out */ 777 wptr = added & etp->et_mask; 778 779 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 780 781 /* Only write the third DWORD */ 782 EFX_POPULATE_DWORD_1(dword, 783 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 784 785 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 786 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 787 wptr, pushed & etp->et_mask); 788 EFSYS_PIO_WRITE_BARRIER(); 789 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 790 etp->et_index, &dword, B_FALSE); 791 } 792 793 #define EFX_MAX_PACE_VALUE 20 794 795 static __checkReturn efx_rc_t 796 siena_tx_qpace( 797 __in efx_txq_t *etp, 798 __in unsigned int ns) 799 { 800 efx_nic_t *enp = etp->et_enp; 801 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 802 efx_oword_t oword; 803 unsigned int pace_val; 804 unsigned int timer_period; 805 efx_rc_t rc; 806 807 if (ns == 0) { 808 pace_val = 0; 809 } else { 810 /* 811 * The pace_val to write into the table is s.t 812 * ns <= timer_period * (2 ^ pace_val) 813 */ 814 timer_period = 104 / encp->enc_clk_mult; 815 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 816 if ((timer_period << pace_val) >= ns) 817 break; 818 } 819 } 820 if (pace_val > EFX_MAX_PACE_VALUE) { 821 rc = EINVAL; 822 goto fail1; 823 } 824 825 /* Update the pacing table */ 826 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 827 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 828 &oword, B_TRUE); 829 830 return (0); 831 832 fail1: 833 EFSYS_PROBE1(fail1, efx_rc_t, rc); 834 835 return (rc); 836 } 837 838 static __checkReturn efx_rc_t 839 siena_tx_qflush( 840 __in efx_txq_t *etp) 841 { 842 efx_nic_t *enp = etp->et_enp; 843 efx_oword_t oword; 844 uint32_t label; 845 846 (void) efx_tx_qpace(etp, 0); 847 848 label = etp->et_index; 849 850 /* Flush the queue */ 851 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 852 FRF_AZ_TX_FLUSH_DESCQ, label); 853 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 854 855 return (0); 856 } 857 858 static void 859 siena_tx_qenable( 860 __in efx_txq_t *etp) 861 { 862 efx_nic_t *enp = etp->et_enp; 863 efx_oword_t oword; 864 865 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 866 etp->et_index, &oword, B_TRUE); 867 868 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 869 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 870 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 871 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 872 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 873 874 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 875 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 876 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 877 878 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 879 etp->et_index, &oword, B_TRUE); 880 } 881 882 static __checkReturn efx_rc_t 883 siena_tx_qcreate( 884 __in efx_nic_t *enp, 885 __in unsigned int index, 886 __in unsigned int label, 887 __in efsys_mem_t *esmp, 888 __in size_t n, 889 __in uint32_t id, 890 __in uint16_t flags, 891 __in efx_evq_t *eep, 892 __in efx_txq_t *etp, 893 __out unsigned int *addedp) 894 { 895 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 896 efx_oword_t oword; 897 uint32_t size; 898 efx_rc_t rc; 899 900 _NOTE(ARGUNUSED(esmp)); 901 902 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 903 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 904 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 905 906 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp))); 907 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 908 909 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { 910 rc = EINVAL; 911 goto fail1; 912 } 913 if (index >= encp->enc_txq_limit) { 914 rc = EINVAL; 915 goto fail2; 916 } 917 for (size = 0; 918 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS); 919 size++) 920 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 921 break; 922 if (id + (1 << size) >= encp->enc_buftbl_limit) { 923 rc = EINVAL; 924 goto fail3; 925 } 926 927 /* Set up the new descriptor queue */ 928 *addedp = 0; 929 930 EFX_POPULATE_OWORD_6(oword, 931 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 932 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 933 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 934 FRF_AZ_TX_DESCQ_LABEL, label, 935 FRF_AZ_TX_DESCQ_SIZE, size, 936 FRF_AZ_TX_DESCQ_TYPE, 0); 937 938 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 939 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 940 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 941 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 942 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 943 944 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 945 etp->et_index, &oword, B_TRUE); 946 947 return (0); 948 949 fail3: 950 EFSYS_PROBE(fail3); 951 fail2: 952 EFSYS_PROBE(fail2); 953 fail1: 954 EFSYS_PROBE1(fail1, efx_rc_t, rc); 955 956 return (rc); 957 } 958 959 __checkReturn efx_rc_t 960 siena_tx_qdesc_post( 961 __in efx_txq_t *etp, 962 __in_ecount(n) efx_desc_t *ed, 963 __in unsigned int n, 964 __in unsigned int completed, 965 __inout unsigned int *addedp) 966 { 967 unsigned int added = *addedp; 968 unsigned int i; 969 efx_rc_t rc; 970 971 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 972 rc = ENOSPC; 973 goto fail1; 974 } 975 976 for (i = 0; i < n; i++) { 977 efx_desc_t *edp = &ed[i]; 978 unsigned int id; 979 size_t offset; 980 981 id = added++ & etp->et_mask; 982 offset = id * sizeof (efx_desc_t); 983 984 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 985 } 986 987 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 988 unsigned int, added, unsigned int, n); 989 990 EFX_TX_QSTAT_INCR(etp, TX_POST); 991 992 *addedp = added; 993 return (0); 994 995 fail1: 996 EFSYS_PROBE1(fail1, efx_rc_t, rc); 997 return (rc); 998 } 999 1000 void 1001 siena_tx_qdesc_dma_create( 1002 __in efx_txq_t *etp, 1003 __in efsys_dma_addr_t addr, 1004 __in size_t size, 1005 __in boolean_t eop, 1006 __out efx_desc_t *edp) 1007 { 1008 /* Fragments must not span 4k boundaries. */ 1009 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size); 1010 1011 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1012 efsys_dma_addr_t, addr, 1013 size_t, size, boolean_t, eop); 1014 1015 EFX_POPULATE_QWORD_4(edp->ed_eq, 1016 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1017 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1018 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1019 (uint32_t)(addr & 0xffffffff), 1020 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1021 (uint32_t)(addr >> 32)); 1022 } 1023 1024 #endif /* EFSYS_OPT_SIENA */ 1025 1026 #if EFSYS_OPT_QSTATS 1027 #if EFSYS_OPT_NAMES 1028 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */ 1029 static const char *__efx_tx_qstat_name[] = { 1030 "post", 1031 "post_pio", 1032 }; 1033 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1034 1035 const char * 1036 efx_tx_qstat_name( 1037 __in efx_nic_t *enp, 1038 __in unsigned int id) 1039 { 1040 _NOTE(ARGUNUSED(enp)) 1041 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1042 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1043 1044 return (__efx_tx_qstat_name[id]); 1045 } 1046 #endif /* EFSYS_OPT_NAMES */ 1047 #endif /* EFSYS_OPT_QSTATS */ 1048 1049 #if EFSYS_OPT_SIENA 1050 1051 #if EFSYS_OPT_QSTATS 1052 static void 1053 siena_tx_qstats_update( 1054 __in efx_txq_t *etp, 1055 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1056 { 1057 unsigned int id; 1058 1059 for (id = 0; id < TX_NQSTATS; id++) { 1060 efsys_stat_t *essp = &stat[id]; 1061 1062 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1063 etp->et_stat[id] = 0; 1064 } 1065 } 1066 #endif /* EFSYS_OPT_QSTATS */ 1067 1068 static void 1069 siena_tx_qdestroy( 1070 __in efx_txq_t *etp) 1071 { 1072 efx_nic_t *enp = etp->et_enp; 1073 efx_oword_t oword; 1074 1075 /* Purge descriptor queue */ 1076 EFX_ZERO_OWORD(oword); 1077 1078 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1079 etp->et_index, &oword, B_TRUE); 1080 } 1081 1082 static void 1083 siena_tx_fini( 1084 __in efx_nic_t *enp) 1085 { 1086 _NOTE(ARGUNUSED(enp)) 1087 } 1088 1089 #endif /* EFSYS_OPT_SIENA */ 1090