1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "efx.h" 37 #include "efx_impl.h" 38 39 #if EFSYS_OPT_QSTATS 40 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 41 do { \ 42 (_etp)->et_stat[_stat]++; \ 43 _NOTE(CONSTANTCONDITION) \ 44 } while (B_FALSE) 45 #else 46 #define EFX_TX_QSTAT_INCR(_etp, _stat) 47 #endif 48 49 #if EFSYS_OPT_SIENA 50 51 static __checkReturn efx_rc_t 52 siena_tx_init( 53 __in efx_nic_t *enp); 54 55 static void 56 siena_tx_fini( 57 __in efx_nic_t *enp); 58 59 static __checkReturn efx_rc_t 60 siena_tx_qcreate( 61 __in efx_nic_t *enp, 62 __in unsigned int index, 63 __in unsigned int label, 64 __in efsys_mem_t *esmp, 65 __in size_t n, 66 __in uint32_t id, 67 __in uint16_t flags, 68 __in efx_evq_t *eep, 69 __in efx_txq_t *etp, 70 __out unsigned int *addedp); 71 72 static void 73 siena_tx_qdestroy( 74 __in efx_txq_t *etp); 75 76 static __checkReturn efx_rc_t 77 siena_tx_qpost( 78 __in efx_txq_t *etp, 79 __in_ecount(n) efx_buffer_t *eb, 80 __in unsigned int n, 81 __in unsigned int completed, 82 __inout unsigned int *addedp); 83 84 static void 85 siena_tx_qpush( 86 __in efx_txq_t *etp, 87 __in unsigned int added, 88 __in unsigned int pushed); 89 90 static __checkReturn efx_rc_t 91 siena_tx_qpace( 92 __in efx_txq_t *etp, 93 __in unsigned int ns); 94 95 static __checkReturn efx_rc_t 96 siena_tx_qflush( 97 __in efx_txq_t *etp); 98 99 static void 100 siena_tx_qenable( 101 __in efx_txq_t *etp); 102 103 __checkReturn efx_rc_t 104 siena_tx_qdesc_post( 105 __in efx_txq_t *etp, 106 __in_ecount(n) efx_desc_t *ed, 107 __in unsigned int n, 108 __in unsigned int completed, 109 __inout unsigned int *addedp); 110 111 void 112 siena_tx_qdesc_dma_create( 113 __in efx_txq_t *etp, 114 __in efsys_dma_addr_t addr, 115 __in size_t size, 116 __in boolean_t eop, 117 __out efx_desc_t *edp); 118 119 #if EFSYS_OPT_QSTATS 120 static void 121 siena_tx_qstats_update( 122 __in efx_txq_t *etp, 123 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 124 #endif 125 126 #endif /* EFSYS_OPT_SIENA */ 127 128 129 #if EFSYS_OPT_SIENA 130 static const efx_tx_ops_t __efx_tx_siena_ops = { 131 siena_tx_init, /* etxo_init */ 132 siena_tx_fini, /* etxo_fini */ 133 siena_tx_qcreate, /* etxo_qcreate */ 134 siena_tx_qdestroy, /* etxo_qdestroy */ 135 siena_tx_qpost, /* etxo_qpost */ 136 siena_tx_qpush, /* etxo_qpush */ 137 siena_tx_qpace, /* etxo_qpace */ 138 siena_tx_qflush, /* etxo_qflush */ 139 siena_tx_qenable, /* etxo_qenable */ 140 NULL, /* etxo_qpio_enable */ 141 NULL, /* etxo_qpio_disable */ 142 NULL, /* etxo_qpio_write */ 143 NULL, /* etxo_qpio_post */ 144 siena_tx_qdesc_post, /* etxo_qdesc_post */ 145 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 146 NULL, /* etxo_qdesc_tso_create */ 147 NULL, /* etxo_qdesc_tso2_create */ 148 NULL, /* etxo_qdesc_vlantci_create */ 149 #if EFSYS_OPT_QSTATS 150 siena_tx_qstats_update, /* etxo_qstats_update */ 151 #endif 152 }; 153 #endif /* EFSYS_OPT_SIENA */ 154 155 #if EFSYS_OPT_HUNTINGTON 156 static const efx_tx_ops_t __efx_tx_hunt_ops = { 157 ef10_tx_init, /* etxo_init */ 158 ef10_tx_fini, /* etxo_fini */ 159 ef10_tx_qcreate, /* etxo_qcreate */ 160 ef10_tx_qdestroy, /* etxo_qdestroy */ 161 ef10_tx_qpost, /* etxo_qpost */ 162 ef10_tx_qpush, /* etxo_qpush */ 163 ef10_tx_qpace, /* etxo_qpace */ 164 ef10_tx_qflush, /* etxo_qflush */ 165 ef10_tx_qenable, /* etxo_qenable */ 166 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 167 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 168 ef10_tx_qpio_write, /* etxo_qpio_write */ 169 ef10_tx_qpio_post, /* etxo_qpio_post */ 170 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 171 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 172 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 173 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 174 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 175 #if EFSYS_OPT_QSTATS 176 ef10_tx_qstats_update, /* etxo_qstats_update */ 177 #endif 178 }; 179 #endif /* EFSYS_OPT_HUNTINGTON */ 180 181 #if EFSYS_OPT_MEDFORD 182 static const efx_tx_ops_t __efx_tx_medford_ops = { 183 ef10_tx_init, /* etxo_init */ 184 ef10_tx_fini, /* etxo_fini */ 185 ef10_tx_qcreate, /* etxo_qcreate */ 186 ef10_tx_qdestroy, /* etxo_qdestroy */ 187 ef10_tx_qpost, /* etxo_qpost */ 188 ef10_tx_qpush, /* etxo_qpush */ 189 ef10_tx_qpace, /* etxo_qpace */ 190 ef10_tx_qflush, /* etxo_qflush */ 191 ef10_tx_qenable, /* etxo_qenable */ 192 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 193 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 194 ef10_tx_qpio_write, /* etxo_qpio_write */ 195 ef10_tx_qpio_post, /* etxo_qpio_post */ 196 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 198 NULL, /* etxo_qdesc_tso_create */ 199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 201 #if EFSYS_OPT_QSTATS 202 ef10_tx_qstats_update, /* etxo_qstats_update */ 203 #endif 204 }; 205 #endif /* EFSYS_OPT_MEDFORD */ 206 207 __checkReturn efx_rc_t 208 efx_tx_init( 209 __in efx_nic_t *enp) 210 { 211 const efx_tx_ops_t *etxop; 212 efx_rc_t rc; 213 214 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 215 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 216 217 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 218 rc = EINVAL; 219 goto fail1; 220 } 221 222 if (enp->en_mod_flags & EFX_MOD_TX) { 223 rc = EINVAL; 224 goto fail2; 225 } 226 227 switch (enp->en_family) { 228 #if EFSYS_OPT_SIENA 229 case EFX_FAMILY_SIENA: 230 etxop = &__efx_tx_siena_ops; 231 break; 232 #endif /* EFSYS_OPT_SIENA */ 233 234 #if EFSYS_OPT_HUNTINGTON 235 case EFX_FAMILY_HUNTINGTON: 236 etxop = &__efx_tx_hunt_ops; 237 break; 238 #endif /* EFSYS_OPT_HUNTINGTON */ 239 240 #if EFSYS_OPT_MEDFORD 241 case EFX_FAMILY_MEDFORD: 242 etxop = &__efx_tx_medford_ops; 243 break; 244 #endif /* EFSYS_OPT_MEDFORD */ 245 246 default: 247 EFSYS_ASSERT(0); 248 rc = ENOTSUP; 249 goto fail3; 250 } 251 252 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 253 254 if ((rc = etxop->etxo_init(enp)) != 0) 255 goto fail4; 256 257 enp->en_etxop = etxop; 258 enp->en_mod_flags |= EFX_MOD_TX; 259 return (0); 260 261 fail4: 262 EFSYS_PROBE(fail4); 263 fail3: 264 EFSYS_PROBE(fail3); 265 fail2: 266 EFSYS_PROBE(fail2); 267 fail1: 268 EFSYS_PROBE1(fail1, efx_rc_t, rc); 269 270 enp->en_etxop = NULL; 271 enp->en_mod_flags &= ~EFX_MOD_TX; 272 return (rc); 273 } 274 275 void 276 efx_tx_fini( 277 __in efx_nic_t *enp) 278 { 279 const efx_tx_ops_t *etxop = enp->en_etxop; 280 281 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 282 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 283 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 284 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 285 286 etxop->etxo_fini(enp); 287 288 enp->en_etxop = NULL; 289 enp->en_mod_flags &= ~EFX_MOD_TX; 290 } 291 292 __checkReturn efx_rc_t 293 efx_tx_qcreate( 294 __in efx_nic_t *enp, 295 __in unsigned int index, 296 __in unsigned int label, 297 __in efsys_mem_t *esmp, 298 __in size_t n, 299 __in uint32_t id, 300 __in uint16_t flags, 301 __in efx_evq_t *eep, 302 __deref_out efx_txq_t **etpp, 303 __out unsigned int *addedp) 304 { 305 const efx_tx_ops_t *etxop = enp->en_etxop; 306 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 307 efx_txq_t *etp; 308 efx_rc_t rc; 309 310 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 311 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 312 313 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 314 315 /* Allocate an TXQ object */ 316 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 317 318 if (etp == NULL) { 319 rc = ENOMEM; 320 goto fail1; 321 } 322 323 etp->et_magic = EFX_TXQ_MAGIC; 324 etp->et_enp = enp; 325 etp->et_index = index; 326 etp->et_mask = n - 1; 327 etp->et_esmp = esmp; 328 329 /* Initial descriptor index may be modified by etxo_qcreate */ 330 *addedp = 0; 331 332 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 333 n, id, flags, eep, etp, addedp)) != 0) 334 goto fail2; 335 336 enp->en_tx_qcount++; 337 *etpp = etp; 338 339 return (0); 340 341 fail2: 342 EFSYS_PROBE(fail2); 343 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 344 fail1: 345 EFSYS_PROBE1(fail1, efx_rc_t, rc); 346 return (rc); 347 } 348 349 void 350 efx_tx_qdestroy( 351 __in efx_txq_t *etp) 352 { 353 efx_nic_t *enp = etp->et_enp; 354 const efx_tx_ops_t *etxop = enp->en_etxop; 355 356 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 357 358 EFSYS_ASSERT(enp->en_tx_qcount != 0); 359 --enp->en_tx_qcount; 360 361 etxop->etxo_qdestroy(etp); 362 363 /* Free the TXQ object */ 364 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 365 } 366 367 __checkReturn efx_rc_t 368 efx_tx_qpost( 369 __in efx_txq_t *etp, 370 __in_ecount(n) efx_buffer_t *eb, 371 __in unsigned int n, 372 __in unsigned int completed, 373 __inout unsigned int *addedp) 374 { 375 efx_nic_t *enp = etp->et_enp; 376 const efx_tx_ops_t *etxop = enp->en_etxop; 377 efx_rc_t rc; 378 379 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 380 381 if ((rc = etxop->etxo_qpost(etp, eb, 382 n, completed, addedp)) != 0) 383 goto fail1; 384 385 return (0); 386 387 fail1: 388 EFSYS_PROBE1(fail1, efx_rc_t, rc); 389 return (rc); 390 } 391 392 void 393 efx_tx_qpush( 394 __in efx_txq_t *etp, 395 __in unsigned int added, 396 __in unsigned int pushed) 397 { 398 efx_nic_t *enp = etp->et_enp; 399 const efx_tx_ops_t *etxop = enp->en_etxop; 400 401 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 402 403 etxop->etxo_qpush(etp, added, pushed); 404 } 405 406 __checkReturn efx_rc_t 407 efx_tx_qpace( 408 __in efx_txq_t *etp, 409 __in unsigned int ns) 410 { 411 efx_nic_t *enp = etp->et_enp; 412 const efx_tx_ops_t *etxop = enp->en_etxop; 413 efx_rc_t rc; 414 415 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 416 417 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 418 goto fail1; 419 420 return (0); 421 422 fail1: 423 EFSYS_PROBE1(fail1, efx_rc_t, rc); 424 return (rc); 425 } 426 427 __checkReturn efx_rc_t 428 efx_tx_qflush( 429 __in efx_txq_t *etp) 430 { 431 efx_nic_t *enp = etp->et_enp; 432 const efx_tx_ops_t *etxop = enp->en_etxop; 433 efx_rc_t rc; 434 435 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 436 437 if ((rc = etxop->etxo_qflush(etp)) != 0) 438 goto fail1; 439 440 return (0); 441 442 fail1: 443 EFSYS_PROBE1(fail1, efx_rc_t, rc); 444 return (rc); 445 } 446 447 void 448 efx_tx_qenable( 449 __in efx_txq_t *etp) 450 { 451 efx_nic_t *enp = etp->et_enp; 452 const efx_tx_ops_t *etxop = enp->en_etxop; 453 454 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 455 456 etxop->etxo_qenable(etp); 457 } 458 459 __checkReturn efx_rc_t 460 efx_tx_qpio_enable( 461 __in efx_txq_t *etp) 462 { 463 efx_nic_t *enp = etp->et_enp; 464 const efx_tx_ops_t *etxop = enp->en_etxop; 465 efx_rc_t rc; 466 467 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 468 469 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 470 rc = ENOTSUP; 471 goto fail1; 472 } 473 if (etxop->etxo_qpio_enable == NULL) { 474 rc = ENOTSUP; 475 goto fail2; 476 } 477 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 478 goto fail3; 479 480 return (0); 481 482 fail3: 483 EFSYS_PROBE(fail3); 484 fail2: 485 EFSYS_PROBE(fail2); 486 fail1: 487 EFSYS_PROBE1(fail1, efx_rc_t, rc); 488 return (rc); 489 } 490 491 void 492 efx_tx_qpio_disable( 493 __in efx_txq_t *etp) 494 { 495 efx_nic_t *enp = etp->et_enp; 496 const efx_tx_ops_t *etxop = enp->en_etxop; 497 498 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 499 500 if (etxop->etxo_qpio_disable != NULL) 501 etxop->etxo_qpio_disable(etp); 502 } 503 504 __checkReturn efx_rc_t 505 efx_tx_qpio_write( 506 __in efx_txq_t *etp, 507 __in_ecount(buf_length) uint8_t *buffer, 508 __in size_t buf_length, 509 __in size_t pio_buf_offset) 510 { 511 efx_nic_t *enp = etp->et_enp; 512 const efx_tx_ops_t *etxop = enp->en_etxop; 513 efx_rc_t rc; 514 515 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 516 517 if (etxop->etxo_qpio_write != NULL) { 518 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 519 pio_buf_offset)) != 0) 520 goto fail1; 521 return (0); 522 } 523 524 return (ENOTSUP); 525 526 fail1: 527 EFSYS_PROBE1(fail1, efx_rc_t, rc); 528 return (rc); 529 } 530 531 __checkReturn efx_rc_t 532 efx_tx_qpio_post( 533 __in efx_txq_t *etp, 534 __in size_t pkt_length, 535 __in unsigned int completed, 536 __inout unsigned int *addedp) 537 { 538 efx_nic_t *enp = etp->et_enp; 539 const efx_tx_ops_t *etxop = enp->en_etxop; 540 efx_rc_t rc; 541 542 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 543 544 if (etxop->etxo_qpio_post != NULL) { 545 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 546 addedp)) != 0) 547 goto fail1; 548 return (0); 549 } 550 551 return (ENOTSUP); 552 553 fail1: 554 EFSYS_PROBE1(fail1, efx_rc_t, rc); 555 return (rc); 556 } 557 558 __checkReturn efx_rc_t 559 efx_tx_qdesc_post( 560 __in efx_txq_t *etp, 561 __in_ecount(n) efx_desc_t *ed, 562 __in unsigned int n, 563 __in unsigned int completed, 564 __inout unsigned int *addedp) 565 { 566 efx_nic_t *enp = etp->et_enp; 567 const efx_tx_ops_t *etxop = enp->en_etxop; 568 efx_rc_t rc; 569 570 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 571 572 if ((rc = etxop->etxo_qdesc_post(etp, ed, 573 n, completed, addedp)) != 0) 574 goto fail1; 575 576 return (0); 577 578 fail1: 579 EFSYS_PROBE1(fail1, efx_rc_t, rc); 580 return (rc); 581 } 582 583 void 584 efx_tx_qdesc_dma_create( 585 __in efx_txq_t *etp, 586 __in efsys_dma_addr_t addr, 587 __in size_t size, 588 __in boolean_t eop, 589 __out efx_desc_t *edp) 590 { 591 efx_nic_t *enp = etp->et_enp; 592 const efx_tx_ops_t *etxop = enp->en_etxop; 593 594 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 595 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 596 597 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 598 } 599 600 void 601 efx_tx_qdesc_tso_create( 602 __in efx_txq_t *etp, 603 __in uint16_t ipv4_id, 604 __in uint32_t tcp_seq, 605 __in uint8_t tcp_flags, 606 __out efx_desc_t *edp) 607 { 608 efx_nic_t *enp = etp->et_enp; 609 const efx_tx_ops_t *etxop = enp->en_etxop; 610 611 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 612 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 613 614 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 615 } 616 617 void 618 efx_tx_qdesc_tso2_create( 619 __in efx_txq_t *etp, 620 __in uint16_t ipv4_id, 621 __in uint32_t tcp_seq, 622 __in uint16_t mss, 623 __out_ecount(count) efx_desc_t *edp, 624 __in int count) 625 { 626 efx_nic_t *enp = etp->et_enp; 627 const efx_tx_ops_t *etxop = enp->en_etxop; 628 629 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 630 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 631 632 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count); 633 } 634 635 void 636 efx_tx_qdesc_vlantci_create( 637 __in efx_txq_t *etp, 638 __in uint16_t tci, 639 __out efx_desc_t *edp) 640 { 641 efx_nic_t *enp = etp->et_enp; 642 const efx_tx_ops_t *etxop = enp->en_etxop; 643 644 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 645 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 646 647 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 648 } 649 650 651 #if EFSYS_OPT_QSTATS 652 void 653 efx_tx_qstats_update( 654 __in efx_txq_t *etp, 655 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 656 { 657 efx_nic_t *enp = etp->et_enp; 658 const efx_tx_ops_t *etxop = enp->en_etxop; 659 660 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 661 662 etxop->etxo_qstats_update(etp, stat); 663 } 664 #endif 665 666 667 #if EFSYS_OPT_SIENA 668 669 static __checkReturn efx_rc_t 670 siena_tx_init( 671 __in efx_nic_t *enp) 672 { 673 efx_oword_t oword; 674 675 /* 676 * Disable the timer-based TX DMA backoff and allow TX DMA to be 677 * controlled by the RX FIFO fill level (although always allow a 678 * minimal trickle). 679 */ 680 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 682 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 683 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 684 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 685 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 686 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 687 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 688 689 /* 690 * Filter all packets less than 14 bytes to avoid parsing 691 * errors. 692 */ 693 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 694 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 695 696 /* 697 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 698 * descriptors (which is bad). 699 */ 700 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 701 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 702 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 703 704 return (0); 705 } 706 707 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 708 do { \ 709 unsigned int id; \ 710 size_t offset; \ 711 efx_qword_t qword; \ 712 \ 713 id = (_added)++ & (_etp)->et_mask; \ 714 offset = id * sizeof (efx_qword_t); \ 715 \ 716 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 717 unsigned int, id, efsys_dma_addr_t, (_addr), \ 718 size_t, (_size), boolean_t, (_eop)); \ 719 \ 720 EFX_POPULATE_QWORD_4(qword, \ 721 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 722 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 723 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 724 (uint32_t)((_addr) & 0xffffffff), \ 725 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 726 (uint32_t)((_addr) >> 32)); \ 727 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 728 \ 729 _NOTE(CONSTANTCONDITION) \ 730 } while (B_FALSE) 731 732 static __checkReturn efx_rc_t 733 siena_tx_qpost( 734 __in efx_txq_t *etp, 735 __in_ecount(n) efx_buffer_t *eb, 736 __in unsigned int n, 737 __in unsigned int completed, 738 __inout unsigned int *addedp) 739 { 740 unsigned int added = *addedp; 741 unsigned int i; 742 int rc = ENOSPC; 743 744 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 745 goto fail1; 746 747 for (i = 0; i < n; i++) { 748 efx_buffer_t *ebp = &eb[i]; 749 efsys_dma_addr_t start = ebp->eb_addr; 750 size_t size = ebp->eb_size; 751 efsys_dma_addr_t end = start + size; 752 753 /* 754 * Fragments must not span 4k boundaries. 755 * Here it is a stricter requirement than the maximum length. 756 */ 757 EFSYS_ASSERT(P2ROUNDUP(start + 1, 758 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); 759 760 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 761 } 762 763 EFX_TX_QSTAT_INCR(etp, TX_POST); 764 765 *addedp = added; 766 return (0); 767 768 fail1: 769 EFSYS_PROBE1(fail1, efx_rc_t, rc); 770 771 return (rc); 772 } 773 774 static void 775 siena_tx_qpush( 776 __in efx_txq_t *etp, 777 __in unsigned int added, 778 __in unsigned int pushed) 779 { 780 efx_nic_t *enp = etp->et_enp; 781 uint32_t wptr; 782 efx_dword_t dword; 783 efx_oword_t oword; 784 785 /* Push the populated descriptors out */ 786 wptr = added & etp->et_mask; 787 788 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 789 790 /* Only write the third DWORD */ 791 EFX_POPULATE_DWORD_1(dword, 792 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 793 794 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 795 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 796 wptr, pushed & etp->et_mask); 797 EFSYS_PIO_WRITE_BARRIER(); 798 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 799 etp->et_index, &dword, B_FALSE); 800 } 801 802 #define EFX_MAX_PACE_VALUE 20 803 #define EFX_TX_PACE_CLOCK_BASE 104 804 805 static __checkReturn efx_rc_t 806 siena_tx_qpace( 807 __in efx_txq_t *etp, 808 __in unsigned int ns) 809 { 810 efx_nic_t *enp = etp->et_enp; 811 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 812 efx_oword_t oword; 813 unsigned int pace_val; 814 unsigned int timer_period; 815 efx_rc_t rc; 816 817 if (ns == 0) { 818 pace_val = 0; 819 } else { 820 /* 821 * The pace_val to write into the table is s.t 822 * ns <= timer_period * (2 ^ pace_val) 823 */ 824 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 825 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 826 if ((timer_period << pace_val) >= ns) 827 break; 828 } 829 } 830 if (pace_val > EFX_MAX_PACE_VALUE) { 831 rc = EINVAL; 832 goto fail1; 833 } 834 835 /* Update the pacing table */ 836 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 837 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 838 &oword, B_TRUE); 839 840 return (0); 841 842 fail1: 843 EFSYS_PROBE1(fail1, efx_rc_t, rc); 844 845 return (rc); 846 } 847 848 static __checkReturn efx_rc_t 849 siena_tx_qflush( 850 __in efx_txq_t *etp) 851 { 852 efx_nic_t *enp = etp->et_enp; 853 efx_oword_t oword; 854 uint32_t label; 855 856 efx_tx_qpace(etp, 0); 857 858 label = etp->et_index; 859 860 /* Flush the queue */ 861 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 862 FRF_AZ_TX_FLUSH_DESCQ, label); 863 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 864 865 return (0); 866 } 867 868 static void 869 siena_tx_qenable( 870 __in efx_txq_t *etp) 871 { 872 efx_nic_t *enp = etp->et_enp; 873 efx_oword_t oword; 874 875 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 876 etp->et_index, &oword, B_TRUE); 877 878 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 879 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 880 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 881 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 882 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 883 884 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 885 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 886 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 887 888 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 889 etp->et_index, &oword, B_TRUE); 890 } 891 892 static __checkReturn efx_rc_t 893 siena_tx_qcreate( 894 __in efx_nic_t *enp, 895 __in unsigned int index, 896 __in unsigned int label, 897 __in efsys_mem_t *esmp, 898 __in size_t n, 899 __in uint32_t id, 900 __in uint16_t flags, 901 __in efx_evq_t *eep, 902 __in efx_txq_t *etp, 903 __out unsigned int *addedp) 904 { 905 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 906 efx_oword_t oword; 907 uint32_t size; 908 efx_rc_t rc; 909 910 _NOTE(ARGUNUSED(esmp)) 911 912 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 913 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 914 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 915 916 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp))); 917 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 918 919 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { 920 rc = EINVAL; 921 goto fail1; 922 } 923 if (index >= encp->enc_txq_limit) { 924 rc = EINVAL; 925 goto fail2; 926 } 927 for (size = 0; 928 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS); 929 size++) 930 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 931 break; 932 if (id + (1 << size) >= encp->enc_buftbl_limit) { 933 rc = EINVAL; 934 goto fail3; 935 } 936 937 /* Set up the new descriptor queue */ 938 *addedp = 0; 939 940 EFX_POPULATE_OWORD_6(oword, 941 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 942 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 943 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 944 FRF_AZ_TX_DESCQ_LABEL, label, 945 FRF_AZ_TX_DESCQ_SIZE, size, 946 FRF_AZ_TX_DESCQ_TYPE, 0); 947 948 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 949 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 950 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 951 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 952 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 953 954 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 955 etp->et_index, &oword, B_TRUE); 956 957 return (0); 958 959 fail3: 960 EFSYS_PROBE(fail3); 961 fail2: 962 EFSYS_PROBE(fail2); 963 fail1: 964 EFSYS_PROBE1(fail1, efx_rc_t, rc); 965 966 return (rc); 967 } 968 969 __checkReturn efx_rc_t 970 siena_tx_qdesc_post( 971 __in efx_txq_t *etp, 972 __in_ecount(n) efx_desc_t *ed, 973 __in unsigned int n, 974 __in unsigned int completed, 975 __inout unsigned int *addedp) 976 { 977 unsigned int added = *addedp; 978 unsigned int i; 979 efx_rc_t rc; 980 981 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 982 rc = ENOSPC; 983 goto fail1; 984 } 985 986 for (i = 0; i < n; i++) { 987 efx_desc_t *edp = &ed[i]; 988 unsigned int id; 989 size_t offset; 990 991 id = added++ & etp->et_mask; 992 offset = id * sizeof (efx_desc_t); 993 994 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 995 } 996 997 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 998 unsigned int, added, unsigned int, n); 999 1000 EFX_TX_QSTAT_INCR(etp, TX_POST); 1001 1002 *addedp = added; 1003 return (0); 1004 1005 fail1: 1006 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1007 return (rc); 1008 } 1009 1010 void 1011 siena_tx_qdesc_dma_create( 1012 __in efx_txq_t *etp, 1013 __in efsys_dma_addr_t addr, 1014 __in size_t size, 1015 __in boolean_t eop, 1016 __out efx_desc_t *edp) 1017 { 1018 /* 1019 * Fragments must not span 4k boundaries. 1020 * Here it is a stricter requirement than the maximum length. 1021 */ 1022 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 1023 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); 1024 1025 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1026 efsys_dma_addr_t, addr, 1027 size_t, size, boolean_t, eop); 1028 1029 EFX_POPULATE_QWORD_4(edp->ed_eq, 1030 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1031 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1032 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1033 (uint32_t)(addr & 0xffffffff), 1034 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1035 (uint32_t)(addr >> 32)); 1036 } 1037 1038 #endif /* EFSYS_OPT_SIENA */ 1039 1040 #if EFSYS_OPT_QSTATS 1041 #if EFSYS_OPT_NAMES 1042 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ 1043 static const char * const __efx_tx_qstat_name[] = { 1044 "post", 1045 "post_pio", 1046 }; 1047 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1048 1049 const char * 1050 efx_tx_qstat_name( 1051 __in efx_nic_t *enp, 1052 __in unsigned int id) 1053 { 1054 _NOTE(ARGUNUSED(enp)) 1055 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1056 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1057 1058 return (__efx_tx_qstat_name[id]); 1059 } 1060 #endif /* EFSYS_OPT_NAMES */ 1061 #endif /* EFSYS_OPT_QSTATS */ 1062 1063 #if EFSYS_OPT_SIENA 1064 1065 #if EFSYS_OPT_QSTATS 1066 static void 1067 siena_tx_qstats_update( 1068 __in efx_txq_t *etp, 1069 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1070 { 1071 unsigned int id; 1072 1073 for (id = 0; id < TX_NQSTATS; id++) { 1074 efsys_stat_t *essp = &stat[id]; 1075 1076 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1077 etp->et_stat[id] = 0; 1078 } 1079 } 1080 #endif /* EFSYS_OPT_QSTATS */ 1081 1082 static void 1083 siena_tx_qdestroy( 1084 __in efx_txq_t *etp) 1085 { 1086 efx_nic_t *enp = etp->et_enp; 1087 efx_oword_t oword; 1088 1089 /* Purge descriptor queue */ 1090 EFX_ZERO_OWORD(oword); 1091 1092 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1093 etp->et_index, &oword, B_TRUE); 1094 } 1095 1096 static void 1097 siena_tx_fini( 1098 __in efx_nic_t *enp) 1099 { 1100 _NOTE(ARGUNUSED(enp)) 1101 } 1102 1103 #endif /* EFSYS_OPT_SIENA */ 1104