1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "efx.h" 37 #include "efx_impl.h" 38 39 #if EFSYS_OPT_QSTATS 40 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 41 do { \ 42 (_etp)->et_stat[_stat]++; \ 43 _NOTE(CONSTANTCONDITION) \ 44 } while (B_FALSE) 45 #else 46 #define EFX_TX_QSTAT_INCR(_etp, _stat) 47 #endif 48 49 #if EFSYS_OPT_SIENA 50 51 static __checkReturn efx_rc_t 52 siena_tx_init( 53 __in efx_nic_t *enp); 54 55 static void 56 siena_tx_fini( 57 __in efx_nic_t *enp); 58 59 static __checkReturn efx_rc_t 60 siena_tx_qcreate( 61 __in efx_nic_t *enp, 62 __in unsigned int index, 63 __in unsigned int label, 64 __in efsys_mem_t *esmp, 65 __in size_t ndescs, 66 __in uint32_t id, 67 __in uint16_t flags, 68 __in efx_evq_t *eep, 69 __in efx_txq_t *etp, 70 __out unsigned int *addedp); 71 72 static void 73 siena_tx_qdestroy( 74 __in efx_txq_t *etp); 75 76 static __checkReturn efx_rc_t 77 siena_tx_qpost( 78 __in efx_txq_t *etp, 79 __in_ecount(ndescs) efx_buffer_t *eb, 80 __in unsigned int ndescs, 81 __in unsigned int completed, 82 __inout unsigned int *addedp); 83 84 static void 85 siena_tx_qpush( 86 __in efx_txq_t *etp, 87 __in unsigned int added, 88 __in unsigned int pushed); 89 90 static __checkReturn efx_rc_t 91 siena_tx_qpace( 92 __in efx_txq_t *etp, 93 __in unsigned int ns); 94 95 static __checkReturn efx_rc_t 96 siena_tx_qflush( 97 __in efx_txq_t *etp); 98 99 static void 100 siena_tx_qenable( 101 __in efx_txq_t *etp); 102 103 __checkReturn efx_rc_t 104 siena_tx_qdesc_post( 105 __in efx_txq_t *etp, 106 __in_ecount(ndescs) efx_desc_t *ed, 107 __in unsigned int ndescs, 108 __in unsigned int completed, 109 __inout unsigned int *addedp); 110 111 void 112 siena_tx_qdesc_dma_create( 113 __in efx_txq_t *etp, 114 __in efsys_dma_addr_t addr, 115 __in size_t size, 116 __in boolean_t eop, 117 __out efx_desc_t *edp); 118 119 #if EFSYS_OPT_QSTATS 120 static void 121 siena_tx_qstats_update( 122 __in efx_txq_t *etp, 123 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 124 #endif 125 126 #endif /* EFSYS_OPT_SIENA */ 127 128 129 #if EFSYS_OPT_SIENA 130 static const efx_tx_ops_t __efx_tx_siena_ops = { 131 siena_tx_init, /* etxo_init */ 132 siena_tx_fini, /* etxo_fini */ 133 siena_tx_qcreate, /* etxo_qcreate */ 134 siena_tx_qdestroy, /* etxo_qdestroy */ 135 siena_tx_qpost, /* etxo_qpost */ 136 siena_tx_qpush, /* etxo_qpush */ 137 siena_tx_qpace, /* etxo_qpace */ 138 siena_tx_qflush, /* etxo_qflush */ 139 siena_tx_qenable, /* etxo_qenable */ 140 NULL, /* etxo_qpio_enable */ 141 NULL, /* etxo_qpio_disable */ 142 NULL, /* etxo_qpio_write */ 143 NULL, /* etxo_qpio_post */ 144 siena_tx_qdesc_post, /* etxo_qdesc_post */ 145 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 146 NULL, /* etxo_qdesc_tso_create */ 147 NULL, /* etxo_qdesc_tso2_create */ 148 NULL, /* etxo_qdesc_vlantci_create */ 149 #if EFSYS_OPT_QSTATS 150 siena_tx_qstats_update, /* etxo_qstats_update */ 151 #endif 152 }; 153 #endif /* EFSYS_OPT_SIENA */ 154 155 #if EFSYS_OPT_HUNTINGTON 156 static const efx_tx_ops_t __efx_tx_hunt_ops = { 157 ef10_tx_init, /* etxo_init */ 158 ef10_tx_fini, /* etxo_fini */ 159 ef10_tx_qcreate, /* etxo_qcreate */ 160 ef10_tx_qdestroy, /* etxo_qdestroy */ 161 ef10_tx_qpost, /* etxo_qpost */ 162 ef10_tx_qpush, /* etxo_qpush */ 163 ef10_tx_qpace, /* etxo_qpace */ 164 ef10_tx_qflush, /* etxo_qflush */ 165 ef10_tx_qenable, /* etxo_qenable */ 166 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 167 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 168 ef10_tx_qpio_write, /* etxo_qpio_write */ 169 ef10_tx_qpio_post, /* etxo_qpio_post */ 170 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 171 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 172 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 173 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 174 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 175 #if EFSYS_OPT_QSTATS 176 ef10_tx_qstats_update, /* etxo_qstats_update */ 177 #endif 178 }; 179 #endif /* EFSYS_OPT_HUNTINGTON */ 180 181 #if EFSYS_OPT_MEDFORD 182 static const efx_tx_ops_t __efx_tx_medford_ops = { 183 ef10_tx_init, /* etxo_init */ 184 ef10_tx_fini, /* etxo_fini */ 185 ef10_tx_qcreate, /* etxo_qcreate */ 186 ef10_tx_qdestroy, /* etxo_qdestroy */ 187 ef10_tx_qpost, /* etxo_qpost */ 188 ef10_tx_qpush, /* etxo_qpush */ 189 ef10_tx_qpace, /* etxo_qpace */ 190 ef10_tx_qflush, /* etxo_qflush */ 191 ef10_tx_qenable, /* etxo_qenable */ 192 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 193 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 194 ef10_tx_qpio_write, /* etxo_qpio_write */ 195 ef10_tx_qpio_post, /* etxo_qpio_post */ 196 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 198 NULL, /* etxo_qdesc_tso_create */ 199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 201 #if EFSYS_OPT_QSTATS 202 ef10_tx_qstats_update, /* etxo_qstats_update */ 203 #endif 204 }; 205 #endif /* EFSYS_OPT_MEDFORD */ 206 207 __checkReturn efx_rc_t 208 efx_tx_init( 209 __in efx_nic_t *enp) 210 { 211 const efx_tx_ops_t *etxop; 212 efx_rc_t rc; 213 214 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 215 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 216 217 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 218 rc = EINVAL; 219 goto fail1; 220 } 221 222 if (enp->en_mod_flags & EFX_MOD_TX) { 223 rc = EINVAL; 224 goto fail2; 225 } 226 227 switch (enp->en_family) { 228 #if EFSYS_OPT_SIENA 229 case EFX_FAMILY_SIENA: 230 etxop = &__efx_tx_siena_ops; 231 break; 232 #endif /* EFSYS_OPT_SIENA */ 233 234 #if EFSYS_OPT_HUNTINGTON 235 case EFX_FAMILY_HUNTINGTON: 236 etxop = &__efx_tx_hunt_ops; 237 break; 238 #endif /* EFSYS_OPT_HUNTINGTON */ 239 240 #if EFSYS_OPT_MEDFORD 241 case EFX_FAMILY_MEDFORD: 242 etxop = &__efx_tx_medford_ops; 243 break; 244 #endif /* EFSYS_OPT_MEDFORD */ 245 246 default: 247 EFSYS_ASSERT(0); 248 rc = ENOTSUP; 249 goto fail3; 250 } 251 252 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 253 254 if ((rc = etxop->etxo_init(enp)) != 0) 255 goto fail4; 256 257 enp->en_etxop = etxop; 258 enp->en_mod_flags |= EFX_MOD_TX; 259 return (0); 260 261 fail4: 262 EFSYS_PROBE(fail4); 263 fail3: 264 EFSYS_PROBE(fail3); 265 fail2: 266 EFSYS_PROBE(fail2); 267 fail1: 268 EFSYS_PROBE1(fail1, efx_rc_t, rc); 269 270 enp->en_etxop = NULL; 271 enp->en_mod_flags &= ~EFX_MOD_TX; 272 return (rc); 273 } 274 275 void 276 efx_tx_fini( 277 __in efx_nic_t *enp) 278 { 279 const efx_tx_ops_t *etxop = enp->en_etxop; 280 281 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 282 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 283 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 284 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 285 286 etxop->etxo_fini(enp); 287 288 enp->en_etxop = NULL; 289 enp->en_mod_flags &= ~EFX_MOD_TX; 290 } 291 292 __checkReturn efx_rc_t 293 efx_tx_qcreate( 294 __in efx_nic_t *enp, 295 __in unsigned int index, 296 __in unsigned int label, 297 __in efsys_mem_t *esmp, 298 __in size_t ndescs, 299 __in uint32_t id, 300 __in uint16_t flags, 301 __in efx_evq_t *eep, 302 __deref_out efx_txq_t **etpp, 303 __out unsigned int *addedp) 304 { 305 const efx_tx_ops_t *etxop = enp->en_etxop; 306 efx_txq_t *etp; 307 efx_rc_t rc; 308 309 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 310 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 311 312 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, 313 enp->en_nic_cfg.enc_txq_limit); 314 315 /* Allocate an TXQ object */ 316 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 317 318 if (etp == NULL) { 319 rc = ENOMEM; 320 goto fail1; 321 } 322 323 etp->et_magic = EFX_TXQ_MAGIC; 324 etp->et_enp = enp; 325 etp->et_index = index; 326 etp->et_mask = ndescs - 1; 327 etp->et_esmp = esmp; 328 329 /* Initial descriptor index may be modified by etxo_qcreate */ 330 *addedp = 0; 331 332 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 333 ndescs, id, flags, eep, etp, addedp)) != 0) 334 goto fail2; 335 336 enp->en_tx_qcount++; 337 *etpp = etp; 338 339 return (0); 340 341 fail2: 342 EFSYS_PROBE(fail2); 343 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 344 fail1: 345 EFSYS_PROBE1(fail1, efx_rc_t, rc); 346 return (rc); 347 } 348 349 void 350 efx_tx_qdestroy( 351 __in efx_txq_t *etp) 352 { 353 efx_nic_t *enp = etp->et_enp; 354 const efx_tx_ops_t *etxop = enp->en_etxop; 355 356 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 357 358 EFSYS_ASSERT(enp->en_tx_qcount != 0); 359 --enp->en_tx_qcount; 360 361 etxop->etxo_qdestroy(etp); 362 363 /* Free the TXQ object */ 364 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 365 } 366 367 __checkReturn efx_rc_t 368 efx_tx_qpost( 369 __in efx_txq_t *etp, 370 __in_ecount(ndescs) efx_buffer_t *eb, 371 __in unsigned int ndescs, 372 __in unsigned int completed, 373 __inout unsigned int *addedp) 374 { 375 efx_nic_t *enp = etp->et_enp; 376 const efx_tx_ops_t *etxop = enp->en_etxop; 377 efx_rc_t rc; 378 379 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 380 381 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0) 382 goto fail1; 383 384 return (0); 385 386 fail1: 387 EFSYS_PROBE1(fail1, efx_rc_t, rc); 388 return (rc); 389 } 390 391 void 392 efx_tx_qpush( 393 __in efx_txq_t *etp, 394 __in unsigned int added, 395 __in unsigned int pushed) 396 { 397 efx_nic_t *enp = etp->et_enp; 398 const efx_tx_ops_t *etxop = enp->en_etxop; 399 400 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 401 402 etxop->etxo_qpush(etp, added, pushed); 403 } 404 405 __checkReturn efx_rc_t 406 efx_tx_qpace( 407 __in efx_txq_t *etp, 408 __in unsigned int ns) 409 { 410 efx_nic_t *enp = etp->et_enp; 411 const efx_tx_ops_t *etxop = enp->en_etxop; 412 efx_rc_t rc; 413 414 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 415 416 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 417 goto fail1; 418 419 return (0); 420 421 fail1: 422 EFSYS_PROBE1(fail1, efx_rc_t, rc); 423 return (rc); 424 } 425 426 __checkReturn efx_rc_t 427 efx_tx_qflush( 428 __in efx_txq_t *etp) 429 { 430 efx_nic_t *enp = etp->et_enp; 431 const efx_tx_ops_t *etxop = enp->en_etxop; 432 efx_rc_t rc; 433 434 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 435 436 if ((rc = etxop->etxo_qflush(etp)) != 0) 437 goto fail1; 438 439 return (0); 440 441 fail1: 442 EFSYS_PROBE1(fail1, efx_rc_t, rc); 443 return (rc); 444 } 445 446 void 447 efx_tx_qenable( 448 __in efx_txq_t *etp) 449 { 450 efx_nic_t *enp = etp->et_enp; 451 const efx_tx_ops_t *etxop = enp->en_etxop; 452 453 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 454 455 etxop->etxo_qenable(etp); 456 } 457 458 __checkReturn efx_rc_t 459 efx_tx_qpio_enable( 460 __in efx_txq_t *etp) 461 { 462 efx_nic_t *enp = etp->et_enp; 463 const efx_tx_ops_t *etxop = enp->en_etxop; 464 efx_rc_t rc; 465 466 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 467 468 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 469 rc = ENOTSUP; 470 goto fail1; 471 } 472 if (etxop->etxo_qpio_enable == NULL) { 473 rc = ENOTSUP; 474 goto fail2; 475 } 476 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 477 goto fail3; 478 479 return (0); 480 481 fail3: 482 EFSYS_PROBE(fail3); 483 fail2: 484 EFSYS_PROBE(fail2); 485 fail1: 486 EFSYS_PROBE1(fail1, efx_rc_t, rc); 487 return (rc); 488 } 489 490 void 491 efx_tx_qpio_disable( 492 __in efx_txq_t *etp) 493 { 494 efx_nic_t *enp = etp->et_enp; 495 const efx_tx_ops_t *etxop = enp->en_etxop; 496 497 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 498 499 if (etxop->etxo_qpio_disable != NULL) 500 etxop->etxo_qpio_disable(etp); 501 } 502 503 __checkReturn efx_rc_t 504 efx_tx_qpio_write( 505 __in efx_txq_t *etp, 506 __in_ecount(buf_length) uint8_t *buffer, 507 __in size_t buf_length, 508 __in size_t pio_buf_offset) 509 { 510 efx_nic_t *enp = etp->et_enp; 511 const efx_tx_ops_t *etxop = enp->en_etxop; 512 efx_rc_t rc; 513 514 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 515 516 if (etxop->etxo_qpio_write != NULL) { 517 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 518 pio_buf_offset)) != 0) 519 goto fail1; 520 return (0); 521 } 522 523 return (ENOTSUP); 524 525 fail1: 526 EFSYS_PROBE1(fail1, efx_rc_t, rc); 527 return (rc); 528 } 529 530 __checkReturn efx_rc_t 531 efx_tx_qpio_post( 532 __in efx_txq_t *etp, 533 __in size_t pkt_length, 534 __in unsigned int completed, 535 __inout unsigned int *addedp) 536 { 537 efx_nic_t *enp = etp->et_enp; 538 const efx_tx_ops_t *etxop = enp->en_etxop; 539 efx_rc_t rc; 540 541 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 542 543 if (etxop->etxo_qpio_post != NULL) { 544 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 545 addedp)) != 0) 546 goto fail1; 547 return (0); 548 } 549 550 return (ENOTSUP); 551 552 fail1: 553 EFSYS_PROBE1(fail1, efx_rc_t, rc); 554 return (rc); 555 } 556 557 __checkReturn efx_rc_t 558 efx_tx_qdesc_post( 559 __in efx_txq_t *etp, 560 __in_ecount(ndescs) efx_desc_t *ed, 561 __in unsigned int ndescs, 562 __in unsigned int completed, 563 __inout unsigned int *addedp) 564 { 565 efx_nic_t *enp = etp->et_enp; 566 const efx_tx_ops_t *etxop = enp->en_etxop; 567 efx_rc_t rc; 568 569 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 570 571 if ((rc = etxop->etxo_qdesc_post(etp, ed, 572 ndescs, completed, addedp)) != 0) 573 goto fail1; 574 575 return (0); 576 577 fail1: 578 EFSYS_PROBE1(fail1, efx_rc_t, rc); 579 return (rc); 580 } 581 582 void 583 efx_tx_qdesc_dma_create( 584 __in efx_txq_t *etp, 585 __in efsys_dma_addr_t addr, 586 __in size_t size, 587 __in boolean_t eop, 588 __out efx_desc_t *edp) 589 { 590 efx_nic_t *enp = etp->et_enp; 591 const efx_tx_ops_t *etxop = enp->en_etxop; 592 593 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 594 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 595 596 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 597 } 598 599 void 600 efx_tx_qdesc_tso_create( 601 __in efx_txq_t *etp, 602 __in uint16_t ipv4_id, 603 __in uint32_t tcp_seq, 604 __in uint8_t tcp_flags, 605 __out efx_desc_t *edp) 606 { 607 efx_nic_t *enp = etp->et_enp; 608 const efx_tx_ops_t *etxop = enp->en_etxop; 609 610 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 611 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 612 613 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 614 } 615 616 void 617 efx_tx_qdesc_tso2_create( 618 __in efx_txq_t *etp, 619 __in uint16_t ipv4_id, 620 __in uint32_t tcp_seq, 621 __in uint16_t mss, 622 __out_ecount(count) efx_desc_t *edp, 623 __in int count) 624 { 625 efx_nic_t *enp = etp->et_enp; 626 const efx_tx_ops_t *etxop = enp->en_etxop; 627 628 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 629 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 630 631 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count); 632 } 633 634 void 635 efx_tx_qdesc_vlantci_create( 636 __in efx_txq_t *etp, 637 __in uint16_t tci, 638 __out efx_desc_t *edp) 639 { 640 efx_nic_t *enp = etp->et_enp; 641 const efx_tx_ops_t *etxop = enp->en_etxop; 642 643 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 644 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 645 646 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 647 } 648 649 650 #if EFSYS_OPT_QSTATS 651 void 652 efx_tx_qstats_update( 653 __in efx_txq_t *etp, 654 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 655 { 656 efx_nic_t *enp = etp->et_enp; 657 const efx_tx_ops_t *etxop = enp->en_etxop; 658 659 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 660 661 etxop->etxo_qstats_update(etp, stat); 662 } 663 #endif 664 665 666 #if EFSYS_OPT_SIENA 667 668 static __checkReturn efx_rc_t 669 siena_tx_init( 670 __in efx_nic_t *enp) 671 { 672 efx_oword_t oword; 673 674 /* 675 * Disable the timer-based TX DMA backoff and allow TX DMA to be 676 * controlled by the RX FIFO fill level (although always allow a 677 * minimal trickle). 678 */ 679 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 680 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 682 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 683 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 684 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 685 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 686 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 687 688 /* 689 * Filter all packets less than 14 bytes to avoid parsing 690 * errors. 691 */ 692 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 693 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 694 695 /* 696 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 697 * descriptors (which is bad). 698 */ 699 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 700 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 701 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 702 703 return (0); 704 } 705 706 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 707 do { \ 708 unsigned int id; \ 709 size_t offset; \ 710 efx_qword_t qword; \ 711 \ 712 id = (_added)++ & (_etp)->et_mask; \ 713 offset = id * sizeof (efx_qword_t); \ 714 \ 715 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 716 unsigned int, id, efsys_dma_addr_t, (_addr), \ 717 size_t, (_size), boolean_t, (_eop)); \ 718 \ 719 EFX_POPULATE_QWORD_4(qword, \ 720 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 721 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 722 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 723 (uint32_t)((_addr) & 0xffffffff), \ 724 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 725 (uint32_t)((_addr) >> 32)); \ 726 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 727 \ 728 _NOTE(CONSTANTCONDITION) \ 729 } while (B_FALSE) 730 731 static __checkReturn efx_rc_t 732 siena_tx_qpost( 733 __in efx_txq_t *etp, 734 __in_ecount(ndescs) efx_buffer_t *eb, 735 __in unsigned int ndescs, 736 __in unsigned int completed, 737 __inout unsigned int *addedp) 738 { 739 unsigned int added = *addedp; 740 unsigned int i; 741 int rc = ENOSPC; 742 743 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) 744 goto fail1; 745 746 for (i = 0; i < ndescs; i++) { 747 efx_buffer_t *ebp = &eb[i]; 748 efsys_dma_addr_t start = ebp->eb_addr; 749 size_t size = ebp->eb_size; 750 efsys_dma_addr_t end = start + size; 751 752 /* 753 * Fragments must not span 4k boundaries. 754 * Here it is a stricter requirement than the maximum length. 755 */ 756 EFSYS_ASSERT(P2ROUNDUP(start + 1, 757 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); 758 759 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 760 } 761 762 EFX_TX_QSTAT_INCR(etp, TX_POST); 763 764 *addedp = added; 765 return (0); 766 767 fail1: 768 EFSYS_PROBE1(fail1, efx_rc_t, rc); 769 770 return (rc); 771 } 772 773 static void 774 siena_tx_qpush( 775 __in efx_txq_t *etp, 776 __in unsigned int added, 777 __in unsigned int pushed) 778 { 779 efx_nic_t *enp = etp->et_enp; 780 uint32_t wptr; 781 efx_dword_t dword; 782 efx_oword_t oword; 783 784 /* Push the populated descriptors out */ 785 wptr = added & etp->et_mask; 786 787 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 788 789 /* Only write the third DWORD */ 790 EFX_POPULATE_DWORD_1(dword, 791 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 792 793 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 794 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 795 wptr, pushed & etp->et_mask); 796 EFSYS_PIO_WRITE_BARRIER(); 797 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 798 etp->et_index, &dword, B_FALSE); 799 } 800 801 #define EFX_MAX_PACE_VALUE 20 802 #define EFX_TX_PACE_CLOCK_BASE 104 803 804 static __checkReturn efx_rc_t 805 siena_tx_qpace( 806 __in efx_txq_t *etp, 807 __in unsigned int ns) 808 { 809 efx_nic_t *enp = etp->et_enp; 810 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 811 efx_oword_t oword; 812 unsigned int pace_val; 813 unsigned int timer_period; 814 efx_rc_t rc; 815 816 if (ns == 0) { 817 pace_val = 0; 818 } else { 819 /* 820 * The pace_val to write into the table is s.t 821 * ns <= timer_period * (2 ^ pace_val) 822 */ 823 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 824 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 825 if ((timer_period << pace_val) >= ns) 826 break; 827 } 828 } 829 if (pace_val > EFX_MAX_PACE_VALUE) { 830 rc = EINVAL; 831 goto fail1; 832 } 833 834 /* Update the pacing table */ 835 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 836 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 837 &oword, B_TRUE); 838 839 return (0); 840 841 fail1: 842 EFSYS_PROBE1(fail1, efx_rc_t, rc); 843 844 return (rc); 845 } 846 847 static __checkReturn efx_rc_t 848 siena_tx_qflush( 849 __in efx_txq_t *etp) 850 { 851 efx_nic_t *enp = etp->et_enp; 852 efx_oword_t oword; 853 uint32_t label; 854 855 efx_tx_qpace(etp, 0); 856 857 label = etp->et_index; 858 859 /* Flush the queue */ 860 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 861 FRF_AZ_TX_FLUSH_DESCQ, label); 862 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 863 864 return (0); 865 } 866 867 static void 868 siena_tx_qenable( 869 __in efx_txq_t *etp) 870 { 871 efx_nic_t *enp = etp->et_enp; 872 efx_oword_t oword; 873 874 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 875 etp->et_index, &oword, B_TRUE); 876 877 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 878 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 879 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 880 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 881 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 882 883 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 884 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 885 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 886 887 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 888 etp->et_index, &oword, B_TRUE); 889 } 890 891 static __checkReturn efx_rc_t 892 siena_tx_qcreate( 893 __in efx_nic_t *enp, 894 __in unsigned int index, 895 __in unsigned int label, 896 __in efsys_mem_t *esmp, 897 __in size_t ndescs, 898 __in uint32_t id, 899 __in uint16_t flags, 900 __in efx_evq_t *eep, 901 __in efx_txq_t *etp, 902 __out unsigned int *addedp) 903 { 904 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 905 efx_oword_t oword; 906 uint32_t size; 907 uint16_t inner_csum; 908 efx_rc_t rc; 909 910 _NOTE(ARGUNUSED(esmp)) 911 912 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 913 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 914 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 915 916 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs)); 917 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 918 919 if (!ISP2(ndescs) || 920 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) { 921 rc = EINVAL; 922 goto fail1; 923 } 924 if (index >= encp->enc_txq_limit) { 925 rc = EINVAL; 926 goto fail2; 927 } 928 for (size = 0; 929 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS); 930 size++) 931 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS)) 932 break; 933 if (id + (1 << size) >= encp->enc_buftbl_limit) { 934 rc = EINVAL; 935 goto fail3; 936 } 937 938 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP; 939 if ((flags & inner_csum) != 0) { 940 rc = EINVAL; 941 goto fail4; 942 } 943 944 /* Set up the new descriptor queue */ 945 *addedp = 0; 946 947 EFX_POPULATE_OWORD_6(oword, 948 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 949 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 950 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 951 FRF_AZ_TX_DESCQ_LABEL, label, 952 FRF_AZ_TX_DESCQ_SIZE, size, 953 FRF_AZ_TX_DESCQ_TYPE, 0); 954 955 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 956 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 957 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 958 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 959 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 960 961 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 962 etp->et_index, &oword, B_TRUE); 963 964 return (0); 965 966 fail4: 967 EFSYS_PROBE(fail4); 968 fail3: 969 EFSYS_PROBE(fail3); 970 fail2: 971 EFSYS_PROBE(fail2); 972 fail1: 973 EFSYS_PROBE1(fail1, efx_rc_t, rc); 974 975 return (rc); 976 } 977 978 __checkReturn efx_rc_t 979 siena_tx_qdesc_post( 980 __in efx_txq_t *etp, 981 __in_ecount(ndescs) efx_desc_t *ed, 982 __in unsigned int ndescs, 983 __in unsigned int completed, 984 __inout unsigned int *addedp) 985 { 986 unsigned int added = *addedp; 987 unsigned int i; 988 efx_rc_t rc; 989 990 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 991 rc = ENOSPC; 992 goto fail1; 993 } 994 995 for (i = 0; i < ndescs; i++) { 996 efx_desc_t *edp = &ed[i]; 997 unsigned int id; 998 size_t offset; 999 1000 id = added++ & etp->et_mask; 1001 offset = id * sizeof (efx_desc_t); 1002 1003 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 1004 } 1005 1006 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 1007 unsigned int, added, unsigned int, ndescs); 1008 1009 EFX_TX_QSTAT_INCR(etp, TX_POST); 1010 1011 *addedp = added; 1012 return (0); 1013 1014 fail1: 1015 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1016 return (rc); 1017 } 1018 1019 void 1020 siena_tx_qdesc_dma_create( 1021 __in efx_txq_t *etp, 1022 __in efsys_dma_addr_t addr, 1023 __in size_t size, 1024 __in boolean_t eop, 1025 __out efx_desc_t *edp) 1026 { 1027 /* 1028 * Fragments must not span 4k boundaries. 1029 * Here it is a stricter requirement than the maximum length. 1030 */ 1031 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 1032 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); 1033 1034 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1035 efsys_dma_addr_t, addr, 1036 size_t, size, boolean_t, eop); 1037 1038 EFX_POPULATE_QWORD_4(edp->ed_eq, 1039 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1040 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1041 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1042 (uint32_t)(addr & 0xffffffff), 1043 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1044 (uint32_t)(addr >> 32)); 1045 } 1046 1047 #endif /* EFSYS_OPT_SIENA */ 1048 1049 #if EFSYS_OPT_QSTATS 1050 #if EFSYS_OPT_NAMES 1051 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ 1052 static const char * const __efx_tx_qstat_name[] = { 1053 "post", 1054 "post_pio", 1055 }; 1056 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1057 1058 const char * 1059 efx_tx_qstat_name( 1060 __in efx_nic_t *enp, 1061 __in unsigned int id) 1062 { 1063 _NOTE(ARGUNUSED(enp)) 1064 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1065 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1066 1067 return (__efx_tx_qstat_name[id]); 1068 } 1069 #endif /* EFSYS_OPT_NAMES */ 1070 #endif /* EFSYS_OPT_QSTATS */ 1071 1072 #if EFSYS_OPT_SIENA 1073 1074 #if EFSYS_OPT_QSTATS 1075 static void 1076 siena_tx_qstats_update( 1077 __in efx_txq_t *etp, 1078 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1079 { 1080 unsigned int id; 1081 1082 for (id = 0; id < TX_NQSTATS; id++) { 1083 efsys_stat_t *essp = &stat[id]; 1084 1085 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1086 etp->et_stat[id] = 0; 1087 } 1088 } 1089 #endif /* EFSYS_OPT_QSTATS */ 1090 1091 static void 1092 siena_tx_qdestroy( 1093 __in efx_txq_t *etp) 1094 { 1095 efx_nic_t *enp = etp->et_enp; 1096 efx_oword_t oword; 1097 1098 /* Purge descriptor queue */ 1099 EFX_ZERO_OWORD(oword); 1100 1101 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1102 etp->et_index, &oword, B_TRUE); 1103 } 1104 1105 static void 1106 siena_tx_fini( 1107 __in efx_nic_t *enp) 1108 { 1109 _NOTE(ARGUNUSED(enp)) 1110 } 1111 1112 #endif /* EFSYS_OPT_SIENA */ 1113