1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33 #include <sys/cdefs.h> 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 #if EFSYS_OPT_QSTATS 38 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 39 do { \ 40 (_etp)->et_stat[_stat]++; \ 41 _NOTE(CONSTANTCONDITION) \ 42 } while (B_FALSE) 43 #else 44 #define EFX_TX_QSTAT_INCR(_etp, _stat) 45 #endif 46 47 #if EFSYS_OPT_SIENA 48 49 static __checkReturn efx_rc_t 50 siena_tx_init( 51 __in efx_nic_t *enp); 52 53 static void 54 siena_tx_fini( 55 __in efx_nic_t *enp); 56 57 static __checkReturn efx_rc_t 58 siena_tx_qcreate( 59 __in efx_nic_t *enp, 60 __in unsigned int index, 61 __in unsigned int label, 62 __in efsys_mem_t *esmp, 63 __in size_t ndescs, 64 __in uint32_t id, 65 __in uint16_t flags, 66 __in efx_evq_t *eep, 67 __in efx_txq_t *etp, 68 __out unsigned int *addedp); 69 70 static void 71 siena_tx_qdestroy( 72 __in efx_txq_t *etp); 73 74 static __checkReturn efx_rc_t 75 siena_tx_qpost( 76 __in efx_txq_t *etp, 77 __in_ecount(ndescs) efx_buffer_t *eb, 78 __in unsigned int ndescs, 79 __in unsigned int completed, 80 __inout unsigned int *addedp); 81 82 static void 83 siena_tx_qpush( 84 __in efx_txq_t *etp, 85 __in unsigned int added, 86 __in unsigned int pushed); 87 88 static __checkReturn efx_rc_t 89 siena_tx_qpace( 90 __in efx_txq_t *etp, 91 __in unsigned int ns); 92 93 static __checkReturn efx_rc_t 94 siena_tx_qflush( 95 __in efx_txq_t *etp); 96 97 static void 98 siena_tx_qenable( 99 __in efx_txq_t *etp); 100 101 __checkReturn efx_rc_t 102 siena_tx_qdesc_post( 103 __in efx_txq_t *etp, 104 __in_ecount(ndescs) efx_desc_t *ed, 105 __in unsigned int ndescs, 106 __in unsigned int completed, 107 __inout unsigned int *addedp); 108 109 void 110 siena_tx_qdesc_dma_create( 111 __in efx_txq_t *etp, 112 __in efsys_dma_addr_t addr, 113 __in size_t size, 114 __in boolean_t eop, 115 __out efx_desc_t *edp); 116 117 #if EFSYS_OPT_QSTATS 118 static void 119 siena_tx_qstats_update( 120 __in efx_txq_t *etp, 121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 122 #endif 123 124 #endif /* EFSYS_OPT_SIENA */ 125 126 #if EFSYS_OPT_SIENA 127 static const efx_tx_ops_t __efx_tx_siena_ops = { 128 siena_tx_init, /* etxo_init */ 129 siena_tx_fini, /* etxo_fini */ 130 siena_tx_qcreate, /* etxo_qcreate */ 131 siena_tx_qdestroy, /* etxo_qdestroy */ 132 siena_tx_qpost, /* etxo_qpost */ 133 siena_tx_qpush, /* etxo_qpush */ 134 siena_tx_qpace, /* etxo_qpace */ 135 siena_tx_qflush, /* etxo_qflush */ 136 siena_tx_qenable, /* etxo_qenable */ 137 NULL, /* etxo_qpio_enable */ 138 NULL, /* etxo_qpio_disable */ 139 NULL, /* etxo_qpio_write */ 140 NULL, /* etxo_qpio_post */ 141 siena_tx_qdesc_post, /* etxo_qdesc_post */ 142 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 143 NULL, /* etxo_qdesc_tso_create */ 144 NULL, /* etxo_qdesc_tso2_create */ 145 NULL, /* etxo_qdesc_vlantci_create */ 146 NULL, /* etxo_qdesc_checksum_create */ 147 #if EFSYS_OPT_QSTATS 148 siena_tx_qstats_update, /* etxo_qstats_update */ 149 #endif 150 }; 151 #endif /* EFSYS_OPT_SIENA */ 152 153 #if EFSYS_OPT_HUNTINGTON 154 static const efx_tx_ops_t __efx_tx_hunt_ops = { 155 ef10_tx_init, /* etxo_init */ 156 ef10_tx_fini, /* etxo_fini */ 157 ef10_tx_qcreate, /* etxo_qcreate */ 158 ef10_tx_qdestroy, /* etxo_qdestroy */ 159 ef10_tx_qpost, /* etxo_qpost */ 160 ef10_tx_qpush, /* etxo_qpush */ 161 ef10_tx_qpace, /* etxo_qpace */ 162 ef10_tx_qflush, /* etxo_qflush */ 163 ef10_tx_qenable, /* etxo_qenable */ 164 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 165 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 166 ef10_tx_qpio_write, /* etxo_qpio_write */ 167 ef10_tx_qpio_post, /* etxo_qpio_post */ 168 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 169 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 170 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 171 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 172 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 173 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 174 #if EFSYS_OPT_QSTATS 175 ef10_tx_qstats_update, /* etxo_qstats_update */ 176 #endif 177 }; 178 #endif /* EFSYS_OPT_HUNTINGTON */ 179 180 #if EFSYS_OPT_MEDFORD 181 static const efx_tx_ops_t __efx_tx_medford_ops = { 182 ef10_tx_init, /* etxo_init */ 183 ef10_tx_fini, /* etxo_fini */ 184 ef10_tx_qcreate, /* etxo_qcreate */ 185 ef10_tx_qdestroy, /* etxo_qdestroy */ 186 ef10_tx_qpost, /* etxo_qpost */ 187 ef10_tx_qpush, /* etxo_qpush */ 188 ef10_tx_qpace, /* etxo_qpace */ 189 ef10_tx_qflush, /* etxo_qflush */ 190 ef10_tx_qenable, /* etxo_qenable */ 191 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 192 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 193 ef10_tx_qpio_write, /* etxo_qpio_write */ 194 ef10_tx_qpio_post, /* etxo_qpio_post */ 195 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 196 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 197 NULL, /* etxo_qdesc_tso_create */ 198 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 199 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 200 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 201 #if EFSYS_OPT_QSTATS 202 ef10_tx_qstats_update, /* etxo_qstats_update */ 203 #endif 204 }; 205 #endif /* EFSYS_OPT_MEDFORD */ 206 207 #if EFSYS_OPT_MEDFORD2 208 static const efx_tx_ops_t __efx_tx_medford2_ops = { 209 ef10_tx_init, /* etxo_init */ 210 ef10_tx_fini, /* etxo_fini */ 211 ef10_tx_qcreate, /* etxo_qcreate */ 212 ef10_tx_qdestroy, /* etxo_qdestroy */ 213 ef10_tx_qpost, /* etxo_qpost */ 214 ef10_tx_qpush, /* etxo_qpush */ 215 ef10_tx_qpace, /* etxo_qpace */ 216 ef10_tx_qflush, /* etxo_qflush */ 217 ef10_tx_qenable, /* etxo_qenable */ 218 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 219 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 220 ef10_tx_qpio_write, /* etxo_qpio_write */ 221 ef10_tx_qpio_post, /* etxo_qpio_post */ 222 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 223 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 224 NULL, /* etxo_qdesc_tso_create */ 225 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 226 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 227 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 228 #if EFSYS_OPT_QSTATS 229 ef10_tx_qstats_update, /* etxo_qstats_update */ 230 #endif 231 }; 232 #endif /* EFSYS_OPT_MEDFORD2 */ 233 234 __checkReturn efx_rc_t 235 efx_tx_init( 236 __in efx_nic_t *enp) 237 { 238 const efx_tx_ops_t *etxop; 239 efx_rc_t rc; 240 241 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 242 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 243 244 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 245 rc = EINVAL; 246 goto fail1; 247 } 248 249 if (enp->en_mod_flags & EFX_MOD_TX) { 250 rc = EINVAL; 251 goto fail2; 252 } 253 254 switch (enp->en_family) { 255 #if EFSYS_OPT_SIENA 256 case EFX_FAMILY_SIENA: 257 etxop = &__efx_tx_siena_ops; 258 break; 259 #endif /* EFSYS_OPT_SIENA */ 260 261 #if EFSYS_OPT_HUNTINGTON 262 case EFX_FAMILY_HUNTINGTON: 263 etxop = &__efx_tx_hunt_ops; 264 break; 265 #endif /* EFSYS_OPT_HUNTINGTON */ 266 267 #if EFSYS_OPT_MEDFORD 268 case EFX_FAMILY_MEDFORD: 269 etxop = &__efx_tx_medford_ops; 270 break; 271 #endif /* EFSYS_OPT_MEDFORD */ 272 273 #if EFSYS_OPT_MEDFORD2 274 case EFX_FAMILY_MEDFORD2: 275 etxop = &__efx_tx_medford2_ops; 276 break; 277 #endif /* EFSYS_OPT_MEDFORD2 */ 278 279 default: 280 EFSYS_ASSERT(0); 281 rc = ENOTSUP; 282 goto fail3; 283 } 284 285 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 286 287 if ((rc = etxop->etxo_init(enp)) != 0) 288 goto fail4; 289 290 enp->en_etxop = etxop; 291 enp->en_mod_flags |= EFX_MOD_TX; 292 return (0); 293 294 fail4: 295 EFSYS_PROBE(fail4); 296 fail3: 297 EFSYS_PROBE(fail3); 298 fail2: 299 EFSYS_PROBE(fail2); 300 fail1: 301 EFSYS_PROBE1(fail1, efx_rc_t, rc); 302 303 enp->en_etxop = NULL; 304 enp->en_mod_flags &= ~EFX_MOD_TX; 305 return (rc); 306 } 307 308 void 309 efx_tx_fini( 310 __in efx_nic_t *enp) 311 { 312 const efx_tx_ops_t *etxop = enp->en_etxop; 313 314 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 315 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 316 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 317 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 318 319 etxop->etxo_fini(enp); 320 321 enp->en_etxop = NULL; 322 enp->en_mod_flags &= ~EFX_MOD_TX; 323 } 324 325 __checkReturn efx_rc_t 326 efx_tx_qcreate( 327 __in efx_nic_t *enp, 328 __in unsigned int index, 329 __in unsigned int label, 330 __in efsys_mem_t *esmp, 331 __in size_t ndescs, 332 __in uint32_t id, 333 __in uint16_t flags, 334 __in efx_evq_t *eep, 335 __deref_out efx_txq_t **etpp, 336 __out unsigned int *addedp) 337 { 338 const efx_tx_ops_t *etxop = enp->en_etxop; 339 efx_txq_t *etp; 340 efx_rc_t rc; 341 342 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 343 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 344 345 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, 346 enp->en_nic_cfg.enc_txq_limit); 347 348 /* Allocate an TXQ object */ 349 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 350 351 if (etp == NULL) { 352 rc = ENOMEM; 353 goto fail1; 354 } 355 356 etp->et_magic = EFX_TXQ_MAGIC; 357 etp->et_enp = enp; 358 etp->et_index = index; 359 etp->et_mask = ndescs - 1; 360 etp->et_esmp = esmp; 361 362 /* Initial descriptor index may be modified by etxo_qcreate */ 363 *addedp = 0; 364 365 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 366 ndescs, id, flags, eep, etp, addedp)) != 0) 367 goto fail2; 368 369 enp->en_tx_qcount++; 370 *etpp = etp; 371 372 return (0); 373 374 fail2: 375 EFSYS_PROBE(fail2); 376 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 377 fail1: 378 EFSYS_PROBE1(fail1, efx_rc_t, rc); 379 return (rc); 380 } 381 382 void 383 efx_tx_qdestroy( 384 __in efx_txq_t *etp) 385 { 386 efx_nic_t *enp = etp->et_enp; 387 const efx_tx_ops_t *etxop = enp->en_etxop; 388 389 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 390 391 EFSYS_ASSERT(enp->en_tx_qcount != 0); 392 --enp->en_tx_qcount; 393 394 etxop->etxo_qdestroy(etp); 395 396 /* Free the TXQ object */ 397 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 398 } 399 400 __checkReturn efx_rc_t 401 efx_tx_qpost( 402 __in efx_txq_t *etp, 403 __in_ecount(ndescs) efx_buffer_t *eb, 404 __in unsigned int ndescs, 405 __in unsigned int completed, 406 __inout unsigned int *addedp) 407 { 408 efx_nic_t *enp = etp->et_enp; 409 const efx_tx_ops_t *etxop = enp->en_etxop; 410 efx_rc_t rc; 411 412 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 413 414 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0) 415 goto fail1; 416 417 return (0); 418 419 fail1: 420 EFSYS_PROBE1(fail1, efx_rc_t, rc); 421 return (rc); 422 } 423 424 void 425 efx_tx_qpush( 426 __in efx_txq_t *etp, 427 __in unsigned int added, 428 __in unsigned int pushed) 429 { 430 efx_nic_t *enp = etp->et_enp; 431 const efx_tx_ops_t *etxop = enp->en_etxop; 432 433 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 434 435 etxop->etxo_qpush(etp, added, pushed); 436 } 437 438 __checkReturn efx_rc_t 439 efx_tx_qpace( 440 __in efx_txq_t *etp, 441 __in unsigned int ns) 442 { 443 efx_nic_t *enp = etp->et_enp; 444 const efx_tx_ops_t *etxop = enp->en_etxop; 445 efx_rc_t rc; 446 447 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 448 449 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 450 goto fail1; 451 452 return (0); 453 454 fail1: 455 EFSYS_PROBE1(fail1, efx_rc_t, rc); 456 return (rc); 457 } 458 459 __checkReturn efx_rc_t 460 efx_tx_qflush( 461 __in efx_txq_t *etp) 462 { 463 efx_nic_t *enp = etp->et_enp; 464 const efx_tx_ops_t *etxop = enp->en_etxop; 465 efx_rc_t rc; 466 467 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 468 469 if ((rc = etxop->etxo_qflush(etp)) != 0) 470 goto fail1; 471 472 return (0); 473 474 fail1: 475 EFSYS_PROBE1(fail1, efx_rc_t, rc); 476 return (rc); 477 } 478 479 void 480 efx_tx_qenable( 481 __in efx_txq_t *etp) 482 { 483 efx_nic_t *enp = etp->et_enp; 484 const efx_tx_ops_t *etxop = enp->en_etxop; 485 486 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 487 488 etxop->etxo_qenable(etp); 489 } 490 491 __checkReturn efx_rc_t 492 efx_tx_qpio_enable( 493 __in efx_txq_t *etp) 494 { 495 efx_nic_t *enp = etp->et_enp; 496 const efx_tx_ops_t *etxop = enp->en_etxop; 497 efx_rc_t rc; 498 499 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 500 501 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 502 rc = ENOTSUP; 503 goto fail1; 504 } 505 if (etxop->etxo_qpio_enable == NULL) { 506 rc = ENOTSUP; 507 goto fail2; 508 } 509 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 510 goto fail3; 511 512 return (0); 513 514 fail3: 515 EFSYS_PROBE(fail3); 516 fail2: 517 EFSYS_PROBE(fail2); 518 fail1: 519 EFSYS_PROBE1(fail1, efx_rc_t, rc); 520 return (rc); 521 } 522 523 void 524 efx_tx_qpio_disable( 525 __in efx_txq_t *etp) 526 { 527 efx_nic_t *enp = etp->et_enp; 528 const efx_tx_ops_t *etxop = enp->en_etxop; 529 530 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 531 532 if (etxop->etxo_qpio_disable != NULL) 533 etxop->etxo_qpio_disable(etp); 534 } 535 536 __checkReturn efx_rc_t 537 efx_tx_qpio_write( 538 __in efx_txq_t *etp, 539 __in_ecount(buf_length) uint8_t *buffer, 540 __in size_t buf_length, 541 __in size_t pio_buf_offset) 542 { 543 efx_nic_t *enp = etp->et_enp; 544 const efx_tx_ops_t *etxop = enp->en_etxop; 545 efx_rc_t rc; 546 547 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 548 549 if (etxop->etxo_qpio_write != NULL) { 550 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 551 pio_buf_offset)) != 0) 552 goto fail1; 553 return (0); 554 } 555 556 return (ENOTSUP); 557 558 fail1: 559 EFSYS_PROBE1(fail1, efx_rc_t, rc); 560 return (rc); 561 } 562 563 __checkReturn efx_rc_t 564 efx_tx_qpio_post( 565 __in efx_txq_t *etp, 566 __in size_t pkt_length, 567 __in unsigned int completed, 568 __inout unsigned int *addedp) 569 { 570 efx_nic_t *enp = etp->et_enp; 571 const efx_tx_ops_t *etxop = enp->en_etxop; 572 efx_rc_t rc; 573 574 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 575 576 if (etxop->etxo_qpio_post != NULL) { 577 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 578 addedp)) != 0) 579 goto fail1; 580 return (0); 581 } 582 583 return (ENOTSUP); 584 585 fail1: 586 EFSYS_PROBE1(fail1, efx_rc_t, rc); 587 return (rc); 588 } 589 590 __checkReturn efx_rc_t 591 efx_tx_qdesc_post( 592 __in efx_txq_t *etp, 593 __in_ecount(ndescs) efx_desc_t *ed, 594 __in unsigned int ndescs, 595 __in unsigned int completed, 596 __inout unsigned int *addedp) 597 { 598 efx_nic_t *enp = etp->et_enp; 599 const efx_tx_ops_t *etxop = enp->en_etxop; 600 601 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 602 603 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp)); 604 } 605 606 void 607 efx_tx_qdesc_dma_create( 608 __in efx_txq_t *etp, 609 __in efsys_dma_addr_t addr, 610 __in size_t size, 611 __in boolean_t eop, 612 __out efx_desc_t *edp) 613 { 614 efx_nic_t *enp = etp->et_enp; 615 const efx_tx_ops_t *etxop = enp->en_etxop; 616 617 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 618 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 619 620 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 621 } 622 623 void 624 efx_tx_qdesc_tso_create( 625 __in efx_txq_t *etp, 626 __in uint16_t ipv4_id, 627 __in uint32_t tcp_seq, 628 __in uint8_t tcp_flags, 629 __out efx_desc_t *edp) 630 { 631 efx_nic_t *enp = etp->et_enp; 632 const efx_tx_ops_t *etxop = enp->en_etxop; 633 634 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 635 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 636 637 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 638 } 639 640 void 641 efx_tx_qdesc_tso2_create( 642 __in efx_txq_t *etp, 643 __in uint16_t ipv4_id, 644 __in uint16_t outer_ipv4_id, 645 __in uint32_t tcp_seq, 646 __in uint16_t mss, 647 __out_ecount(count) efx_desc_t *edp, 648 __in int count) 649 { 650 efx_nic_t *enp = etp->et_enp; 651 const efx_tx_ops_t *etxop = enp->en_etxop; 652 653 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 654 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 655 656 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id, 657 tcp_seq, mss, edp, count); 658 } 659 660 void 661 efx_tx_qdesc_vlantci_create( 662 __in efx_txq_t *etp, 663 __in uint16_t tci, 664 __out efx_desc_t *edp) 665 { 666 efx_nic_t *enp = etp->et_enp; 667 const efx_tx_ops_t *etxop = enp->en_etxop; 668 669 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 670 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 671 672 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 673 } 674 675 void 676 efx_tx_qdesc_checksum_create( 677 __in efx_txq_t *etp, 678 __in uint16_t flags, 679 __out efx_desc_t *edp) 680 { 681 efx_nic_t *enp = etp->et_enp; 682 const efx_tx_ops_t *etxop = enp->en_etxop; 683 684 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 685 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL); 686 687 etxop->etxo_qdesc_checksum_create(etp, flags, edp); 688 } 689 690 #if EFSYS_OPT_QSTATS 691 void 692 efx_tx_qstats_update( 693 __in efx_txq_t *etp, 694 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 695 { 696 efx_nic_t *enp = etp->et_enp; 697 const efx_tx_ops_t *etxop = enp->en_etxop; 698 699 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 700 701 etxop->etxo_qstats_update(etp, stat); 702 } 703 #endif 704 705 #if EFSYS_OPT_SIENA 706 707 static __checkReturn efx_rc_t 708 siena_tx_init( 709 __in efx_nic_t *enp) 710 { 711 efx_oword_t oword; 712 713 /* 714 * Disable the timer-based TX DMA backoff and allow TX DMA to be 715 * controlled by the RX FIFO fill level (although always allow a 716 * minimal trickle). 717 */ 718 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 719 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 720 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 721 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 722 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 723 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 724 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 725 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 726 727 /* 728 * Filter all packets less than 14 bytes to avoid parsing 729 * errors. 730 */ 731 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 732 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 733 734 /* 735 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 736 * descriptors (which is bad). 737 */ 738 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 739 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 740 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 741 742 return (0); 743 } 744 745 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 746 do { \ 747 unsigned int id; \ 748 size_t offset; \ 749 efx_qword_t qword; \ 750 \ 751 id = (_added)++ & (_etp)->et_mask; \ 752 offset = id * sizeof (efx_qword_t); \ 753 \ 754 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 755 unsigned int, id, efsys_dma_addr_t, (_addr), \ 756 size_t, (_size), boolean_t, (_eop)); \ 757 \ 758 EFX_POPULATE_QWORD_4(qword, \ 759 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 760 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 761 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 762 (uint32_t)((_addr) & 0xffffffff), \ 763 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 764 (uint32_t)((_addr) >> 32)); \ 765 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 766 \ 767 _NOTE(CONSTANTCONDITION) \ 768 } while (B_FALSE) 769 770 static __checkReturn efx_rc_t 771 siena_tx_qpost( 772 __in efx_txq_t *etp, 773 __in_ecount(ndescs) efx_buffer_t *eb, 774 __in unsigned int ndescs, 775 __in unsigned int completed, 776 __inout unsigned int *addedp) 777 { 778 unsigned int added = *addedp; 779 unsigned int i; 780 781 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) 782 return (ENOSPC); 783 784 for (i = 0; i < ndescs; i++) { 785 efx_buffer_t *ebp = &eb[i]; 786 efsys_dma_addr_t start = ebp->eb_addr; 787 size_t size = ebp->eb_size; 788 efsys_dma_addr_t end = start + size; 789 790 /* 791 * Fragments must not span 4k boundaries. 792 * Here it is a stricter requirement than the maximum length. 793 */ 794 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1, 795 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); 796 797 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 798 } 799 800 EFX_TX_QSTAT_INCR(etp, TX_POST); 801 802 *addedp = added; 803 return (0); 804 } 805 806 static void 807 siena_tx_qpush( 808 __in efx_txq_t *etp, 809 __in unsigned int added, 810 __in unsigned int pushed) 811 { 812 efx_nic_t *enp = etp->et_enp; 813 uint32_t wptr; 814 efx_dword_t dword; 815 efx_oword_t oword; 816 817 /* Push the populated descriptors out */ 818 wptr = added & etp->et_mask; 819 820 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 821 822 /* Only write the third DWORD */ 823 EFX_POPULATE_DWORD_1(dword, 824 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 825 826 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 827 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 828 wptr, pushed & etp->et_mask); 829 EFSYS_PIO_WRITE_BARRIER(); 830 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 831 etp->et_index, &dword, B_FALSE); 832 } 833 834 #define EFX_MAX_PACE_VALUE 20 835 #define EFX_TX_PACE_CLOCK_BASE 104 836 837 static __checkReturn efx_rc_t 838 siena_tx_qpace( 839 __in efx_txq_t *etp, 840 __in unsigned int ns) 841 { 842 efx_nic_t *enp = etp->et_enp; 843 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 844 efx_oword_t oword; 845 unsigned int pace_val; 846 unsigned int timer_period; 847 efx_rc_t rc; 848 849 if (ns == 0) { 850 pace_val = 0; 851 } else { 852 /* 853 * The pace_val to write into the table is s.t 854 * ns <= timer_period * (2 ^ pace_val) 855 */ 856 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 857 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 858 if ((timer_period << pace_val) >= ns) 859 break; 860 } 861 } 862 if (pace_val > EFX_MAX_PACE_VALUE) { 863 rc = EINVAL; 864 goto fail1; 865 } 866 867 /* Update the pacing table */ 868 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 869 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 870 &oword, B_TRUE); 871 872 return (0); 873 874 fail1: 875 EFSYS_PROBE1(fail1, efx_rc_t, rc); 876 877 return (rc); 878 } 879 880 static __checkReturn efx_rc_t 881 siena_tx_qflush( 882 __in efx_txq_t *etp) 883 { 884 efx_nic_t *enp = etp->et_enp; 885 efx_oword_t oword; 886 uint32_t label; 887 888 efx_tx_qpace(etp, 0); 889 890 label = etp->et_index; 891 892 /* Flush the queue */ 893 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 894 FRF_AZ_TX_FLUSH_DESCQ, label); 895 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 896 897 return (0); 898 } 899 900 static void 901 siena_tx_qenable( 902 __in efx_txq_t *etp) 903 { 904 efx_nic_t *enp = etp->et_enp; 905 efx_oword_t oword; 906 907 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 908 etp->et_index, &oword, B_TRUE); 909 910 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 911 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 912 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 913 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 914 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 915 916 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 917 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 918 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 919 920 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 921 etp->et_index, &oword, B_TRUE); 922 } 923 924 static __checkReturn efx_rc_t 925 siena_tx_qcreate( 926 __in efx_nic_t *enp, 927 __in unsigned int index, 928 __in unsigned int label, 929 __in efsys_mem_t *esmp, 930 __in size_t ndescs, 931 __in uint32_t id, 932 __in uint16_t flags, 933 __in efx_evq_t *eep, 934 __in efx_txq_t *etp, 935 __out unsigned int *addedp) 936 { 937 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 938 efx_oword_t oword; 939 uint32_t size; 940 uint16_t inner_csum; 941 efx_rc_t rc; 942 943 _NOTE(ARGUNUSED(esmp)) 944 945 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 946 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 947 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 948 949 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs)); 950 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 951 952 if (!ISP2(ndescs) || 953 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) { 954 rc = EINVAL; 955 goto fail1; 956 } 957 if (index >= encp->enc_txq_limit) { 958 rc = EINVAL; 959 goto fail2; 960 } 961 for (size = 0; 962 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS); 963 size++) 964 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS)) 965 break; 966 if (id + (1 << size) >= encp->enc_buftbl_limit) { 967 rc = EINVAL; 968 goto fail3; 969 } 970 971 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP; 972 if ((flags & inner_csum) != 0) { 973 rc = EINVAL; 974 goto fail4; 975 } 976 977 /* Set up the new descriptor queue */ 978 *addedp = 0; 979 980 EFX_POPULATE_OWORD_6(oword, 981 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 982 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 983 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 984 FRF_AZ_TX_DESCQ_LABEL, label, 985 FRF_AZ_TX_DESCQ_SIZE, size, 986 FRF_AZ_TX_DESCQ_TYPE, 0); 987 988 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 989 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 990 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 991 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 992 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 993 994 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 995 etp->et_index, &oword, B_TRUE); 996 997 return (0); 998 999 fail4: 1000 EFSYS_PROBE(fail4); 1001 fail3: 1002 EFSYS_PROBE(fail3); 1003 fail2: 1004 EFSYS_PROBE(fail2); 1005 fail1: 1006 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1007 1008 return (rc); 1009 } 1010 1011 __checkReturn efx_rc_t 1012 siena_tx_qdesc_post( 1013 __in efx_txq_t *etp, 1014 __in_ecount(ndescs) efx_desc_t *ed, 1015 __in unsigned int ndescs, 1016 __in unsigned int completed, 1017 __inout unsigned int *addedp) 1018 { 1019 unsigned int added = *addedp; 1020 unsigned int i; 1021 efx_rc_t rc; 1022 1023 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 1024 rc = ENOSPC; 1025 goto fail1; 1026 } 1027 1028 for (i = 0; i < ndescs; i++) { 1029 efx_desc_t *edp = &ed[i]; 1030 unsigned int id; 1031 size_t offset; 1032 1033 id = added++ & etp->et_mask; 1034 offset = id * sizeof (efx_desc_t); 1035 1036 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 1037 } 1038 1039 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 1040 unsigned int, added, unsigned int, ndescs); 1041 1042 EFX_TX_QSTAT_INCR(etp, TX_POST); 1043 1044 *addedp = added; 1045 return (0); 1046 1047 fail1: 1048 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1049 return (rc); 1050 } 1051 1052 void 1053 siena_tx_qdesc_dma_create( 1054 __in efx_txq_t *etp, 1055 __in efsys_dma_addr_t addr, 1056 __in size_t size, 1057 __in boolean_t eop, 1058 __out efx_desc_t *edp) 1059 { 1060 /* 1061 * Fragments must not span 4k boundaries. 1062 * Here it is a stricter requirement than the maximum length. 1063 */ 1064 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1, 1065 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); 1066 1067 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1068 efsys_dma_addr_t, addr, 1069 size_t, size, boolean_t, eop); 1070 1071 EFX_POPULATE_QWORD_4(edp->ed_eq, 1072 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1073 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1074 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1075 (uint32_t)(addr & 0xffffffff), 1076 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1077 (uint32_t)(addr >> 32)); 1078 } 1079 1080 #endif /* EFSYS_OPT_SIENA */ 1081 1082 #if EFSYS_OPT_QSTATS 1083 #if EFSYS_OPT_NAMES 1084 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ 1085 static const char * const __efx_tx_qstat_name[] = { 1086 "post", 1087 "post_pio", 1088 }; 1089 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1090 1091 const char * 1092 efx_tx_qstat_name( 1093 __in efx_nic_t *enp, 1094 __in unsigned int id) 1095 { 1096 _NOTE(ARGUNUSED(enp)) 1097 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1098 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1099 1100 return (__efx_tx_qstat_name[id]); 1101 } 1102 #endif /* EFSYS_OPT_NAMES */ 1103 #endif /* EFSYS_OPT_QSTATS */ 1104 1105 #if EFSYS_OPT_SIENA 1106 1107 #if EFSYS_OPT_QSTATS 1108 static void 1109 siena_tx_qstats_update( 1110 __in efx_txq_t *etp, 1111 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1112 { 1113 unsigned int id; 1114 1115 for (id = 0; id < TX_NQSTATS; id++) { 1116 efsys_stat_t *essp = &stat[id]; 1117 1118 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1119 etp->et_stat[id] = 0; 1120 } 1121 } 1122 #endif /* EFSYS_OPT_QSTATS */ 1123 1124 static void 1125 siena_tx_qdestroy( 1126 __in efx_txq_t *etp) 1127 { 1128 efx_nic_t *enp = etp->et_enp; 1129 efx_oword_t oword; 1130 1131 /* Purge descriptor queue */ 1132 EFX_ZERO_OWORD(oword); 1133 1134 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1135 etp->et_index, &oword, B_TRUE); 1136 } 1137 1138 static void 1139 siena_tx_fini( 1140 __in efx_nic_t *enp) 1141 { 1142 _NOTE(ARGUNUSED(enp)) 1143 } 1144 1145 #endif /* EFSYS_OPT_SIENA */ 1146