1 /*- 2 * Copyright (c) 2007-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 #if EFSYS_OPT_QSTATS 38 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 39 do { \ 40 (_etp)->et_stat[_stat]++; \ 41 _NOTE(CONSTANTCONDITION) \ 42 } while (B_FALSE) 43 #else 44 #define EFX_TX_QSTAT_INCR(_etp, _stat) 45 #endif 46 47 #if EFSYS_OPT_SIENA 48 49 static __checkReturn efx_rc_t 50 siena_tx_init( 51 __in efx_nic_t *enp); 52 53 static void 54 siena_tx_fini( 55 __in efx_nic_t *enp); 56 57 static __checkReturn efx_rc_t 58 siena_tx_qcreate( 59 __in efx_nic_t *enp, 60 __in unsigned int index, 61 __in unsigned int label, 62 __in efsys_mem_t *esmp, 63 __in size_t n, 64 __in uint32_t id, 65 __in uint16_t flags, 66 __in efx_evq_t *eep, 67 __in efx_txq_t *etp, 68 __out unsigned int *addedp); 69 70 static void 71 siena_tx_qdestroy( 72 __in efx_txq_t *etp); 73 74 static __checkReturn efx_rc_t 75 siena_tx_qpost( 76 __in efx_txq_t *etp, 77 __in_ecount(n) efx_buffer_t *eb, 78 __in unsigned int n, 79 __in unsigned int completed, 80 __inout unsigned int *addedp); 81 82 static void 83 siena_tx_qpush( 84 __in efx_txq_t *etp, 85 __in unsigned int added, 86 __in unsigned int pushed); 87 88 static __checkReturn efx_rc_t 89 siena_tx_qpace( 90 __in efx_txq_t *etp, 91 __in unsigned int ns); 92 93 static __checkReturn efx_rc_t 94 siena_tx_qflush( 95 __in efx_txq_t *etp); 96 97 static void 98 siena_tx_qenable( 99 __in efx_txq_t *etp); 100 101 __checkReturn efx_rc_t 102 siena_tx_qdesc_post( 103 __in efx_txq_t *etp, 104 __in_ecount(n) efx_desc_t *ed, 105 __in unsigned int n, 106 __in unsigned int completed, 107 __inout unsigned int *addedp); 108 109 void 110 siena_tx_qdesc_dma_create( 111 __in efx_txq_t *etp, 112 __in efsys_dma_addr_t addr, 113 __in size_t size, 114 __in boolean_t eop, 115 __out efx_desc_t *edp); 116 117 #if EFSYS_OPT_QSTATS 118 static void 119 siena_tx_qstats_update( 120 __in efx_txq_t *etp, 121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 122 #endif 123 124 #endif /* EFSYS_OPT_SIENA */ 125 126 127 #if EFSYS_OPT_SIENA 128 static const efx_tx_ops_t __efx_tx_siena_ops = { 129 siena_tx_init, /* etxo_init */ 130 siena_tx_fini, /* etxo_fini */ 131 siena_tx_qcreate, /* etxo_qcreate */ 132 siena_tx_qdestroy, /* etxo_qdestroy */ 133 siena_tx_qpost, /* etxo_qpost */ 134 siena_tx_qpush, /* etxo_qpush */ 135 siena_tx_qpace, /* etxo_qpace */ 136 siena_tx_qflush, /* etxo_qflush */ 137 siena_tx_qenable, /* etxo_qenable */ 138 NULL, /* etxo_qpio_enable */ 139 NULL, /* etxo_qpio_disable */ 140 NULL, /* etxo_qpio_write */ 141 NULL, /* etxo_qpio_post */ 142 siena_tx_qdesc_post, /* etxo_qdesc_post */ 143 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 144 NULL, /* etxo_qdesc_tso_create */ 145 NULL, /* etxo_qdesc_tso2_create */ 146 NULL, /* etxo_qdesc_vlantci_create */ 147 #if EFSYS_OPT_QSTATS 148 siena_tx_qstats_update, /* etxo_qstats_update */ 149 #endif 150 }; 151 #endif /* EFSYS_OPT_SIENA */ 152 153 #if EFSYS_OPT_HUNTINGTON 154 static const efx_tx_ops_t __efx_tx_hunt_ops = { 155 ef10_tx_init, /* etxo_init */ 156 ef10_tx_fini, /* etxo_fini */ 157 ef10_tx_qcreate, /* etxo_qcreate */ 158 ef10_tx_qdestroy, /* etxo_qdestroy */ 159 ef10_tx_qpost, /* etxo_qpost */ 160 ef10_tx_qpush, /* etxo_qpush */ 161 ef10_tx_qpace, /* etxo_qpace */ 162 ef10_tx_qflush, /* etxo_qflush */ 163 ef10_tx_qenable, /* etxo_qenable */ 164 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 165 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 166 ef10_tx_qpio_write, /* etxo_qpio_write */ 167 ef10_tx_qpio_post, /* etxo_qpio_post */ 168 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 169 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 170 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 171 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 172 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 173 #if EFSYS_OPT_QSTATS 174 ef10_tx_qstats_update, /* etxo_qstats_update */ 175 #endif 176 }; 177 #endif /* EFSYS_OPT_HUNTINGTON */ 178 179 #if EFSYS_OPT_MEDFORD 180 static const efx_tx_ops_t __efx_tx_medford_ops = { 181 ef10_tx_init, /* etxo_init */ 182 ef10_tx_fini, /* etxo_fini */ 183 ef10_tx_qcreate, /* etxo_qcreate */ 184 ef10_tx_qdestroy, /* etxo_qdestroy */ 185 ef10_tx_qpost, /* etxo_qpost */ 186 ef10_tx_qpush, /* etxo_qpush */ 187 ef10_tx_qpace, /* etxo_qpace */ 188 ef10_tx_qflush, /* etxo_qflush */ 189 ef10_tx_qenable, /* etxo_qenable */ 190 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 191 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 192 ef10_tx_qpio_write, /* etxo_qpio_write */ 193 ef10_tx_qpio_post, /* etxo_qpio_post */ 194 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 195 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 196 NULL, /* etxo_qdesc_tso_create */ 197 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 198 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 199 #if EFSYS_OPT_QSTATS 200 ef10_tx_qstats_update, /* etxo_qstats_update */ 201 #endif 202 }; 203 #endif /* EFSYS_OPT_MEDFORD */ 204 205 __checkReturn efx_rc_t 206 efx_tx_init( 207 __in efx_nic_t *enp) 208 { 209 const efx_tx_ops_t *etxop; 210 efx_rc_t rc; 211 212 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 213 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 214 215 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 216 rc = EINVAL; 217 goto fail1; 218 } 219 220 if (enp->en_mod_flags & EFX_MOD_TX) { 221 rc = EINVAL; 222 goto fail2; 223 } 224 225 switch (enp->en_family) { 226 #if EFSYS_OPT_SIENA 227 case EFX_FAMILY_SIENA: 228 etxop = &__efx_tx_siena_ops; 229 break; 230 #endif /* EFSYS_OPT_SIENA */ 231 232 #if EFSYS_OPT_HUNTINGTON 233 case EFX_FAMILY_HUNTINGTON: 234 etxop = &__efx_tx_hunt_ops; 235 break; 236 #endif /* EFSYS_OPT_HUNTINGTON */ 237 238 #if EFSYS_OPT_MEDFORD 239 case EFX_FAMILY_MEDFORD: 240 etxop = &__efx_tx_medford_ops; 241 break; 242 #endif /* EFSYS_OPT_MEDFORD */ 243 244 default: 245 EFSYS_ASSERT(0); 246 rc = ENOTSUP; 247 goto fail3; 248 } 249 250 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 251 252 if ((rc = etxop->etxo_init(enp)) != 0) 253 goto fail4; 254 255 enp->en_etxop = etxop; 256 enp->en_mod_flags |= EFX_MOD_TX; 257 return (0); 258 259 fail4: 260 EFSYS_PROBE(fail4); 261 fail3: 262 EFSYS_PROBE(fail3); 263 fail2: 264 EFSYS_PROBE(fail2); 265 fail1: 266 EFSYS_PROBE1(fail1, efx_rc_t, rc); 267 268 enp->en_etxop = NULL; 269 enp->en_mod_flags &= ~EFX_MOD_TX; 270 return (rc); 271 } 272 273 void 274 efx_tx_fini( 275 __in efx_nic_t *enp) 276 { 277 const efx_tx_ops_t *etxop = enp->en_etxop; 278 279 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 280 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 281 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 282 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 283 284 etxop->etxo_fini(enp); 285 286 enp->en_etxop = NULL; 287 enp->en_mod_flags &= ~EFX_MOD_TX; 288 } 289 290 __checkReturn efx_rc_t 291 efx_tx_qcreate( 292 __in efx_nic_t *enp, 293 __in unsigned int index, 294 __in unsigned int label, 295 __in efsys_mem_t *esmp, 296 __in size_t n, 297 __in uint32_t id, 298 __in uint16_t flags, 299 __in efx_evq_t *eep, 300 __deref_out efx_txq_t **etpp, 301 __out unsigned int *addedp) 302 { 303 const efx_tx_ops_t *etxop = enp->en_etxop; 304 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 305 efx_txq_t *etp; 306 efx_rc_t rc; 307 308 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 309 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 310 311 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 312 313 /* Allocate an TXQ object */ 314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 315 316 if (etp == NULL) { 317 rc = ENOMEM; 318 goto fail1; 319 } 320 321 etp->et_magic = EFX_TXQ_MAGIC; 322 etp->et_enp = enp; 323 etp->et_index = index; 324 etp->et_mask = n - 1; 325 etp->et_esmp = esmp; 326 327 /* Initial descriptor index may be modified by etxo_qcreate */ 328 *addedp = 0; 329 330 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 331 n, id, flags, eep, etp, addedp)) != 0) 332 goto fail2; 333 334 enp->en_tx_qcount++; 335 *etpp = etp; 336 337 return (0); 338 339 fail2: 340 EFSYS_PROBE(fail2); 341 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 342 fail1: 343 EFSYS_PROBE1(fail1, efx_rc_t, rc); 344 return (rc); 345 } 346 347 void 348 efx_tx_qdestroy( 349 __in efx_txq_t *etp) 350 { 351 efx_nic_t *enp = etp->et_enp; 352 const efx_tx_ops_t *etxop = enp->en_etxop; 353 354 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 355 356 EFSYS_ASSERT(enp->en_tx_qcount != 0); 357 --enp->en_tx_qcount; 358 359 etxop->etxo_qdestroy(etp); 360 361 /* Free the TXQ object */ 362 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 363 } 364 365 __checkReturn efx_rc_t 366 efx_tx_qpost( 367 __in efx_txq_t *etp, 368 __in_ecount(n) efx_buffer_t *eb, 369 __in unsigned int n, 370 __in unsigned int completed, 371 __inout unsigned int *addedp) 372 { 373 efx_nic_t *enp = etp->et_enp; 374 const efx_tx_ops_t *etxop = enp->en_etxop; 375 efx_rc_t rc; 376 377 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 378 379 if ((rc = etxop->etxo_qpost(etp, eb, 380 n, completed, addedp)) != 0) 381 goto fail1; 382 383 return (0); 384 385 fail1: 386 EFSYS_PROBE1(fail1, efx_rc_t, rc); 387 return (rc); 388 } 389 390 void 391 efx_tx_qpush( 392 __in efx_txq_t *etp, 393 __in unsigned int added, 394 __in unsigned int pushed) 395 { 396 efx_nic_t *enp = etp->et_enp; 397 const efx_tx_ops_t *etxop = enp->en_etxop; 398 399 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 400 401 etxop->etxo_qpush(etp, added, pushed); 402 } 403 404 __checkReturn efx_rc_t 405 efx_tx_qpace( 406 __in efx_txq_t *etp, 407 __in unsigned int ns) 408 { 409 efx_nic_t *enp = etp->et_enp; 410 const efx_tx_ops_t *etxop = enp->en_etxop; 411 efx_rc_t rc; 412 413 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 414 415 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 416 goto fail1; 417 418 return (0); 419 420 fail1: 421 EFSYS_PROBE1(fail1, efx_rc_t, rc); 422 return (rc); 423 } 424 425 __checkReturn efx_rc_t 426 efx_tx_qflush( 427 __in efx_txq_t *etp) 428 { 429 efx_nic_t *enp = etp->et_enp; 430 const efx_tx_ops_t *etxop = enp->en_etxop; 431 efx_rc_t rc; 432 433 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 434 435 if ((rc = etxop->etxo_qflush(etp)) != 0) 436 goto fail1; 437 438 return (0); 439 440 fail1: 441 EFSYS_PROBE1(fail1, efx_rc_t, rc); 442 return (rc); 443 } 444 445 void 446 efx_tx_qenable( 447 __in efx_txq_t *etp) 448 { 449 efx_nic_t *enp = etp->et_enp; 450 const efx_tx_ops_t *etxop = enp->en_etxop; 451 452 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 453 454 etxop->etxo_qenable(etp); 455 } 456 457 __checkReturn efx_rc_t 458 efx_tx_qpio_enable( 459 __in efx_txq_t *etp) 460 { 461 efx_nic_t *enp = etp->et_enp; 462 const efx_tx_ops_t *etxop = enp->en_etxop; 463 efx_rc_t rc; 464 465 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 466 467 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 468 rc = ENOTSUP; 469 goto fail1; 470 } 471 if (etxop->etxo_qpio_enable == NULL) { 472 rc = ENOTSUP; 473 goto fail2; 474 } 475 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 476 goto fail3; 477 478 return (0); 479 480 fail3: 481 EFSYS_PROBE(fail3); 482 fail2: 483 EFSYS_PROBE(fail2); 484 fail1: 485 EFSYS_PROBE1(fail1, efx_rc_t, rc); 486 return (rc); 487 } 488 489 void 490 efx_tx_qpio_disable( 491 __in efx_txq_t *etp) 492 { 493 efx_nic_t *enp = etp->et_enp; 494 const efx_tx_ops_t *etxop = enp->en_etxop; 495 496 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 497 498 if (etxop->etxo_qpio_disable != NULL) 499 etxop->etxo_qpio_disable(etp); 500 } 501 502 __checkReturn efx_rc_t 503 efx_tx_qpio_write( 504 __in efx_txq_t *etp, 505 __in_ecount(buf_length) uint8_t *buffer, 506 __in size_t buf_length, 507 __in size_t pio_buf_offset) 508 { 509 efx_nic_t *enp = etp->et_enp; 510 const efx_tx_ops_t *etxop = enp->en_etxop; 511 efx_rc_t rc; 512 513 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 514 515 if (etxop->etxo_qpio_write != NULL) { 516 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 517 pio_buf_offset)) != 0) 518 goto fail1; 519 return (0); 520 } 521 522 return (ENOTSUP); 523 524 fail1: 525 EFSYS_PROBE1(fail1, efx_rc_t, rc); 526 return (rc); 527 } 528 529 __checkReturn efx_rc_t 530 efx_tx_qpio_post( 531 __in efx_txq_t *etp, 532 __in size_t pkt_length, 533 __in unsigned int completed, 534 __inout unsigned int *addedp) 535 { 536 efx_nic_t *enp = etp->et_enp; 537 const efx_tx_ops_t *etxop = enp->en_etxop; 538 efx_rc_t rc; 539 540 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 541 542 if (etxop->etxo_qpio_post != NULL) { 543 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 544 addedp)) != 0) 545 goto fail1; 546 return (0); 547 } 548 549 return (ENOTSUP); 550 551 fail1: 552 EFSYS_PROBE1(fail1, efx_rc_t, rc); 553 return (rc); 554 } 555 556 __checkReturn efx_rc_t 557 efx_tx_qdesc_post( 558 __in efx_txq_t *etp, 559 __in_ecount(n) efx_desc_t *ed, 560 __in unsigned int n, 561 __in unsigned int completed, 562 __inout unsigned int *addedp) 563 { 564 efx_nic_t *enp = etp->et_enp; 565 const efx_tx_ops_t *etxop = enp->en_etxop; 566 efx_rc_t rc; 567 568 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 569 570 if ((rc = etxop->etxo_qdesc_post(etp, ed, 571 n, completed, addedp)) != 0) 572 goto fail1; 573 574 return (0); 575 576 fail1: 577 EFSYS_PROBE1(fail1, efx_rc_t, rc); 578 return (rc); 579 } 580 581 void 582 efx_tx_qdesc_dma_create( 583 __in efx_txq_t *etp, 584 __in efsys_dma_addr_t addr, 585 __in size_t size, 586 __in boolean_t eop, 587 __out efx_desc_t *edp) 588 { 589 efx_nic_t *enp = etp->et_enp; 590 const efx_tx_ops_t *etxop = enp->en_etxop; 591 592 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 593 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 594 595 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 596 } 597 598 void 599 efx_tx_qdesc_tso_create( 600 __in efx_txq_t *etp, 601 __in uint16_t ipv4_id, 602 __in uint32_t tcp_seq, 603 __in uint8_t tcp_flags, 604 __out efx_desc_t *edp) 605 { 606 efx_nic_t *enp = etp->et_enp; 607 const efx_tx_ops_t *etxop = enp->en_etxop; 608 609 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 610 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 611 612 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 613 } 614 615 void 616 efx_tx_qdesc_tso2_create( 617 __in efx_txq_t *etp, 618 __in uint16_t ipv4_id, 619 __in uint32_t tcp_seq, 620 __in uint16_t mss, 621 __out_ecount(count) efx_desc_t *edp, 622 __in int count) 623 { 624 efx_nic_t *enp = etp->et_enp; 625 const efx_tx_ops_t *etxop = enp->en_etxop; 626 627 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 628 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 629 630 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count); 631 } 632 633 void 634 efx_tx_qdesc_vlantci_create( 635 __in efx_txq_t *etp, 636 __in uint16_t tci, 637 __out efx_desc_t *edp) 638 { 639 efx_nic_t *enp = etp->et_enp; 640 const efx_tx_ops_t *etxop = enp->en_etxop; 641 642 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 643 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 644 645 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 646 } 647 648 649 #if EFSYS_OPT_QSTATS 650 void 651 efx_tx_qstats_update( 652 __in efx_txq_t *etp, 653 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 654 { 655 efx_nic_t *enp = etp->et_enp; 656 const efx_tx_ops_t *etxop = enp->en_etxop; 657 658 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 659 660 etxop->etxo_qstats_update(etp, stat); 661 } 662 #endif 663 664 665 #if EFSYS_OPT_SIENA 666 667 static __checkReturn efx_rc_t 668 siena_tx_init( 669 __in efx_nic_t *enp) 670 { 671 efx_oword_t oword; 672 673 /* 674 * Disable the timer-based TX DMA backoff and allow TX DMA to be 675 * controlled by the RX FIFO fill level (although always allow a 676 * minimal trickle). 677 */ 678 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 679 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 680 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 682 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 683 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 684 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 685 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 686 687 /* 688 * Filter all packets less than 14 bytes to avoid parsing 689 * errors. 690 */ 691 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 692 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 693 694 /* 695 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 696 * descriptors (which is bad). 697 */ 698 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 699 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 700 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 701 702 return (0); 703 } 704 705 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 706 do { \ 707 unsigned int id; \ 708 size_t offset; \ 709 efx_qword_t qword; \ 710 \ 711 id = (_added)++ & (_etp)->et_mask; \ 712 offset = id * sizeof (efx_qword_t); \ 713 \ 714 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 715 unsigned int, id, efsys_dma_addr_t, (_addr), \ 716 size_t, (_size), boolean_t, (_eop)); \ 717 \ 718 EFX_POPULATE_QWORD_4(qword, \ 719 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 720 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 721 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 722 (uint32_t)((_addr) & 0xffffffff), \ 723 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 724 (uint32_t)((_addr) >> 32)); \ 725 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 726 \ 727 _NOTE(CONSTANTCONDITION) \ 728 } while (B_FALSE) 729 730 static __checkReturn efx_rc_t 731 siena_tx_qpost( 732 __in efx_txq_t *etp, 733 __in_ecount(n) efx_buffer_t *eb, 734 __in unsigned int n, 735 __in unsigned int completed, 736 __inout unsigned int *addedp) 737 { 738 unsigned int added = *addedp; 739 unsigned int i; 740 int rc = ENOSPC; 741 742 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 743 goto fail1; 744 745 for (i = 0; i < n; i++) { 746 efx_buffer_t *ebp = &eb[i]; 747 efsys_dma_addr_t start = ebp->eb_addr; 748 size_t size = ebp->eb_size; 749 efsys_dma_addr_t end = start + size; 750 751 /* 752 * Fragments must not span 4k boundaries. 753 * Here it is a stricter requirement than the maximum length. 754 */ 755 EFSYS_ASSERT(P2ROUNDUP(start + 1, 756 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); 757 758 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 759 } 760 761 EFX_TX_QSTAT_INCR(etp, TX_POST); 762 763 *addedp = added; 764 return (0); 765 766 fail1: 767 EFSYS_PROBE1(fail1, efx_rc_t, rc); 768 769 return (rc); 770 } 771 772 static void 773 siena_tx_qpush( 774 __in efx_txq_t *etp, 775 __in unsigned int added, 776 __in unsigned int pushed) 777 { 778 efx_nic_t *enp = etp->et_enp; 779 uint32_t wptr; 780 efx_dword_t dword; 781 efx_oword_t oword; 782 783 /* Push the populated descriptors out */ 784 wptr = added & etp->et_mask; 785 786 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 787 788 /* Only write the third DWORD */ 789 EFX_POPULATE_DWORD_1(dword, 790 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 791 792 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 793 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 794 wptr, pushed & etp->et_mask); 795 EFSYS_PIO_WRITE_BARRIER(); 796 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 797 etp->et_index, &dword, B_FALSE); 798 } 799 800 #define EFX_MAX_PACE_VALUE 20 801 #define EFX_TX_PACE_CLOCK_BASE 104 802 803 static __checkReturn efx_rc_t 804 siena_tx_qpace( 805 __in efx_txq_t *etp, 806 __in unsigned int ns) 807 { 808 efx_nic_t *enp = etp->et_enp; 809 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 810 efx_oword_t oword; 811 unsigned int pace_val; 812 unsigned int timer_period; 813 efx_rc_t rc; 814 815 if (ns == 0) { 816 pace_val = 0; 817 } else { 818 /* 819 * The pace_val to write into the table is s.t 820 * ns <= timer_period * (2 ^ pace_val) 821 */ 822 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 823 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 824 if ((timer_period << pace_val) >= ns) 825 break; 826 } 827 } 828 if (pace_val > EFX_MAX_PACE_VALUE) { 829 rc = EINVAL; 830 goto fail1; 831 } 832 833 /* Update the pacing table */ 834 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 835 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 836 &oword, B_TRUE); 837 838 return (0); 839 840 fail1: 841 EFSYS_PROBE1(fail1, efx_rc_t, rc); 842 843 return (rc); 844 } 845 846 static __checkReturn efx_rc_t 847 siena_tx_qflush( 848 __in efx_txq_t *etp) 849 { 850 efx_nic_t *enp = etp->et_enp; 851 efx_oword_t oword; 852 uint32_t label; 853 854 efx_tx_qpace(etp, 0); 855 856 label = etp->et_index; 857 858 /* Flush the queue */ 859 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 860 FRF_AZ_TX_FLUSH_DESCQ, label); 861 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 862 863 return (0); 864 } 865 866 static void 867 siena_tx_qenable( 868 __in efx_txq_t *etp) 869 { 870 efx_nic_t *enp = etp->et_enp; 871 efx_oword_t oword; 872 873 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 874 etp->et_index, &oword, B_TRUE); 875 876 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 877 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 878 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 879 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 880 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 881 882 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 883 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 884 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 885 886 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 887 etp->et_index, &oword, B_TRUE); 888 } 889 890 static __checkReturn efx_rc_t 891 siena_tx_qcreate( 892 __in efx_nic_t *enp, 893 __in unsigned int index, 894 __in unsigned int label, 895 __in efsys_mem_t *esmp, 896 __in size_t n, 897 __in uint32_t id, 898 __in uint16_t flags, 899 __in efx_evq_t *eep, 900 __in efx_txq_t *etp, 901 __out unsigned int *addedp) 902 { 903 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 904 efx_oword_t oword; 905 uint32_t size; 906 efx_rc_t rc; 907 908 _NOTE(ARGUNUSED(esmp)) 909 910 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 911 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 912 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 913 914 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp))); 915 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 916 917 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { 918 rc = EINVAL; 919 goto fail1; 920 } 921 if (index >= encp->enc_txq_limit) { 922 rc = EINVAL; 923 goto fail2; 924 } 925 for (size = 0; 926 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS); 927 size++) 928 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 929 break; 930 if (id + (1 << size) >= encp->enc_buftbl_limit) { 931 rc = EINVAL; 932 goto fail3; 933 } 934 935 /* Set up the new descriptor queue */ 936 *addedp = 0; 937 938 EFX_POPULATE_OWORD_6(oword, 939 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 940 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 941 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 942 FRF_AZ_TX_DESCQ_LABEL, label, 943 FRF_AZ_TX_DESCQ_SIZE, size, 944 FRF_AZ_TX_DESCQ_TYPE, 0); 945 946 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 947 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 948 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 949 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 950 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 951 952 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 953 etp->et_index, &oword, B_TRUE); 954 955 return (0); 956 957 fail3: 958 EFSYS_PROBE(fail3); 959 fail2: 960 EFSYS_PROBE(fail2); 961 fail1: 962 EFSYS_PROBE1(fail1, efx_rc_t, rc); 963 964 return (rc); 965 } 966 967 __checkReturn efx_rc_t 968 siena_tx_qdesc_post( 969 __in efx_txq_t *etp, 970 __in_ecount(n) efx_desc_t *ed, 971 __in unsigned int n, 972 __in unsigned int completed, 973 __inout unsigned int *addedp) 974 { 975 unsigned int added = *addedp; 976 unsigned int i; 977 efx_rc_t rc; 978 979 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 980 rc = ENOSPC; 981 goto fail1; 982 } 983 984 for (i = 0; i < n; i++) { 985 efx_desc_t *edp = &ed[i]; 986 unsigned int id; 987 size_t offset; 988 989 id = added++ & etp->et_mask; 990 offset = id * sizeof (efx_desc_t); 991 992 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 993 } 994 995 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 996 unsigned int, added, unsigned int, n); 997 998 EFX_TX_QSTAT_INCR(etp, TX_POST); 999 1000 *addedp = added; 1001 return (0); 1002 1003 fail1: 1004 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1005 return (rc); 1006 } 1007 1008 void 1009 siena_tx_qdesc_dma_create( 1010 __in efx_txq_t *etp, 1011 __in efsys_dma_addr_t addr, 1012 __in size_t size, 1013 __in boolean_t eop, 1014 __out efx_desc_t *edp) 1015 { 1016 /* 1017 * Fragments must not span 4k boundaries. 1018 * Here it is a stricter requirement than the maximum length. 1019 */ 1020 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 1021 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); 1022 1023 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1024 efsys_dma_addr_t, addr, 1025 size_t, size, boolean_t, eop); 1026 1027 EFX_POPULATE_QWORD_4(edp->ed_eq, 1028 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1029 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1030 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1031 (uint32_t)(addr & 0xffffffff), 1032 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1033 (uint32_t)(addr >> 32)); 1034 } 1035 1036 #endif /* EFSYS_OPT_SIENA */ 1037 1038 #if EFSYS_OPT_QSTATS 1039 #if EFSYS_OPT_NAMES 1040 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ 1041 static const char * const __efx_tx_qstat_name[] = { 1042 "post", 1043 "post_pio", 1044 }; 1045 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1046 1047 const char * 1048 efx_tx_qstat_name( 1049 __in efx_nic_t *enp, 1050 __in unsigned int id) 1051 { 1052 _NOTE(ARGUNUSED(enp)) 1053 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1054 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1055 1056 return (__efx_tx_qstat_name[id]); 1057 } 1058 #endif /* EFSYS_OPT_NAMES */ 1059 #endif /* EFSYS_OPT_QSTATS */ 1060 1061 #if EFSYS_OPT_SIENA 1062 1063 #if EFSYS_OPT_QSTATS 1064 static void 1065 siena_tx_qstats_update( 1066 __in efx_txq_t *etp, 1067 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1068 { 1069 unsigned int id; 1070 1071 for (id = 0; id < TX_NQSTATS; id++) { 1072 efsys_stat_t *essp = &stat[id]; 1073 1074 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1075 etp->et_stat[id] = 0; 1076 } 1077 } 1078 #endif /* EFSYS_OPT_QSTATS */ 1079 1080 static void 1081 siena_tx_qdestroy( 1082 __in efx_txq_t *etp) 1083 { 1084 efx_nic_t *enp = etp->et_enp; 1085 efx_oword_t oword; 1086 1087 /* Purge descriptor queue */ 1088 EFX_ZERO_OWORD(oword); 1089 1090 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1091 etp->et_index, &oword, B_TRUE); 1092 } 1093 1094 static void 1095 siena_tx_fini( 1096 __in efx_nic_t *enp) 1097 { 1098 _NOTE(ARGUNUSED(enp)) 1099 } 1100 1101 #endif /* EFSYS_OPT_SIENA */ 1102