1 /*- 2 * Copyright (c) 2007-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 #if EFSYS_OPT_QSTATS 38 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 39 do { \ 40 (_etp)->et_stat[_stat]++; \ 41 _NOTE(CONSTANTCONDITION) \ 42 } while (B_FALSE) 43 #else 44 #define EFX_TX_QSTAT_INCR(_etp, _stat) 45 #endif 46 47 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 48 49 static __checkReturn efx_rc_t 50 falconsiena_tx_init( 51 __in efx_nic_t *enp); 52 53 static void 54 falconsiena_tx_fini( 55 __in efx_nic_t *enp); 56 57 static __checkReturn efx_rc_t 58 falconsiena_tx_qcreate( 59 __in efx_nic_t *enp, 60 __in unsigned int index, 61 __in unsigned int label, 62 __in efsys_mem_t *esmp, 63 __in size_t n, 64 __in uint32_t id, 65 __in uint16_t flags, 66 __in efx_evq_t *eep, 67 __in efx_txq_t *etp, 68 __out unsigned int *addedp); 69 70 static void 71 falconsiena_tx_qdestroy( 72 __in efx_txq_t *etp); 73 74 static __checkReturn efx_rc_t 75 falconsiena_tx_qpost( 76 __in efx_txq_t *etp, 77 __in_ecount(n) efx_buffer_t *eb, 78 __in unsigned int n, 79 __in unsigned int completed, 80 __inout unsigned int *addedp); 81 82 static void 83 falconsiena_tx_qpush( 84 __in efx_txq_t *etp, 85 __in unsigned int added, 86 __in unsigned int pushed); 87 88 static __checkReturn efx_rc_t 89 falconsiena_tx_qpace( 90 __in efx_txq_t *etp, 91 __in unsigned int ns); 92 93 static __checkReturn efx_rc_t 94 falconsiena_tx_qflush( 95 __in efx_txq_t *etp); 96 97 static void 98 falconsiena_tx_qenable( 99 __in efx_txq_t *etp); 100 101 __checkReturn efx_rc_t 102 falconsiena_tx_qdesc_post( 103 __in efx_txq_t *etp, 104 __in_ecount(n) efx_desc_t *ed, 105 __in unsigned int n, 106 __in unsigned int completed, 107 __inout unsigned int *addedp); 108 109 void 110 falconsiena_tx_qdesc_dma_create( 111 __in efx_txq_t *etp, 112 __in efsys_dma_addr_t addr, 113 __in size_t size, 114 __in boolean_t eop, 115 __out efx_desc_t *edp); 116 117 #if EFSYS_OPT_QSTATS 118 static void 119 falconsiena_tx_qstats_update( 120 __in efx_txq_t *etp, 121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 122 #endif 123 124 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 125 126 127 #if EFSYS_OPT_FALCON 128 static efx_tx_ops_t __efx_tx_falcon_ops = { 129 falconsiena_tx_init, /* etxo_init */ 130 falconsiena_tx_fini, /* etxo_fini */ 131 falconsiena_tx_qcreate, /* etxo_qcreate */ 132 falconsiena_tx_qdestroy, /* etxo_qdestroy */ 133 falconsiena_tx_qpost, /* etxo_qpost */ 134 falconsiena_tx_qpush, /* etxo_qpush */ 135 falconsiena_tx_qpace, /* etxo_qpace */ 136 falconsiena_tx_qflush, /* etxo_qflush */ 137 falconsiena_tx_qenable, /* etxo_qenable */ 138 NULL, /* etxo_qpio_enable */ 139 NULL, /* etxo_qpio_disable */ 140 NULL, /* etxo_qpio_write */ 141 NULL, /* etxo_qpio_post */ 142 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */ 143 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 144 NULL, /* etxo_qdesc_tso_create */ 145 NULL, /* etxo_qdesc_vlantci_create */ 146 #if EFSYS_OPT_QSTATS 147 falconsiena_tx_qstats_update, /* etxo_qstats_update */ 148 #endif 149 }; 150 #endif /* EFSYS_OPT_FALCON */ 151 152 #if EFSYS_OPT_SIENA 153 static efx_tx_ops_t __efx_tx_siena_ops = { 154 falconsiena_tx_init, /* etxo_init */ 155 falconsiena_tx_fini, /* etxo_fini */ 156 falconsiena_tx_qcreate, /* etxo_qcreate */ 157 falconsiena_tx_qdestroy, /* etxo_qdestroy */ 158 falconsiena_tx_qpost, /* etxo_qpost */ 159 falconsiena_tx_qpush, /* etxo_qpush */ 160 falconsiena_tx_qpace, /* etxo_qpace */ 161 falconsiena_tx_qflush, /* etxo_qflush */ 162 falconsiena_tx_qenable, /* etxo_qenable */ 163 NULL, /* etxo_qpio_enable */ 164 NULL, /* etxo_qpio_disable */ 165 NULL, /* etxo_qpio_write */ 166 NULL, /* etxo_qpio_post */ 167 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */ 168 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 169 NULL, /* etxo_qdesc_tso_create */ 170 NULL, /* etxo_qdesc_vlantci_create */ 171 #if EFSYS_OPT_QSTATS 172 falconsiena_tx_qstats_update, /* etxo_qstats_update */ 173 #endif 174 }; 175 #endif /* EFSYS_OPT_SIENA */ 176 177 #if EFSYS_OPT_HUNTINGTON 178 static efx_tx_ops_t __efx_tx_hunt_ops = { 179 ef10_tx_init, /* etxo_init */ 180 ef10_tx_fini, /* etxo_fini */ 181 ef10_tx_qcreate, /* etxo_qcreate */ 182 ef10_tx_qdestroy, /* etxo_qdestroy */ 183 ef10_tx_qpost, /* etxo_qpost */ 184 ef10_tx_qpush, /* etxo_qpush */ 185 ef10_tx_qpace, /* etxo_qpace */ 186 ef10_tx_qflush, /* etxo_qflush */ 187 ef10_tx_qenable, /* etxo_qenable */ 188 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 189 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 190 ef10_tx_qpio_write, /* etxo_qpio_write */ 191 ef10_tx_qpio_post, /* etxo_qpio_post */ 192 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 193 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 194 hunt_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 195 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 196 #if EFSYS_OPT_QSTATS 197 ef10_tx_qstats_update, /* etxo_qstats_update */ 198 #endif 199 }; 200 #endif /* EFSYS_OPT_HUNTINGTON */ 201 202 #if EFSYS_OPT_MEDFORD 203 static efx_tx_ops_t __efx_tx_medford_ops = { 204 ef10_tx_init, /* etxo_init */ 205 ef10_tx_fini, /* etxo_fini */ 206 ef10_tx_qcreate, /* etxo_qcreate */ 207 ef10_tx_qdestroy, /* etxo_qdestroy */ 208 ef10_tx_qpost, /* etxo_qpost */ 209 ef10_tx_qpush, /* etxo_qpush */ 210 ef10_tx_qpace, /* etxo_qpace */ 211 ef10_tx_qflush, /* etxo_qflush */ 212 ef10_tx_qenable, /* etxo_qenable */ 213 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 214 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 215 ef10_tx_qpio_write, /* etxo_qpio_write */ 216 ef10_tx_qpio_post, /* etxo_qpio_post */ 217 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 218 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 219 NULL, /* etxo_qdesc_tso_create */ 220 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 221 #if EFSYS_OPT_QSTATS 222 ef10_tx_qstats_update, /* etxo_qstats_update */ 223 #endif 224 }; 225 #endif /* EFSYS_OPT_MEDFORD */ 226 227 __checkReturn efx_rc_t 228 efx_tx_init( 229 __in efx_nic_t *enp) 230 { 231 efx_tx_ops_t *etxop; 232 efx_rc_t rc; 233 234 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 235 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 236 237 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 238 rc = EINVAL; 239 goto fail1; 240 } 241 242 if (enp->en_mod_flags & EFX_MOD_TX) { 243 rc = EINVAL; 244 goto fail2; 245 } 246 247 switch (enp->en_family) { 248 #if EFSYS_OPT_FALCON 249 case EFX_FAMILY_FALCON: 250 etxop = (efx_tx_ops_t *)&__efx_tx_falcon_ops; 251 break; 252 #endif /* EFSYS_OPT_FALCON */ 253 254 #if EFSYS_OPT_SIENA 255 case EFX_FAMILY_SIENA: 256 etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops; 257 break; 258 #endif /* EFSYS_OPT_SIENA */ 259 260 #if EFSYS_OPT_HUNTINGTON 261 case EFX_FAMILY_HUNTINGTON: 262 etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops; 263 break; 264 #endif /* EFSYS_OPT_HUNTINGTON */ 265 266 #if EFSYS_OPT_MEDFORD 267 case EFX_FAMILY_MEDFORD: 268 etxop = (efx_tx_ops_t *)&__efx_tx_medford_ops; 269 break; 270 #endif /* EFSYS_OPT_MEDFORD */ 271 272 default: 273 EFSYS_ASSERT(0); 274 rc = ENOTSUP; 275 goto fail3; 276 } 277 278 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 279 280 if ((rc = etxop->etxo_init(enp)) != 0) 281 goto fail4; 282 283 enp->en_etxop = etxop; 284 enp->en_mod_flags |= EFX_MOD_TX; 285 return (0); 286 287 fail4: 288 EFSYS_PROBE(fail4); 289 fail3: 290 EFSYS_PROBE(fail3); 291 fail2: 292 EFSYS_PROBE(fail2); 293 fail1: 294 EFSYS_PROBE1(fail1, efx_rc_t, rc); 295 296 enp->en_etxop = NULL; 297 enp->en_mod_flags &= ~EFX_MOD_TX; 298 return (rc); 299 } 300 301 void 302 efx_tx_fini( 303 __in efx_nic_t *enp) 304 { 305 efx_tx_ops_t *etxop = enp->en_etxop; 306 307 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 308 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 309 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 310 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 311 312 etxop->etxo_fini(enp); 313 314 enp->en_etxop = NULL; 315 enp->en_mod_flags &= ~EFX_MOD_TX; 316 } 317 318 __checkReturn efx_rc_t 319 efx_tx_qcreate( 320 __in efx_nic_t *enp, 321 __in unsigned int index, 322 __in unsigned int label, 323 __in efsys_mem_t *esmp, 324 __in size_t n, 325 __in uint32_t id, 326 __in uint16_t flags, 327 __in efx_evq_t *eep, 328 __deref_out efx_txq_t **etpp, 329 __out unsigned int *addedp) 330 { 331 efx_tx_ops_t *etxop = enp->en_etxop; 332 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 333 efx_txq_t *etp; 334 efx_rc_t rc; 335 336 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 337 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 338 339 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 340 341 /* Allocate an TXQ object */ 342 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 343 344 if (etp == NULL) { 345 rc = ENOMEM; 346 goto fail1; 347 } 348 349 etp->et_magic = EFX_TXQ_MAGIC; 350 etp->et_enp = enp; 351 etp->et_index = index; 352 etp->et_mask = n - 1; 353 etp->et_esmp = esmp; 354 355 /* Initial descriptor index may be modified by etxo_qcreate */ 356 *addedp = 0; 357 358 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 359 n, id, flags, eep, etp, addedp)) != 0) 360 goto fail2; 361 362 enp->en_tx_qcount++; 363 *etpp = etp; 364 365 return (0); 366 367 fail2: 368 EFSYS_PROBE(fail2); 369 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 370 fail1: 371 EFSYS_PROBE1(fail1, efx_rc_t, rc); 372 return (rc); 373 } 374 375 void 376 efx_tx_qdestroy( 377 __in efx_txq_t *etp) 378 { 379 efx_nic_t *enp = etp->et_enp; 380 efx_tx_ops_t *etxop = enp->en_etxop; 381 382 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 383 384 EFSYS_ASSERT(enp->en_tx_qcount != 0); 385 --enp->en_tx_qcount; 386 387 etxop->etxo_qdestroy(etp); 388 389 /* Free the TXQ object */ 390 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 391 } 392 393 __checkReturn efx_rc_t 394 efx_tx_qpost( 395 __in efx_txq_t *etp, 396 __in_ecount(n) efx_buffer_t *eb, 397 __in unsigned int n, 398 __in unsigned int completed, 399 __inout unsigned int *addedp) 400 { 401 efx_nic_t *enp = etp->et_enp; 402 efx_tx_ops_t *etxop = enp->en_etxop; 403 efx_rc_t rc; 404 405 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 406 407 if ((rc = etxop->etxo_qpost(etp, eb, 408 n, completed, addedp)) != 0) 409 goto fail1; 410 411 return (0); 412 413 fail1: 414 EFSYS_PROBE1(fail1, efx_rc_t, rc); 415 return (rc); 416 } 417 418 void 419 efx_tx_qpush( 420 __in efx_txq_t *etp, 421 __in unsigned int added, 422 __in unsigned int pushed) 423 { 424 efx_nic_t *enp = etp->et_enp; 425 efx_tx_ops_t *etxop = enp->en_etxop; 426 427 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 428 429 etxop->etxo_qpush(etp, added, pushed); 430 } 431 432 __checkReturn efx_rc_t 433 efx_tx_qpace( 434 __in efx_txq_t *etp, 435 __in unsigned int ns) 436 { 437 efx_nic_t *enp = etp->et_enp; 438 efx_tx_ops_t *etxop = enp->en_etxop; 439 efx_rc_t rc; 440 441 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 442 443 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 444 goto fail1; 445 446 return (0); 447 448 fail1: 449 EFSYS_PROBE1(fail1, efx_rc_t, rc); 450 return (rc); 451 } 452 453 __checkReturn efx_rc_t 454 efx_tx_qflush( 455 __in efx_txq_t *etp) 456 { 457 efx_nic_t *enp = etp->et_enp; 458 efx_tx_ops_t *etxop = enp->en_etxop; 459 efx_rc_t rc; 460 461 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 462 463 if ((rc = etxop->etxo_qflush(etp)) != 0) 464 goto fail1; 465 466 return (0); 467 468 fail1: 469 EFSYS_PROBE1(fail1, efx_rc_t, rc); 470 return (rc); 471 } 472 473 void 474 efx_tx_qenable( 475 __in efx_txq_t *etp) 476 { 477 efx_nic_t *enp = etp->et_enp; 478 efx_tx_ops_t *etxop = enp->en_etxop; 479 480 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 481 482 etxop->etxo_qenable(etp); 483 } 484 485 __checkReturn efx_rc_t 486 efx_tx_qpio_enable( 487 __in efx_txq_t *etp) 488 { 489 efx_nic_t *enp = etp->et_enp; 490 efx_tx_ops_t *etxop = enp->en_etxop; 491 efx_rc_t rc; 492 493 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 494 495 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 496 rc = ENOTSUP; 497 goto fail1; 498 } 499 if (etxop->etxo_qpio_enable == NULL) { 500 rc = ENOTSUP; 501 goto fail2; 502 } 503 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 504 goto fail3; 505 506 return (0); 507 508 fail3: 509 EFSYS_PROBE(fail3); 510 fail2: 511 EFSYS_PROBE(fail2); 512 fail1: 513 EFSYS_PROBE1(fail1, efx_rc_t, rc); 514 return (rc); 515 } 516 517 void 518 efx_tx_qpio_disable( 519 __in efx_txq_t *etp) 520 { 521 efx_nic_t *enp = etp->et_enp; 522 efx_tx_ops_t *etxop = enp->en_etxop; 523 524 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 525 526 if (etxop->etxo_qpio_disable != NULL) 527 etxop->etxo_qpio_disable(etp); 528 } 529 530 __checkReturn efx_rc_t 531 efx_tx_qpio_write( 532 __in efx_txq_t *etp, 533 __in_ecount(buf_length) uint8_t *buffer, 534 __in size_t buf_length, 535 __in size_t pio_buf_offset) 536 { 537 efx_nic_t *enp = etp->et_enp; 538 efx_tx_ops_t *etxop = enp->en_etxop; 539 efx_rc_t rc; 540 541 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 542 543 if (etxop->etxo_qpio_write != NULL) { 544 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 545 pio_buf_offset)) != 0) 546 goto fail1; 547 return (0); 548 } 549 550 return (ENOTSUP); 551 552 fail1: 553 EFSYS_PROBE1(fail1, efx_rc_t, rc); 554 return (rc); 555 } 556 557 __checkReturn efx_rc_t 558 efx_tx_qpio_post( 559 __in efx_txq_t *etp, 560 __in size_t pkt_length, 561 __in unsigned int completed, 562 __inout unsigned int *addedp) 563 { 564 efx_nic_t *enp = etp->et_enp; 565 efx_tx_ops_t *etxop = enp->en_etxop; 566 efx_rc_t rc; 567 568 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 569 570 if (etxop->etxo_qpio_post != NULL) { 571 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 572 addedp)) != 0) 573 goto fail1; 574 return (0); 575 } 576 577 return (ENOTSUP); 578 579 fail1: 580 EFSYS_PROBE1(fail1, efx_rc_t, rc); 581 return (rc); 582 } 583 584 __checkReturn efx_rc_t 585 efx_tx_qdesc_post( 586 __in efx_txq_t *etp, 587 __in_ecount(n) efx_desc_t *ed, 588 __in unsigned int n, 589 __in unsigned int completed, 590 __inout unsigned int *addedp) 591 { 592 efx_nic_t *enp = etp->et_enp; 593 efx_tx_ops_t *etxop = enp->en_etxop; 594 efx_rc_t rc; 595 596 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 597 598 if ((rc = etxop->etxo_qdesc_post(etp, ed, 599 n, completed, addedp)) != 0) 600 goto fail1; 601 602 return (0); 603 604 fail1: 605 EFSYS_PROBE1(fail1, efx_rc_t, rc); 606 return (rc); 607 } 608 609 void 610 efx_tx_qdesc_dma_create( 611 __in efx_txq_t *etp, 612 __in efsys_dma_addr_t addr, 613 __in size_t size, 614 __in boolean_t eop, 615 __out efx_desc_t *edp) 616 { 617 efx_nic_t *enp = etp->et_enp; 618 efx_tx_ops_t *etxop = enp->en_etxop; 619 620 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 621 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 622 623 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 624 } 625 626 void 627 efx_tx_qdesc_tso_create( 628 __in efx_txq_t *etp, 629 __in uint16_t ipv4_id, 630 __in uint32_t tcp_seq, 631 __in uint8_t tcp_flags, 632 __out efx_desc_t *edp) 633 { 634 efx_nic_t *enp = etp->et_enp; 635 efx_tx_ops_t *etxop = enp->en_etxop; 636 637 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 638 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 639 640 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 641 } 642 643 void 644 efx_tx_qdesc_vlantci_create( 645 __in efx_txq_t *etp, 646 __in uint16_t tci, 647 __out efx_desc_t *edp) 648 { 649 efx_nic_t *enp = etp->et_enp; 650 efx_tx_ops_t *etxop = enp->en_etxop; 651 652 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 653 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 654 655 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 656 } 657 658 659 #if EFSYS_OPT_QSTATS 660 void 661 efx_tx_qstats_update( 662 __in efx_txq_t *etp, 663 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 664 { 665 efx_nic_t *enp = etp->et_enp; 666 efx_tx_ops_t *etxop = enp->en_etxop; 667 668 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 669 670 etxop->etxo_qstats_update(etp, stat); 671 } 672 #endif 673 674 675 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 676 677 static __checkReturn efx_rc_t 678 falconsiena_tx_init( 679 __in efx_nic_t *enp) 680 { 681 efx_oword_t oword; 682 683 /* 684 * Disable the timer-based TX DMA backoff and allow TX DMA to be 685 * controlled by the RX FIFO fill level (although always allow a 686 * minimal trickle). 687 */ 688 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 689 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 690 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 691 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 692 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 693 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 694 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 695 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 696 697 /* 698 * Filter all packets less than 14 bytes to avoid parsing 699 * errors. 700 */ 701 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 702 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 703 704 /* 705 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 706 * descriptors (which is bad). 707 */ 708 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 709 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 710 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 711 712 return (0); 713 } 714 715 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 716 do { \ 717 unsigned int id; \ 718 size_t offset; \ 719 efx_qword_t qword; \ 720 \ 721 id = (_added)++ & (_etp)->et_mask; \ 722 offset = id * sizeof (efx_qword_t); \ 723 \ 724 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 725 unsigned int, id, efsys_dma_addr_t, (_addr), \ 726 size_t, (_size), boolean_t, (_eop)); \ 727 \ 728 EFX_POPULATE_QWORD_4(qword, \ 729 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 730 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 731 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 732 (uint32_t)((_addr) & 0xffffffff), \ 733 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 734 (uint32_t)((_addr) >> 32)); \ 735 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 736 \ 737 _NOTE(CONSTANTCONDITION) \ 738 } while (B_FALSE) 739 740 static __checkReturn efx_rc_t 741 falconsiena_tx_qpost( 742 __in efx_txq_t *etp, 743 __in_ecount(n) efx_buffer_t *eb, 744 __in unsigned int n, 745 __in unsigned int completed, 746 __inout unsigned int *addedp) 747 { 748 unsigned int added = *addedp; 749 unsigned int i; 750 int rc = ENOSPC; 751 752 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 753 goto fail1; 754 755 for (i = 0; i < n; i++) { 756 efx_buffer_t *ebp = &eb[i]; 757 efsys_dma_addr_t start = ebp->eb_addr; 758 size_t size = ebp->eb_size; 759 efsys_dma_addr_t end = start + size; 760 761 /* Fragments must not span 4k boundaries. */ 762 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 763 764 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 765 } 766 767 EFX_TX_QSTAT_INCR(etp, TX_POST); 768 769 *addedp = added; 770 return (0); 771 772 fail1: 773 EFSYS_PROBE1(fail1, efx_rc_t, rc); 774 775 return (rc); 776 } 777 778 static void 779 falconsiena_tx_qpush( 780 __in efx_txq_t *etp, 781 __in unsigned int added, 782 __in unsigned int pushed) 783 { 784 efx_nic_t *enp = etp->et_enp; 785 uint32_t wptr; 786 efx_dword_t dword; 787 efx_oword_t oword; 788 789 /* Push the populated descriptors out */ 790 wptr = added & etp->et_mask; 791 792 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 793 794 /* Only write the third DWORD */ 795 EFX_POPULATE_DWORD_1(dword, 796 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 797 798 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 799 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 800 wptr, pushed & etp->et_mask); 801 EFSYS_PIO_WRITE_BARRIER(); 802 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 803 etp->et_index, &dword, B_FALSE); 804 } 805 806 #define EFX_MAX_PACE_VALUE 20 807 #define EFX_TX_PACE_CLOCK_BASE 104 808 809 static __checkReturn efx_rc_t 810 falconsiena_tx_qpace( 811 __in efx_txq_t *etp, 812 __in unsigned int ns) 813 { 814 efx_nic_t *enp = etp->et_enp; 815 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 816 efx_oword_t oword; 817 unsigned int pace_val; 818 unsigned int timer_period; 819 efx_rc_t rc; 820 821 if (ns == 0) { 822 pace_val = 0; 823 } else { 824 /* 825 * The pace_val to write into the table is s.t 826 * ns <= timer_period * (2 ^ pace_val) 827 */ 828 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 829 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 830 if ((timer_period << pace_val) >= ns) 831 break; 832 } 833 } 834 if (pace_val > EFX_MAX_PACE_VALUE) { 835 rc = EINVAL; 836 goto fail1; 837 } 838 839 /* Update the pacing table */ 840 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 841 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 842 &oword, B_TRUE); 843 844 return (0); 845 846 fail1: 847 EFSYS_PROBE1(fail1, efx_rc_t, rc); 848 849 return (rc); 850 } 851 852 static __checkReturn efx_rc_t 853 falconsiena_tx_qflush( 854 __in efx_txq_t *etp) 855 { 856 efx_nic_t *enp = etp->et_enp; 857 efx_oword_t oword; 858 uint32_t label; 859 860 efx_tx_qpace(etp, 0); 861 862 label = etp->et_index; 863 864 /* Flush the queue */ 865 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 866 FRF_AZ_TX_FLUSH_DESCQ, label); 867 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 868 869 return (0); 870 } 871 872 static void 873 falconsiena_tx_qenable( 874 __in efx_txq_t *etp) 875 { 876 efx_nic_t *enp = etp->et_enp; 877 efx_oword_t oword; 878 879 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 880 etp->et_index, &oword, B_TRUE); 881 882 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 883 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 884 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 885 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 886 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 887 888 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 889 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 890 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 891 892 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 893 etp->et_index, &oword, B_TRUE); 894 } 895 896 static __checkReturn efx_rc_t 897 falconsiena_tx_qcreate( 898 __in efx_nic_t *enp, 899 __in unsigned int index, 900 __in unsigned int label, 901 __in efsys_mem_t *esmp, 902 __in size_t n, 903 __in uint32_t id, 904 __in uint16_t flags, 905 __in efx_evq_t *eep, 906 __in efx_txq_t *etp, 907 __out unsigned int *addedp) 908 { 909 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 910 efx_oword_t oword; 911 uint32_t size; 912 efx_rc_t rc; 913 914 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 915 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 916 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 917 918 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp))); 919 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 920 921 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { 922 rc = EINVAL; 923 goto fail1; 924 } 925 if (index >= encp->enc_txq_limit) { 926 rc = EINVAL; 927 goto fail2; 928 } 929 for (size = 0; 930 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS); 931 size++) 932 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 933 break; 934 if (id + (1 << size) >= encp->enc_buftbl_limit) { 935 rc = EINVAL; 936 goto fail3; 937 } 938 939 /* Set up the new descriptor queue */ 940 *addedp = 0; 941 942 EFX_POPULATE_OWORD_6(oword, 943 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 944 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 945 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 946 FRF_AZ_TX_DESCQ_LABEL, label, 947 FRF_AZ_TX_DESCQ_SIZE, size, 948 FRF_AZ_TX_DESCQ_TYPE, 0); 949 950 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 951 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 952 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 953 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 954 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 955 956 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 957 etp->et_index, &oword, B_TRUE); 958 959 return (0); 960 961 fail3: 962 EFSYS_PROBE(fail3); 963 fail2: 964 EFSYS_PROBE(fail2); 965 fail1: 966 EFSYS_PROBE1(fail1, efx_rc_t, rc); 967 968 return (rc); 969 } 970 971 __checkReturn efx_rc_t 972 falconsiena_tx_qdesc_post( 973 __in efx_txq_t *etp, 974 __in_ecount(n) efx_desc_t *ed, 975 __in unsigned int n, 976 __in unsigned int completed, 977 __inout unsigned int *addedp) 978 { 979 unsigned int added = *addedp; 980 unsigned int i; 981 efx_rc_t rc; 982 983 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 984 rc = ENOSPC; 985 goto fail1; 986 } 987 988 for (i = 0; i < n; i++) { 989 efx_desc_t *edp = &ed[i]; 990 unsigned int id; 991 size_t offset; 992 993 id = added++ & etp->et_mask; 994 offset = id * sizeof (efx_desc_t); 995 996 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 997 } 998 999 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 1000 unsigned int, added, unsigned int, n); 1001 1002 EFX_TX_QSTAT_INCR(etp, TX_POST); 1003 1004 *addedp = added; 1005 return (0); 1006 1007 fail1: 1008 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1009 return (rc); 1010 } 1011 1012 void 1013 falconsiena_tx_qdesc_dma_create( 1014 __in efx_txq_t *etp, 1015 __in efsys_dma_addr_t addr, 1016 __in size_t size, 1017 __in boolean_t eop, 1018 __out efx_desc_t *edp) 1019 { 1020 /* Fragments must not span 4k boundaries. */ 1021 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size); 1022 1023 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1024 efsys_dma_addr_t, addr, 1025 size_t, size, boolean_t, eop); 1026 1027 EFX_POPULATE_QWORD_4(edp->ed_eq, 1028 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1029 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1030 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1031 (uint32_t)(addr & 0xffffffff), 1032 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1033 (uint32_t)(addr >> 32)); 1034 } 1035 1036 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 1037 1038 #if EFSYS_OPT_QSTATS 1039 #if EFSYS_OPT_NAMES 1040 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */ 1041 static const char *__efx_tx_qstat_name[] = { 1042 "post", 1043 "post_pio", 1044 }; 1045 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1046 1047 const char * 1048 efx_tx_qstat_name( 1049 __in efx_nic_t *enp, 1050 __in unsigned int id) 1051 { 1052 _NOTE(ARGUNUSED(enp)) 1053 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1054 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1055 1056 return (__efx_tx_qstat_name[id]); 1057 } 1058 #endif /* EFSYS_OPT_NAMES */ 1059 #endif /* EFSYS_OPT_QSTATS */ 1060 1061 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 1062 1063 #if EFSYS_OPT_QSTATS 1064 static void 1065 falconsiena_tx_qstats_update( 1066 __in efx_txq_t *etp, 1067 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1068 { 1069 unsigned int id; 1070 1071 for (id = 0; id < TX_NQSTATS; id++) { 1072 efsys_stat_t *essp = &stat[id]; 1073 1074 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1075 etp->et_stat[id] = 0; 1076 } 1077 } 1078 #endif /* EFSYS_OPT_QSTATS */ 1079 1080 static void 1081 falconsiena_tx_qdestroy( 1082 __in efx_txq_t *etp) 1083 { 1084 efx_nic_t *enp = etp->et_enp; 1085 efx_oword_t oword; 1086 1087 /* Purge descriptor queue */ 1088 EFX_ZERO_OWORD(oword); 1089 1090 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1091 etp->et_index, &oword, B_TRUE); 1092 } 1093 1094 static void 1095 falconsiena_tx_fini( 1096 __in efx_nic_t *enp) 1097 { 1098 _NOTE(ARGUNUSED(enp)) 1099 } 1100 1101 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 1102