1 /*- 2 * Copyright (c) 2007-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 #if EFSYS_OPT_QSTATS 38 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 39 do { \ 40 (_etp)->et_stat[_stat]++; \ 41 _NOTE(CONSTANTCONDITION) \ 42 } while (B_FALSE) 43 #else 44 #define EFX_TX_QSTAT_INCR(_etp, _stat) 45 #endif 46 47 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 48 49 static __checkReturn efx_rc_t 50 falconsiena_tx_init( 51 __in efx_nic_t *enp); 52 53 static void 54 falconsiena_tx_fini( 55 __in efx_nic_t *enp); 56 57 static __checkReturn efx_rc_t 58 falconsiena_tx_qcreate( 59 __in efx_nic_t *enp, 60 __in unsigned int index, 61 __in unsigned int label, 62 __in efsys_mem_t *esmp, 63 __in size_t n, 64 __in uint32_t id, 65 __in uint16_t flags, 66 __in efx_evq_t *eep, 67 __in efx_txq_t *etp, 68 __out unsigned int *addedp); 69 70 static void 71 falconsiena_tx_qdestroy( 72 __in efx_txq_t *etp); 73 74 static __checkReturn efx_rc_t 75 falconsiena_tx_qpost( 76 __in efx_txq_t *etp, 77 __in_ecount(n) efx_buffer_t *eb, 78 __in unsigned int n, 79 __in unsigned int completed, 80 __inout unsigned int *addedp); 81 82 static void 83 falconsiena_tx_qpush( 84 __in efx_txq_t *etp, 85 __in unsigned int added, 86 __in unsigned int pushed); 87 88 static __checkReturn efx_rc_t 89 falconsiena_tx_qpace( 90 __in efx_txq_t *etp, 91 __in unsigned int ns); 92 93 static __checkReturn efx_rc_t 94 falconsiena_tx_qflush( 95 __in efx_txq_t *etp); 96 97 static void 98 falconsiena_tx_qenable( 99 __in efx_txq_t *etp); 100 101 __checkReturn efx_rc_t 102 falconsiena_tx_qdesc_post( 103 __in efx_txq_t *etp, 104 __in_ecount(n) efx_desc_t *ed, 105 __in unsigned int n, 106 __in unsigned int completed, 107 __inout unsigned int *addedp); 108 109 void 110 falconsiena_tx_qdesc_dma_create( 111 __in efx_txq_t *etp, 112 __in efsys_dma_addr_t addr, 113 __in size_t size, 114 __in boolean_t eop, 115 __out efx_desc_t *edp); 116 117 #if EFSYS_OPT_QSTATS 118 static void 119 falconsiena_tx_qstats_update( 120 __in efx_txq_t *etp, 121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 122 #endif 123 124 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 125 126 127 #if EFSYS_OPT_FALCON 128 static efx_tx_ops_t __efx_tx_falcon_ops = { 129 falconsiena_tx_init, /* etxo_init */ 130 falconsiena_tx_fini, /* etxo_fini */ 131 falconsiena_tx_qcreate, /* etxo_qcreate */ 132 falconsiena_tx_qdestroy, /* etxo_qdestroy */ 133 falconsiena_tx_qpost, /* etxo_qpost */ 134 falconsiena_tx_qpush, /* etxo_qpush */ 135 falconsiena_tx_qpace, /* etxo_qpace */ 136 falconsiena_tx_qflush, /* etxo_qflush */ 137 falconsiena_tx_qenable, /* etxo_qenable */ 138 NULL, /* etxo_qpio_enable */ 139 NULL, /* etxo_qpio_disable */ 140 NULL, /* etxo_qpio_write */ 141 NULL, /* etxo_qpio_post */ 142 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */ 143 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 144 NULL, /* etxo_qdesc_tso_create */ 145 NULL, /* etxo_qdesc_tso2_create */ 146 NULL, /* etxo_qdesc_vlantci_create */ 147 #if EFSYS_OPT_QSTATS 148 falconsiena_tx_qstats_update, /* etxo_qstats_update */ 149 #endif 150 }; 151 #endif /* EFSYS_OPT_FALCON */ 152 153 #if EFSYS_OPT_SIENA 154 static efx_tx_ops_t __efx_tx_siena_ops = { 155 falconsiena_tx_init, /* etxo_init */ 156 falconsiena_tx_fini, /* etxo_fini */ 157 falconsiena_tx_qcreate, /* etxo_qcreate */ 158 falconsiena_tx_qdestroy, /* etxo_qdestroy */ 159 falconsiena_tx_qpost, /* etxo_qpost */ 160 falconsiena_tx_qpush, /* etxo_qpush */ 161 falconsiena_tx_qpace, /* etxo_qpace */ 162 falconsiena_tx_qflush, /* etxo_qflush */ 163 falconsiena_tx_qenable, /* etxo_qenable */ 164 NULL, /* etxo_qpio_enable */ 165 NULL, /* etxo_qpio_disable */ 166 NULL, /* etxo_qpio_write */ 167 NULL, /* etxo_qpio_post */ 168 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */ 169 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 170 NULL, /* etxo_qdesc_tso_create */ 171 NULL, /* etxo_qdesc_tso2_create */ 172 NULL, /* etxo_qdesc_vlantci_create */ 173 #if EFSYS_OPT_QSTATS 174 falconsiena_tx_qstats_update, /* etxo_qstats_update */ 175 #endif 176 }; 177 #endif /* EFSYS_OPT_SIENA */ 178 179 #if EFSYS_OPT_HUNTINGTON 180 static efx_tx_ops_t __efx_tx_hunt_ops = { 181 ef10_tx_init, /* etxo_init */ 182 ef10_tx_fini, /* etxo_fini */ 183 ef10_tx_qcreate, /* etxo_qcreate */ 184 ef10_tx_qdestroy, /* etxo_qdestroy */ 185 ef10_tx_qpost, /* etxo_qpost */ 186 ef10_tx_qpush, /* etxo_qpush */ 187 ef10_tx_qpace, /* etxo_qpace */ 188 ef10_tx_qflush, /* etxo_qflush */ 189 ef10_tx_qenable, /* etxo_qenable */ 190 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 191 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 192 ef10_tx_qpio_write, /* etxo_qpio_write */ 193 ef10_tx_qpio_post, /* etxo_qpio_post */ 194 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 195 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 196 hunt_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 197 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 198 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 199 #if EFSYS_OPT_QSTATS 200 ef10_tx_qstats_update, /* etxo_qstats_update */ 201 #endif 202 }; 203 #endif /* EFSYS_OPT_HUNTINGTON */ 204 205 #if EFSYS_OPT_MEDFORD 206 static efx_tx_ops_t __efx_tx_medford_ops = { 207 ef10_tx_init, /* etxo_init */ 208 ef10_tx_fini, /* etxo_fini */ 209 ef10_tx_qcreate, /* etxo_qcreate */ 210 ef10_tx_qdestroy, /* etxo_qdestroy */ 211 ef10_tx_qpost, /* etxo_qpost */ 212 ef10_tx_qpush, /* etxo_qpush */ 213 ef10_tx_qpace, /* etxo_qpace */ 214 ef10_tx_qflush, /* etxo_qflush */ 215 ef10_tx_qenable, /* etxo_qenable */ 216 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 217 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 218 ef10_tx_qpio_write, /* etxo_qpio_write */ 219 ef10_tx_qpio_post, /* etxo_qpio_post */ 220 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 221 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 222 NULL, /* etxo_qdesc_tso_create */ 223 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 224 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 225 #if EFSYS_OPT_QSTATS 226 ef10_tx_qstats_update, /* etxo_qstats_update */ 227 #endif 228 }; 229 #endif /* EFSYS_OPT_MEDFORD */ 230 231 __checkReturn efx_rc_t 232 efx_tx_init( 233 __in efx_nic_t *enp) 234 { 235 efx_tx_ops_t *etxop; 236 efx_rc_t rc; 237 238 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 239 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 240 241 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 242 rc = EINVAL; 243 goto fail1; 244 } 245 246 if (enp->en_mod_flags & EFX_MOD_TX) { 247 rc = EINVAL; 248 goto fail2; 249 } 250 251 switch (enp->en_family) { 252 #if EFSYS_OPT_FALCON 253 case EFX_FAMILY_FALCON: 254 etxop = (efx_tx_ops_t *)&__efx_tx_falcon_ops; 255 break; 256 #endif /* EFSYS_OPT_FALCON */ 257 258 #if EFSYS_OPT_SIENA 259 case EFX_FAMILY_SIENA: 260 etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops; 261 break; 262 #endif /* EFSYS_OPT_SIENA */ 263 264 #if EFSYS_OPT_HUNTINGTON 265 case EFX_FAMILY_HUNTINGTON: 266 etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops; 267 break; 268 #endif /* EFSYS_OPT_HUNTINGTON */ 269 270 #if EFSYS_OPT_MEDFORD 271 case EFX_FAMILY_MEDFORD: 272 etxop = (efx_tx_ops_t *)&__efx_tx_medford_ops; 273 break; 274 #endif /* EFSYS_OPT_MEDFORD */ 275 276 default: 277 EFSYS_ASSERT(0); 278 rc = ENOTSUP; 279 goto fail3; 280 } 281 282 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 283 284 if ((rc = etxop->etxo_init(enp)) != 0) 285 goto fail4; 286 287 enp->en_etxop = etxop; 288 enp->en_mod_flags |= EFX_MOD_TX; 289 return (0); 290 291 fail4: 292 EFSYS_PROBE(fail4); 293 fail3: 294 EFSYS_PROBE(fail3); 295 fail2: 296 EFSYS_PROBE(fail2); 297 fail1: 298 EFSYS_PROBE1(fail1, efx_rc_t, rc); 299 300 enp->en_etxop = NULL; 301 enp->en_mod_flags &= ~EFX_MOD_TX; 302 return (rc); 303 } 304 305 void 306 efx_tx_fini( 307 __in efx_nic_t *enp) 308 { 309 efx_tx_ops_t *etxop = enp->en_etxop; 310 311 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 312 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 313 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 314 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 315 316 etxop->etxo_fini(enp); 317 318 enp->en_etxop = NULL; 319 enp->en_mod_flags &= ~EFX_MOD_TX; 320 } 321 322 __checkReturn efx_rc_t 323 efx_tx_qcreate( 324 __in efx_nic_t *enp, 325 __in unsigned int index, 326 __in unsigned int label, 327 __in efsys_mem_t *esmp, 328 __in size_t n, 329 __in uint32_t id, 330 __in uint16_t flags, 331 __in efx_evq_t *eep, 332 __deref_out efx_txq_t **etpp, 333 __out unsigned int *addedp) 334 { 335 efx_tx_ops_t *etxop = enp->en_etxop; 336 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 337 efx_txq_t *etp; 338 efx_rc_t rc; 339 340 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 341 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 342 343 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 344 345 /* Allocate an TXQ object */ 346 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 347 348 if (etp == NULL) { 349 rc = ENOMEM; 350 goto fail1; 351 } 352 353 etp->et_magic = EFX_TXQ_MAGIC; 354 etp->et_enp = enp; 355 etp->et_index = index; 356 etp->et_mask = n - 1; 357 etp->et_esmp = esmp; 358 359 /* Initial descriptor index may be modified by etxo_qcreate */ 360 *addedp = 0; 361 362 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 363 n, id, flags, eep, etp, addedp)) != 0) 364 goto fail2; 365 366 enp->en_tx_qcount++; 367 *etpp = etp; 368 369 return (0); 370 371 fail2: 372 EFSYS_PROBE(fail2); 373 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 374 fail1: 375 EFSYS_PROBE1(fail1, efx_rc_t, rc); 376 return (rc); 377 } 378 379 void 380 efx_tx_qdestroy( 381 __in efx_txq_t *etp) 382 { 383 efx_nic_t *enp = etp->et_enp; 384 efx_tx_ops_t *etxop = enp->en_etxop; 385 386 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 387 388 EFSYS_ASSERT(enp->en_tx_qcount != 0); 389 --enp->en_tx_qcount; 390 391 etxop->etxo_qdestroy(etp); 392 393 /* Free the TXQ object */ 394 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 395 } 396 397 __checkReturn efx_rc_t 398 efx_tx_qpost( 399 __in efx_txq_t *etp, 400 __in_ecount(n) efx_buffer_t *eb, 401 __in unsigned int n, 402 __in unsigned int completed, 403 __inout unsigned int *addedp) 404 { 405 efx_nic_t *enp = etp->et_enp; 406 efx_tx_ops_t *etxop = enp->en_etxop; 407 efx_rc_t rc; 408 409 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 410 411 if ((rc = etxop->etxo_qpost(etp, eb, 412 n, completed, addedp)) != 0) 413 goto fail1; 414 415 return (0); 416 417 fail1: 418 EFSYS_PROBE1(fail1, efx_rc_t, rc); 419 return (rc); 420 } 421 422 void 423 efx_tx_qpush( 424 __in efx_txq_t *etp, 425 __in unsigned int added, 426 __in unsigned int pushed) 427 { 428 efx_nic_t *enp = etp->et_enp; 429 efx_tx_ops_t *etxop = enp->en_etxop; 430 431 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 432 433 etxop->etxo_qpush(etp, added, pushed); 434 } 435 436 __checkReturn efx_rc_t 437 efx_tx_qpace( 438 __in efx_txq_t *etp, 439 __in unsigned int ns) 440 { 441 efx_nic_t *enp = etp->et_enp; 442 efx_tx_ops_t *etxop = enp->en_etxop; 443 efx_rc_t rc; 444 445 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 446 447 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 448 goto fail1; 449 450 return (0); 451 452 fail1: 453 EFSYS_PROBE1(fail1, efx_rc_t, rc); 454 return (rc); 455 } 456 457 __checkReturn efx_rc_t 458 efx_tx_qflush( 459 __in efx_txq_t *etp) 460 { 461 efx_nic_t *enp = etp->et_enp; 462 efx_tx_ops_t *etxop = enp->en_etxop; 463 efx_rc_t rc; 464 465 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 466 467 if ((rc = etxop->etxo_qflush(etp)) != 0) 468 goto fail1; 469 470 return (0); 471 472 fail1: 473 EFSYS_PROBE1(fail1, efx_rc_t, rc); 474 return (rc); 475 } 476 477 void 478 efx_tx_qenable( 479 __in efx_txq_t *etp) 480 { 481 efx_nic_t *enp = etp->et_enp; 482 efx_tx_ops_t *etxop = enp->en_etxop; 483 484 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 485 486 etxop->etxo_qenable(etp); 487 } 488 489 __checkReturn efx_rc_t 490 efx_tx_qpio_enable( 491 __in efx_txq_t *etp) 492 { 493 efx_nic_t *enp = etp->et_enp; 494 efx_tx_ops_t *etxop = enp->en_etxop; 495 efx_rc_t rc; 496 497 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 498 499 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 500 rc = ENOTSUP; 501 goto fail1; 502 } 503 if (etxop->etxo_qpio_enable == NULL) { 504 rc = ENOTSUP; 505 goto fail2; 506 } 507 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 508 goto fail3; 509 510 return (0); 511 512 fail3: 513 EFSYS_PROBE(fail3); 514 fail2: 515 EFSYS_PROBE(fail2); 516 fail1: 517 EFSYS_PROBE1(fail1, efx_rc_t, rc); 518 return (rc); 519 } 520 521 void 522 efx_tx_qpio_disable( 523 __in efx_txq_t *etp) 524 { 525 efx_nic_t *enp = etp->et_enp; 526 efx_tx_ops_t *etxop = enp->en_etxop; 527 528 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 529 530 if (etxop->etxo_qpio_disable != NULL) 531 etxop->etxo_qpio_disable(etp); 532 } 533 534 __checkReturn efx_rc_t 535 efx_tx_qpio_write( 536 __in efx_txq_t *etp, 537 __in_ecount(buf_length) uint8_t *buffer, 538 __in size_t buf_length, 539 __in size_t pio_buf_offset) 540 { 541 efx_nic_t *enp = etp->et_enp; 542 efx_tx_ops_t *etxop = enp->en_etxop; 543 efx_rc_t rc; 544 545 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 546 547 if (etxop->etxo_qpio_write != NULL) { 548 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 549 pio_buf_offset)) != 0) 550 goto fail1; 551 return (0); 552 } 553 554 return (ENOTSUP); 555 556 fail1: 557 EFSYS_PROBE1(fail1, efx_rc_t, rc); 558 return (rc); 559 } 560 561 __checkReturn efx_rc_t 562 efx_tx_qpio_post( 563 __in efx_txq_t *etp, 564 __in size_t pkt_length, 565 __in unsigned int completed, 566 __inout unsigned int *addedp) 567 { 568 efx_nic_t *enp = etp->et_enp; 569 efx_tx_ops_t *etxop = enp->en_etxop; 570 efx_rc_t rc; 571 572 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 573 574 if (etxop->etxo_qpio_post != NULL) { 575 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 576 addedp)) != 0) 577 goto fail1; 578 return (0); 579 } 580 581 return (ENOTSUP); 582 583 fail1: 584 EFSYS_PROBE1(fail1, efx_rc_t, rc); 585 return (rc); 586 } 587 588 __checkReturn efx_rc_t 589 efx_tx_qdesc_post( 590 __in efx_txq_t *etp, 591 __in_ecount(n) efx_desc_t *ed, 592 __in unsigned int n, 593 __in unsigned int completed, 594 __inout unsigned int *addedp) 595 { 596 efx_nic_t *enp = etp->et_enp; 597 efx_tx_ops_t *etxop = enp->en_etxop; 598 efx_rc_t rc; 599 600 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 601 602 if ((rc = etxop->etxo_qdesc_post(etp, ed, 603 n, completed, addedp)) != 0) 604 goto fail1; 605 606 return (0); 607 608 fail1: 609 EFSYS_PROBE1(fail1, efx_rc_t, rc); 610 return (rc); 611 } 612 613 void 614 efx_tx_qdesc_dma_create( 615 __in efx_txq_t *etp, 616 __in efsys_dma_addr_t addr, 617 __in size_t size, 618 __in boolean_t eop, 619 __out efx_desc_t *edp) 620 { 621 efx_nic_t *enp = etp->et_enp; 622 efx_tx_ops_t *etxop = enp->en_etxop; 623 624 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 625 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 626 627 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 628 } 629 630 void 631 efx_tx_qdesc_tso_create( 632 __in efx_txq_t *etp, 633 __in uint16_t ipv4_id, 634 __in uint32_t tcp_seq, 635 __in uint8_t tcp_flags, 636 __out efx_desc_t *edp) 637 { 638 efx_nic_t *enp = etp->et_enp; 639 efx_tx_ops_t *etxop = enp->en_etxop; 640 641 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 642 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 643 644 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 645 } 646 647 void 648 efx_tx_qdesc_tso2_create( 649 __in efx_txq_t *etp, 650 __in uint16_t ipv4_id, 651 __in uint32_t tcp_seq, 652 __in uint16_t mss, 653 __out_ecount(count) efx_desc_t *edp, 654 __in int count) 655 { 656 efx_nic_t *enp = etp->et_enp; 657 efx_tx_ops_t *etxop = enp->en_etxop; 658 659 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 660 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 661 662 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count); 663 } 664 665 void 666 efx_tx_qdesc_vlantci_create( 667 __in efx_txq_t *etp, 668 __in uint16_t tci, 669 __out efx_desc_t *edp) 670 { 671 efx_nic_t *enp = etp->et_enp; 672 efx_tx_ops_t *etxop = enp->en_etxop; 673 674 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 675 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 676 677 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 678 } 679 680 681 #if EFSYS_OPT_QSTATS 682 void 683 efx_tx_qstats_update( 684 __in efx_txq_t *etp, 685 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 686 { 687 efx_nic_t *enp = etp->et_enp; 688 efx_tx_ops_t *etxop = enp->en_etxop; 689 690 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 691 692 etxop->etxo_qstats_update(etp, stat); 693 } 694 #endif 695 696 697 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 698 699 static __checkReturn efx_rc_t 700 falconsiena_tx_init( 701 __in efx_nic_t *enp) 702 { 703 efx_oword_t oword; 704 705 /* 706 * Disable the timer-based TX DMA backoff and allow TX DMA to be 707 * controlled by the RX FIFO fill level (although always allow a 708 * minimal trickle). 709 */ 710 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 711 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 712 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 713 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 714 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 715 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 716 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 717 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 718 719 /* 720 * Filter all packets less than 14 bytes to avoid parsing 721 * errors. 722 */ 723 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 724 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 725 726 /* 727 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 728 * descriptors (which is bad). 729 */ 730 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 731 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 732 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 733 734 return (0); 735 } 736 737 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 738 do { \ 739 unsigned int id; \ 740 size_t offset; \ 741 efx_qword_t qword; \ 742 \ 743 id = (_added)++ & (_etp)->et_mask; \ 744 offset = id * sizeof (efx_qword_t); \ 745 \ 746 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 747 unsigned int, id, efsys_dma_addr_t, (_addr), \ 748 size_t, (_size), boolean_t, (_eop)); \ 749 \ 750 EFX_POPULATE_QWORD_4(qword, \ 751 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 752 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 753 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 754 (uint32_t)((_addr) & 0xffffffff), \ 755 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 756 (uint32_t)((_addr) >> 32)); \ 757 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 758 \ 759 _NOTE(CONSTANTCONDITION) \ 760 } while (B_FALSE) 761 762 static __checkReturn efx_rc_t 763 falconsiena_tx_qpost( 764 __in efx_txq_t *etp, 765 __in_ecount(n) efx_buffer_t *eb, 766 __in unsigned int n, 767 __in unsigned int completed, 768 __inout unsigned int *addedp) 769 { 770 unsigned int added = *addedp; 771 unsigned int i; 772 int rc = ENOSPC; 773 774 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 775 goto fail1; 776 777 for (i = 0; i < n; i++) { 778 efx_buffer_t *ebp = &eb[i]; 779 efsys_dma_addr_t start = ebp->eb_addr; 780 size_t size = ebp->eb_size; 781 efsys_dma_addr_t end = start + size; 782 783 /* Fragments must not span 4k boundaries. */ 784 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 785 786 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 787 } 788 789 EFX_TX_QSTAT_INCR(etp, TX_POST); 790 791 *addedp = added; 792 return (0); 793 794 fail1: 795 EFSYS_PROBE1(fail1, efx_rc_t, rc); 796 797 return (rc); 798 } 799 800 static void 801 falconsiena_tx_qpush( 802 __in efx_txq_t *etp, 803 __in unsigned int added, 804 __in unsigned int pushed) 805 { 806 efx_nic_t *enp = etp->et_enp; 807 uint32_t wptr; 808 efx_dword_t dword; 809 efx_oword_t oword; 810 811 /* Push the populated descriptors out */ 812 wptr = added & etp->et_mask; 813 814 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 815 816 /* Only write the third DWORD */ 817 EFX_POPULATE_DWORD_1(dword, 818 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 819 820 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 821 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 822 wptr, pushed & etp->et_mask); 823 EFSYS_PIO_WRITE_BARRIER(); 824 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 825 etp->et_index, &dword, B_FALSE); 826 } 827 828 #define EFX_MAX_PACE_VALUE 20 829 #define EFX_TX_PACE_CLOCK_BASE 104 830 831 static __checkReturn efx_rc_t 832 falconsiena_tx_qpace( 833 __in efx_txq_t *etp, 834 __in unsigned int ns) 835 { 836 efx_nic_t *enp = etp->et_enp; 837 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 838 efx_oword_t oword; 839 unsigned int pace_val; 840 unsigned int timer_period; 841 efx_rc_t rc; 842 843 if (ns == 0) { 844 pace_val = 0; 845 } else { 846 /* 847 * The pace_val to write into the table is s.t 848 * ns <= timer_period * (2 ^ pace_val) 849 */ 850 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 851 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 852 if ((timer_period << pace_val) >= ns) 853 break; 854 } 855 } 856 if (pace_val > EFX_MAX_PACE_VALUE) { 857 rc = EINVAL; 858 goto fail1; 859 } 860 861 /* Update the pacing table */ 862 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 863 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 864 &oword, B_TRUE); 865 866 return (0); 867 868 fail1: 869 EFSYS_PROBE1(fail1, efx_rc_t, rc); 870 871 return (rc); 872 } 873 874 static __checkReturn efx_rc_t 875 falconsiena_tx_qflush( 876 __in efx_txq_t *etp) 877 { 878 efx_nic_t *enp = etp->et_enp; 879 efx_oword_t oword; 880 uint32_t label; 881 882 efx_tx_qpace(etp, 0); 883 884 label = etp->et_index; 885 886 /* Flush the queue */ 887 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 888 FRF_AZ_TX_FLUSH_DESCQ, label); 889 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 890 891 return (0); 892 } 893 894 static void 895 falconsiena_tx_qenable( 896 __in efx_txq_t *etp) 897 { 898 efx_nic_t *enp = etp->et_enp; 899 efx_oword_t oword; 900 901 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 902 etp->et_index, &oword, B_TRUE); 903 904 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 905 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 906 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 907 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 908 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 909 910 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 911 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 912 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 913 914 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 915 etp->et_index, &oword, B_TRUE); 916 } 917 918 static __checkReturn efx_rc_t 919 falconsiena_tx_qcreate( 920 __in efx_nic_t *enp, 921 __in unsigned int index, 922 __in unsigned int label, 923 __in efsys_mem_t *esmp, 924 __in size_t n, 925 __in uint32_t id, 926 __in uint16_t flags, 927 __in efx_evq_t *eep, 928 __in efx_txq_t *etp, 929 __out unsigned int *addedp) 930 { 931 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 932 efx_oword_t oword; 933 uint32_t size; 934 efx_rc_t rc; 935 936 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 937 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 938 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 939 940 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp))); 941 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 942 943 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { 944 rc = EINVAL; 945 goto fail1; 946 } 947 if (index >= encp->enc_txq_limit) { 948 rc = EINVAL; 949 goto fail2; 950 } 951 for (size = 0; 952 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS); 953 size++) 954 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 955 break; 956 if (id + (1 << size) >= encp->enc_buftbl_limit) { 957 rc = EINVAL; 958 goto fail3; 959 } 960 961 /* Set up the new descriptor queue */ 962 *addedp = 0; 963 964 EFX_POPULATE_OWORD_6(oword, 965 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 966 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 967 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 968 FRF_AZ_TX_DESCQ_LABEL, label, 969 FRF_AZ_TX_DESCQ_SIZE, size, 970 FRF_AZ_TX_DESCQ_TYPE, 0); 971 972 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 973 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 974 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 975 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 976 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 977 978 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 979 etp->et_index, &oword, B_TRUE); 980 981 return (0); 982 983 fail3: 984 EFSYS_PROBE(fail3); 985 fail2: 986 EFSYS_PROBE(fail2); 987 fail1: 988 EFSYS_PROBE1(fail1, efx_rc_t, rc); 989 990 return (rc); 991 } 992 993 __checkReturn efx_rc_t 994 falconsiena_tx_qdesc_post( 995 __in efx_txq_t *etp, 996 __in_ecount(n) efx_desc_t *ed, 997 __in unsigned int n, 998 __in unsigned int completed, 999 __inout unsigned int *addedp) 1000 { 1001 unsigned int added = *addedp; 1002 unsigned int i; 1003 efx_rc_t rc; 1004 1005 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 1006 rc = ENOSPC; 1007 goto fail1; 1008 } 1009 1010 for (i = 0; i < n; i++) { 1011 efx_desc_t *edp = &ed[i]; 1012 unsigned int id; 1013 size_t offset; 1014 1015 id = added++ & etp->et_mask; 1016 offset = id * sizeof (efx_desc_t); 1017 1018 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 1019 } 1020 1021 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 1022 unsigned int, added, unsigned int, n); 1023 1024 EFX_TX_QSTAT_INCR(etp, TX_POST); 1025 1026 *addedp = added; 1027 return (0); 1028 1029 fail1: 1030 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1031 return (rc); 1032 } 1033 1034 void 1035 falconsiena_tx_qdesc_dma_create( 1036 __in efx_txq_t *etp, 1037 __in efsys_dma_addr_t addr, 1038 __in size_t size, 1039 __in boolean_t eop, 1040 __out efx_desc_t *edp) 1041 { 1042 /* Fragments must not span 4k boundaries. */ 1043 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size); 1044 1045 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1046 efsys_dma_addr_t, addr, 1047 size_t, size, boolean_t, eop); 1048 1049 EFX_POPULATE_QWORD_4(edp->ed_eq, 1050 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1051 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1052 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1053 (uint32_t)(addr & 0xffffffff), 1054 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1055 (uint32_t)(addr >> 32)); 1056 } 1057 1058 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 1059 1060 #if EFSYS_OPT_QSTATS 1061 #if EFSYS_OPT_NAMES 1062 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */ 1063 static const char *__efx_tx_qstat_name[] = { 1064 "post", 1065 "post_pio", 1066 }; 1067 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1068 1069 const char * 1070 efx_tx_qstat_name( 1071 __in efx_nic_t *enp, 1072 __in unsigned int id) 1073 { 1074 _NOTE(ARGUNUSED(enp)) 1075 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1076 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1077 1078 return (__efx_tx_qstat_name[id]); 1079 } 1080 #endif /* EFSYS_OPT_NAMES */ 1081 #endif /* EFSYS_OPT_QSTATS */ 1082 1083 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 1084 1085 #if EFSYS_OPT_QSTATS 1086 static void 1087 falconsiena_tx_qstats_update( 1088 __in efx_txq_t *etp, 1089 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1090 { 1091 unsigned int id; 1092 1093 for (id = 0; id < TX_NQSTATS; id++) { 1094 efsys_stat_t *essp = &stat[id]; 1095 1096 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1097 etp->et_stat[id] = 0; 1098 } 1099 } 1100 #endif /* EFSYS_OPT_QSTATS */ 1101 1102 static void 1103 falconsiena_tx_qdestroy( 1104 __in efx_txq_t *etp) 1105 { 1106 efx_nic_t *enp = etp->et_enp; 1107 efx_oword_t oword; 1108 1109 /* Purge descriptor queue */ 1110 EFX_ZERO_OWORD(oword); 1111 1112 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1113 etp->et_index, &oword, B_TRUE); 1114 } 1115 1116 static void 1117 falconsiena_tx_fini( 1118 __in efx_nic_t *enp) 1119 { 1120 _NOTE(ARGUNUSED(enp)) 1121 } 1122 1123 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 1124