1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "efx.h" 37 #include "efx_impl.h" 38 39 40 #if EFSYS_OPT_SIENA 41 42 static __checkReturn efx_rc_t 43 siena_rx_init( 44 __in efx_nic_t *enp); 45 46 static void 47 siena_rx_fini( 48 __in efx_nic_t *enp); 49 50 #if EFSYS_OPT_RX_SCATTER 51 static __checkReturn efx_rc_t 52 siena_rx_scatter_enable( 53 __in efx_nic_t *enp, 54 __in unsigned int buf_size); 55 #endif /* EFSYS_OPT_RX_SCATTER */ 56 57 #if EFSYS_OPT_RX_SCALE 58 static __checkReturn efx_rc_t 59 siena_rx_scale_mode_set( 60 __in efx_nic_t *enp, 61 __in uint32_t rss_context, 62 __in efx_rx_hash_alg_t alg, 63 __in efx_rx_hash_type_t type, 64 __in boolean_t insert); 65 66 static __checkReturn efx_rc_t 67 siena_rx_scale_key_set( 68 __in efx_nic_t *enp, 69 __in uint32_t rss_context, 70 __in_ecount(n) uint8_t *key, 71 __in size_t n); 72 73 static __checkReturn efx_rc_t 74 siena_rx_scale_tbl_set( 75 __in efx_nic_t *enp, 76 __in uint32_t rss_context, 77 __in_ecount(n) unsigned int *table, 78 __in size_t n); 79 80 static __checkReturn uint32_t 81 siena_rx_prefix_hash( 82 __in efx_nic_t *enp, 83 __in efx_rx_hash_alg_t func, 84 __in uint8_t *buffer); 85 86 #endif /* EFSYS_OPT_RX_SCALE */ 87 88 static __checkReturn efx_rc_t 89 siena_rx_prefix_pktlen( 90 __in efx_nic_t *enp, 91 __in uint8_t *buffer, 92 __out uint16_t *lengthp); 93 94 static void 95 siena_rx_qpost( 96 __in efx_rxq_t *erp, 97 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 98 __in size_t size, 99 __in unsigned int ndescs, 100 __in unsigned int completed, 101 __in unsigned int added); 102 103 static void 104 siena_rx_qpush( 105 __in efx_rxq_t *erp, 106 __in unsigned int added, 107 __inout unsigned int *pushedp); 108 109 #if EFSYS_OPT_RX_PACKED_STREAM 110 static void 111 siena_rx_qpush_ps_credits( 112 __in efx_rxq_t *erp); 113 114 static __checkReturn uint8_t * 115 siena_rx_qps_packet_info( 116 __in efx_rxq_t *erp, 117 __in uint8_t *buffer, 118 __in uint32_t buffer_length, 119 __in uint32_t current_offset, 120 __out uint16_t *lengthp, 121 __out uint32_t *next_offsetp, 122 __out uint32_t *timestamp); 123 #endif 124 125 static __checkReturn efx_rc_t 126 siena_rx_qflush( 127 __in efx_rxq_t *erp); 128 129 static void 130 siena_rx_qenable( 131 __in efx_rxq_t *erp); 132 133 static __checkReturn efx_rc_t 134 siena_rx_qcreate( 135 __in efx_nic_t *enp, 136 __in unsigned int index, 137 __in unsigned int label, 138 __in efx_rxq_type_t type, 139 __in uint32_t type_data, 140 __in efsys_mem_t *esmp, 141 __in size_t ndescs, 142 __in uint32_t id, 143 __in unsigned int flags, 144 __in efx_evq_t *eep, 145 __in efx_rxq_t *erp); 146 147 static void 148 siena_rx_qdestroy( 149 __in efx_rxq_t *erp); 150 151 #endif /* EFSYS_OPT_SIENA */ 152 153 154 #if EFSYS_OPT_SIENA 155 static const efx_rx_ops_t __efx_rx_siena_ops = { 156 siena_rx_init, /* erxo_init */ 157 siena_rx_fini, /* erxo_fini */ 158 #if EFSYS_OPT_RX_SCATTER 159 siena_rx_scatter_enable, /* erxo_scatter_enable */ 160 #endif 161 #if EFSYS_OPT_RX_SCALE 162 NULL, /* erxo_scale_context_alloc */ 163 NULL, /* erxo_scale_context_free */ 164 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 165 siena_rx_scale_key_set, /* erxo_scale_key_set */ 166 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 167 siena_rx_prefix_hash, /* erxo_prefix_hash */ 168 #endif 169 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 170 siena_rx_qpost, /* erxo_qpost */ 171 siena_rx_qpush, /* erxo_qpush */ 172 #if EFSYS_OPT_RX_PACKED_STREAM 173 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 174 siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 175 #endif 176 siena_rx_qflush, /* erxo_qflush */ 177 siena_rx_qenable, /* erxo_qenable */ 178 siena_rx_qcreate, /* erxo_qcreate */ 179 siena_rx_qdestroy, /* erxo_qdestroy */ 180 }; 181 #endif /* EFSYS_OPT_SIENA */ 182 183 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 184 static const efx_rx_ops_t __efx_rx_ef10_ops = { 185 ef10_rx_init, /* erxo_init */ 186 ef10_rx_fini, /* erxo_fini */ 187 #if EFSYS_OPT_RX_SCATTER 188 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 189 #endif 190 #if EFSYS_OPT_RX_SCALE 191 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 192 ef10_rx_scale_context_free, /* erxo_scale_context_free */ 193 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 194 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 195 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 196 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 197 #endif 198 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 199 ef10_rx_qpost, /* erxo_qpost */ 200 ef10_rx_qpush, /* erxo_qpush */ 201 #if EFSYS_OPT_RX_PACKED_STREAM 202 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 203 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 204 #endif 205 ef10_rx_qflush, /* erxo_qflush */ 206 ef10_rx_qenable, /* erxo_qenable */ 207 ef10_rx_qcreate, /* erxo_qcreate */ 208 ef10_rx_qdestroy, /* erxo_qdestroy */ 209 }; 210 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 211 212 213 __checkReturn efx_rc_t 214 efx_rx_init( 215 __inout efx_nic_t *enp) 216 { 217 const efx_rx_ops_t *erxop; 218 efx_rc_t rc; 219 220 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 221 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 222 223 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 224 rc = EINVAL; 225 goto fail1; 226 } 227 228 if (enp->en_mod_flags & EFX_MOD_RX) { 229 rc = EINVAL; 230 goto fail2; 231 } 232 233 switch (enp->en_family) { 234 #if EFSYS_OPT_SIENA 235 case EFX_FAMILY_SIENA: 236 erxop = &__efx_rx_siena_ops; 237 break; 238 #endif /* EFSYS_OPT_SIENA */ 239 240 #if EFSYS_OPT_HUNTINGTON 241 case EFX_FAMILY_HUNTINGTON: 242 erxop = &__efx_rx_ef10_ops; 243 break; 244 #endif /* EFSYS_OPT_HUNTINGTON */ 245 246 #if EFSYS_OPT_MEDFORD 247 case EFX_FAMILY_MEDFORD: 248 erxop = &__efx_rx_ef10_ops; 249 break; 250 #endif /* EFSYS_OPT_MEDFORD */ 251 252 default: 253 EFSYS_ASSERT(0); 254 rc = ENOTSUP; 255 goto fail3; 256 } 257 258 if ((rc = erxop->erxo_init(enp)) != 0) 259 goto fail4; 260 261 enp->en_erxop = erxop; 262 enp->en_mod_flags |= EFX_MOD_RX; 263 return (0); 264 265 fail4: 266 EFSYS_PROBE(fail4); 267 fail3: 268 EFSYS_PROBE(fail3); 269 fail2: 270 EFSYS_PROBE(fail2); 271 fail1: 272 EFSYS_PROBE1(fail1, efx_rc_t, rc); 273 274 enp->en_erxop = NULL; 275 enp->en_mod_flags &= ~EFX_MOD_RX; 276 return (rc); 277 } 278 279 void 280 efx_rx_fini( 281 __in efx_nic_t *enp) 282 { 283 const efx_rx_ops_t *erxop = enp->en_erxop; 284 285 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 286 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 287 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 288 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 289 290 erxop->erxo_fini(enp); 291 292 enp->en_erxop = NULL; 293 enp->en_mod_flags &= ~EFX_MOD_RX; 294 } 295 296 #if EFSYS_OPT_RX_SCATTER 297 __checkReturn efx_rc_t 298 efx_rx_scatter_enable( 299 __in efx_nic_t *enp, 300 __in unsigned int buf_size) 301 { 302 const efx_rx_ops_t *erxop = enp->en_erxop; 303 efx_rc_t rc; 304 305 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 306 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 307 308 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 309 goto fail1; 310 311 return (0); 312 313 fail1: 314 EFSYS_PROBE1(fail1, efx_rc_t, rc); 315 return (rc); 316 } 317 #endif /* EFSYS_OPT_RX_SCATTER */ 318 319 #if EFSYS_OPT_RX_SCALE 320 __checkReturn efx_rc_t 321 efx_rx_hash_default_support_get( 322 __in efx_nic_t *enp, 323 __out efx_rx_hash_support_t *supportp) 324 { 325 efx_rc_t rc; 326 327 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 328 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 329 330 if (supportp == NULL) { 331 rc = EINVAL; 332 goto fail1; 333 } 334 335 /* 336 * Report the hashing support the client gets by default if it 337 * does not allocate an RSS context itself. 338 */ 339 *supportp = enp->en_hash_support; 340 341 return (0); 342 343 fail1: 344 EFSYS_PROBE1(fail1, efx_rc_t, rc); 345 346 return (rc); 347 } 348 349 __checkReturn efx_rc_t 350 efx_rx_scale_default_support_get( 351 __in efx_nic_t *enp, 352 __out efx_rx_scale_context_type_t *typep) 353 { 354 efx_rc_t rc; 355 356 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 357 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 358 359 if (typep == NULL) { 360 rc = EINVAL; 361 goto fail1; 362 } 363 364 /* 365 * Report the RSS support the client gets by default if it 366 * does not allocate an RSS context itself. 367 */ 368 *typep = enp->en_rss_context_type; 369 370 return (0); 371 372 fail1: 373 EFSYS_PROBE1(fail1, efx_rc_t, rc); 374 375 return (rc); 376 } 377 #endif /* EFSYS_OPT_RX_SCALE */ 378 379 #if EFSYS_OPT_RX_SCALE 380 __checkReturn efx_rc_t 381 efx_rx_scale_context_alloc( 382 __in efx_nic_t *enp, 383 __in efx_rx_scale_context_type_t type, 384 __in uint32_t num_queues, 385 __out uint32_t *rss_contextp) 386 { 387 const efx_rx_ops_t *erxop = enp->en_erxop; 388 efx_rc_t rc; 389 390 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 391 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 392 393 if (erxop->erxo_scale_context_alloc == NULL) { 394 rc = ENOTSUP; 395 goto fail1; 396 } 397 if ((rc = erxop->erxo_scale_context_alloc(enp, type, 398 num_queues, rss_contextp)) != 0) { 399 goto fail2; 400 } 401 402 return (0); 403 404 fail2: 405 EFSYS_PROBE(fail2); 406 fail1: 407 EFSYS_PROBE1(fail1, efx_rc_t, rc); 408 return (rc); 409 } 410 #endif /* EFSYS_OPT_RX_SCALE */ 411 412 #if EFSYS_OPT_RX_SCALE 413 __checkReturn efx_rc_t 414 efx_rx_scale_context_free( 415 __in efx_nic_t *enp, 416 __in uint32_t rss_context) 417 { 418 const efx_rx_ops_t *erxop = enp->en_erxop; 419 efx_rc_t rc; 420 421 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 422 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 423 424 if (erxop->erxo_scale_context_free == NULL) { 425 rc = ENOTSUP; 426 goto fail1; 427 } 428 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 429 goto fail2; 430 431 return (0); 432 433 fail2: 434 EFSYS_PROBE(fail2); 435 fail1: 436 EFSYS_PROBE1(fail1, efx_rc_t, rc); 437 return (rc); 438 } 439 #endif /* EFSYS_OPT_RX_SCALE */ 440 441 #if EFSYS_OPT_RX_SCALE 442 __checkReturn efx_rc_t 443 efx_rx_scale_mode_set( 444 __in efx_nic_t *enp, 445 __in uint32_t rss_context, 446 __in efx_rx_hash_alg_t alg, 447 __in efx_rx_hash_type_t type, 448 __in boolean_t insert) 449 { 450 const efx_rx_ops_t *erxop = enp->en_erxop; 451 efx_rc_t rc; 452 453 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 454 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 455 456 if (erxop->erxo_scale_mode_set != NULL) { 457 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 458 type, insert)) != 0) 459 goto fail1; 460 } 461 462 return (0); 463 464 fail1: 465 EFSYS_PROBE1(fail1, efx_rc_t, rc); 466 return (rc); 467 } 468 #endif /* EFSYS_OPT_RX_SCALE */ 469 470 #if EFSYS_OPT_RX_SCALE 471 __checkReturn efx_rc_t 472 efx_rx_scale_key_set( 473 __in efx_nic_t *enp, 474 __in uint32_t rss_context, 475 __in_ecount(n) uint8_t *key, 476 __in size_t n) 477 { 478 const efx_rx_ops_t *erxop = enp->en_erxop; 479 efx_rc_t rc; 480 481 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 482 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 483 484 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 485 goto fail1; 486 487 return (0); 488 489 fail1: 490 EFSYS_PROBE1(fail1, efx_rc_t, rc); 491 492 return (rc); 493 } 494 #endif /* EFSYS_OPT_RX_SCALE */ 495 496 #if EFSYS_OPT_RX_SCALE 497 __checkReturn efx_rc_t 498 efx_rx_scale_tbl_set( 499 __in efx_nic_t *enp, 500 __in uint32_t rss_context, 501 __in_ecount(n) unsigned int *table, 502 __in size_t n) 503 { 504 const efx_rx_ops_t *erxop = enp->en_erxop; 505 efx_rc_t rc; 506 507 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 508 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 509 510 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) 511 goto fail1; 512 513 return (0); 514 515 fail1: 516 EFSYS_PROBE1(fail1, efx_rc_t, rc); 517 518 return (rc); 519 } 520 #endif /* EFSYS_OPT_RX_SCALE */ 521 522 void 523 efx_rx_qpost( 524 __in efx_rxq_t *erp, 525 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 526 __in size_t size, 527 __in unsigned int ndescs, 528 __in unsigned int completed, 529 __in unsigned int added) 530 { 531 efx_nic_t *enp = erp->er_enp; 532 const efx_rx_ops_t *erxop = enp->en_erxop; 533 534 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 535 536 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added); 537 } 538 539 #if EFSYS_OPT_RX_PACKED_STREAM 540 541 void 542 efx_rx_qpush_ps_credits( 543 __in efx_rxq_t *erp) 544 { 545 efx_nic_t *enp = erp->er_enp; 546 const efx_rx_ops_t *erxop = enp->en_erxop; 547 548 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 549 550 erxop->erxo_qpush_ps_credits(erp); 551 } 552 553 __checkReturn uint8_t * 554 efx_rx_qps_packet_info( 555 __in efx_rxq_t *erp, 556 __in uint8_t *buffer, 557 __in uint32_t buffer_length, 558 __in uint32_t current_offset, 559 __out uint16_t *lengthp, 560 __out uint32_t *next_offsetp, 561 __out uint32_t *timestamp) 562 { 563 efx_nic_t *enp = erp->er_enp; 564 const efx_rx_ops_t *erxop = enp->en_erxop; 565 566 return (erxop->erxo_qps_packet_info(erp, buffer, 567 buffer_length, current_offset, lengthp, 568 next_offsetp, timestamp)); 569 } 570 571 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 572 573 void 574 efx_rx_qpush( 575 __in efx_rxq_t *erp, 576 __in unsigned int added, 577 __inout unsigned int *pushedp) 578 { 579 efx_nic_t *enp = erp->er_enp; 580 const efx_rx_ops_t *erxop = enp->en_erxop; 581 582 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 583 584 erxop->erxo_qpush(erp, added, pushedp); 585 } 586 587 __checkReturn efx_rc_t 588 efx_rx_qflush( 589 __in efx_rxq_t *erp) 590 { 591 efx_nic_t *enp = erp->er_enp; 592 const efx_rx_ops_t *erxop = enp->en_erxop; 593 efx_rc_t rc; 594 595 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 596 597 if ((rc = erxop->erxo_qflush(erp)) != 0) 598 goto fail1; 599 600 return (0); 601 602 fail1: 603 EFSYS_PROBE1(fail1, efx_rc_t, rc); 604 605 return (rc); 606 } 607 608 void 609 efx_rx_qenable( 610 __in efx_rxq_t *erp) 611 { 612 efx_nic_t *enp = erp->er_enp; 613 const efx_rx_ops_t *erxop = enp->en_erxop; 614 615 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 616 617 erxop->erxo_qenable(erp); 618 } 619 620 static __checkReturn efx_rc_t 621 efx_rx_qcreate_internal( 622 __in efx_nic_t *enp, 623 __in unsigned int index, 624 __in unsigned int label, 625 __in efx_rxq_type_t type, 626 __in uint32_t type_data, 627 __in efsys_mem_t *esmp, 628 __in size_t ndescs, 629 __in uint32_t id, 630 __in unsigned int flags, 631 __in efx_evq_t *eep, 632 __deref_out efx_rxq_t **erpp) 633 { 634 const efx_rx_ops_t *erxop = enp->en_erxop; 635 efx_rxq_t *erp; 636 efx_rc_t rc; 637 638 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 639 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 640 641 /* Allocate an RXQ object */ 642 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 643 644 if (erp == NULL) { 645 rc = ENOMEM; 646 goto fail1; 647 } 648 649 erp->er_magic = EFX_RXQ_MAGIC; 650 erp->er_enp = enp; 651 erp->er_index = index; 652 erp->er_mask = ndescs - 1; 653 erp->er_esmp = esmp; 654 655 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp, 656 ndescs, id, flags, eep, erp)) != 0) 657 goto fail2; 658 659 enp->en_rx_qcount++; 660 *erpp = erp; 661 662 return (0); 663 664 fail2: 665 EFSYS_PROBE(fail2); 666 667 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 668 fail1: 669 EFSYS_PROBE1(fail1, efx_rc_t, rc); 670 671 return (rc); 672 } 673 674 __checkReturn efx_rc_t 675 efx_rx_qcreate( 676 __in efx_nic_t *enp, 677 __in unsigned int index, 678 __in unsigned int label, 679 __in efx_rxq_type_t type, 680 __in efsys_mem_t *esmp, 681 __in size_t ndescs, 682 __in uint32_t id, 683 __in unsigned int flags, 684 __in efx_evq_t *eep, 685 __deref_out efx_rxq_t **erpp) 686 { 687 return efx_rx_qcreate_internal(enp, index, label, type, 0, esmp, ndescs, 688 id, flags, eep, erpp); 689 } 690 691 #if EFSYS_OPT_RX_PACKED_STREAM 692 693 __checkReturn efx_rc_t 694 efx_rx_qcreate_packed_stream( 695 __in efx_nic_t *enp, 696 __in unsigned int index, 697 __in unsigned int label, 698 __in uint32_t ps_buf_size, 699 __in efsys_mem_t *esmp, 700 __in size_t ndescs, 701 __in efx_evq_t *eep, 702 __deref_out efx_rxq_t **erpp) 703 { 704 return efx_rx_qcreate_internal(enp, index, label, 705 EFX_RXQ_TYPE_PACKED_STREAM, ps_buf_size, esmp, ndescs, 706 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp); 707 } 708 709 #endif 710 711 void 712 efx_rx_qdestroy( 713 __in efx_rxq_t *erp) 714 { 715 efx_nic_t *enp = erp->er_enp; 716 const efx_rx_ops_t *erxop = enp->en_erxop; 717 718 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 719 720 erxop->erxo_qdestroy(erp); 721 } 722 723 __checkReturn efx_rc_t 724 efx_pseudo_hdr_pkt_length_get( 725 __in efx_rxq_t *erp, 726 __in uint8_t *buffer, 727 __out uint16_t *lengthp) 728 { 729 efx_nic_t *enp = erp->er_enp; 730 const efx_rx_ops_t *erxop = enp->en_erxop; 731 732 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 733 734 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 735 } 736 737 #if EFSYS_OPT_RX_SCALE 738 __checkReturn uint32_t 739 efx_pseudo_hdr_hash_get( 740 __in efx_rxq_t *erp, 741 __in efx_rx_hash_alg_t func, 742 __in uint8_t *buffer) 743 { 744 efx_nic_t *enp = erp->er_enp; 745 const efx_rx_ops_t *erxop = enp->en_erxop; 746 747 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 748 749 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 750 return (erxop->erxo_prefix_hash(enp, func, buffer)); 751 } 752 #endif /* EFSYS_OPT_RX_SCALE */ 753 754 #if EFSYS_OPT_SIENA 755 756 static __checkReturn efx_rc_t 757 siena_rx_init( 758 __in efx_nic_t *enp) 759 { 760 efx_oword_t oword; 761 unsigned int index; 762 763 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 764 765 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 766 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 767 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 768 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 769 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 770 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 771 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 772 773 /* Zero the RSS table */ 774 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 775 index++) { 776 EFX_ZERO_OWORD(oword); 777 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 778 index, &oword, B_TRUE); 779 } 780 781 #if EFSYS_OPT_RX_SCALE 782 /* The RSS key and indirection table are writable. */ 783 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 784 785 /* Hardware can insert RX hash with/without RSS */ 786 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 787 #endif /* EFSYS_OPT_RX_SCALE */ 788 789 return (0); 790 } 791 792 #if EFSYS_OPT_RX_SCATTER 793 static __checkReturn efx_rc_t 794 siena_rx_scatter_enable( 795 __in efx_nic_t *enp, 796 __in unsigned int buf_size) 797 { 798 unsigned int nbuf32; 799 efx_oword_t oword; 800 efx_rc_t rc; 801 802 nbuf32 = buf_size / 32; 803 if ((nbuf32 == 0) || 804 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 805 ((buf_size % 32) != 0)) { 806 rc = EINVAL; 807 goto fail1; 808 } 809 810 if (enp->en_rx_qcount > 0) { 811 rc = EBUSY; 812 goto fail2; 813 } 814 815 /* Set scatter buffer size */ 816 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 817 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 818 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 819 820 /* Enable scatter for packets not matching a filter */ 821 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 822 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 823 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 824 825 return (0); 826 827 fail2: 828 EFSYS_PROBE(fail2); 829 fail1: 830 EFSYS_PROBE1(fail1, efx_rc_t, rc); 831 832 return (rc); 833 } 834 #endif /* EFSYS_OPT_RX_SCATTER */ 835 836 837 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 838 do { \ 839 efx_oword_t oword; \ 840 \ 841 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 842 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 843 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 844 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 845 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 846 (_insert) ? 1 : 0); \ 847 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 848 \ 849 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 850 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 851 &oword); \ 852 EFX_SET_OWORD_FIELD(oword, \ 853 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 854 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 855 &oword); \ 856 } \ 857 \ 858 _NOTE(CONSTANTCONDITION) \ 859 } while (B_FALSE) 860 861 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 862 do { \ 863 efx_oword_t oword; \ 864 \ 865 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 866 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 867 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 868 (_ip) ? 1 : 0); \ 869 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 870 (_tcp) ? 0 : 1); \ 871 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 872 (_insert) ? 1 : 0); \ 873 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 874 \ 875 _NOTE(CONSTANTCONDITION) \ 876 } while (B_FALSE) 877 878 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 879 do { \ 880 efx_oword_t oword; \ 881 \ 882 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 883 EFX_SET_OWORD_FIELD(oword, \ 884 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 885 EFX_SET_OWORD_FIELD(oword, \ 886 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 887 EFX_SET_OWORD_FIELD(oword, \ 888 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 889 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 890 \ 891 (_rc) = 0; \ 892 \ 893 _NOTE(CONSTANTCONDITION) \ 894 } while (B_FALSE) 895 896 897 #if EFSYS_OPT_RX_SCALE 898 899 static __checkReturn efx_rc_t 900 siena_rx_scale_mode_set( 901 __in efx_nic_t *enp, 902 __in uint32_t rss_context, 903 __in efx_rx_hash_alg_t alg, 904 __in efx_rx_hash_type_t type, 905 __in boolean_t insert) 906 { 907 efx_rc_t rc; 908 909 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 910 rc = EINVAL; 911 goto fail1; 912 } 913 914 switch (alg) { 915 case EFX_RX_HASHALG_LFSR: 916 EFX_RX_LFSR_HASH(enp, insert); 917 break; 918 919 case EFX_RX_HASHALG_TOEPLITZ: 920 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 921 type & EFX_RX_HASH_IPV4, 922 type & EFX_RX_HASH_TCPIPV4); 923 924 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 925 type & EFX_RX_HASH_IPV6, 926 type & EFX_RX_HASH_TCPIPV6, 927 rc); 928 if (rc != 0) 929 goto fail2; 930 931 break; 932 933 default: 934 rc = EINVAL; 935 goto fail3; 936 } 937 938 return (0); 939 940 fail3: 941 EFSYS_PROBE(fail3); 942 fail2: 943 EFSYS_PROBE(fail2); 944 fail1: 945 EFSYS_PROBE1(fail1, efx_rc_t, rc); 946 947 EFX_RX_LFSR_HASH(enp, B_FALSE); 948 949 return (rc); 950 } 951 #endif 952 953 #if EFSYS_OPT_RX_SCALE 954 static __checkReturn efx_rc_t 955 siena_rx_scale_key_set( 956 __in efx_nic_t *enp, 957 __in uint32_t rss_context, 958 __in_ecount(n) uint8_t *key, 959 __in size_t n) 960 { 961 efx_oword_t oword; 962 unsigned int byte; 963 unsigned int offset; 964 efx_rc_t rc; 965 966 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 967 rc = EINVAL; 968 goto fail1; 969 } 970 971 byte = 0; 972 973 /* Write Toeplitz IPv4 hash key */ 974 EFX_ZERO_OWORD(oword); 975 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 976 offset > 0 && byte < n; 977 --offset) 978 oword.eo_u8[offset - 1] = key[byte++]; 979 980 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 981 982 byte = 0; 983 984 /* Verify Toeplitz IPv4 hash key */ 985 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 986 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 987 offset > 0 && byte < n; 988 --offset) { 989 if (oword.eo_u8[offset - 1] != key[byte++]) { 990 rc = EFAULT; 991 goto fail2; 992 } 993 } 994 995 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 996 goto done; 997 998 byte = 0; 999 1000 /* Write Toeplitz IPv6 hash key 3 */ 1001 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1002 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1003 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1004 offset > 0 && byte < n; 1005 --offset) 1006 oword.eo_u8[offset - 1] = key[byte++]; 1007 1008 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1009 1010 /* Write Toeplitz IPv6 hash key 2 */ 1011 EFX_ZERO_OWORD(oword); 1012 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1013 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1014 offset > 0 && byte < n; 1015 --offset) 1016 oword.eo_u8[offset - 1] = key[byte++]; 1017 1018 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1019 1020 /* Write Toeplitz IPv6 hash key 1 */ 1021 EFX_ZERO_OWORD(oword); 1022 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1023 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1024 offset > 0 && byte < n; 1025 --offset) 1026 oword.eo_u8[offset - 1] = key[byte++]; 1027 1028 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1029 1030 byte = 0; 1031 1032 /* Verify Toeplitz IPv6 hash key 3 */ 1033 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1034 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1035 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1036 offset > 0 && byte < n; 1037 --offset) { 1038 if (oword.eo_u8[offset - 1] != key[byte++]) { 1039 rc = EFAULT; 1040 goto fail3; 1041 } 1042 } 1043 1044 /* Verify Toeplitz IPv6 hash key 2 */ 1045 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1046 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1047 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1048 offset > 0 && byte < n; 1049 --offset) { 1050 if (oword.eo_u8[offset - 1] != key[byte++]) { 1051 rc = EFAULT; 1052 goto fail4; 1053 } 1054 } 1055 1056 /* Verify Toeplitz IPv6 hash key 1 */ 1057 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1058 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1059 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1060 offset > 0 && byte < n; 1061 --offset) { 1062 if (oword.eo_u8[offset - 1] != key[byte++]) { 1063 rc = EFAULT; 1064 goto fail5; 1065 } 1066 } 1067 1068 done: 1069 return (0); 1070 1071 fail5: 1072 EFSYS_PROBE(fail5); 1073 fail4: 1074 EFSYS_PROBE(fail4); 1075 fail3: 1076 EFSYS_PROBE(fail3); 1077 fail2: 1078 EFSYS_PROBE(fail2); 1079 fail1: 1080 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1081 1082 return (rc); 1083 } 1084 #endif 1085 1086 #if EFSYS_OPT_RX_SCALE 1087 static __checkReturn efx_rc_t 1088 siena_rx_scale_tbl_set( 1089 __in efx_nic_t *enp, 1090 __in uint32_t rss_context, 1091 __in_ecount(n) unsigned int *table, 1092 __in size_t n) 1093 { 1094 efx_oword_t oword; 1095 int index; 1096 efx_rc_t rc; 1097 1098 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1099 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1100 1101 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1102 rc = EINVAL; 1103 goto fail1; 1104 } 1105 1106 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1107 rc = EINVAL; 1108 goto fail2; 1109 } 1110 1111 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1112 uint32_t byte; 1113 1114 /* Calculate the entry to place in the table */ 1115 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1116 1117 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1118 1119 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1120 1121 /* Write the table */ 1122 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1123 index, &oword, B_TRUE); 1124 } 1125 1126 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1127 uint32_t byte; 1128 1129 /* Determine if we're starting a new batch */ 1130 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1131 1132 /* Read the table */ 1133 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1134 index, &oword, B_TRUE); 1135 1136 /* Verify the entry */ 1137 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1138 rc = EFAULT; 1139 goto fail3; 1140 } 1141 } 1142 1143 return (0); 1144 1145 fail3: 1146 EFSYS_PROBE(fail3); 1147 fail2: 1148 EFSYS_PROBE(fail2); 1149 fail1: 1150 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1151 1152 return (rc); 1153 } 1154 #endif 1155 1156 /* 1157 * Falcon/Siena pseudo-header 1158 * -------------------------- 1159 * 1160 * Receive packets are prefixed by an optional 16 byte pseudo-header. 1161 * The pseudo-header is a byte array of one of the forms: 1162 * 1163 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 1164 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 1165 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 1166 * 1167 * where: 1168 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 1169 * LL.LL LFSR hash (16-bit big-endian) 1170 */ 1171 1172 #if EFSYS_OPT_RX_SCALE 1173 static __checkReturn uint32_t 1174 siena_rx_prefix_hash( 1175 __in efx_nic_t *enp, 1176 __in efx_rx_hash_alg_t func, 1177 __in uint8_t *buffer) 1178 { 1179 _NOTE(ARGUNUSED(enp)) 1180 1181 switch (func) { 1182 case EFX_RX_HASHALG_TOEPLITZ: 1183 return ((buffer[12] << 24) | 1184 (buffer[13] << 16) | 1185 (buffer[14] << 8) | 1186 buffer[15]); 1187 1188 case EFX_RX_HASHALG_LFSR: 1189 return ((buffer[14] << 8) | buffer[15]); 1190 1191 default: 1192 EFSYS_ASSERT(0); 1193 return (0); 1194 } 1195 } 1196 #endif /* EFSYS_OPT_RX_SCALE */ 1197 1198 static __checkReturn efx_rc_t 1199 siena_rx_prefix_pktlen( 1200 __in efx_nic_t *enp, 1201 __in uint8_t *buffer, 1202 __out uint16_t *lengthp) 1203 { 1204 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1205 1206 /* Not supported by Falcon/Siena hardware */ 1207 EFSYS_ASSERT(0); 1208 return (ENOTSUP); 1209 } 1210 1211 1212 static void 1213 siena_rx_qpost( 1214 __in efx_rxq_t *erp, 1215 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 1216 __in size_t size, 1217 __in unsigned int ndescs, 1218 __in unsigned int completed, 1219 __in unsigned int added) 1220 { 1221 efx_qword_t qword; 1222 unsigned int i; 1223 unsigned int offset; 1224 unsigned int id; 1225 1226 /* The client driver must not overfill the queue */ 1227 EFSYS_ASSERT3U(added - completed + ndescs, <=, 1228 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1229 1230 id = added & (erp->er_mask); 1231 for (i = 0; i < ndescs; i++) { 1232 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1233 unsigned int, id, efsys_dma_addr_t, addrp[i], 1234 size_t, size); 1235 1236 EFX_POPULATE_QWORD_3(qword, 1237 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1238 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1239 (uint32_t)(addrp[i] & 0xffffffff), 1240 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1241 (uint32_t)(addrp[i] >> 32)); 1242 1243 offset = id * sizeof (efx_qword_t); 1244 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1245 1246 id = (id + 1) & (erp->er_mask); 1247 } 1248 } 1249 1250 static void 1251 siena_rx_qpush( 1252 __in efx_rxq_t *erp, 1253 __in unsigned int added, 1254 __inout unsigned int *pushedp) 1255 { 1256 efx_nic_t *enp = erp->er_enp; 1257 unsigned int pushed = *pushedp; 1258 uint32_t wptr; 1259 efx_oword_t oword; 1260 efx_dword_t dword; 1261 1262 /* All descriptors are pushed */ 1263 *pushedp = added; 1264 1265 /* Push the populated descriptors out */ 1266 wptr = added & erp->er_mask; 1267 1268 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1269 1270 /* Only write the third DWORD */ 1271 EFX_POPULATE_DWORD_1(dword, 1272 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1273 1274 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1275 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1276 wptr, pushed & erp->er_mask); 1277 EFSYS_PIO_WRITE_BARRIER(); 1278 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1279 erp->er_index, &dword, B_FALSE); 1280 } 1281 1282 #if EFSYS_OPT_RX_PACKED_STREAM 1283 static void 1284 siena_rx_qpush_ps_credits( 1285 __in efx_rxq_t *erp) 1286 { 1287 /* Not supported by Siena hardware */ 1288 EFSYS_ASSERT(0); 1289 } 1290 1291 static uint8_t * 1292 siena_rx_qps_packet_info( 1293 __in efx_rxq_t *erp, 1294 __in uint8_t *buffer, 1295 __in uint32_t buffer_length, 1296 __in uint32_t current_offset, 1297 __out uint16_t *lengthp, 1298 __out uint32_t *next_offsetp, 1299 __out uint32_t *timestamp) 1300 { 1301 /* Not supported by Siena hardware */ 1302 EFSYS_ASSERT(0); 1303 1304 return (NULL); 1305 } 1306 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1307 1308 static __checkReturn efx_rc_t 1309 siena_rx_qflush( 1310 __in efx_rxq_t *erp) 1311 { 1312 efx_nic_t *enp = erp->er_enp; 1313 efx_oword_t oword; 1314 uint32_t label; 1315 1316 label = erp->er_index; 1317 1318 /* Flush the queue */ 1319 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1320 FRF_AZ_RX_FLUSH_DESCQ, label); 1321 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1322 1323 return (0); 1324 } 1325 1326 static void 1327 siena_rx_qenable( 1328 __in efx_rxq_t *erp) 1329 { 1330 efx_nic_t *enp = erp->er_enp; 1331 efx_oword_t oword; 1332 1333 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1334 1335 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1336 erp->er_index, &oword, B_TRUE); 1337 1338 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1339 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1340 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1341 1342 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1343 erp->er_index, &oword, B_TRUE); 1344 } 1345 1346 static __checkReturn efx_rc_t 1347 siena_rx_qcreate( 1348 __in efx_nic_t *enp, 1349 __in unsigned int index, 1350 __in unsigned int label, 1351 __in efx_rxq_type_t type, 1352 __in uint32_t type_data, 1353 __in efsys_mem_t *esmp, 1354 __in size_t ndescs, 1355 __in uint32_t id, 1356 __in unsigned int flags, 1357 __in efx_evq_t *eep, 1358 __in efx_rxq_t *erp) 1359 { 1360 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1361 efx_oword_t oword; 1362 uint32_t size; 1363 boolean_t jumbo = B_FALSE; 1364 efx_rc_t rc; 1365 1366 _NOTE(ARGUNUSED(esmp)) 1367 _NOTE(ARGUNUSED(type_data)) 1368 1369 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1370 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1371 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1372 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1373 1374 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 1375 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 1376 1377 if (!ISP2(ndescs) || 1378 (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) { 1379 rc = EINVAL; 1380 goto fail1; 1381 } 1382 if (index >= encp->enc_rxq_limit) { 1383 rc = EINVAL; 1384 goto fail2; 1385 } 1386 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1387 size++) 1388 if ((1 << size) == (int)(ndescs / EFX_RXQ_MINNDESCS)) 1389 break; 1390 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1391 rc = EINVAL; 1392 goto fail3; 1393 } 1394 1395 switch (type) { 1396 case EFX_RXQ_TYPE_DEFAULT: 1397 break; 1398 1399 default: 1400 rc = EINVAL; 1401 goto fail4; 1402 } 1403 1404 if (flags & EFX_RXQ_FLAG_SCATTER) { 1405 #if EFSYS_OPT_RX_SCATTER 1406 jumbo = B_TRUE; 1407 #else 1408 rc = EINVAL; 1409 goto fail5; 1410 #endif /* EFSYS_OPT_RX_SCATTER */ 1411 } 1412 1413 /* Set up the new descriptor queue */ 1414 EFX_POPULATE_OWORD_7(oword, 1415 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1416 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1417 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1418 FRF_AZ_RX_DESCQ_LABEL, label, 1419 FRF_AZ_RX_DESCQ_SIZE, size, 1420 FRF_AZ_RX_DESCQ_TYPE, 0, 1421 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1422 1423 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1424 erp->er_index, &oword, B_TRUE); 1425 1426 return (0); 1427 1428 #if !EFSYS_OPT_RX_SCATTER 1429 fail5: 1430 EFSYS_PROBE(fail5); 1431 #endif 1432 fail4: 1433 EFSYS_PROBE(fail4); 1434 fail3: 1435 EFSYS_PROBE(fail3); 1436 fail2: 1437 EFSYS_PROBE(fail2); 1438 fail1: 1439 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1440 1441 return (rc); 1442 } 1443 1444 static void 1445 siena_rx_qdestroy( 1446 __in efx_rxq_t *erp) 1447 { 1448 efx_nic_t *enp = erp->er_enp; 1449 efx_oword_t oword; 1450 1451 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1452 --enp->en_rx_qcount; 1453 1454 /* Purge descriptor queue */ 1455 EFX_ZERO_OWORD(oword); 1456 1457 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1458 erp->er_index, &oword, B_TRUE); 1459 1460 /* Free the RXQ object */ 1461 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1462 } 1463 1464 static void 1465 siena_rx_fini( 1466 __in efx_nic_t *enp) 1467 { 1468 _NOTE(ARGUNUSED(enp)) 1469 } 1470 1471 #endif /* EFSYS_OPT_SIENA */ 1472