1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "efx.h" 37 #include "efx_impl.h" 38 39 40 #if EFSYS_OPT_SIENA 41 42 static __checkReturn efx_rc_t 43 siena_rx_init( 44 __in efx_nic_t *enp); 45 46 static void 47 siena_rx_fini( 48 __in efx_nic_t *enp); 49 50 #if EFSYS_OPT_RX_SCATTER 51 static __checkReturn efx_rc_t 52 siena_rx_scatter_enable( 53 __in efx_nic_t *enp, 54 __in unsigned int buf_size); 55 #endif /* EFSYS_OPT_RX_SCATTER */ 56 57 #if EFSYS_OPT_RX_SCALE 58 static __checkReturn efx_rc_t 59 siena_rx_scale_mode_set( 60 __in efx_nic_t *enp, 61 __in uint32_t rss_context, 62 __in efx_rx_hash_alg_t alg, 63 __in efx_rx_hash_type_t type, 64 __in boolean_t insert); 65 66 static __checkReturn efx_rc_t 67 siena_rx_scale_key_set( 68 __in efx_nic_t *enp, 69 __in uint32_t rss_context, 70 __in_ecount(n) uint8_t *key, 71 __in size_t n); 72 73 static __checkReturn efx_rc_t 74 siena_rx_scale_tbl_set( 75 __in efx_nic_t *enp, 76 __in uint32_t rss_context, 77 __in_ecount(n) unsigned int *table, 78 __in size_t n); 79 80 static __checkReturn uint32_t 81 siena_rx_prefix_hash( 82 __in efx_nic_t *enp, 83 __in efx_rx_hash_alg_t func, 84 __in uint8_t *buffer); 85 86 #endif /* EFSYS_OPT_RX_SCALE */ 87 88 static __checkReturn efx_rc_t 89 siena_rx_prefix_pktlen( 90 __in efx_nic_t *enp, 91 __in uint8_t *buffer, 92 __out uint16_t *lengthp); 93 94 static void 95 siena_rx_qpost( 96 __in efx_rxq_t *erp, 97 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 98 __in size_t size, 99 __in unsigned int ndescs, 100 __in unsigned int completed, 101 __in unsigned int added); 102 103 static void 104 siena_rx_qpush( 105 __in efx_rxq_t *erp, 106 __in unsigned int added, 107 __inout unsigned int *pushedp); 108 109 #if EFSYS_OPT_RX_PACKED_STREAM 110 static void 111 siena_rx_qpush_ps_credits( 112 __in efx_rxq_t *erp); 113 114 static __checkReturn uint8_t * 115 siena_rx_qps_packet_info( 116 __in efx_rxq_t *erp, 117 __in uint8_t *buffer, 118 __in uint32_t buffer_length, 119 __in uint32_t current_offset, 120 __out uint16_t *lengthp, 121 __out uint32_t *next_offsetp, 122 __out uint32_t *timestamp); 123 #endif 124 125 static __checkReturn efx_rc_t 126 siena_rx_qflush( 127 __in efx_rxq_t *erp); 128 129 static void 130 siena_rx_qenable( 131 __in efx_rxq_t *erp); 132 133 static __checkReturn efx_rc_t 134 siena_rx_qcreate( 135 __in efx_nic_t *enp, 136 __in unsigned int index, 137 __in unsigned int label, 138 __in efx_rxq_type_t type, 139 __in efsys_mem_t *esmp, 140 __in size_t ndescs, 141 __in uint32_t id, 142 __in efx_evq_t *eep, 143 __in efx_rxq_t *erp); 144 145 static void 146 siena_rx_qdestroy( 147 __in efx_rxq_t *erp); 148 149 #endif /* EFSYS_OPT_SIENA */ 150 151 152 #if EFSYS_OPT_SIENA 153 static const efx_rx_ops_t __efx_rx_siena_ops = { 154 siena_rx_init, /* erxo_init */ 155 siena_rx_fini, /* erxo_fini */ 156 #if EFSYS_OPT_RX_SCATTER 157 siena_rx_scatter_enable, /* erxo_scatter_enable */ 158 #endif 159 #if EFSYS_OPT_RX_SCALE 160 NULL, /* erxo_scale_context_alloc */ 161 NULL, /* erxo_scale_context_free */ 162 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 163 siena_rx_scale_key_set, /* erxo_scale_key_set */ 164 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 165 siena_rx_prefix_hash, /* erxo_prefix_hash */ 166 #endif 167 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 168 siena_rx_qpost, /* erxo_qpost */ 169 siena_rx_qpush, /* erxo_qpush */ 170 #if EFSYS_OPT_RX_PACKED_STREAM 171 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 172 siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 173 #endif 174 siena_rx_qflush, /* erxo_qflush */ 175 siena_rx_qenable, /* erxo_qenable */ 176 siena_rx_qcreate, /* erxo_qcreate */ 177 siena_rx_qdestroy, /* erxo_qdestroy */ 178 }; 179 #endif /* EFSYS_OPT_SIENA */ 180 181 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 182 static const efx_rx_ops_t __efx_rx_ef10_ops = { 183 ef10_rx_init, /* erxo_init */ 184 ef10_rx_fini, /* erxo_fini */ 185 #if EFSYS_OPT_RX_SCATTER 186 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 187 #endif 188 #if EFSYS_OPT_RX_SCALE 189 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 190 ef10_rx_scale_context_free, /* erxo_scale_context_free */ 191 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 192 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 193 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 194 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 195 #endif 196 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 197 ef10_rx_qpost, /* erxo_qpost */ 198 ef10_rx_qpush, /* erxo_qpush */ 199 #if EFSYS_OPT_RX_PACKED_STREAM 200 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 201 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 202 #endif 203 ef10_rx_qflush, /* erxo_qflush */ 204 ef10_rx_qenable, /* erxo_qenable */ 205 ef10_rx_qcreate, /* erxo_qcreate */ 206 ef10_rx_qdestroy, /* erxo_qdestroy */ 207 }; 208 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 209 210 211 __checkReturn efx_rc_t 212 efx_rx_init( 213 __inout efx_nic_t *enp) 214 { 215 const efx_rx_ops_t *erxop; 216 efx_rc_t rc; 217 218 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 219 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 220 221 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 222 rc = EINVAL; 223 goto fail1; 224 } 225 226 if (enp->en_mod_flags & EFX_MOD_RX) { 227 rc = EINVAL; 228 goto fail2; 229 } 230 231 switch (enp->en_family) { 232 #if EFSYS_OPT_SIENA 233 case EFX_FAMILY_SIENA: 234 erxop = &__efx_rx_siena_ops; 235 break; 236 #endif /* EFSYS_OPT_SIENA */ 237 238 #if EFSYS_OPT_HUNTINGTON 239 case EFX_FAMILY_HUNTINGTON: 240 erxop = &__efx_rx_ef10_ops; 241 break; 242 #endif /* EFSYS_OPT_HUNTINGTON */ 243 244 #if EFSYS_OPT_MEDFORD 245 case EFX_FAMILY_MEDFORD: 246 erxop = &__efx_rx_ef10_ops; 247 break; 248 #endif /* EFSYS_OPT_MEDFORD */ 249 250 default: 251 EFSYS_ASSERT(0); 252 rc = ENOTSUP; 253 goto fail3; 254 } 255 256 if ((rc = erxop->erxo_init(enp)) != 0) 257 goto fail4; 258 259 enp->en_erxop = erxop; 260 enp->en_mod_flags |= EFX_MOD_RX; 261 return (0); 262 263 fail4: 264 EFSYS_PROBE(fail4); 265 fail3: 266 EFSYS_PROBE(fail3); 267 fail2: 268 EFSYS_PROBE(fail2); 269 fail1: 270 EFSYS_PROBE1(fail1, efx_rc_t, rc); 271 272 enp->en_erxop = NULL; 273 enp->en_mod_flags &= ~EFX_MOD_RX; 274 return (rc); 275 } 276 277 void 278 efx_rx_fini( 279 __in efx_nic_t *enp) 280 { 281 const efx_rx_ops_t *erxop = enp->en_erxop; 282 283 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 284 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 285 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 286 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 287 288 erxop->erxo_fini(enp); 289 290 enp->en_erxop = NULL; 291 enp->en_mod_flags &= ~EFX_MOD_RX; 292 } 293 294 #if EFSYS_OPT_RX_SCATTER 295 __checkReturn efx_rc_t 296 efx_rx_scatter_enable( 297 __in efx_nic_t *enp, 298 __in unsigned int buf_size) 299 { 300 const efx_rx_ops_t *erxop = enp->en_erxop; 301 efx_rc_t rc; 302 303 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 304 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 305 306 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 307 goto fail1; 308 309 return (0); 310 311 fail1: 312 EFSYS_PROBE1(fail1, efx_rc_t, rc); 313 return (rc); 314 } 315 #endif /* EFSYS_OPT_RX_SCATTER */ 316 317 #if EFSYS_OPT_RX_SCALE 318 __checkReturn efx_rc_t 319 efx_rx_hash_default_support_get( 320 __in efx_nic_t *enp, 321 __out efx_rx_hash_support_t *supportp) 322 { 323 efx_rc_t rc; 324 325 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 326 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 327 328 if (supportp == NULL) { 329 rc = EINVAL; 330 goto fail1; 331 } 332 333 /* 334 * Report the hashing support the client gets by default if it 335 * does not allocate an RSS context itself. 336 */ 337 *supportp = enp->en_hash_support; 338 339 return (0); 340 341 fail1: 342 EFSYS_PROBE1(fail1, efx_rc_t, rc); 343 344 return (rc); 345 } 346 347 __checkReturn efx_rc_t 348 efx_rx_scale_default_support_get( 349 __in efx_nic_t *enp, 350 __out efx_rx_scale_context_type_t *typep) 351 { 352 efx_rc_t rc; 353 354 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 355 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 356 357 if (typep == NULL) { 358 rc = EINVAL; 359 goto fail1; 360 } 361 362 /* 363 * Report the RSS support the client gets by default if it 364 * does not allocate an RSS context itself. 365 */ 366 *typep = enp->en_rss_context_type; 367 368 return (0); 369 370 fail1: 371 EFSYS_PROBE1(fail1, efx_rc_t, rc); 372 373 return (rc); 374 } 375 #endif /* EFSYS_OPT_RX_SCALE */ 376 377 #if EFSYS_OPT_RX_SCALE 378 __checkReturn efx_rc_t 379 efx_rx_scale_context_alloc( 380 __in efx_nic_t *enp, 381 __in efx_rx_scale_context_type_t type, 382 __in uint32_t num_queues, 383 __out uint32_t *rss_contextp) 384 { 385 const efx_rx_ops_t *erxop = enp->en_erxop; 386 efx_rc_t rc; 387 388 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 389 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 390 391 if (erxop->erxo_scale_context_alloc == NULL) { 392 rc = ENOTSUP; 393 goto fail1; 394 } 395 if ((rc = erxop->erxo_scale_context_alloc(enp, type, 396 num_queues, rss_contextp)) != 0) { 397 goto fail2; 398 } 399 400 return (0); 401 402 fail2: 403 EFSYS_PROBE(fail2); 404 fail1: 405 EFSYS_PROBE1(fail1, efx_rc_t, rc); 406 return (rc); 407 } 408 #endif /* EFSYS_OPT_RX_SCALE */ 409 410 #if EFSYS_OPT_RX_SCALE 411 __checkReturn efx_rc_t 412 efx_rx_scale_context_free( 413 __in efx_nic_t *enp, 414 __in uint32_t rss_context) 415 { 416 const efx_rx_ops_t *erxop = enp->en_erxop; 417 efx_rc_t rc; 418 419 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 420 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 421 422 if (erxop->erxo_scale_context_free == NULL) { 423 rc = ENOTSUP; 424 goto fail1; 425 } 426 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 427 goto fail2; 428 429 return (0); 430 431 fail2: 432 EFSYS_PROBE(fail2); 433 fail1: 434 EFSYS_PROBE1(fail1, efx_rc_t, rc); 435 return (rc); 436 } 437 #endif /* EFSYS_OPT_RX_SCALE */ 438 439 #if EFSYS_OPT_RX_SCALE 440 __checkReturn efx_rc_t 441 efx_rx_scale_mode_set( 442 __in efx_nic_t *enp, 443 __in uint32_t rss_context, 444 __in efx_rx_hash_alg_t alg, 445 __in efx_rx_hash_type_t type, 446 __in boolean_t insert) 447 { 448 const efx_rx_ops_t *erxop = enp->en_erxop; 449 efx_rc_t rc; 450 451 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 452 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 453 454 if (erxop->erxo_scale_mode_set != NULL) { 455 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 456 type, insert)) != 0) 457 goto fail1; 458 } 459 460 return (0); 461 462 fail1: 463 EFSYS_PROBE1(fail1, efx_rc_t, rc); 464 return (rc); 465 } 466 #endif /* EFSYS_OPT_RX_SCALE */ 467 468 #if EFSYS_OPT_RX_SCALE 469 __checkReturn efx_rc_t 470 efx_rx_scale_key_set( 471 __in efx_nic_t *enp, 472 __in uint32_t rss_context, 473 __in_ecount(n) uint8_t *key, 474 __in size_t n) 475 { 476 const efx_rx_ops_t *erxop = enp->en_erxop; 477 efx_rc_t rc; 478 479 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 480 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 481 482 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 483 goto fail1; 484 485 return (0); 486 487 fail1: 488 EFSYS_PROBE1(fail1, efx_rc_t, rc); 489 490 return (rc); 491 } 492 #endif /* EFSYS_OPT_RX_SCALE */ 493 494 #if EFSYS_OPT_RX_SCALE 495 __checkReturn efx_rc_t 496 efx_rx_scale_tbl_set( 497 __in efx_nic_t *enp, 498 __in uint32_t rss_context, 499 __in_ecount(n) unsigned int *table, 500 __in size_t n) 501 { 502 const efx_rx_ops_t *erxop = enp->en_erxop; 503 efx_rc_t rc; 504 505 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 506 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 507 508 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) 509 goto fail1; 510 511 return (0); 512 513 fail1: 514 EFSYS_PROBE1(fail1, efx_rc_t, rc); 515 516 return (rc); 517 } 518 #endif /* EFSYS_OPT_RX_SCALE */ 519 520 void 521 efx_rx_qpost( 522 __in efx_rxq_t *erp, 523 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 524 __in size_t size, 525 __in unsigned int ndescs, 526 __in unsigned int completed, 527 __in unsigned int added) 528 { 529 efx_nic_t *enp = erp->er_enp; 530 const efx_rx_ops_t *erxop = enp->en_erxop; 531 532 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 533 534 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added); 535 } 536 537 #if EFSYS_OPT_RX_PACKED_STREAM 538 539 void 540 efx_rx_qpush_ps_credits( 541 __in efx_rxq_t *erp) 542 { 543 efx_nic_t *enp = erp->er_enp; 544 const efx_rx_ops_t *erxop = enp->en_erxop; 545 546 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 547 548 erxop->erxo_qpush_ps_credits(erp); 549 } 550 551 __checkReturn uint8_t * 552 efx_rx_qps_packet_info( 553 __in efx_rxq_t *erp, 554 __in uint8_t *buffer, 555 __in uint32_t buffer_length, 556 __in uint32_t current_offset, 557 __out uint16_t *lengthp, 558 __out uint32_t *next_offsetp, 559 __out uint32_t *timestamp) 560 { 561 efx_nic_t *enp = erp->er_enp; 562 const efx_rx_ops_t *erxop = enp->en_erxop; 563 564 return (erxop->erxo_qps_packet_info(erp, buffer, 565 buffer_length, current_offset, lengthp, 566 next_offsetp, timestamp)); 567 } 568 569 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 570 571 void 572 efx_rx_qpush( 573 __in efx_rxq_t *erp, 574 __in unsigned int added, 575 __inout unsigned int *pushedp) 576 { 577 efx_nic_t *enp = erp->er_enp; 578 const efx_rx_ops_t *erxop = enp->en_erxop; 579 580 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 581 582 erxop->erxo_qpush(erp, added, pushedp); 583 } 584 585 __checkReturn efx_rc_t 586 efx_rx_qflush( 587 __in efx_rxq_t *erp) 588 { 589 efx_nic_t *enp = erp->er_enp; 590 const efx_rx_ops_t *erxop = enp->en_erxop; 591 efx_rc_t rc; 592 593 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 594 595 if ((rc = erxop->erxo_qflush(erp)) != 0) 596 goto fail1; 597 598 return (0); 599 600 fail1: 601 EFSYS_PROBE1(fail1, efx_rc_t, rc); 602 603 return (rc); 604 } 605 606 void 607 efx_rx_qenable( 608 __in efx_rxq_t *erp) 609 { 610 efx_nic_t *enp = erp->er_enp; 611 const efx_rx_ops_t *erxop = enp->en_erxop; 612 613 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 614 615 erxop->erxo_qenable(erp); 616 } 617 618 __checkReturn efx_rc_t 619 efx_rx_qcreate( 620 __in efx_nic_t *enp, 621 __in unsigned int index, 622 __in unsigned int label, 623 __in efx_rxq_type_t type, 624 __in efsys_mem_t *esmp, 625 __in size_t ndescs, 626 __in uint32_t id, 627 __in efx_evq_t *eep, 628 __deref_out efx_rxq_t **erpp) 629 { 630 const efx_rx_ops_t *erxop = enp->en_erxop; 631 efx_rxq_t *erp; 632 efx_rc_t rc; 633 634 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 635 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 636 637 /* Allocate an RXQ object */ 638 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 639 640 if (erp == NULL) { 641 rc = ENOMEM; 642 goto fail1; 643 } 644 645 erp->er_magic = EFX_RXQ_MAGIC; 646 erp->er_enp = enp; 647 erp->er_index = index; 648 erp->er_mask = ndescs - 1; 649 erp->er_esmp = esmp; 650 651 if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, ndescs, id, 652 eep, erp)) != 0) 653 goto fail2; 654 655 enp->en_rx_qcount++; 656 *erpp = erp; 657 658 return (0); 659 660 fail2: 661 EFSYS_PROBE(fail2); 662 663 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 664 fail1: 665 EFSYS_PROBE1(fail1, efx_rc_t, rc); 666 667 return (rc); 668 } 669 670 void 671 efx_rx_qdestroy( 672 __in efx_rxq_t *erp) 673 { 674 efx_nic_t *enp = erp->er_enp; 675 const efx_rx_ops_t *erxop = enp->en_erxop; 676 677 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 678 679 erxop->erxo_qdestroy(erp); 680 } 681 682 __checkReturn efx_rc_t 683 efx_pseudo_hdr_pkt_length_get( 684 __in efx_rxq_t *erp, 685 __in uint8_t *buffer, 686 __out uint16_t *lengthp) 687 { 688 efx_nic_t *enp = erp->er_enp; 689 const efx_rx_ops_t *erxop = enp->en_erxop; 690 691 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 692 693 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 694 } 695 696 #if EFSYS_OPT_RX_SCALE 697 __checkReturn uint32_t 698 efx_pseudo_hdr_hash_get( 699 __in efx_rxq_t *erp, 700 __in efx_rx_hash_alg_t func, 701 __in uint8_t *buffer) 702 { 703 efx_nic_t *enp = erp->er_enp; 704 const efx_rx_ops_t *erxop = enp->en_erxop; 705 706 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 707 708 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 709 return (erxop->erxo_prefix_hash(enp, func, buffer)); 710 } 711 #endif /* EFSYS_OPT_RX_SCALE */ 712 713 #if EFSYS_OPT_SIENA 714 715 static __checkReturn efx_rc_t 716 siena_rx_init( 717 __in efx_nic_t *enp) 718 { 719 efx_oword_t oword; 720 unsigned int index; 721 722 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 723 724 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 725 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 726 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 727 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 728 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 729 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 730 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 731 732 /* Zero the RSS table */ 733 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 734 index++) { 735 EFX_ZERO_OWORD(oword); 736 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 737 index, &oword, B_TRUE); 738 } 739 740 #if EFSYS_OPT_RX_SCALE 741 /* The RSS key and indirection table are writable. */ 742 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 743 744 /* Hardware can insert RX hash with/without RSS */ 745 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 746 #endif /* EFSYS_OPT_RX_SCALE */ 747 748 return (0); 749 } 750 751 #if EFSYS_OPT_RX_SCATTER 752 static __checkReturn efx_rc_t 753 siena_rx_scatter_enable( 754 __in efx_nic_t *enp, 755 __in unsigned int buf_size) 756 { 757 unsigned int nbuf32; 758 efx_oword_t oword; 759 efx_rc_t rc; 760 761 nbuf32 = buf_size / 32; 762 if ((nbuf32 == 0) || 763 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 764 ((buf_size % 32) != 0)) { 765 rc = EINVAL; 766 goto fail1; 767 } 768 769 if (enp->en_rx_qcount > 0) { 770 rc = EBUSY; 771 goto fail2; 772 } 773 774 /* Set scatter buffer size */ 775 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 776 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 777 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 778 779 /* Enable scatter for packets not matching a filter */ 780 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 781 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 782 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 783 784 return (0); 785 786 fail2: 787 EFSYS_PROBE(fail2); 788 fail1: 789 EFSYS_PROBE1(fail1, efx_rc_t, rc); 790 791 return (rc); 792 } 793 #endif /* EFSYS_OPT_RX_SCATTER */ 794 795 796 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 797 do { \ 798 efx_oword_t oword; \ 799 \ 800 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 801 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 802 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 803 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 804 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 805 (_insert) ? 1 : 0); \ 806 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 807 \ 808 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 809 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 810 &oword); \ 811 EFX_SET_OWORD_FIELD(oword, \ 812 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 813 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 814 &oword); \ 815 } \ 816 \ 817 _NOTE(CONSTANTCONDITION) \ 818 } while (B_FALSE) 819 820 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 821 do { \ 822 efx_oword_t oword; \ 823 \ 824 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 825 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 826 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 827 (_ip) ? 1 : 0); \ 828 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 829 (_tcp) ? 0 : 1); \ 830 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 831 (_insert) ? 1 : 0); \ 832 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 833 \ 834 _NOTE(CONSTANTCONDITION) \ 835 } while (B_FALSE) 836 837 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 838 do { \ 839 efx_oword_t oword; \ 840 \ 841 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 842 EFX_SET_OWORD_FIELD(oword, \ 843 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 844 EFX_SET_OWORD_FIELD(oword, \ 845 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 846 EFX_SET_OWORD_FIELD(oword, \ 847 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 848 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 849 \ 850 (_rc) = 0; \ 851 \ 852 _NOTE(CONSTANTCONDITION) \ 853 } while (B_FALSE) 854 855 856 #if EFSYS_OPT_RX_SCALE 857 858 static __checkReturn efx_rc_t 859 siena_rx_scale_mode_set( 860 __in efx_nic_t *enp, 861 __in uint32_t rss_context, 862 __in efx_rx_hash_alg_t alg, 863 __in efx_rx_hash_type_t type, 864 __in boolean_t insert) 865 { 866 efx_rc_t rc; 867 868 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 869 rc = EINVAL; 870 goto fail1; 871 } 872 873 switch (alg) { 874 case EFX_RX_HASHALG_LFSR: 875 EFX_RX_LFSR_HASH(enp, insert); 876 break; 877 878 case EFX_RX_HASHALG_TOEPLITZ: 879 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 880 type & EFX_RX_HASH_IPV4, 881 type & EFX_RX_HASH_TCPIPV4); 882 883 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 884 type & EFX_RX_HASH_IPV6, 885 type & EFX_RX_HASH_TCPIPV6, 886 rc); 887 if (rc != 0) 888 goto fail2; 889 890 break; 891 892 default: 893 rc = EINVAL; 894 goto fail3; 895 } 896 897 return (0); 898 899 fail3: 900 EFSYS_PROBE(fail3); 901 fail2: 902 EFSYS_PROBE(fail2); 903 fail1: 904 EFSYS_PROBE1(fail1, efx_rc_t, rc); 905 906 EFX_RX_LFSR_HASH(enp, B_FALSE); 907 908 return (rc); 909 } 910 #endif 911 912 #if EFSYS_OPT_RX_SCALE 913 static __checkReturn efx_rc_t 914 siena_rx_scale_key_set( 915 __in efx_nic_t *enp, 916 __in uint32_t rss_context, 917 __in_ecount(n) uint8_t *key, 918 __in size_t n) 919 { 920 efx_oword_t oword; 921 unsigned int byte; 922 unsigned int offset; 923 efx_rc_t rc; 924 925 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 926 rc = EINVAL; 927 goto fail1; 928 } 929 930 byte = 0; 931 932 /* Write Toeplitz IPv4 hash key */ 933 EFX_ZERO_OWORD(oword); 934 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 935 offset > 0 && byte < n; 936 --offset) 937 oword.eo_u8[offset - 1] = key[byte++]; 938 939 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 940 941 byte = 0; 942 943 /* Verify Toeplitz IPv4 hash key */ 944 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 945 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 946 offset > 0 && byte < n; 947 --offset) { 948 if (oword.eo_u8[offset - 1] != key[byte++]) { 949 rc = EFAULT; 950 goto fail2; 951 } 952 } 953 954 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 955 goto done; 956 957 byte = 0; 958 959 /* Write Toeplitz IPv6 hash key 3 */ 960 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 961 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 962 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 963 offset > 0 && byte < n; 964 --offset) 965 oword.eo_u8[offset - 1] = key[byte++]; 966 967 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 968 969 /* Write Toeplitz IPv6 hash key 2 */ 970 EFX_ZERO_OWORD(oword); 971 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 972 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 973 offset > 0 && byte < n; 974 --offset) 975 oword.eo_u8[offset - 1] = key[byte++]; 976 977 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 978 979 /* Write Toeplitz IPv6 hash key 1 */ 980 EFX_ZERO_OWORD(oword); 981 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 982 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 983 offset > 0 && byte < n; 984 --offset) 985 oword.eo_u8[offset - 1] = key[byte++]; 986 987 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 988 989 byte = 0; 990 991 /* Verify Toeplitz IPv6 hash key 3 */ 992 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 993 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 994 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 995 offset > 0 && byte < n; 996 --offset) { 997 if (oword.eo_u8[offset - 1] != key[byte++]) { 998 rc = EFAULT; 999 goto fail3; 1000 } 1001 } 1002 1003 /* Verify Toeplitz IPv6 hash key 2 */ 1004 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1005 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1006 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1007 offset > 0 && byte < n; 1008 --offset) { 1009 if (oword.eo_u8[offset - 1] != key[byte++]) { 1010 rc = EFAULT; 1011 goto fail4; 1012 } 1013 } 1014 1015 /* Verify Toeplitz IPv6 hash key 1 */ 1016 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1017 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1018 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1019 offset > 0 && byte < n; 1020 --offset) { 1021 if (oword.eo_u8[offset - 1] != key[byte++]) { 1022 rc = EFAULT; 1023 goto fail5; 1024 } 1025 } 1026 1027 done: 1028 return (0); 1029 1030 fail5: 1031 EFSYS_PROBE(fail5); 1032 fail4: 1033 EFSYS_PROBE(fail4); 1034 fail3: 1035 EFSYS_PROBE(fail3); 1036 fail2: 1037 EFSYS_PROBE(fail2); 1038 fail1: 1039 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1040 1041 return (rc); 1042 } 1043 #endif 1044 1045 #if EFSYS_OPT_RX_SCALE 1046 static __checkReturn efx_rc_t 1047 siena_rx_scale_tbl_set( 1048 __in efx_nic_t *enp, 1049 __in uint32_t rss_context, 1050 __in_ecount(n) unsigned int *table, 1051 __in size_t n) 1052 { 1053 efx_oword_t oword; 1054 int index; 1055 efx_rc_t rc; 1056 1057 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1058 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1059 1060 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1061 rc = EINVAL; 1062 goto fail1; 1063 } 1064 1065 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1066 rc = EINVAL; 1067 goto fail2; 1068 } 1069 1070 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1071 uint32_t byte; 1072 1073 /* Calculate the entry to place in the table */ 1074 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1075 1076 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1077 1078 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1079 1080 /* Write the table */ 1081 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1082 index, &oword, B_TRUE); 1083 } 1084 1085 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1086 uint32_t byte; 1087 1088 /* Determine if we're starting a new batch */ 1089 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1090 1091 /* Read the table */ 1092 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1093 index, &oword, B_TRUE); 1094 1095 /* Verify the entry */ 1096 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1097 rc = EFAULT; 1098 goto fail3; 1099 } 1100 } 1101 1102 return (0); 1103 1104 fail3: 1105 EFSYS_PROBE(fail3); 1106 fail2: 1107 EFSYS_PROBE(fail2); 1108 fail1: 1109 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1110 1111 return (rc); 1112 } 1113 #endif 1114 1115 /* 1116 * Falcon/Siena pseudo-header 1117 * -------------------------- 1118 * 1119 * Receive packets are prefixed by an optional 16 byte pseudo-header. 1120 * The pseudo-header is a byte array of one of the forms: 1121 * 1122 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 1123 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 1124 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 1125 * 1126 * where: 1127 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 1128 * LL.LL LFSR hash (16-bit big-endian) 1129 */ 1130 1131 #if EFSYS_OPT_RX_SCALE 1132 static __checkReturn uint32_t 1133 siena_rx_prefix_hash( 1134 __in efx_nic_t *enp, 1135 __in efx_rx_hash_alg_t func, 1136 __in uint8_t *buffer) 1137 { 1138 _NOTE(ARGUNUSED(enp)) 1139 1140 switch (func) { 1141 case EFX_RX_HASHALG_TOEPLITZ: 1142 return ((buffer[12] << 24) | 1143 (buffer[13] << 16) | 1144 (buffer[14] << 8) | 1145 buffer[15]); 1146 1147 case EFX_RX_HASHALG_LFSR: 1148 return ((buffer[14] << 8) | buffer[15]); 1149 1150 default: 1151 EFSYS_ASSERT(0); 1152 return (0); 1153 } 1154 } 1155 #endif /* EFSYS_OPT_RX_SCALE */ 1156 1157 static __checkReturn efx_rc_t 1158 siena_rx_prefix_pktlen( 1159 __in efx_nic_t *enp, 1160 __in uint8_t *buffer, 1161 __out uint16_t *lengthp) 1162 { 1163 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1164 1165 /* Not supported by Falcon/Siena hardware */ 1166 EFSYS_ASSERT(0); 1167 return (ENOTSUP); 1168 } 1169 1170 1171 static void 1172 siena_rx_qpost( 1173 __in efx_rxq_t *erp, 1174 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 1175 __in size_t size, 1176 __in unsigned int ndescs, 1177 __in unsigned int completed, 1178 __in unsigned int added) 1179 { 1180 efx_qword_t qword; 1181 unsigned int i; 1182 unsigned int offset; 1183 unsigned int id; 1184 1185 /* The client driver must not overfill the queue */ 1186 EFSYS_ASSERT3U(added - completed + ndescs, <=, 1187 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1188 1189 id = added & (erp->er_mask); 1190 for (i = 0; i < ndescs; i++) { 1191 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1192 unsigned int, id, efsys_dma_addr_t, addrp[i], 1193 size_t, size); 1194 1195 EFX_POPULATE_QWORD_3(qword, 1196 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1197 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1198 (uint32_t)(addrp[i] & 0xffffffff), 1199 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1200 (uint32_t)(addrp[i] >> 32)); 1201 1202 offset = id * sizeof (efx_qword_t); 1203 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1204 1205 id = (id + 1) & (erp->er_mask); 1206 } 1207 } 1208 1209 static void 1210 siena_rx_qpush( 1211 __in efx_rxq_t *erp, 1212 __in unsigned int added, 1213 __inout unsigned int *pushedp) 1214 { 1215 efx_nic_t *enp = erp->er_enp; 1216 unsigned int pushed = *pushedp; 1217 uint32_t wptr; 1218 efx_oword_t oword; 1219 efx_dword_t dword; 1220 1221 /* All descriptors are pushed */ 1222 *pushedp = added; 1223 1224 /* Push the populated descriptors out */ 1225 wptr = added & erp->er_mask; 1226 1227 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1228 1229 /* Only write the third DWORD */ 1230 EFX_POPULATE_DWORD_1(dword, 1231 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1232 1233 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1234 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1235 wptr, pushed & erp->er_mask); 1236 EFSYS_PIO_WRITE_BARRIER(); 1237 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1238 erp->er_index, &dword, B_FALSE); 1239 } 1240 1241 #if EFSYS_OPT_RX_PACKED_STREAM 1242 static void 1243 siena_rx_qpush_ps_credits( 1244 __in efx_rxq_t *erp) 1245 { 1246 /* Not supported by Siena hardware */ 1247 EFSYS_ASSERT(0); 1248 } 1249 1250 static uint8_t * 1251 siena_rx_qps_packet_info( 1252 __in efx_rxq_t *erp, 1253 __in uint8_t *buffer, 1254 __in uint32_t buffer_length, 1255 __in uint32_t current_offset, 1256 __out uint16_t *lengthp, 1257 __out uint32_t *next_offsetp, 1258 __out uint32_t *timestamp) 1259 { 1260 /* Not supported by Siena hardware */ 1261 EFSYS_ASSERT(0); 1262 1263 return (NULL); 1264 } 1265 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1266 1267 static __checkReturn efx_rc_t 1268 siena_rx_qflush( 1269 __in efx_rxq_t *erp) 1270 { 1271 efx_nic_t *enp = erp->er_enp; 1272 efx_oword_t oword; 1273 uint32_t label; 1274 1275 label = erp->er_index; 1276 1277 /* Flush the queue */ 1278 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1279 FRF_AZ_RX_FLUSH_DESCQ, label); 1280 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1281 1282 return (0); 1283 } 1284 1285 static void 1286 siena_rx_qenable( 1287 __in efx_rxq_t *erp) 1288 { 1289 efx_nic_t *enp = erp->er_enp; 1290 efx_oword_t oword; 1291 1292 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1293 1294 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1295 erp->er_index, &oword, B_TRUE); 1296 1297 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1298 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1299 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1300 1301 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1302 erp->er_index, &oword, B_TRUE); 1303 } 1304 1305 static __checkReturn efx_rc_t 1306 siena_rx_qcreate( 1307 __in efx_nic_t *enp, 1308 __in unsigned int index, 1309 __in unsigned int label, 1310 __in efx_rxq_type_t type, 1311 __in efsys_mem_t *esmp, 1312 __in size_t ndescs, 1313 __in uint32_t id, 1314 __in efx_evq_t *eep, 1315 __in efx_rxq_t *erp) 1316 { 1317 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1318 efx_oword_t oword; 1319 uint32_t size; 1320 boolean_t jumbo; 1321 efx_rc_t rc; 1322 1323 _NOTE(ARGUNUSED(esmp)) 1324 1325 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1326 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1327 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1328 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1329 1330 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 1331 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 1332 1333 if (!ISP2(ndescs) || 1334 (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) { 1335 rc = EINVAL; 1336 goto fail1; 1337 } 1338 if (index >= encp->enc_rxq_limit) { 1339 rc = EINVAL; 1340 goto fail2; 1341 } 1342 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1343 size++) 1344 if ((1 << size) == (int)(ndescs / EFX_RXQ_MINNDESCS)) 1345 break; 1346 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1347 rc = EINVAL; 1348 goto fail3; 1349 } 1350 1351 switch (type) { 1352 case EFX_RXQ_TYPE_DEFAULT: 1353 jumbo = B_FALSE; 1354 break; 1355 1356 #if EFSYS_OPT_RX_SCATTER 1357 case EFX_RXQ_TYPE_SCATTER: 1358 if (enp->en_family < EFX_FAMILY_SIENA) { 1359 rc = EINVAL; 1360 goto fail4; 1361 } 1362 jumbo = B_TRUE; 1363 break; 1364 #endif /* EFSYS_OPT_RX_SCATTER */ 1365 1366 default: 1367 rc = EINVAL; 1368 goto fail4; 1369 } 1370 1371 /* Set up the new descriptor queue */ 1372 EFX_POPULATE_OWORD_7(oword, 1373 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1374 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1375 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1376 FRF_AZ_RX_DESCQ_LABEL, label, 1377 FRF_AZ_RX_DESCQ_SIZE, size, 1378 FRF_AZ_RX_DESCQ_TYPE, 0, 1379 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1380 1381 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1382 erp->er_index, &oword, B_TRUE); 1383 1384 return (0); 1385 1386 fail4: 1387 EFSYS_PROBE(fail4); 1388 fail3: 1389 EFSYS_PROBE(fail3); 1390 fail2: 1391 EFSYS_PROBE(fail2); 1392 fail1: 1393 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1394 1395 return (rc); 1396 } 1397 1398 static void 1399 siena_rx_qdestroy( 1400 __in efx_rxq_t *erp) 1401 { 1402 efx_nic_t *enp = erp->er_enp; 1403 efx_oword_t oword; 1404 1405 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1406 --enp->en_rx_qcount; 1407 1408 /* Purge descriptor queue */ 1409 EFX_ZERO_OWORD(oword); 1410 1411 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1412 erp->er_index, &oword, B_TRUE); 1413 1414 /* Free the RXQ object */ 1415 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1416 } 1417 1418 static void 1419 siena_rx_fini( 1420 __in efx_nic_t *enp) 1421 { 1422 _NOTE(ARGUNUSED(enp)) 1423 } 1424 1425 #endif /* EFSYS_OPT_SIENA */ 1426