1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33 #include <sys/cdefs.h> 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 #if EFSYS_OPT_SIENA 38 39 static __checkReturn efx_rc_t 40 siena_rx_init( 41 __in efx_nic_t *enp); 42 43 static void 44 siena_rx_fini( 45 __in efx_nic_t *enp); 46 47 #if EFSYS_OPT_RX_SCATTER 48 static __checkReturn efx_rc_t 49 siena_rx_scatter_enable( 50 __in efx_nic_t *enp, 51 __in unsigned int buf_size); 52 #endif /* EFSYS_OPT_RX_SCATTER */ 53 54 #if EFSYS_OPT_RX_SCALE 55 static __checkReturn efx_rc_t 56 siena_rx_scale_mode_set( 57 __in efx_nic_t *enp, 58 __in uint32_t rss_context, 59 __in efx_rx_hash_alg_t alg, 60 __in efx_rx_hash_type_t type, 61 __in boolean_t insert); 62 63 static __checkReturn efx_rc_t 64 siena_rx_scale_key_set( 65 __in efx_nic_t *enp, 66 __in uint32_t rss_context, 67 __in_ecount(n) uint8_t *key, 68 __in size_t n); 69 70 static __checkReturn efx_rc_t 71 siena_rx_scale_tbl_set( 72 __in efx_nic_t *enp, 73 __in uint32_t rss_context, 74 __in_ecount(n) unsigned int *table, 75 __in size_t n); 76 77 static __checkReturn uint32_t 78 siena_rx_prefix_hash( 79 __in efx_nic_t *enp, 80 __in efx_rx_hash_alg_t func, 81 __in uint8_t *buffer); 82 83 #endif /* EFSYS_OPT_RX_SCALE */ 84 85 static __checkReturn efx_rc_t 86 siena_rx_prefix_pktlen( 87 __in efx_nic_t *enp, 88 __in uint8_t *buffer, 89 __out uint16_t *lengthp); 90 91 static void 92 siena_rx_qpost( 93 __in efx_rxq_t *erp, 94 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 95 __in size_t size, 96 __in unsigned int ndescs, 97 __in unsigned int completed, 98 __in unsigned int added); 99 100 static void 101 siena_rx_qpush( 102 __in efx_rxq_t *erp, 103 __in unsigned int added, 104 __inout unsigned int *pushedp); 105 106 #if EFSYS_OPT_RX_PACKED_STREAM 107 static void 108 siena_rx_qpush_ps_credits( 109 __in efx_rxq_t *erp); 110 111 static __checkReturn uint8_t * 112 siena_rx_qps_packet_info( 113 __in efx_rxq_t *erp, 114 __in uint8_t *buffer, 115 __in uint32_t buffer_length, 116 __in uint32_t current_offset, 117 __out uint16_t *lengthp, 118 __out uint32_t *next_offsetp, 119 __out uint32_t *timestamp); 120 #endif 121 122 static __checkReturn efx_rc_t 123 siena_rx_qflush( 124 __in efx_rxq_t *erp); 125 126 static void 127 siena_rx_qenable( 128 __in efx_rxq_t *erp); 129 130 static __checkReturn efx_rc_t 131 siena_rx_qcreate( 132 __in efx_nic_t *enp, 133 __in unsigned int index, 134 __in unsigned int label, 135 __in efx_rxq_type_t type, 136 __in_opt const efx_rxq_type_data_t *type_data, 137 __in efsys_mem_t *esmp, 138 __in size_t ndescs, 139 __in uint32_t id, 140 __in unsigned int flags, 141 __in efx_evq_t *eep, 142 __in efx_rxq_t *erp); 143 144 static void 145 siena_rx_qdestroy( 146 __in efx_rxq_t *erp); 147 148 #endif /* EFSYS_OPT_SIENA */ 149 150 #if EFSYS_OPT_SIENA 151 static const efx_rx_ops_t __efx_rx_siena_ops = { 152 siena_rx_init, /* erxo_init */ 153 siena_rx_fini, /* erxo_fini */ 154 #if EFSYS_OPT_RX_SCATTER 155 siena_rx_scatter_enable, /* erxo_scatter_enable */ 156 #endif 157 #if EFSYS_OPT_RX_SCALE 158 NULL, /* erxo_scale_context_alloc */ 159 NULL, /* erxo_scale_context_free */ 160 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 161 siena_rx_scale_key_set, /* erxo_scale_key_set */ 162 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 163 siena_rx_prefix_hash, /* erxo_prefix_hash */ 164 #endif 165 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 166 siena_rx_qpost, /* erxo_qpost */ 167 siena_rx_qpush, /* erxo_qpush */ 168 #if EFSYS_OPT_RX_PACKED_STREAM 169 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 170 siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 171 #endif 172 siena_rx_qflush, /* erxo_qflush */ 173 siena_rx_qenable, /* erxo_qenable */ 174 siena_rx_qcreate, /* erxo_qcreate */ 175 siena_rx_qdestroy, /* erxo_qdestroy */ 176 }; 177 #endif /* EFSYS_OPT_SIENA */ 178 179 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 180 static const efx_rx_ops_t __efx_rx_ef10_ops = { 181 ef10_rx_init, /* erxo_init */ 182 ef10_rx_fini, /* erxo_fini */ 183 #if EFSYS_OPT_RX_SCATTER 184 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 185 #endif 186 #if EFSYS_OPT_RX_SCALE 187 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 188 ef10_rx_scale_context_free, /* erxo_scale_context_free */ 189 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 190 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 191 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 192 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 193 #endif 194 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 195 ef10_rx_qpost, /* erxo_qpost */ 196 ef10_rx_qpush, /* erxo_qpush */ 197 #if EFSYS_OPT_RX_PACKED_STREAM 198 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 199 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 200 #endif 201 ef10_rx_qflush, /* erxo_qflush */ 202 ef10_rx_qenable, /* erxo_qenable */ 203 ef10_rx_qcreate, /* erxo_qcreate */ 204 ef10_rx_qdestroy, /* erxo_qdestroy */ 205 }; 206 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ 207 208 __checkReturn efx_rc_t 209 efx_rx_init( 210 __inout efx_nic_t *enp) 211 { 212 const efx_rx_ops_t *erxop; 213 efx_rc_t rc; 214 215 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 216 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 217 218 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 219 rc = EINVAL; 220 goto fail1; 221 } 222 223 if (enp->en_mod_flags & EFX_MOD_RX) { 224 rc = EINVAL; 225 goto fail2; 226 } 227 228 switch (enp->en_family) { 229 #if EFSYS_OPT_SIENA 230 case EFX_FAMILY_SIENA: 231 erxop = &__efx_rx_siena_ops; 232 break; 233 #endif /* EFSYS_OPT_SIENA */ 234 235 #if EFSYS_OPT_HUNTINGTON 236 case EFX_FAMILY_HUNTINGTON: 237 erxop = &__efx_rx_ef10_ops; 238 break; 239 #endif /* EFSYS_OPT_HUNTINGTON */ 240 241 #if EFSYS_OPT_MEDFORD 242 case EFX_FAMILY_MEDFORD: 243 erxop = &__efx_rx_ef10_ops; 244 break; 245 #endif /* EFSYS_OPT_MEDFORD */ 246 247 #if EFSYS_OPT_MEDFORD2 248 case EFX_FAMILY_MEDFORD2: 249 erxop = &__efx_rx_ef10_ops; 250 break; 251 #endif /* EFSYS_OPT_MEDFORD2 */ 252 253 default: 254 EFSYS_ASSERT(0); 255 rc = ENOTSUP; 256 goto fail3; 257 } 258 259 if ((rc = erxop->erxo_init(enp)) != 0) 260 goto fail4; 261 262 enp->en_erxop = erxop; 263 enp->en_mod_flags |= EFX_MOD_RX; 264 return (0); 265 266 fail4: 267 EFSYS_PROBE(fail4); 268 fail3: 269 EFSYS_PROBE(fail3); 270 fail2: 271 EFSYS_PROBE(fail2); 272 fail1: 273 EFSYS_PROBE1(fail1, efx_rc_t, rc); 274 275 enp->en_erxop = NULL; 276 enp->en_mod_flags &= ~EFX_MOD_RX; 277 return (rc); 278 } 279 280 void 281 efx_rx_fini( 282 __in efx_nic_t *enp) 283 { 284 const efx_rx_ops_t *erxop = enp->en_erxop; 285 286 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 287 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 288 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 289 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 290 291 erxop->erxo_fini(enp); 292 293 enp->en_erxop = NULL; 294 enp->en_mod_flags &= ~EFX_MOD_RX; 295 } 296 297 #if EFSYS_OPT_RX_SCATTER 298 __checkReturn efx_rc_t 299 efx_rx_scatter_enable( 300 __in efx_nic_t *enp, 301 __in unsigned int buf_size) 302 { 303 const efx_rx_ops_t *erxop = enp->en_erxop; 304 efx_rc_t rc; 305 306 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 307 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 308 309 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 310 goto fail1; 311 312 return (0); 313 314 fail1: 315 EFSYS_PROBE1(fail1, efx_rc_t, rc); 316 return (rc); 317 } 318 #endif /* EFSYS_OPT_RX_SCATTER */ 319 320 #if EFSYS_OPT_RX_SCALE 321 __checkReturn efx_rc_t 322 efx_rx_scale_hash_flags_get( 323 __in efx_nic_t *enp, 324 __in efx_rx_hash_alg_t hash_alg, 325 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp, 326 __in unsigned int max_nflags, 327 __out unsigned int *nflagsp) 328 { 329 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 330 unsigned int nflags = 0; 331 efx_rc_t rc; 332 333 if (flagsp == NULL || nflagsp == NULL) { 334 rc = EINVAL; 335 goto fail1; 336 } 337 338 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) { 339 nflags = 0; 340 goto done; 341 } 342 343 /* Helper to add flags word to flags array without buffer overflow */ 344 #define INSERT_FLAGS(_flags) \ 345 do { \ 346 if (nflags >= max_nflags) { \ 347 rc = E2BIG; \ 348 goto fail2; \ 349 } \ 350 *(flagsp + nflags) = (_flags); \ 351 nflags++; \ 352 \ 353 _NOTE(CONSTANTCONDITION) \ 354 } while (B_FALSE) 355 356 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) { 357 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE)); 358 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE)); 359 } 360 361 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) && 362 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) { 363 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST)); 364 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC)); 365 366 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST)); 367 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC)); 368 369 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE)); 370 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST)); 371 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC)); 372 373 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE)); 374 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST)); 375 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC)); 376 } 377 378 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE)); 379 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE)); 380 381 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE)); 382 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE)); 383 384 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) { 385 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST)); 386 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC)); 387 388 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST)); 389 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC)); 390 391 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE)); 392 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST)); 393 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC)); 394 395 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE)); 396 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST)); 397 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC)); 398 399 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST)); 400 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC)); 401 402 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST)); 403 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC)); 404 } 405 406 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE)); 407 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE)); 408 409 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE)); 410 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE)); 411 412 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE)); 413 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE)); 414 415 #undef INSERT_FLAGS 416 417 done: 418 *nflagsp = nflags; 419 return (0); 420 421 fail2: 422 EFSYS_PROBE(fail2); 423 fail1: 424 EFSYS_PROBE1(fail1, efx_rc_t, rc); 425 426 return (rc); 427 } 428 429 __checkReturn efx_rc_t 430 efx_rx_hash_default_support_get( 431 __in efx_nic_t *enp, 432 __out efx_rx_hash_support_t *supportp) 433 { 434 efx_rc_t rc; 435 436 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 437 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 438 439 if (supportp == NULL) { 440 rc = EINVAL; 441 goto fail1; 442 } 443 444 /* 445 * Report the hashing support the client gets by default if it 446 * does not allocate an RSS context itself. 447 */ 448 *supportp = enp->en_hash_support; 449 450 return (0); 451 452 fail1: 453 EFSYS_PROBE1(fail1, efx_rc_t, rc); 454 455 return (rc); 456 } 457 458 __checkReturn efx_rc_t 459 efx_rx_scale_default_support_get( 460 __in efx_nic_t *enp, 461 __out efx_rx_scale_context_type_t *typep) 462 { 463 efx_rc_t rc; 464 465 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 466 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 467 468 if (typep == NULL) { 469 rc = EINVAL; 470 goto fail1; 471 } 472 473 /* 474 * Report the RSS support the client gets by default if it 475 * does not allocate an RSS context itself. 476 */ 477 *typep = enp->en_rss_context_type; 478 479 return (0); 480 481 fail1: 482 EFSYS_PROBE1(fail1, efx_rc_t, rc); 483 484 return (rc); 485 } 486 #endif /* EFSYS_OPT_RX_SCALE */ 487 488 #if EFSYS_OPT_RX_SCALE 489 __checkReturn efx_rc_t 490 efx_rx_scale_context_alloc( 491 __in efx_nic_t *enp, 492 __in efx_rx_scale_context_type_t type, 493 __in uint32_t num_queues, 494 __out uint32_t *rss_contextp) 495 { 496 const efx_rx_ops_t *erxop = enp->en_erxop; 497 efx_rc_t rc; 498 499 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 500 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 501 502 if (erxop->erxo_scale_context_alloc == NULL) { 503 rc = ENOTSUP; 504 goto fail1; 505 } 506 if ((rc = erxop->erxo_scale_context_alloc(enp, type, 507 num_queues, rss_contextp)) != 0) { 508 goto fail2; 509 } 510 511 return (0); 512 513 fail2: 514 EFSYS_PROBE(fail2); 515 fail1: 516 EFSYS_PROBE1(fail1, efx_rc_t, rc); 517 return (rc); 518 } 519 #endif /* EFSYS_OPT_RX_SCALE */ 520 521 #if EFSYS_OPT_RX_SCALE 522 __checkReturn efx_rc_t 523 efx_rx_scale_context_free( 524 __in efx_nic_t *enp, 525 __in uint32_t rss_context) 526 { 527 const efx_rx_ops_t *erxop = enp->en_erxop; 528 efx_rc_t rc; 529 530 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 531 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 532 533 if (erxop->erxo_scale_context_free == NULL) { 534 rc = ENOTSUP; 535 goto fail1; 536 } 537 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 538 goto fail2; 539 540 return (0); 541 542 fail2: 543 EFSYS_PROBE(fail2); 544 fail1: 545 EFSYS_PROBE1(fail1, efx_rc_t, rc); 546 return (rc); 547 } 548 #endif /* EFSYS_OPT_RX_SCALE */ 549 550 #if EFSYS_OPT_RX_SCALE 551 __checkReturn efx_rc_t 552 efx_rx_scale_mode_set( 553 __in efx_nic_t *enp, 554 __in uint32_t rss_context, 555 __in efx_rx_hash_alg_t alg, 556 __in efx_rx_hash_type_t type, 557 __in boolean_t insert) 558 { 559 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 560 const efx_rx_ops_t *erxop = enp->en_erxop; 561 efx_rx_hash_type_t type_check; 562 unsigned int i; 563 efx_rc_t rc; 564 565 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 566 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 567 568 /* 569 * Legacy flags and modern bits cannot be 570 * used at the same time in the hash type. 571 */ 572 if ((type & EFX_RX_HASH_LEGACY_MASK) && 573 (type & ~EFX_RX_HASH_LEGACY_MASK)) { 574 rc = EINVAL; 575 goto fail1; 576 } 577 578 /* 579 * If RSS hash type is represented by additional bits 580 * in the value, the latter need to be verified since 581 * not all bit combinations are valid RSS modes. Also, 582 * depending on the firmware, some valid combinations 583 * may be unsupported. Discern additional bits in the 584 * type value and try to recognise valid combinations. 585 * If some bits remain unrecognised, report the error. 586 */ 587 type_check = type & ~EFX_RX_HASH_LEGACY_MASK; 588 if (type_check != 0) { 589 unsigned int type_flags[EFX_RX_HASH_NFLAGS]; 590 unsigned int type_nflags; 591 592 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags, 593 EFX_ARRAY_SIZE(type_flags), &type_nflags); 594 if (rc != 0) 595 goto fail2; 596 597 for (i = 0; i < type_nflags; ++i) { 598 if ((type_check & type_flags[i]) == type_flags[i]) 599 type_check &= ~(type_flags[i]); 600 } 601 602 if (type_check != 0) { 603 rc = EINVAL; 604 goto fail3; 605 } 606 } 607 608 /* 609 * Translate EFX_RX_HASH() flags to their legacy counterparts 610 * provided that the FW claims no support for additional modes. 611 */ 612 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) { 613 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | 614 EFX_RX_HASH(IPV4_TCP, 2TUPLE); 615 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | 616 EFX_RX_HASH(IPV6_TCP, 2TUPLE); 617 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE); 618 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE); 619 620 if ((type & t_ipv4) == t_ipv4) 621 type |= EFX_RX_HASH_IPV4; 622 if ((type & t_ipv6) == t_ipv6) 623 type |= EFX_RX_HASH_IPV6; 624 625 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) { 626 if ((type & t_ipv4_tcp) == t_ipv4_tcp) 627 type |= EFX_RX_HASH_TCPIPV4; 628 if ((type & t_ipv6_tcp) == t_ipv6_tcp) 629 type |= EFX_RX_HASH_TCPIPV6; 630 } 631 632 type &= EFX_RX_HASH_LEGACY_MASK; 633 } 634 635 if (erxop->erxo_scale_mode_set != NULL) { 636 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 637 type, insert)) != 0) 638 goto fail4; 639 } 640 641 return (0); 642 643 fail4: 644 EFSYS_PROBE(fail4); 645 fail3: 646 EFSYS_PROBE(fail3); 647 fail2: 648 EFSYS_PROBE(fail2); 649 fail1: 650 EFSYS_PROBE1(fail1, efx_rc_t, rc); 651 return (rc); 652 } 653 #endif /* EFSYS_OPT_RX_SCALE */ 654 655 #if EFSYS_OPT_RX_SCALE 656 __checkReturn efx_rc_t 657 efx_rx_scale_key_set( 658 __in efx_nic_t *enp, 659 __in uint32_t rss_context, 660 __in_ecount(n) uint8_t *key, 661 __in size_t n) 662 { 663 const efx_rx_ops_t *erxop = enp->en_erxop; 664 efx_rc_t rc; 665 666 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 667 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 668 669 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 670 goto fail1; 671 672 return (0); 673 674 fail1: 675 EFSYS_PROBE1(fail1, efx_rc_t, rc); 676 677 return (rc); 678 } 679 #endif /* EFSYS_OPT_RX_SCALE */ 680 681 #if EFSYS_OPT_RX_SCALE 682 __checkReturn efx_rc_t 683 efx_rx_scale_tbl_set( 684 __in efx_nic_t *enp, 685 __in uint32_t rss_context, 686 __in_ecount(n) unsigned int *table, 687 __in size_t n) 688 { 689 const efx_rx_ops_t *erxop = enp->en_erxop; 690 efx_rc_t rc; 691 692 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 693 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 694 695 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) 696 goto fail1; 697 698 return (0); 699 700 fail1: 701 EFSYS_PROBE1(fail1, efx_rc_t, rc); 702 703 return (rc); 704 } 705 #endif /* EFSYS_OPT_RX_SCALE */ 706 707 void 708 efx_rx_qpost( 709 __in efx_rxq_t *erp, 710 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 711 __in size_t size, 712 __in unsigned int ndescs, 713 __in unsigned int completed, 714 __in unsigned int added) 715 { 716 efx_nic_t *enp = erp->er_enp; 717 const efx_rx_ops_t *erxop = enp->en_erxop; 718 719 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 720 721 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added); 722 } 723 724 #if EFSYS_OPT_RX_PACKED_STREAM 725 726 void 727 efx_rx_qpush_ps_credits( 728 __in efx_rxq_t *erp) 729 { 730 efx_nic_t *enp = erp->er_enp; 731 const efx_rx_ops_t *erxop = enp->en_erxop; 732 733 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 734 735 erxop->erxo_qpush_ps_credits(erp); 736 } 737 738 __checkReturn uint8_t * 739 efx_rx_qps_packet_info( 740 __in efx_rxq_t *erp, 741 __in uint8_t *buffer, 742 __in uint32_t buffer_length, 743 __in uint32_t current_offset, 744 __out uint16_t *lengthp, 745 __out uint32_t *next_offsetp, 746 __out uint32_t *timestamp) 747 { 748 efx_nic_t *enp = erp->er_enp; 749 const efx_rx_ops_t *erxop = enp->en_erxop; 750 751 return (erxop->erxo_qps_packet_info(erp, buffer, 752 buffer_length, current_offset, lengthp, 753 next_offsetp, timestamp)); 754 } 755 756 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 757 758 void 759 efx_rx_qpush( 760 __in efx_rxq_t *erp, 761 __in unsigned int added, 762 __inout unsigned int *pushedp) 763 { 764 efx_nic_t *enp = erp->er_enp; 765 const efx_rx_ops_t *erxop = enp->en_erxop; 766 767 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 768 769 erxop->erxo_qpush(erp, added, pushedp); 770 } 771 772 __checkReturn efx_rc_t 773 efx_rx_qflush( 774 __in efx_rxq_t *erp) 775 { 776 efx_nic_t *enp = erp->er_enp; 777 const efx_rx_ops_t *erxop = enp->en_erxop; 778 efx_rc_t rc; 779 780 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 781 782 if ((rc = erxop->erxo_qflush(erp)) != 0) 783 goto fail1; 784 785 return (0); 786 787 fail1: 788 EFSYS_PROBE1(fail1, efx_rc_t, rc); 789 790 return (rc); 791 } 792 793 void 794 efx_rx_qenable( 795 __in efx_rxq_t *erp) 796 { 797 efx_nic_t *enp = erp->er_enp; 798 const efx_rx_ops_t *erxop = enp->en_erxop; 799 800 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 801 802 erxop->erxo_qenable(erp); 803 } 804 805 static __checkReturn efx_rc_t 806 efx_rx_qcreate_internal( 807 __in efx_nic_t *enp, 808 __in unsigned int index, 809 __in unsigned int label, 810 __in efx_rxq_type_t type, 811 __in_opt const efx_rxq_type_data_t *type_data, 812 __in efsys_mem_t *esmp, 813 __in size_t ndescs, 814 __in uint32_t id, 815 __in unsigned int flags, 816 __in efx_evq_t *eep, 817 __deref_out efx_rxq_t **erpp) 818 { 819 const efx_rx_ops_t *erxop = enp->en_erxop; 820 efx_rxq_t *erp; 821 efx_rc_t rc; 822 823 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 824 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 825 826 /* Allocate an RXQ object */ 827 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 828 829 if (erp == NULL) { 830 rc = ENOMEM; 831 goto fail1; 832 } 833 834 erp->er_magic = EFX_RXQ_MAGIC; 835 erp->er_enp = enp; 836 erp->er_index = index; 837 erp->er_mask = ndescs - 1; 838 erp->er_esmp = esmp; 839 840 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp, 841 ndescs, id, flags, eep, erp)) != 0) 842 goto fail2; 843 844 enp->en_rx_qcount++; 845 *erpp = erp; 846 847 return (0); 848 849 fail2: 850 EFSYS_PROBE(fail2); 851 852 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 853 fail1: 854 EFSYS_PROBE1(fail1, efx_rc_t, rc); 855 856 return (rc); 857 } 858 859 __checkReturn efx_rc_t 860 efx_rx_qcreate( 861 __in efx_nic_t *enp, 862 __in unsigned int index, 863 __in unsigned int label, 864 __in efx_rxq_type_t type, 865 __in efsys_mem_t *esmp, 866 __in size_t ndescs, 867 __in uint32_t id, 868 __in unsigned int flags, 869 __in efx_evq_t *eep, 870 __deref_out efx_rxq_t **erpp) 871 { 872 return efx_rx_qcreate_internal(enp, index, label, type, NULL, 873 esmp, ndescs, id, flags, eep, erpp); 874 } 875 876 #if EFSYS_OPT_RX_PACKED_STREAM 877 878 __checkReturn efx_rc_t 879 efx_rx_qcreate_packed_stream( 880 __in efx_nic_t *enp, 881 __in unsigned int index, 882 __in unsigned int label, 883 __in uint32_t ps_buf_size, 884 __in efsys_mem_t *esmp, 885 __in size_t ndescs, 886 __in efx_evq_t *eep, 887 __deref_out efx_rxq_t **erpp) 888 { 889 efx_rxq_type_data_t type_data; 890 891 memset(&type_data, 0, sizeof (type_data)); 892 893 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size; 894 895 return efx_rx_qcreate_internal(enp, index, label, 896 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs, 897 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp); 898 } 899 900 #endif 901 902 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 903 904 __checkReturn efx_rc_t 905 efx_rx_qcreate_es_super_buffer( 906 __in efx_nic_t *enp, 907 __in unsigned int index, 908 __in unsigned int label, 909 __in uint32_t n_bufs_per_desc, 910 __in uint32_t max_dma_len, 911 __in uint32_t buf_stride, 912 __in uint32_t hol_block_timeout, 913 __in efsys_mem_t *esmp, 914 __in size_t ndescs, 915 __in unsigned int flags, 916 __in efx_evq_t *eep, 917 __deref_out efx_rxq_t **erpp) 918 { 919 efx_rc_t rc; 920 efx_rxq_type_data_t type_data; 921 922 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 923 rc = EINVAL; 924 goto fail1; 925 } 926 927 memset(&type_data, 0, sizeof (type_data)); 928 929 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc; 930 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len; 931 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride; 932 type_data.ertd_es_super_buffer.eessb_hol_block_timeout = 933 hol_block_timeout; 934 935 rc = efx_rx_qcreate_internal(enp, index, label, 936 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs, 937 0 /* id unused on EF10 */, flags, eep, erpp); 938 if (rc != 0) 939 goto fail2; 940 941 return (0); 942 943 fail2: 944 EFSYS_PROBE(fail2); 945 fail1: 946 EFSYS_PROBE1(fail1, efx_rc_t, rc); 947 948 return (rc); 949 } 950 951 #endif 952 953 void 954 efx_rx_qdestroy( 955 __in efx_rxq_t *erp) 956 { 957 efx_nic_t *enp = erp->er_enp; 958 const efx_rx_ops_t *erxop = enp->en_erxop; 959 960 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 961 962 erxop->erxo_qdestroy(erp); 963 } 964 965 __checkReturn efx_rc_t 966 efx_pseudo_hdr_pkt_length_get( 967 __in efx_rxq_t *erp, 968 __in uint8_t *buffer, 969 __out uint16_t *lengthp) 970 { 971 efx_nic_t *enp = erp->er_enp; 972 const efx_rx_ops_t *erxop = enp->en_erxop; 973 974 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 975 976 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 977 } 978 979 #if EFSYS_OPT_RX_SCALE 980 __checkReturn uint32_t 981 efx_pseudo_hdr_hash_get( 982 __in efx_rxq_t *erp, 983 __in efx_rx_hash_alg_t func, 984 __in uint8_t *buffer) 985 { 986 efx_nic_t *enp = erp->er_enp; 987 const efx_rx_ops_t *erxop = enp->en_erxop; 988 989 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 990 991 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 992 return (erxop->erxo_prefix_hash(enp, func, buffer)); 993 } 994 #endif /* EFSYS_OPT_RX_SCALE */ 995 996 #if EFSYS_OPT_SIENA 997 998 static __checkReturn efx_rc_t 999 siena_rx_init( 1000 __in efx_nic_t *enp) 1001 { 1002 efx_oword_t oword; 1003 unsigned int index; 1004 1005 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1006 1007 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 1008 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 1009 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 1010 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 1011 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 1012 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 1013 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1014 1015 /* Zero the RSS table */ 1016 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 1017 index++) { 1018 EFX_ZERO_OWORD(oword); 1019 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1020 index, &oword, B_TRUE); 1021 } 1022 1023 #if EFSYS_OPT_RX_SCALE 1024 /* The RSS key and indirection table are writable. */ 1025 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 1026 1027 /* Hardware can insert RX hash with/without RSS */ 1028 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 1029 #endif /* EFSYS_OPT_RX_SCALE */ 1030 1031 return (0); 1032 } 1033 1034 #if EFSYS_OPT_RX_SCATTER 1035 static __checkReturn efx_rc_t 1036 siena_rx_scatter_enable( 1037 __in efx_nic_t *enp, 1038 __in unsigned int buf_size) 1039 { 1040 unsigned int nbuf32; 1041 efx_oword_t oword; 1042 efx_rc_t rc; 1043 1044 nbuf32 = buf_size / 32; 1045 IF ((NBUF32 == 0) || 1046 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 1047 ((buf_size % 32) != 0)) { 1048 rc = EINVAL; 1049 goto fail1; 1050 } 1051 1052 if (enp->en_rx_qcount > 0) { 1053 rc = EBUSY; 1054 goto fail2; 1055 } 1056 1057 /* Set scatter buffer size */ 1058 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1059 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 1060 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1061 1062 /* Enable scatter for packets not matching a filter */ 1063 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1064 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 1065 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1066 1067 return (0); 1068 1069 fail2: 1070 EFSYS_PROBE(fail2); 1071 fail1: 1072 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1073 1074 return (rc); 1075 } 1076 #endif /* EFSYS_OPT_RX_SCATTER */ 1077 1078 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 1079 do { \ 1080 efx_oword_t oword; \ 1081 \ 1082 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1083 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 1084 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 1085 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 1086 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1087 (_insert) ? 1 : 0); \ 1088 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1089 \ 1090 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 1091 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1092 &oword); \ 1093 EFX_SET_OWORD_FIELD(oword, \ 1094 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 1095 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1096 &oword); \ 1097 } \ 1098 \ 1099 _NOTE(CONSTANTCONDITION) \ 1100 } while (B_FALSE) 1101 1102 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 1103 do { \ 1104 efx_oword_t oword; \ 1105 \ 1106 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1107 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 1108 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 1109 (_ip) ? 1 : 0); \ 1110 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 1111 (_tcp) ? 0 : 1); \ 1112 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1113 (_insert) ? 1 : 0); \ 1114 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1115 \ 1116 _NOTE(CONSTANTCONDITION) \ 1117 } while (B_FALSE) 1118 1119 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 1120 do { \ 1121 efx_oword_t oword; \ 1122 \ 1123 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1124 EFX_SET_OWORD_FIELD(oword, \ 1125 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 1126 EFX_SET_OWORD_FIELD(oword, \ 1127 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 1128 EFX_SET_OWORD_FIELD(oword, \ 1129 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 1130 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1131 \ 1132 (_rc) = 0; \ 1133 \ 1134 _NOTE(CONSTANTCONDITION) \ 1135 } while (B_FALSE) 1136 1137 #if EFSYS_OPT_RX_SCALE 1138 1139 static __checkReturn efx_rc_t 1140 siena_rx_scale_mode_set( 1141 __in efx_nic_t *enp, 1142 __in uint32_t rss_context, 1143 __in efx_rx_hash_alg_t alg, 1144 __in efx_rx_hash_type_t type, 1145 __in boolean_t insert) 1146 { 1147 efx_rc_t rc; 1148 1149 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1150 rc = EINVAL; 1151 goto fail1; 1152 } 1153 1154 switch (alg) { 1155 case EFX_RX_HASHALG_LFSR: 1156 EFX_RX_LFSR_HASH(enp, insert); 1157 break; 1158 1159 case EFX_RX_HASHALG_TOEPLITZ: 1160 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 1161 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE, 1162 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE); 1163 1164 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 1165 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE, 1166 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE, 1167 rc); 1168 if (rc != 0) 1169 goto fail2; 1170 1171 break; 1172 1173 default: 1174 rc = EINVAL; 1175 goto fail3; 1176 } 1177 1178 return (0); 1179 1180 fail3: 1181 EFSYS_PROBE(fail3); 1182 fail2: 1183 EFSYS_PROBE(fail2); 1184 fail1: 1185 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1186 1187 EFX_RX_LFSR_HASH(enp, B_FALSE); 1188 1189 return (rc); 1190 } 1191 #endif 1192 1193 #if EFSYS_OPT_RX_SCALE 1194 static __checkReturn efx_rc_t 1195 siena_rx_scale_key_set( 1196 __in efx_nic_t *enp, 1197 __in uint32_t rss_context, 1198 __in_ecount(n) uint8_t *key, 1199 __in size_t n) 1200 { 1201 efx_oword_t oword; 1202 unsigned int byte; 1203 unsigned int offset; 1204 efx_rc_t rc; 1205 1206 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1207 rc = EINVAL; 1208 goto fail1; 1209 } 1210 1211 byte = 0; 1212 1213 /* Write Toeplitz IPv4 hash key */ 1214 EFX_ZERO_OWORD(oword); 1215 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1216 offset > 0 && byte < n; 1217 --offset) 1218 oword.eo_u8[offset - 1] = key[byte++]; 1219 1220 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1221 1222 byte = 0; 1223 1224 /* Verify Toeplitz IPv4 hash key */ 1225 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1226 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1227 offset > 0 && byte < n; 1228 --offset) { 1229 if (oword.eo_u8[offset - 1] != key[byte++]) { 1230 rc = EFAULT; 1231 goto fail2; 1232 } 1233 } 1234 1235 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 1236 goto done; 1237 1238 byte = 0; 1239 1240 /* Write Toeplitz IPv6 hash key 3 */ 1241 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1242 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1243 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1244 offset > 0 && byte < n; 1245 --offset) 1246 oword.eo_u8[offset - 1] = key[byte++]; 1247 1248 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1249 1250 /* Write Toeplitz IPv6 hash key 2 */ 1251 EFX_ZERO_OWORD(oword); 1252 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1253 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1254 offset > 0 && byte < n; 1255 --offset) 1256 oword.eo_u8[offset - 1] = key[byte++]; 1257 1258 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1259 1260 /* Write Toeplitz IPv6 hash key 1 */ 1261 EFX_ZERO_OWORD(oword); 1262 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1263 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1264 offset > 0 && byte < n; 1265 --offset) 1266 oword.eo_u8[offset - 1] = key[byte++]; 1267 1268 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1269 1270 byte = 0; 1271 1272 /* Verify Toeplitz IPv6 hash key 3 */ 1273 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1274 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1275 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1276 offset > 0 && byte < n; 1277 --offset) { 1278 if (oword.eo_u8[offset - 1] != key[byte++]) { 1279 rc = EFAULT; 1280 goto fail3; 1281 } 1282 } 1283 1284 /* Verify Toeplitz IPv6 hash key 2 */ 1285 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1286 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1287 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1288 offset > 0 && byte < n; 1289 --offset) { 1290 if (oword.eo_u8[offset - 1] != key[byte++]) { 1291 rc = EFAULT; 1292 goto fail4; 1293 } 1294 } 1295 1296 /* Verify Toeplitz IPv6 hash key 1 */ 1297 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1298 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1299 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1300 offset > 0 && byte < n; 1301 --offset) { 1302 if (oword.eo_u8[offset - 1] != key[byte++]) { 1303 rc = EFAULT; 1304 goto fail5; 1305 } 1306 } 1307 1308 done: 1309 return (0); 1310 1311 fail5: 1312 EFSYS_PROBE(fail5); 1313 fail4: 1314 EFSYS_PROBE(fail4); 1315 fail3: 1316 EFSYS_PROBE(fail3); 1317 fail2: 1318 EFSYS_PROBE(fail2); 1319 fail1: 1320 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1321 1322 return (rc); 1323 } 1324 #endif 1325 1326 #if EFSYS_OPT_RX_SCALE 1327 static __checkReturn efx_rc_t 1328 siena_rx_scale_tbl_set( 1329 __in efx_nic_t *enp, 1330 __in uint32_t rss_context, 1331 __in_ecount(n) unsigned int *table, 1332 __in size_t n) 1333 { 1334 efx_oword_t oword; 1335 int index; 1336 efx_rc_t rc; 1337 1338 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1339 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1340 1341 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1342 rc = EINVAL; 1343 goto fail1; 1344 } 1345 1346 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1347 rc = EINVAL; 1348 goto fail2; 1349 } 1350 1351 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1352 uint32_t byte; 1353 1354 /* Calculate the entry to place in the table */ 1355 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1356 1357 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1358 1359 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1360 1361 /* Write the table */ 1362 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1363 index, &oword, B_TRUE); 1364 } 1365 1366 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1367 uint32_t byte; 1368 1369 /* Determine if we're starting a new batch */ 1370 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1371 1372 /* Read the table */ 1373 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1374 index, &oword, B_TRUE); 1375 1376 /* Verify the entry */ 1377 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1378 rc = EFAULT; 1379 goto fail3; 1380 } 1381 } 1382 1383 return (0); 1384 1385 fail3: 1386 EFSYS_PROBE(fail3); 1387 fail2: 1388 EFSYS_PROBE(fail2); 1389 fail1: 1390 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1391 1392 return (rc); 1393 } 1394 #endif 1395 1396 /* 1397 * Falcon/Siena pseudo-header 1398 * -------------------------- 1399 * 1400 * Receive packets are prefixed by an optional 16 byte pseudo-header. 1401 * The pseudo-header is a byte array of one of the forms: 1402 * 1403 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 1404 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 1405 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 1406 * 1407 * where: 1408 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 1409 * LL.LL LFSR hash (16-bit big-endian) 1410 */ 1411 1412 #if EFSYS_OPT_RX_SCALE 1413 static __checkReturn uint32_t 1414 siena_rx_prefix_hash( 1415 __in efx_nic_t *enp, 1416 __in efx_rx_hash_alg_t func, 1417 __in uint8_t *buffer) 1418 { 1419 _NOTE(ARGUNUSED(enp)) 1420 1421 switch (func) { 1422 case EFX_RX_HASHALG_TOEPLITZ: 1423 return ((buffer[12] << 24) | 1424 (buffer[13] << 16) | 1425 (buffer[14] << 8) | 1426 buffer[15]); 1427 1428 case EFX_RX_HASHALG_LFSR: 1429 return ((buffer[14] << 8) | buffer[15]); 1430 1431 default: 1432 EFSYS_ASSERT(0); 1433 return (0); 1434 } 1435 } 1436 #endif /* EFSYS_OPT_RX_SCALE */ 1437 1438 static __checkReturn efx_rc_t 1439 siena_rx_prefix_pktlen( 1440 __in efx_nic_t *enp, 1441 __in uint8_t *buffer, 1442 __out uint16_t *lengthp) 1443 { 1444 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1445 1446 /* Not supported by Falcon/Siena hardware */ 1447 EFSYS_ASSERT(0); 1448 return (ENOTSUP); 1449 } 1450 1451 static void 1452 siena_rx_qpost( 1453 __in efx_rxq_t *erp, 1454 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 1455 __in size_t size, 1456 __in unsigned int ndescs, 1457 __in unsigned int completed, 1458 __in unsigned int added) 1459 { 1460 efx_qword_t qword; 1461 unsigned int i; 1462 unsigned int offset; 1463 unsigned int id; 1464 1465 /* The client driver must not overfill the queue */ 1466 EFSYS_ASSERT3U(added - completed + ndescs, <=, 1467 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1468 1469 id = added & (erp->er_mask); 1470 for (i = 0; i < ndescs; i++) { 1471 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1472 unsigned int, id, efsys_dma_addr_t, addrp[i], 1473 size_t, size); 1474 1475 EFX_POPULATE_QWORD_3(qword, 1476 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1477 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1478 (uint32_t)(addrp[i] & 0xffffffff), 1479 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1480 (uint32_t)(addrp[i] >> 32)); 1481 1482 offset = id * sizeof (efx_qword_t); 1483 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1484 1485 id = (id + 1) & (erp->er_mask); 1486 } 1487 } 1488 1489 static void 1490 siena_rx_qpush( 1491 __in efx_rxq_t *erp, 1492 __in unsigned int added, 1493 __inout unsigned int *pushedp) 1494 { 1495 efx_nic_t *enp = erp->er_enp; 1496 unsigned int pushed = *pushedp; 1497 uint32_t wptr; 1498 efx_oword_t oword; 1499 efx_dword_t dword; 1500 1501 /* All descriptors are pushed */ 1502 *pushedp = added; 1503 1504 /* Push the populated descriptors out */ 1505 wptr = added & erp->er_mask; 1506 1507 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1508 1509 /* Only write the third DWORD */ 1510 EFX_POPULATE_DWORD_1(dword, 1511 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1512 1513 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1514 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1515 wptr, pushed & erp->er_mask); 1516 EFSYS_PIO_WRITE_BARRIER(); 1517 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1518 erp->er_index, &dword, B_FALSE); 1519 } 1520 1521 #if EFSYS_OPT_RX_PACKED_STREAM 1522 static void 1523 siena_rx_qpush_ps_credits( 1524 __in efx_rxq_t *erp) 1525 { 1526 /* Not supported by Siena hardware */ 1527 EFSYS_ASSERT(0); 1528 } 1529 1530 static uint8_t * 1531 siena_rx_qps_packet_info( 1532 __in efx_rxq_t *erp, 1533 __in uint8_t *buffer, 1534 __in uint32_t buffer_length, 1535 __in uint32_t current_offset, 1536 __out uint16_t *lengthp, 1537 __out uint32_t *next_offsetp, 1538 __out uint32_t *timestamp) 1539 { 1540 /* Not supported by Siena hardware */ 1541 EFSYS_ASSERT(0); 1542 1543 return (NULL); 1544 } 1545 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1546 1547 static __checkReturn efx_rc_t 1548 siena_rx_qflush( 1549 __in efx_rxq_t *erp) 1550 { 1551 efx_nic_t *enp = erp->er_enp; 1552 efx_oword_t oword; 1553 uint32_t label; 1554 1555 label = erp->er_index; 1556 1557 /* Flush the queue */ 1558 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1559 FRF_AZ_RX_FLUSH_DESCQ, label); 1560 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1561 1562 return (0); 1563 } 1564 1565 static void 1566 siena_rx_qenable( 1567 __in efx_rxq_t *erp) 1568 { 1569 efx_nic_t *enp = erp->er_enp; 1570 efx_oword_t oword; 1571 1572 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1573 1574 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1575 erp->er_index, &oword, B_TRUE); 1576 1577 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1578 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1579 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1580 1581 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1582 erp->er_index, &oword, B_TRUE); 1583 } 1584 1585 static __checkReturn efx_rc_t 1586 siena_rx_qcreate( 1587 __in efx_nic_t *enp, 1588 __in unsigned int index, 1589 __in unsigned int label, 1590 __in efx_rxq_type_t type, 1591 __in_opt const efx_rxq_type_data_t *type_data, 1592 __in efsys_mem_t *esmp, 1593 __in size_t ndescs, 1594 __in uint32_t id, 1595 __in unsigned int flags, 1596 __in efx_evq_t *eep, 1597 __in efx_rxq_t *erp) 1598 { 1599 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1600 efx_oword_t oword; 1601 uint32_t size; 1602 boolean_t jumbo = B_FALSE; 1603 efx_rc_t rc; 1604 1605 _NOTE(ARGUNUSED(esmp)) 1606 _NOTE(ARGUNUSED(type_data)) 1607 1608 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1609 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1610 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1611 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1612 1613 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 1614 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 1615 1616 if (!ISP2(ndescs) || 1617 (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) { 1618 rc = EINVAL; 1619 goto fail1; 1620 } 1621 if (index >= encp->enc_rxq_limit) { 1622 rc = EINVAL; 1623 goto fail2; 1624 } 1625 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1626 size++) 1627 if ((1 << size) == (int)(ndescs / EFX_RXQ_MINNDESCS)) 1628 break; 1629 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1630 rc = EINVAL; 1631 goto fail3; 1632 } 1633 1634 switch (type) { 1635 case EFX_RXQ_TYPE_DEFAULT: 1636 break; 1637 1638 default: 1639 rc = EINVAL; 1640 goto fail4; 1641 } 1642 1643 if (flags & EFX_RXQ_FLAG_SCATTER) { 1644 #if EFSYS_OPT_RX_SCATTER 1645 jumbo = B_TRUE; 1646 #else 1647 rc = EINVAL; 1648 goto fail5; 1649 #endif /* EFSYS_OPT_RX_SCATTER */ 1650 } 1651 1652 /* Set up the new descriptor queue */ 1653 EFX_POPULATE_OWORD_7(oword, 1654 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1655 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1656 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1657 FRF_AZ_RX_DESCQ_LABEL, label, 1658 FRF_AZ_RX_DESCQ_SIZE, size, 1659 FRF_AZ_RX_DESCQ_TYPE, 0, 1660 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1661 1662 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1663 erp->er_index, &oword, B_TRUE); 1664 1665 return (0); 1666 1667 #if !EFSYS_OPT_RX_SCATTER 1668 fail5: 1669 EFSYS_PROBE(fail5); 1670 #endif 1671 fail4: 1672 EFSYS_PROBE(fail4); 1673 fail3: 1674 EFSYS_PROBE(fail3); 1675 fail2: 1676 EFSYS_PROBE(fail2); 1677 fail1: 1678 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1679 1680 return (rc); 1681 } 1682 1683 static void 1684 siena_rx_qdestroy( 1685 __in efx_rxq_t *erp) 1686 { 1687 efx_nic_t *enp = erp->er_enp; 1688 efx_oword_t oword; 1689 1690 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1691 --enp->en_rx_qcount; 1692 1693 /* Purge descriptor queue */ 1694 EFX_ZERO_OWORD(oword); 1695 1696 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1697 erp->er_index, &oword, B_TRUE); 1698 1699 /* Free the RXQ object */ 1700 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1701 } 1702 1703 static void 1704 siena_rx_fini( 1705 __in efx_nic_t *enp) 1706 { 1707 _NOTE(ARGUNUSED(enp)) 1708 } 1709 1710 #endif /* EFSYS_OPT_SIENA */ 1711