1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "efx.h" 37 #include "efx_impl.h" 38 39 40 #if EFSYS_OPT_SIENA 41 42 static __checkReturn efx_rc_t 43 siena_rx_init( 44 __in efx_nic_t *enp); 45 46 static void 47 siena_rx_fini( 48 __in efx_nic_t *enp); 49 50 #if EFSYS_OPT_RX_SCATTER 51 static __checkReturn efx_rc_t 52 siena_rx_scatter_enable( 53 __in efx_nic_t *enp, 54 __in unsigned int buf_size); 55 #endif /* EFSYS_OPT_RX_SCATTER */ 56 57 #if EFSYS_OPT_RX_SCALE 58 static __checkReturn efx_rc_t 59 siena_rx_scale_mode_set( 60 __in efx_nic_t *enp, 61 __in uint32_t rss_context, 62 __in efx_rx_hash_alg_t alg, 63 __in efx_rx_hash_type_t type, 64 __in boolean_t insert); 65 66 static __checkReturn efx_rc_t 67 siena_rx_scale_key_set( 68 __in efx_nic_t *enp, 69 __in uint32_t rss_context, 70 __in_ecount(n) uint8_t *key, 71 __in size_t n); 72 73 static __checkReturn efx_rc_t 74 siena_rx_scale_tbl_set( 75 __in efx_nic_t *enp, 76 __in uint32_t rss_context, 77 __in_ecount(n) unsigned int *table, 78 __in size_t n); 79 80 static __checkReturn uint32_t 81 siena_rx_prefix_hash( 82 __in efx_nic_t *enp, 83 __in efx_rx_hash_alg_t func, 84 __in uint8_t *buffer); 85 86 #endif /* EFSYS_OPT_RX_SCALE */ 87 88 static __checkReturn efx_rc_t 89 siena_rx_prefix_pktlen( 90 __in efx_nic_t *enp, 91 __in uint8_t *buffer, 92 __out uint16_t *lengthp); 93 94 static void 95 siena_rx_qpost( 96 __in efx_rxq_t *erp, 97 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 98 __in size_t size, 99 __in unsigned int ndescs, 100 __in unsigned int completed, 101 __in unsigned int added); 102 103 static void 104 siena_rx_qpush( 105 __in efx_rxq_t *erp, 106 __in unsigned int added, 107 __inout unsigned int *pushedp); 108 109 #if EFSYS_OPT_RX_PACKED_STREAM 110 static void 111 siena_rx_qpush_ps_credits( 112 __in efx_rxq_t *erp); 113 114 static __checkReturn uint8_t * 115 siena_rx_qps_packet_info( 116 __in efx_rxq_t *erp, 117 __in uint8_t *buffer, 118 __in uint32_t buffer_length, 119 __in uint32_t current_offset, 120 __out uint16_t *lengthp, 121 __out uint32_t *next_offsetp, 122 __out uint32_t *timestamp); 123 #endif 124 125 static __checkReturn efx_rc_t 126 siena_rx_qflush( 127 __in efx_rxq_t *erp); 128 129 static void 130 siena_rx_qenable( 131 __in efx_rxq_t *erp); 132 133 static __checkReturn efx_rc_t 134 siena_rx_qcreate( 135 __in efx_nic_t *enp, 136 __in unsigned int index, 137 __in unsigned int label, 138 __in efx_rxq_type_t type, 139 __in_opt const efx_rxq_type_data_t *type_data, 140 __in efsys_mem_t *esmp, 141 __in size_t ndescs, 142 __in uint32_t id, 143 __in unsigned int flags, 144 __in efx_evq_t *eep, 145 __in efx_rxq_t *erp); 146 147 static void 148 siena_rx_qdestroy( 149 __in efx_rxq_t *erp); 150 151 #endif /* EFSYS_OPT_SIENA */ 152 153 154 #if EFSYS_OPT_SIENA 155 static const efx_rx_ops_t __efx_rx_siena_ops = { 156 siena_rx_init, /* erxo_init */ 157 siena_rx_fini, /* erxo_fini */ 158 #if EFSYS_OPT_RX_SCATTER 159 siena_rx_scatter_enable, /* erxo_scatter_enable */ 160 #endif 161 #if EFSYS_OPT_RX_SCALE 162 NULL, /* erxo_scale_context_alloc */ 163 NULL, /* erxo_scale_context_free */ 164 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 165 siena_rx_scale_key_set, /* erxo_scale_key_set */ 166 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 167 siena_rx_prefix_hash, /* erxo_prefix_hash */ 168 #endif 169 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 170 siena_rx_qpost, /* erxo_qpost */ 171 siena_rx_qpush, /* erxo_qpush */ 172 #if EFSYS_OPT_RX_PACKED_STREAM 173 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 174 siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 175 #endif 176 siena_rx_qflush, /* erxo_qflush */ 177 siena_rx_qenable, /* erxo_qenable */ 178 siena_rx_qcreate, /* erxo_qcreate */ 179 siena_rx_qdestroy, /* erxo_qdestroy */ 180 }; 181 #endif /* EFSYS_OPT_SIENA */ 182 183 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 184 static const efx_rx_ops_t __efx_rx_ef10_ops = { 185 ef10_rx_init, /* erxo_init */ 186 ef10_rx_fini, /* erxo_fini */ 187 #if EFSYS_OPT_RX_SCATTER 188 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 189 #endif 190 #if EFSYS_OPT_RX_SCALE 191 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 192 ef10_rx_scale_context_free, /* erxo_scale_context_free */ 193 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 194 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 195 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 196 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 197 #endif 198 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 199 ef10_rx_qpost, /* erxo_qpost */ 200 ef10_rx_qpush, /* erxo_qpush */ 201 #if EFSYS_OPT_RX_PACKED_STREAM 202 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 203 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 204 #endif 205 ef10_rx_qflush, /* erxo_qflush */ 206 ef10_rx_qenable, /* erxo_qenable */ 207 ef10_rx_qcreate, /* erxo_qcreate */ 208 ef10_rx_qdestroy, /* erxo_qdestroy */ 209 }; 210 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ 211 212 213 __checkReturn efx_rc_t 214 efx_rx_init( 215 __inout efx_nic_t *enp) 216 { 217 const efx_rx_ops_t *erxop; 218 efx_rc_t rc; 219 220 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 221 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 222 223 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 224 rc = EINVAL; 225 goto fail1; 226 } 227 228 if (enp->en_mod_flags & EFX_MOD_RX) { 229 rc = EINVAL; 230 goto fail2; 231 } 232 233 switch (enp->en_family) { 234 #if EFSYS_OPT_SIENA 235 case EFX_FAMILY_SIENA: 236 erxop = &__efx_rx_siena_ops; 237 break; 238 #endif /* EFSYS_OPT_SIENA */ 239 240 #if EFSYS_OPT_HUNTINGTON 241 case EFX_FAMILY_HUNTINGTON: 242 erxop = &__efx_rx_ef10_ops; 243 break; 244 #endif /* EFSYS_OPT_HUNTINGTON */ 245 246 #if EFSYS_OPT_MEDFORD 247 case EFX_FAMILY_MEDFORD: 248 erxop = &__efx_rx_ef10_ops; 249 break; 250 #endif /* EFSYS_OPT_MEDFORD */ 251 252 #if EFSYS_OPT_MEDFORD2 253 case EFX_FAMILY_MEDFORD2: 254 erxop = &__efx_rx_ef10_ops; 255 break; 256 #endif /* EFSYS_OPT_MEDFORD2 */ 257 258 default: 259 EFSYS_ASSERT(0); 260 rc = ENOTSUP; 261 goto fail3; 262 } 263 264 if ((rc = erxop->erxo_init(enp)) != 0) 265 goto fail4; 266 267 enp->en_erxop = erxop; 268 enp->en_mod_flags |= EFX_MOD_RX; 269 return (0); 270 271 fail4: 272 EFSYS_PROBE(fail4); 273 fail3: 274 EFSYS_PROBE(fail3); 275 fail2: 276 EFSYS_PROBE(fail2); 277 fail1: 278 EFSYS_PROBE1(fail1, efx_rc_t, rc); 279 280 enp->en_erxop = NULL; 281 enp->en_mod_flags &= ~EFX_MOD_RX; 282 return (rc); 283 } 284 285 void 286 efx_rx_fini( 287 __in efx_nic_t *enp) 288 { 289 const efx_rx_ops_t *erxop = enp->en_erxop; 290 291 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 292 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 293 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 294 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 295 296 erxop->erxo_fini(enp); 297 298 enp->en_erxop = NULL; 299 enp->en_mod_flags &= ~EFX_MOD_RX; 300 } 301 302 #if EFSYS_OPT_RX_SCATTER 303 __checkReturn efx_rc_t 304 efx_rx_scatter_enable( 305 __in efx_nic_t *enp, 306 __in unsigned int buf_size) 307 { 308 const efx_rx_ops_t *erxop = enp->en_erxop; 309 efx_rc_t rc; 310 311 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 312 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 313 314 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 315 goto fail1; 316 317 return (0); 318 319 fail1: 320 EFSYS_PROBE1(fail1, efx_rc_t, rc); 321 return (rc); 322 } 323 #endif /* EFSYS_OPT_RX_SCATTER */ 324 325 #if EFSYS_OPT_RX_SCALE 326 __checkReturn efx_rc_t 327 efx_rx_scale_hash_flags_get( 328 __in efx_nic_t *enp, 329 __in efx_rx_hash_alg_t hash_alg, 330 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp, 331 __in unsigned int max_nflags, 332 __out unsigned int *nflagsp) 333 { 334 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 335 unsigned int nflags = 0; 336 efx_rc_t rc; 337 338 if (flagsp == NULL || nflagsp == NULL) { 339 rc = EINVAL; 340 goto fail1; 341 } 342 343 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) { 344 nflags = 0; 345 goto done; 346 } 347 348 /* Helper to add flags word to flags array without buffer overflow */ 349 #define INSERT_FLAGS(_flags) \ 350 do { \ 351 if (nflags >= max_nflags) { \ 352 rc = E2BIG; \ 353 goto fail2; \ 354 } \ 355 *(flagsp + nflags) = (_flags); \ 356 nflags++; \ 357 \ 358 _NOTE(CONSTANTCONDITION) \ 359 } while (B_FALSE) 360 361 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) { 362 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE)); 363 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE)); 364 } 365 366 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) && 367 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) { 368 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST)); 369 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC)); 370 371 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST)); 372 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC)); 373 374 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE)); 375 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST)); 376 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC)); 377 378 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE)); 379 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST)); 380 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC)); 381 } 382 383 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE)); 384 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE)); 385 386 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE)); 387 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE)); 388 389 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) { 390 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST)); 391 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC)); 392 393 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST)); 394 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC)); 395 396 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE)); 397 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST)); 398 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC)); 399 400 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE)); 401 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST)); 402 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC)); 403 404 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST)); 405 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC)); 406 407 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST)); 408 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC)); 409 } 410 411 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE)); 412 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE)); 413 414 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE)); 415 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE)); 416 417 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE)); 418 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE)); 419 420 #undef INSERT_FLAGS 421 422 done: 423 *nflagsp = nflags; 424 return (0); 425 426 fail2: 427 EFSYS_PROBE(fail2); 428 fail1: 429 EFSYS_PROBE1(fail1, efx_rc_t, rc); 430 431 return (rc); 432 } 433 434 __checkReturn efx_rc_t 435 efx_rx_hash_default_support_get( 436 __in efx_nic_t *enp, 437 __out efx_rx_hash_support_t *supportp) 438 { 439 efx_rc_t rc; 440 441 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 442 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 443 444 if (supportp == NULL) { 445 rc = EINVAL; 446 goto fail1; 447 } 448 449 /* 450 * Report the hashing support the client gets by default if it 451 * does not allocate an RSS context itself. 452 */ 453 *supportp = enp->en_hash_support; 454 455 return (0); 456 457 fail1: 458 EFSYS_PROBE1(fail1, efx_rc_t, rc); 459 460 return (rc); 461 } 462 463 __checkReturn efx_rc_t 464 efx_rx_scale_default_support_get( 465 __in efx_nic_t *enp, 466 __out efx_rx_scale_context_type_t *typep) 467 { 468 efx_rc_t rc; 469 470 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 471 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 472 473 if (typep == NULL) { 474 rc = EINVAL; 475 goto fail1; 476 } 477 478 /* 479 * Report the RSS support the client gets by default if it 480 * does not allocate an RSS context itself. 481 */ 482 *typep = enp->en_rss_context_type; 483 484 return (0); 485 486 fail1: 487 EFSYS_PROBE1(fail1, efx_rc_t, rc); 488 489 return (rc); 490 } 491 #endif /* EFSYS_OPT_RX_SCALE */ 492 493 #if EFSYS_OPT_RX_SCALE 494 __checkReturn efx_rc_t 495 efx_rx_scale_context_alloc( 496 __in efx_nic_t *enp, 497 __in efx_rx_scale_context_type_t type, 498 __in uint32_t num_queues, 499 __out uint32_t *rss_contextp) 500 { 501 const efx_rx_ops_t *erxop = enp->en_erxop; 502 efx_rc_t rc; 503 504 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 505 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 506 507 if (erxop->erxo_scale_context_alloc == NULL) { 508 rc = ENOTSUP; 509 goto fail1; 510 } 511 if ((rc = erxop->erxo_scale_context_alloc(enp, type, 512 num_queues, rss_contextp)) != 0) { 513 goto fail2; 514 } 515 516 return (0); 517 518 fail2: 519 EFSYS_PROBE(fail2); 520 fail1: 521 EFSYS_PROBE1(fail1, efx_rc_t, rc); 522 return (rc); 523 } 524 #endif /* EFSYS_OPT_RX_SCALE */ 525 526 #if EFSYS_OPT_RX_SCALE 527 __checkReturn efx_rc_t 528 efx_rx_scale_context_free( 529 __in efx_nic_t *enp, 530 __in uint32_t rss_context) 531 { 532 const efx_rx_ops_t *erxop = enp->en_erxop; 533 efx_rc_t rc; 534 535 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 536 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 537 538 if (erxop->erxo_scale_context_free == NULL) { 539 rc = ENOTSUP; 540 goto fail1; 541 } 542 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 543 goto fail2; 544 545 return (0); 546 547 fail2: 548 EFSYS_PROBE(fail2); 549 fail1: 550 EFSYS_PROBE1(fail1, efx_rc_t, rc); 551 return (rc); 552 } 553 #endif /* EFSYS_OPT_RX_SCALE */ 554 555 #if EFSYS_OPT_RX_SCALE 556 __checkReturn efx_rc_t 557 efx_rx_scale_mode_set( 558 __in efx_nic_t *enp, 559 __in uint32_t rss_context, 560 __in efx_rx_hash_alg_t alg, 561 __in efx_rx_hash_type_t type, 562 __in boolean_t insert) 563 { 564 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 565 const efx_rx_ops_t *erxop = enp->en_erxop; 566 efx_rx_hash_type_t type_check; 567 unsigned int i; 568 efx_rc_t rc; 569 570 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 571 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 572 573 /* 574 * Legacy flags and modern bits cannot be 575 * used at the same time in the hash type. 576 */ 577 if ((type & EFX_RX_HASH_LEGACY_MASK) && 578 (type & ~EFX_RX_HASH_LEGACY_MASK)) { 579 rc = EINVAL; 580 goto fail1; 581 } 582 583 /* 584 * If RSS hash type is represented by additional bits 585 * in the value, the latter need to be verified since 586 * not all bit combinations are valid RSS modes. Also, 587 * depending on the firmware, some valid combinations 588 * may be unsupported. Discern additional bits in the 589 * type value and try to recognise valid combinations. 590 * If some bits remain unrecognised, report the error. 591 */ 592 type_check = type & ~EFX_RX_HASH_LEGACY_MASK; 593 if (type_check != 0) { 594 unsigned int type_flags[EFX_RX_HASH_NFLAGS]; 595 unsigned int type_nflags; 596 597 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags, 598 EFX_ARRAY_SIZE(type_flags), &type_nflags); 599 if (rc != 0) 600 goto fail2; 601 602 for (i = 0; i < type_nflags; ++i) { 603 if ((type_check & type_flags[i]) == type_flags[i]) 604 type_check &= ~(type_flags[i]); 605 } 606 607 if (type_check != 0) { 608 rc = EINVAL; 609 goto fail3; 610 } 611 } 612 613 /* 614 * Translate EFX_RX_HASH() flags to their legacy counterparts 615 * provided that the FW claims no support for additional modes. 616 */ 617 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) { 618 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | 619 EFX_RX_HASH(IPV4_TCP, 2TUPLE); 620 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | 621 EFX_RX_HASH(IPV6_TCP, 2TUPLE); 622 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE); 623 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE); 624 625 if ((type & t_ipv4) == t_ipv4) 626 type |= EFX_RX_HASH_IPV4; 627 if ((type & t_ipv6) == t_ipv6) 628 type |= EFX_RX_HASH_IPV6; 629 630 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) { 631 if ((type & t_ipv4_tcp) == t_ipv4_tcp) 632 type |= EFX_RX_HASH_TCPIPV4; 633 if ((type & t_ipv6_tcp) == t_ipv6_tcp) 634 type |= EFX_RX_HASH_TCPIPV6; 635 } 636 637 type &= EFX_RX_HASH_LEGACY_MASK; 638 } 639 640 if (erxop->erxo_scale_mode_set != NULL) { 641 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 642 type, insert)) != 0) 643 goto fail4; 644 } 645 646 return (0); 647 648 fail4: 649 EFSYS_PROBE(fail4); 650 fail3: 651 EFSYS_PROBE(fail3); 652 fail2: 653 EFSYS_PROBE(fail2); 654 fail1: 655 EFSYS_PROBE1(fail1, efx_rc_t, rc); 656 return (rc); 657 } 658 #endif /* EFSYS_OPT_RX_SCALE */ 659 660 #if EFSYS_OPT_RX_SCALE 661 __checkReturn efx_rc_t 662 efx_rx_scale_key_set( 663 __in efx_nic_t *enp, 664 __in uint32_t rss_context, 665 __in_ecount(n) uint8_t *key, 666 __in size_t n) 667 { 668 const efx_rx_ops_t *erxop = enp->en_erxop; 669 efx_rc_t rc; 670 671 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 672 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 673 674 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 675 goto fail1; 676 677 return (0); 678 679 fail1: 680 EFSYS_PROBE1(fail1, efx_rc_t, rc); 681 682 return (rc); 683 } 684 #endif /* EFSYS_OPT_RX_SCALE */ 685 686 #if EFSYS_OPT_RX_SCALE 687 __checkReturn efx_rc_t 688 efx_rx_scale_tbl_set( 689 __in efx_nic_t *enp, 690 __in uint32_t rss_context, 691 __in_ecount(n) unsigned int *table, 692 __in size_t n) 693 { 694 const efx_rx_ops_t *erxop = enp->en_erxop; 695 efx_rc_t rc; 696 697 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 698 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 699 700 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) 701 goto fail1; 702 703 return (0); 704 705 fail1: 706 EFSYS_PROBE1(fail1, efx_rc_t, rc); 707 708 return (rc); 709 } 710 #endif /* EFSYS_OPT_RX_SCALE */ 711 712 void 713 efx_rx_qpost( 714 __in efx_rxq_t *erp, 715 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 716 __in size_t size, 717 __in unsigned int ndescs, 718 __in unsigned int completed, 719 __in unsigned int added) 720 { 721 efx_nic_t *enp = erp->er_enp; 722 const efx_rx_ops_t *erxop = enp->en_erxop; 723 724 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 725 726 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added); 727 } 728 729 #if EFSYS_OPT_RX_PACKED_STREAM 730 731 void 732 efx_rx_qpush_ps_credits( 733 __in efx_rxq_t *erp) 734 { 735 efx_nic_t *enp = erp->er_enp; 736 const efx_rx_ops_t *erxop = enp->en_erxop; 737 738 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 739 740 erxop->erxo_qpush_ps_credits(erp); 741 } 742 743 __checkReturn uint8_t * 744 efx_rx_qps_packet_info( 745 __in efx_rxq_t *erp, 746 __in uint8_t *buffer, 747 __in uint32_t buffer_length, 748 __in uint32_t current_offset, 749 __out uint16_t *lengthp, 750 __out uint32_t *next_offsetp, 751 __out uint32_t *timestamp) 752 { 753 efx_nic_t *enp = erp->er_enp; 754 const efx_rx_ops_t *erxop = enp->en_erxop; 755 756 return (erxop->erxo_qps_packet_info(erp, buffer, 757 buffer_length, current_offset, lengthp, 758 next_offsetp, timestamp)); 759 } 760 761 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 762 763 void 764 efx_rx_qpush( 765 __in efx_rxq_t *erp, 766 __in unsigned int added, 767 __inout unsigned int *pushedp) 768 { 769 efx_nic_t *enp = erp->er_enp; 770 const efx_rx_ops_t *erxop = enp->en_erxop; 771 772 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 773 774 erxop->erxo_qpush(erp, added, pushedp); 775 } 776 777 __checkReturn efx_rc_t 778 efx_rx_qflush( 779 __in efx_rxq_t *erp) 780 { 781 efx_nic_t *enp = erp->er_enp; 782 const efx_rx_ops_t *erxop = enp->en_erxop; 783 efx_rc_t rc; 784 785 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 786 787 if ((rc = erxop->erxo_qflush(erp)) != 0) 788 goto fail1; 789 790 return (0); 791 792 fail1: 793 EFSYS_PROBE1(fail1, efx_rc_t, rc); 794 795 return (rc); 796 } 797 798 void 799 efx_rx_qenable( 800 __in efx_rxq_t *erp) 801 { 802 efx_nic_t *enp = erp->er_enp; 803 const efx_rx_ops_t *erxop = enp->en_erxop; 804 805 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 806 807 erxop->erxo_qenable(erp); 808 } 809 810 static __checkReturn efx_rc_t 811 efx_rx_qcreate_internal( 812 __in efx_nic_t *enp, 813 __in unsigned int index, 814 __in unsigned int label, 815 __in efx_rxq_type_t type, 816 __in_opt const efx_rxq_type_data_t *type_data, 817 __in efsys_mem_t *esmp, 818 __in size_t ndescs, 819 __in uint32_t id, 820 __in unsigned int flags, 821 __in efx_evq_t *eep, 822 __deref_out efx_rxq_t **erpp) 823 { 824 const efx_rx_ops_t *erxop = enp->en_erxop; 825 efx_rxq_t *erp; 826 efx_rc_t rc; 827 828 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 829 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 830 831 /* Allocate an RXQ object */ 832 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 833 834 if (erp == NULL) { 835 rc = ENOMEM; 836 goto fail1; 837 } 838 839 erp->er_magic = EFX_RXQ_MAGIC; 840 erp->er_enp = enp; 841 erp->er_index = index; 842 erp->er_mask = ndescs - 1; 843 erp->er_esmp = esmp; 844 845 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp, 846 ndescs, id, flags, eep, erp)) != 0) 847 goto fail2; 848 849 enp->en_rx_qcount++; 850 *erpp = erp; 851 852 return (0); 853 854 fail2: 855 EFSYS_PROBE(fail2); 856 857 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 858 fail1: 859 EFSYS_PROBE1(fail1, efx_rc_t, rc); 860 861 return (rc); 862 } 863 864 __checkReturn efx_rc_t 865 efx_rx_qcreate( 866 __in efx_nic_t *enp, 867 __in unsigned int index, 868 __in unsigned int label, 869 __in efx_rxq_type_t type, 870 __in efsys_mem_t *esmp, 871 __in size_t ndescs, 872 __in uint32_t id, 873 __in unsigned int flags, 874 __in efx_evq_t *eep, 875 __deref_out efx_rxq_t **erpp) 876 { 877 return efx_rx_qcreate_internal(enp, index, label, type, NULL, 878 esmp, ndescs, id, flags, eep, erpp); 879 } 880 881 #if EFSYS_OPT_RX_PACKED_STREAM 882 883 __checkReturn efx_rc_t 884 efx_rx_qcreate_packed_stream( 885 __in efx_nic_t *enp, 886 __in unsigned int index, 887 __in unsigned int label, 888 __in uint32_t ps_buf_size, 889 __in efsys_mem_t *esmp, 890 __in size_t ndescs, 891 __in efx_evq_t *eep, 892 __deref_out efx_rxq_t **erpp) 893 { 894 efx_rxq_type_data_t type_data; 895 896 memset(&type_data, 0, sizeof (type_data)); 897 898 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size; 899 900 return efx_rx_qcreate_internal(enp, index, label, 901 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs, 902 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp); 903 } 904 905 #endif 906 907 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 908 909 __checkReturn efx_rc_t 910 efx_rx_qcreate_es_super_buffer( 911 __in efx_nic_t *enp, 912 __in unsigned int index, 913 __in unsigned int label, 914 __in uint32_t n_bufs_per_desc, 915 __in uint32_t max_dma_len, 916 __in uint32_t buf_stride, 917 __in uint32_t hol_block_timeout, 918 __in efsys_mem_t *esmp, 919 __in size_t ndescs, 920 __in unsigned int flags, 921 __in efx_evq_t *eep, 922 __deref_out efx_rxq_t **erpp) 923 { 924 efx_rc_t rc; 925 efx_rxq_type_data_t type_data; 926 927 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 928 rc = EINVAL; 929 goto fail1; 930 } 931 932 memset(&type_data, 0, sizeof (type_data)); 933 934 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc; 935 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len; 936 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride; 937 type_data.ertd_es_super_buffer.eessb_hol_block_timeout = 938 hol_block_timeout; 939 940 rc = efx_rx_qcreate_internal(enp, index, label, 941 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs, 942 0 /* id unused on EF10 */, flags, eep, erpp); 943 if (rc != 0) 944 goto fail2; 945 946 return (0); 947 948 fail2: 949 EFSYS_PROBE(fail2); 950 fail1: 951 EFSYS_PROBE1(fail1, efx_rc_t, rc); 952 953 return (rc); 954 } 955 956 #endif 957 958 959 void 960 efx_rx_qdestroy( 961 __in efx_rxq_t *erp) 962 { 963 efx_nic_t *enp = erp->er_enp; 964 const efx_rx_ops_t *erxop = enp->en_erxop; 965 966 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 967 968 erxop->erxo_qdestroy(erp); 969 } 970 971 __checkReturn efx_rc_t 972 efx_pseudo_hdr_pkt_length_get( 973 __in efx_rxq_t *erp, 974 __in uint8_t *buffer, 975 __out uint16_t *lengthp) 976 { 977 efx_nic_t *enp = erp->er_enp; 978 const efx_rx_ops_t *erxop = enp->en_erxop; 979 980 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 981 982 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 983 } 984 985 #if EFSYS_OPT_RX_SCALE 986 __checkReturn uint32_t 987 efx_pseudo_hdr_hash_get( 988 __in efx_rxq_t *erp, 989 __in efx_rx_hash_alg_t func, 990 __in uint8_t *buffer) 991 { 992 efx_nic_t *enp = erp->er_enp; 993 const efx_rx_ops_t *erxop = enp->en_erxop; 994 995 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 996 997 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 998 return (erxop->erxo_prefix_hash(enp, func, buffer)); 999 } 1000 #endif /* EFSYS_OPT_RX_SCALE */ 1001 1002 #if EFSYS_OPT_SIENA 1003 1004 static __checkReturn efx_rc_t 1005 siena_rx_init( 1006 __in efx_nic_t *enp) 1007 { 1008 efx_oword_t oword; 1009 unsigned int index; 1010 1011 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1012 1013 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 1014 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 1015 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 1016 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 1017 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 1018 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 1019 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1020 1021 /* Zero the RSS table */ 1022 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 1023 index++) { 1024 EFX_ZERO_OWORD(oword); 1025 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1026 index, &oword, B_TRUE); 1027 } 1028 1029 #if EFSYS_OPT_RX_SCALE 1030 /* The RSS key and indirection table are writable. */ 1031 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 1032 1033 /* Hardware can insert RX hash with/without RSS */ 1034 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 1035 #endif /* EFSYS_OPT_RX_SCALE */ 1036 1037 return (0); 1038 } 1039 1040 #if EFSYS_OPT_RX_SCATTER 1041 static __checkReturn efx_rc_t 1042 siena_rx_scatter_enable( 1043 __in efx_nic_t *enp, 1044 __in unsigned int buf_size) 1045 { 1046 unsigned int nbuf32; 1047 efx_oword_t oword; 1048 efx_rc_t rc; 1049 1050 nbuf32 = buf_size / 32; 1051 IF ((NBUF32 == 0) || 1052 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 1053 ((buf_size % 32) != 0)) { 1054 rc = EINVAL; 1055 goto fail1; 1056 } 1057 1058 if (enp->en_rx_qcount > 0) { 1059 rc = EBUSY; 1060 goto fail2; 1061 } 1062 1063 /* Set scatter buffer size */ 1064 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1065 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 1066 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1067 1068 /* Enable scatter for packets not matching a filter */ 1069 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1070 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 1071 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1072 1073 return (0); 1074 1075 fail2: 1076 EFSYS_PROBE(fail2); 1077 fail1: 1078 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1079 1080 return (rc); 1081 } 1082 #endif /* EFSYS_OPT_RX_SCATTER */ 1083 1084 1085 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 1086 do { \ 1087 efx_oword_t oword; \ 1088 \ 1089 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1090 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 1091 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 1092 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 1093 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1094 (_insert) ? 1 : 0); \ 1095 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1096 \ 1097 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 1098 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1099 &oword); \ 1100 EFX_SET_OWORD_FIELD(oword, \ 1101 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 1102 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1103 &oword); \ 1104 } \ 1105 \ 1106 _NOTE(CONSTANTCONDITION) \ 1107 } while (B_FALSE) 1108 1109 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 1110 do { \ 1111 efx_oword_t oword; \ 1112 \ 1113 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1114 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 1115 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 1116 (_ip) ? 1 : 0); \ 1117 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 1118 (_tcp) ? 0 : 1); \ 1119 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1120 (_insert) ? 1 : 0); \ 1121 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1122 \ 1123 _NOTE(CONSTANTCONDITION) \ 1124 } while (B_FALSE) 1125 1126 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 1127 do { \ 1128 efx_oword_t oword; \ 1129 \ 1130 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1131 EFX_SET_OWORD_FIELD(oword, \ 1132 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 1133 EFX_SET_OWORD_FIELD(oword, \ 1134 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 1135 EFX_SET_OWORD_FIELD(oword, \ 1136 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 1137 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1138 \ 1139 (_rc) = 0; \ 1140 \ 1141 _NOTE(CONSTANTCONDITION) \ 1142 } while (B_FALSE) 1143 1144 1145 #if EFSYS_OPT_RX_SCALE 1146 1147 static __checkReturn efx_rc_t 1148 siena_rx_scale_mode_set( 1149 __in efx_nic_t *enp, 1150 __in uint32_t rss_context, 1151 __in efx_rx_hash_alg_t alg, 1152 __in efx_rx_hash_type_t type, 1153 __in boolean_t insert) 1154 { 1155 efx_rc_t rc; 1156 1157 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1158 rc = EINVAL; 1159 goto fail1; 1160 } 1161 1162 switch (alg) { 1163 case EFX_RX_HASHALG_LFSR: 1164 EFX_RX_LFSR_HASH(enp, insert); 1165 break; 1166 1167 case EFX_RX_HASHALG_TOEPLITZ: 1168 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 1169 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE, 1170 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE); 1171 1172 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 1173 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE, 1174 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE, 1175 rc); 1176 if (rc != 0) 1177 goto fail2; 1178 1179 break; 1180 1181 default: 1182 rc = EINVAL; 1183 goto fail3; 1184 } 1185 1186 return (0); 1187 1188 fail3: 1189 EFSYS_PROBE(fail3); 1190 fail2: 1191 EFSYS_PROBE(fail2); 1192 fail1: 1193 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1194 1195 EFX_RX_LFSR_HASH(enp, B_FALSE); 1196 1197 return (rc); 1198 } 1199 #endif 1200 1201 #if EFSYS_OPT_RX_SCALE 1202 static __checkReturn efx_rc_t 1203 siena_rx_scale_key_set( 1204 __in efx_nic_t *enp, 1205 __in uint32_t rss_context, 1206 __in_ecount(n) uint8_t *key, 1207 __in size_t n) 1208 { 1209 efx_oword_t oword; 1210 unsigned int byte; 1211 unsigned int offset; 1212 efx_rc_t rc; 1213 1214 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1215 rc = EINVAL; 1216 goto fail1; 1217 } 1218 1219 byte = 0; 1220 1221 /* Write Toeplitz IPv4 hash key */ 1222 EFX_ZERO_OWORD(oword); 1223 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1224 offset > 0 && byte < n; 1225 --offset) 1226 oword.eo_u8[offset - 1] = key[byte++]; 1227 1228 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1229 1230 byte = 0; 1231 1232 /* Verify Toeplitz IPv4 hash key */ 1233 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1234 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1235 offset > 0 && byte < n; 1236 --offset) { 1237 if (oword.eo_u8[offset - 1] != key[byte++]) { 1238 rc = EFAULT; 1239 goto fail2; 1240 } 1241 } 1242 1243 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 1244 goto done; 1245 1246 byte = 0; 1247 1248 /* Write Toeplitz IPv6 hash key 3 */ 1249 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1250 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1251 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1252 offset > 0 && byte < n; 1253 --offset) 1254 oword.eo_u8[offset - 1] = key[byte++]; 1255 1256 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1257 1258 /* Write Toeplitz IPv6 hash key 2 */ 1259 EFX_ZERO_OWORD(oword); 1260 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1261 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1262 offset > 0 && byte < n; 1263 --offset) 1264 oword.eo_u8[offset - 1] = key[byte++]; 1265 1266 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1267 1268 /* Write Toeplitz IPv6 hash key 1 */ 1269 EFX_ZERO_OWORD(oword); 1270 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1271 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1272 offset > 0 && byte < n; 1273 --offset) 1274 oword.eo_u8[offset - 1] = key[byte++]; 1275 1276 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1277 1278 byte = 0; 1279 1280 /* Verify Toeplitz IPv6 hash key 3 */ 1281 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1282 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1283 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1284 offset > 0 && byte < n; 1285 --offset) { 1286 if (oword.eo_u8[offset - 1] != key[byte++]) { 1287 rc = EFAULT; 1288 goto fail3; 1289 } 1290 } 1291 1292 /* Verify Toeplitz IPv6 hash key 2 */ 1293 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1294 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1295 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1296 offset > 0 && byte < n; 1297 --offset) { 1298 if (oword.eo_u8[offset - 1] != key[byte++]) { 1299 rc = EFAULT; 1300 goto fail4; 1301 } 1302 } 1303 1304 /* Verify Toeplitz IPv6 hash key 1 */ 1305 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1306 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1307 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1308 offset > 0 && byte < n; 1309 --offset) { 1310 if (oword.eo_u8[offset - 1] != key[byte++]) { 1311 rc = EFAULT; 1312 goto fail5; 1313 } 1314 } 1315 1316 done: 1317 return (0); 1318 1319 fail5: 1320 EFSYS_PROBE(fail5); 1321 fail4: 1322 EFSYS_PROBE(fail4); 1323 fail3: 1324 EFSYS_PROBE(fail3); 1325 fail2: 1326 EFSYS_PROBE(fail2); 1327 fail1: 1328 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1329 1330 return (rc); 1331 } 1332 #endif 1333 1334 #if EFSYS_OPT_RX_SCALE 1335 static __checkReturn efx_rc_t 1336 siena_rx_scale_tbl_set( 1337 __in efx_nic_t *enp, 1338 __in uint32_t rss_context, 1339 __in_ecount(n) unsigned int *table, 1340 __in size_t n) 1341 { 1342 efx_oword_t oword; 1343 int index; 1344 efx_rc_t rc; 1345 1346 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1347 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1348 1349 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1350 rc = EINVAL; 1351 goto fail1; 1352 } 1353 1354 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1355 rc = EINVAL; 1356 goto fail2; 1357 } 1358 1359 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1360 uint32_t byte; 1361 1362 /* Calculate the entry to place in the table */ 1363 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1364 1365 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1366 1367 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1368 1369 /* Write the table */ 1370 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1371 index, &oword, B_TRUE); 1372 } 1373 1374 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1375 uint32_t byte; 1376 1377 /* Determine if we're starting a new batch */ 1378 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1379 1380 /* Read the table */ 1381 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1382 index, &oword, B_TRUE); 1383 1384 /* Verify the entry */ 1385 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1386 rc = EFAULT; 1387 goto fail3; 1388 } 1389 } 1390 1391 return (0); 1392 1393 fail3: 1394 EFSYS_PROBE(fail3); 1395 fail2: 1396 EFSYS_PROBE(fail2); 1397 fail1: 1398 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1399 1400 return (rc); 1401 } 1402 #endif 1403 1404 /* 1405 * Falcon/Siena pseudo-header 1406 * -------------------------- 1407 * 1408 * Receive packets are prefixed by an optional 16 byte pseudo-header. 1409 * The pseudo-header is a byte array of one of the forms: 1410 * 1411 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 1412 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 1413 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 1414 * 1415 * where: 1416 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 1417 * LL.LL LFSR hash (16-bit big-endian) 1418 */ 1419 1420 #if EFSYS_OPT_RX_SCALE 1421 static __checkReturn uint32_t 1422 siena_rx_prefix_hash( 1423 __in efx_nic_t *enp, 1424 __in efx_rx_hash_alg_t func, 1425 __in uint8_t *buffer) 1426 { 1427 _NOTE(ARGUNUSED(enp)) 1428 1429 switch (func) { 1430 case EFX_RX_HASHALG_TOEPLITZ: 1431 return ((buffer[12] << 24) | 1432 (buffer[13] << 16) | 1433 (buffer[14] << 8) | 1434 buffer[15]); 1435 1436 case EFX_RX_HASHALG_LFSR: 1437 return ((buffer[14] << 8) | buffer[15]); 1438 1439 default: 1440 EFSYS_ASSERT(0); 1441 return (0); 1442 } 1443 } 1444 #endif /* EFSYS_OPT_RX_SCALE */ 1445 1446 static __checkReturn efx_rc_t 1447 siena_rx_prefix_pktlen( 1448 __in efx_nic_t *enp, 1449 __in uint8_t *buffer, 1450 __out uint16_t *lengthp) 1451 { 1452 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1453 1454 /* Not supported by Falcon/Siena hardware */ 1455 EFSYS_ASSERT(0); 1456 return (ENOTSUP); 1457 } 1458 1459 1460 static void 1461 siena_rx_qpost( 1462 __in efx_rxq_t *erp, 1463 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 1464 __in size_t size, 1465 __in unsigned int ndescs, 1466 __in unsigned int completed, 1467 __in unsigned int added) 1468 { 1469 efx_qword_t qword; 1470 unsigned int i; 1471 unsigned int offset; 1472 unsigned int id; 1473 1474 /* The client driver must not overfill the queue */ 1475 EFSYS_ASSERT3U(added - completed + ndescs, <=, 1476 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1477 1478 id = added & (erp->er_mask); 1479 for (i = 0; i < ndescs; i++) { 1480 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1481 unsigned int, id, efsys_dma_addr_t, addrp[i], 1482 size_t, size); 1483 1484 EFX_POPULATE_QWORD_3(qword, 1485 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1486 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1487 (uint32_t)(addrp[i] & 0xffffffff), 1488 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1489 (uint32_t)(addrp[i] >> 32)); 1490 1491 offset = id * sizeof (efx_qword_t); 1492 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1493 1494 id = (id + 1) & (erp->er_mask); 1495 } 1496 } 1497 1498 static void 1499 siena_rx_qpush( 1500 __in efx_rxq_t *erp, 1501 __in unsigned int added, 1502 __inout unsigned int *pushedp) 1503 { 1504 efx_nic_t *enp = erp->er_enp; 1505 unsigned int pushed = *pushedp; 1506 uint32_t wptr; 1507 efx_oword_t oword; 1508 efx_dword_t dword; 1509 1510 /* All descriptors are pushed */ 1511 *pushedp = added; 1512 1513 /* Push the populated descriptors out */ 1514 wptr = added & erp->er_mask; 1515 1516 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1517 1518 /* Only write the third DWORD */ 1519 EFX_POPULATE_DWORD_1(dword, 1520 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1521 1522 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1523 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1524 wptr, pushed & erp->er_mask); 1525 EFSYS_PIO_WRITE_BARRIER(); 1526 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1527 erp->er_index, &dword, B_FALSE); 1528 } 1529 1530 #if EFSYS_OPT_RX_PACKED_STREAM 1531 static void 1532 siena_rx_qpush_ps_credits( 1533 __in efx_rxq_t *erp) 1534 { 1535 /* Not supported by Siena hardware */ 1536 EFSYS_ASSERT(0); 1537 } 1538 1539 static uint8_t * 1540 siena_rx_qps_packet_info( 1541 __in efx_rxq_t *erp, 1542 __in uint8_t *buffer, 1543 __in uint32_t buffer_length, 1544 __in uint32_t current_offset, 1545 __out uint16_t *lengthp, 1546 __out uint32_t *next_offsetp, 1547 __out uint32_t *timestamp) 1548 { 1549 /* Not supported by Siena hardware */ 1550 EFSYS_ASSERT(0); 1551 1552 return (NULL); 1553 } 1554 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1555 1556 static __checkReturn efx_rc_t 1557 siena_rx_qflush( 1558 __in efx_rxq_t *erp) 1559 { 1560 efx_nic_t *enp = erp->er_enp; 1561 efx_oword_t oword; 1562 uint32_t label; 1563 1564 label = erp->er_index; 1565 1566 /* Flush the queue */ 1567 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1568 FRF_AZ_RX_FLUSH_DESCQ, label); 1569 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1570 1571 return (0); 1572 } 1573 1574 static void 1575 siena_rx_qenable( 1576 __in efx_rxq_t *erp) 1577 { 1578 efx_nic_t *enp = erp->er_enp; 1579 efx_oword_t oword; 1580 1581 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1582 1583 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1584 erp->er_index, &oword, B_TRUE); 1585 1586 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1587 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1588 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1589 1590 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1591 erp->er_index, &oword, B_TRUE); 1592 } 1593 1594 static __checkReturn efx_rc_t 1595 siena_rx_qcreate( 1596 __in efx_nic_t *enp, 1597 __in unsigned int index, 1598 __in unsigned int label, 1599 __in efx_rxq_type_t type, 1600 __in_opt const efx_rxq_type_data_t *type_data, 1601 __in efsys_mem_t *esmp, 1602 __in size_t ndescs, 1603 __in uint32_t id, 1604 __in unsigned int flags, 1605 __in efx_evq_t *eep, 1606 __in efx_rxq_t *erp) 1607 { 1608 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1609 efx_oword_t oword; 1610 uint32_t size; 1611 boolean_t jumbo = B_FALSE; 1612 efx_rc_t rc; 1613 1614 _NOTE(ARGUNUSED(esmp)) 1615 _NOTE(ARGUNUSED(type_data)) 1616 1617 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1618 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1619 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1620 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1621 1622 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 1623 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 1624 1625 if (!ISP2(ndescs) || 1626 (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) { 1627 rc = EINVAL; 1628 goto fail1; 1629 } 1630 if (index >= encp->enc_rxq_limit) { 1631 rc = EINVAL; 1632 goto fail2; 1633 } 1634 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1635 size++) 1636 if ((1 << size) == (int)(ndescs / EFX_RXQ_MINNDESCS)) 1637 break; 1638 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1639 rc = EINVAL; 1640 goto fail3; 1641 } 1642 1643 switch (type) { 1644 case EFX_RXQ_TYPE_DEFAULT: 1645 break; 1646 1647 default: 1648 rc = EINVAL; 1649 goto fail4; 1650 } 1651 1652 if (flags & EFX_RXQ_FLAG_SCATTER) { 1653 #if EFSYS_OPT_RX_SCATTER 1654 jumbo = B_TRUE; 1655 #else 1656 rc = EINVAL; 1657 goto fail5; 1658 #endif /* EFSYS_OPT_RX_SCATTER */ 1659 } 1660 1661 /* Set up the new descriptor queue */ 1662 EFX_POPULATE_OWORD_7(oword, 1663 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1664 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1665 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1666 FRF_AZ_RX_DESCQ_LABEL, label, 1667 FRF_AZ_RX_DESCQ_SIZE, size, 1668 FRF_AZ_RX_DESCQ_TYPE, 0, 1669 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1670 1671 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1672 erp->er_index, &oword, B_TRUE); 1673 1674 return (0); 1675 1676 #if !EFSYS_OPT_RX_SCATTER 1677 fail5: 1678 EFSYS_PROBE(fail5); 1679 #endif 1680 fail4: 1681 EFSYS_PROBE(fail4); 1682 fail3: 1683 EFSYS_PROBE(fail3); 1684 fail2: 1685 EFSYS_PROBE(fail2); 1686 fail1: 1687 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1688 1689 return (rc); 1690 } 1691 1692 static void 1693 siena_rx_qdestroy( 1694 __in efx_rxq_t *erp) 1695 { 1696 efx_nic_t *enp = erp->er_enp; 1697 efx_oword_t oword; 1698 1699 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1700 --enp->en_rx_qcount; 1701 1702 /* Purge descriptor queue */ 1703 EFX_ZERO_OWORD(oword); 1704 1705 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1706 erp->er_index, &oword, B_TRUE); 1707 1708 /* Free the RXQ object */ 1709 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1710 } 1711 1712 static void 1713 siena_rx_fini( 1714 __in efx_nic_t *enp) 1715 { 1716 _NOTE(ARGUNUSED(enp)) 1717 } 1718 1719 #endif /* EFSYS_OPT_SIENA */ 1720