1 /*- 2 * Copyright (c) 2007-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 38 #if EFSYS_OPT_SIENA 39 40 static __checkReturn efx_rc_t 41 siena_rx_init( 42 __in efx_nic_t *enp); 43 44 static void 45 siena_rx_fini( 46 __in efx_nic_t *enp); 47 48 #if EFSYS_OPT_RX_SCATTER 49 static __checkReturn efx_rc_t 50 siena_rx_scatter_enable( 51 __in efx_nic_t *enp, 52 __in unsigned int buf_size); 53 #endif /* EFSYS_OPT_RX_SCATTER */ 54 55 #if EFSYS_OPT_RX_SCALE 56 static __checkReturn efx_rc_t 57 siena_rx_scale_mode_set( 58 __in efx_nic_t *enp, 59 __in efx_rx_hash_alg_t alg, 60 __in efx_rx_hash_type_t type, 61 __in boolean_t insert); 62 63 static __checkReturn efx_rc_t 64 siena_rx_scale_key_set( 65 __in efx_nic_t *enp, 66 __in_ecount(n) uint8_t *key, 67 __in size_t n); 68 69 static __checkReturn efx_rc_t 70 siena_rx_scale_tbl_set( 71 __in efx_nic_t *enp, 72 __in_ecount(n) unsigned int *table, 73 __in size_t n); 74 75 static __checkReturn uint32_t 76 siena_rx_prefix_hash( 77 __in efx_nic_t *enp, 78 __in efx_rx_hash_alg_t func, 79 __in uint8_t *buffer); 80 81 #endif /* EFSYS_OPT_RX_SCALE */ 82 83 static __checkReturn efx_rc_t 84 siena_rx_prefix_pktlen( 85 __in efx_nic_t *enp, 86 __in uint8_t *buffer, 87 __out uint16_t *lengthp); 88 89 static void 90 siena_rx_qpost( 91 __in efx_rxq_t *erp, 92 __in_ecount(n) efsys_dma_addr_t *addrp, 93 __in size_t size, 94 __in unsigned int n, 95 __in unsigned int completed, 96 __in unsigned int added); 97 98 static void 99 siena_rx_qpush( 100 __in efx_rxq_t *erp, 101 __in unsigned int added, 102 __inout unsigned int *pushedp); 103 104 static __checkReturn efx_rc_t 105 siena_rx_qflush( 106 __in efx_rxq_t *erp); 107 108 static void 109 siena_rx_qenable( 110 __in efx_rxq_t *erp); 111 112 static __checkReturn efx_rc_t 113 siena_rx_qcreate( 114 __in efx_nic_t *enp, 115 __in unsigned int index, 116 __in unsigned int label, 117 __in efx_rxq_type_t type, 118 __in efsys_mem_t *esmp, 119 __in size_t n, 120 __in uint32_t id, 121 __in efx_evq_t *eep, 122 __in efx_rxq_t *erp); 123 124 static void 125 siena_rx_qdestroy( 126 __in efx_rxq_t *erp); 127 128 #endif /* EFSYS_OPT_SIENA */ 129 130 131 #if EFSYS_OPT_SIENA 132 static const efx_rx_ops_t __efx_rx_siena_ops = { 133 siena_rx_init, /* erxo_init */ 134 siena_rx_fini, /* erxo_fini */ 135 #if EFSYS_OPT_RX_SCATTER 136 siena_rx_scatter_enable, /* erxo_scatter_enable */ 137 #endif 138 #if EFSYS_OPT_RX_SCALE 139 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 140 siena_rx_scale_key_set, /* erxo_scale_key_set */ 141 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 142 siena_rx_prefix_hash, /* erxo_prefix_hash */ 143 #endif 144 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 145 siena_rx_qpost, /* erxo_qpost */ 146 siena_rx_qpush, /* erxo_qpush */ 147 siena_rx_qflush, /* erxo_qflush */ 148 siena_rx_qenable, /* erxo_qenable */ 149 siena_rx_qcreate, /* erxo_qcreate */ 150 siena_rx_qdestroy, /* erxo_qdestroy */ 151 }; 152 #endif /* EFSYS_OPT_SIENA */ 153 154 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 155 static const efx_rx_ops_t __efx_rx_ef10_ops = { 156 ef10_rx_init, /* erxo_init */ 157 ef10_rx_fini, /* erxo_fini */ 158 #if EFSYS_OPT_RX_SCATTER 159 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 160 #endif 161 #if EFSYS_OPT_RX_SCALE 162 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 163 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 164 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 165 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 166 #endif 167 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 168 ef10_rx_qpost, /* erxo_qpost */ 169 ef10_rx_qpush, /* erxo_qpush */ 170 ef10_rx_qflush, /* erxo_qflush */ 171 ef10_rx_qenable, /* erxo_qenable */ 172 ef10_rx_qcreate, /* erxo_qcreate */ 173 ef10_rx_qdestroy, /* erxo_qdestroy */ 174 }; 175 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 176 177 178 __checkReturn efx_rc_t 179 efx_rx_init( 180 __inout efx_nic_t *enp) 181 { 182 const efx_rx_ops_t *erxop; 183 efx_rc_t rc; 184 185 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 186 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 187 188 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 189 rc = EINVAL; 190 goto fail1; 191 } 192 193 if (enp->en_mod_flags & EFX_MOD_RX) { 194 rc = EINVAL; 195 goto fail2; 196 } 197 198 switch (enp->en_family) { 199 #if EFSYS_OPT_SIENA 200 case EFX_FAMILY_SIENA: 201 erxop = &__efx_rx_siena_ops; 202 break; 203 #endif /* EFSYS_OPT_SIENA */ 204 205 #if EFSYS_OPT_HUNTINGTON 206 case EFX_FAMILY_HUNTINGTON: 207 erxop = &__efx_rx_ef10_ops; 208 break; 209 #endif /* EFSYS_OPT_HUNTINGTON */ 210 211 #if EFSYS_OPT_MEDFORD 212 case EFX_FAMILY_MEDFORD: 213 erxop = &__efx_rx_ef10_ops; 214 break; 215 #endif /* EFSYS_OPT_MEDFORD */ 216 217 default: 218 EFSYS_ASSERT(0); 219 rc = ENOTSUP; 220 goto fail3; 221 } 222 223 if ((rc = erxop->erxo_init(enp)) != 0) 224 goto fail4; 225 226 enp->en_erxop = erxop; 227 enp->en_mod_flags |= EFX_MOD_RX; 228 return (0); 229 230 fail4: 231 EFSYS_PROBE(fail4); 232 fail3: 233 EFSYS_PROBE(fail3); 234 fail2: 235 EFSYS_PROBE(fail2); 236 fail1: 237 EFSYS_PROBE1(fail1, efx_rc_t, rc); 238 239 enp->en_erxop = NULL; 240 enp->en_mod_flags &= ~EFX_MOD_RX; 241 return (rc); 242 } 243 244 void 245 efx_rx_fini( 246 __in efx_nic_t *enp) 247 { 248 const efx_rx_ops_t *erxop = enp->en_erxop; 249 250 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 251 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 252 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 253 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 254 255 erxop->erxo_fini(enp); 256 257 enp->en_erxop = NULL; 258 enp->en_mod_flags &= ~EFX_MOD_RX; 259 } 260 261 #if EFSYS_OPT_RX_SCATTER 262 __checkReturn efx_rc_t 263 efx_rx_scatter_enable( 264 __in efx_nic_t *enp, 265 __in unsigned int buf_size) 266 { 267 const efx_rx_ops_t *erxop = enp->en_erxop; 268 efx_rc_t rc; 269 270 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 271 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 272 273 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 274 goto fail1; 275 276 return (0); 277 278 fail1: 279 EFSYS_PROBE1(fail1, efx_rc_t, rc); 280 return (rc); 281 } 282 #endif /* EFSYS_OPT_RX_SCATTER */ 283 284 #if EFSYS_OPT_RX_SCALE 285 __checkReturn efx_rc_t 286 efx_rx_hash_support_get( 287 __in efx_nic_t *enp, 288 __out efx_rx_hash_support_t *supportp) 289 { 290 efx_rc_t rc; 291 292 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 293 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 294 295 if (supportp == NULL) { 296 rc = EINVAL; 297 goto fail1; 298 } 299 300 /* Report if resources are available to insert RX hash value */ 301 *supportp = enp->en_hash_support; 302 303 return (0); 304 305 fail1: 306 EFSYS_PROBE1(fail1, efx_rc_t, rc); 307 308 return (rc); 309 } 310 311 __checkReturn efx_rc_t 312 efx_rx_scale_support_get( 313 __in efx_nic_t *enp, 314 __out efx_rx_scale_support_t *supportp) 315 { 316 efx_rc_t rc; 317 318 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 319 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 320 321 if (supportp == NULL) { 322 rc = EINVAL; 323 goto fail1; 324 } 325 326 /* Report if resources are available to support RSS */ 327 *supportp = enp->en_rss_support; 328 329 return (0); 330 331 fail1: 332 EFSYS_PROBE1(fail1, efx_rc_t, rc); 333 334 return (rc); 335 } 336 337 __checkReturn efx_rc_t 338 efx_rx_scale_mode_set( 339 __in efx_nic_t *enp, 340 __in efx_rx_hash_alg_t alg, 341 __in efx_rx_hash_type_t type, 342 __in boolean_t insert) 343 { 344 const efx_rx_ops_t *erxop = enp->en_erxop; 345 efx_rc_t rc; 346 347 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 348 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 349 350 if (erxop->erxo_scale_mode_set != NULL) { 351 if ((rc = erxop->erxo_scale_mode_set(enp, alg, 352 type, insert)) != 0) 353 goto fail1; 354 } 355 356 return (0); 357 358 fail1: 359 EFSYS_PROBE1(fail1, efx_rc_t, rc); 360 return (rc); 361 } 362 #endif /* EFSYS_OPT_RX_SCALE */ 363 364 #if EFSYS_OPT_RX_SCALE 365 __checkReturn efx_rc_t 366 efx_rx_scale_key_set( 367 __in efx_nic_t *enp, 368 __in_ecount(n) uint8_t *key, 369 __in size_t n) 370 { 371 const efx_rx_ops_t *erxop = enp->en_erxop; 372 efx_rc_t rc; 373 374 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 375 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 376 377 if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0) 378 goto fail1; 379 380 return (0); 381 382 fail1: 383 EFSYS_PROBE1(fail1, efx_rc_t, rc); 384 385 return (rc); 386 } 387 #endif /* EFSYS_OPT_RX_SCALE */ 388 389 #if EFSYS_OPT_RX_SCALE 390 __checkReturn efx_rc_t 391 efx_rx_scale_tbl_set( 392 __in efx_nic_t *enp, 393 __in_ecount(n) unsigned int *table, 394 __in size_t n) 395 { 396 const efx_rx_ops_t *erxop = enp->en_erxop; 397 efx_rc_t rc; 398 399 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 400 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 401 402 if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0) 403 goto fail1; 404 405 return (0); 406 407 fail1: 408 EFSYS_PROBE1(fail1, efx_rc_t, rc); 409 410 return (rc); 411 } 412 #endif /* EFSYS_OPT_RX_SCALE */ 413 414 void 415 efx_rx_qpost( 416 __in efx_rxq_t *erp, 417 __in_ecount(n) efsys_dma_addr_t *addrp, 418 __in size_t size, 419 __in unsigned int n, 420 __in unsigned int completed, 421 __in unsigned int added) 422 { 423 efx_nic_t *enp = erp->er_enp; 424 const efx_rx_ops_t *erxop = enp->en_erxop; 425 426 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 427 428 erxop->erxo_qpost(erp, addrp, size, n, completed, added); 429 } 430 431 void 432 efx_rx_qpush( 433 __in efx_rxq_t *erp, 434 __in unsigned int added, 435 __inout unsigned int *pushedp) 436 { 437 efx_nic_t *enp = erp->er_enp; 438 const efx_rx_ops_t *erxop = enp->en_erxop; 439 440 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 441 442 erxop->erxo_qpush(erp, added, pushedp); 443 } 444 445 __checkReturn efx_rc_t 446 efx_rx_qflush( 447 __in efx_rxq_t *erp) 448 { 449 efx_nic_t *enp = erp->er_enp; 450 const efx_rx_ops_t *erxop = enp->en_erxop; 451 efx_rc_t rc; 452 453 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 454 455 if ((rc = erxop->erxo_qflush(erp)) != 0) 456 goto fail1; 457 458 return (0); 459 460 fail1: 461 EFSYS_PROBE1(fail1, efx_rc_t, rc); 462 463 return (rc); 464 } 465 466 void 467 efx_rx_qenable( 468 __in efx_rxq_t *erp) 469 { 470 efx_nic_t *enp = erp->er_enp; 471 const efx_rx_ops_t *erxop = enp->en_erxop; 472 473 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 474 475 erxop->erxo_qenable(erp); 476 } 477 478 __checkReturn efx_rc_t 479 efx_rx_qcreate( 480 __in efx_nic_t *enp, 481 __in unsigned int index, 482 __in unsigned int label, 483 __in efx_rxq_type_t type, 484 __in efsys_mem_t *esmp, 485 __in size_t n, 486 __in uint32_t id, 487 __in efx_evq_t *eep, 488 __deref_out efx_rxq_t **erpp) 489 { 490 const efx_rx_ops_t *erxop = enp->en_erxop; 491 efx_rxq_t *erp; 492 efx_rc_t rc; 493 494 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 495 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 496 497 /* Allocate an RXQ object */ 498 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 499 500 if (erp == NULL) { 501 rc = ENOMEM; 502 goto fail1; 503 } 504 505 erp->er_magic = EFX_RXQ_MAGIC; 506 erp->er_enp = enp; 507 erp->er_index = index; 508 erp->er_mask = n - 1; 509 erp->er_esmp = esmp; 510 511 if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id, 512 eep, erp)) != 0) 513 goto fail2; 514 515 enp->en_rx_qcount++; 516 *erpp = erp; 517 518 return (0); 519 520 fail2: 521 EFSYS_PROBE(fail2); 522 523 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 524 fail1: 525 EFSYS_PROBE1(fail1, efx_rc_t, rc); 526 527 return (rc); 528 } 529 530 void 531 efx_rx_qdestroy( 532 __in efx_rxq_t *erp) 533 { 534 efx_nic_t *enp = erp->er_enp; 535 const efx_rx_ops_t *erxop = enp->en_erxop; 536 537 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 538 539 erxop->erxo_qdestroy(erp); 540 } 541 542 __checkReturn efx_rc_t 543 efx_psuedo_hdr_pkt_length_get( 544 __in efx_nic_t *enp, 545 __in uint8_t *buffer, 546 __out uint16_t *lengthp) 547 { 548 const efx_rx_ops_t *erxop = enp->en_erxop; 549 550 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 551 } 552 553 #if EFSYS_OPT_RX_SCALE 554 __checkReturn uint32_t 555 efx_psuedo_hdr_hash_get( 556 __in efx_nic_t *enp, 557 __in efx_rx_hash_alg_t func, 558 __in uint8_t *buffer) 559 { 560 const efx_rx_ops_t *erxop = enp->en_erxop; 561 562 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 563 return (erxop->erxo_prefix_hash(enp, func, buffer)); 564 } 565 #endif /* EFSYS_OPT_RX_SCALE */ 566 567 #if EFSYS_OPT_SIENA 568 569 static __checkReturn efx_rc_t 570 siena_rx_init( 571 __in efx_nic_t *enp) 572 { 573 efx_oword_t oword; 574 unsigned int index; 575 576 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 577 578 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 579 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 580 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 581 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 582 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 583 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 584 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 585 586 /* Zero the RSS table */ 587 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 588 index++) { 589 EFX_ZERO_OWORD(oword); 590 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 591 index, &oword, B_TRUE); 592 } 593 594 #if EFSYS_OPT_RX_SCALE 595 /* The RSS key and indirection table are writable. */ 596 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; 597 598 /* Hardware can insert RX hash with/without RSS */ 599 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 600 #endif /* EFSYS_OPT_RX_SCALE */ 601 602 return (0); 603 } 604 605 #if EFSYS_OPT_RX_SCATTER 606 static __checkReturn efx_rc_t 607 siena_rx_scatter_enable( 608 __in efx_nic_t *enp, 609 __in unsigned int buf_size) 610 { 611 unsigned int nbuf32; 612 efx_oword_t oword; 613 efx_rc_t rc; 614 615 nbuf32 = buf_size / 32; 616 if ((nbuf32 == 0) || 617 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 618 ((buf_size % 32) != 0)) { 619 rc = EINVAL; 620 goto fail1; 621 } 622 623 if (enp->en_rx_qcount > 0) { 624 rc = EBUSY; 625 goto fail2; 626 } 627 628 /* Set scatter buffer size */ 629 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 630 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 631 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 632 633 /* Enable scatter for packets not matching a filter */ 634 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 635 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 636 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 637 638 return (0); 639 640 fail2: 641 EFSYS_PROBE(fail2); 642 fail1: 643 EFSYS_PROBE1(fail1, efx_rc_t, rc); 644 645 return (rc); 646 } 647 #endif /* EFSYS_OPT_RX_SCATTER */ 648 649 650 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 651 do { \ 652 efx_oword_t oword; \ 653 \ 654 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 655 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 656 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 657 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 658 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 659 (_insert) ? 1 : 0); \ 660 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 661 \ 662 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 663 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 664 &oword); \ 665 EFX_SET_OWORD_FIELD(oword, \ 666 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 667 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 668 &oword); \ 669 } \ 670 \ 671 _NOTE(CONSTANTCONDITION) \ 672 } while (B_FALSE) 673 674 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 675 do { \ 676 efx_oword_t oword; \ 677 \ 678 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 679 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 680 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 681 (_ip) ? 1 : 0); \ 682 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 683 (_tcp) ? 0 : 1); \ 684 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 685 (_insert) ? 1 : 0); \ 686 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 687 \ 688 _NOTE(CONSTANTCONDITION) \ 689 } while (B_FALSE) 690 691 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 692 do { \ 693 efx_oword_t oword; \ 694 \ 695 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 696 EFX_SET_OWORD_FIELD(oword, \ 697 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 698 EFX_SET_OWORD_FIELD(oword, \ 699 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 700 EFX_SET_OWORD_FIELD(oword, \ 701 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 702 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 703 \ 704 (_rc) = 0; \ 705 \ 706 _NOTE(CONSTANTCONDITION) \ 707 } while (B_FALSE) 708 709 710 #if EFSYS_OPT_RX_SCALE 711 712 static __checkReturn efx_rc_t 713 siena_rx_scale_mode_set( 714 __in efx_nic_t *enp, 715 __in efx_rx_hash_alg_t alg, 716 __in efx_rx_hash_type_t type, 717 __in boolean_t insert) 718 { 719 efx_rc_t rc; 720 721 switch (alg) { 722 case EFX_RX_HASHALG_LFSR: 723 EFX_RX_LFSR_HASH(enp, insert); 724 break; 725 726 case EFX_RX_HASHALG_TOEPLITZ: 727 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 728 type & (1 << EFX_RX_HASH_IPV4), 729 type & (1 << EFX_RX_HASH_TCPIPV4)); 730 731 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 732 type & (1 << EFX_RX_HASH_IPV6), 733 type & (1 << EFX_RX_HASH_TCPIPV6), 734 rc); 735 if (rc != 0) 736 goto fail1; 737 738 break; 739 740 default: 741 rc = EINVAL; 742 goto fail2; 743 } 744 745 return (0); 746 747 fail2: 748 EFSYS_PROBE(fail2); 749 fail1: 750 EFSYS_PROBE1(fail1, efx_rc_t, rc); 751 752 EFX_RX_LFSR_HASH(enp, B_FALSE); 753 754 return (rc); 755 } 756 #endif 757 758 #if EFSYS_OPT_RX_SCALE 759 static __checkReturn efx_rc_t 760 siena_rx_scale_key_set( 761 __in efx_nic_t *enp, 762 __in_ecount(n) uint8_t *key, 763 __in size_t n) 764 { 765 efx_oword_t oword; 766 unsigned int byte; 767 unsigned int offset; 768 efx_rc_t rc; 769 770 byte = 0; 771 772 /* Write Toeplitz IPv4 hash key */ 773 EFX_ZERO_OWORD(oword); 774 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 775 offset > 0 && byte < n; 776 --offset) 777 oword.eo_u8[offset - 1] = key[byte++]; 778 779 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 780 781 byte = 0; 782 783 /* Verify Toeplitz IPv4 hash key */ 784 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 785 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 786 offset > 0 && byte < n; 787 --offset) { 788 if (oword.eo_u8[offset - 1] != key[byte++]) { 789 rc = EFAULT; 790 goto fail1; 791 } 792 } 793 794 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 795 goto done; 796 797 byte = 0; 798 799 /* Write Toeplitz IPv6 hash key 3 */ 800 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 801 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 802 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 803 offset > 0 && byte < n; 804 --offset) 805 oword.eo_u8[offset - 1] = key[byte++]; 806 807 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 808 809 /* Write Toeplitz IPv6 hash key 2 */ 810 EFX_ZERO_OWORD(oword); 811 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 812 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 813 offset > 0 && byte < n; 814 --offset) 815 oword.eo_u8[offset - 1] = key[byte++]; 816 817 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 818 819 /* Write Toeplitz IPv6 hash key 1 */ 820 EFX_ZERO_OWORD(oword); 821 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 822 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 823 offset > 0 && byte < n; 824 --offset) 825 oword.eo_u8[offset - 1] = key[byte++]; 826 827 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 828 829 byte = 0; 830 831 /* Verify Toeplitz IPv6 hash key 3 */ 832 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 833 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 834 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 835 offset > 0 && byte < n; 836 --offset) { 837 if (oword.eo_u8[offset - 1] != key[byte++]) { 838 rc = EFAULT; 839 goto fail2; 840 } 841 } 842 843 /* Verify Toeplitz IPv6 hash key 2 */ 844 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 845 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 846 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 847 offset > 0 && byte < n; 848 --offset) { 849 if (oword.eo_u8[offset - 1] != key[byte++]) { 850 rc = EFAULT; 851 goto fail3; 852 } 853 } 854 855 /* Verify Toeplitz IPv6 hash key 1 */ 856 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 857 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 858 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 859 offset > 0 && byte < n; 860 --offset) { 861 if (oword.eo_u8[offset - 1] != key[byte++]) { 862 rc = EFAULT; 863 goto fail4; 864 } 865 } 866 867 done: 868 return (0); 869 870 fail4: 871 EFSYS_PROBE(fail4); 872 fail3: 873 EFSYS_PROBE(fail3); 874 fail2: 875 EFSYS_PROBE(fail2); 876 fail1: 877 EFSYS_PROBE1(fail1, efx_rc_t, rc); 878 879 return (rc); 880 } 881 #endif 882 883 #if EFSYS_OPT_RX_SCALE 884 static __checkReturn efx_rc_t 885 siena_rx_scale_tbl_set( 886 __in efx_nic_t *enp, 887 __in_ecount(n) unsigned int *table, 888 __in size_t n) 889 { 890 efx_oword_t oword; 891 int index; 892 efx_rc_t rc; 893 894 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 895 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 896 897 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 898 rc = EINVAL; 899 goto fail1; 900 } 901 902 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 903 uint32_t byte; 904 905 /* Calculate the entry to place in the table */ 906 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 907 908 EFSYS_PROBE2(table, int, index, uint32_t, byte); 909 910 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 911 912 /* Write the table */ 913 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 914 index, &oword, B_TRUE); 915 } 916 917 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 918 uint32_t byte; 919 920 /* Determine if we're starting a new batch */ 921 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 922 923 /* Read the table */ 924 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 925 index, &oword, B_TRUE); 926 927 /* Verify the entry */ 928 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 929 rc = EFAULT; 930 goto fail2; 931 } 932 } 933 934 return (0); 935 936 fail2: 937 EFSYS_PROBE(fail2); 938 fail1: 939 EFSYS_PROBE1(fail1, efx_rc_t, rc); 940 941 return (rc); 942 } 943 #endif 944 945 /* 946 * Falcon/Siena psuedo-header 947 * -------------------------- 948 * 949 * Receive packets are prefixed by an optional 16 byte pseudo-header. 950 * The psuedo-header is a byte array of one of the forms: 951 * 952 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 953 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 954 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 955 * 956 * where: 957 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 958 * LL.LL LFSR hash (16-bit big-endian) 959 */ 960 961 #if EFSYS_OPT_RX_SCALE 962 static __checkReturn uint32_t 963 siena_rx_prefix_hash( 964 __in efx_nic_t *enp, 965 __in efx_rx_hash_alg_t func, 966 __in uint8_t *buffer) 967 { 968 _NOTE(ARGUNUSED(enp)) 969 970 switch (func) { 971 case EFX_RX_HASHALG_TOEPLITZ: 972 return ((buffer[12] << 24) | 973 (buffer[13] << 16) | 974 (buffer[14] << 8) | 975 buffer[15]); 976 977 case EFX_RX_HASHALG_LFSR: 978 return ((buffer[14] << 8) | buffer[15]); 979 980 default: 981 EFSYS_ASSERT(0); 982 return (0); 983 } 984 } 985 #endif /* EFSYS_OPT_RX_SCALE */ 986 987 static __checkReturn efx_rc_t 988 siena_rx_prefix_pktlen( 989 __in efx_nic_t *enp, 990 __in uint8_t *buffer, 991 __out uint16_t *lengthp) 992 { 993 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 994 995 /* Not supported by Falcon/Siena hardware */ 996 EFSYS_ASSERT(0); 997 return (ENOTSUP); 998 } 999 1000 1001 static void 1002 siena_rx_qpost( 1003 __in efx_rxq_t *erp, 1004 __in_ecount(n) efsys_dma_addr_t *addrp, 1005 __in size_t size, 1006 __in unsigned int n, 1007 __in unsigned int completed, 1008 __in unsigned int added) 1009 { 1010 efx_qword_t qword; 1011 unsigned int i; 1012 unsigned int offset; 1013 unsigned int id; 1014 1015 /* The client driver must not overfill the queue */ 1016 EFSYS_ASSERT3U(added - completed + n, <=, 1017 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1018 1019 id = added & (erp->er_mask); 1020 for (i = 0; i < n; i++) { 1021 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1022 unsigned int, id, efsys_dma_addr_t, addrp[i], 1023 size_t, size); 1024 1025 EFX_POPULATE_QWORD_3(qword, 1026 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1027 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1028 (uint32_t)(addrp[i] & 0xffffffff), 1029 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1030 (uint32_t)(addrp[i] >> 32)); 1031 1032 offset = id * sizeof (efx_qword_t); 1033 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1034 1035 id = (id + 1) & (erp->er_mask); 1036 } 1037 } 1038 1039 static void 1040 siena_rx_qpush( 1041 __in efx_rxq_t *erp, 1042 __in unsigned int added, 1043 __inout unsigned int *pushedp) 1044 { 1045 efx_nic_t *enp = erp->er_enp; 1046 unsigned int pushed = *pushedp; 1047 uint32_t wptr; 1048 efx_oword_t oword; 1049 efx_dword_t dword; 1050 1051 /* All descriptors are pushed */ 1052 *pushedp = added; 1053 1054 /* Push the populated descriptors out */ 1055 wptr = added & erp->er_mask; 1056 1057 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1058 1059 /* Only write the third DWORD */ 1060 EFX_POPULATE_DWORD_1(dword, 1061 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1062 1063 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1064 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1065 wptr, pushed & erp->er_mask); 1066 EFSYS_PIO_WRITE_BARRIER(); 1067 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1068 erp->er_index, &dword, B_FALSE); 1069 } 1070 1071 static __checkReturn efx_rc_t 1072 siena_rx_qflush( 1073 __in efx_rxq_t *erp) 1074 { 1075 efx_nic_t *enp = erp->er_enp; 1076 efx_oword_t oword; 1077 uint32_t label; 1078 1079 label = erp->er_index; 1080 1081 /* Flush the queue */ 1082 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1083 FRF_AZ_RX_FLUSH_DESCQ, label); 1084 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1085 1086 return (0); 1087 } 1088 1089 static void 1090 siena_rx_qenable( 1091 __in efx_rxq_t *erp) 1092 { 1093 efx_nic_t *enp = erp->er_enp; 1094 efx_oword_t oword; 1095 1096 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1097 1098 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1099 erp->er_index, &oword, B_TRUE); 1100 1101 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1102 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1103 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1104 1105 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1106 erp->er_index, &oword, B_TRUE); 1107 } 1108 1109 static __checkReturn efx_rc_t 1110 siena_rx_qcreate( 1111 __in efx_nic_t *enp, 1112 __in unsigned int index, 1113 __in unsigned int label, 1114 __in efx_rxq_type_t type, 1115 __in efsys_mem_t *esmp, 1116 __in size_t n, 1117 __in uint32_t id, 1118 __in efx_evq_t *eep, 1119 __in efx_rxq_t *erp) 1120 { 1121 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1122 efx_oword_t oword; 1123 uint32_t size; 1124 boolean_t jumbo; 1125 efx_rc_t rc; 1126 1127 _NOTE(ARGUNUSED(esmp)) 1128 1129 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1130 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1131 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1132 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1133 1134 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 1135 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 1136 1137 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 1138 rc = EINVAL; 1139 goto fail1; 1140 } 1141 if (index >= encp->enc_rxq_limit) { 1142 rc = EINVAL; 1143 goto fail2; 1144 } 1145 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 1146 size++) 1147 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS)) 1148 break; 1149 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1150 rc = EINVAL; 1151 goto fail3; 1152 } 1153 1154 switch (type) { 1155 case EFX_RXQ_TYPE_DEFAULT: 1156 jumbo = B_FALSE; 1157 break; 1158 1159 #if EFSYS_OPT_RX_SCATTER 1160 case EFX_RXQ_TYPE_SCATTER: 1161 if (enp->en_family < EFX_FAMILY_SIENA) { 1162 rc = EINVAL; 1163 goto fail4; 1164 } 1165 jumbo = B_TRUE; 1166 break; 1167 #endif /* EFSYS_OPT_RX_SCATTER */ 1168 1169 default: 1170 rc = EINVAL; 1171 goto fail4; 1172 } 1173 1174 /* Set up the new descriptor queue */ 1175 EFX_POPULATE_OWORD_7(oword, 1176 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1177 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1178 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1179 FRF_AZ_RX_DESCQ_LABEL, label, 1180 FRF_AZ_RX_DESCQ_SIZE, size, 1181 FRF_AZ_RX_DESCQ_TYPE, 0, 1182 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1183 1184 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1185 erp->er_index, &oword, B_TRUE); 1186 1187 return (0); 1188 1189 fail4: 1190 EFSYS_PROBE(fail4); 1191 fail3: 1192 EFSYS_PROBE(fail3); 1193 fail2: 1194 EFSYS_PROBE(fail2); 1195 fail1: 1196 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1197 1198 return (rc); 1199 } 1200 1201 static void 1202 siena_rx_qdestroy( 1203 __in efx_rxq_t *erp) 1204 { 1205 efx_nic_t *enp = erp->er_enp; 1206 efx_oword_t oword; 1207 1208 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1209 --enp->en_rx_qcount; 1210 1211 /* Purge descriptor queue */ 1212 EFX_ZERO_OWORD(oword); 1213 1214 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1215 erp->er_index, &oword, B_TRUE); 1216 1217 /* Free the RXQ object */ 1218 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1219 } 1220 1221 static void 1222 siena_rx_fini( 1223 __in efx_nic_t *enp) 1224 { 1225 _NOTE(ARGUNUSED(enp)) 1226 } 1227 1228 #endif /* EFSYS_OPT_SIENA */ 1229