1 /*- 2 * Copyright 2007-2009 Solarflare Communications Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include <sys/cdefs.h> 27 __FBSDID("$FreeBSD$"); 28 29 #include "efsys.h" 30 #include "efx.h" 31 #include "efx_types.h" 32 #include "efx_regs.h" 33 #include "efx_impl.h" 34 35 __checkReturn int 36 efx_rx_init( 37 __in efx_nic_t *enp) 38 { 39 efx_oword_t oword; 40 unsigned int index; 41 int rc; 42 43 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 44 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 45 46 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 47 rc = EINVAL; 48 goto fail1; 49 } 50 51 if (enp->en_mod_flags & EFX_MOD_RX) { 52 rc = EINVAL; 53 goto fail2; 54 } 55 56 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 57 58 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 59 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 60 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 61 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 62 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 63 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 64 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 65 66 /* Zero the RSS table */ 67 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 68 index++) { 69 EFX_ZERO_OWORD(oword); 70 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 71 index, &oword); 72 } 73 74 enp->en_mod_flags |= EFX_MOD_RX; 75 return (0); 76 77 fail2: 78 EFSYS_PROBE(fail2); 79 fail1: 80 EFSYS_PROBE1(fail1, int, rc); 81 82 return (rc); 83 } 84 85 #if EFSYS_OPT_RX_HDR_SPLIT 86 __checkReturn int 87 efx_rx_hdr_split_enable( 88 __in efx_nic_t *enp, 89 __in unsigned int hdr_buf_size, 90 __in unsigned int pld_buf_size) 91 { 92 unsigned int nhdr32; 93 unsigned int npld32; 94 efx_oword_t oword; 95 int rc; 96 97 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 98 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 99 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA); 100 101 nhdr32 = hdr_buf_size / 32; 102 if ((nhdr32 == 0) || 103 (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) || 104 ((hdr_buf_size % 32) != 0)) { 105 rc = EINVAL; 106 goto fail1; 107 } 108 109 npld32 = pld_buf_size / 32; 110 if ((npld32 == 0) || 111 (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) || 112 ((pld_buf_size % 32) != 0)) { 113 rc = EINVAL; 114 goto fail2; 115 } 116 117 if (enp->en_rx_qcount > 0) { 118 rc = EBUSY; 119 goto fail3; 120 } 121 122 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 123 124 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1); 125 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32); 126 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32); 127 128 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 129 130 return (0); 131 132 fail3: 133 EFSYS_PROBE(fail3); 134 fail2: 135 EFSYS_PROBE(fail2); 136 fail1: 137 EFSYS_PROBE1(fail1, int, rc); 138 139 return (rc); 140 } 141 #endif /* EFSYS_OPT_RX_HDR_SPLIT */ 142 143 144 #if EFSYS_OPT_RX_SCATTER 145 __checkReturn int 146 efx_rx_scatter_enable( 147 __in efx_nic_t *enp, 148 __in unsigned int buf_size) 149 { 150 unsigned int nbuf32; 151 efx_oword_t oword; 152 int rc; 153 154 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 155 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 156 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON); 157 158 nbuf32 = buf_size / 32; 159 if ((nbuf32 == 0) || 160 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 161 ((buf_size % 32) != 0)) { 162 rc = EINVAL; 163 goto fail1; 164 } 165 166 if (enp->en_rx_qcount > 0) { 167 rc = EBUSY; 168 goto fail2; 169 } 170 171 /* Set scatter buffer size */ 172 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 173 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 174 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 175 176 /* Enable scatter for packets not matching a filter */ 177 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 178 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 179 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 180 181 return (0); 182 183 fail2: 184 EFSYS_PROBE(fail2); 185 fail1: 186 EFSYS_PROBE1(fail1, int, rc); 187 188 return (rc); 189 } 190 #endif /* EFSYS_OPT_RX_SCATTER */ 191 192 193 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 194 do { \ 195 efx_oword_t oword; \ 196 \ 197 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 198 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 199 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 200 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 201 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 202 (_insert) ? 1 : 0); \ 203 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 204 \ 205 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 206 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 207 &oword); \ 208 EFX_SET_OWORD_FIELD(oword, \ 209 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 210 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 211 &oword); \ 212 } \ 213 \ 214 _NOTE(CONSTANTCONDITION) \ 215 } while (B_FALSE) 216 217 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 218 do { \ 219 efx_oword_t oword; \ 220 \ 221 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 222 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 223 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 224 (_ip) ? 1 : 0); \ 225 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 226 (_tcp) ? 0 : 1); \ 227 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 228 (_insert) ? 1 : 0); \ 229 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 230 \ 231 _NOTE(CONSTANTCONDITION) \ 232 } while (B_FALSE) 233 234 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 235 do { \ 236 efx_oword_t oword; \ 237 \ 238 if ((_enp)->en_family == EFX_FAMILY_FALCON) { \ 239 (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0; \ 240 break; \ 241 } \ 242 \ 243 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 244 EFX_SET_OWORD_FIELD(oword, \ 245 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 246 EFX_SET_OWORD_FIELD(oword, \ 247 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 248 EFX_SET_OWORD_FIELD(oword, \ 249 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 250 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 251 \ 252 (_rc) = 0; \ 253 \ 254 _NOTE(CONSTANTCONDITION) \ 255 } while (B_FALSE) 256 257 258 #if EFSYS_OPT_RX_SCALE 259 __checkReturn int 260 efx_rx_scale_mode_set( 261 __in efx_nic_t *enp, 262 __in efx_rx_hash_alg_t alg, 263 __in efx_rx_hash_type_t type, 264 __in boolean_t insert) 265 { 266 int rc; 267 268 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 269 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 270 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON); 271 272 switch (alg) { 273 case EFX_RX_HASHALG_LFSR: 274 EFX_RX_LFSR_HASH(enp, insert); 275 break; 276 277 case EFX_RX_HASHALG_TOEPLITZ: 278 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 279 type & (1 << EFX_RX_HASH_IPV4), 280 type & (1 << EFX_RX_HASH_TCPIPV4)); 281 282 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 283 type & (1 << EFX_RX_HASH_IPV6), 284 type & (1 << EFX_RX_HASH_TCPIPV6), 285 rc); 286 if (rc != 0) 287 goto fail1; 288 289 break; 290 291 default: 292 rc = EINVAL; 293 goto fail2; 294 } 295 296 return (0); 297 298 fail2: 299 EFSYS_PROBE(fail2); 300 fail1: 301 EFSYS_PROBE1(fail1, int, rc); 302 303 EFX_RX_LFSR_HASH(enp, B_FALSE); 304 305 return (rc); 306 } 307 #endif 308 309 #if EFSYS_OPT_RX_SCALE 310 __checkReturn int 311 efx_rx_scale_toeplitz_ipv4_key_set( 312 __in efx_nic_t *enp, 313 __in_ecount(n) uint8_t *key, 314 __in size_t n) 315 { 316 efx_oword_t oword; 317 unsigned int byte; 318 unsigned int offset; 319 int rc; 320 321 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 322 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 323 324 byte = 0; 325 326 /* Write toeplitz hash key */ 327 EFX_ZERO_OWORD(oword); 328 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 329 offset > 0 && byte < n; 330 --offset) 331 oword.eo_u8[offset - 1] = key[byte++]; 332 333 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 334 335 byte = 0; 336 337 /* Verify toeplitz hash key */ 338 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 339 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 340 offset > 0 && byte < n; 341 --offset) { 342 if (oword.eo_u8[offset - 1] != key[byte++]) { 343 rc = EFAULT; 344 goto fail1; 345 } 346 } 347 348 return (0); 349 350 fail1: 351 EFSYS_PROBE1(fail1, int, rc); 352 353 return (rc); 354 } 355 #endif 356 357 #if EFSYS_OPT_RX_SCALE 358 __checkReturn int 359 efx_rx_scale_toeplitz_ipv6_key_set( 360 __in efx_nic_t *enp, 361 __in_ecount(n) uint8_t *key, 362 __in size_t n) 363 { 364 efx_oword_t oword; 365 unsigned int byte; 366 int offset; 367 int rc; 368 369 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 370 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 371 372 byte = 0; 373 374 /* Write toeplitz hash key 3 */ 375 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 376 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 377 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 378 offset > 0 && byte < n; 379 --offset) 380 oword.eo_u8[offset - 1] = key[byte++]; 381 382 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 383 384 /* Write toeplitz hash key 2 */ 385 EFX_ZERO_OWORD(oword); 386 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 387 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 388 offset > 0 && byte < n; 389 --offset) 390 oword.eo_u8[offset - 1] = key[byte++]; 391 392 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 393 394 /* Write toeplitz hash key 1 */ 395 EFX_ZERO_OWORD(oword); 396 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 397 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 398 offset > 0 && byte < n; 399 --offset) 400 oword.eo_u8[offset - 1] = key[byte++]; 401 402 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 403 404 byte = 0; 405 406 /* Verify toeplitz hash key 3 */ 407 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 408 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 409 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 410 offset > 0 && byte < n; 411 --offset) { 412 if (oword.eo_u8[offset - 1] != key[byte++]) { 413 rc = EFAULT; 414 goto fail1; 415 } 416 } 417 418 /* Verify toeplitz hash key 2 */ 419 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 420 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 421 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 422 offset > 0 && byte < n; 423 --offset) { 424 if (oword.eo_u8[offset - 1] != key[byte++]) { 425 rc = EFAULT; 426 goto fail2; 427 } 428 } 429 430 /* Verify toeplitz hash key 1 */ 431 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 432 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 433 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 434 offset > 0 && byte < n; 435 --offset) { 436 if (oword.eo_u8[offset - 1] != key[byte++]) { 437 rc = EFAULT; 438 goto fail3; 439 } 440 } 441 442 return (0); 443 444 fail3: 445 EFSYS_PROBE(fail3); 446 fail2: 447 EFSYS_PROBE(fail2); 448 fail1: 449 EFSYS_PROBE1(fail1, int, rc); 450 451 return (rc); 452 } 453 #endif 454 455 #if EFSYS_OPT_RX_SCALE 456 __checkReturn int 457 efx_rx_scale_tbl_set( 458 __in efx_nic_t *enp, 459 __in_ecount(n) unsigned int *table, 460 __in size_t n) 461 { 462 efx_oword_t oword; 463 int index; 464 int rc; 465 466 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 467 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 468 469 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 470 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 471 472 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 473 rc = EINVAL; 474 goto fail1; 475 } 476 477 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 478 uint32_t byte; 479 480 /* Calculate the entry to place in the table */ 481 byte = (uint32_t)table[index % n]; 482 483 EFSYS_PROBE2(table, int, index, uint32_t, byte); 484 485 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 486 487 /* Write the table */ 488 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 489 index, &oword); 490 } 491 492 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 493 uint32_t byte; 494 495 /* Determine if we're starting a new batch */ 496 byte = (uint32_t)table[index % n]; 497 498 /* Read the table */ 499 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 500 index, &oword); 501 502 /* Verify the entry */ 503 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 504 rc = EFAULT; 505 goto fail2; 506 } 507 } 508 509 return (0); 510 511 fail2: 512 EFSYS_PROBE(fail2); 513 fail1: 514 EFSYS_PROBE1(fail1, int, rc); 515 516 return (rc); 517 } 518 #endif 519 520 #if EFSYS_OPT_FILTER 521 extern __checkReturn int 522 efx_rx_filter_insert( 523 __in efx_rxq_t *erp, 524 __inout efx_filter_spec_t *spec) 525 { 526 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 527 EFSYS_ASSERT3P(spec, !=, NULL); 528 529 spec->efs_dmaq_id = (uint16_t)erp->er_index; 530 return (efx_filter_insert_filter(erp->er_enp, spec, B_FALSE)); 531 } 532 #endif 533 534 #if EFSYS_OPT_FILTER 535 extern __checkReturn int 536 efx_rx_filter_remove( 537 __in efx_rxq_t *erp, 538 __inout efx_filter_spec_t *spec) 539 { 540 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 541 EFSYS_ASSERT3P(spec, !=, NULL); 542 543 spec->efs_dmaq_id = (uint16_t)erp->er_index; 544 return (efx_filter_remove_filter(erp->er_enp, spec)); 545 } 546 #endif 547 548 extern void 549 efx_rx_qpost( 550 __in efx_rxq_t *erp, 551 __in_ecount(n) efsys_dma_addr_t *addrp, 552 __in size_t size, 553 __in unsigned int n, 554 __in unsigned int completed, 555 __in unsigned int added) 556 { 557 efx_qword_t qword; 558 unsigned int i; 559 unsigned int offset; 560 unsigned int id; 561 562 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 563 564 /* The client driver must not overfill the queue */ 565 EFSYS_ASSERT3U(added - completed + n, <=, 566 EFX_RXQ_LIMIT(erp->er_mask + 1)); 567 568 id = added & (erp->er_mask); 569 for (i = 0; i < n; i++) { 570 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 571 unsigned int, id, efsys_dma_addr_t, addrp[i], 572 size_t, size); 573 574 EFX_POPULATE_QWORD_3(qword, 575 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 576 FSF_AZ_RX_KER_BUF_ADDR_DW0, 577 (uint32_t)(addrp[i] & 0xffffffff), 578 FSF_AZ_RX_KER_BUF_ADDR_DW1, 579 (uint32_t)(addrp[i] >> 32)); 580 581 offset = id * sizeof (efx_qword_t); 582 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 583 584 id = (id + 1) & (erp->er_mask); 585 } 586 } 587 588 void 589 efx_rx_qpush( 590 __in efx_rxq_t *erp, 591 __in unsigned int added) 592 { 593 efx_nic_t *enp = erp->er_enp; 594 uint32_t wptr; 595 efx_oword_t oword; 596 efx_dword_t dword; 597 598 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 599 600 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 601 EFSYS_PIO_WRITE_BARRIER(); 602 603 /* Push the populated descriptors out */ 604 wptr = added & erp->er_mask; 605 606 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 607 608 /* Only write the third DWORD */ 609 EFX_POPULATE_DWORD_1(dword, 610 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 611 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 612 erp->er_index, &dword, B_FALSE); 613 } 614 615 void 616 efx_rx_qflush( 617 __in efx_rxq_t *erp) 618 { 619 efx_nic_t *enp = erp->er_enp; 620 efx_oword_t oword; 621 uint32_t label; 622 623 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 624 625 label = erp->er_index; 626 627 /* Flush the queue */ 628 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 629 FRF_AZ_RX_FLUSH_DESCQ, label); 630 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 631 } 632 633 void 634 efx_rx_qenable( 635 __in efx_rxq_t *erp) 636 { 637 efx_nic_t *enp = erp->er_enp; 638 efx_oword_t oword; 639 640 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 641 642 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 643 erp->er_index, &oword); 644 645 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 646 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 647 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 648 649 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 650 erp->er_index, &oword); 651 } 652 653 __checkReturn int 654 efx_rx_qcreate( 655 __in efx_nic_t *enp, 656 __in unsigned int index, 657 __in unsigned int label, 658 __in efx_rxq_type_t type, 659 __in efsys_mem_t *esmp, 660 __in size_t n, 661 __in uint32_t id, 662 __in efx_evq_t *eep, 663 __deref_out efx_rxq_t **erpp) 664 { 665 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 666 efx_rxq_t *erp; 667 efx_oword_t oword; 668 uint32_t size; 669 boolean_t split; 670 boolean_t jumbo; 671 int rc; 672 673 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 674 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 675 676 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 677 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 678 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 679 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 680 681 if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) { 682 rc = EINVAL; 683 goto fail1; 684 } 685 if (index >= encp->enc_rxq_limit) { 686 rc = EINVAL; 687 goto fail2; 688 } 689 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); 690 size++) 691 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS)) 692 break; 693 if (id + (1 << size) >= encp->enc_buftbl_limit) { 694 rc = EINVAL; 695 goto fail3; 696 } 697 698 switch (type) { 699 case EFX_RXQ_TYPE_DEFAULT: 700 split = B_FALSE; 701 jumbo = B_FALSE; 702 break; 703 704 #if EFSYS_OPT_RX_HDR_SPLIT 705 case EFX_RXQ_TYPE_SPLIT_HEADER: 706 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) { 707 rc = EINVAL; 708 goto fail4; 709 } 710 split = B_TRUE; 711 jumbo = B_TRUE; 712 break; 713 714 case EFX_RXQ_TYPE_SPLIT_PAYLOAD: 715 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) { 716 rc = EINVAL; 717 goto fail4; 718 } 719 split = B_FALSE; 720 jumbo = B_TRUE; 721 break; 722 #endif /* EFSYS_OPT_RX_HDR_SPLIT */ 723 724 #if EFSYS_OPT_RX_SCATTER 725 case EFX_RXQ_TYPE_SCATTER: 726 if (enp->en_family < EFX_FAMILY_SIENA) { 727 rc = EINVAL; 728 goto fail4; 729 } 730 split = B_FALSE; 731 jumbo = B_TRUE; 732 break; 733 #endif /* EFSYS_OPT_RX_SCATTER */ 734 735 default: 736 rc = EINVAL; 737 goto fail4; 738 } 739 740 /* Allocate an RXQ object */ 741 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 742 743 if (erp == NULL) { 744 rc = ENOMEM; 745 goto fail5; 746 } 747 748 erp->er_magic = EFX_RXQ_MAGIC; 749 erp->er_enp = enp; 750 erp->er_index = index; 751 erp->er_mask = n - 1; 752 erp->er_esmp = esmp; 753 754 /* Set up the new descriptor queue */ 755 EFX_POPULATE_OWORD_10(oword, 756 FRF_CZ_RX_HDR_SPLIT, split, 757 FRF_AZ_RX_ISCSI_DDIG_EN, 0, 758 FRF_AZ_RX_ISCSI_HDIG_EN, 0, 759 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 760 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 761 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 762 FRF_AZ_RX_DESCQ_LABEL, label, 763 FRF_AZ_RX_DESCQ_SIZE, size, 764 FRF_AZ_RX_DESCQ_TYPE, 0, 765 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 766 767 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 768 erp->er_index, &oword); 769 770 enp->en_rx_qcount++; 771 *erpp = erp; 772 return (0); 773 774 fail5: 775 EFSYS_PROBE(fail5); 776 fail4: 777 EFSYS_PROBE(fail4); 778 fail3: 779 EFSYS_PROBE(fail3); 780 fail2: 781 EFSYS_PROBE(fail2); 782 fail1: 783 EFSYS_PROBE1(fail1, int, rc); 784 785 return (rc); 786 } 787 788 void 789 efx_rx_qdestroy( 790 __in efx_rxq_t *erp) 791 { 792 efx_nic_t *enp = erp->er_enp; 793 efx_oword_t oword; 794 795 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 796 797 EFSYS_ASSERT(enp->en_rx_qcount != 0); 798 --enp->en_rx_qcount; 799 800 /* Purge descriptor queue */ 801 EFX_ZERO_OWORD(oword); 802 803 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 804 erp->er_index, &oword); 805 806 /* Free the RXQ object */ 807 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 808 } 809 810 void 811 efx_rx_fini( 812 __in efx_nic_t *enp) 813 { 814 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 815 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 816 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 817 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 818 819 enp->en_mod_flags &= ~EFX_MOD_RX; 820 } 821