1 /*- 2 * Copyright (c) 2012-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 38 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 39 40 41 static __checkReturn efx_rc_t 42 efx_mcdi_init_rxq( 43 __in efx_nic_t *enp, 44 __in uint32_t ndescs, 45 __in uint32_t target_evq, 46 __in uint32_t label, 47 __in uint32_t instance, 48 __in efsys_mem_t *esmp, 49 __in boolean_t disable_scatter, 50 __in boolean_t want_inner_classes, 51 __in uint32_t ps_bufsize) 52 { 53 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 54 efx_mcdi_req_t req; 55 uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN, 56 MC_CMD_INIT_RXQ_EXT_OUT_LEN)]; 57 int npages = EFX_RXQ_NBUFS(ndescs); 58 int i; 59 efx_qword_t *dma_addr; 60 uint64_t addr; 61 efx_rc_t rc; 62 uint32_t dma_mode; 63 boolean_t want_outer_classes; 64 65 EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS); 66 67 if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_RXQ_SIZE(ndescs))) { 68 rc = EINVAL; 69 goto fail1; 70 } 71 72 if (ps_bufsize > 0) 73 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM; 74 else 75 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET; 76 77 if (encp->enc_tunnel_encapsulations_supported != 0 && 78 !want_inner_classes) { 79 /* 80 * WANT_OUTER_CLASSES can only be specified on hardware which 81 * supports tunnel encapsulation offloads, even though it is 82 * effectively the behaviour the hardware gives. 83 * 84 * Also, on hardware which does support such offloads, older 85 * firmware rejects the flag if the offloads are not supported 86 * by the current firmware variant, which means this may fail if 87 * the capabilities are not updated when the firmware variant 88 * changes. This is not an issue on newer firmware, as it was 89 * changed in bug 69842 (v6.4.2.1007) to permit this flag to be 90 * specified on all firmware variants. 91 */ 92 want_outer_classes = B_TRUE; 93 } else { 94 want_outer_classes = B_FALSE; 95 } 96 97 (void) memset(payload, 0, sizeof (payload)); 98 req.emr_cmd = MC_CMD_INIT_RXQ; 99 req.emr_in_buf = payload; 100 req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN; 101 req.emr_out_buf = payload; 102 req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN; 103 104 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs); 105 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq); 106 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label); 107 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance); 108 MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS, 109 INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0, 110 INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0, 111 INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0, 112 INIT_RXQ_EXT_IN_CRC_MODE, 0, 113 INIT_RXQ_EXT_IN_FLAG_PREFIX, 1, 114 INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter, 115 INIT_RXQ_EXT_IN_DMA_MODE, 116 dma_mode, 117 INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize, 118 INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes); 119 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0); 120 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 121 122 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); 123 addr = EFSYS_MEM_ADDR(esmp); 124 125 for (i = 0; i < npages; i++) { 126 EFX_POPULATE_QWORD_2(*dma_addr, 127 EFX_DWORD_1, (uint32_t)(addr >> 32), 128 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 129 130 dma_addr++; 131 addr += EFX_BUF_SIZE; 132 } 133 134 efx_mcdi_execute(enp, &req); 135 136 if (req.emr_rc != 0) { 137 rc = req.emr_rc; 138 goto fail2; 139 } 140 141 return (0); 142 143 fail2: 144 EFSYS_PROBE(fail2); 145 fail1: 146 EFSYS_PROBE1(fail1, efx_rc_t, rc); 147 148 return (rc); 149 } 150 151 static __checkReturn efx_rc_t 152 efx_mcdi_fini_rxq( 153 __in efx_nic_t *enp, 154 __in uint32_t instance) 155 { 156 efx_mcdi_req_t req; 157 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN, 158 MC_CMD_FINI_RXQ_OUT_LEN)]; 159 efx_rc_t rc; 160 161 (void) memset(payload, 0, sizeof (payload)); 162 req.emr_cmd = MC_CMD_FINI_RXQ; 163 req.emr_in_buf = payload; 164 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; 165 req.emr_out_buf = payload; 166 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; 167 168 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); 169 170 efx_mcdi_execute_quiet(enp, &req); 171 172 if (req.emr_rc != 0) { 173 rc = req.emr_rc; 174 goto fail1; 175 } 176 177 return (0); 178 179 fail1: 180 /* 181 * EALREADY is not an error, but indicates that the MC has rebooted and 182 * that the RXQ has already been destroyed. 183 */ 184 if (rc != EALREADY) 185 EFSYS_PROBE1(fail1, efx_rc_t, rc); 186 187 return (rc); 188 } 189 190 #if EFSYS_OPT_RX_SCALE 191 static __checkReturn efx_rc_t 192 efx_mcdi_rss_context_alloc( 193 __in efx_nic_t *enp, 194 __in efx_rx_scale_context_type_t type, 195 __in uint32_t num_queues, 196 __out uint32_t *rss_contextp) 197 { 198 efx_mcdi_req_t req; 199 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 200 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; 201 uint32_t rss_context; 202 uint32_t context_type; 203 efx_rc_t rc; 204 205 if (num_queues > EFX_MAXRSS) { 206 rc = EINVAL; 207 goto fail1; 208 } 209 210 switch (type) { 211 case EFX_RX_SCALE_EXCLUSIVE: 212 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 213 break; 214 case EFX_RX_SCALE_SHARED: 215 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 216 break; 217 default: 218 rc = EINVAL; 219 goto fail2; 220 } 221 222 (void) memset(payload, 0, sizeof (payload)); 223 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 224 req.emr_in_buf = payload; 225 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 226 req.emr_out_buf = payload; 227 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 228 229 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 230 EVB_PORT_ID_ASSIGNED); 231 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 232 233 /* 234 * For exclusive contexts, NUM_QUEUES is only used to validate 235 * indirection table offsets. 236 * For shared contexts, the provided context will spread traffic over 237 * NUM_QUEUES many queues. 238 */ 239 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 240 241 efx_mcdi_execute(enp, &req); 242 243 if (req.emr_rc != 0) { 244 rc = req.emr_rc; 245 goto fail3; 246 } 247 248 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 249 rc = EMSGSIZE; 250 goto fail4; 251 } 252 253 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 254 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 255 rc = ENOENT; 256 goto fail5; 257 } 258 259 *rss_contextp = rss_context; 260 261 return (0); 262 263 fail5: 264 EFSYS_PROBE(fail5); 265 fail4: 266 EFSYS_PROBE(fail4); 267 fail3: 268 EFSYS_PROBE(fail3); 269 fail2: 270 EFSYS_PROBE(fail2); 271 fail1: 272 EFSYS_PROBE1(fail1, efx_rc_t, rc); 273 274 return (rc); 275 } 276 #endif /* EFSYS_OPT_RX_SCALE */ 277 278 #if EFSYS_OPT_RX_SCALE 279 static efx_rc_t 280 efx_mcdi_rss_context_free( 281 __in efx_nic_t *enp, 282 __in uint32_t rss_context) 283 { 284 efx_mcdi_req_t req; 285 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 286 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; 287 efx_rc_t rc; 288 289 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 290 rc = EINVAL; 291 goto fail1; 292 } 293 294 (void) memset(payload, 0, sizeof (payload)); 295 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 296 req.emr_in_buf = payload; 297 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 298 req.emr_out_buf = payload; 299 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 300 301 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 302 303 efx_mcdi_execute_quiet(enp, &req); 304 305 if (req.emr_rc != 0) { 306 rc = req.emr_rc; 307 goto fail2; 308 } 309 310 return (0); 311 312 fail2: 313 EFSYS_PROBE(fail2); 314 fail1: 315 EFSYS_PROBE1(fail1, efx_rc_t, rc); 316 317 return (rc); 318 } 319 #endif /* EFSYS_OPT_RX_SCALE */ 320 321 #if EFSYS_OPT_RX_SCALE 322 static efx_rc_t 323 efx_mcdi_rss_context_set_flags( 324 __in efx_nic_t *enp, 325 __in uint32_t rss_context, 326 __in efx_rx_hash_type_t type) 327 { 328 efx_mcdi_req_t req; 329 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 330 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; 331 efx_rc_t rc; 332 333 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 334 rc = EINVAL; 335 goto fail1; 336 } 337 338 (void) memset(payload, 0, sizeof (payload)); 339 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 340 req.emr_in_buf = payload; 341 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 342 req.emr_out_buf = payload; 343 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 344 345 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 346 rss_context); 347 348 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 349 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 350 (type & EFX_RX_HASH_IPV4) ? 1 : 0, 351 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 352 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, 353 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 354 (type & EFX_RX_HASH_IPV6) ? 1 : 0, 355 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 356 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0); 357 358 efx_mcdi_execute(enp, &req); 359 360 if (req.emr_rc != 0) { 361 rc = req.emr_rc; 362 goto fail2; 363 } 364 365 return (0); 366 367 fail2: 368 EFSYS_PROBE(fail2); 369 fail1: 370 EFSYS_PROBE1(fail1, efx_rc_t, rc); 371 372 return (rc); 373 } 374 #endif /* EFSYS_OPT_RX_SCALE */ 375 376 #if EFSYS_OPT_RX_SCALE 377 static efx_rc_t 378 efx_mcdi_rss_context_set_key( 379 __in efx_nic_t *enp, 380 __in uint32_t rss_context, 381 __in_ecount(n) uint8_t *key, 382 __in size_t n) 383 { 384 efx_mcdi_req_t req; 385 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 386 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; 387 efx_rc_t rc; 388 389 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 390 rc = EINVAL; 391 goto fail1; 392 } 393 394 (void) memset(payload, 0, sizeof (payload)); 395 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 396 req.emr_in_buf = payload; 397 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 398 req.emr_out_buf = payload; 399 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 400 401 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 402 rss_context); 403 404 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 405 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 406 rc = EINVAL; 407 goto fail2; 408 } 409 410 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 411 key, n); 412 413 efx_mcdi_execute(enp, &req); 414 415 if (req.emr_rc != 0) { 416 rc = req.emr_rc; 417 goto fail3; 418 } 419 420 return (0); 421 422 fail3: 423 EFSYS_PROBE(fail3); 424 fail2: 425 EFSYS_PROBE(fail2); 426 fail1: 427 EFSYS_PROBE1(fail1, efx_rc_t, rc); 428 429 return (rc); 430 } 431 #endif /* EFSYS_OPT_RX_SCALE */ 432 433 #if EFSYS_OPT_RX_SCALE 434 static efx_rc_t 435 efx_mcdi_rss_context_set_table( 436 __in efx_nic_t *enp, 437 __in uint32_t rss_context, 438 __in_ecount(n) unsigned int *table, 439 __in size_t n) 440 { 441 efx_mcdi_req_t req; 442 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 443 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; 444 uint8_t *req_table; 445 int i, rc; 446 447 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 448 rc = EINVAL; 449 goto fail1; 450 } 451 452 (void) memset(payload, 0, sizeof (payload)); 453 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 454 req.emr_in_buf = payload; 455 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 456 req.emr_out_buf = payload; 457 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 458 459 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 460 rss_context); 461 462 req_table = 463 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 464 465 for (i = 0; 466 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 467 i++) { 468 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 469 } 470 471 efx_mcdi_execute(enp, &req); 472 473 if (req.emr_rc != 0) { 474 rc = req.emr_rc; 475 goto fail2; 476 } 477 478 return (0); 479 480 fail2: 481 EFSYS_PROBE(fail2); 482 fail1: 483 EFSYS_PROBE1(fail1, efx_rc_t, rc); 484 485 return (rc); 486 } 487 #endif /* EFSYS_OPT_RX_SCALE */ 488 489 490 __checkReturn efx_rc_t 491 ef10_rx_init( 492 __in efx_nic_t *enp) 493 { 494 #if EFSYS_OPT_RX_SCALE 495 496 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 497 &enp->en_rss_context) == 0) { 498 /* 499 * Allocated an exclusive RSS context, which allows both the 500 * indirection table and key to be modified. 501 */ 502 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 503 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 504 } else { 505 /* 506 * Failed to allocate an exclusive RSS context. Continue 507 * operation without support for RSS. The pseudo-header in 508 * received packets will not contain a Toeplitz hash value. 509 */ 510 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 511 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 512 } 513 514 #endif /* EFSYS_OPT_RX_SCALE */ 515 516 return (0); 517 } 518 519 #if EFSYS_OPT_RX_SCATTER 520 __checkReturn efx_rc_t 521 ef10_rx_scatter_enable( 522 __in efx_nic_t *enp, 523 __in unsigned int buf_size) 524 { 525 _NOTE(ARGUNUSED(enp, buf_size)) 526 return (0); 527 } 528 #endif /* EFSYS_OPT_RX_SCATTER */ 529 530 #if EFSYS_OPT_RX_SCALE 531 __checkReturn efx_rc_t 532 ef10_rx_scale_context_alloc( 533 __in efx_nic_t *enp, 534 __in efx_rx_scale_context_type_t type, 535 __in uint32_t num_queues, 536 __out uint32_t *rss_contextp) 537 { 538 efx_rc_t rc; 539 540 rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp); 541 if (rc != 0) 542 goto fail1; 543 544 return (0); 545 546 fail1: 547 EFSYS_PROBE1(fail1, efx_rc_t, rc); 548 return (rc); 549 } 550 #endif /* EFSYS_OPT_RX_SCALE */ 551 552 #if EFSYS_OPT_RX_SCALE 553 __checkReturn efx_rc_t 554 ef10_rx_scale_context_free( 555 __in efx_nic_t *enp, 556 __in uint32_t rss_context) 557 { 558 efx_rc_t rc; 559 560 rc = efx_mcdi_rss_context_free(enp, rss_context); 561 if (rc != 0) 562 goto fail1; 563 564 return (0); 565 566 fail1: 567 EFSYS_PROBE1(fail1, efx_rc_t, rc); 568 return (rc); 569 } 570 #endif /* EFSYS_OPT_RX_SCALE */ 571 572 #if EFSYS_OPT_RX_SCALE 573 __checkReturn efx_rc_t 574 ef10_rx_scale_mode_set( 575 __in efx_nic_t *enp, 576 __in uint32_t rss_context, 577 __in efx_rx_hash_alg_t alg, 578 __in efx_rx_hash_type_t type, 579 __in boolean_t insert) 580 { 581 efx_rc_t rc; 582 583 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); 584 EFSYS_ASSERT3U(insert, ==, B_TRUE); 585 586 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { 587 rc = EINVAL; 588 goto fail1; 589 } 590 591 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 592 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 593 rc = ENOTSUP; 594 goto fail2; 595 } 596 rss_context = enp->en_rss_context; 597 } 598 599 if ((rc = efx_mcdi_rss_context_set_flags(enp, 600 rss_context, type)) != 0) 601 goto fail3; 602 603 return (0); 604 605 fail3: 606 EFSYS_PROBE(fail3); 607 fail2: 608 EFSYS_PROBE(fail2); 609 fail1: 610 EFSYS_PROBE1(fail1, efx_rc_t, rc); 611 612 return (rc); 613 } 614 #endif /* EFSYS_OPT_RX_SCALE */ 615 616 #if EFSYS_OPT_RX_SCALE 617 __checkReturn efx_rc_t 618 ef10_rx_scale_key_set( 619 __in efx_nic_t *enp, 620 __in uint32_t rss_context, 621 __in_ecount(n) uint8_t *key, 622 __in size_t n) 623 { 624 efx_rc_t rc; 625 626 EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE == 627 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 628 629 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 630 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 631 rc = ENOTSUP; 632 goto fail1; 633 } 634 rss_context = enp->en_rss_context; 635 } 636 637 if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0) 638 goto fail2; 639 640 return (0); 641 642 fail2: 643 EFSYS_PROBE(fail2); 644 fail1: 645 EFSYS_PROBE1(fail1, efx_rc_t, rc); 646 647 return (rc); 648 } 649 #endif /* EFSYS_OPT_RX_SCALE */ 650 651 #if EFSYS_OPT_RX_SCALE 652 __checkReturn efx_rc_t 653 ef10_rx_scale_tbl_set( 654 __in efx_nic_t *enp, 655 __in uint32_t rss_context, 656 __in_ecount(n) unsigned int *table, 657 __in size_t n) 658 { 659 efx_rc_t rc; 660 661 662 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 663 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 664 rc = ENOTSUP; 665 goto fail1; 666 } 667 rss_context = enp->en_rss_context; 668 } 669 670 if ((rc = efx_mcdi_rss_context_set_table(enp, 671 rss_context, table, n)) != 0) 672 goto fail2; 673 674 return (0); 675 676 fail2: 677 EFSYS_PROBE(fail2); 678 fail1: 679 EFSYS_PROBE1(fail1, efx_rc_t, rc); 680 681 return (rc); 682 } 683 #endif /* EFSYS_OPT_RX_SCALE */ 684 685 686 /* 687 * EF10 RX pseudo-header 688 * --------------------- 689 * 690 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 691 * 692 * +00: Toeplitz hash value. 693 * (32bit little-endian) 694 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 695 * (16bit big-endian) 696 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 697 * (16bit big-endian) 698 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 699 * (16bit little-endian) 700 * +10: MAC timestamp. Zero if timestamping is not enabled. 701 * (32bit little-endian) 702 * 703 * See "The RX Pseudo-header" in SF-109306-TC. 704 */ 705 706 __checkReturn efx_rc_t 707 ef10_rx_prefix_pktlen( 708 __in efx_nic_t *enp, 709 __in uint8_t *buffer, 710 __out uint16_t *lengthp) 711 { 712 _NOTE(ARGUNUSED(enp)) 713 714 /* 715 * The RX pseudo-header contains the packet length, excluding the 716 * pseudo-header. If the hardware receive datapath was operating in 717 * cut-through mode then the length in the RX pseudo-header will be 718 * zero, and the packet length must be obtained from the DMA length 719 * reported in the RX event. 720 */ 721 *lengthp = buffer[8] | (buffer[9] << 8); 722 return (0); 723 } 724 725 #if EFSYS_OPT_RX_SCALE 726 __checkReturn uint32_t 727 ef10_rx_prefix_hash( 728 __in efx_nic_t *enp, 729 __in efx_rx_hash_alg_t func, 730 __in uint8_t *buffer) 731 { 732 _NOTE(ARGUNUSED(enp)) 733 734 switch (func) { 735 case EFX_RX_HASHALG_TOEPLITZ: 736 return (buffer[0] | 737 (buffer[1] << 8) | 738 (buffer[2] << 16) | 739 (buffer[3] << 24)); 740 741 default: 742 EFSYS_ASSERT(0); 743 return (0); 744 } 745 } 746 #endif /* EFSYS_OPT_RX_SCALE */ 747 748 #if EFSYS_OPT_RX_PACKED_STREAM 749 /* 750 * Fake length for RXQ descriptors in packed stream mode 751 * to make hardware happy 752 */ 753 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 754 #endif 755 756 void 757 ef10_rx_qpost( 758 __in efx_rxq_t *erp, 759 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 760 __in size_t size, 761 __in unsigned int ndescs, 762 __in unsigned int completed, 763 __in unsigned int added) 764 { 765 efx_qword_t qword; 766 unsigned int i; 767 unsigned int offset; 768 unsigned int id; 769 770 _NOTE(ARGUNUSED(completed)) 771 772 #if EFSYS_OPT_RX_PACKED_STREAM 773 /* 774 * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT 775 * and equal to 0 after applying mask. Hardware does not like it. 776 */ 777 if (erp->er_ev_qstate->eers_rx_packed_stream) 778 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE; 779 #endif 780 781 /* The client driver must not overfill the queue */ 782 EFSYS_ASSERT3U(added - completed + ndescs, <=, 783 EFX_RXQ_LIMIT(erp->er_mask + 1)); 784 785 id = added & (erp->er_mask); 786 for (i = 0; i < ndescs; i++) { 787 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 788 unsigned int, id, efsys_dma_addr_t, addrp[i], 789 size_t, size); 790 791 EFX_POPULATE_QWORD_3(qword, 792 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 793 ESF_DZ_RX_KER_BUF_ADDR_DW0, 794 (uint32_t)(addrp[i] & 0xffffffff), 795 ESF_DZ_RX_KER_BUF_ADDR_DW1, 796 (uint32_t)(addrp[i] >> 32)); 797 798 offset = id * sizeof (efx_qword_t); 799 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 800 801 id = (id + 1) & (erp->er_mask); 802 } 803 } 804 805 void 806 ef10_rx_qpush( 807 __in efx_rxq_t *erp, 808 __in unsigned int added, 809 __inout unsigned int *pushedp) 810 { 811 efx_nic_t *enp = erp->er_enp; 812 unsigned int pushed = *pushedp; 813 uint32_t wptr; 814 efx_dword_t dword; 815 816 /* Hardware has alignment restriction for WPTR */ 817 wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN); 818 if (pushed == wptr) 819 return; 820 821 *pushedp = wptr; 822 823 /* Push the populated descriptors out */ 824 wptr &= erp->er_mask; 825 826 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 827 828 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 829 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 830 wptr, pushed & erp->er_mask); 831 EFSYS_PIO_WRITE_BARRIER(); 832 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 833 erp->er_index, &dword, B_FALSE); 834 } 835 836 #if EFSYS_OPT_RX_PACKED_STREAM 837 838 void 839 ef10_rx_qpush_ps_credits( 840 __in efx_rxq_t *erp) 841 { 842 efx_nic_t *enp = erp->er_enp; 843 efx_dword_t dword; 844 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 845 uint32_t credits; 846 847 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 848 849 if (rxq_state->eers_rx_packed_stream_credits == 0) 850 return; 851 852 /* 853 * It is a bug if we think that FW has utilized more 854 * credits than it is allowed to have (maximum). However, 855 * make sure that we do not credit more than maximum anyway. 856 */ 857 credits = MIN(rxq_state->eers_rx_packed_stream_credits, 858 EFX_RX_PACKED_STREAM_MAX_CREDITS); 859 EFX_POPULATE_DWORD_3(dword, 860 ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, 861 ERF_DZ_RX_DESC_MAGIC_CMD, 862 ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, 863 ERF_DZ_RX_DESC_MAGIC_DATA, credits); 864 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 865 erp->er_index, &dword, B_FALSE); 866 867 rxq_state->eers_rx_packed_stream_credits = 0; 868 } 869 870 /* 871 * In accordance with SF-112241-TC the received data has the following layout: 872 * - 8 byte pseudo-header which consist of: 873 * - 4 byte little-endian timestamp 874 * - 2 byte little-endian captured length in bytes 875 * - 2 byte little-endian original packet length in bytes 876 * - captured packet bytes 877 * - optional padding to align to 64 bytes boundary 878 * - 64 bytes scratch space for the host software 879 */ 880 __checkReturn uint8_t * 881 ef10_rx_qps_packet_info( 882 __in efx_rxq_t *erp, 883 __in uint8_t *buffer, 884 __in uint32_t buffer_length, 885 __in uint32_t current_offset, 886 __out uint16_t *lengthp, 887 __out uint32_t *next_offsetp, 888 __out uint32_t *timestamp) 889 { 890 uint16_t buf_len; 891 uint8_t *pkt_start; 892 efx_qword_t *qwordp; 893 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 894 895 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 896 897 buffer += current_offset; 898 pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; 899 900 qwordp = (efx_qword_t *)buffer; 901 *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); 902 *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); 903 buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); 904 905 buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, 906 EFX_RX_PACKED_STREAM_ALIGNMENT); 907 *next_offsetp = 908 current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; 909 910 EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); 911 EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); 912 913 if ((*next_offsetp ^ current_offset) & 914 EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) 915 rxq_state->eers_rx_packed_stream_credits++; 916 917 return (pkt_start); 918 } 919 920 921 #endif 922 923 __checkReturn efx_rc_t 924 ef10_rx_qflush( 925 __in efx_rxq_t *erp) 926 { 927 efx_nic_t *enp = erp->er_enp; 928 efx_rc_t rc; 929 930 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 931 goto fail1; 932 933 return (0); 934 935 fail1: 936 /* 937 * EALREADY is not an error, but indicates that the MC has rebooted and 938 * that the RXQ has already been destroyed. Callers need to know that 939 * the RXQ flush has completed to avoid waiting until timeout for a 940 * flush done event that will not be delivered. 941 */ 942 if (rc != EALREADY) 943 EFSYS_PROBE1(fail1, efx_rc_t, rc); 944 945 return (rc); 946 } 947 948 void 949 ef10_rx_qenable( 950 __in efx_rxq_t *erp) 951 { 952 /* FIXME */ 953 _NOTE(ARGUNUSED(erp)) 954 /* FIXME */ 955 } 956 957 __checkReturn efx_rc_t 958 ef10_rx_qcreate( 959 __in efx_nic_t *enp, 960 __in unsigned int index, 961 __in unsigned int label, 962 __in efx_rxq_type_t type, 963 __in uint32_t type_data, 964 __in efsys_mem_t *esmp, 965 __in size_t ndescs, 966 __in uint32_t id, 967 __in unsigned int flags, 968 __in efx_evq_t *eep, 969 __in efx_rxq_t *erp) 970 { 971 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 972 efx_rc_t rc; 973 boolean_t disable_scatter; 974 boolean_t want_inner_classes; 975 unsigned int ps_buf_size; 976 977 _NOTE(ARGUNUSED(id, erp, type_data)) 978 979 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 980 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 981 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 982 983 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 984 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 985 986 if (!ISP2(ndescs) || 987 (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) { 988 rc = EINVAL; 989 goto fail1; 990 } 991 if (index >= encp->enc_rxq_limit) { 992 rc = EINVAL; 993 goto fail2; 994 } 995 996 switch (type) { 997 case EFX_RXQ_TYPE_DEFAULT: 998 ps_buf_size = 0; 999 break; 1000 #if EFSYS_OPT_RX_PACKED_STREAM 1001 case EFX_RXQ_TYPE_PACKED_STREAM: 1002 switch (type_data) { 1003 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M: 1004 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; 1005 break; 1006 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K: 1007 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; 1008 break; 1009 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K: 1010 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; 1011 break; 1012 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K: 1013 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; 1014 break; 1015 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K: 1016 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; 1017 break; 1018 default: 1019 rc = ENOTSUP; 1020 goto fail3; 1021 } 1022 break; 1023 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1024 default: 1025 rc = ENOTSUP; 1026 goto fail4; 1027 } 1028 1029 #if EFSYS_OPT_RX_PACKED_STREAM 1030 if (ps_buf_size != 0) { 1031 /* Check if datapath firmware supports packed stream mode */ 1032 if (encp->enc_rx_packed_stream_supported == B_FALSE) { 1033 rc = ENOTSUP; 1034 goto fail5; 1035 } 1036 /* Check if packed stream allows configurable buffer sizes */ 1037 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) && 1038 (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { 1039 rc = ENOTSUP; 1040 goto fail6; 1041 } 1042 } 1043 #else /* EFSYS_OPT_RX_PACKED_STREAM */ 1044 EFSYS_ASSERT(ps_buf_size == 0); 1045 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1046 1047 /* Scatter can only be disabled if the firmware supports doing so */ 1048 if (flags & EFX_RXQ_FLAG_SCATTER) 1049 disable_scatter = B_FALSE; 1050 else 1051 disable_scatter = encp->enc_rx_disable_scatter_supported; 1052 1053 if (flags & EFX_RXQ_FLAG_INNER_CLASSES) 1054 want_inner_classes = B_TRUE; 1055 else 1056 want_inner_classes = B_FALSE; 1057 1058 if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index, 1059 esmp, disable_scatter, want_inner_classes, 1060 ps_buf_size)) != 0) 1061 goto fail7; 1062 1063 erp->er_eep = eep; 1064 erp->er_label = label; 1065 1066 ef10_ev_rxlabel_init(eep, erp, label, type); 1067 1068 erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label]; 1069 1070 return (0); 1071 1072 fail7: 1073 EFSYS_PROBE(fail7); 1074 #if EFSYS_OPT_RX_PACKED_STREAM 1075 fail6: 1076 EFSYS_PROBE(fail6); 1077 fail5: 1078 EFSYS_PROBE(fail5); 1079 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1080 fail4: 1081 EFSYS_PROBE(fail4); 1082 #if EFSYS_OPT_RX_PACKED_STREAM 1083 fail3: 1084 EFSYS_PROBE(fail3); 1085 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1086 fail2: 1087 EFSYS_PROBE(fail2); 1088 fail1: 1089 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1090 1091 return (rc); 1092 } 1093 1094 void 1095 ef10_rx_qdestroy( 1096 __in efx_rxq_t *erp) 1097 { 1098 efx_nic_t *enp = erp->er_enp; 1099 efx_evq_t *eep = erp->er_eep; 1100 unsigned int label = erp->er_label; 1101 1102 ef10_ev_rxlabel_fini(eep, label); 1103 1104 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1105 --enp->en_rx_qcount; 1106 1107 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1108 } 1109 1110 void 1111 ef10_rx_fini( 1112 __in efx_nic_t *enp) 1113 { 1114 #if EFSYS_OPT_RX_SCALE 1115 if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE) 1116 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 1117 enp->en_rss_context = 0; 1118 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 1119 #else 1120 _NOTE(ARGUNUSED(enp)) 1121 #endif /* EFSYS_OPT_RX_SCALE */ 1122 } 1123 1124 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ 1125