1 /*- 2 * Copyright (c) 2012-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 38 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 39 40 41 static __checkReturn efx_rc_t 42 efx_mcdi_init_rxq( 43 __in efx_nic_t *enp, 44 __in uint32_t ndescs, 45 __in uint32_t target_evq, 46 __in uint32_t label, 47 __in uint32_t instance, 48 __in efsys_mem_t *esmp, 49 __in boolean_t disable_scatter, 50 __in boolean_t want_inner_classes, 51 __in uint32_t ps_bufsize) 52 { 53 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 54 efx_mcdi_req_t req; 55 uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN, 56 MC_CMD_INIT_RXQ_EXT_OUT_LEN)]; 57 int npages = EFX_RXQ_NBUFS(ndescs); 58 int i; 59 efx_qword_t *dma_addr; 60 uint64_t addr; 61 efx_rc_t rc; 62 uint32_t dma_mode; 63 boolean_t want_outer_classes; 64 65 EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS); 66 67 if (ps_bufsize > 0) 68 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM; 69 else 70 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET; 71 72 if (encp->enc_tunnel_encapsulations_supported != 0 && 73 !want_inner_classes) { 74 /* 75 * WANT_OUTER_CLASSES can only be specified on hardware which 76 * supports tunnel encapsulation offloads, even though it is 77 * effectively the behaviour the hardware gives. 78 * 79 * Also, on hardware which does support such offloads, older 80 * firmware rejects the flag if the offloads are not supported 81 * by the current firmware variant, which means this may fail if 82 * the capabilities are not updated when the firmware variant 83 * changes. This is not an issue on newer firmware, as it was 84 * changed in bug 69842 (v6.4.2.1007) to permit this flag to be 85 * specified on all firmware variants. 86 */ 87 want_outer_classes = B_TRUE; 88 } else { 89 want_outer_classes = B_FALSE; 90 } 91 92 (void) memset(payload, 0, sizeof (payload)); 93 req.emr_cmd = MC_CMD_INIT_RXQ; 94 req.emr_in_buf = payload; 95 req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN; 96 req.emr_out_buf = payload; 97 req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN; 98 99 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs); 100 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq); 101 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label); 102 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance); 103 MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS, 104 INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0, 105 INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0, 106 INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0, 107 INIT_RXQ_EXT_IN_CRC_MODE, 0, 108 INIT_RXQ_EXT_IN_FLAG_PREFIX, 1, 109 INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter, 110 INIT_RXQ_EXT_IN_DMA_MODE, 111 dma_mode, 112 INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize, 113 INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes); 114 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0); 115 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 116 117 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); 118 addr = EFSYS_MEM_ADDR(esmp); 119 120 for (i = 0; i < npages; i++) { 121 EFX_POPULATE_QWORD_2(*dma_addr, 122 EFX_DWORD_1, (uint32_t)(addr >> 32), 123 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 124 125 dma_addr++; 126 addr += EFX_BUF_SIZE; 127 } 128 129 efx_mcdi_execute(enp, &req); 130 131 if (req.emr_rc != 0) { 132 rc = req.emr_rc; 133 goto fail1; 134 } 135 136 return (0); 137 138 fail1: 139 EFSYS_PROBE1(fail1, efx_rc_t, rc); 140 141 return (rc); 142 } 143 144 static __checkReturn efx_rc_t 145 efx_mcdi_fini_rxq( 146 __in efx_nic_t *enp, 147 __in uint32_t instance) 148 { 149 efx_mcdi_req_t req; 150 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN, 151 MC_CMD_FINI_RXQ_OUT_LEN)]; 152 efx_rc_t rc; 153 154 (void) memset(payload, 0, sizeof (payload)); 155 req.emr_cmd = MC_CMD_FINI_RXQ; 156 req.emr_in_buf = payload; 157 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; 158 req.emr_out_buf = payload; 159 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; 160 161 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); 162 163 efx_mcdi_execute_quiet(enp, &req); 164 165 if (req.emr_rc != 0) { 166 rc = req.emr_rc; 167 goto fail1; 168 } 169 170 return (0); 171 172 fail1: 173 /* 174 * EALREADY is not an error, but indicates that the MC has rebooted and 175 * that the RXQ has already been destroyed. 176 */ 177 if (rc != EALREADY) 178 EFSYS_PROBE1(fail1, efx_rc_t, rc); 179 180 return (rc); 181 } 182 183 #if EFSYS_OPT_RX_SCALE 184 static __checkReturn efx_rc_t 185 efx_mcdi_rss_context_alloc( 186 __in efx_nic_t *enp, 187 __in efx_rx_scale_context_type_t type, 188 __in uint32_t num_queues, 189 __out uint32_t *rss_contextp) 190 { 191 efx_mcdi_req_t req; 192 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 193 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; 194 uint32_t rss_context; 195 uint32_t context_type; 196 efx_rc_t rc; 197 198 if (num_queues > EFX_MAXRSS) { 199 rc = EINVAL; 200 goto fail1; 201 } 202 203 switch (type) { 204 case EFX_RX_SCALE_EXCLUSIVE: 205 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 206 break; 207 case EFX_RX_SCALE_SHARED: 208 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 209 break; 210 default: 211 rc = EINVAL; 212 goto fail2; 213 } 214 215 (void) memset(payload, 0, sizeof (payload)); 216 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 217 req.emr_in_buf = payload; 218 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 219 req.emr_out_buf = payload; 220 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 221 222 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 223 EVB_PORT_ID_ASSIGNED); 224 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 225 226 /* 227 * For exclusive contexts, NUM_QUEUES is only used to validate 228 * indirection table offsets. 229 * For shared contexts, the provided context will spread traffic over 230 * NUM_QUEUES many queues. 231 */ 232 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 233 234 efx_mcdi_execute(enp, &req); 235 236 if (req.emr_rc != 0) { 237 rc = req.emr_rc; 238 goto fail3; 239 } 240 241 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 242 rc = EMSGSIZE; 243 goto fail4; 244 } 245 246 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 247 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 248 rc = ENOENT; 249 goto fail5; 250 } 251 252 *rss_contextp = rss_context; 253 254 return (0); 255 256 fail5: 257 EFSYS_PROBE(fail5); 258 fail4: 259 EFSYS_PROBE(fail4); 260 fail3: 261 EFSYS_PROBE(fail3); 262 fail2: 263 EFSYS_PROBE(fail2); 264 fail1: 265 EFSYS_PROBE1(fail1, efx_rc_t, rc); 266 267 return (rc); 268 } 269 #endif /* EFSYS_OPT_RX_SCALE */ 270 271 #if EFSYS_OPT_RX_SCALE 272 static efx_rc_t 273 efx_mcdi_rss_context_free( 274 __in efx_nic_t *enp, 275 __in uint32_t rss_context) 276 { 277 efx_mcdi_req_t req; 278 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 279 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; 280 efx_rc_t rc; 281 282 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 283 rc = EINVAL; 284 goto fail1; 285 } 286 287 (void) memset(payload, 0, sizeof (payload)); 288 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 289 req.emr_in_buf = payload; 290 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 291 req.emr_out_buf = payload; 292 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 293 294 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 295 296 efx_mcdi_execute_quiet(enp, &req); 297 298 if (req.emr_rc != 0) { 299 rc = req.emr_rc; 300 goto fail2; 301 } 302 303 return (0); 304 305 fail2: 306 EFSYS_PROBE(fail2); 307 fail1: 308 EFSYS_PROBE1(fail1, efx_rc_t, rc); 309 310 return (rc); 311 } 312 #endif /* EFSYS_OPT_RX_SCALE */ 313 314 #if EFSYS_OPT_RX_SCALE 315 static efx_rc_t 316 efx_mcdi_rss_context_set_flags( 317 __in efx_nic_t *enp, 318 __in uint32_t rss_context, 319 __in efx_rx_hash_type_t type) 320 { 321 efx_mcdi_req_t req; 322 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 323 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; 324 efx_rc_t rc; 325 326 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 327 rc = EINVAL; 328 goto fail1; 329 } 330 331 (void) memset(payload, 0, sizeof (payload)); 332 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 333 req.emr_in_buf = payload; 334 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 335 req.emr_out_buf = payload; 336 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 337 338 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 339 rss_context); 340 341 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 342 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 343 (type & EFX_RX_HASH_IPV4) ? 1 : 0, 344 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 345 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, 346 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 347 (type & EFX_RX_HASH_IPV6) ? 1 : 0, 348 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 349 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0); 350 351 efx_mcdi_execute(enp, &req); 352 353 if (req.emr_rc != 0) { 354 rc = req.emr_rc; 355 goto fail2; 356 } 357 358 return (0); 359 360 fail2: 361 EFSYS_PROBE(fail2); 362 fail1: 363 EFSYS_PROBE1(fail1, efx_rc_t, rc); 364 365 return (rc); 366 } 367 #endif /* EFSYS_OPT_RX_SCALE */ 368 369 #if EFSYS_OPT_RX_SCALE 370 static efx_rc_t 371 efx_mcdi_rss_context_set_key( 372 __in efx_nic_t *enp, 373 __in uint32_t rss_context, 374 __in_ecount(n) uint8_t *key, 375 __in size_t n) 376 { 377 efx_mcdi_req_t req; 378 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 379 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; 380 efx_rc_t rc; 381 382 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 383 rc = EINVAL; 384 goto fail1; 385 } 386 387 (void) memset(payload, 0, sizeof (payload)); 388 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 389 req.emr_in_buf = payload; 390 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 391 req.emr_out_buf = payload; 392 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 393 394 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 395 rss_context); 396 397 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 398 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 399 rc = EINVAL; 400 goto fail2; 401 } 402 403 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 404 key, n); 405 406 efx_mcdi_execute(enp, &req); 407 408 if (req.emr_rc != 0) { 409 rc = req.emr_rc; 410 goto fail3; 411 } 412 413 return (0); 414 415 fail3: 416 EFSYS_PROBE(fail3); 417 fail2: 418 EFSYS_PROBE(fail2); 419 fail1: 420 EFSYS_PROBE1(fail1, efx_rc_t, rc); 421 422 return (rc); 423 } 424 #endif /* EFSYS_OPT_RX_SCALE */ 425 426 #if EFSYS_OPT_RX_SCALE 427 static efx_rc_t 428 efx_mcdi_rss_context_set_table( 429 __in efx_nic_t *enp, 430 __in uint32_t rss_context, 431 __in_ecount(n) unsigned int *table, 432 __in size_t n) 433 { 434 efx_mcdi_req_t req; 435 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 436 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; 437 uint8_t *req_table; 438 int i, rc; 439 440 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 441 rc = EINVAL; 442 goto fail1; 443 } 444 445 (void) memset(payload, 0, sizeof (payload)); 446 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 447 req.emr_in_buf = payload; 448 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 449 req.emr_out_buf = payload; 450 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 451 452 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 453 rss_context); 454 455 req_table = 456 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 457 458 for (i = 0; 459 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 460 i++) { 461 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 462 } 463 464 efx_mcdi_execute(enp, &req); 465 466 if (req.emr_rc != 0) { 467 rc = req.emr_rc; 468 goto fail2; 469 } 470 471 return (0); 472 473 fail2: 474 EFSYS_PROBE(fail2); 475 fail1: 476 EFSYS_PROBE1(fail1, efx_rc_t, rc); 477 478 return (rc); 479 } 480 #endif /* EFSYS_OPT_RX_SCALE */ 481 482 483 __checkReturn efx_rc_t 484 ef10_rx_init( 485 __in efx_nic_t *enp) 486 { 487 #if EFSYS_OPT_RX_SCALE 488 489 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 490 &enp->en_rss_context) == 0) { 491 /* 492 * Allocated an exclusive RSS context, which allows both the 493 * indirection table and key to be modified. 494 */ 495 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 496 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 497 } else { 498 /* 499 * Failed to allocate an exclusive RSS context. Continue 500 * operation without support for RSS. The pseudo-header in 501 * received packets will not contain a Toeplitz hash value. 502 */ 503 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 504 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 505 } 506 507 #endif /* EFSYS_OPT_RX_SCALE */ 508 509 return (0); 510 } 511 512 #if EFSYS_OPT_RX_SCATTER 513 __checkReturn efx_rc_t 514 ef10_rx_scatter_enable( 515 __in efx_nic_t *enp, 516 __in unsigned int buf_size) 517 { 518 _NOTE(ARGUNUSED(enp, buf_size)) 519 return (0); 520 } 521 #endif /* EFSYS_OPT_RX_SCATTER */ 522 523 #if EFSYS_OPT_RX_SCALE 524 __checkReturn efx_rc_t 525 ef10_rx_scale_context_alloc( 526 __in efx_nic_t *enp, 527 __in efx_rx_scale_context_type_t type, 528 __in uint32_t num_queues, 529 __out uint32_t *rss_contextp) 530 { 531 efx_rc_t rc; 532 533 rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp); 534 if (rc != 0) 535 goto fail1; 536 537 return (0); 538 539 fail1: 540 EFSYS_PROBE1(fail1, efx_rc_t, rc); 541 return (rc); 542 } 543 #endif /* EFSYS_OPT_RX_SCALE */ 544 545 #if EFSYS_OPT_RX_SCALE 546 __checkReturn efx_rc_t 547 ef10_rx_scale_context_free( 548 __in efx_nic_t *enp, 549 __in uint32_t rss_context) 550 { 551 efx_rc_t rc; 552 553 rc = efx_mcdi_rss_context_free(enp, rss_context); 554 if (rc != 0) 555 goto fail1; 556 557 return (0); 558 559 fail1: 560 EFSYS_PROBE1(fail1, efx_rc_t, rc); 561 return (rc); 562 } 563 #endif /* EFSYS_OPT_RX_SCALE */ 564 565 #if EFSYS_OPT_RX_SCALE 566 __checkReturn efx_rc_t 567 ef10_rx_scale_mode_set( 568 __in efx_nic_t *enp, 569 __in uint32_t rss_context, 570 __in efx_rx_hash_alg_t alg, 571 __in efx_rx_hash_type_t type, 572 __in boolean_t insert) 573 { 574 efx_rc_t rc; 575 576 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); 577 EFSYS_ASSERT3U(insert, ==, B_TRUE); 578 579 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { 580 rc = EINVAL; 581 goto fail1; 582 } 583 584 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 585 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 586 rc = ENOTSUP; 587 goto fail2; 588 } 589 rss_context = enp->en_rss_context; 590 } 591 592 if ((rc = efx_mcdi_rss_context_set_flags(enp, 593 rss_context, type)) != 0) 594 goto fail3; 595 596 return (0); 597 598 fail3: 599 EFSYS_PROBE(fail3); 600 fail2: 601 EFSYS_PROBE(fail2); 602 fail1: 603 EFSYS_PROBE1(fail1, efx_rc_t, rc); 604 605 return (rc); 606 } 607 #endif /* EFSYS_OPT_RX_SCALE */ 608 609 #if EFSYS_OPT_RX_SCALE 610 __checkReturn efx_rc_t 611 ef10_rx_scale_key_set( 612 __in efx_nic_t *enp, 613 __in uint32_t rss_context, 614 __in_ecount(n) uint8_t *key, 615 __in size_t n) 616 { 617 efx_rc_t rc; 618 619 EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE == 620 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 621 622 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 623 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 624 rc = ENOTSUP; 625 goto fail1; 626 } 627 rss_context = enp->en_rss_context; 628 } 629 630 if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0) 631 goto fail2; 632 633 return (0); 634 635 fail2: 636 EFSYS_PROBE(fail2); 637 fail1: 638 EFSYS_PROBE1(fail1, efx_rc_t, rc); 639 640 return (rc); 641 } 642 #endif /* EFSYS_OPT_RX_SCALE */ 643 644 #if EFSYS_OPT_RX_SCALE 645 __checkReturn efx_rc_t 646 ef10_rx_scale_tbl_set( 647 __in efx_nic_t *enp, 648 __in uint32_t rss_context, 649 __in_ecount(n) unsigned int *table, 650 __in size_t n) 651 { 652 efx_rc_t rc; 653 654 655 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 656 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 657 rc = ENOTSUP; 658 goto fail1; 659 } 660 rss_context = enp->en_rss_context; 661 } 662 663 if ((rc = efx_mcdi_rss_context_set_table(enp, 664 rss_context, table, n)) != 0) 665 goto fail2; 666 667 return (0); 668 669 fail2: 670 EFSYS_PROBE(fail2); 671 fail1: 672 EFSYS_PROBE1(fail1, efx_rc_t, rc); 673 674 return (rc); 675 } 676 #endif /* EFSYS_OPT_RX_SCALE */ 677 678 679 /* 680 * EF10 RX pseudo-header 681 * --------------------- 682 * 683 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 684 * 685 * +00: Toeplitz hash value. 686 * (32bit little-endian) 687 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 688 * (16bit big-endian) 689 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 690 * (16bit big-endian) 691 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 692 * (16bit little-endian) 693 * +10: MAC timestamp. Zero if timestamping is not enabled. 694 * (32bit little-endian) 695 * 696 * See "The RX Pseudo-header" in SF-109306-TC. 697 */ 698 699 __checkReturn efx_rc_t 700 ef10_rx_prefix_pktlen( 701 __in efx_nic_t *enp, 702 __in uint8_t *buffer, 703 __out uint16_t *lengthp) 704 { 705 _NOTE(ARGUNUSED(enp)) 706 707 /* 708 * The RX pseudo-header contains the packet length, excluding the 709 * pseudo-header. If the hardware receive datapath was operating in 710 * cut-through mode then the length in the RX pseudo-header will be 711 * zero, and the packet length must be obtained from the DMA length 712 * reported in the RX event. 713 */ 714 *lengthp = buffer[8] | (buffer[9] << 8); 715 return (0); 716 } 717 718 #if EFSYS_OPT_RX_SCALE 719 __checkReturn uint32_t 720 ef10_rx_prefix_hash( 721 __in efx_nic_t *enp, 722 __in efx_rx_hash_alg_t func, 723 __in uint8_t *buffer) 724 { 725 _NOTE(ARGUNUSED(enp)) 726 727 switch (func) { 728 case EFX_RX_HASHALG_TOEPLITZ: 729 return (buffer[0] | 730 (buffer[1] << 8) | 731 (buffer[2] << 16) | 732 (buffer[3] << 24)); 733 734 default: 735 EFSYS_ASSERT(0); 736 return (0); 737 } 738 } 739 #endif /* EFSYS_OPT_RX_SCALE */ 740 741 #if EFSYS_OPT_RX_PACKED_STREAM 742 /* 743 * Fake length for RXQ descriptors in packed stream mode 744 * to make hardware happy 745 */ 746 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 747 #endif 748 749 void 750 ef10_rx_qpost( 751 __in efx_rxq_t *erp, 752 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 753 __in size_t size, 754 __in unsigned int ndescs, 755 __in unsigned int completed, 756 __in unsigned int added) 757 { 758 efx_qword_t qword; 759 unsigned int i; 760 unsigned int offset; 761 unsigned int id; 762 763 _NOTE(ARGUNUSED(completed)) 764 765 #if EFSYS_OPT_RX_PACKED_STREAM 766 /* 767 * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT 768 * and equal to 0 after applying mask. Hardware does not like it. 769 */ 770 if (erp->er_ev_qstate->eers_rx_packed_stream) 771 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE; 772 #endif 773 774 /* The client driver must not overfill the queue */ 775 EFSYS_ASSERT3U(added - completed + ndescs, <=, 776 EFX_RXQ_LIMIT(erp->er_mask + 1)); 777 778 id = added & (erp->er_mask); 779 for (i = 0; i < ndescs; i++) { 780 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 781 unsigned int, id, efsys_dma_addr_t, addrp[i], 782 size_t, size); 783 784 EFX_POPULATE_QWORD_3(qword, 785 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 786 ESF_DZ_RX_KER_BUF_ADDR_DW0, 787 (uint32_t)(addrp[i] & 0xffffffff), 788 ESF_DZ_RX_KER_BUF_ADDR_DW1, 789 (uint32_t)(addrp[i] >> 32)); 790 791 offset = id * sizeof (efx_qword_t); 792 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 793 794 id = (id + 1) & (erp->er_mask); 795 } 796 } 797 798 void 799 ef10_rx_qpush( 800 __in efx_rxq_t *erp, 801 __in unsigned int added, 802 __inout unsigned int *pushedp) 803 { 804 efx_nic_t *enp = erp->er_enp; 805 unsigned int pushed = *pushedp; 806 uint32_t wptr; 807 efx_dword_t dword; 808 809 /* Hardware has alignment restriction for WPTR */ 810 wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN); 811 if (pushed == wptr) 812 return; 813 814 *pushedp = wptr; 815 816 /* Push the populated descriptors out */ 817 wptr &= erp->er_mask; 818 819 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 820 821 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 822 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 823 wptr, pushed & erp->er_mask); 824 EFSYS_PIO_WRITE_BARRIER(); 825 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 826 erp->er_index, &dword, B_FALSE); 827 } 828 829 #if EFSYS_OPT_RX_PACKED_STREAM 830 831 void 832 ef10_rx_qpush_ps_credits( 833 __in efx_rxq_t *erp) 834 { 835 efx_nic_t *enp = erp->er_enp; 836 efx_dword_t dword; 837 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 838 uint32_t credits; 839 840 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 841 842 if (rxq_state->eers_rx_packed_stream_credits == 0) 843 return; 844 845 /* 846 * It is a bug if we think that FW has utilized more 847 * credits than it is allowed to have (maximum). However, 848 * make sure that we do not credit more than maximum anyway. 849 */ 850 credits = MIN(rxq_state->eers_rx_packed_stream_credits, 851 EFX_RX_PACKED_STREAM_MAX_CREDITS); 852 EFX_POPULATE_DWORD_3(dword, 853 ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, 854 ERF_DZ_RX_DESC_MAGIC_CMD, 855 ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, 856 ERF_DZ_RX_DESC_MAGIC_DATA, credits); 857 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 858 erp->er_index, &dword, B_FALSE); 859 860 rxq_state->eers_rx_packed_stream_credits = 0; 861 } 862 863 /* 864 * In accordance with SF-112241-TC the received data has the following layout: 865 * - 8 byte pseudo-header which consist of: 866 * - 4 byte little-endian timestamp 867 * - 2 byte little-endian captured length in bytes 868 * - 2 byte little-endian original packet length in bytes 869 * - captured packet bytes 870 * - optional padding to align to 64 bytes boundary 871 * - 64 bytes scratch space for the host software 872 */ 873 __checkReturn uint8_t * 874 ef10_rx_qps_packet_info( 875 __in efx_rxq_t *erp, 876 __in uint8_t *buffer, 877 __in uint32_t buffer_length, 878 __in uint32_t current_offset, 879 __out uint16_t *lengthp, 880 __out uint32_t *next_offsetp, 881 __out uint32_t *timestamp) 882 { 883 uint16_t buf_len; 884 uint8_t *pkt_start; 885 efx_qword_t *qwordp; 886 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 887 888 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 889 890 buffer += current_offset; 891 pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; 892 893 qwordp = (efx_qword_t *)buffer; 894 *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); 895 *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); 896 buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); 897 898 buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, 899 EFX_RX_PACKED_STREAM_ALIGNMENT); 900 *next_offsetp = 901 current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; 902 903 EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); 904 EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); 905 906 if ((*next_offsetp ^ current_offset) & 907 EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) 908 rxq_state->eers_rx_packed_stream_credits++; 909 910 return (pkt_start); 911 } 912 913 914 #endif 915 916 __checkReturn efx_rc_t 917 ef10_rx_qflush( 918 __in efx_rxq_t *erp) 919 { 920 efx_nic_t *enp = erp->er_enp; 921 efx_rc_t rc; 922 923 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 924 goto fail1; 925 926 return (0); 927 928 fail1: 929 /* 930 * EALREADY is not an error, but indicates that the MC has rebooted and 931 * that the RXQ has already been destroyed. Callers need to know that 932 * the RXQ flush has completed to avoid waiting until timeout for a 933 * flush done event that will not be delivered. 934 */ 935 if (rc != EALREADY) 936 EFSYS_PROBE1(fail1, efx_rc_t, rc); 937 938 return (rc); 939 } 940 941 void 942 ef10_rx_qenable( 943 __in efx_rxq_t *erp) 944 { 945 /* FIXME */ 946 _NOTE(ARGUNUSED(erp)) 947 /* FIXME */ 948 } 949 950 __checkReturn efx_rc_t 951 ef10_rx_qcreate( 952 __in efx_nic_t *enp, 953 __in unsigned int index, 954 __in unsigned int label, 955 __in efx_rxq_type_t type, 956 __in uint32_t type_data, 957 __in efsys_mem_t *esmp, 958 __in size_t ndescs, 959 __in uint32_t id, 960 __in unsigned int flags, 961 __in efx_evq_t *eep, 962 __in efx_rxq_t *erp) 963 { 964 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 965 efx_rc_t rc; 966 boolean_t disable_scatter; 967 boolean_t want_inner_classes; 968 unsigned int ps_buf_size; 969 970 _NOTE(ARGUNUSED(id, erp, type_data)) 971 972 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 973 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 974 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 975 976 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 977 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 978 979 if (!ISP2(ndescs) || 980 (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) { 981 rc = EINVAL; 982 goto fail1; 983 } 984 if (index >= encp->enc_rxq_limit) { 985 rc = EINVAL; 986 goto fail2; 987 } 988 989 switch (type) { 990 case EFX_RXQ_TYPE_DEFAULT: 991 ps_buf_size = 0; 992 break; 993 #if EFSYS_OPT_RX_PACKED_STREAM 994 case EFX_RXQ_TYPE_PACKED_STREAM: 995 switch (type_data) { 996 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M: 997 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; 998 break; 999 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K: 1000 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; 1001 break; 1002 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K: 1003 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; 1004 break; 1005 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K: 1006 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; 1007 break; 1008 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K: 1009 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; 1010 break; 1011 default: 1012 rc = ENOTSUP; 1013 goto fail3; 1014 } 1015 break; 1016 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1017 default: 1018 rc = ENOTSUP; 1019 goto fail4; 1020 } 1021 1022 #if EFSYS_OPT_RX_PACKED_STREAM 1023 if (ps_buf_size != 0) { 1024 /* Check if datapath firmware supports packed stream mode */ 1025 if (encp->enc_rx_packed_stream_supported == B_FALSE) { 1026 rc = ENOTSUP; 1027 goto fail5; 1028 } 1029 /* Check if packed stream allows configurable buffer sizes */ 1030 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) && 1031 (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { 1032 rc = ENOTSUP; 1033 goto fail6; 1034 } 1035 } 1036 #else /* EFSYS_OPT_RX_PACKED_STREAM */ 1037 EFSYS_ASSERT(ps_buf_size == 0); 1038 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1039 1040 /* Scatter can only be disabled if the firmware supports doing so */ 1041 if (flags & EFX_RXQ_FLAG_SCATTER) 1042 disable_scatter = B_FALSE; 1043 else 1044 disable_scatter = encp->enc_rx_disable_scatter_supported; 1045 1046 if (flags & EFX_RXQ_FLAG_INNER_CLASSES) 1047 want_inner_classes = B_TRUE; 1048 else 1049 want_inner_classes = B_FALSE; 1050 1051 if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index, 1052 esmp, disable_scatter, want_inner_classes, 1053 ps_buf_size)) != 0) 1054 goto fail7; 1055 1056 erp->er_eep = eep; 1057 erp->er_label = label; 1058 1059 ef10_ev_rxlabel_init(eep, erp, label, type); 1060 1061 erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label]; 1062 1063 return (0); 1064 1065 fail7: 1066 EFSYS_PROBE(fail7); 1067 #if EFSYS_OPT_RX_PACKED_STREAM 1068 fail6: 1069 EFSYS_PROBE(fail6); 1070 fail5: 1071 EFSYS_PROBE(fail5); 1072 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1073 fail4: 1074 EFSYS_PROBE(fail4); 1075 #if EFSYS_OPT_RX_PACKED_STREAM 1076 fail3: 1077 EFSYS_PROBE(fail3); 1078 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1079 fail2: 1080 EFSYS_PROBE(fail2); 1081 fail1: 1082 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1083 1084 return (rc); 1085 } 1086 1087 void 1088 ef10_rx_qdestroy( 1089 __in efx_rxq_t *erp) 1090 { 1091 efx_nic_t *enp = erp->er_enp; 1092 efx_evq_t *eep = erp->er_eep; 1093 unsigned int label = erp->er_label; 1094 1095 ef10_ev_rxlabel_fini(eep, label); 1096 1097 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1098 --enp->en_rx_qcount; 1099 1100 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1101 } 1102 1103 void 1104 ef10_rx_fini( 1105 __in efx_nic_t *enp) 1106 { 1107 #if EFSYS_OPT_RX_SCALE 1108 if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE) 1109 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 1110 enp->en_rss_context = 0; 1111 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 1112 #else 1113 _NOTE(ARGUNUSED(enp)) 1114 #endif /* EFSYS_OPT_RX_SCALE */ 1115 } 1116 1117 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 1118