1 /*- 2 * Copyright (c) 2012-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 38 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 39 40 41 static __checkReturn efx_rc_t 42 efx_mcdi_init_rxq( 43 __in efx_nic_t *enp, 44 __in uint32_t size, 45 __in uint32_t target_evq, 46 __in uint32_t label, 47 __in uint32_t instance, 48 __in efsys_mem_t *esmp, 49 __in boolean_t disable_scatter, 50 __in uint32_t ps_bufsize) 51 { 52 efx_mcdi_req_t req; 53 uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN, 54 MC_CMD_INIT_RXQ_EXT_OUT_LEN)]; 55 int npages = EFX_RXQ_NBUFS(size); 56 int i; 57 efx_qword_t *dma_addr; 58 uint64_t addr; 59 efx_rc_t rc; 60 uint32_t dma_mode; 61 62 /* If this changes, then the payload size might need to change. */ 63 EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0); 64 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS); 65 66 if (ps_bufsize > 0) 67 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM; 68 else 69 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET; 70 71 (void) memset(payload, 0, sizeof (payload)); 72 req.emr_cmd = MC_CMD_INIT_RXQ; 73 req.emr_in_buf = payload; 74 req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN; 75 req.emr_out_buf = payload; 76 req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN; 77 78 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, size); 79 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq); 80 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label); 81 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance); 82 MCDI_IN_POPULATE_DWORD_8(req, INIT_RXQ_EXT_IN_FLAGS, 83 INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0, 84 INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0, 85 INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0, 86 INIT_RXQ_EXT_IN_CRC_MODE, 0, 87 INIT_RXQ_EXT_IN_FLAG_PREFIX, 1, 88 INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter, 89 INIT_RXQ_EXT_IN_DMA_MODE, 90 dma_mode, 91 INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize); 92 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0); 93 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 94 95 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); 96 addr = EFSYS_MEM_ADDR(esmp); 97 98 for (i = 0; i < npages; i++) { 99 EFX_POPULATE_QWORD_2(*dma_addr, 100 EFX_DWORD_1, (uint32_t)(addr >> 32), 101 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 102 103 dma_addr++; 104 addr += EFX_BUF_SIZE; 105 } 106 107 efx_mcdi_execute(enp, &req); 108 109 if (req.emr_rc != 0) { 110 rc = req.emr_rc; 111 goto fail1; 112 } 113 114 return (0); 115 116 fail1: 117 EFSYS_PROBE1(fail1, efx_rc_t, rc); 118 119 return (rc); 120 } 121 122 static __checkReturn efx_rc_t 123 efx_mcdi_fini_rxq( 124 __in efx_nic_t *enp, 125 __in uint32_t instance) 126 { 127 efx_mcdi_req_t req; 128 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN, 129 MC_CMD_FINI_RXQ_OUT_LEN)]; 130 efx_rc_t rc; 131 132 (void) memset(payload, 0, sizeof (payload)); 133 req.emr_cmd = MC_CMD_FINI_RXQ; 134 req.emr_in_buf = payload; 135 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; 136 req.emr_out_buf = payload; 137 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; 138 139 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); 140 141 efx_mcdi_execute_quiet(enp, &req); 142 143 if (req.emr_rc != 0) { 144 rc = req.emr_rc; 145 goto fail1; 146 } 147 148 return (0); 149 150 fail1: 151 /* 152 * EALREADY is not an error, but indicates that the MC has rebooted and 153 * that the RXQ has already been destroyed. 154 */ 155 if (rc != EALREADY) 156 EFSYS_PROBE1(fail1, efx_rc_t, rc); 157 158 return (rc); 159 } 160 161 #if EFSYS_OPT_RX_SCALE 162 static __checkReturn efx_rc_t 163 efx_mcdi_rss_context_alloc( 164 __in efx_nic_t *enp, 165 __in efx_rx_scale_context_type_t type, 166 __in uint32_t num_queues, 167 __out uint32_t *rss_contextp) 168 { 169 efx_mcdi_req_t req; 170 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 171 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; 172 uint32_t rss_context; 173 uint32_t context_type; 174 efx_rc_t rc; 175 176 if (num_queues > EFX_MAXRSS) { 177 rc = EINVAL; 178 goto fail1; 179 } 180 181 switch (type) { 182 case EFX_RX_SCALE_EXCLUSIVE: 183 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 184 break; 185 case EFX_RX_SCALE_SHARED: 186 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 187 break; 188 default: 189 rc = EINVAL; 190 goto fail2; 191 } 192 193 (void) memset(payload, 0, sizeof (payload)); 194 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 195 req.emr_in_buf = payload; 196 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 197 req.emr_out_buf = payload; 198 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 199 200 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 201 EVB_PORT_ID_ASSIGNED); 202 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 203 /* NUM_QUEUES is only used to validate indirection table offsets */ 204 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 205 206 efx_mcdi_execute(enp, &req); 207 208 if (req.emr_rc != 0) { 209 rc = req.emr_rc; 210 goto fail3; 211 } 212 213 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 214 rc = EMSGSIZE; 215 goto fail4; 216 } 217 218 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 219 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 220 rc = ENOENT; 221 goto fail5; 222 } 223 224 *rss_contextp = rss_context; 225 226 return (0); 227 228 fail5: 229 EFSYS_PROBE(fail5); 230 fail4: 231 EFSYS_PROBE(fail4); 232 fail3: 233 EFSYS_PROBE(fail3); 234 fail2: 235 EFSYS_PROBE(fail2); 236 fail1: 237 EFSYS_PROBE1(fail1, efx_rc_t, rc); 238 239 return (rc); 240 } 241 #endif /* EFSYS_OPT_RX_SCALE */ 242 243 #if EFSYS_OPT_RX_SCALE 244 static efx_rc_t 245 efx_mcdi_rss_context_free( 246 __in efx_nic_t *enp, 247 __in uint32_t rss_context) 248 { 249 efx_mcdi_req_t req; 250 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 251 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; 252 efx_rc_t rc; 253 254 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 255 rc = EINVAL; 256 goto fail1; 257 } 258 259 (void) memset(payload, 0, sizeof (payload)); 260 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 261 req.emr_in_buf = payload; 262 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 263 req.emr_out_buf = payload; 264 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 265 266 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 267 268 efx_mcdi_execute_quiet(enp, &req); 269 270 if (req.emr_rc != 0) { 271 rc = req.emr_rc; 272 goto fail2; 273 } 274 275 return (0); 276 277 fail2: 278 EFSYS_PROBE(fail2); 279 fail1: 280 EFSYS_PROBE1(fail1, efx_rc_t, rc); 281 282 return (rc); 283 } 284 #endif /* EFSYS_OPT_RX_SCALE */ 285 286 #if EFSYS_OPT_RX_SCALE 287 static efx_rc_t 288 efx_mcdi_rss_context_set_flags( 289 __in efx_nic_t *enp, 290 __in uint32_t rss_context, 291 __in efx_rx_hash_type_t type) 292 { 293 efx_mcdi_req_t req; 294 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 295 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; 296 efx_rc_t rc; 297 298 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 299 rc = EINVAL; 300 goto fail1; 301 } 302 303 (void) memset(payload, 0, sizeof (payload)); 304 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 305 req.emr_in_buf = payload; 306 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 307 req.emr_out_buf = payload; 308 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 309 310 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 311 rss_context); 312 313 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 314 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 315 (type & EFX_RX_HASH_IPV4) ? 1 : 0, 316 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 317 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, 318 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 319 (type & EFX_RX_HASH_IPV6) ? 1 : 0, 320 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 321 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0); 322 323 efx_mcdi_execute(enp, &req); 324 325 if (req.emr_rc != 0) { 326 rc = req.emr_rc; 327 goto fail2; 328 } 329 330 return (0); 331 332 fail2: 333 EFSYS_PROBE(fail2); 334 fail1: 335 EFSYS_PROBE1(fail1, efx_rc_t, rc); 336 337 return (rc); 338 } 339 #endif /* EFSYS_OPT_RX_SCALE */ 340 341 #if EFSYS_OPT_RX_SCALE 342 static efx_rc_t 343 efx_mcdi_rss_context_set_key( 344 __in efx_nic_t *enp, 345 __in uint32_t rss_context, 346 __in_ecount(n) uint8_t *key, 347 __in size_t n) 348 { 349 efx_mcdi_req_t req; 350 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 351 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; 352 efx_rc_t rc; 353 354 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 355 rc = EINVAL; 356 goto fail1; 357 } 358 359 (void) memset(payload, 0, sizeof (payload)); 360 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 361 req.emr_in_buf = payload; 362 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 363 req.emr_out_buf = payload; 364 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 365 366 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 367 rss_context); 368 369 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 370 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 371 rc = EINVAL; 372 goto fail2; 373 } 374 375 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 376 key, n); 377 378 efx_mcdi_execute(enp, &req); 379 380 if (req.emr_rc != 0) { 381 rc = req.emr_rc; 382 goto fail3; 383 } 384 385 return (0); 386 387 fail3: 388 EFSYS_PROBE(fail3); 389 fail2: 390 EFSYS_PROBE(fail2); 391 fail1: 392 EFSYS_PROBE1(fail1, efx_rc_t, rc); 393 394 return (rc); 395 } 396 #endif /* EFSYS_OPT_RX_SCALE */ 397 398 #if EFSYS_OPT_RX_SCALE 399 static efx_rc_t 400 efx_mcdi_rss_context_set_table( 401 __in efx_nic_t *enp, 402 __in uint32_t rss_context, 403 __in_ecount(n) unsigned int *table, 404 __in size_t n) 405 { 406 efx_mcdi_req_t req; 407 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 408 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; 409 uint8_t *req_table; 410 int i, rc; 411 412 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 413 rc = EINVAL; 414 goto fail1; 415 } 416 417 (void) memset(payload, 0, sizeof (payload)); 418 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 419 req.emr_in_buf = payload; 420 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 421 req.emr_out_buf = payload; 422 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 423 424 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 425 rss_context); 426 427 req_table = 428 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 429 430 for (i = 0; 431 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 432 i++) { 433 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 434 } 435 436 efx_mcdi_execute(enp, &req); 437 438 if (req.emr_rc != 0) { 439 rc = req.emr_rc; 440 goto fail2; 441 } 442 443 return (0); 444 445 fail2: 446 EFSYS_PROBE(fail2); 447 fail1: 448 EFSYS_PROBE1(fail1, efx_rc_t, rc); 449 450 return (rc); 451 } 452 #endif /* EFSYS_OPT_RX_SCALE */ 453 454 455 __checkReturn efx_rc_t 456 ef10_rx_init( 457 __in efx_nic_t *enp) 458 { 459 #if EFSYS_OPT_RX_SCALE 460 461 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 462 &enp->en_rss_context) == 0) { 463 /* 464 * Allocated an exclusive RSS context, which allows both the 465 * indirection table and key to be modified. 466 */ 467 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 468 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 469 } else { 470 /* 471 * Failed to allocate an exclusive RSS context. Continue 472 * operation without support for RSS. The pseudo-header in 473 * received packets will not contain a Toeplitz hash value. 474 */ 475 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 476 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 477 } 478 479 #endif /* EFSYS_OPT_RX_SCALE */ 480 481 return (0); 482 } 483 484 #if EFSYS_OPT_RX_SCATTER 485 __checkReturn efx_rc_t 486 ef10_rx_scatter_enable( 487 __in efx_nic_t *enp, 488 __in unsigned int buf_size) 489 { 490 _NOTE(ARGUNUSED(enp, buf_size)) 491 return (0); 492 } 493 #endif /* EFSYS_OPT_RX_SCATTER */ 494 495 #if EFSYS_OPT_RX_SCALE 496 __checkReturn efx_rc_t 497 ef10_rx_scale_context_alloc( 498 __in efx_nic_t *enp, 499 __in efx_rx_scale_context_type_t type, 500 __in uint32_t num_queues, 501 __out uint32_t *rss_contextp) 502 { 503 efx_rc_t rc; 504 505 rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp); 506 if (rc != 0) 507 goto fail1; 508 509 return (0); 510 511 fail1: 512 EFSYS_PROBE1(fail1, efx_rc_t, rc); 513 return (rc); 514 } 515 #endif /* EFSYS_OPT_RX_SCALE */ 516 517 #if EFSYS_OPT_RX_SCALE 518 __checkReturn efx_rc_t 519 ef10_rx_scale_context_free( 520 __in efx_nic_t *enp, 521 __in uint32_t rss_context) 522 { 523 efx_rc_t rc; 524 525 rc = efx_mcdi_rss_context_free(enp, rss_context); 526 if (rc != 0) 527 goto fail1; 528 529 return (0); 530 531 fail1: 532 EFSYS_PROBE1(fail1, efx_rc_t, rc); 533 return (rc); 534 } 535 #endif /* EFSYS_OPT_RX_SCALE */ 536 537 #if EFSYS_OPT_RX_SCALE 538 __checkReturn efx_rc_t 539 ef10_rx_scale_mode_set( 540 __in efx_nic_t *enp, 541 __in uint32_t rss_context, 542 __in efx_rx_hash_alg_t alg, 543 __in efx_rx_hash_type_t type, 544 __in boolean_t insert) 545 { 546 efx_rc_t rc; 547 548 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); 549 EFSYS_ASSERT3U(insert, ==, B_TRUE); 550 551 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { 552 rc = EINVAL; 553 goto fail1; 554 } 555 556 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 557 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 558 rc = ENOTSUP; 559 goto fail2; 560 } 561 rss_context = enp->en_rss_context; 562 } 563 564 if ((rc = efx_mcdi_rss_context_set_flags(enp, 565 rss_context, type)) != 0) 566 goto fail3; 567 568 return (0); 569 570 fail3: 571 EFSYS_PROBE(fail3); 572 fail2: 573 EFSYS_PROBE(fail2); 574 fail1: 575 EFSYS_PROBE1(fail1, efx_rc_t, rc); 576 577 return (rc); 578 } 579 #endif /* EFSYS_OPT_RX_SCALE */ 580 581 #if EFSYS_OPT_RX_SCALE 582 __checkReturn efx_rc_t 583 ef10_rx_scale_key_set( 584 __in efx_nic_t *enp, 585 __in uint32_t rss_context, 586 __in_ecount(n) uint8_t *key, 587 __in size_t n) 588 { 589 efx_rc_t rc; 590 591 EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE == 592 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 593 594 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 595 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 596 rc = ENOTSUP; 597 goto fail1; 598 } 599 rss_context = enp->en_rss_context; 600 } 601 602 if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0) 603 goto fail2; 604 605 return (0); 606 607 fail2: 608 EFSYS_PROBE(fail2); 609 fail1: 610 EFSYS_PROBE1(fail1, efx_rc_t, rc); 611 612 return (rc); 613 } 614 #endif /* EFSYS_OPT_RX_SCALE */ 615 616 #if EFSYS_OPT_RX_SCALE 617 __checkReturn efx_rc_t 618 ef10_rx_scale_tbl_set( 619 __in efx_nic_t *enp, 620 __in uint32_t rss_context, 621 __in_ecount(n) unsigned int *table, 622 __in size_t n) 623 { 624 efx_rc_t rc; 625 626 627 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 628 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 629 rc = ENOTSUP; 630 goto fail1; 631 } 632 rss_context = enp->en_rss_context; 633 } 634 635 if ((rc = efx_mcdi_rss_context_set_table(enp, 636 rss_context, table, n)) != 0) 637 goto fail2; 638 639 return (0); 640 641 fail2: 642 EFSYS_PROBE(fail2); 643 fail1: 644 EFSYS_PROBE1(fail1, efx_rc_t, rc); 645 646 return (rc); 647 } 648 #endif /* EFSYS_OPT_RX_SCALE */ 649 650 651 /* 652 * EF10 RX pseudo-header 653 * --------------------- 654 * 655 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 656 * 657 * +00: Toeplitz hash value. 658 * (32bit little-endian) 659 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 660 * (16bit big-endian) 661 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 662 * (16bit big-endian) 663 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 664 * (16bit little-endian) 665 * +10: MAC timestamp. Zero if timestamping is not enabled. 666 * (32bit little-endian) 667 * 668 * See "The RX Pseudo-header" in SF-109306-TC. 669 */ 670 671 __checkReturn efx_rc_t 672 ef10_rx_prefix_pktlen( 673 __in efx_nic_t *enp, 674 __in uint8_t *buffer, 675 __out uint16_t *lengthp) 676 { 677 _NOTE(ARGUNUSED(enp)) 678 679 /* 680 * The RX pseudo-header contains the packet length, excluding the 681 * pseudo-header. If the hardware receive datapath was operating in 682 * cut-through mode then the length in the RX pseudo-header will be 683 * zero, and the packet length must be obtained from the DMA length 684 * reported in the RX event. 685 */ 686 *lengthp = buffer[8] | (buffer[9] << 8); 687 return (0); 688 } 689 690 #if EFSYS_OPT_RX_SCALE 691 __checkReturn uint32_t 692 ef10_rx_prefix_hash( 693 __in efx_nic_t *enp, 694 __in efx_rx_hash_alg_t func, 695 __in uint8_t *buffer) 696 { 697 _NOTE(ARGUNUSED(enp)) 698 699 switch (func) { 700 case EFX_RX_HASHALG_TOEPLITZ: 701 return (buffer[0] | 702 (buffer[1] << 8) | 703 (buffer[2] << 16) | 704 (buffer[3] << 24)); 705 706 default: 707 EFSYS_ASSERT(0); 708 return (0); 709 } 710 } 711 #endif /* EFSYS_OPT_RX_SCALE */ 712 713 #if EFSYS_OPT_RX_PACKED_STREAM 714 /* 715 * Fake length for RXQ descriptors in packed stream mode 716 * to make hardware happy 717 */ 718 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 719 #endif 720 721 void 722 ef10_rx_qpost( 723 __in efx_rxq_t *erp, 724 __in_ecount(n) efsys_dma_addr_t *addrp, 725 __in size_t size, 726 __in unsigned int n, 727 __in unsigned int completed, 728 __in unsigned int added) 729 { 730 efx_qword_t qword; 731 unsigned int i; 732 unsigned int offset; 733 unsigned int id; 734 735 #if EFSYS_OPT_RX_PACKED_STREAM 736 /* 737 * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT 738 * and equal to 0 after applying mask. Hardware does not like it. 739 */ 740 if (erp->er_ev_qstate->eers_rx_packed_stream) 741 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE; 742 #endif 743 744 /* The client driver must not overfill the queue */ 745 EFSYS_ASSERT3U(added - completed + n, <=, 746 EFX_RXQ_LIMIT(erp->er_mask + 1)); 747 748 id = added & (erp->er_mask); 749 for (i = 0; i < n; i++) { 750 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 751 unsigned int, id, efsys_dma_addr_t, addrp[i], 752 size_t, size); 753 754 EFX_POPULATE_QWORD_3(qword, 755 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 756 ESF_DZ_RX_KER_BUF_ADDR_DW0, 757 (uint32_t)(addrp[i] & 0xffffffff), 758 ESF_DZ_RX_KER_BUF_ADDR_DW1, 759 (uint32_t)(addrp[i] >> 32)); 760 761 offset = id * sizeof (efx_qword_t); 762 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 763 764 id = (id + 1) & (erp->er_mask); 765 } 766 } 767 768 void 769 ef10_rx_qpush( 770 __in efx_rxq_t *erp, 771 __in unsigned int added, 772 __inout unsigned int *pushedp) 773 { 774 efx_nic_t *enp = erp->er_enp; 775 unsigned int pushed = *pushedp; 776 uint32_t wptr; 777 efx_dword_t dword; 778 779 /* Hardware has alignment restriction for WPTR */ 780 wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN); 781 if (pushed == wptr) 782 return; 783 784 *pushedp = wptr; 785 786 /* Push the populated descriptors out */ 787 wptr &= erp->er_mask; 788 789 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 790 791 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 792 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 793 wptr, pushed & erp->er_mask); 794 EFSYS_PIO_WRITE_BARRIER(); 795 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 796 erp->er_index, &dword, B_FALSE); 797 } 798 799 #if EFSYS_OPT_RX_PACKED_STREAM 800 801 void 802 ef10_rx_qpush_ps_credits( 803 __in efx_rxq_t *erp) 804 { 805 efx_nic_t *enp = erp->er_enp; 806 efx_dword_t dword; 807 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 808 uint32_t credits; 809 810 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 811 812 if (rxq_state->eers_rx_packed_stream_credits == 0) 813 return; 814 815 /* 816 * It is a bug if we think that FW has utilized more 817 * credits than it is allowed to have (maximum). However, 818 * make sure that we do not credit more than maximum anyway. 819 */ 820 credits = MIN(rxq_state->eers_rx_packed_stream_credits, 821 EFX_RX_PACKED_STREAM_MAX_CREDITS); 822 EFX_POPULATE_DWORD_3(dword, 823 ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, 824 ERF_DZ_RX_DESC_MAGIC_CMD, 825 ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, 826 ERF_DZ_RX_DESC_MAGIC_DATA, credits); 827 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 828 erp->er_index, &dword, B_FALSE); 829 830 rxq_state->eers_rx_packed_stream_credits = 0; 831 } 832 833 /* 834 * In accordance with SF-112241-TC the received data has the following layout: 835 * - 8 byte pseudo-header which consist of: 836 * - 4 byte little-endian timestamp 837 * - 2 byte little-endian captured length in bytes 838 * - 2 byte little-endian original packet length in bytes 839 * - captured packet bytes 840 * - optional padding to align to 64 bytes boundary 841 * - 64 bytes scratch space for the host software 842 */ 843 __checkReturn uint8_t * 844 ef10_rx_qps_packet_info( 845 __in efx_rxq_t *erp, 846 __in uint8_t *buffer, 847 __in uint32_t buffer_length, 848 __in uint32_t current_offset, 849 __out uint16_t *lengthp, 850 __out uint32_t *next_offsetp, 851 __out uint32_t *timestamp) 852 { 853 uint16_t buf_len; 854 uint8_t *pkt_start; 855 efx_qword_t *qwordp; 856 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 857 858 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 859 860 buffer += current_offset; 861 pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; 862 863 qwordp = (efx_qword_t *)buffer; 864 *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); 865 *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); 866 buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); 867 868 buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, 869 EFX_RX_PACKED_STREAM_ALIGNMENT); 870 *next_offsetp = 871 current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; 872 873 EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); 874 EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); 875 876 if ((*next_offsetp ^ current_offset) & 877 EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) 878 rxq_state->eers_rx_packed_stream_credits++; 879 880 return (pkt_start); 881 } 882 883 884 #endif 885 886 __checkReturn efx_rc_t 887 ef10_rx_qflush( 888 __in efx_rxq_t *erp) 889 { 890 efx_nic_t *enp = erp->er_enp; 891 efx_rc_t rc; 892 893 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 894 goto fail1; 895 896 return (0); 897 898 fail1: 899 /* 900 * EALREADY is not an error, but indicates that the MC has rebooted and 901 * that the RXQ has already been destroyed. Callers need to know that 902 * the RXQ flush has completed to avoid waiting until timeout for a 903 * flush done event that will not be delivered. 904 */ 905 if (rc != EALREADY) 906 EFSYS_PROBE1(fail1, efx_rc_t, rc); 907 908 return (rc); 909 } 910 911 void 912 ef10_rx_qenable( 913 __in efx_rxq_t *erp) 914 { 915 /* FIXME */ 916 _NOTE(ARGUNUSED(erp)) 917 /* FIXME */ 918 } 919 920 __checkReturn efx_rc_t 921 ef10_rx_qcreate( 922 __in efx_nic_t *enp, 923 __in unsigned int index, 924 __in unsigned int label, 925 __in efx_rxq_type_t type, 926 __in efsys_mem_t *esmp, 927 __in size_t n, 928 __in uint32_t id, 929 __in efx_evq_t *eep, 930 __in efx_rxq_t *erp) 931 { 932 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 933 efx_rc_t rc; 934 boolean_t disable_scatter; 935 unsigned int ps_buf_size; 936 937 _NOTE(ARGUNUSED(id, erp)) 938 939 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 940 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 941 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 942 943 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 944 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 945 946 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 947 rc = EINVAL; 948 goto fail1; 949 } 950 if (index >= encp->enc_rxq_limit) { 951 rc = EINVAL; 952 goto fail2; 953 } 954 955 switch (type) { 956 case EFX_RXQ_TYPE_DEFAULT: 957 case EFX_RXQ_TYPE_SCATTER: 958 ps_buf_size = 0; 959 break; 960 #if EFSYS_OPT_RX_PACKED_STREAM 961 case EFX_RXQ_TYPE_PACKED_STREAM_1M: 962 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; 963 break; 964 case EFX_RXQ_TYPE_PACKED_STREAM_512K: 965 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; 966 break; 967 case EFX_RXQ_TYPE_PACKED_STREAM_256K: 968 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; 969 break; 970 case EFX_RXQ_TYPE_PACKED_STREAM_128K: 971 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; 972 break; 973 case EFX_RXQ_TYPE_PACKED_STREAM_64K: 974 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; 975 break; 976 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 977 default: 978 rc = ENOTSUP; 979 goto fail3; 980 } 981 982 #if EFSYS_OPT_RX_PACKED_STREAM 983 if (ps_buf_size != 0) { 984 /* Check if datapath firmware supports packed stream mode */ 985 if (encp->enc_rx_packed_stream_supported == B_FALSE) { 986 rc = ENOTSUP; 987 goto fail4; 988 } 989 /* Check if packed stream allows configurable buffer sizes */ 990 if ((type != EFX_RXQ_TYPE_PACKED_STREAM_1M) && 991 (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { 992 rc = ENOTSUP; 993 goto fail5; 994 } 995 } 996 #else /* EFSYS_OPT_RX_PACKED_STREAM */ 997 EFSYS_ASSERT(ps_buf_size == 0); 998 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 999 1000 /* Scatter can only be disabled if the firmware supports doing so */ 1001 if (type == EFX_RXQ_TYPE_SCATTER) 1002 disable_scatter = B_FALSE; 1003 else 1004 disable_scatter = encp->enc_rx_disable_scatter_supported; 1005 1006 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index, 1007 esmp, disable_scatter, ps_buf_size)) != 0) 1008 goto fail6; 1009 1010 erp->er_eep = eep; 1011 erp->er_label = label; 1012 1013 ef10_ev_rxlabel_init(eep, erp, label, type); 1014 1015 erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label]; 1016 1017 return (0); 1018 1019 fail6: 1020 EFSYS_PROBE(fail6); 1021 #if EFSYS_OPT_RX_PACKED_STREAM 1022 fail5: 1023 EFSYS_PROBE(fail5); 1024 fail4: 1025 EFSYS_PROBE(fail4); 1026 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1027 fail3: 1028 EFSYS_PROBE(fail3); 1029 fail2: 1030 EFSYS_PROBE(fail2); 1031 fail1: 1032 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1033 1034 return (rc); 1035 } 1036 1037 void 1038 ef10_rx_qdestroy( 1039 __in efx_rxq_t *erp) 1040 { 1041 efx_nic_t *enp = erp->er_enp; 1042 efx_evq_t *eep = erp->er_eep; 1043 unsigned int label = erp->er_label; 1044 1045 ef10_ev_rxlabel_fini(eep, label); 1046 1047 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1048 --enp->en_rx_qcount; 1049 1050 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1051 } 1052 1053 void 1054 ef10_rx_fini( 1055 __in efx_nic_t *enp) 1056 { 1057 #if EFSYS_OPT_RX_SCALE 1058 if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE) 1059 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 1060 enp->en_rss_context = 0; 1061 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 1062 #else 1063 _NOTE(ARGUNUSED(enp)) 1064 #endif /* EFSYS_OPT_RX_SCALE */ 1065 } 1066 1067 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 1068