1 /*- 2 * Copyright (c) 2012-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 38 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 39 40 41 static __checkReturn efx_rc_t 42 efx_mcdi_init_rxq( 43 __in efx_nic_t *enp, 44 __in uint32_t size, 45 __in uint32_t target_evq, 46 __in uint32_t label, 47 __in uint32_t instance, 48 __in efsys_mem_t *esmp, 49 __in boolean_t disable_scatter) 50 { 51 efx_mcdi_req_t req; 52 uint8_t payload[ 53 MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS))]; 54 int npages = EFX_RXQ_NBUFS(size); 55 int i; 56 efx_qword_t *dma_addr; 57 uint64_t addr; 58 efx_rc_t rc; 59 60 /* If this changes, then the payload size might need to change. */ 61 EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0); 62 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS); 63 64 (void) memset(payload, 0, sizeof (payload)); 65 req.emr_cmd = MC_CMD_INIT_RXQ; 66 req.emr_in_buf = payload; 67 req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages); 68 req.emr_out_buf = payload; 69 req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN; 70 71 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size); 72 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq); 73 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label); 74 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance); 75 MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS, 76 INIT_RXQ_IN_FLAG_BUFF_MODE, 0, 77 INIT_RXQ_IN_FLAG_HDR_SPLIT, 0, 78 INIT_RXQ_IN_FLAG_TIMESTAMP, 0, 79 INIT_RXQ_IN_CRC_MODE, 0, 80 INIT_RXQ_IN_FLAG_PREFIX, 1, 81 INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter); 82 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0); 83 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 84 85 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); 86 addr = EFSYS_MEM_ADDR(esmp); 87 88 for (i = 0; i < npages; i++) { 89 EFX_POPULATE_QWORD_2(*dma_addr, 90 EFX_DWORD_1, (uint32_t)(addr >> 32), 91 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 92 93 dma_addr++; 94 addr += EFX_BUF_SIZE; 95 } 96 97 efx_mcdi_execute(enp, &req); 98 99 if (req.emr_rc != 0) { 100 rc = req.emr_rc; 101 goto fail1; 102 } 103 104 return (0); 105 106 fail1: 107 EFSYS_PROBE1(fail1, efx_rc_t, rc); 108 109 return (rc); 110 } 111 112 static __checkReturn efx_rc_t 113 efx_mcdi_fini_rxq( 114 __in efx_nic_t *enp, 115 __in uint32_t instance) 116 { 117 efx_mcdi_req_t req; 118 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN, 119 MC_CMD_FINI_RXQ_OUT_LEN)]; 120 efx_rc_t rc; 121 122 (void) memset(payload, 0, sizeof (payload)); 123 req.emr_cmd = MC_CMD_FINI_RXQ; 124 req.emr_in_buf = payload; 125 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; 126 req.emr_out_buf = payload; 127 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; 128 129 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); 130 131 efx_mcdi_execute_quiet(enp, &req); 132 133 if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) { 134 rc = req.emr_rc; 135 goto fail1; 136 } 137 138 return (0); 139 140 fail1: 141 EFSYS_PROBE1(fail1, efx_rc_t, rc); 142 143 return (rc); 144 } 145 146 #if EFSYS_OPT_RX_SCALE 147 static __checkReturn efx_rc_t 148 efx_mcdi_rss_context_alloc( 149 __in efx_nic_t *enp, 150 __in efx_rx_scale_support_t scale_support, 151 __in uint32_t num_queues, 152 __out uint32_t *rss_contextp) 153 { 154 efx_mcdi_req_t req; 155 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 156 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; 157 uint32_t rss_context; 158 uint32_t context_type; 159 efx_rc_t rc; 160 161 if (num_queues > EFX_MAXRSS) { 162 rc = EINVAL; 163 goto fail1; 164 } 165 166 switch (scale_support) { 167 case EFX_RX_SCALE_EXCLUSIVE: 168 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 169 break; 170 case EFX_RX_SCALE_SHARED: 171 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 172 break; 173 default: 174 rc = EINVAL; 175 goto fail2; 176 } 177 178 (void) memset(payload, 0, sizeof (payload)); 179 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 180 req.emr_in_buf = payload; 181 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 182 req.emr_out_buf = payload; 183 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 184 185 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 186 EVB_PORT_ID_ASSIGNED); 187 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 188 /* NUM_QUEUES is only used to validate indirection table offsets */ 189 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 190 191 efx_mcdi_execute(enp, &req); 192 193 if (req.emr_rc != 0) { 194 rc = req.emr_rc; 195 goto fail3; 196 } 197 198 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 199 rc = EMSGSIZE; 200 goto fail4; 201 } 202 203 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 204 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 205 rc = ENOENT; 206 goto fail5; 207 } 208 209 *rss_contextp = rss_context; 210 211 return (0); 212 213 fail5: 214 EFSYS_PROBE(fail5); 215 fail4: 216 EFSYS_PROBE(fail4); 217 fail3: 218 EFSYS_PROBE(fail3); 219 fail2: 220 EFSYS_PROBE(fail2); 221 fail1: 222 EFSYS_PROBE1(fail1, efx_rc_t, rc); 223 224 return (rc); 225 } 226 #endif /* EFSYS_OPT_RX_SCALE */ 227 228 #if EFSYS_OPT_RX_SCALE 229 static efx_rc_t 230 efx_mcdi_rss_context_free( 231 __in efx_nic_t *enp, 232 __in uint32_t rss_context) 233 { 234 efx_mcdi_req_t req; 235 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 236 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; 237 efx_rc_t rc; 238 239 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 240 rc = EINVAL; 241 goto fail1; 242 } 243 244 (void) memset(payload, 0, sizeof (payload)); 245 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 246 req.emr_in_buf = payload; 247 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 248 req.emr_out_buf = payload; 249 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 250 251 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 252 253 efx_mcdi_execute_quiet(enp, &req); 254 255 if (req.emr_rc != 0) { 256 rc = req.emr_rc; 257 goto fail2; 258 } 259 260 return (0); 261 262 fail2: 263 EFSYS_PROBE(fail2); 264 fail1: 265 EFSYS_PROBE1(fail1, efx_rc_t, rc); 266 267 return (rc); 268 } 269 #endif /* EFSYS_OPT_RX_SCALE */ 270 271 #if EFSYS_OPT_RX_SCALE 272 static efx_rc_t 273 efx_mcdi_rss_context_set_flags( 274 __in efx_nic_t *enp, 275 __in uint32_t rss_context, 276 __in efx_rx_hash_type_t type) 277 { 278 efx_mcdi_req_t req; 279 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 280 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; 281 efx_rc_t rc; 282 283 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 284 rc = EINVAL; 285 goto fail1; 286 } 287 288 (void) memset(payload, 0, sizeof (payload)); 289 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 290 req.emr_in_buf = payload; 291 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 292 req.emr_out_buf = payload; 293 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 294 295 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 296 rss_context); 297 298 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 299 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 300 (type & EFX_RX_HASH_IPV4) ? 1 : 0, 301 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 302 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, 303 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 304 (type & EFX_RX_HASH_IPV6) ? 1 : 0, 305 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 306 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0); 307 308 efx_mcdi_execute(enp, &req); 309 310 if (req.emr_rc != 0) { 311 rc = req.emr_rc; 312 goto fail2; 313 } 314 315 return (0); 316 317 fail2: 318 EFSYS_PROBE(fail2); 319 fail1: 320 EFSYS_PROBE1(fail1, efx_rc_t, rc); 321 322 return (rc); 323 } 324 #endif /* EFSYS_OPT_RX_SCALE */ 325 326 #if EFSYS_OPT_RX_SCALE 327 static efx_rc_t 328 efx_mcdi_rss_context_set_key( 329 __in efx_nic_t *enp, 330 __in uint32_t rss_context, 331 __in_ecount(n) uint8_t *key, 332 __in size_t n) 333 { 334 efx_mcdi_req_t req; 335 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 336 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; 337 efx_rc_t rc; 338 339 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 340 rc = EINVAL; 341 goto fail1; 342 } 343 344 (void) memset(payload, 0, sizeof (payload)); 345 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 346 req.emr_in_buf = payload; 347 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 348 req.emr_out_buf = payload; 349 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 350 351 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 352 rss_context); 353 354 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 355 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 356 rc = EINVAL; 357 goto fail2; 358 } 359 360 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 361 key, n); 362 363 efx_mcdi_execute(enp, &req); 364 365 if (req.emr_rc != 0) { 366 rc = req.emr_rc; 367 goto fail3; 368 } 369 370 return (0); 371 372 fail3: 373 EFSYS_PROBE(fail3); 374 fail2: 375 EFSYS_PROBE(fail2); 376 fail1: 377 EFSYS_PROBE1(fail1, efx_rc_t, rc); 378 379 return (rc); 380 } 381 #endif /* EFSYS_OPT_RX_SCALE */ 382 383 #if EFSYS_OPT_RX_SCALE 384 static efx_rc_t 385 efx_mcdi_rss_context_set_table( 386 __in efx_nic_t *enp, 387 __in uint32_t rss_context, 388 __in_ecount(n) unsigned int *table, 389 __in size_t n) 390 { 391 efx_mcdi_req_t req; 392 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 393 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; 394 uint8_t *req_table; 395 int i, rc; 396 397 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 398 rc = EINVAL; 399 goto fail1; 400 } 401 402 (void) memset(payload, 0, sizeof (payload)); 403 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 404 req.emr_in_buf = payload; 405 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 406 req.emr_out_buf = payload; 407 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 408 409 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 410 rss_context); 411 412 req_table = 413 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 414 415 for (i = 0; 416 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 417 i++) { 418 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 419 } 420 421 efx_mcdi_execute(enp, &req); 422 423 if (req.emr_rc != 0) { 424 rc = req.emr_rc; 425 goto fail2; 426 } 427 428 return (0); 429 430 fail2: 431 EFSYS_PROBE(fail2); 432 fail1: 433 EFSYS_PROBE1(fail1, efx_rc_t, rc); 434 435 return (rc); 436 } 437 #endif /* EFSYS_OPT_RX_SCALE */ 438 439 440 __checkReturn efx_rc_t 441 ef10_rx_init( 442 __in efx_nic_t *enp) 443 { 444 #if EFSYS_OPT_RX_SCALE 445 446 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 447 &enp->en_rss_context) == 0) { 448 /* 449 * Allocated an exclusive RSS context, which allows both the 450 * indirection table and key to be modified. 451 */ 452 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; 453 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 454 } else { 455 /* 456 * Failed to allocate an exclusive RSS context. Continue 457 * operation without support for RSS. The pseudo-header in 458 * received packets will not contain a Toeplitz hash value. 459 */ 460 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 461 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 462 } 463 464 #endif /* EFSYS_OPT_RX_SCALE */ 465 466 return (0); 467 } 468 469 #if EFSYS_OPT_RX_SCATTER 470 __checkReturn efx_rc_t 471 ef10_rx_scatter_enable( 472 __in efx_nic_t *enp, 473 __in unsigned int buf_size) 474 { 475 _NOTE(ARGUNUSED(enp, buf_size)) 476 return (0); 477 } 478 #endif /* EFSYS_OPT_RX_SCATTER */ 479 480 #if EFSYS_OPT_RX_SCALE 481 __checkReturn efx_rc_t 482 ef10_rx_scale_mode_set( 483 __in efx_nic_t *enp, 484 __in efx_rx_hash_alg_t alg, 485 __in efx_rx_hash_type_t type, 486 __in boolean_t insert) 487 { 488 efx_rc_t rc; 489 490 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); 491 EFSYS_ASSERT3U(insert, ==, B_TRUE); 492 493 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { 494 rc = EINVAL; 495 goto fail1; 496 } 497 498 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 499 rc = ENOTSUP; 500 goto fail2; 501 } 502 503 if ((rc = efx_mcdi_rss_context_set_flags(enp, 504 enp->en_rss_context, type)) != 0) 505 goto fail3; 506 507 return (0); 508 509 fail3: 510 EFSYS_PROBE(fail3); 511 fail2: 512 EFSYS_PROBE(fail2); 513 fail1: 514 EFSYS_PROBE1(fail1, efx_rc_t, rc); 515 516 return (rc); 517 } 518 #endif /* EFSYS_OPT_RX_SCALE */ 519 520 #if EFSYS_OPT_RX_SCALE 521 __checkReturn efx_rc_t 522 ef10_rx_scale_key_set( 523 __in efx_nic_t *enp, 524 __in_ecount(n) uint8_t *key, 525 __in size_t n) 526 { 527 efx_rc_t rc; 528 529 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 530 rc = ENOTSUP; 531 goto fail1; 532 } 533 534 if ((rc = efx_mcdi_rss_context_set_key(enp, 535 enp->en_rss_context, key, n)) != 0) 536 goto fail2; 537 538 return (0); 539 540 fail2: 541 EFSYS_PROBE(fail2); 542 fail1: 543 EFSYS_PROBE1(fail1, efx_rc_t, rc); 544 545 return (rc); 546 } 547 #endif /* EFSYS_OPT_RX_SCALE */ 548 549 #if EFSYS_OPT_RX_SCALE 550 __checkReturn efx_rc_t 551 ef10_rx_scale_tbl_set( 552 __in efx_nic_t *enp, 553 __in_ecount(n) unsigned int *table, 554 __in size_t n) 555 { 556 efx_rc_t rc; 557 558 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 559 rc = ENOTSUP; 560 goto fail1; 561 } 562 563 if ((rc = efx_mcdi_rss_context_set_table(enp, 564 enp->en_rss_context, table, n)) != 0) 565 goto fail2; 566 567 return (0); 568 569 fail2: 570 EFSYS_PROBE(fail2); 571 fail1: 572 EFSYS_PROBE1(fail1, efx_rc_t, rc); 573 574 return (rc); 575 } 576 #endif /* EFSYS_OPT_RX_SCALE */ 577 578 579 /* 580 * EF10 RX pseudo-header 581 * --------------------- 582 * 583 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 584 * 585 * +00: Toeplitz hash value. 586 * (32bit little-endian) 587 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 588 * (16bit big-endian) 589 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 590 * (16bit big-endian) 591 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 592 * (16bit little-endian) 593 * +10: MAC timestamp. Zero if timestamping is not enabled. 594 * (32bit little-endian) 595 * 596 * See "The RX Pseudo-header" in SF-109306-TC. 597 */ 598 599 __checkReturn efx_rc_t 600 ef10_rx_prefix_pktlen( 601 __in efx_nic_t *enp, 602 __in uint8_t *buffer, 603 __out uint16_t *lengthp) 604 { 605 _NOTE(ARGUNUSED(enp)) 606 607 /* 608 * The RX pseudo-header contains the packet length, excluding the 609 * pseudo-header. If the hardware receive datapath was operating in 610 * cut-through mode then the length in the RX pseudo-header will be 611 * zero, and the packet length must be obtained from the DMA length 612 * reported in the RX event. 613 */ 614 *lengthp = buffer[8] | (buffer[9] << 8); 615 return (0); 616 } 617 618 #if EFSYS_OPT_RX_SCALE 619 __checkReturn uint32_t 620 ef10_rx_prefix_hash( 621 __in efx_nic_t *enp, 622 __in efx_rx_hash_alg_t func, 623 __in uint8_t *buffer) 624 { 625 _NOTE(ARGUNUSED(enp)) 626 627 switch (func) { 628 case EFX_RX_HASHALG_TOEPLITZ: 629 return (buffer[0] | 630 (buffer[1] << 8) | 631 (buffer[2] << 16) | 632 (buffer[3] << 24)); 633 634 default: 635 EFSYS_ASSERT(0); 636 return (0); 637 } 638 } 639 #endif /* EFSYS_OPT_RX_SCALE */ 640 641 void 642 ef10_rx_qpost( 643 __in efx_rxq_t *erp, 644 __in_ecount(n) efsys_dma_addr_t *addrp, 645 __in size_t size, 646 __in unsigned int n, 647 __in unsigned int completed, 648 __in unsigned int added) 649 { 650 efx_qword_t qword; 651 unsigned int i; 652 unsigned int offset; 653 unsigned int id; 654 655 /* The client driver must not overfill the queue */ 656 EFSYS_ASSERT3U(added - completed + n, <=, 657 EFX_RXQ_LIMIT(erp->er_mask + 1)); 658 659 id = added & (erp->er_mask); 660 for (i = 0; i < n; i++) { 661 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 662 unsigned int, id, efsys_dma_addr_t, addrp[i], 663 size_t, size); 664 665 EFX_POPULATE_QWORD_3(qword, 666 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 667 ESF_DZ_RX_KER_BUF_ADDR_DW0, 668 (uint32_t)(addrp[i] & 0xffffffff), 669 ESF_DZ_RX_KER_BUF_ADDR_DW1, 670 (uint32_t)(addrp[i] >> 32)); 671 672 offset = id * sizeof (efx_qword_t); 673 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 674 675 id = (id + 1) & (erp->er_mask); 676 } 677 } 678 679 void 680 ef10_rx_qpush( 681 __in efx_rxq_t *erp, 682 __in unsigned int added, 683 __inout unsigned int *pushedp) 684 { 685 efx_nic_t *enp = erp->er_enp; 686 unsigned int pushed = *pushedp; 687 uint32_t wptr; 688 efx_dword_t dword; 689 690 /* Hardware has alignment restriction for WPTR */ 691 wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN); 692 if (pushed == wptr) 693 return; 694 695 *pushedp = wptr; 696 697 /* Push the populated descriptors out */ 698 wptr &= erp->er_mask; 699 700 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 701 702 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 703 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 704 wptr, pushed & erp->er_mask); 705 EFSYS_PIO_WRITE_BARRIER(); 706 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 707 erp->er_index, &dword, B_FALSE); 708 } 709 710 __checkReturn efx_rc_t 711 ef10_rx_qflush( 712 __in efx_rxq_t *erp) 713 { 714 efx_nic_t *enp = erp->er_enp; 715 efx_rc_t rc; 716 717 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 718 goto fail1; 719 720 return (0); 721 722 fail1: 723 EFSYS_PROBE1(fail1, efx_rc_t, rc); 724 725 return (rc); 726 } 727 728 void 729 ef10_rx_qenable( 730 __in efx_rxq_t *erp) 731 { 732 /* FIXME */ 733 _NOTE(ARGUNUSED(erp)) 734 /* FIXME */ 735 } 736 737 __checkReturn efx_rc_t 738 ef10_rx_qcreate( 739 __in efx_nic_t *enp, 740 __in unsigned int index, 741 __in unsigned int label, 742 __in efx_rxq_type_t type, 743 __in efsys_mem_t *esmp, 744 __in size_t n, 745 __in uint32_t id, 746 __in efx_evq_t *eep, 747 __in efx_rxq_t *erp) 748 { 749 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 750 efx_rc_t rc; 751 boolean_t disable_scatter; 752 753 _NOTE(ARGUNUSED(id, erp)) 754 755 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 756 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 757 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 758 759 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 760 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 761 762 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 763 rc = EINVAL; 764 goto fail1; 765 } 766 if (index >= encp->enc_rxq_limit) { 767 rc = EINVAL; 768 goto fail2; 769 } 770 771 /* Scatter can only be disabled if the firmware supports doing so */ 772 if (type == EFX_RXQ_TYPE_SCATTER) 773 disable_scatter = B_FALSE; 774 else 775 disable_scatter = encp->enc_rx_disable_scatter_supported; 776 777 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index, 778 esmp, disable_scatter)) != 0) 779 goto fail3; 780 781 erp->er_eep = eep; 782 erp->er_label = label; 783 784 ef10_ev_rxlabel_init(eep, erp, label); 785 786 return (0); 787 788 fail3: 789 EFSYS_PROBE(fail3); 790 fail2: 791 EFSYS_PROBE(fail2); 792 fail1: 793 EFSYS_PROBE1(fail1, efx_rc_t, rc); 794 795 return (rc); 796 } 797 798 void 799 ef10_rx_qdestroy( 800 __in efx_rxq_t *erp) 801 { 802 efx_nic_t *enp = erp->er_enp; 803 efx_evq_t *eep = erp->er_eep; 804 unsigned int label = erp->er_label; 805 806 ef10_ev_rxlabel_fini(eep, label); 807 808 EFSYS_ASSERT(enp->en_rx_qcount != 0); 809 --enp->en_rx_qcount; 810 811 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 812 } 813 814 void 815 ef10_rx_fini( 816 __in efx_nic_t *enp) 817 { 818 #if EFSYS_OPT_RX_SCALE 819 if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) { 820 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 821 } 822 enp->en_rss_context = 0; 823 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 824 #else 825 _NOTE(ARGUNUSED(enp)) 826 #endif /* EFSYS_OPT_RX_SCALE */ 827 } 828 829 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 830