1 /*- 2 * Copyright (c) 2012-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 37 38 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 39 40 41 static __checkReturn efx_rc_t 42 efx_mcdi_init_rxq( 43 __in efx_nic_t *enp, 44 __in uint32_t size, 45 __in uint32_t target_evq, 46 __in uint32_t label, 47 __in uint32_t instance, 48 __in efsys_mem_t *esmp, 49 __in boolean_t disable_scatter) 50 { 51 efx_mcdi_req_t req; 52 uint8_t payload[ 53 MAX(MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS)), 54 MC_CMD_INIT_RXQ_OUT_LEN)]; 55 int npages = EFX_RXQ_NBUFS(size); 56 int i; 57 efx_qword_t *dma_addr; 58 uint64_t addr; 59 efx_rc_t rc; 60 61 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS); 62 63 (void) memset(payload, 0, sizeof (payload)); 64 req.emr_cmd = MC_CMD_INIT_RXQ; 65 req.emr_in_buf = payload; 66 req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages); 67 req.emr_out_buf = payload; 68 req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN; 69 70 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size); 71 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq); 72 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label); 73 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance); 74 MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS, 75 INIT_RXQ_IN_FLAG_BUFF_MODE, 0, 76 INIT_RXQ_IN_FLAG_HDR_SPLIT, 0, 77 INIT_RXQ_IN_FLAG_TIMESTAMP, 0, 78 INIT_RXQ_IN_CRC_MODE, 0, 79 INIT_RXQ_IN_FLAG_PREFIX, 1, 80 INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter); 81 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0); 82 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 83 84 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); 85 addr = EFSYS_MEM_ADDR(esmp); 86 87 for (i = 0; i < npages; i++) { 88 EFX_POPULATE_QWORD_2(*dma_addr, 89 EFX_DWORD_1, (uint32_t)(addr >> 32), 90 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 91 92 dma_addr++; 93 addr += EFX_BUF_SIZE; 94 } 95 96 efx_mcdi_execute(enp, &req); 97 98 if (req.emr_rc != 0) { 99 rc = req.emr_rc; 100 goto fail1; 101 } 102 103 return (0); 104 105 fail1: 106 EFSYS_PROBE1(fail1, efx_rc_t, rc); 107 108 return (rc); 109 } 110 111 static __checkReturn efx_rc_t 112 efx_mcdi_fini_rxq( 113 __in efx_nic_t *enp, 114 __in uint32_t instance) 115 { 116 efx_mcdi_req_t req; 117 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN, 118 MC_CMD_FINI_RXQ_OUT_LEN)]; 119 efx_rc_t rc; 120 121 (void) memset(payload, 0, sizeof (payload)); 122 req.emr_cmd = MC_CMD_FINI_RXQ; 123 req.emr_in_buf = payload; 124 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; 125 req.emr_out_buf = payload; 126 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; 127 128 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); 129 130 efx_mcdi_execute_quiet(enp, &req); 131 132 if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) { 133 rc = req.emr_rc; 134 goto fail1; 135 } 136 137 return (0); 138 139 fail1: 140 EFSYS_PROBE1(fail1, efx_rc_t, rc); 141 142 return (rc); 143 } 144 145 #if EFSYS_OPT_RX_SCALE 146 static __checkReturn efx_rc_t 147 efx_mcdi_rss_context_alloc( 148 __in efx_nic_t *enp, 149 __in efx_rx_scale_support_t scale_support, 150 __in uint32_t num_queues, 151 __out uint32_t *rss_contextp) 152 { 153 efx_mcdi_req_t req; 154 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 155 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; 156 uint32_t rss_context; 157 uint32_t context_type; 158 efx_rc_t rc; 159 160 if (num_queues > EFX_MAXRSS) { 161 rc = EINVAL; 162 goto fail1; 163 } 164 165 switch (scale_support) { 166 case EFX_RX_SCALE_EXCLUSIVE: 167 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 168 break; 169 case EFX_RX_SCALE_SHARED: 170 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 171 break; 172 default: 173 rc = EINVAL; 174 goto fail2; 175 } 176 177 (void) memset(payload, 0, sizeof (payload)); 178 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 179 req.emr_in_buf = payload; 180 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 181 req.emr_out_buf = payload; 182 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 183 184 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 185 EVB_PORT_ID_ASSIGNED); 186 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 187 /* NUM_QUEUES is only used to validate indirection table offsets */ 188 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 189 190 efx_mcdi_execute(enp, &req); 191 192 if (req.emr_rc != 0) { 193 rc = req.emr_rc; 194 goto fail3; 195 } 196 197 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 198 rc = EMSGSIZE; 199 goto fail4; 200 } 201 202 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 203 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 204 rc = ENOENT; 205 goto fail5; 206 } 207 208 *rss_contextp = rss_context; 209 210 return (0); 211 212 fail5: 213 EFSYS_PROBE(fail5); 214 fail4: 215 EFSYS_PROBE(fail4); 216 fail3: 217 EFSYS_PROBE(fail3); 218 fail2: 219 EFSYS_PROBE(fail2); 220 fail1: 221 EFSYS_PROBE1(fail1, efx_rc_t, rc); 222 223 return (rc); 224 } 225 #endif /* EFSYS_OPT_RX_SCALE */ 226 227 #if EFSYS_OPT_RX_SCALE 228 static efx_rc_t 229 efx_mcdi_rss_context_free( 230 __in efx_nic_t *enp, 231 __in uint32_t rss_context) 232 { 233 efx_mcdi_req_t req; 234 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 235 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; 236 efx_rc_t rc; 237 238 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 239 rc = EINVAL; 240 goto fail1; 241 } 242 243 (void) memset(payload, 0, sizeof (payload)); 244 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 245 req.emr_in_buf = payload; 246 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 247 req.emr_out_buf = payload; 248 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 249 250 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 251 252 efx_mcdi_execute_quiet(enp, &req); 253 254 if (req.emr_rc != 0) { 255 rc = req.emr_rc; 256 goto fail2; 257 } 258 259 return (0); 260 261 fail2: 262 EFSYS_PROBE(fail2); 263 fail1: 264 EFSYS_PROBE1(fail1, efx_rc_t, rc); 265 266 return (rc); 267 } 268 #endif /* EFSYS_OPT_RX_SCALE */ 269 270 #if EFSYS_OPT_RX_SCALE 271 static efx_rc_t 272 efx_mcdi_rss_context_set_flags( 273 __in efx_nic_t *enp, 274 __in uint32_t rss_context, 275 __in efx_rx_hash_type_t type) 276 { 277 efx_mcdi_req_t req; 278 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 279 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; 280 efx_rc_t rc; 281 282 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 283 rc = EINVAL; 284 goto fail1; 285 } 286 287 (void) memset(payload, 0, sizeof (payload)); 288 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 289 req.emr_in_buf = payload; 290 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 291 req.emr_out_buf = payload; 292 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 293 294 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 295 rss_context); 296 297 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 298 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 299 (type & (1U << EFX_RX_HASH_IPV4)) ? 1 : 0, 300 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 301 (type & (1U << EFX_RX_HASH_TCPIPV4)) ? 1 : 0, 302 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 303 (type & (1U << EFX_RX_HASH_IPV6)) ? 1 : 0, 304 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 305 (type & (1U << EFX_RX_HASH_TCPIPV6)) ? 1 : 0); 306 307 efx_mcdi_execute(enp, &req); 308 309 if (req.emr_rc != 0) { 310 rc = req.emr_rc; 311 goto fail2; 312 } 313 314 return (0); 315 316 fail2: 317 EFSYS_PROBE(fail2); 318 fail1: 319 EFSYS_PROBE1(fail1, efx_rc_t, rc); 320 321 return (rc); 322 } 323 #endif /* EFSYS_OPT_RX_SCALE */ 324 325 #if EFSYS_OPT_RX_SCALE 326 static efx_rc_t 327 efx_mcdi_rss_context_set_key( 328 __in efx_nic_t *enp, 329 __in uint32_t rss_context, 330 __in_ecount(n) uint8_t *key, 331 __in size_t n) 332 { 333 efx_mcdi_req_t req; 334 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 335 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; 336 efx_rc_t rc; 337 338 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 339 rc = EINVAL; 340 goto fail1; 341 } 342 343 (void) memset(payload, 0, sizeof (payload)); 344 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 345 req.emr_in_buf = payload; 346 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 347 req.emr_out_buf = payload; 348 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 349 350 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 351 rss_context); 352 353 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 354 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 355 rc = EINVAL; 356 goto fail2; 357 } 358 359 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 360 key, n); 361 362 efx_mcdi_execute(enp, &req); 363 364 if (req.emr_rc != 0) { 365 rc = req.emr_rc; 366 goto fail3; 367 } 368 369 return (0); 370 371 fail3: 372 EFSYS_PROBE(fail3); 373 fail2: 374 EFSYS_PROBE(fail2); 375 fail1: 376 EFSYS_PROBE1(fail1, efx_rc_t, rc); 377 378 return (rc); 379 } 380 #endif /* EFSYS_OPT_RX_SCALE */ 381 382 #if EFSYS_OPT_RX_SCALE 383 static efx_rc_t 384 efx_mcdi_rss_context_set_table( 385 __in efx_nic_t *enp, 386 __in uint32_t rss_context, 387 __in_ecount(n) unsigned int *table, 388 __in size_t n) 389 { 390 efx_mcdi_req_t req; 391 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 392 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; 393 uint8_t *req_table; 394 int i, rc; 395 396 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 397 rc = EINVAL; 398 goto fail1; 399 } 400 401 (void) memset(payload, 0, sizeof (payload)); 402 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 403 req.emr_in_buf = payload; 404 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 405 req.emr_out_buf = payload; 406 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 407 408 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 409 rss_context); 410 411 req_table = 412 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 413 414 for (i = 0; 415 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 416 i++) { 417 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 418 } 419 420 efx_mcdi_execute(enp, &req); 421 422 if (req.emr_rc != 0) { 423 rc = req.emr_rc; 424 goto fail2; 425 } 426 427 return (0); 428 429 fail2: 430 EFSYS_PROBE(fail2); 431 fail1: 432 EFSYS_PROBE1(fail1, efx_rc_t, rc); 433 434 return (rc); 435 } 436 #endif /* EFSYS_OPT_RX_SCALE */ 437 438 439 __checkReturn efx_rc_t 440 ef10_rx_init( 441 __in efx_nic_t *enp) 442 { 443 #if EFSYS_OPT_RX_SCALE 444 445 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 446 &enp->en_rss_context) == 0) { 447 /* 448 * Allocated an exclusive RSS context, which allows both the 449 * indirection table and key to be modified. 450 */ 451 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; 452 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 453 } else { 454 /* 455 * Failed to allocate an exclusive RSS context. Continue 456 * operation without support for RSS. The pseudo-header in 457 * received packets will not contain a Toeplitz hash value. 458 */ 459 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 460 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 461 } 462 463 #endif /* EFSYS_OPT_RX_SCALE */ 464 465 return (0); 466 } 467 468 #if EFSYS_OPT_RX_SCATTER 469 __checkReturn efx_rc_t 470 ef10_rx_scatter_enable( 471 __in efx_nic_t *enp, 472 __in unsigned int buf_size) 473 { 474 _NOTE(ARGUNUSED(enp, buf_size)) 475 return (0); 476 } 477 #endif /* EFSYS_OPT_RX_SCATTER */ 478 479 #if EFSYS_OPT_RX_SCALE 480 __checkReturn efx_rc_t 481 ef10_rx_scale_mode_set( 482 __in efx_nic_t *enp, 483 __in efx_rx_hash_alg_t alg, 484 __in efx_rx_hash_type_t type, 485 __in boolean_t insert) 486 { 487 efx_rc_t rc; 488 489 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); 490 EFSYS_ASSERT3U(insert, ==, B_TRUE); 491 492 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { 493 rc = EINVAL; 494 goto fail1; 495 } 496 497 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 498 rc = ENOTSUP; 499 goto fail2; 500 } 501 502 if ((rc = efx_mcdi_rss_context_set_flags(enp, 503 enp->en_rss_context, type)) != 0) 504 goto fail3; 505 506 return (0); 507 508 fail3: 509 EFSYS_PROBE(fail3); 510 fail2: 511 EFSYS_PROBE(fail2); 512 fail1: 513 EFSYS_PROBE1(fail1, efx_rc_t, rc); 514 515 return (rc); 516 } 517 #endif /* EFSYS_OPT_RX_SCALE */ 518 519 #if EFSYS_OPT_RX_SCALE 520 __checkReturn efx_rc_t 521 ef10_rx_scale_key_set( 522 __in efx_nic_t *enp, 523 __in_ecount(n) uint8_t *key, 524 __in size_t n) 525 { 526 efx_rc_t rc; 527 528 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 529 rc = ENOTSUP; 530 goto fail1; 531 } 532 533 if ((rc = efx_mcdi_rss_context_set_key(enp, 534 enp->en_rss_context, key, n)) != 0) 535 goto fail2; 536 537 return (0); 538 539 fail2: 540 EFSYS_PROBE(fail2); 541 fail1: 542 EFSYS_PROBE1(fail1, efx_rc_t, rc); 543 544 return (rc); 545 } 546 #endif /* EFSYS_OPT_RX_SCALE */ 547 548 #if EFSYS_OPT_RX_SCALE 549 __checkReturn efx_rc_t 550 ef10_rx_scale_tbl_set( 551 __in efx_nic_t *enp, 552 __in_ecount(n) unsigned int *table, 553 __in size_t n) 554 { 555 efx_rc_t rc; 556 557 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 558 rc = ENOTSUP; 559 goto fail1; 560 } 561 562 if ((rc = efx_mcdi_rss_context_set_table(enp, 563 enp->en_rss_context, table, n)) != 0) 564 goto fail2; 565 566 return (0); 567 568 fail2: 569 EFSYS_PROBE(fail2); 570 fail1: 571 EFSYS_PROBE1(fail1, efx_rc_t, rc); 572 573 return (rc); 574 } 575 #endif /* EFSYS_OPT_RX_SCALE */ 576 577 578 /* 579 * EF10 RX pseudo-header 580 * --------------------- 581 * 582 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 583 * 584 * +00: Toeplitz hash value. 585 * (32bit little-endian) 586 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 587 * (16bit big-endian) 588 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 589 * (16bit big-endian) 590 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 591 * (16bit little-endian) 592 * +10: MAC timestamp. Zero if timestamping is not enabled. 593 * (32bit little-endian) 594 * 595 * See "The RX Pseudo-header" in SF-109306-TC. 596 */ 597 598 __checkReturn efx_rc_t 599 ef10_rx_prefix_pktlen( 600 __in efx_nic_t *enp, 601 __in uint8_t *buffer, 602 __out uint16_t *lengthp) 603 { 604 /* 605 * The RX pseudo-header contains the packet length, excluding the 606 * pseudo-header. If the hardware receive datapath was operating in 607 * cut-through mode then the length in the RX pseudo-header will be 608 * zero, and the packet length must be obtained from the DMA length 609 * reported in the RX event. 610 */ 611 *lengthp = buffer[8] | (buffer[9] << 8); 612 return (0); 613 } 614 615 #if EFSYS_OPT_RX_SCALE 616 __checkReturn uint32_t 617 ef10_rx_prefix_hash( 618 __in efx_nic_t *enp, 619 __in efx_rx_hash_alg_t func, 620 __in uint8_t *buffer) 621 { 622 switch (func) { 623 case EFX_RX_HASHALG_TOEPLITZ: 624 return (buffer[0] | 625 (buffer[1] << 8) | 626 (buffer[2] << 16) | 627 (buffer[3] << 24)); 628 629 default: 630 EFSYS_ASSERT(0); 631 return (0); 632 } 633 } 634 #endif /* EFSYS_OPT_RX_SCALE */ 635 636 void 637 ef10_rx_qpost( 638 __in efx_rxq_t *erp, 639 __in_ecount(n) efsys_dma_addr_t *addrp, 640 __in size_t size, 641 __in unsigned int n, 642 __in unsigned int completed, 643 __in unsigned int added) 644 { 645 efx_qword_t qword; 646 unsigned int i; 647 unsigned int offset; 648 unsigned int id; 649 650 /* The client driver must not overfill the queue */ 651 EFSYS_ASSERT3U(added - completed + n, <=, 652 EFX_RXQ_LIMIT(erp->er_mask + 1)); 653 654 id = added & (erp->er_mask); 655 for (i = 0; i < n; i++) { 656 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 657 unsigned int, id, efsys_dma_addr_t, addrp[i], 658 size_t, size); 659 660 EFX_POPULATE_QWORD_3(qword, 661 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 662 ESF_DZ_RX_KER_BUF_ADDR_DW0, 663 (uint32_t)(addrp[i] & 0xffffffff), 664 ESF_DZ_RX_KER_BUF_ADDR_DW1, 665 (uint32_t)(addrp[i] >> 32)); 666 667 offset = id * sizeof (efx_qword_t); 668 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 669 670 id = (id + 1) & (erp->er_mask); 671 } 672 } 673 674 void 675 ef10_rx_qpush( 676 __in efx_rxq_t *erp, 677 __in unsigned int added, 678 __inout unsigned int *pushedp) 679 { 680 efx_nic_t *enp = erp->er_enp; 681 unsigned int pushed = *pushedp; 682 uint32_t wptr; 683 efx_dword_t dword; 684 685 /* Hardware has alignment restriction for WPTR */ 686 wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN); 687 if (pushed == wptr) 688 return; 689 690 *pushedp = wptr; 691 692 /* Push the populated descriptors out */ 693 wptr &= erp->er_mask; 694 695 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 696 697 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 698 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 699 wptr, pushed & erp->er_mask); 700 EFSYS_PIO_WRITE_BARRIER(); 701 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 702 erp->er_index, &dword, B_FALSE); 703 } 704 705 __checkReturn efx_rc_t 706 ef10_rx_qflush( 707 __in efx_rxq_t *erp) 708 { 709 efx_nic_t *enp = erp->er_enp; 710 efx_rc_t rc; 711 712 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 713 goto fail1; 714 715 return (0); 716 717 fail1: 718 EFSYS_PROBE1(fail1, efx_rc_t, rc); 719 720 return (rc); 721 } 722 723 void 724 ef10_rx_qenable( 725 __in efx_rxq_t *erp) 726 { 727 /* FIXME */ 728 _NOTE(ARGUNUSED(erp)) 729 /* FIXME */ 730 } 731 732 __checkReturn efx_rc_t 733 ef10_rx_qcreate( 734 __in efx_nic_t *enp, 735 __in unsigned int index, 736 __in unsigned int label, 737 __in efx_rxq_type_t type, 738 __in efsys_mem_t *esmp, 739 __in size_t n, 740 __in uint32_t id, 741 __in efx_evq_t *eep, 742 __in efx_rxq_t *erp) 743 { 744 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 745 efx_rc_t rc; 746 boolean_t disable_scatter; 747 748 _NOTE(ARGUNUSED(erp)) 749 750 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 751 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 752 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 753 754 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 755 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 756 757 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 758 rc = EINVAL; 759 goto fail1; 760 } 761 if (index >= encp->enc_rxq_limit) { 762 rc = EINVAL; 763 goto fail2; 764 } 765 766 /* Scatter can only be disabled if the firmware supports doing so */ 767 if ((type != EFX_RXQ_TYPE_SCATTER) && 768 enp->en_nic_cfg.enc_rx_disable_scatter_supported) { 769 disable_scatter = B_TRUE; 770 } else { 771 disable_scatter = B_FALSE; 772 } 773 774 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index, 775 esmp, disable_scatter)) != 0) 776 goto fail3; 777 778 erp->er_eep = eep; 779 erp->er_label = label; 780 781 ef10_ev_rxlabel_init(eep, erp, label); 782 783 return (0); 784 785 fail3: 786 EFSYS_PROBE(fail3); 787 fail2: 788 EFSYS_PROBE(fail2); 789 fail1: 790 EFSYS_PROBE1(fail1, efx_rc_t, rc); 791 792 return (rc); 793 } 794 795 void 796 ef10_rx_qdestroy( 797 __in efx_rxq_t *erp) 798 { 799 efx_nic_t *enp = erp->er_enp; 800 efx_evq_t *eep = erp->er_eep; 801 unsigned int label = erp->er_label; 802 803 ef10_ev_rxlabel_fini(eep, label); 804 805 EFSYS_ASSERT(enp->en_rx_qcount != 0); 806 --enp->en_rx_qcount; 807 808 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 809 } 810 811 void 812 ef10_rx_fini( 813 __in efx_nic_t *enp) 814 { 815 #if EFSYS_OPT_RX_SCALE 816 if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) { 817 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 818 } 819 enp->en_rss_context = 0; 820 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 821 #else 822 _NOTE(ARGUNUSED(enp)) 823 #endif /* EFSYS_OPT_RX_SCALE */ 824 } 825 826 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 827