1 /****************************************************************************** 2 3 Copyright (c) 2013-2018, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 34 /* 35 ** IXL driver TX/RX Routines: 36 ** This was separated to allow usage by 37 ** both the PF and VF drivers. 38 */ 39 40 #ifndef IXL_STANDALONE_BUILD 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #include "opt_rss.h" 44 #endif 45 46 #include "ixl.h" 47 48 #ifdef RSS 49 #include <net/rss_config.h> 50 #endif 51 52 /* Local Prototypes */ 53 static u8 ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype); 54 55 static int ixl_isc_txd_encap(void *arg, if_pkt_info_t pi); 56 static void ixl_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 57 static int ixl_isc_txd_credits_update_hwb(void *arg, uint16_t txqid, bool clear); 58 static int ixl_isc_txd_credits_update_dwb(void *arg, uint16_t txqid, bool clear); 59 60 static void ixl_isc_rxd_refill(void *arg, if_rxd_update_t iru); 61 static void ixl_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, 62 qidx_t pidx); 63 static int ixl_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, 64 qidx_t budget); 65 static int ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 66 67 struct if_txrx ixl_txrx_hwb = { 68 ixl_isc_txd_encap, 69 ixl_isc_txd_flush, 70 ixl_isc_txd_credits_update_hwb, 71 ixl_isc_rxd_available, 72 ixl_isc_rxd_pkt_get, 73 ixl_isc_rxd_refill, 74 ixl_isc_rxd_flush, 75 NULL 76 }; 77 78 struct if_txrx ixl_txrx_dwb = { 79 ixl_isc_txd_encap, 80 ixl_isc_txd_flush, 81 ixl_isc_txd_credits_update_dwb, 82 ixl_isc_rxd_available, 83 ixl_isc_rxd_pkt_get, 84 ixl_isc_rxd_refill, 85 ixl_isc_rxd_flush, 86 NULL 87 }; 88 89 /** 90 * i40e_vc_stat_str - convert virtchnl status err code to a string 91 * @hw: pointer to the HW structure 92 * @stat_err: the status error code to convert 93 **/ 94 const char * 95 i40e_vc_stat_str(struct i40e_hw *hw, enum virtchnl_status_code stat_err) 96 { 97 switch (stat_err) { 98 case VIRTCHNL_STATUS_SUCCESS: 99 return "OK"; 100 case VIRTCHNL_ERR_PARAM: 101 return "VIRTCHNL_ERR_PARAM"; 102 case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH: 103 return "VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH"; 104 case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR: 105 return "VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR"; 106 case VIRTCHNL_STATUS_ERR_INVALID_VF_ID: 107 return "VIRTCHNL_STATUS_ERR_INVALID_VF_ID"; 108 case VIRTCHNL_STATUS_NOT_SUPPORTED: 109 return "VIRTCHNL_STATUS_NOT_SUPPORTED"; 110 } 111 112 snprintf(hw->err_str, sizeof(hw->err_str), "%d", stat_err); 113 return hw->err_str; 114 } 115 116 void 117 ixl_debug_core(device_t dev, u32 enabled_mask, u32 mask, char *fmt, ...) 118 { 119 va_list args; 120 121 if (!(mask & enabled_mask)) 122 return; 123 124 /* Re-implement device_printf() */ 125 device_print_prettyname(dev); 126 va_start(args, fmt); 127 vprintf(fmt, args); 128 va_end(args); 129 } 130 131 static bool 132 ixl_is_tx_desc_done(struct tx_ring *txr, int idx) 133 { 134 u64 descriptor; 135 136 descriptor = le64toh(txr->tx_base[idx].cmd_type_offset_bsz); 137 return (((descriptor >> I40E_TXD_QW1_DTYPE_SHIFT) 138 & I40E_TXD_QW1_DTYPE_MASK) == I40E_TX_DESC_DTYPE_DESC_DONE); 139 } 140 141 static int 142 ixl_tso_detect_sparse(bus_dma_segment_t *segs, int nsegs, if_pkt_info_t pi) 143 { 144 int count, curseg, i, hlen, segsz, seglen, tsolen; 145 146 if (nsegs <= IXL_MAX_TX_SEGS-2) 147 return (0); 148 segsz = pi->ipi_tso_segsz; 149 curseg = count = 0; 150 151 hlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 152 tsolen = pi->ipi_len - hlen; 153 154 i = 0; 155 curseg = segs[0].ds_len; 156 while (hlen > 0) { 157 count++; 158 if (count > IXL_MAX_TX_SEGS - 2) 159 return (1); 160 if (curseg == 0) { 161 i++; 162 if (__predict_false(i == nsegs)) 163 return (1); 164 165 curseg = segs[i].ds_len; 166 } 167 seglen = min(curseg, hlen); 168 curseg -= seglen; 169 hlen -= seglen; 170 // printf("H:seglen = %d, count=%d\n", seglen, count); 171 } 172 while (tsolen > 0) { 173 segsz = pi->ipi_tso_segsz; 174 while (segsz > 0 && tsolen != 0) { 175 count++; 176 if (count > IXL_MAX_TX_SEGS - 2) { 177 // printf("bad: count = %d\n", count); 178 return (1); 179 } 180 if (curseg == 0) { 181 i++; 182 if (__predict_false(i == nsegs)) { 183 // printf("bad: tsolen = %d", tsolen); 184 return (1); 185 } 186 curseg = segs[i].ds_len; 187 } 188 seglen = min(curseg, segsz); 189 segsz -= seglen; 190 curseg -= seglen; 191 tsolen -= seglen; 192 // printf("D:seglen = %d, count=%d\n", seglen, count); 193 } 194 count = 0; 195 } 196 197 return (0); 198 } 199 200 /********************************************************************* 201 * 202 * Setup descriptor for hw offloads 203 * 204 **********************************************************************/ 205 206 static void 207 ixl_tx_setup_offload(struct ixl_tx_queue *que, 208 if_pkt_info_t pi, u32 *cmd, u32 *off) 209 { 210 switch (pi->ipi_etype) { 211 #ifdef INET 212 case ETHERTYPE_IP: 213 if (pi->ipi_csum_flags & IXL_CSUM_IPV4) 214 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; 215 else 216 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; 217 break; 218 #endif 219 #ifdef INET6 220 case ETHERTYPE_IPV6: 221 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; 222 break; 223 #endif 224 default: 225 break; 226 } 227 228 *off |= (pi->ipi_ehdrlen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 229 *off |= (pi->ipi_ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 230 231 switch (pi->ipi_ipproto) { 232 case IPPROTO_TCP: 233 if (pi->ipi_csum_flags & IXL_CSUM_TCP) { 234 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 235 *off |= (pi->ipi_tcp_hlen >> 2) << 236 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 237 /* Check for NO_HEAD MDD event */ 238 MPASS(pi->ipi_tcp_hlen != 0); 239 } 240 break; 241 case IPPROTO_UDP: 242 if (pi->ipi_csum_flags & IXL_CSUM_UDP) { 243 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; 244 *off |= (sizeof(struct udphdr) >> 2) << 245 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 246 } 247 break; 248 case IPPROTO_SCTP: 249 if (pi->ipi_csum_flags & IXL_CSUM_SCTP) { 250 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; 251 *off |= (sizeof(struct sctphdr) >> 2) << 252 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 253 } 254 /* Fall Thru */ 255 default: 256 break; 257 } 258 } 259 260 /********************************************************************** 261 * 262 * Setup context for hardware segmentation offload (TSO) 263 * 264 **********************************************************************/ 265 static int 266 ixl_tso_setup(struct tx_ring *txr, if_pkt_info_t pi) 267 { 268 if_softc_ctx_t scctx; 269 struct i40e_tx_context_desc *TXD; 270 u32 cmd, mss, type, tsolen; 271 int idx, total_hdr_len; 272 u64 type_cmd_tso_mss; 273 274 idx = pi->ipi_pidx; 275 TXD = (struct i40e_tx_context_desc *) &txr->tx_base[idx]; 276 total_hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 277 tsolen = pi->ipi_len - total_hdr_len; 278 scctx = txr->que->vsi->shared; 279 280 type = I40E_TX_DESC_DTYPE_CONTEXT; 281 cmd = I40E_TX_CTX_DESC_TSO; 282 /* 283 * TSO MSS must not be less than 64; this prevents a 284 * BAD_LSO_MSS MDD event when the MSS is too small. 285 */ 286 if (pi->ipi_tso_segsz < IXL_MIN_TSO_MSS) { 287 txr->mss_too_small++; 288 pi->ipi_tso_segsz = IXL_MIN_TSO_MSS; 289 } 290 mss = pi->ipi_tso_segsz; 291 292 /* Check for BAD_LS0_MSS MDD event (mss too large) */ 293 MPASS(mss <= IXL_MAX_TSO_MSS); 294 /* Check for NO_HEAD MDD event (header lengths are 0) */ 295 MPASS(pi->ipi_ehdrlen != 0); 296 MPASS(pi->ipi_ip_hlen != 0); 297 /* Partial check for BAD_LSO_LEN MDD event */ 298 MPASS(tsolen != 0); 299 /* Partial check for WRONG_SIZE MDD event (during TSO) */ 300 MPASS(total_hdr_len + mss <= IXL_MAX_FRAME); 301 302 type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | 303 ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | 304 ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | 305 ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); 306 TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); 307 308 TXD->tunneling_params = htole32(0); 309 txr->que->tso++; 310 311 return ((idx + 1) & (scctx->isc_ntxd[0]-1)); 312 } 313 314 /********************************************************************* 315 * 316 * This routine maps the mbufs to tx descriptors, allowing the 317 * TX engine to transmit the packets. 318 * - return 0 on success, positive on failure 319 * 320 **********************************************************************/ 321 static int 322 ixl_isc_txd_encap(void *arg, if_pkt_info_t pi) 323 { 324 struct ixl_vsi *vsi = arg; 325 if_softc_ctx_t scctx = vsi->shared; 326 struct ixl_tx_queue *que = &vsi->tx_queues[pi->ipi_qsidx]; 327 struct tx_ring *txr = &que->txr; 328 int nsegs = pi->ipi_nsegs; 329 bus_dma_segment_t *segs = pi->ipi_segs; 330 struct i40e_tx_desc *txd = NULL; 331 int i, j, mask, pidx_last; 332 u32 cmd, off, tx_intr, txd_cmd; 333 334 cmd = off = 0; 335 i = pi->ipi_pidx; 336 337 tx_intr = (pi->ipi_flags & IPI_TX_INTR); 338 txd_cmd = I40E_TX_DESC_CMD_EOP; 339 if (tx_intr) 340 txd_cmd |= I40E_TX_DESC_CMD_RS; 341 342 /* Set up the TSO/CSUM offload */ 343 if (pi->ipi_csum_flags & CSUM_OFFLOAD) { 344 /* Set up the TSO context descriptor if required */ 345 if (pi->ipi_csum_flags & CSUM_TSO) { 346 /* Prevent MAX_BUFF MDD event (for TSO) */ 347 if (ixl_tso_detect_sparse(segs, nsegs, pi)) 348 return (EFBIG); 349 i = ixl_tso_setup(txr, pi); 350 } 351 ixl_tx_setup_offload(que, pi, &cmd, &off); 352 } 353 if (pi->ipi_mflags & M_VLANTAG) 354 cmd |= I40E_TX_DESC_CMD_IL2TAG1; 355 356 cmd |= I40E_TX_DESC_CMD_ICRC; 357 mask = scctx->isc_ntxd[0] - 1; 358 /* Check for WRONG_SIZE MDD event */ 359 MPASS(pi->ipi_len >= IXL_MIN_FRAME); 360 #ifdef INVARIANTS 361 if (!(pi->ipi_csum_flags & CSUM_TSO)) 362 MPASS(pi->ipi_len <= IXL_MAX_FRAME); 363 #endif 364 for (j = 0; j < nsegs; j++) { 365 bus_size_t seglen; 366 367 txd = &txr->tx_base[i]; 368 seglen = segs[j].ds_len; 369 370 /* Check for ZERO_BSIZE MDD event */ 371 MPASS(seglen != 0); 372 373 txd->buffer_addr = htole64(segs[j].ds_addr); 374 txd->cmd_type_offset_bsz = 375 htole64(I40E_TX_DESC_DTYPE_DATA 376 | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) 377 | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) 378 | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) 379 | ((u64)pi->ipi_vtag << I40E_TXD_QW1_L2TAG1_SHIFT)); 380 381 txr->tx_bytes += seglen; 382 pidx_last = i; 383 i = (i+1) & mask; 384 } 385 /* Set the last descriptor for report */ 386 txd->cmd_type_offset_bsz |= 387 htole64((u64)txd_cmd << I40E_TXD_QW1_CMD_SHIFT); 388 /* Add to report status array (if using TX interrupts) */ 389 if (!vsi->enable_head_writeback && tx_intr) { 390 txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; 391 txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & mask; 392 MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); 393 } 394 pi->ipi_new_pidx = i; 395 396 ++txr->tx_packets; 397 return (0); 398 } 399 400 static void 401 ixl_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 402 { 403 struct ixl_vsi *vsi = arg; 404 struct tx_ring *txr = &vsi->tx_queues[txqid].txr; 405 406 /* 407 * Advance the Transmit Descriptor Tail (Tdt), this tells the 408 * hardware that this frame is available to transmit. 409 */ 410 /* Check for ENDLESS_TX MDD event */ 411 MPASS(pidx < vsi->shared->isc_ntxd[0]); 412 wr32(vsi->hw, txr->tail, pidx); 413 } 414 415 416 /********************************************************************* 417 * 418 * (Re)Initialize a queue transmit ring by clearing its memory. 419 * 420 **********************************************************************/ 421 void 422 ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que) 423 { 424 struct tx_ring *txr = &que->txr; 425 426 /* Clear the old ring contents */ 427 bzero((void *)txr->tx_base, 428 (sizeof(struct i40e_tx_desc)) * 429 (vsi->shared->isc_ntxd[0] + (vsi->enable_head_writeback ? 1 : 0))); 430 431 wr32(vsi->hw, txr->tail, 0); 432 } 433 434 /* 435 * ixl_get_tx_head - Retrieve the value from the 436 * location the HW records its HEAD index 437 */ 438 static inline u32 439 ixl_get_tx_head(struct ixl_tx_queue *que) 440 { 441 if_softc_ctx_t scctx = que->vsi->shared; 442 struct tx_ring *txr = &que->txr; 443 void *head = &txr->tx_base[scctx->isc_ntxd[0]]; 444 445 return LE32_TO_CPU(*(volatile __le32 *)head); 446 } 447 448 static int 449 ixl_isc_txd_credits_update_hwb(void *arg, uint16_t qid, bool clear) 450 { 451 struct ixl_vsi *vsi = arg; 452 if_softc_ctx_t scctx = vsi->shared; 453 struct ixl_tx_queue *que = &vsi->tx_queues[qid]; 454 struct tx_ring *txr = &que->txr; 455 int head, credits; 456 457 /* Get the Head WB value */ 458 head = ixl_get_tx_head(que); 459 460 credits = head - txr->tx_cidx_processed; 461 if (credits < 0) 462 credits += scctx->isc_ntxd[0]; 463 if (clear) 464 txr->tx_cidx_processed = head; 465 466 return (credits); 467 } 468 469 static int 470 ixl_isc_txd_credits_update_dwb(void *arg, uint16_t txqid, bool clear) 471 { 472 struct ixl_vsi *vsi = arg; 473 struct ixl_tx_queue *tx_que = &vsi->tx_queues[txqid]; 474 if_softc_ctx_t scctx = vsi->shared; 475 struct tx_ring *txr = &tx_que->txr; 476 477 qidx_t processed = 0; 478 qidx_t cur, prev, ntxd, rs_cidx; 479 int32_t delta; 480 bool is_done; 481 482 rs_cidx = txr->tx_rs_cidx; 483 #if 0 484 device_printf(iflib_get_dev(vsi->ctx), "%s: (q%d) rs_cidx %d, txr->tx_rs_pidx %d\n", __func__, 485 txr->me, rs_cidx, txr->tx_rs_pidx); 486 #endif 487 if (rs_cidx == txr->tx_rs_pidx) 488 return (0); 489 cur = txr->tx_rsq[rs_cidx]; 490 MPASS(cur != QIDX_INVALID); 491 is_done = ixl_is_tx_desc_done(txr, cur); 492 493 if (!is_done) 494 return (0); 495 496 /* If clear is false just let caller know that there 497 * are descriptors to reclaim */ 498 if (!clear) 499 return (1); 500 501 prev = txr->tx_cidx_processed; 502 ntxd = scctx->isc_ntxd[0]; 503 do { 504 MPASS(prev != cur); 505 delta = (int32_t)cur - (int32_t)prev; 506 if (delta < 0) 507 delta += ntxd; 508 MPASS(delta > 0); 509 #if 0 510 device_printf(iflib_get_dev(vsi->ctx), 511 "%s: (q%d) cidx_processed=%u cur=%u clear=%d delta=%d\n", 512 __func__, txr->me, prev, cur, clear, delta); 513 #endif 514 processed += delta; 515 prev = cur; 516 rs_cidx = (rs_cidx + 1) & (ntxd-1); 517 if (rs_cidx == txr->tx_rs_pidx) 518 break; 519 cur = txr->tx_rsq[rs_cidx]; 520 MPASS(cur != QIDX_INVALID); 521 is_done = ixl_is_tx_desc_done(txr, cur); 522 } while (is_done); 523 524 txr->tx_rs_cidx = rs_cidx; 525 txr->tx_cidx_processed = prev; 526 527 #if 0 528 device_printf(iflib_get_dev(vsi->ctx), "%s: (q%d) processed %d\n", __func__, txr->me, processed); 529 #endif 530 return (processed); 531 } 532 533 static void 534 ixl_isc_rxd_refill(void *arg, if_rxd_update_t iru) 535 { 536 struct ixl_vsi *vsi = arg; 537 if_softc_ctx_t scctx = vsi->shared; 538 struct rx_ring *rxr = &((vsi->rx_queues[iru->iru_qsidx]).rxr); 539 uint64_t *paddrs; 540 uint32_t next_pidx, pidx; 541 uint16_t count; 542 int i; 543 544 paddrs = iru->iru_paddrs; 545 pidx = iru->iru_pidx; 546 count = iru->iru_count; 547 548 for (i = 0, next_pidx = pidx; i < count; i++) { 549 rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); 550 if (++next_pidx == scctx->isc_nrxd[0]) 551 next_pidx = 0; 552 } 553 } 554 555 static void 556 ixl_isc_rxd_flush(void * arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) 557 { 558 struct ixl_vsi *vsi = arg; 559 struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; 560 561 wr32(vsi->hw, rxr->tail, pidx); 562 } 563 564 static int 565 ixl_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 566 { 567 struct ixl_vsi *vsi = arg; 568 struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; 569 union i40e_rx_desc *rxd; 570 u64 qword; 571 uint32_t status; 572 int cnt, i, nrxd; 573 574 nrxd = vsi->shared->isc_nrxd[0]; 575 576 for (cnt = 0, i = idx; cnt < nrxd - 1 && cnt <= budget;) { 577 rxd = &rxr->rx_base[i]; 578 qword = le64toh(rxd->wb.qword1.status_error_len); 579 status = (qword & I40E_RXD_QW1_STATUS_MASK) 580 >> I40E_RXD_QW1_STATUS_SHIFT; 581 582 if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) 583 break; 584 if (++i == nrxd) 585 i = 0; 586 if (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)) 587 cnt++; 588 } 589 590 return (cnt); 591 } 592 593 /* 594 ** i40e_ptype_to_hash: parse the packet type 595 ** to determine the appropriate hash. 596 */ 597 static inline int 598 ixl_ptype_to_hash(u8 ptype) 599 { 600 struct i40e_rx_ptype_decoded decoded; 601 602 decoded = decode_rx_desc_ptype(ptype); 603 604 if (!decoded.known) 605 return M_HASHTYPE_OPAQUE; 606 607 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2) 608 return M_HASHTYPE_OPAQUE; 609 610 /* Note: anything that gets to this point is IP */ 611 if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { 612 switch (decoded.inner_prot) { 613 case I40E_RX_PTYPE_INNER_PROT_TCP: 614 return M_HASHTYPE_RSS_TCP_IPV6; 615 case I40E_RX_PTYPE_INNER_PROT_UDP: 616 return M_HASHTYPE_RSS_UDP_IPV6; 617 default: 618 return M_HASHTYPE_RSS_IPV6; 619 } 620 } 621 if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) { 622 switch (decoded.inner_prot) { 623 case I40E_RX_PTYPE_INNER_PROT_TCP: 624 return M_HASHTYPE_RSS_TCP_IPV4; 625 case I40E_RX_PTYPE_INNER_PROT_UDP: 626 return M_HASHTYPE_RSS_UDP_IPV4; 627 default: 628 return M_HASHTYPE_RSS_IPV4; 629 } 630 } 631 /* We should never get here!! */ 632 return M_HASHTYPE_OPAQUE; 633 } 634 635 /********************************************************************* 636 * 637 * This routine executes in ithread context. It sends data which has been 638 * dma'ed into host memory to upper layer. 639 * 640 * Returns 0 upon success, errno on failure 641 * 642 *********************************************************************/ 643 static int 644 ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 645 { 646 struct ixl_vsi *vsi = arg; 647 if_softc_ctx_t scctx = vsi->shared; 648 struct ixl_rx_queue *que = &vsi->rx_queues[ri->iri_qsidx]; 649 struct rx_ring *rxr = &que->rxr; 650 union i40e_rx_desc *cur; 651 u32 status, error, fltstat; 652 u16 plen; 653 u64 qword; 654 u8 ptype; 655 bool eop; 656 int i, cidx; 657 658 cidx = ri->iri_cidx; 659 i = 0; 660 do { 661 /* 5 descriptor receive limit */ 662 MPASS(i < IXL_MAX_RX_SEGS); 663 664 cur = &rxr->rx_base[cidx]; 665 qword = le64toh(cur->wb.qword1.status_error_len); 666 status = (qword & I40E_RXD_QW1_STATUS_MASK) 667 >> I40E_RXD_QW1_STATUS_SHIFT; 668 error = (qword & I40E_RXD_QW1_ERROR_MASK) 669 >> I40E_RXD_QW1_ERROR_SHIFT; 670 plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) 671 >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; 672 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) 673 >> I40E_RXD_QW1_PTYPE_SHIFT; 674 675 /* we should never be called without a valid descriptor */ 676 MPASS((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) != 0); 677 678 ri->iri_len += plen; 679 rxr->rx_bytes += plen; 680 681 cur->wb.qword1.status_error_len = 0; 682 eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); 683 684 /* 685 ** Make sure bad packets are discarded, 686 ** note that only EOP descriptor has valid 687 ** error results. 688 */ 689 if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { 690 rxr->desc_errs++; 691 return (EBADMSG); 692 } 693 ri->iri_frags[i].irf_flid = 0; 694 ri->iri_frags[i].irf_idx = cidx; 695 ri->iri_frags[i].irf_len = plen; 696 if (++cidx == vsi->shared->isc_nrxd[0]) 697 cidx = 0; 698 i++; 699 } while (!eop); 700 701 /* capture data for dynamic ITR adjustment */ 702 rxr->packets++; 703 rxr->rx_packets++; 704 705 if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0) 706 rxr->csum_errs += ixl_rx_checksum(ri, status, error, ptype); 707 fltstat = (status >> I40E_RX_DESC_STATUS_FLTSTAT_SHIFT); 708 if ((fltstat & I40E_RX_DESC_FLTSTAT_RSS_HASH) == 709 I40E_RX_DESC_FLTSTAT_RSS_HASH) { 710 ri->iri_flowid = le32toh(cur->wb.qword0.hi_dword.rss); 711 ri->iri_rsstype = ixl_ptype_to_hash(ptype); 712 } 713 if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) { 714 ri->iri_vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); 715 ri->iri_flags |= M_VLANTAG; 716 } 717 ri->iri_nfrags = i; 718 return (0); 719 } 720 721 /********************************************************************* 722 * 723 * Verify that the hardware indicated that the checksum is valid. 724 * Inform the stack about the status of checksum so that stack 725 * doesn't spend time verifying the checksum. 726 * 727 *********************************************************************/ 728 static u8 729 ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype) 730 { 731 struct i40e_rx_ptype_decoded decoded; 732 733 ri->iri_csum_flags = 0; 734 735 /* No L3 or L4 checksum was calculated */ 736 if (!(status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT))) 737 return (0); 738 739 decoded = decode_rx_desc_ptype(ptype); 740 741 /* IPv6 with extension headers likely have bad csum */ 742 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 743 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { 744 if (status & 745 (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { 746 ri->iri_csum_flags = 0; 747 return (1); 748 } 749 } 750 751 ri->iri_csum_flags |= CSUM_L3_CALC; 752 753 /* IPv4 checksum error */ 754 if (error & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT)) 755 return (1); 756 757 ri->iri_csum_flags |= CSUM_L3_VALID; 758 ri->iri_csum_flags |= CSUM_L4_CALC; 759 760 /* L4 checksum error */ 761 if (error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)) 762 return (1); 763 764 ri->iri_csum_flags |= CSUM_L4_VALID; 765 ri->iri_csum_data |= htons(0xffff); 766 767 return (0); 768 } 769 770 /* Set Report Status queue fields to 0 */ 771 void 772 ixl_init_tx_rsqs(struct ixl_vsi *vsi) 773 { 774 if_softc_ctx_t scctx = vsi->shared; 775 struct ixl_tx_queue *tx_que; 776 int i, j; 777 778 for (i = 0, tx_que = vsi->tx_queues; i < vsi->num_tx_queues; i++, tx_que++) { 779 struct tx_ring *txr = &tx_que->txr; 780 781 txr->tx_rs_cidx = txr->tx_rs_pidx; 782 783 /* Initialize the last processed descriptor to be the end of 784 * the ring, rather than the start, so that we avoid an 785 * off-by-one error when calculating how many descriptors are 786 * done in the credits_update function. 787 */ 788 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 789 790 for (j = 0; j < scctx->isc_ntxd[0]; j++) 791 txr->tx_rsq[j] = QIDX_INVALID; 792 } 793 } 794 795 void 796 ixl_init_tx_cidx(struct ixl_vsi *vsi) 797 { 798 if_softc_ctx_t scctx = vsi->shared; 799 struct ixl_tx_queue *tx_que; 800 int i; 801 802 for (i = 0, tx_que = vsi->tx_queues; i < vsi->num_tx_queues; i++, tx_que++) { 803 struct tx_ring *txr = &tx_que->txr; 804 805 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 806 } 807 } 808 809 /* 810 * Input: bitmap of enum virtchnl_link_speed 811 */ 812 u64 813 ixl_max_vc_speed_to_value(u8 link_speeds) 814 { 815 if (link_speeds & VIRTCHNL_LINK_SPEED_40GB) 816 return IF_Gbps(40); 817 if (link_speeds & VIRTCHNL_LINK_SPEED_25GB) 818 return IF_Gbps(25); 819 if (link_speeds & VIRTCHNL_LINK_SPEED_20GB) 820 return IF_Gbps(20); 821 if (link_speeds & VIRTCHNL_LINK_SPEED_10GB) 822 return IF_Gbps(10); 823 if (link_speeds & VIRTCHNL_LINK_SPEED_1GB) 824 return IF_Gbps(1); 825 if (link_speeds & VIRTCHNL_LINK_SPEED_100MB) 826 return IF_Mbps(100); 827 else 828 /* Minimum supported link speed */ 829 return IF_Mbps(100); 830 } 831 832 void 833 ixl_add_vsi_sysctls(device_t dev, struct ixl_vsi *vsi, 834 struct sysctl_ctx_list *ctx, const char *sysctl_name) 835 { 836 struct sysctl_oid *tree; 837 struct sysctl_oid_list *child; 838 struct sysctl_oid_list *vsi_list; 839 840 tree = device_get_sysctl_tree(dev); 841 child = SYSCTL_CHILDREN(tree); 842 vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name, 843 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "VSI Number"); 844 vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); 845 846 ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats); 847 } 848 849 void 850 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, 851 struct sysctl_oid_list *child, 852 struct i40e_eth_stats *eth_stats) 853 { 854 struct ixl_sysctl_info ctls[] = 855 { 856 {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, 857 {ð_stats->rx_unicast, "ucast_pkts_rcvd", 858 "Unicast Packets Received"}, 859 {ð_stats->rx_multicast, "mcast_pkts_rcvd", 860 "Multicast Packets Received"}, 861 {ð_stats->rx_broadcast, "bcast_pkts_rcvd", 862 "Broadcast Packets Received"}, 863 {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"}, 864 {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, 865 {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, 866 {ð_stats->tx_multicast, "mcast_pkts_txd", 867 "Multicast Packets Transmitted"}, 868 {ð_stats->tx_broadcast, "bcast_pkts_txd", 869 "Broadcast Packets Transmitted"}, 870 // end 871 {0,0,0} 872 }; 873 874 struct ixl_sysctl_info *entry = ctls; 875 while (entry->stat != 0) 876 { 877 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name, 878 CTLFLAG_RD, entry->stat, 879 entry->description); 880 entry++; 881 } 882 } 883 884 void 885 ixl_vsi_add_queues_stats(struct ixl_vsi *vsi, struct sysctl_ctx_list *ctx) 886 { 887 struct sysctl_oid_list *vsi_list, *queue_list; 888 struct sysctl_oid *queue_node; 889 char queue_namebuf[IXL_QUEUE_NAME_LEN]; 890 891 struct ixl_rx_queue *rx_que; 892 struct ixl_tx_queue *tx_que; 893 struct tx_ring *txr; 894 struct rx_ring *rxr; 895 896 vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); 897 898 /* Queue statistics */ 899 for (int q = 0; q < vsi->num_rx_queues; q++) { 900 bzero(queue_namebuf, sizeof(queue_namebuf)); 901 snprintf(queue_namebuf, sizeof(queue_namebuf), "rxq%02d", q); 902 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, 903 OID_AUTO, queue_namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, 904 NULL, "RX Queue #"); 905 queue_list = SYSCTL_CHILDREN(queue_node); 906 907 rx_que = &(vsi->rx_queues[q]); 908 rxr = &(rx_que->rxr); 909 910 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", 911 CTLFLAG_RD, &(rx_que->irqs), 912 "irqs on this queue (both Tx and Rx)"); 913 914 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "packets", 915 CTLFLAG_RD, &(rxr->rx_packets), 916 "Queue Packets Received"); 917 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "bytes", 918 CTLFLAG_RD, &(rxr->rx_bytes), 919 "Queue Bytes Received"); 920 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "desc_err", 921 CTLFLAG_RD, &(rxr->desc_errs), 922 "Queue Rx Descriptor Errors"); 923 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "itr", 924 CTLFLAG_RD, &(rxr->itr), 0, 925 "Queue Rx ITR Interval"); 926 } 927 for (int q = 0; q < vsi->num_tx_queues; q++) { 928 bzero(queue_namebuf, sizeof(queue_namebuf)); 929 snprintf(queue_namebuf, sizeof(queue_namebuf), "txq%02d", q); 930 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, 931 OID_AUTO, queue_namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, 932 NULL, "TX Queue #"); 933 queue_list = SYSCTL_CHILDREN(queue_node); 934 935 tx_que = &(vsi->tx_queues[q]); 936 txr = &(tx_que->txr); 937 938 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso", 939 CTLFLAG_RD, &(tx_que->tso), 940 "TSO"); 941 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mss_too_small", 942 CTLFLAG_RD, &(txr->mss_too_small), 943 "TSO sends with an MSS less than 64"); 944 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "packets", 945 CTLFLAG_RD, &(txr->tx_packets), 946 "Queue Packets Transmitted"); 947 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "bytes", 948 CTLFLAG_RD, &(txr->tx_bytes), 949 "Queue Bytes Transmitted"); 950 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "itr", 951 CTLFLAG_RD, &(txr->itr), 0, 952 "Queue Tx ITR Interval"); 953 } 954 } 955