1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 5 * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* $FreeBSD$ */ 31 #include "if_em.h" 32 33 #ifdef RSS 34 #include <net/rss_config.h> 35 #include <netinet/in_rss.h> 36 #endif 37 38 #ifdef VERBOSE_DEBUG 39 #define DPRINTF device_printf 40 #else 41 #define DPRINTF(...) 42 #endif 43 44 /********************************************************************* 45 * Local Function prototypes 46 *********************************************************************/ 47 static int em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, 48 uint32_t *txd_upper, uint32_t *txd_lower); 49 static int em_transmit_checksum_setup(struct e1000_softc *sc, 50 if_pkt_info_t pi, uint32_t *txd_upper, uint32_t *txd_lower); 51 static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); 52 static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 53 static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); 54 static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru); 55 static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, 56 qidx_t pidx); 57 static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, 58 qidx_t budget); 59 static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 60 61 static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru); 62 63 static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, 64 qidx_t budget); 65 static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 66 67 static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t); 68 static int em_determine_rsstype(uint32_t pkt_info); 69 extern int em_intr(void *arg); 70 71 struct if_txrx em_txrx = { 72 .ift_txd_encap = em_isc_txd_encap, 73 .ift_txd_flush = em_isc_txd_flush, 74 .ift_txd_credits_update = em_isc_txd_credits_update, 75 .ift_rxd_available = em_isc_rxd_available, 76 .ift_rxd_pkt_get = em_isc_rxd_pkt_get, 77 .ift_rxd_refill = em_isc_rxd_refill, 78 .ift_rxd_flush = em_isc_rxd_flush, 79 .ift_legacy_intr = em_intr 80 }; 81 82 struct if_txrx lem_txrx = { 83 .ift_txd_encap = em_isc_txd_encap, 84 .ift_txd_flush = em_isc_txd_flush, 85 .ift_txd_credits_update = em_isc_txd_credits_update, 86 .ift_rxd_available = lem_isc_rxd_available, 87 .ift_rxd_pkt_get = lem_isc_rxd_pkt_get, 88 .ift_rxd_refill = lem_isc_rxd_refill, 89 .ift_rxd_flush = em_isc_rxd_flush, 90 .ift_legacy_intr = em_intr 91 }; 92 93 extern if_shared_ctx_t em_sctx; 94 95 void 96 em_dump_rs(struct e1000_softc *sc) 97 { 98 if_softc_ctx_t scctx = sc->shared; 99 struct em_tx_queue *que; 100 struct tx_ring *txr; 101 qidx_t i, ntxd, qid, cur; 102 int16_t rs_cidx; 103 uint8_t status; 104 105 printf("\n"); 106 ntxd = scctx->isc_ntxd[0]; 107 for (qid = 0; qid < sc->tx_num_queues; qid++) { 108 que = &sc->tx_queues[qid]; 109 txr = &que->txr; 110 rs_cidx = txr->tx_rs_cidx; 111 if (rs_cidx != txr->tx_rs_pidx) { 112 cur = txr->tx_rsq[rs_cidx]; 113 status = txr->tx_base[cur].upper.fields.status; 114 if (!(status & E1000_TXD_STAT_DD)) 115 printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur); 116 } else { 117 rs_cidx = (rs_cidx-1)&(ntxd-1); 118 cur = txr->tx_rsq[rs_cidx]; 119 printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", qid, rs_cidx, cur); 120 } 121 printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, 122 txr->tx_rs_pidx); 123 for (i = 0; i < ntxd; i++) { 124 if (txr->tx_base[i].upper.fields.status & E1000_TXD_STAT_DD) 125 printf("%d set ", i); 126 } 127 printf("\n"); 128 } 129 } 130 131 /********************************************************************** 132 * 133 * Setup work for hardware segmentation offload (TSO) on 134 * adapters using advanced tx descriptors 135 * 136 **********************************************************************/ 137 static int 138 em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, uint32_t *txd_upper, 139 uint32_t *txd_lower) 140 { 141 if_softc_ctx_t scctx = sc->shared; 142 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 143 struct tx_ring *txr = &que->txr; 144 struct e1000_hw *hw = &sc->hw; 145 struct e1000_context_desc *TXD; 146 int cur, hdr_len; 147 uint32_t cmd_type_len; 148 149 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 150 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 151 E1000_TXD_DTYP_D | /* Data descr type */ 152 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 153 154 /* IP and/or TCP header checksum calculation and insertion. */ 155 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 156 157 cur = pi->ipi_pidx; 158 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 159 160 /* 161 * Start offset for header checksum calculation. 162 * End offset for header checksum calculation. 163 * Offset of place put the checksum. 164 */ 165 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 166 TXD->lower_setup.ip_fields.ipcse = 167 htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); 168 TXD->lower_setup.ip_fields.ipcso = 169 pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 170 171 /* 172 * Start offset for payload checksum calculation. 173 * End offset for payload checksum calculation. 174 * Offset of place to put the checksum. 175 */ 176 TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 177 TXD->upper_setup.tcp_fields.tucse = 0; 178 TXD->upper_setup.tcp_fields.tucso = 179 pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum); 180 181 /* 182 * Payload size per packet w/o any headers. 183 * Length of all headers up to payload. 184 */ 185 TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); 186 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 187 188 /* 189 * 8254x SDM4.0 page 45, and PCIe GbE SDM2.5 page 63 190 * - Set up basic TUCMDs 191 * - Enable IP bit on 82544 192 * - For others IP bit on indicates IPv4, while off indicates IPv6 193 */ 194 cmd_type_len = sc->txd_cmd | 195 E1000_TXD_CMD_DEXT | /* Extended descr */ 196 E1000_TXD_CMD_TSE | /* TSE context */ 197 E1000_TXD_CMD_TCP; /* Do TCP checksum */ 198 if (hw->mac.type == e1000_82544) 199 cmd_type_len |= E1000_TXD_CMD_IP; 200 else if (pi->ipi_etype == ETHERTYPE_IP) 201 cmd_type_len |= E1000_TXD_CMD_IP; 202 TXD->cmd_and_length = htole32(cmd_type_len | 203 (pi->ipi_len - hdr_len)); /* Total len */ 204 205 txr->tx_tso = true; 206 207 if (++cur == scctx->isc_ntxd[0]) { 208 cur = 0; 209 } 210 DPRINTF(iflib_get_dev(sc->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, 211 pi->ipi_pidx, cur); 212 return (cur); 213 } 214 215 #define TSO_WORKAROUND 4 216 #define DONT_FORCE_CTX 1 217 218 219 /********************************************************************* 220 * The offload context is protocol specific (TCP/UDP) and thus 221 * only needs to be set when the protocol changes. The occasion 222 * of a context change can be a performance detriment, and 223 * might be better just disabled. The reason arises in the way 224 * in which the controller supports pipelined requests from the 225 * Tx data DMA. Up to four requests can be pipelined, and they may 226 * belong to the same packet or to multiple packets. However all 227 * requests for one packet are issued before a request is issued 228 * for a subsequent packet and if a request for the next packet 229 * requires a context change, that request will be stalled 230 * until the previous request completes. This means setting up 231 * a new context effectively disables pipelined Tx data DMA which 232 * in turn greatly slow down performance to send small sized 233 * frames. 234 **********************************************************************/ 235 236 static int 237 em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi, 238 uint32_t *txd_upper, uint32_t *txd_lower) 239 { 240 struct e1000_context_desc *TXD = NULL; 241 if_softc_ctx_t scctx = sc->shared; 242 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 243 struct tx_ring *txr = &que->txr; 244 int csum_flags = pi->ipi_csum_flags; 245 int cur, hdr_len; 246 uint32_t cmd; 247 248 cur = pi->ipi_pidx; 249 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 250 cmd = sc->txd_cmd; 251 252 /* 253 * The 82574L can only remember the *last* context used 254 * regardless of queue that it was use for. We cannot reuse 255 * contexts on this hardware platform and must generate a new 256 * context every time. 82574L hardware spec, section 7.2.6, 257 * second note. 258 */ 259 if (DONT_FORCE_CTX && 260 sc->tx_num_queues == 1 && 261 txr->csum_lhlen == pi->ipi_ehdrlen && 262 txr->csum_iphlen == pi->ipi_ip_hlen && 263 txr->csum_flags == csum_flags) { 264 /* 265 * Same csum offload context as the previous packets; 266 * just return. 267 */ 268 *txd_upper = txr->csum_txd_upper; 269 *txd_lower = txr->csum_txd_lower; 270 return (cur); 271 } 272 273 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 274 if (csum_flags & CSUM_IP) { 275 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 276 /* 277 * Start offset for header checksum calculation. 278 * End offset for header checksum calculation. 279 * Offset of place to put the checksum. 280 */ 281 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 282 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); 283 TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + 284 offsetof(struct ip, ip_sum); 285 cmd |= E1000_TXD_CMD_IP; 286 } 287 288 if (csum_flags & (CSUM_TCP|CSUM_UDP)) { 289 uint8_t tucso; 290 291 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 292 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 293 294 if (csum_flags & CSUM_TCP) { 295 tucso = hdr_len + offsetof(struct tcphdr, th_sum); 296 cmd |= E1000_TXD_CMD_TCP; 297 } else 298 tucso = hdr_len + offsetof(struct udphdr, uh_sum); 299 TXD->upper_setup.tcp_fields.tucss = hdr_len; 300 TXD->upper_setup.tcp_fields.tucse = htole16(0); 301 TXD->upper_setup.tcp_fields.tucso = tucso; 302 } 303 304 txr->csum_lhlen = pi->ipi_ehdrlen; 305 txr->csum_iphlen = pi->ipi_ip_hlen; 306 txr->csum_flags = csum_flags; 307 txr->csum_txd_upper = *txd_upper; 308 txr->csum_txd_lower = *txd_lower; 309 310 TXD->tcp_seg_setup.data = htole32(0); 311 TXD->cmd_and_length = 312 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 313 314 if (++cur == scctx->isc_ntxd[0]) { 315 cur = 0; 316 } 317 DPRINTF(iflib_get_dev(sc->ctx), 318 "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n", 319 csum_flags, *txd_upper, *txd_lower, hdr_len, cmd); 320 return (cur); 321 } 322 323 static int 324 em_isc_txd_encap(void *arg, if_pkt_info_t pi) 325 { 326 struct e1000_softc *sc = arg; 327 if_softc_ctx_t scctx = sc->shared; 328 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 329 struct tx_ring *txr = &que->txr; 330 bus_dma_segment_t *segs = pi->ipi_segs; 331 int nsegs = pi->ipi_nsegs; 332 int csum_flags = pi->ipi_csum_flags; 333 int i, j, first, pidx_last; 334 uint32_t txd_flags, txd_upper = 0, txd_lower = 0; 335 336 struct e1000_tx_desc *ctxd = NULL; 337 bool do_tso, tso_desc; 338 qidx_t ntxd; 339 340 txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; 341 i = first = pi->ipi_pidx; 342 do_tso = (csum_flags & CSUM_TSO); 343 tso_desc = false; 344 ntxd = scctx->isc_ntxd[0]; 345 /* 346 * TSO Hardware workaround, if this packet is not 347 * TSO, and is only a single descriptor long, and 348 * it follows a TSO burst, then we need to add a 349 * sentinel descriptor to prevent premature writeback. 350 */ 351 if ((!do_tso) && (txr->tx_tso == true)) { 352 if (nsegs == 1) 353 tso_desc = true; 354 txr->tx_tso = false; 355 } 356 357 /* Do hardware assists */ 358 if (do_tso) { 359 i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); 360 tso_desc = true; 361 } else if (csum_flags & EM_CSUM_OFFLOAD) { 362 i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower); 363 } 364 365 if (pi->ipi_mflags & M_VLANTAG) { 366 /* Set the vlan id. */ 367 txd_upper |= htole16(pi->ipi_vtag) << 16; 368 /* Tell hardware to add tag */ 369 txd_lower |= htole32(E1000_TXD_CMD_VLE); 370 } 371 372 DPRINTF(iflib_get_dev(sc->ctx), 373 "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); 374 /* XXX sc->pcix_82544 -- lem_fill_descriptors */ 375 376 /* Set up our transmit descriptors */ 377 for (j = 0; j < nsegs; j++) { 378 bus_size_t seg_len; 379 bus_addr_t seg_addr; 380 uint32_t cmd; 381 382 ctxd = &txr->tx_base[i]; 383 seg_addr = segs[j].ds_addr; 384 seg_len = segs[j].ds_len; 385 cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; 386 387 /* 388 * TSO Workaround: 389 * If this is the last descriptor, we want to 390 * split it so we have a small final sentinel 391 */ 392 if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { 393 seg_len -= TSO_WORKAROUND; 394 ctxd->buffer_addr = htole64(seg_addr); 395 ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 396 ctxd->upper.data = htole32(txd_upper); 397 398 if (++i == scctx->isc_ntxd[0]) 399 i = 0; 400 401 /* Now make the sentinel */ 402 ctxd = &txr->tx_base[i]; 403 ctxd->buffer_addr = htole64(seg_addr + seg_len); 404 ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND); 405 ctxd->upper.data = htole32(txd_upper); 406 pidx_last = i; 407 if (++i == scctx->isc_ntxd[0]) 408 i = 0; 409 DPRINTF(iflib_get_dev(sc->ctx), 410 "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", 411 pidx_last, i, scctx->isc_ntxd[0]); 412 } else { 413 ctxd->buffer_addr = htole64(seg_addr); 414 ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 415 ctxd->upper.data = htole32(txd_upper); 416 pidx_last = i; 417 if (++i == scctx->isc_ntxd[0]) 418 i = 0; 419 DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", 420 pidx_last, i, scctx->isc_ntxd[0]); 421 } 422 } 423 424 /* 425 * Last Descriptor of Packet 426 * needs End Of Packet (EOP) 427 * and Report Status (RS) 428 */ 429 if (txd_flags && nsegs) { 430 txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; 431 DPRINTF(iflib_get_dev(sc->ctx), 432 "setting to RS on %d rs_pidx %d first: %d\n", 433 pidx_last, txr->tx_rs_pidx, first); 434 txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); 435 MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); 436 } 437 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags); 438 DPRINTF(iflib_get_dev(sc->ctx), 439 "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i); 440 pi->ipi_new_pidx = i; 441 442 return (0); 443 } 444 445 static void 446 em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 447 { 448 struct e1000_softc *sc = arg; 449 struct em_tx_queue *que = &sc->tx_queues[txqid]; 450 struct tx_ring *txr = &que->txr; 451 452 E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx); 453 } 454 455 static int 456 em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) 457 { 458 struct e1000_softc *sc = arg; 459 if_softc_ctx_t scctx = sc->shared; 460 struct em_tx_queue *que = &sc->tx_queues[txqid]; 461 struct tx_ring *txr = &que->txr; 462 463 qidx_t processed = 0; 464 int updated; 465 qidx_t cur, prev, ntxd, rs_cidx; 466 int32_t delta; 467 uint8_t status; 468 469 rs_cidx = txr->tx_rs_cidx; 470 if (rs_cidx == txr->tx_rs_pidx) 471 return (0); 472 cur = txr->tx_rsq[rs_cidx]; 473 MPASS(cur != QIDX_INVALID); 474 status = txr->tx_base[cur].upper.fields.status; 475 updated = !!(status & E1000_TXD_STAT_DD); 476 477 if (!updated) 478 return (0); 479 480 /* If clear is false just let caller know that there 481 * are descriptors to reclaim */ 482 if (!clear) 483 return (1); 484 485 prev = txr->tx_cidx_processed; 486 ntxd = scctx->isc_ntxd[0]; 487 do { 488 MPASS(prev != cur); 489 delta = (int32_t)cur - (int32_t)prev; 490 if (delta < 0) 491 delta += ntxd; 492 MPASS(delta > 0); 493 DPRINTF(iflib_get_dev(sc->ctx), 494 "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n", 495 __FUNCTION__, prev, cur, clear, delta); 496 497 processed += delta; 498 prev = cur; 499 rs_cidx = (rs_cidx + 1) & (ntxd-1); 500 if (rs_cidx == txr->tx_rs_pidx) 501 break; 502 cur = txr->tx_rsq[rs_cidx]; 503 MPASS(cur != QIDX_INVALID); 504 status = txr->tx_base[cur].upper.fields.status; 505 } while ((status & E1000_TXD_STAT_DD)); 506 507 txr->tx_rs_cidx = rs_cidx; 508 txr->tx_cidx_processed = prev; 509 return(processed); 510 } 511 512 static void 513 lem_isc_rxd_refill(void *arg, if_rxd_update_t iru) 514 { 515 struct e1000_softc *sc = arg; 516 if_softc_ctx_t scctx = sc->shared; 517 struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; 518 struct rx_ring *rxr = &que->rxr; 519 struct e1000_rx_desc *rxd; 520 uint64_t *paddrs; 521 uint32_t next_pidx, pidx; 522 uint16_t count; 523 int i; 524 525 paddrs = iru->iru_paddrs; 526 pidx = iru->iru_pidx; 527 count = iru->iru_count; 528 529 for (i = 0, next_pidx = pidx; i < count; i++) { 530 rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; 531 rxd->buffer_addr = htole64(paddrs[i]); 532 /* status bits must be cleared */ 533 rxd->status = 0; 534 535 if (++next_pidx == scctx->isc_nrxd[0]) 536 next_pidx = 0; 537 } 538 } 539 540 static void 541 em_isc_rxd_refill(void *arg, if_rxd_update_t iru) 542 { 543 struct e1000_softc *sc = arg; 544 if_softc_ctx_t scctx = sc->shared; 545 uint16_t rxqid = iru->iru_qsidx; 546 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 547 struct rx_ring *rxr = &que->rxr; 548 union e1000_rx_desc_extended *rxd; 549 uint64_t *paddrs; 550 uint32_t next_pidx, pidx; 551 uint16_t count; 552 int i; 553 554 paddrs = iru->iru_paddrs; 555 pidx = iru->iru_pidx; 556 count = iru->iru_count; 557 558 for (i = 0, next_pidx = pidx; i < count; i++) { 559 rxd = &rxr->rx_base[next_pidx]; 560 rxd->read.buffer_addr = htole64(paddrs[i]); 561 /* DD bits must be cleared */ 562 rxd->wb.upper.status_error = 0; 563 564 if (++next_pidx == scctx->isc_nrxd[0]) 565 next_pidx = 0; 566 } 567 } 568 569 static void 570 em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, 571 qidx_t pidx) 572 { 573 struct e1000_softc *sc = arg; 574 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 575 struct rx_ring *rxr = &que->rxr; 576 577 E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); 578 } 579 580 static int 581 lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 582 { 583 struct e1000_softc *sc = arg; 584 if_softc_ctx_t scctx = sc->shared; 585 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 586 struct rx_ring *rxr = &que->rxr; 587 struct e1000_rx_desc *rxd; 588 uint32_t staterr = 0; 589 int cnt, i; 590 591 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 592 rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; 593 staterr = rxd->status; 594 595 if ((staterr & E1000_RXD_STAT_DD) == 0) 596 break; 597 if (++i == scctx->isc_nrxd[0]) 598 i = 0; 599 if (staterr & E1000_RXD_STAT_EOP) 600 cnt++; 601 } 602 return (cnt); 603 } 604 605 static int 606 em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 607 { 608 struct e1000_softc *sc = arg; 609 if_softc_ctx_t scctx = sc->shared; 610 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 611 struct rx_ring *rxr = &que->rxr; 612 union e1000_rx_desc_extended *rxd; 613 uint32_t staterr = 0; 614 int cnt, i; 615 616 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 617 rxd = &rxr->rx_base[i]; 618 staterr = le32toh(rxd->wb.upper.status_error); 619 620 if ((staterr & E1000_RXD_STAT_DD) == 0) 621 break; 622 if (++i == scctx->isc_nrxd[0]) 623 i = 0; 624 if (staterr & E1000_RXD_STAT_EOP) 625 cnt++; 626 } 627 return (cnt); 628 } 629 630 static int 631 lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 632 { 633 struct e1000_softc *sc = arg; 634 if_softc_ctx_t scctx = sc->shared; 635 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx]; 636 struct rx_ring *rxr = &que->rxr; 637 struct e1000_rx_desc *rxd; 638 uint16_t len; 639 uint32_t status, errors; 640 bool eop; 641 int i, cidx; 642 643 status = errors = i = 0; 644 cidx = ri->iri_cidx; 645 646 do { 647 rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; 648 status = rxd->status; 649 errors = rxd->errors; 650 651 /* Error Checking then decrement count */ 652 MPASS ((status & E1000_RXD_STAT_DD) != 0); 653 654 len = le16toh(rxd->length); 655 ri->iri_len += len; 656 657 eop = (status & E1000_RXD_STAT_EOP) != 0; 658 659 /* Make sure bad packets are discarded */ 660 if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 661 sc->dropped_pkts++; 662 /* XXX fixup if common */ 663 return (EBADMSG); 664 } 665 666 ri->iri_frags[i].irf_flid = 0; 667 ri->iri_frags[i].irf_idx = cidx; 668 ri->iri_frags[i].irf_len = len; 669 /* Zero out the receive descriptors status. */ 670 rxd->status = 0; 671 672 if (++cidx == scctx->isc_nrxd[0]) 673 cidx = 0; 674 i++; 675 } while (!eop); 676 677 if (scctx->isc_capenable & IFCAP_RXCSUM) 678 em_receive_checksum(status, errors, ri); 679 680 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING && 681 status & E1000_RXD_STAT_VP) { 682 ri->iri_vtag = le16toh(rxd->special & E1000_RXD_SPC_VLAN_MASK); 683 ri->iri_flags |= M_VLANTAG; 684 } 685 686 ri->iri_nfrags = i; 687 688 return (0); 689 } 690 691 static int 692 em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 693 { 694 struct e1000_softc *sc = arg; 695 if_softc_ctx_t scctx = sc->shared; 696 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx]; 697 struct rx_ring *rxr = &que->rxr; 698 union e1000_rx_desc_extended *rxd; 699 700 uint16_t len; 701 uint32_t pkt_info; 702 uint32_t staterr; 703 bool eop; 704 int i, cidx; 705 706 staterr = i = 0; 707 cidx = ri->iri_cidx; 708 709 do { 710 rxd = &rxr->rx_base[cidx]; 711 staterr = le32toh(rxd->wb.upper.status_error); 712 pkt_info = le32toh(rxd->wb.lower.mrq); 713 714 /* Error Checking then decrement count */ 715 MPASS ((staterr & E1000_RXD_STAT_DD) != 0); 716 717 len = le16toh(rxd->wb.upper.length); 718 ri->iri_len += len; 719 720 eop = (staterr & E1000_RXD_STAT_EOP) != 0; 721 722 /* Make sure bad packets are discarded */ 723 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { 724 sc->dropped_pkts++; 725 return EBADMSG; 726 } 727 728 ri->iri_frags[i].irf_flid = 0; 729 ri->iri_frags[i].irf_idx = cidx; 730 ri->iri_frags[i].irf_len = len; 731 /* Zero out the receive descriptors status. */ 732 rxd->wb.upper.status_error &= htole32(~0xFF); 733 734 if (++cidx == scctx->isc_nrxd[0]) 735 cidx = 0; 736 i++; 737 } while (!eop); 738 739 if (scctx->isc_capenable & IFCAP_RXCSUM) 740 em_receive_checksum(staterr, staterr >> 24, ri); 741 742 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING && 743 staterr & E1000_RXD_STAT_VP) { 744 ri->iri_vtag = le16toh(rxd->wb.upper.vlan); 745 ri->iri_flags |= M_VLANTAG; 746 } 747 748 ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); 749 ri->iri_rsstype = em_determine_rsstype(pkt_info); 750 751 ri->iri_nfrags = i; 752 return (0); 753 } 754 755 /********************************************************************* 756 * 757 * Verify that the hardware indicated that the checksum is valid. 758 * Inform the stack about the status of checksum so that stack 759 * doesn't spend time verifying the checksum. 760 * 761 *********************************************************************/ 762 static void 763 em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri) 764 { 765 if (__predict_false(status & E1000_RXD_STAT_IXSM)) 766 return; 767 768 /* If there is a layer 3 or 4 error we are done */ 769 if (__predict_false(errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE))) 770 return; 771 772 /* IP Checksum Good */ 773 if (status & E1000_RXD_STAT_IPCS) 774 ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); 775 776 /* Valid L4E checksum */ 777 if (__predict_true(status & 778 (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) { 779 ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 780 ri->iri_csum_data = htons(0xffff); 781 } 782 } 783 784 /******************************************************************** 785 * 786 * Parse the packet type to determine the appropriate hash 787 * 788 ******************************************************************/ 789 static int 790 em_determine_rsstype(uint32_t pkt_info) 791 { 792 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { 793 case E1000_RXDADV_RSSTYPE_IPV4_TCP: 794 return M_HASHTYPE_RSS_TCP_IPV4; 795 case E1000_RXDADV_RSSTYPE_IPV4: 796 return M_HASHTYPE_RSS_IPV4; 797 case E1000_RXDADV_RSSTYPE_IPV6_TCP: 798 return M_HASHTYPE_RSS_TCP_IPV6; 799 case E1000_RXDADV_RSSTYPE_IPV6_EX: 800 return M_HASHTYPE_RSS_IPV6_EX; 801 case E1000_RXDADV_RSSTYPE_IPV6: 802 return M_HASHTYPE_RSS_IPV6; 803 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: 804 return M_HASHTYPE_RSS_TCP_IPV6_EX; 805 default: 806 return M_HASHTYPE_OPAQUE; 807 } 808 } 809