1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 5 * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* $FreeBSD$ */ 31 #include "if_em.h" 32 33 #ifdef RSS 34 #include <net/rss_config.h> 35 #include <netinet/in_rss.h> 36 #endif 37 38 #ifdef VERBOSE_DEBUG 39 #define DPRINTF device_printf 40 #else 41 #define DPRINTF(...) 42 #endif 43 44 /********************************************************************* 45 * Local Function prototypes 46 *********************************************************************/ 47 static int em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, 48 uint32_t *txd_upper, uint32_t *txd_lower); 49 static int em_transmit_checksum_setup(struct e1000_softc *sc, 50 if_pkt_info_t pi, uint32_t *txd_upper, uint32_t *txd_lower); 51 static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); 52 static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 53 static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); 54 static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru); 55 static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, 56 qidx_t pidx); 57 static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, 58 qidx_t budget); 59 static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 60 61 static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru); 62 63 static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, 64 qidx_t budget); 65 static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 66 67 static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t); 68 static int em_determine_rsstype(uint32_t pkt_info); 69 extern int em_intr(void *arg); 70 71 struct if_txrx em_txrx = { 72 .ift_txd_encap = em_isc_txd_encap, 73 .ift_txd_flush = em_isc_txd_flush, 74 .ift_txd_credits_update = em_isc_txd_credits_update, 75 .ift_rxd_available = em_isc_rxd_available, 76 .ift_rxd_pkt_get = em_isc_rxd_pkt_get, 77 .ift_rxd_refill = em_isc_rxd_refill, 78 .ift_rxd_flush = em_isc_rxd_flush, 79 .ift_legacy_intr = em_intr 80 }; 81 82 struct if_txrx lem_txrx = { 83 .ift_txd_encap = em_isc_txd_encap, 84 .ift_txd_flush = em_isc_txd_flush, 85 .ift_txd_credits_update = em_isc_txd_credits_update, 86 .ift_rxd_available = lem_isc_rxd_available, 87 .ift_rxd_pkt_get = lem_isc_rxd_pkt_get, 88 .ift_rxd_refill = lem_isc_rxd_refill, 89 .ift_rxd_flush = em_isc_rxd_flush, 90 .ift_legacy_intr = em_intr 91 }; 92 93 extern if_shared_ctx_t em_sctx; 94 95 void 96 em_dump_rs(struct e1000_softc *sc) 97 { 98 if_softc_ctx_t scctx = sc->shared; 99 struct em_tx_queue *que; 100 struct tx_ring *txr; 101 qidx_t i, ntxd, qid, cur; 102 int16_t rs_cidx; 103 uint8_t status; 104 105 printf("\n"); 106 ntxd = scctx->isc_ntxd[0]; 107 for (qid = 0; qid < sc->tx_num_queues; qid++) { 108 que = &sc->tx_queues[qid]; 109 txr = &que->txr; 110 rs_cidx = txr->tx_rs_cidx; 111 if (rs_cidx != txr->tx_rs_pidx) { 112 cur = txr->tx_rsq[rs_cidx]; 113 status = txr->tx_base[cur].upper.fields.status; 114 if (!(status & E1000_TXD_STAT_DD)) 115 printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur); 116 } else { 117 rs_cidx = (rs_cidx-1)&(ntxd-1); 118 cur = txr->tx_rsq[rs_cidx]; 119 printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", qid, rs_cidx, cur); 120 } 121 printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, 122 txr->tx_rs_pidx); 123 for (i = 0; i < ntxd; i++) { 124 if (txr->tx_base[i].upper.fields.status & E1000_TXD_STAT_DD) 125 printf("%d set ", i); 126 } 127 printf("\n"); 128 } 129 } 130 131 /********************************************************************** 132 * 133 * Setup work for hardware segmentation offload (TSO) on 134 * adapters using advanced tx descriptors 135 * 136 **********************************************************************/ 137 static int 138 em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, uint32_t *txd_upper, 139 uint32_t *txd_lower) 140 { 141 if_softc_ctx_t scctx = sc->shared; 142 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 143 struct tx_ring *txr = &que->txr; 144 struct e1000_context_desc *TXD; 145 int cur, hdr_len; 146 uint32_t cmd_type_len; 147 148 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 149 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 150 E1000_TXD_DTYP_D | /* Data descr type */ 151 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 152 153 cur = pi->ipi_pidx; 154 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 155 156 /* 157 * ipcss - Start offset for header checksum calculation. 158 * ipcse - End offset for header checksum calculation. 159 * ipcso - Offset of place to put the checksum. 160 */ 161 switch(pi->ipi_etype) { 162 case ETHERTYPE_IP: 163 /* IP and/or TCP header checksum calculation and insertion. */ 164 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 165 166 TXD->lower_setup.ip_fields.ipcse = 167 htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); 168 break; 169 case ETHERTYPE_IPV6: 170 /* TCP header checksum calculation and insertion. */ 171 *txd_upper = E1000_TXD_POPTS_TXSM << 8; 172 173 TXD->lower_setup.ip_fields.ipcse = htole16(0); 174 break; 175 default: 176 break; 177 } 178 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 179 TXD->lower_setup.ip_fields.ipcso = 180 pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 181 182 /* 183 * tucss - Start offset for payload checksum calculation. 184 * tucse - End offset for payload checksum calculation. 185 * tucso - Offset of place to put the checksum. 186 */ 187 TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 188 TXD->upper_setup.tcp_fields.tucse = 0; 189 TXD->upper_setup.tcp_fields.tucso = 190 pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum); 191 192 /* 193 * Payload size per packet w/o any headers. 194 * Length of all headers up to payload. 195 */ 196 TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); 197 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 198 199 /* 200 * "PCI/PCI-X SDM 4.0" page 45, and "PCIe GbE SDM 2.5" page 63 201 * - Set up basic TUCMDs 202 * - For others IP bit on indicates IPv4, while off indicates IPv6 203 */ 204 cmd_type_len = sc->txd_cmd | 205 E1000_TXD_CMD_DEXT | /* Extended descr */ 206 E1000_TXD_CMD_TSE | /* TSE context */ 207 E1000_TXD_CMD_TCP; /* Do TCP checksum */ 208 if (pi->ipi_etype == ETHERTYPE_IP) 209 cmd_type_len |= E1000_TXD_CMD_IP; 210 TXD->cmd_and_length = htole32(cmd_type_len | 211 (pi->ipi_len - hdr_len)); /* Total len */ 212 213 txr->tx_tso = true; 214 215 if (++cur == scctx->isc_ntxd[0]) { 216 cur = 0; 217 } 218 DPRINTF(iflib_get_dev(sc->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, 219 pi->ipi_pidx, cur); 220 return (cur); 221 } 222 223 /********************************************************************* 224 * The offload context is protocol specific (TCP/UDP) and thus 225 * only needs to be set when the protocol changes. The occasion 226 * of a context change can be a performance detriment, and 227 * might be better just disabled. The reason arises in the way 228 * in which the controller supports pipelined requests from the 229 * Tx data DMA. Up to four requests can be pipelined, and they may 230 * belong to the same packet or to multiple packets. However all 231 * requests for one packet are issued before a request is issued 232 * for a subsequent packet and if a request for the next packet 233 * requires a context change, that request will be stalled 234 * until the previous request completes. This means setting up 235 * a new context effectively disables pipelined Tx data DMA which 236 * in turn greatly slow down performance to send small sized 237 * frames. 238 **********************************************************************/ 239 #define DONT_FORCE_CTX 1 240 241 static int 242 em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi, 243 uint32_t *txd_upper, uint32_t *txd_lower) 244 { 245 struct e1000_context_desc *TXD = NULL; 246 if_softc_ctx_t scctx = sc->shared; 247 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 248 struct tx_ring *txr = &que->txr; 249 int csum_flags = pi->ipi_csum_flags; 250 int cur, hdr_len; 251 uint32_t cmd; 252 253 cur = pi->ipi_pidx; 254 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 255 cmd = sc->txd_cmd; 256 257 /* 258 * The 82574L can only remember the *last* context used 259 * regardless of queue that it was use for. We cannot reuse 260 * contexts on this hardware platform and must generate a new 261 * context every time. 82574L hardware spec, section 7.2.6, 262 * second note. 263 */ 264 if (DONT_FORCE_CTX && 265 sc->tx_num_queues == 1 && 266 txr->csum_lhlen == pi->ipi_ehdrlen && 267 txr->csum_iphlen == pi->ipi_ip_hlen && 268 txr->csum_flags == csum_flags) { 269 /* 270 * Same csum offload context as the previous packets; 271 * just return. 272 */ 273 *txd_upper = txr->csum_txd_upper; 274 *txd_lower = txr->csum_txd_lower; 275 return (cur); 276 } 277 278 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 279 /* 280 * ipcss - Start offset for header checksum calculation. 281 * ipcse - End offset for header checksum calculation. 282 * ipcso - Offset of place to put the checksum. 283 * 284 * We set ipcsX values regardless of IP version to work around HW issues 285 * and ipcse must be 0 for IPv6 per "PCIe GbE SDM 2.5" page 61. 286 * IXSM controls whether it's inserted. 287 */ 288 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 289 TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + 290 offsetof(struct ip, ip_sum); 291 if (csum_flags & CSUM_IP) { 292 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 293 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); 294 cmd |= E1000_TXD_CMD_IP; 295 } else if (csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP)) 296 TXD->lower_setup.ip_fields.ipcse = htole16(0); 297 298 /* 299 * tucss - Start offset for payload checksum calculation. 300 * tucse - End offset for payload checksum calculation. 301 * tucso - Offset of place to put the checksum. 302 */ 303 if (csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_IP6_TCP | CSUM_IP6_UDP)) { 304 uint8_t tucso; 305 306 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 307 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 308 309 if (csum_flags & (CSUM_TCP | CSUM_IP6_TCP)) { 310 tucso = hdr_len + offsetof(struct tcphdr, th_sum); 311 cmd |= E1000_TXD_CMD_TCP; 312 } else 313 tucso = hdr_len + offsetof(struct udphdr, uh_sum); 314 TXD->upper_setup.tcp_fields.tucss = hdr_len; 315 TXD->upper_setup.tcp_fields.tucse = htole16(0); 316 TXD->upper_setup.tcp_fields.tucso = tucso; 317 } 318 319 txr->csum_lhlen = pi->ipi_ehdrlen; 320 txr->csum_iphlen = pi->ipi_ip_hlen; 321 txr->csum_flags = csum_flags; 322 txr->csum_txd_upper = *txd_upper; 323 txr->csum_txd_lower = *txd_lower; 324 325 TXD->tcp_seg_setup.data = htole32(0); 326 TXD->cmd_and_length = 327 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 328 329 if (++cur == scctx->isc_ntxd[0]) { 330 cur = 0; 331 } 332 DPRINTF(iflib_get_dev(sc->ctx), 333 "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n", 334 csum_flags, *txd_upper, *txd_lower, hdr_len, cmd); 335 return (cur); 336 } 337 338 #define TSO_WORKAROUND 4 /* TSO sentinel descriptor */ 339 340 static int 341 em_isc_txd_encap(void *arg, if_pkt_info_t pi) 342 { 343 struct e1000_softc *sc = arg; 344 if_softc_ctx_t scctx = sc->shared; 345 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 346 struct tx_ring *txr = &que->txr; 347 bus_dma_segment_t *segs = pi->ipi_segs; 348 int nsegs = pi->ipi_nsegs; 349 int csum_flags = pi->ipi_csum_flags; 350 int i, j, first, pidx_last; 351 uint32_t txd_flags, txd_upper = 0, txd_lower = 0; 352 353 struct e1000_tx_desc *ctxd = NULL; 354 bool do_tso, tso_desc; 355 qidx_t ntxd; 356 357 txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; 358 i = first = pi->ipi_pidx; 359 do_tso = (csum_flags & CSUM_TSO); 360 tso_desc = false; 361 ntxd = scctx->isc_ntxd[0]; 362 /* 363 * TSO Hardware workaround, if this packet is not 364 * TSO, and is only a single descriptor long, and 365 * it follows a TSO burst, then we need to add a 366 * sentinel descriptor to prevent premature writeback. 367 */ 368 if ((!do_tso) && (txr->tx_tso == true)) { 369 if (nsegs == 1) 370 tso_desc = true; 371 txr->tx_tso = false; 372 } 373 374 /* Do hardware assists */ 375 if (do_tso) { 376 i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); 377 tso_desc = true; 378 } else if (csum_flags & EM_CSUM_OFFLOAD) { 379 i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower); 380 } 381 382 if (pi->ipi_mflags & M_VLANTAG) { 383 /* Set the vlan id. */ 384 txd_upper |= htole16(pi->ipi_vtag) << 16; 385 /* Tell hardware to add tag */ 386 txd_lower |= htole32(E1000_TXD_CMD_VLE); 387 } 388 389 DPRINTF(iflib_get_dev(sc->ctx), 390 "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); 391 /* XXX sc->pcix_82544 -- lem_fill_descriptors */ 392 393 /* Set up our transmit descriptors */ 394 for (j = 0; j < nsegs; j++) { 395 bus_size_t seg_len; 396 bus_addr_t seg_addr; 397 uint32_t cmd; 398 399 ctxd = &txr->tx_base[i]; 400 seg_addr = segs[j].ds_addr; 401 seg_len = segs[j].ds_len; 402 cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; 403 404 /* 405 * TSO Workaround: 406 * If this is the last descriptor, we want to 407 * split it so we have a small final sentinel 408 */ 409 if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { 410 seg_len -= TSO_WORKAROUND; 411 ctxd->buffer_addr = htole64(seg_addr); 412 ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 413 ctxd->upper.data = htole32(txd_upper); 414 415 if (++i == scctx->isc_ntxd[0]) 416 i = 0; 417 418 /* Now make the sentinel */ 419 ctxd = &txr->tx_base[i]; 420 ctxd->buffer_addr = htole64(seg_addr + seg_len); 421 ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND); 422 ctxd->upper.data = htole32(txd_upper); 423 pidx_last = i; 424 if (++i == scctx->isc_ntxd[0]) 425 i = 0; 426 DPRINTF(iflib_get_dev(sc->ctx), 427 "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", 428 pidx_last, i, scctx->isc_ntxd[0]); 429 } else { 430 ctxd->buffer_addr = htole64(seg_addr); 431 ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 432 ctxd->upper.data = htole32(txd_upper); 433 pidx_last = i; 434 if (++i == scctx->isc_ntxd[0]) 435 i = 0; 436 DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", 437 pidx_last, i, scctx->isc_ntxd[0]); 438 } 439 } 440 441 /* 442 * Last Descriptor of Packet 443 * needs End Of Packet (EOP) 444 * and Report Status (RS) 445 */ 446 if (txd_flags && nsegs) { 447 txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; 448 DPRINTF(iflib_get_dev(sc->ctx), 449 "setting to RS on %d rs_pidx %d first: %d\n", 450 pidx_last, txr->tx_rs_pidx, first); 451 txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); 452 MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); 453 } 454 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags); 455 DPRINTF(iflib_get_dev(sc->ctx), 456 "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i); 457 pi->ipi_new_pidx = i; 458 459 return (0); 460 } 461 462 static void 463 em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 464 { 465 struct e1000_softc *sc = arg; 466 struct em_tx_queue *que = &sc->tx_queues[txqid]; 467 struct tx_ring *txr = &que->txr; 468 469 E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx); 470 } 471 472 static int 473 em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) 474 { 475 struct e1000_softc *sc = arg; 476 if_softc_ctx_t scctx = sc->shared; 477 struct em_tx_queue *que = &sc->tx_queues[txqid]; 478 struct tx_ring *txr = &que->txr; 479 480 qidx_t processed = 0; 481 int updated; 482 qidx_t cur, prev, ntxd, rs_cidx; 483 int32_t delta; 484 uint8_t status; 485 486 rs_cidx = txr->tx_rs_cidx; 487 if (rs_cidx == txr->tx_rs_pidx) 488 return (0); 489 cur = txr->tx_rsq[rs_cidx]; 490 MPASS(cur != QIDX_INVALID); 491 status = txr->tx_base[cur].upper.fields.status; 492 updated = !!(status & E1000_TXD_STAT_DD); 493 494 if (!updated) 495 return (0); 496 497 /* If clear is false just let caller know that there 498 * are descriptors to reclaim */ 499 if (!clear) 500 return (1); 501 502 prev = txr->tx_cidx_processed; 503 ntxd = scctx->isc_ntxd[0]; 504 do { 505 MPASS(prev != cur); 506 delta = (int32_t)cur - (int32_t)prev; 507 if (delta < 0) 508 delta += ntxd; 509 MPASS(delta > 0); 510 DPRINTF(iflib_get_dev(sc->ctx), 511 "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n", 512 __FUNCTION__, prev, cur, clear, delta); 513 514 processed += delta; 515 prev = cur; 516 rs_cidx = (rs_cidx + 1) & (ntxd-1); 517 if (rs_cidx == txr->tx_rs_pidx) 518 break; 519 cur = txr->tx_rsq[rs_cidx]; 520 MPASS(cur != QIDX_INVALID); 521 status = txr->tx_base[cur].upper.fields.status; 522 } while ((status & E1000_TXD_STAT_DD)); 523 524 txr->tx_rs_cidx = rs_cidx; 525 txr->tx_cidx_processed = prev; 526 return(processed); 527 } 528 529 static void 530 lem_isc_rxd_refill(void *arg, if_rxd_update_t iru) 531 { 532 struct e1000_softc *sc = arg; 533 if_softc_ctx_t scctx = sc->shared; 534 struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; 535 struct rx_ring *rxr = &que->rxr; 536 struct e1000_rx_desc *rxd; 537 uint64_t *paddrs; 538 uint32_t next_pidx, pidx; 539 uint16_t count; 540 int i; 541 542 paddrs = iru->iru_paddrs; 543 pidx = iru->iru_pidx; 544 count = iru->iru_count; 545 546 for (i = 0, next_pidx = pidx; i < count; i++) { 547 rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; 548 rxd->buffer_addr = htole64(paddrs[i]); 549 /* status bits must be cleared */ 550 rxd->status = 0; 551 552 if (++next_pidx == scctx->isc_nrxd[0]) 553 next_pidx = 0; 554 } 555 } 556 557 static void 558 em_isc_rxd_refill(void *arg, if_rxd_update_t iru) 559 { 560 struct e1000_softc *sc = arg; 561 if_softc_ctx_t scctx = sc->shared; 562 uint16_t rxqid = iru->iru_qsidx; 563 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 564 struct rx_ring *rxr = &que->rxr; 565 union e1000_rx_desc_extended *rxd; 566 uint64_t *paddrs; 567 uint32_t next_pidx, pidx; 568 uint16_t count; 569 int i; 570 571 paddrs = iru->iru_paddrs; 572 pidx = iru->iru_pidx; 573 count = iru->iru_count; 574 575 for (i = 0, next_pidx = pidx; i < count; i++) { 576 rxd = &rxr->rx_base[next_pidx]; 577 rxd->read.buffer_addr = htole64(paddrs[i]); 578 /* DD bits must be cleared */ 579 rxd->wb.upper.status_error = 0; 580 581 if (++next_pidx == scctx->isc_nrxd[0]) 582 next_pidx = 0; 583 } 584 } 585 586 static void 587 em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, 588 qidx_t pidx) 589 { 590 struct e1000_softc *sc = arg; 591 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 592 struct rx_ring *rxr = &que->rxr; 593 594 E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); 595 } 596 597 static int 598 lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 599 { 600 struct e1000_softc *sc = arg; 601 if_softc_ctx_t scctx = sc->shared; 602 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 603 struct rx_ring *rxr = &que->rxr; 604 struct e1000_rx_desc *rxd; 605 uint32_t staterr = 0; 606 int cnt, i; 607 608 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 609 rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; 610 staterr = rxd->status; 611 612 if ((staterr & E1000_RXD_STAT_DD) == 0) 613 break; 614 if (++i == scctx->isc_nrxd[0]) 615 i = 0; 616 if (staterr & E1000_RXD_STAT_EOP) 617 cnt++; 618 } 619 return (cnt); 620 } 621 622 static int 623 em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 624 { 625 struct e1000_softc *sc = arg; 626 if_softc_ctx_t scctx = sc->shared; 627 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 628 struct rx_ring *rxr = &que->rxr; 629 union e1000_rx_desc_extended *rxd; 630 uint32_t staterr = 0; 631 int cnt, i; 632 633 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 634 rxd = &rxr->rx_base[i]; 635 staterr = le32toh(rxd->wb.upper.status_error); 636 637 if ((staterr & E1000_RXD_STAT_DD) == 0) 638 break; 639 if (++i == scctx->isc_nrxd[0]) 640 i = 0; 641 if (staterr & E1000_RXD_STAT_EOP) 642 cnt++; 643 } 644 return (cnt); 645 } 646 647 static int 648 lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 649 { 650 struct e1000_softc *sc = arg; 651 if_softc_ctx_t scctx = sc->shared; 652 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx]; 653 struct rx_ring *rxr = &que->rxr; 654 struct e1000_rx_desc *rxd; 655 uint16_t len; 656 uint32_t status, errors; 657 bool eop; 658 int i, cidx; 659 660 status = errors = i = 0; 661 cidx = ri->iri_cidx; 662 663 do { 664 rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; 665 status = rxd->status; 666 errors = rxd->errors; 667 668 /* Error Checking then decrement count */ 669 MPASS ((status & E1000_RXD_STAT_DD) != 0); 670 671 len = le16toh(rxd->length); 672 ri->iri_len += len; 673 674 eop = (status & E1000_RXD_STAT_EOP) != 0; 675 676 /* Make sure bad packets are discarded */ 677 if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 678 sc->dropped_pkts++; 679 /* XXX fixup if common */ 680 return (EBADMSG); 681 } 682 683 ri->iri_frags[i].irf_flid = 0; 684 ri->iri_frags[i].irf_idx = cidx; 685 ri->iri_frags[i].irf_len = len; 686 /* Zero out the receive descriptors status. */ 687 rxd->status = 0; 688 689 if (++cidx == scctx->isc_nrxd[0]) 690 cidx = 0; 691 i++; 692 } while (!eop); 693 694 if (scctx->isc_capenable & IFCAP_RXCSUM) 695 em_receive_checksum(status, errors, ri); 696 697 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING && 698 status & E1000_RXD_STAT_VP) { 699 ri->iri_vtag = le16toh(rxd->special & E1000_RXD_SPC_VLAN_MASK); 700 ri->iri_flags |= M_VLANTAG; 701 } 702 703 ri->iri_nfrags = i; 704 705 return (0); 706 } 707 708 static int 709 em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 710 { 711 struct e1000_softc *sc = arg; 712 if_softc_ctx_t scctx = sc->shared; 713 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx]; 714 struct rx_ring *rxr = &que->rxr; 715 union e1000_rx_desc_extended *rxd; 716 717 uint16_t len; 718 uint32_t pkt_info; 719 uint32_t staterr; 720 bool eop; 721 int i, cidx; 722 723 staterr = i = 0; 724 cidx = ri->iri_cidx; 725 726 do { 727 rxd = &rxr->rx_base[cidx]; 728 staterr = le32toh(rxd->wb.upper.status_error); 729 pkt_info = le32toh(rxd->wb.lower.mrq); 730 731 /* Error Checking then decrement count */ 732 MPASS ((staterr & E1000_RXD_STAT_DD) != 0); 733 734 len = le16toh(rxd->wb.upper.length); 735 ri->iri_len += len; 736 737 eop = (staterr & E1000_RXD_STAT_EOP) != 0; 738 739 /* Make sure bad packets are discarded */ 740 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { 741 sc->dropped_pkts++; 742 return EBADMSG; 743 } 744 745 ri->iri_frags[i].irf_flid = 0; 746 ri->iri_frags[i].irf_idx = cidx; 747 ri->iri_frags[i].irf_len = len; 748 /* Zero out the receive descriptors status. */ 749 rxd->wb.upper.status_error &= htole32(~0xFF); 750 751 if (++cidx == scctx->isc_nrxd[0]) 752 cidx = 0; 753 i++; 754 } while (!eop); 755 756 if (scctx->isc_capenable & IFCAP_RXCSUM) 757 em_receive_checksum(staterr, staterr >> 24, ri); 758 759 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING && 760 staterr & E1000_RXD_STAT_VP) { 761 ri->iri_vtag = le16toh(rxd->wb.upper.vlan); 762 ri->iri_flags |= M_VLANTAG; 763 } 764 765 ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); 766 ri->iri_rsstype = em_determine_rsstype(pkt_info); 767 768 ri->iri_nfrags = i; 769 return (0); 770 } 771 772 /********************************************************************* 773 * 774 * Verify that the hardware indicated that the checksum is valid. 775 * Inform the stack about the status of checksum so that stack 776 * doesn't spend time verifying the checksum. 777 * 778 *********************************************************************/ 779 static void 780 em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri) 781 { 782 if (__predict_false(status & E1000_RXD_STAT_IXSM)) 783 return; 784 785 /* If there is a layer 3 or 4 error we are done */ 786 if (__predict_false(errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE))) 787 return; 788 789 /* IP Checksum Good */ 790 if (status & E1000_RXD_STAT_IPCS) 791 ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); 792 793 /* Valid L4E checksum */ 794 if (__predict_true(status & 795 (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) { 796 ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 797 ri->iri_csum_data = htons(0xffff); 798 } 799 } 800 801 /******************************************************************** 802 * 803 * Parse the packet type to determine the appropriate hash 804 * 805 ******************************************************************/ 806 static int 807 em_determine_rsstype(uint32_t pkt_info) 808 { 809 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { 810 case E1000_RXDADV_RSSTYPE_IPV4_TCP: 811 return M_HASHTYPE_RSS_TCP_IPV4; 812 case E1000_RXDADV_RSSTYPE_IPV4: 813 return M_HASHTYPE_RSS_IPV4; 814 case E1000_RXDADV_RSSTYPE_IPV6_TCP: 815 return M_HASHTYPE_RSS_TCP_IPV6; 816 case E1000_RXDADV_RSSTYPE_IPV6_EX: 817 return M_HASHTYPE_RSS_IPV6_EX; 818 case E1000_RXDADV_RSSTYPE_IPV6: 819 return M_HASHTYPE_RSS_IPV6; 820 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: 821 return M_HASHTYPE_RSS_TCP_IPV6_EX; 822 default: 823 return M_HASHTYPE_OPAQUE; 824 } 825 } 826