1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 5 * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "if_em.h" 31 32 #include <net/rss_config.h> 33 #include <netinet/in_rss.h> 34 35 #ifdef VERBOSE_DEBUG 36 #define DPRINTF device_printf 37 #else 38 #define DPRINTF(...) 39 #endif 40 41 /********************************************************************* 42 * Local Function prototypes 43 *********************************************************************/ 44 static int em_tso_setup(struct e1000_softc *, if_pkt_info_t, uint32_t *, 45 uint32_t *); 46 static int em_transmit_checksum_setup(struct e1000_softc *, if_pkt_info_t, 47 uint32_t *, uint32_t *); 48 static int em_isc_txd_encap(void *, if_pkt_info_t); 49 static void em_isc_txd_flush(void *, uint16_t, qidx_t); 50 static int em_isc_txd_credits_update(void *, uint16_t, bool); 51 static void em_isc_rxd_refill(void *, if_rxd_update_t); 52 static void em_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t); 53 static int em_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t); 54 static int em_isc_rxd_pkt_get(void *, if_rxd_info_t); 55 56 static void lem_isc_rxd_refill(void *, if_rxd_update_t); 57 58 static int lem_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t); 59 static int lem_isc_rxd_pkt_get(void *, if_rxd_info_t); 60 61 static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t); 62 static int em_determine_rsstype(uint32_t); 63 extern int em_intr(void *); 64 65 struct if_txrx em_txrx = { 66 .ift_txd_encap = em_isc_txd_encap, 67 .ift_txd_flush = em_isc_txd_flush, 68 .ift_txd_credits_update = em_isc_txd_credits_update, 69 .ift_rxd_available = em_isc_rxd_available, 70 .ift_rxd_pkt_get = em_isc_rxd_pkt_get, 71 .ift_rxd_refill = em_isc_rxd_refill, 72 .ift_rxd_flush = em_isc_rxd_flush, 73 .ift_legacy_intr = em_intr 74 }; 75 76 struct if_txrx lem_txrx = { 77 .ift_txd_encap = em_isc_txd_encap, 78 .ift_txd_flush = em_isc_txd_flush, 79 .ift_txd_credits_update = em_isc_txd_credits_update, 80 .ift_rxd_available = lem_isc_rxd_available, 81 .ift_rxd_pkt_get = lem_isc_rxd_pkt_get, 82 .ift_rxd_refill = lem_isc_rxd_refill, 83 .ift_rxd_flush = em_isc_rxd_flush, 84 .ift_legacy_intr = em_intr 85 }; 86 87 extern if_shared_ctx_t em_sctx; 88 89 void 90 em_dump_rs(struct e1000_softc *sc) 91 { 92 if_softc_ctx_t scctx = sc->shared; 93 struct em_tx_queue *que; 94 struct tx_ring *txr; 95 qidx_t i, ntxd, qid, cur; 96 int16_t rs_cidx; 97 uint8_t status; 98 99 printf("\n"); 100 ntxd = scctx->isc_ntxd[0]; 101 for (qid = 0; qid < sc->tx_num_queues; qid++) { 102 que = &sc->tx_queues[qid]; 103 txr = &que->txr; 104 rs_cidx = txr->tx_rs_cidx; 105 if (rs_cidx != txr->tx_rs_pidx) { 106 cur = txr->tx_rsq[rs_cidx]; 107 status = txr->tx_base[cur].upper.fields.status; 108 if (!(status & E1000_TXD_STAT_DD)) 109 printf("qid[%d]->tx_rsq[%d]: %d clear ", 110 qid, rs_cidx, cur); 111 } else { 112 rs_cidx = (rs_cidx-1)&(ntxd-1); 113 cur = txr->tx_rsq[rs_cidx]; 114 printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", 115 qid, rs_cidx, cur); 116 } 117 printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, 118 txr->tx_rs_pidx); 119 for (i = 0; i < ntxd; i++) { 120 if (txr->tx_base[i].upper.fields.status & 121 E1000_TXD_STAT_DD) 122 printf("%d set ", i); 123 } 124 printf("\n"); 125 } 126 } 127 128 /********************************************************************** 129 * 130 * Setup work for hardware segmentation offload (TSO) on 131 * adapters using advanced tx descriptors 132 * 133 **********************************************************************/ 134 static int 135 em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, uint32_t *txd_upper, 136 uint32_t *txd_lower) 137 { 138 if_softc_ctx_t scctx = sc->shared; 139 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 140 struct tx_ring *txr = &que->txr; 141 struct e1000_context_desc *TXD; 142 int cur, hdr_len; 143 uint32_t cmd_type_len; 144 145 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 146 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 147 E1000_TXD_DTYP_D | /* Data descr type */ 148 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 149 150 cur = pi->ipi_pidx; 151 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 152 153 /* 154 * ipcss - Start offset for header checksum calculation. 155 * ipcse - End offset for header checksum calculation. 156 * ipcso - Offset of place to put the checksum. 157 */ 158 switch(pi->ipi_etype) { 159 case ETHERTYPE_IP: 160 /* IP and/or TCP header checksum calculation and insertion. */ 161 *txd_upper = 162 (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 163 164 TXD->lower_setup.ip_fields.ipcse = 165 htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); 166 break; 167 case ETHERTYPE_IPV6: 168 /* TCP header checksum calculation and insertion. */ 169 *txd_upper = E1000_TXD_POPTS_TXSM << 8; 170 171 TXD->lower_setup.ip_fields.ipcse = htole16(0); 172 break; 173 default: 174 break; 175 } 176 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 177 TXD->lower_setup.ip_fields.ipcso = 178 pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 179 180 /* 181 * tucss - Start offset for payload checksum calculation. 182 * tucse - End offset for payload checksum calculation. 183 * tucso - Offset of place to put the checksum. 184 */ 185 TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 186 TXD->upper_setup.tcp_fields.tucse = 0; 187 TXD->upper_setup.tcp_fields.tucso = 188 pi->ipi_ehdrlen + pi->ipi_ip_hlen + 189 offsetof(struct tcphdr, th_sum); 190 191 /* 192 * Payload size per packet w/o any headers. 193 * Length of all headers up to payload. 194 */ 195 TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); 196 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 197 198 /* 199 * "PCI/PCI-X SDM 4.0" page 45, and "PCIe GbE SDM 2.5" page 63 200 * - Set up basic TUCMDs 201 * - For others IP bit on indicates IPv4, while off indicates IPv6 202 */ 203 cmd_type_len = sc->txd_cmd | 204 E1000_TXD_CMD_DEXT | /* Extended descr */ 205 E1000_TXD_CMD_TSE | /* TSE context */ 206 E1000_TXD_CMD_TCP; /* Do TCP checksum */ 207 if (pi->ipi_etype == ETHERTYPE_IP) 208 cmd_type_len |= E1000_TXD_CMD_IP; 209 TXD->cmd_and_length = htole32(cmd_type_len | 210 (pi->ipi_len - hdr_len)); /* Total len */ 211 212 txr->tx_tso = true; 213 214 if (++cur == scctx->isc_ntxd[0]) { 215 cur = 0; 216 } 217 DPRINTF(iflib_get_dev(sc->ctx), "%s: pidx: %d cur: %d\n", 218 __FUNCTION__, pi->ipi_pidx, cur); 219 return (cur); 220 } 221 222 /********************************************************************* 223 * The offload context is protocol specific (TCP/UDP) and thus 224 * only needs to be set when the protocol changes. The occasion 225 * of a context change can be a performance detriment, and 226 * might be better just disabled. The reason arises in the way 227 * in which the controller supports pipelined requests from the 228 * Tx data DMA. Up to four requests can be pipelined, and they may 229 * belong to the same packet or to multiple packets. However all 230 * requests for one packet are issued before a request is issued 231 * for a subsequent packet and if a request for the next packet 232 * requires a context change, that request will be stalled 233 * until the previous request completes. This means setting up 234 * a new context effectively disables pipelined Tx data DMA which 235 * in turn greatly slow down performance to send small sized 236 * frames. 237 **********************************************************************/ 238 #define DONT_FORCE_CTX 1 239 240 static int 241 em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi, 242 uint32_t *txd_upper, uint32_t *txd_lower) 243 { 244 struct e1000_context_desc *TXD = NULL; 245 if_softc_ctx_t scctx = sc->shared; 246 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 247 struct tx_ring *txr = &que->txr; 248 int csum_flags = pi->ipi_csum_flags; 249 int cur, hdr_len; 250 uint32_t cmd; 251 252 cur = pi->ipi_pidx; 253 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 254 cmd = sc->txd_cmd; 255 256 /* 257 * The 82574L can only remember the *last* context used 258 * regardless of queue that it was use for. We cannot reuse 259 * contexts on this hardware platform and must generate a new 260 * context every time. 82574L hardware spec, section 7.2.6, 261 * second note. 262 */ 263 if (DONT_FORCE_CTX && 264 sc->tx_num_queues == 1 && 265 txr->csum_lhlen == pi->ipi_ehdrlen && 266 txr->csum_iphlen == pi->ipi_ip_hlen && 267 txr->csum_flags == csum_flags) { 268 /* 269 * Same csum offload context as the previous packets; 270 * just return. 271 */ 272 *txd_upper = txr->csum_txd_upper; 273 *txd_lower = txr->csum_txd_lower; 274 return (cur); 275 } 276 277 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 278 /* 279 * ipcss - Start offset for header checksum calculation. 280 * ipcse - End offset for header checksum calculation. 281 * ipcso - Offset of place to put the checksum. 282 * 283 * We set ipcsX values regardless of IP version to work around HW 284 * issues and ipcse must be 0 for IPv6 per "PCIe GbE SDM 2.5" page 61. 285 * IXSM controls whether it's inserted. 286 */ 287 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 288 TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + 289 offsetof(struct ip, ip_sum); 290 if (csum_flags & CSUM_IP) { 291 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 292 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len - 1); 293 cmd |= E1000_TXD_CMD_IP; 294 } else if (csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP)) 295 TXD->lower_setup.ip_fields.ipcse = htole16(0); 296 297 /* 298 * tucss - Start offset for payload checksum calculation. 299 * tucse - End offset for payload checksum calculation. 300 * tucso - Offset of place to put the checksum. 301 */ 302 if (csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_IP6_TCP | 303 CSUM_IP6_UDP)) { 304 uint8_t tucso; 305 306 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 307 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 308 309 if (csum_flags & (CSUM_TCP | CSUM_IP6_TCP)) { 310 tucso = hdr_len + offsetof(struct tcphdr, th_sum); 311 cmd |= E1000_TXD_CMD_TCP; 312 } else 313 tucso = hdr_len + offsetof(struct udphdr, uh_sum); 314 TXD->upper_setup.tcp_fields.tucss = hdr_len; 315 TXD->upper_setup.tcp_fields.tucse = htole16(0); 316 TXD->upper_setup.tcp_fields.tucso = tucso; 317 } 318 319 txr->csum_lhlen = pi->ipi_ehdrlen; 320 txr->csum_iphlen = pi->ipi_ip_hlen; 321 txr->csum_flags = csum_flags; 322 txr->csum_txd_upper = *txd_upper; 323 txr->csum_txd_lower = *txd_lower; 324 325 TXD->tcp_seg_setup.data = htole32(0); 326 TXD->cmd_and_length = 327 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 328 329 if (++cur == scctx->isc_ntxd[0]) { 330 cur = 0; 331 } 332 DPRINTF(iflib_get_dev(sc->ctx), 333 "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x" 334 " hdr_len=%d cmd=%x\n", 335 csum_flags, *txd_upper, *txd_lower, hdr_len, cmd); 336 return (cur); 337 } 338 339 #define TSO_WORKAROUND 4 /* TSO sentinel descriptor */ 340 341 static int 342 em_isc_txd_encap(void *arg, if_pkt_info_t pi) 343 { 344 struct e1000_softc *sc = arg; 345 if_softc_ctx_t scctx = sc->shared; 346 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 347 struct tx_ring *txr = &que->txr; 348 bus_dma_segment_t *segs = pi->ipi_segs; 349 int nsegs = pi->ipi_nsegs; 350 int csum_flags = pi->ipi_csum_flags; 351 int i, j, first, pidx_last; 352 uint32_t txd_flags, txd_upper = 0, txd_lower = 0; 353 354 struct e1000_tx_desc *ctxd = NULL; 355 bool do_tso, tso_desc; 356 qidx_t ntxd; 357 358 txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; 359 i = first = pi->ipi_pidx; 360 do_tso = (csum_flags & CSUM_TSO); 361 tso_desc = false; 362 ntxd = scctx->isc_ntxd[0]; 363 /* 364 * TSO Hardware workaround, if this packet is not 365 * TSO, and is only a single descriptor long, and 366 * it follows a TSO burst, then we need to add a 367 * sentinel descriptor to prevent premature writeback. 368 */ 369 if ((!do_tso) && (txr->tx_tso == true)) { 370 if (nsegs == 1) 371 tso_desc = true; 372 txr->tx_tso = false; 373 } 374 375 /* Do hardware assists */ 376 if (do_tso) { 377 i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); 378 tso_desc = true; 379 } else if (csum_flags & EM_CSUM_OFFLOAD) { 380 i = em_transmit_checksum_setup(sc, pi, &txd_upper, 381 &txd_lower); 382 } 383 384 if (pi->ipi_mflags & M_VLANTAG) { 385 /* Set the vlan id. */ 386 txd_upper |= htole16(pi->ipi_vtag) << 16; 387 /* Tell hardware to add tag */ 388 txd_lower |= htole32(E1000_TXD_CMD_VLE); 389 } 390 391 DPRINTF(iflib_get_dev(sc->ctx), 392 "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); 393 /* XXX sc->pcix_82544 -- lem_fill_descriptors */ 394 395 /* Set up our transmit descriptors */ 396 for (j = 0; j < nsegs; j++) { 397 bus_size_t seg_len; 398 bus_addr_t seg_addr; 399 uint32_t cmd; 400 401 ctxd = &txr->tx_base[i]; 402 seg_addr = segs[j].ds_addr; 403 seg_len = segs[j].ds_len; 404 cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; 405 406 /* 407 * TSO Workaround: 408 * If this is the last descriptor, we want to 409 * split it so we have a small final sentinel 410 */ 411 if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { 412 seg_len -= TSO_WORKAROUND; 413 ctxd->buffer_addr = htole64(seg_addr); 414 ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 415 ctxd->upper.data = htole32(txd_upper); 416 417 if (++i == scctx->isc_ntxd[0]) 418 i = 0; 419 420 /* Now make the sentinel */ 421 ctxd = &txr->tx_base[i]; 422 ctxd->buffer_addr = htole64(seg_addr + seg_len); 423 ctxd->lower.data = 424 htole32(cmd | txd_lower | TSO_WORKAROUND); 425 ctxd->upper.data = htole32(txd_upper); 426 pidx_last = i; 427 if (++i == scctx->isc_ntxd[0]) 428 i = 0; 429 DPRINTF(iflib_get_dev(sc->ctx), 430 "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", 431 pidx_last, i, scctx->isc_ntxd[0]); 432 } else { 433 ctxd->buffer_addr = htole64(seg_addr); 434 ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 435 ctxd->upper.data = htole32(txd_upper); 436 pidx_last = i; 437 if (++i == scctx->isc_ntxd[0]) 438 i = 0; 439 DPRINTF(iflib_get_dev(sc->ctx), 440 "pidx_last=%d i=%d ntxd[0]=%d\n", 441 pidx_last, i, scctx->isc_ntxd[0]); 442 } 443 } 444 445 /* 446 * Last Descriptor of Packet 447 * needs End Of Packet (EOP) 448 * and Report Status (RS) 449 */ 450 if (txd_flags && nsegs) { 451 txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; 452 DPRINTF(iflib_get_dev(sc->ctx), 453 "setting to RS on %d rs_pidx %d first: %d\n", 454 pidx_last, txr->tx_rs_pidx, first); 455 txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); 456 MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); 457 } 458 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags); 459 DPRINTF(iflib_get_dev(sc->ctx), 460 "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", 461 first, pidx_last, i); 462 pi->ipi_new_pidx = i; 463 464 /* Sent data accounting for AIM */ 465 txr->tx_bytes += pi->ipi_len; 466 ++txr->tx_packets; 467 468 return (0); 469 } 470 471 static void 472 em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 473 { 474 struct e1000_softc *sc = arg; 475 struct em_tx_queue *que = &sc->tx_queues[txqid]; 476 struct tx_ring *txr = &que->txr; 477 478 E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx); 479 } 480 481 static int 482 em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) 483 { 484 struct e1000_softc *sc = arg; 485 if_softc_ctx_t scctx = sc->shared; 486 struct em_tx_queue *que = &sc->tx_queues[txqid]; 487 struct tx_ring *txr = &que->txr; 488 489 qidx_t processed = 0; 490 int updated; 491 qidx_t cur, prev, ntxd, rs_cidx; 492 int32_t delta; 493 uint8_t status; 494 495 rs_cidx = txr->tx_rs_cidx; 496 if (rs_cidx == txr->tx_rs_pidx) 497 return (0); 498 cur = txr->tx_rsq[rs_cidx]; 499 MPASS(cur != QIDX_INVALID); 500 status = txr->tx_base[cur].upper.fields.status; 501 updated = !!(status & E1000_TXD_STAT_DD); 502 503 if (!updated) 504 return (0); 505 506 /* If clear is false just let caller know that there 507 * are descriptors to reclaim */ 508 if (!clear) 509 return (1); 510 511 prev = txr->tx_cidx_processed; 512 ntxd = scctx->isc_ntxd[0]; 513 do { 514 MPASS(prev != cur); 515 delta = (int32_t)cur - (int32_t)prev; 516 if (delta < 0) 517 delta += ntxd; 518 MPASS(delta > 0); 519 DPRINTF(iflib_get_dev(sc->ctx), 520 "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n", 521 __FUNCTION__, prev, cur, clear, delta); 522 523 processed += delta; 524 prev = cur; 525 rs_cidx = (rs_cidx + 1) & (ntxd-1); 526 if (rs_cidx == txr->tx_rs_pidx) 527 break; 528 cur = txr->tx_rsq[rs_cidx]; 529 MPASS(cur != QIDX_INVALID); 530 status = txr->tx_base[cur].upper.fields.status; 531 } while ((status & E1000_TXD_STAT_DD)); 532 533 txr->tx_rs_cidx = rs_cidx; 534 txr->tx_cidx_processed = prev; 535 return(processed); 536 } 537 538 static void 539 lem_isc_rxd_refill(void *arg, if_rxd_update_t iru) 540 { 541 struct e1000_softc *sc = arg; 542 if_softc_ctx_t scctx = sc->shared; 543 struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; 544 struct rx_ring *rxr = &que->rxr; 545 struct e1000_rx_desc *rxd; 546 uint64_t *paddrs; 547 uint32_t next_pidx, pidx; 548 uint16_t count; 549 int i; 550 551 paddrs = iru->iru_paddrs; 552 pidx = iru->iru_pidx; 553 count = iru->iru_count; 554 555 for (i = 0, next_pidx = pidx; i < count; i++) { 556 rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; 557 rxd->buffer_addr = htole64(paddrs[i]); 558 /* status bits must be cleared */ 559 rxd->status = 0; 560 561 if (++next_pidx == scctx->isc_nrxd[0]) 562 next_pidx = 0; 563 } 564 } 565 566 static void 567 em_isc_rxd_refill(void *arg, if_rxd_update_t iru) 568 { 569 struct e1000_softc *sc = arg; 570 if_softc_ctx_t scctx = sc->shared; 571 uint16_t rxqid = iru->iru_qsidx; 572 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 573 struct rx_ring *rxr = &que->rxr; 574 union e1000_rx_desc_extended *rxd; 575 uint64_t *paddrs; 576 uint32_t next_pidx, pidx; 577 uint16_t count; 578 int i; 579 580 paddrs = iru->iru_paddrs; 581 pidx = iru->iru_pidx; 582 count = iru->iru_count; 583 584 for (i = 0, next_pidx = pidx; i < count; i++) { 585 rxd = &rxr->rx_base[next_pidx]; 586 rxd->read.buffer_addr = htole64(paddrs[i]); 587 /* DD bits must be cleared */ 588 rxd->wb.upper.status_error = 0; 589 590 if (++next_pidx == scctx->isc_nrxd[0]) 591 next_pidx = 0; 592 } 593 } 594 595 static void 596 em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, 597 qidx_t pidx) 598 { 599 struct e1000_softc *sc = arg; 600 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 601 struct rx_ring *rxr = &que->rxr; 602 603 E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); 604 } 605 606 static int 607 lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 608 { 609 struct e1000_softc *sc = arg; 610 if_softc_ctx_t scctx = sc->shared; 611 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 612 struct rx_ring *rxr = &que->rxr; 613 struct e1000_rx_desc *rxd; 614 uint32_t staterr = 0; 615 int cnt, i; 616 617 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 618 rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; 619 staterr = rxd->status; 620 621 if ((staterr & E1000_RXD_STAT_DD) == 0) 622 break; 623 if (++i == scctx->isc_nrxd[0]) 624 i = 0; 625 if (staterr & E1000_RXD_STAT_EOP) 626 cnt++; 627 } 628 return (cnt); 629 } 630 631 static int 632 em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 633 { 634 struct e1000_softc *sc = arg; 635 if_softc_ctx_t scctx = sc->shared; 636 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 637 struct rx_ring *rxr = &que->rxr; 638 union e1000_rx_desc_extended *rxd; 639 uint32_t staterr = 0; 640 int cnt, i; 641 642 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 643 rxd = &rxr->rx_base[i]; 644 staterr = le32toh(rxd->wb.upper.status_error); 645 646 if ((staterr & E1000_RXD_STAT_DD) == 0) 647 break; 648 if (++i == scctx->isc_nrxd[0]) 649 i = 0; 650 if (staterr & E1000_RXD_STAT_EOP) 651 cnt++; 652 } 653 return (cnt); 654 } 655 656 static int 657 lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 658 { 659 struct e1000_softc *sc = arg; 660 if_softc_ctx_t scctx = sc->shared; 661 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx]; 662 struct rx_ring *rxr = &que->rxr; 663 struct e1000_rx_desc *rxd; 664 uint16_t len; 665 uint32_t status, errors; 666 bool eop; 667 int i, cidx; 668 669 status = errors = i = 0; 670 cidx = ri->iri_cidx; 671 672 do { 673 rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; 674 status = rxd->status; 675 errors = rxd->errors; 676 677 /* Error Checking then decrement count */ 678 MPASS ((status & E1000_RXD_STAT_DD) != 0); 679 680 len = le16toh(rxd->length); 681 ri->iri_len += len; 682 rxr->rx_bytes += ri->iri_len; 683 684 eop = (status & E1000_RXD_STAT_EOP) != 0; 685 686 /* Make sure bad packets are discarded */ 687 if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 688 sc->dropped_pkts++; 689 /* XXX fixup if common */ 690 return (EBADMSG); 691 } 692 693 ri->iri_frags[i].irf_flid = 0; 694 ri->iri_frags[i].irf_idx = cidx; 695 ri->iri_frags[i].irf_len = len; 696 /* Zero out the receive descriptors status. */ 697 rxd->status = 0; 698 699 if (++cidx == scctx->isc_nrxd[0]) 700 cidx = 0; 701 i++; 702 } while (!eop); 703 704 rxr->rx_packets++; 705 706 if (scctx->isc_capenable & IFCAP_RXCSUM) 707 em_receive_checksum(status, errors, ri); 708 709 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING && 710 status & E1000_RXD_STAT_VP) { 711 ri->iri_vtag = le16toh(rxd->special); 712 ri->iri_flags |= M_VLANTAG; 713 } 714 715 ri->iri_nfrags = i; 716 717 return (0); 718 } 719 720 static int 721 em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 722 { 723 struct e1000_softc *sc = arg; 724 if_softc_ctx_t scctx = sc->shared; 725 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx]; 726 struct rx_ring *rxr = &que->rxr; 727 union e1000_rx_desc_extended *rxd; 728 729 uint16_t len; 730 uint32_t pkt_info; 731 uint32_t staterr; 732 bool eop; 733 int i, cidx; 734 735 staterr = i = 0; 736 cidx = ri->iri_cidx; 737 738 do { 739 rxd = &rxr->rx_base[cidx]; 740 staterr = le32toh(rxd->wb.upper.status_error); 741 pkt_info = le32toh(rxd->wb.lower.mrq); 742 743 /* Error Checking then decrement count */ 744 MPASS ((staterr & E1000_RXD_STAT_DD) != 0); 745 746 len = le16toh(rxd->wb.upper.length); 747 ri->iri_len += len; 748 rxr->rx_bytes += ri->iri_len; 749 750 eop = (staterr & E1000_RXD_STAT_EOP) != 0; 751 752 /* Make sure bad packets are discarded */ 753 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { 754 sc->dropped_pkts++; 755 return EBADMSG; 756 } 757 758 ri->iri_frags[i].irf_flid = 0; 759 ri->iri_frags[i].irf_idx = cidx; 760 ri->iri_frags[i].irf_len = len; 761 /* Zero out the receive descriptors status. */ 762 rxd->wb.upper.status_error &= htole32(~0xFF); 763 764 if (++cidx == scctx->isc_nrxd[0]) 765 cidx = 0; 766 i++; 767 } while (!eop); 768 769 rxr->rx_packets++; 770 771 if (scctx->isc_capenable & IFCAP_RXCSUM) 772 em_receive_checksum(staterr, staterr >> 24, ri); 773 774 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING && 775 staterr & E1000_RXD_STAT_VP) { 776 ri->iri_vtag = le16toh(rxd->wb.upper.vlan); 777 ri->iri_flags |= M_VLANTAG; 778 } 779 780 ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); 781 ri->iri_rsstype = em_determine_rsstype(pkt_info); 782 783 ri->iri_nfrags = i; 784 return (0); 785 } 786 787 /********************************************************************* 788 * 789 * Verify that the hardware indicated that the checksum is valid. 790 * Inform the stack about the status of checksum so that stack 791 * doesn't spend time verifying the checksum. 792 * 793 *********************************************************************/ 794 static void 795 em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri) 796 { 797 if (__predict_false(status & E1000_RXD_STAT_IXSM)) 798 return; 799 800 /* If there is a layer 3 or 4 error we are done */ 801 if (__predict_false(errors & (E1000_RXD_ERR_IPE | 802 E1000_RXD_ERR_TCPE))) 803 return; 804 805 /* IP Checksum Good */ 806 if (status & E1000_RXD_STAT_IPCS) 807 ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); 808 809 /* Valid L4E checksum */ 810 if (__predict_true(status & 811 (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) { 812 ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 813 ri->iri_csum_data = htons(0xffff); 814 } 815 } 816 817 /******************************************************************** 818 * 819 * Parse the packet type to determine the appropriate hash 820 * 821 ******************************************************************/ 822 static int 823 em_determine_rsstype(uint32_t pkt_info) 824 { 825 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { 826 case E1000_RXDADV_RSSTYPE_IPV4_TCP: 827 return M_HASHTYPE_RSS_TCP_IPV4; 828 case E1000_RXDADV_RSSTYPE_IPV4: 829 return M_HASHTYPE_RSS_IPV4; 830 case E1000_RXDADV_RSSTYPE_IPV6_TCP: 831 return M_HASHTYPE_RSS_TCP_IPV6; 832 case E1000_RXDADV_RSSTYPE_IPV6_EX: 833 return M_HASHTYPE_RSS_IPV6_EX; 834 case E1000_RXDADV_RSSTYPE_IPV6: 835 return M_HASHTYPE_RSS_IPV6; 836 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: 837 return M_HASHTYPE_RSS_TCP_IPV6_EX; 838 default: 839 return M_HASHTYPE_OPAQUE; 840 } 841 } 842