1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 5 * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "if_em.h" 31 32 #include <net/rss_config.h> 33 #include <netinet/in_rss.h> 34 35 #ifdef VERBOSE_DEBUG 36 #define DPRINTF device_printf 37 #else 38 #define DPRINTF(...) 39 #endif 40 41 /********************************************************************* 42 * Local Function prototypes 43 *********************************************************************/ 44 static int em_tso_setup(struct e1000_softc *, if_pkt_info_t, uint32_t *, 45 uint32_t *); 46 static int em_transmit_checksum_setup(struct e1000_softc *, if_pkt_info_t, 47 uint32_t *, uint32_t *); 48 static int em_isc_txd_encap(void *, if_pkt_info_t); 49 static void em_isc_txd_flush(void *, uint16_t, qidx_t); 50 static int em_isc_txd_credits_update(void *, uint16_t, bool); 51 static void em_isc_rxd_refill(void *, if_rxd_update_t); 52 static void em_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t); 53 static int em_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t); 54 static int em_isc_rxd_pkt_get(void *, if_rxd_info_t); 55 56 static void lem_isc_rxd_refill(void *, if_rxd_update_t); 57 58 static int lem_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t); 59 static int lem_isc_rxd_pkt_get(void *, if_rxd_info_t); 60 61 static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t); 62 static int em_determine_rsstype(uint32_t); 63 extern int em_intr(void *); 64 65 struct if_txrx em_txrx = { 66 .ift_txd_encap = em_isc_txd_encap, 67 .ift_txd_flush = em_isc_txd_flush, 68 .ift_txd_credits_update = em_isc_txd_credits_update, 69 .ift_rxd_available = em_isc_rxd_available, 70 .ift_rxd_pkt_get = em_isc_rxd_pkt_get, 71 .ift_rxd_refill = em_isc_rxd_refill, 72 .ift_rxd_flush = em_isc_rxd_flush, 73 .ift_legacy_intr = em_intr 74 }; 75 76 struct if_txrx lem_txrx = { 77 .ift_txd_encap = em_isc_txd_encap, 78 .ift_txd_flush = em_isc_txd_flush, 79 .ift_txd_credits_update = em_isc_txd_credits_update, 80 .ift_rxd_available = lem_isc_rxd_available, 81 .ift_rxd_pkt_get = lem_isc_rxd_pkt_get, 82 .ift_rxd_refill = lem_isc_rxd_refill, 83 .ift_rxd_flush = em_isc_rxd_flush, 84 .ift_legacy_intr = em_intr 85 }; 86 87 extern if_shared_ctx_t em_sctx; 88 89 void 90 em_dump_rs(struct e1000_softc *sc) 91 { 92 if_softc_ctx_t scctx = sc->shared; 93 struct em_tx_queue *que; 94 struct tx_ring *txr; 95 qidx_t i, ntxd, qid, cur; 96 int16_t rs_cidx; 97 uint8_t status; 98 99 printf("\n"); 100 ntxd = scctx->isc_ntxd[0]; 101 for (qid = 0; qid < sc->tx_num_queues; qid++) { 102 que = &sc->tx_queues[qid]; 103 txr = &que->txr; 104 rs_cidx = txr->tx_rs_cidx; 105 if (rs_cidx != txr->tx_rs_pidx) { 106 cur = txr->tx_rsq[rs_cidx]; 107 status = txr->tx_base[cur].upper.fields.status; 108 if (!(status & E1000_TXD_STAT_DD)) 109 printf("qid[%d]->tx_rsq[%d]: %d clear ", 110 qid, rs_cidx, cur); 111 } else { 112 rs_cidx = (rs_cidx-1)&(ntxd-1); 113 cur = txr->tx_rsq[rs_cidx]; 114 printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", 115 qid, rs_cidx, cur); 116 } 117 printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, 118 txr->tx_rs_pidx); 119 for (i = 0; i < ntxd; i++) { 120 if (txr->tx_base[i].upper.fields.status & 121 E1000_TXD_STAT_DD) 122 printf("%d set ", i); 123 } 124 printf("\n"); 125 } 126 } 127 128 /********************************************************************** 129 * 130 * Setup work for hardware segmentation offload (TSO) on 131 * adapters using advanced tx descriptors 132 * 133 **********************************************************************/ 134 static int 135 em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, uint32_t *txd_upper, 136 uint32_t *txd_lower) 137 { 138 if_softc_ctx_t scctx = sc->shared; 139 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 140 struct tx_ring *txr = &que->txr; 141 struct e1000_context_desc *TXD; 142 int cur, hdr_len; 143 uint32_t cmd_type_len; 144 145 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 146 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 147 E1000_TXD_DTYP_D | /* Data descr type */ 148 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 149 150 cur = pi->ipi_pidx; 151 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 152 153 /* 154 * ipcss - Start offset for header checksum calculation. 155 * ipcse - End offset for header checksum calculation. 156 * ipcso - Offset of place to put the checksum. 157 */ 158 switch(pi->ipi_etype) { 159 case ETHERTYPE_IP: 160 /* IP and/or TCP header checksum calculation and insertion. */ 161 *txd_upper = 162 (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 163 164 TXD->lower_setup.ip_fields.ipcse = 165 htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); 166 break; 167 case ETHERTYPE_IPV6: 168 /* TCP header checksum calculation and insertion. */ 169 *txd_upper = E1000_TXD_POPTS_TXSM << 8; 170 171 TXD->lower_setup.ip_fields.ipcse = htole16(0); 172 break; 173 default: 174 break; 175 } 176 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 177 TXD->lower_setup.ip_fields.ipcso = 178 pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 179 180 /* 181 * tucss - Start offset for payload checksum calculation. 182 * tucse - End offset for payload checksum calculation. 183 * tucso - Offset of place to put the checksum. 184 */ 185 TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 186 TXD->upper_setup.tcp_fields.tucse = 0; 187 TXD->upper_setup.tcp_fields.tucso = 188 pi->ipi_ehdrlen + pi->ipi_ip_hlen + 189 offsetof(struct tcphdr, th_sum); 190 191 /* 192 * Payload size per packet w/o any headers. 193 * Length of all headers up to payload. 194 */ 195 TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); 196 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 197 198 /* 199 * "PCI/PCI-X SDM 4.0" page 45, and "PCIe GbE SDM 2.5" page 63 200 * - Set up basic TUCMDs 201 * - For others IP bit on indicates IPv4, while off indicates IPv6 202 */ 203 cmd_type_len = sc->txd_cmd | 204 E1000_TXD_CMD_DEXT | /* Extended descr */ 205 E1000_TXD_CMD_TSE | /* TSE context */ 206 E1000_TXD_CMD_TCP; /* Do TCP checksum */ 207 if (pi->ipi_etype == ETHERTYPE_IP) 208 cmd_type_len |= E1000_TXD_CMD_IP; 209 TXD->cmd_and_length = htole32(cmd_type_len | 210 (pi->ipi_len - hdr_len)); /* Total len */ 211 212 txr->tx_tso = true; 213 214 if (++cur == scctx->isc_ntxd[0]) { 215 cur = 0; 216 } 217 DPRINTF(iflib_get_dev(sc->ctx), "%s: pidx: %d cur: %d\n", 218 __FUNCTION__, pi->ipi_pidx, cur); 219 return (cur); 220 } 221 222 /********************************************************************* 223 * The offload context is protocol specific (TCP/UDP) and thus 224 * only needs to be set when the protocol changes. The occasion 225 * of a context change can be a performance detriment, and 226 * might be better just disabled. The reason arises in the way 227 * in which the controller supports pipelined requests from the 228 * Tx data DMA. Up to four requests can be pipelined, and they may 229 * belong to the same packet or to multiple packets. However all 230 * requests for one packet are issued before a request is issued 231 * for a subsequent packet and if a request for the next packet 232 * requires a context change, that request will be stalled 233 * until the previous request completes. This means setting up 234 * a new context effectively disables pipelined Tx data DMA which 235 * in turn greatly slow down performance to send small sized 236 * frames. 237 **********************************************************************/ 238 #define DONT_FORCE_CTX 1 239 240 static int 241 em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi, 242 uint32_t *txd_upper, uint32_t *txd_lower) 243 { 244 struct e1000_context_desc *TXD = NULL; 245 if_softc_ctx_t scctx = sc->shared; 246 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 247 struct tx_ring *txr = &que->txr; 248 int csum_flags = pi->ipi_csum_flags; 249 int cur, hdr_len; 250 uint32_t cmd; 251 252 cur = pi->ipi_pidx; 253 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 254 cmd = sc->txd_cmd; 255 256 /* 257 * The 82574L can only remember the *last* context used 258 * regardless of queue that it was use for. We cannot reuse 259 * contexts on this hardware platform and must generate a new 260 * context every time. 82574L hardware spec, section 7.2.6, 261 * second note. 262 */ 263 if (DONT_FORCE_CTX && 264 sc->tx_num_queues == 1 && 265 txr->csum_lhlen == pi->ipi_ehdrlen && 266 txr->csum_iphlen == pi->ipi_ip_hlen && 267 txr->csum_flags == csum_flags) { 268 /* 269 * Same csum offload context as the previous packets; 270 * just return. 271 */ 272 *txd_upper = txr->csum_txd_upper; 273 *txd_lower = txr->csum_txd_lower; 274 return (cur); 275 } 276 277 TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 278 /* 279 * ipcss - Start offset for header checksum calculation. 280 * ipcse - End offset for header checksum calculation. 281 * ipcso - Offset of place to put the checksum. 282 * 283 * We set ipcsX values regardless of IP version to work around HW 284 * issues and ipcse must be 0 for IPv6 per "PCIe GbE SDM 2.5" page 61. 285 * IXSM controls whether it's inserted. 286 */ 287 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 288 TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + 289 offsetof(struct ip, ip_sum); 290 if (csum_flags & CSUM_IP) { 291 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 292 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len - 1); 293 cmd |= E1000_TXD_CMD_IP; 294 } else if (csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP)) 295 TXD->lower_setup.ip_fields.ipcse = htole16(0); 296 297 /* 298 * tucss - Start offset for payload checksum calculation. 299 * tucse - End offset for payload checksum calculation. 300 * tucso - Offset of place to put the checksum. 301 */ 302 if (csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_IP6_TCP | 303 CSUM_IP6_UDP)) { 304 uint8_t tucso; 305 306 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 307 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 308 309 if (csum_flags & (CSUM_TCP | CSUM_IP6_TCP)) { 310 tucso = hdr_len + offsetof(struct tcphdr, th_sum); 311 cmd |= E1000_TXD_CMD_TCP; 312 } else 313 tucso = hdr_len + offsetof(struct udphdr, uh_sum); 314 TXD->upper_setup.tcp_fields.tucss = hdr_len; 315 TXD->upper_setup.tcp_fields.tucse = htole16(0); 316 TXD->upper_setup.tcp_fields.tucso = tucso; 317 } 318 319 txr->csum_lhlen = pi->ipi_ehdrlen; 320 txr->csum_iphlen = pi->ipi_ip_hlen; 321 txr->csum_flags = csum_flags; 322 txr->csum_txd_upper = *txd_upper; 323 txr->csum_txd_lower = *txd_lower; 324 325 TXD->tcp_seg_setup.data = htole32(0); 326 TXD->cmd_and_length = 327 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 328 329 if (++cur == scctx->isc_ntxd[0]) { 330 cur = 0; 331 } 332 DPRINTF(iflib_get_dev(sc->ctx), 333 "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x" 334 " hdr_len=%d cmd=%x\n", 335 csum_flags, *txd_upper, *txd_lower, hdr_len, cmd); 336 return (cur); 337 } 338 339 #define TSO_WORKAROUND 4 /* TSO sentinel descriptor */ 340 341 static int 342 em_isc_txd_encap(void *arg, if_pkt_info_t pi) 343 { 344 struct e1000_softc *sc = arg; 345 if_softc_ctx_t scctx = sc->shared; 346 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 347 struct tx_ring *txr = &que->txr; 348 bus_dma_segment_t *segs = pi->ipi_segs; 349 int nsegs = pi->ipi_nsegs; 350 int csum_flags = pi->ipi_csum_flags; 351 int i, j, first, pidx_last; 352 uint32_t txd_flags, txd_upper = 0, txd_lower = 0; 353 354 struct e1000_tx_desc *ctxd = NULL; 355 bool do_tso, tso_desc; 356 qidx_t ntxd; 357 358 txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; 359 i = first = pi->ipi_pidx; 360 do_tso = (csum_flags & CSUM_TSO); 361 tso_desc = false; 362 ntxd = scctx->isc_ntxd[0]; 363 /* 364 * TSO Hardware workaround, if this packet is not 365 * TSO, and is only a single descriptor long, and 366 * it follows a TSO burst, then we need to add a 367 * sentinel descriptor to prevent premature writeback. 368 */ 369 if ((!do_tso) && (txr->tx_tso == true)) { 370 if (nsegs == 1) 371 tso_desc = true; 372 txr->tx_tso = false; 373 } 374 375 /* Do hardware assists */ 376 if (do_tso) { 377 i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); 378 tso_desc = true; 379 } else if (csum_flags & EM_CSUM_OFFLOAD) { 380 i = em_transmit_checksum_setup(sc, pi, &txd_upper, 381 &txd_lower); 382 } 383 384 if (pi->ipi_mflags & M_VLANTAG) { 385 /* Set the vlan id. */ 386 txd_upper |= htole16(pi->ipi_vtag) << 16; 387 /* Tell hardware to add tag */ 388 txd_lower |= htole32(E1000_TXD_CMD_VLE); 389 } 390 391 DPRINTF(iflib_get_dev(sc->ctx), 392 "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); 393 /* XXX sc->pcix_82544 -- lem_fill_descriptors */ 394 395 /* Set up our transmit descriptors */ 396 for (j = 0; j < nsegs; j++) { 397 bus_size_t seg_len; 398 bus_addr_t seg_addr; 399 uint32_t cmd; 400 401 ctxd = &txr->tx_base[i]; 402 seg_addr = segs[j].ds_addr; 403 seg_len = segs[j].ds_len; 404 cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; 405 406 /* 407 * TSO Workaround: 408 * If this is the last descriptor, we want to 409 * split it so we have a small final sentinel 410 */ 411 if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { 412 seg_len -= TSO_WORKAROUND; 413 ctxd->buffer_addr = htole64(seg_addr); 414 ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 415 ctxd->upper.data = htole32(txd_upper); 416 417 if (++i == scctx->isc_ntxd[0]) 418 i = 0; 419 420 /* Now make the sentinel */ 421 ctxd = &txr->tx_base[i]; 422 ctxd->buffer_addr = htole64(seg_addr + seg_len); 423 ctxd->lower.data = 424 htole32(cmd | txd_lower | TSO_WORKAROUND); 425 ctxd->upper.data = htole32(txd_upper); 426 pidx_last = i; 427 if (++i == scctx->isc_ntxd[0]) 428 i = 0; 429 DPRINTF(iflib_get_dev(sc->ctx), 430 "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", 431 pidx_last, i, scctx->isc_ntxd[0]); 432 } else { 433 ctxd->buffer_addr = htole64(seg_addr); 434 ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 435 ctxd->upper.data = htole32(txd_upper); 436 pidx_last = i; 437 if (++i == scctx->isc_ntxd[0]) 438 i = 0; 439 DPRINTF(iflib_get_dev(sc->ctx), 440 "pidx_last=%d i=%d ntxd[0]=%d\n", 441 pidx_last, i, scctx->isc_ntxd[0]); 442 } 443 } 444 445 /* 446 * Last Descriptor of Packet 447 * needs End Of Packet (EOP) 448 * and Report Status (RS) 449 */ 450 if (txd_flags && nsegs) { 451 txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; 452 DPRINTF(iflib_get_dev(sc->ctx), 453 "setting to RS on %d rs_pidx %d first: %d\n", 454 pidx_last, txr->tx_rs_pidx, first); 455 txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); 456 MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); 457 } 458 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags); 459 DPRINTF(iflib_get_dev(sc->ctx), 460 "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", 461 first, pidx_last, i); 462 pi->ipi_new_pidx = i; 463 464 /* 465 * Sent data accounting for AIM. For TSO, ipi_len is the whole 466 * unsegmented payload, which is not a size the moderation 467 * calculation can use. Count the segments the hardware will put on 468 * the wire and the header each of them carries, so that the average 469 * it sees is a wire packet. 470 */ 471 if (do_tso && pi->ipi_tso_segsz != 0) { 472 u32 hdrlen, segs; 473 474 hdrlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 475 if (pi->ipi_len > hdrlen) { 476 segs = howmany(pi->ipi_len - hdrlen, pi->ipi_tso_segsz); 477 txr->tx_bytes += pi->ipi_len + (segs - 1) * hdrlen; 478 txr->tx_packets += segs; 479 return (0); 480 } 481 } 482 483 txr->tx_bytes += pi->ipi_len; 484 ++txr->tx_packets; 485 return (0); 486 } 487 488 static void 489 em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 490 { 491 struct e1000_softc *sc = arg; 492 struct em_tx_queue *que = &sc->tx_queues[txqid]; 493 struct tx_ring *txr = &que->txr; 494 495 E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx); 496 if (sc->hw.mac.type >= e1000_82540) 497 em_aim_publish(txr); 498 } 499 500 static int 501 em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) 502 { 503 struct e1000_softc *sc = arg; 504 if_softc_ctx_t scctx = sc->shared; 505 struct em_tx_queue *que = &sc->tx_queues[txqid]; 506 struct tx_ring *txr = &que->txr; 507 508 qidx_t processed = 0; 509 int updated; 510 qidx_t cur, prev, ntxd, rs_cidx; 511 int32_t delta; 512 uint8_t status; 513 514 rs_cidx = txr->tx_rs_cidx; 515 if (rs_cidx == txr->tx_rs_pidx) 516 return (0); 517 cur = txr->tx_rsq[rs_cidx]; 518 MPASS(cur != QIDX_INVALID); 519 status = txr->tx_base[cur].upper.fields.status; 520 updated = !!(status & E1000_TXD_STAT_DD); 521 522 if (!updated) 523 return (0); 524 525 /* If clear is false just let caller know that there 526 * are descriptors to reclaim */ 527 if (!clear) 528 return (1); 529 530 prev = txr->tx_cidx_processed; 531 ntxd = scctx->isc_ntxd[0]; 532 do { 533 MPASS(prev != cur); 534 delta = (int32_t)cur - (int32_t)prev; 535 if (delta < 0) 536 delta += ntxd; 537 MPASS(delta > 0); 538 DPRINTF(iflib_get_dev(sc->ctx), 539 "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n", 540 __FUNCTION__, prev, cur, clear, delta); 541 542 processed += delta; 543 prev = cur; 544 rs_cidx = (rs_cidx + 1) & (ntxd-1); 545 if (rs_cidx == txr->tx_rs_pidx) 546 break; 547 cur = txr->tx_rsq[rs_cidx]; 548 MPASS(cur != QIDX_INVALID); 549 status = txr->tx_base[cur].upper.fields.status; 550 } while ((status & E1000_TXD_STAT_DD)); 551 552 txr->tx_rs_cidx = rs_cidx; 553 txr->tx_cidx_processed = prev; 554 return(processed); 555 } 556 557 static void 558 lem_isc_rxd_refill(void *arg, if_rxd_update_t iru) 559 { 560 struct e1000_softc *sc = arg; 561 if_softc_ctx_t scctx = sc->shared; 562 struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; 563 struct rx_ring *rxr = &que->rxr; 564 struct e1000_rx_desc *rxd; 565 uint64_t *paddrs; 566 uint32_t next_pidx, pidx; 567 uint16_t count; 568 int i; 569 570 paddrs = iru->iru_paddrs; 571 pidx = iru->iru_pidx; 572 count = iru->iru_count; 573 574 for (i = 0, next_pidx = pidx; i < count; i++) { 575 rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; 576 rxd->buffer_addr = htole64(paddrs[i]); 577 /* status bits must be cleared */ 578 rxd->status = 0; 579 580 if (++next_pidx == scctx->isc_nrxd[0]) 581 next_pidx = 0; 582 } 583 } 584 585 static void 586 em_isc_rxd_refill(void *arg, if_rxd_update_t iru) 587 { 588 struct e1000_softc *sc = arg; 589 if_softc_ctx_t scctx = sc->shared; 590 uint16_t rxqid = iru->iru_qsidx; 591 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 592 struct rx_ring *rxr = &que->rxr; 593 union e1000_rx_desc_extended *rxd; 594 uint64_t *paddrs; 595 uint32_t next_pidx, pidx; 596 uint16_t count; 597 int i; 598 599 paddrs = iru->iru_paddrs; 600 pidx = iru->iru_pidx; 601 count = iru->iru_count; 602 603 for (i = 0, next_pidx = pidx; i < count; i++) { 604 rxd = &rxr->rx_base[next_pidx]; 605 rxd->read.buffer_addr = htole64(paddrs[i]); 606 /* DD bits must be cleared */ 607 rxd->wb.upper.status_error = 0; 608 609 if (++next_pidx == scctx->isc_nrxd[0]) 610 next_pidx = 0; 611 } 612 } 613 614 static void 615 em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, 616 qidx_t pidx) 617 { 618 struct e1000_softc *sc = arg; 619 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 620 struct rx_ring *rxr = &que->rxr; 621 622 E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); 623 if (sc->hw.mac.type >= e1000_82540) 624 em_aim_publish_rx(rxr); 625 } 626 627 static int 628 lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 629 { 630 struct e1000_softc *sc = arg; 631 if_softc_ctx_t scctx = sc->shared; 632 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 633 struct rx_ring *rxr = &que->rxr; 634 struct e1000_rx_desc *rxd; 635 uint32_t staterr = 0; 636 int cnt, i; 637 638 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 639 rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; 640 staterr = rxd->status; 641 642 if ((staterr & E1000_RXD_STAT_DD) == 0) 643 break; 644 if (++i == scctx->isc_nrxd[0]) 645 i = 0; 646 if (staterr & E1000_RXD_STAT_EOP) 647 cnt++; 648 } 649 return (cnt); 650 } 651 652 static int 653 em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 654 { 655 struct e1000_softc *sc = arg; 656 if_softc_ctx_t scctx = sc->shared; 657 struct em_rx_queue *que = &sc->rx_queues[rxqid]; 658 struct rx_ring *rxr = &que->rxr; 659 union e1000_rx_desc_extended *rxd; 660 uint32_t staterr = 0; 661 int cnt, i; 662 663 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 664 rxd = &rxr->rx_base[i]; 665 staterr = le32toh(rxd->wb.upper.status_error); 666 667 if ((staterr & E1000_RXD_STAT_DD) == 0) 668 break; 669 if (++i == scctx->isc_nrxd[0]) 670 i = 0; 671 if (staterr & E1000_RXD_STAT_EOP) 672 cnt++; 673 } 674 return (cnt); 675 } 676 677 static int 678 lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 679 { 680 struct e1000_softc *sc = arg; 681 if_softc_ctx_t scctx = sc->shared; 682 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx]; 683 struct rx_ring *rxr = &que->rxr; 684 struct e1000_rx_desc *rxd; 685 uint16_t len; 686 uint32_t status, errors; 687 bool eop; 688 int i, cidx; 689 690 status = errors = i = 0; 691 cidx = ri->iri_cidx; 692 693 do { 694 rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; 695 status = rxd->status; 696 errors = rxd->errors; 697 698 /* Error Checking then decrement count */ 699 MPASS ((status & E1000_RXD_STAT_DD) != 0); 700 701 len = le16toh(rxd->length); 702 ri->iri_len += len; 703 704 eop = (status & E1000_RXD_STAT_EOP) != 0; 705 706 /* Make sure bad packets are discarded */ 707 if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 708 sc->dropped_pkts++; 709 /* XXX fixup if common */ 710 return (EBADMSG); 711 } 712 713 ri->iri_frags[i].irf_flid = 0; 714 ri->iri_frags[i].irf_idx = cidx; 715 ri->iri_frags[i].irf_len = len; 716 /* Zero out the receive descriptors status. */ 717 rxd->status = 0; 718 719 if (++cidx == scctx->isc_nrxd[0]) 720 cidx = 0; 721 i++; 722 } while (!eop); 723 724 rxr->rx_bytes += ri->iri_len; 725 rxr->rx_packets++; 726 727 if (scctx->isc_capenable & IFCAP_RXCSUM) 728 em_receive_checksum(status, errors, ri); 729 730 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING && 731 status & E1000_RXD_STAT_VP) { 732 ri->iri_vtag = le16toh(rxd->special); 733 ri->iri_flags |= M_VLANTAG; 734 } 735 736 ri->iri_nfrags = i; 737 738 return (0); 739 } 740 741 static int 742 em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 743 { 744 struct e1000_softc *sc = arg; 745 if_softc_ctx_t scctx = sc->shared; 746 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx]; 747 struct rx_ring *rxr = &que->rxr; 748 union e1000_rx_desc_extended *rxd; 749 750 uint16_t len; 751 uint32_t pkt_info; 752 uint32_t staterr; 753 bool eop; 754 int i, cidx; 755 756 staterr = i = 0; 757 cidx = ri->iri_cidx; 758 759 do { 760 rxd = &rxr->rx_base[cidx]; 761 staterr = le32toh(rxd->wb.upper.status_error); 762 pkt_info = le32toh(rxd->wb.lower.mrq); 763 764 /* Error Checking then decrement count */ 765 MPASS ((staterr & E1000_RXD_STAT_DD) != 0); 766 767 len = le16toh(rxd->wb.upper.length); 768 ri->iri_len += len; 769 770 eop = (staterr & E1000_RXD_STAT_EOP) != 0; 771 772 /* Make sure bad packets are discarded */ 773 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { 774 sc->dropped_pkts++; 775 return EBADMSG; 776 } 777 778 ri->iri_frags[i].irf_flid = 0; 779 ri->iri_frags[i].irf_idx = cidx; 780 ri->iri_frags[i].irf_len = len; 781 /* Zero out the receive descriptors status. */ 782 rxd->wb.upper.status_error &= htole32(~0xFF); 783 784 if (++cidx == scctx->isc_nrxd[0]) 785 cidx = 0; 786 i++; 787 } while (!eop); 788 789 rxr->rx_bytes += ri->iri_len; 790 rxr->rx_packets++; 791 792 if (scctx->isc_capenable & IFCAP_RXCSUM) 793 em_receive_checksum(staterr, staterr >> 24, ri); 794 795 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING && 796 staterr & E1000_RXD_STAT_VP) { 797 ri->iri_vtag = le16toh(rxd->wb.upper.vlan); 798 ri->iri_flags |= M_VLANTAG; 799 } 800 801 ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); 802 ri->iri_rsstype = em_determine_rsstype(pkt_info); 803 804 ri->iri_nfrags = i; 805 return (0); 806 } 807 808 /********************************************************************* 809 * 810 * Verify that the hardware indicated that the checksum is valid. 811 * Inform the stack about the status of checksum so that stack 812 * doesn't spend time verifying the checksum. 813 * 814 *********************************************************************/ 815 static void 816 em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri) 817 { 818 if (__predict_false(status & E1000_RXD_STAT_IXSM)) 819 return; 820 821 /* If there is a layer 3 or 4 error we are done */ 822 if (__predict_false(errors & (E1000_RXD_ERR_IPE | 823 E1000_RXD_ERR_TCPE))) 824 return; 825 826 /* IP Checksum Good */ 827 if (status & E1000_RXD_STAT_IPCS) 828 ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); 829 830 /* Valid L4E checksum */ 831 if (__predict_true(status & 832 (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) { 833 ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 834 ri->iri_csum_data = htons(0xffff); 835 } 836 } 837 838 /******************************************************************** 839 * 840 * Parse the packet type to determine the appropriate hash 841 * 842 ******************************************************************/ 843 static int 844 em_determine_rsstype(uint32_t pkt_info) 845 { 846 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { 847 case E1000_RXDADV_RSSTYPE_IPV4_TCP: 848 return M_HASHTYPE_RSS_TCP_IPV4; 849 case E1000_RXDADV_RSSTYPE_IPV4: 850 return M_HASHTYPE_RSS_IPV4; 851 case E1000_RXDADV_RSSTYPE_IPV6_TCP: 852 return M_HASHTYPE_RSS_TCP_IPV6; 853 case E1000_RXDADV_RSSTYPE_IPV6_EX: 854 return M_HASHTYPE_RSS_IPV6_EX; 855 case E1000_RXDADV_RSSTYPE_IPV6: 856 return M_HASHTYPE_RSS_IPV6; 857 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: 858 return M_HASHTYPE_RSS_TCP_IPV6_EX; 859 default: 860 return M_HASHTYPE_NONE; 861 } 862 } 863