1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org>
5 * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "if_em.h"
31
32 #include <net/rss_config.h>
33 #include <netinet/in_rss.h>
34
35 #ifdef VERBOSE_DEBUG
36 #define DPRINTF device_printf
37 #else
38 #define DPRINTF(...)
39 #endif
40
41 /*********************************************************************
42 * Local Function prototypes
43 *********************************************************************/
44 static int em_tso_setup(struct e1000_softc *, if_pkt_info_t, uint32_t *,
45 uint32_t *);
46 static int em_transmit_checksum_setup(struct e1000_softc *, if_pkt_info_t,
47 uint32_t *, uint32_t *);
48 static int em_isc_txd_encap(void *, if_pkt_info_t);
49 static void em_isc_txd_flush(void *, uint16_t, qidx_t);
50 static int em_isc_txd_credits_update(void *, uint16_t, bool);
51 static void em_isc_rxd_refill(void *, if_rxd_update_t);
52 static void em_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t);
53 static int em_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t);
54 static int em_isc_rxd_pkt_get(void *, if_rxd_info_t);
55
56 static void lem_isc_rxd_refill(void *, if_rxd_update_t);
57
58 static int lem_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t);
59 static int lem_isc_rxd_pkt_get(void *, if_rxd_info_t);
60
61 static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t);
62 static int em_determine_rsstype(uint32_t);
63 extern int em_intr(void *);
64
65 struct if_txrx em_txrx = {
66 .ift_txd_encap = em_isc_txd_encap,
67 .ift_txd_flush = em_isc_txd_flush,
68 .ift_txd_credits_update = em_isc_txd_credits_update,
69 .ift_rxd_available = em_isc_rxd_available,
70 .ift_rxd_pkt_get = em_isc_rxd_pkt_get,
71 .ift_rxd_refill = em_isc_rxd_refill,
72 .ift_rxd_flush = em_isc_rxd_flush,
73 .ift_legacy_intr = em_intr
74 };
75
76 struct if_txrx lem_txrx = {
77 .ift_txd_encap = em_isc_txd_encap,
78 .ift_txd_flush = em_isc_txd_flush,
79 .ift_txd_credits_update = em_isc_txd_credits_update,
80 .ift_rxd_available = lem_isc_rxd_available,
81 .ift_rxd_pkt_get = lem_isc_rxd_pkt_get,
82 .ift_rxd_refill = lem_isc_rxd_refill,
83 .ift_rxd_flush = em_isc_rxd_flush,
84 .ift_legacy_intr = em_intr
85 };
86
87 extern if_shared_ctx_t em_sctx;
88
89 void
em_dump_rs(struct e1000_softc * sc)90 em_dump_rs(struct e1000_softc *sc)
91 {
92 if_softc_ctx_t scctx = sc->shared;
93 struct em_tx_queue *que;
94 struct tx_ring *txr;
95 qidx_t i, ntxd, qid, cur;
96 int16_t rs_cidx;
97 uint8_t status;
98
99 if (sc->tx_queues == NULL) {
100 device_printf(sc->dev, "queue state is unavailable\n");
101 return;
102 }
103
104 printf("\n");
105 ntxd = scctx->isc_ntxd[0];
106 for (qid = 0; qid < sc->tx_num_queues; qid++) {
107 que = &sc->tx_queues[qid];
108 txr = &que->txr;
109 rs_cidx = txr->tx_rs_cidx;
110 if (rs_cidx != txr->tx_rs_pidx) {
111 cur = txr->tx_rsq[rs_cidx];
112 status = txr->tx_base[cur].upper.fields.status;
113 if (!(status & E1000_TXD_STAT_DD))
114 printf("qid[%d]->tx_rsq[%d]: %d clear ",
115 qid, rs_cidx, cur);
116 } else {
117 rs_cidx = (rs_cidx-1)&(ntxd-1);
118 cur = txr->tx_rsq[rs_cidx];
119 printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ",
120 qid, rs_cidx, cur);
121 }
122 printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed,
123 txr->tx_rs_pidx);
124 for (i = 0; i < ntxd; i++) {
125 if (txr->tx_base[i].upper.fields.status &
126 E1000_TXD_STAT_DD)
127 printf("%d set ", i);
128 }
129 printf("\n");
130 }
131 }
132
133 /**********************************************************************
134 *
135 * Setup work for hardware segmentation offload (TSO) on
136 * adapters using advanced tx descriptors
137 *
138 **********************************************************************/
139 static int
em_tso_setup(struct e1000_softc * sc,if_pkt_info_t pi,uint32_t * txd_upper,uint32_t * txd_lower)140 em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, uint32_t *txd_upper,
141 uint32_t *txd_lower)
142 {
143 if_softc_ctx_t scctx = sc->shared;
144 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
145 struct tx_ring *txr = &que->txr;
146 struct e1000_context_desc *TXD;
147 int cur, hdr_len;
148 uint32_t cmd_type_len;
149
150 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
151 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
152 E1000_TXD_DTYP_D | /* Data descr type */
153 E1000_TXD_CMD_TSE); /* Do TSE on this packet */
154
155 cur = pi->ipi_pidx;
156 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
157
158 /*
159 * ipcss - Start offset for header checksum calculation.
160 * ipcse - End offset for header checksum calculation.
161 * ipcso - Offset of place to put the checksum.
162 */
163 switch(pi->ipi_etype) {
164 case ETHERTYPE_IP:
165 /* IP and/or TCP header checksum calculation and insertion. */
166 *txd_upper =
167 (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
168
169 TXD->lower_setup.ip_fields.ipcse =
170 htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1);
171 break;
172 case ETHERTYPE_IPV6:
173 /* TCP header checksum calculation and insertion. */
174 *txd_upper = E1000_TXD_POPTS_TXSM << 8;
175
176 TXD->lower_setup.ip_fields.ipcse = htole16(0);
177 break;
178 default:
179 break;
180 }
181 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
182 TXD->lower_setup.ip_fields.ipcso =
183 pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);
184
185 /*
186 * tucss - Start offset for payload checksum calculation.
187 * tucse - End offset for payload checksum calculation.
188 * tucso - Offset of place to put the checksum.
189 */
190 TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
191 TXD->upper_setup.tcp_fields.tucse = 0;
192 TXD->upper_setup.tcp_fields.tucso =
193 pi->ipi_ehdrlen + pi->ipi_ip_hlen +
194 offsetof(struct tcphdr, th_sum);
195
196 /*
197 * Payload size per packet w/o any headers.
198 * Length of all headers up to payload.
199 */
200 TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz);
201 TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
202
203 /*
204 * "PCI/PCI-X SDM 4.0" page 45, and "PCIe GbE SDM 2.5" page 63
205 * - Set up basic TUCMDs
206 * - For others IP bit on indicates IPv4, while off indicates IPv6
207 */
208 cmd_type_len = sc->txd_cmd |
209 E1000_TXD_CMD_DEXT | /* Extended descr */
210 E1000_TXD_CMD_TSE | /* TSE context */
211 E1000_TXD_CMD_TCP; /* Do TCP checksum */
212 if (pi->ipi_etype == ETHERTYPE_IP)
213 cmd_type_len |= E1000_TXD_CMD_IP;
214 TXD->cmd_and_length = htole32(cmd_type_len |
215 (pi->ipi_len - hdr_len)); /* Total len */
216
217 /*
218 * 82540EP and 82545GM erratum 3 and 82546GB erratum 1 require
219 * a second descriptor for the final four bytes of an LSO packet.
220 */
221 txr->tx_tso = sc->hw.mac.type < e1000_82571;
222
223 if (++cur == scctx->isc_ntxd[0]) {
224 cur = 0;
225 }
226 DPRINTF(iflib_get_dev(sc->ctx), "%s: pidx: %d cur: %d\n",
227 __FUNCTION__, pi->ipi_pidx, cur);
228 return (cur);
229 }
230
231 /*********************************************************************
232 * The offload context is protocol specific (TCP/UDP) and thus
233 * only needs to be set when the protocol changes. The occasion
234 * of a context change can be a performance detriment, and
235 * might be better just disabled. The reason arises in the way
236 * in which the controller supports pipelined requests from the
237 * Tx data DMA. Up to four requests can be pipelined, and they may
238 * belong to the same packet or to multiple packets. However all
239 * requests for one packet are issued before a request is issued
240 * for a subsequent packet and if a request for the next packet
241 * requires a context change, that request will be stalled
242 * until the previous request completes. This means setting up
243 * a new context effectively disables pipelined Tx data DMA which
244 * in turn greatly slow down performance to send small sized
245 * frames.
246 **********************************************************************/
247 #define DONT_FORCE_CTX 1
248
249 static int
em_transmit_checksum_setup(struct e1000_softc * sc,if_pkt_info_t pi,uint32_t * txd_upper,uint32_t * txd_lower)250 em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi,
251 uint32_t *txd_upper, uint32_t *txd_lower)
252 {
253 struct e1000_context_desc *TXD = NULL;
254 if_softc_ctx_t scctx = sc->shared;
255 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
256 struct tx_ring *txr = &que->txr;
257 int csum_flags = pi->ipi_csum_flags;
258 int cur, hdr_len;
259 uint32_t cmd;
260
261 cur = pi->ipi_pidx;
262 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
263 cmd = sc->txd_cmd;
264
265 /*
266 * The 82574L can only remember the *last* context used
267 * regardless of queue that it was use for. We cannot reuse
268 * contexts on this hardware platform and must generate a new
269 * context every time. 82574L hardware spec, section 7.2.6,
270 * second note.
271 */
272 if (DONT_FORCE_CTX &&
273 sc->tx_num_queues == 1 &&
274 txr->csum_lhlen == pi->ipi_ehdrlen &&
275 txr->csum_iphlen == pi->ipi_ip_hlen &&
276 txr->csum_flags == csum_flags) {
277 /*
278 * Same csum offload context as the previous packets;
279 * just return.
280 */
281 *txd_upper = txr->csum_txd_upper;
282 *txd_lower = txr->csum_txd_lower;
283 return (cur);
284 }
285
286 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
287 /*
288 * ipcss - Start offset for header checksum calculation.
289 * ipcse - End offset for header checksum calculation.
290 * ipcso - Offset of place to put the checksum.
291 *
292 * We set ipcsX values regardless of IP version to work around HW
293 * issues and ipcse must be 0 for IPv6 per "PCIe GbE SDM 2.5" page 61.
294 * IXSM controls whether it's inserted.
295 */
296 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
297 TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen +
298 offsetof(struct ip, ip_sum);
299 if (csum_flags & CSUM_IP) {
300 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
301 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len - 1);
302 cmd |= E1000_TXD_CMD_IP;
303 } else if (csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP))
304 TXD->lower_setup.ip_fields.ipcse = htole16(0);
305
306 /*
307 * tucss - Start offset for payload checksum calculation.
308 * tucse - End offset for payload checksum calculation.
309 * tucso - Offset of place to put the checksum.
310 */
311 if (csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_IP6_TCP |
312 CSUM_IP6_UDP)) {
313 uint8_t tucso;
314
315 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
316 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
317
318 if (csum_flags & (CSUM_TCP | CSUM_IP6_TCP)) {
319 tucso = hdr_len + offsetof(struct tcphdr, th_sum);
320 cmd |= E1000_TXD_CMD_TCP;
321 } else
322 tucso = hdr_len + offsetof(struct udphdr, uh_sum);
323 TXD->upper_setup.tcp_fields.tucss = hdr_len;
324 TXD->upper_setup.tcp_fields.tucse = htole16(0);
325 TXD->upper_setup.tcp_fields.tucso = tucso;
326 }
327
328 txr->csum_lhlen = pi->ipi_ehdrlen;
329 txr->csum_iphlen = pi->ipi_ip_hlen;
330 txr->csum_flags = csum_flags;
331 txr->csum_txd_upper = *txd_upper;
332 txr->csum_txd_lower = *txd_lower;
333
334 TXD->tcp_seg_setup.data = htole32(0);
335 TXD->cmd_and_length =
336 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);
337
338 if (++cur == scctx->isc_ntxd[0]) {
339 cur = 0;
340 }
341 DPRINTF(iflib_get_dev(sc->ctx),
342 "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x"
343 " hdr_len=%d cmd=%x\n",
344 csum_flags, *txd_upper, *txd_lower, hdr_len, cmd);
345 return (cur);
346 }
347
348 #define TSO_WORKAROUND 4 /* TSO sentinel descriptor */
349
350 static int
em_isc_txd_encap(void * arg,if_pkt_info_t pi)351 em_isc_txd_encap(void *arg, if_pkt_info_t pi)
352 {
353 struct e1000_softc *sc = arg;
354 if_softc_ctx_t scctx = sc->shared;
355 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
356 struct tx_ring *txr = &que->txr;
357 bus_dma_segment_t *segs = pi->ipi_segs;
358 int nsegs = pi->ipi_nsegs;
359 int csum_flags = pi->ipi_csum_flags;
360 int i, j, first, pidx_last;
361 uint32_t txd_flags, txd_upper = 0, txd_lower = 0;
362
363 struct e1000_tx_desc *ctxd = NULL;
364 bool do_tso, tso_desc;
365 qidx_t ntxd;
366
367 txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0;
368 i = first = pi->ipi_pidx;
369 do_tso = (csum_flags & CSUM_TSO);
370 tso_desc = false;
371 ntxd = scctx->isc_ntxd[0];
372 /*
373 * TSO Hardware workaround, if this packet is not
374 * TSO, and is only a single descriptor long, and
375 * it follows a TSO burst, then we need to add a
376 * sentinel descriptor to prevent premature writeback.
377 */
378 if ((!do_tso) && (txr->tx_tso == true)) {
379 if (nsegs == 1)
380 tso_desc = true;
381 txr->tx_tso = false;
382 }
383
384 /* Do hardware assists */
385 if (do_tso) {
386 i = em_tso_setup(sc, pi, &txd_upper, &txd_lower);
387 tso_desc = txr->tx_tso;
388 } else if (csum_flags & EM_CSUM_OFFLOAD) {
389 i = em_transmit_checksum_setup(sc, pi, &txd_upper,
390 &txd_lower);
391 }
392
393 if (pi->ipi_mflags & M_VLANTAG) {
394 /* Set the vlan id. */
395 txd_upper |= htole16(pi->ipi_vtag) << 16;
396 /* Tell hardware to add tag */
397 txd_lower |= htole32(E1000_TXD_CMD_VLE);
398 }
399
400 DPRINTF(iflib_get_dev(sc->ctx),
401 "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i);
402 /* XXX sc->pcix_82544 -- lem_fill_descriptors */
403
404 /* Set up our transmit descriptors */
405 for (j = 0; j < nsegs; j++) {
406 bus_size_t seg_len;
407 bus_addr_t seg_addr;
408 uint32_t cmd;
409
410 ctxd = &txr->tx_base[i];
411 seg_addr = segs[j].ds_addr;
412 seg_len = segs[j].ds_len;
413 cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd;
414
415 /*
416 * TSO Workaround:
417 * If this is the last descriptor, we want to
418 * split it so we have a small final sentinel
419 */
420 if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
421 seg_len -= TSO_WORKAROUND;
422 ctxd->buffer_addr = htole64(seg_addr);
423 ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
424 ctxd->upper.data = htole32(txd_upper);
425
426 if (++i == scctx->isc_ntxd[0])
427 i = 0;
428
429 /* Now make the sentinel */
430 ctxd = &txr->tx_base[i];
431 ctxd->buffer_addr = htole64(seg_addr + seg_len);
432 ctxd->lower.data =
433 htole32(cmd | txd_lower | TSO_WORKAROUND);
434 ctxd->upper.data = htole32(txd_upper);
435 pidx_last = i;
436 if (++i == scctx->isc_ntxd[0])
437 i = 0;
438 DPRINTF(iflib_get_dev(sc->ctx),
439 "TSO path pidx_last=%d i=%d ntxd[0]=%d\n",
440 pidx_last, i, scctx->isc_ntxd[0]);
441 } else {
442 ctxd->buffer_addr = htole64(seg_addr);
443 ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
444 ctxd->upper.data = htole32(txd_upper);
445 pidx_last = i;
446 if (++i == scctx->isc_ntxd[0])
447 i = 0;
448 DPRINTF(iflib_get_dev(sc->ctx),
449 "pidx_last=%d i=%d ntxd[0]=%d\n",
450 pidx_last, i, scctx->isc_ntxd[0]);
451 }
452 }
453
454 /*
455 * Last Descriptor of Packet
456 * needs End Of Packet (EOP)
457 * and Report Status (RS)
458 */
459 if (txd_flags && nsegs) {
460 txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
461 DPRINTF(iflib_get_dev(sc->ctx),
462 "setting to RS on %d rs_pidx %d first: %d\n",
463 pidx_last, txr->tx_rs_pidx, first);
464 txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1);
465 MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx);
466 }
467 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags);
468 DPRINTF(iflib_get_dev(sc->ctx),
469 "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n",
470 first, pidx_last, i);
471 pi->ipi_new_pidx = i;
472
473 /*
474 * Sent data accounting for AIM. For TSO, ipi_len is the whole
475 * unsegmented payload, which is not a size the moderation
476 * calculation can use. Count the segments the hardware will put on
477 * the wire and the header each of them carries, so that the average
478 * it sees is a wire packet.
479 */
480 if (do_tso && pi->ipi_tso_segsz != 0) {
481 u32 hdrlen, segs;
482
483 hdrlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
484 if (pi->ipi_len > hdrlen) {
485 segs = howmany(pi->ipi_len - hdrlen, pi->ipi_tso_segsz);
486 txr->tx_bytes += pi->ipi_len + (segs - 1) * hdrlen;
487 txr->tx_packets += segs;
488 return (0);
489 }
490 }
491
492 txr->tx_bytes += pi->ipi_len;
493 ++txr->tx_packets;
494 return (0);
495 }
496
497 static void
em_isc_txd_flush(void * arg,uint16_t txqid,qidx_t pidx)498 em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
499 {
500 struct e1000_softc *sc = arg;
501 struct em_tx_queue *que = &sc->tx_queues[txqid];
502 struct tx_ring *txr = &que->txr;
503
504 E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx);
505 if (sc->hw.mac.type >= e1000_82540)
506 em_aim_publish(txr);
507 }
508
509 static int
em_isc_txd_credits_update(void * arg,uint16_t txqid,bool clear)510 em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
511 {
512 struct e1000_softc *sc = arg;
513 if_softc_ctx_t scctx = sc->shared;
514 struct em_tx_queue *que = &sc->tx_queues[txqid];
515 struct tx_ring *txr = &que->txr;
516
517 qidx_t processed = 0;
518 int updated;
519 qidx_t cur, prev, ntxd, rs_cidx;
520 int32_t delta;
521 uint8_t status;
522
523 rs_cidx = txr->tx_rs_cidx;
524 if (rs_cidx == txr->tx_rs_pidx)
525 return (0);
526 cur = txr->tx_rsq[rs_cidx];
527 MPASS(cur != QIDX_INVALID);
528 status = txr->tx_base[cur].upper.fields.status;
529 updated = !!(status & E1000_TXD_STAT_DD);
530
531 if (!updated)
532 return (0);
533
534 /* If clear is false just let caller know that there
535 * are descriptors to reclaim */
536 if (!clear)
537 return (1);
538
539 prev = txr->tx_cidx_processed;
540 ntxd = scctx->isc_ntxd[0];
541 do {
542 MPASS(prev != cur);
543 delta = (int32_t)cur - (int32_t)prev;
544 if (delta < 0)
545 delta += ntxd;
546 MPASS(delta > 0);
547 DPRINTF(iflib_get_dev(sc->ctx),
548 "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n",
549 __FUNCTION__, prev, cur, clear, delta);
550
551 processed += delta;
552 prev = cur;
553 rs_cidx = (rs_cidx + 1) & (ntxd-1);
554 if (rs_cidx == txr->tx_rs_pidx)
555 break;
556 cur = txr->tx_rsq[rs_cidx];
557 MPASS(cur != QIDX_INVALID);
558 status = txr->tx_base[cur].upper.fields.status;
559 } while ((status & E1000_TXD_STAT_DD));
560
561 txr->tx_rs_cidx = rs_cidx;
562 txr->tx_cidx_processed = prev;
563 return(processed);
564 }
565
566 static void
lem_isc_rxd_refill(void * arg,if_rxd_update_t iru)567 lem_isc_rxd_refill(void *arg, if_rxd_update_t iru)
568 {
569 struct e1000_softc *sc = arg;
570 if_softc_ctx_t scctx = sc->shared;
571 struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx];
572 struct rx_ring *rxr = &que->rxr;
573 struct e1000_rx_desc *rxd;
574 uint64_t *paddrs;
575 uint32_t next_pidx, pidx;
576 uint16_t count;
577 int i;
578
579 paddrs = iru->iru_paddrs;
580 pidx = iru->iru_pidx;
581 count = iru->iru_count;
582
583 for (i = 0, next_pidx = pidx; i < count; i++) {
584 rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx];
585 rxd->buffer_addr = htole64(paddrs[i]);
586 /* status bits must be cleared */
587 rxd->status = 0;
588
589 if (++next_pidx == scctx->isc_nrxd[0])
590 next_pidx = 0;
591 }
592 }
593
594 static void
em_isc_rxd_refill(void * arg,if_rxd_update_t iru)595 em_isc_rxd_refill(void *arg, if_rxd_update_t iru)
596 {
597 struct e1000_softc *sc = arg;
598 if_softc_ctx_t scctx = sc->shared;
599 uint16_t rxqid = iru->iru_qsidx;
600 struct em_rx_queue *que = &sc->rx_queues[rxqid];
601 struct rx_ring *rxr = &que->rxr;
602 union e1000_rx_desc_extended *rxd;
603 uint64_t *paddrs;
604 uint32_t next_pidx, pidx;
605 uint16_t count;
606 int i;
607
608 paddrs = iru->iru_paddrs;
609 pidx = iru->iru_pidx;
610 count = iru->iru_count;
611
612 for (i = 0, next_pidx = pidx; i < count; i++) {
613 rxd = &rxr->rx_base[next_pidx];
614 rxd->read.buffer_addr = htole64(paddrs[i]);
615 /* DD bits must be cleared */
616 rxd->wb.upper.status_error = 0;
617
618 if (++next_pidx == scctx->isc_nrxd[0])
619 next_pidx = 0;
620 }
621 }
622
623 static void
em_isc_rxd_flush(void * arg,uint16_t rxqid,uint8_t flid __unused,qidx_t pidx)624 em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
625 qidx_t pidx)
626 {
627 struct e1000_softc *sc = arg;
628 struct em_rx_queue *que = &sc->rx_queues[rxqid];
629 struct rx_ring *rxr = &que->rxr;
630
631 E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
632 if (sc->hw.mac.type >= e1000_82540)
633 em_aim_publish_rx(rxr);
634 }
635
636 static int
lem_isc_rxd_available(void * arg,uint16_t rxqid,qidx_t idx,qidx_t budget)637 lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
638 {
639 struct e1000_softc *sc = arg;
640 if_softc_ctx_t scctx = sc->shared;
641 struct em_rx_queue *que = &sc->rx_queues[rxqid];
642 struct rx_ring *rxr = &que->rxr;
643 struct e1000_rx_desc *rxd;
644 uint32_t staterr = 0;
645 int cnt, i;
646
647 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
648 rxd = (struct e1000_rx_desc *)&rxr->rx_base[i];
649 staterr = rxd->status;
650
651 if ((staterr & E1000_RXD_STAT_DD) == 0)
652 break;
653 if (++i == scctx->isc_nrxd[0])
654 i = 0;
655 if (staterr & E1000_RXD_STAT_EOP)
656 cnt++;
657 }
658 return (cnt);
659 }
660
661 static int
em_isc_rxd_available(void * arg,uint16_t rxqid,qidx_t idx,qidx_t budget)662 em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
663 {
664 struct e1000_softc *sc = arg;
665 if_softc_ctx_t scctx = sc->shared;
666 struct em_rx_queue *que = &sc->rx_queues[rxqid];
667 struct rx_ring *rxr = &que->rxr;
668 union e1000_rx_desc_extended *rxd;
669 uint32_t staterr = 0;
670 int cnt, i;
671
672 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
673 rxd = &rxr->rx_base[i];
674 staterr = le32toh(rxd->wb.upper.status_error);
675
676 if ((staterr & E1000_RXD_STAT_DD) == 0)
677 break;
678 if (++i == scctx->isc_nrxd[0])
679 i = 0;
680 if (staterr & E1000_RXD_STAT_EOP)
681 cnt++;
682 }
683 return (cnt);
684 }
685
686 static int
lem_isc_rxd_pkt_get(void * arg,if_rxd_info_t ri)687 lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
688 {
689 struct e1000_softc *sc = arg;
690 if_softc_ctx_t scctx = sc->shared;
691 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
692 struct rx_ring *rxr = &que->rxr;
693 struct e1000_rx_desc *rxd;
694 uint16_t len;
695 uint32_t status, errors;
696 bool eop;
697 int i, cidx;
698
699 status = errors = i = 0;
700 cidx = ri->iri_cidx;
701
702 do {
703 rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx];
704 status = rxd->status;
705 errors = rxd->errors;
706
707 /* Error Checking then decrement count */
708 MPASS ((status & E1000_RXD_STAT_DD) != 0);
709
710 len = le16toh(rxd->length);
711 ri->iri_len += len;
712
713 eop = (status & E1000_RXD_STAT_EOP) != 0;
714
715 /* Make sure bad packets are discarded */
716 if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
717 sc->dropped_pkts++;
718 /* XXX fixup if common */
719 return (EBADMSG);
720 }
721
722 ri->iri_frags[i].irf_flid = 0;
723 ri->iri_frags[i].irf_idx = cidx;
724 ri->iri_frags[i].irf_len = len;
725 /* Zero out the receive descriptors status. */
726 rxd->status = 0;
727
728 if (++cidx == scctx->isc_nrxd[0])
729 cidx = 0;
730 i++;
731 } while (!eop);
732
733 rxr->rx_bytes += ri->iri_len;
734 rxr->rx_packets++;
735
736 if (scctx->isc_capenable & IFCAP_RXCSUM)
737 em_receive_checksum(status, errors, ri);
738
739 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING &&
740 status & E1000_RXD_STAT_VP) {
741 ri->iri_vtag = le16toh(rxd->special);
742 ri->iri_flags |= M_VLANTAG;
743 }
744
745 ri->iri_nfrags = i;
746
747 return (0);
748 }
749
750 static int
em_isc_rxd_pkt_get(void * arg,if_rxd_info_t ri)751 em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
752 {
753 struct e1000_softc *sc = arg;
754 if_softc_ctx_t scctx = sc->shared;
755 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
756 struct rx_ring *rxr = &que->rxr;
757 union e1000_rx_desc_extended *rxd;
758
759 uint16_t len;
760 uint32_t pkt_info;
761 uint32_t staterr;
762 bool eop;
763 int i, cidx;
764
765 staterr = i = 0;
766 cidx = ri->iri_cidx;
767
768 do {
769 rxd = &rxr->rx_base[cidx];
770 staterr = le32toh(rxd->wb.upper.status_error);
771 pkt_info = le32toh(rxd->wb.lower.mrq);
772
773 /* Error Checking then decrement count */
774 MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
775
776 len = le16toh(rxd->wb.upper.length);
777 ri->iri_len += len;
778
779 eop = (staterr & E1000_RXD_STAT_EOP) != 0;
780
781 /* Make sure bad packets are discarded */
782 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
783 sc->dropped_pkts++;
784 return EBADMSG;
785 }
786
787 ri->iri_frags[i].irf_flid = 0;
788 ri->iri_frags[i].irf_idx = cidx;
789 ri->iri_frags[i].irf_len = len;
790 /* Zero out the receive descriptors status. */
791 rxd->wb.upper.status_error &= htole32(~0xFF);
792
793 if (++cidx == scctx->isc_nrxd[0])
794 cidx = 0;
795 i++;
796 } while (!eop);
797
798 rxr->rx_bytes += ri->iri_len;
799 rxr->rx_packets++;
800
801 if (scctx->isc_capenable & IFCAP_RXCSUM)
802 em_receive_checksum(staterr, staterr >> 24, ri);
803
804 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING &&
805 staterr & E1000_RXD_STAT_VP) {
806 ri->iri_vtag = le16toh(rxd->wb.upper.vlan);
807 ri->iri_flags |= M_VLANTAG;
808 }
809
810 ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss);
811 ri->iri_rsstype = em_determine_rsstype(pkt_info);
812
813 ri->iri_nfrags = i;
814 return (0);
815 }
816
817 /*********************************************************************
818 *
819 * Verify that the hardware indicated that the checksum is valid.
820 * Inform the stack about the status of checksum so that stack
821 * doesn't spend time verifying the checksum.
822 *
823 *********************************************************************/
824 static void
em_receive_checksum(uint16_t status,uint8_t errors,if_rxd_info_t ri)825 em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri)
826 {
827 if (__predict_false(status & E1000_RXD_STAT_IXSM))
828 return;
829
830 /* If there is a layer 3 or 4 error we are done */
831 if (__predict_false(errors & (E1000_RXD_ERR_IPE |
832 E1000_RXD_ERR_TCPE)))
833 return;
834
835 /* IP Checksum Good */
836 if (status & E1000_RXD_STAT_IPCS)
837 ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
838
839 /* Valid L4E checksum */
840 if (__predict_true(status &
841 (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) {
842 ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
843 ri->iri_csum_data = htons(0xffff);
844 }
845 }
846
847 /********************************************************************
848 *
849 * Parse the packet type to determine the appropriate hash
850 *
851 ******************************************************************/
852 static int
em_determine_rsstype(uint32_t pkt_info)853 em_determine_rsstype(uint32_t pkt_info)
854 {
855 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
856 case E1000_RXDADV_RSSTYPE_IPV4_TCP:
857 return M_HASHTYPE_RSS_TCP_IPV4;
858 case E1000_RXDADV_RSSTYPE_IPV4:
859 return M_HASHTYPE_RSS_IPV4;
860 case E1000_RXDADV_RSSTYPE_IPV6_TCP:
861 return M_HASHTYPE_RSS_TCP_IPV6;
862 case E1000_RXDADV_RSSTYPE_IPV6_EX:
863 return M_HASHTYPE_RSS_IPV6_EX;
864 case E1000_RXDADV_RSSTYPE_IPV6:
865 return M_HASHTYPE_RSS_IPV6;
866 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
867 return M_HASHTYPE_RSS_TCP_IPV6_EX;
868 case E1000_RXDADV_RSSTYPE_IPV4_UDP:
869 return M_HASHTYPE_RSS_UDP_IPV4;
870 case E1000_RXDADV_RSSTYPE_IPV6_UDP:
871 return M_HASHTYPE_RSS_UDP_IPV6;
872 case E1000_RXDADV_RSSTYPE_IPV6_UDP_EX:
873 return M_HASHTYPE_RSS_UDP_IPV6_EX;
874 default:
875 return M_HASHTYPE_NONE;
876 }
877 }
878