1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org>
5 * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "if_em.h"
31
32 #ifdef RSS
33 #include <net/rss_config.h>
34 #include <netinet/in_rss.h>
35 #endif
36
37 #ifdef VERBOSE_DEBUG
38 #define DPRINTF device_printf
39 #else
40 #define DPRINTF(...)
41 #endif
42
43 /*********************************************************************
44 * Local Function prototypes
45 *********************************************************************/
46 static int em_tso_setup(struct e1000_softc *, if_pkt_info_t, uint32_t *,
47 uint32_t *);
48 static int em_transmit_checksum_setup(struct e1000_softc *, if_pkt_info_t,
49 uint32_t *, uint32_t *);
50 static int em_isc_txd_encap(void *, if_pkt_info_t);
51 static void em_isc_txd_flush(void *, uint16_t, qidx_t);
52 static int em_isc_txd_credits_update(void *, uint16_t, bool);
53 static void em_isc_rxd_refill(void *, if_rxd_update_t);
54 static void em_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t);
55 static int em_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t);
56 static int em_isc_rxd_pkt_get(void *, if_rxd_info_t);
57
58 static void lem_isc_rxd_refill(void *, if_rxd_update_t);
59
60 static int lem_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t);
61 static int lem_isc_rxd_pkt_get(void *, if_rxd_info_t);
62
63 static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t);
64 static int em_determine_rsstype(uint32_t);
65 extern int em_intr(void *);
66
67 struct if_txrx em_txrx = {
68 .ift_txd_encap = em_isc_txd_encap,
69 .ift_txd_flush = em_isc_txd_flush,
70 .ift_txd_credits_update = em_isc_txd_credits_update,
71 .ift_rxd_available = em_isc_rxd_available,
72 .ift_rxd_pkt_get = em_isc_rxd_pkt_get,
73 .ift_rxd_refill = em_isc_rxd_refill,
74 .ift_rxd_flush = em_isc_rxd_flush,
75 .ift_legacy_intr = em_intr
76 };
77
78 struct if_txrx lem_txrx = {
79 .ift_txd_encap = em_isc_txd_encap,
80 .ift_txd_flush = em_isc_txd_flush,
81 .ift_txd_credits_update = em_isc_txd_credits_update,
82 .ift_rxd_available = lem_isc_rxd_available,
83 .ift_rxd_pkt_get = lem_isc_rxd_pkt_get,
84 .ift_rxd_refill = lem_isc_rxd_refill,
85 .ift_rxd_flush = em_isc_rxd_flush,
86 .ift_legacy_intr = em_intr
87 };
88
89 extern if_shared_ctx_t em_sctx;
90
91 void
em_dump_rs(struct e1000_softc * sc)92 em_dump_rs(struct e1000_softc *sc)
93 {
94 if_softc_ctx_t scctx = sc->shared;
95 struct em_tx_queue *que;
96 struct tx_ring *txr;
97 qidx_t i, ntxd, qid, cur;
98 int16_t rs_cidx;
99 uint8_t status;
100
101 printf("\n");
102 ntxd = scctx->isc_ntxd[0];
103 for (qid = 0; qid < sc->tx_num_queues; qid++) {
104 que = &sc->tx_queues[qid];
105 txr = &que->txr;
106 rs_cidx = txr->tx_rs_cidx;
107 if (rs_cidx != txr->tx_rs_pidx) {
108 cur = txr->tx_rsq[rs_cidx];
109 status = txr->tx_base[cur].upper.fields.status;
110 if (!(status & E1000_TXD_STAT_DD))
111 printf("qid[%d]->tx_rsq[%d]: %d clear ",
112 qid, rs_cidx, cur);
113 } else {
114 rs_cidx = (rs_cidx-1)&(ntxd-1);
115 cur = txr->tx_rsq[rs_cidx];
116 printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ",
117 qid, rs_cidx, cur);
118 }
119 printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed,
120 txr->tx_rs_pidx);
121 for (i = 0; i < ntxd; i++) {
122 if (txr->tx_base[i].upper.fields.status &
123 E1000_TXD_STAT_DD)
124 printf("%d set ", i);
125 }
126 printf("\n");
127 }
128 }
129
130 /**********************************************************************
131 *
132 * Setup work for hardware segmentation offload (TSO) on
133 * adapters using advanced tx descriptors
134 *
135 **********************************************************************/
136 static int
em_tso_setup(struct e1000_softc * sc,if_pkt_info_t pi,uint32_t * txd_upper,uint32_t * txd_lower)137 em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, uint32_t *txd_upper,
138 uint32_t *txd_lower)
139 {
140 if_softc_ctx_t scctx = sc->shared;
141 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
142 struct tx_ring *txr = &que->txr;
143 struct e1000_context_desc *TXD;
144 int cur, hdr_len;
145 uint32_t cmd_type_len;
146
147 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
148 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
149 E1000_TXD_DTYP_D | /* Data descr type */
150 E1000_TXD_CMD_TSE); /* Do TSE on this packet */
151
152 cur = pi->ipi_pidx;
153 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
154
155 /*
156 * ipcss - Start offset for header checksum calculation.
157 * ipcse - End offset for header checksum calculation.
158 * ipcso - Offset of place to put the checksum.
159 */
160 switch(pi->ipi_etype) {
161 case ETHERTYPE_IP:
162 /* IP and/or TCP header checksum calculation and insertion. */
163 *txd_upper =
164 (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
165
166 TXD->lower_setup.ip_fields.ipcse =
167 htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1);
168 break;
169 case ETHERTYPE_IPV6:
170 /* TCP header checksum calculation and insertion. */
171 *txd_upper = E1000_TXD_POPTS_TXSM << 8;
172
173 TXD->lower_setup.ip_fields.ipcse = htole16(0);
174 break;
175 default:
176 break;
177 }
178 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
179 TXD->lower_setup.ip_fields.ipcso =
180 pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);
181
182 /*
183 * tucss - Start offset for payload checksum calculation.
184 * tucse - End offset for payload checksum calculation.
185 * tucso - Offset of place to put the checksum.
186 */
187 TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
188 TXD->upper_setup.tcp_fields.tucse = 0;
189 TXD->upper_setup.tcp_fields.tucso =
190 pi->ipi_ehdrlen + pi->ipi_ip_hlen +
191 offsetof(struct tcphdr, th_sum);
192
193 /*
194 * Payload size per packet w/o any headers.
195 * Length of all headers up to payload.
196 */
197 TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz);
198 TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
199
200 /*
201 * "PCI/PCI-X SDM 4.0" page 45, and "PCIe GbE SDM 2.5" page 63
202 * - Set up basic TUCMDs
203 * - For others IP bit on indicates IPv4, while off indicates IPv6
204 */
205 cmd_type_len = sc->txd_cmd |
206 E1000_TXD_CMD_DEXT | /* Extended descr */
207 E1000_TXD_CMD_TSE | /* TSE context */
208 E1000_TXD_CMD_TCP; /* Do TCP checksum */
209 if (pi->ipi_etype == ETHERTYPE_IP)
210 cmd_type_len |= E1000_TXD_CMD_IP;
211 TXD->cmd_and_length = htole32(cmd_type_len |
212 (pi->ipi_len - hdr_len)); /* Total len */
213
214 txr->tx_tso = true;
215
216 if (++cur == scctx->isc_ntxd[0]) {
217 cur = 0;
218 }
219 DPRINTF(iflib_get_dev(sc->ctx), "%s: pidx: %d cur: %d\n",
220 __FUNCTION__, pi->ipi_pidx, cur);
221 return (cur);
222 }
223
224 /*********************************************************************
225 * The offload context is protocol specific (TCP/UDP) and thus
226 * only needs to be set when the protocol changes. The occasion
227 * of a context change can be a performance detriment, and
228 * might be better just disabled. The reason arises in the way
229 * in which the controller supports pipelined requests from the
230 * Tx data DMA. Up to four requests can be pipelined, and they may
231 * belong to the same packet or to multiple packets. However all
232 * requests for one packet are issued before a request is issued
233 * for a subsequent packet and if a request for the next packet
234 * requires a context change, that request will be stalled
235 * until the previous request completes. This means setting up
236 * a new context effectively disables pipelined Tx data DMA which
237 * in turn greatly slow down performance to send small sized
238 * frames.
239 **********************************************************************/
240 #define DONT_FORCE_CTX 1
241
242 static int
em_transmit_checksum_setup(struct e1000_softc * sc,if_pkt_info_t pi,uint32_t * txd_upper,uint32_t * txd_lower)243 em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi,
244 uint32_t *txd_upper, uint32_t *txd_lower)
245 {
246 struct e1000_context_desc *TXD = NULL;
247 if_softc_ctx_t scctx = sc->shared;
248 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
249 struct tx_ring *txr = &que->txr;
250 int csum_flags = pi->ipi_csum_flags;
251 int cur, hdr_len;
252 uint32_t cmd;
253
254 cur = pi->ipi_pidx;
255 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
256 cmd = sc->txd_cmd;
257
258 /*
259 * The 82574L can only remember the *last* context used
260 * regardless of queue that it was use for. We cannot reuse
261 * contexts on this hardware platform and must generate a new
262 * context every time. 82574L hardware spec, section 7.2.6,
263 * second note.
264 */
265 if (DONT_FORCE_CTX &&
266 sc->tx_num_queues == 1 &&
267 txr->csum_lhlen == pi->ipi_ehdrlen &&
268 txr->csum_iphlen == pi->ipi_ip_hlen &&
269 txr->csum_flags == csum_flags) {
270 /*
271 * Same csum offload context as the previous packets;
272 * just return.
273 */
274 *txd_upper = txr->csum_txd_upper;
275 *txd_lower = txr->csum_txd_lower;
276 return (cur);
277 }
278
279 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
280 /*
281 * ipcss - Start offset for header checksum calculation.
282 * ipcse - End offset for header checksum calculation.
283 * ipcso - Offset of place to put the checksum.
284 *
285 * We set ipcsX values regardless of IP version to work around HW
286 * issues and ipcse must be 0 for IPv6 per "PCIe GbE SDM 2.5" page 61.
287 * IXSM controls whether it's inserted.
288 */
289 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
290 TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen +
291 offsetof(struct ip, ip_sum);
292 if (csum_flags & CSUM_IP) {
293 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
294 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len - 1);
295 cmd |= E1000_TXD_CMD_IP;
296 } else if (csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP))
297 TXD->lower_setup.ip_fields.ipcse = htole16(0);
298
299 /*
300 * tucss - Start offset for payload checksum calculation.
301 * tucse - End offset for payload checksum calculation.
302 * tucso - Offset of place to put the checksum.
303 */
304 if (csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_IP6_TCP |
305 CSUM_IP6_UDP)) {
306 uint8_t tucso;
307
308 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
309 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
310
311 if (csum_flags & (CSUM_TCP | CSUM_IP6_TCP)) {
312 tucso = hdr_len + offsetof(struct tcphdr, th_sum);
313 cmd |= E1000_TXD_CMD_TCP;
314 } else
315 tucso = hdr_len + offsetof(struct udphdr, uh_sum);
316 TXD->upper_setup.tcp_fields.tucss = hdr_len;
317 TXD->upper_setup.tcp_fields.tucse = htole16(0);
318 TXD->upper_setup.tcp_fields.tucso = tucso;
319 }
320
321 txr->csum_lhlen = pi->ipi_ehdrlen;
322 txr->csum_iphlen = pi->ipi_ip_hlen;
323 txr->csum_flags = csum_flags;
324 txr->csum_txd_upper = *txd_upper;
325 txr->csum_txd_lower = *txd_lower;
326
327 TXD->tcp_seg_setup.data = htole32(0);
328 TXD->cmd_and_length =
329 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);
330
331 if (++cur == scctx->isc_ntxd[0]) {
332 cur = 0;
333 }
334 DPRINTF(iflib_get_dev(sc->ctx),
335 "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x"
336 " hdr_len=%d cmd=%x\n",
337 csum_flags, *txd_upper, *txd_lower, hdr_len, cmd);
338 return (cur);
339 }
340
341 #define TSO_WORKAROUND 4 /* TSO sentinel descriptor */
342
343 static int
em_isc_txd_encap(void * arg,if_pkt_info_t pi)344 em_isc_txd_encap(void *arg, if_pkt_info_t pi)
345 {
346 struct e1000_softc *sc = arg;
347 if_softc_ctx_t scctx = sc->shared;
348 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
349 struct tx_ring *txr = &que->txr;
350 bus_dma_segment_t *segs = pi->ipi_segs;
351 int nsegs = pi->ipi_nsegs;
352 int csum_flags = pi->ipi_csum_flags;
353 int i, j, first, pidx_last;
354 uint32_t txd_flags, txd_upper = 0, txd_lower = 0;
355
356 struct e1000_tx_desc *ctxd = NULL;
357 bool do_tso, tso_desc;
358 qidx_t ntxd;
359
360 txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0;
361 i = first = pi->ipi_pidx;
362 do_tso = (csum_flags & CSUM_TSO);
363 tso_desc = false;
364 ntxd = scctx->isc_ntxd[0];
365 /*
366 * TSO Hardware workaround, if this packet is not
367 * TSO, and is only a single descriptor long, and
368 * it follows a TSO burst, then we need to add a
369 * sentinel descriptor to prevent premature writeback.
370 */
371 if ((!do_tso) && (txr->tx_tso == true)) {
372 if (nsegs == 1)
373 tso_desc = true;
374 txr->tx_tso = false;
375 }
376
377 /* Do hardware assists */
378 if (do_tso) {
379 i = em_tso_setup(sc, pi, &txd_upper, &txd_lower);
380 tso_desc = true;
381 } else if (csum_flags & EM_CSUM_OFFLOAD) {
382 i = em_transmit_checksum_setup(sc, pi, &txd_upper,
383 &txd_lower);
384 }
385
386 if (pi->ipi_mflags & M_VLANTAG) {
387 /* Set the vlan id. */
388 txd_upper |= htole16(pi->ipi_vtag) << 16;
389 /* Tell hardware to add tag */
390 txd_lower |= htole32(E1000_TXD_CMD_VLE);
391 }
392
393 DPRINTF(iflib_get_dev(sc->ctx),
394 "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i);
395 /* XXX sc->pcix_82544 -- lem_fill_descriptors */
396
397 /* Set up our transmit descriptors */
398 for (j = 0; j < nsegs; j++) {
399 bus_size_t seg_len;
400 bus_addr_t seg_addr;
401 uint32_t cmd;
402
403 ctxd = &txr->tx_base[i];
404 seg_addr = segs[j].ds_addr;
405 seg_len = segs[j].ds_len;
406 cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd;
407
408 /*
409 * TSO Workaround:
410 * If this is the last descriptor, we want to
411 * split it so we have a small final sentinel
412 */
413 if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
414 seg_len -= TSO_WORKAROUND;
415 ctxd->buffer_addr = htole64(seg_addr);
416 ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
417 ctxd->upper.data = htole32(txd_upper);
418
419 if (++i == scctx->isc_ntxd[0])
420 i = 0;
421
422 /* Now make the sentinel */
423 ctxd = &txr->tx_base[i];
424 ctxd->buffer_addr = htole64(seg_addr + seg_len);
425 ctxd->lower.data =
426 htole32(cmd | txd_lower | TSO_WORKAROUND);
427 ctxd->upper.data = htole32(txd_upper);
428 pidx_last = i;
429 if (++i == scctx->isc_ntxd[0])
430 i = 0;
431 DPRINTF(iflib_get_dev(sc->ctx),
432 "TSO path pidx_last=%d i=%d ntxd[0]=%d\n",
433 pidx_last, i, scctx->isc_ntxd[0]);
434 } else {
435 ctxd->buffer_addr = htole64(seg_addr);
436 ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
437 ctxd->upper.data = htole32(txd_upper);
438 pidx_last = i;
439 if (++i == scctx->isc_ntxd[0])
440 i = 0;
441 DPRINTF(iflib_get_dev(sc->ctx),
442 "pidx_last=%d i=%d ntxd[0]=%d\n",
443 pidx_last, i, scctx->isc_ntxd[0]);
444 }
445 }
446
447 /*
448 * Last Descriptor of Packet
449 * needs End Of Packet (EOP)
450 * and Report Status (RS)
451 */
452 if (txd_flags && nsegs) {
453 txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
454 DPRINTF(iflib_get_dev(sc->ctx),
455 "setting to RS on %d rs_pidx %d first: %d\n",
456 pidx_last, txr->tx_rs_pidx, first);
457 txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1);
458 MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx);
459 }
460 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags);
461 DPRINTF(iflib_get_dev(sc->ctx),
462 "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n",
463 first, pidx_last, i);
464 pi->ipi_new_pidx = i;
465
466 /* Sent data accounting for AIM */
467 txr->tx_bytes += pi->ipi_len;
468 ++txr->tx_packets;
469
470 return (0);
471 }
472
473 static void
em_isc_txd_flush(void * arg,uint16_t txqid,qidx_t pidx)474 em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
475 {
476 struct e1000_softc *sc = arg;
477 struct em_tx_queue *que = &sc->tx_queues[txqid];
478 struct tx_ring *txr = &que->txr;
479
480 E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx);
481 }
482
483 static int
em_isc_txd_credits_update(void * arg,uint16_t txqid,bool clear)484 em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
485 {
486 struct e1000_softc *sc = arg;
487 if_softc_ctx_t scctx = sc->shared;
488 struct em_tx_queue *que = &sc->tx_queues[txqid];
489 struct tx_ring *txr = &que->txr;
490
491 qidx_t processed = 0;
492 int updated;
493 qidx_t cur, prev, ntxd, rs_cidx;
494 int32_t delta;
495 uint8_t status;
496
497 rs_cidx = txr->tx_rs_cidx;
498 if (rs_cidx == txr->tx_rs_pidx)
499 return (0);
500 cur = txr->tx_rsq[rs_cidx];
501 MPASS(cur != QIDX_INVALID);
502 status = txr->tx_base[cur].upper.fields.status;
503 updated = !!(status & E1000_TXD_STAT_DD);
504
505 if (!updated)
506 return (0);
507
508 /* If clear is false just let caller know that there
509 * are descriptors to reclaim */
510 if (!clear)
511 return (1);
512
513 prev = txr->tx_cidx_processed;
514 ntxd = scctx->isc_ntxd[0];
515 do {
516 MPASS(prev != cur);
517 delta = (int32_t)cur - (int32_t)prev;
518 if (delta < 0)
519 delta += ntxd;
520 MPASS(delta > 0);
521 DPRINTF(iflib_get_dev(sc->ctx),
522 "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n",
523 __FUNCTION__, prev, cur, clear, delta);
524
525 processed += delta;
526 prev = cur;
527 rs_cidx = (rs_cidx + 1) & (ntxd-1);
528 if (rs_cidx == txr->tx_rs_pidx)
529 break;
530 cur = txr->tx_rsq[rs_cidx];
531 MPASS(cur != QIDX_INVALID);
532 status = txr->tx_base[cur].upper.fields.status;
533 } while ((status & E1000_TXD_STAT_DD));
534
535 txr->tx_rs_cidx = rs_cidx;
536 txr->tx_cidx_processed = prev;
537 return(processed);
538 }
539
540 static void
lem_isc_rxd_refill(void * arg,if_rxd_update_t iru)541 lem_isc_rxd_refill(void *arg, if_rxd_update_t iru)
542 {
543 struct e1000_softc *sc = arg;
544 if_softc_ctx_t scctx = sc->shared;
545 struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx];
546 struct rx_ring *rxr = &que->rxr;
547 struct e1000_rx_desc *rxd;
548 uint64_t *paddrs;
549 uint32_t next_pidx, pidx;
550 uint16_t count;
551 int i;
552
553 paddrs = iru->iru_paddrs;
554 pidx = iru->iru_pidx;
555 count = iru->iru_count;
556
557 for (i = 0, next_pidx = pidx; i < count; i++) {
558 rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx];
559 rxd->buffer_addr = htole64(paddrs[i]);
560 /* status bits must be cleared */
561 rxd->status = 0;
562
563 if (++next_pidx == scctx->isc_nrxd[0])
564 next_pidx = 0;
565 }
566 }
567
568 static void
em_isc_rxd_refill(void * arg,if_rxd_update_t iru)569 em_isc_rxd_refill(void *arg, if_rxd_update_t iru)
570 {
571 struct e1000_softc *sc = arg;
572 if_softc_ctx_t scctx = sc->shared;
573 uint16_t rxqid = iru->iru_qsidx;
574 struct em_rx_queue *que = &sc->rx_queues[rxqid];
575 struct rx_ring *rxr = &que->rxr;
576 union e1000_rx_desc_extended *rxd;
577 uint64_t *paddrs;
578 uint32_t next_pidx, pidx;
579 uint16_t count;
580 int i;
581
582 paddrs = iru->iru_paddrs;
583 pidx = iru->iru_pidx;
584 count = iru->iru_count;
585
586 for (i = 0, next_pidx = pidx; i < count; i++) {
587 rxd = &rxr->rx_base[next_pidx];
588 rxd->read.buffer_addr = htole64(paddrs[i]);
589 /* DD bits must be cleared */
590 rxd->wb.upper.status_error = 0;
591
592 if (++next_pidx == scctx->isc_nrxd[0])
593 next_pidx = 0;
594 }
595 }
596
597 static void
em_isc_rxd_flush(void * arg,uint16_t rxqid,uint8_t flid __unused,qidx_t pidx)598 em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
599 qidx_t pidx)
600 {
601 struct e1000_softc *sc = arg;
602 struct em_rx_queue *que = &sc->rx_queues[rxqid];
603 struct rx_ring *rxr = &que->rxr;
604
605 E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
606 }
607
608 static int
lem_isc_rxd_available(void * arg,uint16_t rxqid,qidx_t idx,qidx_t budget)609 lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
610 {
611 struct e1000_softc *sc = arg;
612 if_softc_ctx_t scctx = sc->shared;
613 struct em_rx_queue *que = &sc->rx_queues[rxqid];
614 struct rx_ring *rxr = &que->rxr;
615 struct e1000_rx_desc *rxd;
616 uint32_t staterr = 0;
617 int cnt, i;
618
619 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
620 rxd = (struct e1000_rx_desc *)&rxr->rx_base[i];
621 staterr = rxd->status;
622
623 if ((staterr & E1000_RXD_STAT_DD) == 0)
624 break;
625 if (++i == scctx->isc_nrxd[0])
626 i = 0;
627 if (staterr & E1000_RXD_STAT_EOP)
628 cnt++;
629 }
630 return (cnt);
631 }
632
633 static int
em_isc_rxd_available(void * arg,uint16_t rxqid,qidx_t idx,qidx_t budget)634 em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
635 {
636 struct e1000_softc *sc = arg;
637 if_softc_ctx_t scctx = sc->shared;
638 struct em_rx_queue *que = &sc->rx_queues[rxqid];
639 struct rx_ring *rxr = &que->rxr;
640 union e1000_rx_desc_extended *rxd;
641 uint32_t staterr = 0;
642 int cnt, i;
643
644 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
645 rxd = &rxr->rx_base[i];
646 staterr = le32toh(rxd->wb.upper.status_error);
647
648 if ((staterr & E1000_RXD_STAT_DD) == 0)
649 break;
650 if (++i == scctx->isc_nrxd[0])
651 i = 0;
652 if (staterr & E1000_RXD_STAT_EOP)
653 cnt++;
654 }
655 return (cnt);
656 }
657
658 static int
lem_isc_rxd_pkt_get(void * arg,if_rxd_info_t ri)659 lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
660 {
661 struct e1000_softc *sc = arg;
662 if_softc_ctx_t scctx = sc->shared;
663 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
664 struct rx_ring *rxr = &que->rxr;
665 struct e1000_rx_desc *rxd;
666 uint16_t len;
667 uint32_t status, errors;
668 bool eop;
669 int i, cidx;
670
671 status = errors = i = 0;
672 cidx = ri->iri_cidx;
673
674 do {
675 rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx];
676 status = rxd->status;
677 errors = rxd->errors;
678
679 /* Error Checking then decrement count */
680 MPASS ((status & E1000_RXD_STAT_DD) != 0);
681
682 len = le16toh(rxd->length);
683 ri->iri_len += len;
684 rxr->rx_bytes += ri->iri_len;
685
686 eop = (status & E1000_RXD_STAT_EOP) != 0;
687
688 /* Make sure bad packets are discarded */
689 if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
690 sc->dropped_pkts++;
691 /* XXX fixup if common */
692 return (EBADMSG);
693 }
694
695 ri->iri_frags[i].irf_flid = 0;
696 ri->iri_frags[i].irf_idx = cidx;
697 ri->iri_frags[i].irf_len = len;
698 /* Zero out the receive descriptors status. */
699 rxd->status = 0;
700
701 if (++cidx == scctx->isc_nrxd[0])
702 cidx = 0;
703 i++;
704 } while (!eop);
705
706 rxr->rx_packets++;
707
708 if (scctx->isc_capenable & IFCAP_RXCSUM)
709 em_receive_checksum(status, errors, ri);
710
711 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING &&
712 status & E1000_RXD_STAT_VP) {
713 ri->iri_vtag =
714 le16toh(rxd->special & E1000_RXD_SPC_VLAN_MASK);
715 ri->iri_flags |= M_VLANTAG;
716 }
717
718 ri->iri_nfrags = i;
719
720 return (0);
721 }
722
723 static int
em_isc_rxd_pkt_get(void * arg,if_rxd_info_t ri)724 em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
725 {
726 struct e1000_softc *sc = arg;
727 if_softc_ctx_t scctx = sc->shared;
728 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
729 struct rx_ring *rxr = &que->rxr;
730 union e1000_rx_desc_extended *rxd;
731
732 uint16_t len;
733 uint32_t pkt_info;
734 uint32_t staterr;
735 bool eop;
736 int i, cidx;
737
738 staterr = i = 0;
739 cidx = ri->iri_cidx;
740
741 do {
742 rxd = &rxr->rx_base[cidx];
743 staterr = le32toh(rxd->wb.upper.status_error);
744 pkt_info = le32toh(rxd->wb.lower.mrq);
745
746 /* Error Checking then decrement count */
747 MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
748
749 len = le16toh(rxd->wb.upper.length);
750 ri->iri_len += len;
751 rxr->rx_bytes += ri->iri_len;
752
753 eop = (staterr & E1000_RXD_STAT_EOP) != 0;
754
755 /* Make sure bad packets are discarded */
756 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
757 sc->dropped_pkts++;
758 return EBADMSG;
759 }
760
761 ri->iri_frags[i].irf_flid = 0;
762 ri->iri_frags[i].irf_idx = cidx;
763 ri->iri_frags[i].irf_len = len;
764 /* Zero out the receive descriptors status. */
765 rxd->wb.upper.status_error &= htole32(~0xFF);
766
767 if (++cidx == scctx->isc_nrxd[0])
768 cidx = 0;
769 i++;
770 } while (!eop);
771
772 rxr->rx_packets++;
773
774 if (scctx->isc_capenable & IFCAP_RXCSUM)
775 em_receive_checksum(staterr, staterr >> 24, ri);
776
777 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING &&
778 staterr & E1000_RXD_STAT_VP) {
779 ri->iri_vtag = le16toh(rxd->wb.upper.vlan);
780 ri->iri_flags |= M_VLANTAG;
781 }
782
783 ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss);
784 ri->iri_rsstype = em_determine_rsstype(pkt_info);
785
786 ri->iri_nfrags = i;
787 return (0);
788 }
789
790 /*********************************************************************
791 *
792 * Verify that the hardware indicated that the checksum is valid.
793 * Inform the stack about the status of checksum so that stack
794 * doesn't spend time verifying the checksum.
795 *
796 *********************************************************************/
797 static void
em_receive_checksum(uint16_t status,uint8_t errors,if_rxd_info_t ri)798 em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri)
799 {
800 if (__predict_false(status & E1000_RXD_STAT_IXSM))
801 return;
802
803 /* If there is a layer 3 or 4 error we are done */
804 if (__predict_false(errors & (E1000_RXD_ERR_IPE |
805 E1000_RXD_ERR_TCPE)))
806 return;
807
808 /* IP Checksum Good */
809 if (status & E1000_RXD_STAT_IPCS)
810 ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
811
812 /* Valid L4E checksum */
813 if (__predict_true(status &
814 (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) {
815 ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
816 ri->iri_csum_data = htons(0xffff);
817 }
818 }
819
820 /********************************************************************
821 *
822 * Parse the packet type to determine the appropriate hash
823 *
824 ******************************************************************/
825 static int
em_determine_rsstype(uint32_t pkt_info)826 em_determine_rsstype(uint32_t pkt_info)
827 {
828 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
829 case E1000_RXDADV_RSSTYPE_IPV4_TCP:
830 return M_HASHTYPE_RSS_TCP_IPV4;
831 case E1000_RXDADV_RSSTYPE_IPV4:
832 return M_HASHTYPE_RSS_IPV4;
833 case E1000_RXDADV_RSSTYPE_IPV6_TCP:
834 return M_HASHTYPE_RSS_TCP_IPV6;
835 case E1000_RXDADV_RSSTYPE_IPV6_EX:
836 return M_HASHTYPE_RSS_IPV6_EX;
837 case E1000_RXDADV_RSSTYPE_IPV6:
838 return M_HASHTYPE_RSS_IPV6;
839 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
840 return M_HASHTYPE_RSS_TCP_IPV6_EX;
841 default:
842 return M_HASHTYPE_OPAQUE;
843 }
844 }
845