1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org>
5 * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "if_em.h"
31
32 #include <net/rss_config.h>
33 #include <netinet/in_rss.h>
34
35 #ifdef VERBOSE_DEBUG
36 #define DPRINTF device_printf
37 #else
38 #define DPRINTF(...)
39 #endif
40
41 /*********************************************************************
42 * Local Function prototypes
43 *********************************************************************/
44 static int em_tso_setup(struct e1000_softc *, if_pkt_info_t, uint32_t *,
45 uint32_t *);
46 static int em_transmit_checksum_setup(struct e1000_softc *, if_pkt_info_t,
47 uint32_t *, uint32_t *);
48 static int em_isc_txd_encap(void *, if_pkt_info_t);
49 static void em_isc_txd_flush(void *, uint16_t, qidx_t);
50 static int em_isc_txd_credits_update(void *, uint16_t, bool);
51 static void em_isc_rxd_refill(void *, if_rxd_update_t);
52 static void em_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t);
53 static int em_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t);
54 static int em_isc_rxd_pkt_get(void *, if_rxd_info_t);
55
56 static void lem_isc_rxd_refill(void *, if_rxd_update_t);
57
58 static int lem_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t);
59 static int lem_isc_rxd_pkt_get(void *, if_rxd_info_t);
60
61 static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t);
62 static int em_determine_rsstype(uint32_t);
63 extern int em_intr(void *);
64
65 struct if_txrx em_txrx = {
66 .ift_txd_encap = em_isc_txd_encap,
67 .ift_txd_flush = em_isc_txd_flush,
68 .ift_txd_credits_update = em_isc_txd_credits_update,
69 .ift_rxd_available = em_isc_rxd_available,
70 .ift_rxd_pkt_get = em_isc_rxd_pkt_get,
71 .ift_rxd_refill = em_isc_rxd_refill,
72 .ift_rxd_flush = em_isc_rxd_flush,
73 .ift_legacy_intr = em_intr
74 };
75
76 struct if_txrx lem_txrx = {
77 .ift_txd_encap = em_isc_txd_encap,
78 .ift_txd_flush = em_isc_txd_flush,
79 .ift_txd_credits_update = em_isc_txd_credits_update,
80 .ift_rxd_available = lem_isc_rxd_available,
81 .ift_rxd_pkt_get = lem_isc_rxd_pkt_get,
82 .ift_rxd_refill = lem_isc_rxd_refill,
83 .ift_rxd_flush = em_isc_rxd_flush,
84 .ift_legacy_intr = em_intr
85 };
86
87 extern if_shared_ctx_t em_sctx;
88
89 void
em_dump_rs(struct e1000_softc * sc)90 em_dump_rs(struct e1000_softc *sc)
91 {
92 if_softc_ctx_t scctx = sc->shared;
93 struct em_tx_queue *que;
94 struct tx_ring *txr;
95 qidx_t i, ntxd, qid, cur;
96 int16_t rs_cidx;
97 uint8_t status;
98
99 printf("\n");
100 ntxd = scctx->isc_ntxd[0];
101 for (qid = 0; qid < sc->tx_num_queues; qid++) {
102 que = &sc->tx_queues[qid];
103 txr = &que->txr;
104 rs_cidx = txr->tx_rs_cidx;
105 if (rs_cidx != txr->tx_rs_pidx) {
106 cur = txr->tx_rsq[rs_cidx];
107 status = txr->tx_base[cur].upper.fields.status;
108 if (!(status & E1000_TXD_STAT_DD))
109 printf("qid[%d]->tx_rsq[%d]: %d clear ",
110 qid, rs_cidx, cur);
111 } else {
112 rs_cidx = (rs_cidx-1)&(ntxd-1);
113 cur = txr->tx_rsq[rs_cidx];
114 printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ",
115 qid, rs_cidx, cur);
116 }
117 printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed,
118 txr->tx_rs_pidx);
119 for (i = 0; i < ntxd; i++) {
120 if (txr->tx_base[i].upper.fields.status &
121 E1000_TXD_STAT_DD)
122 printf("%d set ", i);
123 }
124 printf("\n");
125 }
126 }
127
128 /**********************************************************************
129 *
130 * Setup work for hardware segmentation offload (TSO) on
131 * adapters using advanced tx descriptors
132 *
133 **********************************************************************/
134 static int
em_tso_setup(struct e1000_softc * sc,if_pkt_info_t pi,uint32_t * txd_upper,uint32_t * txd_lower)135 em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, uint32_t *txd_upper,
136 uint32_t *txd_lower)
137 {
138 if_softc_ctx_t scctx = sc->shared;
139 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
140 struct tx_ring *txr = &que->txr;
141 struct e1000_context_desc *TXD;
142 int cur, hdr_len;
143 uint32_t cmd_type_len;
144
145 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
146 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
147 E1000_TXD_DTYP_D | /* Data descr type */
148 E1000_TXD_CMD_TSE); /* Do TSE on this packet */
149
150 cur = pi->ipi_pidx;
151 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
152
153 /*
154 * ipcss - Start offset for header checksum calculation.
155 * ipcse - End offset for header checksum calculation.
156 * ipcso - Offset of place to put the checksum.
157 */
158 switch(pi->ipi_etype) {
159 case ETHERTYPE_IP:
160 /* IP and/or TCP header checksum calculation and insertion. */
161 *txd_upper =
162 (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
163
164 TXD->lower_setup.ip_fields.ipcse =
165 htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1);
166 break;
167 case ETHERTYPE_IPV6:
168 /* TCP header checksum calculation and insertion. */
169 *txd_upper = E1000_TXD_POPTS_TXSM << 8;
170
171 TXD->lower_setup.ip_fields.ipcse = htole16(0);
172 break;
173 default:
174 break;
175 }
176 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
177 TXD->lower_setup.ip_fields.ipcso =
178 pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);
179
180 /*
181 * tucss - Start offset for payload checksum calculation.
182 * tucse - End offset for payload checksum calculation.
183 * tucso - Offset of place to put the checksum.
184 */
185 TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
186 TXD->upper_setup.tcp_fields.tucse = 0;
187 TXD->upper_setup.tcp_fields.tucso =
188 pi->ipi_ehdrlen + pi->ipi_ip_hlen +
189 offsetof(struct tcphdr, th_sum);
190
191 /*
192 * Payload size per packet w/o any headers.
193 * Length of all headers up to payload.
194 */
195 TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz);
196 TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
197
198 /*
199 * "PCI/PCI-X SDM 4.0" page 45, and "PCIe GbE SDM 2.5" page 63
200 * - Set up basic TUCMDs
201 * - For others IP bit on indicates IPv4, while off indicates IPv6
202 */
203 cmd_type_len = sc->txd_cmd |
204 E1000_TXD_CMD_DEXT | /* Extended descr */
205 E1000_TXD_CMD_TSE | /* TSE context */
206 E1000_TXD_CMD_TCP; /* Do TCP checksum */
207 if (pi->ipi_etype == ETHERTYPE_IP)
208 cmd_type_len |= E1000_TXD_CMD_IP;
209 TXD->cmd_and_length = htole32(cmd_type_len |
210 (pi->ipi_len - hdr_len)); /* Total len */
211
212 txr->tx_tso = true;
213
214 if (++cur == scctx->isc_ntxd[0]) {
215 cur = 0;
216 }
217 DPRINTF(iflib_get_dev(sc->ctx), "%s: pidx: %d cur: %d\n",
218 __FUNCTION__, pi->ipi_pidx, cur);
219 return (cur);
220 }
221
222 /*********************************************************************
223 * The offload context is protocol specific (TCP/UDP) and thus
224 * only needs to be set when the protocol changes. The occasion
225 * of a context change can be a performance detriment, and
226 * might be better just disabled. The reason arises in the way
227 * in which the controller supports pipelined requests from the
228 * Tx data DMA. Up to four requests can be pipelined, and they may
229 * belong to the same packet or to multiple packets. However all
230 * requests for one packet are issued before a request is issued
231 * for a subsequent packet and if a request for the next packet
232 * requires a context change, that request will be stalled
233 * until the previous request completes. This means setting up
234 * a new context effectively disables pipelined Tx data DMA which
235 * in turn greatly slow down performance to send small sized
236 * frames.
237 **********************************************************************/
238 #define DONT_FORCE_CTX 1
239
240 static int
em_transmit_checksum_setup(struct e1000_softc * sc,if_pkt_info_t pi,uint32_t * txd_upper,uint32_t * txd_lower)241 em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi,
242 uint32_t *txd_upper, uint32_t *txd_lower)
243 {
244 struct e1000_context_desc *TXD = NULL;
245 if_softc_ctx_t scctx = sc->shared;
246 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
247 struct tx_ring *txr = &que->txr;
248 int csum_flags = pi->ipi_csum_flags;
249 int cur, hdr_len;
250 uint32_t cmd;
251
252 cur = pi->ipi_pidx;
253 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
254 cmd = sc->txd_cmd;
255
256 /*
257 * The 82574L can only remember the *last* context used
258 * regardless of queue that it was use for. We cannot reuse
259 * contexts on this hardware platform and must generate a new
260 * context every time. 82574L hardware spec, section 7.2.6,
261 * second note.
262 */
263 if (DONT_FORCE_CTX &&
264 sc->tx_num_queues == 1 &&
265 txr->csum_lhlen == pi->ipi_ehdrlen &&
266 txr->csum_iphlen == pi->ipi_ip_hlen &&
267 txr->csum_flags == csum_flags) {
268 /*
269 * Same csum offload context as the previous packets;
270 * just return.
271 */
272 *txd_upper = txr->csum_txd_upper;
273 *txd_lower = txr->csum_txd_lower;
274 return (cur);
275 }
276
277 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
278 /*
279 * ipcss - Start offset for header checksum calculation.
280 * ipcse - End offset for header checksum calculation.
281 * ipcso - Offset of place to put the checksum.
282 *
283 * We set ipcsX values regardless of IP version to work around HW
284 * issues and ipcse must be 0 for IPv6 per "PCIe GbE SDM 2.5" page 61.
285 * IXSM controls whether it's inserted.
286 */
287 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
288 TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen +
289 offsetof(struct ip, ip_sum);
290 if (csum_flags & CSUM_IP) {
291 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
292 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len - 1);
293 cmd |= E1000_TXD_CMD_IP;
294 } else if (csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP))
295 TXD->lower_setup.ip_fields.ipcse = htole16(0);
296
297 /*
298 * tucss - Start offset for payload checksum calculation.
299 * tucse - End offset for payload checksum calculation.
300 * tucso - Offset of place to put the checksum.
301 */
302 if (csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_IP6_TCP |
303 CSUM_IP6_UDP)) {
304 uint8_t tucso;
305
306 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
307 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
308
309 if (csum_flags & (CSUM_TCP | CSUM_IP6_TCP)) {
310 tucso = hdr_len + offsetof(struct tcphdr, th_sum);
311 cmd |= E1000_TXD_CMD_TCP;
312 } else
313 tucso = hdr_len + offsetof(struct udphdr, uh_sum);
314 TXD->upper_setup.tcp_fields.tucss = hdr_len;
315 TXD->upper_setup.tcp_fields.tucse = htole16(0);
316 TXD->upper_setup.tcp_fields.tucso = tucso;
317 }
318
319 txr->csum_lhlen = pi->ipi_ehdrlen;
320 txr->csum_iphlen = pi->ipi_ip_hlen;
321 txr->csum_flags = csum_flags;
322 txr->csum_txd_upper = *txd_upper;
323 txr->csum_txd_lower = *txd_lower;
324
325 TXD->tcp_seg_setup.data = htole32(0);
326 TXD->cmd_and_length =
327 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);
328
329 if (++cur == scctx->isc_ntxd[0]) {
330 cur = 0;
331 }
332 DPRINTF(iflib_get_dev(sc->ctx),
333 "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x"
334 " hdr_len=%d cmd=%x\n",
335 csum_flags, *txd_upper, *txd_lower, hdr_len, cmd);
336 return (cur);
337 }
338
339 #define TSO_WORKAROUND 4 /* TSO sentinel descriptor */
340
341 static int
em_isc_txd_encap(void * arg,if_pkt_info_t pi)342 em_isc_txd_encap(void *arg, if_pkt_info_t pi)
343 {
344 struct e1000_softc *sc = arg;
345 if_softc_ctx_t scctx = sc->shared;
346 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
347 struct tx_ring *txr = &que->txr;
348 bus_dma_segment_t *segs = pi->ipi_segs;
349 int nsegs = pi->ipi_nsegs;
350 int csum_flags = pi->ipi_csum_flags;
351 int i, j, first, pidx_last;
352 uint32_t txd_flags, txd_upper = 0, txd_lower = 0;
353
354 struct e1000_tx_desc *ctxd = NULL;
355 bool do_tso, tso_desc;
356 qidx_t ntxd;
357
358 txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0;
359 i = first = pi->ipi_pidx;
360 do_tso = (csum_flags & CSUM_TSO);
361 tso_desc = false;
362 ntxd = scctx->isc_ntxd[0];
363 /*
364 * TSO Hardware workaround, if this packet is not
365 * TSO, and is only a single descriptor long, and
366 * it follows a TSO burst, then we need to add a
367 * sentinel descriptor to prevent premature writeback.
368 */
369 if ((!do_tso) && (txr->tx_tso == true)) {
370 if (nsegs == 1)
371 tso_desc = true;
372 txr->tx_tso = false;
373 }
374
375 /* Do hardware assists */
376 if (do_tso) {
377 i = em_tso_setup(sc, pi, &txd_upper, &txd_lower);
378 tso_desc = true;
379 } else if (csum_flags & EM_CSUM_OFFLOAD) {
380 i = em_transmit_checksum_setup(sc, pi, &txd_upper,
381 &txd_lower);
382 }
383
384 if (pi->ipi_mflags & M_VLANTAG) {
385 /* Set the vlan id. */
386 txd_upper |= htole16(pi->ipi_vtag) << 16;
387 /* Tell hardware to add tag */
388 txd_lower |= htole32(E1000_TXD_CMD_VLE);
389 }
390
391 DPRINTF(iflib_get_dev(sc->ctx),
392 "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i);
393 /* XXX sc->pcix_82544 -- lem_fill_descriptors */
394
395 /* Set up our transmit descriptors */
396 for (j = 0; j < nsegs; j++) {
397 bus_size_t seg_len;
398 bus_addr_t seg_addr;
399 uint32_t cmd;
400
401 ctxd = &txr->tx_base[i];
402 seg_addr = segs[j].ds_addr;
403 seg_len = segs[j].ds_len;
404 cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd;
405
406 /*
407 * TSO Workaround:
408 * If this is the last descriptor, we want to
409 * split it so we have a small final sentinel
410 */
411 if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
412 seg_len -= TSO_WORKAROUND;
413 ctxd->buffer_addr = htole64(seg_addr);
414 ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
415 ctxd->upper.data = htole32(txd_upper);
416
417 if (++i == scctx->isc_ntxd[0])
418 i = 0;
419
420 /* Now make the sentinel */
421 ctxd = &txr->tx_base[i];
422 ctxd->buffer_addr = htole64(seg_addr + seg_len);
423 ctxd->lower.data =
424 htole32(cmd | txd_lower | TSO_WORKAROUND);
425 ctxd->upper.data = htole32(txd_upper);
426 pidx_last = i;
427 if (++i == scctx->isc_ntxd[0])
428 i = 0;
429 DPRINTF(iflib_get_dev(sc->ctx),
430 "TSO path pidx_last=%d i=%d ntxd[0]=%d\n",
431 pidx_last, i, scctx->isc_ntxd[0]);
432 } else {
433 ctxd->buffer_addr = htole64(seg_addr);
434 ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
435 ctxd->upper.data = htole32(txd_upper);
436 pidx_last = i;
437 if (++i == scctx->isc_ntxd[0])
438 i = 0;
439 DPRINTF(iflib_get_dev(sc->ctx),
440 "pidx_last=%d i=%d ntxd[0]=%d\n",
441 pidx_last, i, scctx->isc_ntxd[0]);
442 }
443 }
444
445 /*
446 * Last Descriptor of Packet
447 * needs End Of Packet (EOP)
448 * and Report Status (RS)
449 */
450 if (txd_flags && nsegs) {
451 txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
452 DPRINTF(iflib_get_dev(sc->ctx),
453 "setting to RS on %d rs_pidx %d first: %d\n",
454 pidx_last, txr->tx_rs_pidx, first);
455 txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1);
456 MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx);
457 }
458 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags);
459 DPRINTF(iflib_get_dev(sc->ctx),
460 "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n",
461 first, pidx_last, i);
462 pi->ipi_new_pidx = i;
463
464 /* Sent data accounting for AIM */
465 txr->tx_bytes += pi->ipi_len;
466 ++txr->tx_packets;
467
468 return (0);
469 }
470
471 static void
em_isc_txd_flush(void * arg,uint16_t txqid,qidx_t pidx)472 em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
473 {
474 struct e1000_softc *sc = arg;
475 struct em_tx_queue *que = &sc->tx_queues[txqid];
476 struct tx_ring *txr = &que->txr;
477
478 E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx);
479 }
480
481 static int
em_isc_txd_credits_update(void * arg,uint16_t txqid,bool clear)482 em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
483 {
484 struct e1000_softc *sc = arg;
485 if_softc_ctx_t scctx = sc->shared;
486 struct em_tx_queue *que = &sc->tx_queues[txqid];
487 struct tx_ring *txr = &que->txr;
488
489 qidx_t processed = 0;
490 int updated;
491 qidx_t cur, prev, ntxd, rs_cidx;
492 int32_t delta;
493 uint8_t status;
494
495 rs_cidx = txr->tx_rs_cidx;
496 if (rs_cidx == txr->tx_rs_pidx)
497 return (0);
498 cur = txr->tx_rsq[rs_cidx];
499 MPASS(cur != QIDX_INVALID);
500 status = txr->tx_base[cur].upper.fields.status;
501 updated = !!(status & E1000_TXD_STAT_DD);
502
503 if (!updated)
504 return (0);
505
506 /* If clear is false just let caller know that there
507 * are descriptors to reclaim */
508 if (!clear)
509 return (1);
510
511 prev = txr->tx_cidx_processed;
512 ntxd = scctx->isc_ntxd[0];
513 do {
514 MPASS(prev != cur);
515 delta = (int32_t)cur - (int32_t)prev;
516 if (delta < 0)
517 delta += ntxd;
518 MPASS(delta > 0);
519 DPRINTF(iflib_get_dev(sc->ctx),
520 "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n",
521 __FUNCTION__, prev, cur, clear, delta);
522
523 processed += delta;
524 prev = cur;
525 rs_cidx = (rs_cidx + 1) & (ntxd-1);
526 if (rs_cidx == txr->tx_rs_pidx)
527 break;
528 cur = txr->tx_rsq[rs_cidx];
529 MPASS(cur != QIDX_INVALID);
530 status = txr->tx_base[cur].upper.fields.status;
531 } while ((status & E1000_TXD_STAT_DD));
532
533 txr->tx_rs_cidx = rs_cidx;
534 txr->tx_cidx_processed = prev;
535 return(processed);
536 }
537
538 static void
lem_isc_rxd_refill(void * arg,if_rxd_update_t iru)539 lem_isc_rxd_refill(void *arg, if_rxd_update_t iru)
540 {
541 struct e1000_softc *sc = arg;
542 if_softc_ctx_t scctx = sc->shared;
543 struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx];
544 struct rx_ring *rxr = &que->rxr;
545 struct e1000_rx_desc *rxd;
546 uint64_t *paddrs;
547 uint32_t next_pidx, pidx;
548 uint16_t count;
549 int i;
550
551 paddrs = iru->iru_paddrs;
552 pidx = iru->iru_pidx;
553 count = iru->iru_count;
554
555 for (i = 0, next_pidx = pidx; i < count; i++) {
556 rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx];
557 rxd->buffer_addr = htole64(paddrs[i]);
558 /* status bits must be cleared */
559 rxd->status = 0;
560
561 if (++next_pidx == scctx->isc_nrxd[0])
562 next_pidx = 0;
563 }
564 }
565
566 static void
em_isc_rxd_refill(void * arg,if_rxd_update_t iru)567 em_isc_rxd_refill(void *arg, if_rxd_update_t iru)
568 {
569 struct e1000_softc *sc = arg;
570 if_softc_ctx_t scctx = sc->shared;
571 uint16_t rxqid = iru->iru_qsidx;
572 struct em_rx_queue *que = &sc->rx_queues[rxqid];
573 struct rx_ring *rxr = &que->rxr;
574 union e1000_rx_desc_extended *rxd;
575 uint64_t *paddrs;
576 uint32_t next_pidx, pidx;
577 uint16_t count;
578 int i;
579
580 paddrs = iru->iru_paddrs;
581 pidx = iru->iru_pidx;
582 count = iru->iru_count;
583
584 for (i = 0, next_pidx = pidx; i < count; i++) {
585 rxd = &rxr->rx_base[next_pidx];
586 rxd->read.buffer_addr = htole64(paddrs[i]);
587 /* DD bits must be cleared */
588 rxd->wb.upper.status_error = 0;
589
590 if (++next_pidx == scctx->isc_nrxd[0])
591 next_pidx = 0;
592 }
593 }
594
595 static void
em_isc_rxd_flush(void * arg,uint16_t rxqid,uint8_t flid __unused,qidx_t pidx)596 em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
597 qidx_t pidx)
598 {
599 struct e1000_softc *sc = arg;
600 struct em_rx_queue *que = &sc->rx_queues[rxqid];
601 struct rx_ring *rxr = &que->rxr;
602
603 E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
604 }
605
606 static int
lem_isc_rxd_available(void * arg,uint16_t rxqid,qidx_t idx,qidx_t budget)607 lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
608 {
609 struct e1000_softc *sc = arg;
610 if_softc_ctx_t scctx = sc->shared;
611 struct em_rx_queue *que = &sc->rx_queues[rxqid];
612 struct rx_ring *rxr = &que->rxr;
613 struct e1000_rx_desc *rxd;
614 uint32_t staterr = 0;
615 int cnt, i;
616
617 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
618 rxd = (struct e1000_rx_desc *)&rxr->rx_base[i];
619 staterr = rxd->status;
620
621 if ((staterr & E1000_RXD_STAT_DD) == 0)
622 break;
623 if (++i == scctx->isc_nrxd[0])
624 i = 0;
625 if (staterr & E1000_RXD_STAT_EOP)
626 cnt++;
627 }
628 return (cnt);
629 }
630
631 static int
em_isc_rxd_available(void * arg,uint16_t rxqid,qidx_t idx,qidx_t budget)632 em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
633 {
634 struct e1000_softc *sc = arg;
635 if_softc_ctx_t scctx = sc->shared;
636 struct em_rx_queue *que = &sc->rx_queues[rxqid];
637 struct rx_ring *rxr = &que->rxr;
638 union e1000_rx_desc_extended *rxd;
639 uint32_t staterr = 0;
640 int cnt, i;
641
642 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
643 rxd = &rxr->rx_base[i];
644 staterr = le32toh(rxd->wb.upper.status_error);
645
646 if ((staterr & E1000_RXD_STAT_DD) == 0)
647 break;
648 if (++i == scctx->isc_nrxd[0])
649 i = 0;
650 if (staterr & E1000_RXD_STAT_EOP)
651 cnt++;
652 }
653 return (cnt);
654 }
655
656 static int
lem_isc_rxd_pkt_get(void * arg,if_rxd_info_t ri)657 lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
658 {
659 struct e1000_softc *sc = arg;
660 if_softc_ctx_t scctx = sc->shared;
661 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
662 struct rx_ring *rxr = &que->rxr;
663 struct e1000_rx_desc *rxd;
664 uint16_t len;
665 uint32_t status, errors;
666 bool eop;
667 int i, cidx;
668
669 status = errors = i = 0;
670 cidx = ri->iri_cidx;
671
672 do {
673 rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx];
674 status = rxd->status;
675 errors = rxd->errors;
676
677 /* Error Checking then decrement count */
678 MPASS ((status & E1000_RXD_STAT_DD) != 0);
679
680 len = le16toh(rxd->length);
681 ri->iri_len += len;
682 rxr->rx_bytes += ri->iri_len;
683
684 eop = (status & E1000_RXD_STAT_EOP) != 0;
685
686 /* Make sure bad packets are discarded */
687 if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
688 sc->dropped_pkts++;
689 /* XXX fixup if common */
690 return (EBADMSG);
691 }
692
693 ri->iri_frags[i].irf_flid = 0;
694 ri->iri_frags[i].irf_idx = cidx;
695 ri->iri_frags[i].irf_len = len;
696 /* Zero out the receive descriptors status. */
697 rxd->status = 0;
698
699 if (++cidx == scctx->isc_nrxd[0])
700 cidx = 0;
701 i++;
702 } while (!eop);
703
704 rxr->rx_packets++;
705
706 if (scctx->isc_capenable & IFCAP_RXCSUM)
707 em_receive_checksum(status, errors, ri);
708
709 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING &&
710 status & E1000_RXD_STAT_VP) {
711 ri->iri_vtag = le16toh(rxd->special);
712 ri->iri_flags |= M_VLANTAG;
713 }
714
715 ri->iri_nfrags = i;
716
717 return (0);
718 }
719
720 static int
em_isc_rxd_pkt_get(void * arg,if_rxd_info_t ri)721 em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
722 {
723 struct e1000_softc *sc = arg;
724 if_softc_ctx_t scctx = sc->shared;
725 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
726 struct rx_ring *rxr = &que->rxr;
727 union e1000_rx_desc_extended *rxd;
728
729 uint16_t len;
730 uint32_t pkt_info;
731 uint32_t staterr;
732 bool eop;
733 int i, cidx;
734
735 staterr = i = 0;
736 cidx = ri->iri_cidx;
737
738 do {
739 rxd = &rxr->rx_base[cidx];
740 staterr = le32toh(rxd->wb.upper.status_error);
741 pkt_info = le32toh(rxd->wb.lower.mrq);
742
743 /* Error Checking then decrement count */
744 MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
745
746 len = le16toh(rxd->wb.upper.length);
747 ri->iri_len += len;
748 rxr->rx_bytes += ri->iri_len;
749
750 eop = (staterr & E1000_RXD_STAT_EOP) != 0;
751
752 /* Make sure bad packets are discarded */
753 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
754 sc->dropped_pkts++;
755 return EBADMSG;
756 }
757
758 ri->iri_frags[i].irf_flid = 0;
759 ri->iri_frags[i].irf_idx = cidx;
760 ri->iri_frags[i].irf_len = len;
761 /* Zero out the receive descriptors status. */
762 rxd->wb.upper.status_error &= htole32(~0xFF);
763
764 if (++cidx == scctx->isc_nrxd[0])
765 cidx = 0;
766 i++;
767 } while (!eop);
768
769 rxr->rx_packets++;
770
771 if (scctx->isc_capenable & IFCAP_RXCSUM)
772 em_receive_checksum(staterr, staterr >> 24, ri);
773
774 if (scctx->isc_capenable & IFCAP_VLAN_HWTAGGING &&
775 staterr & E1000_RXD_STAT_VP) {
776 ri->iri_vtag = le16toh(rxd->wb.upper.vlan);
777 ri->iri_flags |= M_VLANTAG;
778 }
779
780 ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss);
781 ri->iri_rsstype = em_determine_rsstype(pkt_info);
782
783 ri->iri_nfrags = i;
784 return (0);
785 }
786
787 /*********************************************************************
788 *
789 * Verify that the hardware indicated that the checksum is valid.
790 * Inform the stack about the status of checksum so that stack
791 * doesn't spend time verifying the checksum.
792 *
793 *********************************************************************/
794 static void
em_receive_checksum(uint16_t status,uint8_t errors,if_rxd_info_t ri)795 em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri)
796 {
797 if (__predict_false(status & E1000_RXD_STAT_IXSM))
798 return;
799
800 /* If there is a layer 3 or 4 error we are done */
801 if (__predict_false(errors & (E1000_RXD_ERR_IPE |
802 E1000_RXD_ERR_TCPE)))
803 return;
804
805 /* IP Checksum Good */
806 if (status & E1000_RXD_STAT_IPCS)
807 ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
808
809 /* Valid L4E checksum */
810 if (__predict_true(status &
811 (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) {
812 ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
813 ri->iri_csum_data = htons(0xffff);
814 }
815 }
816
817 /********************************************************************
818 *
819 * Parse the packet type to determine the appropriate hash
820 *
821 ******************************************************************/
822 static int
em_determine_rsstype(uint32_t pkt_info)823 em_determine_rsstype(uint32_t pkt_info)
824 {
825 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
826 case E1000_RXDADV_RSSTYPE_IPV4_TCP:
827 return M_HASHTYPE_RSS_TCP_IPV4;
828 case E1000_RXDADV_RSSTYPE_IPV4:
829 return M_HASHTYPE_RSS_IPV4;
830 case E1000_RXDADV_RSSTYPE_IPV6_TCP:
831 return M_HASHTYPE_RSS_TCP_IPV6;
832 case E1000_RXDADV_RSSTYPE_IPV6_EX:
833 return M_HASHTYPE_RSS_IPV6_EX;
834 case E1000_RXDADV_RSSTYPE_IPV6:
835 return M_HASHTYPE_RSS_IPV6;
836 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
837 return M_HASHTYPE_RSS_TCP_IPV6_EX;
838 default:
839 return M_HASHTYPE_OPAQUE;
840 }
841 }
842