1 /*-
2 * Copyright (c) 2026 Justin Hibbits
3 * Copyright (c) 2012 Semihalf.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/kernel.h>
31 #include <sys/module.h>
32 #include <sys/bus.h>
33 #include <sys/rman.h>
34 #include <sys/malloc.h>
35 #include <sys/mbuf.h>
36 #include <sys/smp.h>
37 #include <sys/socket.h>
38 #include <sys/sockio.h>
39 #include <sys/sysctl.h>
40
41 #include <net/ethernet.h>
42 #include <net/if.h>
43 #include <net/if_dl.h>
44 #include <net/if_media.h>
45 #include <net/if_types.h>
46 #include <net/if_arp.h>
47 #include <netinet/ip.h>
48 #include <netinet/ip6.h>
49 #include <netinet/tcp_lro.h>
50
51 #include <dev/mii/mii.h>
52 #include <dev/mii/miivar.h>
53
54 #include <vm/vm.h>
55 #include <vm/pmap.h>
56
57 #include "miibus_if.h"
58
59 #include "bman.h"
60 #include "dpaa_common.h"
61 #include "dpaa_eth.h"
62 #include "fman.h"
63 #include "fman_keygen.h"
64 #include "fman_parser.h"
65 #include "fman_port.h"
66 #include "fman_if.h"
67 #include "fman_port_if.h"
68 #include "if_dtsec.h"
69 #include "qman.h"
70 #include "qman_var.h"
71 #include "qman_portal_if.h"
72
73
74 #define DPAA_ETH_LOCK(sc) mtx_lock(&(sc)->sc_lock)
75 #define DPAA_ETH_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock)
76 #define DPAA_ETH_LOCK_ASSERT(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED)
77
78 /*
79 * On 64-bit Book-E the direct map is always present, and the driver's
80 * UMA zones plus page-sized mbuf clusters live in it. Bypass the
81 * page-table walk in pmap_kextract() for those; fall back for
82 * MJUM9BYTES/MJUM16BYTES clusters, which are kmem_alloc_contig()'d
83 * into KVA.
84 */
85 static inline vm_paddr_t
dpaa_eth_va_to_phys(vm_offset_t va)86 dpaa_eth_va_to_phys(vm_offset_t va)
87 {
88 if (__predict_true(va >= DMAP_BASE_ADDRESS && va <= DMAP_MAX_ADDRESS))
89 return (DMAP_TO_PHYS(va));
90 return (pmap_kextract(va));
91 }
92
93 /**
94 * @group dTSEC RM private defines.
95 * @{
96 */
97 #define DTSEC_BPOOLS_USED (1)
98 #define DTSEC_MAX_TX_QUEUE_LEN 256
99 /*
100 * Sample the hardware TX FQ counter every Nth packet. The FQ counter is
101 * 24 bits and the soft cap above is 256, so overshoot by N is trivial.
102 */
103 #define DTSEC_MAX_TX_QUEUE_CHECK_INTERVAL 32
104 /*
105 * Confirmation callback drain-detection. Fast path (TX not backpressured)
106 * skips the MC call entirely; when flagged, we sample every Nth callback to
107 * detect the drain-to-zero transition.
108 */
109 #define DTSEC_TX_CONF_CHECK_INTERVAL 32
110
111 struct dpaa_eth_frame_info {
112 struct fman_internal_context fi_ic;
113 struct mbuf *fi_mbuf;
114 struct dpaa_sgte fi_sgt[DPAA_NUM_OF_SG_TABLE_ENTRY];
115 };
116
117 /*
118 * Loader-tunable override for the per-port RX FQ count. 0 (default)
119 * means "auto". Otherwise must be a power of two.
120 */
121 static int dpaa_eth_nrxfqs_tunable = 0;
122 SYSCTL_NODE(_hw, OID_AUTO, dpaa, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
123 "DPAA driver tunables");
124 static SYSCTL_NODE(_hw_dpaa, OID_AUTO, eth, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
125 "DPAA Ethernet driver");
126 SYSCTL_INT(_hw_dpaa_eth, OID_AUTO, nrxfqs, CTLFLAG_RDTUN,
127 &dpaa_eth_nrxfqs_tunable, 0,
128 "Per-port RX FQ count override (0=auto, else power-of-2 in [1,8])");
129
130 enum dpaa_eth_pool_params {
131 DTSEC_RM_POOL_RX_LOW_MARK = 16,
132 DTSEC_RM_POOL_RX_HIGH_MARK = 64,
133 DTSEC_RM_POOL_RX_MAX_SIZE = 256,
134 /*
135 * MAX_SIZE is a soft cap set well below the BMan hardware pool
136 * limit, so sampling the depth every N put-backs per CPU is safe:
137 * worst-case overshoot is N * ncpus buffers, still tiny vs. the
138 * hardware pool.
139 */
140 DTSEC_RM_POOL_RX_CHECK_INTERVAL = 32,
141
142 DTSEC_RM_POOL_FI_LOW_MARK = 16,
143 DTSEC_RM_POOL_FI_HIGH_MARK = 64,
144 DTSEC_RM_POOL_FI_MAX_SIZE = 256,
145 };
146
147 enum dpaa_eth_fq_params {
148 DTSEC_RM_FQR_RX_WQ = 1,
149 DTSEC_RM_FQR_TX_WQ = 1,
150 DTSEC_RM_FQR_TX_CONF_WQ = 1
151 };
152 /** @} */
153
154
155 /**
156 * @group dTSEC Frame Info routines.
157 * @{
158 */
159 void
dpaa_eth_fi_pool_free(struct dpaa_eth_softc * sc)160 dpaa_eth_fi_pool_free(struct dpaa_eth_softc *sc)
161 {
162
163 if (sc->sc_fi_zone != NULL)
164 uma_zdestroy(sc->sc_fi_zone);
165 }
166
167 int
dpaa_eth_fi_pool_init(struct dpaa_eth_softc * sc)168 dpaa_eth_fi_pool_init(struct dpaa_eth_softc *sc)
169 {
170
171 snprintf(sc->sc_fi_zname, sizeof(sc->sc_fi_zname), "%s: Frame Info",
172 device_get_nameunit(sc->sc_dev));
173
174 sc->sc_fi_zone = uma_zcreate(sc->sc_fi_zname,
175 sizeof(struct dpaa_eth_frame_info), NULL, NULL, NULL, NULL,
176 UMA_ALIGN_PTR, 0);
177
178 return (0);
179 }
180
181 static struct dpaa_eth_frame_info *
dpaa_eth_fi_alloc(struct dpaa_eth_softc * sc)182 dpaa_eth_fi_alloc(struct dpaa_eth_softc *sc)
183 {
184 struct dpaa_eth_frame_info *fi;
185
186 fi = uma_zalloc(sc->sc_fi_zone, M_NOWAIT | M_ZERO);
187
188 return (fi);
189 }
190
191 static void
dpaa_eth_fi_free(struct dpaa_eth_softc * sc,struct dpaa_eth_frame_info * fi)192 dpaa_eth_fi_free(struct dpaa_eth_softc *sc, struct dpaa_eth_frame_info *fi)
193 {
194
195 uma_zfree(sc->sc_fi_zone, fi);
196 }
197 /** @} */
198
199
200 /**
201 * @group dTSEC FMan PORT routines.
202 * @{
203 */
204 int
dpaa_eth_fm_port_rx_init(struct dpaa_eth_softc * sc)205 dpaa_eth_fm_port_rx_init(struct dpaa_eth_softc *sc)
206 {
207 struct fman_port_params params;
208 int error;
209
210 /*
211 * dflt/err FQID is the base of the RSS range: non-hashable
212 * frames (ARP, IP fragments, non-IP) fall through to FQ #0.
213 */
214 params.dflt_fqid = sc->sc_rx_fqid_base;
215 params.err_fqid = sc->sc_rx_fqid_base;
216 params.rx_params.num_pools = 1;
217 params.rx_params.bpools[0].bpid = bman_get_bpid(sc->sc_rx_pool);
218 params.rx_params.bpools[0].size = MCLBYTES;
219 error = FMAN_PORT_CONFIG(sc->sc_rx_port, ¶ms);
220 error = FMAN_PORT_INIT(sc->sc_rx_port);
221 if (error != 0) {
222 device_printf(sc->sc_dev, "couldn't initialize FM Port RX.\n");
223 return (ENXIO);
224 }
225
226 /*
227 * The RX port's own FMan hardware port ID (cell-index in the
228 * OFW node) is the index KG uses for scheme-binding. It was
229 * previously left at zero, which made every port fight for KG
230 * port 0 -- only the first attach won, the rest got EBUSY.
231 */
232 sc->sc_port_rx_hw_id = fman_port_get_id(sc->sc_rx_port);
233
234 /*
235 * Wire up FMan KeyGen 5-tuple hashing across the sc_nrxfqs FQs
236 * created in dpaa_eth_fq_rx_init. Skipped for the trivial N=1
237 * case (single-core system or forced fallback): with one FQ
238 * there's nothing to distribute. Only flip the port's parser
239 * output to KG on success -- routing to KG with no bound scheme
240 * drops frames.
241 */
242 if (sc->sc_nrxfqs > 1) {
243 struct fman_softc *fman_sc =
244 device_get_softc(device_get_parent(sc->sc_rx_port));
245
246 error = fman_kg_alloc_hash_scheme(fman_sc,
247 sc->sc_port_rx_hw_id, sc->sc_rx_fqid_base,
248 sc->sc_nrxfqs);
249 if (error != 0) {
250 device_printf(sc->sc_dev,
251 "fman_kg_alloc_hash_scheme failed: %d\n", error);
252 /*
253 * Non-fatal: FMan port still delivers to
254 * dflt_fqid == sc_rx_fqid_base (FQ #0), which is
255 * a valid single-queue fallback. Leave RFPNE at
256 * the BMI-enqueue default.
257 */
258 } else {
259 fman_port_rx_use_kg(sc->sc_rx_port, true);
260 }
261 }
262
263 return (0);
264 }
265
266 int
dpaa_eth_fm_port_tx_init(struct dpaa_eth_softc * sc)267 dpaa_eth_fm_port_tx_init(struct dpaa_eth_softc *sc)
268 {
269 struct fman_port_params params;
270 int error;
271
272 params.dflt_fqid = sc->sc_tx_conf_fqid;
273 params.err_fqid = sc->sc_tx_conf_fqid;
274
275 error = FMAN_PORT_CONFIG(sc->sc_tx_port, ¶ms);
276 error = FMAN_PORT_INIT(sc->sc_tx_port);
277 if (error != 0) {
278 device_printf(sc->sc_dev, "couldn't initialize FM Port TX.\n");
279 return (ENXIO);
280 }
281
282 return (0);
283 }
284 /** @} */
285
286
287 /**
288 * @group dTSEC buffer pools routines.
289 * @{
290 */
291 static int
dpaa_eth_pool_rx_put_buffer(struct dpaa_eth_softc * sc,uint8_t * buffer,void * context)292 dpaa_eth_pool_rx_put_buffer(struct dpaa_eth_softc *sc, uint8_t *buffer,
293 void *context)
294 {
295
296 uma_zfree(sc->sc_rx_zone, buffer);
297
298 return (0);
299 }
300
301 static int
dtsec_add_buffers(struct dpaa_eth_softc * sc,int count)302 dtsec_add_buffers(struct dpaa_eth_softc *sc, int count)
303 {
304 struct bman_buffer bufs[8] = {};
305 int err;
306 int c;
307
308 while (count > 0) {
309 c = min(8, count);
310 for (int i = 0; i < c; i++) {
311 void *b;
312 vm_paddr_t pa;
313
314 b = uma_zalloc(sc->sc_rx_zone, M_NOWAIT);
315 if (b == NULL)
316 return (ENOMEM);
317 pa = DMAP_TO_PHYS((vm_offset_t)b);
318 bufs[i].buf_hi = (pa >> 32);
319 bufs[i].buf_lo = (pa & 0xffffffff);
320 }
321
322 err = bman_put_buffers(sc->sc_rx_pool, bufs, c);
323 if (err != 0)
324 return (err);
325 count -= c;
326 }
327
328 return (0);
329 }
330
331 static void
dpaa_eth_pool_rx_depleted(void * h_App,bool in)332 dpaa_eth_pool_rx_depleted(void *h_App, bool in)
333 {
334 struct dpaa_eth_softc *sc;
335 unsigned int count;
336
337 sc = h_App;
338
339 if (!in)
340 return;
341
342 while (1) {
343 count = bman_count(sc->sc_rx_pool);
344 if (count > DTSEC_RM_POOL_RX_HIGH_MARK)
345 return;
346
347 /* Can only release 8 buffers at a time */
348 count = min(DTSEC_RM_POOL_RX_HIGH_MARK - count + 8, 8);
349 if (dtsec_add_buffers(sc, count) != 0)
350 return;
351 }
352 }
353
354 void
dpaa_eth_pool_rx_free(struct dpaa_eth_softc * sc)355 dpaa_eth_pool_rx_free(struct dpaa_eth_softc *sc)
356 {
357
358 if (sc->sc_rx_pool != NULL)
359 bman_pool_destroy(sc->sc_rx_pool);
360
361 if (sc->sc_rx_zone != NULL)
362 uma_zdestroy(sc->sc_rx_zone);
363
364 free(sc->sc_rx_pool_check_cnt, M_DEVBUF);
365 sc->sc_rx_pool_check_cnt = NULL;
366 }
367
368 int
dpaa_eth_pool_rx_init(struct dpaa_eth_softc * sc)369 dpaa_eth_pool_rx_init(struct dpaa_eth_softc *sc)
370 {
371
372 /* MCLBYTES must be less than PAGE_SIZE */
373 CTASSERT(MCLBYTES < PAGE_SIZE);
374
375 snprintf(sc->sc_rx_zname, sizeof(sc->sc_rx_zname), "%s: RX Buffers",
376 device_get_nameunit(sc->sc_dev));
377
378 sc->sc_rx_zone = uma_zcreate(sc->sc_rx_zname, MCLBYTES, NULL,
379 NULL, NULL, NULL, MCLBYTES - 1, 0);
380
381 sc->sc_rx_pool_check_cnt = malloc_aligned(
382 (mp_maxid + 1) * sizeof(struct dpaa_pcpu_cnt),
383 CACHE_LINE_SIZE, M_DEVBUF, M_WAITOK | M_ZERO);
384
385 sc->sc_rx_pool = bman_pool_create(&sc->sc_rx_bpid, MCLBYTES,
386 DTSEC_RM_POOL_RX_MAX_SIZE, DTSEC_RM_POOL_RX_LOW_MARK,
387 DTSEC_RM_POOL_RX_HIGH_MARK, 0, 0, dpaa_eth_pool_rx_depleted, sc);
388 if (sc->sc_rx_pool == NULL) {
389 device_printf(sc->sc_dev, "NULL rx pool somehow\n");
390 dpaa_eth_pool_rx_free(sc);
391 return (EIO);
392 }
393
394 dtsec_add_buffers(sc, DTSEC_RM_POOL_RX_HIGH_MARK);
395
396 return (0);
397 }
398 /** @} */
399
400
401 /**
402 * @group dTSEC Frame Queue Range routines.
403 * @{
404 */
405 static void
dpaa_eth_fq_mext_free(struct mbuf * m)406 dpaa_eth_fq_mext_free(struct mbuf *m)
407 {
408 struct dpaa_eth_softc *sc;
409 void *buffer;
410
411 buffer = m->m_ext.ext_arg1;
412 sc = m->m_ext.ext_arg2;
413 /*
414 * Sloppy per-CPU sampling: no pin, no atomic. A stray migration
415 * between the curcpu read and the increment can only mis-attribute
416 * one bump to the wrong CPU's counter; the sampling rate stays
417 * within the acceptable slop window.
418 */
419 if ((++sc->sc_rx_pool_check_cnt[curcpu].cnt &
420 (DTSEC_RM_POOL_RX_CHECK_INTERVAL - 1)) == 0 &&
421 bman_count(sc->sc_rx_pool) > DTSEC_RM_POOL_RX_MAX_SIZE)
422 dpaa_eth_pool_rx_put_buffer(sc, buffer, NULL);
423 else
424 bman_put_buffer(sc->sc_rx_pool,
425 DMAP_TO_PHYS((vm_offset_t)buffer), sc->sc_rx_bpid);
426 }
427
428 static int
dpaa_eth_update_csum_flags(struct qman_fd * frame,struct fman_parse_result * prs,struct mbuf * m)429 dpaa_eth_update_csum_flags(struct qman_fd *frame,
430 struct fman_parse_result *prs, struct mbuf *m)
431 {
432 uint16_t l3r = be16toh(prs->l3r);
433
434 /* TODO: nested protocols? */
435 if ((l3r & L3R_FIRST_IP_M) != 0) {
436 m->m_pkthdr.csum_flags |= CSUM_L3_CALC;
437 if ((l3r & L3R_FIRST_ERROR) == 0)
438 m->m_pkthdr.csum_flags |= CSUM_L3_VALID;
439 }
440 if (frame->cmd_stat & DPAA_FD_RX_STATUS_L4CV) {
441 m->m_pkthdr.csum_flags |= CSUM_L4_CALC;
442 m->m_pkthdr.csum_data = 0xffff;
443 if ((prs->l4r & L4R_TYPE_M) != 0 &&
444 (prs->l4r & L4R_ERR) == 0)
445 m->m_pkthdr.csum_flags |= CSUM_L4_VALID;
446 }
447
448 return (0);
449 }
450
451 static int
dpaa_eth_fq_rx_callback(device_t portal,struct qman_fq * fq,struct qman_fd * frame,void * app)452 dpaa_eth_fq_rx_callback(device_t portal, struct qman_fq *fq,
453 struct qman_fd *frame, void *app)
454 {
455 struct dpaa_eth_rx_fq *rxfq;
456 struct dpaa_eth_softc *sc;
457 struct mbuf *m;
458 struct fman_internal_context *frame_ic;
459 void *frame_va;
460
461 m = NULL;
462 rxfq = app;
463 sc = rxfq->sc;
464 rxfq->frames_in++;
465
466 frame_va = DPAA_FD_GET_ADDR(frame);
467 frame_ic = frame_va; /* internal context at head of the frame */
468 /* Only simple (single- or multi-) frames are supported. */
469 KASSERT(frame->format == 0 || frame->format == 4,
470 ("%s(): Got unsupported frame format 0x%02X!", __func__,
471 frame->format));
472
473 if ((frame->cmd_stat & DPAA_FD_CMD_STAT_ERR_M) != 0) {
474 device_printf(sc->sc_dev, "RX error: 0x%08X\n",
475 frame->cmd_stat);
476 goto err;
477 }
478
479 m = m_gethdr(M_NOWAIT, MT_HEADER);
480 if (m == NULL)
481 goto err;
482
483 if (frame->format == 0) {
484 /* Single-frame format */
485 m_extadd(m, (char *)frame_va + frame->offset, frame->length,
486 dpaa_eth_fq_mext_free, frame_va, sc, 0, EXT_NET_DRV);
487 } else {
488 struct dpaa_sgte *sgt =
489 (struct dpaa_sgte *)(char *)frame_va + frame->offset;
490 /* Simple multi-frame format */
491 for (int i = 0; i < DPAA_NUM_OF_SG_TABLE_ENTRY; i++) {
492 if (sgt[i].length > 0)
493 m_extadd(m, PHYS_TO_DMAP(sgt[i].addr),
494 sgt[i].length, dpaa_eth_fq_mext_free,
495 PHYS_TO_DMAP(sgt[i].addr), sc, 0,
496 EXT_NET_DRV);
497 if (sgt[i].final)
498 break;
499 }
500 /* Free the SGT buffer, it's no longer needed. */
501 bman_put_buffer(sc->sc_rx_pool, frame->addr, sc->sc_rx_bpid);
502 }
503
504 if (if_getcapenable(sc->sc_ifnet) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
505 dpaa_eth_update_csum_flags(frame, &frame_ic->prs, m);
506
507 m->m_pkthdr.rcvif = sc->sc_ifnet;
508 m->m_len = frame->length;
509 m_fixhdr(m);
510
511 /*
512 * Offer to LRO first.
513 */
514 if (rxfq->lro_inited &&
515 (if_getcapenable(sc->sc_ifnet) & IFCAP_LRO) != 0 &&
516 (m->m_pkthdr.csum_flags & (CSUM_L4_CALC | CSUM_L4_VALID)) ==
517 (CSUM_L4_CALC | CSUM_L4_VALID) &&
518 tcp_lro_rx(&rxfq->lro, m, 0) == 0)
519 return (1);
520
521 m->m_nextpkt = NULL;
522 *rxfq->rx_tailp = m;
523 rxfq->rx_tailp = &m->m_nextpkt;
524
525 return (1);
526
527 err:
528 bman_put_buffer(sc->sc_rx_pool, frame->addr, sc->sc_rx_bpid);
529 if (m != NULL)
530 m_freem(m);
531
532 return (1);
533 }
534
535 /*
536 * Post-poll flush hook invoked once per QMan portal poll on any
537 * RX FQ that dispatched at least one frame this cycle. Runs on
538 * the FQ's affine CPU, outside the DQRR dispatch loop.
539 */
540 static void
dpaa_eth_fq_rx_flush(struct qman_fq * fq __unused,void * ctx)541 dpaa_eth_fq_rx_flush(struct qman_fq *fq __unused, void *ctx)
542 {
543 struct dpaa_eth_rx_fq *rxfq = ctx;
544
545 if (rxfq->rx_head != NULL) {
546 struct mbuf *chain = rxfq->rx_head;
547
548 rxfq->rx_head = NULL;
549 rxfq->rx_tailp = &rxfq->rx_head;
550 if_input(rxfq->sc->sc_ifnet, chain);
551 }
552 /*
553 * Flush LRO whenever initialised. If the user disabled
554 * IFCAP_LRO between the last callback and this flush, entries
555 * queued in that window still need to be drained.
556 */
557 if (rxfq->lro_inited)
558 tcp_lro_flush_all(&rxfq->lro);
559 }
560
561 static int
dpaa_eth_fq_tx_confirm_callback(device_t portal,struct qman_fq * fq,struct qman_fd * frame,void * app)562 dpaa_eth_fq_tx_confirm_callback(device_t portal, struct qman_fq *fq,
563 struct qman_fd *frame, void *app)
564 {
565 struct dpaa_eth_frame_info *fi;
566 struct dpaa_eth_softc *sc;
567
568 sc = app;
569
570 if ((frame->cmd_stat & DPAA_FD_TX_STAT_ERR_M) != 0)
571 device_printf(sc->sc_dev, "TX error: 0x%08X\n",
572 frame->cmd_stat);
573
574 /*
575 * We are storing struct dpaa_eth_frame_info in first entry
576 * of scatter-gather table.
577 */
578 fi = (struct dpaa_eth_frame_info *)PHYS_TO_DMAP(frame->addr);
579
580 /* Free transmitted frame */
581 m_freem(fi->fi_mbuf);
582 dpaa_eth_fi_free(sc, fi);
583
584 /*
585 * Fast path: TX isn't backpressured, so there's nothing to
586 * restart. Acquire load pairs with the release store on the
587 * TX path so a concurrent set of the flag is observed here.
588 */
589 if (atomic_load_acq_int(&sc->sc_tx_fq_full) == 0)
590 return (1);
591
592 /* Rate-limit the MC round-trip to detect drain-to-zero. */
593 if ((sc->sc_tx_conf_check_cnt++ &
594 (DTSEC_TX_CONF_CHECK_INTERVAL - 1)) != 0)
595 return (1);
596 if (qman_fq_get_counter(sc->sc_tx_conf_fq, QMAN_COUNTER_FRAME) != 0)
597 return (1);
598
599 DPAA_ETH_LOCK(sc);
600 if (sc->sc_tx_fq_full) {
601 atomic_store_rel_int(&sc->sc_tx_fq_full, 0);
602 dpaa_eth_if_start_locked(sc);
603 }
604 DPAA_ETH_UNLOCK(sc);
605
606 return (1);
607 }
608
609 void
dpaa_eth_fq_rx_free(struct dpaa_eth_softc * sc)610 dpaa_eth_fq_rx_free(struct dpaa_eth_softc *sc)
611 {
612 int i;
613
614 /*
615 * Tear down the KG scheme first so no new frames land on FQs
616 * about to be retired. Point the parser output back at BMI-
617 * enqueue before freeing the scheme so the port keeps
618 * delivering to dflt_fqid instead of dropping through an
619 * emptied KG.
620 */
621 if (sc->sc_nrxfqs > 1 && sc->sc_rx_port != NULL) {
622 struct fman_softc *fman_sc =
623 device_get_softc(device_get_parent(sc->sc_rx_port));
624
625 fman_port_rx_use_kg(sc->sc_rx_port, false);
626 (void)fman_kg_free_hash_scheme(fman_sc,
627 sc->sc_port_rx_hw_id);
628 }
629
630 if (sc->sc_rx_fqs != NULL) {
631 for (i = 0; i < sc->sc_nrxfqs; i++) {
632 if (sc->sc_rx_fqs[i].fq != NULL)
633 qman_fq_free(sc->sc_rx_fqs[i].fq);
634 /*
635 * Any pending non-LRO mbufs on the batch chain
636 * are dropped here rather than delivered late.
637 */
638 if (sc->sc_rx_fqs[i].rx_head != NULL)
639 m_freem(sc->sc_rx_fqs[i].rx_head);
640 if (sc->sc_rx_fqs[i].lro_inited)
641 tcp_lro_free(&sc->sc_rx_fqs[i].lro);
642 }
643 free(sc->sc_rx_fqs, M_DEVBUF);
644 sc->sc_rx_fqs = NULL;
645 }
646 if (sc->sc_nrxfqs > 0) {
647 qman_free_fqid_range(sc->sc_rx_fqid_base, sc->sc_nrxfqs);
648 sc->sc_nrxfqs = 0;
649 sc->sc_rx_fqid_base = 0;
650 }
651 }
652
653 int
dpaa_eth_fq_rx_init(struct dpaa_eth_softc * sc)654 dpaa_eth_fq_rx_init(struct dpaa_eth_softc *sc)
655 {
656 struct qman_fq *fq;
657 uint32_t base_fqid;
658 int align, nfqs;
659 int error, i;
660
661 if (dpaa_eth_nrxfqs_tunable > 0)
662 nfqs = dpaa_eth_nrxfqs_tunable;
663 else
664 nfqs = mp_ncpus;
665
666 align = 1 << ilog2(nfqs);
667 error = qman_alloc_fqid_range(nfqs, align, &base_fqid);
668 if (error != 0) {
669 device_printf(sc->sc_dev,
670 "could not reserve %d contiguous FQIDs (aligned): %d\n",
671 nfqs, error);
672 return (EIO);
673 }
674
675 sc->sc_nrxfqs = nfqs;
676 sc->sc_rx_fqid_base = base_fqid;
677 sc->sc_rx_fqs = malloc(nfqs * sizeof(*sc->sc_rx_fqs),
678 M_DEVBUF, M_WAITOK | M_ZERO);
679
680 /*
681 * Create the N RX FQs, one per per-CPU channel. QMan portal
682 * attach has already subscribed each portal to its own
683 * per-CPU channel, so frames land on the right core without
684 * per-driver static-dequeue plumbing.
685 *
686 * Stash 1 cacheline of frame annotation (parse result / IC)
687 * and 1 of frame data head into the destination core's cache
688 * when QMan dequeues an RX frame -- the RX callback reads
689 * both.
690 */
691 for (i = 0; i < nfqs; i++) {
692 int chan = qman_percpu_channel(i);
693
694 if (chan == -1) {
695 device_printf(sc->sc_dev,
696 "no per-CPU QMan channel for CPU %d\n", i);
697 error = EIO;
698 goto err;
699 }
700 fq = qman_fq_create(1, chan, DTSEC_RM_FQR_RX_WQ,
701 /*force_fqid=*/true, base_fqid + i,
702 false, false, true, false, 0, 0, 0, 1, 1);
703 if (fq == NULL) {
704 device_printf(sc->sc_dev,
705 "could not create RX FQ %d (fqid 0x%x)\n",
706 i, base_fqid + i);
707 error = EIO;
708 goto err;
709 }
710 sc->sc_rx_fqs[i].fq = fq;
711 sc->sc_rx_fqs[i].fqid = base_fqid + i;
712 sc->sc_rx_fqs[i].cpu = i;
713 sc->sc_rx_fqs[i].sc = sc;
714 sc->sc_rx_fqs[i].rx_head = NULL;
715 sc->sc_rx_fqs[i].rx_tailp = &sc->sc_rx_fqs[i].rx_head;
716
717 /* Best-effort LRO per FQ. */
718 if (tcp_lro_init(&sc->sc_rx_fqs[i].lro) == 0) {
719 sc->sc_rx_fqs[i].lro.ifp = sc->sc_ifnet;
720 sc->sc_rx_fqs[i].lro_inited = true;
721 }
722
723 error = qman_fq_register_cb(fq, dpaa_eth_fq_rx_callback,
724 &sc->sc_rx_fqs[i]);
725 if (error != 0) {
726 device_printf(sc->sc_dev,
727 "could not register RX callback for FQ %d\n", i);
728 goto err;
729 }
730 (void)qman_fq_register_flush_cb(fq, dpaa_eth_fq_rx_flush);
731 }
732
733 /*
734 * Expose per-FQ observability under dev.<port>.rx_fq.<i>.{cpu,
735 * fqid, frames}. The sysctl_ctx owned by sc_dev handles all
736 * teardown at device detach, so nothing to unwind on the free
737 * path.
738 */
739 {
740 struct sysctl_ctx_list *ctx =
741 device_get_sysctl_ctx(sc->sc_dev);
742 struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev);
743 struct sysctl_oid *rxnode = SYSCTL_ADD_NODE(ctx,
744 SYSCTL_CHILDREN(tree), OID_AUTO, "rx_fq",
745 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
746 "Per-FQ RX statistics");
747
748 for (i = 0; i < nfqs; i++) {
749 char name[8];
750 struct sysctl_oid *fqnode;
751
752 snprintf(name, sizeof(name), "%d", i);
753 fqnode = SYSCTL_ADD_NODE(ctx,
754 SYSCTL_CHILDREN(rxnode), OID_AUTO, name,
755 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
756 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(fqnode),
757 OID_AUTO, "fqid", CTLFLAG_RD,
758 &sc->sc_rx_fqs[i].fqid, 0, "FQID");
759 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(fqnode),
760 OID_AUTO, "cpu", CTLFLAG_RD,
761 &sc->sc_rx_fqs[i].cpu, 0,
762 "CPU affine to this FQ");
763 SYSCTL_ADD_UQUAD(ctx, SYSCTL_CHILDREN(fqnode),
764 OID_AUTO, "frames", CTLFLAG_RD,
765 &sc->sc_rx_fqs[i].frames_in,
766 "Frames dispatched to the RX callback");
767 }
768 }
769
770 if (bootverbose) {
771 device_printf(sc->sc_dev,
772 "RSS: %d RX FQ%s starting at fqid 0x%x, %s\n",
773 nfqs, nfqs == 1 ? "" : "s", base_fqid,
774 nfqs == 1 ? "no distribution" :
775 "IP 5-tuple hash via KeyGen");
776 }
777
778 return (0);
779
780 err:
781 dpaa_eth_fq_rx_free(sc);
782 return (error);
783 }
784
785 void
dpaa_eth_fq_tx_free(struct dpaa_eth_softc * sc)786 dpaa_eth_fq_tx_free(struct dpaa_eth_softc *sc)
787 {
788 int cpu;
789
790 if (sc->sc_tx_fq)
791 qman_fq_free(sc->sc_tx_fq);
792
793 if (sc->sc_tx_conf_fq)
794 qman_fq_free(sc->sc_tx_conf_fq);
795
796 /*
797 * The port's pool channel now hosts only TX confirms (RX has
798 * moved to per-CPU pool channels). Unsubscribe every portal
799 * and release the channel here rather than in fq_rx_free.
800 */
801 if (sc->sc_rx_channel != 0) {
802 CPU_FOREACH(cpu) {
803 device_t portal = DPCPU_ID_GET(cpu, qman_affine_portal);
804 QMAN_PORTAL_STATIC_DEQUEUE_RM_CHANNEL(portal,
805 sc->sc_rx_channel);
806 }
807 qman_free_channel(sc->sc_rx_channel);
808 sc->sc_rx_channel = 0;
809 }
810 }
811
812 int
dpaa_eth_fq_tx_init(struct dpaa_eth_softc * sc)813 dpaa_eth_fq_tx_init(struct dpaa_eth_softc *sc)
814 {
815 int error;
816 int cpu;
817 void *fq;
818
819 /* TX Frame Queue */
820 fq = qman_fq_create(1, sc->sc_port_tx_qman_chan,
821 DTSEC_RM_FQR_TX_WQ, false, 0, false, false, true, false, 0, 0, 0,
822 0, 0);
823 if (fq == NULL) {
824 device_printf(sc->sc_dev, "could not create default TX queue"
825 "\n");
826 return (EIO);
827 }
828
829 sc->sc_tx_fq = fq;
830
831 /*
832 * TX confirms need a pool channel with at least one subscriber.
833 * Historically the RX path allocated and subscribed sc_rx_channel
834 * for both RX and TX confirms; now RX uses per-CPU channels, so
835 * this path is the sole owner. Allocate + subscribe here.
836 * (Confirm handling doesn't benefit from CPU pinning today; a
837 * follow-on could route TX confirms to the enqueue CPU via the
838 * same per-CPU channel infrastructure.)
839 */
840 if (sc->sc_rx_channel == 0) {
841 sc->sc_rx_channel = qman_alloc_channel();
842 CPU_FOREACH(cpu) {
843 device_t portal = DPCPU_ID_GET(cpu, qman_affine_portal);
844 QMAN_PORTAL_STATIC_DEQUEUE_CHANNEL(portal,
845 sc->sc_rx_channel);
846 }
847 }
848
849 /* TX Confirmation Frame Queue */
850 fq = qman_fq_create(1, sc->sc_rx_channel,
851 DTSEC_RM_FQR_TX_CONF_WQ, false, 0, false, false, true, false, 0, 0,
852 0, 0, 0);
853 if (fq == NULL) {
854 device_printf(sc->sc_dev, "could not create TX confirmation "
855 "queue\n");
856 dpaa_eth_fq_tx_free(sc);
857 return (EIO);
858 }
859
860 sc->sc_tx_conf_fq = fq;
861 sc->sc_tx_conf_fqid = qman_fq_get_fqid(fq);
862
863 error = qman_fq_register_cb(fq, dpaa_eth_fq_tx_confirm_callback, sc);
864 if (error != 0) {
865 device_printf(sc->sc_dev, "could not register TX confirmation "
866 "callback\n");
867 dpaa_eth_fq_tx_free(sc);
868 return (EIO);
869 }
870
871 return (0);
872 }
873 /** @} */
874
875 /* Returns the cmd_stat field for the frame descriptor */
876 static uint32_t
dpaa_eth_tx_add_csum(struct dpaa_eth_frame_info * fi)877 dpaa_eth_tx_add_csum(struct dpaa_eth_frame_info *fi)
878 {
879 struct mbuf *m = fi->fi_mbuf;
880 struct fman_parse_result *prs = &fi->fi_ic.prs;
881 uint32_t csum_flags = m->m_pkthdr.csum_flags;
882 uint8_t ether_size = ETHER_HDR_LEN;
883
884 if ((csum_flags & CSUM_FLAGS_TX) == 0)
885 return (0);
886
887 if (m->m_flags & M_VLANTAG)
888 ether_size += ETHER_VLAN_ENCAP_LEN;
889 if (csum_flags & CSUM_IP)
890 prs->l3r = L3R_FIRST_IPV4;
891 if (csum_flags & CSUM_IP_UDP) {
892 prs->l4r = L4R_TYPE_UDP;
893 prs->l4_off = ether_size + sizeof(struct ip);
894 } else if (csum_flags & CSUM_IP_TCP) {
895 prs->l4r = L4R_TYPE_TCP;
896 prs->l4_off = ether_size + sizeof(struct ip);
897 } else if (csum_flags & CSUM_IP6_UDP) {
898 prs->l3r = L3R_FIRST_IPV6;
899 prs->l4r = L4R_TYPE_UDP;
900 prs->l4_off = ether_size + sizeof(struct ip6_hdr);
901 } else if (csum_flags & CSUM_IP6_TCP) {
902 prs->l3r = L3R_FIRST_IPV6;
903 prs->l4r = L4R_TYPE_TCP;
904 prs->l4_off = ether_size + sizeof(struct ip6_hdr);
905 }
906
907 prs->ip_off[0] = ether_size;
908
909 return (DPAA_FD_TX_CMD_RPD | DPAA_FD_TX_CMD_DTC);
910 }
911
912 /**
913 * @group dTSEC IFnet routines.
914 * @{
915 */
916 void
dpaa_eth_if_start_locked(struct dpaa_eth_softc * sc)917 dpaa_eth_if_start_locked(struct dpaa_eth_softc *sc)
918 {
919 vm_size_t psize, ssize;
920 struct dpaa_eth_frame_info *fi;
921 unsigned int i;
922 struct mbuf *m0, *m;
923 vm_offset_t vaddr;
924 struct dpaa_fd fd;
925
926 DPAA_ETH_LOCK_ASSERT(sc);
927 /* TODO: IFF_DRV_OACTIVE */
928
929 if ((sc->sc_mii->mii_media_status & IFM_ACTIVE) == 0)
930 return;
931
932 if ((if_getdrvflags(sc->sc_ifnet) & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
933 return;
934
935 if (sc->sc_tx_fq_full)
936 return;
937
938 while (!if_sendq_empty(sc->sc_ifnet)) {
939 if ((sc->sc_tx_queue_check_cnt++ &
940 (DTSEC_MAX_TX_QUEUE_CHECK_INTERVAL - 1)) == 0 &&
941 qman_fq_get_counter(sc->sc_tx_fq, QMAN_COUNTER_FRAME) >=
942 DTSEC_MAX_TX_QUEUE_LEN) {
943 atomic_store_rel_int(&sc->sc_tx_fq_full, 1);
944 sc->sc_tx_queue_check_cnt = 0;
945 return;
946 }
947
948 fi = dpaa_eth_fi_alloc(sc);
949 if (fi == NULL)
950 return;
951
952 m0 = if_dequeue(sc->sc_ifnet);
953 if (m0 == NULL) {
954 dpaa_eth_fi_free(sc, fi);
955 return;
956 }
957
958 i = 0;
959 psize = 0;
960 fi->fi_mbuf = m0;
961
962 for (m = m0; m != NULL && i < DPAA_NUM_OF_SG_TABLE_ENTRY;
963 m = m->m_next) {
964 vm_size_t rem;
965
966 if (m->m_len == 0)
967 continue;
968
969 vaddr = (vm_offset_t)m->m_data;
970 rem = m->m_len;
971
972 /*
973 * Fast path: the whole segment lives inside one
974 * page. Covers every default-cluster mbuf
975 * (MCLBYTES < PAGE_SIZE) and skips the split loop
976 * in the common case.
977 */
978 if ((vaddr & PAGE_MASK) + rem <= PAGE_SIZE) {
979 fi->fi_sgt[i].addr = dpaa_eth_va_to_phys(vaddr);
980 fi->fi_sgt[i].length = rem;
981 fi->fi_sgt[i].extension = 0;
982 fi->fi_sgt[i].final = 0;
983 fi->fi_sgt[i].bpid = 0;
984 fi->fi_sgt[i].offset = 0;
985 psize += rem;
986 i++;
987 continue;
988 }
989
990 /*
991 * Slow path: mbuf crosses at least one page
992 * boundary (jumbo cluster, or an unusually-offset
993 * external buffer). Emit one SGT entry per
994 * contiguous physical span.
995 */
996 while (rem > 0 && i < DPAA_NUM_OF_SG_TABLE_ENTRY) {
997 ssize = PAGE_SIZE - (vaddr & PAGE_MASK);
998 if (rem < ssize)
999 ssize = rem;
1000
1001 fi->fi_sgt[i].addr = dpaa_eth_va_to_phys(vaddr);
1002 fi->fi_sgt[i].length = ssize;
1003 fi->fi_sgt[i].extension = 0;
1004 fi->fi_sgt[i].final = 0;
1005 fi->fi_sgt[i].bpid = 0;
1006 fi->fi_sgt[i].offset = 0;
1007
1008 rem -= ssize;
1009 vaddr += ssize;
1010 psize += ssize;
1011 i++;
1012 }
1013
1014 if (rem > 0) /* SGT full mid-mbuf */
1015 break;
1016 }
1017
1018 /*
1019 * Reject the frame if we didn't consume the whole chain
1020 * (SGT full mid-frame) or if the chain produced no SGT
1021 * entries at all (all-zero-length mbufs).
1022 */
1023 if (m != NULL || i == 0) {
1024 dpaa_eth_fi_free(sc, fi);
1025 m_freem(m0);
1026 continue;
1027 }
1028
1029 fi->fi_sgt[i - 1].final = 1;
1030
1031 fd.addr = DMAP_TO_PHYS((vm_offset_t)fi);
1032 fd.length = psize;
1033 fd.format = DPAA_FD_FORMAT_SHORT_MBSF;
1034
1035 fd.liodn = 0;
1036 fd.bpid = 0;
1037 fd.eliodn = 0;
1038 fd.offset = offsetof(struct dpaa_eth_frame_info, fi_sgt);
1039 fd.cmd_stat = dpaa_eth_tx_add_csum(fi);
1040
1041 DPAA_ETH_UNLOCK(sc);
1042 if (qman_fq_enqueue(sc->sc_tx_fq, &fd) != 0) {
1043 dpaa_eth_fi_free(sc, fi);
1044 m_freem(m0);
1045 }
1046 DPAA_ETH_LOCK(sc);
1047 }
1048 }
1049 /** @} */
1050