1 /*- 2 * Copyright (c) 2026 Justin Hibbits 3 * Copyright (c) 2012 Semihalf. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/kernel.h> 31 #include <sys/module.h> 32 #include <sys/bus.h> 33 #include <sys/rman.h> 34 #include <sys/malloc.h> 35 #include <sys/mbuf.h> 36 #include <sys/smp.h> 37 #include <sys/socket.h> 38 #include <sys/sockio.h> 39 #include <sys/sysctl.h> 40 41 #include <net/ethernet.h> 42 #include <net/if.h> 43 #include <net/if_dl.h> 44 #include <net/if_media.h> 45 #include <net/if_types.h> 46 #include <net/if_arp.h> 47 #include <netinet/ip.h> 48 #include <netinet/ip6.h> 49 #include <netinet/tcp_lro.h> 50 51 #include <dev/mii/mii.h> 52 #include <dev/mii/miivar.h> 53 54 #include <vm/vm.h> 55 #include <vm/pmap.h> 56 57 #include "miibus_if.h" 58 59 #include "bman.h" 60 #include "dpaa_common.h" 61 #include "dpaa_eth.h" 62 #include "fman.h" 63 #include "fman_keygen.h" 64 #include "fman_parser.h" 65 #include "fman_port.h" 66 #include "fman_if.h" 67 #include "fman_port_if.h" 68 #include "if_dtsec.h" 69 #include "qman.h" 70 #include "qman_var.h" 71 #include "qman_portal_if.h" 72 73 74 #define DPAA_ETH_LOCK(sc) mtx_lock(&(sc)->sc_lock) 75 #define DPAA_ETH_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) 76 #define DPAA_ETH_LOCK_ASSERT(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) 77 78 /* 79 * On 64-bit Book-E the direct map is always present, and the driver's 80 * UMA zones plus page-sized mbuf clusters live in it. Bypass the 81 * page-table walk in pmap_kextract() for those; fall back for 82 * MJUM9BYTES/MJUM16BYTES clusters, which are kmem_alloc_contig()'d 83 * into KVA. 84 */ 85 static inline vm_paddr_t 86 dpaa_eth_va_to_phys(vm_offset_t va) 87 { 88 if (__predict_true(va >= DMAP_BASE_ADDRESS && va <= DMAP_MAX_ADDRESS)) 89 return (DMAP_TO_PHYS(va)); 90 return (pmap_kextract(va)); 91 } 92 93 /** 94 * @group dTSEC RM private defines. 95 * @{ 96 */ 97 #define DTSEC_BPOOLS_USED (1) 98 #define DTSEC_MAX_TX_QUEUE_LEN 256 99 /* 100 * Sample the hardware TX FQ counter every Nth packet. The FQ counter is 101 * 24 bits and the soft cap above is 256, so overshoot by N is trivial. 102 */ 103 #define DTSEC_MAX_TX_QUEUE_CHECK_INTERVAL 32 104 /* 105 * Confirmation callback drain-detection. Fast path (TX not backpressured) 106 * skips the MC call entirely; when flagged, we sample every Nth callback to 107 * detect the drain-to-zero transition. 108 */ 109 #define DTSEC_TX_CONF_CHECK_INTERVAL 32 110 111 struct dpaa_eth_frame_info { 112 struct fman_internal_context fi_ic; 113 struct mbuf *fi_mbuf; 114 struct dpaa_sgte fi_sgt[DPAA_NUM_OF_SG_TABLE_ENTRY]; 115 }; 116 117 /* 118 * Loader-tunable override for the per-port RX FQ count. 0 (default) 119 * means "auto". Otherwise must be a power of two. 120 */ 121 static int dpaa_eth_nrxfqs_tunable = 0; 122 SYSCTL_NODE(_hw, OID_AUTO, dpaa, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 123 "DPAA driver tunables"); 124 static SYSCTL_NODE(_hw_dpaa, OID_AUTO, eth, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 125 "DPAA Ethernet driver"); 126 SYSCTL_INT(_hw_dpaa_eth, OID_AUTO, nrxfqs, CTLFLAG_RDTUN, 127 &dpaa_eth_nrxfqs_tunable, 0, 128 "Per-port RX FQ count override (0=auto, else power-of-2 in [1,8])"); 129 130 enum dpaa_eth_pool_params { 131 DTSEC_RM_POOL_RX_LOW_MARK = 16, 132 DTSEC_RM_POOL_RX_HIGH_MARK = 64, 133 DTSEC_RM_POOL_RX_MAX_SIZE = 256, 134 /* 135 * MAX_SIZE is a soft cap set well below the BMan hardware pool 136 * limit, so sampling the depth every N put-backs per CPU is safe: 137 * worst-case overshoot is N * ncpus buffers, still tiny vs. the 138 * hardware pool. 139 */ 140 DTSEC_RM_POOL_RX_CHECK_INTERVAL = 32, 141 142 DTSEC_RM_POOL_FI_LOW_MARK = 16, 143 DTSEC_RM_POOL_FI_HIGH_MARK = 64, 144 DTSEC_RM_POOL_FI_MAX_SIZE = 256, 145 }; 146 147 enum dpaa_eth_fq_params { 148 DTSEC_RM_FQR_RX_WQ = 1, 149 DTSEC_RM_FQR_TX_WQ = 1, 150 DTSEC_RM_FQR_TX_CONF_WQ = 1 151 }; 152 /** @} */ 153 154 155 /** 156 * @group dTSEC Frame Info routines. 157 * @{ 158 */ 159 void 160 dpaa_eth_fi_pool_free(struct dpaa_eth_softc *sc) 161 { 162 163 if (sc->sc_fi_zone != NULL) 164 uma_zdestroy(sc->sc_fi_zone); 165 } 166 167 int 168 dpaa_eth_fi_pool_init(struct dpaa_eth_softc *sc) 169 { 170 171 snprintf(sc->sc_fi_zname, sizeof(sc->sc_fi_zname), "%s: Frame Info", 172 device_get_nameunit(sc->sc_dev)); 173 174 sc->sc_fi_zone = uma_zcreate(sc->sc_fi_zname, 175 sizeof(struct dpaa_eth_frame_info), NULL, NULL, NULL, NULL, 176 UMA_ALIGN_PTR, 0); 177 178 return (0); 179 } 180 181 static struct dpaa_eth_frame_info * 182 dpaa_eth_fi_alloc(struct dpaa_eth_softc *sc) 183 { 184 struct dpaa_eth_frame_info *fi; 185 186 fi = uma_zalloc(sc->sc_fi_zone, M_NOWAIT | M_ZERO); 187 188 return (fi); 189 } 190 191 static void 192 dpaa_eth_fi_free(struct dpaa_eth_softc *sc, struct dpaa_eth_frame_info *fi) 193 { 194 195 uma_zfree(sc->sc_fi_zone, fi); 196 } 197 /** @} */ 198 199 200 /** 201 * @group dTSEC FMan PORT routines. 202 * @{ 203 */ 204 int 205 dpaa_eth_fm_port_rx_init(struct dpaa_eth_softc *sc) 206 { 207 struct fman_port_params params; 208 int error; 209 210 /* 211 * dflt/err FQID is the base of the RSS range: non-hashable 212 * frames (ARP, IP fragments, non-IP) fall through to FQ #0. 213 */ 214 params.dflt_fqid = sc->sc_rx_fqid_base; 215 params.err_fqid = sc->sc_rx_fqid_base; 216 params.rx_params.num_pools = 1; 217 params.rx_params.bpools[0].bpid = bman_get_bpid(sc->sc_rx_pool); 218 params.rx_params.bpools[0].size = MCLBYTES; 219 error = FMAN_PORT_CONFIG(sc->sc_rx_port, ¶ms); 220 error = FMAN_PORT_INIT(sc->sc_rx_port); 221 if (error != 0) { 222 device_printf(sc->sc_dev, "couldn't initialize FM Port RX.\n"); 223 return (ENXIO); 224 } 225 226 /* 227 * The RX port's own FMan hardware port ID (cell-index in the 228 * OFW node) is the index KG uses for scheme-binding. It was 229 * previously left at zero, which made every port fight for KG 230 * port 0 -- only the first attach won, the rest got EBUSY. 231 */ 232 sc->sc_port_rx_hw_id = fman_port_get_id(sc->sc_rx_port); 233 234 /* 235 * Wire up FMan KeyGen 5-tuple hashing across the sc_nrxfqs FQs 236 * created in dpaa_eth_fq_rx_init. Skipped for the trivial N=1 237 * case (single-core system or forced fallback): with one FQ 238 * there's nothing to distribute. Only flip the port's parser 239 * output to KG on success -- routing to KG with no bound scheme 240 * drops frames. 241 */ 242 if (sc->sc_nrxfqs > 1) { 243 struct fman_softc *fman_sc = 244 device_get_softc(device_get_parent(sc->sc_rx_port)); 245 246 error = fman_kg_alloc_hash_scheme(fman_sc, 247 sc->sc_port_rx_hw_id, sc->sc_rx_fqid_base, 248 sc->sc_nrxfqs); 249 if (error != 0) { 250 device_printf(sc->sc_dev, 251 "fman_kg_alloc_hash_scheme failed: %d\n", error); 252 /* 253 * Non-fatal: FMan port still delivers to 254 * dflt_fqid == sc_rx_fqid_base (FQ #0), which is 255 * a valid single-queue fallback. Leave RFPNE at 256 * the BMI-enqueue default. 257 */ 258 } else { 259 fman_port_rx_use_kg(sc->sc_rx_port, true); 260 } 261 } 262 263 return (0); 264 } 265 266 int 267 dpaa_eth_fm_port_tx_init(struct dpaa_eth_softc *sc) 268 { 269 struct fman_port_params params; 270 int error; 271 272 params.dflt_fqid = sc->sc_tx_conf_fqid; 273 params.err_fqid = sc->sc_tx_conf_fqid; 274 275 error = FMAN_PORT_CONFIG(sc->sc_tx_port, ¶ms); 276 error = FMAN_PORT_INIT(sc->sc_tx_port); 277 if (error != 0) { 278 device_printf(sc->sc_dev, "couldn't initialize FM Port TX.\n"); 279 return (ENXIO); 280 } 281 282 return (0); 283 } 284 /** @} */ 285 286 287 /** 288 * @group dTSEC buffer pools routines. 289 * @{ 290 */ 291 static int 292 dpaa_eth_pool_rx_put_buffer(struct dpaa_eth_softc *sc, uint8_t *buffer, 293 void *context) 294 { 295 296 uma_zfree(sc->sc_rx_zone, buffer); 297 298 return (0); 299 } 300 301 static int 302 dtsec_add_buffers(struct dpaa_eth_softc *sc, int count) 303 { 304 struct bman_buffer bufs[8] = {}; 305 int err; 306 int c; 307 308 while (count > 0) { 309 c = min(8, count); 310 for (int i = 0; i < c; i++) { 311 void *b; 312 vm_paddr_t pa; 313 314 b = uma_zalloc(sc->sc_rx_zone, M_NOWAIT); 315 if (b == NULL) 316 return (ENOMEM); 317 pa = DMAP_TO_PHYS((vm_offset_t)b); 318 bufs[i].buf_hi = (pa >> 32); 319 bufs[i].buf_lo = (pa & 0xffffffff); 320 } 321 322 err = bman_put_buffers(sc->sc_rx_pool, bufs, c); 323 if (err != 0) 324 return (err); 325 count -= c; 326 } 327 328 return (0); 329 } 330 331 static void 332 dpaa_eth_pool_rx_depleted(void *h_App, bool in) 333 { 334 struct dpaa_eth_softc *sc; 335 unsigned int count; 336 337 sc = h_App; 338 339 if (!in) 340 return; 341 342 while (1) { 343 count = bman_count(sc->sc_rx_pool); 344 if (count > DTSEC_RM_POOL_RX_HIGH_MARK) 345 return; 346 347 /* Can only release 8 buffers at a time */ 348 count = min(DTSEC_RM_POOL_RX_HIGH_MARK - count + 8, 8); 349 if (dtsec_add_buffers(sc, count) != 0) 350 return; 351 } 352 } 353 354 void 355 dpaa_eth_pool_rx_free(struct dpaa_eth_softc *sc) 356 { 357 358 if (sc->sc_rx_pool != NULL) 359 bman_pool_destroy(sc->sc_rx_pool); 360 361 if (sc->sc_rx_zone != NULL) 362 uma_zdestroy(sc->sc_rx_zone); 363 364 free(sc->sc_rx_pool_check_cnt, M_DEVBUF); 365 sc->sc_rx_pool_check_cnt = NULL; 366 } 367 368 int 369 dpaa_eth_pool_rx_init(struct dpaa_eth_softc *sc) 370 { 371 372 /* MCLBYTES must be less than PAGE_SIZE */ 373 CTASSERT(MCLBYTES < PAGE_SIZE); 374 375 snprintf(sc->sc_rx_zname, sizeof(sc->sc_rx_zname), "%s: RX Buffers", 376 device_get_nameunit(sc->sc_dev)); 377 378 sc->sc_rx_zone = uma_zcreate(sc->sc_rx_zname, MCLBYTES, NULL, 379 NULL, NULL, NULL, MCLBYTES - 1, 0); 380 381 sc->sc_rx_pool_check_cnt = malloc_aligned( 382 (mp_maxid + 1) * sizeof(struct dpaa_pcpu_cnt), 383 CACHE_LINE_SIZE, M_DEVBUF, M_WAITOK | M_ZERO); 384 385 sc->sc_rx_pool = bman_pool_create(&sc->sc_rx_bpid, MCLBYTES, 386 DTSEC_RM_POOL_RX_MAX_SIZE, DTSEC_RM_POOL_RX_LOW_MARK, 387 DTSEC_RM_POOL_RX_HIGH_MARK, 0, 0, dpaa_eth_pool_rx_depleted, sc); 388 if (sc->sc_rx_pool == NULL) { 389 device_printf(sc->sc_dev, "NULL rx pool somehow\n"); 390 dpaa_eth_pool_rx_free(sc); 391 return (EIO); 392 } 393 394 dtsec_add_buffers(sc, DTSEC_RM_POOL_RX_HIGH_MARK); 395 396 return (0); 397 } 398 /** @} */ 399 400 401 /** 402 * @group dTSEC Frame Queue Range routines. 403 * @{ 404 */ 405 static void 406 dpaa_eth_fq_mext_free(struct mbuf *m) 407 { 408 struct dpaa_eth_softc *sc; 409 void *buffer; 410 411 buffer = m->m_ext.ext_arg1; 412 sc = m->m_ext.ext_arg2; 413 /* 414 * Sloppy per-CPU sampling: no pin, no atomic. A stray migration 415 * between the curcpu read and the increment can only mis-attribute 416 * one bump to the wrong CPU's counter; the sampling rate stays 417 * within the acceptable slop window. 418 */ 419 if ((++sc->sc_rx_pool_check_cnt[curcpu].cnt & 420 (DTSEC_RM_POOL_RX_CHECK_INTERVAL - 1)) == 0 && 421 bman_count(sc->sc_rx_pool) > DTSEC_RM_POOL_RX_MAX_SIZE) 422 dpaa_eth_pool_rx_put_buffer(sc, buffer, NULL); 423 else 424 bman_put_buffer(sc->sc_rx_pool, 425 DMAP_TO_PHYS((vm_offset_t)buffer), sc->sc_rx_bpid); 426 } 427 428 static int 429 dpaa_eth_update_csum_flags(struct qman_fd *frame, 430 struct fman_parse_result *prs, struct mbuf *m) 431 { 432 uint16_t l3r = be16toh(prs->l3r); 433 434 /* TODO: nested protocols? */ 435 if ((l3r & L3R_FIRST_IP_M) != 0) { 436 m->m_pkthdr.csum_flags |= CSUM_L3_CALC; 437 if ((l3r & L3R_FIRST_ERROR) == 0) 438 m->m_pkthdr.csum_flags |= CSUM_L3_VALID; 439 } 440 if (frame->cmd_stat & DPAA_FD_RX_STATUS_L4CV) { 441 m->m_pkthdr.csum_flags |= CSUM_L4_CALC; 442 m->m_pkthdr.csum_data = 0xffff; 443 if ((prs->l4r & L4R_TYPE_M) != 0 && 444 (prs->l4r & L4R_ERR) == 0) 445 m->m_pkthdr.csum_flags |= CSUM_L4_VALID; 446 } 447 448 return (0); 449 } 450 451 static int 452 dpaa_eth_fq_rx_callback(device_t portal, struct qman_fq *fq, 453 struct qman_fd *frame, void *app) 454 { 455 struct dpaa_eth_rx_fq *rxfq; 456 struct dpaa_eth_softc *sc; 457 struct mbuf *m; 458 struct fman_internal_context *frame_ic; 459 void *frame_va; 460 461 m = NULL; 462 rxfq = app; 463 sc = rxfq->sc; 464 rxfq->frames_in++; 465 466 frame_va = DPAA_FD_GET_ADDR(frame); 467 frame_ic = frame_va; /* internal context at head of the frame */ 468 /* Only simple (single- or multi-) frames are supported. */ 469 KASSERT(frame->format == 0 || frame->format == 4, 470 ("%s(): Got unsupported frame format 0x%02X!", __func__, 471 frame->format)); 472 473 if ((frame->cmd_stat & DPAA_FD_CMD_STAT_ERR_M) != 0) { 474 device_printf(sc->sc_dev, "RX error: 0x%08X\n", 475 frame->cmd_stat); 476 goto err; 477 } 478 479 m = m_gethdr(M_NOWAIT, MT_HEADER); 480 if (m == NULL) 481 goto err; 482 483 if (frame->format == 0) { 484 /* Single-frame format */ 485 m_extadd(m, (char *)frame_va + frame->offset, frame->length, 486 dpaa_eth_fq_mext_free, frame_va, sc, 0, EXT_NET_DRV); 487 } else { 488 struct dpaa_sgte *sgt = 489 (struct dpaa_sgte *)(char *)frame_va + frame->offset; 490 /* Simple multi-frame format */ 491 for (int i = 0; i < DPAA_NUM_OF_SG_TABLE_ENTRY; i++) { 492 if (sgt[i].length > 0) 493 m_extadd(m, PHYS_TO_DMAP(sgt[i].addr), 494 sgt[i].length, dpaa_eth_fq_mext_free, 495 PHYS_TO_DMAP(sgt[i].addr), sc, 0, 496 EXT_NET_DRV); 497 if (sgt[i].final) 498 break; 499 } 500 /* Free the SGT buffer, it's no longer needed. */ 501 bman_put_buffer(sc->sc_rx_pool, frame->addr, sc->sc_rx_bpid); 502 } 503 504 if (if_getcapenable(sc->sc_ifnet) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 505 dpaa_eth_update_csum_flags(frame, &frame_ic->prs, m); 506 507 m->m_pkthdr.rcvif = sc->sc_ifnet; 508 m->m_len = frame->length; 509 m_fixhdr(m); 510 511 /* 512 * Offer to LRO first. 513 */ 514 if (rxfq->lro_inited && 515 (if_getcapenable(sc->sc_ifnet) & IFCAP_LRO) != 0 && 516 (m->m_pkthdr.csum_flags & (CSUM_L4_CALC | CSUM_L4_VALID)) == 517 (CSUM_L4_CALC | CSUM_L4_VALID) && 518 tcp_lro_rx(&rxfq->lro, m, 0) == 0) 519 return (1); 520 521 m->m_nextpkt = NULL; 522 *rxfq->rx_tailp = m; 523 rxfq->rx_tailp = &m->m_nextpkt; 524 525 return (1); 526 527 err: 528 bman_put_buffer(sc->sc_rx_pool, frame->addr, sc->sc_rx_bpid); 529 if (m != NULL) 530 m_freem(m); 531 532 return (1); 533 } 534 535 /* 536 * Post-poll flush hook invoked once per QMan portal poll on any 537 * RX FQ that dispatched at least one frame this cycle. Runs on 538 * the FQ's affine CPU, outside the DQRR dispatch loop. 539 */ 540 static void 541 dpaa_eth_fq_rx_flush(struct qman_fq *fq __unused, void *ctx) 542 { 543 struct dpaa_eth_rx_fq *rxfq = ctx; 544 545 if (rxfq->rx_head != NULL) { 546 struct mbuf *chain = rxfq->rx_head; 547 548 rxfq->rx_head = NULL; 549 rxfq->rx_tailp = &rxfq->rx_head; 550 if_input(rxfq->sc->sc_ifnet, chain); 551 } 552 /* 553 * Flush LRO whenever initialised. If the user disabled 554 * IFCAP_LRO between the last callback and this flush, entries 555 * queued in that window still need to be drained. 556 */ 557 if (rxfq->lro_inited) 558 tcp_lro_flush_all(&rxfq->lro); 559 } 560 561 static int 562 dpaa_eth_fq_tx_confirm_callback(device_t portal, struct qman_fq *fq, 563 struct qman_fd *frame, void *app) 564 { 565 struct dpaa_eth_frame_info *fi; 566 struct dpaa_eth_softc *sc; 567 568 sc = app; 569 570 if ((frame->cmd_stat & DPAA_FD_TX_STAT_ERR_M) != 0) 571 device_printf(sc->sc_dev, "TX error: 0x%08X\n", 572 frame->cmd_stat); 573 574 /* 575 * We are storing struct dpaa_eth_frame_info in first entry 576 * of scatter-gather table. 577 */ 578 fi = (struct dpaa_eth_frame_info *)PHYS_TO_DMAP(frame->addr); 579 580 /* Free transmitted frame */ 581 m_freem(fi->fi_mbuf); 582 dpaa_eth_fi_free(sc, fi); 583 584 /* 585 * Fast path: TX isn't backpressured, so there's nothing to 586 * restart. Acquire load pairs with the release store on the 587 * TX path so a concurrent set of the flag is observed here. 588 */ 589 if (atomic_load_acq_int(&sc->sc_tx_fq_full) == 0) 590 return (1); 591 592 /* Rate-limit the MC round-trip to detect drain-to-zero. */ 593 if ((sc->sc_tx_conf_check_cnt++ & 594 (DTSEC_TX_CONF_CHECK_INTERVAL - 1)) != 0) 595 return (1); 596 if (qman_fq_get_counter(sc->sc_tx_conf_fq, QMAN_COUNTER_FRAME) != 0) 597 return (1); 598 599 DPAA_ETH_LOCK(sc); 600 if (sc->sc_tx_fq_full) { 601 atomic_store_rel_int(&sc->sc_tx_fq_full, 0); 602 dpaa_eth_if_start_locked(sc); 603 } 604 DPAA_ETH_UNLOCK(sc); 605 606 return (1); 607 } 608 609 void 610 dpaa_eth_fq_rx_free(struct dpaa_eth_softc *sc) 611 { 612 int i; 613 614 /* 615 * Tear down the KG scheme first so no new frames land on FQs 616 * about to be retired. Point the parser output back at BMI- 617 * enqueue before freeing the scheme so the port keeps 618 * delivering to dflt_fqid instead of dropping through an 619 * emptied KG. 620 */ 621 if (sc->sc_nrxfqs > 1 && sc->sc_rx_port != NULL) { 622 struct fman_softc *fman_sc = 623 device_get_softc(device_get_parent(sc->sc_rx_port)); 624 625 fman_port_rx_use_kg(sc->sc_rx_port, false); 626 (void)fman_kg_free_hash_scheme(fman_sc, 627 sc->sc_port_rx_hw_id); 628 } 629 630 if (sc->sc_rx_fqs != NULL) { 631 for (i = 0; i < sc->sc_nrxfqs; i++) { 632 if (sc->sc_rx_fqs[i].fq != NULL) 633 qman_fq_free(sc->sc_rx_fqs[i].fq); 634 /* 635 * Any pending non-LRO mbufs on the batch chain 636 * are dropped here rather than delivered late. 637 */ 638 if (sc->sc_rx_fqs[i].rx_head != NULL) 639 m_freem(sc->sc_rx_fqs[i].rx_head); 640 if (sc->sc_rx_fqs[i].lro_inited) 641 tcp_lro_free(&sc->sc_rx_fqs[i].lro); 642 } 643 free(sc->sc_rx_fqs, M_DEVBUF); 644 sc->sc_rx_fqs = NULL; 645 } 646 if (sc->sc_nrxfqs > 0) { 647 qman_free_fqid_range(sc->sc_rx_fqid_base, sc->sc_nrxfqs); 648 sc->sc_nrxfqs = 0; 649 sc->sc_rx_fqid_base = 0; 650 } 651 } 652 653 int 654 dpaa_eth_fq_rx_init(struct dpaa_eth_softc *sc) 655 { 656 struct qman_fq *fq; 657 uint32_t base_fqid; 658 int align, nfqs; 659 int error, i; 660 661 if (dpaa_eth_nrxfqs_tunable > 0) 662 nfqs = dpaa_eth_nrxfqs_tunable; 663 else 664 nfqs = mp_ncpus; 665 666 align = 1 << ilog2(nfqs); 667 error = qman_alloc_fqid_range(nfqs, align, &base_fqid); 668 if (error != 0) { 669 device_printf(sc->sc_dev, 670 "could not reserve %d contiguous FQIDs (aligned): %d\n", 671 nfqs, error); 672 return (EIO); 673 } 674 675 sc->sc_nrxfqs = nfqs; 676 sc->sc_rx_fqid_base = base_fqid; 677 sc->sc_rx_fqs = malloc(nfqs * sizeof(*sc->sc_rx_fqs), 678 M_DEVBUF, M_WAITOK | M_ZERO); 679 680 /* 681 * Create the N RX FQs, one per per-CPU channel. QMan portal 682 * attach has already subscribed each portal to its own 683 * per-CPU channel, so frames land on the right core without 684 * per-driver static-dequeue plumbing. 685 * 686 * Stash 1 cacheline of frame annotation (parse result / IC) 687 * and 1 of frame data head into the destination core's cache 688 * when QMan dequeues an RX frame -- the RX callback reads 689 * both. 690 */ 691 for (i = 0; i < nfqs; i++) { 692 int chan = qman_percpu_channel(i); 693 694 if (chan == -1) { 695 device_printf(sc->sc_dev, 696 "no per-CPU QMan channel for CPU %d\n", i); 697 error = EIO; 698 goto err; 699 } 700 fq = qman_fq_create(1, chan, DTSEC_RM_FQR_RX_WQ, 701 /*force_fqid=*/true, base_fqid + i, 702 false, false, true, false, 0, 0, 0, 1, 1, 0, 0); 703 if (fq == NULL) { 704 device_printf(sc->sc_dev, 705 "could not create RX FQ %d (fqid 0x%x)\n", 706 i, base_fqid + i); 707 error = EIO; 708 goto err; 709 } 710 sc->sc_rx_fqs[i].fq = fq; 711 sc->sc_rx_fqs[i].fqid = base_fqid + i; 712 sc->sc_rx_fqs[i].cpu = i; 713 sc->sc_rx_fqs[i].sc = sc; 714 sc->sc_rx_fqs[i].rx_head = NULL; 715 sc->sc_rx_fqs[i].rx_tailp = &sc->sc_rx_fqs[i].rx_head; 716 717 /* Best-effort LRO per FQ. */ 718 if (tcp_lro_init(&sc->sc_rx_fqs[i].lro) == 0) { 719 sc->sc_rx_fqs[i].lro.ifp = sc->sc_ifnet; 720 sc->sc_rx_fqs[i].lro_inited = true; 721 } 722 723 error = qman_fq_register_cb(fq, dpaa_eth_fq_rx_callback, 724 &sc->sc_rx_fqs[i]); 725 if (error != 0) { 726 device_printf(sc->sc_dev, 727 "could not register RX callback for FQ %d\n", i); 728 goto err; 729 } 730 (void)qman_fq_register_flush_cb(fq, dpaa_eth_fq_rx_flush); 731 } 732 733 /* 734 * Expose per-FQ observability under dev.<port>.rx_fq.<i>.{cpu, 735 * fqid, frames}. The sysctl_ctx owned by sc_dev handles all 736 * teardown at device detach, so nothing to unwind on the free 737 * path. 738 */ 739 { 740 struct sysctl_ctx_list *ctx = 741 device_get_sysctl_ctx(sc->sc_dev); 742 struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev); 743 struct sysctl_oid *rxnode = SYSCTL_ADD_NODE(ctx, 744 SYSCTL_CHILDREN(tree), OID_AUTO, "rx_fq", 745 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 746 "Per-FQ RX statistics"); 747 748 for (i = 0; i < nfqs; i++) { 749 char name[8]; 750 struct sysctl_oid *fqnode; 751 752 snprintf(name, sizeof(name), "%d", i); 753 fqnode = SYSCTL_ADD_NODE(ctx, 754 SYSCTL_CHILDREN(rxnode), OID_AUTO, name, 755 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 756 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(fqnode), 757 OID_AUTO, "fqid", CTLFLAG_RD, 758 &sc->sc_rx_fqs[i].fqid, 0, "FQID"); 759 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(fqnode), 760 OID_AUTO, "cpu", CTLFLAG_RD, 761 &sc->sc_rx_fqs[i].cpu, 0, 762 "CPU affine to this FQ"); 763 SYSCTL_ADD_UQUAD(ctx, SYSCTL_CHILDREN(fqnode), 764 OID_AUTO, "frames", CTLFLAG_RD, 765 &sc->sc_rx_fqs[i].frames_in, 766 "Frames dispatched to the RX callback"); 767 } 768 } 769 770 if (bootverbose) { 771 device_printf(sc->sc_dev, 772 "RSS: %d RX FQ%s starting at fqid 0x%x, %s\n", 773 nfqs, nfqs == 1 ? "" : "s", base_fqid, 774 nfqs == 1 ? "no distribution" : 775 "IP 5-tuple hash via KeyGen"); 776 } 777 778 return (0); 779 780 err: 781 dpaa_eth_fq_rx_free(sc); 782 return (error); 783 } 784 785 void 786 dpaa_eth_fq_tx_free(struct dpaa_eth_softc *sc) 787 { 788 int cpu; 789 790 if (sc->sc_tx_fq) 791 qman_fq_free(sc->sc_tx_fq); 792 793 if (sc->sc_tx_conf_fq) 794 qman_fq_free(sc->sc_tx_conf_fq); 795 796 /* 797 * The port's pool channel now hosts only TX confirms (RX has 798 * moved to per-CPU pool channels). Unsubscribe every portal 799 * and release the channel here rather than in fq_rx_free. 800 */ 801 if (sc->sc_rx_channel != 0) { 802 CPU_FOREACH(cpu) { 803 device_t portal = DPCPU_ID_GET(cpu, qman_affine_portal); 804 QMAN_PORTAL_STATIC_DEQUEUE_RM_CHANNEL(portal, 805 sc->sc_rx_channel); 806 } 807 qman_free_channel(sc->sc_rx_channel); 808 sc->sc_rx_channel = 0; 809 } 810 } 811 812 int 813 dpaa_eth_fq_tx_init(struct dpaa_eth_softc *sc) 814 { 815 int error; 816 int cpu; 817 void *fq; 818 819 /* TX Frame Queue */ 820 fq = qman_fq_create(1, sc->sc_port_tx_qman_chan, 821 DTSEC_RM_FQR_TX_WQ, false, 0, false, false, true, false, 0, 0, 0, 822 0, 0, 0, 0); 823 if (fq == NULL) { 824 device_printf(sc->sc_dev, "could not create default TX queue" 825 "\n"); 826 return (EIO); 827 } 828 829 sc->sc_tx_fq = fq; 830 831 /* 832 * TX confirms need a pool channel with at least one subscriber. 833 * Historically the RX path allocated and subscribed sc_rx_channel 834 * for both RX and TX confirms; now RX uses per-CPU channels, so 835 * this path is the sole owner. Allocate + subscribe here. 836 * (Confirm handling doesn't benefit from CPU pinning today; a 837 * follow-on could route TX confirms to the enqueue CPU via the 838 * same per-CPU channel infrastructure.) 839 */ 840 if (sc->sc_rx_channel == 0) { 841 sc->sc_rx_channel = qman_alloc_channel(); 842 CPU_FOREACH(cpu) { 843 device_t portal = DPCPU_ID_GET(cpu, qman_affine_portal); 844 QMAN_PORTAL_STATIC_DEQUEUE_CHANNEL(portal, 845 sc->sc_rx_channel); 846 } 847 } 848 849 /* TX Confirmation Frame Queue */ 850 fq = qman_fq_create(1, sc->sc_rx_channel, 851 DTSEC_RM_FQR_TX_CONF_WQ, false, 0, false, false, true, false, 0, 0, 852 0, 0, 0, 0, 0); 853 if (fq == NULL) { 854 device_printf(sc->sc_dev, "could not create TX confirmation " 855 "queue\n"); 856 dpaa_eth_fq_tx_free(sc); 857 return (EIO); 858 } 859 860 sc->sc_tx_conf_fq = fq; 861 sc->sc_tx_conf_fqid = qman_fq_get_fqid(fq); 862 863 error = qman_fq_register_cb(fq, dpaa_eth_fq_tx_confirm_callback, sc); 864 if (error != 0) { 865 device_printf(sc->sc_dev, "could not register TX confirmation " 866 "callback\n"); 867 dpaa_eth_fq_tx_free(sc); 868 return (EIO); 869 } 870 871 return (0); 872 } 873 /** @} */ 874 875 /* Returns the cmd_stat field for the frame descriptor */ 876 static uint32_t 877 dpaa_eth_tx_add_csum(struct dpaa_eth_frame_info *fi) 878 { 879 struct mbuf *m = fi->fi_mbuf; 880 struct fman_parse_result *prs = &fi->fi_ic.prs; 881 uint32_t csum_flags = m->m_pkthdr.csum_flags; 882 uint8_t ether_size = ETHER_HDR_LEN; 883 884 if ((csum_flags & CSUM_FLAGS_TX) == 0) 885 return (0); 886 887 if (m->m_flags & M_VLANTAG) 888 ether_size += ETHER_VLAN_ENCAP_LEN; 889 if (csum_flags & CSUM_IP) 890 prs->l3r = L3R_FIRST_IPV4; 891 if (csum_flags & CSUM_IP_UDP) { 892 prs->l4r = L4R_TYPE_UDP; 893 prs->l4_off = ether_size + sizeof(struct ip); 894 } else if (csum_flags & CSUM_IP_TCP) { 895 prs->l4r = L4R_TYPE_TCP; 896 prs->l4_off = ether_size + sizeof(struct ip); 897 } else if (csum_flags & CSUM_IP6_UDP) { 898 prs->l3r = L3R_FIRST_IPV6; 899 prs->l4r = L4R_TYPE_UDP; 900 prs->l4_off = ether_size + sizeof(struct ip6_hdr); 901 } else if (csum_flags & CSUM_IP6_TCP) { 902 prs->l3r = L3R_FIRST_IPV6; 903 prs->l4r = L4R_TYPE_TCP; 904 prs->l4_off = ether_size + sizeof(struct ip6_hdr); 905 } 906 907 prs->ip_off[0] = ether_size; 908 909 return (DPAA_FD_TX_CMD_RPD | DPAA_FD_TX_CMD_DTC); 910 } 911 912 /** 913 * @group dTSEC IFnet routines. 914 * @{ 915 */ 916 void 917 dpaa_eth_if_start_locked(struct dpaa_eth_softc *sc) 918 { 919 vm_size_t psize, ssize; 920 struct dpaa_eth_frame_info *fi; 921 unsigned int i; 922 struct mbuf *m0, *m; 923 vm_offset_t vaddr; 924 struct dpaa_fd fd; 925 926 DPAA_ETH_LOCK_ASSERT(sc); 927 /* TODO: IFF_DRV_OACTIVE */ 928 929 if ((sc->sc_mii->mii_media_status & IFM_ACTIVE) == 0) 930 return; 931 932 if ((if_getdrvflags(sc->sc_ifnet) & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 933 return; 934 935 if (sc->sc_tx_fq_full) 936 return; 937 938 while (!if_sendq_empty(sc->sc_ifnet)) { 939 if ((sc->sc_tx_queue_check_cnt++ & 940 (DTSEC_MAX_TX_QUEUE_CHECK_INTERVAL - 1)) == 0 && 941 qman_fq_get_counter(sc->sc_tx_fq, QMAN_COUNTER_FRAME) >= 942 DTSEC_MAX_TX_QUEUE_LEN) { 943 atomic_store_rel_int(&sc->sc_tx_fq_full, 1); 944 sc->sc_tx_queue_check_cnt = 0; 945 return; 946 } 947 948 fi = dpaa_eth_fi_alloc(sc); 949 if (fi == NULL) 950 return; 951 952 m0 = if_dequeue(sc->sc_ifnet); 953 if (m0 == NULL) { 954 dpaa_eth_fi_free(sc, fi); 955 return; 956 } 957 958 i = 0; 959 psize = 0; 960 fi->fi_mbuf = m0; 961 962 for (m = m0; m != NULL && i < DPAA_NUM_OF_SG_TABLE_ENTRY; 963 m = m->m_next) { 964 vm_size_t rem; 965 966 if (m->m_len == 0) 967 continue; 968 969 vaddr = (vm_offset_t)m->m_data; 970 rem = m->m_len; 971 972 /* 973 * Fast path: the whole segment lives inside one 974 * page. Covers every default-cluster mbuf 975 * (MCLBYTES < PAGE_SIZE) and skips the split loop 976 * in the common case. 977 */ 978 if ((vaddr & PAGE_MASK) + rem <= PAGE_SIZE) { 979 fi->fi_sgt[i].addr = dpaa_eth_va_to_phys(vaddr); 980 fi->fi_sgt[i].length = rem; 981 fi->fi_sgt[i].extension = 0; 982 fi->fi_sgt[i].final = 0; 983 fi->fi_sgt[i].bpid = 0; 984 fi->fi_sgt[i].offset = 0; 985 psize += rem; 986 i++; 987 continue; 988 } 989 990 /* 991 * Slow path: mbuf crosses at least one page 992 * boundary (jumbo cluster, or an unusually-offset 993 * external buffer). Emit one SGT entry per 994 * contiguous physical span. 995 */ 996 while (rem > 0 && i < DPAA_NUM_OF_SG_TABLE_ENTRY) { 997 ssize = PAGE_SIZE - (vaddr & PAGE_MASK); 998 if (rem < ssize) 999 ssize = rem; 1000 1001 fi->fi_sgt[i].addr = dpaa_eth_va_to_phys(vaddr); 1002 fi->fi_sgt[i].length = ssize; 1003 fi->fi_sgt[i].extension = 0; 1004 fi->fi_sgt[i].final = 0; 1005 fi->fi_sgt[i].bpid = 0; 1006 fi->fi_sgt[i].offset = 0; 1007 1008 rem -= ssize; 1009 vaddr += ssize; 1010 psize += ssize; 1011 i++; 1012 } 1013 1014 if (rem > 0) /* SGT full mid-mbuf */ 1015 break; 1016 } 1017 1018 /* 1019 * Reject the frame if we didn't consume the whole chain 1020 * (SGT full mid-frame) or if the chain produced no SGT 1021 * entries at all (all-zero-length mbufs). 1022 */ 1023 if (m != NULL || i == 0) { 1024 dpaa_eth_fi_free(sc, fi); 1025 m_freem(m0); 1026 continue; 1027 } 1028 1029 fi->fi_sgt[i - 1].final = 1; 1030 1031 fd.addr = DMAP_TO_PHYS((vm_offset_t)fi); 1032 fd.length = psize; 1033 fd.format = DPAA_FD_FORMAT_SHORT_MBSF; 1034 1035 fd.liodn = 0; 1036 fd.bpid = 0; 1037 fd.eliodn = 0; 1038 fd.offset = offsetof(struct dpaa_eth_frame_info, fi_sgt); 1039 fd.cmd_stat = dpaa_eth_tx_add_csum(fi); 1040 1041 DPAA_ETH_UNLOCK(sc); 1042 if (qman_fq_enqueue(sc->sc_tx_fq, &fd) != 0) { 1043 dpaa_eth_fi_free(sc, fi); 1044 m_freem(m0); 1045 } 1046 DPAA_ETH_LOCK(sc); 1047 } 1048 } 1049 /** @} */ 1050