1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright © 2023 Dmitry Salychev 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* 29 * QBMan channel to process ingress traffic (Rx, Tx confirmation, Rx error). 30 * 31 * NOTE: Several WQs are organized into a single channel. 32 */ 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/bus.h> 38 #include <sys/rman.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/mutex.h> 42 #include <sys/socket.h> 43 #include <sys/sockio.h> 44 #include <sys/sysctl.h> 45 #include <sys/mbuf.h> 46 #include <sys/taskqueue.h> 47 #include <sys/sysctl.h> 48 #include <sys/buf_ring.h> 49 #include <sys/smp.h> 50 #include <sys/proc.h> 51 52 #include <machine/bus.h> 53 #include <machine/resource.h> 54 #include <machine/atomic.h> 55 #include <machine/vmparam.h> 56 57 #include <net/ethernet.h> 58 #include <net/bpf.h> 59 #include <net/if.h> 60 #include <net/if_dl.h> 61 #include <net/if_media.h> 62 #include <net/if_types.h> 63 #include <net/if_var.h> 64 65 #include "dpaa2_types.h" 66 #include "dpaa2_channel.h" 67 #include "dpaa2_ni.h" 68 #include "dpaa2_mc.h" 69 #include "dpaa2_mc_if.h" 70 #include "dpaa2_mcp.h" 71 #include "dpaa2_io.h" 72 #include "dpaa2_con.h" 73 #include "dpaa2_buf.h" 74 #include "dpaa2_swp.h" 75 #include "dpaa2_swp_if.h" 76 #include "dpaa2_bp.h" 77 #include "dpaa2_cmd_if.h" 78 79 MALLOC_DEFINE(M_DPAA2_CH, "dpaa2_ch", "DPAA2 QBMan Channel"); 80 81 #define RX_SEG_N (1u) 82 #define RX_SEG_SZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE) 83 #define RX_SEG_MAXSZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE) 84 CTASSERT(RX_SEG_SZ % PAGE_SIZE == 0); 85 CTASSERT(RX_SEG_MAXSZ % PAGE_SIZE == 0); 86 87 #define TX_SEG_N (16u) /* XXX-DSL: does DPAA2 limit exist? */ 88 #define TX_SEG_SZ (PAGE_SIZE) 89 #define TX_SEG_MAXSZ (TX_SEG_N * TX_SEG_SZ) 90 CTASSERT(TX_SEG_SZ % PAGE_SIZE == 0); 91 CTASSERT(TX_SEG_MAXSZ % PAGE_SIZE == 0); 92 93 #define SGT_SEG_N (1u) 94 #define SGT_SEG_SZ (PAGE_SIZE) 95 #define SGT_SEG_MAXSZ (PAGE_SIZE) 96 CTASSERT(SGT_SEG_SZ % PAGE_SIZE == 0); 97 CTASSERT(SGT_SEG_MAXSZ % PAGE_SIZE == 0); 98 99 static int dpaa2_chan_setup_dma(device_t, struct dpaa2_channel *, bus_size_t); 100 static int dpaa2_chan_alloc_storage(device_t, struct dpaa2_channel *, bus_size_t, 101 int, bus_size_t); 102 static void dpaa2_chan_bp_task(void *, int); 103 104 /** 105 * @brief Сonfigures QBMan channel and registers data availability notifications. 106 */ 107 int 108 dpaa2_chan_setup(device_t dev, device_t iodev, device_t condev, device_t bpdev, 109 struct dpaa2_channel **channel, uint32_t flowid, task_fn_t cleanup_task_fn) 110 { 111 device_t pdev = device_get_parent(dev); 112 device_t child = dev; 113 struct dpaa2_ni_softc *sc = device_get_softc(dev); 114 struct dpaa2_io_softc *iosc = device_get_softc(iodev); 115 struct dpaa2_con_softc *consc = device_get_softc(condev); 116 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 117 struct dpaa2_devinfo *ioinfo = device_get_ivars(iodev); 118 struct dpaa2_devinfo *coninfo = device_get_ivars(condev); 119 struct dpaa2_con_notif_cfg notif_cfg; 120 struct dpaa2_io_notif_ctx *ctx; 121 struct dpaa2_channel *ch = NULL; 122 struct dpaa2_cmd cmd; 123 uint16_t rctk, contk; 124 int error; 125 126 DPAA2_CMD_INIT(&cmd); 127 128 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rctk); 129 if (error) { 130 device_printf(dev, "%s: failed to open DPRC: id=%d, error=%d\n", 131 __func__, rcinfo->id, error); 132 goto fail_rc_open; 133 } 134 error = DPAA2_CMD_CON_OPEN(dev, child, &cmd, coninfo->id, &contk); 135 if (error) { 136 device_printf(dev, "%s: failed to open DPCON: id=%d, error=%d\n", 137 __func__, coninfo->id, error); 138 goto fail_con_open; 139 } 140 141 error = DPAA2_CMD_CON_ENABLE(dev, child, &cmd); 142 if (error) { 143 device_printf(dev, "%s: failed to enable channel: dpcon_id=%d, " 144 "chan_id=%d\n", __func__, coninfo->id, consc->attr.chan_id); 145 goto fail_con_enable; 146 } 147 148 ch = malloc(sizeof(struct dpaa2_channel), M_DPAA2_CH, M_WAITOK | M_ZERO); 149 ch->ni_dev = dev; 150 ch->io_dev = iodev; 151 ch->con_dev = condev; 152 ch->id = consc->attr.chan_id; 153 ch->flowid = flowid; 154 ch->tx_frames = 0; /* for debug purposes */ 155 ch->tx_dropped = 0; /* for debug purposes */ 156 ch->store_sz = 0; 157 ch->store_idx = 0; 158 ch->recycled_n = 0; 159 ch->rxq_n = 0; 160 161 NET_TASK_INIT(&ch->cleanup_task, 0, cleanup_task_fn, ch); 162 NET_TASK_INIT(&ch->bp_task, 0, dpaa2_chan_bp_task, ch); 163 164 ch->cleanup_tq = taskqueue_create("dpaa2_ch cleanup", M_WAITOK, 165 taskqueue_thread_enqueue, &ch->cleanup_tq); 166 taskqueue_start_threads_cpuset(&ch->cleanup_tq, 1, PI_NET, 167 &iosc->cpu_mask, "dpaa2_ch%d cleanup", ch->id); 168 169 error = dpaa2_chan_setup_dma(dev, ch, sc->buf_align); 170 if (error != 0) { 171 device_printf(dev, "%s: failed to setup DMA\n", __func__); 172 goto fail_dma_setup; 173 } 174 175 mtx_init(&ch->xmit_mtx, "dpaa2_ch_xmit", NULL, MTX_DEF); 176 177 ch->xmit_br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF, M_NOWAIT, 178 &ch->xmit_mtx); 179 if (ch->xmit_br == NULL) { 180 device_printf(dev, "%s: buf_ring_alloc() failed\n", __func__); 181 error = ENOMEM; 182 goto fail_buf_ring; 183 } 184 185 DPAA2_BUF_INIT(&ch->store); 186 187 /* Register the new notification context */ 188 ctx = &ch->ctx; 189 ctx->qman_ctx = (uint64_t)ctx; 190 ctx->cdan_en = true; 191 ctx->fq_chan_id = ch->id; 192 ctx->io_dev = ch->io_dev; 193 ctx->channel = ch; 194 error = DPAA2_SWP_CONF_WQ_CHANNEL(ch->io_dev, ctx); 195 if (error) { 196 device_printf(dev, "%s: failed to register CDAN context\n", 197 __func__); 198 goto fail_dpcon_notif; 199 } 200 201 /* Register DPCON notification within Management Complex */ 202 notif_cfg.dpio_id = ioinfo->id; 203 notif_cfg.prior = 0; 204 notif_cfg.qman_ctx = ctx->qman_ctx; 205 error = DPAA2_CMD_CON_SET_NOTIF(dev, child, &cmd, ¬if_cfg); 206 if (error) { 207 device_printf(dev, "%s: failed to register DPCON " 208 "notifications: dpcon_id=%d, chan_id=%d\n", __func__, 209 coninfo->id, consc->attr.chan_id); 210 goto fail_dpcon_notif; 211 } 212 213 /* Allocate initial # of Rx buffers and a channel storage */ 214 error = dpaa2_buf_seed_pool(dev, bpdev, ch, DPAA2_NI_BUFS_INIT, 215 DPAA2_RX_BUF_SIZE, NULL); 216 if (error) { 217 device_printf(dev, "%s: failed to seed buffer pool\n", 218 __func__); 219 goto fail_dpcon_notif; 220 } 221 error = dpaa2_chan_alloc_storage(dev, ch, DPAA2_ETH_STORE_SIZE, 222 BUS_DMA_NOWAIT, sc->buf_align); 223 if (error != 0) { 224 device_printf(dev, "%s: failed to allocate channel storage\n", 225 __func__); 226 goto fail_dpcon_notif; 227 } else { 228 ch->store_sz = DPAA2_ETH_STORE_FRAMES; 229 } 230 231 /* Prepare queues for the channel */ 232 error = dpaa2_chan_setup_fq(dev, ch, DPAA2_NI_QUEUE_TX_CONF); 233 if (error) { 234 device_printf(dev, "%s: failed to prepare TxConf queue: " 235 "error=%d\n", __func__, error); 236 goto fail_fq_setup; 237 } 238 error = dpaa2_chan_setup_fq(dev, ch, DPAA2_NI_QUEUE_RX); 239 if (error) { 240 device_printf(dev, "%s: failed to prepare Rx queue: error=%d\n", 241 __func__, error); 242 goto fail_fq_setup; 243 } 244 245 if (bootverbose) { 246 device_printf(dev, "channel: dpio_id=%d dpcon_id=%d chan_id=%d, " 247 "priorities=%d\n", ioinfo->id, coninfo->id, ch->id, 248 consc->attr.prior_num); 249 } 250 251 *channel = ch; 252 253 (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd); 254 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rctk)); 255 256 return (0); 257 258 fail_fq_setup: 259 if (ch->store.vaddr != NULL) { 260 bus_dmamem_free(ch->store.dmat, ch->store.vaddr, ch->store.dmap); 261 } 262 if (ch->store.dmat != NULL) { 263 bus_dma_tag_destroy(ch->store.dmat); 264 } 265 ch->store.dmat = NULL; 266 ch->store.vaddr = NULL; 267 ch->store.paddr = 0; 268 ch->store.nseg = 0; 269 fail_dpcon_notif: 270 buf_ring_free(ch->xmit_br, M_DEVBUF); 271 fail_buf_ring: 272 mtx_destroy(&ch->xmit_mtx); 273 fail_dma_setup: 274 /* while (taskqueue_cancel(ch->cleanup_tq, &ch->cleanup_task, NULL)) { */ 275 /* taskqueue_drain(ch->cleanup_tq, &ch->cleanup_task); */ 276 /* } */ 277 /* taskqueue_free(ch->cleanup_tq); */ 278 (void)DPAA2_CMD_CON_DISABLE(dev, child, DPAA2_CMD_TK(&cmd, contk)); 279 fail_con_enable: 280 (void)DPAA2_CMD_CON_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, contk)); 281 fail_con_open: 282 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rctk)); 283 fail_rc_open: 284 return (error); 285 } 286 287 /** 288 * @brief Performs an initial configuration of the frame queue. 289 */ 290 int 291 dpaa2_chan_setup_fq(device_t dev, struct dpaa2_channel *ch, 292 enum dpaa2_ni_queue_type queue_type) 293 { 294 struct dpaa2_ni_softc *sc = device_get_softc(dev); 295 struct dpaa2_ni_fq *fq; 296 297 switch (queue_type) { 298 case DPAA2_NI_QUEUE_TX_CONF: 299 /* One queue per channel */ 300 fq = &ch->txc_queue; 301 fq->chan = ch; 302 fq->flowid = ch->flowid; 303 fq->tc = 0; /* ignored */ 304 fq->type = queue_type; 305 break; 306 case DPAA2_NI_QUEUE_RX: 307 KASSERT(sc->attr.num.rx_tcs <= DPAA2_MAX_TCS, 308 ("too many Rx traffic classes: rx_tcs=%d\n", 309 sc->attr.num.rx_tcs)); 310 311 /* One queue per Rx traffic class within a channel */ 312 for (int i = 0; i < sc->attr.num.rx_tcs; i++) { 313 fq = &ch->rx_queues[i]; 314 fq->chan = ch; 315 fq->flowid = ch->flowid; 316 fq->tc = (uint8_t) i; 317 fq->type = queue_type; 318 319 ch->rxq_n++; 320 } 321 break; 322 case DPAA2_NI_QUEUE_RX_ERR: 323 /* One queue per network interface */ 324 fq = &sc->rxe_queue; 325 fq->chan = ch; 326 fq->flowid = 0; /* ignored */ 327 fq->tc = 0; /* ignored */ 328 fq->type = queue_type; 329 break; 330 default: 331 device_printf(dev, "%s: unexpected frame queue type: %d\n", 332 __func__, queue_type); 333 return (EINVAL); 334 } 335 336 return (0); 337 } 338 339 /** 340 * @brief Obtain the next dequeue response from the channel storage. 341 */ 342 int 343 dpaa2_chan_next_frame(struct dpaa2_channel *ch, struct dpaa2_dq **dq) 344 { 345 struct dpaa2_buf *buf = &ch->store; 346 struct dpaa2_dq *msgs = (struct dpaa2_dq *)buf->vaddr; 347 struct dpaa2_dq *msg = &msgs[ch->store_idx]; 348 int rc = EINPROGRESS; 349 350 ch->store_idx++; 351 352 if (msg->fdr.desc.stat & DPAA2_DQ_STAT_EXPIRED) { 353 rc = EALREADY; /* VDQ command is expired */ 354 ch->store_idx = 0; 355 if (!(msg->fdr.desc.stat & DPAA2_DQ_STAT_VALIDFRAME)) { 356 msg = NULL; /* Null response, FD is invalid */ 357 } 358 } 359 if (msg != NULL && (msg->fdr.desc.stat & DPAA2_DQ_STAT_FQEMPTY)) { 360 rc = ENOENT; /* FQ is empty */ 361 ch->store_idx = 0; 362 } 363 364 if (dq != NULL) { 365 *dq = msg; 366 } 367 368 return (rc); 369 } 370 371 static int 372 dpaa2_chan_setup_dma(device_t dev, struct dpaa2_channel *ch, 373 bus_size_t alignment) 374 { 375 int error; 376 377 mtx_init(&ch->dma_mtx, "dpaa2_ch_dma_mtx", NULL, MTX_DEF); 378 379 error = bus_dma_tag_create( 380 bus_get_dma_tag(dev), /* parent */ 381 alignment, 0, /* alignment, boundary */ 382 BUS_SPACE_MAXADDR, /* low restricted addr */ 383 BUS_SPACE_MAXADDR, /* high restricted addr */ 384 NULL, NULL, /* filter, filterarg */ 385 RX_SEG_MAXSZ, /* maxsize */ 386 RX_SEG_N, /* nsegments */ 387 RX_SEG_SZ, /* maxsegsize */ 388 0, /* flags */ 389 NULL, /* lockfunc */ 390 NULL, /* lockarg */ 391 &ch->rx_dmat); 392 if (error) { 393 device_printf(dev, "%s: failed to create rx_dmat\n", __func__); 394 goto fail_rx_tag; 395 } 396 397 error = bus_dma_tag_create( 398 bus_get_dma_tag(dev), /* parent */ 399 alignment, 0, /* alignment, boundary */ 400 BUS_SPACE_MAXADDR, /* low restricted addr */ 401 BUS_SPACE_MAXADDR, /* high restricted addr */ 402 NULL, NULL, /* filter, filterarg */ 403 TX_SEG_MAXSZ, /* maxsize */ 404 TX_SEG_N, /* nsegments */ 405 TX_SEG_SZ, /* maxsegsize */ 406 0, /* flags */ 407 NULL, /* lockfunc */ 408 NULL, /* lockarg */ 409 &ch->tx_dmat); 410 if (error) { 411 device_printf(dev, "%s: failed to create tx_dmat\n", __func__); 412 goto fail_tx_tag; 413 } 414 415 error = bus_dma_tag_create( 416 bus_get_dma_tag(dev), /* parent */ 417 alignment, 0, /* alignment, boundary */ 418 BUS_SPACE_MAXADDR, /* low restricted addr */ 419 BUS_SPACE_MAXADDR, /* high restricted addr */ 420 NULL, NULL, /* filter, filterarg */ 421 SGT_SEG_MAXSZ, /* maxsize */ 422 SGT_SEG_N, /* nsegments */ 423 SGT_SEG_SZ, /* maxsegsize */ 424 0, /* flags */ 425 NULL, /* lockfunc */ 426 NULL, /* lockarg */ 427 &ch->sgt_dmat); 428 if (error) { 429 device_printf(dev, "%s: failed to create sgt_dmat\n", __func__); 430 goto fail_sgt_tag; 431 } 432 433 return (0); 434 435 fail_sgt_tag: 436 bus_dma_tag_destroy(ch->tx_dmat); 437 fail_tx_tag: 438 bus_dma_tag_destroy(ch->rx_dmat); 439 fail_rx_tag: 440 mtx_destroy(&ch->dma_mtx); 441 ch->rx_dmat = NULL; 442 ch->tx_dmat = NULL; 443 ch->sgt_dmat = NULL; 444 445 return (error); 446 } 447 448 /** 449 * @brief Allocate a DMA-mapped storage to keep responses from VDQ command. 450 */ 451 static int 452 dpaa2_chan_alloc_storage(device_t dev, struct dpaa2_channel *ch, bus_size_t size, 453 int mapflags, bus_size_t alignment) 454 { 455 struct dpaa2_buf *buf = &ch->store; 456 uint32_t maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE; 457 int error; 458 459 error = bus_dma_tag_create( 460 bus_get_dma_tag(dev), /* parent */ 461 alignment, 0, /* alignment, boundary */ 462 BUS_SPACE_MAXADDR, /* low restricted addr */ 463 BUS_SPACE_MAXADDR, /* high restricted addr */ 464 NULL, NULL, /* filter, filterarg */ 465 maxsize, /* maxsize */ 466 1, /* nsegments */ 467 maxsize, /* maxsegsize */ 468 BUS_DMA_ALLOCNOW, /* flags */ 469 NULL, /* lockfunc */ 470 NULL, /* lockarg */ 471 &buf->dmat); 472 if (error != 0) { 473 device_printf(dev, "%s: failed to create DMA tag\n", __func__); 474 goto fail_tag; 475 } 476 477 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, 478 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); 479 if (error != 0) { 480 device_printf(dev, "%s: failed to allocate storage memory\n", 481 __func__); 482 goto fail_map_create; 483 } 484 485 buf->paddr = 0; 486 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, size, 487 dpaa2_dmamap_oneseg_cb, &buf->paddr, mapflags); 488 if (error != 0) { 489 device_printf(dev, "%s: failed to map storage memory\n", 490 __func__); 491 goto fail_map_load; 492 } 493 494 bus_dmamap_sync(buf->dmat, buf->dmap, 495 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 496 497 buf->nseg = 1; 498 499 return (0); 500 501 fail_map_load: 502 bus_dmamem_free(buf->dmat, buf->vaddr, buf->dmap); 503 fail_map_create: 504 bus_dma_tag_destroy(buf->dmat); 505 fail_tag: 506 buf->dmat = NULL; 507 buf->vaddr = NULL; 508 buf->paddr = 0; 509 buf->nseg = 0; 510 511 return (error); 512 } 513 514 /** 515 * @brief Release new buffers to the buffer pool if necessary. 516 */ 517 static void 518 dpaa2_chan_bp_task(void *arg, int count) 519 { 520 struct dpaa2_channel *ch = (struct dpaa2_channel *)arg; 521 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); 522 struct dpaa2_bp_softc *bpsc; 523 struct dpaa2_bp_conf bpconf; 524 const int buf_num = DPAA2_ATOMIC_READ(&sc->buf_num); 525 device_t bpdev; 526 int error; 527 528 /* There's only one buffer pool for now */ 529 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 530 bpsc = device_get_softc(bpdev); 531 532 /* Get state of the buffer pool */ 533 error = DPAA2_SWP_QUERY_BP(ch->io_dev, bpsc->attr.bpid, &bpconf); 534 if (error) { 535 device_printf(sc->dev, "%s: DPAA2_SWP_QUERY_BP() failed: " 536 "error=%d\n", __func__, error); 537 return; 538 } 539 540 /* Double allocated Rx buffers if amount of free buffers is < 25% */ 541 if (bpconf.free_bufn < (buf_num >> 2)) { 542 mtx_assert(&ch->dma_mtx, MA_NOTOWNED); 543 mtx_lock(&ch->dma_mtx); 544 (void)dpaa2_buf_seed_pool(ch->ni_dev, bpdev, ch, buf_num, 545 DPAA2_RX_BUF_SIZE, &ch->dma_mtx); 546 mtx_unlock(&ch->dma_mtx); 547 548 DPAA2_ATOMIC_XCHG(&sc->buf_free, bpconf.free_bufn); 549 } 550 } 551