1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright © 2023 Dmitry Salychev 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* 29 * QBMan channel to process ingress traffic (Rx, Tx confirmation, Rx error). 30 * 31 * NOTE: Several WQs are organized into a single channel. 32 */ 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/bus.h> 38 #include <sys/rman.h> 39 #include <sys/module.h> 40 #include <sys/malloc.h> 41 #include <sys/mutex.h> 42 #include <sys/socket.h> 43 #include <sys/sockio.h> 44 #include <sys/sysctl.h> 45 #include <sys/mbuf.h> 46 #include <sys/taskqueue.h> 47 #include <sys/sysctl.h> 48 #include <sys/buf_ring.h> 49 #include <sys/smp.h> 50 #include <sys/proc.h> 51 52 #include <machine/bus.h> 53 #include <machine/resource.h> 54 #include <machine/atomic.h> 55 #include <machine/vmparam.h> 56 57 #include <net/ethernet.h> 58 #include <net/bpf.h> 59 #include <net/if.h> 60 #include <net/if_dl.h> 61 #include <net/if_media.h> 62 #include <net/if_types.h> 63 #include <net/if_var.h> 64 65 #include "dpaa2_types.h" 66 #include "dpaa2_channel.h" 67 #include "dpaa2_ni.h" 68 #include "dpaa2_mc.h" 69 #include "dpaa2_mc_if.h" 70 #include "dpaa2_mcp.h" 71 #include "dpaa2_io.h" 72 #include "dpaa2_con.h" 73 #include "dpaa2_buf.h" 74 #include "dpaa2_swp.h" 75 #include "dpaa2_swp_if.h" 76 #include "dpaa2_bp.h" 77 #include "dpaa2_cmd_if.h" 78 79 MALLOC_DEFINE(M_DPAA2_CH, "dpaa2_ch", "DPAA2 QBMan Channel"); 80 81 #define TX_SEG_N (16u) /* XXX-DSL: does DPAA2 limit exist? */ 82 #define TX_SEG_SZ (PAGE_SIZE) 83 #define TX_SEG_MAXSZ (TX_SEG_N * TX_SEG_SZ) 84 CTASSERT(TX_SEG_SZ % PAGE_SIZE == 0); 85 CTASSERT(TX_SEG_MAXSZ % PAGE_SIZE == 0); 86 87 #define SGT_SEG_N (1u) 88 #define SGT_SEG_SZ (PAGE_SIZE) 89 #define SGT_SEG_MAXSZ (PAGE_SIZE) 90 CTASSERT(SGT_SEG_SZ % PAGE_SIZE == 0); 91 CTASSERT(SGT_SEG_MAXSZ % PAGE_SIZE == 0); 92 93 static int dpaa2_chan_setup_dma(device_t, struct dpaa2_channel *, bus_size_t); 94 static int dpaa2_chan_alloc_storage(device_t, struct dpaa2_channel *, bus_size_t, 95 int, bus_size_t); 96 static void dpaa2_chan_bp_task(void *, int); 97 98 /** 99 * @brief Сonfigures QBMan channel and registers data availability notifications. 100 */ 101 int 102 dpaa2_chan_setup(device_t dev, device_t iodev, device_t condev, device_t bpdev, 103 struct dpaa2_channel **channel, uint32_t flowid, task_fn_t cleanup_task_fn) 104 { 105 device_t pdev = device_get_parent(dev); 106 device_t child = dev; 107 struct dpaa2_ni_softc *sc = device_get_softc(dev); 108 struct dpaa2_io_softc *iosc = device_get_softc(iodev); 109 struct dpaa2_con_softc *consc = device_get_softc(condev); 110 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 111 struct dpaa2_devinfo *ioinfo = device_get_ivars(iodev); 112 struct dpaa2_devinfo *coninfo = device_get_ivars(condev); 113 struct dpaa2_con_notif_cfg notif_cfg; 114 struct dpaa2_io_notif_ctx *ctx; 115 struct dpaa2_channel *ch = NULL; 116 struct dpaa2_cmd cmd; 117 uint16_t rctk, contk; 118 int error; 119 120 DPAA2_CMD_INIT(&cmd); 121 122 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rctk); 123 if (error) { 124 device_printf(dev, "%s: failed to open DPRC: id=%d, error=%d\n", 125 __func__, rcinfo->id, error); 126 goto fail_rc_open; 127 } 128 error = DPAA2_CMD_CON_OPEN(dev, child, &cmd, coninfo->id, &contk); 129 if (error) { 130 device_printf(dev, "%s: failed to open DPCON: id=%d, error=%d\n", 131 __func__, coninfo->id, error); 132 goto fail_con_open; 133 } 134 135 error = DPAA2_CMD_CON_ENABLE(dev, child, &cmd); 136 if (error) { 137 device_printf(dev, "%s: failed to enable channel: dpcon_id=%d, " 138 "chan_id=%d\n", __func__, coninfo->id, consc->attr.chan_id); 139 goto fail_con_enable; 140 } 141 142 ch = malloc(sizeof(struct dpaa2_channel), M_DPAA2_CH, M_WAITOK | M_ZERO); 143 ch->ni_dev = dev; 144 ch->io_dev = iodev; 145 ch->con_dev = condev; 146 ch->id = consc->attr.chan_id; 147 ch->flowid = flowid; 148 ch->tx_frames = 0; /* for debug purposes */ 149 ch->tx_dropped = 0; /* for debug purposes */ 150 ch->store_sz = 0; 151 ch->store_idx = 0; 152 ch->recycled_n = 0; 153 ch->rxq_n = 0; 154 155 NET_TASK_INIT(&ch->cleanup_task, 0, cleanup_task_fn, ch); 156 NET_TASK_INIT(&ch->bp_task, 0, dpaa2_chan_bp_task, ch); 157 158 ch->cleanup_tq = taskqueue_create("dpaa2_ch cleanup", M_WAITOK, 159 taskqueue_thread_enqueue, &ch->cleanup_tq); 160 taskqueue_start_threads_cpuset(&ch->cleanup_tq, 1, PI_NET, 161 &iosc->cpu_mask, "dpaa2_ch%d cleanup", ch->id); 162 163 error = dpaa2_chan_setup_dma(dev, ch, sc->buf_align); 164 if (error != 0) { 165 device_printf(dev, "%s: failed to setup DMA\n", __func__); 166 goto fail_dma_setup; 167 } 168 169 mtx_init(&ch->xmit_mtx, "dpaa2_ch_xmit", NULL, MTX_DEF); 170 171 ch->xmit_br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF, M_NOWAIT, 172 &ch->xmit_mtx); 173 if (ch->xmit_br == NULL) { 174 device_printf(dev, "%s: buf_ring_alloc() failed\n", __func__); 175 error = ENOMEM; 176 goto fail_buf_ring; 177 } 178 179 DPAA2_BUF_INIT(&ch->store); 180 181 /* Register the new notification context */ 182 ctx = &ch->ctx; 183 ctx->qman_ctx = (uint64_t)ctx; 184 ctx->cdan_en = true; 185 ctx->fq_chan_id = ch->id; 186 ctx->io_dev = ch->io_dev; 187 ctx->channel = ch; 188 error = DPAA2_SWP_CONF_WQ_CHANNEL(ch->io_dev, ctx); 189 if (error) { 190 device_printf(dev, "%s: failed to register CDAN context\n", 191 __func__); 192 goto fail_dpcon_notif; 193 } 194 195 /* Register DPCON notification within Management Complex */ 196 notif_cfg.dpio_id = ioinfo->id; 197 notif_cfg.prior = 0; 198 notif_cfg.qman_ctx = ctx->qman_ctx; 199 error = DPAA2_CMD_CON_SET_NOTIF(dev, child, &cmd, ¬if_cfg); 200 if (error) { 201 device_printf(dev, "%s: failed to register DPCON " 202 "notifications: dpcon_id=%d, chan_id=%d\n", __func__, 203 coninfo->id, consc->attr.chan_id); 204 goto fail_dpcon_notif; 205 } 206 207 /* Allocate initial # of Rx buffers and a channel storage */ 208 error = dpaa2_buf_seed_pool(dev, bpdev, ch, DPAA2_NI_BUFS_INIT, 209 DPAA2_RX_BUF_SIZE); 210 if (error) { 211 device_printf(dev, "%s: failed to seed buffer pool\n", 212 __func__); 213 goto fail_dpcon_notif; 214 } 215 error = dpaa2_chan_alloc_storage(dev, ch, DPAA2_ETH_STORE_SIZE, 216 BUS_DMA_NOWAIT, sc->buf_align); 217 if (error != 0) { 218 device_printf(dev, "%s: failed to allocate channel storage\n", 219 __func__); 220 goto fail_dpcon_notif; 221 } else { 222 ch->store_sz = DPAA2_ETH_STORE_FRAMES; 223 } 224 225 /* Prepare queues for the channel */ 226 error = dpaa2_chan_setup_fq(dev, ch, DPAA2_NI_QUEUE_TX_CONF); 227 if (error) { 228 device_printf(dev, "%s: failed to prepare TxConf queue: " 229 "error=%d\n", __func__, error); 230 goto fail_fq_setup; 231 } 232 error = dpaa2_chan_setup_fq(dev, ch, DPAA2_NI_QUEUE_RX); 233 if (error) { 234 device_printf(dev, "%s: failed to prepare Rx queue: error=%d\n", 235 __func__, error); 236 goto fail_fq_setup; 237 } 238 239 if (bootverbose) { 240 device_printf(dev, "channel: dpio_id=%d dpcon_id=%d chan_id=%d, " 241 "priorities=%d\n", ioinfo->id, coninfo->id, ch->id, 242 consc->attr.prior_num); 243 } 244 245 *channel = ch; 246 247 (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd); 248 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rctk)); 249 250 return (0); 251 252 fail_fq_setup: 253 if (ch->store.vaddr != NULL) { 254 bus_dmamem_free(ch->store.dmat, ch->store.vaddr, ch->store.dmap); 255 } 256 if (ch->store.dmat != NULL) { 257 bus_dma_tag_destroy(ch->store.dmat); 258 } 259 ch->store.dmat = NULL; 260 ch->store.vaddr = NULL; 261 ch->store.paddr = 0; 262 ch->store.nseg = 0; 263 fail_dpcon_notif: 264 buf_ring_free(ch->xmit_br, M_DEVBUF); 265 fail_buf_ring: 266 mtx_destroy(&ch->xmit_mtx); 267 fail_dma_setup: 268 /* while (taskqueue_cancel(ch->cleanup_tq, &ch->cleanup_task, NULL)) { */ 269 /* taskqueue_drain(ch->cleanup_tq, &ch->cleanup_task); */ 270 /* } */ 271 /* taskqueue_free(ch->cleanup_tq); */ 272 (void)DPAA2_CMD_CON_DISABLE(dev, child, DPAA2_CMD_TK(&cmd, contk)); 273 fail_con_enable: 274 (void)DPAA2_CMD_CON_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, contk)); 275 fail_con_open: 276 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rctk)); 277 fail_rc_open: 278 return (error); 279 } 280 281 /** 282 * @brief Performs an initial configuration of the frame queue. 283 */ 284 int 285 dpaa2_chan_setup_fq(device_t dev, struct dpaa2_channel *ch, 286 enum dpaa2_ni_queue_type queue_type) 287 { 288 struct dpaa2_ni_softc *sc = device_get_softc(dev); 289 struct dpaa2_ni_fq *fq; 290 291 switch (queue_type) { 292 case DPAA2_NI_QUEUE_TX_CONF: 293 /* One queue per channel */ 294 fq = &ch->txc_queue; 295 fq->chan = ch; 296 fq->flowid = ch->flowid; 297 fq->tc = 0; /* ignored */ 298 fq->type = queue_type; 299 break; 300 case DPAA2_NI_QUEUE_RX: 301 KASSERT(sc->attr.num.rx_tcs <= DPAA2_MAX_TCS, 302 ("too many Rx traffic classes: rx_tcs=%d\n", 303 sc->attr.num.rx_tcs)); 304 305 /* One queue per Rx traffic class within a channel */ 306 for (int i = 0; i < sc->attr.num.rx_tcs; i++) { 307 fq = &ch->rx_queues[i]; 308 fq->chan = ch; 309 fq->flowid = ch->flowid; 310 fq->tc = (uint8_t) i; 311 fq->type = queue_type; 312 313 ch->rxq_n++; 314 } 315 break; 316 case DPAA2_NI_QUEUE_RX_ERR: 317 /* One queue per network interface */ 318 fq = &sc->rxe_queue; 319 fq->chan = ch; 320 fq->flowid = 0; /* ignored */ 321 fq->tc = 0; /* ignored */ 322 fq->type = queue_type; 323 break; 324 default: 325 device_printf(dev, "%s: unexpected frame queue type: %d\n", 326 __func__, queue_type); 327 return (EINVAL); 328 } 329 330 return (0); 331 } 332 333 /** 334 * @brief Obtain the next dequeue response from the channel storage. 335 */ 336 int 337 dpaa2_chan_next_frame(struct dpaa2_channel *ch, struct dpaa2_dq **dq) 338 { 339 struct dpaa2_buf *buf = &ch->store; 340 struct dpaa2_dq *msgs = (struct dpaa2_dq *)buf->vaddr; 341 struct dpaa2_dq *msg = &msgs[ch->store_idx]; 342 int rc = EINPROGRESS; 343 344 ch->store_idx++; 345 346 if (msg->fdr.desc.stat & DPAA2_DQ_STAT_EXPIRED) { 347 rc = EALREADY; /* VDQ command is expired */ 348 ch->store_idx = 0; 349 if (!(msg->fdr.desc.stat & DPAA2_DQ_STAT_VALIDFRAME)) { 350 msg = NULL; /* Null response, FD is invalid */ 351 } 352 } 353 if (msg != NULL && (msg->fdr.desc.stat & DPAA2_DQ_STAT_FQEMPTY)) { 354 rc = ENOENT; /* FQ is empty */ 355 ch->store_idx = 0; 356 } 357 358 if (dq != NULL) { 359 *dq = msg; 360 } 361 362 return (rc); 363 } 364 365 static int 366 dpaa2_chan_setup_dma(device_t dev, struct dpaa2_channel *ch, 367 bus_size_t alignment) 368 { 369 int error; 370 371 mtx_init(&ch->dma_mtx, "dpaa2_ch_dma_mtx", NULL, MTX_DEF); 372 373 error = bus_dma_tag_create( 374 bus_get_dma_tag(dev), /* parent */ 375 alignment, 0, /* alignment, boundary */ 376 BUS_SPACE_MAXADDR, /* low restricted addr */ 377 BUS_SPACE_MAXADDR, /* high restricted addr */ 378 NULL, NULL, /* filter, filterarg */ 379 RX_SEG_MAXSZ, /* maxsize */ 380 RX_SEG_N, /* nsegments */ 381 RX_SEG_SZ, /* maxsegsize */ 382 0, /* flags */ 383 NULL, /* lockfunc */ 384 NULL, /* lockarg */ 385 &ch->rx_dmat); 386 if (error) { 387 device_printf(dev, "%s: failed to create rx_dmat\n", __func__); 388 goto fail_rx_tag; 389 } 390 391 error = bus_dma_tag_create( 392 bus_get_dma_tag(dev), /* parent */ 393 alignment, 0, /* alignment, boundary */ 394 BUS_SPACE_MAXADDR, /* low restricted addr */ 395 BUS_SPACE_MAXADDR, /* high restricted addr */ 396 NULL, NULL, /* filter, filterarg */ 397 TX_SEG_MAXSZ, /* maxsize */ 398 TX_SEG_N, /* nsegments */ 399 TX_SEG_SZ, /* maxsegsize */ 400 0, /* flags */ 401 NULL, /* lockfunc */ 402 NULL, /* lockarg */ 403 &ch->tx_dmat); 404 if (error) { 405 device_printf(dev, "%s: failed to create tx_dmat\n", __func__); 406 goto fail_tx_tag; 407 } 408 409 error = bus_dma_tag_create( 410 bus_get_dma_tag(dev), /* parent */ 411 alignment, 0, /* alignment, boundary */ 412 BUS_SPACE_MAXADDR, /* low restricted addr */ 413 BUS_SPACE_MAXADDR, /* high restricted addr */ 414 NULL, NULL, /* filter, filterarg */ 415 SGT_SEG_MAXSZ, /* maxsize */ 416 SGT_SEG_N, /* nsegments */ 417 SGT_SEG_SZ, /* maxsegsize */ 418 0, /* flags */ 419 NULL, /* lockfunc */ 420 NULL, /* lockarg */ 421 &ch->sgt_dmat); 422 if (error) { 423 device_printf(dev, "%s: failed to create sgt_dmat\n", __func__); 424 goto fail_sgt_tag; 425 } 426 427 return (0); 428 429 fail_sgt_tag: 430 bus_dma_tag_destroy(ch->tx_dmat); 431 fail_tx_tag: 432 bus_dma_tag_destroy(ch->rx_dmat); 433 fail_rx_tag: 434 mtx_destroy(&ch->dma_mtx); 435 ch->rx_dmat = NULL; 436 ch->tx_dmat = NULL; 437 ch->sgt_dmat = NULL; 438 439 return (error); 440 } 441 442 /** 443 * @brief Allocate a DMA-mapped storage to keep responses from VDQ command. 444 */ 445 static int 446 dpaa2_chan_alloc_storage(device_t dev, struct dpaa2_channel *ch, bus_size_t size, 447 int mapflags, bus_size_t alignment) 448 { 449 struct dpaa2_buf *buf = &ch->store; 450 uint32_t maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE; 451 int error; 452 453 error = bus_dma_tag_create( 454 bus_get_dma_tag(dev), /* parent */ 455 alignment, 0, /* alignment, boundary */ 456 BUS_SPACE_MAXADDR, /* low restricted addr */ 457 BUS_SPACE_MAXADDR, /* high restricted addr */ 458 NULL, NULL, /* filter, filterarg */ 459 maxsize, /* maxsize */ 460 1, /* nsegments */ 461 maxsize, /* maxsegsize */ 462 BUS_DMA_ALLOCNOW, /* flags */ 463 NULL, /* lockfunc */ 464 NULL, /* lockarg */ 465 &buf->dmat); 466 if (error != 0) { 467 device_printf(dev, "%s: failed to create DMA tag\n", __func__); 468 goto fail_tag; 469 } 470 471 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, 472 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); 473 if (error != 0) { 474 device_printf(dev, "%s: failed to allocate storage memory\n", 475 __func__); 476 goto fail_map_create; 477 } 478 479 buf->paddr = 0; 480 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, size, 481 dpaa2_dmamap_oneseg_cb, &buf->paddr, mapflags); 482 if (error != 0) { 483 device_printf(dev, "%s: failed to map storage memory\n", 484 __func__); 485 goto fail_map_load; 486 } 487 488 bus_dmamap_sync(buf->dmat, buf->dmap, 489 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 490 491 buf->nseg = 1; 492 493 return (0); 494 495 fail_map_load: 496 bus_dmamem_free(buf->dmat, buf->vaddr, buf->dmap); 497 fail_map_create: 498 bus_dma_tag_destroy(buf->dmat); 499 fail_tag: 500 buf->dmat = NULL; 501 buf->vaddr = NULL; 502 buf->paddr = 0; 503 buf->nseg = 0; 504 505 return (error); 506 } 507 508 /** 509 * @brief Release new buffers to the buffer pool if necessary. 510 */ 511 static void 512 dpaa2_chan_bp_task(void *arg, int count) 513 { 514 struct dpaa2_channel *ch = (struct dpaa2_channel *)arg; 515 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); 516 struct dpaa2_bp_softc *bpsc; 517 struct dpaa2_bp_conf bpconf; 518 const int buf_num = DPAA2_ATOMIC_READ(&sc->buf_num); 519 device_t bpdev; 520 int error; 521 522 /* There's only one buffer pool for now */ 523 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 524 bpsc = device_get_softc(bpdev); 525 526 /* Get state of the buffer pool */ 527 error = DPAA2_SWP_QUERY_BP(ch->io_dev, bpsc->attr.bpid, &bpconf); 528 if (error) { 529 device_printf(sc->dev, "%s: DPAA2_SWP_QUERY_BP() failed: " 530 "error=%d\n", __func__, error); 531 return; 532 } 533 534 /* Double allocated Rx buffers if amount of free buffers is < 25% */ 535 if (bpconf.free_bufn < (buf_num >> 2)) { 536 (void)dpaa2_buf_seed_pool(ch->ni_dev, bpdev, ch, buf_num, 537 DPAA2_RX_BUF_SIZE); 538 DPAA2_ATOMIC_XCHG(&sc->buf_free, bpconf.free_bufn); 539 } 540 } 541