1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright © 2021-2022 Dmitry Salychev 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 /* 30 * QBMan command interface and the DPAA2 I/O (DPIO) driver. 31 * 32 * The DPIO object allows configuration of the QBMan software portal with 33 * optional notification capabilities. 34 * 35 * Software portals are used by the driver to communicate with the QBMan. The 36 * DPIO object’s main purpose is to enable the driver to perform I/O – enqueue 37 * and dequeue operations, as well as buffer release and acquire operations – 38 * using QBMan. 39 */ 40 41 #include "opt_rss.h" 42 43 #include <sys/param.h> 44 #include <sys/kernel.h> 45 #include <sys/bus.h> 46 #include <sys/rman.h> 47 #include <sys/module.h> 48 #include <sys/malloc.h> 49 #include <sys/lock.h> 50 #include <sys/mutex.h> 51 #include <sys/_cpuset.h> 52 #include <sys/cpuset.h> 53 #include <sys/sysctl.h> 54 #include <sys/taskqueue.h> 55 #include <sys/smp.h> 56 57 #include <vm/vm.h> 58 59 #include <machine/bus.h> 60 #include <machine/resource.h> 61 62 #include <dev/pci/pcivar.h> 63 64 #ifdef RSS 65 #include <net/rss_config.h> 66 #endif 67 68 #include "pcib_if.h" 69 #include "pci_if.h" 70 71 #include "dpaa2_mc.h" 72 #include "dpaa2_mcp.h" 73 #include "dpaa2_swp.h" 74 #include "dpaa2_swp_if.h" 75 #include "dpaa2_cmd_if.h" 76 #include "dpaa2_io.h" 77 #include "dpaa2_ni.h" 78 #include "dpaa2_channel.h" 79 80 #define DPIO_IRQ_INDEX 0 /* index of the only DPIO IRQ */ 81 #define DPIO_POLL_MAX 32 82 83 #define SWP_HOLDOFF_MS_MIN 100 84 #define SWP_HOLDOFF_MS_MAX 10000 85 86 /* 87 * Memory: 88 * 0: cache-enabled part of the QBMan software portal. 89 * 1: cache-inhibited part of the QBMan software portal. 90 * 2: control registers of the QBMan software portal? 91 * 92 * Note that MSI should be allocated separately using pseudo-PCI interface. 93 */ 94 struct resource_spec dpaa2_io_spec[] = { 95 /* 96 * System Memory resources. 97 */ 98 #define MEM_RES_NUM (3u) 99 #define MEM_RID_OFF (0u) 100 #define MEM_RID(rid) ((rid) + MEM_RID_OFF) 101 { SYS_RES_MEMORY, MEM_RID(0), RF_ACTIVE | RF_UNMAPPED }, 102 { SYS_RES_MEMORY, MEM_RID(1), RF_ACTIVE | RF_UNMAPPED }, 103 { SYS_RES_MEMORY, MEM_RID(2), RF_ACTIVE | RF_UNMAPPED | RF_OPTIONAL }, 104 /* 105 * DPMCP resources. 106 * 107 * NOTE: MC command portals (MCPs) are used to send commands to, and 108 * receive responses from, the MC firmware. One portal per DPIO. 109 */ 110 #define MCP_RES_NUM (1u) 111 #define MCP_RID_OFF (MEM_RID_OFF + MEM_RES_NUM) 112 #define MCP_RID(rid) ((rid) + MCP_RID_OFF) 113 /* --- */ 114 { DPAA2_DEV_MCP, MCP_RID(0), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 115 /* --- */ 116 RESOURCE_SPEC_END 117 }; 118 119 /* Configuration routines. */ 120 static int dpaa2_io_setup_irqs(device_t dev); 121 static int dpaa2_io_release_irqs(device_t dev); 122 static int dpaa2_io_setup_msi(struct dpaa2_io_softc *sc); 123 static int dpaa2_io_release_msi(struct dpaa2_io_softc *sc); 124 125 /* Interrupt handlers */ 126 static void dpaa2_io_intr(void *arg); 127 128 static int 129 dpaa2_io_probe(device_t dev) 130 { 131 /* DPIO device will be added by a parent resource container itself. */ 132 device_set_desc(dev, "DPAA2 I/O"); 133 return (BUS_PROBE_DEFAULT); 134 } 135 136 static int 137 dpaa2_io_detach(device_t dev) 138 { 139 device_t pdev = device_get_parent(dev); 140 device_t child = dev; 141 struct dpaa2_io_softc *sc = device_get_softc(dev); 142 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 143 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 144 struct dpaa2_cmd cmd; 145 uint16_t rc_token, io_token; 146 int error; 147 148 DPAA2_CMD_INIT(&cmd); 149 150 /* Tear down interrupt handler and release IRQ resources. */ 151 dpaa2_io_release_irqs(dev); 152 153 /* Free software portal helper object. */ 154 dpaa2_swp_free_portal(sc->swp); 155 156 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 157 if (error) { 158 device_printf(dev, "%s: failed to open DPRC: error=%d\n", 159 __func__, error); 160 goto err_exit; 161 } 162 error = DPAA2_CMD_IO_OPEN(dev, child, &cmd, dinfo->id, &io_token); 163 if (error) { 164 device_printf(dev, "%s: failed to open DPIO: id=%d, error=%d\n", 165 __func__, dinfo->id, error); 166 goto close_rc; 167 } 168 169 error = DPAA2_CMD_IO_DISABLE(dev, child, &cmd); 170 if (error && bootverbose) { 171 device_printf(dev, "%s: failed to disable DPIO: id=%d, " 172 "error=%d\n", __func__, dinfo->id, error); 173 } 174 175 (void)DPAA2_CMD_IO_CLOSE(dev, child, &cmd); 176 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 177 178 /* Unmap memory resources of the portal. */ 179 for (int i = 0; i < MEM_RES_NUM; i++) { 180 if (sc->res[MEM_RID(i)] == NULL) { 181 continue; 182 } 183 error = bus_unmap_resource(sc->dev, SYS_RES_MEMORY, 184 sc->res[MEM_RID(i)], &sc->map[MEM_RID(i)]); 185 if (error && bootverbose) { 186 device_printf(dev, "%s: failed to unmap memory " 187 "resource: rid=%d, error=%d\n", __func__, MEM_RID(i), 188 error); 189 } 190 } 191 192 /* Release allocated resources. */ 193 bus_release_resources(dev, dpaa2_io_spec, sc->res); 194 195 return (0); 196 197 close_rc: 198 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 199 err_exit: 200 return (error); 201 } 202 203 static int 204 dpaa2_io_collect_irq_holdoff(SYSCTL_HANDLER_ARGS) 205 { 206 struct dpaa2_io_softc *sc; 207 struct dpaa2_swp *swp; 208 uint32_t error, new_value; 209 210 sc = (struct dpaa2_io_softc *)arg1; 211 swp = sc->swp; 212 error = SYSCTL_OUT(req, &sc->irq_holdoff, 213 sizeof(sc->irq_holdoff)); 214 if (error || req->newptr == NULL) 215 return error; 216 error = SYSCTL_IN(req, &new_value, sizeof(new_value)); 217 if (error) 218 return error; 219 if ((new_value < SWP_HOLDOFF_MS_MIN) || (new_value > SWP_HOLDOFF_MS_MAX)) 220 return EINVAL; 221 error = dpaa2_swp_set_irq_coalescing(swp, swp->dqrr.ring_size - 1, 222 new_value); 223 if (error) 224 return error; 225 sc->irq_holdoff = new_value; 226 return 0; 227 } 228 229 static void 230 dpaa2_io_setup_sysctls(struct dpaa2_io_softc *sc) 231 { 232 struct sysctl_ctx_list *ctx; 233 struct sysctl_oid *node; 234 struct sysctl_oid_list *parent; 235 236 ctx = device_get_sysctl_ctx(sc->dev); 237 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 238 239 node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "io", 240 CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "io control"); 241 parent = SYSCTL_CHILDREN(node); 242 243 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "irq_holdoff", 244 CTLTYPE_U32 | CTLFLAG_RW, sc, 0, dpaa2_io_collect_irq_holdoff, "IU", 245 "SWP holdoff time in ms"); 246 } 247 248 static int 249 dpaa2_io_attach(device_t dev) 250 { 251 device_t pdev = device_get_parent(dev); 252 device_t child = dev; 253 device_t mcp_dev; 254 struct dpaa2_io_softc *sc = device_get_softc(dev); 255 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 256 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 257 struct dpaa2_devinfo *mcp_dinfo; 258 struct dpaa2_cmd cmd; 259 struct resource_map_request req; 260 struct { 261 vm_memattr_t memattr; 262 char *label; 263 } map_args[MEM_RES_NUM] = { 264 { VM_MEMATTR_WRITE_BACK, "cache-enabled part" }, 265 { VM_MEMATTR_DEVICE, "cache-inhibited part" }, 266 { VM_MEMATTR_DEVICE, "control registers" } 267 }; 268 uint16_t rc_token, io_token; 269 int error; 270 271 sc->dev = dev; 272 sc->swp = NULL; 273 sc->intr = NULL; 274 sc->irq_resource = NULL; 275 sc->irq_holdoff = 120; 276 277 /* Allocate resources. */ 278 error = bus_alloc_resources(sc->dev, dpaa2_io_spec, sc->res); 279 if (error) { 280 device_printf(dev, "%s: failed to allocate resources: " 281 "error=%d\n", __func__, error); 282 return (ENXIO); 283 } 284 285 /* Set allocated MC portal up. */ 286 mcp_dev = (device_t) rman_get_start(sc->res[MCP_RID(0)]); 287 mcp_dinfo = device_get_ivars(mcp_dev); 288 dinfo->portal = mcp_dinfo->portal; 289 290 /* Map memory resources of the portal. */ 291 for (int i = 0; i < MEM_RES_NUM; i++) { 292 if (sc->res[MEM_RID(i)] == NULL) { 293 continue; 294 } 295 296 resource_init_map_request(&req); 297 req.memattr = map_args[i].memattr; 298 error = bus_map_resource(sc->dev, SYS_RES_MEMORY, 299 sc->res[MEM_RID(i)], &req, &sc->map[MEM_RID(i)]); 300 if (error) { 301 device_printf(dev, "%s: failed to map %s: error=%d\n", 302 __func__, map_args[i].label, error); 303 goto err_exit; 304 } 305 } 306 307 DPAA2_CMD_INIT(&cmd); 308 309 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 310 if (error) { 311 device_printf(dev, "%s: failed to open DPRC: error=%d\n", 312 __func__, error); 313 goto err_exit; 314 } 315 error = DPAA2_CMD_IO_OPEN(dev, child, &cmd, dinfo->id, &io_token); 316 if (error) { 317 device_printf(dev, "%s: failed to open DPIO: id=%d, error=%d\n", 318 __func__, dinfo->id, error); 319 goto close_rc; 320 } 321 error = DPAA2_CMD_IO_RESET(dev, child, &cmd); 322 if (error) { 323 device_printf(dev, "%s: failed to reset DPIO: id=%d, error=%d\n", 324 __func__, dinfo->id, error); 325 goto close_io; 326 } 327 error = DPAA2_CMD_IO_GET_ATTRIBUTES(dev, child, &cmd, &sc->attr); 328 if (error) { 329 device_printf(dev, "%s: failed to get DPIO attributes: id=%d, " 330 "error=%d\n", __func__, dinfo->id, error); 331 goto close_io; 332 } 333 error = DPAA2_CMD_IO_ENABLE(dev, child, &cmd); 334 if (error) { 335 device_printf(dev, "%s: failed to enable DPIO: id=%d, " 336 "error=%d\n", __func__, dinfo->id, error); 337 goto close_io; 338 } 339 340 /* Prepare descriptor of the QBMan software portal. */ 341 sc->swp_desc.dpio_dev = dev; 342 sc->swp_desc.swp_version = sc->attr.swp_version; 343 sc->swp_desc.swp_clk = sc->attr.swp_clk; 344 sc->swp_desc.swp_id = sc->attr.swp_id; 345 sc->swp_desc.has_notif = sc->attr.priors_num ? true : false; 346 sc->swp_desc.has_8prio = sc->attr.priors_num == 8u ? true : false; 347 348 sc->swp_desc.cena_res = sc->res[0]; 349 sc->swp_desc.cena_map = &sc->map[0]; 350 sc->swp_desc.cinh_res = sc->res[1]; 351 sc->swp_desc.cinh_map = &sc->map[1]; 352 353 /* 354 * Compute how many 256 QBMAN cycles fit into one ns. This is because 355 * the interrupt timeout period register needs to be specified in QBMAN 356 * clock cycles in increments of 256. 357 */ 358 sc->swp_desc.swp_cycles_ratio = 256000 / 359 (sc->swp_desc.swp_clk / 1000000); 360 361 dpaa2_io_setup_sysctls(sc); 362 363 /* Initialize QBMan software portal. */ 364 error = dpaa2_swp_init_portal(sc, DPAA2_SWP_DEF); 365 if (error) { 366 device_printf(dev, "%s: failed to initialize dpaa2_swp: " 367 "error=%d\n", __func__, error); 368 goto err_exit; 369 } 370 371 error = dpaa2_io_setup_irqs(dev); 372 if (error) { 373 device_printf(dev, "%s: failed to setup IRQs: error=%d\n", 374 __func__, error); 375 goto err_exit; 376 } 377 378 if (bootverbose) { 379 device_printf(dev, "dpio_id=%d, swp_id=%d, chan_mode=%s, " 380 "notif_priors=%d, swp_version=0x%x\n", 381 sc->attr.id, sc->attr.swp_id, 382 sc->attr.chan_mode == DPAA2_IO_LOCAL_CHANNEL 383 ? "local_channel" : "no_channel", sc->attr.priors_num, 384 sc->attr.swp_version); 385 } 386 387 (void)DPAA2_CMD_IO_CLOSE(dev, child, &cmd); 388 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 389 return (0); 390 391 close_io: 392 (void)DPAA2_CMD_IO_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, io_token)); 393 close_rc: 394 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 395 err_exit: 396 dpaa2_io_detach(dev); 397 return (ENXIO); 398 } 399 400 /** 401 * @brief Enqueue multiple frames to a frame queue using one FQID. 402 */ 403 static int 404 dpaa2_io_enq_multiple_fq(device_t iodev, uint32_t fqid, 405 struct dpaa2_fd *fd, int frames_n) 406 { 407 struct dpaa2_io_softc *sc = device_get_softc(iodev); 408 struct dpaa2_swp *swp = sc->swp; 409 struct dpaa2_eq_desc ed; 410 uint32_t flags = 0; 411 412 memset(&ed, 0, sizeof(ed)); 413 414 /* Setup enqueue descriptor. */ 415 dpaa2_swp_set_ed_norp(&ed, false); 416 dpaa2_swp_set_ed_fq(&ed, fqid); 417 418 return (dpaa2_swp_enq_mult(swp, &ed, fd, &flags, frames_n)); 419 } 420 421 /** 422 * @brief Configure the channel data availability notification (CDAN) 423 * in a particular WQ channel paired with DPIO. 424 */ 425 static int 426 dpaa2_io_conf_wq_channel(device_t iodev, struct dpaa2_io_notif_ctx *ctx) 427 { 428 struct dpaa2_io_softc *sc = device_get_softc(iodev); 429 430 /* Enable generation of the CDAN notifications. */ 431 if (ctx->cdan_en) { 432 return (dpaa2_swp_conf_wq_channel(sc->swp, ctx->fq_chan_id, 433 DPAA2_WQCHAN_WE_EN | DPAA2_WQCHAN_WE_CTX, ctx->cdan_en, 434 ctx->qman_ctx)); 435 } 436 437 return (0); 438 } 439 440 /** 441 * @brief Query current configuration/state of the buffer pool. 442 */ 443 static int 444 dpaa2_io_query_bp(device_t iodev, uint16_t bpid, struct dpaa2_bp_conf *conf) 445 { 446 struct dpaa2_io_softc *sc = device_get_softc(iodev); 447 448 return (dpaa2_swp_query_bp(sc->swp, bpid, conf)); 449 } 450 451 /** 452 * @brief Release one or more buffer pointers to the QBMan buffer pool. 453 */ 454 static int 455 dpaa2_io_release_bufs(device_t iodev, uint16_t bpid, bus_addr_t *buf, 456 uint32_t buf_num) 457 { 458 struct dpaa2_io_softc *sc = device_get_softc(iodev); 459 460 return (dpaa2_swp_release_bufs(sc->swp, bpid, buf, buf_num)); 461 } 462 463 /** 464 * @brief Configure DPNI object to generate interrupts. 465 */ 466 static int 467 dpaa2_io_setup_irqs(device_t dev) 468 { 469 struct dpaa2_io_softc *sc = device_get_softc(dev); 470 int error; 471 472 /* 473 * Setup interrupts generated by the software portal. 474 */ 475 dpaa2_swp_set_intr_trigger(sc->swp, DPAA2_SWP_INTR_DQRI); 476 dpaa2_swp_clear_intr_status(sc->swp, 0xFFFFFFFFu); 477 478 /* Configure IRQs. */ 479 error = dpaa2_io_setup_msi(sc); 480 if (error) { 481 device_printf(dev, "%s: failed to allocate MSI: error=%d\n", 482 __func__, error); 483 return (error); 484 } 485 if ((sc->irq_resource = bus_alloc_resource_any(dev, SYS_RES_IRQ, 486 &sc->irq_rid[0], RF_ACTIVE | RF_SHAREABLE)) == NULL) { 487 device_printf(dev, "%s: failed to allocate IRQ resource\n", 488 __func__); 489 return (ENXIO); 490 } 491 if (bus_setup_intr(dev, sc->irq_resource, INTR_TYPE_NET | INTR_MPSAFE | 492 INTR_ENTROPY, NULL, dpaa2_io_intr, sc, &sc->intr)) { 493 device_printf(dev, "%s: failed to setup IRQ resource\n", 494 __func__); 495 return (ENXIO); 496 } 497 498 /* Wrap DPIO ID around number of CPUs/RSS buckets */ 499 #ifdef RSS 500 sc->cpu = rss_getcpu(sc->attr.id % rss_getnumbuckets()); 501 #else 502 sc->cpu = sc->attr.id % mp_ncpus; 503 #endif 504 CPU_SETOF(sc->cpu, &sc->cpu_mask); 505 bus_bind_intr(dev, sc->irq_resource, sc->cpu); 506 507 /* 508 * Setup and enable Static Dequeue Command to receive CDANs from 509 * channel 0. 510 */ 511 if (sc->swp_desc.has_notif) 512 dpaa2_swp_set_push_dequeue(sc->swp, 0, true); 513 514 return (0); 515 } 516 517 static int 518 dpaa2_io_release_irqs(device_t dev) 519 { 520 struct dpaa2_io_softc *sc = device_get_softc(dev); 521 522 /* Disable receiving CDANs from channel 0. */ 523 if (sc->swp_desc.has_notif) 524 dpaa2_swp_set_push_dequeue(sc->swp, 0, false); 525 526 /* Release IRQ resources. */ 527 if (sc->intr != NULL) 528 bus_teardown_intr(dev, sc->irq_resource, &sc->intr); 529 if (sc->irq_resource != NULL) 530 bus_release_resource(dev, SYS_RES_IRQ, sc->irq_rid[0], 531 sc->irq_resource); 532 533 (void)dpaa2_io_release_msi(device_get_softc(dev)); 534 535 /* Configure software portal to stop generating interrupts. */ 536 dpaa2_swp_set_intr_trigger(sc->swp, 0); 537 dpaa2_swp_clear_intr_status(sc->swp, 0xFFFFFFFFu); 538 539 return (0); 540 } 541 542 /** 543 * @brief Allocate MSI interrupts for this DPAA2 I/O object. 544 */ 545 static int 546 dpaa2_io_setup_msi(struct dpaa2_io_softc *sc) 547 { 548 int val; 549 550 val = pci_msi_count(sc->dev); 551 if (val < DPAA2_IO_MSI_COUNT) 552 device_printf(sc->dev, "MSI: actual=%d, expected=%d\n", val, 553 DPAA2_IO_MSI_COUNT); 554 val = MIN(val, DPAA2_IO_MSI_COUNT); 555 556 if (pci_alloc_msi(sc->dev, &val) != 0) 557 return (EINVAL); 558 559 for (int i = 0; i < val; i++) 560 sc->irq_rid[i] = i + 1; 561 562 return (0); 563 } 564 565 static int 566 dpaa2_io_release_msi(struct dpaa2_io_softc *sc) 567 { 568 int error; 569 570 error = pci_release_msi(sc->dev); 571 if (error) { 572 device_printf(sc->dev, "%s: failed to release MSI: error=%d/n", 573 __func__, error); 574 return (error); 575 } 576 577 return (0); 578 } 579 580 /** 581 * @brief DPAA2 I/O interrupt handler. 582 */ 583 static void 584 dpaa2_io_intr(void *arg) 585 { 586 struct dpaa2_io_softc *sc = (struct dpaa2_io_softc *) arg; 587 /* struct dpaa2_ni_softc *nisc = NULL; */ 588 struct dpaa2_io_notif_ctx *ctx[DPIO_POLL_MAX]; 589 struct dpaa2_channel *chan; 590 struct dpaa2_dq dq; 591 uint32_t idx, status; 592 uint16_t flags; 593 int rc, cdan_n = 0; 594 595 status = dpaa2_swp_read_intr_status(sc->swp); 596 if (status == 0) { 597 return; 598 } 599 600 DPAA2_SWP_LOCK(sc->swp, &flags); 601 if (flags & DPAA2_SWP_DESTROYED) { 602 /* Terminate operation if portal is destroyed. */ 603 DPAA2_SWP_UNLOCK(sc->swp); 604 return; 605 } 606 607 for (int i = 0; i < DPIO_POLL_MAX; i++) { 608 rc = dpaa2_swp_dqrr_next_locked(sc->swp, &dq, &idx); 609 if (rc) { 610 break; 611 } 612 613 if ((dq.common.verb & DPAA2_DQRR_RESULT_MASK) == 614 DPAA2_DQRR_RESULT_CDAN) { 615 ctx[cdan_n++] = (struct dpaa2_io_notif_ctx *) dq.scn.ctx; 616 } else { 617 /* TODO: Report unknown DQRR entry. */ 618 } 619 dpaa2_swp_write_reg(sc->swp, DPAA2_SWP_CINH_DCAP, idx); 620 } 621 DPAA2_SWP_UNLOCK(sc->swp); 622 623 for (int i = 0; i < cdan_n; i++) { 624 chan = (struct dpaa2_channel *)ctx[i]->channel; 625 /* nisc = device_get_softc(chan->ni_dev); */ 626 taskqueue_enqueue(chan->cleanup_tq, &chan->cleanup_task); 627 } 628 629 /* Enable software portal interrupts back */ 630 dpaa2_swp_clear_intr_status(sc->swp, status); 631 dpaa2_swp_write_reg(sc->swp, DPAA2_SWP_CINH_IIR, 0); 632 } 633 634 static device_method_t dpaa2_io_methods[] = { 635 /* Device interface */ 636 DEVMETHOD(device_probe, dpaa2_io_probe), 637 DEVMETHOD(device_attach, dpaa2_io_attach), 638 DEVMETHOD(device_detach, dpaa2_io_detach), 639 640 /* QBMan software portal interface */ 641 DEVMETHOD(dpaa2_swp_enq_multiple_fq, dpaa2_io_enq_multiple_fq), 642 DEVMETHOD(dpaa2_swp_conf_wq_channel, dpaa2_io_conf_wq_channel), 643 DEVMETHOD(dpaa2_swp_query_bp, dpaa2_io_query_bp), 644 DEVMETHOD(dpaa2_swp_release_bufs, dpaa2_io_release_bufs), 645 646 DEVMETHOD_END 647 }; 648 649 static driver_t dpaa2_io_driver = { 650 "dpaa2_io", 651 dpaa2_io_methods, 652 sizeof(struct dpaa2_io_softc), 653 }; 654 655 DRIVER_MODULE(dpaa2_io, dpaa2_rc, dpaa2_io_driver, 0, 0); 656