1 /*
2 * Copyright (c) 2026 Justin Hibbits
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6 /*-
7 * Copyright (c) 2011-2012 Semihalf.
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/bus.h>
36 #include <sys/lock.h>
37 #include <sys/malloc.h>
38 #include <sys/module.h>
39 #include <sys/mutex.h>
40 #include <sys/proc.h>
41 #include <sys/pcpu.h>
42 #include <sys/rman.h>
43 #include <sys/sched.h>
44 #include <sys/smp.h>
45
46 #include <machine/bus.h>
47 #include <machine/resource.h>
48 #include <machine/tlb.h>
49
50 #include "dpaa_common.h"
51 #include "portals.h"
52 #include "qman.h"
53 #include "qman_var.h"
54 #include "qman_portal_if.h"
55
56 /* Registers */
57 #define QCSP_IO_CFG(n) (0x004 + (n) * 16)
58 #define IO_CFG_SDEST_M 0x00ff0000
59 #define IO_CFG_SDEST_S 16
60 #define QMAN_DCP_CFG(n) (0x300 + (n) * 0x10)
61 #define DCP_CFG_ED 0x00000100
62 #define DCP_CFG_ED_3 0x00001000
63 #define QMAN_PFDR_FP_LWIT 0x410
64 #define QMAN_PFDR_CFG 0x414
65 #define QMAN_SFDR_CFG 0x500
66 #define QMAN_MCR 0xb00
67 #define MCR_INIT_PFDR 0x01000000
68 #define MCR_READ_PFDR 0x02000000
69 #define MCR_READ_SFDR 0x03000000
70 #define MCR_QUERY_FQD_FILL 0x10000000
71 #define MCR_QUERY_FQD_TAGS 0x11000000
72 #define MCR_QUERY_FQD_CACHE 0x12000000
73 #define MCR_QUERY_WQ 0x20000000
74 #define MCR_RSLT_OK 0xf0000000
75 #define MCR_RSLT_OK_DATA 0xf1000000
76 #define MCR_RSLT_ABRT_INV 0xf4000000
77 #define MCR_RSLT_ABRT_DIS 0xf8000000
78 #define MCR_RSLT_ABRT_IDX 0xff000000
79 #define MCR_RSLT_ABRT_MASK 0xff000000
80 #define QMAN_MCP0 0xb04
81 #define QMAN_MCP1 0xb08
82 #define QMAN_IP_REV_1 0xbf8
83 #define IP_MJ_M 0x0000ff00
84 #define IP_MJ_S 8
85 #define IP_MN_M 0x000000ff
86 #define QMAN_FQD_BARE 0xc00
87 #define QMAN_FQD_BAR 0xc04
88 #define QMAN_FQD_AR 0xc10
89 #define AR_EN 0x80000000
90 #define QMAN_PFDR_BARE 0xc20
91 #define QMAN_PFDR_BAR 0xc24
92 #define QMAN_PFDR_AR 0xc30
93 #define QMAN_QCSP_BARE 0xc80
94 #define QMAN_QCSP_BAR 0xc84
95 #define QMAN_QCSP_AR 0xc90
96 #define QMAN_CI_SCHED_CFG 0xd00
97 #define CI_SCHED_CFG_SW 0x80000000
98 #define CI_SCHED_CFG_SRCCIV 0x04000000 /* Recommended */
99 #define CI_SCHED_CFG_SRQ_W_M 0x00000700
100 #define CI_SCHED_CFG_SRQ_W_S 8
101 #define CI_SCHED_CFG_RW_W_M 0x00000070
102 #define CI_SCHED_CFG_RW_W_S 4
103 #define CI_SCHED_CFG_BMAN_W_M 0x00000007
104 #define QMAN_ERR_ISR 0xe00
105 #define QMAN_ERR_IER 0xe04
106 #define QCSP_IO_CFG_3(n) (0x1004 + (n) * 16)
107
108 /* Software portals. Cache-inhibited registers */
109
110 #define QCSP_DQRR_PDQCR 0x05c
111
112 /* Software portals. Cache-enabled registers */
113
114 /*
115 * Context_A stashing config.
116 */
117 #define QM_STASHING_EXCL_ANNOTATION 0x04
118 #define QM_STASHING_EXCL_DATA 0x02
119 #define QM_STASHING_EXCL_CONTEXT 0x01
120
121 #define QMAN_CHANNEL_POOL1_REV1 0x21
122 #define QMAN_CHANNEL_POOL1_REV3 0x401
123
124 #define QMAN_PFDR_MAX 0xfffeff
125
126 /* P1023 has only 3 pool channels, but we don't support that SoC. */
127 #define QMAN_POOL_CHANNELS 15
128
129 /* P1023 only supports 64 congestion groups... */
130 #define QMAN_CGRS 256
131
132 static struct qman_softc *qman_sc;
133
134 static MALLOC_DEFINE(M_QMAN, "qman", "DPAA Queue Manager structures");
135
136 int qman_channel_base;
137 int qman_total_fqids;
138 struct qman_fq **qman_fq_list;
139
140 /* Entries sorted right-to-left in bit order of the ISR */
141 static const char * const qman_errors[] = {
142 "Invalid enqueue queue",
143 "Invalid enqueue channel!",
144 "Invalid enqueue state",
145 "Invalid enqueue overflow",
146 "Invalid enqueue configuration",
147 NULL,
148 NULL,
149 NULL,
150 "Invalid dequeue queue",
151 "Invalid dequeue source",
152 "Invalid dequeue FQ",
153 "Invalid dequeue direct connect portal",
154 NULL,
155 NULL,
156 NULL,
157 NULL,
158 "Invalid command verb",
159 "Invalid FQ flow control state",
160 NULL,
161 NULL,
162 NULL,
163 NULL,
164 NULL,
165 "Insufficient free PFDRs",
166 "Single-bit ECC error",
167 "Multi-bit ECC error",
168 "PFDR low watermark",
169 "Invalid target transaction",
170 "Initiator data error",
171 NULL,
172 NULL
173 };
174
175 static void
qman_isr(void * arg)176 qman_isr(void *arg)
177 {
178 struct qman_softc *sc = arg;
179 uint32_t ier, isr, isr_bit;
180 int i;
181
182 ier = bus_read_4(sc->sc_rres, QMAN_ERR_IER);
183 isr = bus_read_4(sc->sc_rres, QMAN_ERR_ISR);
184
185 if ((ier & isr) == 0)
186 return;
187
188 isr_bit = (isr & ier);
189 for (i = 0; isr_bit != 0; i++, isr_bit >>= 1) {
190 if (isr_bit & 1)
191 device_printf(sc->sc_dev, "%s", qman_errors[i]);
192 }
193
194 bus_write_4(sc->sc_rres, QMAN_ERR_ISR, isr);
195 }
196
197
198 /* Set up reserved memory configuration for PFDR and FQD, per `off`. */
199 static int
qman_set_memory(struct qman_softc * sc,vm_paddr_t pa,vm_size_t size,bus_size_t off)200 qman_set_memory(struct qman_softc *sc, vm_paddr_t pa,
201 vm_size_t size, bus_size_t off)
202 {
203 uint32_t bar, bare;
204 vm_paddr_t old_bar;
205
206 /*
207 * Register offsets:
208 * 0 - BARE
209 * 4 - BAR
210 * 0x10 - AR
211 */
212 bare = bus_read_4(sc->sc_rres, off);
213 bar = bus_read_4(sc->sc_rres, off + 4);
214 old_bar = (vm_paddr_t)bare << 32 | bar;
215
216 if (old_bar != 0 && old_bar != pa) {
217 device_printf(sc->sc_dev, "QMan BAR already initialized!\n");
218 return (ENOMEM);
219 } else if (old_bar == pa)
220 return (EEXIST);
221
222 /*
223 * Zero the memory and flush cache through DMAP. QMan accesses the
224 * memory as non-coherent.
225 */
226 memset((void *)PHYS_TO_DMAP(pa), 0, size);
227 cpu_flush_dcache((void *)PHYS_TO_DMAP(pa), size);
228
229 bus_write_4(sc->sc_rres, off, pa >> 32);
230 bus_write_4(sc->sc_rres, off + 4, (uint32_t)pa);
231 bus_write_4(sc->sc_rres, off + 0x10, AR_EN | (ilog2(size) - 1));
232
233 return (0);
234 }
235
236 /*
237 * Set up PFDR structures. Some things to keep in mind:
238 * - npfdr is the total number of PFDRs in the private memory. PFDRs are 64
239 * bytes in size, so npfdr is (pfdr_sz/64).
240 * - PFDR 0-7 are reserved, so the base starts at 8, not 0, so we adjust
241 * internally.
242 * - The second parameter is the last PFDR, not the number of PFDRs, so needs to
243 * be adjusted down one more, so subtract 9.
244 */
245 static int
qman_setup_pfdr(struct qman_softc * sc,int npfdr)246 qman_setup_pfdr(struct qman_softc *sc, int npfdr)
247 {
248 uint32_t res;
249
250 npfdr = min(npfdr, QMAN_PFDR_MAX);
251 bus_write_4(sc->sc_rres, QMAN_MCP0, 8);
252 bus_write_4(sc->sc_rres, QMAN_MCP1, npfdr - 9);
253 bus_write_4(sc->sc_rres, QMAN_MCR, MCR_INIT_PFDR);
254
255 for (int timeout = 100000; timeout > 0; timeout--) {
256 DELAY(1);
257 res = bus_read_4(sc->sc_rres, QMAN_MCR);
258 if (res >= MCR_RSLT_OK)
259 break;
260 }
261
262 if (res < MCR_RSLT_OK)
263 return (EBUSY);
264 if (res == MCR_RSLT_OK)
265 return (0);
266
267 return (ENXIO);
268 }
269
270 int
qman_attach(device_t dev)271 qman_attach(device_t dev)
272 {
273 struct qman_softc *sc;
274 int error;
275 vm_paddr_t fqd_pa, pfdr_pa;
276 vm_size_t fqd_sz, pfdr_sz;
277 int qman_channel_pool1 = QMAN_CHANNEL_POOL1_REV1;
278 uint32_t ver;
279 uint32_t nfqd;
280 bool qman3 = false;
281
282 sc = device_get_softc(dev);
283 sc->sc_dev = dev;
284 qman_sc = sc;
285
286 /* Allocate resources */
287 sc->sc_rrid = 0;
288 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 0, RF_ACTIVE);
289 if (sc->sc_rres == NULL) {
290 device_printf(dev, "could not allocate memory.\n");
291 goto err;
292 }
293
294 sc->sc_irid = 0;
295 sc->sc_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ,
296 &sc->sc_irid, RF_ACTIVE | RF_SHAREABLE);
297 if (sc->sc_ires == NULL) {
298 device_printf(dev, "could not allocate error interrupt.\n");
299 goto err;
300 }
301 error = dpaa_map_private_memory(dev, 0, "fsl,qman-fqd",
302 &fqd_pa, &fqd_sz);
303 error = dpaa_map_private_memory(dev, 1, "fsl,qman-pfdr",
304 &pfdr_pa, &pfdr_sz);
305
306 bzero((void *)PHYS_TO_DMAP(fqd_pa), fqd_sz);
307 cpu_flush_dcache((void *)PHYS_TO_DMAP(fqd_pa), fqd_sz);
308 /*
309 * FQDs are 64 bytes in size, with 24 bit pointers, so FQIDs are 24
310 * bits, fits fine in a uint32_t.
311 */
312 nfqd = fqd_sz / 64;
313 qman_total_fqids = nfqd;
314 qman_fq_list = malloc(nfqd * sizeof(struct qman_fq *), M_QMAN,
315 M_WAITOK);
316
317 error = qman_set_memory(sc, fqd_pa, fqd_sz, QMAN_FQD_BARE);
318 if (error != 0 && error != EEXIST)
319 goto err;
320 error = qman_set_memory(sc, pfdr_pa, pfdr_sz, QMAN_PFDR_BARE);
321 if (error != 0 && error != EEXIST)
322 goto err;
323 if (error == 0) {
324 /* Initialize PFDRs if it hasn't been initialized before */
325 error = qman_setup_pfdr(sc, pfdr_sz / 64);
326 if (error != 0)
327 goto err;
328 /* Magic constant from documentation */
329 bus_write_4(sc->sc_rres, QMAN_PFDR_CFG, 64);
330 }
331
332 bus_write_4(sc->sc_rres, QMAN_ERR_ISR, 0xffffffff);
333 bus_write_4(sc->sc_rres, QMAN_ERR_IER, 0xffffffff);
334
335 ver = bus_read_4(sc->sc_rres, QMAN_IP_REV_1);
336 sc->sc_qman_major = ((ver & IP_MJ_M) >> IP_MJ_S);
337 if (sc->sc_qman_major >= 3)
338 qman3 = true;
339
340 if (qman3)
341 qman_channel_pool1 = QMAN_CHANNEL_POOL1_REV3;
342 qman_channel_base = qman_channel_pool1;
343
344 sc->sc_qman_base_channel = qman_channel_pool1;
345
346 sc->sc_fqalloc =
347 vmem_create("qman-fqalloc", 1, nfqd - 1, 1, 0, M_WAITOK);
348 sc->sc_qpalloc =
349 vmem_create("qman-qpalloc", qman_channel_pool1,
350 QMAN_POOL_CHANNELS, 1, 0, M_WAITOK);
351 sc->sc_cgalloc = vmem_create("qman-cgalloc", 0, QMAN_CGRS,
352 1, 0, M_WAITOK);
353
354 if (bus_setup_intr(dev, sc->sc_ires, INTR_TYPE_NET, NULL, qman_isr,
355 sc, &sc->sc_intr_cookie) != 0)
356 goto err;
357
358 if (error != 0) {
359 device_printf(dev, "could not be initialized\n");
360 goto err;
361 }
362 bus_write_4(sc->sc_rres, QMAN_DCP_CFG(0),
363 qman3 ? DCP_CFG_ED_3 : DCP_CFG_ED);
364 bus_write_4(sc->sc_rres, QMAN_DCP_CFG(1),
365 qman3 ? DCP_CFG_ED_3 : DCP_CFG_ED);
366
367 bus_write_4(sc->sc_rres, 0xd00, 0x80000322);
368
369 /* TODO: DO we need a taskqueue? Allocate here if so */
370
371 return (0);
372
373 err:
374 qman_detach(dev);
375 return (ENXIO);
376 }
377
378 int
qman_detach(device_t dev)379 qman_detach(device_t dev)
380 {
381 struct qman_softc *sc;
382
383 sc = device_get_softc(dev);
384
385 if (sc->sc_fqalloc != NULL)
386 vmem_destroy(sc->sc_fqalloc);
387 if (sc->sc_qpalloc != NULL)
388 vmem_destroy(sc->sc_qpalloc);
389 if (sc->sc_cgalloc != NULL)
390 vmem_destroy(sc->sc_cgalloc);
391
392 if (sc->sc_intr_cookie != NULL)
393 bus_teardown_intr(dev, sc->sc_ires, sc->sc_intr_cookie);
394
395 if (sc->sc_ires != NULL)
396 bus_release_resource(dev, SYS_RES_IRQ,
397 sc->sc_irid, sc->sc_ires);
398
399 if (sc->sc_rres != NULL)
400 bus_release_resource(dev, SYS_RES_MEMORY,
401 sc->sc_rrid, sc->sc_rres);
402
403 free(qman_fq_list, M_QMAN);
404 qman_fq_list = NULL;
405
406 return (0);
407 }
408
409 int
qman_suspend(device_t dev)410 qman_suspend(device_t dev)
411 {
412
413 return (0);
414 }
415
416 int
qman_resume(device_t dev)417 qman_resume(device_t dev)
418 {
419
420 return (0);
421 }
422
423 int
qman_shutdown(device_t dev)424 qman_shutdown(device_t dev)
425 {
426
427 return (0);
428 }
429
430 int
qman_alloc_channel(void)431 qman_alloc_channel(void)
432 {
433 struct qman_softc *sc = qman_sc;
434 vmem_addr_t channel;
435
436 vmem_alloc(sc->sc_qpalloc, 1, M_BESTFIT | M_WAITOK, &channel);
437
438 return (channel);
439 }
440
441 void
qman_free_channel(int channel)442 qman_free_channel(int channel)
443 {
444 struct qman_softc *sc = qman_sc;
445
446 vmem_free(sc->sc_qpalloc, channel, 1);
447 }
448
449 int
qman_percpu_channel(int cpu)450 qman_percpu_channel(int cpu)
451 {
452 device_t portal;
453 struct qman_portal_softc *sc;
454
455 portal = DPCPU_ID_GET(cpu, qman_affine_portal);
456 if (portal == NULL)
457 return (-1);
458
459 sc = device_get_softc(portal);
460
461 return (sc->sc_affine_channel);
462 }
463
464 int
qman_alloc_fqid_range(uint32_t count,uint32_t align,uint32_t * basep)465 qman_alloc_fqid_range(uint32_t count, uint32_t align, uint32_t *basep)
466 {
467 struct qman_softc *sc = qman_sc;
468 vmem_addr_t base;
469 int error;
470
471 error = vmem_xalloc(sc->sc_fqalloc, count, align, 0, 0,
472 VMEM_ADDR_MIN, VMEM_ADDR_MAX, M_BESTFIT | M_NOWAIT, &base);
473 if (error != 0)
474 return (error);
475 *basep = base;
476 return (0);
477 }
478
479 void
qman_free_fqid_range(uint32_t base,uint32_t count)480 qman_free_fqid_range(uint32_t base, uint32_t count)
481 {
482 struct qman_softc *sc = qman_sc;
483
484 vmem_free(sc->sc_fqalloc, base, count);
485 }
486
487 /**
488 * @group QMan API functions implementation.
489 * @{
490 */
491
492 struct qman_fq *
qman_fq_from_index(uint32_t fqid)493 qman_fq_from_index(uint32_t fqid)
494 {
495 if (fqid > qman_total_fqids)
496 return (NULL);
497 return (qman_fq_list[fqid]);
498 }
499
500 /* Allocate and initialize an FQ Range */
501 struct qman_fq *
qman_fq_create(uint32_t fqids_num,int channel,uint8_t wq,bool force_fqid,uint32_t fqid_or_align,bool init_parked,bool hold_active,bool prefer_in_cache,bool congst_avoid_ena,void * congst_group,int8_t overhead_accounting_len,uint32_t tail_drop_threshold,uint8_t annotation_cl,uint8_t data_cl,uint64_t context_a_opaque,uint32_t context_b_opaque)502 qman_fq_create(uint32_t fqids_num, int channel, uint8_t wq,
503 bool force_fqid, uint32_t fqid_or_align, bool init_parked,
504 bool hold_active, bool prefer_in_cache, bool congst_avoid_ena,
505 void *congst_group, int8_t overhead_accounting_len,
506 uint32_t tail_drop_threshold,
507 uint8_t annotation_cl, uint8_t data_cl,
508 uint64_t context_a_opaque, uint32_t context_b_opaque)
509 {
510 union qman_mc_command cmd;
511 struct qman_softc *sc;
512 union qman_mc_result *res;
513 struct qman_fq *fqh;
514 device_t portal;
515 vmem_addr_t fqid_base;
516 uint8_t rslt;
517
518 sc = qman_sc;
519
520 bzero(&cmd, sizeof(cmd));
521 if (force_fqid) {
522 /*
523 * Caller has already reserved the FQID (typically via
524 * qman_alloc_fqid_range() for a KeyGen distribution range)
525 * and passes the concrete FQID via fqid_or_align. Do not
526 * touch the vmem allocator; just record the base for
527 * downstream book-keeping.
528 */
529 fqid_base = fqid_or_align;
530 } else {
531 int error;
532
533 error = vmem_alloc(sc->sc_fqalloc, fqids_num,
534 M_BESTFIT | M_WAITOK, &fqid_base);
535 if (error != 0) {
536 device_printf(sc->sc_dev,
537 "qman_fq_create: no FQID range of %u\n",
538 fqids_num);
539 return (NULL);
540 }
541 }
542 cmd.init_fq.fqid = fqid_base;
543 cmd.init_fq.count = fqids_num - 1;
544 cmd.init_fq.dest_chan = channel;
545 cmd.init_fq.dest_wq = wq;
546 cmd.init_fq.we_mask = QCSP_INIT_FQ_WE_DEST_WQ | QCSP_INIT_FQ_WE_FQ_CTRL;
547 if (init_parked)
548 cmd.init_fq.verb = QCSP_VERB_INIT_FQ_PARK;
549 else
550 cmd.init_fq.verb = QCSP_VERB_INIT_FQ_SCHED;
551 cmd.init_fq.fq_ctrl = (prefer_in_cache ? QM_FQCTRL_LIC : 0) |
552 (hold_active ? QM_FQCTRL_HOLDACTIVE : 0) |
553 (congst_avoid_ena ? QM_FQCTRL_AVOIDBLOCK : 0);
554
555 /*
556 * Configure hardware cache stashing: on dequeue, QMan will push
557 * the requested number of cachelines of frame annotation and/or
558 * frame data into the destination core's cache, hiding memory
559 * latency for the RX callback.
560 */
561 if (annotation_cl != 0 || data_cl != 0) {
562 uint64_t excl, cl;
563
564 excl = (annotation_cl != 0 ? QM_STASHING_EXCL_ANNOTATION : 0) |
565 (data_cl != 0 ? QM_STASHING_EXCL_DATA : 0);
566 cl = ((annotation_cl & 3) << 4) | ((data_cl & 3) << 2);
567 /* excl in wire byte 0 (bits 63-56), cl in wire byte 1. */
568 cmd.init_fq.context_a = (excl << 56) | (cl << 48);
569 cmd.init_fq.fq_ctrl |= QM_FQCTRL_CTXASTASH;
570 cmd.init_fq.we_mask |= QCSP_INIT_FQ_WE_CONTEXT_A;
571 } else if (context_a_opaque != 0) {
572 /*
573 * Opaque ContextA: caller (e.g. SEC in QI mode) needs the
574 * FQ's ContextA read literally by the consumer. CTXASTASH
575 * stays off so QMan doesn't reinterpret it as stashing
576 * config.
577 */
578 cmd.init_fq.context_a = context_a_opaque;
579 cmd.init_fq.we_mask |= QCSP_INIT_FQ_WE_CONTEXT_A;
580 }
581
582 if (context_b_opaque != 0) {
583 cmd.init_fq.context_b = context_b_opaque;
584 cmd.init_fq.we_mask |= QCSP_INIT_FQ_WE_CONTEXT_B;
585 }
586
587 critical_enter();
588
589 /* Ensure we have got QMan port initialized */
590 portal = DPCPU_GET(qman_affine_portal);
591 res = QMAN_PORTAL_MC_SEND_RAW(portal, &cmd);
592
593 rslt = 0;
594 if (res != NULL)
595 rslt = res->init_fq.rslt;
596
597 critical_exit();
598 if (res == NULL || rslt != QMAN_MC_RES_OK) {
599 if (!force_fqid)
600 vmem_free(sc->sc_fqalloc, fqid_base, fqids_num);
601 goto err;
602 }
603
604 fqh = malloc(sizeof(*fqh), M_QMAN, M_WAITOK | M_ZERO);
605 fqh->fqid = fqid_base;
606 fqh->fqid_count = fqids_num;
607 fqh->force_fqid = force_fqid;
608
609 for (uint32_t i = 0; i < fqids_num; i++)
610 qman_fq_list[fqid_base + i] = fqh;
611
612 return (fqh);
613
614 err:
615
616 return (NULL);
617 }
618
619 static int
qman_fq_retire_one(device_t portal,uint32_t fqid)620 qman_fq_retire_one(device_t portal, uint32_t fqid)
621 {
622 union qman_mc_command cmd;
623 union qman_mc_result *rr;
624
625 bzero(&cmd, sizeof(cmd));
626
627 cmd.alter_fqs.verb = QCSP_VERB_ALTER_FQ_RETIRE;
628 cmd.alter_fqs.fqid = fqid;
629 rr = QMAN_PORTAL_MC_SEND_RAW(portal, &cmd);
630 if (rr == NULL)
631 return (ETIMEDOUT);
632
633 if (rr->alter_fqs.rslt == QMAN_MC_RES_OK) {
634 if (rr->alter_fqs.fqs & QMAN_MC_AFQS_NE) {
635 /* TODO: Drain.... */
636 }
637 return (0);
638 }
639
640 return (0);
641 }
642
643 int
qman_fq_free(struct qman_fq * fq)644 qman_fq_free(struct qman_fq *fq)
645 {
646 struct qman_softc *sc;
647 device_t portal;
648 int error;
649
650 sc = qman_sc;
651
652 critical_enter();
653 portal = DPCPU_GET(qman_affine_portal);
654 for (uint32_t i = 0; i < fq->fqid_count; i++) {
655 error = qman_fq_retire_one(portal, fq->fqid + i);
656 if (error != 0) {
657 critical_exit();
658 return (error);
659 }
660 }
661 /* TODO: Take FQ out of service. */
662 critical_exit();
663
664 if (!fq->force_fqid)
665 vmem_free(sc->sc_fqalloc, fq->fqid, fq->fqid_count);
666 for (uint32_t i = 0; i < fq->fqid_count; i++)
667 qman_fq_list[fq->fqid + i] = NULL;
668 free(fq, M_QMAN);
669
670 return (0);
671 }
672
673 int
qman_fq_register_cb(struct qman_fq * fq,qman_cb_dqrr callback,void * ctx)674 qman_fq_register_cb(struct qman_fq *fq, qman_cb_dqrr callback,
675 void *ctx)
676 {
677 fq->cb.dqrr = callback;
678 fq->cb.ctx = ctx;
679
680 return (0);
681 }
682
683 int
qman_fq_register_flush_cb(struct qman_fq * fq,qman_cb_flush flush)684 qman_fq_register_flush_cb(struct qman_fq *fq, qman_cb_flush flush)
685 {
686 fq->cb.flush = flush;
687 return (0);
688 }
689
690 int
qman_fq_enqueue(struct qman_fq * fq,struct dpaa_fd * frame)691 qman_fq_enqueue(struct qman_fq *fq, struct dpaa_fd *frame)
692 {
693 struct qman_softc *sc;
694 int error;
695 void *portal;
696
697 sc = qman_sc;
698 critical_enter();
699
700 /* Ensure we have got QMan port initialized */
701 portal = DPCPU_GET(qman_affine_portal);
702 if (portal == NULL) {
703 device_printf(sc->sc_dev, "could not setup QMan portal\n");
704 critical_exit();
705 return (ENXIO);
706 }
707
708 error = QMAN_PORTAL_ENQUEUE(portal, fq, frame);
709
710 critical_exit();
711
712 return (error);
713 }
714
715 uint32_t
qman_fq_get_fqid(struct qman_fq * fq)716 qman_fq_get_fqid(struct qman_fq *fq)
717 {
718 return (fq->fqid);
719 }
720
721
722 uint32_t
qman_fq_get_counter(struct qman_fq * fq,int counter)723 qman_fq_get_counter(struct qman_fq *fq, int counter)
724 {
725 union qman_mc_result *cmd_res;
726 union qman_mc_command command;
727 device_t portal;
728 u_int ret = 0;
729
730 bzero(&command, sizeof(command));
731 command.query_fq_np.verb = QCSP_VERB_QUERY_FQ_NP;
732 command.query_fq_np.fqid = fq->fqid;
733 critical_enter();
734 portal = DPCPU_GET(qman_affine_portal);
735 cmd_res = QMAN_PORTAL_MC_SEND_RAW(portal, &command);
736 if (counter == QMAN_COUNTER_FRAME)
737 ret = cmd_res->query_fq_np.frm_cnt;
738 else if (counter == QMAN_COUNTER_BYTES)
739 ret = cmd_res->query_fq_np.byte_cnt;
740
741 critical_exit();
742
743 return (ret);
744 }
745
746 void
qman_set_sdest(uint16_t channel,int cpu)747 qman_set_sdest(uint16_t channel, int cpu)
748 {
749 struct qman_softc *sc = qman_sc;
750 uint32_t reg;
751
752 if (sc->sc_qman_major >= 3) {
753 reg = bus_read_4(sc->sc_rres, QCSP_IO_CFG_3(channel));
754 reg &= IO_CFG_SDEST_M;
755 reg |= (cpu << IO_CFG_SDEST_S);
756 bus_write_4(sc->sc_rres, QCSP_IO_CFG_3(channel), reg);
757 } else {
758 reg = bus_read_4(sc->sc_rres, QCSP_IO_CFG(channel));
759 reg &= IO_CFG_SDEST_M;
760 reg |= (cpu << IO_CFG_SDEST_S);
761 bus_write_4(sc->sc_rres, QCSP_IO_CFG(channel), reg);
762 }
763 }
764
765 /*
766 * TODO: add polling and/or congestion support.
767 */
768
769 /** @} */
770