1 /*
2 * Copyright (c) 2026 Justin Hibbits
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6
7 #include "opt_platform.h"
8
9 #include <sys/param.h>
10 #include <sys/systm.h>
11 #include <sys/kernel.h>
12 #include <sys/bus.h>
13 #include <sys/lock.h>
14 #include <sys/module.h>
15 #include <sys/mutex.h>
16 #include <sys/proc.h>
17 #include <sys/pcpu.h>
18 #include <sys/sched.h>
19 #include <ddb/ddb.h>
20
21 #include <machine/bus.h>
22 #include <machine/tlb.h>
23
24 #include <dev/ofw/ofw_bus.h>
25 #include <dev/ofw/ofw_bus_subr.h>
26
27 #include <powerpc/mpc85xx/mpc85xx.h>
28
29 #include "qman.h"
30 #include "portals.h"
31 #include "qman_var.h"
32
33
34 /* Cache-enabled registers */
35 #define QCSP_EQCR_N(n) (0x0000 + (n * 64))
36 #define QMAN_EQCR_COUNT 8
37 #define QCSP_DQRR_N(n) (0x1000 + (n * 64))
38 #define QMAN_DQRR_COUNT 16
39 #define QCSP_MR_N(n) (0x2000 + (n * 64))
40 #define QMAN_MR_COUNT 8
41 #define QCSP_CR 0x3800
42 #define QCSP_RR(n) (0x3900 + 0x40 * (n))
43
44 #define QCSP_EQCR_PI_CENA 0x0000
45 #define EQCR_PI_VP 0x00000008
46 #define EQCR_PI_PI_M 0x00000007
47 #define QCSP_EQCR_CI_CENA 0x0004
48 #define EQCR_CI_C 0x00000008
49 #define EQCR_CI_CI_M 0x00000007
50 #define QCSP_DQRR_PI_CENA 0x0000
51 #define DQRR_PI_VP 0x00000010
52 #define DQRR_PI_PI_M 0x0000000f
53 #define QCSP_DQRR_CI_CENA 0x0004
54 #define DQRR_CI_C 0x00000010
55 #define DQRR_CI_CI_M 0x0000000f
56
57 #define QMAN_MC_VERB_VBIT 0x80
58
59
60 /* Cache-inhibited registers */
61 #define QCSP_EQCR_PI_CINH 0x0000
62 #define QCSP_EQCR_CI_CINH 0x0004
63 #define QCSP_DQRR_PI_CINH 0x0040
64 #define QCSP_DQRR_CI_CINH 0x0044
65 #define QCSP_EQCR_ITR 0x0008
66 #define QCSP_DQRR_ITR 0x0048
67 #define QCSP_DQRR_SDQCR 0x0054
68 #define SDQCR_SS 0x40000000
69 #define SDQCR_FC 0x20000000
70 #define SDQCR_DP 0x10000000
71 #define SDQCR_DCT_NUL 0x00000000
72 #define SDQCR_DCT_PRI_PREC 0x01000000
73 #define SDQCR_DCT_ACTIVE_WQ 0x02000000
74 #define SDQCR_DCT_ACTIVE_FQ_O 0x03000000
75 #define SDQCR_DCT_M 0x03000000
76 #define SDQCR_TOKEN_M 0x00ff0000
77 #define SDQCR_TOKEN_S 16
78 #define DQRR_DQ_SRC_M 0x0000ffff
79 #define DQRR_DQ_SRC_DCP 0x00008000
80 #define SDQCR_DQ_SRC_CHAN(n) (0x8000 >> (n + 1))
81 #define QCSP_DQRR_VDQCR 0x0058
82 #define QCSP_DQRR_PDQCR 0x005c
83 #define QCSP_MR_ITR 0x0088
84 #define QCSP_CFG 0x0100
85 #define CFG_EST_M 0x70000000
86 #define CFG_EST_S 28
87 #define CFG_EP 0x04000000
88 #define CFG_EPM_M 0x03000000
89 #define CFG_EPM_PI_CI 0x00000000
90 #define CFG_EPM_PI_CE 0x01000000
91 #define CFG_EPM_VB1 0x02000000
92 #define CFG_EPM_VB2 0x03000000
93 #define CFG_DQRR_MF_M 0x00f00000
94 #define CFG_DQRR_MF_S 20
95 #define CFG_DP 0x00040000
96 #define CFG_DCM_C_M 0x00030000
97 #define CFG_DCM_CI_CI 0x00000000
98 #define CFG_DCM_CI_CE 0x00010000
99 #define CFG_DCM_DCA1 0x00020000
100 #define CFG_DCM_DCA2 0x00030000
101 #define CFG_SD 0x00000200
102 #define CFG_MM 0x00000100
103 #define CFG_RE 0x00000080
104 #define CFG_RP 0x00000040
105 #define CFG_SE 0x00000020
106 #define CFG_SP 0x00000010
107 #define CFG_SDEST_M 0x00000007
108 #define QCSP_ISR 0x0e00
109 #define QM_PIRQ_CSCI 0x00100000
110 #define QM_PIRQ_EQCI 0x00080000
111 #define QM_PIRQ_EQRI 0x00040000
112 #define QM_PIRQ_DQRI 0x00020000
113 #define QM_PIRQ_MRI 0x00010000
114 #define QM_PIRQ_DQ_AVAIL_M 0x0000ffff
115 #define QCSP_IER 0x0e04
116 #define QCSP_ISDR 0x0e08
117 #define QCSP_IIR 0xe0c
118
119 #define QM_EQCR_VERB_CMD_ENQUEUE 0x01
120 #define QM_EQCR_VERB_BIT_INT 0x04
121
122 #define DEF_SDQCR_TOKEN 0xab
123
124 static void qman_portal_loop_rings(struct qman_portal_softc *sc);
125 static void qman_portal_isr(void *);
126
127 DPCPU_DEFINE(device_t, qman_affine_portal);
128 DPAA_RING(qman_eqcr, QMAN_EQCR_COUNT, QCSP_EQCR_PI_CENA, QCSP_EQCR_CI_CENA,
129 QCSP_EQCR_PI_CINH, QCSP_EQCR_CI_CINH);
130 DPAA_RING(qman_dqrr, QMAN_DQRR_COUNT, QCSP_DQRR_PI_CENA, QCSP_DQRR_CI_CENA,
131 QCSP_DQRR_PI_CINH, QCSP_DQRR_CI_CINH);
132
133 /*
134 * pmode: one of the CFG_EPM constants.
135 * stash_prio: 0 or CFG_EP
136 * stash_thresh: 0-7
137 */
138 static int
qman_eqcr_init(struct qman_portal_softc * sc,int pmode,u_int stash_thresh,u_int stash_prio)139 qman_eqcr_init(struct qman_portal_softc *sc, int pmode, u_int stash_thresh,
140 u_int stash_prio)
141 {
142 struct resource *regs = sc->sc_base.sc_mres[1];
143 uint32_t reg;
144
145 sc->sc_eqcr.ring =
146 (struct qman_eqcr_entry *)(sc->sc_base.sc_ce_va + QCSP_EQCR_N(0));
147 qman_eqcr_ring_init(&sc->sc_eqcr, &sc->sc_base);
148 reg = bus_read_4(regs, QCSP_CFG);
149 reg &= 0x00ffffff;
150 reg |= pmode;
151 reg |= ((stash_thresh << CFG_EST_S) & CFG_EST_M);
152 reg |= stash_prio;
153
154 bus_write_4(regs, QCSP_CFG, reg);
155 return (0);
156 }
157
158 static int
qman_dqrr_init(struct qman_portal_softc * sc)159 qman_dqrr_init(struct qman_portal_softc *sc)
160 {
161 struct resource *regs = sc->sc_base.sc_mres[1];
162 uint32_t reg;
163
164 /* Dequeue from the direct-connect channel and pool 0, up to 3 frames */
165 bus_write_4(regs, QCSP_DQRR_SDQCR,
166 SDQCR_FC | SDQCR_DP | SDQCR_DCT_PRI_PREC |
167 (DEF_SDQCR_TOKEN << SDQCR_TOKEN_S) |
168 DQRR_DQ_SRC_DCP | SDQCR_DQ_SRC_CHAN(0));
169 bus_write_4(regs, QCSP_DQRR_VDQCR, 0);
170 bus_write_4(regs, QCSP_DQRR_PDQCR, 0);
171
172 sc->sc_dqrr.ring =
173 (struct qman_dqrr_entry *)(sc->sc_base.sc_ce_va + QCSP_DQRR_N(0));
174 qman_dqrr_ring_init(&sc->sc_dqrr, &sc->sc_base);
175
176 /* Set DQRR max fill to 15 */
177 reg = bus_read_4(regs, QCSP_CFG);
178 reg |= (0xf << CFG_DQRR_MF_S);
179 bus_write_4(regs, QCSP_CFG, reg);
180
181 for (int i = 0; i < QMAN_DQRR_COUNT; i++)
182 __asm __volatile ("dcbi 0,%0" :: "r"(&sc->sc_dqrr.ring[i]) : "memory");
183
184 return (0);
185 }
186
187 int
qman_portal_attach(device_t dev,int cpu)188 qman_portal_attach(device_t dev, int cpu)
189 {
190 struct qman_portal_softc *sc = device_get_softc(dev);
191 union qman_mc_command *cr;
192 pcell_t cell;
193 phandle_t node;
194
195 sc->sc_base.sc_cpu = cpu;
196 dpaa_portal_alloc_res(dev, cpu);
197
198 qman_eqcr_init(sc, CFG_EPM_VB1, 0, 0);
199 qman_dqrr_init(sc);
200 bus_setup_intr(dev, sc->sc_base.sc_ires, INTR_TYPE_NET | INTR_MPSAFE,
201 NULL, qman_portal_isr, sc, &sc->sc_base.sc_intr_cookie);
202 bus_bind_intr(dev, sc->sc_base.sc_ires, cpu);
203
204 node = ofw_bus_get_node(dev);
205 if (OF_getencprop(node, "cell-index", &cell, sizeof(cell)) <= 0) {
206 device_printf(dev, "missing 'cell-index' property\n");
207 return (ENXIO);
208 }
209 sc->sc_affine_channel = cell;
210 DPCPU_ID_SET(cpu, qman_affine_portal, dev);
211 bus_write_4(sc->sc_base.sc_mres[1], QCSP_IER,
212 QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI | QM_PIRQ_CSCI |
213 QM_PIRQ_DQRI);
214 bus_write_4(sc->sc_base.sc_mres[1], QCSP_ISDR, 0);
215
216 /* Initialize the MC polarity bit, it may not be 0. */
217 cr = (union qman_mc_command *)(sc->sc_base.sc_ce_va + QCSP_CR);
218 sc->sc_mc.polarity =
219 (cr->common.verb & QMAN_MC_VERB_VBIT) ^ QMAN_MC_VERB_VBIT;
220 /* TODO: LIODN. Fake it for now */
221
222 qman_set_sdest(sc->sc_affine_channel, cpu);
223
224 return (0);
225 }
226
227
228 int
qman_portal_detach(device_t dev)229 qman_portal_detach(device_t dev)
230 {
231 struct qman_portal_softc *sc;
232 int i;
233
234 sc = device_get_softc(dev);
235
236 /* TODO: Unmap TLB regions */
237 thread_lock(curthread);
238 sched_bind(curthread, sc->sc_base.sc_cpu);
239 thread_unlock(curthread);
240
241 if (sc->sc_base.sc_ires != NULL)
242 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_base.sc_ires);
243
244 for (i = 0; i < nitems(sc->sc_base.sc_mres); i++) {
245 if (sc->sc_base.sc_mres[i] != NULL)
246 bus_release_resource(dev, SYS_RES_MEMORY,
247 i, sc->sc_base.sc_mres[i]);
248 }
249 thread_lock(curthread);
250 sched_unbind(curthread);
251 thread_unlock(curthread);
252
253
254 return (0);
255 }
256
257 static void
qman_portal_isr(void * arg)258 qman_portal_isr(void *arg)
259 {
260 struct qman_portal_softc *sc = arg;
261
262 qman_portal_loop_rings(sc);
263 }
264
265 int
qman_portal_fq_enqueue(device_t dev,struct qman_fq * fq,struct dpaa_fd * frame)266 qman_portal_fq_enqueue(device_t dev, struct qman_fq *fq, struct dpaa_fd *frame)
267 {
268 struct qman_portal_softc *sc = device_get_softc(dev);
269 struct qman_eqcr_entry *eqcr;
270
271 /* Get available... */
272 eqcr = qman_eqcr_start(&sc->sc_eqcr, &sc->sc_base);
273 if (eqcr == NULL)
274 return (EBUSY);
275 eqcr->fd = *frame;
276 eqcr->fqid = fq->fqid;
277 qman_eqcr_commit(&sc->sc_eqcr, QM_EQCR_VERB_CMD_ENQUEUE);
278
279 return (0);
280 }
281
282 static int
qman_portal_loop_dqrr(struct qman_portal_softc * sc)283 qman_portal_loop_dqrr(struct qman_portal_softc *sc)
284 {
285 struct qman_dqrr_entry *dqrr;
286 struct qman_dqrr_entry *base;
287 struct qman_fq *fq;
288 int ci = bus_read_4(sc->sc_base.sc_mres[1], QCSP_DQRR_CI_CINH) &
289 DQRR_CI_CI_M;
290 int pi = bus_read_4(sc->sc_base.sc_mres[1], QCSP_DQRR_PI_CINH) &
291 DQRR_PI_PI_M;
292
293 base = sc->sc_dqrr.ring;
294 do {
295 dqrr = &base[ci];
296 dpaa_flush_line(dqrr);
297 dpaa_touch_line(dqrr);
298 if ((dqrr->stat & QMAN_DQRR_STAT_HAS_FRAME)) {
299 fq = qman_fq_from_index(dqrr->fqid);
300 if (fq != NULL && fq->cb.dqrr != NULL) {
301 fq->cb.dqrr(sc->sc_base.sc_dev, fq,
302 &dqrr->fd, fq->cb.ctx);
303 }
304 } else
305 break;
306 ci = (ci + 1) & DQRR_CI_CI_M;
307 bus_write_4(sc->sc_base.sc_mres[1], QCSP_DQRR_CI_CINH, ci);
308 } while (ci != pi);
309
310 return (0);
311 }
312
313 static void
qman_portal_loop_rings(struct qman_portal_softc * sc)314 qman_portal_loop_rings(struct qman_portal_softc *sc)
315 {
316 uint32_t isr;
317
318 isr = bus_read_4(sc->sc_base.sc_mres[1], QCSP_ISR);
319
320 /* Handle DQRR first. */
321 if ((isr & QM_PIRQ_DQRI)) {
322 qman_portal_loop_dqrr(sc);
323 }
324 if ((isr & QM_PIRQ_CSCI)) {
325 }
326 if ((isr & QM_PIRQ_EQRI)) {
327 qman_eqcr_update(&sc->sc_eqcr, &sc->sc_base);
328 }
329 bus_write_4(sc->sc_base.sc_mres[1], QCSP_ISR, isr);
330 }
331
332 /* MC commands */
333
334 /* Assumes pinned */
335 union qman_mc_result *
qman_portal_mc_send_raw(device_t dev,union qman_mc_command * c)336 qman_portal_mc_send_raw(device_t dev, union qman_mc_command *c)
337 {
338 struct qman_portal_softc *sc;
339 int res_idx;
340 union qman_mc_result *rr;
341 union qman_mc_command *cr;
342 int timeout = 10000;
343 uint8_t verb;
344
345 sc = device_get_softc(dev);
346
347 verb = c->common.verb;
348 c->common.verb = 0;
349 cr = (union qman_mc_command *)(sc->sc_base.sc_ce_va + QCSP_CR);
350 dpaa_zero_line(cr);
351 *cr = *c;
352 dpaa_lw_barrier();
353 cr->common.verb = verb | sc->sc_mc.polarity;
354 res_idx = (sc->sc_mc.polarity ? 1 : 0);
355 sc->sc_mc.polarity ^= QMAN_MC_VERB_VBIT;
356 dpaa_flush_line(cr);
357 dpaa_touch_line(cr);
358
359 rr = (union qman_mc_result *)(sc->sc_base.sc_ce_va + QCSP_RR(res_idx));
360 for (; timeout > 0; --timeout) {
361 dpaa_flush_line(rr);
362 if (rr->common.verb != 0)
363 break;
364 }
365 if (timeout == 0)
366 return (NULL);
367 return (rr);
368 }
369
370 void
qman_portal_static_dequeue_channel(device_t dev,int channel)371 qman_portal_static_dequeue_channel(device_t dev, int channel)
372 {
373 struct qman_portal_softc *sc = device_get_softc(dev);
374 uint32_t reg;
375
376 reg = bus_read_4(sc->sc_base.sc_mres[1], QCSP_DQRR_SDQCR);
377 reg |= (1 << (15 - (channel - qman_channel_base)));
378 bus_write_4(sc->sc_base.sc_mres[1], QCSP_DQRR_SDQCR, reg);
379 }
380
381 void
qman_portal_static_dequeue_rm_channel(device_t dev,int channel)382 qman_portal_static_dequeue_rm_channel(device_t dev, int channel)
383 {
384 struct qman_portal_softc *sc = device_get_softc(dev);
385 uint32_t reg;
386
387 reg = bus_read_4(sc->sc_base.sc_mres[1], QCSP_DQRR_SDQCR);
388 reg &= ~(1 << (15 - (channel - qman_channel_base)));
389 bus_write_4(sc->sc_base.sc_mres[1], QCSP_DQRR_SDQCR, reg);
390 }
391
DB_SHOW_COMMAND(fqid,qman_show_fqid)392 DB_SHOW_COMMAND(fqid, qman_show_fqid)
393 {
394 union qman_mc_command cmd;
395 union qman_mc_result *res;
396 union qman_mc_result save_res;
397 device_t portal;
398
399 if (!have_addr)
400 return;
401
402 bzero(&cmd, sizeof(cmd));
403 cmd.query_fq_np.fqid = addr;
404
405 /* Ensure we have got QMan port initialized */
406 portal = DPCPU_GET(qman_affine_portal);
407 res = qman_portal_mc_send_raw(portal, &cmd);
408
409 if (res != NULL)
410 save_res = *res;
411
412 /* Dump all NP fields */
413 if (res != NULL && save_res.query_fq_np.rslt == 0xf0) {
414 db_printf("FQID: %d\n", (int)addr);
415 db_printf(" State: %x\n", save_res.query_fq_np.state);
416 db_printf(" Link: %x\n", save_res.query_fq_np.fqd_link);
417 db_printf(" ODP_SEQ: %x\n", save_res.query_fq_np.odp_seq);
418 db_printf(" ORP_NESN: %x\n", save_res.query_fq_np.orp_nesn);
419 db_printf(" ORP_EA_HSEQ: %x\n",
420 save_res.query_fq_np.orp_ea_hseq);
421 db_printf(" ORP_EA_TSEQ: %x\n",
422 save_res.query_fq_np.orp_ea_tseq);
423 db_printf(" ORP_EA_HPTR: %x\n",
424 save_res.query_fq_np.orp_ea_hptr);
425 db_printf(" ORP_EA_TPTR: %x\n",
426 save_res.query_fq_np.orp_ea_tptr);
427 db_printf(" pfdr_hptr: %x\n", save_res.query_fq_np.pfdr_hptr);
428 db_printf(" pfdr_tptr: %x\n", save_res.query_fq_np.pfdr_tptr);
429 db_printf(" IS: %x\n", save_res.query_fq_np.is);
430 db_printf(" ICS_SURP: %x\n", save_res.query_fq_np.ics_surp);
431 db_printf(" byte_cnt: %x\n", save_res.query_fq_np.byte_cnt);
432 db_printf(" frm_cnt: %x\n", save_res.query_fq_np.frm_cnt);
433 db_printf(" ra1_sfdr: %x\n", save_res.query_fq_np.ra1_sfdr);
434 db_printf(" ra2_sfdr: %x\n", save_res.query_fq_np.ra2_sfdr);
435 db_printf(" od1_sfdr: %x\n", save_res.query_fq_np.od1_sfdr);
436 db_printf(" od2_sfdr: %x\n", save_res.query_fq_np.od2_sfdr);
437 db_printf(" od3_sfdr: %x\n", save_res.query_fq_np.od3_sfdr);
438 }
439 }
440