1 /*
2 * Copyright (c) 2026 Justin Hibbits
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6
7 #include "opt_platform.h"
8
9 #include <sys/param.h>
10 #include <sys/systm.h>
11 #include <sys/kernel.h>
12 #include <sys/bus.h>
13 #include <sys/cpuset.h>
14 #include <sys/interrupt.h>
15 #include <sys/lock.h>
16 #include <sys/module.h>
17 #include <sys/mutex.h>
18 #include <sys/proc.h>
19 #include <sys/pcpu.h>
20 #include <sys/sched.h>
21
22 #include <machine/bus.h>
23 #include <machine/tlb.h>
24
25 #include <dev/ofw/ofw_bus.h>
26 #include <dev/ofw/ofw_bus_subr.h>
27
28 #include <powerpc/mpc85xx/mpc85xx.h>
29
30 #include "bman.h"
31 #include "bman_var.h"
32 #include "portals.h"
33
34 #define BCSP_CFG 0x0100
35 #define CFG_RPM_M 0x00000003
36 #define CFG_RPM_PI 0x00000000
37 #define CFG_RPM_PE 0x00000001
38 #define CFG_RPM_VBM 0x00000002
39 #define BCSP_SCN0 0x0200
40 #define BCSP_SCN1 0x0204
41 #define BCSP_ISR 0x0e00
42 #define BCSP_IER 0x0e04
43 #define BCSP_ISDR 0x0e08
44 #define INTR_RCDI 0x00000004
45 #define INTR_RCRI 0x00000002
46 #define INTR_BSCN 0x00000001
47
48 #define BMAN_CE_CR 0x0000
49 #define BMAN_CE_RR0 0x0100
50 #define BMAN_CE_RR1 0x0140
51 #define BMAN_CE_RR(n) (BMAN_CE_RR0 + 0x40 * (n))
52 #define BMAN_CE_RCR 0x1000
53 #define BCSP_RCR_PI_CENA 0x3000
54 #define BCSP_RCR_CI_CENA 0x3100
55 #define BCSP_RCR_PI_CINH 0x000
56 #define BCSP_RCR_CI_CINH 0x004
57
58 #define BMAN_MC_VERB_VBIT 0x80
59 #define BMAN_MC_VERB_ACQUIRE 0x10
60 #define BMAN_MC_VERB_QUERY 0x40
61 #define BMAN_RCR_VERB_BPID0 0x20
62 #define BMAN_RCR_VERB_BPID_BUF 0x30
63
64 struct bman_mc_command {
65 uint8_t verb;
66 uint8_t cd;
67 uint8_t rsvd[62];
68 };
69
70 union bman_mc_result {
71 struct {
72 uint8_t verb;
73 uint8_t cd;
74 uint8_t rsvd[62];
75 };
76 struct {
77 uint64_t rsvd_q1[5];
78 uint64_t bp_as;
79 uint64_t rsvd_q2;
80 uint64_t bp_ds;
81 };
82 struct bman_buffer bufs[8];
83 };
84
85 struct bman_rcr_entry {
86 union {
87 struct {
88 uint8_t verb;
89 uint8_t bpid;
90 uint8_t rsvd[62];
91 };
92 struct bman_buffer bufs[8];
93 };
94 };
95
96 static void bman_portal_isr(void *arg);
97
98 static union bman_mc_result *bman_mc_send(struct bman_portal_softc *p,
99 uint8_t verb, uint8_t cd);
100
101 DPCPU_DEFINE(struct bman_portal_softc *, bman_affine_portal);
102
103 DPAA_RING(bman_rcr, 8, BCSP_RCR_PI_CENA, BCSP_RCR_CI_CENA,
104 BCSP_RCR_PI_CINH, BCSP_RCR_CI_CINH);
105
106 static uint32_t
bm_ci_read(struct bman_portal_softc * sc,bus_size_t off)107 bm_ci_read(struct bman_portal_softc *sc, bus_size_t off)
108 {
109 return (bus_read_4(sc->sc_base.sc_mres[1], off));
110 }
111
112 static void
bm_ci_write(struct bman_portal_softc * sc,bus_size_t off,uint32_t val)113 bm_ci_write(struct bman_portal_softc *sc, bus_size_t off, uint32_t val)
114 {
115 bus_write_4(sc->sc_base.sc_mres[1], off, val);
116 }
117
118 int
bman_portal_attach(device_t dev,int cpu)119 bman_portal_attach(device_t dev, int cpu)
120 {
121 struct bman_portal_softc *sc = device_get_softc(dev);
122
123 sc->sc_base.sc_cpu = cpu;
124 dpaa_portal_alloc_res(dev, cpu);
125
126 bm_ci_write(sc, BCSP_ISDR, 0);
127 bm_ci_write(sc, BCSP_IER, INTR_RCRI | INTR_BSCN);
128 bus_setup_intr(dev, sc->sc_base.sc_ires, INTR_TYPE_NET | INTR_MPSAFE,
129 NULL, bman_portal_isr, sc, &sc->sc_base.sc_intr_cookie);
130 bus_bind_intr(dev, sc->sc_base.sc_ires, cpu);
131
132 /* Select valid-bit mode for rings */
133 bus_write_4(sc->sc_base.sc_mres[1], BCSP_CFG, CFG_RPM_VBM);
134 /* Disable pool depletion notifications. */
135 bm_ci_write(sc, BCSP_SCN0, 0);
136 bm_ci_write(sc, BCSP_SCN1, 0);
137
138 DPCPU_ID_SET(cpu, bman_affine_portal, sc);
139
140 sc->sc_rcr.ring =
141 (struct bman_rcr_entry *)(sc->sc_base.sc_ce_va + BMAN_CE_RCR);
142 bman_rcr_ring_init(&sc->sc_rcr, &sc->sc_base);
143 /* Starting MC polarity is always 1 */
144 sc->mc.polarity = BMAN_MC_VERB_VBIT;
145
146 return (0);
147 }
148
149 int
bman_portal_detach(device_t dev)150 bman_portal_detach(device_t dev)
151 {
152 struct bman_portal_softc *sc;
153 int i;
154
155 sc = device_get_softc(dev);
156
157
158 /* TODO: Unmap TLB regions */
159 thread_lock(curthread);
160 sched_bind(curthread, sc->sc_base.sc_cpu);
161 thread_unlock(curthread);
162
163 if (sc->sc_base.sc_ires != NULL)
164 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_base.sc_ires);
165
166 for (i = 0; i < nitems(sc->sc_base.sc_mres); i++) {
167 if (sc->sc_base.sc_mres[i] != NULL)
168 bus_release_resource(dev, SYS_RES_MEMORY,
169 i, sc->sc_base.sc_mres[i]);
170 }
171 thread_lock(curthread);
172 sched_unbind(curthread);
173 thread_unlock(curthread);
174
175
176 return (0);
177 }
178
179 static uint64_t
bman_query(struct bman_portal_softc * sc,bool depletion)180 bman_query(struct bman_portal_softc *sc, bool depletion)
181 {
182 union bman_mc_result *mc_res;
183 uint64_t res;
184
185 critical_enter();
186 mc_res = bman_mc_send(sc, BMAN_MC_VERB_QUERY, 0);
187 if (mc_res == NULL)
188 goto err;
189
190 if (depletion)
191 res = mc_res->bp_ds;
192 else
193 res = mc_res->bp_as;
194 critical_exit();
195
196 return (res);
197
198 err:
199 critical_exit();
200 device_printf(sc->sc_base.sc_dev, "Timeout querying depltetion\n");
201 return (0);
202 }
203
204 static void
bman_portal_isr(void * arg)205 bman_portal_isr(void *arg)
206 {
207 struct bman_portal_softc *sc = arg;
208 uint32_t intrs;
209
210 intrs = bm_ci_read(sc, BCSP_ISR);
211
212 /* Release Command Ring interrupt. */
213 if (intrs & INTR_RCRI) {
214 bman_rcr_update(&sc->sc_rcr, &sc->sc_base);
215 }
216 /* Buffer Pool State Change Notification. */
217 if (intrs & INTR_BSCN) {
218 struct bman_pool *pool;
219 uint64_t res = bman_query(sc, true);
220 if (__predict_true(res != 0)) {
221 int idx = flsll(res);
222 pool = sc->sc_pools[64 - idx];
223 KASSERT(pool != NULL,
224 ("state change on unassociated bpid %d\n", idx));
225 pool->dep_cb(pool->arg, true);
226 }
227 }
228
229 bm_ci_write(sc, BCSP_ISR, intrs);
230 }
231
232 /* RCR */
233
234 int
bman_release(struct bman_pool * pool,const struct bman_buffer * bufs,uint8_t count)235 bman_release(struct bman_pool *pool, const struct bman_buffer *bufs,
236 uint8_t count)
237 {
238 struct bman_portal_softc *portal;
239 struct bman_rcr_entry *rcr;
240
241 if (count > 8)
242 return (EINVAL);
243
244 critical_enter();
245 portal = DPCPU_GET(bman_affine_portal);
246 rcr = bman_rcr_start(&portal->sc_rcr, &portal->sc_base);
247 bzero(rcr, sizeof(*rcr));
248
249 /* This should be safe, because bpid must be less than 256. */
250 for (int i = 0; i < count; i++)
251 rcr->bufs[i] = bufs[i];
252 rcr->bufs[0].bpid = pool->bpid;
253 bman_rcr_commit(&portal->sc_rcr, BMAN_RCR_VERB_BPID0 | count);
254 critical_exit();
255
256 return (0);
257 }
258
259 /* MC commands */
260 /* Assumes pinned */
261 static union bman_mc_result *
bman_mc_send(struct bman_portal_softc * p,uint8_t verb,uint8_t cd)262 bman_mc_send(struct bman_portal_softc *p, uint8_t verb, uint8_t cd)
263 {
264 int res_idx;
265 struct bman_mc_command *command;
266 union bman_mc_result *rr;
267 uintptr_t ce_va = p->sc_base.sc_ce_va;
268
269 command = (struct bman_mc_command *)(ce_va + BMAN_CE_CR);
270 dpaa_zero_line(command);
271 command->cd = cd;
272 dpaa_lw_barrier();
273 command->verb = verb | p->mc.polarity;
274 res_idx = (p->mc.polarity ? 1 : 0);
275 p->mc.polarity ^= BMAN_MC_VERB_VBIT;
276 dpaa_flush_line(command);
277
278 rr = (union bman_mc_result *)(ce_va + BMAN_CE_RR(res_idx));
279 for (;;) {
280 if (rr->verb != 0)
281 break;
282 dpaa_flush_line(rr);
283 }
284 return (rr);
285 }
286
287 int
bman_acquire(struct bman_pool * pool,struct bman_buffer * bufs,uint8_t count)288 bman_acquire(struct bman_pool *pool, struct bman_buffer *bufs, uint8_t count)
289 {
290 union bman_mc_result *rr;
291
292 if (count > 8 || count == 0)
293 return (EINVAL);
294 critical_enter();
295 rr = bman_mc_send(DPCPU_GET(bman_affine_portal),
296 BMAN_MC_VERB_ACQUIRE | count,
297 pool->bpid);
298 critical_exit();
299
300 if (rr == NULL)
301 return (ETIMEDOUT);
302 if ((rr->verb & ~BMAN_MC_VERB_VBIT) == 0)
303 return (ENOMEM);
304
305 memcpy(bufs, rr, count * sizeof(*bufs));
306
307 return (0);
308 }
309
310 /*
311 * Enable pool state change notifications on this portal. This requires the
312 * pool to already be configured with the callback to handle state changes.
313 */
314 void
bman_portal_enable_scn(struct bman_portal_softc * sc,struct bman_pool * pool)315 bman_portal_enable_scn(struct bman_portal_softc *sc, struct bman_pool *pool)
316 {
317 uint32_t reg, reg_ptr;
318
319 if (pool->bpid >= 32)
320 reg_ptr = BCSP_SCN1;
321 else
322 reg_ptr = BCSP_SCN0;
323 reg = bm_ci_read(sc, reg_ptr);
324 reg |= (1 << (31 - pool->bpid));
325 bm_ci_write(sc, reg_ptr, reg);
326 sc->sc_pools[pool->bpid] = pool;
327 }
328