1 /*-
2 * Copyright (c) 2026 Justin Hibbits
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 *
6 * FMan KeyGen (KG) support. The KG block is the FMan sub-unit that
7 * classifies incoming frames into destination FQIDs via user-defined
8 * "schemes". Each scheme extracts a key from the frame's parse
9 * result, hashes it, and enqueues to FQID = base | (hash & mask).
10 */
11
12 #include <sys/param.h>
13 #include <sys/systm.h>
14 #include <sys/bus.h>
15 #include <sys/rman.h>
16
17 #include <machine/bus.h>
18
19 #include "fman.h"
20 #include "fman_keygen.h"
21
22 #define FMAN_KG_OFFSET 0xc1000
23
24 /*
25 * Register offsets, relative to FMAN_KG_OFFSET (0xc1000) inside the
26 * FMan register block.
27 */
28 #define FMKG_GCR 0x000 /* general configuration */
29 #define FMKG_GCR_EN 0x80000000
30 #define FMKG_EER 0x00C /* error event */
31 #define FMKG_EER_DOUBLE_ECC 0x80000000
32 #define FMKG_EER_KEYSIZE_OVF 0x40000000
33 #define FMKG_EEER 0x010 /* error event enable */
34 #define FMKG_SEER 0x01C /* scheme error event */
35 #define FMKG_SEEER 0x020 /* scheme error event enable */
36 #define FMKG_GSR 0x024 /* global status */
37 #define FMKG_GSR_BSY 0x80000000
38 #define FMKG_TPC 0x028 /* total packet counter */
39 #define FMKG_SERC 0x02C /* soft error capture */
40 #define FMKG_FDOR 0x034 /* frame data offset */
41 #define FMKG_GDV0R 0x038 /* global default value 0 */
42 #define FMKG_GDV1R 0x03C /* global default value 1 */
43 #define FMKG_FEER 0x044 /* force error event */
44 #define FMKG_AR 0x1FC /* action register */
45 #define FMKG_AR_GO 0x80000000
46 #define FMKG_AR_READ 0x40000000 /* clear = write */
47 #define FMKG_AR_ERR 0x20000000
48 #define FMKG_AR_SEL_CLS_PLAN 0x01000000
49 #define FMKG_AR_SEL_PORT 0x02000000 /* clear = scheme entry */
50 #define FMKG_AR_PORT_WSEL_SP 0x00008000
51 #define FMKG_AR_PORT_WSEL_CPP 0x00004000
52 #define FMKG_AR_SCM_WSEL_UPDCNT 0x00008000
53 #define FMKG_AR_NUM_SHIFT 16 /* scheme id here */
54
55 /*
56 * Indirect access sub-region (union of "scheme entry" and "port
57 * entry" views, discriminated by the last AR selector written).
58 */
59 #define FMKG_IND 0x100
60
61 /* Scheme-view offsets, relative to FMKG_IND. */
62 #define FMKG_SE_MODE 0x00 /* mode + NIA */
63 #define FMKG_SE_MODE_EN 0x80000000
64 #define FMKG_SE_EKFC 0x04 /* extract known fields command */
65 #define FMKG_EKFC_IPSRC1 0x00100000
66 #define FMKG_EKFC_IPDST1 0x00080000
67 #define FMKG_EKFC_L4PSRC 0x00000004
68 #define FMKG_EKFC_L4PDST 0x00000002
69 #define FMKG_EKFC_IPSEC_SPI 0x00000200
70 #define FMKG_SE_EKDV 0x08 /* extract known default value */
71 #define FMKG_EKDV_IP_ADDR_SHIFT 18
72 #define FMKG_EKDV_L4_PORT_SHIFT 8
73 #define FMKG_EKDV_USE_DV0 2
74 #define FMKG_EKDV_USE_DV1 3
75 #define FMKG_SE_BMCH 0x0C /* bit mask high */
76 #define FMKG_SE_BMCL 0x10 /* bit mask low */
77 #define FMKG_SE_FQB 0x14 /* frame queue base */
78 #define FMKG_SE_HC 0x18 /* hash command */
79 #define FMKG_SE_HC_SYM 0x40000000 /* symmetric hash */
80 #define FMKG_SE_HC_SHIFT_SHIFT 24
81 #define FMKG_SE_HC_MASK_SHIFT 0
82 #define FMKG_SE_PPC 0x1C /* policer profile command */
83 #define FMKG_SE_GEC(i) (0x20 + (i) * 4) /* generic extract command [0..7] */
84 #define FMKG_SE_SPC 0x40 /* statistic packet counter */
85 #define FMKG_SE_DV0 0x44 /* default value 0 */
86 #define FMKG_SE_DV1 0x48 /* default value 1 */
87 #define FMKG_SE_CCBS 0x4C /* coarse classification */
88 #define FMKG_SE_MV 0x50 /* match vector */
89 #define FMKG_SE_OM 0x54 /* operation mode */
90 #define FMKG_SE_VSP 0x58 /* virtual storage profile */
91 #define FMKG_SE_VSP_NO_KSP_EN 0x80000000
92
93 /* Port-view offsets, relative to FMKG_IND. */
94 #define FMKG_PE_SP 0x00 /* port scheme partition */
95 #define FMKG_PE_CPP 0x04 /* port classification-plan partition */
96
97 /*
98 * Default NIA for post-KeyGen
99 */
100 #define FMKG_ENQ_KG_DFLT_NIA \
101 (NIA_ENG_BMI | NIA_BMI_AC_ENQ_FRAME)
102
103 /*
104 * Arbitrary fillers used for the "missing field" fallback slots.
105 * The exact values don't matter for correctness -- the hash still
106 * distributes uniformly across FQs -- but a non-zero pattern makes
107 * two frames missing the same field hash to a consistent bucket.
108 */
109 #define FMKG_DFLT_IPv4_ADDR 0x0a0a0a0a
110 #define FMKG_DFLT_L4_PORT 0x0b0b0b0b
111
112 /* Sizing. */
113 #define FMKG_MAX_SCHEMES 32
114 #define FMKG_MAX_HW_PORTS 64
115
116 /*
117 * AR-write acknowledge wait. Linux busy-loops with no bound; we
118 * cap at ~1 ms of DELAY(1) to survive a wedged block without
119 * hanging the boot.
120 */
121 #define FMKG_AR_WAIT_US 1000
122
123 static inline uint32_t
kg_read(struct fman_softc * sc,bus_size_t off)124 kg_read(struct fman_softc *sc, bus_size_t off)
125 {
126 return (bus_read_4(sc->mem_res, FMAN_KG_OFFSET + off));
127 }
128
129 static inline void
kg_write(struct fman_softc * sc,bus_size_t off,uint32_t val)130 kg_write(struct fman_softc *sc, bus_size_t off, uint32_t val)
131 {
132 bus_write_4(sc->mem_res, FMAN_KG_OFFSET + off, val);
133 }
134
135 static int
kg_ar_write(struct fman_softc * sc,uint32_t ar)136 kg_ar_write(struct fman_softc *sc, uint32_t ar)
137 {
138 int i;
139
140 kg_write(sc, FMKG_AR, ar);
141 for (i = 0; i < FMKG_AR_WAIT_US; i++) {
142 ar = kg_read(sc, FMKG_AR);
143 if ((ar & FMKG_AR_GO) == 0)
144 break;
145 DELAY(1);
146 }
147 if ((ar & FMKG_AR_GO) != 0) {
148 device_printf(sc->sc_base.dev,
149 "fman_kg: AR stuck busy (0x%08x)\n", ar);
150 return (ETIMEDOUT);
151 }
152 if ((ar & FMKG_AR_ERR) != 0) {
153 device_printf(sc->sc_base.dev,
154 "fman_kg: AR reported error (0x%08x)\n", ar);
155 return (EIO);
156 }
157 return (0);
158 }
159
160 /*
161 * Populate the indirect scheme registers for an RSS-style hash
162 * scheme and commit it via an AR-write.
163 */
164 static int
kg_program_scheme(struct fman_softc * sc,uint8_t scheme_id,uint32_t base_fqid,uint32_t nfqs)165 kg_program_scheme(struct fman_softc *sc, uint8_t scheme_id,
166 uint32_t base_fqid, uint32_t nfqs)
167 {
168 uint32_t ekdv;
169 int error;
170
171 kg_write(sc, FMKG_IND + FMKG_SE_MODE,
172 FMKG_SE_MODE_EN | FMKG_ENQ_KG_DFLT_NIA);
173
174 kg_write(sc, FMKG_IND + FMKG_SE_EKFC,
175 FMKG_EKFC_IPSRC1 | FMKG_EKFC_IPDST1 |
176 FMKG_EKFC_L4PSRC | FMKG_EKFC_L4PDST |
177 FMKG_EKFC_IPSEC_SPI);
178
179 ekdv = (FMKG_EKDV_USE_DV0 << FMKG_EKDV_IP_ADDR_SHIFT) |
180 (FMKG_EKDV_USE_DV1 << FMKG_EKDV_L4_PORT_SHIFT);
181 kg_write(sc, FMKG_IND + FMKG_SE_EKDV, ekdv);
182 kg_write(sc, FMKG_IND + FMKG_SE_DV0, FMKG_DFLT_IPv4_ADDR);
183 kg_write(sc, FMKG_IND + FMKG_SE_DV1, FMKG_DFLT_L4_PORT);
184
185 kg_write(sc, FMKG_IND + FMKG_SE_FQB, base_fqid);
186 /*
187 * FQID mask width = (nfqs - 1), no hash right-shift. Symmetric
188 * hash is deliberately off: per NXP's own driver notes,
189 * spreading breaks with SYM set even though the extraction key
190 * is nominally symmetric.
191 */
192 kg_write(sc, FMKG_IND + FMKG_SE_HC,
193 (nfqs - 1) << FMKG_SE_HC_MASK_SHIFT);
194
195 /*
196 * TODO: Revisit these registers later, if necessary.
197 */
198 kg_write(sc, FMKG_IND + FMKG_SE_BMCH, 0);
199 kg_write(sc, FMKG_IND + FMKG_SE_BMCL, 0);
200 kg_write(sc, FMKG_IND + FMKG_SE_PPC, 0);
201 kg_write(sc, FMKG_IND + FMKG_SE_SPC, 0);
202 for (int i = 0; i < 8; i++)
203 kg_write(sc, FMKG_IND + FMKG_SE_GEC(i), 0);
204 kg_write(sc, FMKG_IND + FMKG_SE_CCBS, 0);
205 kg_write(sc, FMKG_IND + FMKG_SE_MV, 0); /* indirect scheme */
206 kg_write(sc, FMKG_IND + FMKG_SE_OM, 0);
207 /*
208 * Don't let the scheme override the port's Virtual Storage
209 * Profile.
210 */
211 kg_write(sc, FMKG_IND + FMKG_SE_VSP, FMKG_SE_VSP_NO_KSP_EN);
212 error = kg_ar_write(sc, FMKG_AR_GO |
213 (scheme_id << FMKG_AR_NUM_SHIFT) | FMKG_AR_SCM_WSEL_UPDCNT);
214 if (error != 0)
215 device_printf(sc->sc_base.dev,
216 "fman_kg: scheme %u program failed\n", scheme_id);
217 return (error);
218 }
219
220 static int
kg_disable_scheme(struct fman_softc * sc,uint8_t scheme_id)221 kg_disable_scheme(struct fman_softc *sc, uint8_t scheme_id)
222 {
223 kg_write(sc, FMKG_IND + FMKG_SE_MODE, 0);
224 return (kg_ar_write(sc, FMKG_AR_GO | (scheme_id << FMKG_AR_NUM_SHIFT)));
225 }
226
227 /*
228 * Read-modify-write the port's scheme-partition bitmap to add or
229 * remove @scheme_id.
230 */
231 static int
kg_bind_scheme(struct fman_softc * sc,uint8_t hw_port_id,uint8_t scheme_id,bool bind)232 kg_bind_scheme(struct fman_softc *sc, uint8_t hw_port_id, uint8_t scheme_id,
233 bool bind)
234 {
235 uint32_t sp;
236 int error;
237
238 error = kg_ar_write(sc,
239 FMKG_AR_GO | FMKG_AR_READ | FMKG_AR_SEL_PORT |
240 hw_port_id | FMKG_AR_PORT_WSEL_SP);
241 if (error != 0)
242 return (error);
243
244 sp = kg_read(sc, FMKG_IND + FMKG_PE_SP);
245 if (bind)
246 sp |= (1U << (31 - scheme_id));
247 else
248 sp &= ~(1U << (31 - scheme_id));
249 kg_write(sc, FMKG_IND + FMKG_PE_SP, sp);
250
251 return (kg_ar_write(sc,
252 FMKG_AR_GO | FMKG_AR_SEL_PORT | hw_port_id | FMKG_AR_PORT_WSEL_SP));
253 }
254
255 int
fman_kg_init(struct fman_softc * sc)256 fman_kg_init(struct fman_softc *sc)
257 {
258 int error, i;
259
260 sc->sc_kg_schemes_used = 0;
261 for (i = 0; i < FMKG_MAX_HW_PORTS; i++)
262 sc->sc_kg_port_scheme[i] = -1;
263
264 kg_write(sc, FMKG_GCR, FMKG_ENQ_KG_DFLT_NIA);
265 kg_write(sc, FMKG_EER,
266 FMKG_EER_DOUBLE_ECC | FMKG_EER_KEYSIZE_OVF);
267
268 kg_write(sc, FMKG_FDOR, 0);
269 kg_write(sc, FMKG_GDV0R, 0);
270 kg_write(sc, FMKG_GDV1R, 0);
271
272 for (i = 0; i < FMKG_MAX_HW_PORTS; i++) {
273 kg_write(sc, FMKG_IND + FMKG_PE_SP, 0);
274 error = kg_ar_write(sc, FMKG_AR_GO | FMKG_AR_SEL_PORT |
275 i | FMKG_AR_PORT_WSEL_SP);
276 if (error != 0)
277 return (error);
278
279 kg_write(sc, FMKG_IND + FMKG_PE_CPP, 0);
280 error = kg_ar_write(sc, FMKG_AR_GO | FMKG_AR_SEL_PORT |
281 i | FMKG_AR_PORT_WSEL_CPP);
282 if (error != 0)
283 return (error);
284 }
285
286 /* Enable per-scheme error events. */
287 kg_write(sc, FMKG_SEER, 0xffffffff);
288 kg_write(sc, FMKG_SEEER, 0xffffffff);
289
290 /* TODO: Error handling interrupts -- register with FMan */
291
292 /* Enable the block. */
293 kg_write(sc, FMKG_GCR, kg_read(sc, FMKG_GCR) | FMKG_GCR_EN);
294
295 return (0);
296 }
297
298 void
fman_kg_fini(struct fman_softc * sc)299 fman_kg_fini(struct fman_softc *sc)
300 {
301 /* No need for a full teardown, just disable the module. */
302 kg_write(sc, FMKG_GCR, 0);
303 while ((kg_read(sc, FMKG_GSR) & FMKG_GSR_BSY) != 0)
304 DELAY(1);
305 }
306
307 int
fman_kg_alloc_hash_scheme(struct fman_softc * sc,int hw_port_id,uint32_t base_fqid,uint32_t nfqs)308 fman_kg_alloc_hash_scheme(struct fman_softc *sc, int hw_port_id,
309 uint32_t base_fqid, uint32_t nfqs)
310 {
311 uint8_t scheme_id;
312 int error, i;
313
314 if (hw_port_id < 0 || hw_port_id >= FMKG_MAX_HW_PORTS)
315 return (EINVAL);
316 if (base_fqid == 0 || (base_fqid & ~0x00ffffff) != 0)
317 return (EINVAL);
318 if (nfqs == 0 || (nfqs & (nfqs - 1)) != 0)
319 return (EINVAL); /* not a power of two */
320 if ((base_fqid & (nfqs - 1)) != 0)
321 return (EINVAL); /* base not aligned to nfqs */
322 if (sc->sc_kg_port_scheme[hw_port_id] != -1)
323 return (EBUSY);
324
325 for (i = 0; i < FMKG_MAX_SCHEMES; i++) {
326 if ((sc->sc_kg_schemes_used & (1U << i)) == 0) {
327 scheme_id = i;
328 break;
329 }
330 }
331 if (i == FMKG_MAX_SCHEMES)
332 return (ENOSPC);
333
334 error = kg_program_scheme(sc, scheme_id, base_fqid, nfqs);
335 if (error != 0)
336 return (error);
337
338 error = kg_bind_scheme(sc, hw_port_id, scheme_id, true);
339 if (error != 0) {
340 (void)kg_disable_scheme(sc, scheme_id);
341 return (error);
342 }
343
344 sc->sc_kg_schemes_used |= (1U << scheme_id);
345 sc->sc_kg_port_scheme[hw_port_id] = scheme_id;
346 return (0);
347 }
348
349 int
fman_kg_free_hash_scheme(struct fman_softc * sc,int hw_port_id)350 fman_kg_free_hash_scheme(struct fman_softc *sc, int hw_port_id)
351 {
352 int8_t scheme_id;
353 int error;
354
355 if (hw_port_id < 0 || hw_port_id >= FMKG_MAX_HW_PORTS)
356 return (EINVAL);
357 scheme_id = sc->sc_kg_port_scheme[hw_port_id];
358 if (scheme_id == -1)
359 return (ENOENT);
360
361 error = kg_bind_scheme(sc, hw_port_id, scheme_id, false);
362 if (error != 0)
363 return (error);
364 error = kg_disable_scheme(sc, scheme_id);
365 if (error != 0)
366 return (error);
367
368 sc->sc_kg_schemes_used &= ~(1U << scheme_id);
369 sc->sc_kg_port_scheme[hw_port_id] = -1;
370 return (0);
371 }
372