1 /*
2 * Copyright 2026 Justin Hibbits <jhibbits@FreeBSD.org>
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6 #include "sec_var.h"
7
8 /*
9 * A job that doesn't complete in 5 seconds (should take microseconds or less)
10 * is considered a failure.
11 */
12 #define SEC_JOB_TIMEOUT (5 * hz)
13
14 /*
15 * Job Ring register offsets, relative to the JR's base within SEC's
16 * CCSR window.
17 */
18 #define JR_IRBAR_MS 0x00 /* Input ring base, upper (64-bit reg) */
19 #define JR_IRBAR_LS 0x04 /* Input ring base, lower */
20 #define JR_IRSR 0x0c /* Input ring size (ring entries) */
21 #define JR_IRSAR 0x14 /* Input ring slots available (add-to) */
22 #define JR_IRJAR 0x1c /* Input ring jobs added (bump on enqueue) */
23 #define JR_ORBAR_MS 0x20 /* Output ring base, upper */
24 #define JR_ORBAR_LS 0x24 /* Output ring base, lower */
25 #define JR_ORSR 0x2c /* Output ring size */
26 #define JR_ORJRR 0x34 /* Output ring jobs removed */
27 #define JR_ORSFR 0x3c /* Output ring slots full */
28 #define JR_JRSTAR 0x44 /* Output status (per-job termination) */
29 #define JR_JRINTR 0x4c /* Interrupt status (W1C) */
30 #define JRINTR_JRI 0x00000001 /* JR interrupt asserted */
31 #define JRINTR_JRE 0x00000002 /* JR error */
32 /*
33 * HALT tracks a flush requested through JRCR: 01b while SEC
34 * is still draining, 10b once every job has reached the output ring.
35 * Writing the field's high bit clears it and lets the ring run again.
36 */
37 #define JRINTR_HALT_M 0x0000000c
38 #define JRINTR_HALT_ONGOING 0x00000004
39 #define JRINTR_HALT_DONE 0x00000008
40 #define JR_JRCFGR_MS 0x50 /* Configuration, upper */
41 #define JR_JRCFGR_LS 0x54 /* Configuration, lower */
42 #define JRCFGR_LS_IMSK 0x00000001 /* Mask interrupts (1=masked) */
43 #define JRCFGR_LS_ICEN 0x00000002 /* Interrupt coalescing enable */
44 #define JR_JRCR 0x6c /* Command: flush/reset */
45 #define JRCR_RESET 0x00000001 /* Flush, or reset if halted */
46 #define JR_IRRIR 0x5c /* Input ring read index (RO) */
47 #define JR_ORWIR 0x64 /* Output ring write index (RO) */
48
49 #define JR_RING_SIZE 16 /* power of 2, small for scaffolding */
50 #define JR_RING_MASK (JR_RING_SIZE - 1)
51
52 #define JR_RD4(sec, jr, off) bus_read_4(sec->sc_rres, jr->jr_off + off)
53 #define JR_WR4(sec, jr, off, v) \
54 bus_write_4(sec->sc_rres, jr->jr_off + off, v)
55
56 static void sec_jr_intr(void *arg);
57
58 #define FOREACH_JOB_RING(node) \
59 for (phandle_t child = OF_child(node); child != 0; \
60 child = OF_peer(child)) \
61 if ((ofw_bus_node_is_compatible(child, \
62 "fsl,sec-v5.0-job-ring") || \
63 ofw_bus_node_is_compatible(child, \
64 "fsl,sec-v4.0-job-ring")) && \
65 ofw_bus_node_status_okay(child) && \
66 OF_getproplen(child, "reg") == 2 * sizeof(pcell_t))
67 /*
68 * Job Ring helpers.
69 */
70
71 static int
sec_jr_count(struct sec_softc * sc)72 sec_jr_count(struct sec_softc *sc)
73 {
74 phandle_t node = ofw_bus_get_node(sc->sc_dev);
75 int n = 0;
76
77 FOREACH_JOB_RING(node)
78 n++;
79
80 return (n);
81 }
82
83 static void
sec_jr_dma_cb(void * arg,bus_dma_segment_t * segs,int nsegs,int error)84 sec_jr_dma_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
85 {
86 vm_paddr_t *pa = arg;
87 *pa = error == 0 && nsegs == 1 ? segs[0].ds_addr : 0;
88 }
89
90 static int
sec_jr_irq_setup(struct sec_softc * sc,struct sec_jr * jr,u_int idx)91 sec_jr_irq_setup(struct sec_softc *sc, struct sec_jr *jr, u_int idx)
92 {
93 device_t dev = sc->sc_dev;
94 struct resource_list *rl;
95 phandle_t iparent;
96 pcell_t *cells;
97 int ncells, irqnum;
98
99 if (jr->jr_node == 0)
100 return (ENXIO);
101
102 if (ofw_bus_intr_by_rid(dev, jr->jr_node, 0, &iparent, &ncells,
103 &cells) != 0)
104 return (ENXIO);
105 irqnum = ofw_bus_map_intr(dev, iparent, ncells, cells);
106 OF_prop_free(cells);
107 if (irqnum <= 0)
108 return (ENXIO);
109
110 rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
111 jr->jr_irid = 1 + idx; /* rid 0 is the SEC top-level error IRQ */
112 resource_list_add(rl, SYS_RES_IRQ, jr->jr_irid, irqnum, irqnum, 1);
113
114 jr->jr_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ,
115 &jr->jr_irid, RF_ACTIVE);
116 if (jr->jr_ires == NULL)
117 return (ENXIO);
118
119 if (bus_setup_intr(dev, jr->jr_ires, INTR_TYPE_MISC | INTR_MPSAFE,
120 NULL, sec_jr_intr, jr, &jr->jr_icookie) != 0)
121 return (ENXIO);
122
123 /*
124 * Enable JR interrupts (IMSK=0, ICEN=0 = fire on every completion).
125 * Reset default is already IMSK=0, but be explicit.
126 */
127 JR_WR4(sc, jr, JR_JRCFGR_LS, 0);
128
129 return (0);
130 }
131
132 /*
133 * Watchdog for a wedged ring. Nothing else reclaims a job that never
134 * reaches the output ring, so its caller would wait forever.
135 *
136 * Writing JRCR[RESET] while RESET reads 0 flushes the ring: jobs already
137 * in the holding tanks or DECOs are terminated onto the output ring with
138 * an error status, and the ordinary completion path reclaims them. So
139 * this only starts the flush and later clears HALT. Jobs merely stalled
140 * in the input ring resume from there.
141 */
142 static void
sec_jr_watchdog(void * arg)143 sec_jr_watchdog(void *arg)
144 {
145 struct sec_jr *jr = arg;
146 struct sec_softc *sc = jr->jr_sc;
147 struct sec_job *job;
148 uint32_t intr;
149
150 if (jr->jr_dying)
151 return;
152
153 if (jr->jr_flushing) {
154 intr = JR_RD4(sc, jr, JR_JRINTR);
155 if ((intr & JRINTR_HALT_M) == JRINTR_HALT_DONE) {
156 JR_WR4(sc, jr, JR_JRINTR, JRINTR_HALT_DONE);
157 jr->jr_flushing = false;
158 device_printf(sc->sc_dev,
159 "job ring at %#x resumed after flush\n",
160 jr->jr_off);
161 }
162 } else if ((job = TAILQ_FIRST(&jr->jr_active)) != NULL &&
163 (int)(ticks - job->job_deadline) >= 0) {
164 device_printf(sc->sc_dev,
165 "job ring at %#x stalled with %u job%s outstanding, "
166 "flushing\n", jr->jr_off, jr->jr_inflight,
167 jr->jr_inflight != 1 ? "s" : "");
168 JR_WR4(sc, jr, JR_JRCR, JRCR_RESET);
169 jr->jr_flushing = true;
170 }
171
172 callout_reset(&jr->jr_wdog, hz, sec_jr_watchdog, jr);
173 }
174
175 /* A per-job non-zero status arrives in the OR entry, not through JRE. */
176 static void
sec_jr_intr(void * arg)177 sec_jr_intr(void *arg)
178 {
179 struct sec_jr *jr = arg;
180 struct sec_softc *sc = jr->jr_sc;
181 struct sec_or_entry done[JR_RING_SIZE];
182 uint32_t i, intr, n, tail;
183 int blocked;
184
185 mtx_lock(&jr->jr_lock);
186
187 intr = JR_RD4(sc, jr, JR_JRINTR);
188 if ((intr & (JRINTR_JRI | JRINTR_JRE)) == 0) {
189 mtx_unlock(&jr->jr_lock);
190 return;
191 }
192 if ((intr & JRINTR_JRE) != 0)
193 device_printf(sc->sc_dev, "JR error, JRINTR=%#x\n", intr);
194
195 /*
196 * Acknowledge before draining. A job completing between the ORSFR
197 * read and the W1C would otherwise have its interrupt cleared along
198 * with the one being serviced, and would sit there with nothing
199 * left to raise it again. Acknowledging first costs at worst a
200 * spurious interrupt that finds the ring empty.
201 */
202 JR_WR4(sc, jr, JR_JRINTR, intr & (JRINTR_JRI | JRINTR_JRE));
203
204 /*
205 * Completion has to run with jr_lock dropped, since crypto_done()
206 * can dispatch the next request straight back into sec_process().
207 */
208 n = JR_RD4(sc, jr, JR_ORSFR);
209 if (n > JR_RING_SIZE)
210 n = JR_RING_SIZE;
211 for (i = 0; i < n; i++) {
212 tail = (jr->jr_or_tail + i) & JR_RING_MASK;
213 done[i] = jr->jr_or[tail];
214 TAILQ_REMOVE(&jr->jr_active, (struct sec_job *)
215 PHYS_TO_DMAP((vm_paddr_t)done[i].desc_addr), job_link);
216 }
217 if (n != 0) {
218 /* Finish reading the entries before freeing their slots. */
219 atomic_thread_fence_rel();
220 jr->jr_or_tail += n;
221 JR_WR4(sc, jr, JR_ORJRR, n);
222 jr->jr_inflight -= n;
223 }
224
225 blocked = 0;
226 if (jr->jr_blocked != 0 && jr->jr_inflight < JR_RING_SIZE) {
227 blocked = jr->jr_blocked;
228 jr->jr_blocked = 0;
229 }
230
231 mtx_unlock(&jr->jr_lock);
232
233 if (blocked != 0)
234 crypto_unblock(sc->sc_cid, blocked);
235
236 for (i = 0; i < n; i++)
237 sec_complete_one(sc, done[i].desc_addr, done[i].status);
238 }
239
240 static int
sec_jr_init(struct sec_softc * sc,struct sec_jr * jr)241 sec_jr_init(struct sec_softc *sc, struct sec_jr *jr)
242 {
243 void *ring_va;
244 size_t ir_bytes = JR_RING_SIZE * sizeof(uint64_t);
245 size_t or_bytes = JR_RING_SIZE * sizeof(struct sec_or_entry);
246 size_t total = ir_bytes + or_bytes;
247
248 mtx_init(&jr->jr_lock, device_get_nameunit(sc->sc_dev), NULL, MTX_DEF);
249 TAILQ_INIT(&jr->jr_active);
250 callout_init_mtx(&jr->jr_wdog, &jr->jr_lock, 0);
251
252 if (bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 64, 0,
253 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
254 total, 1, total, BUS_DMA_ALLOCNOW, NULL, NULL,
255 &jr->jr_ring_tag) != 0)
256 return (ENOMEM);
257 if (bus_dmamem_alloc(jr->jr_ring_tag, &ring_va,
258 BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT,
259 &jr->jr_map) != 0)
260 return (ENOMEM);
261
262 jr->jr_ir = ring_va;
263 jr->jr_or = (struct sec_or_entry *)((uint8_t *)ring_va + ir_bytes);
264
265 if (bus_dmamap_load(jr->jr_ring_tag, jr->jr_map, ring_va,
266 total, sec_jr_dma_cb, &jr->jr_ir_pa,
267 BUS_DMA_NOWAIT) != 0 || jr->jr_ir_pa == 0)
268 return (ENOMEM);
269 jr->jr_or_pa = jr->jr_ir_pa + ir_bytes;
270
271 JR_WR4(sc, jr, JR_IRBAR_MS, (uint32_t)(jr->jr_ir_pa >> 32));
272 JR_WR4(sc, jr, JR_IRBAR_LS, (uint32_t)jr->jr_ir_pa);
273 JR_WR4(sc, jr, JR_IRSR, JR_RING_SIZE);
274
275 JR_WR4(sc, jr, JR_ORBAR_MS, (uint32_t)(jr->jr_or_pa >> 32));
276 JR_WR4(sc, jr, JR_ORBAR_LS, (uint32_t)jr->jr_or_pa);
277 JR_WR4(sc, jr, JR_ORSR, JR_RING_SIZE);
278
279 /* Enable the ring by writing IRSAR = ring size (all slots free). */
280 JR_WR4(sc, jr, JR_IRSAR, JR_RING_SIZE);
281
282 mtx_lock(&jr->jr_lock);
283 callout_reset(&jr->jr_wdog, hz, sec_jr_watchdog, jr);
284 mtx_unlock(&jr->jr_lock);
285
286 return (0);
287 }
288
289 int
sec_init_rings(struct sec_softc * sc)290 sec_init_rings(struct sec_softc *sc)
291 {
292 uint32_t reg[2];
293 int err, i, njrs;
294
295 njrs = sec_jr_count(sc);
296 if (njrs == 0)
297 return (0);
298
299 sc->sc_jr = mallocarray(njrs, sizeof(struct sec_jr), M_SEC,
300 M_WAITOK | M_ZERO);
301
302 i = 0;
303 FOREACH_JOB_RING(ofw_bus_get_node(sc->sc_dev)) {
304 struct sec_jr *jr = &sc->sc_jr[i];
305
306 OF_getencprop(child, "reg", reg, sizeof(reg));
307 jr->jr_sc = sc;
308 jr->jr_node = child;
309 /* Offset within SEC's CCSR window. */
310 jr->jr_off = reg[0];
311
312 err = sec_jr_init(sc, jr);
313 if (err != 0)
314 goto fail;
315
316 err = sec_jr_irq_setup(sc, jr, i);
317 if (err != 0) {
318 device_printf(sc->sc_dev,
319 "could not install JR%u interrupt\n", i);
320 goto fail;
321 }
322 i++;
323 }
324
325 sc->sc_njr = njrs;
326
327 return (njrs);
328
329 fail:
330 /*
331 * Teardown copes with a partly built ring, so running it over the
332 * whole array also cleans up the one that failed.
333 */
334 for (i = 0; i < njrs; i++)
335 sec_jr_teardown(sc, &sc->sc_jr[i]);
336 free(sc->sc_jr, M_SEC);
337 sc->sc_jr = NULL;
338
339 return (0);
340 }
341
342 void
sec_jr_teardown(struct sec_softc * sc,struct sec_jr * jr)343 sec_jr_teardown(struct sec_softc *sc, struct sec_jr *jr)
344 {
345
346 if (mtx_initialized(&jr->jr_lock)) {
347 mtx_lock(&jr->jr_lock);
348 jr->jr_dying = true;
349 callout_stop(&jr->jr_wdog);
350 mtx_unlock(&jr->jr_lock);
351 callout_drain(&jr->jr_wdog);
352 }
353
354 if (jr->jr_ring_tag != NULL) {
355 /* Halt the JR by writing 0 to input ring size. */
356 if (sc->sc_rres != NULL)
357 JR_WR4(sc, jr, JR_IRSR, 0);
358
359 if (jr->jr_ir != NULL) {
360 bus_dmamap_unload(jr->jr_ring_tag, jr->jr_map);
361 bus_dmamem_free(jr->jr_ring_tag, jr->jr_ir,
362 jr->jr_map);
363 }
364 bus_dma_tag_destroy(jr->jr_ring_tag);
365 }
366
367 /* sec_jr_init() can fail after taking the lock but before the tag. */
368 if (mtx_initialized(&jr->jr_lock))
369 mtx_destroy(&jr->jr_lock);
370 }
371
372 int
sec_jr_submit_job(struct sec_softc * sc,struct sec_jr * jr,struct sec_job * job)373 sec_jr_submit_job(struct sec_softc *sc, struct sec_jr *jr, struct sec_job *job)
374 {
375 vm_paddr_t job_pa;
376 int slot;
377
378 job_pa = pmap_kextract((vm_offset_t)job);
379 mtx_lock(&jr->jr_lock);
380
381 if (jr->jr_inflight >= JR_RING_SIZE) {
382 jr->jr_blocked = CRYPTO_SYMQ;
383 mtx_unlock(&jr->jr_lock);
384 return (ERESTART);
385 }
386
387 slot = jr->jr_ir_head & JR_RING_MASK;
388 jr->jr_ir[slot] = (uint64_t)job_pa;
389 jr->jr_ir_head++;
390 jr->jr_inflight++;
391 job->job_deadline = ticks + SEC_JOB_TIMEOUT;
392 TAILQ_INSERT_TAIL(&jr->jr_active, job, job_link);
393 /* The ring entry must be visible before the doorbell. */
394 atomic_thread_fence_rel();
395 JR_WR4(sc, jr, JR_IRJAR, 1);
396
397 mtx_unlock(&jr->jr_lock);
398
399 return (0);
400 }
401