1 /* 2 * Copyright 2026 Justin Hibbits <jhibbits@FreeBSD.org> 3 * 4 * SPDX-License-Identifier: BSD-2-Clause 5 */ 6 #include "sec_var.h" 7 8 /* 9 * A job that doesn't complete in 5 seconds (should take microseconds or less) 10 * is considered a failure. 11 */ 12 #define SEC_JOB_TIMEOUT (5 * hz) 13 14 /* 15 * Job Ring register offsets, relative to the JR's base within SEC's 16 * CCSR window. 17 */ 18 #define JR_IRBAR_MS 0x00 /* Input ring base, upper (64-bit reg) */ 19 #define JR_IRBAR_LS 0x04 /* Input ring base, lower */ 20 #define JR_IRSR 0x0c /* Input ring size (ring entries) */ 21 #define JR_IRSAR 0x14 /* Input ring slots available (add-to) */ 22 #define JR_IRJAR 0x1c /* Input ring jobs added (bump on enqueue) */ 23 #define JR_ORBAR_MS 0x20 /* Output ring base, upper */ 24 #define JR_ORBAR_LS 0x24 /* Output ring base, lower */ 25 #define JR_ORSR 0x2c /* Output ring size */ 26 #define JR_ORJRR 0x34 /* Output ring jobs removed */ 27 #define JR_ORSFR 0x3c /* Output ring slots full */ 28 #define JR_JRSTAR 0x44 /* Output status (per-job termination) */ 29 #define JR_JRINTR 0x4c /* Interrupt status (W1C) */ 30 #define JRINTR_JRI 0x00000001 /* JR interrupt asserted */ 31 #define JRINTR_JRE 0x00000002 /* JR error */ 32 /* 33 * HALT tracks a flush requested through JRCR: 01b while SEC 34 * is still draining, 10b once every job has reached the output ring. 35 * Writing the field's high bit clears it and lets the ring run again. 36 */ 37 #define JRINTR_HALT_M 0x0000000c 38 #define JRINTR_HALT_ONGOING 0x00000004 39 #define JRINTR_HALT_DONE 0x00000008 40 #define JR_JRCFGR_MS 0x50 /* Configuration, upper */ 41 #define JR_JRCFGR_LS 0x54 /* Configuration, lower */ 42 #define JRCFGR_LS_IMSK 0x00000001 /* Mask interrupts (1=masked) */ 43 #define JRCFGR_LS_ICEN 0x00000002 /* Interrupt coalescing enable */ 44 #define JR_JRCR 0x6c /* Command: flush/reset */ 45 #define JRCR_RESET 0x00000001 /* Flush, or reset if halted */ 46 #define JR_IRRIR 0x5c /* Input ring read index (RO) */ 47 #define JR_ORWIR 0x64 /* Output ring write index (RO) */ 48 49 #define JR_RING_SIZE 16 /* power of 2, small for scaffolding */ 50 #define JR_RING_MASK (JR_RING_SIZE - 1) 51 52 #define JR_RD4(sec, jr, off) bus_read_4(sec->sc_rres, jr->jr_off + off) 53 #define JR_WR4(sec, jr, off, v) \ 54 bus_write_4(sec->sc_rres, jr->jr_off + off, v) 55 56 static void sec_jr_intr(void *arg); 57 58 #define FOREACH_JOB_RING(node) \ 59 for (phandle_t child = OF_child(node); child != 0; \ 60 child = OF_peer(child)) \ 61 if ((ofw_bus_node_is_compatible(child, \ 62 "fsl,sec-v5.0-job-ring") || \ 63 ofw_bus_node_is_compatible(child, \ 64 "fsl,sec-v4.0-job-ring")) && \ 65 ofw_bus_node_status_okay(child) && \ 66 OF_getproplen(child, "reg") == 2 * sizeof(pcell_t)) 67 /* 68 * Job Ring helpers. 69 */ 70 71 static int 72 sec_jr_count(struct sec_softc *sc) 73 { 74 phandle_t node = ofw_bus_get_node(sc->sc_dev); 75 int n = 0; 76 77 FOREACH_JOB_RING(node) 78 n++; 79 80 return (n); 81 } 82 83 static void 84 sec_jr_dma_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 85 { 86 vm_paddr_t *pa = arg; 87 *pa = error == 0 && nsegs == 1 ? segs[0].ds_addr : 0; 88 } 89 90 static int 91 sec_jr_irq_setup(struct sec_softc *sc, struct sec_jr *jr, u_int idx) 92 { 93 device_t dev = sc->sc_dev; 94 struct resource_list *rl; 95 phandle_t iparent; 96 pcell_t *cells; 97 int ncells, irqnum; 98 99 if (jr->jr_node == 0) 100 return (ENXIO); 101 102 if (ofw_bus_intr_by_rid(dev, jr->jr_node, 0, &iparent, &ncells, 103 &cells) != 0) 104 return (ENXIO); 105 irqnum = ofw_bus_map_intr(dev, iparent, ncells, cells); 106 OF_prop_free(cells); 107 if (irqnum <= 0) 108 return (ENXIO); 109 110 rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev); 111 jr->jr_irid = 1 + idx; /* rid 0 is the SEC top-level error IRQ */ 112 resource_list_add(rl, SYS_RES_IRQ, jr->jr_irid, irqnum, irqnum, 1); 113 114 jr->jr_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, 115 &jr->jr_irid, RF_ACTIVE); 116 if (jr->jr_ires == NULL) 117 return (ENXIO); 118 119 if (bus_setup_intr(dev, jr->jr_ires, INTR_TYPE_MISC | INTR_MPSAFE, 120 NULL, sec_jr_intr, jr, &jr->jr_icookie) != 0) 121 return (ENXIO); 122 123 /* 124 * Enable JR interrupts (IMSK=0, ICEN=0 = fire on every completion). 125 * Reset default is already IMSK=0, but be explicit. 126 */ 127 JR_WR4(sc, jr, JR_JRCFGR_LS, 0); 128 129 return (0); 130 } 131 132 /* 133 * Watchdog for a wedged ring. Nothing else reclaims a job that never 134 * reaches the output ring, so its caller would wait forever. 135 * 136 * Writing JRCR[RESET] while RESET reads 0 flushes the ring: jobs already 137 * in the holding tanks or DECOs are terminated onto the output ring with 138 * an error status, and the ordinary completion path reclaims them. So 139 * this only starts the flush and later clears HALT. Jobs merely stalled 140 * in the input ring resume from there. 141 */ 142 static void 143 sec_jr_watchdog(void *arg) 144 { 145 struct sec_jr *jr = arg; 146 struct sec_softc *sc = jr->jr_sc; 147 struct sec_job *job; 148 uint32_t intr; 149 150 if (jr->jr_dying) 151 return; 152 153 if (jr->jr_flushing) { 154 intr = JR_RD4(sc, jr, JR_JRINTR); 155 if ((intr & JRINTR_HALT_M) == JRINTR_HALT_DONE) { 156 JR_WR4(sc, jr, JR_JRINTR, JRINTR_HALT_DONE); 157 jr->jr_flushing = false; 158 device_printf(sc->sc_dev, 159 "job ring at %#x resumed after flush\n", 160 jr->jr_off); 161 } 162 } else if ((job = TAILQ_FIRST(&jr->jr_active)) != NULL && 163 (int)(ticks - job->job_deadline) >= 0) { 164 device_printf(sc->sc_dev, 165 "job ring at %#x stalled with %u job%s outstanding, " 166 "flushing\n", jr->jr_off, jr->jr_inflight, 167 jr->jr_inflight != 1 ? "s" : ""); 168 JR_WR4(sc, jr, JR_JRCR, JRCR_RESET); 169 jr->jr_flushing = true; 170 } 171 172 callout_reset(&jr->jr_wdog, hz, sec_jr_watchdog, jr); 173 } 174 175 /* A per-job non-zero status arrives in the OR entry, not through JRE. */ 176 static void 177 sec_jr_intr(void *arg) 178 { 179 struct sec_jr *jr = arg; 180 struct sec_softc *sc = jr->jr_sc; 181 struct sec_or_entry done[JR_RING_SIZE]; 182 uint32_t i, intr, n, tail; 183 int blocked; 184 185 mtx_lock(&jr->jr_lock); 186 187 intr = JR_RD4(sc, jr, JR_JRINTR); 188 if ((intr & (JRINTR_JRI | JRINTR_JRE)) == 0) { 189 mtx_unlock(&jr->jr_lock); 190 return; 191 } 192 if ((intr & JRINTR_JRE) != 0) 193 device_printf(sc->sc_dev, "JR error, JRINTR=%#x\n", intr); 194 195 /* 196 * Acknowledge before draining. A job completing between the ORSFR 197 * read and the W1C would otherwise have its interrupt cleared along 198 * with the one being serviced, and would sit there with nothing 199 * left to raise it again. Acknowledging first costs at worst a 200 * spurious interrupt that finds the ring empty. 201 */ 202 JR_WR4(sc, jr, JR_JRINTR, intr & (JRINTR_JRI | JRINTR_JRE)); 203 204 /* 205 * Completion has to run with jr_lock dropped, since crypto_done() 206 * can dispatch the next request straight back into sec_process(). 207 */ 208 n = JR_RD4(sc, jr, JR_ORSFR); 209 if (n > JR_RING_SIZE) 210 n = JR_RING_SIZE; 211 for (i = 0; i < n; i++) { 212 tail = (jr->jr_or_tail + i) & JR_RING_MASK; 213 done[i] = jr->jr_or[tail]; 214 TAILQ_REMOVE(&jr->jr_active, (struct sec_job *) 215 PHYS_TO_DMAP((vm_paddr_t)done[i].desc_addr), job_link); 216 } 217 if (n != 0) { 218 /* Finish reading the entries before freeing their slots. */ 219 atomic_thread_fence_rel(); 220 jr->jr_or_tail += n; 221 JR_WR4(sc, jr, JR_ORJRR, n); 222 jr->jr_inflight -= n; 223 } 224 225 blocked = 0; 226 if (jr->jr_blocked != 0 && jr->jr_inflight < JR_RING_SIZE) { 227 blocked = jr->jr_blocked; 228 jr->jr_blocked = 0; 229 } 230 231 mtx_unlock(&jr->jr_lock); 232 233 if (blocked != 0) 234 crypto_unblock(sc->sc_cid, blocked); 235 236 for (i = 0; i < n; i++) 237 sec_complete_one(sc, done[i].desc_addr, done[i].status); 238 } 239 240 static int 241 sec_jr_init(struct sec_softc *sc, struct sec_jr *jr) 242 { 243 void *ring_va; 244 size_t ir_bytes = JR_RING_SIZE * sizeof(uint64_t); 245 size_t or_bytes = JR_RING_SIZE * sizeof(struct sec_or_entry); 246 size_t total = ir_bytes + or_bytes; 247 248 mtx_init(&jr->jr_lock, device_get_nameunit(sc->sc_dev), NULL, MTX_DEF); 249 TAILQ_INIT(&jr->jr_active); 250 callout_init_mtx(&jr->jr_wdog, &jr->jr_lock, 0); 251 252 if (bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 64, 0, 253 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 254 total, 1, total, BUS_DMA_ALLOCNOW, NULL, NULL, 255 &jr->jr_ring_tag) != 0) 256 return (ENOMEM); 257 if (bus_dmamem_alloc(jr->jr_ring_tag, &ring_va, 258 BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, 259 &jr->jr_map) != 0) 260 return (ENOMEM); 261 262 jr->jr_ir = ring_va; 263 jr->jr_or = (struct sec_or_entry *)((uint8_t *)ring_va + ir_bytes); 264 265 if (bus_dmamap_load(jr->jr_ring_tag, jr->jr_map, ring_va, 266 total, sec_jr_dma_cb, &jr->jr_ir_pa, 267 BUS_DMA_NOWAIT) != 0 || jr->jr_ir_pa == 0) 268 return (ENOMEM); 269 jr->jr_or_pa = jr->jr_ir_pa + ir_bytes; 270 271 JR_WR4(sc, jr, JR_IRBAR_MS, (uint32_t)(jr->jr_ir_pa >> 32)); 272 JR_WR4(sc, jr, JR_IRBAR_LS, (uint32_t)jr->jr_ir_pa); 273 JR_WR4(sc, jr, JR_IRSR, JR_RING_SIZE); 274 275 JR_WR4(sc, jr, JR_ORBAR_MS, (uint32_t)(jr->jr_or_pa >> 32)); 276 JR_WR4(sc, jr, JR_ORBAR_LS, (uint32_t)jr->jr_or_pa); 277 JR_WR4(sc, jr, JR_ORSR, JR_RING_SIZE); 278 279 /* Enable the ring by writing IRSAR = ring size (all slots free). */ 280 JR_WR4(sc, jr, JR_IRSAR, JR_RING_SIZE); 281 282 mtx_lock(&jr->jr_lock); 283 callout_reset(&jr->jr_wdog, hz, sec_jr_watchdog, jr); 284 mtx_unlock(&jr->jr_lock); 285 286 return (0); 287 } 288 289 int 290 sec_init_rings(struct sec_softc *sc) 291 { 292 uint32_t reg[2]; 293 int err, i, njrs; 294 295 njrs = sec_jr_count(sc); 296 if (njrs == 0) 297 return (0); 298 299 sc->sc_jr = mallocarray(njrs, sizeof(struct sec_jr), M_SEC, 300 M_WAITOK | M_ZERO); 301 302 i = 0; 303 FOREACH_JOB_RING(ofw_bus_get_node(sc->sc_dev)) { 304 struct sec_jr *jr = &sc->sc_jr[i]; 305 306 OF_getencprop(child, "reg", reg, sizeof(reg)); 307 jr->jr_sc = sc; 308 jr->jr_node = child; 309 /* Offset within SEC's CCSR window. */ 310 jr->jr_off = reg[0]; 311 312 err = sec_jr_init(sc, jr); 313 if (err != 0) 314 goto fail; 315 316 err = sec_jr_irq_setup(sc, jr, i); 317 if (err != 0) { 318 device_printf(sc->sc_dev, 319 "could not install JR%u interrupt\n", i); 320 goto fail; 321 } 322 i++; 323 } 324 325 sc->sc_njr = njrs; 326 327 return (njrs); 328 329 fail: 330 /* 331 * Teardown copes with a partly built ring, so running it over the 332 * whole array also cleans up the one that failed. 333 */ 334 for (i = 0; i < njrs; i++) 335 sec_jr_teardown(sc, &sc->sc_jr[i]); 336 free(sc->sc_jr, M_SEC); 337 sc->sc_jr = NULL; 338 339 return (0); 340 } 341 342 void 343 sec_jr_teardown(struct sec_softc *sc, struct sec_jr *jr) 344 { 345 346 if (mtx_initialized(&jr->jr_lock)) { 347 mtx_lock(&jr->jr_lock); 348 jr->jr_dying = true; 349 callout_stop(&jr->jr_wdog); 350 mtx_unlock(&jr->jr_lock); 351 callout_drain(&jr->jr_wdog); 352 } 353 354 if (jr->jr_ring_tag != NULL) { 355 /* Halt the JR by writing 0 to input ring size. */ 356 if (sc->sc_rres != NULL) 357 JR_WR4(sc, jr, JR_IRSR, 0); 358 359 if (jr->jr_ir != NULL) { 360 bus_dmamap_unload(jr->jr_ring_tag, jr->jr_map); 361 bus_dmamem_free(jr->jr_ring_tag, jr->jr_ir, 362 jr->jr_map); 363 } 364 bus_dma_tag_destroy(jr->jr_ring_tag); 365 } 366 367 /* sec_jr_init() can fail after taking the lock but before the tag. */ 368 if (mtx_initialized(&jr->jr_lock)) 369 mtx_destroy(&jr->jr_lock); 370 } 371 372 int 373 sec_jr_submit_job(struct sec_softc *sc, struct sec_jr *jr, struct sec_job *job) 374 { 375 vm_paddr_t job_pa; 376 int slot; 377 378 job_pa = pmap_kextract((vm_offset_t)job); 379 mtx_lock(&jr->jr_lock); 380 381 if (jr->jr_inflight >= JR_RING_SIZE) { 382 jr->jr_blocked = CRYPTO_SYMQ; 383 mtx_unlock(&jr->jr_lock); 384 return (ERESTART); 385 } 386 387 slot = jr->jr_ir_head & JR_RING_MASK; 388 jr->jr_ir[slot] = (uint64_t)job_pa; 389 jr->jr_ir_head++; 390 jr->jr_inflight++; 391 job->job_deadline = ticks + SEC_JOB_TIMEOUT; 392 TAILQ_INSERT_TAIL(&jr->jr_active, job, job_link); 393 /* The ring entry must be visible before the doorbell. */ 394 atomic_thread_fence_rel(); 395 JR_WR4(sc, jr, JR_IRJAR, 1); 396 397 mtx_unlock(&jr->jr_lock); 398 399 return (0); 400 } 401