1 /*
2 * Copyright (c) 2026 Justin Hibbits <jhibbits@FreeBSD.org>
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6
7 #include <sys/param.h>
8 #include <sys/bus.h>
9 #include <sys/callout.h>
10 #include <sys/kernel.h>
11 #include <sys/lock.h>
12 #include <sys/malloc.h>
13 #include <sys/module.h>
14 #include <sys/mutex.h>
15 #include <sys/queue.h>
16 #include <sys/rman.h>
17 #include <sys/smp.h>
18
19 #include <machine/atomic.h>
20 #include <machine/bus.h>
21 #include <machine/resource.h>
22
23 #include <vm/vm.h>
24 #include <vm/pmap.h>
25
26 #include <sys/endian.h>
27
28 #include <dev/ofw/ofw_bus.h>
29 #include <dev/ofw/ofw_bus_subr.h>
30 #include <opencrypto/cryptodev.h>
31 #include <opencrypto/xform_auth.h>
32
33 #include "sec_var.h"
34 #include "cryptodev_if.h"
35
36 /*
37 * Most of this work is based on the T2080 Security (SEC) Reference Manual.
38 *
39 * The driver uses the Job Ring interface for all jobs. The QI interface can be
40 * added if IPSec, OVPN, or kTLS acceleration is added.
41 */
42
43 /* From T2080 Security Reference Manual */
44 #define SEC_MAX_SHDESC_WORDS 62
45
46 #define SEC_MAX_JR 4 /* T2080 exposes four Job Rings */
47
48 /* CCSR register offsets. */
49 #define SEC_MCFGR 0x0004
50 #define MCFGR_SWRST 0x80000000 /* Software reset */
51 #define MCFGR_WDE 0x40000000 /* DECO watchdog enable */
52 #define MCFGR_WDF 0x20000000 /* Watchdog fast (test only) */
53 #define MCFGR_DMARST 0x10000000 /* DMA reset (with SWRST) */
54 #define MCFGR_WRHD 0x08000000 /* Write handoff disable */
55 #define MCFGR_DJPC 0x00200000 /* Disable job perf ctrs */
56 #define MCFGR_DBPC 0x00100000 /* Disable byte perf ctrs */
57 #define MCFGR_PS 0x00010000 /* Large pointers */
58 #define MCFGR_ARCACHE_M 0x0000f000 /* AXI read cache attrs */
59 #define MCFGR_AWCACHE_M 0x00000f00 /* AXI write cache attrs */
60 #define MCFGR_AXIPRI 0x00000008 /* AXI master priority */
61 #define MCFGR_LARGE_BURST 0x00000004 /* Enable 256B bursts */
62 #define SEC_SCFGR 0x000c
63 #define SCFGR_VIRT_EN 0x00008000 /* Virtualization enabled */
64
65 #define SEC_RDSTA 0x06c0 /* RNG DRNG Status */
66 #define RDSTA_IF0 0x00000001 /* State handle 0 up */
67 #define RDSTA_IF1 0x00000002 /* State handle 1 up */
68 #define RDSTA_ERRCODE_M 0x000f0000
69 #define RDSTA_ERRCODE_S 16
70 #define RDSTA_CE 0x00100000 /* Catastrophic error */
71
72 /* DECO direct-access registers */
73 #define SEC_DECORR 0x009c /* DECO Request Register */
74 #define DECORR_DEN0 0x00010000 /* DECO0 enable (RO, bit 16) */
75 #define DECORR_RQD0 0x00000001 /* DECO0 request */
76 #define SEC_D0LIODNR_MS 0x00a0
77 #define SEC_D0LIODNR_LS 0x00a4
78 #define SEC_D0JQCR_MS 0x8800 /* JQCR upper: WHL/FOUR/SOB */
79 #define DAJQCR_MS_WHL 0x20000000 /* Whole descriptor loaded */
80 #define DAJQCR_MS_FOUR 0x10000000 /* >= 4 words in first burst */
81 #define DAJQCR_MS_SOB 0x00010000 /* Shared/burst loaded */
82 #define DAJQCR_MS_SRC_M 0x00000700 /* Job source */
83 #define DAJQCR_MS_SRC_S 8
84 #define SEC_D0JQCR_LS 0x8804
85 #define SEC_D0DAR_MS 0x8808 /* Descriptor address, upper */
86 #define SEC_D0DAR_LS 0x880c
87 #define SEC_D0DESB(n) (0x8a00 + (n) * 4) /* n = 0..63 */
88 #define SEC_D0DDR 0x8e04 /* Debug status */
89 #define DADDR_VALID 0x80000000 /* Job currently running */
90 #define DADDR_DECO_STATE_M 0x00f00000 /* Main state machine */
91 #define DADDR_DECO_STATE_S 20
92
93 /* Fault-address registers. */
94 #define SEC_FAR_HI 0x0fc0 /* Fault Address, upper */
95 #define SEC_FAR_LO 0x0fc4 /* Fault Address, lower */
96 #define SEC_FALR 0x0fc8 /* Fault Address LIODN */
97 #define SEC_FADR 0x0fcc /* Fault Address Detail */
98 #define FADR_FERR_M 0xc0000000 /* AXI error response */
99 #define FADR_FERR_S 30
100 #define FADR_FSZ_EXT_M 0x00070000 /* Transfer size high 3 bits */
101 #define FADR_FSZ_EXT_S 16
102 #define FADR_DTYP 0x00008000 /* 0=message, 1=control */
103 #define FADR_JSRC_M 0x00007000 /* Job source */
104 #define FADR_JSRC_S 12
105 #define FADR_BLKID_M 0x00000f00 /* SEC internal block ID */
106 #define FADR_BLKID_S 8
107 #define FADR_TYP 0x00000080 /* 0=read, 1=write */
108 #define FADR_FSZ_M 0x0000007f /* Transfer size low 7 bits */
109
110 #define SEC_RD4(sc, off) bus_read_4((sc)->sc_rres, (off))
111 #define SEC_WR4(sc, off, v) bus_write_4((sc)->sc_rres, (off), (v))
112
113 /* Descriptor command components */
114 /* SEQ commands are intended for network protocols */
115 #define CMD_DESC(n) ((n) << 27)
116 #define CMD_KEY 0x00 /* Pointer/key follows descriptor */
117 #define CMD_SEQ_KEY 0x01
118 #define KEY_CLASS_M 0x06000000
119 #define KEY_CLASS_1 0x02000000
120 #define KEY_CLASS_2 0x04000000
121 #define KEY_SGF 0x01000000 /* KEY - Pointer to SGT */
122 #define KEY_VLF 0x01000000 /* SK - variable length */
123 #define KEY_IMM 0x00800000 /* KEY - Key follows descriptor */
124 #define KEY_AIDF 0x00800000 /* SK - Already in Input FIFO */
125 #define KEY_ENC 0x00400000 /* Key is encrypted */
126 #define KEY_NWB 0x00200000 /* No write back */
127 #define KEY_EKT 0x00100000 /* Encrypted Key Type:
128 * 0 - AES-CCB
129 * 1 - AES-CCM
130 */
131 #define KEY_KDEST_M 0x00030000 /* Key Destination */
132 #define KEY_KDEST_REG 0x00000000 /* Dest is Key register */
133 #define KEY_KDEST_PKHA 0x00010000 /* Dest is PKHA E-memory */
134 #define KEY_KDEST_AFHA 0x00020000 /* Dest is AFHA S-Box */
135 #define KEY_KDEST_MDHA_SPLIT 0x00030000 /* Key is MDHA split key */
136 #define KEY_TK 0x00008000 /* Trusted Key */
137 #define KEY_LENGTH_M 0x000003ff /* Key length */
138 #define CMD_LOAD 0x02
139 #define CMD_SEQ_LOAD 0x03
140 #define LOAD_CLASS_M 0x06000000
141 #define LOAD_CLASS_1 0x02000000
142 #define LOAD_CLASS_2 0x04000000
143 #define LOAD_CLASS_3 0x06000000
144 #define LOAD_SGF 0x01000000 /* LOAD - Pointer to SGT */
145 #define LOAD_VLF 0x01000000 /* SL - variable length */
146 #define LOAD_IMM 0x00800000 /* LOAD - Data follows descriptor */
147 #define LOAD_DST_M 0x007f0000 /* Destination register */
148 #define LOAD_DST_S 16
149 #define LOAD_KSR 0x00010000 /* Key Size Register (C1/C2) */
150 #define LOAD_DSR 0x00020000 /* Data Size Register (C1/C2) */
151 #define LOAD_ICVS 0x00030000 /* ICV Size Register (C1/C2) */
152 #define LOAD_LSR 0x00040000 /* LIODN Status Register (C3) */
153 #define LOAD_DCTRL2 0x00050000 /* DECO Control Register 2(C3) */
154 #define LOAD_CCTRL 0x00060000 /* CHA Control Register (C1) */
155 #define LOAD_DCTRL 0x00060000 /* DECO Control Register (C3) */
156 #define LOAD_ICTRL 0x00070000 /* IRQ Control Register (C0) */
157 #define LOAD_DPOVRD 0x00070000 /* DECO Protocol Override (C3) */
158 #define LOAD_CLRW 0x00080000 /* Clear Written Register (C0) */
159 #define LOAD_MATH0W 0x00080000 /* DECO Math Register 0 (C3) */
160 #define LOAD_MATH1W 0x00090000 /* DECO Math Register 1 (C3) */
161 #define LOAD_MATH2W 0x000a0000 /* DECO Math Register 2 (C3) */
162 #define LOAD_CISEL 0x000a0000 /* CHA Instance Select Reg (C0) */
163 #define LOAD_AADSZ 0x000b0000 /* AAD Size Register (C1) */
164 #define LOAD_MAT3W 0x000b0000 /* DECO Math Register 3 (C3) */
165 #define LOAD_C1VSZ 0x000c0000 /* Class 1 IV SIze Register (C1) */
166 #define LOAD_ALTDS1 0x000f0000 /* Alternate Data Size C1 (C1) */
167 #define LOAD_PKASZ 0x00100000 /* PKHA A Size Register (C1) */
168 #define LOAD_PKBSZ 0x00110000 /* PKHA B Size Register (C1) */
169 #define LOAD_PKNSZ 0x00120000 /* PKHA N Size Register (C1) */
170 #define LOAD_PKESZ 0x00130000 /* PKHA E Size Register (C1) */
171 #define LOAD_CTX 0x00200000 /* Context Register (C1/C2) */
172 #define LOAD_KEY 0x00400000 /* Key Register (C1/C2) */
173 #define LOAD_DESC_BUF 0x00400000 /* DECO Descriptor Buffer (C3) */
174 #define LOAD_NFSL 0x00700000 /* NFIFO and size registers (C0) */
175 #define LOAD_NFSM 0x00710000 /* NFIFO and size registers (C0) */
176 #define LOAD_NFL 0x00720000 /* NFIFO (C0) */
177 #define LOAD_NFM 0x00730000 /* NFIFO (C0) */
178 #define LOAD_SL 0x00740000 /* Size register(s) (C0) */
179 #define LOAD_SM 0x00750000 /* Size register(s) (C0) */
180 #define LOAD_IDFNS 0x00760000 /* Input Data FIFO Nibble Shift (C0) */
181 #define LOAD_ODFNS 0x00770000 /* Output Data FIFO Nibble Shift (C0) */
182 #define LOAD_AUXDATA 0x00780000 /* Aux Data FIFO (C0) */
183 #define LOAD_NFIFO 0x007a0000 /* NFIFO (C0) */
184 #define LOAD_IFIFO 0x007c0000 /* Input Data FIFO (C0) */
185 #define LOAD_OFIFO 0x007e0000 /* Output Data FIFO (C0) */
186 #define LOAD_LENGTH_M 0x000000ff /* Data length (8 bits) */
187 #define LOAD_OFFSET_S 8 /* OFFSET field shift (bits 8-15) */
188 #define CMD_FIFO_LOAD 0x04
189 #define CMD_SEQ_FIFO_LOAD 0x05
190 #define CMD_STORE 0x0a
191 #define CMD_SEQ_STORE 0x0b
192 #define CMD_FIFO_STORE 0x0c
193 #define CMD_SEQ_FIFO_STORE 0x0d
194 #define CMD_MOVE 0x0e
195 #define CMD_MOVE_LEN 0x0f
196 #define CMD_OPERATION 0x10
197 #define OPTYPE_M 0x07000000
198 #define OPTYPE_S 24
199 #define OPTYPE_CLASS1_ALG 0x02000000
200 #define OPTYPE_CLASS2_ALG 0x04000000
201 #define ALG_S 16
202 #define CMD_ALGORITHM(m, n) ((m) | ((n) << ALG_S))
203 /* Class 1 algorithms */
204 #define ALG_AES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x10)
205 #define ALG_DES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x20)
206 #define ALG_3DES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x21)
207 #define ALG_ARC4 CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x30)
208 #define ALG_RNG CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x50)
209 #define ALG_SNOW3G_F8 CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x60)
210 #define ALG_KASUMI CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x70)
211 #define ALG_ZUC_ENC CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0xb0)
212 /* Class 2 algorithms */
213 #define ALG_MD5 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x40)
214 #define ALG_SHA1 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x41)
215 #define ALG_SHA224 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x42)
216 #define ALG_SHA256 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x43)
217 #define ALG_SHA384 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x44)
218 #define ALG_SHA512 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x45)
219 #define ALG_CRC CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x90)
220 #define ALG_SNOW3G_F9 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0xa0)
221 #define ALG_ZUC_AUTH CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0xc0)
222 /* AAI (Additional Algorithm Information) codes. */
223 #define AAI_S 4
224 /* AES modes */
225 #define AAI_AES_CTR (0x00 << AAI_S)
226 #define AAI_AES_CBC (0x10 << AAI_S)
227 #define AAI_AES_ECB (0x20 << AAI_S)
228 #define AAI_AES_CFB (0x30 << AAI_S)
229 #define AAI_AES_OFB (0x40 << AAI_S)
230 #define AAI_AES_XTS (0x50 << AAI_S)
231 #define AAI_AES_CMAC (0x60 << AAI_S)
232 #define AAI_AES_XCBC_MAC (0x70 << AAI_S)
233 #define AAI_AES_CCM (0x80 << AAI_S)
234 #define AAI_AES_GCM (0x90 << AAI_S)
235 #define AAI_AES_DK (0x100 << AAI_S) /* Decrypt-key derive */
236 /* DES/3DES modes */
237 #define AAI_DES_CBC (0x10 << AAI_S)
238 #define AAI_DES_ECB (0x20 << AAI_S)
239 /* MDHA modes */
240 #define AAI_HASH (0x00 << AAI_S)
241 #define AAI_HMAC (0x01 << AAI_S)
242 #define AAI_HMAC_PRECOMP (0x04 << AAI_S) /* Precomputed IPAD/OPAD */
243 /* Algorithm State field (bits 2-3): what phase to run */
244 #define AS_S 2
245 #define AS_UPDATE (0x0 << AS_S)
246 #define AS_INIT (0x1 << AS_S)
247 #define AS_FINAL (0x2 << AS_S)
248 #define AS_INIT_FINAL (0x3 << AS_S)
249 /* RNG-specific: State-Handle field. */
250 #define OP_RNG_SH_S 4
251 #define OP_RNG_SH(n) ((n) << OP_RNG_SH_S)
252 /* Direction / ICV */
253 #define OP_ICV 0x00000002
254 #define OP_ENC 0x00000001
255
256 /* SEQ FIFO LOAD command bits. */
257 #define FIFOLD_CLASS_1 0x02000000 /* CLASS = 01b (Class 1) */
258 #define FIFOLD_CLASS_2 0x04000000 /* CLASS = 10b (Class 2) */
259 #define FIFOLD_CLASS_BOTH 0x06000000 /* CLASS = 11b (snooping) */
260 #define FIFOLD_VLF 0x01000000 /* Variable-length flag */
261 /*
262 * Input data type: top 3 bits = type,
263 * bottom 3 bits = LC2/LC1/FC1 flags.
264 */
265 #define FIFOLD_TYPE_S 16
266 #define FIFOLD_TYPE_MSG (0x10 << FIFOLD_TYPE_S) /* 010_000 */
267 /* Class 1 output fed straight into Class 2, i.e. MAC over ciphertext. */
268 #define FIFOLD_TYPE_MSG_C1OUT (0x18 << FIFOLD_TYPE_S) /* 011_000 */
269 #define FIFOLD_TYPE_IV (0x20 << FIFOLD_TYPE_S) /* 100_000 */
270 #define FIFOLD_TYPE_AAD (0x30 << FIFOLD_TYPE_S) /* 110_000 */
271 #define FIFOLD_TYPE_ICV (0x38 << FIFOLD_TYPE_S) /* 111_000 */
272 #define FIFOLD_FC1 (0x01 << FIFOLD_TYPE_S) /* Flush class 1 */
273 #define FIFOLD_LC1 (0x02 << FIFOLD_TYPE_S) /* Last for Class 1 */
274 #define FIFOLD_LC2 (0x04 << FIFOLD_TYPE_S) /* Last for Class 2 */
275 /* Length moves to a 32-bit word after the command. */
276 #define FIFO_EXT 0x00400000
277
278 /* SEQ FIFO STORE command bits. */
279 #define FIFOST_VLF 0x01000000
280 #define FIFOST_TYPE_S 16
281 #define FIFOST_TYPE_MSG_DATA (0x30 << FIFOST_TYPE_S)
282
283 #define CMD_SIGNATURE 0x12
284 #define CMD_JUMP 0x14
285 #define CMD_MATH 0x15
286 #define MATH_FN_ADD (0x0 << 20) /* SRC0 + SRC1 */
287 #define MATH_SRC0_SIL (0x8 << 16) /* Sequence In Length */
288 #define MATH_SRC1_ZERO (0xF << 12) /* Constant zero */
289 #define MATH_DEST_VSIL (0xA << 8) /* Variable SIL */
290 #define MATH_DEST_VSOL (0xB << 8) /* Variable SOL */
291 #define MATH_LEN_4 0x4
292 /* J - Job Descriptor, S - Shared Descriptor */
293 #define CMD_DESC_HEADER 0x16
294 #define HEADER_EXT 0x04000000 /* Has Extension (J) */
295 #define HEADER_RSL 0x02000000 /* Require SEQ LIODN (J) */
296 #define HEADER_DNR 0x01000000 /* Do Not Run (J/S) */
297 #define HEADER_ONE 0x00800000 /* Must be 1 (J/S) */
298 #define HEADER_START_INDEX(n) ((n) << 16) /* Start Index (J/S) */
299 #define HEADER_SHR_DESC_L(n) ((n) << 16) /* Shared Desc len (J) */
300 /* Bit 16 must be 0 */
301 #define HEADER_TDES_M 0x00006000 /* Trusted Descriptor Mask (J) */
302 #define HEADER_TDES 0x00004000 /* Trusted Descriptor (J) */
303 #define HEADER_TDES_CAND 0x00006000 /* Candidate Trust Desc (J) */
304 #define HEADER_SHR 0x00001000 /* Has Shared Descriptor (J) */
305 #define HEADER_REO 0x00000800 /* Reverse Execution Order (J) */
306 #define HEADER_SHARE_M 0x00000700 /* Share State (J/S) */
307 #define HEADER_SHARE_WAIT 0x00000100 /* Wait to share (J/S) */
308 #define HEADER_SHARE_SERIAL 0x00000200 /* Serialize (J/S) */
309 #define HEADER_SHARE_ALWAYS 0x00000300 /* Always share (stateless) (J/S) */
310 #define HEADER_SHARE_DEFER 0x00000400 /* Defer to shared desc (J) */
311 #define HEADER_DESCLEN_M 0x0000007f /* Descriptor length */
312 #define HEADER_DESCLEN_S 0
313 #define HEADER_EXT_FTD 0x00000100 /* Fake Trusted Descriptor */
314 #define HEADER_EXT_DSELVALID 0x00000080 /* DECO_SELECT field valid */
315 #define HEADER_EXT_DSEL_M 0x0000000f /* DECO Select */
316 #define CMD_SHARED_HEADER 0x17
317 #define HEADER_RIF 0x02000000 /* Read Input Frame */
318 #define HEADER_CIF 0x00002000 /* Clear Input FIFO */
319 #define HEADER_SC 0x00001000 /* Save Context */
320 #define HEADER_PD 0x00000800 /* Propagate DNR */
321 #define CMD_MATHI 0x1d
322 #define CMD_SEQ_IN_PTR 0x1e
323 #define SEQ_SGF 0x01000000 /* Pointer is SGT (bit 7 NXP) */
324 #define SEQ_EXT 0x00400000 /* 32-bit extended length (bit 9 NXP) */
325 #define CMD_SEQ_OUT_PTR 0x1f
326
327 /* Shared descriptor container. */
328 struct sec_context {
329 uint32_t shd[SEC_MAX_SHDESC_WORDS];
330 };
331
332
333 /*
334 * Session state: one shared descriptor per direction. The shared
335 * descriptor holds just KEY + OPERATION; the per-job JD adds LOAD-IV
336 * and SEQ_IN_PTR / SEQ_OUT_PTR inline.
337 */
338 #define SEC_MAX_SPLIT_KEY 128 /* SHA-512 AES-ECB encrypted */
339
340 #define SEC_CCM_AAD_MAX 0xfeff
341
342 struct sec_session {
343 struct sec_softc *sess_sc;
344 struct sec_context ctx[2]; /* [0]=dec, [1]=enc */
345 uint32_t sdlen[2]; /* words per direction */
346 uint8_t digestlen; /* HMAC output size (0 if none) */
347 uint8_t skeylen; /* HMAC split key size (0 if none) */
348 uint8_t skey[SEC_MAX_SPLIT_KEY];
349 };
350
351 static device_probe_t sec_probe;
352 static device_attach_t sec_attach;
353 static device_detach_t sec_detach;
354 static cryptodev_probesession_t sec_probe_session;
355 static cryptodev_newsession_t sec_new_session;
356 static cryptodev_freesession_t sec_free_session;
357 static cryptodev_process_t sec_process;
358
359 static void sec_intr(void *);
360
361 /* Register-level bring-up. Filled in from the SEC reference manual. */
362 static int sec_reset(struct sec_softc *);
363 static int sec_rng_init(struct sec_softc *);
364
365 static struct ofw_compat_data compats[] = {
366 { "fsl,sec-v5.2", 52 },
367 { "fsl,sec-v5.0", 50 },
368 { "fsl,sec-v4.0", 40 },
369 { NULL, 0 }
370 };
371
372 static device_method_t sec_methods[] = {
373 /* Device methods */
374 DEVMETHOD(device_probe, sec_probe),
375 DEVMETHOD(device_attach, sec_attach),
376 DEVMETHOD(device_detach, sec_detach),
377
378 /* Cryptodev methods */
379 DEVMETHOD(cryptodev_probesession, sec_probe_session),
380 DEVMETHOD(cryptodev_newsession, sec_new_session),
381 DEVMETHOD(cryptodev_freesession, sec_free_session),
382 DEVMETHOD(cryptodev_process, sec_process),
383
384 DEVMETHOD_END
385 };
386
387 static DEFINE_CLASS_0(sec, sec_driver, sec_methods, sizeof(struct sec_softc));
388 DRIVER_MODULE(sec, simplebus, sec_driver, NULL, NULL);
389 MODULE_DEPEND(sec, crypto, 1, 1, 1);
390
391 MALLOC_DEFINE(M_SEC, "sec", "SEC driver");
392
393 static int
sec_probe(device_t dev)394 sec_probe(device_t dev)
395 {
396 const struct ofw_compat_data *cd;
397
398 cd = ofw_bus_search_compatible(dev, compats);
399 if (cd->ocd_data == 0)
400 return (ENXIO);
401
402 device_set_descf(dev, "Freescale Security Engine v%d.%d",
403 (int)cd->ocd_data / 10, (int)cd->ocd_data % 10);
404
405 return (BUS_PROBE_DEFAULT);
406 }
407
408 static int
sec_attach(device_t dev)409 sec_attach(device_t dev)
410 {
411 struct sec_softc *sc = device_get_softc(dev);
412 const struct ofw_compat_data *cd;
413
414 sc->sc_dev = dev;
415 sc->sc_cid = -1;
416
417 cd = ofw_bus_search_compatible(dev, compats);
418 sc->sc_version = cd->ocd_data;
419
420 sc->sc_rrid = 0;
421 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
422 RF_ACTIVE);
423 if (sc->sc_rres == NULL) {
424 device_printf(dev, "could not allocate register resource\n");
425 goto fail;
426 }
427
428 /* TODO: Error IRQ handling. */
429 sc->sc_irid = 0;
430 sc->sc_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irid,
431 RF_ACTIVE | RF_SHAREABLE);
432 if (sc->sc_ires == NULL) {
433 device_printf(dev, "could not allocate error interrupt\n");
434 goto fail;
435 }
436
437 if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
438 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
439 SEC_MAX_SIZE, SEC_MAX_SEGMENTS, SEC_MAX_SIZE, BUS_DMA_ALLOCNOW,
440 NULL, NULL, &sc->sc_dmatag) != 0) {
441 device_printf(dev, "could not create DMA tag\n");
442 goto fail;
443 }
444
445 if (sec_reset(sc) != 0) {
446 device_printf(dev, "SEC reset failed\n");
447 goto fail;
448 }
449 if (sec_rng_init(sc) != 0) {
450 device_printf(dev, "SEC RNG instantiation failed\n");
451 goto fail;
452 }
453 if (sec_init_rings(sc) == 0) {
454 device_printf(dev, "SEC job ring init failed\n");
455 goto fail;
456 }
457
458 /*
459 * Clear any fault-address latch left over from the bootloader before
460 * enabling the error IRQ. FADR, FAR_HI/LO, and FALR must all be read
461 * before they're all cleared, per the RM.
462 */
463 (void)SEC_RD4(sc, SEC_FADR);
464 (void)SEC_RD4(sc, SEC_FAR_HI);
465 (void)SEC_RD4(sc, SEC_FAR_LO);
466 (void)SEC_RD4(sc, SEC_FALR);
467
468 if (bus_setup_intr(dev, sc->sc_ires, INTR_TYPE_MISC | INTR_MPSAFE,
469 NULL, sec_intr, sc, &sc->sc_icookie) != 0) {
470 device_printf(dev, "could not install error interrupt\n");
471 goto fail;
472 }
473
474 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session),
475 CRYPTOCAP_F_HARDWARE);
476 if (sc->sc_cid < 0) {
477 device_printf(dev, "could not get crypto driver id\n");
478 goto fail;
479 }
480
481 return (0);
482
483 fail:
484 sec_detach(dev);
485 return (ENXIO);
486 }
487
488 static int
sec_detach(device_t dev)489 sec_detach(device_t dev)
490 {
491 struct sec_softc *sc = device_get_softc(dev);
492 u_int i;
493
494 if (sc->sc_cid >= 0)
495 crypto_unregister_all(sc->sc_cid);
496
497 /* Silence the rings before halting them. */
498 for (i = 0; i < sc->sc_njr; i++) {
499 struct sec_jr *jr = &sc->sc_jr[i];
500
501 if (jr->jr_icookie != NULL)
502 bus_teardown_intr(dev, jr->jr_ires, jr->jr_icookie);
503 if (jr->jr_ires != NULL)
504 bus_release_resource(dev, SYS_RES_IRQ, jr->jr_irid,
505 jr->jr_ires);
506 sec_jr_teardown(sc, jr);
507 }
508 free(sc->sc_jr, M_SEC);
509
510 if (sc->sc_dmatag != NULL)
511 bus_dma_tag_destroy(sc->sc_dmatag);
512 if (sc->sc_icookie != NULL)
513 bus_teardown_intr(dev, sc->sc_ires, sc->sc_icookie);
514 if (sc->sc_ires != NULL)
515 bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irid,
516 sc->sc_ires);
517 if (sc->sc_rres != NULL)
518 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid,
519 sc->sc_rres);
520
521 return (0);
522 }
523
524 static const char *sec_ferr_str[] = {
525 "OKAY", "reserved", "SLVERR", "DECERR",
526 };
527
528 static const char *sec_jsrc_str[] = {
529 "JR0", "JR1", "JR2", "JR3", "RTIC", "QI", "rsvd6", "rsvd7",
530 };
531
532 static void
sec_intr(void * arg)533 sec_intr(void *arg)
534 {
535 struct sec_softc *sc = arg;
536 uint32_t fadr, falr;
537 uint64_t far;
538
539 fadr = SEC_RD4(sc, SEC_FADR);
540 if ((fadr & FADR_FERR_M) != 0) {
541 /*
542 * All fault registers are latched by hardware until all are
543 * read, in any order.
544 */
545 far = (uint64_t)SEC_RD4(sc, SEC_FAR_HI) << 32;
546 far |= SEC_RD4(sc, SEC_FAR_LO);
547 falr = SEC_RD4(sc, SEC_FALR);
548
549 device_printf(sc->sc_dev,
550 "bus fault: FADR=%#x FAR=%#jx FALR=%#x "
551 "(%s, %s, src=%s, blkid=%#x, %s, size=%u)\n",
552 fadr, (uintmax_t)far, falr,
553 sec_ferr_str[(fadr & FADR_FERR_M) >> FADR_FERR_S],
554 (fadr & FADR_DTYP) ? "control" : "message",
555 sec_jsrc_str[(fadr & FADR_JSRC_M) >> FADR_JSRC_S],
556 (fadr & FADR_BLKID_M) >> FADR_BLKID_S,
557 (fadr & FADR_TYP) ? "write" : "read",
558 (unsigned)(((fadr & FADR_FSZ_EXT_M) >>
559 (FADR_FSZ_EXT_S - 7)) | (fadr & FADR_FSZ_M)));
560 }
561
562 }
563
564 /*
565 * Decode a SEC job termination status word.
566 *
567 * Bits 0-3 (MSB) are the "source" of the report; the remaining bits are
568 * source-specific. Zero means clean completion.
569 *
570 * Two cases we care to distinguish:
571 * - CCB (source 2), ERRID field bits 28-31
572 * value 0xA is "ICV check failed" -> EBADMSG.
573 * - DECO (source 4), Error Code bits 24-31
574 * values F0h/F1h/FFh are informational warnings (IPsec TTL,
575 * 3GPP HFN, output-length rollover). The job actually completed,
576 * so map those to success.
577 *
578 * Everything else is logged and reported as EIO. Real per-code
579 * decoding of DECO/QI errors can be layered on as we hit them.
580 */
581 #define SEC_STAT_SOURCE(s) (((s) >> 28) & 0xf)
582 #define SEC_SRC_NONE 0x0
583 #define SEC_SRC_CCB 0x2
584 #define SEC_SRC_DECO 0x4
585 #define SEC_SRC_QI 0x5
586 #define SEC_SRC_JR 0x6
587 #define SEC_CCB_ERR_ICV_FAIL 0x0a
588 #define SEC_DECO_ERR_WARN_MIN 0xf0
589
590 static int
sec_decode_status(struct sec_softc * sc,uint32_t status)591 sec_decode_status(struct sec_softc *sc, uint32_t status)
592 {
593 uint32_t source;
594
595 if (status == 0)
596 return (0);
597
598 source = SEC_STAT_SOURCE(status);
599
600 switch (source) {
601 case SEC_SRC_CCB:
602 if ((status & 0xf) == SEC_CCB_ERR_ICV_FAIL)
603 return (EBADMSG);
604 break;
605 case SEC_SRC_DECO:
606 if ((status & 0xff) >= SEC_DECO_ERR_WARN_MIN)
607 return (0);
608 break;
609 }
610
611 device_printf(sc->sc_dev,
612 "job termination status %#x (source %#x)\n", status, source);
613 return (EIO);
614 }
615
616 /*
617 * Complete one job that SEC has finished processing.
618 */
619 void
sec_complete_one(struct sec_softc * sc,uint64_t desc_pa,uint32_t status)620 sec_complete_one(struct sec_softc *sc, uint64_t desc_pa, uint32_t status)
621 {
622 struct sec_job *job;
623 struct cryptop *crp;
624 const struct crypto_session_params *csp;
625 uint8_t expected[SEC_MAX_DIGEST];
626 int dlen;
627
628 job = (struct sec_job *)PHYS_TO_DMAP((vm_paddr_t)desc_pa);
629 crp = job->crp;
630
631 crp->crp_etype = sec_decode_status(sc, status);
632
633 bus_dmamap_sync(sc->sc_dmatag, job->map,
634 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
635 bus_dmamap_unload(sc->sc_dmatag, job->map);
636 bus_dmamap_destroy(sc->sc_dmatag, job->map);
637
638 if (crp->crp_etype == 0) {
639 csp = crypto_get_params(crp->crp_session);
640 dlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen :
641 job->sess->digestlen;
642 switch (csp->csp_mode) {
643 case CSP_MODE_DIGEST:
644 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
645 crypto_copydata(crp, crp->crp_digest_start,
646 dlen, expected);
647 if (timingsafe_bcmp(job->digest, expected,
648 dlen) != 0)
649 crp->crp_etype = EBADMSG;
650 } else {
651 crypto_copyback(crp, crp->crp_digest_start,
652 dlen, job->digest);
653 }
654 break;
655 case CSP_MODE_ETA:
656 if ((crp->crp_op & CRYPTO_OP_ENCRYPT) != 0) {
657 crypto_copyback(crp, crp->crp_digest_start,
658 dlen, job->digest);
659 break;
660 }
661 crypto_copydata(crp, crp->crp_digest_start, dlen,
662 expected);
663 if (timingsafe_bcmp(job->digest, expected, dlen) != 0)
664 crp->crp_etype = EBADMSG;
665 break;
666 case CSP_MODE_AEAD:
667 if ((crp->crp_op & CRYPTO_OP_ENCRYPT) != 0)
668 crypto_copyback(crp, crp->crp_digest_start,
669 dlen, job->digest);
670 break;
671 }
672 }
673
674 crypto_done(crp);
675 free(job, M_SEC);
676 }
677
678 static bool
check_cipher(const struct crypto_session_params * csp)679 check_cipher(const struct crypto_session_params *csp)
680 {
681
682 switch (csp->csp_cipher_alg) {
683 case CRYPTO_AES_CBC:
684 case CRYPTO_AES_ICM:
685 if (csp->csp_ivlen != AES_BLOCK_LEN)
686 return (false);
687 return (csp->csp_cipher_klen == 16 ||
688 csp->csp_cipher_klen == 24 ||
689 csp->csp_cipher_klen == 32);
690 case CRYPTO_AES_XTS:
691 if (csp->csp_ivlen != AES_XTS_IV_LEN)
692 return (false);
693 return (csp->csp_cipher_klen == 32 ||
694 csp->csp_cipher_klen == 64);
695 default:
696 return (false);
697 }
698 }
699
700 static bool
check_aead(const struct crypto_session_params * csp)701 check_aead(const struct crypto_session_params *csp)
702 {
703
704 switch (csp->csp_cipher_alg) {
705 case CRYPTO_AES_NIST_GCM_16:
706 if (csp->csp_auth_mlen != 0 &&
707 csp->csp_auth_mlen != AES_GMAC_HASH_LEN)
708 return (false);
709 return (csp->csp_cipher_klen == 16 ||
710 csp->csp_cipher_klen == 24 ||
711 csp->csp_cipher_klen == 32);
712 case CRYPTO_AES_CCM_16:
713 return (csp->csp_cipher_klen == 16 ||
714 csp->csp_cipher_klen == 24 ||
715 csp->csp_cipher_klen == 32);
716 default:
717 return (false);
718 }
719 }
720
721 /*
722 * Map an opencrypto auth_alg to its SEC selector and digest length.
723 * skeylen is zero for a plain hash, which is what tells the two apart.
724 */
725 static bool
sec_hash_params(int auth_alg,uint32_t * alg,uint8_t * dlen,uint8_t * skeylen)726 sec_hash_params(int auth_alg, uint32_t *alg, uint8_t *dlen, uint8_t *skeylen)
727 {
728
729 switch (auth_alg) {
730 case CRYPTO_SHA1_HMAC:
731 *alg = ALG_SHA1; *dlen = 20; *skeylen = 40; return (true);
732 case CRYPTO_SHA2_224_HMAC:
733 *alg = ALG_SHA224; *dlen = 28; *skeylen = 64; return (true);
734 case CRYPTO_SHA2_256_HMAC:
735 *alg = ALG_SHA256; *dlen = 32; *skeylen = 64; return (true);
736 case CRYPTO_SHA2_384_HMAC:
737 *alg = ALG_SHA384; *dlen = 48; *skeylen = 128; return (true);
738 case CRYPTO_SHA2_512_HMAC:
739 *alg = ALG_SHA512; *dlen = 64; *skeylen = 128; return (true);
740 case CRYPTO_SHA1:
741 *alg = ALG_SHA1; *dlen = 20; *skeylen = 0; return (true);
742 case CRYPTO_SHA2_224:
743 *alg = ALG_SHA224; *dlen = 28; *skeylen = 0; return (true);
744 case CRYPTO_SHA2_256:
745 *alg = ALG_SHA256; *dlen = 32; *skeylen = 0; return (true);
746 case CRYPTO_SHA2_384:
747 *alg = ALG_SHA384; *dlen = 48; *skeylen = 0; return (true);
748 case CRYPTO_SHA2_512:
749 *alg = ALG_SHA512; *dlen = 64; *skeylen = 0; return (true);
750 }
751 return (false);
752 }
753
754 static bool
check_digest(const struct crypto_session_params * csp)755 check_digest(const struct crypto_session_params *csp)
756 {
757 uint32_t alg;
758 uint8_t dlen, skeylen;
759
760 /* GMAC is AESA rather than MDHA, so it has its own constraints. */
761 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
762 if (csp->csp_ivlen != AES_GCM_IV_LEN)
763 return (false);
764 if (csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
765 return (false);
766 return (csp->csp_auth_klen == 16 ||
767 csp->csp_auth_klen == 24 ||
768 csp->csp_auth_klen == 32);
769 }
770
771 if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen))
772 return (false);
773 /* Keyed variants require a key; plain hashes must not carry one. */
774 if ((skeylen != 0) != (csp->csp_auth_klen != 0))
775 return (false);
776 return (csp->csp_auth_mlen <= dlen);
777 }
778
779 static bool
check_eta(const struct crypto_session_params * csp)780 check_eta(const struct crypto_session_params *csp)
781 {
782
783 /*
784 * ESN appends four bytes from crp_esn to the MAC input, which the
785 * descriptor has no way to splice in, so refuse rather than
786 * authenticate the wrong span.
787 */
788 if ((csp->csp_flags & CSP_F_ESN) != 0)
789 return (false);
790 /*
791 * XTS carries its tweak in the class 1 context and pairs with no
792 * MAC; its shared descriptor is shaped differently.
793 */
794 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
795 return (false);
796 /* The MAC half has to be keyed; a bare hash authenticates nothing. */
797 if (csp->csp_auth_klen == 0)
798 return (false);
799 return (check_cipher(csp) && check_digest(csp));
800 }
801
802 /*
803 * Software split-key generator: computes the HMAC ipad/opad hash-state
804 * halves in software and packs them big-endian for SEC's Class 2 KEY
805 * register.
806 *
807 * Runs the CPU through one SHA block per pad (two total). Much cheaper than
808 * the round trip through the job ring for setup.
809 */
810 static void
sec_pack_state32(uint8_t * dst,const uint32_t * src,unsigned int nbytes)811 sec_pack_state32(uint8_t *dst, const uint32_t *src, unsigned int nbytes)
812 {
813 unsigned int i;
814
815 for (i = 0; i < nbytes; i += 4)
816 be32enc(dst + i, src[i / 4]);
817 }
818
819 static void
sec_pack_state64(uint8_t * dst,const uint64_t * src,unsigned int nbytes)820 sec_pack_state64(uint8_t *dst, const uint64_t *src, unsigned int nbytes)
821 {
822 unsigned int i;
823
824 for (i = 0; i < nbytes; i += 8)
825 be64enc(dst + i, src[i / 8]);
826 }
827
828 static void
sec_sw_gen_split_key(const struct crypto_session_params * csp,uint8_t * out,size_t out_len)829 sec_sw_gen_split_key(const struct crypto_session_params *csp,
830 uint8_t *out, size_t out_len)
831 {
832 union authctx ictx, octx;
833 const struct auth_hash *axf;
834 uint8_t half;
835
836 axf = crypto_auth_hash(csp);
837 hmac_init_ipad(axf, csp->csp_auth_key, csp->csp_auth_klen, &ictx);
838 hmac_init_opad(axf, csp->csp_auth_key, csp->csp_auth_klen, &octx);
839
840 KASSERT(out_len % 2 == 0, ("split key len must be even"));
841 half = out_len / 2;
842
843 switch (csp->csp_auth_alg) {
844 case CRYPTO_SHA1_HMAC:
845 sec_pack_state32(out, ictx.sha1ctx.h.b32, half);
846 sec_pack_state32(out + half, octx.sha1ctx.h.b32, half);
847 break;
848 case CRYPTO_SHA2_224_HMAC:
849 sec_pack_state32(out, ictx.sha224ctx.state, half);
850 sec_pack_state32(out + half, octx.sha224ctx.state, half);
851 break;
852 case CRYPTO_SHA2_256_HMAC:
853 sec_pack_state32(out, ictx.sha256ctx.state, half);
854 sec_pack_state32(out + half, octx.sha256ctx.state, half);
855 break;
856 case CRYPTO_SHA2_384_HMAC:
857 sec_pack_state64(out, ictx.sha384ctx.state, half);
858 sec_pack_state64(out + half, octx.sha384ctx.state, half);
859 break;
860 case CRYPTO_SHA2_512_HMAC:
861 sec_pack_state64(out, ictx.sha512ctx.state, half);
862 sec_pack_state64(out + half, octx.sha512ctx.state, half);
863 break;
864 }
865
866 explicit_bzero(&ictx, sizeof(ictx));
867 explicit_bzero(&octx, sizeof(octx));
868 }
869
870 /*
871 * Descriptor builder. Word 0 is the HEADER and is filled in last, since its
872 * length field is only known once the body has been emitted.
873 */
874 struct sec_desc_builder {
875 uint32_t *desc;
876 unsigned int idx; /* next word to write */
877 unsigned int max;
878 int err;
879 };
880
881 static inline void
sec_desc_init(struct sec_desc_builder * b,uint32_t * desc,unsigned int max)882 sec_desc_init(struct sec_desc_builder *b, uint32_t *desc, unsigned int max)
883 {
884
885 b->desc = desc;
886 b->idx = 1; /* reserve word 0 for the HEADER */
887 b->max = max;
888 b->err = 0;
889 }
890
891 static inline void
sec_desc_word(struct sec_desc_builder * b,uint32_t w)892 sec_desc_word(struct sec_desc_builder *b, uint32_t w)
893 {
894
895 if (b->err != 0)
896 return;
897 if (b->idx >= b->max) {
898 b->err = ENOSPC;
899 return;
900 }
901 b->desc[b->idx++] = w;
902 }
903
904 /* Emit a KEY command with the key inline after it. */
905 static inline void
sec_desc_key_imm(struct sec_desc_builder * b,uint32_t class,const void * key,unsigned int keylen)906 sec_desc_key_imm(struct sec_desc_builder *b, uint32_t class,
907 const void *key, unsigned int keylen)
908 {
909 unsigned int nwords = howmany(keylen, sizeof(uint32_t));
910
911 if (b->err != 0)
912 return;
913 if (b->idx + 1 + nwords > b->max) {
914 b->err = ENOSPC;
915 return;
916 }
917 b->desc[b->idx++] = CMD_DESC(CMD_KEY) | class | KEY_IMM |
918 (keylen & KEY_LENGTH_M);
919 memcpy(&b->desc[b->idx], key, keylen);
920 b->idx += nwords;
921 }
922
923 static int
sec_desc_finalize_shared(struct sec_desc_builder * b,uint32_t flags,uint32_t * sdlenp)924 sec_desc_finalize_shared(struct sec_desc_builder *b, uint32_t flags,
925 uint32_t *sdlenp)
926 {
927
928 if (b->err != 0)
929 return (b->err);
930 if (b->idx > SEC_MAX_SHDESC_WORDS)
931 return (ENOSPC);
932 b->desc[0] = CMD_DESC(CMD_SHARED_HEADER) | HEADER_ONE |
933 (flags & (HEADER_SHARE_M | HEADER_SC)) |
934 (b->idx & HEADER_DESCLEN_M);
935 *sdlenp = b->idx;
936 return (0);
937 }
938
939 static int
sec_desc_finalize_job(struct sec_desc_builder * b,uint32_t word,uint32_t * dlenp)940 sec_desc_finalize_job(struct sec_desc_builder *b, uint32_t word,
941 uint32_t *dlenp)
942 {
943
944 if (b->err != 0)
945 return (b->err);
946 if (b->idx > SEC_MAX_DESC_WORDS)
947 return (ENOSPC);
948 b->desc[0] = CMD_DESC(CMD_DESC_HEADER) | HEADER_ONE |
949 word | (b->idx & HEADER_DESCLEN_M);
950 *dlenp = b->idx;
951 return (0);
952 }
953
954
955 /*
956 * Job descriptor builder conveniences.
957 */
958
959 static inline void
sec_jd_ptr(struct sec_desc_builder * b,vm_paddr_t pa)960 sec_jd_ptr(struct sec_desc_builder *b, vm_paddr_t pa)
961 {
962 sec_desc_word(b, (uint32_t)(pa >> 32));
963 sec_desc_word(b, (uint32_t)pa);
964 }
965
966 /* Build a SEQ_IN/SEQ_OUT descriptor command. */
967 static inline void
sec_jd_seq(struct sec_desc_builder * b,bool inout,uint32_t flags,vm_paddr_t ptr,uint32_t len)968 sec_jd_seq(struct sec_desc_builder *b, bool inout, uint32_t flags,
969 vm_paddr_t ptr, uint32_t len)
970 {
971 sec_desc_word(b,
972 CMD_DESC(inout ? CMD_SEQ_OUT_PTR : CMD_SEQ_IN_PTR) | flags);
973 sec_jd_ptr(b, ptr);
974 sec_desc_word(b, len);
975 }
976
977 static inline void
sec_jd_load(struct sec_desc_builder * b,bool seq,uint32_t class,uint32_t flags,uint32_t dst,uint32_t off,uint32_t len,vm_paddr_t ptr)978 sec_jd_load(struct sec_desc_builder *b, bool seq, uint32_t class,
979 uint32_t flags, uint32_t dst, uint32_t off, uint32_t len, vm_paddr_t ptr)
980 {
981 uint32_t cmd = seq ? CMD_SEQ_LOAD : CMD_LOAD;
982
983 sec_desc_word(b, CMD_DESC(cmd) | class | flags | dst |
984 (off << LOAD_OFFSET_S) | (len & LOAD_LENGTH_M));
985 if (!seq)
986 sec_jd_ptr(b, ptr);
987 }
988
989 static inline void
sec_jd_store(struct sec_desc_builder * b,bool seq,uint32_t class,uint32_t src,uint32_t off,uint32_t len,vm_paddr_t ptr)990 sec_jd_store(struct sec_desc_builder *b, bool seq, uint32_t class, uint32_t src,
991 uint32_t off, uint32_t len, vm_paddr_t ptr)
992 {
993 uint32_t cmd = seq ? CMD_SEQ_STORE : CMD_STORE;
994
995 sec_desc_word(b, CMD_DESC(cmd) | class | src |
996 (off << LOAD_OFFSET_S) | (len & LOAD_LENGTH_M));
997 if (!seq)
998 sec_jd_ptr(b, ptr);
999 }
1000
1001 static inline void
sec_jd_fifo(struct sec_desc_builder * b,uint32_t cmd,uint32_t len)1002 sec_jd_fifo(struct sec_desc_builder *b, uint32_t cmd, uint32_t len)
1003 {
1004
1005 if (len > 0xffff) {
1006 sec_desc_word(b, cmd | FIFO_EXT);
1007 sec_desc_word(b, len);
1008 } else {
1009 sec_desc_word(b, cmd | len);
1010 }
1011 }
1012
1013 /*
1014 * AES-XTS Class 1 context layout (byte offsets into the CTX register).
1015 * The 16-byte tweak is split either side of the sector-size field.
1016 */
1017 #define SEC_XTS_CTX_TWEAK_LO 0x20
1018 #define SEC_XTS_CTX_SECTOR 0x28
1019 #define SEC_XTS_CTX_TWEAK_HI 0x30
1020
1021 /*
1022 * Sector size tells the hardware how often to re-derive the tweak.
1023 * opencrypto's XTS runs one continuous tweak over the whole request, so
1024 * this only needs to exceed any payload we accept; sec_jd_build_cipher
1025 * rejects requests that would cross the boundary.
1026 */
1027 #define SEC_XTS_SECTOR_SIZE 0x8000
1028
1029 /*
1030 * Build the CCM context block and formatted-AAD length prefix.
1031 *
1032 * The hardware wants B0 in context dwords 0-1 and the initial counter
1033 * CTR0 in dwords 2-3, with dwords 4-6 zeroed because AS is
1034 * INITIALIZE/FINALIZE. Both blocks are laid out per RFC 3610: with a
1035 * nonce of n bytes, the length field occupies the trailing L = 15 - n
1036 * bytes and the flags byte carries L-1 plus, for B0, the encoded tag
1037 * size and an AAD-present flag.
1038 *
1039 * The AAD itself is prefixed with its length and then zero-padded to a
1040 * 16-byte boundary by the hardware, which pads AAD and IV FIFO loads
1041 * when the flush-class-1 bit is set.
1042 */
1043 static int
sec_ccm_prep(struct sec_job * job,const struct crypto_session_params * csp)1044 sec_ccm_prep(struct sec_job *job, const struct crypto_session_params *csp)
1045 {
1046 uint8_t *b0 = job->ccm_ctx;
1047 uint8_t *ctr0 = job->ccm_ctx + 16;
1048 uint32_t aadlen = job->crp->crp_aad_length;
1049 uint64_t paylen = job->crp->crp_payload_length;
1050 u_int i, lfield = 15 - csp->csp_ivlen;
1051
1052 if (aadlen > SEC_CCM_AAD_MAX)
1053 return (EOPNOTSUPP);
1054 /*
1055 * B0 carries the payload length in its trailing lfield bytes, so
1056 * the nonce is what really caps the payload: a 13-byte nonce
1057 * leaves two bytes and stops at 64 KB, while the usual 12-byte one
1058 * leaves three and reaches 16 MB.
1059 */
1060 if (lfield < sizeof(paylen) && paylen >= (uint64_t)1 << (8 * lfield))
1061 return (EOPNOTSUPP);
1062
1063 memset(job->ccm_ctx, 0, sizeof(job->ccm_ctx));
1064
1065 b0[0] = (aadlen > 0 ? 0x40 : 0x00) |
1066 (((job->sess->digestlen - 2) / 2) << 3) | (lfield - 1);
1067 memcpy(b0 + 1, job->iv, csp->csp_ivlen);
1068 for (i = 0; i < lfield; i++)
1069 b0[15 - i] = (paylen >> (8 * i)) & 0xff;
1070
1071 ctr0[0] = lfield - 1;
1072 memcpy(ctr0 + 1, job->iv, csp->csp_ivlen);
1073
1074 be16enc(job->ccm_alen, aadlen);
1075 return (0);
1076 }
1077
1078 /*
1079 * Bytes of IV the input sequence carries. XTS is the odd one out:
1080 * opencrypto's IV is just the 8-byte block number, but the hardware
1081 * loads both halves of the 16-byte tweak from the sequence.
1082 */
1083 static uint32_t
sec_cipher_ivlen(const struct crypto_session_params * csp)1084 sec_cipher_ivlen(const struct crypto_session_params *csp)
1085 {
1086
1087 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
1088 return (AES_BLOCK_LEN);
1089 return (csp->csp_ivlen);
1090 }
1091
1092 /*
1093 * Expand opencrypto's 8-byte XTS IV in place into the 16-byte tweak the
1094 * hardware expects. The IV holds a block number in host order
1095 * (xform_aes_xts.c:aes_xts_reinit) and the tweak is that number's
1096 * little-endian encoding followed by zeroes.
1097 */
1098 static void
sec_xts_tweak(uint8_t * iv)1099 sec_xts_tweak(uint8_t *iv)
1100 {
1101 uint64_t blocknum;
1102
1103 memcpy(&blocknum, iv, sizeof(blocknum));
1104 le64enc(iv, blocknum);
1105 memset(iv + sizeof(blocknum), 0, AES_BLOCK_LEN - sizeof(blocknum));
1106 }
1107
1108 static uint32_t
sec_cipher_ctx_offset(uint32_t cipher_alg)1109 sec_cipher_ctx_offset(uint32_t cipher_alg)
1110 {
1111 switch (cipher_alg) {
1112 case CRYPTO_AES_ICM:
1113 return (16);
1114 };
1115
1116 return (0);
1117 }
1118
1119 /* Per-mode shared-descriptor builders. */
1120 /*
1121 * Cipher shared descriptor has the following format:
1122 * [0] - Header
1123 * [1..klen] - KEY descriptor + key
1124 * [XTS:..5] -- XTS specific
1125 * [0..2] - LOAD XTS context
1126 * [3..4] - LOAD XTS tweak
1127 * [!XTS:1] -- Load IV into Context register
1128 * [] - Operation
1129 * [] - MATH - Move SIL register to VSIL for FIFO IN
1130 * [] - MATH - Move SOL register to VSOL for FIFO OUT
1131 * [] - FIFO LOAD
1132 * [] - FIFO STORE
1133 */
1134 static int
sec_shd_build_cipher(struct sec_session * sess,const struct crypto_session_params * csp,int enc,uint32_t * sdlenp)1135 sec_shd_build_cipher(struct sec_session *sess,
1136 const struct crypto_session_params *csp, int enc, uint32_t *sdlenp)
1137 {
1138 struct sec_desc_builder b;
1139 uint32_t flags, op;
1140 uint32_t ctx_offset;
1141
1142 switch (csp->csp_cipher_alg) {
1143 case CRYPTO_AES_CBC:
1144 op = CMD_DESC(CMD_OPERATION) | ALG_AES |
1145 AAI_AES_CBC | AS_INIT_FINAL;
1146 break;
1147 case CRYPTO_AES_ICM:
1148 op = CMD_DESC(CMD_OPERATION) | ALG_AES |
1149 AAI_AES_CTR | AS_INIT_FINAL;
1150 break;
1151 case CRYPTO_AES_XTS:
1152 op = CMD_DESC(CMD_OPERATION) | ALG_AES |
1153 AAI_AES_XTS | AS_INIT_FINAL;
1154 break;
1155 default:
1156 return (EOPNOTSUPP);
1157 }
1158
1159 ctx_offset = sec_cipher_ctx_offset(csp->csp_cipher_alg);
1160
1161 if (enc)
1162 op |= OP_ENC;
1163
1164 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
1165
1166 if (csp->csp_cipher_klen > 0)
1167 sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_cipher_key,
1168 csp->csp_cipher_klen);
1169
1170 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) {
1171 sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_CTX,
1172 SEC_XTS_CTX_SECTOR, 8, SEC_XTS_SECTOR_SIZE);
1173
1174 sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX,
1175 SEC_XTS_CTX_TWEAK_LO, 8, 0);
1176 sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX,
1177 SEC_XTS_CTX_TWEAK_HI, 8, 0);
1178 } else {
1179 sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX, ctx_offset,
1180 csp->csp_ivlen, 0);
1181 }
1182
1183 sec_desc_word(&b, op);
1184
1185 /*
1186 * Copy SIL into VSIL and VSOL so the following VLF-flagged FIFO
1187 * commands know how many bytes to move. VLF reads the VS*L
1188 * registers, so we need to get the values from the SEQ registers
1189 * the SEQ IN/OUT PTR descriptors populate.
1190 */
1191 sec_desc_word(&b, CMD_DESC(CMD_MATH) | MATH_FN_ADD | MATH_SRC0_SIL |
1192 MATH_SRC1_ZERO | MATH_DEST_VSIL | MATH_LEN_4);
1193 sec_desc_word(&b, CMD_DESC(CMD_MATH) | MATH_FN_ADD | MATH_SRC0_SIL |
1194 MATH_SRC1_ZERO | MATH_DEST_VSOL | MATH_LEN_4);
1195
1196 sec_desc_word(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
1197 FIFOLD_VLF | FIFOLD_TYPE_MSG | FIFOLD_LC1);
1198
1199 sec_desc_word(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) | FIFOST_VLF |
1200 FIFOST_TYPE_MSG_DATA);
1201
1202 /* XTS keeps its tweak in the context, so the CCB has to save it. */
1203 flags = HEADER_SHARE_SERIAL;
1204 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
1205 flags |= HEADER_SC;
1206
1207 return (sec_desc_finalize_shared(&b, flags, sdlenp));
1208 }
1209
1210 /*
1211 * Digest shared descriptor. The split key is computed in software at
1212 * session setup, so MDHA is told it is precomputed and skips the
1213 * ipad/opad expansion.
1214 */
1215 static int
sec_shd_build_digest(struct sec_session * sess,const struct crypto_session_params * csp,int enc,uint32_t * sdlenp)1216 sec_shd_build_digest(struct sec_session *sess,
1217 const struct crypto_session_params *csp, int enc, uint32_t *sdlenp)
1218 {
1219 struct sec_desc_builder b;
1220 uint32_t alg;
1221 uint8_t dlen, skeylen;
1222
1223 /*
1224 * GMAC runs on AESA, not MDHA: the shared descriptor is just the
1225 * class 1 key, and the JD drives it as GCM with no message.
1226 */
1227 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
1228 sess->digestlen = csp->csp_auth_mlen != 0 ?
1229 csp->csp_auth_mlen : AES_GMAC_HASH_LEN;
1230 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
1231 sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_auth_key,
1232 csp->csp_auth_klen);
1233 return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL,
1234 sdlenp));
1235 }
1236
1237 if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen))
1238 return (EOPNOTSUPP);
1239 sess->digestlen = dlen;
1240
1241 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
1242
1243 /*
1244 * A plain hash takes no key at all; the keyed variants load the
1245 * precomputed ipad || opad blob as an MDHA split key, which is
1246 * what AAI_HMAC_PRECOMP tells MDHA to expect.
1247 */
1248 if (skeylen != 0) {
1249 unsigned int nwords = howmany(skeylen, 4);
1250
1251 b.desc[b.idx++] = CMD_DESC(CMD_KEY) | KEY_CLASS_2 |
1252 KEY_KDEST_MDHA_SPLIT | KEY_IMM |
1253 (skeylen & KEY_LENGTH_M);
1254 memcpy(&b.desc[b.idx], sess->skey, skeylen);
1255 if (skeylen % 4 != 0)
1256 memset((uint8_t *)&b.desc[b.idx] + skeylen, 0,
1257 nwords * 4 - skeylen);
1258 b.idx += nwords;
1259 }
1260
1261 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | alg |
1262 (skeylen != 0 ? AAI_HMAC_PRECOMP : AAI_HASH) | AS_INIT_FINAL);
1263
1264 /* VLF FIFO_LOAD needs VSIL, which SEQ_IN_PTR doesn't populate. */
1265 sec_desc_word(&b, CMD_DESC(CMD_MATH) | MATH_FN_ADD | MATH_SRC0_SIL |
1266 MATH_SRC1_ZERO | MATH_DEST_VSIL | MATH_LEN_4);
1267
1268 sec_desc_word(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_2 |
1269 FIFOLD_VLF | FIFOLD_TYPE_MSG | FIFOLD_LC2);
1270
1271 /*
1272 * Drain the completed hash from the Class 2 CCB Context register.
1273 * SEQ_STORE with class 2 + SRC=CTX (0x20) blocks until MDHA is
1274 * done.
1275 */
1276 sec_jd_store(&b, true, LOAD_CLASS_2, LOAD_CTX, 0, dlen, 0);
1277
1278 return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, sdlenp));
1279 }
1280
1281 /*
1282 * AEAD shared descriptor. AAD and payload lengths vary per job, so
1283 * everything but the key lives in the JD. Execution order is not
1284 * reversed here: the key has to be loaded before the JD drives data.
1285 *
1286 * When ICV is set the ENC bit must be clear, which is the only
1287 * difference between the two direction slots.
1288 */
1289 static int
sec_shd_build_aead(struct sec_session * sess,const struct crypto_session_params * csp,int enc,uint32_t * sdlenp)1290 sec_shd_build_aead(struct sec_session *sess,
1291 const struct crypto_session_params *csp, int enc, uint32_t *sdlenp)
1292 {
1293 struct sec_desc_builder b;
1294
1295 switch (csp->csp_cipher_alg) {
1296 case CRYPTO_AES_NIST_GCM_16:
1297 case CRYPTO_AES_CCM_16:
1298 break;
1299 default:
1300 return (EOPNOTSUPP);
1301 }
1302 (void)enc;
1303
1304 /* Cache tag length once (both directions share). */
1305 sess->digestlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen : 16;
1306
1307 /*
1308 * SHD holds just the AES key.
1309 */
1310 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
1311 if (csp->csp_cipher_klen > 0)
1312 sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_cipher_key,
1313 csp->csp_cipher_klen);
1314 return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, sdlenp));
1315 }
1316
1317 /*
1318 * Shared descriptor for encrypt-then-auth: both keys and both mode
1319 * registers, nothing else.
1320 */
1321 static int
sec_shd_build_eta(struct sec_session * sess,const struct crypto_session_params * csp,int enc,uint32_t * sdlenp)1322 sec_shd_build_eta(struct sec_session *sess,
1323 const struct crypto_session_params *csp, int enc, uint32_t *sdlenp)
1324 {
1325 struct sec_desc_builder b;
1326 uint32_t alg, op;
1327 uint8_t dlen, skeylen;
1328
1329 if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen))
1330 return (EOPNOTSUPP);
1331 sess->digestlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen : dlen;
1332
1333 switch (csp->csp_cipher_alg) {
1334 case CRYPTO_AES_CBC:
1335 op = CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_CBC |
1336 AS_INIT_FINAL;
1337 break;
1338 case CRYPTO_AES_ICM:
1339 op = CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_CTR |
1340 AS_INIT_FINAL;
1341 break;
1342 default:
1343 return (EOPNOTSUPP);
1344 }
1345 if (enc)
1346 op |= OP_ENC;
1347
1348 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
1349
1350 sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_cipher_key,
1351 csp->csp_cipher_klen);
1352
1353 /* Class 2 takes the precomputed ipad/opad blob, as for plain HMAC. */
1354 {
1355 unsigned int nwords = howmany(skeylen, 4);
1356
1357 b.desc[b.idx++] = CMD_DESC(CMD_KEY) | KEY_CLASS_2 |
1358 KEY_KDEST_MDHA_SPLIT | KEY_IMM | (skeylen & KEY_LENGTH_M);
1359 memcpy(&b.desc[b.idx], sess->skey, skeylen);
1360 if (skeylen % 4 != 0)
1361 memset((uint8_t *)&b.desc[b.idx] + skeylen, 0,
1362 nwords * 4 - skeylen);
1363 b.idx += nwords;
1364 }
1365
1366 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | alg | AAI_HMAC_PRECOMP |
1367 AS_INIT_FINAL);
1368 sec_desc_word(&b, op);
1369
1370 return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, sdlenp));
1371 }
1372
1373 static int
sec_probe_session(device_t dev,const struct crypto_session_params * csp)1374 sec_probe_session(device_t dev, const struct crypto_session_params *csp)
1375 {
1376
1377 switch (csp->csp_mode) {
1378 case CSP_MODE_CIPHER:
1379 if (!check_cipher(csp))
1380 return (EINVAL);
1381 break;
1382 case CSP_MODE_DIGEST:
1383 if (!check_digest(csp))
1384 return (EINVAL);
1385 break;
1386 case CSP_MODE_AEAD:
1387 if (!check_aead(csp))
1388 return (EINVAL);
1389 break;
1390 case CSP_MODE_ETA:
1391 if (!check_eta(csp))
1392 return (EINVAL);
1393 break;
1394 default:
1395 return (EINVAL);
1396 }
1397 return (CRYPTODEV_PROBE_HARDWARE);
1398 }
1399
1400 static int
sec_new_session(device_t dev,crypto_session_t session,const struct crypto_session_params * csp)1401 sec_new_session(device_t dev, crypto_session_t session,
1402 const struct crypto_session_params *csp)
1403 {
1404 struct sec_softc *sc = device_get_softc(dev);
1405 struct sec_session *sess;
1406 uint32_t sdlen;
1407 int enc, error;
1408
1409 sess = crypto_get_driver_session(session);
1410 sess->sess_sc = sc;
1411
1412 if ((csp->csp_mode == CSP_MODE_DIGEST ||
1413 csp->csp_mode == CSP_MODE_ETA) && csp->csp_auth_klen > 0 &&
1414 csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) {
1415 uint32_t alg;
1416 uint8_t dlen, skeylen;
1417
1418 if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen))
1419 return (EOPNOTSUPP);
1420 (void)alg;
1421 sec_sw_gen_split_key(csp, sess->skey, skeylen);
1422 sess->skeylen = skeylen;
1423 }
1424
1425 for (enc = 0; enc <= 1; enc++) {
1426 switch (csp->csp_mode) {
1427 case CSP_MODE_CIPHER:
1428 error = sec_shd_build_cipher(sess, csp, enc, &sdlen);
1429 break;
1430 case CSP_MODE_DIGEST:
1431 error = sec_shd_build_digest(sess, csp, enc, &sdlen);
1432 break;
1433 case CSP_MODE_AEAD:
1434 error = sec_shd_build_aead(sess, csp, enc, &sdlen);
1435 break;
1436 case CSP_MODE_ETA:
1437 error = sec_shd_build_eta(sess, csp, enc, &sdlen);
1438 break;
1439 default:
1440 return (EINVAL);
1441 }
1442 if (error != 0)
1443 return (error);
1444 sess->sdlen[enc] = sdlen;
1445 }
1446 return (0);
1447 }
1448
1449 static void
sec_free_session(device_t dev,crypto_session_t session)1450 sec_free_session(device_t dev, crypto_session_t session)
1451 {
1452 /* Nothing to do here. */
1453 }
1454
1455 static void
sec_load_cb(void * arg,bus_dma_segment_t * segs,int nsegs,int error)1456 sec_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
1457 {
1458 struct sec_job *job = arg;
1459
1460 if (error != 0) {
1461 job->nsegs = 0;
1462 return;
1463 }
1464 KASSERT(nsegs <= SEC_MAX_SEGMENTS,
1465 ("SEC job segment overflow: %d > %d", nsegs, SEC_MAX_SEGMENTS));
1466 memcpy(job->segs, segs, nsegs * sizeof(segs[0]));
1467 job->nsegs = nsegs;
1468 }
1469
1470 /*
1471 * Append the segments covering [start, start + len) of the mapped buffer.
1472 * Returns the next free index, or -1 if the table would overflow or the
1473 * range runs past the mapping.
1474 */
1475 static int
dpaa_sgte_append(struct sec_job * job,struct dpaa_sgte * sgt,int i,int max,uint32_t start,uint32_t len)1476 dpaa_sgte_append(struct sec_job *job, struct dpaa_sgte *sgt, int i, int max,
1477 uint32_t start, uint32_t len)
1478 {
1479 int s;
1480
1481 for (s = 0; s < job->nsegs && len > 0; s++) {
1482 bus_addr_t addr = job->segs[s].ds_addr;
1483 bus_size_t seglen = job->segs[s].ds_len;
1484 uint32_t take;
1485
1486 if (start >= seglen) {
1487 start -= seglen;
1488 continue;
1489 }
1490 addr += start;
1491 seglen -= start;
1492 start = 0;
1493
1494 take = seglen > len ? len : seglen;
1495 len -= take;
1496
1497 if (i >= max)
1498 return (-1);
1499 sgt[i].addr = addr;
1500 sgt[i].extension = 0;
1501 sgt[i].final = 0;
1502 sgt[i].length = take;
1503 sgt[i].bpid = 0;
1504 sgt[i].offset = 0;
1505 i++;
1506 }
1507 if (len != 0)
1508 return (-1);
1509 return (i);
1510 }
1511
1512 /*
1513 * Populate the SGTs from the DMA-loaded segment list. Entry order is
1514 * what the descriptor's SEQ commands consume, so it is fixed per mode:
1515 * CIPHER:
1516 * in_sgt[0] = job->iv
1517 * in_sgt[1..n] = payload segments
1518 * out_sgt[0..n-1] = payload segments (in-place)
1519 * DIGEST (HMAC, no IV):
1520 * in_sgt[0..n-1] = payload segments
1521 * out_sgt unused (JD points SEQ_OUT_PTR directly at job->digest).
1522 * AEAD and ETA:
1523 * in_sgt[0] = job->iv, or job->ccm_ctx for CCM
1524 * in_sgt[1..A] = AAD (crp_aad, or crp_buf at crp_aad_start)
1525 * in_sgt[A+1..N] = payload segments
1526 * in_sgt[N+1..] = (decrypt only) received tag from crp_buf
1527 * out_sgt[0..] = payload segments (in-place); the tag goes to
1528 * job->digest via a separate STORE.
1529 * The final SGT entry in each populated table gets F=1.
1530 */
1531 static int
sec_job_build_sgts(struct sec_job * job,const struct crypto_session_params * csp)1532 sec_job_build_sgts(struct sec_job *job, const struct crypto_session_params *csp)
1533 {
1534 struct cryptop *crp = job->crp;
1535 uint32_t skip = crp->crp_payload_start;
1536 uint32_t left = crp->crp_payload_length;
1537 int i, out_i;
1538 int iv_slot = csp->csp_ivlen > 0 ? 1 : 0;
1539
1540 if (csp->csp_mode == CSP_MODE_AEAD ||
1541 csp->csp_mode == CSP_MODE_ETA) {
1542 const int inmax = 1 + SEC_MAX_SEGMENTS;
1543 bool encrypt = (crp->crp_op & CRYPTO_OP_ENCRYPT) != 0;
1544 bool ccm = csp->csp_mode == CSP_MODE_AEAD &&
1545 csp->csp_cipher_alg == CRYPTO_AES_CCM_16;
1546 int in_i = 0, npay, pay_i;
1547
1548 /* IV, or for CCM the B0 || CTR0 context block. */
1549 if (ccm) {
1550 job->in_sgt[in_i].addr =
1551 pmap_kextract((vm_offset_t)job->ccm_ctx);
1552 job->in_sgt[in_i].length = SEC_CCM_CTX_LEN;
1553 } else {
1554 job->in_sgt[in_i].addr =
1555 pmap_kextract((vm_offset_t)job->iv);
1556 job->in_sgt[in_i].length = csp->csp_ivlen;
1557 }
1558 job->in_sgt[in_i].extension = 0;
1559 job->in_sgt[in_i].final = 0;
1560 job->in_sgt[in_i].bpid = 0;
1561 job->in_sgt[in_i].offset = 0;
1562 in_i++;
1563
1564 if (crp->crp_aad_length > 0) {
1565 /* CCM feeds the AAD length ahead of the AAD. */
1566 if (ccm) {
1567 job->in_sgt[in_i].addr = pmap_kextract(
1568 (vm_offset_t)job->ccm_alen);
1569 job->in_sgt[in_i].extension = 0;
1570 job->in_sgt[in_i].final = 0;
1571 job->in_sgt[in_i].length =
1572 sizeof(job->ccm_alen);
1573 job->in_sgt[in_i].bpid = 0;
1574 job->in_sgt[in_i].offset = 0;
1575 in_i++;
1576 }
1577 if (crp->crp_aad != NULL) {
1578 /*
1579 * A dedicated AAD buffer is not part of the
1580 * crp mapping; it is small enough that one
1581 * entry always covers it.
1582 */
1583 job->in_sgt[in_i].addr = pmap_kextract(
1584 (vm_offset_t)crp->crp_aad);
1585 job->in_sgt[in_i].extension = 0;
1586 job->in_sgt[in_i].final = 0;
1587 job->in_sgt[in_i].length = crp->crp_aad_length;
1588 job->in_sgt[in_i].bpid = 0;
1589 job->in_sgt[in_i].offset = 0;
1590 in_i++;
1591 } else {
1592 in_i = dpaa_sgte_append(job, job->in_sgt, in_i,
1593 inmax, crp->crp_aad_start,
1594 crp->crp_aad_length);
1595 if (in_i < 0)
1596 return (E2BIG);
1597 }
1598 }
1599
1600 pay_i = in_i;
1601 in_i = dpaa_sgte_append(job, job->in_sgt, in_i, inmax, skip,
1602 left);
1603 if (in_i < 0)
1604 return (E2BIG);
1605 npay = in_i - pay_i;
1606 if (npay == 0)
1607 return (EINVAL);
1608
1609 /*
1610 * AEAD decrypt hands the received tag to the CHA for its
1611 * own compare; ETA drains the MAC to job->digest instead
1612 * and compares in software, so it needs no entry here.
1613 */
1614 if (!encrypt && csp->csp_mode == CSP_MODE_AEAD) {
1615 in_i = dpaa_sgte_append(job, job->in_sgt, in_i, inmax,
1616 crp->crp_digest_start, job->sess->digestlen);
1617 if (in_i < 0)
1618 return (E2BIG);
1619 }
1620 job->in_sgt[in_i - 1].final = 1;
1621
1622 /* Output mirrors the payload segments, in place. */
1623 memcpy(job->out_sgt, &job->in_sgt[pay_i],
1624 npay * sizeof(job->out_sgt[0]));
1625 job->out_sgt[npay - 1].final = 1;
1626 return (0);
1627 }
1628
1629 if (iv_slot) {
1630 job->in_sgt[0].addr = pmap_kextract((vm_offset_t)job->iv);
1631 job->in_sgt[0].extension = 0;
1632 job->in_sgt[0].final = 0;
1633 job->in_sgt[0].length = sec_cipher_ivlen(csp);
1634 job->in_sgt[0].bpid = 0;
1635 job->in_sgt[0].offset = 0;
1636 }
1637
1638 out_i = 0;
1639 for (i = 0; i < job->nsegs && left > 0; i++) {
1640 bus_addr_t addr = job->segs[i].ds_addr;
1641 bus_size_t len = job->segs[i].ds_len;
1642 uint32_t take;
1643
1644 if (skip >= len) {
1645 skip -= len;
1646 continue;
1647 }
1648 addr += skip;
1649 len -= skip;
1650 skip = 0;
1651
1652 take = (len > left) ? left : len;
1653 left -= take;
1654
1655 if (out_i >= SEC_MAX_SEGMENTS)
1656 return (E2BIG);
1657
1658 job->in_sgt[iv_slot + out_i].addr = addr;
1659 job->in_sgt[iv_slot + out_i].extension = 0;
1660 job->in_sgt[iv_slot + out_i].final = 0;
1661 job->in_sgt[iv_slot + out_i].length = take;
1662 job->in_sgt[iv_slot + out_i].bpid = 0;
1663 job->in_sgt[iv_slot + out_i].offset = 0;
1664
1665 if (iv_slot)
1666 job->out_sgt[out_i] = job->in_sgt[iv_slot + out_i];
1667 out_i++;
1668 }
1669 if (left != 0)
1670 return (EINVAL);
1671 if (out_i == 0)
1672 return (EINVAL);
1673
1674 job->in_sgt[iv_slot + out_i - 1].final = 1;
1675 if (iv_slot)
1676 job->out_sgt[out_i - 1].final = 1;
1677 return (0);
1678 }
1679
1680 /*
1681 * JD for the cipher modes. The shared descriptor runs the pipeline, so
1682 * the JD only points at the SGTs.
1683 *
1684 * Both sequences use SGF and EXT unconditionally. Always using a table
1685 * keeps the builder from caring how many segments there are, and the
1686 * 16-bit length in the command word is too small for the payloads geli
1687 * and kTLS hand down.
1688 */
1689 static int
sec_jd_build_cipher(struct sec_job * job,const struct crypto_session_params * csp)1690 sec_jd_build_cipher(struct sec_job *job,
1691 const struct crypto_session_params *csp)
1692 {
1693 struct sec_desc_builder b;
1694 struct sec_session *sess = job->sess;
1695 int enc = CRYPTO_OP_IS_ENCRYPT(job->crp->crp_op);
1696 uint32_t sdlen = sess->sdlen[enc];
1697 uint32_t desclen;
1698 uint32_t in_len = sec_cipher_ivlen(csp) + job->crp->crp_payload_length;
1699 vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa;
1700
1701 /*
1702 * The hardware restarts the tweak every SEC_XTS_SECTOR_SIZE bytes;
1703 * opencrypto expects one continuous tweak, so anything that would
1704 * cross the boundary has to go back to software.
1705 */
1706 if (csp->csp_cipher_alg == CRYPTO_AES_XTS &&
1707 job->crp->crp_payload_length > SEC_XTS_SECTOR_SIZE)
1708 return (EOPNOTSUPP);
1709
1710 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd);
1711 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
1712 out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt);
1713
1714 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
1715 sec_jd_ptr(&b, shd_pa);
1716
1717 sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa,
1718 job->crp->crp_payload_length);
1719
1720 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
1721
1722 /*
1723 * HEADER_REO (Reverse Execution Order) makes SEC run the JD
1724 * commands FIRST (SEQ_OUT_PTR / SEQ_IN_PTR set up the input and
1725 * output sequences), then fall into the shared descriptor. The
1726 * shared descriptor's SEQ_LOAD / SEQ_FIFO_LOAD / SEQ_FIFO_STORE
1727 * commands depend on those sequences being programmed. Without
1728 * this bit the shared desc runs first and SEQ_LOAD hits an
1729 * uninitialized input sequence, and DECO reports an invalid
1730 * sequence command (error 0x10).
1731 */
1732 return (sec_desc_finalize_job(&b, HEADER_SHR |
1733 HEADER_REO | HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_DEFER,
1734 &desclen));
1735 }
1736
1737 /*
1738 * JD for the digest modes. There is no IV to prepend and the output is
1739 * a small fixed buffer, so SEQ_OUT_PTR addresses it directly.
1740 */
1741 static int
sec_jd_build_digest(struct sec_job * job)1742 sec_jd_build_digest(struct sec_job *job)
1743 {
1744 struct sec_desc_builder b;
1745 struct sec_session *sess = job->sess;
1746 uint32_t sdlen = sess->sdlen[0];
1747 uint32_t desclen;
1748 vm_paddr_t shd_pa, in_sgt_pa, digest_pa;
1749
1750 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[0].shd);
1751 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
1752 digest_pa = pmap_kextract((vm_offset_t)job->digest);
1753
1754 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
1755 sec_jd_ptr(&b, shd_pa);
1756
1757 sec_jd_seq(&b, true, SEQ_EXT, digest_pa, sess->digestlen);
1758 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa,
1759 job->crp->crp_payload_length);
1760
1761 return (sec_desc_finalize_job(&b,
1762 HEADER_SHR | HEADER_REO | HEADER_SHR_DESC_L(sdlen) |
1763 HEADER_SHARE_DEFER, &desclen));
1764 }
1765
1766 /*
1767 * JD for AEAD (AES-GCM).
1768 *
1769 * The shared descriptor holds only the class 1 key and runs first, so
1770 * the job descriptor sets up both sequences and drives all of the data.
1771 * The data size counts the IV and AAD rounded up to 16 bytes even though
1772 * the FIFO loads supply them unpadded; SEC pads them internally.
1773 *
1774 * [0] - Header
1775 * [1..2] - Shared descriptor pointer
1776 * [3..6] - SEQ OUT PTR - ciphertext only, the tag leaves via STORE
1777 * [7..10] - SEQ IN PTR - iv + aad + payload, and the tag when decrypting
1778 * [11] - Operation
1779 * [12] - LOAD Class 1 Data Size, which starts processing
1780 * [13] - FIFO LOAD IV
1781 * [14] - FIFO LOAD AAD
1782 * [15] - FIFO STORE ciphertext
1783 * [16] - FIFO LOAD message
1784 * [encrypt:17..19] - STORE the computed tag to job->digest
1785 * [decrypt:17] - FIFO LOAD received ICV
1786 */
1787 static int
sec_jd_build_aead(struct sec_job * job)1788 sec_jd_build_aead(struct sec_job *job)
1789 {
1790 struct sec_desc_builder b;
1791 struct sec_session *sess = job->sess;
1792 const struct crypto_session_params *csp;
1793 uint32_t sdlen = sess->sdlen[0];
1794 uint32_t desclen;
1795 vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa;
1796 uint32_t ivlen, aadlen, paylen, taglen;
1797 uint32_t padded_iv, padded_aad, dsr_val;
1798 uint32_t in_len, out_len;
1799 vm_paddr_t digest_pa;
1800 int enc;
1801
1802 csp = crypto_get_params(job->crp->crp_session);
1803 enc = (job->crp->crp_op & CRYPTO_OP_ENCRYPT) != 0;
1804 ivlen = csp->csp_ivlen;
1805 aadlen = job->crp->crp_aad_length;
1806 paylen = job->crp->crp_payload_length;
1807 taglen = sess->digestlen;
1808
1809 padded_iv = roundup(ivlen, 16);
1810 padded_aad = roundup(aadlen, 16);
1811 dsr_val = padded_iv + padded_aad + paylen;
1812 in_len = ivlen + aadlen + paylen + (enc ? 0 : taglen);
1813 /* Output sequence is ciphertext only; tag goes via direct STORE. */
1814 out_len = paylen;
1815 digest_pa = pmap_kextract((vm_offset_t)job->digest);
1816
1817 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd);
1818 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
1819 out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt);
1820
1821 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
1822 sec_jd_ptr(&b, shd_pa);
1823
1824 sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, out_len);
1825 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
1826
1827 /*
1828 * Writing the data size starts processing, so OPERATION has to arm
1829 * the CHA in GCM mode before the DSR load below.
1830 */
1831 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_GCM |
1832 AS_INIT_FINAL | (enc ? OP_ENC : OP_ICV));
1833
1834 sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_DSR, 0, 8,
1835 (uint64_t)dsr_val << 32);
1836
1837 /* IV: FC1 so SEC pads to 16 without ending class 1 input. */
1838 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
1839 FIFOLD_TYPE_IV | FIFOLD_FC1, ivlen);
1840 /*
1841 * Always emit an AAD FIFO_LOAD (even with length 0) so SEC gets
1842 * an explicit "AAD phase done" signal via FC1.
1843 */
1844 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
1845 FIFOLD_TYPE_AAD | FIFOLD_FC1, aadlen);
1846 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) |
1847 FIFOST_TYPE_MSG_DATA, paylen);
1848
1849 /* MSG: LC1 for encrypt (last class-1 input), FC1 for decrypt. */
1850 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
1851 FIFOLD_TYPE_MSG | (enc ? FIFOLD_LC1 : FIFOLD_FC1), paylen);
1852 if (enc)
1853 sec_jd_store(&b, false, LOAD_CLASS_1, LOAD_CTX, 0, taglen,
1854 digest_pa);
1855 else
1856 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
1857 FIFOLD_CLASS_1 | FIFOLD_TYPE_ICV | FIFOLD_LC1, taglen);
1858
1859 /* SHR=1, NO REO, so the shd (KEY only) runs first, then the JD. */
1860 return (sec_desc_finalize_job(&b, HEADER_SHR |
1861 HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen));
1862 }
1863
1864 /*
1865 * JD for AES-CCM.
1866 */
1867 static int
sec_jd_build_ccm(struct sec_job * job)1868 sec_jd_build_ccm(struct sec_job *job)
1869 {
1870 struct sec_desc_builder b;
1871 struct sec_session *sess = job->sess;
1872 uint32_t sdlen = sess->sdlen[0];
1873 uint32_t desclen, aadlen, paylen, taglen, in_len;
1874 vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa, digest_pa;
1875 int enc;
1876
1877 enc = (job->crp->crp_op & CRYPTO_OP_ENCRYPT) != 0;
1878 aadlen = job->crp->crp_aad_length;
1879 paylen = job->crp->crp_payload_length;
1880 taglen = sess->digestlen;
1881
1882 in_len = SEC_CCM_CTX_LEN + paylen + (enc ? 0 : taglen);
1883 if (aadlen > 0)
1884 in_len += sizeof(job->ccm_alen) + aadlen;
1885
1886 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd);
1887 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
1888 out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt);
1889 digest_pa = pmap_kextract((vm_offset_t)job->digest);
1890
1891 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
1892 sec_jd_ptr(&b, shd_pa);
1893
1894 sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, paylen);
1895 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
1896
1897 /* B0 || CTR0 || zeroed result dwords, from the head of the input. */
1898 sec_desc_word(&b, CMD_DESC(CMD_SEQ_LOAD) | LOAD_CLASS_1 | LOAD_CTX |
1899 (SEC_CCM_CTX_LEN & LOAD_LENGTH_M));
1900
1901 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_CCM |
1902 AS_INIT_FINAL | (enc ? OP_ENC : OP_ICV));
1903
1904 /* Writing the data size starts the operation. */
1905 sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_DSR, 0, 8,
1906 (uint64_t)paylen << 32);
1907
1908 /* Length-prefixed AAD; the hardware pads it out to 16 bytes. */
1909 if (aadlen > 0)
1910 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
1911 FIFOLD_CLASS_1 | FIFOLD_TYPE_AAD | FIFOLD_FC1,
1912 sizeof(job->ccm_alen) + aadlen);
1913
1914 /* Arm the drain before the message, as for GCM. */
1915 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) |
1916 FIFOST_TYPE_MSG_DATA, paylen);
1917
1918 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
1919 FIFOLD_TYPE_MSG | (enc ? FIFOLD_LC1 : FIFOLD_FC1), paylen);
1920
1921 if (enc)
1922 sec_jd_store(&b, false, LOAD_CLASS_1, LOAD_CTX, 32, taglen,
1923 digest_pa);
1924 else
1925 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
1926 FIFOLD_CLASS_1 | FIFOLD_TYPE_ICV | FIFOLD_LC1, taglen);
1927
1928 return (sec_desc_finalize_job(&b, HEADER_SHR |
1929 HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen));
1930 }
1931
1932 /*
1933 * JD for encrypt-then-auth.
1934 */
1935 static int
sec_jd_build_eta(struct sec_job * job)1936 sec_jd_build_eta(struct sec_job *job)
1937 {
1938 struct sec_desc_builder b;
1939 struct sec_session *sess = job->sess;
1940 const struct crypto_session_params *csp;
1941 uint32_t sdlen, desclen, aadlen, paylen, ivlen, in_len;
1942 uint32_t ctx_offset;
1943 vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa, digest_pa;
1944 int enc;
1945
1946 csp = crypto_get_params(job->crp->crp_session);
1947 enc = CRYPTO_OP_IS_ENCRYPT(job->crp->crp_op);
1948 sdlen = sess->sdlen[enc];
1949 ivlen = csp->csp_ivlen;
1950 aadlen = job->crp->crp_aad_length;
1951 paylen = job->crp->crp_payload_length;
1952
1953 in_len = ivlen + aadlen + paylen;
1954
1955 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd);
1956 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
1957 out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt);
1958 digest_pa = pmap_kextract((vm_offset_t)job->digest);
1959
1960 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
1961 sec_jd_ptr(&b, shd_pa);
1962
1963 sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, paylen);
1964 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
1965
1966 ctx_offset = sec_cipher_ctx_offset(csp->csp_cipher_alg);
1967
1968 /* IV into the class 1 context; also drops SIL by ivlen. */
1969 sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX, ctx_offset,
1970 ivlen, 0);
1971
1972 /* AAD is authenticated only, so class 2 alone. */
1973 if (aadlen > 0)
1974 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
1975 FIFOLD_CLASS_2 | FIFOLD_TYPE_MSG, aadlen);
1976
1977 /* Arm the ciphertext drain before feeding the message. */
1978 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) |
1979 FIFOST_TYPE_MSG_DATA, paylen);
1980
1981 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
1982 FIFOLD_CLASS_BOTH | FIFOLD_LC1 | FIFOLD_LC2 |
1983 (enc ? FIFOLD_TYPE_MSG_C1OUT : FIFOLD_TYPE_MSG), paylen);
1984
1985 /* Drain the MAC to job->digest; the caller compares or copies back. */
1986 sec_jd_store(&b, false, LOAD_CLASS_2, LOAD_CTX, 0,
1987 sess->digestlen, digest_pa);
1988
1989 return (sec_desc_finalize_job(&b, HEADER_SHR |
1990 HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen));
1991 }
1992
1993 /*
1994 * JD for AES-GMAC: GCM with nothing to encrypt.
1995 */
1996 static int
sec_jd_build_gmac(struct sec_job * job)1997 sec_jd_build_gmac(struct sec_job *job)
1998 {
1999 struct sec_desc_builder b;
2000 struct sec_session *sess = job->sess;
2001 const struct crypto_session_params *csp;
2002 uint32_t sdlen = sess->sdlen[0];
2003 uint32_t desclen, ivlen, datalen, dsr_val, in_len;
2004 vm_paddr_t shd_pa, in_sgt_pa, digest_pa;
2005
2006 csp = crypto_get_params(job->crp->crp_session);
2007 ivlen = csp->csp_ivlen;
2008 datalen = job->crp->crp_payload_length;
2009
2010 /*
2011 * The digest-mode SGT maps the payload only, so AAD has nowhere
2012 * to come from.
2013 */
2014 if (job->crp->crp_aad_length != 0)
2015 return (EOPNOTSUPP);
2016
2017 dsr_val = roundup(ivlen, 16) + roundup(datalen, 16);
2018 in_len = ivlen + datalen;
2019
2020 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[0].shd);
2021 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
2022 digest_pa = pmap_kextract((vm_offset_t)job->digest);
2023
2024 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
2025 sec_jd_ptr(&b, shd_pa);
2026
2027 /* No output sequence: the tag leaves through an inline STORE. */
2028 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
2029
2030 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_GCM |
2031 AS_INIT_FINAL | OP_ENC);
2032
2033 sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_DSR, 0, 8,
2034 (uint64_t)dsr_val << 32);
2035
2036 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
2037 FIFOLD_TYPE_IV | FIFOLD_FC1, ivlen);
2038 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
2039 FIFOLD_TYPE_AAD | FIFOLD_LC1, datalen);
2040
2041 sec_jd_store(&b, false, LOAD_CLASS_1, LOAD_CTX, 0,
2042 sess->digestlen, digest_pa);
2043
2044 return (sec_desc_finalize_job(&b, HEADER_SHR |
2045 HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen));
2046 }
2047
2048 static int
sec_process(device_t dev,struct cryptop * crp,int hint)2049 sec_process(device_t dev, struct cryptop *crp, int hint)
2050 {
2051 struct sec_softc *sc = device_get_softc(dev);
2052 struct sec_session *sess = crypto_get_driver_session(crp->crp_session);
2053 const struct crypto_session_params *csp;
2054 struct sec_job *job;
2055 struct sec_jr *jr;
2056 int error;
2057
2058 job = malloc(sizeof(*job), M_SEC, M_NOWAIT | M_ZERO);
2059 if (job == NULL) {
2060 crp->crp_etype = ENOMEM;
2061 crypto_done(crp);
2062 return (0);
2063 }
2064 job->crp = crp;
2065 job->sess = sess;
2066
2067 error = bus_dmamap_create(sc->sc_dmatag, 0, &job->map);
2068 if (error != 0)
2069 goto fail_free;
2070
2071 error = bus_dmamap_load_crp(sc->sc_dmatag, job->map, crp,
2072 sec_load_cb, job, BUS_DMA_NOWAIT);
2073 if (error != 0 || job->nsegs == 0) {
2074 if (error == 0)
2075 error = EIO;
2076 goto fail_destroy;
2077 }
2078
2079 if (crp->crp_payload_length == 0) {
2080 error = EINVAL;
2081 goto fail_unload;
2082 }
2083
2084 csp = crypto_get_params(crp->crp_session);
2085 if (csp->csp_ivlen > 0)
2086 crypto_read_iv(crp, job->iv);
2087 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
2088 sec_xts_tweak(job->iv);
2089 if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) {
2090 error = sec_ccm_prep(job, csp);
2091 if (error != 0)
2092 goto fail_unload;
2093 }
2094
2095 error = sec_job_build_sgts(job, csp);
2096 if (error != 0)
2097 goto fail_unload;
2098
2099 bus_dmamap_sync(sc->sc_dmatag, job->map,
2100 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2101
2102 switch (csp->csp_mode) {
2103 case CSP_MODE_DIGEST:
2104 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC)
2105 error = sec_jd_build_gmac(job);
2106 else
2107 error = sec_jd_build_digest(job);
2108 break;
2109 case CSP_MODE_AEAD:
2110 if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16)
2111 error = sec_jd_build_ccm(job);
2112 else
2113 error = sec_jd_build_aead(job);
2114 break;
2115 case CSP_MODE_ETA:
2116 error = sec_jd_build_eta(job);
2117 break;
2118 default:
2119 error = sec_jd_build_cipher(job, csp);
2120 break;
2121 }
2122 if (error != 0)
2123 goto fail_unload;
2124
2125
2126 /*
2127 * Hand the job to a ring and return; sec_jr_intr() completes it.
2128 */
2129 jr = &sc->sc_jr[curcpu % sc->sc_njr];
2130 sec_jr_submit_job(sc, jr, job);
2131 return (0);
2132
2133 fail_unload:
2134 bus_dmamap_sync(sc->sc_dmatag, job->map,
2135 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2136 bus_dmamap_unload(sc->sc_dmatag, job->map);
2137 fail_destroy:
2138 bus_dmamap_destroy(sc->sc_dmatag, job->map);
2139 fail_free:
2140 free(job, M_SEC);
2141 /* ERESTART means opencrypto retries this crp, so don't complete it. */
2142 if (error == ERESTART)
2143 return (ERESTART);
2144 crp->crp_etype = error;
2145 crypto_done(crp);
2146 return (0);
2147 }
2148
2149 static int
sec_reset(struct sec_softc * sc)2150 sec_reset(struct sec_softc *sc)
2151 {
2152 uint32_t mcfgr;
2153 int i;
2154
2155 /*
2156 * Preserve cache-attribute fields (AWCACHE/ARCACHE) and burst
2157 * settings across the reset. The MCFGR write overwrites those
2158 * along with SWRST.
2159 */
2160 mcfgr = SEC_RD4(sc, SEC_MCFGR);
2161 SEC_WR4(sc, SEC_MCFGR, mcfgr | MCFGR_SWRST);
2162
2163 /* Poll SWRST for self-clear. */
2164 for (i = 0; i < 10000; i++) {
2165 if ((SEC_RD4(sc, SEC_MCFGR) & MCFGR_SWRST) == 0)
2166 break;
2167 DELAY(10);
2168 }
2169 if ((SEC_RD4(sc, SEC_MCFGR) & MCFGR_SWRST) != 0) {
2170 device_printf(sc->sc_dev, "MCFGR.SWRST did not clear\n");
2171 return (EIO);
2172 }
2173
2174 /*
2175 * Post-reset configuration: 40-bit pointers, DECO watchdog on,
2176 * large bursts. Preserve whatever cache attributes the bootloader
2177 * left in place.
2178 */
2179 mcfgr &= (MCFGR_ARCACHE_M | MCFGR_AWCACHE_M);
2180 mcfgr |= MCFGR_PS | MCFGR_WDE | MCFGR_LARGE_BURST;
2181 SEC_WR4(sc, SEC_MCFGR, mcfgr);
2182
2183 return (0);
2184 }
2185
2186 /*
2187 * Instantiate one RNG state handle via DECO0 direct access.
2188 */
2189 static int
sec_deco_rng_init(struct sec_softc * sc,int sh)2190 sec_deco_rng_init(struct sec_softc *sc, int sh)
2191 {
2192 uint32_t jd[2];
2193 uint32_t reg, decorr, scfgr;
2194 int i;
2195
2196 jd[0] = CMD_DESC(CMD_DESC_HEADER) | HEADER_ONE |
2197 (2 & HEADER_DESCLEN_M);
2198 jd[1] = CMD_DESC(CMD_OPERATION) | ALG_RNG | AS_INIT | OP_RNG_SH(sh);
2199
2200 /* Request DECO0 and wait for the grant (DEN0=1). */
2201 SEC_WR4(sc, SEC_DECORR, DECORR_RQD0);
2202 decorr = SEC_RD4(sc, SEC_DECORR);
2203 for (i = 0; i < 10000; i++) {
2204 decorr = SEC_RD4(sc, SEC_DECORR);
2205 if ((decorr & DECORR_DEN0) != 0)
2206 break;
2207 DELAY(10);
2208 }
2209 if ((decorr & DECORR_DEN0) == 0) {
2210 scfgr = SEC_RD4(sc, SEC_SCFGR);
2211 device_printf(sc->sc_dev,
2212 "DECO0 acquire timeout (DECORR=%#x SCFGR=%#x%s)\n",
2213 decorr, scfgr,
2214 (scfgr & SCFGR_VIRT_EN) ? " VIRT_EN" : "");
2215 SEC_WR4(sc, SEC_DECORR, 0);
2216 return (ETIMEDOUT);
2217 }
2218
2219 SEC_WR4(sc, SEC_D0DESB(0), jd[0]);
2220 SEC_WR4(sc, SEC_D0DESB(1), jd[1]);
2221
2222 SEC_WR4(sc, SEC_D0JQCR_MS, DAJQCR_MS_WHL);
2223
2224 /* Wait for job completion */
2225 reg = 0;
2226 for (i = 0; i < 100000; i++) {
2227 reg = SEC_RD4(sc, SEC_D0DDR);
2228 if ((reg & DADDR_VALID) == 0)
2229 break;
2230 DELAY(10);
2231 }
2232
2233 /* Release DECO0 either way. */
2234 SEC_WR4(sc, SEC_DECORR, 0);
2235
2236 if ((reg & DADDR_VALID) != 0) {
2237 device_printf(sc->sc_dev,
2238 "RNG SH%d instantiate timeout (D0DDR=%#x)\n", sh, reg);
2239 return (ETIMEDOUT);
2240 }
2241 if (((reg & DADDR_DECO_STATE_M) >> DADDR_DECO_STATE_S) != 0) {
2242 device_printf(sc->sc_dev,
2243 "RNG SH%d instantiate error (D0DDR=%#x, DECO_STATE=%u)\n",
2244 sh, reg,
2245 (reg & DADDR_DECO_STATE_M) >> DADDR_DECO_STATE_S);
2246 return (EIO);
2247 }
2248 return (0);
2249 }
2250
2251 static int
sec_rng_init(struct sec_softc * sc)2252 sec_rng_init(struct sec_softc *sc)
2253 {
2254 uint32_t rdsta;
2255 int error, sh;
2256
2257 /*
2258 * SEC v4/v5 requires the DRNG state handles to be instantiated
2259 * before any class-1 (AES/DES/RNG) job will execute. This is typically
2260 * done by the bootloader, but finish what it didn't.
2261 */
2262 rdsta = SEC_RD4(sc, SEC_RDSTA);
2263
2264 if ((rdsta & RDSTA_CE) != 0) {
2265 device_printf(sc->sc_dev,
2266 "RNG catastrophic error (RDSTA=%#x, ERRCODE=%u)\n",
2267 rdsta, (rdsta & RDSTA_ERRCODE_M) >> RDSTA_ERRCODE_S);
2268 return (EIO);
2269 }
2270
2271 /* Instantiate anything the bootloader didn't. */
2272 for (sh = 0; sh <= 1; sh++) {
2273 uint32_t bit = (sh == 0) ? RDSTA_IF0 : RDSTA_IF1;
2274
2275 if ((rdsta & bit) != 0)
2276 continue;
2277 error = sec_deco_rng_init(sc, sh);
2278 if (error != 0)
2279 return (error);
2280 }
2281
2282 /* Verify the handles are now up. */
2283 rdsta = SEC_RD4(sc, SEC_RDSTA);
2284 if ((rdsta & (RDSTA_IF0 | RDSTA_IF1)) !=
2285 (RDSTA_IF0 | RDSTA_IF1)) {
2286 device_printf(sc->sc_dev,
2287 "RNG instantiation left RDSTA=%#x\n", rdsta);
2288 return (EIO);
2289 }
2290 return (0);
2291 }
2292