1 /* 2 * Copyright (c) 2026 Justin Hibbits <jhibbits@FreeBSD.org> 3 * 4 * SPDX-License-Identifier: BSD-2-Clause 5 */ 6 7 #include <sys/param.h> 8 #include <sys/bus.h> 9 #include <sys/callout.h> 10 #include <sys/kernel.h> 11 #include <sys/lock.h> 12 #include <sys/malloc.h> 13 #include <sys/module.h> 14 #include <sys/mutex.h> 15 #include <sys/queue.h> 16 #include <sys/rman.h> 17 #include <sys/smp.h> 18 19 #include <machine/atomic.h> 20 #include <machine/bus.h> 21 #include <machine/resource.h> 22 23 #include <vm/vm.h> 24 #include <vm/pmap.h> 25 26 #include <sys/endian.h> 27 28 #include <dev/ofw/ofw_bus.h> 29 #include <dev/ofw/ofw_bus_subr.h> 30 #include <opencrypto/cryptodev.h> 31 #include <opencrypto/xform_auth.h> 32 33 #include "sec_var.h" 34 #include "cryptodev_if.h" 35 36 /* 37 * Most of this work is based on the T2080 Security (SEC) Reference Manual. 38 * 39 * The driver uses the Job Ring interface for all jobs. The QI interface can be 40 * added if IPSec, OVPN, or kTLS acceleration is added. 41 */ 42 43 /* From T2080 Security Reference Manual */ 44 #define SEC_MAX_SHDESC_WORDS 62 45 46 #define SEC_MAX_JR 4 /* T2080 exposes four Job Rings */ 47 48 /* CCSR register offsets. */ 49 #define SEC_MCFGR 0x0004 50 #define MCFGR_SWRST 0x80000000 /* Software reset */ 51 #define MCFGR_WDE 0x40000000 /* DECO watchdog enable */ 52 #define MCFGR_WDF 0x20000000 /* Watchdog fast (test only) */ 53 #define MCFGR_DMARST 0x10000000 /* DMA reset (with SWRST) */ 54 #define MCFGR_WRHD 0x08000000 /* Write handoff disable */ 55 #define MCFGR_DJPC 0x00200000 /* Disable job perf ctrs */ 56 #define MCFGR_DBPC 0x00100000 /* Disable byte perf ctrs */ 57 #define MCFGR_PS 0x00010000 /* Large pointers */ 58 #define MCFGR_ARCACHE_M 0x0000f000 /* AXI read cache attrs */ 59 #define MCFGR_AWCACHE_M 0x00000f00 /* AXI write cache attrs */ 60 #define MCFGR_AXIPRI 0x00000008 /* AXI master priority */ 61 #define MCFGR_LARGE_BURST 0x00000004 /* Enable 256B bursts */ 62 #define SEC_SCFGR 0x000c 63 #define SCFGR_VIRT_EN 0x00008000 /* Virtualization enabled */ 64 65 #define SEC_RDSTA 0x06c0 /* RNG DRNG Status */ 66 #define RDSTA_IF0 0x00000001 /* State handle 0 up */ 67 #define RDSTA_IF1 0x00000002 /* State handle 1 up */ 68 #define RDSTA_ERRCODE_M 0x000f0000 69 #define RDSTA_ERRCODE_S 16 70 #define RDSTA_CE 0x00100000 /* Catastrophic error */ 71 72 /* DECO direct-access registers */ 73 #define SEC_DECORR 0x009c /* DECO Request Register */ 74 #define DECORR_DEN0 0x00010000 /* DECO0 enable (RO, bit 16) */ 75 #define DECORR_RQD0 0x00000001 /* DECO0 request */ 76 #define SEC_D0LIODNR_MS 0x00a0 77 #define SEC_D0LIODNR_LS 0x00a4 78 #define SEC_D0JQCR_MS 0x8800 /* JQCR upper: WHL/FOUR/SOB */ 79 #define DAJQCR_MS_WHL 0x20000000 /* Whole descriptor loaded */ 80 #define DAJQCR_MS_FOUR 0x10000000 /* >= 4 words in first burst */ 81 #define DAJQCR_MS_SOB 0x00010000 /* Shared/burst loaded */ 82 #define DAJQCR_MS_SRC_M 0x00000700 /* Job source */ 83 #define DAJQCR_MS_SRC_S 8 84 #define SEC_D0JQCR_LS 0x8804 85 #define SEC_D0DAR_MS 0x8808 /* Descriptor address, upper */ 86 #define SEC_D0DAR_LS 0x880c 87 #define SEC_D0DESB(n) (0x8a00 + (n) * 4) /* n = 0..63 */ 88 #define SEC_D0DDR 0x8e04 /* Debug status */ 89 #define DADDR_VALID 0x80000000 /* Job currently running */ 90 #define DADDR_DECO_STATE_M 0x00f00000 /* Main state machine */ 91 #define DADDR_DECO_STATE_S 20 92 93 /* Fault-address registers. */ 94 #define SEC_FAR_HI 0x0fc0 /* Fault Address, upper */ 95 #define SEC_FAR_LO 0x0fc4 /* Fault Address, lower */ 96 #define SEC_FALR 0x0fc8 /* Fault Address LIODN */ 97 #define SEC_FADR 0x0fcc /* Fault Address Detail */ 98 #define FADR_FERR_M 0xc0000000 /* AXI error response */ 99 #define FADR_FERR_S 30 100 #define FADR_FSZ_EXT_M 0x00070000 /* Transfer size high 3 bits */ 101 #define FADR_FSZ_EXT_S 16 102 #define FADR_DTYP 0x00008000 /* 0=message, 1=control */ 103 #define FADR_JSRC_M 0x00007000 /* Job source */ 104 #define FADR_JSRC_S 12 105 #define FADR_BLKID_M 0x00000f00 /* SEC internal block ID */ 106 #define FADR_BLKID_S 8 107 #define FADR_TYP 0x00000080 /* 0=read, 1=write */ 108 #define FADR_FSZ_M 0x0000007f /* Transfer size low 7 bits */ 109 110 #define SEC_RD4(sc, off) bus_read_4((sc)->sc_rres, (off)) 111 #define SEC_WR4(sc, off, v) bus_write_4((sc)->sc_rres, (off), (v)) 112 113 /* Descriptor command components */ 114 /* SEQ commands are intended for network protocols */ 115 #define CMD_DESC(n) ((n) << 27) 116 #define CMD_KEY 0x00 /* Pointer/key follows descriptor */ 117 #define CMD_SEQ_KEY 0x01 118 #define KEY_CLASS_M 0x06000000 119 #define KEY_CLASS_1 0x02000000 120 #define KEY_CLASS_2 0x04000000 121 #define KEY_SGF 0x01000000 /* KEY - Pointer to SGT */ 122 #define KEY_VLF 0x01000000 /* SK - variable length */ 123 #define KEY_IMM 0x00800000 /* KEY - Key follows descriptor */ 124 #define KEY_AIDF 0x00800000 /* SK - Already in Input FIFO */ 125 #define KEY_ENC 0x00400000 /* Key is encrypted */ 126 #define KEY_NWB 0x00200000 /* No write back */ 127 #define KEY_EKT 0x00100000 /* Encrypted Key Type: 128 * 0 - AES-CCB 129 * 1 - AES-CCM 130 */ 131 #define KEY_KDEST_M 0x00030000 /* Key Destination */ 132 #define KEY_KDEST_REG 0x00000000 /* Dest is Key register */ 133 #define KEY_KDEST_PKHA 0x00010000 /* Dest is PKHA E-memory */ 134 #define KEY_KDEST_AFHA 0x00020000 /* Dest is AFHA S-Box */ 135 #define KEY_KDEST_MDHA_SPLIT 0x00030000 /* Key is MDHA split key */ 136 #define KEY_TK 0x00008000 /* Trusted Key */ 137 #define KEY_LENGTH_M 0x000003ff /* Key length */ 138 #define CMD_LOAD 0x02 139 #define CMD_SEQ_LOAD 0x03 140 #define LOAD_CLASS_M 0x06000000 141 #define LOAD_CLASS_1 0x02000000 142 #define LOAD_CLASS_2 0x04000000 143 #define LOAD_CLASS_3 0x06000000 144 #define LOAD_SGF 0x01000000 /* LOAD - Pointer to SGT */ 145 #define LOAD_VLF 0x01000000 /* SL - variable length */ 146 #define LOAD_IMM 0x00800000 /* LOAD - Data follows descriptor */ 147 #define LOAD_DST_M 0x007f0000 /* Destination register */ 148 #define LOAD_DST_S 16 149 #define LOAD_KSR 0x00010000 /* Key Size Register (C1/C2) */ 150 #define LOAD_DSR 0x00020000 /* Data Size Register (C1/C2) */ 151 #define LOAD_ICVS 0x00030000 /* ICV Size Register (C1/C2) */ 152 #define LOAD_LSR 0x00040000 /* LIODN Status Register (C3) */ 153 #define LOAD_DCTRL2 0x00050000 /* DECO Control Register 2(C3) */ 154 #define LOAD_CCTRL 0x00060000 /* CHA Control Register (C1) */ 155 #define LOAD_DCTRL 0x00060000 /* DECO Control Register (C3) */ 156 #define LOAD_ICTRL 0x00070000 /* IRQ Control Register (C0) */ 157 #define LOAD_DPOVRD 0x00070000 /* DECO Protocol Override (C3) */ 158 #define LOAD_CLRW 0x00080000 /* Clear Written Register (C0) */ 159 #define LOAD_MATH0W 0x00080000 /* DECO Math Register 0 (C3) */ 160 #define LOAD_MATH1W 0x00090000 /* DECO Math Register 1 (C3) */ 161 #define LOAD_MATH2W 0x000a0000 /* DECO Math Register 2 (C3) */ 162 #define LOAD_CISEL 0x000a0000 /* CHA Instance Select Reg (C0) */ 163 #define LOAD_AADSZ 0x000b0000 /* AAD Size Register (C1) */ 164 #define LOAD_MAT3W 0x000b0000 /* DECO Math Register 3 (C3) */ 165 #define LOAD_C1VSZ 0x000c0000 /* Class 1 IV SIze Register (C1) */ 166 #define LOAD_ALTDS1 0x000f0000 /* Alternate Data Size C1 (C1) */ 167 #define LOAD_PKASZ 0x00100000 /* PKHA A Size Register (C1) */ 168 #define LOAD_PKBSZ 0x00110000 /* PKHA B Size Register (C1) */ 169 #define LOAD_PKNSZ 0x00120000 /* PKHA N Size Register (C1) */ 170 #define LOAD_PKESZ 0x00130000 /* PKHA E Size Register (C1) */ 171 #define LOAD_CTX 0x00200000 /* Context Register (C1/C2) */ 172 #define LOAD_KEY 0x00400000 /* Key Register (C1/C2) */ 173 #define LOAD_DESC_BUF 0x00400000 /* DECO Descriptor Buffer (C3) */ 174 #define LOAD_NFSL 0x00700000 /* NFIFO and size registers (C0) */ 175 #define LOAD_NFSM 0x00710000 /* NFIFO and size registers (C0) */ 176 #define LOAD_NFL 0x00720000 /* NFIFO (C0) */ 177 #define LOAD_NFM 0x00730000 /* NFIFO (C0) */ 178 #define LOAD_SL 0x00740000 /* Size register(s) (C0) */ 179 #define LOAD_SM 0x00750000 /* Size register(s) (C0) */ 180 #define LOAD_IDFNS 0x00760000 /* Input Data FIFO Nibble Shift (C0) */ 181 #define LOAD_ODFNS 0x00770000 /* Output Data FIFO Nibble Shift (C0) */ 182 #define LOAD_AUXDATA 0x00780000 /* Aux Data FIFO (C0) */ 183 #define LOAD_NFIFO 0x007a0000 /* NFIFO (C0) */ 184 #define LOAD_IFIFO 0x007c0000 /* Input Data FIFO (C0) */ 185 #define LOAD_OFIFO 0x007e0000 /* Output Data FIFO (C0) */ 186 #define LOAD_LENGTH_M 0x000000ff /* Data length (8 bits) */ 187 #define LOAD_OFFSET_S 8 /* OFFSET field shift (bits 8-15) */ 188 #define CMD_FIFO_LOAD 0x04 189 #define CMD_SEQ_FIFO_LOAD 0x05 190 #define CMD_STORE 0x0a 191 #define CMD_SEQ_STORE 0x0b 192 #define CMD_FIFO_STORE 0x0c 193 #define CMD_SEQ_FIFO_STORE 0x0d 194 #define CMD_MOVE 0x0e 195 #define CMD_MOVE_LEN 0x0f 196 #define CMD_OPERATION 0x10 197 #define OPTYPE_M 0x07000000 198 #define OPTYPE_S 24 199 #define OPTYPE_CLASS1_ALG 0x02000000 200 #define OPTYPE_CLASS2_ALG 0x04000000 201 #define ALG_S 16 202 #define CMD_ALGORITHM(m, n) ((m) | ((n) << ALG_S)) 203 /* Class 1 algorithms */ 204 #define ALG_AES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x10) 205 #define ALG_DES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x20) 206 #define ALG_3DES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x21) 207 #define ALG_ARC4 CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x30) 208 #define ALG_RNG CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x50) 209 #define ALG_SNOW3G_F8 CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x60) 210 #define ALG_KASUMI CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x70) 211 #define ALG_ZUC_ENC CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0xb0) 212 /* Class 2 algorithms */ 213 #define ALG_MD5 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x40) 214 #define ALG_SHA1 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x41) 215 #define ALG_SHA224 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x42) 216 #define ALG_SHA256 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x43) 217 #define ALG_SHA384 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x44) 218 #define ALG_SHA512 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x45) 219 #define ALG_CRC CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x90) 220 #define ALG_SNOW3G_F9 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0xa0) 221 #define ALG_ZUC_AUTH CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0xc0) 222 /* AAI (Additional Algorithm Information) codes. */ 223 #define AAI_S 4 224 /* AES modes */ 225 #define AAI_AES_CTR (0x00 << AAI_S) 226 #define AAI_AES_CBC (0x10 << AAI_S) 227 #define AAI_AES_ECB (0x20 << AAI_S) 228 #define AAI_AES_CFB (0x30 << AAI_S) 229 #define AAI_AES_OFB (0x40 << AAI_S) 230 #define AAI_AES_XTS (0x50 << AAI_S) 231 #define AAI_AES_CMAC (0x60 << AAI_S) 232 #define AAI_AES_XCBC_MAC (0x70 << AAI_S) 233 #define AAI_AES_CCM (0x80 << AAI_S) 234 #define AAI_AES_GCM (0x90 << AAI_S) 235 #define AAI_AES_DK (0x100 << AAI_S) /* Decrypt-key derive */ 236 /* DES/3DES modes */ 237 #define AAI_DES_CBC (0x10 << AAI_S) 238 #define AAI_DES_ECB (0x20 << AAI_S) 239 /* MDHA modes */ 240 #define AAI_HASH (0x00 << AAI_S) 241 #define AAI_HMAC (0x01 << AAI_S) 242 #define AAI_HMAC_PRECOMP (0x04 << AAI_S) /* Precomputed IPAD/OPAD */ 243 /* Algorithm State field (bits 2-3): what phase to run */ 244 #define AS_S 2 245 #define AS_UPDATE (0x0 << AS_S) 246 #define AS_INIT (0x1 << AS_S) 247 #define AS_FINAL (0x2 << AS_S) 248 #define AS_INIT_FINAL (0x3 << AS_S) 249 /* RNG-specific: State-Handle field. */ 250 #define OP_RNG_SH_S 4 251 #define OP_RNG_SH(n) ((n) << OP_RNG_SH_S) 252 /* Direction / ICV */ 253 #define OP_ICV 0x00000002 254 #define OP_ENC 0x00000001 255 256 /* SEQ FIFO LOAD command bits. */ 257 #define FIFOLD_CLASS_1 0x02000000 /* CLASS = 01b (Class 1) */ 258 #define FIFOLD_CLASS_2 0x04000000 /* CLASS = 10b (Class 2) */ 259 #define FIFOLD_CLASS_BOTH 0x06000000 /* CLASS = 11b (snooping) */ 260 #define FIFOLD_VLF 0x01000000 /* Variable-length flag */ 261 /* 262 * Input data type: top 3 bits = type, 263 * bottom 3 bits = LC2/LC1/FC1 flags. 264 */ 265 #define FIFOLD_TYPE_S 16 266 #define FIFOLD_TYPE_MSG (0x10 << FIFOLD_TYPE_S) /* 010_000 */ 267 /* Class 1 output fed straight into Class 2, i.e. MAC over ciphertext. */ 268 #define FIFOLD_TYPE_MSG_C1OUT (0x18 << FIFOLD_TYPE_S) /* 011_000 */ 269 #define FIFOLD_TYPE_IV (0x20 << FIFOLD_TYPE_S) /* 100_000 */ 270 #define FIFOLD_TYPE_AAD (0x30 << FIFOLD_TYPE_S) /* 110_000 */ 271 #define FIFOLD_TYPE_ICV (0x38 << FIFOLD_TYPE_S) /* 111_000 */ 272 #define FIFOLD_FC1 (0x01 << FIFOLD_TYPE_S) /* Flush class 1 */ 273 #define FIFOLD_LC1 (0x02 << FIFOLD_TYPE_S) /* Last for Class 1 */ 274 #define FIFOLD_LC2 (0x04 << FIFOLD_TYPE_S) /* Last for Class 2 */ 275 /* Length moves to a 32-bit word after the command. */ 276 #define FIFO_EXT 0x00400000 277 278 /* SEQ FIFO STORE command bits. */ 279 #define FIFOST_VLF 0x01000000 280 #define FIFOST_TYPE_S 16 281 #define FIFOST_TYPE_MSG_DATA (0x30 << FIFOST_TYPE_S) 282 283 #define CMD_SIGNATURE 0x12 284 #define CMD_JUMP 0x14 285 #define CMD_MATH 0x15 286 #define MATH_FN_ADD (0x0 << 20) /* SRC0 + SRC1 */ 287 #define MATH_SRC0_SIL (0x8 << 16) /* Sequence In Length */ 288 #define MATH_SRC1_ZERO (0xF << 12) /* Constant zero */ 289 #define MATH_DEST_VSIL (0xA << 8) /* Variable SIL */ 290 #define MATH_DEST_VSOL (0xB << 8) /* Variable SOL */ 291 #define MATH_LEN_4 0x4 292 /* J - Job Descriptor, S - Shared Descriptor */ 293 #define CMD_DESC_HEADER 0x16 294 #define HEADER_EXT 0x04000000 /* Has Extension (J) */ 295 #define HEADER_RSL 0x02000000 /* Require SEQ LIODN (J) */ 296 #define HEADER_DNR 0x01000000 /* Do Not Run (J/S) */ 297 #define HEADER_ONE 0x00800000 /* Must be 1 (J/S) */ 298 #define HEADER_START_INDEX(n) ((n) << 16) /* Start Index (J/S) */ 299 #define HEADER_SHR_DESC_L(n) ((n) << 16) /* Shared Desc len (J) */ 300 /* Bit 16 must be 0 */ 301 #define HEADER_TDES_M 0x00006000 /* Trusted Descriptor Mask (J) */ 302 #define HEADER_TDES 0x00004000 /* Trusted Descriptor (J) */ 303 #define HEADER_TDES_CAND 0x00006000 /* Candidate Trust Desc (J) */ 304 #define HEADER_SHR 0x00001000 /* Has Shared Descriptor (J) */ 305 #define HEADER_REO 0x00000800 /* Reverse Execution Order (J) */ 306 #define HEADER_SHARE_M 0x00000700 /* Share State (J/S) */ 307 #define HEADER_SHARE_WAIT 0x00000100 /* Wait to share (J/S) */ 308 #define HEADER_SHARE_SERIAL 0x00000200 /* Serialize (J/S) */ 309 #define HEADER_SHARE_ALWAYS 0x00000300 /* Always share (stateless) (J/S) */ 310 #define HEADER_SHARE_DEFER 0x00000400 /* Defer to shared desc (J) */ 311 #define HEADER_DESCLEN_M 0x0000007f /* Descriptor length */ 312 #define HEADER_DESCLEN_S 0 313 #define HEADER_EXT_FTD 0x00000100 /* Fake Trusted Descriptor */ 314 #define HEADER_EXT_DSELVALID 0x00000080 /* DECO_SELECT field valid */ 315 #define HEADER_EXT_DSEL_M 0x0000000f /* DECO Select */ 316 #define CMD_SHARED_HEADER 0x17 317 #define HEADER_RIF 0x02000000 /* Read Input Frame */ 318 #define HEADER_CIF 0x00002000 /* Clear Input FIFO */ 319 #define HEADER_SC 0x00001000 /* Save Context */ 320 #define HEADER_PD 0x00000800 /* Propagate DNR */ 321 #define CMD_MATHI 0x1d 322 #define CMD_SEQ_IN_PTR 0x1e 323 #define SEQ_SGF 0x01000000 /* Pointer is SGT (bit 7 NXP) */ 324 #define SEQ_EXT 0x00400000 /* 32-bit extended length (bit 9 NXP) */ 325 #define CMD_SEQ_OUT_PTR 0x1f 326 327 /* Shared descriptor container. */ 328 struct sec_context { 329 uint32_t shd[SEC_MAX_SHDESC_WORDS]; 330 }; 331 332 333 /* 334 * Session state: one shared descriptor per direction. The shared 335 * descriptor holds just KEY + OPERATION; the per-job JD adds LOAD-IV 336 * and SEQ_IN_PTR / SEQ_OUT_PTR inline. 337 */ 338 #define SEC_MAX_SPLIT_KEY 128 /* SHA-512 AES-ECB encrypted */ 339 340 #define SEC_CCM_AAD_MAX 0xfeff 341 342 struct sec_session { 343 struct sec_softc *sess_sc; 344 struct sec_context ctx[2]; /* [0]=dec, [1]=enc */ 345 uint32_t sdlen[2]; /* words per direction */ 346 uint8_t digestlen; /* HMAC output size (0 if none) */ 347 uint8_t skeylen; /* HMAC split key size (0 if none) */ 348 uint8_t skey[SEC_MAX_SPLIT_KEY]; 349 }; 350 351 static device_probe_t sec_probe; 352 static device_attach_t sec_attach; 353 static device_detach_t sec_detach; 354 static cryptodev_probesession_t sec_probe_session; 355 static cryptodev_newsession_t sec_new_session; 356 static cryptodev_freesession_t sec_free_session; 357 static cryptodev_process_t sec_process; 358 359 static void sec_intr(void *); 360 361 /* Register-level bring-up. Filled in from the SEC reference manual. */ 362 static int sec_reset(struct sec_softc *); 363 static int sec_rng_init(struct sec_softc *); 364 365 static struct ofw_compat_data compats[] = { 366 { "fsl,sec-v5.2", 52 }, 367 { "fsl,sec-v5.0", 50 }, 368 { "fsl,sec-v4.0", 40 }, 369 { NULL, 0 } 370 }; 371 372 static device_method_t sec_methods[] = { 373 /* Device methods */ 374 DEVMETHOD(device_probe, sec_probe), 375 DEVMETHOD(device_attach, sec_attach), 376 DEVMETHOD(device_detach, sec_detach), 377 378 /* Cryptodev methods */ 379 DEVMETHOD(cryptodev_probesession, sec_probe_session), 380 DEVMETHOD(cryptodev_newsession, sec_new_session), 381 DEVMETHOD(cryptodev_freesession, sec_free_session), 382 DEVMETHOD(cryptodev_process, sec_process), 383 384 DEVMETHOD_END 385 }; 386 387 static DEFINE_CLASS_0(sec, sec_driver, sec_methods, sizeof(struct sec_softc)); 388 DRIVER_MODULE(sec, simplebus, sec_driver, NULL, NULL); 389 MODULE_DEPEND(sec, crypto, 1, 1, 1); 390 391 MALLOC_DEFINE(M_SEC, "sec", "SEC driver"); 392 393 static int 394 sec_probe(device_t dev) 395 { 396 const struct ofw_compat_data *cd; 397 398 cd = ofw_bus_search_compatible(dev, compats); 399 if (cd->ocd_data == 0) 400 return (ENXIO); 401 402 device_set_descf(dev, "Freescale Security Engine v%d.%d", 403 (int)cd->ocd_data / 10, (int)cd->ocd_data % 10); 404 405 return (BUS_PROBE_DEFAULT); 406 } 407 408 static int 409 sec_attach(device_t dev) 410 { 411 struct sec_softc *sc = device_get_softc(dev); 412 const struct ofw_compat_data *cd; 413 414 sc->sc_dev = dev; 415 sc->sc_cid = -1; 416 417 cd = ofw_bus_search_compatible(dev, compats); 418 sc->sc_version = cd->ocd_data; 419 420 sc->sc_rrid = 0; 421 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 422 RF_ACTIVE); 423 if (sc->sc_rres == NULL) { 424 device_printf(dev, "could not allocate register resource\n"); 425 goto fail; 426 } 427 428 /* TODO: Error IRQ handling. */ 429 sc->sc_irid = 0; 430 sc->sc_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irid, 431 RF_ACTIVE | RF_SHAREABLE); 432 if (sc->sc_ires == NULL) { 433 device_printf(dev, "could not allocate error interrupt\n"); 434 goto fail; 435 } 436 437 if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, 438 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 439 SEC_MAX_SIZE, SEC_MAX_SEGMENTS, SEC_MAX_SIZE, BUS_DMA_ALLOCNOW, 440 NULL, NULL, &sc->sc_dmatag) != 0) { 441 device_printf(dev, "could not create DMA tag\n"); 442 goto fail; 443 } 444 445 if (sec_reset(sc) != 0) { 446 device_printf(dev, "SEC reset failed\n"); 447 goto fail; 448 } 449 if (sec_rng_init(sc) != 0) { 450 device_printf(dev, "SEC RNG instantiation failed\n"); 451 goto fail; 452 } 453 if (sec_init_rings(sc) == 0) { 454 device_printf(dev, "SEC job ring init failed\n"); 455 goto fail; 456 } 457 458 /* 459 * Clear any fault-address latch left over from the bootloader before 460 * enabling the error IRQ. FADR, FAR_HI/LO, and FALR must all be read 461 * before they're all cleared, per the RM. 462 */ 463 (void)SEC_RD4(sc, SEC_FADR); 464 (void)SEC_RD4(sc, SEC_FAR_HI); 465 (void)SEC_RD4(sc, SEC_FAR_LO); 466 (void)SEC_RD4(sc, SEC_FALR); 467 468 if (bus_setup_intr(dev, sc->sc_ires, INTR_TYPE_MISC | INTR_MPSAFE, 469 NULL, sec_intr, sc, &sc->sc_icookie) != 0) { 470 device_printf(dev, "could not install error interrupt\n"); 471 goto fail; 472 } 473 474 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session), 475 CRYPTOCAP_F_HARDWARE); 476 if (sc->sc_cid < 0) { 477 device_printf(dev, "could not get crypto driver id\n"); 478 goto fail; 479 } 480 481 return (0); 482 483 fail: 484 sec_detach(dev); 485 return (ENXIO); 486 } 487 488 static int 489 sec_detach(device_t dev) 490 { 491 struct sec_softc *sc = device_get_softc(dev); 492 u_int i; 493 494 if (sc->sc_cid >= 0) 495 crypto_unregister_all(sc->sc_cid); 496 497 /* Silence the rings before halting them. */ 498 for (i = 0; i < sc->sc_njr; i++) { 499 struct sec_jr *jr = &sc->sc_jr[i]; 500 501 if (jr->jr_icookie != NULL) 502 bus_teardown_intr(dev, jr->jr_ires, jr->jr_icookie); 503 if (jr->jr_ires != NULL) 504 bus_release_resource(dev, SYS_RES_IRQ, jr->jr_irid, 505 jr->jr_ires); 506 sec_jr_teardown(sc, jr); 507 } 508 free(sc->sc_jr, M_SEC); 509 510 if (sc->sc_dmatag != NULL) 511 bus_dma_tag_destroy(sc->sc_dmatag); 512 if (sc->sc_icookie != NULL) 513 bus_teardown_intr(dev, sc->sc_ires, sc->sc_icookie); 514 if (sc->sc_ires != NULL) 515 bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irid, 516 sc->sc_ires); 517 if (sc->sc_rres != NULL) 518 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 519 sc->sc_rres); 520 521 return (0); 522 } 523 524 static const char *sec_ferr_str[] = { 525 "OKAY", "reserved", "SLVERR", "DECERR", 526 }; 527 528 static const char *sec_jsrc_str[] = { 529 "JR0", "JR1", "JR2", "JR3", "RTIC", "QI", "rsvd6", "rsvd7", 530 }; 531 532 static void 533 sec_intr(void *arg) 534 { 535 struct sec_softc *sc = arg; 536 uint32_t fadr, falr; 537 uint64_t far; 538 539 fadr = SEC_RD4(sc, SEC_FADR); 540 if ((fadr & FADR_FERR_M) != 0) { 541 /* 542 * All fault registers are latched by hardware until all are 543 * read, in any order. 544 */ 545 far = (uint64_t)SEC_RD4(sc, SEC_FAR_HI) << 32; 546 far |= SEC_RD4(sc, SEC_FAR_LO); 547 falr = SEC_RD4(sc, SEC_FALR); 548 549 device_printf(sc->sc_dev, 550 "bus fault: FADR=%#x FAR=%#jx FALR=%#x " 551 "(%s, %s, src=%s, blkid=%#x, %s, size=%u)\n", 552 fadr, (uintmax_t)far, falr, 553 sec_ferr_str[(fadr & FADR_FERR_M) >> FADR_FERR_S], 554 (fadr & FADR_DTYP) ? "control" : "message", 555 sec_jsrc_str[(fadr & FADR_JSRC_M) >> FADR_JSRC_S], 556 (fadr & FADR_BLKID_M) >> FADR_BLKID_S, 557 (fadr & FADR_TYP) ? "write" : "read", 558 (unsigned)(((fadr & FADR_FSZ_EXT_M) >> 559 (FADR_FSZ_EXT_S - 7)) | (fadr & FADR_FSZ_M))); 560 } 561 562 } 563 564 /* 565 * Decode a SEC job termination status word. 566 * 567 * Bits 0-3 (MSB) are the "source" of the report; the remaining bits are 568 * source-specific. Zero means clean completion. 569 * 570 * Two cases we care to distinguish: 571 * - CCB (source 2), ERRID field bits 28-31 572 * value 0xA is "ICV check failed" -> EBADMSG. 573 * - DECO (source 4), Error Code bits 24-31 574 * values F0h/F1h/FFh are informational warnings (IPsec TTL, 575 * 3GPP HFN, output-length rollover). The job actually completed, 576 * so map those to success. 577 * 578 * Everything else is logged and reported as EIO. Real per-code 579 * decoding of DECO/QI errors can be layered on as we hit them. 580 */ 581 #define SEC_STAT_SOURCE(s) (((s) >> 28) & 0xf) 582 #define SEC_SRC_NONE 0x0 583 #define SEC_SRC_CCB 0x2 584 #define SEC_SRC_DECO 0x4 585 #define SEC_SRC_QI 0x5 586 #define SEC_SRC_JR 0x6 587 #define SEC_CCB_ERR_ICV_FAIL 0x0a 588 #define SEC_DECO_ERR_WARN_MIN 0xf0 589 590 static int 591 sec_decode_status(struct sec_softc *sc, uint32_t status) 592 { 593 uint32_t source; 594 595 if (status == 0) 596 return (0); 597 598 source = SEC_STAT_SOURCE(status); 599 600 switch (source) { 601 case SEC_SRC_CCB: 602 if ((status & 0xf) == SEC_CCB_ERR_ICV_FAIL) 603 return (EBADMSG); 604 break; 605 case SEC_SRC_DECO: 606 if ((status & 0xff) >= SEC_DECO_ERR_WARN_MIN) 607 return (0); 608 break; 609 } 610 611 device_printf(sc->sc_dev, 612 "job termination status %#x (source %#x)\n", status, source); 613 return (EIO); 614 } 615 616 /* 617 * Complete one job that SEC has finished processing. 618 */ 619 void 620 sec_complete_one(struct sec_softc *sc, uint64_t desc_pa, uint32_t status) 621 { 622 struct sec_job *job; 623 struct cryptop *crp; 624 const struct crypto_session_params *csp; 625 uint8_t expected[SEC_MAX_DIGEST]; 626 int dlen; 627 628 job = (struct sec_job *)PHYS_TO_DMAP((vm_paddr_t)desc_pa); 629 crp = job->crp; 630 631 crp->crp_etype = sec_decode_status(sc, status); 632 633 bus_dmamap_sync(sc->sc_dmatag, job->map, 634 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 635 bus_dmamap_unload(sc->sc_dmatag, job->map); 636 bus_dmamap_destroy(sc->sc_dmatag, job->map); 637 638 if (crp->crp_etype == 0) { 639 csp = crypto_get_params(crp->crp_session); 640 dlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen : 641 job->sess->digestlen; 642 switch (csp->csp_mode) { 643 case CSP_MODE_DIGEST: 644 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { 645 crypto_copydata(crp, crp->crp_digest_start, 646 dlen, expected); 647 if (timingsafe_bcmp(job->digest, expected, 648 dlen) != 0) 649 crp->crp_etype = EBADMSG; 650 } else { 651 crypto_copyback(crp, crp->crp_digest_start, 652 dlen, job->digest); 653 } 654 break; 655 case CSP_MODE_ETA: 656 if ((crp->crp_op & CRYPTO_OP_ENCRYPT) != 0) { 657 crypto_copyback(crp, crp->crp_digest_start, 658 dlen, job->digest); 659 break; 660 } 661 crypto_copydata(crp, crp->crp_digest_start, dlen, 662 expected); 663 if (timingsafe_bcmp(job->digest, expected, dlen) != 0) 664 crp->crp_etype = EBADMSG; 665 break; 666 case CSP_MODE_AEAD: 667 if ((crp->crp_op & CRYPTO_OP_ENCRYPT) != 0) 668 crypto_copyback(crp, crp->crp_digest_start, 669 dlen, job->digest); 670 break; 671 } 672 } 673 674 crypto_done(crp); 675 free(job, M_SEC); 676 } 677 678 static bool 679 check_cipher(const struct crypto_session_params *csp) 680 { 681 682 switch (csp->csp_cipher_alg) { 683 case CRYPTO_AES_CBC: 684 case CRYPTO_AES_ICM: 685 if (csp->csp_ivlen != AES_BLOCK_LEN) 686 return (false); 687 return (csp->csp_cipher_klen == 16 || 688 csp->csp_cipher_klen == 24 || 689 csp->csp_cipher_klen == 32); 690 case CRYPTO_AES_XTS: 691 if (csp->csp_ivlen != AES_XTS_IV_LEN) 692 return (false); 693 return (csp->csp_cipher_klen == 32 || 694 csp->csp_cipher_klen == 64); 695 default: 696 return (false); 697 } 698 } 699 700 static bool 701 check_aead(const struct crypto_session_params *csp) 702 { 703 704 switch (csp->csp_cipher_alg) { 705 case CRYPTO_AES_NIST_GCM_16: 706 if (csp->csp_auth_mlen != 0 && 707 csp->csp_auth_mlen != AES_GMAC_HASH_LEN) 708 return (false); 709 return (csp->csp_cipher_klen == 16 || 710 csp->csp_cipher_klen == 24 || 711 csp->csp_cipher_klen == 32); 712 case CRYPTO_AES_CCM_16: 713 return (csp->csp_cipher_klen == 16 || 714 csp->csp_cipher_klen == 24 || 715 csp->csp_cipher_klen == 32); 716 default: 717 return (false); 718 } 719 } 720 721 /* 722 * Map an opencrypto auth_alg to its SEC selector and digest length. 723 * skeylen is zero for a plain hash, which is what tells the two apart. 724 */ 725 static bool 726 sec_hash_params(int auth_alg, uint32_t *alg, uint8_t *dlen, uint8_t *skeylen) 727 { 728 729 switch (auth_alg) { 730 case CRYPTO_SHA1_HMAC: 731 *alg = ALG_SHA1; *dlen = 20; *skeylen = 40; return (true); 732 case CRYPTO_SHA2_224_HMAC: 733 *alg = ALG_SHA224; *dlen = 28; *skeylen = 64; return (true); 734 case CRYPTO_SHA2_256_HMAC: 735 *alg = ALG_SHA256; *dlen = 32; *skeylen = 64; return (true); 736 case CRYPTO_SHA2_384_HMAC: 737 *alg = ALG_SHA384; *dlen = 48; *skeylen = 128; return (true); 738 case CRYPTO_SHA2_512_HMAC: 739 *alg = ALG_SHA512; *dlen = 64; *skeylen = 128; return (true); 740 case CRYPTO_SHA1: 741 *alg = ALG_SHA1; *dlen = 20; *skeylen = 0; return (true); 742 case CRYPTO_SHA2_224: 743 *alg = ALG_SHA224; *dlen = 28; *skeylen = 0; return (true); 744 case CRYPTO_SHA2_256: 745 *alg = ALG_SHA256; *dlen = 32; *skeylen = 0; return (true); 746 case CRYPTO_SHA2_384: 747 *alg = ALG_SHA384; *dlen = 48; *skeylen = 0; return (true); 748 case CRYPTO_SHA2_512: 749 *alg = ALG_SHA512; *dlen = 64; *skeylen = 0; return (true); 750 } 751 return (false); 752 } 753 754 static bool 755 check_digest(const struct crypto_session_params *csp) 756 { 757 uint32_t alg; 758 uint8_t dlen, skeylen; 759 760 /* GMAC is AESA rather than MDHA, so it has its own constraints. */ 761 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { 762 if (csp->csp_ivlen != AES_GCM_IV_LEN) 763 return (false); 764 if (csp->csp_auth_mlen > AES_GMAC_HASH_LEN) 765 return (false); 766 return (csp->csp_auth_klen == 16 || 767 csp->csp_auth_klen == 24 || 768 csp->csp_auth_klen == 32); 769 } 770 771 if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen)) 772 return (false); 773 /* Keyed variants require a key; plain hashes must not carry one. */ 774 if ((skeylen != 0) != (csp->csp_auth_klen != 0)) 775 return (false); 776 return (csp->csp_auth_mlen <= dlen); 777 } 778 779 static bool 780 check_eta(const struct crypto_session_params *csp) 781 { 782 783 /* 784 * ESN appends four bytes from crp_esn to the MAC input, which the 785 * descriptor has no way to splice in, so refuse rather than 786 * authenticate the wrong span. 787 */ 788 if ((csp->csp_flags & CSP_F_ESN) != 0) 789 return (false); 790 /* 791 * XTS carries its tweak in the class 1 context and pairs with no 792 * MAC; its shared descriptor is shaped differently. 793 */ 794 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 795 return (false); 796 /* The MAC half has to be keyed; a bare hash authenticates nothing. */ 797 if (csp->csp_auth_klen == 0) 798 return (false); 799 return (check_cipher(csp) && check_digest(csp)); 800 } 801 802 /* 803 * Software split-key generator: computes the HMAC ipad/opad hash-state 804 * halves in software and packs them big-endian for SEC's Class 2 KEY 805 * register. 806 * 807 * Runs the CPU through one SHA block per pad (two total). Much cheaper than 808 * the round trip through the job ring for setup. 809 */ 810 static void 811 sec_pack_state32(uint8_t *dst, const uint32_t *src, unsigned int nbytes) 812 { 813 unsigned int i; 814 815 for (i = 0; i < nbytes; i += 4) 816 be32enc(dst + i, src[i / 4]); 817 } 818 819 static void 820 sec_pack_state64(uint8_t *dst, const uint64_t *src, unsigned int nbytes) 821 { 822 unsigned int i; 823 824 for (i = 0; i < nbytes; i += 8) 825 be64enc(dst + i, src[i / 8]); 826 } 827 828 static void 829 sec_sw_gen_split_key(const struct crypto_session_params *csp, 830 uint8_t *out, size_t out_len) 831 { 832 union authctx ictx, octx; 833 const struct auth_hash *axf; 834 uint8_t half; 835 836 axf = crypto_auth_hash(csp); 837 hmac_init_ipad(axf, csp->csp_auth_key, csp->csp_auth_klen, &ictx); 838 hmac_init_opad(axf, csp->csp_auth_key, csp->csp_auth_klen, &octx); 839 840 KASSERT(out_len % 2 == 0, ("split key len must be even")); 841 half = out_len / 2; 842 843 switch (csp->csp_auth_alg) { 844 case CRYPTO_SHA1_HMAC: 845 sec_pack_state32(out, ictx.sha1ctx.h.b32, half); 846 sec_pack_state32(out + half, octx.sha1ctx.h.b32, half); 847 break; 848 case CRYPTO_SHA2_224_HMAC: 849 sec_pack_state32(out, ictx.sha224ctx.state, half); 850 sec_pack_state32(out + half, octx.sha224ctx.state, half); 851 break; 852 case CRYPTO_SHA2_256_HMAC: 853 sec_pack_state32(out, ictx.sha256ctx.state, half); 854 sec_pack_state32(out + half, octx.sha256ctx.state, half); 855 break; 856 case CRYPTO_SHA2_384_HMAC: 857 sec_pack_state64(out, ictx.sha384ctx.state, half); 858 sec_pack_state64(out + half, octx.sha384ctx.state, half); 859 break; 860 case CRYPTO_SHA2_512_HMAC: 861 sec_pack_state64(out, ictx.sha512ctx.state, half); 862 sec_pack_state64(out + half, octx.sha512ctx.state, half); 863 break; 864 } 865 866 explicit_bzero(&ictx, sizeof(ictx)); 867 explicit_bzero(&octx, sizeof(octx)); 868 } 869 870 /* 871 * Descriptor builder. Word 0 is the HEADER and is filled in last, since its 872 * length field is only known once the body has been emitted. 873 */ 874 struct sec_desc_builder { 875 uint32_t *desc; 876 unsigned int idx; /* next word to write */ 877 unsigned int max; 878 int err; 879 }; 880 881 static inline void 882 sec_desc_init(struct sec_desc_builder *b, uint32_t *desc, unsigned int max) 883 { 884 885 b->desc = desc; 886 b->idx = 1; /* reserve word 0 for the HEADER */ 887 b->max = max; 888 b->err = 0; 889 } 890 891 static inline void 892 sec_desc_word(struct sec_desc_builder *b, uint32_t w) 893 { 894 895 if (b->err != 0) 896 return; 897 if (b->idx >= b->max) { 898 b->err = ENOSPC; 899 return; 900 } 901 b->desc[b->idx++] = w; 902 } 903 904 /* Emit a KEY command with the key inline after it. */ 905 static inline void 906 sec_desc_key_imm(struct sec_desc_builder *b, uint32_t class, 907 const void *key, unsigned int keylen) 908 { 909 unsigned int nwords = howmany(keylen, sizeof(uint32_t)); 910 911 if (b->err != 0) 912 return; 913 if (b->idx + 1 + nwords > b->max) { 914 b->err = ENOSPC; 915 return; 916 } 917 b->desc[b->idx++] = CMD_DESC(CMD_KEY) | class | KEY_IMM | 918 (keylen & KEY_LENGTH_M); 919 memcpy(&b->desc[b->idx], key, keylen); 920 b->idx += nwords; 921 } 922 923 static int 924 sec_desc_finalize_shared(struct sec_desc_builder *b, uint32_t flags, 925 uint32_t *sdlenp) 926 { 927 928 if (b->err != 0) 929 return (b->err); 930 if (b->idx > SEC_MAX_SHDESC_WORDS) 931 return (ENOSPC); 932 b->desc[0] = CMD_DESC(CMD_SHARED_HEADER) | HEADER_ONE | 933 (flags & (HEADER_SHARE_M | HEADER_SC)) | 934 (b->idx & HEADER_DESCLEN_M); 935 *sdlenp = b->idx; 936 return (0); 937 } 938 939 static int 940 sec_desc_finalize_job(struct sec_desc_builder *b, uint32_t word, 941 uint32_t *dlenp) 942 { 943 944 if (b->err != 0) 945 return (b->err); 946 if (b->idx > SEC_MAX_DESC_WORDS) 947 return (ENOSPC); 948 b->desc[0] = CMD_DESC(CMD_DESC_HEADER) | HEADER_ONE | 949 word | (b->idx & HEADER_DESCLEN_M); 950 *dlenp = b->idx; 951 return (0); 952 } 953 954 955 /* 956 * Job descriptor builder conveniences. 957 */ 958 959 static inline void 960 sec_jd_ptr(struct sec_desc_builder *b, vm_paddr_t pa) 961 { 962 sec_desc_word(b, (uint32_t)(pa >> 32)); 963 sec_desc_word(b, (uint32_t)pa); 964 } 965 966 /* Build a SEQ_IN/SEQ_OUT descriptor command. */ 967 static inline void 968 sec_jd_seq(struct sec_desc_builder *b, bool inout, uint32_t flags, 969 vm_paddr_t ptr, uint32_t len) 970 { 971 sec_desc_word(b, 972 CMD_DESC(inout ? CMD_SEQ_OUT_PTR : CMD_SEQ_IN_PTR) | flags); 973 sec_jd_ptr(b, ptr); 974 sec_desc_word(b, len); 975 } 976 977 static inline void 978 sec_jd_load(struct sec_desc_builder *b, bool seq, uint32_t class, 979 uint32_t flags, uint32_t dst, uint32_t off, uint32_t len, vm_paddr_t ptr) 980 { 981 uint32_t cmd = seq ? CMD_SEQ_LOAD : CMD_LOAD; 982 983 sec_desc_word(b, CMD_DESC(cmd) | class | flags | dst | 984 (off << LOAD_OFFSET_S) | (len & LOAD_LENGTH_M)); 985 if (!seq) 986 sec_jd_ptr(b, ptr); 987 } 988 989 static inline void 990 sec_jd_store(struct sec_desc_builder *b, bool seq, uint32_t class, uint32_t src, 991 uint32_t off, uint32_t len, vm_paddr_t ptr) 992 { 993 uint32_t cmd = seq ? CMD_SEQ_STORE : CMD_STORE; 994 995 sec_desc_word(b, CMD_DESC(cmd) | class | src | 996 (off << LOAD_OFFSET_S) | (len & LOAD_LENGTH_M)); 997 if (!seq) 998 sec_jd_ptr(b, ptr); 999 } 1000 1001 static inline void 1002 sec_jd_fifo(struct sec_desc_builder *b, uint32_t cmd, uint32_t len) 1003 { 1004 1005 if (len > 0xffff) { 1006 sec_desc_word(b, cmd | FIFO_EXT); 1007 sec_desc_word(b, len); 1008 } else { 1009 sec_desc_word(b, cmd | len); 1010 } 1011 } 1012 1013 /* 1014 * AES-XTS Class 1 context layout (byte offsets into the CTX register). 1015 * The 16-byte tweak is split either side of the sector-size field. 1016 */ 1017 #define SEC_XTS_CTX_TWEAK_LO 0x20 1018 #define SEC_XTS_CTX_SECTOR 0x28 1019 #define SEC_XTS_CTX_TWEAK_HI 0x30 1020 1021 /* 1022 * Sector size tells the hardware how often to re-derive the tweak. 1023 * opencrypto's XTS runs one continuous tweak over the whole request, so 1024 * this only needs to exceed any payload we accept; sec_jd_build_cipher 1025 * rejects requests that would cross the boundary. 1026 */ 1027 #define SEC_XTS_SECTOR_SIZE 0x8000 1028 1029 /* 1030 * Build the CCM context block and formatted-AAD length prefix. 1031 * 1032 * The hardware wants B0 in context dwords 0-1 and the initial counter 1033 * CTR0 in dwords 2-3, with dwords 4-6 zeroed because AS is 1034 * INITIALIZE/FINALIZE. Both blocks are laid out per RFC 3610: with a 1035 * nonce of n bytes, the length field occupies the trailing L = 15 - n 1036 * bytes and the flags byte carries L-1 plus, for B0, the encoded tag 1037 * size and an AAD-present flag. 1038 * 1039 * The AAD itself is prefixed with its length and then zero-padded to a 1040 * 16-byte boundary by the hardware, which pads AAD and IV FIFO loads 1041 * when the flush-class-1 bit is set. 1042 */ 1043 static int 1044 sec_ccm_prep(struct sec_job *job, const struct crypto_session_params *csp) 1045 { 1046 uint8_t *b0 = job->ccm_ctx; 1047 uint8_t *ctr0 = job->ccm_ctx + 16; 1048 uint32_t aadlen = job->crp->crp_aad_length; 1049 uint64_t paylen = job->crp->crp_payload_length; 1050 u_int i, lfield = 15 - csp->csp_ivlen; 1051 1052 if (aadlen > SEC_CCM_AAD_MAX) 1053 return (EOPNOTSUPP); 1054 /* 1055 * B0 carries the payload length in its trailing lfield bytes, so 1056 * the nonce is what really caps the payload: a 13-byte nonce 1057 * leaves two bytes and stops at 64 KB, while the usual 12-byte one 1058 * leaves three and reaches 16 MB. 1059 */ 1060 if (lfield < sizeof(paylen) && paylen >= (uint64_t)1 << (8 * lfield)) 1061 return (EOPNOTSUPP); 1062 1063 memset(job->ccm_ctx, 0, sizeof(job->ccm_ctx)); 1064 1065 b0[0] = (aadlen > 0 ? 0x40 : 0x00) | 1066 (((job->sess->digestlen - 2) / 2) << 3) | (lfield - 1); 1067 memcpy(b0 + 1, job->iv, csp->csp_ivlen); 1068 for (i = 0; i < lfield; i++) 1069 b0[15 - i] = (paylen >> (8 * i)) & 0xff; 1070 1071 ctr0[0] = lfield - 1; 1072 memcpy(ctr0 + 1, job->iv, csp->csp_ivlen); 1073 1074 be16enc(job->ccm_alen, aadlen); 1075 return (0); 1076 } 1077 1078 /* 1079 * Bytes of IV the input sequence carries. XTS is the odd one out: 1080 * opencrypto's IV is just the 8-byte block number, but the hardware 1081 * loads both halves of the 16-byte tweak from the sequence. 1082 */ 1083 static uint32_t 1084 sec_cipher_ivlen(const struct crypto_session_params *csp) 1085 { 1086 1087 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 1088 return (AES_BLOCK_LEN); 1089 return (csp->csp_ivlen); 1090 } 1091 1092 /* 1093 * Expand opencrypto's 8-byte XTS IV in place into the 16-byte tweak the 1094 * hardware expects. The IV holds a block number in host order 1095 * (xform_aes_xts.c:aes_xts_reinit) and the tweak is that number's 1096 * little-endian encoding followed by zeroes. 1097 */ 1098 static void 1099 sec_xts_tweak(uint8_t *iv) 1100 { 1101 uint64_t blocknum; 1102 1103 memcpy(&blocknum, iv, sizeof(blocknum)); 1104 le64enc(iv, blocknum); 1105 memset(iv + sizeof(blocknum), 0, AES_BLOCK_LEN - sizeof(blocknum)); 1106 } 1107 1108 static uint32_t 1109 sec_cipher_ctx_offset(uint32_t cipher_alg) 1110 { 1111 switch (cipher_alg) { 1112 case CRYPTO_AES_ICM: 1113 return (16); 1114 }; 1115 1116 return (0); 1117 } 1118 1119 /* Per-mode shared-descriptor builders. */ 1120 /* 1121 * Cipher shared descriptor has the following format: 1122 * [0] - Header 1123 * [1..klen] - KEY descriptor + key 1124 * [XTS:..5] -- XTS specific 1125 * [0..2] - LOAD XTS context 1126 * [3..4] - LOAD XTS tweak 1127 * [!XTS:1] -- Load IV into Context register 1128 * [] - Operation 1129 * [] - MATH - Move SIL register to VSIL for FIFO IN 1130 * [] - MATH - Move SOL register to VSOL for FIFO OUT 1131 * [] - FIFO LOAD 1132 * [] - FIFO STORE 1133 */ 1134 static int 1135 sec_shd_build_cipher(struct sec_session *sess, 1136 const struct crypto_session_params *csp, int enc, uint32_t *sdlenp) 1137 { 1138 struct sec_desc_builder b; 1139 uint32_t flags, op; 1140 uint32_t ctx_offset; 1141 1142 switch (csp->csp_cipher_alg) { 1143 case CRYPTO_AES_CBC: 1144 op = CMD_DESC(CMD_OPERATION) | ALG_AES | 1145 AAI_AES_CBC | AS_INIT_FINAL; 1146 break; 1147 case CRYPTO_AES_ICM: 1148 op = CMD_DESC(CMD_OPERATION) | ALG_AES | 1149 AAI_AES_CTR | AS_INIT_FINAL; 1150 break; 1151 case CRYPTO_AES_XTS: 1152 op = CMD_DESC(CMD_OPERATION) | ALG_AES | 1153 AAI_AES_XTS | AS_INIT_FINAL; 1154 break; 1155 default: 1156 return (EOPNOTSUPP); 1157 } 1158 1159 ctx_offset = sec_cipher_ctx_offset(csp->csp_cipher_alg); 1160 1161 if (enc) 1162 op |= OP_ENC; 1163 1164 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS); 1165 1166 if (csp->csp_cipher_klen > 0) 1167 sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_cipher_key, 1168 csp->csp_cipher_klen); 1169 1170 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) { 1171 sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_CTX, 1172 SEC_XTS_CTX_SECTOR, 8, SEC_XTS_SECTOR_SIZE); 1173 1174 sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX, 1175 SEC_XTS_CTX_TWEAK_LO, 8, 0); 1176 sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX, 1177 SEC_XTS_CTX_TWEAK_HI, 8, 0); 1178 } else { 1179 sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX, ctx_offset, 1180 csp->csp_ivlen, 0); 1181 } 1182 1183 sec_desc_word(&b, op); 1184 1185 /* 1186 * Copy SIL into VSIL and VSOL so the following VLF-flagged FIFO 1187 * commands know how many bytes to move. VLF reads the VS*L 1188 * registers, so we need to get the values from the SEQ registers 1189 * the SEQ IN/OUT PTR descriptors populate. 1190 */ 1191 sec_desc_word(&b, CMD_DESC(CMD_MATH) | MATH_FN_ADD | MATH_SRC0_SIL | 1192 MATH_SRC1_ZERO | MATH_DEST_VSIL | MATH_LEN_4); 1193 sec_desc_word(&b, CMD_DESC(CMD_MATH) | MATH_FN_ADD | MATH_SRC0_SIL | 1194 MATH_SRC1_ZERO | MATH_DEST_VSOL | MATH_LEN_4); 1195 1196 sec_desc_word(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 | 1197 FIFOLD_VLF | FIFOLD_TYPE_MSG | FIFOLD_LC1); 1198 1199 sec_desc_word(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) | FIFOST_VLF | 1200 FIFOST_TYPE_MSG_DATA); 1201 1202 /* XTS keeps its tweak in the context, so the CCB has to save it. */ 1203 flags = HEADER_SHARE_SERIAL; 1204 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 1205 flags |= HEADER_SC; 1206 1207 return (sec_desc_finalize_shared(&b, flags, sdlenp)); 1208 } 1209 1210 /* 1211 * Digest shared descriptor. The split key is computed in software at 1212 * session setup, so MDHA is told it is precomputed and skips the 1213 * ipad/opad expansion. 1214 */ 1215 static int 1216 sec_shd_build_digest(struct sec_session *sess, 1217 const struct crypto_session_params *csp, int enc, uint32_t *sdlenp) 1218 { 1219 struct sec_desc_builder b; 1220 uint32_t alg; 1221 uint8_t dlen, skeylen; 1222 1223 /* 1224 * GMAC runs on AESA, not MDHA: the shared descriptor is just the 1225 * class 1 key, and the JD drives it as GCM with no message. 1226 */ 1227 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { 1228 sess->digestlen = csp->csp_auth_mlen != 0 ? 1229 csp->csp_auth_mlen : AES_GMAC_HASH_LEN; 1230 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS); 1231 sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_auth_key, 1232 csp->csp_auth_klen); 1233 return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, 1234 sdlenp)); 1235 } 1236 1237 if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen)) 1238 return (EOPNOTSUPP); 1239 sess->digestlen = dlen; 1240 1241 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS); 1242 1243 /* 1244 * A plain hash takes no key at all; the keyed variants load the 1245 * precomputed ipad || opad blob as an MDHA split key, which is 1246 * what AAI_HMAC_PRECOMP tells MDHA to expect. 1247 */ 1248 if (skeylen != 0) { 1249 unsigned int nwords = howmany(skeylen, 4); 1250 1251 b.desc[b.idx++] = CMD_DESC(CMD_KEY) | KEY_CLASS_2 | 1252 KEY_KDEST_MDHA_SPLIT | KEY_IMM | 1253 (skeylen & KEY_LENGTH_M); 1254 memcpy(&b.desc[b.idx], sess->skey, skeylen); 1255 if (skeylen % 4 != 0) 1256 memset((uint8_t *)&b.desc[b.idx] + skeylen, 0, 1257 nwords * 4 - skeylen); 1258 b.idx += nwords; 1259 } 1260 1261 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | alg | 1262 (skeylen != 0 ? AAI_HMAC_PRECOMP : AAI_HASH) | AS_INIT_FINAL); 1263 1264 /* VLF FIFO_LOAD needs VSIL, which SEQ_IN_PTR doesn't populate. */ 1265 sec_desc_word(&b, CMD_DESC(CMD_MATH) | MATH_FN_ADD | MATH_SRC0_SIL | 1266 MATH_SRC1_ZERO | MATH_DEST_VSIL | MATH_LEN_4); 1267 1268 sec_desc_word(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_2 | 1269 FIFOLD_VLF | FIFOLD_TYPE_MSG | FIFOLD_LC2); 1270 1271 /* 1272 * Drain the completed hash from the Class 2 CCB Context register. 1273 * SEQ_STORE with class 2 + SRC=CTX (0x20) blocks until MDHA is 1274 * done. 1275 */ 1276 sec_jd_store(&b, true, LOAD_CLASS_2, LOAD_CTX, 0, dlen, 0); 1277 1278 return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, sdlenp)); 1279 } 1280 1281 /* 1282 * AEAD shared descriptor. AAD and payload lengths vary per job, so 1283 * everything but the key lives in the JD. Execution order is not 1284 * reversed here: the key has to be loaded before the JD drives data. 1285 * 1286 * When ICV is set the ENC bit must be clear, which is the only 1287 * difference between the two direction slots. 1288 */ 1289 static int 1290 sec_shd_build_aead(struct sec_session *sess, 1291 const struct crypto_session_params *csp, int enc, uint32_t *sdlenp) 1292 { 1293 struct sec_desc_builder b; 1294 1295 switch (csp->csp_cipher_alg) { 1296 case CRYPTO_AES_NIST_GCM_16: 1297 case CRYPTO_AES_CCM_16: 1298 break; 1299 default: 1300 return (EOPNOTSUPP); 1301 } 1302 (void)enc; 1303 1304 /* Cache tag length once (both directions share). */ 1305 sess->digestlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen : 16; 1306 1307 /* 1308 * SHD holds just the AES key. 1309 */ 1310 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS); 1311 if (csp->csp_cipher_klen > 0) 1312 sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_cipher_key, 1313 csp->csp_cipher_klen); 1314 return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, sdlenp)); 1315 } 1316 1317 /* 1318 * Shared descriptor for encrypt-then-auth: both keys and both mode 1319 * registers, nothing else. 1320 */ 1321 static int 1322 sec_shd_build_eta(struct sec_session *sess, 1323 const struct crypto_session_params *csp, int enc, uint32_t *sdlenp) 1324 { 1325 struct sec_desc_builder b; 1326 uint32_t alg, op; 1327 uint8_t dlen, skeylen; 1328 1329 if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen)) 1330 return (EOPNOTSUPP); 1331 sess->digestlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen : dlen; 1332 1333 switch (csp->csp_cipher_alg) { 1334 case CRYPTO_AES_CBC: 1335 op = CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_CBC | 1336 AS_INIT_FINAL; 1337 break; 1338 case CRYPTO_AES_ICM: 1339 op = CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_CTR | 1340 AS_INIT_FINAL; 1341 break; 1342 default: 1343 return (EOPNOTSUPP); 1344 } 1345 if (enc) 1346 op |= OP_ENC; 1347 1348 sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS); 1349 1350 sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_cipher_key, 1351 csp->csp_cipher_klen); 1352 1353 /* Class 2 takes the precomputed ipad/opad blob, as for plain HMAC. */ 1354 { 1355 unsigned int nwords = howmany(skeylen, 4); 1356 1357 b.desc[b.idx++] = CMD_DESC(CMD_KEY) | KEY_CLASS_2 | 1358 KEY_KDEST_MDHA_SPLIT | KEY_IMM | (skeylen & KEY_LENGTH_M); 1359 memcpy(&b.desc[b.idx], sess->skey, skeylen); 1360 if (skeylen % 4 != 0) 1361 memset((uint8_t *)&b.desc[b.idx] + skeylen, 0, 1362 nwords * 4 - skeylen); 1363 b.idx += nwords; 1364 } 1365 1366 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | alg | AAI_HMAC_PRECOMP | 1367 AS_INIT_FINAL); 1368 sec_desc_word(&b, op); 1369 1370 return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, sdlenp)); 1371 } 1372 1373 static int 1374 sec_probe_session(device_t dev, const struct crypto_session_params *csp) 1375 { 1376 1377 switch (csp->csp_mode) { 1378 case CSP_MODE_CIPHER: 1379 if (!check_cipher(csp)) 1380 return (EINVAL); 1381 break; 1382 case CSP_MODE_DIGEST: 1383 if (!check_digest(csp)) 1384 return (EINVAL); 1385 break; 1386 case CSP_MODE_AEAD: 1387 if (!check_aead(csp)) 1388 return (EINVAL); 1389 break; 1390 case CSP_MODE_ETA: 1391 if (!check_eta(csp)) 1392 return (EINVAL); 1393 break; 1394 default: 1395 return (EINVAL); 1396 } 1397 return (CRYPTODEV_PROBE_HARDWARE); 1398 } 1399 1400 static int 1401 sec_new_session(device_t dev, crypto_session_t session, 1402 const struct crypto_session_params *csp) 1403 { 1404 struct sec_softc *sc = device_get_softc(dev); 1405 struct sec_session *sess; 1406 uint32_t sdlen; 1407 int enc, error; 1408 1409 sess = crypto_get_driver_session(session); 1410 sess->sess_sc = sc; 1411 1412 if ((csp->csp_mode == CSP_MODE_DIGEST || 1413 csp->csp_mode == CSP_MODE_ETA) && csp->csp_auth_klen > 0 && 1414 csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) { 1415 uint32_t alg; 1416 uint8_t dlen, skeylen; 1417 1418 if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen)) 1419 return (EOPNOTSUPP); 1420 (void)alg; 1421 sec_sw_gen_split_key(csp, sess->skey, skeylen); 1422 sess->skeylen = skeylen; 1423 } 1424 1425 for (enc = 0; enc <= 1; enc++) { 1426 switch (csp->csp_mode) { 1427 case CSP_MODE_CIPHER: 1428 error = sec_shd_build_cipher(sess, csp, enc, &sdlen); 1429 break; 1430 case CSP_MODE_DIGEST: 1431 error = sec_shd_build_digest(sess, csp, enc, &sdlen); 1432 break; 1433 case CSP_MODE_AEAD: 1434 error = sec_shd_build_aead(sess, csp, enc, &sdlen); 1435 break; 1436 case CSP_MODE_ETA: 1437 error = sec_shd_build_eta(sess, csp, enc, &sdlen); 1438 break; 1439 default: 1440 return (EINVAL); 1441 } 1442 if (error != 0) 1443 return (error); 1444 sess->sdlen[enc] = sdlen; 1445 } 1446 return (0); 1447 } 1448 1449 static void 1450 sec_free_session(device_t dev, crypto_session_t session) 1451 { 1452 /* Nothing to do here. */ 1453 } 1454 1455 static void 1456 sec_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 1457 { 1458 struct sec_job *job = arg; 1459 1460 if (error != 0) { 1461 job->nsegs = 0; 1462 return; 1463 } 1464 KASSERT(nsegs <= SEC_MAX_SEGMENTS, 1465 ("SEC job segment overflow: %d > %d", nsegs, SEC_MAX_SEGMENTS)); 1466 memcpy(job->segs, segs, nsegs * sizeof(segs[0])); 1467 job->nsegs = nsegs; 1468 } 1469 1470 /* 1471 * Append the segments covering [start, start + len) of the mapped buffer. 1472 * Returns the next free index, or -1 if the table would overflow or the 1473 * range runs past the mapping. 1474 */ 1475 static int 1476 dpaa_sgte_append(struct sec_job *job, struct dpaa_sgte *sgt, int i, int max, 1477 uint32_t start, uint32_t len) 1478 { 1479 int s; 1480 1481 for (s = 0; s < job->nsegs && len > 0; s++) { 1482 bus_addr_t addr = job->segs[s].ds_addr; 1483 bus_size_t seglen = job->segs[s].ds_len; 1484 uint32_t take; 1485 1486 if (start >= seglen) { 1487 start -= seglen; 1488 continue; 1489 } 1490 addr += start; 1491 seglen -= start; 1492 start = 0; 1493 1494 take = seglen > len ? len : seglen; 1495 len -= take; 1496 1497 if (i >= max) 1498 return (-1); 1499 sgt[i].addr = addr; 1500 sgt[i].extension = 0; 1501 sgt[i].final = 0; 1502 sgt[i].length = take; 1503 sgt[i].bpid = 0; 1504 sgt[i].offset = 0; 1505 i++; 1506 } 1507 if (len != 0) 1508 return (-1); 1509 return (i); 1510 } 1511 1512 /* 1513 * Populate the SGTs from the DMA-loaded segment list. Entry order is 1514 * what the descriptor's SEQ commands consume, so it is fixed per mode: 1515 * CIPHER: 1516 * in_sgt[0] = job->iv 1517 * in_sgt[1..n] = payload segments 1518 * out_sgt[0..n-1] = payload segments (in-place) 1519 * DIGEST (HMAC, no IV): 1520 * in_sgt[0..n-1] = payload segments 1521 * out_sgt unused (JD points SEQ_OUT_PTR directly at job->digest). 1522 * AEAD and ETA: 1523 * in_sgt[0] = job->iv, or job->ccm_ctx for CCM 1524 * in_sgt[1..A] = AAD (crp_aad, or crp_buf at crp_aad_start) 1525 * in_sgt[A+1..N] = payload segments 1526 * in_sgt[N+1..] = (decrypt only) received tag from crp_buf 1527 * out_sgt[0..] = payload segments (in-place); the tag goes to 1528 * job->digest via a separate STORE. 1529 * The final SGT entry in each populated table gets F=1. 1530 */ 1531 static int 1532 sec_job_build_sgts(struct sec_job *job, const struct crypto_session_params *csp) 1533 { 1534 struct cryptop *crp = job->crp; 1535 uint32_t skip = crp->crp_payload_start; 1536 uint32_t left = crp->crp_payload_length; 1537 int i, out_i; 1538 int iv_slot = csp->csp_ivlen > 0 ? 1 : 0; 1539 1540 if (csp->csp_mode == CSP_MODE_AEAD || 1541 csp->csp_mode == CSP_MODE_ETA) { 1542 const int inmax = 1 + SEC_MAX_SEGMENTS; 1543 bool encrypt = (crp->crp_op & CRYPTO_OP_ENCRYPT) != 0; 1544 bool ccm = csp->csp_mode == CSP_MODE_AEAD && 1545 csp->csp_cipher_alg == CRYPTO_AES_CCM_16; 1546 int in_i = 0, npay, pay_i; 1547 1548 /* IV, or for CCM the B0 || CTR0 context block. */ 1549 if (ccm) { 1550 job->in_sgt[in_i].addr = 1551 pmap_kextract((vm_offset_t)job->ccm_ctx); 1552 job->in_sgt[in_i].length = SEC_CCM_CTX_LEN; 1553 } else { 1554 job->in_sgt[in_i].addr = 1555 pmap_kextract((vm_offset_t)job->iv); 1556 job->in_sgt[in_i].length = csp->csp_ivlen; 1557 } 1558 job->in_sgt[in_i].extension = 0; 1559 job->in_sgt[in_i].final = 0; 1560 job->in_sgt[in_i].bpid = 0; 1561 job->in_sgt[in_i].offset = 0; 1562 in_i++; 1563 1564 if (crp->crp_aad_length > 0) { 1565 /* CCM feeds the AAD length ahead of the AAD. */ 1566 if (ccm) { 1567 job->in_sgt[in_i].addr = pmap_kextract( 1568 (vm_offset_t)job->ccm_alen); 1569 job->in_sgt[in_i].extension = 0; 1570 job->in_sgt[in_i].final = 0; 1571 job->in_sgt[in_i].length = 1572 sizeof(job->ccm_alen); 1573 job->in_sgt[in_i].bpid = 0; 1574 job->in_sgt[in_i].offset = 0; 1575 in_i++; 1576 } 1577 if (crp->crp_aad != NULL) { 1578 /* 1579 * A dedicated AAD buffer is not part of the 1580 * crp mapping; it is small enough that one 1581 * entry always covers it. 1582 */ 1583 job->in_sgt[in_i].addr = pmap_kextract( 1584 (vm_offset_t)crp->crp_aad); 1585 job->in_sgt[in_i].extension = 0; 1586 job->in_sgt[in_i].final = 0; 1587 job->in_sgt[in_i].length = crp->crp_aad_length; 1588 job->in_sgt[in_i].bpid = 0; 1589 job->in_sgt[in_i].offset = 0; 1590 in_i++; 1591 } else { 1592 in_i = dpaa_sgte_append(job, job->in_sgt, in_i, 1593 inmax, crp->crp_aad_start, 1594 crp->crp_aad_length); 1595 if (in_i < 0) 1596 return (E2BIG); 1597 } 1598 } 1599 1600 pay_i = in_i; 1601 in_i = dpaa_sgte_append(job, job->in_sgt, in_i, inmax, skip, 1602 left); 1603 if (in_i < 0) 1604 return (E2BIG); 1605 npay = in_i - pay_i; 1606 if (npay == 0) 1607 return (EINVAL); 1608 1609 /* 1610 * AEAD decrypt hands the received tag to the CHA for its 1611 * own compare; ETA drains the MAC to job->digest instead 1612 * and compares in software, so it needs no entry here. 1613 */ 1614 if (!encrypt && csp->csp_mode == CSP_MODE_AEAD) { 1615 in_i = dpaa_sgte_append(job, job->in_sgt, in_i, inmax, 1616 crp->crp_digest_start, job->sess->digestlen); 1617 if (in_i < 0) 1618 return (E2BIG); 1619 } 1620 job->in_sgt[in_i - 1].final = 1; 1621 1622 /* Output mirrors the payload segments, in place. */ 1623 memcpy(job->out_sgt, &job->in_sgt[pay_i], 1624 npay * sizeof(job->out_sgt[0])); 1625 job->out_sgt[npay - 1].final = 1; 1626 return (0); 1627 } 1628 1629 if (iv_slot) { 1630 job->in_sgt[0].addr = pmap_kextract((vm_offset_t)job->iv); 1631 job->in_sgt[0].extension = 0; 1632 job->in_sgt[0].final = 0; 1633 job->in_sgt[0].length = sec_cipher_ivlen(csp); 1634 job->in_sgt[0].bpid = 0; 1635 job->in_sgt[0].offset = 0; 1636 } 1637 1638 out_i = 0; 1639 for (i = 0; i < job->nsegs && left > 0; i++) { 1640 bus_addr_t addr = job->segs[i].ds_addr; 1641 bus_size_t len = job->segs[i].ds_len; 1642 uint32_t take; 1643 1644 if (skip >= len) { 1645 skip -= len; 1646 continue; 1647 } 1648 addr += skip; 1649 len -= skip; 1650 skip = 0; 1651 1652 take = (len > left) ? left : len; 1653 left -= take; 1654 1655 if (out_i >= SEC_MAX_SEGMENTS) 1656 return (E2BIG); 1657 1658 job->in_sgt[iv_slot + out_i].addr = addr; 1659 job->in_sgt[iv_slot + out_i].extension = 0; 1660 job->in_sgt[iv_slot + out_i].final = 0; 1661 job->in_sgt[iv_slot + out_i].length = take; 1662 job->in_sgt[iv_slot + out_i].bpid = 0; 1663 job->in_sgt[iv_slot + out_i].offset = 0; 1664 1665 if (iv_slot) 1666 job->out_sgt[out_i] = job->in_sgt[iv_slot + out_i]; 1667 out_i++; 1668 } 1669 if (left != 0) 1670 return (EINVAL); 1671 if (out_i == 0) 1672 return (EINVAL); 1673 1674 job->in_sgt[iv_slot + out_i - 1].final = 1; 1675 if (iv_slot) 1676 job->out_sgt[out_i - 1].final = 1; 1677 return (0); 1678 } 1679 1680 /* 1681 * JD for the cipher modes. The shared descriptor runs the pipeline, so 1682 * the JD only points at the SGTs. 1683 * 1684 * Both sequences use SGF and EXT unconditionally. Always using a table 1685 * keeps the builder from caring how many segments there are, and the 1686 * 16-bit length in the command word is too small for the payloads geli 1687 * and kTLS hand down. 1688 */ 1689 static int 1690 sec_jd_build_cipher(struct sec_job *job, 1691 const struct crypto_session_params *csp) 1692 { 1693 struct sec_desc_builder b; 1694 struct sec_session *sess = job->sess; 1695 int enc = CRYPTO_OP_IS_ENCRYPT(job->crp->crp_op); 1696 uint32_t sdlen = sess->sdlen[enc]; 1697 uint32_t desclen; 1698 uint32_t in_len = sec_cipher_ivlen(csp) + job->crp->crp_payload_length; 1699 vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa; 1700 1701 /* 1702 * The hardware restarts the tweak every SEC_XTS_SECTOR_SIZE bytes; 1703 * opencrypto expects one continuous tweak, so anything that would 1704 * cross the boundary has to go back to software. 1705 */ 1706 if (csp->csp_cipher_alg == CRYPTO_AES_XTS && 1707 job->crp->crp_payload_length > SEC_XTS_SECTOR_SIZE) 1708 return (EOPNOTSUPP); 1709 1710 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd); 1711 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt); 1712 out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt); 1713 1714 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS); 1715 sec_jd_ptr(&b, shd_pa); 1716 1717 sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, 1718 job->crp->crp_payload_length); 1719 1720 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len); 1721 1722 /* 1723 * HEADER_REO (Reverse Execution Order) makes SEC run the JD 1724 * commands FIRST (SEQ_OUT_PTR / SEQ_IN_PTR set up the input and 1725 * output sequences), then fall into the shared descriptor. The 1726 * shared descriptor's SEQ_LOAD / SEQ_FIFO_LOAD / SEQ_FIFO_STORE 1727 * commands depend on those sequences being programmed. Without 1728 * this bit the shared desc runs first and SEQ_LOAD hits an 1729 * uninitialized input sequence, and DECO reports an invalid 1730 * sequence command (error 0x10). 1731 */ 1732 return (sec_desc_finalize_job(&b, HEADER_SHR | 1733 HEADER_REO | HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_DEFER, 1734 &desclen)); 1735 } 1736 1737 /* 1738 * JD for the digest modes. There is no IV to prepend and the output is 1739 * a small fixed buffer, so SEQ_OUT_PTR addresses it directly. 1740 */ 1741 static int 1742 sec_jd_build_digest(struct sec_job *job) 1743 { 1744 struct sec_desc_builder b; 1745 struct sec_session *sess = job->sess; 1746 uint32_t sdlen = sess->sdlen[0]; 1747 uint32_t desclen; 1748 vm_paddr_t shd_pa, in_sgt_pa, digest_pa; 1749 1750 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[0].shd); 1751 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt); 1752 digest_pa = pmap_kextract((vm_offset_t)job->digest); 1753 1754 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS); 1755 sec_jd_ptr(&b, shd_pa); 1756 1757 sec_jd_seq(&b, true, SEQ_EXT, digest_pa, sess->digestlen); 1758 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, 1759 job->crp->crp_payload_length); 1760 1761 return (sec_desc_finalize_job(&b, 1762 HEADER_SHR | HEADER_REO | HEADER_SHR_DESC_L(sdlen) | 1763 HEADER_SHARE_DEFER, &desclen)); 1764 } 1765 1766 /* 1767 * JD for AEAD (AES-GCM). 1768 * 1769 * The shared descriptor holds only the class 1 key and runs first, so 1770 * the job descriptor sets up both sequences and drives all of the data. 1771 * The data size counts the IV and AAD rounded up to 16 bytes even though 1772 * the FIFO loads supply them unpadded; SEC pads them internally. 1773 * 1774 * [0] - Header 1775 * [1..2] - Shared descriptor pointer 1776 * [3..6] - SEQ OUT PTR - ciphertext only, the tag leaves via STORE 1777 * [7..10] - SEQ IN PTR - iv + aad + payload, and the tag when decrypting 1778 * [11] - Operation 1779 * [12] - LOAD Class 1 Data Size, which starts processing 1780 * [13] - FIFO LOAD IV 1781 * [14] - FIFO LOAD AAD 1782 * [15] - FIFO STORE ciphertext 1783 * [16] - FIFO LOAD message 1784 * [encrypt:17..19] - STORE the computed tag to job->digest 1785 * [decrypt:17] - FIFO LOAD received ICV 1786 */ 1787 static int 1788 sec_jd_build_aead(struct sec_job *job) 1789 { 1790 struct sec_desc_builder b; 1791 struct sec_session *sess = job->sess; 1792 const struct crypto_session_params *csp; 1793 uint32_t sdlen = sess->sdlen[0]; 1794 uint32_t desclen; 1795 vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa; 1796 uint32_t ivlen, aadlen, paylen, taglen; 1797 uint32_t padded_iv, padded_aad, dsr_val; 1798 uint32_t in_len, out_len; 1799 vm_paddr_t digest_pa; 1800 int enc; 1801 1802 csp = crypto_get_params(job->crp->crp_session); 1803 enc = (job->crp->crp_op & CRYPTO_OP_ENCRYPT) != 0; 1804 ivlen = csp->csp_ivlen; 1805 aadlen = job->crp->crp_aad_length; 1806 paylen = job->crp->crp_payload_length; 1807 taglen = sess->digestlen; 1808 1809 padded_iv = roundup(ivlen, 16); 1810 padded_aad = roundup(aadlen, 16); 1811 dsr_val = padded_iv + padded_aad + paylen; 1812 in_len = ivlen + aadlen + paylen + (enc ? 0 : taglen); 1813 /* Output sequence is ciphertext only; tag goes via direct STORE. */ 1814 out_len = paylen; 1815 digest_pa = pmap_kextract((vm_offset_t)job->digest); 1816 1817 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd); 1818 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt); 1819 out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt); 1820 1821 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS); 1822 sec_jd_ptr(&b, shd_pa); 1823 1824 sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, out_len); 1825 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len); 1826 1827 /* 1828 * Writing the data size starts processing, so OPERATION has to arm 1829 * the CHA in GCM mode before the DSR load below. 1830 */ 1831 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_GCM | 1832 AS_INIT_FINAL | (enc ? OP_ENC : OP_ICV)); 1833 1834 sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_DSR, 0, 8, 1835 (uint64_t)dsr_val << 32); 1836 1837 /* IV: FC1 so SEC pads to 16 without ending class 1 input. */ 1838 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 | 1839 FIFOLD_TYPE_IV | FIFOLD_FC1, ivlen); 1840 /* 1841 * Always emit an AAD FIFO_LOAD (even with length 0) so SEC gets 1842 * an explicit "AAD phase done" signal via FC1. 1843 */ 1844 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 | 1845 FIFOLD_TYPE_AAD | FIFOLD_FC1, aadlen); 1846 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) | 1847 FIFOST_TYPE_MSG_DATA, paylen); 1848 1849 /* MSG: LC1 for encrypt (last class-1 input), FC1 for decrypt. */ 1850 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 | 1851 FIFOLD_TYPE_MSG | (enc ? FIFOLD_LC1 : FIFOLD_FC1), paylen); 1852 if (enc) 1853 sec_jd_store(&b, false, LOAD_CLASS_1, LOAD_CTX, 0, taglen, 1854 digest_pa); 1855 else 1856 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | 1857 FIFOLD_CLASS_1 | FIFOLD_TYPE_ICV | FIFOLD_LC1, taglen); 1858 1859 /* SHR=1, NO REO, so the shd (KEY only) runs first, then the JD. */ 1860 return (sec_desc_finalize_job(&b, HEADER_SHR | 1861 HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen)); 1862 } 1863 1864 /* 1865 * JD for AES-CCM. 1866 */ 1867 static int 1868 sec_jd_build_ccm(struct sec_job *job) 1869 { 1870 struct sec_desc_builder b; 1871 struct sec_session *sess = job->sess; 1872 uint32_t sdlen = sess->sdlen[0]; 1873 uint32_t desclen, aadlen, paylen, taglen, in_len; 1874 vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa, digest_pa; 1875 int enc; 1876 1877 enc = (job->crp->crp_op & CRYPTO_OP_ENCRYPT) != 0; 1878 aadlen = job->crp->crp_aad_length; 1879 paylen = job->crp->crp_payload_length; 1880 taglen = sess->digestlen; 1881 1882 in_len = SEC_CCM_CTX_LEN + paylen + (enc ? 0 : taglen); 1883 if (aadlen > 0) 1884 in_len += sizeof(job->ccm_alen) + aadlen; 1885 1886 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd); 1887 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt); 1888 out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt); 1889 digest_pa = pmap_kextract((vm_offset_t)job->digest); 1890 1891 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS); 1892 sec_jd_ptr(&b, shd_pa); 1893 1894 sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, paylen); 1895 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len); 1896 1897 /* B0 || CTR0 || zeroed result dwords, from the head of the input. */ 1898 sec_desc_word(&b, CMD_DESC(CMD_SEQ_LOAD) | LOAD_CLASS_1 | LOAD_CTX | 1899 (SEC_CCM_CTX_LEN & LOAD_LENGTH_M)); 1900 1901 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_CCM | 1902 AS_INIT_FINAL | (enc ? OP_ENC : OP_ICV)); 1903 1904 /* Writing the data size starts the operation. */ 1905 sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_DSR, 0, 8, 1906 (uint64_t)paylen << 32); 1907 1908 /* Length-prefixed AAD; the hardware pads it out to 16 bytes. */ 1909 if (aadlen > 0) 1910 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | 1911 FIFOLD_CLASS_1 | FIFOLD_TYPE_AAD | FIFOLD_FC1, 1912 sizeof(job->ccm_alen) + aadlen); 1913 1914 /* Arm the drain before the message, as for GCM. */ 1915 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) | 1916 FIFOST_TYPE_MSG_DATA, paylen); 1917 1918 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 | 1919 FIFOLD_TYPE_MSG | (enc ? FIFOLD_LC1 : FIFOLD_FC1), paylen); 1920 1921 if (enc) 1922 sec_jd_store(&b, false, LOAD_CLASS_1, LOAD_CTX, 32, taglen, 1923 digest_pa); 1924 else 1925 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | 1926 FIFOLD_CLASS_1 | FIFOLD_TYPE_ICV | FIFOLD_LC1, taglen); 1927 1928 return (sec_desc_finalize_job(&b, HEADER_SHR | 1929 HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen)); 1930 } 1931 1932 /* 1933 * JD for encrypt-then-auth. 1934 */ 1935 static int 1936 sec_jd_build_eta(struct sec_job *job) 1937 { 1938 struct sec_desc_builder b; 1939 struct sec_session *sess = job->sess; 1940 const struct crypto_session_params *csp; 1941 uint32_t sdlen, desclen, aadlen, paylen, ivlen, in_len; 1942 uint32_t ctx_offset; 1943 vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa, digest_pa; 1944 int enc; 1945 1946 csp = crypto_get_params(job->crp->crp_session); 1947 enc = CRYPTO_OP_IS_ENCRYPT(job->crp->crp_op); 1948 sdlen = sess->sdlen[enc]; 1949 ivlen = csp->csp_ivlen; 1950 aadlen = job->crp->crp_aad_length; 1951 paylen = job->crp->crp_payload_length; 1952 1953 in_len = ivlen + aadlen + paylen; 1954 1955 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd); 1956 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt); 1957 out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt); 1958 digest_pa = pmap_kextract((vm_offset_t)job->digest); 1959 1960 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS); 1961 sec_jd_ptr(&b, shd_pa); 1962 1963 sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, paylen); 1964 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len); 1965 1966 ctx_offset = sec_cipher_ctx_offset(csp->csp_cipher_alg); 1967 1968 /* IV into the class 1 context; also drops SIL by ivlen. */ 1969 sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX, ctx_offset, 1970 ivlen, 0); 1971 1972 /* AAD is authenticated only, so class 2 alone. */ 1973 if (aadlen > 0) 1974 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | 1975 FIFOLD_CLASS_2 | FIFOLD_TYPE_MSG, aadlen); 1976 1977 /* Arm the ciphertext drain before feeding the message. */ 1978 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) | 1979 FIFOST_TYPE_MSG_DATA, paylen); 1980 1981 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | 1982 FIFOLD_CLASS_BOTH | FIFOLD_LC1 | FIFOLD_LC2 | 1983 (enc ? FIFOLD_TYPE_MSG_C1OUT : FIFOLD_TYPE_MSG), paylen); 1984 1985 /* Drain the MAC to job->digest; the caller compares or copies back. */ 1986 sec_jd_store(&b, false, LOAD_CLASS_2, LOAD_CTX, 0, 1987 sess->digestlen, digest_pa); 1988 1989 return (sec_desc_finalize_job(&b, HEADER_SHR | 1990 HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen)); 1991 } 1992 1993 /* 1994 * JD for AES-GMAC: GCM with nothing to encrypt. 1995 */ 1996 static int 1997 sec_jd_build_gmac(struct sec_job *job) 1998 { 1999 struct sec_desc_builder b; 2000 struct sec_session *sess = job->sess; 2001 const struct crypto_session_params *csp; 2002 uint32_t sdlen = sess->sdlen[0]; 2003 uint32_t desclen, ivlen, datalen, dsr_val, in_len; 2004 vm_paddr_t shd_pa, in_sgt_pa, digest_pa; 2005 2006 csp = crypto_get_params(job->crp->crp_session); 2007 ivlen = csp->csp_ivlen; 2008 datalen = job->crp->crp_payload_length; 2009 2010 /* 2011 * The digest-mode SGT maps the payload only, so AAD has nowhere 2012 * to come from. 2013 */ 2014 if (job->crp->crp_aad_length != 0) 2015 return (EOPNOTSUPP); 2016 2017 dsr_val = roundup(ivlen, 16) + roundup(datalen, 16); 2018 in_len = ivlen + datalen; 2019 2020 shd_pa = pmap_kextract((vm_offset_t)sess->ctx[0].shd); 2021 in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt); 2022 digest_pa = pmap_kextract((vm_offset_t)job->digest); 2023 2024 sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS); 2025 sec_jd_ptr(&b, shd_pa); 2026 2027 /* No output sequence: the tag leaves through an inline STORE. */ 2028 sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len); 2029 2030 sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_GCM | 2031 AS_INIT_FINAL | OP_ENC); 2032 2033 sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_DSR, 0, 8, 2034 (uint64_t)dsr_val << 32); 2035 2036 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 | 2037 FIFOLD_TYPE_IV | FIFOLD_FC1, ivlen); 2038 sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 | 2039 FIFOLD_TYPE_AAD | FIFOLD_LC1, datalen); 2040 2041 sec_jd_store(&b, false, LOAD_CLASS_1, LOAD_CTX, 0, 2042 sess->digestlen, digest_pa); 2043 2044 return (sec_desc_finalize_job(&b, HEADER_SHR | 2045 HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen)); 2046 } 2047 2048 static int 2049 sec_process(device_t dev, struct cryptop *crp, int hint) 2050 { 2051 struct sec_softc *sc = device_get_softc(dev); 2052 struct sec_session *sess = crypto_get_driver_session(crp->crp_session); 2053 const struct crypto_session_params *csp; 2054 struct sec_job *job; 2055 struct sec_jr *jr; 2056 int error; 2057 2058 job = malloc(sizeof(*job), M_SEC, M_NOWAIT | M_ZERO); 2059 if (job == NULL) { 2060 crp->crp_etype = ENOMEM; 2061 crypto_done(crp); 2062 return (0); 2063 } 2064 job->crp = crp; 2065 job->sess = sess; 2066 2067 error = bus_dmamap_create(sc->sc_dmatag, 0, &job->map); 2068 if (error != 0) 2069 goto fail_free; 2070 2071 error = bus_dmamap_load_crp(sc->sc_dmatag, job->map, crp, 2072 sec_load_cb, job, BUS_DMA_NOWAIT); 2073 if (error != 0 || job->nsegs == 0) { 2074 if (error == 0) 2075 error = EIO; 2076 goto fail_destroy; 2077 } 2078 2079 if (crp->crp_payload_length == 0) { 2080 error = EINVAL; 2081 goto fail_unload; 2082 } 2083 2084 csp = crypto_get_params(crp->crp_session); 2085 if (csp->csp_ivlen > 0) 2086 crypto_read_iv(crp, job->iv); 2087 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) 2088 sec_xts_tweak(job->iv); 2089 if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) { 2090 error = sec_ccm_prep(job, csp); 2091 if (error != 0) 2092 goto fail_unload; 2093 } 2094 2095 error = sec_job_build_sgts(job, csp); 2096 if (error != 0) 2097 goto fail_unload; 2098 2099 bus_dmamap_sync(sc->sc_dmatag, job->map, 2100 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2101 2102 switch (csp->csp_mode) { 2103 case CSP_MODE_DIGEST: 2104 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) 2105 error = sec_jd_build_gmac(job); 2106 else 2107 error = sec_jd_build_digest(job); 2108 break; 2109 case CSP_MODE_AEAD: 2110 if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) 2111 error = sec_jd_build_ccm(job); 2112 else 2113 error = sec_jd_build_aead(job); 2114 break; 2115 case CSP_MODE_ETA: 2116 error = sec_jd_build_eta(job); 2117 break; 2118 default: 2119 error = sec_jd_build_cipher(job, csp); 2120 break; 2121 } 2122 if (error != 0) 2123 goto fail_unload; 2124 2125 2126 /* 2127 * Hand the job to a ring and return; sec_jr_intr() completes it. 2128 */ 2129 jr = &sc->sc_jr[curcpu % sc->sc_njr]; 2130 sec_jr_submit_job(sc, jr, job); 2131 return (0); 2132 2133 fail_unload: 2134 bus_dmamap_sync(sc->sc_dmatag, job->map, 2135 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2136 bus_dmamap_unload(sc->sc_dmatag, job->map); 2137 fail_destroy: 2138 bus_dmamap_destroy(sc->sc_dmatag, job->map); 2139 fail_free: 2140 free(job, M_SEC); 2141 /* ERESTART means opencrypto retries this crp, so don't complete it. */ 2142 if (error == ERESTART) 2143 return (ERESTART); 2144 crp->crp_etype = error; 2145 crypto_done(crp); 2146 return (0); 2147 } 2148 2149 static int 2150 sec_reset(struct sec_softc *sc) 2151 { 2152 uint32_t mcfgr; 2153 int i; 2154 2155 /* 2156 * Preserve cache-attribute fields (AWCACHE/ARCACHE) and burst 2157 * settings across the reset. The MCFGR write overwrites those 2158 * along with SWRST. 2159 */ 2160 mcfgr = SEC_RD4(sc, SEC_MCFGR); 2161 SEC_WR4(sc, SEC_MCFGR, mcfgr | MCFGR_SWRST); 2162 2163 /* Poll SWRST for self-clear. */ 2164 for (i = 0; i < 10000; i++) { 2165 if ((SEC_RD4(sc, SEC_MCFGR) & MCFGR_SWRST) == 0) 2166 break; 2167 DELAY(10); 2168 } 2169 if ((SEC_RD4(sc, SEC_MCFGR) & MCFGR_SWRST) != 0) { 2170 device_printf(sc->sc_dev, "MCFGR.SWRST did not clear\n"); 2171 return (EIO); 2172 } 2173 2174 /* 2175 * Post-reset configuration: 40-bit pointers, DECO watchdog on, 2176 * large bursts. Preserve whatever cache attributes the bootloader 2177 * left in place. 2178 */ 2179 mcfgr &= (MCFGR_ARCACHE_M | MCFGR_AWCACHE_M); 2180 mcfgr |= MCFGR_PS | MCFGR_WDE | MCFGR_LARGE_BURST; 2181 SEC_WR4(sc, SEC_MCFGR, mcfgr); 2182 2183 return (0); 2184 } 2185 2186 /* 2187 * Instantiate one RNG state handle via DECO0 direct access. 2188 */ 2189 static int 2190 sec_deco_rng_init(struct sec_softc *sc, int sh) 2191 { 2192 uint32_t jd[2]; 2193 uint32_t reg, decorr, scfgr; 2194 int i; 2195 2196 jd[0] = CMD_DESC(CMD_DESC_HEADER) | HEADER_ONE | 2197 (2 & HEADER_DESCLEN_M); 2198 jd[1] = CMD_DESC(CMD_OPERATION) | ALG_RNG | AS_INIT | OP_RNG_SH(sh); 2199 2200 /* Request DECO0 and wait for the grant (DEN0=1). */ 2201 SEC_WR4(sc, SEC_DECORR, DECORR_RQD0); 2202 decorr = SEC_RD4(sc, SEC_DECORR); 2203 for (i = 0; i < 10000; i++) { 2204 decorr = SEC_RD4(sc, SEC_DECORR); 2205 if ((decorr & DECORR_DEN0) != 0) 2206 break; 2207 DELAY(10); 2208 } 2209 if ((decorr & DECORR_DEN0) == 0) { 2210 scfgr = SEC_RD4(sc, SEC_SCFGR); 2211 device_printf(sc->sc_dev, 2212 "DECO0 acquire timeout (DECORR=%#x SCFGR=%#x%s)\n", 2213 decorr, scfgr, 2214 (scfgr & SCFGR_VIRT_EN) ? " VIRT_EN" : ""); 2215 SEC_WR4(sc, SEC_DECORR, 0); 2216 return (ETIMEDOUT); 2217 } 2218 2219 SEC_WR4(sc, SEC_D0DESB(0), jd[0]); 2220 SEC_WR4(sc, SEC_D0DESB(1), jd[1]); 2221 2222 SEC_WR4(sc, SEC_D0JQCR_MS, DAJQCR_MS_WHL); 2223 2224 /* Wait for job completion */ 2225 reg = 0; 2226 for (i = 0; i < 100000; i++) { 2227 reg = SEC_RD4(sc, SEC_D0DDR); 2228 if ((reg & DADDR_VALID) == 0) 2229 break; 2230 DELAY(10); 2231 } 2232 2233 /* Release DECO0 either way. */ 2234 SEC_WR4(sc, SEC_DECORR, 0); 2235 2236 if ((reg & DADDR_VALID) != 0) { 2237 device_printf(sc->sc_dev, 2238 "RNG SH%d instantiate timeout (D0DDR=%#x)\n", sh, reg); 2239 return (ETIMEDOUT); 2240 } 2241 if (((reg & DADDR_DECO_STATE_M) >> DADDR_DECO_STATE_S) != 0) { 2242 device_printf(sc->sc_dev, 2243 "RNG SH%d instantiate error (D0DDR=%#x, DECO_STATE=%u)\n", 2244 sh, reg, 2245 (reg & DADDR_DECO_STATE_M) >> DADDR_DECO_STATE_S); 2246 return (EIO); 2247 } 2248 return (0); 2249 } 2250 2251 static int 2252 sec_rng_init(struct sec_softc *sc) 2253 { 2254 uint32_t rdsta; 2255 int error, sh; 2256 2257 /* 2258 * SEC v4/v5 requires the DRNG state handles to be instantiated 2259 * before any class-1 (AES/DES/RNG) job will execute. This is typically 2260 * done by the bootloader, but finish what it didn't. 2261 */ 2262 rdsta = SEC_RD4(sc, SEC_RDSTA); 2263 2264 if ((rdsta & RDSTA_CE) != 0) { 2265 device_printf(sc->sc_dev, 2266 "RNG catastrophic error (RDSTA=%#x, ERRCODE=%u)\n", 2267 rdsta, (rdsta & RDSTA_ERRCODE_M) >> RDSTA_ERRCODE_S); 2268 return (EIO); 2269 } 2270 2271 /* Instantiate anything the bootloader didn't. */ 2272 for (sh = 0; sh <= 1; sh++) { 2273 uint32_t bit = (sh == 0) ? RDSTA_IF0 : RDSTA_IF1; 2274 2275 if ((rdsta & bit) != 0) 2276 continue; 2277 error = sec_deco_rng_init(sc, sh); 2278 if (error != 0) 2279 return (error); 2280 } 2281 2282 /* Verify the handles are now up. */ 2283 rdsta = SEC_RD4(sc, SEC_RDSTA); 2284 if ((rdsta & (RDSTA_IF0 | RDSTA_IF1)) != 2285 (RDSTA_IF0 | RDSTA_IF1)) { 2286 device_printf(sc->sc_dev, 2287 "RNG instantiation left RDSTA=%#x\n", rdsta); 2288 return (EIO); 2289 } 2290 return (0); 2291 } 2292