1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Shared descriptors for aead, skcipher algorithms 4 * 5 * Copyright 2016-2019 NXP 6 */ 7 8 #include "compat.h" 9 #include "desc_constr.h" 10 #include "caamalg_desc.h" 11 12 /* 13 * For aead functions, read payload and write payload, 14 * both of which are specified in req->src and req->dst 15 */ 16 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 17 { 18 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 19 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 20 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 21 } 22 23 /* Set DK bit in class 1 operation if shared */ 24 static inline void append_dec_op1(u32 *desc, u32 type) 25 { 26 u32 *jump_cmd, *uncond_jump_cmd; 27 28 /* DK bit is valid only for AES */ 29 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { 30 append_operation(desc, type | OP_ALG_AS_INITFINAL | 31 OP_ALG_DECRYPT); 32 return; 33 } 34 35 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 36 append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT); 37 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 38 set_jump_tgt_here(desc, jump_cmd); 39 append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT | 40 OP_ALG_AAI_DK); 41 set_jump_tgt_here(desc, uncond_jump_cmd); 42 } 43 44 /** 45 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor 46 * (non-protocol) with no (null) encryption. 47 * @desc: pointer to buffer used for descriptor construction 48 * @adata: pointer to authentication transform definitions. 49 * A split key is required for SEC Era < 6; the size of the split key 50 * is specified in this case. Valid algorithm values - one of 51 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 52 * with OP_ALG_AAI_HMAC_PRECOMP. 53 * @icvsize: integrity check value (ICV) size (truncated or full) 54 * @era: SEC Era 55 */ 56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, 57 unsigned int icvsize, int era) 58 { 59 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 60 61 init_sh_desc(desc, HDR_SHARE_SERIAL); 62 63 /* Skip if already shared */ 64 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 65 JUMP_COND_SHRD); 66 if (era < 6) { 67 if (adata->key_inline) 68 append_key_as_imm(desc, adata->key_virt, 69 adata->keylen_pad, adata->keylen, 70 CLASS_2 | KEY_DEST_MDHA_SPLIT | 71 KEY_ENC); 72 else 73 append_key(desc, adata->key_dma, adata->keylen, 74 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); 75 } else { 76 append_proto_dkp(desc, adata); 77 } 78 set_jump_tgt_here(desc, key_jump_cmd); 79 80 /* assoclen + cryptlen = seqinlen */ 81 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 82 83 /* Prepare to read and write cryptlen + assoclen bytes */ 84 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 85 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 86 87 /* 88 * MOVE_LEN opcode is not available in all SEC HW revisions, 89 * thus need to do some magic, i.e. self-patch the descriptor 90 * buffer. 91 */ 92 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 93 MOVE_DEST_MATH3 | 94 (0x6 << MOVE_LEN_SHIFT)); 95 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | 96 MOVE_DEST_DESCBUF | 97 MOVE_WAITCOMP | 98 (0x8 << MOVE_LEN_SHIFT)); 99 100 /* Class 2 operation */ 101 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 102 OP_ALG_ENCRYPT); 103 104 /* Read and write cryptlen bytes */ 105 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 106 107 set_move_tgt_here(desc, read_move_cmd); 108 set_move_tgt_here(desc, write_move_cmd); 109 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 110 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 111 MOVE_AUX_LS); 112 113 /* Write ICV */ 114 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 115 LDST_SRCDST_BYTE_CONTEXT); 116 117 print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ", 118 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 119 1); 120 } 121 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); 122 123 /** 124 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor 125 * (non-protocol) with no (null) decryption. 126 * @desc: pointer to buffer used for descriptor construction 127 * @adata: pointer to authentication transform definitions. 128 * A split key is required for SEC Era < 6; the size of the split key 129 * is specified in this case. Valid algorithm values - one of 130 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 131 * with OP_ALG_AAI_HMAC_PRECOMP. 132 * @icvsize: integrity check value (ICV) size (truncated or full) 133 * @era: SEC Era 134 */ 135 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, 136 unsigned int icvsize, int era) 137 { 138 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd; 139 140 init_sh_desc(desc, HDR_SHARE_SERIAL); 141 142 /* Skip if already shared */ 143 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 144 JUMP_COND_SHRD); 145 if (era < 6) { 146 if (adata->key_inline) 147 append_key_as_imm(desc, adata->key_virt, 148 adata->keylen_pad, adata->keylen, 149 CLASS_2 | KEY_DEST_MDHA_SPLIT | 150 KEY_ENC); 151 else 152 append_key(desc, adata->key_dma, adata->keylen, 153 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); 154 } else { 155 append_proto_dkp(desc, adata); 156 } 157 set_jump_tgt_here(desc, key_jump_cmd); 158 159 /* Class 2 operation */ 160 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 161 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 162 163 /* assoclen + cryptlen = seqoutlen */ 164 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 165 166 /* Prepare to read and write cryptlen + assoclen bytes */ 167 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 168 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 169 170 /* 171 * MOVE_LEN opcode is not available in all SEC HW revisions, 172 * thus need to do some magic, i.e. self-patch the descriptor 173 * buffer. 174 */ 175 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 176 MOVE_DEST_MATH2 | 177 (0x6 << MOVE_LEN_SHIFT)); 178 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | 179 MOVE_DEST_DESCBUF | 180 MOVE_WAITCOMP | 181 (0x8 << MOVE_LEN_SHIFT)); 182 183 /* Read and write cryptlen bytes */ 184 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 185 186 /* 187 * Insert a NOP here, since we need at least 4 instructions between 188 * code patching the descriptor buffer and the location being patched. 189 */ 190 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 191 set_jump_tgt_here(desc, jump_cmd); 192 193 set_move_tgt_here(desc, read_move_cmd); 194 set_move_tgt_here(desc, write_move_cmd); 195 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 196 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 197 MOVE_AUX_LS); 198 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 199 200 /* Load ICV */ 201 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | 202 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 203 204 print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ", 205 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 206 1); 207 } 208 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); 209 210 static void init_sh_desc_key_aead(u32 * const desc, 211 struct alginfo * const cdata, 212 struct alginfo * const adata, 213 const bool is_rfc3686, u32 *nonce, int era) 214 { 215 u32 *key_jump_cmd; 216 unsigned int enckeylen = cdata->keylen; 217 218 /* Note: Context registers are saved. */ 219 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 220 221 /* Skip if already shared */ 222 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 223 JUMP_COND_SHRD); 224 225 /* 226 * RFC3686 specific: 227 * | key = {AUTH_KEY, ENC_KEY, NONCE} 228 * | enckeylen = encryption key size + nonce size 229 */ 230 if (is_rfc3686) 231 enckeylen -= CTR_RFC3686_NONCE_SIZE; 232 233 if (era < 6) { 234 if (adata->key_inline) 235 append_key_as_imm(desc, adata->key_virt, 236 adata->keylen_pad, adata->keylen, 237 CLASS_2 | KEY_DEST_MDHA_SPLIT | 238 KEY_ENC); 239 else 240 append_key(desc, adata->key_dma, adata->keylen, 241 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); 242 } else { 243 append_proto_dkp(desc, adata); 244 } 245 246 if (cdata->key_inline) 247 append_key_as_imm(desc, cdata->key_virt, enckeylen, 248 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 249 else 250 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 | 251 KEY_DEST_CLASS_REG); 252 253 /* Load Counter into CONTEXT1 reg */ 254 if (is_rfc3686) { 255 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 256 LDST_CLASS_IND_CCB | 257 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 258 append_move(desc, 259 MOVE_SRC_OUTFIFO | 260 MOVE_DEST_CLASS1CTX | 261 (16 << MOVE_OFFSET_SHIFT) | 262 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 263 } 264 265 set_jump_tgt_here(desc, key_jump_cmd); 266 } 267 268 /** 269 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor 270 * (non-protocol). 271 * @desc: pointer to buffer used for descriptor construction 272 * @cdata: pointer to block cipher transform definitions 273 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 274 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 275 * @adata: pointer to authentication transform definitions. 276 * A split key is required for SEC Era < 6; the size of the split key 277 * is specified in this case. Valid algorithm values - one of 278 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 279 * with OP_ALG_AAI_HMAC_PRECOMP. 280 * @ivsize: initialization vector size 281 * @icvsize: integrity check value (ICV) size (truncated or full) 282 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 283 * @nonce: pointer to rfc3686 nonce 284 * @ctx1_iv_off: IV offset in CONTEXT1 register 285 * @is_qi: true when called from caam/qi 286 * @era: SEC Era 287 */ 288 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, 289 struct alginfo *adata, unsigned int ivsize, 290 unsigned int icvsize, const bool is_rfc3686, 291 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi, 292 int era) 293 { 294 /* Note: Context registers are saved. */ 295 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); 296 297 /* Class 2 operation */ 298 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 299 OP_ALG_ENCRYPT); 300 301 if (is_qi) { 302 u32 *wait_load_cmd; 303 304 /* REG3 = assoclen */ 305 append_seq_load(desc, 4, LDST_CLASS_DECO | 306 LDST_SRCDST_WORD_DECO_MATH3 | 307 (4 << LDST_OFFSET_SHIFT)); 308 309 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 310 JUMP_COND_CALM | JUMP_COND_NCP | 311 JUMP_COND_NOP | JUMP_COND_NIP | 312 JUMP_COND_NIFP); 313 set_jump_tgt_here(desc, wait_load_cmd); 314 315 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 316 LDST_SRCDST_BYTE_CONTEXT | 317 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 318 } 319 320 /* Read and write assoclen bytes */ 321 if (is_qi || era < 3) { 322 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 323 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 324 } else { 325 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 326 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 327 } 328 329 /* Skip assoc data */ 330 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 331 332 /* read assoc before reading payload */ 333 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 334 FIFOLDST_VLF); 335 336 /* Load Counter into CONTEXT1 reg */ 337 if (is_rfc3686) 338 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 339 LDST_SRCDST_BYTE_CONTEXT | 340 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 341 LDST_OFFSET_SHIFT)); 342 343 /* Class 1 operation */ 344 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 345 OP_ALG_ENCRYPT); 346 347 /* Read and write cryptlen bytes */ 348 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 349 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 350 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 351 352 /* Write ICV */ 353 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 354 LDST_SRCDST_BYTE_CONTEXT); 355 356 print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ", 357 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 358 1); 359 } 360 EXPORT_SYMBOL(cnstr_shdsc_aead_encap); 361 362 /** 363 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor 364 * (non-protocol). 365 * @desc: pointer to buffer used for descriptor construction 366 * @cdata: pointer to block cipher transform definitions 367 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 368 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 369 * @adata: pointer to authentication transform definitions. 370 * A split key is required for SEC Era < 6; the size of the split key 371 * is specified in this case. Valid algorithm values - one of 372 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 373 * with OP_ALG_AAI_HMAC_PRECOMP. 374 * @ivsize: initialization vector size 375 * @icvsize: integrity check value (ICV) size (truncated or full) 376 * @geniv: whether to generate Encrypted Chain IV 377 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 378 * @nonce: pointer to rfc3686 nonce 379 * @ctx1_iv_off: IV offset in CONTEXT1 register 380 * @is_qi: true when called from caam/qi 381 * @era: SEC Era 382 */ 383 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, 384 struct alginfo *adata, unsigned int ivsize, 385 unsigned int icvsize, const bool geniv, 386 const bool is_rfc3686, u32 *nonce, 387 const u32 ctx1_iv_off, const bool is_qi, int era) 388 { 389 /* Note: Context registers are saved. */ 390 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); 391 392 /* Class 2 operation */ 393 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 394 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 395 396 if (is_qi) { 397 u32 *wait_load_cmd; 398 399 /* REG3 = assoclen */ 400 append_seq_load(desc, 4, LDST_CLASS_DECO | 401 LDST_SRCDST_WORD_DECO_MATH3 | 402 (4 << LDST_OFFSET_SHIFT)); 403 404 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 405 JUMP_COND_CALM | JUMP_COND_NCP | 406 JUMP_COND_NOP | JUMP_COND_NIP | 407 JUMP_COND_NIFP); 408 set_jump_tgt_here(desc, wait_load_cmd); 409 410 if (!geniv) 411 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 412 LDST_SRCDST_BYTE_CONTEXT | 413 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 414 } 415 416 /* Read and write assoclen bytes */ 417 if (is_qi || era < 3) { 418 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 419 if (geniv) 420 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, 421 ivsize); 422 else 423 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, 424 CAAM_CMD_SZ); 425 } else { 426 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 427 if (geniv) 428 append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM, 429 ivsize); 430 else 431 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, 432 CAAM_CMD_SZ); 433 } 434 435 /* Skip assoc data */ 436 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 437 438 /* read assoc before reading payload */ 439 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 440 KEY_VLF); 441 442 if (geniv) { 443 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 444 LDST_SRCDST_BYTE_CONTEXT | 445 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 446 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | 447 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); 448 } 449 450 /* Load Counter into CONTEXT1 reg */ 451 if (is_rfc3686) 452 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 453 LDST_SRCDST_BYTE_CONTEXT | 454 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 455 LDST_OFFSET_SHIFT)); 456 457 /* Choose operation */ 458 if (ctx1_iv_off) 459 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 460 OP_ALG_DECRYPT); 461 else 462 append_dec_op1(desc, cdata->algtype); 463 464 /* Read and write cryptlen bytes */ 465 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 466 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 467 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 468 469 /* Load ICV */ 470 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | 471 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 472 473 print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ", 474 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 475 1); 476 } 477 EXPORT_SYMBOL(cnstr_shdsc_aead_decap); 478 479 /** 480 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor 481 * (non-protocol) with HW-generated initialization 482 * vector. 483 * @desc: pointer to buffer used for descriptor construction 484 * @cdata: pointer to block cipher transform definitions 485 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 486 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 487 * @adata: pointer to authentication transform definitions. 488 * A split key is required for SEC Era < 6; the size of the split key 489 * is specified in this case. Valid algorithm values - one of 490 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 491 * with OP_ALG_AAI_HMAC_PRECOMP. 492 * @ivsize: initialization vector size 493 * @icvsize: integrity check value (ICV) size (truncated or full) 494 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 495 * @nonce: pointer to rfc3686 nonce 496 * @ctx1_iv_off: IV offset in CONTEXT1 register 497 * @is_qi: true when called from caam/qi 498 * @era: SEC Era 499 */ 500 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, 501 struct alginfo *adata, unsigned int ivsize, 502 unsigned int icvsize, const bool is_rfc3686, 503 u32 *nonce, const u32 ctx1_iv_off, 504 const bool is_qi, int era) 505 { 506 u32 geniv, moveiv; 507 u32 *wait_cmd; 508 509 /* Note: Context registers are saved. */ 510 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); 511 512 if (is_qi) { 513 u32 *wait_load_cmd; 514 515 /* REG3 = assoclen */ 516 append_seq_load(desc, 4, LDST_CLASS_DECO | 517 LDST_SRCDST_WORD_DECO_MATH3 | 518 (4 << LDST_OFFSET_SHIFT)); 519 520 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 521 JUMP_COND_CALM | JUMP_COND_NCP | 522 JUMP_COND_NOP | JUMP_COND_NIP | 523 JUMP_COND_NIFP); 524 set_jump_tgt_here(desc, wait_load_cmd); 525 } 526 527 if (is_rfc3686) { 528 if (is_qi) 529 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 530 LDST_SRCDST_BYTE_CONTEXT | 531 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 532 533 goto copy_iv; 534 } 535 536 /* Generate IV */ 537 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 538 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 539 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); 540 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 541 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 542 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 543 append_move(desc, MOVE_WAITCOMP | 544 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | 545 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 546 (ivsize << MOVE_LEN_SHIFT)); 547 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 548 549 copy_iv: 550 /* Copy IV to class 1 context */ 551 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | 552 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 553 (ivsize << MOVE_LEN_SHIFT)); 554 555 /* Return to encryption */ 556 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 557 OP_ALG_ENCRYPT); 558 559 /* Read and write assoclen bytes */ 560 if (is_qi || era < 3) { 561 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 562 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 563 } else { 564 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 565 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 566 } 567 568 /* Skip assoc data */ 569 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 570 571 /* read assoc before reading payload */ 572 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 573 KEY_VLF); 574 575 /* Copy iv from outfifo to class 2 fifo */ 576 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 577 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); 578 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 579 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 580 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | 581 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 582 583 /* Load Counter into CONTEXT1 reg */ 584 if (is_rfc3686) 585 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 586 LDST_SRCDST_BYTE_CONTEXT | 587 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 588 LDST_OFFSET_SHIFT)); 589 590 /* Class 1 operation */ 591 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 592 OP_ALG_ENCRYPT); 593 594 /* Will write ivsize + cryptlen */ 595 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 596 597 /* Not need to reload iv */ 598 append_seq_fifo_load(desc, ivsize, 599 FIFOLD_CLASS_SKIP); 600 601 /* Will read cryptlen */ 602 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 603 604 /* 605 * Wait for IV transfer (ofifo -> class2) to finish before starting 606 * ciphertext transfer (ofifo -> external memory). 607 */ 608 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP); 609 set_jump_tgt_here(desc, wait_cmd); 610 611 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | 612 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); 613 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 614 615 /* Write ICV */ 616 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 617 LDST_SRCDST_BYTE_CONTEXT); 618 619 print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ", 620 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 621 1); 622 } 623 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); 624 625 /** 626 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor 627 * @desc: pointer to buffer used for descriptor construction 628 * @cdata: pointer to block cipher transform definitions 629 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 630 * @ivsize: initialization vector size 631 * @icvsize: integrity check value (ICV) size (truncated or full) 632 * @is_qi: true when called from caam/qi 633 */ 634 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, 635 unsigned int ivsize, unsigned int icvsize, 636 const bool is_qi) 637 { 638 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1, 639 *zero_assoc_jump_cmd2; 640 641 init_sh_desc(desc, HDR_SHARE_SERIAL); 642 643 /* skip key loading if they are loaded due to sharing */ 644 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 645 JUMP_COND_SHRD); 646 if (cdata->key_inline) 647 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 648 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 649 else 650 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 651 KEY_DEST_CLASS_REG); 652 set_jump_tgt_here(desc, key_jump_cmd); 653 654 /* class 1 operation */ 655 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 656 OP_ALG_ENCRYPT); 657 658 if (is_qi) { 659 u32 *wait_load_cmd; 660 661 /* REG3 = assoclen */ 662 append_seq_load(desc, 4, LDST_CLASS_DECO | 663 LDST_SRCDST_WORD_DECO_MATH3 | 664 (4 << LDST_OFFSET_SHIFT)); 665 666 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 667 JUMP_COND_CALM | JUMP_COND_NCP | 668 JUMP_COND_NOP | JUMP_COND_NIP | 669 JUMP_COND_NIFP); 670 set_jump_tgt_here(desc, wait_load_cmd); 671 672 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM, 673 ivsize); 674 } else { 675 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, 676 CAAM_CMD_SZ); 677 } 678 679 /* if assoclen + cryptlen is ZERO, skip to ICV write */ 680 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | 681 JUMP_COND_MATH_Z); 682 683 if (is_qi) 684 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 685 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 686 687 /* if assoclen is ZERO, skip reading the assoc data */ 688 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 689 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 690 JUMP_COND_MATH_Z); 691 692 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 693 694 /* skip assoc data */ 695 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 696 697 /* cryptlen = seqinlen - assoclen */ 698 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 699 700 /* if cryptlen is ZERO jump to zero-payload commands */ 701 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 702 JUMP_COND_MATH_Z); 703 704 /* read assoc data */ 705 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 706 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 707 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 708 709 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 710 711 /* write encrypted data */ 712 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 713 714 /* read payload data */ 715 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 716 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 717 718 /* jump to ICV writing */ 719 if (is_qi) 720 append_jump(desc, JUMP_TEST_ALL | 4); 721 else 722 append_jump(desc, JUMP_TEST_ALL | 2); 723 724 /* zero-payload commands */ 725 set_jump_tgt_here(desc, zero_payload_jump_cmd); 726 727 /* read assoc data */ 728 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 729 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 730 if (is_qi) 731 /* jump to ICV writing */ 732 append_jump(desc, JUMP_TEST_ALL | 2); 733 734 /* There is no input data */ 735 set_jump_tgt_here(desc, zero_assoc_jump_cmd2); 736 737 if (is_qi) 738 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 739 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 740 FIFOLD_TYPE_LAST1); 741 742 /* write ICV */ 743 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 744 LDST_SRCDST_BYTE_CONTEXT); 745 746 print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ", 747 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 748 1); 749 } 750 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); 751 752 /** 753 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor 754 * @desc: pointer to buffer used for descriptor construction 755 * @cdata: pointer to block cipher transform definitions 756 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 757 * @ivsize: initialization vector size 758 * @icvsize: integrity check value (ICV) size (truncated or full) 759 * @is_qi: true when called from caam/qi 760 */ 761 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, 762 unsigned int ivsize, unsigned int icvsize, 763 const bool is_qi) 764 { 765 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1; 766 767 init_sh_desc(desc, HDR_SHARE_SERIAL); 768 769 /* skip key loading if they are loaded due to sharing */ 770 key_jump_cmd = append_jump(desc, JUMP_JSL | 771 JUMP_TEST_ALL | JUMP_COND_SHRD); 772 if (cdata->key_inline) 773 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 774 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 775 else 776 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 777 KEY_DEST_CLASS_REG); 778 set_jump_tgt_here(desc, key_jump_cmd); 779 780 /* class 1 operation */ 781 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 782 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 783 784 if (is_qi) { 785 u32 *wait_load_cmd; 786 787 /* REG3 = assoclen */ 788 append_seq_load(desc, 4, LDST_CLASS_DECO | 789 LDST_SRCDST_WORD_DECO_MATH3 | 790 (4 << LDST_OFFSET_SHIFT)); 791 792 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 793 JUMP_COND_CALM | JUMP_COND_NCP | 794 JUMP_COND_NOP | JUMP_COND_NIP | 795 JUMP_COND_NIFP); 796 set_jump_tgt_here(desc, wait_load_cmd); 797 798 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 799 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 800 } 801 802 /* if assoclen is ZERO, skip reading the assoc data */ 803 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 804 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 805 JUMP_COND_MATH_Z); 806 807 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 808 809 /* skip assoc data */ 810 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 811 812 /* read assoc data */ 813 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 814 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 815 816 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 817 818 /* cryptlen = seqoutlen - assoclen */ 819 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 820 821 /* jump to zero-payload command if cryptlen is zero */ 822 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 823 JUMP_COND_MATH_Z); 824 825 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 826 827 /* store encrypted data */ 828 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 829 830 /* read payload data */ 831 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 832 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 833 834 /* zero-payload command */ 835 set_jump_tgt_here(desc, zero_payload_jump_cmd); 836 837 /* read ICV */ 838 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 839 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 840 841 print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ", 842 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 843 1); 844 } 845 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); 846 847 /** 848 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor 849 * (non-protocol). 850 * @desc: pointer to buffer used for descriptor construction 851 * @cdata: pointer to block cipher transform definitions 852 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 853 * @ivsize: initialization vector size 854 * @icvsize: integrity check value (ICV) size (truncated or full) 855 * @is_qi: true when called from caam/qi 856 * 857 * Input sequence: AAD | PTXT 858 * Output sequence: AAD | CTXT | ICV 859 * AAD length (assoclen), which includes the IV length, is available in Math3. 860 */ 861 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, 862 unsigned int ivsize, unsigned int icvsize, 863 const bool is_qi) 864 { 865 u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions; 866 init_sh_desc(desc, HDR_SHARE_SERIAL); 867 868 /* Skip key loading if it is loaded due to sharing */ 869 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 870 JUMP_COND_SHRD); 871 if (cdata->key_inline) 872 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 873 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 874 else 875 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 876 KEY_DEST_CLASS_REG); 877 set_jump_tgt_here(desc, key_jump_cmd); 878 879 /* Class 1 operation */ 880 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 881 OP_ALG_ENCRYPT); 882 883 if (is_qi) { 884 u32 *wait_load_cmd; 885 886 /* REG3 = assoclen */ 887 append_seq_load(desc, 4, LDST_CLASS_DECO | 888 LDST_SRCDST_WORD_DECO_MATH3 | 889 (4 << LDST_OFFSET_SHIFT)); 890 891 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 892 JUMP_COND_CALM | JUMP_COND_NCP | 893 JUMP_COND_NOP | JUMP_COND_NIP | 894 JUMP_COND_NIFP); 895 set_jump_tgt_here(desc, wait_load_cmd); 896 897 /* Read salt and IV */ 898 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + 899 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | 900 FIFOLD_TYPE_IV); 901 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 902 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 903 } 904 905 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); 906 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 907 908 /* Skip AAD */ 909 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 910 911 /* Read cryptlen and set this value into VARSEQOUTLEN */ 912 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 913 914 /* If cryptlen is ZERO jump to AAD command */ 915 zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 916 JUMP_COND_MATH_Z); 917 918 /* Read AAD data */ 919 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 920 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 921 922 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 923 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA); 924 925 /* Skip IV */ 926 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); 927 append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ); 928 929 /* Write encrypted data */ 930 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 931 932 /* Read payload data */ 933 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 934 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 935 936 /* Jump instructions to avoid double reading of AAD */ 937 skip_instructions = append_jump(desc, JUMP_TEST_ALL); 938 939 /* There is no input data, cryptlen = 0 */ 940 set_jump_tgt_here(desc, zero_cryptlen_jump_cmd); 941 942 /* Read AAD */ 943 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 944 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 945 946 set_jump_tgt_here(desc, skip_instructions); 947 948 /* Write ICV */ 949 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 950 LDST_SRCDST_BYTE_CONTEXT); 951 952 print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ", 953 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 954 1); 955 } 956 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); 957 958 /** 959 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor 960 * (non-protocol). 961 * @desc: pointer to buffer used for descriptor construction 962 * @cdata: pointer to block cipher transform definitions 963 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 964 * @ivsize: initialization vector size 965 * @icvsize: integrity check value (ICV) size (truncated or full) 966 * @is_qi: true when called from caam/qi 967 */ 968 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, 969 unsigned int ivsize, unsigned int icvsize, 970 const bool is_qi) 971 { 972 u32 *key_jump_cmd; 973 974 init_sh_desc(desc, HDR_SHARE_SERIAL); 975 976 /* Skip key loading if it is loaded due to sharing */ 977 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 978 JUMP_COND_SHRD); 979 if (cdata->key_inline) 980 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 981 cdata->keylen, CLASS_1 | 982 KEY_DEST_CLASS_REG); 983 else 984 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 985 KEY_DEST_CLASS_REG); 986 set_jump_tgt_here(desc, key_jump_cmd); 987 988 /* Class 1 operation */ 989 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 990 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 991 992 if (is_qi) { 993 u32 *wait_load_cmd; 994 995 /* REG3 = assoclen */ 996 append_seq_load(desc, 4, LDST_CLASS_DECO | 997 LDST_SRCDST_WORD_DECO_MATH3 | 998 (4 << LDST_OFFSET_SHIFT)); 999 1000 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1001 JUMP_COND_CALM | JUMP_COND_NCP | 1002 JUMP_COND_NOP | JUMP_COND_NIP | 1003 JUMP_COND_NIFP); 1004 set_jump_tgt_here(desc, wait_load_cmd); 1005 1006 /* Read salt and IV */ 1007 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + 1008 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | 1009 FIFOLD_TYPE_IV); 1010 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 1011 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 1012 } 1013 1014 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); 1015 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 1016 1017 /* Read assoc data */ 1018 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1019 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 1020 1021 /* Skip IV */ 1022 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); 1023 1024 /* Will read cryptlen bytes */ 1025 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); 1026 1027 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 1028 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 1029 1030 /* Skip assoc data */ 1031 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 1032 1033 /* Will write cryptlen bytes */ 1034 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1035 1036 /* Store payload data */ 1037 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1038 1039 /* Read encrypted data */ 1040 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1041 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 1042 1043 /* Read ICV */ 1044 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 1045 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1046 1047 print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ", 1048 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1049 1); 1050 } 1051 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); 1052 1053 /** 1054 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor 1055 * (non-protocol). 1056 * @desc: pointer to buffer used for descriptor construction 1057 * @cdata: pointer to block cipher transform definitions 1058 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 1059 * @ivsize: initialization vector size 1060 * @icvsize: integrity check value (ICV) size (truncated or full) 1061 * @is_qi: true when called from caam/qi 1062 */ 1063 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, 1064 unsigned int ivsize, unsigned int icvsize, 1065 const bool is_qi) 1066 { 1067 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 1068 1069 init_sh_desc(desc, HDR_SHARE_SERIAL); 1070 1071 /* Skip key loading if it is loaded due to sharing */ 1072 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1073 JUMP_COND_SHRD); 1074 if (cdata->key_inline) 1075 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1076 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1077 else 1078 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 1079 KEY_DEST_CLASS_REG); 1080 set_jump_tgt_here(desc, key_jump_cmd); 1081 1082 /* Class 1 operation */ 1083 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1084 OP_ALG_ENCRYPT); 1085 1086 if (is_qi) { 1087 /* assoclen is not needed, skip it */ 1088 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); 1089 1090 /* Read salt and IV */ 1091 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + 1092 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | 1093 FIFOLD_TYPE_IV); 1094 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 1095 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 1096 } 1097 1098 /* assoclen + cryptlen = seqinlen */ 1099 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 1100 1101 /* 1102 * MOVE_LEN opcode is not available in all SEC HW revisions, 1103 * thus need to do some magic, i.e. self-patch the descriptor 1104 * buffer. 1105 */ 1106 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1107 (0x6 << MOVE_LEN_SHIFT)); 1108 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1109 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP); 1110 1111 /* Will read assoclen + cryptlen bytes */ 1112 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1113 1114 /* Will write assoclen + cryptlen bytes */ 1115 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1116 1117 /* Read and write assoclen + cryptlen bytes */ 1118 aead_append_src_dst(desc, FIFOLD_TYPE_AAD); 1119 1120 set_move_tgt_here(desc, read_move_cmd); 1121 set_move_tgt_here(desc, write_move_cmd); 1122 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1123 /* Move payload data to OFIFO */ 1124 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1125 1126 /* Write ICV */ 1127 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 1128 LDST_SRCDST_BYTE_CONTEXT); 1129 1130 print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ", 1131 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1132 1); 1133 } 1134 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); 1135 1136 /** 1137 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor 1138 * (non-protocol). 1139 * @desc: pointer to buffer used for descriptor construction 1140 * @cdata: pointer to block cipher transform definitions 1141 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 1142 * @ivsize: initialization vector size 1143 * @icvsize: integrity check value (ICV) size (truncated or full) 1144 * @is_qi: true when called from caam/qi 1145 */ 1146 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, 1147 unsigned int ivsize, unsigned int icvsize, 1148 const bool is_qi) 1149 { 1150 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 1151 1152 init_sh_desc(desc, HDR_SHARE_SERIAL); 1153 1154 /* Skip key loading if it is loaded due to sharing */ 1155 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1156 JUMP_COND_SHRD); 1157 if (cdata->key_inline) 1158 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1159 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1160 else 1161 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 1162 KEY_DEST_CLASS_REG); 1163 set_jump_tgt_here(desc, key_jump_cmd); 1164 1165 /* Class 1 operation */ 1166 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1167 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1168 1169 if (is_qi) { 1170 /* assoclen is not needed, skip it */ 1171 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); 1172 1173 /* Read salt and IV */ 1174 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + 1175 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | 1176 FIFOLD_TYPE_IV); 1177 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 1178 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 1179 } 1180 1181 /* assoclen + cryptlen = seqoutlen */ 1182 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1183 1184 /* 1185 * MOVE_LEN opcode is not available in all SEC HW revisions, 1186 * thus need to do some magic, i.e. self-patch the descriptor 1187 * buffer. 1188 */ 1189 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1190 (0x6 << MOVE_LEN_SHIFT)); 1191 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1192 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP); 1193 1194 /* Will read assoclen + cryptlen bytes */ 1195 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1196 1197 /* Will write assoclen + cryptlen bytes */ 1198 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1199 1200 /* Store payload data */ 1201 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1202 1203 /* In-snoop assoclen + cryptlen data */ 1204 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | 1205 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); 1206 1207 set_move_tgt_here(desc, read_move_cmd); 1208 set_move_tgt_here(desc, write_move_cmd); 1209 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1210 /* Move payload data to OFIFO */ 1211 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1212 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1213 1214 /* Read ICV */ 1215 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 1216 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1217 1218 print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ", 1219 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1220 1); 1221 } 1222 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); 1223 1224 /** 1225 * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and 1226 * IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared 1227 * descriptor (non-protocol). 1228 * @desc: pointer to buffer used for descriptor construction 1229 * @cdata: pointer to block cipher transform definitions 1230 * Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with 1231 * OP_ALG_AAI_AEAD. 1232 * @adata: pointer to authentication transform definitions 1233 * Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with 1234 * OP_ALG_AAI_AEAD. 1235 * @ivsize: initialization vector size 1236 * @icvsize: integrity check value (ICV) size (truncated or full) 1237 * @encap: true if encapsulation, false if decapsulation 1238 * @is_qi: true when called from caam/qi 1239 */ 1240 void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata, 1241 struct alginfo *adata, unsigned int ivsize, 1242 unsigned int icvsize, const bool encap, 1243 const bool is_qi) 1244 { 1245 u32 *key_jump_cmd, *wait_cmd; 1246 u32 nfifo; 1247 const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE); 1248 1249 /* Note: Context registers are saved. */ 1250 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1251 1252 /* skip key loading if they are loaded due to sharing */ 1253 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1254 JUMP_COND_SHRD); 1255 1256 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, 1257 CLASS_1 | KEY_DEST_CLASS_REG); 1258 1259 /* For IPsec load the salt from keymat in the context register */ 1260 if (is_ipsec) 1261 append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4, 1262 LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | 1263 4 << LDST_OFFSET_SHIFT); 1264 1265 set_jump_tgt_here(desc, key_jump_cmd); 1266 1267 /* Class 2 and 1 operations: Poly & ChaCha */ 1268 if (encap) { 1269 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 1270 OP_ALG_ENCRYPT); 1271 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1272 OP_ALG_ENCRYPT); 1273 } else { 1274 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 1275 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1276 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1277 OP_ALG_DECRYPT); 1278 } 1279 1280 if (is_qi) { 1281 u32 *wait_load_cmd; 1282 u32 ctx1_iv_off = is_ipsec ? 8 : 4; 1283 1284 /* REG3 = assoclen */ 1285 append_seq_load(desc, 4, LDST_CLASS_DECO | 1286 LDST_SRCDST_WORD_DECO_MATH3 | 1287 4 << LDST_OFFSET_SHIFT); 1288 1289 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1290 JUMP_COND_CALM | JUMP_COND_NCP | 1291 JUMP_COND_NOP | JUMP_COND_NIP | 1292 JUMP_COND_NIFP); 1293 set_jump_tgt_here(desc, wait_load_cmd); 1294 1295 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 1296 LDST_SRCDST_BYTE_CONTEXT | 1297 ctx1_iv_off << LDST_OFFSET_SHIFT); 1298 } 1299 1300 /* 1301 * MAGIC with NFIFO 1302 * Read associated data from the input and send them to class1 and 1303 * class2 alignment blocks. From class1 send data to output fifo and 1304 * then write it to memory since we don't need to encrypt AD. 1305 */ 1306 nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 | 1307 NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND; 1308 append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB | 1309 LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3); 1310 1311 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 1312 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 1313 append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO | 1314 FIFOLD_CLASS_CLASS1 | LDST_VLF); 1315 append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK | 1316 MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3); 1317 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF); 1318 1319 /* IPsec - copy IV at the output */ 1320 if (is_ipsec) 1321 append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA | 1322 0x2 << 25); 1323 1324 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL | 1325 JUMP_COND_NOP | JUMP_TEST_ALL); 1326 set_jump_tgt_here(desc, wait_cmd); 1327 1328 if (encap) { 1329 /* Read and write cryptlen bytes */ 1330 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1331 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, 1332 CAAM_CMD_SZ); 1333 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 1334 1335 /* Write ICV */ 1336 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 1337 LDST_SRCDST_BYTE_CONTEXT); 1338 } else { 1339 /* Read and write cryptlen bytes */ 1340 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, 1341 CAAM_CMD_SZ); 1342 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, 1343 CAAM_CMD_SZ); 1344 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 1345 1346 /* Load ICV for verification */ 1347 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | 1348 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 1349 } 1350 1351 print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ", 1352 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1353 1); 1354 } 1355 EXPORT_SYMBOL(cnstr_shdsc_chachapoly); 1356 1357 /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */ 1358 static inline void skcipher_append_src_dst(u32 *desc) 1359 { 1360 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1361 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1362 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 1363 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 1364 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 1365 } 1366 1367 /** 1368 * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor 1369 * @desc: pointer to buffer used for descriptor construction 1370 * @cdata: pointer to block cipher transform definitions 1371 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1372 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128 1373 * - OP_ALG_ALGSEL_CHACHA20 1374 * @ivsize: initialization vector size 1375 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1376 * @ctx1_iv_off: IV offset in CONTEXT1 register 1377 */ 1378 void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, 1379 unsigned int ivsize, const bool is_rfc3686, 1380 const u32 ctx1_iv_off) 1381 { 1382 u32 *key_jump_cmd; 1383 u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT; 1384 bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) == 1385 OP_ALG_ALGSEL_CHACHA20); 1386 1387 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1388 /* Skip if already shared */ 1389 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1390 JUMP_COND_SHRD); 1391 1392 /* Load class1 key only */ 1393 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1394 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1395 1396 /* Load nonce into CONTEXT1 reg */ 1397 if (is_rfc3686) { 1398 const u8 *nonce = cdata->key_virt + cdata->keylen; 1399 1400 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1401 LDST_CLASS_IND_CCB | 1402 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1403 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1404 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1405 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1406 } 1407 1408 set_jump_tgt_here(desc, key_jump_cmd); 1409 1410 /* Load IV, if there is one */ 1411 if (ivsize) 1412 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1413 LDST_CLASS_1_CCB | (ctx1_iv_off << 1414 LDST_OFFSET_SHIFT)); 1415 1416 /* Load counter into CONTEXT1 reg */ 1417 if (is_rfc3686) 1418 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1419 LDST_SRCDST_BYTE_CONTEXT | 1420 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1421 LDST_OFFSET_SHIFT)); 1422 1423 /* Load operation */ 1424 if (is_chacha20) 1425 options |= OP_ALG_AS_FINALIZE; 1426 append_operation(desc, options); 1427 1428 /* Perform operation */ 1429 skcipher_append_src_dst(desc); 1430 1431 /* Store IV */ 1432 if (!is_chacha20 && ivsize) 1433 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1434 LDST_CLASS_1_CCB | (ctx1_iv_off << 1435 LDST_OFFSET_SHIFT)); 1436 1437 print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ", 1438 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1439 1); 1440 } 1441 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap); 1442 1443 /** 1444 * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor 1445 * @desc: pointer to buffer used for descriptor construction 1446 * @cdata: pointer to block cipher transform definitions 1447 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1448 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128 1449 * - OP_ALG_ALGSEL_CHACHA20 1450 * @ivsize: initialization vector size 1451 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1452 * @ctx1_iv_off: IV offset in CONTEXT1 register 1453 */ 1454 void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, 1455 unsigned int ivsize, const bool is_rfc3686, 1456 const u32 ctx1_iv_off) 1457 { 1458 u32 *key_jump_cmd; 1459 bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) == 1460 OP_ALG_ALGSEL_CHACHA20); 1461 1462 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1463 /* Skip if already shared */ 1464 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1465 JUMP_COND_SHRD); 1466 1467 /* Load class1 key only */ 1468 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1469 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1470 1471 /* Load nonce into CONTEXT1 reg */ 1472 if (is_rfc3686) { 1473 const u8 *nonce = cdata->key_virt + cdata->keylen; 1474 1475 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1476 LDST_CLASS_IND_CCB | 1477 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1478 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1479 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1480 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1481 } 1482 1483 set_jump_tgt_here(desc, key_jump_cmd); 1484 1485 /* Load IV, if there is one */ 1486 if (ivsize) 1487 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1488 LDST_CLASS_1_CCB | (ctx1_iv_off << 1489 LDST_OFFSET_SHIFT)); 1490 1491 /* Load counter into CONTEXT1 reg */ 1492 if (is_rfc3686) 1493 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1494 LDST_SRCDST_BYTE_CONTEXT | 1495 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1496 LDST_OFFSET_SHIFT)); 1497 1498 /* Choose operation */ 1499 if (ctx1_iv_off) 1500 append_operation(desc, cdata->algtype | OP_ALG_AS_INIT | 1501 OP_ALG_DECRYPT); 1502 else 1503 append_dec_op1(desc, cdata->algtype); 1504 1505 /* Perform operation */ 1506 skcipher_append_src_dst(desc); 1507 1508 /* Store IV */ 1509 if (!is_chacha20 && ivsize) 1510 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1511 LDST_CLASS_1_CCB | (ctx1_iv_off << 1512 LDST_OFFSET_SHIFT)); 1513 1514 print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ", 1515 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1516 1); 1517 } 1518 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap); 1519 1520 /** 1521 * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor 1522 * @desc: pointer to buffer used for descriptor construction 1523 * @cdata: pointer to block cipher transform definitions 1524 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. 1525 */ 1526 void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) 1527 { 1528 /* 1529 * Set sector size to a big value, practically disabling 1530 * sector size segmentation in xts implementation. We cannot 1531 * take full advantage of this HW feature with existing 1532 * crypto API / dm-crypt SW architecture. 1533 */ 1534 __be64 sector_size = cpu_to_be64(BIT(15)); 1535 u32 *key_jump_cmd; 1536 1537 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1538 /* Skip if already shared */ 1539 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1540 JUMP_COND_SHRD); 1541 1542 /* Load class1 keys only */ 1543 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1544 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1545 1546 /* Load sector size with index 40 bytes (0x28) */ 1547 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | 1548 LDST_SRCDST_BYTE_CONTEXT | 1549 (0x28 << LDST_OFFSET_SHIFT)); 1550 1551 set_jump_tgt_here(desc, key_jump_cmd); 1552 1553 /* 1554 * create sequence for loading the sector index / 16B tweak value 1555 * Lower 8B of IV - sector index / tweak lower half 1556 * Upper 8B of IV - upper half of 16B tweak 1557 */ 1558 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1559 (0x20 << LDST_OFFSET_SHIFT)); 1560 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1561 (0x30 << LDST_OFFSET_SHIFT)); 1562 1563 /* Load operation */ 1564 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1565 OP_ALG_ENCRYPT); 1566 1567 /* Perform operation */ 1568 skcipher_append_src_dst(desc); 1569 1570 /* Store lower 8B and upper 8B of IV */ 1571 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1572 (0x20 << LDST_OFFSET_SHIFT)); 1573 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1574 (0x30 << LDST_OFFSET_SHIFT)); 1575 1576 print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__) 1577 ": ", DUMP_PREFIX_ADDRESS, 16, 4, 1578 desc, desc_bytes(desc), 1); 1579 } 1580 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap); 1581 1582 /** 1583 * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor 1584 * @desc: pointer to buffer used for descriptor construction 1585 * @cdata: pointer to block cipher transform definitions 1586 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. 1587 */ 1588 void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) 1589 { 1590 /* 1591 * Set sector size to a big value, practically disabling 1592 * sector size segmentation in xts implementation. We cannot 1593 * take full advantage of this HW feature with existing 1594 * crypto API / dm-crypt SW architecture. 1595 */ 1596 __be64 sector_size = cpu_to_be64(BIT(15)); 1597 u32 *key_jump_cmd; 1598 1599 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1600 /* Skip if already shared */ 1601 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1602 JUMP_COND_SHRD); 1603 1604 /* Load class1 key only */ 1605 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1606 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1607 1608 /* Load sector size with index 40 bytes (0x28) */ 1609 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | 1610 LDST_SRCDST_BYTE_CONTEXT | 1611 (0x28 << LDST_OFFSET_SHIFT)); 1612 1613 set_jump_tgt_here(desc, key_jump_cmd); 1614 1615 /* 1616 * create sequence for loading the sector index / 16B tweak value 1617 * Lower 8B of IV - sector index / tweak lower half 1618 * Upper 8B of IV - upper half of 16B tweak 1619 */ 1620 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1621 (0x20 << LDST_OFFSET_SHIFT)); 1622 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1623 (0x30 << LDST_OFFSET_SHIFT)); 1624 /* Load operation */ 1625 append_dec_op1(desc, cdata->algtype); 1626 1627 /* Perform operation */ 1628 skcipher_append_src_dst(desc); 1629 1630 /* Store lower 8B and upper 8B of IV */ 1631 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1632 (0x20 << LDST_OFFSET_SHIFT)); 1633 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1634 (0x30 << LDST_OFFSET_SHIFT)); 1635 1636 print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__) 1637 ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, 1638 desc_bytes(desc), 1); 1639 } 1640 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap); 1641 1642 MODULE_LICENSE("GPL"); 1643 MODULE_DESCRIPTION("FSL CAAM descriptor support"); 1644 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 1645