1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * 6 * Based on talitos crypto API driver. 7 * 8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 9 * 10 * --------------- --------------- 11 * | JobDesc #1 |-------------------->| ShareDesc | 12 * | *(packet 1) | | (PDB) | 13 * --------------- |------------->| (hashKey) | 14 * . | | (cipherKey) | 15 * . | |-------->| (operation) | 16 * --------------- | | --------------- 17 * | JobDesc #2 |------| | 18 * | *(packet 2) | | 19 * --------------- | 20 * . | 21 * . | 22 * --------------- | 23 * | JobDesc #3 |------------ 24 * | *(packet 3) | 25 * --------------- 26 * 27 * The SharedDesc never changes for a connection unless rekeyed, but 28 * each packet will likely be in a different place. So all we need 29 * to know to process the packet is where the input is, where the 30 * output goes, and what context we want to process with. Context is 31 * in the SharedDesc, packet references in the JobDesc. 32 * 33 * So, a job desc looks like: 34 * 35 * --------------------- 36 * | Header | 37 * | ShareDesc Pointer | 38 * | SEQ_OUT_PTR | 39 * | (output buffer) | 40 * | (output length) | 41 * | SEQ_IN_PTR | 42 * | (input buffer) | 43 * | (input length) | 44 * --------------------- 45 */ 46 47 #include "compat.h" 48 49 #include "regs.h" 50 #include "intern.h" 51 #include "desc_constr.h" 52 #include "jr.h" 53 #include "error.h" 54 #include "sg_sw_sec4.h" 55 #include "key_gen.h" 56 57 /* 58 * crypto alg 59 */ 60 #define CAAM_CRA_PRIORITY 3000 61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 62 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 63 CTR_RFC3686_NONCE_SIZE + \ 64 SHA512_DIGEST_SIZE * 2) 65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 66 #define CAAM_MAX_IV_LENGTH 16 67 68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 70 CAAM_CMD_SZ * 4) 71 72 /* length of descriptors text */ 73 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) 74 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) 75 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ) 76 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) 77 78 /* Note: Nonce is counted in enckeylen */ 79 #define DESC_AEAD_CTR_RFC3686_LEN (6 * CAAM_CMD_SZ) 80 81 #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) 82 #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ) 83 #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ) 84 85 #define DESC_GCM_BASE (3 * CAAM_CMD_SZ) 86 #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) 87 #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) 88 89 #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) 90 #define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 10 * CAAM_CMD_SZ) 91 #define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 10 * CAAM_CMD_SZ) 92 93 #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) 94 #define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) 95 #define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) 96 97 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) 98 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 99 20 * CAAM_CMD_SZ) 100 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 101 15 * CAAM_CMD_SZ) 102 103 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) 104 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 105 106 #ifdef DEBUG 107 /* for print_hex_dumps with line references */ 108 #define debug(format, arg...) printk(format, arg) 109 #else 110 #define debug(format, arg...) 111 #endif 112 static struct list_head alg_list; 113 114 /* Set DK bit in class 1 operation if shared */ 115 static inline void append_dec_op1(u32 *desc, u32 type) 116 { 117 u32 *jump_cmd, *uncond_jump_cmd; 118 119 /* DK bit is valid only for AES */ 120 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { 121 append_operation(desc, type | OP_ALG_AS_INITFINAL | 122 OP_ALG_DECRYPT); 123 return; 124 } 125 126 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 127 append_operation(desc, type | OP_ALG_AS_INITFINAL | 128 OP_ALG_DECRYPT); 129 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 130 set_jump_tgt_here(desc, jump_cmd); 131 append_operation(desc, type | OP_ALG_AS_INITFINAL | 132 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 133 set_jump_tgt_here(desc, uncond_jump_cmd); 134 } 135 136 /* 137 * For aead functions, read payload and write payload, 138 * both of which are specified in req->src and req->dst 139 */ 140 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 141 { 142 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 143 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 144 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 145 } 146 147 /* 148 * For aead encrypt and decrypt, read iv for both classes 149 */ 150 static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset) 151 { 152 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 153 LDST_SRCDST_BYTE_CONTEXT | 154 (ivoffset << LDST_OFFSET_SHIFT)); 155 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | 156 (ivoffset << MOVE_OFFSET_SHIFT) | ivsize); 157 } 158 159 /* 160 * For ablkcipher encrypt and decrypt, read from req->src and 161 * write to req->dst 162 */ 163 static inline void ablkcipher_append_src_dst(u32 *desc) 164 { 165 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 166 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 167 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 168 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 169 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 170 } 171 172 /* 173 * If all data, including src (with assoc and iv) or dst (with iv only) are 174 * contiguous 175 */ 176 #define GIV_SRC_CONTIG 1 177 #define GIV_DST_CONTIG (1 << 1) 178 179 /* 180 * per-session context 181 */ 182 struct caam_ctx { 183 struct device *jrdev; 184 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 185 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 186 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 187 dma_addr_t sh_desc_enc_dma; 188 dma_addr_t sh_desc_dec_dma; 189 dma_addr_t sh_desc_givenc_dma; 190 u32 class1_alg_type; 191 u32 class2_alg_type; 192 u32 alg_op; 193 u8 key[CAAM_MAX_KEY_SIZE]; 194 dma_addr_t key_dma; 195 unsigned int enckeylen; 196 unsigned int split_key_len; 197 unsigned int split_key_pad_len; 198 unsigned int authsize; 199 }; 200 201 static void append_key_aead(u32 *desc, struct caam_ctx *ctx, 202 int keys_fit_inline, bool is_rfc3686) 203 { 204 u32 *nonce; 205 unsigned int enckeylen = ctx->enckeylen; 206 207 /* 208 * RFC3686 specific: 209 * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} 210 * | enckeylen = encryption key size + nonce size 211 */ 212 if (is_rfc3686) 213 enckeylen -= CTR_RFC3686_NONCE_SIZE; 214 215 if (keys_fit_inline) { 216 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 217 ctx->split_key_len, CLASS_2 | 218 KEY_DEST_MDHA_SPLIT | KEY_ENC); 219 append_key_as_imm(desc, (void *)ctx->key + 220 ctx->split_key_pad_len, enckeylen, 221 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 222 } else { 223 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 224 KEY_DEST_MDHA_SPLIT | KEY_ENC); 225 append_key(desc, ctx->key_dma + ctx->split_key_pad_len, 226 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 227 } 228 229 /* Load Counter into CONTEXT1 reg */ 230 if (is_rfc3686) { 231 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + 232 enckeylen); 233 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 234 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 235 append_move(desc, 236 MOVE_SRC_OUTFIFO | 237 MOVE_DEST_CLASS1CTX | 238 (16 << MOVE_OFFSET_SHIFT) | 239 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 240 } 241 } 242 243 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, 244 int keys_fit_inline, bool is_rfc3686) 245 { 246 u32 *key_jump_cmd; 247 248 /* Note: Context registers are saved. */ 249 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 250 251 /* Skip if already shared */ 252 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 253 JUMP_COND_SHRD); 254 255 append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 256 257 set_jump_tgt_here(desc, key_jump_cmd); 258 } 259 260 static int aead_null_set_sh_desc(struct crypto_aead *aead) 261 { 262 unsigned int ivsize = crypto_aead_ivsize(aead); 263 struct caam_ctx *ctx = crypto_aead_ctx(aead); 264 struct device *jrdev = ctx->jrdev; 265 bool keys_fit_inline = false; 266 u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd; 267 u32 *desc; 268 269 /* 270 * Job Descriptor and Shared Descriptors 271 * must all fit into the 64-word Descriptor h/w Buffer 272 */ 273 if (DESC_AEAD_NULL_ENC_LEN + DESC_JOB_IO_LEN + 274 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 275 keys_fit_inline = true; 276 277 /* old_aead_encrypt shared descriptor */ 278 desc = ctx->sh_desc_enc; 279 280 init_sh_desc(desc, HDR_SHARE_SERIAL); 281 282 /* Skip if already shared */ 283 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 284 JUMP_COND_SHRD); 285 if (keys_fit_inline) 286 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 287 ctx->split_key_len, CLASS_2 | 288 KEY_DEST_MDHA_SPLIT | KEY_ENC); 289 else 290 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 291 KEY_DEST_MDHA_SPLIT | KEY_ENC); 292 set_jump_tgt_here(desc, key_jump_cmd); 293 294 /* cryptlen = seqoutlen - authsize */ 295 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 296 297 /* 298 * NULL encryption; IV is zero 299 * assoclen = (assoclen + cryptlen) - cryptlen 300 */ 301 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 302 303 /* read assoc before reading payload */ 304 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 305 KEY_VLF); 306 307 /* Prepare to read and write cryptlen bytes */ 308 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 309 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 310 311 /* 312 * MOVE_LEN opcode is not available in all SEC HW revisions, 313 * thus need to do some magic, i.e. self-patch the descriptor 314 * buffer. 315 */ 316 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 317 MOVE_DEST_MATH3 | 318 (0x6 << MOVE_LEN_SHIFT)); 319 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | 320 MOVE_DEST_DESCBUF | 321 MOVE_WAITCOMP | 322 (0x8 << MOVE_LEN_SHIFT)); 323 324 /* Class 2 operation */ 325 append_operation(desc, ctx->class2_alg_type | 326 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 327 328 /* Read and write cryptlen bytes */ 329 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 330 331 set_move_tgt_here(desc, read_move_cmd); 332 set_move_tgt_here(desc, write_move_cmd); 333 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 334 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 335 MOVE_AUX_LS); 336 337 /* Write ICV */ 338 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 339 LDST_SRCDST_BYTE_CONTEXT); 340 341 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 342 desc_bytes(desc), 343 DMA_TO_DEVICE); 344 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 345 dev_err(jrdev, "unable to map shared descriptor\n"); 346 return -ENOMEM; 347 } 348 #ifdef DEBUG 349 print_hex_dump(KERN_ERR, 350 "aead null enc shdesc@"__stringify(__LINE__)": ", 351 DUMP_PREFIX_ADDRESS, 16, 4, desc, 352 desc_bytes(desc), 1); 353 #endif 354 355 /* 356 * Job Descriptor and Shared Descriptors 357 * must all fit into the 64-word Descriptor h/w Buffer 358 */ 359 keys_fit_inline = false; 360 if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN + 361 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 362 keys_fit_inline = true; 363 364 desc = ctx->sh_desc_dec; 365 366 /* old_aead_decrypt shared descriptor */ 367 init_sh_desc(desc, HDR_SHARE_SERIAL); 368 369 /* Skip if already shared */ 370 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 371 JUMP_COND_SHRD); 372 if (keys_fit_inline) 373 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 374 ctx->split_key_len, CLASS_2 | 375 KEY_DEST_MDHA_SPLIT | KEY_ENC); 376 else 377 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 378 KEY_DEST_MDHA_SPLIT | KEY_ENC); 379 set_jump_tgt_here(desc, key_jump_cmd); 380 381 /* Class 2 operation */ 382 append_operation(desc, ctx->class2_alg_type | 383 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 384 385 /* assoclen + cryptlen = seqinlen - ivsize - authsize */ 386 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, 387 ctx->authsize + ivsize); 388 /* assoclen = (assoclen + cryptlen) - cryptlen */ 389 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 390 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); 391 392 /* read assoc before reading payload */ 393 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 394 KEY_VLF); 395 396 /* Prepare to read and write cryptlen bytes */ 397 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 398 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 399 400 /* 401 * MOVE_LEN opcode is not available in all SEC HW revisions, 402 * thus need to do some magic, i.e. self-patch the descriptor 403 * buffer. 404 */ 405 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 406 MOVE_DEST_MATH2 | 407 (0x6 << MOVE_LEN_SHIFT)); 408 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | 409 MOVE_DEST_DESCBUF | 410 MOVE_WAITCOMP | 411 (0x8 << MOVE_LEN_SHIFT)); 412 413 /* Read and write cryptlen bytes */ 414 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 415 416 /* 417 * Insert a NOP here, since we need at least 4 instructions between 418 * code patching the descriptor buffer and the location being patched. 419 */ 420 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 421 set_jump_tgt_here(desc, jump_cmd); 422 423 set_move_tgt_here(desc, read_move_cmd); 424 set_move_tgt_here(desc, write_move_cmd); 425 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 426 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 427 MOVE_AUX_LS); 428 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 429 430 /* Load ICV */ 431 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 432 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 433 434 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 435 desc_bytes(desc), 436 DMA_TO_DEVICE); 437 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 438 dev_err(jrdev, "unable to map shared descriptor\n"); 439 return -ENOMEM; 440 } 441 #ifdef DEBUG 442 print_hex_dump(KERN_ERR, 443 "aead null dec shdesc@"__stringify(__LINE__)": ", 444 DUMP_PREFIX_ADDRESS, 16, 4, desc, 445 desc_bytes(desc), 1); 446 #endif 447 448 return 0; 449 } 450 451 static int aead_set_sh_desc(struct crypto_aead *aead) 452 { 453 unsigned int ivsize = crypto_aead_ivsize(aead); 454 struct caam_ctx *ctx = crypto_aead_ctx(aead); 455 struct crypto_tfm *ctfm = crypto_aead_tfm(aead); 456 const char *alg_name = crypto_tfm_alg_name(ctfm); 457 struct device *jrdev = ctx->jrdev; 458 bool keys_fit_inline; 459 u32 geniv, moveiv; 460 u32 ctx1_iv_off = 0; 461 u32 *desc; 462 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 463 OP_ALG_AAI_CTR_MOD128); 464 const bool is_rfc3686 = (ctr_mode && 465 (strstr(alg_name, "rfc3686") != NULL)); 466 467 if (!ctx->authsize) 468 return 0; 469 470 /* NULL encryption / decryption */ 471 if (!ctx->enckeylen) 472 return aead_null_set_sh_desc(aead); 473 474 /* 475 * AES-CTR needs to load IV in CONTEXT1 reg 476 * at an offset of 128bits (16bytes) 477 * CONTEXT1[255:128] = IV 478 */ 479 if (ctr_mode) 480 ctx1_iv_off = 16; 481 482 /* 483 * RFC3686 specific: 484 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 485 */ 486 if (is_rfc3686) 487 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 488 489 /* 490 * Job Descriptor and Shared Descriptors 491 * must all fit into the 64-word Descriptor h/w Buffer 492 */ 493 keys_fit_inline = false; 494 if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + 495 ctx->split_key_pad_len + ctx->enckeylen + 496 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 497 CAAM_DESC_BYTES_MAX) 498 keys_fit_inline = true; 499 500 /* old_aead_encrypt shared descriptor */ 501 desc = ctx->sh_desc_enc; 502 503 /* Note: Context registers are saved. */ 504 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 505 506 /* Class 2 operation */ 507 append_operation(desc, ctx->class2_alg_type | 508 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 509 510 /* cryptlen = seqoutlen - authsize */ 511 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 512 513 /* assoclen + cryptlen = seqinlen - ivsize */ 514 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, ivsize); 515 516 /* assoclen = (assoclen + cryptlen) - cryptlen */ 517 append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); 518 519 /* read assoc before reading payload */ 520 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 521 KEY_VLF); 522 aead_append_ld_iv(desc, ivsize, ctx1_iv_off); 523 524 /* Load Counter into CONTEXT1 reg */ 525 if (is_rfc3686) 526 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 527 LDST_CLASS_1_CCB | 528 LDST_SRCDST_BYTE_CONTEXT | 529 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 530 LDST_OFFSET_SHIFT)); 531 532 /* Class 1 operation */ 533 append_operation(desc, ctx->class1_alg_type | 534 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 535 536 /* Read and write cryptlen bytes */ 537 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 538 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 539 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 540 541 /* Write ICV */ 542 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 543 LDST_SRCDST_BYTE_CONTEXT); 544 545 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 546 desc_bytes(desc), 547 DMA_TO_DEVICE); 548 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 549 dev_err(jrdev, "unable to map shared descriptor\n"); 550 return -ENOMEM; 551 } 552 #ifdef DEBUG 553 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", 554 DUMP_PREFIX_ADDRESS, 16, 4, desc, 555 desc_bytes(desc), 1); 556 #endif 557 558 /* 559 * Job Descriptor and Shared Descriptors 560 * must all fit into the 64-word Descriptor h/w Buffer 561 */ 562 keys_fit_inline = false; 563 if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + 564 ctx->split_key_pad_len + ctx->enckeylen + 565 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 566 CAAM_DESC_BYTES_MAX) 567 keys_fit_inline = true; 568 569 /* old_aead_decrypt shared descriptor */ 570 desc = ctx->sh_desc_dec; 571 572 /* Note: Context registers are saved. */ 573 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 574 575 /* Class 2 operation */ 576 append_operation(desc, ctx->class2_alg_type | 577 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 578 579 /* assoclen + cryptlen = seqinlen - ivsize - authsize */ 580 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, 581 ctx->authsize + ivsize); 582 /* assoclen = (assoclen + cryptlen) - cryptlen */ 583 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 584 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); 585 586 /* read assoc before reading payload */ 587 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 588 KEY_VLF); 589 590 aead_append_ld_iv(desc, ivsize, ctx1_iv_off); 591 592 /* Load Counter into CONTEXT1 reg */ 593 if (is_rfc3686) 594 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 595 LDST_CLASS_1_CCB | 596 LDST_SRCDST_BYTE_CONTEXT | 597 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 598 LDST_OFFSET_SHIFT)); 599 600 /* Choose operation */ 601 if (ctr_mode) 602 append_operation(desc, ctx->class1_alg_type | 603 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 604 else 605 append_dec_op1(desc, ctx->class1_alg_type); 606 607 /* Read and write cryptlen bytes */ 608 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 609 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 610 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 611 612 /* Load ICV */ 613 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 614 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 615 616 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 617 desc_bytes(desc), 618 DMA_TO_DEVICE); 619 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 620 dev_err(jrdev, "unable to map shared descriptor\n"); 621 return -ENOMEM; 622 } 623 #ifdef DEBUG 624 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", 625 DUMP_PREFIX_ADDRESS, 16, 4, desc, 626 desc_bytes(desc), 1); 627 #endif 628 629 /* 630 * Job Descriptor and Shared Descriptors 631 * must all fit into the 64-word Descriptor h/w Buffer 632 */ 633 keys_fit_inline = false; 634 if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + 635 ctx->split_key_pad_len + ctx->enckeylen + 636 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 637 CAAM_DESC_BYTES_MAX) 638 keys_fit_inline = true; 639 640 /* aead_givencrypt shared descriptor */ 641 desc = ctx->sh_desc_givenc; 642 643 /* Note: Context registers are saved. */ 644 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 645 646 /* Generate IV */ 647 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 648 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 649 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); 650 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 651 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 652 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 653 append_move(desc, MOVE_WAITCOMP | 654 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | 655 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 656 (ivsize << MOVE_LEN_SHIFT)); 657 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 658 659 /* Copy IV to class 1 context */ 660 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | 661 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 662 (ivsize << MOVE_LEN_SHIFT)); 663 664 /* Return to encryption */ 665 append_operation(desc, ctx->class2_alg_type | 666 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 667 668 /* ivsize + cryptlen = seqoutlen - authsize */ 669 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 670 671 /* assoclen = seqinlen - (ivsize + cryptlen) */ 672 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 673 674 /* read assoc before reading payload */ 675 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 676 KEY_VLF); 677 678 /* Copy iv from outfifo to class 2 fifo */ 679 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 680 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); 681 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 682 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 683 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | 684 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 685 686 /* Load Counter into CONTEXT1 reg */ 687 if (is_rfc3686) 688 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 689 LDST_CLASS_1_CCB | 690 LDST_SRCDST_BYTE_CONTEXT | 691 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 692 LDST_OFFSET_SHIFT)); 693 694 /* Class 1 operation */ 695 append_operation(desc, ctx->class1_alg_type | 696 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 697 698 /* Will write ivsize + cryptlen */ 699 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 700 701 /* Not need to reload iv */ 702 append_seq_fifo_load(desc, ivsize, 703 FIFOLD_CLASS_SKIP); 704 705 /* Will read cryptlen */ 706 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 707 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 708 709 /* Write ICV */ 710 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 711 LDST_SRCDST_BYTE_CONTEXT); 712 713 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 714 desc_bytes(desc), 715 DMA_TO_DEVICE); 716 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 717 dev_err(jrdev, "unable to map shared descriptor\n"); 718 return -ENOMEM; 719 } 720 #ifdef DEBUG 721 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", 722 DUMP_PREFIX_ADDRESS, 16, 4, desc, 723 desc_bytes(desc), 1); 724 #endif 725 726 return 0; 727 } 728 729 static int aead_setauthsize(struct crypto_aead *authenc, 730 unsigned int authsize) 731 { 732 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 733 734 ctx->authsize = authsize; 735 aead_set_sh_desc(authenc); 736 737 return 0; 738 } 739 740 static int gcm_set_sh_desc(struct crypto_aead *aead) 741 { 742 struct caam_ctx *ctx = crypto_aead_ctx(aead); 743 struct device *jrdev = ctx->jrdev; 744 bool keys_fit_inline = false; 745 u32 *key_jump_cmd, *zero_payload_jump_cmd, 746 *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; 747 u32 *desc; 748 749 if (!ctx->enckeylen || !ctx->authsize) 750 return 0; 751 752 /* 753 * AES GCM encrypt shared descriptor 754 * Job Descriptor and Shared Descriptor 755 * must fit into the 64-word Descriptor h/w Buffer 756 */ 757 if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN + 758 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 759 keys_fit_inline = true; 760 761 desc = ctx->sh_desc_enc; 762 763 init_sh_desc(desc, HDR_SHARE_SERIAL); 764 765 /* skip key loading if they are loaded due to sharing */ 766 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 767 JUMP_COND_SHRD | JUMP_COND_SELF); 768 if (keys_fit_inline) 769 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 770 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 771 else 772 append_key(desc, ctx->key_dma, ctx->enckeylen, 773 CLASS_1 | KEY_DEST_CLASS_REG); 774 set_jump_tgt_here(desc, key_jump_cmd); 775 776 /* class 1 operation */ 777 append_operation(desc, ctx->class1_alg_type | 778 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 779 780 /* if assoclen + cryptlen is ZERO, skip to ICV write */ 781 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 782 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | 783 JUMP_COND_MATH_Z); 784 785 /* if assoclen is ZERO, skip reading the assoc data */ 786 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 787 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 788 JUMP_COND_MATH_Z); 789 790 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 791 792 /* skip assoc data */ 793 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 794 795 /* cryptlen = seqinlen - assoclen */ 796 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 797 798 /* if cryptlen is ZERO jump to zero-payload commands */ 799 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 800 JUMP_COND_MATH_Z); 801 802 /* read assoc data */ 803 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 804 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 805 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 806 807 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 808 809 /* write encrypted data */ 810 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 811 812 /* read payload data */ 813 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 814 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 815 816 /* jump the zero-payload commands */ 817 append_jump(desc, JUMP_TEST_ALL | 2); 818 819 /* zero-payload commands */ 820 set_jump_tgt_here(desc, zero_payload_jump_cmd); 821 822 /* read assoc data */ 823 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 824 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 825 826 /* There is no input data */ 827 set_jump_tgt_here(desc, zero_assoc_jump_cmd2); 828 829 /* write ICV */ 830 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 831 LDST_SRCDST_BYTE_CONTEXT); 832 833 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 834 desc_bytes(desc), 835 DMA_TO_DEVICE); 836 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 837 dev_err(jrdev, "unable to map shared descriptor\n"); 838 return -ENOMEM; 839 } 840 #ifdef DEBUG 841 print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", 842 DUMP_PREFIX_ADDRESS, 16, 4, desc, 843 desc_bytes(desc), 1); 844 #endif 845 846 /* 847 * Job Descriptor and Shared Descriptors 848 * must all fit into the 64-word Descriptor h/w Buffer 849 */ 850 keys_fit_inline = false; 851 if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN + 852 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 853 keys_fit_inline = true; 854 855 desc = ctx->sh_desc_dec; 856 857 init_sh_desc(desc, HDR_SHARE_SERIAL); 858 859 /* skip key loading if they are loaded due to sharing */ 860 key_jump_cmd = append_jump(desc, JUMP_JSL | 861 JUMP_TEST_ALL | JUMP_COND_SHRD | 862 JUMP_COND_SELF); 863 if (keys_fit_inline) 864 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 865 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 866 else 867 append_key(desc, ctx->key_dma, ctx->enckeylen, 868 CLASS_1 | KEY_DEST_CLASS_REG); 869 set_jump_tgt_here(desc, key_jump_cmd); 870 871 /* class 1 operation */ 872 append_operation(desc, ctx->class1_alg_type | 873 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 874 875 /* if assoclen is ZERO, skip reading the assoc data */ 876 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 877 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 878 JUMP_COND_MATH_Z); 879 880 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 881 882 /* skip assoc data */ 883 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 884 885 /* read assoc data */ 886 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 887 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 888 889 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 890 891 /* cryptlen = seqoutlen - assoclen */ 892 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 893 894 /* jump to zero-payload command if cryptlen is zero */ 895 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 896 JUMP_COND_MATH_Z); 897 898 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 899 900 /* store encrypted data */ 901 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 902 903 /* read payload data */ 904 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 905 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 906 907 /* zero-payload command */ 908 set_jump_tgt_here(desc, zero_payload_jump_cmd); 909 910 /* read ICV */ 911 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 912 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 913 914 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 915 desc_bytes(desc), 916 DMA_TO_DEVICE); 917 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 918 dev_err(jrdev, "unable to map shared descriptor\n"); 919 return -ENOMEM; 920 } 921 #ifdef DEBUG 922 print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", 923 DUMP_PREFIX_ADDRESS, 16, 4, desc, 924 desc_bytes(desc), 1); 925 #endif 926 927 return 0; 928 } 929 930 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 931 { 932 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 933 934 ctx->authsize = authsize; 935 gcm_set_sh_desc(authenc); 936 937 return 0; 938 } 939 940 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 941 { 942 struct caam_ctx *ctx = crypto_aead_ctx(aead); 943 struct device *jrdev = ctx->jrdev; 944 bool keys_fit_inline = false; 945 u32 *key_jump_cmd; 946 u32 *desc; 947 948 if (!ctx->enckeylen || !ctx->authsize) 949 return 0; 950 951 /* 952 * RFC4106 encrypt shared descriptor 953 * Job Descriptor and Shared Descriptor 954 * must fit into the 64-word Descriptor h/w Buffer 955 */ 956 if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN + 957 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 958 keys_fit_inline = true; 959 960 desc = ctx->sh_desc_enc; 961 962 init_sh_desc(desc, HDR_SHARE_SERIAL); 963 964 /* Skip key loading if it is loaded due to sharing */ 965 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 966 JUMP_COND_SHRD); 967 if (keys_fit_inline) 968 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 969 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 970 else 971 append_key(desc, ctx->key_dma, ctx->enckeylen, 972 CLASS_1 | KEY_DEST_CLASS_REG); 973 set_jump_tgt_here(desc, key_jump_cmd); 974 975 /* Class 1 operation */ 976 append_operation(desc, ctx->class1_alg_type | 977 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 978 979 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 980 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 981 982 /* Skip assoc data */ 983 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 984 985 /* Read assoc data */ 986 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 987 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 988 989 /* cryptlen = seqoutlen - assoclen */ 990 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 991 992 /* Will read cryptlen bytes */ 993 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 994 995 /* Write encrypted data */ 996 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 997 998 /* Read payload data */ 999 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1000 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 1001 1002 /* Write ICV */ 1003 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 1004 LDST_SRCDST_BYTE_CONTEXT); 1005 1006 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1007 desc_bytes(desc), 1008 DMA_TO_DEVICE); 1009 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1010 dev_err(jrdev, "unable to map shared descriptor\n"); 1011 return -ENOMEM; 1012 } 1013 #ifdef DEBUG 1014 print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ", 1015 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1016 desc_bytes(desc), 1); 1017 #endif 1018 1019 /* 1020 * Job Descriptor and Shared Descriptors 1021 * must all fit into the 64-word Descriptor h/w Buffer 1022 */ 1023 keys_fit_inline = false; 1024 if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN + 1025 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1026 keys_fit_inline = true; 1027 1028 desc = ctx->sh_desc_dec; 1029 1030 init_sh_desc(desc, HDR_SHARE_SERIAL); 1031 1032 /* Skip key loading if it is loaded due to sharing */ 1033 key_jump_cmd = append_jump(desc, JUMP_JSL | 1034 JUMP_TEST_ALL | JUMP_COND_SHRD); 1035 if (keys_fit_inline) 1036 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1037 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1038 else 1039 append_key(desc, ctx->key_dma, ctx->enckeylen, 1040 CLASS_1 | KEY_DEST_CLASS_REG); 1041 set_jump_tgt_here(desc, key_jump_cmd); 1042 1043 /* Class 1 operation */ 1044 append_operation(desc, ctx->class1_alg_type | 1045 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1046 1047 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 1048 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 1049 1050 /* Skip assoc data */ 1051 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 1052 1053 /* Read assoc data */ 1054 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1055 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 1056 1057 /* Will write cryptlen bytes */ 1058 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1059 1060 /* Will read cryptlen bytes */ 1061 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1062 1063 /* Store payload data */ 1064 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1065 1066 /* Read encrypted data */ 1067 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1068 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 1069 1070 /* Read ICV */ 1071 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1072 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1073 1074 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1075 desc_bytes(desc), 1076 DMA_TO_DEVICE); 1077 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1078 dev_err(jrdev, "unable to map shared descriptor\n"); 1079 return -ENOMEM; 1080 } 1081 #ifdef DEBUG 1082 print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ", 1083 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1084 desc_bytes(desc), 1); 1085 #endif 1086 1087 return 0; 1088 } 1089 1090 static int rfc4106_setauthsize(struct crypto_aead *authenc, 1091 unsigned int authsize) 1092 { 1093 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1094 1095 ctx->authsize = authsize; 1096 rfc4106_set_sh_desc(authenc); 1097 1098 return 0; 1099 } 1100 1101 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 1102 { 1103 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1104 struct device *jrdev = ctx->jrdev; 1105 bool keys_fit_inline = false; 1106 u32 *key_jump_cmd; 1107 u32 *read_move_cmd, *write_move_cmd; 1108 u32 *desc; 1109 1110 if (!ctx->enckeylen || !ctx->authsize) 1111 return 0; 1112 1113 /* 1114 * RFC4543 encrypt shared descriptor 1115 * Job Descriptor and Shared Descriptor 1116 * must fit into the 64-word Descriptor h/w Buffer 1117 */ 1118 if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN + 1119 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1120 keys_fit_inline = true; 1121 1122 desc = ctx->sh_desc_enc; 1123 1124 init_sh_desc(desc, HDR_SHARE_SERIAL); 1125 1126 /* Skip key loading if it is loaded due to sharing */ 1127 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1128 JUMP_COND_SHRD); 1129 if (keys_fit_inline) 1130 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1131 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1132 else 1133 append_key(desc, ctx->key_dma, ctx->enckeylen, 1134 CLASS_1 | KEY_DEST_CLASS_REG); 1135 set_jump_tgt_here(desc, key_jump_cmd); 1136 1137 /* Class 1 operation */ 1138 append_operation(desc, ctx->class1_alg_type | 1139 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1140 1141 /* assoclen + cryptlen = seqinlen */ 1142 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 1143 1144 /* 1145 * MOVE_LEN opcode is not available in all SEC HW revisions, 1146 * thus need to do some magic, i.e. self-patch the descriptor 1147 * buffer. 1148 */ 1149 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1150 (0x6 << MOVE_LEN_SHIFT)); 1151 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1152 (0x8 << MOVE_LEN_SHIFT)); 1153 1154 /* Will read assoclen + cryptlen bytes */ 1155 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1156 1157 /* Will write assoclen + cryptlen bytes */ 1158 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1159 1160 /* Read and write assoclen + cryptlen bytes */ 1161 aead_append_src_dst(desc, FIFOLD_TYPE_AAD); 1162 1163 set_move_tgt_here(desc, read_move_cmd); 1164 set_move_tgt_here(desc, write_move_cmd); 1165 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1166 /* Move payload data to OFIFO */ 1167 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1168 1169 /* Write ICV */ 1170 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 1171 LDST_SRCDST_BYTE_CONTEXT); 1172 1173 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1174 desc_bytes(desc), 1175 DMA_TO_DEVICE); 1176 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1177 dev_err(jrdev, "unable to map shared descriptor\n"); 1178 return -ENOMEM; 1179 } 1180 #ifdef DEBUG 1181 print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ", 1182 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1183 desc_bytes(desc), 1); 1184 #endif 1185 1186 /* 1187 * Job Descriptor and Shared Descriptors 1188 * must all fit into the 64-word Descriptor h/w Buffer 1189 */ 1190 keys_fit_inline = false; 1191 if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN + 1192 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1193 keys_fit_inline = true; 1194 1195 desc = ctx->sh_desc_dec; 1196 1197 init_sh_desc(desc, HDR_SHARE_SERIAL); 1198 1199 /* Skip key loading if it is loaded due to sharing */ 1200 key_jump_cmd = append_jump(desc, JUMP_JSL | 1201 JUMP_TEST_ALL | JUMP_COND_SHRD); 1202 if (keys_fit_inline) 1203 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1204 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1205 else 1206 append_key(desc, ctx->key_dma, ctx->enckeylen, 1207 CLASS_1 | KEY_DEST_CLASS_REG); 1208 set_jump_tgt_here(desc, key_jump_cmd); 1209 1210 /* Class 1 operation */ 1211 append_operation(desc, ctx->class1_alg_type | 1212 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1213 1214 /* assoclen + cryptlen = seqoutlen */ 1215 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1216 1217 /* 1218 * MOVE_LEN opcode is not available in all SEC HW revisions, 1219 * thus need to do some magic, i.e. self-patch the descriptor 1220 * buffer. 1221 */ 1222 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1223 (0x6 << MOVE_LEN_SHIFT)); 1224 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1225 (0x8 << MOVE_LEN_SHIFT)); 1226 1227 /* Will read assoclen + cryptlen bytes */ 1228 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1229 1230 /* Will write assoclen + cryptlen bytes */ 1231 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1232 1233 /* Store payload data */ 1234 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1235 1236 /* In-snoop assoclen + cryptlen data */ 1237 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | 1238 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); 1239 1240 set_move_tgt_here(desc, read_move_cmd); 1241 set_move_tgt_here(desc, write_move_cmd); 1242 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1243 /* Move payload data to OFIFO */ 1244 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1245 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1246 1247 /* Read ICV */ 1248 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1249 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1250 1251 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1252 desc_bytes(desc), 1253 DMA_TO_DEVICE); 1254 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1255 dev_err(jrdev, "unable to map shared descriptor\n"); 1256 return -ENOMEM; 1257 } 1258 #ifdef DEBUG 1259 print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ", 1260 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1261 desc_bytes(desc), 1); 1262 #endif 1263 1264 return 0; 1265 } 1266 1267 static int rfc4543_setauthsize(struct crypto_aead *authenc, 1268 unsigned int authsize) 1269 { 1270 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1271 1272 ctx->authsize = authsize; 1273 rfc4543_set_sh_desc(authenc); 1274 1275 return 0; 1276 } 1277 1278 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, 1279 u32 authkeylen) 1280 { 1281 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, 1282 ctx->split_key_pad_len, key_in, authkeylen, 1283 ctx->alg_op); 1284 } 1285 1286 static int aead_setkey(struct crypto_aead *aead, 1287 const u8 *key, unsigned int keylen) 1288 { 1289 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 1290 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 1291 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1292 struct device *jrdev = ctx->jrdev; 1293 struct crypto_authenc_keys keys; 1294 int ret = 0; 1295 1296 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1297 goto badkey; 1298 1299 /* Pick class 2 key length from algorithm submask */ 1300 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 1301 OP_ALG_ALGSEL_SHIFT] * 2; 1302 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 1303 1304 if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) 1305 goto badkey; 1306 1307 #ifdef DEBUG 1308 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 1309 keys.authkeylen + keys.enckeylen, keys.enckeylen, 1310 keys.authkeylen); 1311 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 1312 ctx->split_key_len, ctx->split_key_pad_len); 1313 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1314 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1315 #endif 1316 1317 ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); 1318 if (ret) { 1319 goto badkey; 1320 } 1321 1322 /* postpend encryption key to auth split key */ 1323 memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); 1324 1325 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 1326 keys.enckeylen, DMA_TO_DEVICE); 1327 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1328 dev_err(jrdev, "unable to map key i/o memory\n"); 1329 return -ENOMEM; 1330 } 1331 #ifdef DEBUG 1332 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 1333 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 1334 ctx->split_key_pad_len + keys.enckeylen, 1); 1335 #endif 1336 1337 ctx->enckeylen = keys.enckeylen; 1338 1339 ret = aead_set_sh_desc(aead); 1340 if (ret) { 1341 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + 1342 keys.enckeylen, DMA_TO_DEVICE); 1343 } 1344 1345 return ret; 1346 badkey: 1347 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 1348 return -EINVAL; 1349 } 1350 1351 static int gcm_setkey(struct crypto_aead *aead, 1352 const u8 *key, unsigned int keylen) 1353 { 1354 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1355 struct device *jrdev = ctx->jrdev; 1356 int ret = 0; 1357 1358 #ifdef DEBUG 1359 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1360 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1361 #endif 1362 1363 memcpy(ctx->key, key, keylen); 1364 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1365 DMA_TO_DEVICE); 1366 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1367 dev_err(jrdev, "unable to map key i/o memory\n"); 1368 return -ENOMEM; 1369 } 1370 ctx->enckeylen = keylen; 1371 1372 ret = gcm_set_sh_desc(aead); 1373 if (ret) { 1374 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1375 DMA_TO_DEVICE); 1376 } 1377 1378 return ret; 1379 } 1380 1381 static int rfc4106_setkey(struct crypto_aead *aead, 1382 const u8 *key, unsigned int keylen) 1383 { 1384 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1385 struct device *jrdev = ctx->jrdev; 1386 int ret = 0; 1387 1388 if (keylen < 4) 1389 return -EINVAL; 1390 1391 #ifdef DEBUG 1392 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1393 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1394 #endif 1395 1396 memcpy(ctx->key, key, keylen); 1397 1398 /* 1399 * The last four bytes of the key material are used as the salt value 1400 * in the nonce. Update the AES key length. 1401 */ 1402 ctx->enckeylen = keylen - 4; 1403 1404 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1405 DMA_TO_DEVICE); 1406 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1407 dev_err(jrdev, "unable to map key i/o memory\n"); 1408 return -ENOMEM; 1409 } 1410 1411 ret = rfc4106_set_sh_desc(aead); 1412 if (ret) { 1413 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1414 DMA_TO_DEVICE); 1415 } 1416 1417 return ret; 1418 } 1419 1420 static int rfc4543_setkey(struct crypto_aead *aead, 1421 const u8 *key, unsigned int keylen) 1422 { 1423 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1424 struct device *jrdev = ctx->jrdev; 1425 int ret = 0; 1426 1427 if (keylen < 4) 1428 return -EINVAL; 1429 1430 #ifdef DEBUG 1431 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1432 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1433 #endif 1434 1435 memcpy(ctx->key, key, keylen); 1436 1437 /* 1438 * The last four bytes of the key material are used as the salt value 1439 * in the nonce. Update the AES key length. 1440 */ 1441 ctx->enckeylen = keylen - 4; 1442 1443 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1444 DMA_TO_DEVICE); 1445 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1446 dev_err(jrdev, "unable to map key i/o memory\n"); 1447 return -ENOMEM; 1448 } 1449 1450 ret = rfc4543_set_sh_desc(aead); 1451 if (ret) { 1452 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1453 DMA_TO_DEVICE); 1454 } 1455 1456 return ret; 1457 } 1458 1459 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 1460 const u8 *key, unsigned int keylen) 1461 { 1462 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1463 struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher; 1464 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 1465 const char *alg_name = crypto_tfm_alg_name(tfm); 1466 struct device *jrdev = ctx->jrdev; 1467 int ret = 0; 1468 u32 *key_jump_cmd; 1469 u32 *desc; 1470 u32 *nonce; 1471 u32 geniv; 1472 u32 ctx1_iv_off = 0; 1473 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 1474 OP_ALG_AAI_CTR_MOD128); 1475 const bool is_rfc3686 = (ctr_mode && 1476 (strstr(alg_name, "rfc3686") != NULL)); 1477 1478 #ifdef DEBUG 1479 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1480 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1481 #endif 1482 /* 1483 * AES-CTR needs to load IV in CONTEXT1 reg 1484 * at an offset of 128bits (16bytes) 1485 * CONTEXT1[255:128] = IV 1486 */ 1487 if (ctr_mode) 1488 ctx1_iv_off = 16; 1489 1490 /* 1491 * RFC3686 specific: 1492 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1493 * | *key = {KEY, NONCE} 1494 */ 1495 if (is_rfc3686) { 1496 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1497 keylen -= CTR_RFC3686_NONCE_SIZE; 1498 } 1499 1500 memcpy(ctx->key, key, keylen); 1501 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1502 DMA_TO_DEVICE); 1503 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1504 dev_err(jrdev, "unable to map key i/o memory\n"); 1505 return -ENOMEM; 1506 } 1507 ctx->enckeylen = keylen; 1508 1509 /* ablkcipher_encrypt shared descriptor */ 1510 desc = ctx->sh_desc_enc; 1511 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1512 /* Skip if already shared */ 1513 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1514 JUMP_COND_SHRD); 1515 1516 /* Load class1 key only */ 1517 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1518 ctx->enckeylen, CLASS_1 | 1519 KEY_DEST_CLASS_REG); 1520 1521 /* Load nonce into CONTEXT1 reg */ 1522 if (is_rfc3686) { 1523 nonce = (u32 *)(key + keylen); 1524 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1525 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1526 append_move(desc, MOVE_WAITCOMP | 1527 MOVE_SRC_OUTFIFO | 1528 MOVE_DEST_CLASS1CTX | 1529 (16 << MOVE_OFFSET_SHIFT) | 1530 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1531 } 1532 1533 set_jump_tgt_here(desc, key_jump_cmd); 1534 1535 /* Load iv */ 1536 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1537 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1538 1539 /* Load counter into CONTEXT1 reg */ 1540 if (is_rfc3686) 1541 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1542 LDST_CLASS_1_CCB | 1543 LDST_SRCDST_BYTE_CONTEXT | 1544 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1545 LDST_OFFSET_SHIFT)); 1546 1547 /* Load operation */ 1548 append_operation(desc, ctx->class1_alg_type | 1549 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1550 1551 /* Perform operation */ 1552 ablkcipher_append_src_dst(desc); 1553 1554 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1555 desc_bytes(desc), 1556 DMA_TO_DEVICE); 1557 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1558 dev_err(jrdev, "unable to map shared descriptor\n"); 1559 return -ENOMEM; 1560 } 1561 #ifdef DEBUG 1562 print_hex_dump(KERN_ERR, 1563 "ablkcipher enc shdesc@"__stringify(__LINE__)": ", 1564 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1565 desc_bytes(desc), 1); 1566 #endif 1567 /* ablkcipher_decrypt shared descriptor */ 1568 desc = ctx->sh_desc_dec; 1569 1570 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1571 /* Skip if already shared */ 1572 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1573 JUMP_COND_SHRD); 1574 1575 /* Load class1 key only */ 1576 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1577 ctx->enckeylen, CLASS_1 | 1578 KEY_DEST_CLASS_REG); 1579 1580 /* Load nonce into CONTEXT1 reg */ 1581 if (is_rfc3686) { 1582 nonce = (u32 *)(key + keylen); 1583 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1584 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1585 append_move(desc, MOVE_WAITCOMP | 1586 MOVE_SRC_OUTFIFO | 1587 MOVE_DEST_CLASS1CTX | 1588 (16 << MOVE_OFFSET_SHIFT) | 1589 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1590 } 1591 1592 set_jump_tgt_here(desc, key_jump_cmd); 1593 1594 /* load IV */ 1595 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1596 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1597 1598 /* Load counter into CONTEXT1 reg */ 1599 if (is_rfc3686) 1600 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1601 LDST_CLASS_1_CCB | 1602 LDST_SRCDST_BYTE_CONTEXT | 1603 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1604 LDST_OFFSET_SHIFT)); 1605 1606 /* Choose operation */ 1607 if (ctr_mode) 1608 append_operation(desc, ctx->class1_alg_type | 1609 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 1610 else 1611 append_dec_op1(desc, ctx->class1_alg_type); 1612 1613 /* Perform operation */ 1614 ablkcipher_append_src_dst(desc); 1615 1616 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1617 desc_bytes(desc), 1618 DMA_TO_DEVICE); 1619 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1620 dev_err(jrdev, "unable to map shared descriptor\n"); 1621 return -ENOMEM; 1622 } 1623 1624 #ifdef DEBUG 1625 print_hex_dump(KERN_ERR, 1626 "ablkcipher dec shdesc@"__stringify(__LINE__)": ", 1627 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1628 desc_bytes(desc), 1); 1629 #endif 1630 /* ablkcipher_givencrypt shared descriptor */ 1631 desc = ctx->sh_desc_givenc; 1632 1633 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1634 /* Skip if already shared */ 1635 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1636 JUMP_COND_SHRD); 1637 1638 /* Load class1 key only */ 1639 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1640 ctx->enckeylen, CLASS_1 | 1641 KEY_DEST_CLASS_REG); 1642 1643 /* Load Nonce into CONTEXT1 reg */ 1644 if (is_rfc3686) { 1645 nonce = (u32 *)(key + keylen); 1646 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1647 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1648 append_move(desc, MOVE_WAITCOMP | 1649 MOVE_SRC_OUTFIFO | 1650 MOVE_DEST_CLASS1CTX | 1651 (16 << MOVE_OFFSET_SHIFT) | 1652 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1653 } 1654 set_jump_tgt_here(desc, key_jump_cmd); 1655 1656 /* Generate IV */ 1657 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 1658 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 1659 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT); 1660 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 1661 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 1662 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1663 append_move(desc, MOVE_WAITCOMP | 1664 MOVE_SRC_INFIFO | 1665 MOVE_DEST_CLASS1CTX | 1666 (crt->ivsize << MOVE_LEN_SHIFT) | 1667 (ctx1_iv_off << MOVE_OFFSET_SHIFT)); 1668 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1669 1670 /* Copy generated IV to memory */ 1671 append_seq_store(desc, crt->ivsize, 1672 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1673 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1674 1675 /* Load Counter into CONTEXT1 reg */ 1676 if (is_rfc3686) 1677 append_load_imm_u32(desc, (u32)1, LDST_IMM | 1678 LDST_CLASS_1_CCB | 1679 LDST_SRCDST_BYTE_CONTEXT | 1680 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1681 LDST_OFFSET_SHIFT)); 1682 1683 if (ctx1_iv_off) 1684 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | 1685 (1 << JUMP_OFFSET_SHIFT)); 1686 1687 /* Load operation */ 1688 append_operation(desc, ctx->class1_alg_type | 1689 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1690 1691 /* Perform operation */ 1692 ablkcipher_append_src_dst(desc); 1693 1694 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 1695 desc_bytes(desc), 1696 DMA_TO_DEVICE); 1697 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 1698 dev_err(jrdev, "unable to map shared descriptor\n"); 1699 return -ENOMEM; 1700 } 1701 #ifdef DEBUG 1702 print_hex_dump(KERN_ERR, 1703 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", 1704 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1705 desc_bytes(desc), 1); 1706 #endif 1707 1708 return ret; 1709 } 1710 1711 /* 1712 * aead_edesc - s/w-extended aead descriptor 1713 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 1714 * @assoc_chained: if source is chained 1715 * @src_nents: number of segments in input scatterlist 1716 * @src_chained: if source is chained 1717 * @dst_nents: number of segments in output scatterlist 1718 * @dst_chained: if destination is chained 1719 * @iv_dma: dma address of iv for checking continuity and link table 1720 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1721 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1722 * @sec4_sg_dma: bus physical mapped address of h/w link table 1723 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1724 */ 1725 struct aead_edesc { 1726 int assoc_nents; 1727 bool assoc_chained; 1728 int src_nents; 1729 bool src_chained; 1730 int dst_nents; 1731 bool dst_chained; 1732 dma_addr_t iv_dma; 1733 int sec4_sg_bytes; 1734 dma_addr_t sec4_sg_dma; 1735 struct sec4_sg_entry *sec4_sg; 1736 u32 hw_desc[]; 1737 }; 1738 1739 /* 1740 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 1741 * @src_nents: number of segments in input scatterlist 1742 * @src_chained: if source is chained 1743 * @dst_nents: number of segments in output scatterlist 1744 * @dst_chained: if destination is chained 1745 * @iv_dma: dma address of iv for checking continuity and link table 1746 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1747 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1748 * @sec4_sg_dma: bus physical mapped address of h/w link table 1749 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1750 */ 1751 struct ablkcipher_edesc { 1752 int src_nents; 1753 bool src_chained; 1754 int dst_nents; 1755 bool dst_chained; 1756 dma_addr_t iv_dma; 1757 int sec4_sg_bytes; 1758 dma_addr_t sec4_sg_dma; 1759 struct sec4_sg_entry *sec4_sg; 1760 u32 hw_desc[0]; 1761 }; 1762 1763 static void caam_unmap(struct device *dev, struct scatterlist *src, 1764 struct scatterlist *dst, int src_nents, 1765 bool src_chained, int dst_nents, bool dst_chained, 1766 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 1767 int sec4_sg_bytes) 1768 { 1769 if (dst != src) { 1770 dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE, 1771 src_chained); 1772 dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE, 1773 dst_chained); 1774 } else { 1775 dma_unmap_sg_chained(dev, src, src_nents ? : 1, 1776 DMA_BIDIRECTIONAL, src_chained); 1777 } 1778 1779 if (iv_dma) 1780 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 1781 if (sec4_sg_bytes) 1782 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 1783 DMA_TO_DEVICE); 1784 } 1785 1786 static void aead_unmap(struct device *dev, 1787 struct aead_edesc *edesc, 1788 struct aead_request *req) 1789 { 1790 caam_unmap(dev, req->src, req->dst, 1791 edesc->src_nents, edesc->src_chained, edesc->dst_nents, 1792 edesc->dst_chained, 0, 0, 1793 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1794 } 1795 1796 static void old_aead_unmap(struct device *dev, 1797 struct aead_edesc *edesc, 1798 struct aead_request *req) 1799 { 1800 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1801 int ivsize = crypto_aead_ivsize(aead); 1802 1803 dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents, 1804 DMA_TO_DEVICE, edesc->assoc_chained); 1805 1806 caam_unmap(dev, req->src, req->dst, 1807 edesc->src_nents, edesc->src_chained, edesc->dst_nents, 1808 edesc->dst_chained, edesc->iv_dma, ivsize, 1809 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1810 } 1811 1812 static void ablkcipher_unmap(struct device *dev, 1813 struct ablkcipher_edesc *edesc, 1814 struct ablkcipher_request *req) 1815 { 1816 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1817 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1818 1819 caam_unmap(dev, req->src, req->dst, 1820 edesc->src_nents, edesc->src_chained, edesc->dst_nents, 1821 edesc->dst_chained, edesc->iv_dma, ivsize, 1822 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1823 } 1824 1825 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1826 void *context) 1827 { 1828 struct aead_request *req = context; 1829 struct aead_edesc *edesc; 1830 1831 #ifdef DEBUG 1832 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1833 #endif 1834 1835 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1836 1837 if (err) 1838 caam_jr_strstatus(jrdev, err); 1839 1840 aead_unmap(jrdev, edesc, req); 1841 1842 kfree(edesc); 1843 1844 aead_request_complete(req, err); 1845 } 1846 1847 static void old_aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1848 void *context) 1849 { 1850 struct aead_request *req = context; 1851 struct aead_edesc *edesc; 1852 #ifdef DEBUG 1853 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1854 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1855 int ivsize = crypto_aead_ivsize(aead); 1856 1857 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1858 #endif 1859 1860 edesc = (struct aead_edesc *)((char *)desc - 1861 offsetof(struct aead_edesc, hw_desc)); 1862 1863 if (err) 1864 caam_jr_strstatus(jrdev, err); 1865 1866 old_aead_unmap(jrdev, edesc, req); 1867 1868 #ifdef DEBUG 1869 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", 1870 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 1871 req->assoclen , 1); 1872 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 1873 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, 1874 edesc->src_nents ? 100 : ivsize, 1); 1875 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 1876 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1877 edesc->src_nents ? 100 : req->cryptlen + 1878 ctx->authsize + 4, 1); 1879 #endif 1880 1881 kfree(edesc); 1882 1883 aead_request_complete(req, err); 1884 } 1885 1886 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 1887 void *context) 1888 { 1889 struct aead_request *req = context; 1890 struct aead_edesc *edesc; 1891 1892 #ifdef DEBUG 1893 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1894 #endif 1895 1896 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1897 1898 if (err) 1899 caam_jr_strstatus(jrdev, err); 1900 1901 aead_unmap(jrdev, edesc, req); 1902 1903 /* 1904 * verify hw auth check passed else return -EBADMSG 1905 */ 1906 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 1907 err = -EBADMSG; 1908 1909 kfree(edesc); 1910 1911 aead_request_complete(req, err); 1912 } 1913 1914 static void old_aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 1915 void *context) 1916 { 1917 struct aead_request *req = context; 1918 struct aead_edesc *edesc; 1919 #ifdef DEBUG 1920 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1921 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1922 int ivsize = crypto_aead_ivsize(aead); 1923 1924 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1925 #endif 1926 1927 edesc = (struct aead_edesc *)((char *)desc - 1928 offsetof(struct aead_edesc, hw_desc)); 1929 1930 #ifdef DEBUG 1931 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 1932 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1933 ivsize, 1); 1934 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 1935 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst), 1936 req->cryptlen - ctx->authsize, 1); 1937 #endif 1938 1939 if (err) 1940 caam_jr_strstatus(jrdev, err); 1941 1942 old_aead_unmap(jrdev, edesc, req); 1943 1944 /* 1945 * verify hw auth check passed else return -EBADMSG 1946 */ 1947 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 1948 err = -EBADMSG; 1949 1950 #ifdef DEBUG 1951 print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ", 1952 DUMP_PREFIX_ADDRESS, 16, 4, 1953 ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)), 1954 sizeof(struct iphdr) + req->assoclen + 1955 ((req->cryptlen > 1500) ? 1500 : req->cryptlen) + 1956 ctx->authsize + 36, 1); 1957 if (!err && edesc->sec4_sg_bytes) { 1958 struct scatterlist *sg = sg_last(req->src, edesc->src_nents); 1959 print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ", 1960 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), 1961 sg->length + ctx->authsize + 16, 1); 1962 } 1963 #endif 1964 1965 kfree(edesc); 1966 1967 aead_request_complete(req, err); 1968 } 1969 1970 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1971 void *context) 1972 { 1973 struct ablkcipher_request *req = context; 1974 struct ablkcipher_edesc *edesc; 1975 #ifdef DEBUG 1976 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1977 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1978 1979 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1980 #endif 1981 1982 edesc = (struct ablkcipher_edesc *)((char *)desc - 1983 offsetof(struct ablkcipher_edesc, hw_desc)); 1984 1985 if (err) 1986 caam_jr_strstatus(jrdev, err); 1987 1988 #ifdef DEBUG 1989 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 1990 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1991 edesc->src_nents > 1 ? 100 : ivsize, 1); 1992 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 1993 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1994 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 1995 #endif 1996 1997 ablkcipher_unmap(jrdev, edesc, req); 1998 kfree(edesc); 1999 2000 ablkcipher_request_complete(req, err); 2001 } 2002 2003 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 2004 void *context) 2005 { 2006 struct ablkcipher_request *req = context; 2007 struct ablkcipher_edesc *edesc; 2008 #ifdef DEBUG 2009 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2010 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2011 2012 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 2013 #endif 2014 2015 edesc = (struct ablkcipher_edesc *)((char *)desc - 2016 offsetof(struct ablkcipher_edesc, hw_desc)); 2017 if (err) 2018 caam_jr_strstatus(jrdev, err); 2019 2020 #ifdef DEBUG 2021 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 2022 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2023 ivsize, 1); 2024 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 2025 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2026 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 2027 #endif 2028 2029 ablkcipher_unmap(jrdev, edesc, req); 2030 kfree(edesc); 2031 2032 ablkcipher_request_complete(req, err); 2033 } 2034 2035 /* 2036 * Fill in aead job descriptor 2037 */ 2038 static void old_init_aead_job(u32 *sh_desc, dma_addr_t ptr, 2039 struct aead_edesc *edesc, 2040 struct aead_request *req, 2041 bool all_contig, bool encrypt) 2042 { 2043 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2044 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2045 int ivsize = crypto_aead_ivsize(aead); 2046 int authsize = ctx->authsize; 2047 u32 *desc = edesc->hw_desc; 2048 u32 out_options = 0, in_options; 2049 dma_addr_t dst_dma, src_dma; 2050 int len, sec4_sg_index = 0; 2051 bool is_gcm = false; 2052 2053 #ifdef DEBUG 2054 debug("assoclen %d cryptlen %d authsize %d\n", 2055 req->assoclen, req->cryptlen, authsize); 2056 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", 2057 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 2058 req->assoclen , 1); 2059 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 2060 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 2061 edesc->src_nents ? 100 : ivsize, 1); 2062 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 2063 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2064 edesc->src_nents ? 100 : req->cryptlen, 1); 2065 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", 2066 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 2067 desc_bytes(sh_desc), 1); 2068 #endif 2069 2070 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == 2071 OP_ALG_ALGSEL_AES) && 2072 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) 2073 is_gcm = true; 2074 2075 len = desc_len(sh_desc); 2076 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2077 2078 if (all_contig) { 2079 if (is_gcm) 2080 src_dma = edesc->iv_dma; 2081 else 2082 src_dma = sg_dma_address(req->assoc); 2083 in_options = 0; 2084 } else { 2085 src_dma = edesc->sec4_sg_dma; 2086 sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 + 2087 (edesc->src_nents ? : 1); 2088 in_options = LDST_SGF; 2089 } 2090 2091 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, 2092 in_options); 2093 2094 if (likely(req->src == req->dst)) { 2095 if (all_contig) { 2096 dst_dma = sg_dma_address(req->src); 2097 } else { 2098 dst_dma = src_dma + sizeof(struct sec4_sg_entry) * 2099 ((edesc->assoc_nents ? : 1) + 1); 2100 out_options = LDST_SGF; 2101 } 2102 } else { 2103 if (!edesc->dst_nents) { 2104 dst_dma = sg_dma_address(req->dst); 2105 } else { 2106 dst_dma = edesc->sec4_sg_dma + 2107 sec4_sg_index * 2108 sizeof(struct sec4_sg_entry); 2109 out_options = LDST_SGF; 2110 } 2111 } 2112 if (encrypt) 2113 append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize, 2114 out_options); 2115 else 2116 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize, 2117 out_options); 2118 } 2119 2120 /* 2121 * Fill in aead job descriptor 2122 */ 2123 static void init_aead_job(struct aead_request *req, 2124 struct aead_edesc *edesc, 2125 bool all_contig, bool encrypt) 2126 { 2127 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2128 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2129 int authsize = ctx->authsize; 2130 u32 *desc = edesc->hw_desc; 2131 u32 out_options, in_options; 2132 dma_addr_t dst_dma, src_dma; 2133 int len, sec4_sg_index = 0; 2134 dma_addr_t ptr; 2135 u32 *sh_desc; 2136 2137 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 2138 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 2139 2140 len = desc_len(sh_desc); 2141 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2142 2143 if (all_contig) { 2144 src_dma = sg_dma_address(req->src); 2145 in_options = 0; 2146 } else { 2147 src_dma = edesc->sec4_sg_dma; 2148 sec4_sg_index += edesc->src_nents; 2149 in_options = LDST_SGF; 2150 } 2151 2152 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 2153 in_options); 2154 2155 dst_dma = src_dma; 2156 out_options = in_options; 2157 2158 if (unlikely(req->src != req->dst)) { 2159 if (!edesc->dst_nents) { 2160 dst_dma = sg_dma_address(req->dst); 2161 } else { 2162 dst_dma = edesc->sec4_sg_dma + 2163 sec4_sg_index * 2164 sizeof(struct sec4_sg_entry); 2165 out_options = LDST_SGF; 2166 } 2167 } 2168 2169 if (encrypt) 2170 append_seq_out_ptr(desc, dst_dma, 2171 req->assoclen + req->cryptlen + authsize, 2172 out_options); 2173 else 2174 append_seq_out_ptr(desc, dst_dma, 2175 req->assoclen + req->cryptlen - authsize, 2176 out_options); 2177 2178 /* REG3 = assoclen */ 2179 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 2180 } 2181 2182 static void init_gcm_job(struct aead_request *req, 2183 struct aead_edesc *edesc, 2184 bool all_contig, bool encrypt) 2185 { 2186 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2187 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2188 unsigned int ivsize = crypto_aead_ivsize(aead); 2189 u32 *desc = edesc->hw_desc; 2190 bool generic_gcm = (ivsize == 12); 2191 unsigned int last; 2192 2193 init_aead_job(req, edesc, all_contig, encrypt); 2194 2195 /* BUG This should not be specific to generic GCM. */ 2196 last = 0; 2197 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 2198 last = FIFOLD_TYPE_LAST1; 2199 2200 /* Read GCM IV */ 2201 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 2202 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); 2203 /* Append Salt */ 2204 if (!generic_gcm) 2205 append_data(desc, ctx->key + ctx->enckeylen, 4); 2206 /* Append IV */ 2207 append_data(desc, req->iv, ivsize); 2208 /* End of blank commands */ 2209 } 2210 2211 /* 2212 * Fill in aead givencrypt job descriptor 2213 */ 2214 static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, 2215 struct aead_edesc *edesc, 2216 struct aead_request *req, 2217 int contig) 2218 { 2219 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2220 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2221 int ivsize = crypto_aead_ivsize(aead); 2222 int authsize = ctx->authsize; 2223 u32 *desc = edesc->hw_desc; 2224 u32 out_options = 0, in_options; 2225 dma_addr_t dst_dma, src_dma; 2226 int len, sec4_sg_index = 0; 2227 bool is_gcm = false; 2228 2229 #ifdef DEBUG 2230 debug("assoclen %d cryptlen %d authsize %d\n", 2231 req->assoclen, req->cryptlen, authsize); 2232 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", 2233 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 2234 req->assoclen , 1); 2235 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 2236 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); 2237 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 2238 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2239 edesc->src_nents > 1 ? 100 : req->cryptlen, 1); 2240 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", 2241 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 2242 desc_bytes(sh_desc), 1); 2243 #endif 2244 2245 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == 2246 OP_ALG_ALGSEL_AES) && 2247 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) 2248 is_gcm = true; 2249 2250 len = desc_len(sh_desc); 2251 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2252 2253 if (contig & GIV_SRC_CONTIG) { 2254 if (is_gcm) 2255 src_dma = edesc->iv_dma; 2256 else 2257 src_dma = sg_dma_address(req->assoc); 2258 in_options = 0; 2259 } else { 2260 src_dma = edesc->sec4_sg_dma; 2261 sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents; 2262 in_options = LDST_SGF; 2263 } 2264 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, 2265 in_options); 2266 2267 if (contig & GIV_DST_CONTIG) { 2268 dst_dma = edesc->iv_dma; 2269 } else { 2270 if (likely(req->src == req->dst)) { 2271 dst_dma = src_dma + sizeof(struct sec4_sg_entry) * 2272 (edesc->assoc_nents + 2273 (is_gcm ? 1 + edesc->src_nents : 0)); 2274 out_options = LDST_SGF; 2275 } else { 2276 dst_dma = edesc->sec4_sg_dma + 2277 sec4_sg_index * 2278 sizeof(struct sec4_sg_entry); 2279 out_options = LDST_SGF; 2280 } 2281 } 2282 2283 append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize, 2284 out_options); 2285 } 2286 2287 /* 2288 * Fill in ablkcipher job descriptor 2289 */ 2290 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 2291 struct ablkcipher_edesc *edesc, 2292 struct ablkcipher_request *req, 2293 bool iv_contig) 2294 { 2295 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2296 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2297 u32 *desc = edesc->hw_desc; 2298 u32 out_options = 0, in_options; 2299 dma_addr_t dst_dma, src_dma; 2300 int len, sec4_sg_index = 0; 2301 2302 #ifdef DEBUG 2303 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 2304 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2305 ivsize, 1); 2306 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 2307 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2308 edesc->src_nents ? 100 : req->nbytes, 1); 2309 #endif 2310 2311 len = desc_len(sh_desc); 2312 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2313 2314 if (iv_contig) { 2315 src_dma = edesc->iv_dma; 2316 in_options = 0; 2317 } else { 2318 src_dma = edesc->sec4_sg_dma; 2319 sec4_sg_index += edesc->src_nents + 1; 2320 in_options = LDST_SGF; 2321 } 2322 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 2323 2324 if (likely(req->src == req->dst)) { 2325 if (!edesc->src_nents && iv_contig) { 2326 dst_dma = sg_dma_address(req->src); 2327 } else { 2328 dst_dma = edesc->sec4_sg_dma + 2329 sizeof(struct sec4_sg_entry); 2330 out_options = LDST_SGF; 2331 } 2332 } else { 2333 if (!edesc->dst_nents) { 2334 dst_dma = sg_dma_address(req->dst); 2335 } else { 2336 dst_dma = edesc->sec4_sg_dma + 2337 sec4_sg_index * sizeof(struct sec4_sg_entry); 2338 out_options = LDST_SGF; 2339 } 2340 } 2341 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 2342 } 2343 2344 /* 2345 * Fill in ablkcipher givencrypt job descriptor 2346 */ 2347 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, 2348 struct ablkcipher_edesc *edesc, 2349 struct ablkcipher_request *req, 2350 bool iv_contig) 2351 { 2352 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2353 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2354 u32 *desc = edesc->hw_desc; 2355 u32 out_options, in_options; 2356 dma_addr_t dst_dma, src_dma; 2357 int len, sec4_sg_index = 0; 2358 2359 #ifdef DEBUG 2360 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", 2361 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2362 ivsize, 1); 2363 print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ", 2364 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2365 edesc->src_nents ? 100 : req->nbytes, 1); 2366 #endif 2367 2368 len = desc_len(sh_desc); 2369 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2370 2371 if (!edesc->src_nents) { 2372 src_dma = sg_dma_address(req->src); 2373 in_options = 0; 2374 } else { 2375 src_dma = edesc->sec4_sg_dma; 2376 sec4_sg_index += edesc->src_nents; 2377 in_options = LDST_SGF; 2378 } 2379 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options); 2380 2381 if (iv_contig) { 2382 dst_dma = edesc->iv_dma; 2383 out_options = 0; 2384 } else { 2385 dst_dma = edesc->sec4_sg_dma + 2386 sec4_sg_index * sizeof(struct sec4_sg_entry); 2387 out_options = LDST_SGF; 2388 } 2389 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options); 2390 } 2391 2392 /* 2393 * allocate and map the aead extended descriptor 2394 */ 2395 static struct aead_edesc *old_aead_edesc_alloc(struct aead_request *req, 2396 int desc_bytes, 2397 bool *all_contig_ptr, 2398 bool encrypt) 2399 { 2400 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2401 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2402 struct device *jrdev = ctx->jrdev; 2403 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2404 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 2405 int assoc_nents, src_nents, dst_nents = 0; 2406 struct aead_edesc *edesc; 2407 dma_addr_t iv_dma = 0; 2408 int sgc; 2409 bool all_contig = true; 2410 bool assoc_chained = false, src_chained = false, dst_chained = false; 2411 int ivsize = crypto_aead_ivsize(aead); 2412 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 2413 unsigned int authsize = ctx->authsize; 2414 bool is_gcm = false; 2415 2416 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); 2417 2418 if (unlikely(req->dst != req->src)) { 2419 src_nents = sg_count(req->src, req->cryptlen, &src_chained); 2420 dst_nents = sg_count(req->dst, 2421 req->cryptlen + 2422 (encrypt ? authsize : (-authsize)), 2423 &dst_chained); 2424 } else { 2425 src_nents = sg_count(req->src, 2426 req->cryptlen + 2427 (encrypt ? authsize : 0), 2428 &src_chained); 2429 } 2430 2431 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, 2432 DMA_TO_DEVICE, assoc_chained); 2433 if (likely(req->src == req->dst)) { 2434 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2435 DMA_BIDIRECTIONAL, src_chained); 2436 } else { 2437 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2438 DMA_TO_DEVICE, src_chained); 2439 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 2440 DMA_FROM_DEVICE, dst_chained); 2441 } 2442 2443 iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); 2444 if (dma_mapping_error(jrdev, iv_dma)) { 2445 dev_err(jrdev, "unable to map IV\n"); 2446 return ERR_PTR(-ENOMEM); 2447 } 2448 2449 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == 2450 OP_ALG_ALGSEL_AES) && 2451 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) 2452 is_gcm = true; 2453 2454 /* 2455 * Check if data are contiguous. 2456 * GCM expected input sequence: IV, AAD, text 2457 * All other - expected input sequence: AAD, IV, text 2458 */ 2459 if (is_gcm) 2460 all_contig = (!assoc_nents && 2461 iv_dma + ivsize == sg_dma_address(req->assoc) && 2462 !src_nents && sg_dma_address(req->assoc) + 2463 req->assoclen == sg_dma_address(req->src)); 2464 else 2465 all_contig = (!assoc_nents && sg_dma_address(req->assoc) + 2466 req->assoclen == iv_dma && !src_nents && 2467 iv_dma + ivsize == sg_dma_address(req->src)); 2468 if (!all_contig) { 2469 assoc_nents = assoc_nents ? : 1; 2470 src_nents = src_nents ? : 1; 2471 sec4_sg_len = assoc_nents + 1 + src_nents; 2472 } 2473 2474 sec4_sg_len += dst_nents; 2475 2476 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 2477 2478 /* allocate space for base edesc and hw desc commands, link tables */ 2479 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + 2480 sec4_sg_bytes, GFP_DMA | flags); 2481 if (!edesc) { 2482 dev_err(jrdev, "could not allocate extended descriptor\n"); 2483 return ERR_PTR(-ENOMEM); 2484 } 2485 2486 edesc->assoc_nents = assoc_nents; 2487 edesc->assoc_chained = assoc_chained; 2488 edesc->src_nents = src_nents; 2489 edesc->src_chained = src_chained; 2490 edesc->dst_nents = dst_nents; 2491 edesc->dst_chained = dst_chained; 2492 edesc->iv_dma = iv_dma; 2493 edesc->sec4_sg_bytes = sec4_sg_bytes; 2494 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 2495 desc_bytes; 2496 *all_contig_ptr = all_contig; 2497 2498 sec4_sg_index = 0; 2499 if (!all_contig) { 2500 if (!is_gcm) { 2501 sg_to_sec4_sg_len(req->assoc, req->assoclen, 2502 edesc->sec4_sg + sec4_sg_index); 2503 sec4_sg_index += assoc_nents; 2504 } 2505 2506 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 2507 iv_dma, ivsize, 0); 2508 sec4_sg_index += 1; 2509 2510 if (is_gcm) { 2511 sg_to_sec4_sg_len(req->assoc, req->assoclen, 2512 edesc->sec4_sg + sec4_sg_index); 2513 sec4_sg_index += assoc_nents; 2514 } 2515 2516 sg_to_sec4_sg_last(req->src, 2517 src_nents, 2518 edesc->sec4_sg + 2519 sec4_sg_index, 0); 2520 sec4_sg_index += src_nents; 2521 } 2522 if (dst_nents) { 2523 sg_to_sec4_sg_last(req->dst, dst_nents, 2524 edesc->sec4_sg + sec4_sg_index, 0); 2525 } 2526 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2527 sec4_sg_bytes, DMA_TO_DEVICE); 2528 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2529 dev_err(jrdev, "unable to map S/G table\n"); 2530 return ERR_PTR(-ENOMEM); 2531 } 2532 2533 return edesc; 2534 } 2535 2536 /* 2537 * allocate and map the aead extended descriptor 2538 */ 2539 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 2540 int desc_bytes, bool *all_contig_ptr, 2541 bool encrypt) 2542 { 2543 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2544 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2545 struct device *jrdev = ctx->jrdev; 2546 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2547 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 2548 int src_nents, dst_nents = 0; 2549 struct aead_edesc *edesc; 2550 int sgc; 2551 bool all_contig = true; 2552 bool src_chained = false, dst_chained = false; 2553 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 2554 unsigned int authsize = ctx->authsize; 2555 2556 if (unlikely(req->dst != req->src)) { 2557 src_nents = sg_count(req->src, req->assoclen + req->cryptlen, 2558 &src_chained); 2559 dst_nents = sg_count(req->dst, 2560 req->assoclen + req->cryptlen + 2561 (encrypt ? authsize : (-authsize)), 2562 &dst_chained); 2563 } else { 2564 src_nents = sg_count(req->src, 2565 req->assoclen + req->cryptlen + 2566 (encrypt ? authsize : 0), 2567 &src_chained); 2568 } 2569 2570 /* Check if data are contiguous. */ 2571 all_contig = !src_nents; 2572 if (!all_contig) { 2573 src_nents = src_nents ? : 1; 2574 sec4_sg_len = src_nents; 2575 } 2576 2577 sec4_sg_len += dst_nents; 2578 2579 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 2580 2581 /* allocate space for base edesc and hw desc commands, link tables */ 2582 edesc = kzalloc(sizeof(struct aead_edesc) + desc_bytes + 2583 sec4_sg_bytes, GFP_DMA | flags); 2584 if (!edesc) { 2585 dev_err(jrdev, "could not allocate extended descriptor\n"); 2586 return ERR_PTR(-ENOMEM); 2587 } 2588 2589 if (likely(req->src == req->dst)) { 2590 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2591 DMA_BIDIRECTIONAL, src_chained); 2592 if (unlikely(!sgc)) { 2593 dev_err(jrdev, "unable to map source\n"); 2594 kfree(edesc); 2595 return ERR_PTR(-ENOMEM); 2596 } 2597 } else { 2598 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2599 DMA_TO_DEVICE, src_chained); 2600 if (unlikely(!sgc)) { 2601 dev_err(jrdev, "unable to map source\n"); 2602 kfree(edesc); 2603 return ERR_PTR(-ENOMEM); 2604 } 2605 2606 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 2607 DMA_FROM_DEVICE, dst_chained); 2608 if (unlikely(!sgc)) { 2609 dev_err(jrdev, "unable to map destination\n"); 2610 dma_unmap_sg_chained(jrdev, req->src, src_nents ? : 1, 2611 DMA_TO_DEVICE, src_chained); 2612 kfree(edesc); 2613 return ERR_PTR(-ENOMEM); 2614 } 2615 } 2616 2617 edesc->src_nents = src_nents; 2618 edesc->src_chained = src_chained; 2619 edesc->dst_nents = dst_nents; 2620 edesc->dst_chained = dst_chained; 2621 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 2622 desc_bytes; 2623 *all_contig_ptr = all_contig; 2624 2625 sec4_sg_index = 0; 2626 if (!all_contig) { 2627 sg_to_sec4_sg_last(req->src, src_nents, 2628 edesc->sec4_sg + sec4_sg_index, 0); 2629 sec4_sg_index += src_nents; 2630 } 2631 if (dst_nents) { 2632 sg_to_sec4_sg_last(req->dst, dst_nents, 2633 edesc->sec4_sg + sec4_sg_index, 0); 2634 } 2635 2636 if (!sec4_sg_bytes) 2637 return edesc; 2638 2639 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2640 sec4_sg_bytes, DMA_TO_DEVICE); 2641 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2642 dev_err(jrdev, "unable to map S/G table\n"); 2643 aead_unmap(jrdev, edesc, req); 2644 kfree(edesc); 2645 return ERR_PTR(-ENOMEM); 2646 } 2647 2648 edesc->sec4_sg_bytes = sec4_sg_bytes; 2649 2650 return edesc; 2651 } 2652 2653 static int gcm_encrypt(struct aead_request *req) 2654 { 2655 struct aead_edesc *edesc; 2656 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2657 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2658 struct device *jrdev = ctx->jrdev; 2659 bool all_contig; 2660 u32 *desc; 2661 int ret = 0; 2662 2663 /* allocate extended descriptor */ 2664 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true); 2665 if (IS_ERR(edesc)) 2666 return PTR_ERR(edesc); 2667 2668 /* Create and submit job descriptor */ 2669 init_gcm_job(req, edesc, all_contig, true); 2670 #ifdef DEBUG 2671 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2672 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2673 desc_bytes(edesc->hw_desc), 1); 2674 #endif 2675 2676 desc = edesc->hw_desc; 2677 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 2678 if (!ret) { 2679 ret = -EINPROGRESS; 2680 } else { 2681 aead_unmap(jrdev, edesc, req); 2682 kfree(edesc); 2683 } 2684 2685 return ret; 2686 } 2687 2688 static int old_aead_encrypt(struct aead_request *req) 2689 { 2690 struct aead_edesc *edesc; 2691 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2692 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2693 struct device *jrdev = ctx->jrdev; 2694 bool all_contig; 2695 u32 *desc; 2696 int ret = 0; 2697 2698 /* allocate extended descriptor */ 2699 edesc = old_aead_edesc_alloc(req, DESC_JOB_IO_LEN * 2700 CAAM_CMD_SZ, &all_contig, true); 2701 if (IS_ERR(edesc)) 2702 return PTR_ERR(edesc); 2703 2704 /* Create and submit job descriptor */ 2705 old_init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req, 2706 all_contig, true); 2707 #ifdef DEBUG 2708 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2709 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2710 desc_bytes(edesc->hw_desc), 1); 2711 #endif 2712 2713 desc = edesc->hw_desc; 2714 ret = caam_jr_enqueue(jrdev, desc, old_aead_encrypt_done, req); 2715 if (!ret) { 2716 ret = -EINPROGRESS; 2717 } else { 2718 old_aead_unmap(jrdev, edesc, req); 2719 kfree(edesc); 2720 } 2721 2722 return ret; 2723 } 2724 2725 static int gcm_decrypt(struct aead_request *req) 2726 { 2727 struct aead_edesc *edesc; 2728 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2729 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2730 struct device *jrdev = ctx->jrdev; 2731 bool all_contig; 2732 u32 *desc; 2733 int ret = 0; 2734 2735 /* allocate extended descriptor */ 2736 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false); 2737 if (IS_ERR(edesc)) 2738 return PTR_ERR(edesc); 2739 2740 /* Create and submit job descriptor*/ 2741 init_gcm_job(req, edesc, all_contig, false); 2742 #ifdef DEBUG 2743 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2744 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2745 desc_bytes(edesc->hw_desc), 1); 2746 #endif 2747 2748 desc = edesc->hw_desc; 2749 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 2750 if (!ret) { 2751 ret = -EINPROGRESS; 2752 } else { 2753 aead_unmap(jrdev, edesc, req); 2754 kfree(edesc); 2755 } 2756 2757 return ret; 2758 } 2759 2760 static int old_aead_decrypt(struct aead_request *req) 2761 { 2762 struct aead_edesc *edesc; 2763 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2764 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2765 struct device *jrdev = ctx->jrdev; 2766 bool all_contig; 2767 u32 *desc; 2768 int ret = 0; 2769 2770 /* allocate extended descriptor */ 2771 edesc = old_aead_edesc_alloc(req, DESC_JOB_IO_LEN * 2772 CAAM_CMD_SZ, &all_contig, false); 2773 if (IS_ERR(edesc)) 2774 return PTR_ERR(edesc); 2775 2776 #ifdef DEBUG 2777 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", 2778 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2779 req->cryptlen, 1); 2780 #endif 2781 2782 /* Create and submit job descriptor*/ 2783 old_init_aead_job(ctx->sh_desc_dec, 2784 ctx->sh_desc_dec_dma, edesc, req, all_contig, false); 2785 #ifdef DEBUG 2786 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2787 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2788 desc_bytes(edesc->hw_desc), 1); 2789 #endif 2790 2791 desc = edesc->hw_desc; 2792 ret = caam_jr_enqueue(jrdev, desc, old_aead_decrypt_done, req); 2793 if (!ret) { 2794 ret = -EINPROGRESS; 2795 } else { 2796 old_aead_unmap(jrdev, edesc, req); 2797 kfree(edesc); 2798 } 2799 2800 return ret; 2801 } 2802 2803 /* 2804 * allocate and map the aead extended descriptor for aead givencrypt 2805 */ 2806 static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request 2807 *greq, int desc_bytes, 2808 u32 *contig_ptr) 2809 { 2810 struct aead_request *req = &greq->areq; 2811 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2812 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2813 struct device *jrdev = ctx->jrdev; 2814 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2815 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 2816 int assoc_nents, src_nents, dst_nents = 0; 2817 struct aead_edesc *edesc; 2818 dma_addr_t iv_dma = 0; 2819 int sgc; 2820 u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG; 2821 int ivsize = crypto_aead_ivsize(aead); 2822 bool assoc_chained = false, src_chained = false, dst_chained = false; 2823 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 2824 bool is_gcm = false; 2825 2826 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); 2827 src_nents = sg_count(req->src, req->cryptlen, &src_chained); 2828 2829 if (unlikely(req->dst != req->src)) 2830 dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize, 2831 &dst_chained); 2832 2833 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, 2834 DMA_TO_DEVICE, assoc_chained); 2835 if (likely(req->src == req->dst)) { 2836 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2837 DMA_BIDIRECTIONAL, src_chained); 2838 } else { 2839 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2840 DMA_TO_DEVICE, src_chained); 2841 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 2842 DMA_FROM_DEVICE, dst_chained); 2843 } 2844 2845 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 2846 if (dma_mapping_error(jrdev, iv_dma)) { 2847 dev_err(jrdev, "unable to map IV\n"); 2848 return ERR_PTR(-ENOMEM); 2849 } 2850 2851 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == 2852 OP_ALG_ALGSEL_AES) && 2853 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) 2854 is_gcm = true; 2855 2856 /* 2857 * Check if data are contiguous. 2858 * GCM expected input sequence: IV, AAD, text 2859 * All other - expected input sequence: AAD, IV, text 2860 */ 2861 2862 if (is_gcm) { 2863 if (assoc_nents || iv_dma + ivsize != 2864 sg_dma_address(req->assoc) || src_nents || 2865 sg_dma_address(req->assoc) + req->assoclen != 2866 sg_dma_address(req->src)) 2867 contig &= ~GIV_SRC_CONTIG; 2868 } else { 2869 if (assoc_nents || 2870 sg_dma_address(req->assoc) + req->assoclen != iv_dma || 2871 src_nents || iv_dma + ivsize != sg_dma_address(req->src)) 2872 contig &= ~GIV_SRC_CONTIG; 2873 } 2874 2875 if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst)) 2876 contig &= ~GIV_DST_CONTIG; 2877 2878 if (!(contig & GIV_SRC_CONTIG)) { 2879 assoc_nents = assoc_nents ? : 1; 2880 src_nents = src_nents ? : 1; 2881 sec4_sg_len += assoc_nents + 1 + src_nents; 2882 if (req->src == req->dst && 2883 (src_nents || iv_dma + ivsize != sg_dma_address(req->src))) 2884 contig &= ~GIV_DST_CONTIG; 2885 } 2886 2887 /* 2888 * Add new sg entries for GCM output sequence. 2889 * Expected output sequence: IV, encrypted text. 2890 */ 2891 if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) 2892 sec4_sg_len += 1 + src_nents; 2893 2894 if (unlikely(req->src != req->dst)) { 2895 dst_nents = dst_nents ? : 1; 2896 sec4_sg_len += 1 + dst_nents; 2897 } 2898 2899 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 2900 2901 /* allocate space for base edesc and hw desc commands, link tables */ 2902 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + 2903 sec4_sg_bytes, GFP_DMA | flags); 2904 if (!edesc) { 2905 dev_err(jrdev, "could not allocate extended descriptor\n"); 2906 return ERR_PTR(-ENOMEM); 2907 } 2908 2909 edesc->assoc_nents = assoc_nents; 2910 edesc->assoc_chained = assoc_chained; 2911 edesc->src_nents = src_nents; 2912 edesc->src_chained = src_chained; 2913 edesc->dst_nents = dst_nents; 2914 edesc->dst_chained = dst_chained; 2915 edesc->iv_dma = iv_dma; 2916 edesc->sec4_sg_bytes = sec4_sg_bytes; 2917 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 2918 desc_bytes; 2919 *contig_ptr = contig; 2920 2921 sec4_sg_index = 0; 2922 if (!(contig & GIV_SRC_CONTIG)) { 2923 if (!is_gcm) { 2924 sg_to_sec4_sg_len(req->assoc, req->assoclen, 2925 edesc->sec4_sg + sec4_sg_index); 2926 sec4_sg_index += assoc_nents; 2927 } 2928 2929 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 2930 iv_dma, ivsize, 0); 2931 sec4_sg_index += 1; 2932 2933 if (is_gcm) { 2934 sg_to_sec4_sg_len(req->assoc, req->assoclen, 2935 edesc->sec4_sg + sec4_sg_index); 2936 sec4_sg_index += assoc_nents; 2937 } 2938 2939 sg_to_sec4_sg_last(req->src, src_nents, 2940 edesc->sec4_sg + 2941 sec4_sg_index, 0); 2942 sec4_sg_index += src_nents; 2943 } 2944 2945 if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) { 2946 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 2947 iv_dma, ivsize, 0); 2948 sec4_sg_index += 1; 2949 sg_to_sec4_sg_last(req->src, src_nents, 2950 edesc->sec4_sg + sec4_sg_index, 0); 2951 } 2952 2953 if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) { 2954 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 2955 iv_dma, ivsize, 0); 2956 sec4_sg_index += 1; 2957 sg_to_sec4_sg_last(req->dst, dst_nents, 2958 edesc->sec4_sg + sec4_sg_index, 0); 2959 } 2960 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2961 sec4_sg_bytes, DMA_TO_DEVICE); 2962 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2963 dev_err(jrdev, "unable to map S/G table\n"); 2964 return ERR_PTR(-ENOMEM); 2965 } 2966 2967 return edesc; 2968 } 2969 2970 static int old_aead_givencrypt(struct aead_givcrypt_request *areq) 2971 { 2972 struct aead_request *req = &areq->areq; 2973 struct aead_edesc *edesc; 2974 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2975 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2976 struct device *jrdev = ctx->jrdev; 2977 u32 contig; 2978 u32 *desc; 2979 int ret = 0; 2980 2981 /* allocate extended descriptor */ 2982 edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN * 2983 CAAM_CMD_SZ, &contig); 2984 2985 if (IS_ERR(edesc)) 2986 return PTR_ERR(edesc); 2987 2988 #ifdef DEBUG 2989 print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ", 2990 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2991 req->cryptlen, 1); 2992 #endif 2993 2994 /* Create and submit job descriptor*/ 2995 init_aead_giv_job(ctx->sh_desc_givenc, 2996 ctx->sh_desc_givenc_dma, edesc, req, contig); 2997 #ifdef DEBUG 2998 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2999 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 3000 desc_bytes(edesc->hw_desc), 1); 3001 #endif 3002 3003 desc = edesc->hw_desc; 3004 ret = caam_jr_enqueue(jrdev, desc, old_aead_encrypt_done, req); 3005 if (!ret) { 3006 ret = -EINPROGRESS; 3007 } else { 3008 old_aead_unmap(jrdev, edesc, req); 3009 kfree(edesc); 3010 } 3011 3012 return ret; 3013 } 3014 3015 static int aead_null_givencrypt(struct aead_givcrypt_request *areq) 3016 { 3017 return old_aead_encrypt(&areq->areq); 3018 } 3019 3020 /* 3021 * allocate and map the ablkcipher extended descriptor for ablkcipher 3022 */ 3023 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 3024 *req, int desc_bytes, 3025 bool *iv_contig_out) 3026 { 3027 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 3028 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 3029 struct device *jrdev = ctx->jrdev; 3030 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 3031 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 3032 GFP_KERNEL : GFP_ATOMIC; 3033 int src_nents, dst_nents = 0, sec4_sg_bytes; 3034 struct ablkcipher_edesc *edesc; 3035 dma_addr_t iv_dma = 0; 3036 bool iv_contig = false; 3037 int sgc; 3038 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 3039 bool src_chained = false, dst_chained = false; 3040 int sec4_sg_index; 3041 3042 src_nents = sg_count(req->src, req->nbytes, &src_chained); 3043 3044 if (req->dst != req->src) 3045 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); 3046 3047 if (likely(req->src == req->dst)) { 3048 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 3049 DMA_BIDIRECTIONAL, src_chained); 3050 } else { 3051 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 3052 DMA_TO_DEVICE, src_chained); 3053 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 3054 DMA_FROM_DEVICE, dst_chained); 3055 } 3056 3057 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 3058 if (dma_mapping_error(jrdev, iv_dma)) { 3059 dev_err(jrdev, "unable to map IV\n"); 3060 return ERR_PTR(-ENOMEM); 3061 } 3062 3063 /* 3064 * Check if iv can be contiguous with source and destination. 3065 * If so, include it. If not, create scatterlist. 3066 */ 3067 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) 3068 iv_contig = true; 3069 else 3070 src_nents = src_nents ? : 1; 3071 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 3072 sizeof(struct sec4_sg_entry); 3073 3074 /* allocate space for base edesc and hw desc commands, link tables */ 3075 edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes + 3076 sec4_sg_bytes, GFP_DMA | flags); 3077 if (!edesc) { 3078 dev_err(jrdev, "could not allocate extended descriptor\n"); 3079 return ERR_PTR(-ENOMEM); 3080 } 3081 3082 edesc->src_nents = src_nents; 3083 edesc->src_chained = src_chained; 3084 edesc->dst_nents = dst_nents; 3085 edesc->dst_chained = dst_chained; 3086 edesc->sec4_sg_bytes = sec4_sg_bytes; 3087 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 3088 desc_bytes; 3089 3090 sec4_sg_index = 0; 3091 if (!iv_contig) { 3092 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 3093 sg_to_sec4_sg_last(req->src, src_nents, 3094 edesc->sec4_sg + 1, 0); 3095 sec4_sg_index += 1 + src_nents; 3096 } 3097 3098 if (dst_nents) { 3099 sg_to_sec4_sg_last(req->dst, dst_nents, 3100 edesc->sec4_sg + sec4_sg_index, 0); 3101 } 3102 3103 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 3104 sec4_sg_bytes, DMA_TO_DEVICE); 3105 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 3106 dev_err(jrdev, "unable to map S/G table\n"); 3107 return ERR_PTR(-ENOMEM); 3108 } 3109 3110 edesc->iv_dma = iv_dma; 3111 3112 #ifdef DEBUG 3113 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", 3114 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 3115 sec4_sg_bytes, 1); 3116 #endif 3117 3118 *iv_contig_out = iv_contig; 3119 return edesc; 3120 } 3121 3122 static int ablkcipher_encrypt(struct ablkcipher_request *req) 3123 { 3124 struct ablkcipher_edesc *edesc; 3125 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 3126 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 3127 struct device *jrdev = ctx->jrdev; 3128 bool iv_contig; 3129 u32 *desc; 3130 int ret = 0; 3131 3132 /* allocate extended descriptor */ 3133 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 3134 CAAM_CMD_SZ, &iv_contig); 3135 if (IS_ERR(edesc)) 3136 return PTR_ERR(edesc); 3137 3138 /* Create and submit job descriptor*/ 3139 init_ablkcipher_job(ctx->sh_desc_enc, 3140 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 3141 #ifdef DEBUG 3142 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 3143 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 3144 desc_bytes(edesc->hw_desc), 1); 3145 #endif 3146 desc = edesc->hw_desc; 3147 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 3148 3149 if (!ret) { 3150 ret = -EINPROGRESS; 3151 } else { 3152 ablkcipher_unmap(jrdev, edesc, req); 3153 kfree(edesc); 3154 } 3155 3156 return ret; 3157 } 3158 3159 static int ablkcipher_decrypt(struct ablkcipher_request *req) 3160 { 3161 struct ablkcipher_edesc *edesc; 3162 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 3163 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 3164 struct device *jrdev = ctx->jrdev; 3165 bool iv_contig; 3166 u32 *desc; 3167 int ret = 0; 3168 3169 /* allocate extended descriptor */ 3170 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 3171 CAAM_CMD_SZ, &iv_contig); 3172 if (IS_ERR(edesc)) 3173 return PTR_ERR(edesc); 3174 3175 /* Create and submit job descriptor*/ 3176 init_ablkcipher_job(ctx->sh_desc_dec, 3177 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 3178 desc = edesc->hw_desc; 3179 #ifdef DEBUG 3180 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 3181 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 3182 desc_bytes(edesc->hw_desc), 1); 3183 #endif 3184 3185 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 3186 if (!ret) { 3187 ret = -EINPROGRESS; 3188 } else { 3189 ablkcipher_unmap(jrdev, edesc, req); 3190 kfree(edesc); 3191 } 3192 3193 return ret; 3194 } 3195 3196 /* 3197 * allocate and map the ablkcipher extended descriptor 3198 * for ablkcipher givencrypt 3199 */ 3200 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 3201 struct skcipher_givcrypt_request *greq, 3202 int desc_bytes, 3203 bool *iv_contig_out) 3204 { 3205 struct ablkcipher_request *req = &greq->creq; 3206 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 3207 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 3208 struct device *jrdev = ctx->jrdev; 3209 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 3210 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 3211 GFP_KERNEL : GFP_ATOMIC; 3212 int src_nents, dst_nents = 0, sec4_sg_bytes; 3213 struct ablkcipher_edesc *edesc; 3214 dma_addr_t iv_dma = 0; 3215 bool iv_contig = false; 3216 int sgc; 3217 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 3218 bool src_chained = false, dst_chained = false; 3219 int sec4_sg_index; 3220 3221 src_nents = sg_count(req->src, req->nbytes, &src_chained); 3222 3223 if (unlikely(req->dst != req->src)) 3224 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); 3225 3226 if (likely(req->src == req->dst)) { 3227 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 3228 DMA_BIDIRECTIONAL, src_chained); 3229 } else { 3230 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 3231 DMA_TO_DEVICE, src_chained); 3232 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 3233 DMA_FROM_DEVICE, dst_chained); 3234 } 3235 3236 /* 3237 * Check if iv can be contiguous with source and destination. 3238 * If so, include it. If not, create scatterlist. 3239 */ 3240 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 3241 if (dma_mapping_error(jrdev, iv_dma)) { 3242 dev_err(jrdev, "unable to map IV\n"); 3243 return ERR_PTR(-ENOMEM); 3244 } 3245 3246 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) 3247 iv_contig = true; 3248 else 3249 dst_nents = dst_nents ? : 1; 3250 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 3251 sizeof(struct sec4_sg_entry); 3252 3253 /* allocate space for base edesc and hw desc commands, link tables */ 3254 edesc = kmalloc(sizeof(*edesc) + desc_bytes + 3255 sec4_sg_bytes, GFP_DMA | flags); 3256 if (!edesc) { 3257 dev_err(jrdev, "could not allocate extended descriptor\n"); 3258 return ERR_PTR(-ENOMEM); 3259 } 3260 3261 edesc->src_nents = src_nents; 3262 edesc->src_chained = src_chained; 3263 edesc->dst_nents = dst_nents; 3264 edesc->dst_chained = dst_chained; 3265 edesc->sec4_sg_bytes = sec4_sg_bytes; 3266 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 3267 desc_bytes; 3268 3269 sec4_sg_index = 0; 3270 if (src_nents) { 3271 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); 3272 sec4_sg_index += src_nents; 3273 } 3274 3275 if (!iv_contig) { 3276 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 3277 iv_dma, ivsize, 0); 3278 sec4_sg_index += 1; 3279 sg_to_sec4_sg_last(req->dst, dst_nents, 3280 edesc->sec4_sg + sec4_sg_index, 0); 3281 } 3282 3283 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 3284 sec4_sg_bytes, DMA_TO_DEVICE); 3285 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 3286 dev_err(jrdev, "unable to map S/G table\n"); 3287 return ERR_PTR(-ENOMEM); 3288 } 3289 edesc->iv_dma = iv_dma; 3290 3291 #ifdef DEBUG 3292 print_hex_dump(KERN_ERR, 3293 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ", 3294 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 3295 sec4_sg_bytes, 1); 3296 #endif 3297 3298 *iv_contig_out = iv_contig; 3299 return edesc; 3300 } 3301 3302 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 3303 { 3304 struct ablkcipher_request *req = &creq->creq; 3305 struct ablkcipher_edesc *edesc; 3306 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 3307 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 3308 struct device *jrdev = ctx->jrdev; 3309 bool iv_contig; 3310 u32 *desc; 3311 int ret = 0; 3312 3313 /* allocate extended descriptor */ 3314 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * 3315 CAAM_CMD_SZ, &iv_contig); 3316 if (IS_ERR(edesc)) 3317 return PTR_ERR(edesc); 3318 3319 /* Create and submit job descriptor*/ 3320 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma, 3321 edesc, req, iv_contig); 3322 #ifdef DEBUG 3323 print_hex_dump(KERN_ERR, 3324 "ablkcipher jobdesc@" __stringify(__LINE__) ": ", 3325 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 3326 desc_bytes(edesc->hw_desc), 1); 3327 #endif 3328 desc = edesc->hw_desc; 3329 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 3330 3331 if (!ret) { 3332 ret = -EINPROGRESS; 3333 } else { 3334 ablkcipher_unmap(jrdev, edesc, req); 3335 kfree(edesc); 3336 } 3337 3338 return ret; 3339 } 3340 3341 #define template_aead template_u.aead 3342 #define template_ablkcipher template_u.ablkcipher 3343 struct caam_alg_template { 3344 char name[CRYPTO_MAX_ALG_NAME]; 3345 char driver_name[CRYPTO_MAX_ALG_NAME]; 3346 unsigned int blocksize; 3347 u32 type; 3348 union { 3349 struct ablkcipher_alg ablkcipher; 3350 struct old_aead_alg aead; 3351 } template_u; 3352 u32 class1_alg_type; 3353 u32 class2_alg_type; 3354 u32 alg_op; 3355 }; 3356 3357 static struct caam_alg_template driver_algs[] = { 3358 /* single-pass ipsec_esp descriptor */ 3359 { 3360 .name = "authenc(hmac(md5),ecb(cipher_null))", 3361 .driver_name = "authenc-hmac-md5-ecb-cipher_null-caam", 3362 .blocksize = NULL_BLOCK_SIZE, 3363 .type = CRYPTO_ALG_TYPE_AEAD, 3364 .template_aead = { 3365 .setkey = aead_setkey, 3366 .setauthsize = aead_setauthsize, 3367 .encrypt = old_aead_encrypt, 3368 .decrypt = old_aead_decrypt, 3369 .givencrypt = aead_null_givencrypt, 3370 .geniv = "<built-in>", 3371 .ivsize = NULL_IV_SIZE, 3372 .maxauthsize = MD5_DIGEST_SIZE, 3373 }, 3374 .class1_alg_type = 0, 3375 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 3376 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3377 }, 3378 { 3379 .name = "authenc(hmac(sha1),ecb(cipher_null))", 3380 .driver_name = "authenc-hmac-sha1-ecb-cipher_null-caam", 3381 .blocksize = NULL_BLOCK_SIZE, 3382 .type = CRYPTO_ALG_TYPE_AEAD, 3383 .template_aead = { 3384 .setkey = aead_setkey, 3385 .setauthsize = aead_setauthsize, 3386 .encrypt = old_aead_encrypt, 3387 .decrypt = old_aead_decrypt, 3388 .givencrypt = aead_null_givencrypt, 3389 .geniv = "<built-in>", 3390 .ivsize = NULL_IV_SIZE, 3391 .maxauthsize = SHA1_DIGEST_SIZE, 3392 }, 3393 .class1_alg_type = 0, 3394 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 3395 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3396 }, 3397 { 3398 .name = "authenc(hmac(sha224),ecb(cipher_null))", 3399 .driver_name = "authenc-hmac-sha224-ecb-cipher_null-caam", 3400 .blocksize = NULL_BLOCK_SIZE, 3401 .type = CRYPTO_ALG_TYPE_AEAD, 3402 .template_aead = { 3403 .setkey = aead_setkey, 3404 .setauthsize = aead_setauthsize, 3405 .encrypt = old_aead_encrypt, 3406 .decrypt = old_aead_decrypt, 3407 .givencrypt = aead_null_givencrypt, 3408 .geniv = "<built-in>", 3409 .ivsize = NULL_IV_SIZE, 3410 .maxauthsize = SHA224_DIGEST_SIZE, 3411 }, 3412 .class1_alg_type = 0, 3413 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3414 OP_ALG_AAI_HMAC_PRECOMP, 3415 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3416 }, 3417 { 3418 .name = "authenc(hmac(sha256),ecb(cipher_null))", 3419 .driver_name = "authenc-hmac-sha256-ecb-cipher_null-caam", 3420 .blocksize = NULL_BLOCK_SIZE, 3421 .type = CRYPTO_ALG_TYPE_AEAD, 3422 .template_aead = { 3423 .setkey = aead_setkey, 3424 .setauthsize = aead_setauthsize, 3425 .encrypt = old_aead_encrypt, 3426 .decrypt = old_aead_decrypt, 3427 .givencrypt = aead_null_givencrypt, 3428 .geniv = "<built-in>", 3429 .ivsize = NULL_IV_SIZE, 3430 .maxauthsize = SHA256_DIGEST_SIZE, 3431 }, 3432 .class1_alg_type = 0, 3433 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3434 OP_ALG_AAI_HMAC_PRECOMP, 3435 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3436 }, 3437 { 3438 .name = "authenc(hmac(sha384),ecb(cipher_null))", 3439 .driver_name = "authenc-hmac-sha384-ecb-cipher_null-caam", 3440 .blocksize = NULL_BLOCK_SIZE, 3441 .type = CRYPTO_ALG_TYPE_AEAD, 3442 .template_aead = { 3443 .setkey = aead_setkey, 3444 .setauthsize = aead_setauthsize, 3445 .encrypt = old_aead_encrypt, 3446 .decrypt = old_aead_decrypt, 3447 .givencrypt = aead_null_givencrypt, 3448 .geniv = "<built-in>", 3449 .ivsize = NULL_IV_SIZE, 3450 .maxauthsize = SHA384_DIGEST_SIZE, 3451 }, 3452 .class1_alg_type = 0, 3453 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3454 OP_ALG_AAI_HMAC_PRECOMP, 3455 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3456 }, 3457 { 3458 .name = "authenc(hmac(sha512),ecb(cipher_null))", 3459 .driver_name = "authenc-hmac-sha512-ecb-cipher_null-caam", 3460 .blocksize = NULL_BLOCK_SIZE, 3461 .type = CRYPTO_ALG_TYPE_AEAD, 3462 .template_aead = { 3463 .setkey = aead_setkey, 3464 .setauthsize = aead_setauthsize, 3465 .encrypt = old_aead_encrypt, 3466 .decrypt = old_aead_decrypt, 3467 .givencrypt = aead_null_givencrypt, 3468 .geniv = "<built-in>", 3469 .ivsize = NULL_IV_SIZE, 3470 .maxauthsize = SHA512_DIGEST_SIZE, 3471 }, 3472 .class1_alg_type = 0, 3473 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3474 OP_ALG_AAI_HMAC_PRECOMP, 3475 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3476 }, 3477 { 3478 .name = "authenc(hmac(md5),cbc(aes))", 3479 .driver_name = "authenc-hmac-md5-cbc-aes-caam", 3480 .blocksize = AES_BLOCK_SIZE, 3481 .type = CRYPTO_ALG_TYPE_AEAD, 3482 .template_aead = { 3483 .setkey = aead_setkey, 3484 .setauthsize = aead_setauthsize, 3485 .encrypt = old_aead_encrypt, 3486 .decrypt = old_aead_decrypt, 3487 .givencrypt = old_aead_givencrypt, 3488 .geniv = "<built-in>", 3489 .ivsize = AES_BLOCK_SIZE, 3490 .maxauthsize = MD5_DIGEST_SIZE, 3491 }, 3492 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3493 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 3494 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3495 }, 3496 { 3497 .name = "authenc(hmac(sha1),cbc(aes))", 3498 .driver_name = "authenc-hmac-sha1-cbc-aes-caam", 3499 .blocksize = AES_BLOCK_SIZE, 3500 .type = CRYPTO_ALG_TYPE_AEAD, 3501 .template_aead = { 3502 .setkey = aead_setkey, 3503 .setauthsize = aead_setauthsize, 3504 .encrypt = old_aead_encrypt, 3505 .decrypt = old_aead_decrypt, 3506 .givencrypt = old_aead_givencrypt, 3507 .geniv = "<built-in>", 3508 .ivsize = AES_BLOCK_SIZE, 3509 .maxauthsize = SHA1_DIGEST_SIZE, 3510 }, 3511 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3512 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 3513 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3514 }, 3515 { 3516 .name = "authenc(hmac(sha224),cbc(aes))", 3517 .driver_name = "authenc-hmac-sha224-cbc-aes-caam", 3518 .blocksize = AES_BLOCK_SIZE, 3519 .type = CRYPTO_ALG_TYPE_AEAD, 3520 .template_aead = { 3521 .setkey = aead_setkey, 3522 .setauthsize = aead_setauthsize, 3523 .encrypt = old_aead_encrypt, 3524 .decrypt = old_aead_decrypt, 3525 .givencrypt = old_aead_givencrypt, 3526 .geniv = "<built-in>", 3527 .ivsize = AES_BLOCK_SIZE, 3528 .maxauthsize = SHA224_DIGEST_SIZE, 3529 }, 3530 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3531 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3532 OP_ALG_AAI_HMAC_PRECOMP, 3533 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3534 }, 3535 { 3536 .name = "authenc(hmac(sha256),cbc(aes))", 3537 .driver_name = "authenc-hmac-sha256-cbc-aes-caam", 3538 .blocksize = AES_BLOCK_SIZE, 3539 .type = CRYPTO_ALG_TYPE_AEAD, 3540 .template_aead = { 3541 .setkey = aead_setkey, 3542 .setauthsize = aead_setauthsize, 3543 .encrypt = old_aead_encrypt, 3544 .decrypt = old_aead_decrypt, 3545 .givencrypt = old_aead_givencrypt, 3546 .geniv = "<built-in>", 3547 .ivsize = AES_BLOCK_SIZE, 3548 .maxauthsize = SHA256_DIGEST_SIZE, 3549 }, 3550 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3551 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3552 OP_ALG_AAI_HMAC_PRECOMP, 3553 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3554 }, 3555 { 3556 .name = "authenc(hmac(sha384),cbc(aes))", 3557 .driver_name = "authenc-hmac-sha384-cbc-aes-caam", 3558 .blocksize = AES_BLOCK_SIZE, 3559 .type = CRYPTO_ALG_TYPE_AEAD, 3560 .template_aead = { 3561 .setkey = aead_setkey, 3562 .setauthsize = aead_setauthsize, 3563 .encrypt = old_aead_encrypt, 3564 .decrypt = old_aead_decrypt, 3565 .givencrypt = old_aead_givencrypt, 3566 .geniv = "<built-in>", 3567 .ivsize = AES_BLOCK_SIZE, 3568 .maxauthsize = SHA384_DIGEST_SIZE, 3569 }, 3570 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3571 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3572 OP_ALG_AAI_HMAC_PRECOMP, 3573 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3574 }, 3575 3576 { 3577 .name = "authenc(hmac(sha512),cbc(aes))", 3578 .driver_name = "authenc-hmac-sha512-cbc-aes-caam", 3579 .blocksize = AES_BLOCK_SIZE, 3580 .type = CRYPTO_ALG_TYPE_AEAD, 3581 .template_aead = { 3582 .setkey = aead_setkey, 3583 .setauthsize = aead_setauthsize, 3584 .encrypt = old_aead_encrypt, 3585 .decrypt = old_aead_decrypt, 3586 .givencrypt = old_aead_givencrypt, 3587 .geniv = "<built-in>", 3588 .ivsize = AES_BLOCK_SIZE, 3589 .maxauthsize = SHA512_DIGEST_SIZE, 3590 }, 3591 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3592 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3593 OP_ALG_AAI_HMAC_PRECOMP, 3594 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3595 }, 3596 { 3597 .name = "authenc(hmac(md5),cbc(des3_ede))", 3598 .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam", 3599 .blocksize = DES3_EDE_BLOCK_SIZE, 3600 .type = CRYPTO_ALG_TYPE_AEAD, 3601 .template_aead = { 3602 .setkey = aead_setkey, 3603 .setauthsize = aead_setauthsize, 3604 .encrypt = old_aead_encrypt, 3605 .decrypt = old_aead_decrypt, 3606 .givencrypt = old_aead_givencrypt, 3607 .geniv = "<built-in>", 3608 .ivsize = DES3_EDE_BLOCK_SIZE, 3609 .maxauthsize = MD5_DIGEST_SIZE, 3610 }, 3611 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3612 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 3613 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3614 }, 3615 { 3616 .name = "authenc(hmac(sha1),cbc(des3_ede))", 3617 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", 3618 .blocksize = DES3_EDE_BLOCK_SIZE, 3619 .type = CRYPTO_ALG_TYPE_AEAD, 3620 .template_aead = { 3621 .setkey = aead_setkey, 3622 .setauthsize = aead_setauthsize, 3623 .encrypt = old_aead_encrypt, 3624 .decrypt = old_aead_decrypt, 3625 .givencrypt = old_aead_givencrypt, 3626 .geniv = "<built-in>", 3627 .ivsize = DES3_EDE_BLOCK_SIZE, 3628 .maxauthsize = SHA1_DIGEST_SIZE, 3629 }, 3630 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3631 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 3632 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3633 }, 3634 { 3635 .name = "authenc(hmac(sha224),cbc(des3_ede))", 3636 .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam", 3637 .blocksize = DES3_EDE_BLOCK_SIZE, 3638 .type = CRYPTO_ALG_TYPE_AEAD, 3639 .template_aead = { 3640 .setkey = aead_setkey, 3641 .setauthsize = aead_setauthsize, 3642 .encrypt = old_aead_encrypt, 3643 .decrypt = old_aead_decrypt, 3644 .givencrypt = old_aead_givencrypt, 3645 .geniv = "<built-in>", 3646 .ivsize = DES3_EDE_BLOCK_SIZE, 3647 .maxauthsize = SHA224_DIGEST_SIZE, 3648 }, 3649 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3650 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3651 OP_ALG_AAI_HMAC_PRECOMP, 3652 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3653 }, 3654 { 3655 .name = "authenc(hmac(sha256),cbc(des3_ede))", 3656 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", 3657 .blocksize = DES3_EDE_BLOCK_SIZE, 3658 .type = CRYPTO_ALG_TYPE_AEAD, 3659 .template_aead = { 3660 .setkey = aead_setkey, 3661 .setauthsize = aead_setauthsize, 3662 .encrypt = old_aead_encrypt, 3663 .decrypt = old_aead_decrypt, 3664 .givencrypt = old_aead_givencrypt, 3665 .geniv = "<built-in>", 3666 .ivsize = DES3_EDE_BLOCK_SIZE, 3667 .maxauthsize = SHA256_DIGEST_SIZE, 3668 }, 3669 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3670 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3671 OP_ALG_AAI_HMAC_PRECOMP, 3672 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3673 }, 3674 { 3675 .name = "authenc(hmac(sha384),cbc(des3_ede))", 3676 .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam", 3677 .blocksize = DES3_EDE_BLOCK_SIZE, 3678 .type = CRYPTO_ALG_TYPE_AEAD, 3679 .template_aead = { 3680 .setkey = aead_setkey, 3681 .setauthsize = aead_setauthsize, 3682 .encrypt = old_aead_encrypt, 3683 .decrypt = old_aead_decrypt, 3684 .givencrypt = old_aead_givencrypt, 3685 .geniv = "<built-in>", 3686 .ivsize = DES3_EDE_BLOCK_SIZE, 3687 .maxauthsize = SHA384_DIGEST_SIZE, 3688 }, 3689 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3690 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3691 OP_ALG_AAI_HMAC_PRECOMP, 3692 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3693 }, 3694 { 3695 .name = "authenc(hmac(sha512),cbc(des3_ede))", 3696 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam", 3697 .blocksize = DES3_EDE_BLOCK_SIZE, 3698 .type = CRYPTO_ALG_TYPE_AEAD, 3699 .template_aead = { 3700 .setkey = aead_setkey, 3701 .setauthsize = aead_setauthsize, 3702 .encrypt = old_aead_encrypt, 3703 .decrypt = old_aead_decrypt, 3704 .givencrypt = old_aead_givencrypt, 3705 .geniv = "<built-in>", 3706 .ivsize = DES3_EDE_BLOCK_SIZE, 3707 .maxauthsize = SHA512_DIGEST_SIZE, 3708 }, 3709 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3710 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3711 OP_ALG_AAI_HMAC_PRECOMP, 3712 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3713 }, 3714 { 3715 .name = "authenc(hmac(md5),cbc(des))", 3716 .driver_name = "authenc-hmac-md5-cbc-des-caam", 3717 .blocksize = DES_BLOCK_SIZE, 3718 .type = CRYPTO_ALG_TYPE_AEAD, 3719 .template_aead = { 3720 .setkey = aead_setkey, 3721 .setauthsize = aead_setauthsize, 3722 .encrypt = old_aead_encrypt, 3723 .decrypt = old_aead_decrypt, 3724 .givencrypt = old_aead_givencrypt, 3725 .geniv = "<built-in>", 3726 .ivsize = DES_BLOCK_SIZE, 3727 .maxauthsize = MD5_DIGEST_SIZE, 3728 }, 3729 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3730 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 3731 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3732 }, 3733 { 3734 .name = "authenc(hmac(sha1),cbc(des))", 3735 .driver_name = "authenc-hmac-sha1-cbc-des-caam", 3736 .blocksize = DES_BLOCK_SIZE, 3737 .type = CRYPTO_ALG_TYPE_AEAD, 3738 .template_aead = { 3739 .setkey = aead_setkey, 3740 .setauthsize = aead_setauthsize, 3741 .encrypt = old_aead_encrypt, 3742 .decrypt = old_aead_decrypt, 3743 .givencrypt = old_aead_givencrypt, 3744 .geniv = "<built-in>", 3745 .ivsize = DES_BLOCK_SIZE, 3746 .maxauthsize = SHA1_DIGEST_SIZE, 3747 }, 3748 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3749 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 3750 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3751 }, 3752 { 3753 .name = "authenc(hmac(sha224),cbc(des))", 3754 .driver_name = "authenc-hmac-sha224-cbc-des-caam", 3755 .blocksize = DES_BLOCK_SIZE, 3756 .type = CRYPTO_ALG_TYPE_AEAD, 3757 .template_aead = { 3758 .setkey = aead_setkey, 3759 .setauthsize = aead_setauthsize, 3760 .encrypt = old_aead_encrypt, 3761 .decrypt = old_aead_decrypt, 3762 .givencrypt = old_aead_givencrypt, 3763 .geniv = "<built-in>", 3764 .ivsize = DES_BLOCK_SIZE, 3765 .maxauthsize = SHA224_DIGEST_SIZE, 3766 }, 3767 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3768 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3769 OP_ALG_AAI_HMAC_PRECOMP, 3770 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3771 }, 3772 { 3773 .name = "authenc(hmac(sha256),cbc(des))", 3774 .driver_name = "authenc-hmac-sha256-cbc-des-caam", 3775 .blocksize = DES_BLOCK_SIZE, 3776 .type = CRYPTO_ALG_TYPE_AEAD, 3777 .template_aead = { 3778 .setkey = aead_setkey, 3779 .setauthsize = aead_setauthsize, 3780 .encrypt = old_aead_encrypt, 3781 .decrypt = old_aead_decrypt, 3782 .givencrypt = old_aead_givencrypt, 3783 .geniv = "<built-in>", 3784 .ivsize = DES_BLOCK_SIZE, 3785 .maxauthsize = SHA256_DIGEST_SIZE, 3786 }, 3787 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3788 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3789 OP_ALG_AAI_HMAC_PRECOMP, 3790 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3791 }, 3792 { 3793 .name = "authenc(hmac(sha384),cbc(des))", 3794 .driver_name = "authenc-hmac-sha384-cbc-des-caam", 3795 .blocksize = DES_BLOCK_SIZE, 3796 .type = CRYPTO_ALG_TYPE_AEAD, 3797 .template_aead = { 3798 .setkey = aead_setkey, 3799 .setauthsize = aead_setauthsize, 3800 .encrypt = old_aead_encrypt, 3801 .decrypt = old_aead_decrypt, 3802 .givencrypt = old_aead_givencrypt, 3803 .geniv = "<built-in>", 3804 .ivsize = DES_BLOCK_SIZE, 3805 .maxauthsize = SHA384_DIGEST_SIZE, 3806 }, 3807 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3808 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3809 OP_ALG_AAI_HMAC_PRECOMP, 3810 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3811 }, 3812 { 3813 .name = "authenc(hmac(sha512),cbc(des))", 3814 .driver_name = "authenc-hmac-sha512-cbc-des-caam", 3815 .blocksize = DES_BLOCK_SIZE, 3816 .type = CRYPTO_ALG_TYPE_AEAD, 3817 .template_aead = { 3818 .setkey = aead_setkey, 3819 .setauthsize = aead_setauthsize, 3820 .encrypt = old_aead_encrypt, 3821 .decrypt = old_aead_decrypt, 3822 .givencrypt = old_aead_givencrypt, 3823 .geniv = "<built-in>", 3824 .ivsize = DES_BLOCK_SIZE, 3825 .maxauthsize = SHA512_DIGEST_SIZE, 3826 }, 3827 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3828 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3829 OP_ALG_AAI_HMAC_PRECOMP, 3830 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3831 }, 3832 { 3833 .name = "authenc(hmac(md5),rfc3686(ctr(aes)))", 3834 .driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam", 3835 .blocksize = 1, 3836 .type = CRYPTO_ALG_TYPE_AEAD, 3837 .template_aead = { 3838 .setkey = aead_setkey, 3839 .setauthsize = aead_setauthsize, 3840 .encrypt = old_aead_encrypt, 3841 .decrypt = old_aead_decrypt, 3842 .givencrypt = old_aead_givencrypt, 3843 .geniv = "<built-in>", 3844 .ivsize = CTR_RFC3686_IV_SIZE, 3845 .maxauthsize = MD5_DIGEST_SIZE, 3846 }, 3847 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 3848 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 3849 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3850 }, 3851 { 3852 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 3853 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam", 3854 .blocksize = 1, 3855 .type = CRYPTO_ALG_TYPE_AEAD, 3856 .template_aead = { 3857 .setkey = aead_setkey, 3858 .setauthsize = aead_setauthsize, 3859 .encrypt = old_aead_encrypt, 3860 .decrypt = old_aead_decrypt, 3861 .givencrypt = old_aead_givencrypt, 3862 .geniv = "<built-in>", 3863 .ivsize = CTR_RFC3686_IV_SIZE, 3864 .maxauthsize = SHA1_DIGEST_SIZE, 3865 }, 3866 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 3867 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 3868 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3869 }, 3870 { 3871 .name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 3872 .driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam", 3873 .blocksize = 1, 3874 .type = CRYPTO_ALG_TYPE_AEAD, 3875 .template_aead = { 3876 .setkey = aead_setkey, 3877 .setauthsize = aead_setauthsize, 3878 .encrypt = old_aead_encrypt, 3879 .decrypt = old_aead_decrypt, 3880 .givencrypt = old_aead_givencrypt, 3881 .geniv = "<built-in>", 3882 .ivsize = CTR_RFC3686_IV_SIZE, 3883 .maxauthsize = SHA224_DIGEST_SIZE, 3884 }, 3885 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 3886 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3887 OP_ALG_AAI_HMAC_PRECOMP, 3888 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3889 }, 3890 { 3891 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 3892 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam", 3893 .blocksize = 1, 3894 .type = CRYPTO_ALG_TYPE_AEAD, 3895 .template_aead = { 3896 .setkey = aead_setkey, 3897 .setauthsize = aead_setauthsize, 3898 .encrypt = old_aead_encrypt, 3899 .decrypt = old_aead_decrypt, 3900 .givencrypt = old_aead_givencrypt, 3901 .geniv = "<built-in>", 3902 .ivsize = CTR_RFC3686_IV_SIZE, 3903 .maxauthsize = SHA256_DIGEST_SIZE, 3904 }, 3905 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 3906 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3907 OP_ALG_AAI_HMAC_PRECOMP, 3908 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3909 }, 3910 { 3911 .name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 3912 .driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam", 3913 .blocksize = 1, 3914 .type = CRYPTO_ALG_TYPE_AEAD, 3915 .template_aead = { 3916 .setkey = aead_setkey, 3917 .setauthsize = aead_setauthsize, 3918 .encrypt = old_aead_encrypt, 3919 .decrypt = old_aead_decrypt, 3920 .givencrypt = old_aead_givencrypt, 3921 .geniv = "<built-in>", 3922 .ivsize = CTR_RFC3686_IV_SIZE, 3923 .maxauthsize = SHA384_DIGEST_SIZE, 3924 }, 3925 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 3926 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3927 OP_ALG_AAI_HMAC_PRECOMP, 3928 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3929 }, 3930 { 3931 .name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 3932 .driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam", 3933 .blocksize = 1, 3934 .type = CRYPTO_ALG_TYPE_AEAD, 3935 .template_aead = { 3936 .setkey = aead_setkey, 3937 .setauthsize = aead_setauthsize, 3938 .encrypt = old_aead_encrypt, 3939 .decrypt = old_aead_decrypt, 3940 .givencrypt = old_aead_givencrypt, 3941 .geniv = "<built-in>", 3942 .ivsize = CTR_RFC3686_IV_SIZE, 3943 .maxauthsize = SHA512_DIGEST_SIZE, 3944 }, 3945 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 3946 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3947 OP_ALG_AAI_HMAC_PRECOMP, 3948 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3949 }, 3950 /* ablkcipher descriptor */ 3951 { 3952 .name = "cbc(aes)", 3953 .driver_name = "cbc-aes-caam", 3954 .blocksize = AES_BLOCK_SIZE, 3955 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 3956 .template_ablkcipher = { 3957 .setkey = ablkcipher_setkey, 3958 .encrypt = ablkcipher_encrypt, 3959 .decrypt = ablkcipher_decrypt, 3960 .givencrypt = ablkcipher_givencrypt, 3961 .geniv = "<built-in>", 3962 .min_keysize = AES_MIN_KEY_SIZE, 3963 .max_keysize = AES_MAX_KEY_SIZE, 3964 .ivsize = AES_BLOCK_SIZE, 3965 }, 3966 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3967 }, 3968 { 3969 .name = "cbc(des3_ede)", 3970 .driver_name = "cbc-3des-caam", 3971 .blocksize = DES3_EDE_BLOCK_SIZE, 3972 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 3973 .template_ablkcipher = { 3974 .setkey = ablkcipher_setkey, 3975 .encrypt = ablkcipher_encrypt, 3976 .decrypt = ablkcipher_decrypt, 3977 .givencrypt = ablkcipher_givencrypt, 3978 .geniv = "<built-in>", 3979 .min_keysize = DES3_EDE_KEY_SIZE, 3980 .max_keysize = DES3_EDE_KEY_SIZE, 3981 .ivsize = DES3_EDE_BLOCK_SIZE, 3982 }, 3983 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3984 }, 3985 { 3986 .name = "cbc(des)", 3987 .driver_name = "cbc-des-caam", 3988 .blocksize = DES_BLOCK_SIZE, 3989 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 3990 .template_ablkcipher = { 3991 .setkey = ablkcipher_setkey, 3992 .encrypt = ablkcipher_encrypt, 3993 .decrypt = ablkcipher_decrypt, 3994 .givencrypt = ablkcipher_givencrypt, 3995 .geniv = "<built-in>", 3996 .min_keysize = DES_KEY_SIZE, 3997 .max_keysize = DES_KEY_SIZE, 3998 .ivsize = DES_BLOCK_SIZE, 3999 }, 4000 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 4001 }, 4002 { 4003 .name = "ctr(aes)", 4004 .driver_name = "ctr-aes-caam", 4005 .blocksize = 1, 4006 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 4007 .template_ablkcipher = { 4008 .setkey = ablkcipher_setkey, 4009 .encrypt = ablkcipher_encrypt, 4010 .decrypt = ablkcipher_decrypt, 4011 .geniv = "chainiv", 4012 .min_keysize = AES_MIN_KEY_SIZE, 4013 .max_keysize = AES_MAX_KEY_SIZE, 4014 .ivsize = AES_BLOCK_SIZE, 4015 }, 4016 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 4017 }, 4018 { 4019 .name = "rfc3686(ctr(aes))", 4020 .driver_name = "rfc3686-ctr-aes-caam", 4021 .blocksize = 1, 4022 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 4023 .template_ablkcipher = { 4024 .setkey = ablkcipher_setkey, 4025 .encrypt = ablkcipher_encrypt, 4026 .decrypt = ablkcipher_decrypt, 4027 .givencrypt = ablkcipher_givencrypt, 4028 .geniv = "<built-in>", 4029 .min_keysize = AES_MIN_KEY_SIZE + 4030 CTR_RFC3686_NONCE_SIZE, 4031 .max_keysize = AES_MAX_KEY_SIZE + 4032 CTR_RFC3686_NONCE_SIZE, 4033 .ivsize = CTR_RFC3686_IV_SIZE, 4034 }, 4035 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 4036 } 4037 }; 4038 4039 struct caam_alg_entry { 4040 int class1_alg_type; 4041 int class2_alg_type; 4042 int alg_op; 4043 }; 4044 4045 struct caam_aead_alg { 4046 struct aead_alg aead; 4047 struct caam_alg_entry caam; 4048 bool registered; 4049 }; 4050 4051 static struct caam_aead_alg driver_aeads[] = { 4052 { 4053 .aead = { 4054 .base = { 4055 .cra_name = "rfc4106(gcm(aes))", 4056 .cra_driver_name = "rfc4106-gcm-aes-caam", 4057 .cra_blocksize = 1, 4058 }, 4059 .setkey = rfc4106_setkey, 4060 .setauthsize = rfc4106_setauthsize, 4061 .encrypt = gcm_encrypt, 4062 .decrypt = gcm_decrypt, 4063 .ivsize = 8, 4064 .maxauthsize = AES_BLOCK_SIZE, 4065 }, 4066 .caam = { 4067 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 4068 }, 4069 }, 4070 { 4071 .aead = { 4072 .base = { 4073 .cra_name = "rfc4543(gcm(aes))", 4074 .cra_driver_name = "rfc4543-gcm-aes-caam", 4075 .cra_blocksize = 1, 4076 }, 4077 .setkey = rfc4543_setkey, 4078 .setauthsize = rfc4543_setauthsize, 4079 .encrypt = gcm_encrypt, 4080 .decrypt = gcm_decrypt, 4081 .ivsize = 8, 4082 .maxauthsize = AES_BLOCK_SIZE, 4083 }, 4084 .caam = { 4085 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 4086 }, 4087 }, 4088 /* Galois Counter Mode */ 4089 { 4090 .aead = { 4091 .base = { 4092 .cra_name = "gcm(aes)", 4093 .cra_driver_name = "gcm-aes-caam", 4094 .cra_blocksize = 1, 4095 }, 4096 .setkey = gcm_setkey, 4097 .setauthsize = gcm_setauthsize, 4098 .encrypt = gcm_encrypt, 4099 .decrypt = gcm_decrypt, 4100 .ivsize = 12, 4101 .maxauthsize = AES_BLOCK_SIZE, 4102 }, 4103 .caam = { 4104 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 4105 }, 4106 }, 4107 }; 4108 4109 struct caam_crypto_alg { 4110 struct crypto_alg crypto_alg; 4111 struct list_head entry; 4112 struct caam_alg_entry caam; 4113 }; 4114 4115 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) 4116 { 4117 ctx->jrdev = caam_jr_alloc(); 4118 if (IS_ERR(ctx->jrdev)) { 4119 pr_err("Job Ring Device allocation for transform failed\n"); 4120 return PTR_ERR(ctx->jrdev); 4121 } 4122 4123 /* copy descriptor header template value */ 4124 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 4125 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 4126 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op; 4127 4128 return 0; 4129 } 4130 4131 static int caam_cra_init(struct crypto_tfm *tfm) 4132 { 4133 struct crypto_alg *alg = tfm->__crt_alg; 4134 struct caam_crypto_alg *caam_alg = 4135 container_of(alg, struct caam_crypto_alg, crypto_alg); 4136 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 4137 4138 return caam_init_common(ctx, &caam_alg->caam); 4139 } 4140 4141 static int caam_aead_init(struct crypto_aead *tfm) 4142 { 4143 struct aead_alg *alg = crypto_aead_alg(tfm); 4144 struct caam_aead_alg *caam_alg = 4145 container_of(alg, struct caam_aead_alg, aead); 4146 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 4147 4148 return caam_init_common(ctx, &caam_alg->caam); 4149 } 4150 4151 static void caam_exit_common(struct caam_ctx *ctx) 4152 { 4153 if (ctx->sh_desc_enc_dma && 4154 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) 4155 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, 4156 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 4157 if (ctx->sh_desc_dec_dma && 4158 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) 4159 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, 4160 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); 4161 if (ctx->sh_desc_givenc_dma && 4162 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) 4163 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, 4164 desc_bytes(ctx->sh_desc_givenc), 4165 DMA_TO_DEVICE); 4166 if (ctx->key_dma && 4167 !dma_mapping_error(ctx->jrdev, ctx->key_dma)) 4168 dma_unmap_single(ctx->jrdev, ctx->key_dma, 4169 ctx->enckeylen + ctx->split_key_pad_len, 4170 DMA_TO_DEVICE); 4171 4172 caam_jr_free(ctx->jrdev); 4173 } 4174 4175 static void caam_cra_exit(struct crypto_tfm *tfm) 4176 { 4177 caam_exit_common(crypto_tfm_ctx(tfm)); 4178 } 4179 4180 static void caam_aead_exit(struct crypto_aead *tfm) 4181 { 4182 caam_exit_common(crypto_aead_ctx(tfm)); 4183 } 4184 4185 static void __exit caam_algapi_exit(void) 4186 { 4187 4188 struct caam_crypto_alg *t_alg, *n; 4189 int i; 4190 4191 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4192 struct caam_aead_alg *t_alg = driver_aeads + i; 4193 4194 if (t_alg->registered) 4195 crypto_unregister_aead(&t_alg->aead); 4196 } 4197 4198 if (!alg_list.next) 4199 return; 4200 4201 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 4202 crypto_unregister_alg(&t_alg->crypto_alg); 4203 list_del(&t_alg->entry); 4204 kfree(t_alg); 4205 } 4206 } 4207 4208 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 4209 *template) 4210 { 4211 struct caam_crypto_alg *t_alg; 4212 struct crypto_alg *alg; 4213 4214 t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL); 4215 if (!t_alg) { 4216 pr_err("failed to allocate t_alg\n"); 4217 return ERR_PTR(-ENOMEM); 4218 } 4219 4220 alg = &t_alg->crypto_alg; 4221 4222 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 4223 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4224 template->driver_name); 4225 alg->cra_module = THIS_MODULE; 4226 alg->cra_init = caam_cra_init; 4227 alg->cra_exit = caam_cra_exit; 4228 alg->cra_priority = CAAM_CRA_PRIORITY; 4229 alg->cra_blocksize = template->blocksize; 4230 alg->cra_alignmask = 0; 4231 alg->cra_ctxsize = sizeof(struct caam_ctx); 4232 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 4233 template->type; 4234 switch (template->type) { 4235 case CRYPTO_ALG_TYPE_GIVCIPHER: 4236 alg->cra_type = &crypto_givcipher_type; 4237 alg->cra_ablkcipher = template->template_ablkcipher; 4238 break; 4239 case CRYPTO_ALG_TYPE_ABLKCIPHER: 4240 alg->cra_type = &crypto_ablkcipher_type; 4241 alg->cra_ablkcipher = template->template_ablkcipher; 4242 break; 4243 case CRYPTO_ALG_TYPE_AEAD: 4244 alg->cra_type = &crypto_aead_type; 4245 alg->cra_aead = template->template_aead; 4246 break; 4247 } 4248 4249 t_alg->caam.class1_alg_type = template->class1_alg_type; 4250 t_alg->caam.class2_alg_type = template->class2_alg_type; 4251 t_alg->caam.alg_op = template->alg_op; 4252 4253 return t_alg; 4254 } 4255 4256 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 4257 { 4258 struct aead_alg *alg = &t_alg->aead; 4259 4260 alg->base.cra_module = THIS_MODULE; 4261 alg->base.cra_priority = CAAM_CRA_PRIORITY; 4262 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 4263 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 4264 4265 alg->init = caam_aead_init; 4266 alg->exit = caam_aead_exit; 4267 } 4268 4269 static int __init caam_algapi_init(void) 4270 { 4271 struct device_node *dev_node; 4272 struct platform_device *pdev; 4273 struct device *ctrldev; 4274 void *priv; 4275 int i = 0, err = 0; 4276 bool registered = false; 4277 4278 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 4279 if (!dev_node) { 4280 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 4281 if (!dev_node) 4282 return -ENODEV; 4283 } 4284 4285 pdev = of_find_device_by_node(dev_node); 4286 if (!pdev) { 4287 of_node_put(dev_node); 4288 return -ENODEV; 4289 } 4290 4291 ctrldev = &pdev->dev; 4292 priv = dev_get_drvdata(ctrldev); 4293 of_node_put(dev_node); 4294 4295 /* 4296 * If priv is NULL, it's probably because the caam driver wasn't 4297 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 4298 */ 4299 if (!priv) 4300 return -ENODEV; 4301 4302 4303 INIT_LIST_HEAD(&alg_list); 4304 4305 /* register crypto algorithms the device supports */ 4306 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4307 /* TODO: check if h/w supports alg */ 4308 struct caam_crypto_alg *t_alg; 4309 4310 t_alg = caam_alg_alloc(&driver_algs[i]); 4311 if (IS_ERR(t_alg)) { 4312 err = PTR_ERR(t_alg); 4313 pr_warn("%s alg allocation failed\n", 4314 driver_algs[i].driver_name); 4315 continue; 4316 } 4317 4318 err = crypto_register_alg(&t_alg->crypto_alg); 4319 if (err) { 4320 pr_warn("%s alg registration failed\n", 4321 t_alg->crypto_alg.cra_driver_name); 4322 kfree(t_alg); 4323 continue; 4324 } 4325 4326 list_add_tail(&t_alg->entry, &alg_list); 4327 registered = true; 4328 } 4329 4330 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4331 struct caam_aead_alg *t_alg = driver_aeads + i; 4332 4333 caam_aead_alg_init(t_alg); 4334 4335 err = crypto_register_aead(&t_alg->aead); 4336 if (err) { 4337 pr_warn("%s alg registration failed\n", 4338 t_alg->aead.base.cra_driver_name); 4339 continue; 4340 } 4341 4342 t_alg->registered = true; 4343 registered = true; 4344 } 4345 4346 if (registered) 4347 pr_info("caam algorithms registered in /proc/crypto\n"); 4348 4349 return err; 4350 } 4351 4352 module_init(caam_algapi_init); 4353 module_exit(caam_algapi_exit); 4354 4355 MODULE_LICENSE("GPL"); 4356 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 4357 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 4358