1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * 6 * Based on talitos crypto API driver. 7 * 8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 9 * 10 * --------------- --------------- 11 * | JobDesc #1 |-------------------->| ShareDesc | 12 * | *(packet 1) | | (PDB) | 13 * --------------- |------------->| (hashKey) | 14 * . | | (cipherKey) | 15 * . | |-------->| (operation) | 16 * --------------- | | --------------- 17 * | JobDesc #2 |------| | 18 * | *(packet 2) | | 19 * --------------- | 20 * . | 21 * . | 22 * --------------- | 23 * | JobDesc #3 |------------ 24 * | *(packet 3) | 25 * --------------- 26 * 27 * The SharedDesc never changes for a connection unless rekeyed, but 28 * each packet will likely be in a different place. So all we need 29 * to know to process the packet is where the input is, where the 30 * output goes, and what context we want to process with. Context is 31 * in the SharedDesc, packet references in the JobDesc. 32 * 33 * So, a job desc looks like: 34 * 35 * --------------------- 36 * | Header | 37 * | ShareDesc Pointer | 38 * | SEQ_OUT_PTR | 39 * | (output buffer) | 40 * | (output length) | 41 * | SEQ_IN_PTR | 42 * | (input buffer) | 43 * | (input length) | 44 * --------------------- 45 */ 46 47 #include "compat.h" 48 49 #include "regs.h" 50 #include "intern.h" 51 #include "desc_constr.h" 52 #include "jr.h" 53 #include "error.h" 54 #include "sg_sw_sec4.h" 55 #include "key_gen.h" 56 57 /* 58 * crypto alg 59 */ 60 #define CAAM_CRA_PRIORITY 3000 61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 62 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 63 SHA512_DIGEST_SIZE * 2) 64 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 65 #define CAAM_MAX_IV_LENGTH 16 66 67 /* length of descriptors text */ 68 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) 69 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ) 70 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ) 71 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) 72 73 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) 74 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 75 20 * CAAM_CMD_SZ) 76 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 77 15 * CAAM_CMD_SZ) 78 79 #define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \ 80 CAAM_MAX_KEY_SIZE) 81 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 82 83 #ifdef DEBUG 84 /* for print_hex_dumps with line references */ 85 #define debug(format, arg...) printk(format, arg) 86 #else 87 #define debug(format, arg...) 88 #endif 89 static struct list_head alg_list; 90 91 /* Set DK bit in class 1 operation if shared */ 92 static inline void append_dec_op1(u32 *desc, u32 type) 93 { 94 u32 *jump_cmd, *uncond_jump_cmd; 95 96 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 97 append_operation(desc, type | OP_ALG_AS_INITFINAL | 98 OP_ALG_DECRYPT); 99 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 100 set_jump_tgt_here(desc, jump_cmd); 101 append_operation(desc, type | OP_ALG_AS_INITFINAL | 102 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 103 set_jump_tgt_here(desc, uncond_jump_cmd); 104 } 105 106 /* 107 * Wait for completion of class 1 key loading before allowing 108 * error propagation 109 */ 110 static inline void append_dec_shr_done(u32 *desc) 111 { 112 u32 *jump_cmd; 113 114 jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL); 115 set_jump_tgt_here(desc, jump_cmd); 116 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 117 } 118 119 /* 120 * For aead functions, read payload and write payload, 121 * both of which are specified in req->src and req->dst 122 */ 123 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 124 { 125 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 126 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 127 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 128 } 129 130 /* 131 * For aead encrypt and decrypt, read iv for both classes 132 */ 133 static inline void aead_append_ld_iv(u32 *desc, int ivsize) 134 { 135 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 136 LDST_CLASS_1_CCB | ivsize); 137 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); 138 } 139 140 /* 141 * For ablkcipher encrypt and decrypt, read from req->src and 142 * write to req->dst 143 */ 144 static inline void ablkcipher_append_src_dst(u32 *desc) 145 { 146 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 147 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 148 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 149 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 150 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 151 } 152 153 /* 154 * If all data, including src (with assoc and iv) or dst (with iv only) are 155 * contiguous 156 */ 157 #define GIV_SRC_CONTIG 1 158 #define GIV_DST_CONTIG (1 << 1) 159 160 /* 161 * per-session context 162 */ 163 struct caam_ctx { 164 struct device *jrdev; 165 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 166 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 167 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 168 dma_addr_t sh_desc_enc_dma; 169 dma_addr_t sh_desc_dec_dma; 170 dma_addr_t sh_desc_givenc_dma; 171 u32 class1_alg_type; 172 u32 class2_alg_type; 173 u32 alg_op; 174 u8 key[CAAM_MAX_KEY_SIZE]; 175 dma_addr_t key_dma; 176 unsigned int enckeylen; 177 unsigned int split_key_len; 178 unsigned int split_key_pad_len; 179 unsigned int authsize; 180 }; 181 182 static void append_key_aead(u32 *desc, struct caam_ctx *ctx, 183 int keys_fit_inline) 184 { 185 if (keys_fit_inline) { 186 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 187 ctx->split_key_len, CLASS_2 | 188 KEY_DEST_MDHA_SPLIT | KEY_ENC); 189 append_key_as_imm(desc, (void *)ctx->key + 190 ctx->split_key_pad_len, ctx->enckeylen, 191 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 192 } else { 193 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 194 KEY_DEST_MDHA_SPLIT | KEY_ENC); 195 append_key(desc, ctx->key_dma + ctx->split_key_pad_len, 196 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 197 } 198 } 199 200 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, 201 int keys_fit_inline) 202 { 203 u32 *key_jump_cmd; 204 205 init_sh_desc(desc, HDR_SHARE_SERIAL); 206 207 /* Skip if already shared */ 208 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 209 JUMP_COND_SHRD); 210 211 append_key_aead(desc, ctx, keys_fit_inline); 212 213 set_jump_tgt_here(desc, key_jump_cmd); 214 215 /* Propagate errors from shared to job descriptor */ 216 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 217 } 218 219 static int aead_set_sh_desc(struct crypto_aead *aead) 220 { 221 struct aead_tfm *tfm = &aead->base.crt_aead; 222 struct caam_ctx *ctx = crypto_aead_ctx(aead); 223 struct device *jrdev = ctx->jrdev; 224 bool keys_fit_inline = false; 225 u32 *key_jump_cmd, *jump_cmd; 226 u32 geniv, moveiv; 227 u32 *desc; 228 229 if (!ctx->enckeylen || !ctx->authsize) 230 return 0; 231 232 /* 233 * Job Descriptor and Shared Descriptors 234 * must all fit into the 64-word Descriptor h/w Buffer 235 */ 236 if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + 237 ctx->split_key_pad_len + ctx->enckeylen <= 238 CAAM_DESC_BYTES_MAX) 239 keys_fit_inline = true; 240 241 /* aead_encrypt shared descriptor */ 242 desc = ctx->sh_desc_enc; 243 244 init_sh_desc_key_aead(desc, ctx, keys_fit_inline); 245 246 /* Class 2 operation */ 247 append_operation(desc, ctx->class2_alg_type | 248 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 249 250 /* cryptlen = seqoutlen - authsize */ 251 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 252 253 /* assoclen + cryptlen = seqinlen - ivsize */ 254 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); 255 256 /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */ 257 append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); 258 259 /* read assoc before reading payload */ 260 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 261 KEY_VLF); 262 aead_append_ld_iv(desc, tfm->ivsize); 263 264 /* Class 1 operation */ 265 append_operation(desc, ctx->class1_alg_type | 266 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 267 268 /* Read and write cryptlen bytes */ 269 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 270 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 271 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 272 273 /* Write ICV */ 274 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 275 LDST_SRCDST_BYTE_CONTEXT); 276 277 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 278 desc_bytes(desc), 279 DMA_TO_DEVICE); 280 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 281 dev_err(jrdev, "unable to map shared descriptor\n"); 282 return -ENOMEM; 283 } 284 #ifdef DEBUG 285 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", 286 DUMP_PREFIX_ADDRESS, 16, 4, desc, 287 desc_bytes(desc), 1); 288 #endif 289 290 /* 291 * Job Descriptor and Shared Descriptors 292 * must all fit into the 64-word Descriptor h/w Buffer 293 */ 294 if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + 295 ctx->split_key_pad_len + ctx->enckeylen <= 296 CAAM_DESC_BYTES_MAX) 297 keys_fit_inline = true; 298 299 desc = ctx->sh_desc_dec; 300 301 /* aead_decrypt shared descriptor */ 302 init_sh_desc(desc, HDR_SHARE_SERIAL); 303 304 /* Skip if already shared */ 305 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 306 JUMP_COND_SHRD); 307 308 append_key_aead(desc, ctx, keys_fit_inline); 309 310 /* Only propagate error immediately if shared */ 311 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 312 set_jump_tgt_here(desc, key_jump_cmd); 313 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 314 set_jump_tgt_here(desc, jump_cmd); 315 316 /* Class 2 operation */ 317 append_operation(desc, ctx->class2_alg_type | 318 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 319 320 /* assoclen + cryptlen = seqinlen - ivsize */ 321 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, 322 ctx->authsize + tfm->ivsize) 323 /* assoclen = (assoclen + cryptlen) - cryptlen */ 324 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 325 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); 326 327 /* read assoc before reading payload */ 328 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 329 KEY_VLF); 330 331 aead_append_ld_iv(desc, tfm->ivsize); 332 333 append_dec_op1(desc, ctx->class1_alg_type); 334 335 /* Read and write cryptlen bytes */ 336 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 337 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 338 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 339 340 /* Load ICV */ 341 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 342 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 343 append_dec_shr_done(desc); 344 345 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 346 desc_bytes(desc), 347 DMA_TO_DEVICE); 348 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 349 dev_err(jrdev, "unable to map shared descriptor\n"); 350 return -ENOMEM; 351 } 352 #ifdef DEBUG 353 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", 354 DUMP_PREFIX_ADDRESS, 16, 4, desc, 355 desc_bytes(desc), 1); 356 #endif 357 358 /* 359 * Job Descriptor and Shared Descriptors 360 * must all fit into the 64-word Descriptor h/w Buffer 361 */ 362 if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + 363 ctx->split_key_pad_len + ctx->enckeylen <= 364 CAAM_DESC_BYTES_MAX) 365 keys_fit_inline = true; 366 367 /* aead_givencrypt shared descriptor */ 368 desc = ctx->sh_desc_givenc; 369 370 init_sh_desc_key_aead(desc, ctx, keys_fit_inline); 371 372 /* Generate IV */ 373 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 374 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 375 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); 376 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 377 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 378 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 379 append_move(desc, MOVE_SRC_INFIFO | 380 MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT)); 381 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 382 383 /* Copy IV to class 1 context */ 384 append_move(desc, MOVE_SRC_CLASS1CTX | 385 MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT)); 386 387 /* Return to encryption */ 388 append_operation(desc, ctx->class2_alg_type | 389 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 390 391 /* ivsize + cryptlen = seqoutlen - authsize */ 392 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 393 394 /* assoclen = seqinlen - (ivsize + cryptlen) */ 395 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 396 397 /* read assoc before reading payload */ 398 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 399 KEY_VLF); 400 401 /* Copy iv from class 1 ctx to class 2 fifo*/ 402 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 403 NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); 404 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 405 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 406 append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB | 407 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 408 409 /* Class 1 operation */ 410 append_operation(desc, ctx->class1_alg_type | 411 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 412 413 /* Will write ivsize + cryptlen */ 414 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 415 416 /* Not need to reload iv */ 417 append_seq_fifo_load(desc, tfm->ivsize, 418 FIFOLD_CLASS_SKIP); 419 420 /* Will read cryptlen */ 421 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 422 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 423 424 /* Write ICV */ 425 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 426 LDST_SRCDST_BYTE_CONTEXT); 427 428 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 429 desc_bytes(desc), 430 DMA_TO_DEVICE); 431 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 432 dev_err(jrdev, "unable to map shared descriptor\n"); 433 return -ENOMEM; 434 } 435 #ifdef DEBUG 436 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", 437 DUMP_PREFIX_ADDRESS, 16, 4, desc, 438 desc_bytes(desc), 1); 439 #endif 440 441 return 0; 442 } 443 444 static int aead_setauthsize(struct crypto_aead *authenc, 445 unsigned int authsize) 446 { 447 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 448 449 ctx->authsize = authsize; 450 aead_set_sh_desc(authenc); 451 452 return 0; 453 } 454 455 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, 456 u32 authkeylen) 457 { 458 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, 459 ctx->split_key_pad_len, key_in, authkeylen, 460 ctx->alg_op); 461 } 462 463 static int aead_setkey(struct crypto_aead *aead, 464 const u8 *key, unsigned int keylen) 465 { 466 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 467 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 468 struct caam_ctx *ctx = crypto_aead_ctx(aead); 469 struct device *jrdev = ctx->jrdev; 470 struct rtattr *rta = (void *)key; 471 struct crypto_authenc_key_param *param; 472 unsigned int authkeylen; 473 unsigned int enckeylen; 474 int ret = 0; 475 476 param = RTA_DATA(rta); 477 enckeylen = be32_to_cpu(param->enckeylen); 478 479 key += RTA_ALIGN(rta->rta_len); 480 keylen -= RTA_ALIGN(rta->rta_len); 481 482 if (keylen < enckeylen) 483 goto badkey; 484 485 authkeylen = keylen - enckeylen; 486 487 if (keylen > CAAM_MAX_KEY_SIZE) 488 goto badkey; 489 490 /* Pick class 2 key length from algorithm submask */ 491 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 492 OP_ALG_ALGSEL_SHIFT] * 2; 493 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 494 495 #ifdef DEBUG 496 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 497 keylen, enckeylen, authkeylen); 498 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 499 ctx->split_key_len, ctx->split_key_pad_len); 500 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 501 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 502 #endif 503 504 ret = gen_split_aead_key(ctx, key, authkeylen); 505 if (ret) { 506 goto badkey; 507 } 508 509 /* postpend encryption key to auth split key */ 510 memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen); 511 512 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 513 enckeylen, DMA_TO_DEVICE); 514 if (dma_mapping_error(jrdev, ctx->key_dma)) { 515 dev_err(jrdev, "unable to map key i/o memory\n"); 516 return -ENOMEM; 517 } 518 #ifdef DEBUG 519 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 520 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 521 ctx->split_key_pad_len + enckeylen, 1); 522 #endif 523 524 ctx->enckeylen = enckeylen; 525 526 ret = aead_set_sh_desc(aead); 527 if (ret) { 528 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + 529 enckeylen, DMA_TO_DEVICE); 530 } 531 532 return ret; 533 badkey: 534 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 535 return -EINVAL; 536 } 537 538 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 539 const u8 *key, unsigned int keylen) 540 { 541 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 542 struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher; 543 struct device *jrdev = ctx->jrdev; 544 int ret = 0; 545 u32 *key_jump_cmd, *jump_cmd; 546 u32 *desc; 547 548 #ifdef DEBUG 549 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 550 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 551 #endif 552 553 memcpy(ctx->key, key, keylen); 554 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 555 DMA_TO_DEVICE); 556 if (dma_mapping_error(jrdev, ctx->key_dma)) { 557 dev_err(jrdev, "unable to map key i/o memory\n"); 558 return -ENOMEM; 559 } 560 ctx->enckeylen = keylen; 561 562 /* ablkcipher_encrypt shared descriptor */ 563 desc = ctx->sh_desc_enc; 564 init_sh_desc(desc, HDR_SHARE_SERIAL); 565 /* Skip if already shared */ 566 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 567 JUMP_COND_SHRD); 568 569 /* Load class1 key only */ 570 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 571 ctx->enckeylen, CLASS_1 | 572 KEY_DEST_CLASS_REG); 573 574 set_jump_tgt_here(desc, key_jump_cmd); 575 576 /* Propagate errors from shared to job descriptor */ 577 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 578 579 /* Load iv */ 580 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 581 LDST_CLASS_1_CCB | tfm->ivsize); 582 583 /* Load operation */ 584 append_operation(desc, ctx->class1_alg_type | 585 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 586 587 /* Perform operation */ 588 ablkcipher_append_src_dst(desc); 589 590 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 591 desc_bytes(desc), 592 DMA_TO_DEVICE); 593 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 594 dev_err(jrdev, "unable to map shared descriptor\n"); 595 return -ENOMEM; 596 } 597 #ifdef DEBUG 598 print_hex_dump(KERN_ERR, 599 "ablkcipher enc shdesc@"__stringify(__LINE__)": ", 600 DUMP_PREFIX_ADDRESS, 16, 4, desc, 601 desc_bytes(desc), 1); 602 #endif 603 /* ablkcipher_decrypt shared descriptor */ 604 desc = ctx->sh_desc_dec; 605 606 init_sh_desc(desc, HDR_SHARE_SERIAL); 607 /* Skip if already shared */ 608 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 609 JUMP_COND_SHRD); 610 611 /* Load class1 key only */ 612 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 613 ctx->enckeylen, CLASS_1 | 614 KEY_DEST_CLASS_REG); 615 616 /* For aead, only propagate error immediately if shared */ 617 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 618 set_jump_tgt_here(desc, key_jump_cmd); 619 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 620 set_jump_tgt_here(desc, jump_cmd); 621 622 /* load IV */ 623 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 624 LDST_CLASS_1_CCB | tfm->ivsize); 625 626 /* Choose operation */ 627 append_dec_op1(desc, ctx->class1_alg_type); 628 629 /* Perform operation */ 630 ablkcipher_append_src_dst(desc); 631 632 /* Wait for key to load before allowing propagating error */ 633 append_dec_shr_done(desc); 634 635 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 636 desc_bytes(desc), 637 DMA_TO_DEVICE); 638 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 639 dev_err(jrdev, "unable to map shared descriptor\n"); 640 return -ENOMEM; 641 } 642 643 #ifdef DEBUG 644 print_hex_dump(KERN_ERR, 645 "ablkcipher dec shdesc@"__stringify(__LINE__)": ", 646 DUMP_PREFIX_ADDRESS, 16, 4, desc, 647 desc_bytes(desc), 1); 648 #endif 649 650 return ret; 651 } 652 653 /* 654 * aead_edesc - s/w-extended aead descriptor 655 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 656 * @assoc_chained: if source is chained 657 * @src_nents: number of segments in input scatterlist 658 * @src_chained: if source is chained 659 * @dst_nents: number of segments in output scatterlist 660 * @dst_chained: if destination is chained 661 * @iv_dma: dma address of iv for checking continuity and link table 662 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 663 * @sec4_sg_bytes: length of dma mapped sec4_sg space 664 * @sec4_sg_dma: bus physical mapped address of h/w link table 665 * @hw_desc: the h/w job descriptor followed by any referenced link tables 666 */ 667 struct aead_edesc { 668 int assoc_nents; 669 bool assoc_chained; 670 int src_nents; 671 bool src_chained; 672 int dst_nents; 673 bool dst_chained; 674 dma_addr_t iv_dma; 675 int sec4_sg_bytes; 676 dma_addr_t sec4_sg_dma; 677 struct sec4_sg_entry *sec4_sg; 678 u32 hw_desc[0]; 679 }; 680 681 /* 682 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 683 * @src_nents: number of segments in input scatterlist 684 * @src_chained: if source is chained 685 * @dst_nents: number of segments in output scatterlist 686 * @dst_chained: if destination is chained 687 * @iv_dma: dma address of iv for checking continuity and link table 688 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 689 * @sec4_sg_bytes: length of dma mapped sec4_sg space 690 * @sec4_sg_dma: bus physical mapped address of h/w link table 691 * @hw_desc: the h/w job descriptor followed by any referenced link tables 692 */ 693 struct ablkcipher_edesc { 694 int src_nents; 695 bool src_chained; 696 int dst_nents; 697 bool dst_chained; 698 dma_addr_t iv_dma; 699 int sec4_sg_bytes; 700 dma_addr_t sec4_sg_dma; 701 struct sec4_sg_entry *sec4_sg; 702 u32 hw_desc[0]; 703 }; 704 705 static void caam_unmap(struct device *dev, struct scatterlist *src, 706 struct scatterlist *dst, int src_nents, 707 bool src_chained, int dst_nents, bool dst_chained, 708 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 709 int sec4_sg_bytes) 710 { 711 if (dst != src) { 712 dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE, 713 src_chained); 714 dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE, 715 dst_chained); 716 } else { 717 dma_unmap_sg_chained(dev, src, src_nents ? : 1, 718 DMA_BIDIRECTIONAL, src_chained); 719 } 720 721 if (iv_dma) 722 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 723 if (sec4_sg_bytes) 724 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 725 DMA_TO_DEVICE); 726 } 727 728 static void aead_unmap(struct device *dev, 729 struct aead_edesc *edesc, 730 struct aead_request *req) 731 { 732 struct crypto_aead *aead = crypto_aead_reqtfm(req); 733 int ivsize = crypto_aead_ivsize(aead); 734 735 dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents, 736 DMA_TO_DEVICE, edesc->assoc_chained); 737 738 caam_unmap(dev, req->src, req->dst, 739 edesc->src_nents, edesc->src_chained, edesc->dst_nents, 740 edesc->dst_chained, edesc->iv_dma, ivsize, 741 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 742 } 743 744 static void ablkcipher_unmap(struct device *dev, 745 struct ablkcipher_edesc *edesc, 746 struct ablkcipher_request *req) 747 { 748 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 749 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 750 751 caam_unmap(dev, req->src, req->dst, 752 edesc->src_nents, edesc->src_chained, edesc->dst_nents, 753 edesc->dst_chained, edesc->iv_dma, ivsize, 754 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 755 } 756 757 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 758 void *context) 759 { 760 struct aead_request *req = context; 761 struct aead_edesc *edesc; 762 #ifdef DEBUG 763 struct crypto_aead *aead = crypto_aead_reqtfm(req); 764 struct caam_ctx *ctx = crypto_aead_ctx(aead); 765 int ivsize = crypto_aead_ivsize(aead); 766 767 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 768 #endif 769 770 edesc = (struct aead_edesc *)((char *)desc - 771 offsetof(struct aead_edesc, hw_desc)); 772 773 if (err) { 774 char tmp[CAAM_ERROR_STR_MAX]; 775 776 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 777 } 778 779 aead_unmap(jrdev, edesc, req); 780 781 #ifdef DEBUG 782 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", 783 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 784 req->assoclen , 1); 785 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 786 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, 787 edesc->src_nents ? 100 : ivsize, 1); 788 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 789 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 790 edesc->src_nents ? 100 : req->cryptlen + 791 ctx->authsize + 4, 1); 792 #endif 793 794 kfree(edesc); 795 796 aead_request_complete(req, err); 797 } 798 799 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 800 void *context) 801 { 802 struct aead_request *req = context; 803 struct aead_edesc *edesc; 804 #ifdef DEBUG 805 struct crypto_aead *aead = crypto_aead_reqtfm(req); 806 struct caam_ctx *ctx = crypto_aead_ctx(aead); 807 int ivsize = crypto_aead_ivsize(aead); 808 809 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 810 #endif 811 812 edesc = (struct aead_edesc *)((char *)desc - 813 offsetof(struct aead_edesc, hw_desc)); 814 815 #ifdef DEBUG 816 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 817 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 818 ivsize, 1); 819 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 820 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst), 821 req->cryptlen, 1); 822 #endif 823 824 if (err) { 825 char tmp[CAAM_ERROR_STR_MAX]; 826 827 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 828 } 829 830 aead_unmap(jrdev, edesc, req); 831 832 /* 833 * verify hw auth check passed else return -EBADMSG 834 */ 835 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 836 err = -EBADMSG; 837 838 #ifdef DEBUG 839 print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ", 840 DUMP_PREFIX_ADDRESS, 16, 4, 841 ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)), 842 sizeof(struct iphdr) + req->assoclen + 843 ((req->cryptlen > 1500) ? 1500 : req->cryptlen) + 844 ctx->authsize + 36, 1); 845 if (!err && edesc->sec4_sg_bytes) { 846 struct scatterlist *sg = sg_last(req->src, edesc->src_nents); 847 print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ", 848 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), 849 sg->length + ctx->authsize + 16, 1); 850 } 851 #endif 852 853 kfree(edesc); 854 855 aead_request_complete(req, err); 856 } 857 858 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 859 void *context) 860 { 861 struct ablkcipher_request *req = context; 862 struct ablkcipher_edesc *edesc; 863 #ifdef DEBUG 864 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 865 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 866 867 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 868 #endif 869 870 edesc = (struct ablkcipher_edesc *)((char *)desc - 871 offsetof(struct ablkcipher_edesc, hw_desc)); 872 873 if (err) { 874 char tmp[CAAM_ERROR_STR_MAX]; 875 876 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 877 } 878 879 #ifdef DEBUG 880 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 881 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 882 edesc->src_nents > 1 ? 100 : ivsize, 1); 883 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 884 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 885 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 886 #endif 887 888 ablkcipher_unmap(jrdev, edesc, req); 889 kfree(edesc); 890 891 ablkcipher_request_complete(req, err); 892 } 893 894 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 895 void *context) 896 { 897 struct ablkcipher_request *req = context; 898 struct ablkcipher_edesc *edesc; 899 #ifdef DEBUG 900 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 901 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 902 903 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 904 #endif 905 906 edesc = (struct ablkcipher_edesc *)((char *)desc - 907 offsetof(struct ablkcipher_edesc, hw_desc)); 908 if (err) { 909 char tmp[CAAM_ERROR_STR_MAX]; 910 911 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 912 } 913 914 #ifdef DEBUG 915 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 916 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 917 ivsize, 1); 918 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 919 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 920 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 921 #endif 922 923 ablkcipher_unmap(jrdev, edesc, req); 924 kfree(edesc); 925 926 ablkcipher_request_complete(req, err); 927 } 928 929 /* 930 * Fill in aead job descriptor 931 */ 932 static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, 933 struct aead_edesc *edesc, 934 struct aead_request *req, 935 bool all_contig, bool encrypt) 936 { 937 struct crypto_aead *aead = crypto_aead_reqtfm(req); 938 struct caam_ctx *ctx = crypto_aead_ctx(aead); 939 int ivsize = crypto_aead_ivsize(aead); 940 int authsize = ctx->authsize; 941 u32 *desc = edesc->hw_desc; 942 u32 out_options = 0, in_options; 943 dma_addr_t dst_dma, src_dma; 944 int len, sec4_sg_index = 0; 945 946 #ifdef DEBUG 947 debug("assoclen %d cryptlen %d authsize %d\n", 948 req->assoclen, req->cryptlen, authsize); 949 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", 950 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 951 req->assoclen , 1); 952 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 953 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 954 edesc->src_nents ? 100 : ivsize, 1); 955 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 956 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 957 edesc->src_nents ? 100 : req->cryptlen, 1); 958 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", 959 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 960 desc_bytes(sh_desc), 1); 961 #endif 962 963 len = desc_len(sh_desc); 964 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 965 966 if (all_contig) { 967 src_dma = sg_dma_address(req->assoc); 968 in_options = 0; 969 } else { 970 src_dma = edesc->sec4_sg_dma; 971 sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 + 972 (edesc->src_nents ? : 1); 973 in_options = LDST_SGF; 974 } 975 if (encrypt) 976 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + 977 req->cryptlen - authsize, in_options); 978 else 979 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + 980 req->cryptlen, in_options); 981 982 if (likely(req->src == req->dst)) { 983 if (all_contig) { 984 dst_dma = sg_dma_address(req->src); 985 } else { 986 dst_dma = src_dma + sizeof(struct sec4_sg_entry) * 987 ((edesc->assoc_nents ? : 1) + 1); 988 out_options = LDST_SGF; 989 } 990 } else { 991 if (!edesc->dst_nents) { 992 dst_dma = sg_dma_address(req->dst); 993 } else { 994 dst_dma = edesc->sec4_sg_dma + 995 sec4_sg_index * 996 sizeof(struct sec4_sg_entry); 997 out_options = LDST_SGF; 998 } 999 } 1000 if (encrypt) 1001 append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options); 1002 else 1003 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize, 1004 out_options); 1005 } 1006 1007 /* 1008 * Fill in aead givencrypt job descriptor 1009 */ 1010 static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, 1011 struct aead_edesc *edesc, 1012 struct aead_request *req, 1013 int contig) 1014 { 1015 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1016 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1017 int ivsize = crypto_aead_ivsize(aead); 1018 int authsize = ctx->authsize; 1019 u32 *desc = edesc->hw_desc; 1020 u32 out_options = 0, in_options; 1021 dma_addr_t dst_dma, src_dma; 1022 int len, sec4_sg_index = 0; 1023 1024 #ifdef DEBUG 1025 debug("assoclen %d cryptlen %d authsize %d\n", 1026 req->assoclen, req->cryptlen, authsize); 1027 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", 1028 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 1029 req->assoclen , 1); 1030 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 1031 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); 1032 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 1033 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1034 edesc->src_nents > 1 ? 100 : req->cryptlen, 1); 1035 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", 1036 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 1037 desc_bytes(sh_desc), 1); 1038 #endif 1039 1040 len = desc_len(sh_desc); 1041 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1042 1043 if (contig & GIV_SRC_CONTIG) { 1044 src_dma = sg_dma_address(req->assoc); 1045 in_options = 0; 1046 } else { 1047 src_dma = edesc->sec4_sg_dma; 1048 sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents; 1049 in_options = LDST_SGF; 1050 } 1051 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + 1052 req->cryptlen - authsize, in_options); 1053 1054 if (contig & GIV_DST_CONTIG) { 1055 dst_dma = edesc->iv_dma; 1056 } else { 1057 if (likely(req->src == req->dst)) { 1058 dst_dma = src_dma + sizeof(struct sec4_sg_entry) * 1059 edesc->assoc_nents; 1060 out_options = LDST_SGF; 1061 } else { 1062 dst_dma = edesc->sec4_sg_dma + 1063 sec4_sg_index * 1064 sizeof(struct sec4_sg_entry); 1065 out_options = LDST_SGF; 1066 } 1067 } 1068 1069 append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options); 1070 } 1071 1072 /* 1073 * Fill in ablkcipher job descriptor 1074 */ 1075 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 1076 struct ablkcipher_edesc *edesc, 1077 struct ablkcipher_request *req, 1078 bool iv_contig) 1079 { 1080 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1081 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1082 u32 *desc = edesc->hw_desc; 1083 u32 out_options = 0, in_options; 1084 dma_addr_t dst_dma, src_dma; 1085 int len, sec4_sg_index = 0; 1086 1087 #ifdef DEBUG 1088 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 1089 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1090 ivsize, 1); 1091 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 1092 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1093 edesc->src_nents ? 100 : req->nbytes, 1); 1094 #endif 1095 1096 len = desc_len(sh_desc); 1097 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1098 1099 if (iv_contig) { 1100 src_dma = edesc->iv_dma; 1101 in_options = 0; 1102 } else { 1103 src_dma = edesc->sec4_sg_dma; 1104 sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents; 1105 in_options = LDST_SGF; 1106 } 1107 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 1108 1109 if (likely(req->src == req->dst)) { 1110 if (!edesc->src_nents && iv_contig) { 1111 dst_dma = sg_dma_address(req->src); 1112 } else { 1113 dst_dma = edesc->sec4_sg_dma + 1114 sizeof(struct sec4_sg_entry); 1115 out_options = LDST_SGF; 1116 } 1117 } else { 1118 if (!edesc->dst_nents) { 1119 dst_dma = sg_dma_address(req->dst); 1120 } else { 1121 dst_dma = edesc->sec4_sg_dma + 1122 sec4_sg_index * sizeof(struct sec4_sg_entry); 1123 out_options = LDST_SGF; 1124 } 1125 } 1126 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 1127 } 1128 1129 /* 1130 * allocate and map the aead extended descriptor 1131 */ 1132 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 1133 int desc_bytes, bool *all_contig_ptr) 1134 { 1135 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1136 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1137 struct device *jrdev = ctx->jrdev; 1138 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1139 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1140 int assoc_nents, src_nents, dst_nents = 0; 1141 struct aead_edesc *edesc; 1142 dma_addr_t iv_dma = 0; 1143 int sgc; 1144 bool all_contig = true; 1145 bool assoc_chained = false, src_chained = false, dst_chained = false; 1146 int ivsize = crypto_aead_ivsize(aead); 1147 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 1148 1149 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); 1150 src_nents = sg_count(req->src, req->cryptlen, &src_chained); 1151 1152 if (unlikely(req->dst != req->src)) 1153 dst_nents = sg_count(req->dst, req->cryptlen, &dst_chained); 1154 1155 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, 1156 DMA_TO_DEVICE, assoc_chained); 1157 if (likely(req->src == req->dst)) { 1158 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1159 DMA_BIDIRECTIONAL, src_chained); 1160 } else { 1161 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1162 DMA_TO_DEVICE, src_chained); 1163 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 1164 DMA_FROM_DEVICE, dst_chained); 1165 } 1166 1167 /* Check if data are contiguous */ 1168 iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); 1169 if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != 1170 iv_dma || src_nents || iv_dma + ivsize != 1171 sg_dma_address(req->src)) { 1172 all_contig = false; 1173 assoc_nents = assoc_nents ? : 1; 1174 src_nents = src_nents ? : 1; 1175 sec4_sg_len = assoc_nents + 1 + src_nents; 1176 } 1177 sec4_sg_len += dst_nents; 1178 1179 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1180 1181 /* allocate space for base edesc and hw desc commands, link tables */ 1182 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + 1183 sec4_sg_bytes, GFP_DMA | flags); 1184 if (!edesc) { 1185 dev_err(jrdev, "could not allocate extended descriptor\n"); 1186 return ERR_PTR(-ENOMEM); 1187 } 1188 1189 edesc->assoc_nents = assoc_nents; 1190 edesc->assoc_chained = assoc_chained; 1191 edesc->src_nents = src_nents; 1192 edesc->src_chained = src_chained; 1193 edesc->dst_nents = dst_nents; 1194 edesc->dst_chained = dst_chained; 1195 edesc->iv_dma = iv_dma; 1196 edesc->sec4_sg_bytes = sec4_sg_bytes; 1197 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1198 desc_bytes; 1199 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1200 sec4_sg_bytes, DMA_TO_DEVICE); 1201 *all_contig_ptr = all_contig; 1202 1203 sec4_sg_index = 0; 1204 if (!all_contig) { 1205 sg_to_sec4_sg(req->assoc, 1206 (assoc_nents ? : 1), 1207 edesc->sec4_sg + 1208 sec4_sg_index, 0); 1209 sec4_sg_index += assoc_nents ? : 1; 1210 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 1211 iv_dma, ivsize, 0); 1212 sec4_sg_index += 1; 1213 sg_to_sec4_sg_last(req->src, 1214 (src_nents ? : 1), 1215 edesc->sec4_sg + 1216 sec4_sg_index, 0); 1217 sec4_sg_index += src_nents ? : 1; 1218 } 1219 if (dst_nents) { 1220 sg_to_sec4_sg_last(req->dst, dst_nents, 1221 edesc->sec4_sg + sec4_sg_index, 0); 1222 } 1223 1224 return edesc; 1225 } 1226 1227 static int aead_encrypt(struct aead_request *req) 1228 { 1229 struct aead_edesc *edesc; 1230 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1231 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1232 struct device *jrdev = ctx->jrdev; 1233 bool all_contig; 1234 u32 *desc; 1235 int ret = 0; 1236 1237 req->cryptlen += ctx->authsize; 1238 1239 /* allocate extended descriptor */ 1240 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * 1241 CAAM_CMD_SZ, &all_contig); 1242 if (IS_ERR(edesc)) 1243 return PTR_ERR(edesc); 1244 1245 /* Create and submit job descriptor */ 1246 init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req, 1247 all_contig, true); 1248 #ifdef DEBUG 1249 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1250 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1251 desc_bytes(edesc->hw_desc), 1); 1252 #endif 1253 1254 desc = edesc->hw_desc; 1255 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1256 if (!ret) { 1257 ret = -EINPROGRESS; 1258 } else { 1259 aead_unmap(jrdev, edesc, req); 1260 kfree(edesc); 1261 } 1262 1263 return ret; 1264 } 1265 1266 static int aead_decrypt(struct aead_request *req) 1267 { 1268 struct aead_edesc *edesc; 1269 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1270 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1271 struct device *jrdev = ctx->jrdev; 1272 bool all_contig; 1273 u32 *desc; 1274 int ret = 0; 1275 1276 /* allocate extended descriptor */ 1277 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * 1278 CAAM_CMD_SZ, &all_contig); 1279 if (IS_ERR(edesc)) 1280 return PTR_ERR(edesc); 1281 1282 #ifdef DEBUG 1283 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", 1284 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1285 req->cryptlen, 1); 1286 #endif 1287 1288 /* Create and submit job descriptor*/ 1289 init_aead_job(ctx->sh_desc_dec, 1290 ctx->sh_desc_dec_dma, edesc, req, all_contig, false); 1291 #ifdef DEBUG 1292 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1293 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1294 desc_bytes(edesc->hw_desc), 1); 1295 #endif 1296 1297 desc = edesc->hw_desc; 1298 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 1299 if (!ret) { 1300 ret = -EINPROGRESS; 1301 } else { 1302 aead_unmap(jrdev, edesc, req); 1303 kfree(edesc); 1304 } 1305 1306 return ret; 1307 } 1308 1309 /* 1310 * allocate and map the aead extended descriptor for aead givencrypt 1311 */ 1312 static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request 1313 *greq, int desc_bytes, 1314 u32 *contig_ptr) 1315 { 1316 struct aead_request *req = &greq->areq; 1317 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1318 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1319 struct device *jrdev = ctx->jrdev; 1320 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1321 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1322 int assoc_nents, src_nents, dst_nents = 0; 1323 struct aead_edesc *edesc; 1324 dma_addr_t iv_dma = 0; 1325 int sgc; 1326 u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG; 1327 int ivsize = crypto_aead_ivsize(aead); 1328 bool assoc_chained = false, src_chained = false, dst_chained = false; 1329 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 1330 1331 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); 1332 src_nents = sg_count(req->src, req->cryptlen, &src_chained); 1333 1334 if (unlikely(req->dst != req->src)) 1335 dst_nents = sg_count(req->dst, req->cryptlen, &dst_chained); 1336 1337 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, 1338 DMA_TO_DEVICE, assoc_chained); 1339 if (likely(req->src == req->dst)) { 1340 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1341 DMA_BIDIRECTIONAL, src_chained); 1342 } else { 1343 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1344 DMA_TO_DEVICE, src_chained); 1345 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 1346 DMA_FROM_DEVICE, dst_chained); 1347 } 1348 1349 /* Check if data are contiguous */ 1350 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 1351 if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != 1352 iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src)) 1353 contig &= ~GIV_SRC_CONTIG; 1354 if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst)) 1355 contig &= ~GIV_DST_CONTIG; 1356 if (unlikely(req->src != req->dst)) { 1357 dst_nents = dst_nents ? : 1; 1358 sec4_sg_len += 1; 1359 } 1360 if (!(contig & GIV_SRC_CONTIG)) { 1361 assoc_nents = assoc_nents ? : 1; 1362 src_nents = src_nents ? : 1; 1363 sec4_sg_len += assoc_nents + 1 + src_nents; 1364 if (likely(req->src == req->dst)) 1365 contig &= ~GIV_DST_CONTIG; 1366 } 1367 sec4_sg_len += dst_nents; 1368 1369 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1370 1371 /* allocate space for base edesc and hw desc commands, link tables */ 1372 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + 1373 sec4_sg_bytes, GFP_DMA | flags); 1374 if (!edesc) { 1375 dev_err(jrdev, "could not allocate extended descriptor\n"); 1376 return ERR_PTR(-ENOMEM); 1377 } 1378 1379 edesc->assoc_nents = assoc_nents; 1380 edesc->assoc_chained = assoc_chained; 1381 edesc->src_nents = src_nents; 1382 edesc->src_chained = src_chained; 1383 edesc->dst_nents = dst_nents; 1384 edesc->dst_chained = dst_chained; 1385 edesc->iv_dma = iv_dma; 1386 edesc->sec4_sg_bytes = sec4_sg_bytes; 1387 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1388 desc_bytes; 1389 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1390 sec4_sg_bytes, DMA_TO_DEVICE); 1391 *contig_ptr = contig; 1392 1393 sec4_sg_index = 0; 1394 if (!(contig & GIV_SRC_CONTIG)) { 1395 sg_to_sec4_sg(req->assoc, assoc_nents, 1396 edesc->sec4_sg + 1397 sec4_sg_index, 0); 1398 sec4_sg_index += assoc_nents; 1399 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 1400 iv_dma, ivsize, 0); 1401 sec4_sg_index += 1; 1402 sg_to_sec4_sg_last(req->src, src_nents, 1403 edesc->sec4_sg + 1404 sec4_sg_index, 0); 1405 sec4_sg_index += src_nents; 1406 } 1407 if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) { 1408 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 1409 iv_dma, ivsize, 0); 1410 sec4_sg_index += 1; 1411 sg_to_sec4_sg_last(req->dst, dst_nents, 1412 edesc->sec4_sg + sec4_sg_index, 0); 1413 } 1414 1415 return edesc; 1416 } 1417 1418 static int aead_givencrypt(struct aead_givcrypt_request *areq) 1419 { 1420 struct aead_request *req = &areq->areq; 1421 struct aead_edesc *edesc; 1422 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1423 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1424 struct device *jrdev = ctx->jrdev; 1425 u32 contig; 1426 u32 *desc; 1427 int ret = 0; 1428 1429 req->cryptlen += ctx->authsize; 1430 1431 /* allocate extended descriptor */ 1432 edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN * 1433 CAAM_CMD_SZ, &contig); 1434 1435 if (IS_ERR(edesc)) 1436 return PTR_ERR(edesc); 1437 1438 #ifdef DEBUG 1439 print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ", 1440 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1441 req->cryptlen, 1); 1442 #endif 1443 1444 /* Create and submit job descriptor*/ 1445 init_aead_giv_job(ctx->sh_desc_givenc, 1446 ctx->sh_desc_givenc_dma, edesc, req, contig); 1447 #ifdef DEBUG 1448 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1449 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1450 desc_bytes(edesc->hw_desc), 1); 1451 #endif 1452 1453 desc = edesc->hw_desc; 1454 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1455 if (!ret) { 1456 ret = -EINPROGRESS; 1457 } else { 1458 aead_unmap(jrdev, edesc, req); 1459 kfree(edesc); 1460 } 1461 1462 return ret; 1463 } 1464 1465 /* 1466 * allocate and map the ablkcipher extended descriptor for ablkcipher 1467 */ 1468 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 1469 *req, int desc_bytes, 1470 bool *iv_contig_out) 1471 { 1472 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1473 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1474 struct device *jrdev = ctx->jrdev; 1475 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1476 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 1477 GFP_KERNEL : GFP_ATOMIC; 1478 int src_nents, dst_nents = 0, sec4_sg_bytes; 1479 struct ablkcipher_edesc *edesc; 1480 dma_addr_t iv_dma = 0; 1481 bool iv_contig = false; 1482 int sgc; 1483 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1484 bool src_chained = false, dst_chained = false; 1485 int sec4_sg_index; 1486 1487 src_nents = sg_count(req->src, req->nbytes, &src_chained); 1488 1489 if (req->dst != req->src) 1490 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); 1491 1492 if (likely(req->src == req->dst)) { 1493 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1494 DMA_BIDIRECTIONAL, src_chained); 1495 } else { 1496 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1497 DMA_TO_DEVICE, src_chained); 1498 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 1499 DMA_FROM_DEVICE, dst_chained); 1500 } 1501 1502 /* 1503 * Check if iv can be contiguous with source and destination. 1504 * If so, include it. If not, create scatterlist. 1505 */ 1506 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 1507 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) 1508 iv_contig = true; 1509 else 1510 src_nents = src_nents ? : 1; 1511 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 1512 sizeof(struct sec4_sg_entry); 1513 1514 /* allocate space for base edesc and hw desc commands, link tables */ 1515 edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes + 1516 sec4_sg_bytes, GFP_DMA | flags); 1517 if (!edesc) { 1518 dev_err(jrdev, "could not allocate extended descriptor\n"); 1519 return ERR_PTR(-ENOMEM); 1520 } 1521 1522 edesc->src_nents = src_nents; 1523 edesc->src_chained = src_chained; 1524 edesc->dst_nents = dst_nents; 1525 edesc->dst_chained = dst_chained; 1526 edesc->sec4_sg_bytes = sec4_sg_bytes; 1527 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 1528 desc_bytes; 1529 1530 sec4_sg_index = 0; 1531 if (!iv_contig) { 1532 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 1533 sg_to_sec4_sg_last(req->src, src_nents, 1534 edesc->sec4_sg + 1, 0); 1535 sec4_sg_index += 1 + src_nents; 1536 } 1537 1538 if (dst_nents) { 1539 sg_to_sec4_sg_last(req->dst, dst_nents, 1540 edesc->sec4_sg + sec4_sg_index, 0); 1541 } 1542 1543 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1544 sec4_sg_bytes, DMA_TO_DEVICE); 1545 edesc->iv_dma = iv_dma; 1546 1547 #ifdef DEBUG 1548 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", 1549 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1550 sec4_sg_bytes, 1); 1551 #endif 1552 1553 *iv_contig_out = iv_contig; 1554 return edesc; 1555 } 1556 1557 static int ablkcipher_encrypt(struct ablkcipher_request *req) 1558 { 1559 struct ablkcipher_edesc *edesc; 1560 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1561 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1562 struct device *jrdev = ctx->jrdev; 1563 bool iv_contig; 1564 u32 *desc; 1565 int ret = 0; 1566 1567 /* allocate extended descriptor */ 1568 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1569 CAAM_CMD_SZ, &iv_contig); 1570 if (IS_ERR(edesc)) 1571 return PTR_ERR(edesc); 1572 1573 /* Create and submit job descriptor*/ 1574 init_ablkcipher_job(ctx->sh_desc_enc, 1575 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 1576 #ifdef DEBUG 1577 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 1578 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1579 desc_bytes(edesc->hw_desc), 1); 1580 #endif 1581 desc = edesc->hw_desc; 1582 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 1583 1584 if (!ret) { 1585 ret = -EINPROGRESS; 1586 } else { 1587 ablkcipher_unmap(jrdev, edesc, req); 1588 kfree(edesc); 1589 } 1590 1591 return ret; 1592 } 1593 1594 static int ablkcipher_decrypt(struct ablkcipher_request *req) 1595 { 1596 struct ablkcipher_edesc *edesc; 1597 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1598 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1599 struct device *jrdev = ctx->jrdev; 1600 bool iv_contig; 1601 u32 *desc; 1602 int ret = 0; 1603 1604 /* allocate extended descriptor */ 1605 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1606 CAAM_CMD_SZ, &iv_contig); 1607 if (IS_ERR(edesc)) 1608 return PTR_ERR(edesc); 1609 1610 /* Create and submit job descriptor*/ 1611 init_ablkcipher_job(ctx->sh_desc_dec, 1612 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 1613 desc = edesc->hw_desc; 1614 #ifdef DEBUG 1615 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 1616 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1617 desc_bytes(edesc->hw_desc), 1); 1618 #endif 1619 1620 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 1621 if (!ret) { 1622 ret = -EINPROGRESS; 1623 } else { 1624 ablkcipher_unmap(jrdev, edesc, req); 1625 kfree(edesc); 1626 } 1627 1628 return ret; 1629 } 1630 1631 #define template_aead template_u.aead 1632 #define template_ablkcipher template_u.ablkcipher 1633 struct caam_alg_template { 1634 char name[CRYPTO_MAX_ALG_NAME]; 1635 char driver_name[CRYPTO_MAX_ALG_NAME]; 1636 unsigned int blocksize; 1637 u32 type; 1638 union { 1639 struct ablkcipher_alg ablkcipher; 1640 struct aead_alg aead; 1641 struct blkcipher_alg blkcipher; 1642 struct cipher_alg cipher; 1643 struct compress_alg compress; 1644 struct rng_alg rng; 1645 } template_u; 1646 u32 class1_alg_type; 1647 u32 class2_alg_type; 1648 u32 alg_op; 1649 }; 1650 1651 static struct caam_alg_template driver_algs[] = { 1652 /* single-pass ipsec_esp descriptor */ 1653 { 1654 .name = "authenc(hmac(md5),cbc(aes))", 1655 .driver_name = "authenc-hmac-md5-cbc-aes-caam", 1656 .blocksize = AES_BLOCK_SIZE, 1657 .type = CRYPTO_ALG_TYPE_AEAD, 1658 .template_aead = { 1659 .setkey = aead_setkey, 1660 .setauthsize = aead_setauthsize, 1661 .encrypt = aead_encrypt, 1662 .decrypt = aead_decrypt, 1663 .givencrypt = aead_givencrypt, 1664 .geniv = "<built-in>", 1665 .ivsize = AES_BLOCK_SIZE, 1666 .maxauthsize = MD5_DIGEST_SIZE, 1667 }, 1668 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1669 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 1670 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1671 }, 1672 { 1673 .name = "authenc(hmac(sha1),cbc(aes))", 1674 .driver_name = "authenc-hmac-sha1-cbc-aes-caam", 1675 .blocksize = AES_BLOCK_SIZE, 1676 .type = CRYPTO_ALG_TYPE_AEAD, 1677 .template_aead = { 1678 .setkey = aead_setkey, 1679 .setauthsize = aead_setauthsize, 1680 .encrypt = aead_encrypt, 1681 .decrypt = aead_decrypt, 1682 .givencrypt = aead_givencrypt, 1683 .geniv = "<built-in>", 1684 .ivsize = AES_BLOCK_SIZE, 1685 .maxauthsize = SHA1_DIGEST_SIZE, 1686 }, 1687 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1688 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 1689 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1690 }, 1691 { 1692 .name = "authenc(hmac(sha224),cbc(aes))", 1693 .driver_name = "authenc-hmac-sha224-cbc-aes-caam", 1694 .blocksize = AES_BLOCK_SIZE, 1695 .type = CRYPTO_ALG_TYPE_AEAD, 1696 .template_aead = { 1697 .setkey = aead_setkey, 1698 .setauthsize = aead_setauthsize, 1699 .encrypt = aead_encrypt, 1700 .decrypt = aead_decrypt, 1701 .givencrypt = aead_givencrypt, 1702 .geniv = "<built-in>", 1703 .ivsize = AES_BLOCK_SIZE, 1704 .maxauthsize = SHA224_DIGEST_SIZE, 1705 }, 1706 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1707 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1708 OP_ALG_AAI_HMAC_PRECOMP, 1709 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 1710 }, 1711 { 1712 .name = "authenc(hmac(sha256),cbc(aes))", 1713 .driver_name = "authenc-hmac-sha256-cbc-aes-caam", 1714 .blocksize = AES_BLOCK_SIZE, 1715 .type = CRYPTO_ALG_TYPE_AEAD, 1716 .template_aead = { 1717 .setkey = aead_setkey, 1718 .setauthsize = aead_setauthsize, 1719 .encrypt = aead_encrypt, 1720 .decrypt = aead_decrypt, 1721 .givencrypt = aead_givencrypt, 1722 .geniv = "<built-in>", 1723 .ivsize = AES_BLOCK_SIZE, 1724 .maxauthsize = SHA256_DIGEST_SIZE, 1725 }, 1726 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1727 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1728 OP_ALG_AAI_HMAC_PRECOMP, 1729 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1730 }, 1731 { 1732 .name = "authenc(hmac(sha384),cbc(aes))", 1733 .driver_name = "authenc-hmac-sha384-cbc-aes-caam", 1734 .blocksize = AES_BLOCK_SIZE, 1735 .type = CRYPTO_ALG_TYPE_AEAD, 1736 .template_aead = { 1737 .setkey = aead_setkey, 1738 .setauthsize = aead_setauthsize, 1739 .encrypt = aead_encrypt, 1740 .decrypt = aead_decrypt, 1741 .givencrypt = aead_givencrypt, 1742 .geniv = "<built-in>", 1743 .ivsize = AES_BLOCK_SIZE, 1744 .maxauthsize = SHA384_DIGEST_SIZE, 1745 }, 1746 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1747 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1748 OP_ALG_AAI_HMAC_PRECOMP, 1749 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 1750 }, 1751 1752 { 1753 .name = "authenc(hmac(sha512),cbc(aes))", 1754 .driver_name = "authenc-hmac-sha512-cbc-aes-caam", 1755 .blocksize = AES_BLOCK_SIZE, 1756 .type = CRYPTO_ALG_TYPE_AEAD, 1757 .template_aead = { 1758 .setkey = aead_setkey, 1759 .setauthsize = aead_setauthsize, 1760 .encrypt = aead_encrypt, 1761 .decrypt = aead_decrypt, 1762 .givencrypt = aead_givencrypt, 1763 .geniv = "<built-in>", 1764 .ivsize = AES_BLOCK_SIZE, 1765 .maxauthsize = SHA512_DIGEST_SIZE, 1766 }, 1767 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1768 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1769 OP_ALG_AAI_HMAC_PRECOMP, 1770 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 1771 }, 1772 { 1773 .name = "authenc(hmac(md5),cbc(des3_ede))", 1774 .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam", 1775 .blocksize = DES3_EDE_BLOCK_SIZE, 1776 .type = CRYPTO_ALG_TYPE_AEAD, 1777 .template_aead = { 1778 .setkey = aead_setkey, 1779 .setauthsize = aead_setauthsize, 1780 .encrypt = aead_encrypt, 1781 .decrypt = aead_decrypt, 1782 .givencrypt = aead_givencrypt, 1783 .geniv = "<built-in>", 1784 .ivsize = DES3_EDE_BLOCK_SIZE, 1785 .maxauthsize = MD5_DIGEST_SIZE, 1786 }, 1787 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1788 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 1789 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1790 }, 1791 { 1792 .name = "authenc(hmac(sha1),cbc(des3_ede))", 1793 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", 1794 .blocksize = DES3_EDE_BLOCK_SIZE, 1795 .type = CRYPTO_ALG_TYPE_AEAD, 1796 .template_aead = { 1797 .setkey = aead_setkey, 1798 .setauthsize = aead_setauthsize, 1799 .encrypt = aead_encrypt, 1800 .decrypt = aead_decrypt, 1801 .givencrypt = aead_givencrypt, 1802 .geniv = "<built-in>", 1803 .ivsize = DES3_EDE_BLOCK_SIZE, 1804 .maxauthsize = SHA1_DIGEST_SIZE, 1805 }, 1806 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1807 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 1808 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1809 }, 1810 { 1811 .name = "authenc(hmac(sha224),cbc(des3_ede))", 1812 .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam", 1813 .blocksize = DES3_EDE_BLOCK_SIZE, 1814 .type = CRYPTO_ALG_TYPE_AEAD, 1815 .template_aead = { 1816 .setkey = aead_setkey, 1817 .setauthsize = aead_setauthsize, 1818 .encrypt = aead_encrypt, 1819 .decrypt = aead_decrypt, 1820 .givencrypt = aead_givencrypt, 1821 .geniv = "<built-in>", 1822 .ivsize = DES3_EDE_BLOCK_SIZE, 1823 .maxauthsize = SHA224_DIGEST_SIZE, 1824 }, 1825 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1826 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1827 OP_ALG_AAI_HMAC_PRECOMP, 1828 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 1829 }, 1830 { 1831 .name = "authenc(hmac(sha256),cbc(des3_ede))", 1832 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", 1833 .blocksize = DES3_EDE_BLOCK_SIZE, 1834 .type = CRYPTO_ALG_TYPE_AEAD, 1835 .template_aead = { 1836 .setkey = aead_setkey, 1837 .setauthsize = aead_setauthsize, 1838 .encrypt = aead_encrypt, 1839 .decrypt = aead_decrypt, 1840 .givencrypt = aead_givencrypt, 1841 .geniv = "<built-in>", 1842 .ivsize = DES3_EDE_BLOCK_SIZE, 1843 .maxauthsize = SHA256_DIGEST_SIZE, 1844 }, 1845 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1846 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1847 OP_ALG_AAI_HMAC_PRECOMP, 1848 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1849 }, 1850 { 1851 .name = "authenc(hmac(sha384),cbc(des3_ede))", 1852 .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam", 1853 .blocksize = DES3_EDE_BLOCK_SIZE, 1854 .type = CRYPTO_ALG_TYPE_AEAD, 1855 .template_aead = { 1856 .setkey = aead_setkey, 1857 .setauthsize = aead_setauthsize, 1858 .encrypt = aead_encrypt, 1859 .decrypt = aead_decrypt, 1860 .givencrypt = aead_givencrypt, 1861 .geniv = "<built-in>", 1862 .ivsize = DES3_EDE_BLOCK_SIZE, 1863 .maxauthsize = SHA384_DIGEST_SIZE, 1864 }, 1865 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1866 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1867 OP_ALG_AAI_HMAC_PRECOMP, 1868 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 1869 }, 1870 { 1871 .name = "authenc(hmac(sha512),cbc(des3_ede))", 1872 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam", 1873 .blocksize = DES3_EDE_BLOCK_SIZE, 1874 .type = CRYPTO_ALG_TYPE_AEAD, 1875 .template_aead = { 1876 .setkey = aead_setkey, 1877 .setauthsize = aead_setauthsize, 1878 .encrypt = aead_encrypt, 1879 .decrypt = aead_decrypt, 1880 .givencrypt = aead_givencrypt, 1881 .geniv = "<built-in>", 1882 .ivsize = DES3_EDE_BLOCK_SIZE, 1883 .maxauthsize = SHA512_DIGEST_SIZE, 1884 }, 1885 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1886 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1887 OP_ALG_AAI_HMAC_PRECOMP, 1888 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 1889 }, 1890 { 1891 .name = "authenc(hmac(md5),cbc(des))", 1892 .driver_name = "authenc-hmac-md5-cbc-des-caam", 1893 .blocksize = DES_BLOCK_SIZE, 1894 .type = CRYPTO_ALG_TYPE_AEAD, 1895 .template_aead = { 1896 .setkey = aead_setkey, 1897 .setauthsize = aead_setauthsize, 1898 .encrypt = aead_encrypt, 1899 .decrypt = aead_decrypt, 1900 .givencrypt = aead_givencrypt, 1901 .geniv = "<built-in>", 1902 .ivsize = DES_BLOCK_SIZE, 1903 .maxauthsize = MD5_DIGEST_SIZE, 1904 }, 1905 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1906 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 1907 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1908 }, 1909 { 1910 .name = "authenc(hmac(sha1),cbc(des))", 1911 .driver_name = "authenc-hmac-sha1-cbc-des-caam", 1912 .blocksize = DES_BLOCK_SIZE, 1913 .type = CRYPTO_ALG_TYPE_AEAD, 1914 .template_aead = { 1915 .setkey = aead_setkey, 1916 .setauthsize = aead_setauthsize, 1917 .encrypt = aead_encrypt, 1918 .decrypt = aead_decrypt, 1919 .givencrypt = aead_givencrypt, 1920 .geniv = "<built-in>", 1921 .ivsize = DES_BLOCK_SIZE, 1922 .maxauthsize = SHA1_DIGEST_SIZE, 1923 }, 1924 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1925 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 1926 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1927 }, 1928 { 1929 .name = "authenc(hmac(sha224),cbc(des))", 1930 .driver_name = "authenc-hmac-sha224-cbc-des-caam", 1931 .blocksize = DES_BLOCK_SIZE, 1932 .type = CRYPTO_ALG_TYPE_AEAD, 1933 .template_aead = { 1934 .setkey = aead_setkey, 1935 .setauthsize = aead_setauthsize, 1936 .encrypt = aead_encrypt, 1937 .decrypt = aead_decrypt, 1938 .givencrypt = aead_givencrypt, 1939 .geniv = "<built-in>", 1940 .ivsize = DES_BLOCK_SIZE, 1941 .maxauthsize = SHA224_DIGEST_SIZE, 1942 }, 1943 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1944 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1945 OP_ALG_AAI_HMAC_PRECOMP, 1946 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 1947 }, 1948 { 1949 .name = "authenc(hmac(sha256),cbc(des))", 1950 .driver_name = "authenc-hmac-sha256-cbc-des-caam", 1951 .blocksize = DES_BLOCK_SIZE, 1952 .type = CRYPTO_ALG_TYPE_AEAD, 1953 .template_aead = { 1954 .setkey = aead_setkey, 1955 .setauthsize = aead_setauthsize, 1956 .encrypt = aead_encrypt, 1957 .decrypt = aead_decrypt, 1958 .givencrypt = aead_givencrypt, 1959 .geniv = "<built-in>", 1960 .ivsize = DES_BLOCK_SIZE, 1961 .maxauthsize = SHA256_DIGEST_SIZE, 1962 }, 1963 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1964 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1965 OP_ALG_AAI_HMAC_PRECOMP, 1966 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1967 }, 1968 { 1969 .name = "authenc(hmac(sha384),cbc(des))", 1970 .driver_name = "authenc-hmac-sha384-cbc-des-caam", 1971 .blocksize = DES_BLOCK_SIZE, 1972 .type = CRYPTO_ALG_TYPE_AEAD, 1973 .template_aead = { 1974 .setkey = aead_setkey, 1975 .setauthsize = aead_setauthsize, 1976 .encrypt = aead_encrypt, 1977 .decrypt = aead_decrypt, 1978 .givencrypt = aead_givencrypt, 1979 .geniv = "<built-in>", 1980 .ivsize = DES_BLOCK_SIZE, 1981 .maxauthsize = SHA384_DIGEST_SIZE, 1982 }, 1983 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1984 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1985 OP_ALG_AAI_HMAC_PRECOMP, 1986 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 1987 }, 1988 { 1989 .name = "authenc(hmac(sha512),cbc(des))", 1990 .driver_name = "authenc-hmac-sha512-cbc-des-caam", 1991 .blocksize = DES_BLOCK_SIZE, 1992 .type = CRYPTO_ALG_TYPE_AEAD, 1993 .template_aead = { 1994 .setkey = aead_setkey, 1995 .setauthsize = aead_setauthsize, 1996 .encrypt = aead_encrypt, 1997 .decrypt = aead_decrypt, 1998 .givencrypt = aead_givencrypt, 1999 .geniv = "<built-in>", 2000 .ivsize = DES_BLOCK_SIZE, 2001 .maxauthsize = SHA512_DIGEST_SIZE, 2002 }, 2003 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2004 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2005 OP_ALG_AAI_HMAC_PRECOMP, 2006 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 2007 }, 2008 /* ablkcipher descriptor */ 2009 { 2010 .name = "cbc(aes)", 2011 .driver_name = "cbc-aes-caam", 2012 .blocksize = AES_BLOCK_SIZE, 2013 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2014 .template_ablkcipher = { 2015 .setkey = ablkcipher_setkey, 2016 .encrypt = ablkcipher_encrypt, 2017 .decrypt = ablkcipher_decrypt, 2018 .geniv = "eseqiv", 2019 .min_keysize = AES_MIN_KEY_SIZE, 2020 .max_keysize = AES_MAX_KEY_SIZE, 2021 .ivsize = AES_BLOCK_SIZE, 2022 }, 2023 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2024 }, 2025 { 2026 .name = "cbc(des3_ede)", 2027 .driver_name = "cbc-3des-caam", 2028 .blocksize = DES3_EDE_BLOCK_SIZE, 2029 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2030 .template_ablkcipher = { 2031 .setkey = ablkcipher_setkey, 2032 .encrypt = ablkcipher_encrypt, 2033 .decrypt = ablkcipher_decrypt, 2034 .geniv = "eseqiv", 2035 .min_keysize = DES3_EDE_KEY_SIZE, 2036 .max_keysize = DES3_EDE_KEY_SIZE, 2037 .ivsize = DES3_EDE_BLOCK_SIZE, 2038 }, 2039 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2040 }, 2041 { 2042 .name = "cbc(des)", 2043 .driver_name = "cbc-des-caam", 2044 .blocksize = DES_BLOCK_SIZE, 2045 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2046 .template_ablkcipher = { 2047 .setkey = ablkcipher_setkey, 2048 .encrypt = ablkcipher_encrypt, 2049 .decrypt = ablkcipher_decrypt, 2050 .geniv = "eseqiv", 2051 .min_keysize = DES_KEY_SIZE, 2052 .max_keysize = DES_KEY_SIZE, 2053 .ivsize = DES_BLOCK_SIZE, 2054 }, 2055 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2056 } 2057 }; 2058 2059 struct caam_crypto_alg { 2060 struct list_head entry; 2061 int class1_alg_type; 2062 int class2_alg_type; 2063 int alg_op; 2064 struct crypto_alg crypto_alg; 2065 }; 2066 2067 static int caam_cra_init(struct crypto_tfm *tfm) 2068 { 2069 struct crypto_alg *alg = tfm->__crt_alg; 2070 struct caam_crypto_alg *caam_alg = 2071 container_of(alg, struct caam_crypto_alg, crypto_alg); 2072 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2073 2074 ctx->jrdev = caam_jr_alloc(); 2075 if (IS_ERR(ctx->jrdev)) { 2076 pr_err("Job Ring Device allocation for transform failed\n"); 2077 return PTR_ERR(ctx->jrdev); 2078 } 2079 2080 /* copy descriptor header template value */ 2081 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type; 2082 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type; 2083 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op; 2084 2085 return 0; 2086 } 2087 2088 static void caam_cra_exit(struct crypto_tfm *tfm) 2089 { 2090 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2091 2092 if (ctx->sh_desc_enc_dma && 2093 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) 2094 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, 2095 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 2096 if (ctx->sh_desc_dec_dma && 2097 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) 2098 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, 2099 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); 2100 if (ctx->sh_desc_givenc_dma && 2101 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) 2102 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, 2103 desc_bytes(ctx->sh_desc_givenc), 2104 DMA_TO_DEVICE); 2105 2106 caam_jr_free(ctx->jrdev); 2107 } 2108 2109 static void __exit caam_algapi_exit(void) 2110 { 2111 2112 struct caam_crypto_alg *t_alg, *n; 2113 2114 if (!alg_list.next) 2115 return; 2116 2117 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 2118 crypto_unregister_alg(&t_alg->crypto_alg); 2119 list_del(&t_alg->entry); 2120 kfree(t_alg); 2121 } 2122 } 2123 2124 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 2125 *template) 2126 { 2127 struct caam_crypto_alg *t_alg; 2128 struct crypto_alg *alg; 2129 2130 t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL); 2131 if (!t_alg) { 2132 pr_err("failed to allocate t_alg\n"); 2133 return ERR_PTR(-ENOMEM); 2134 } 2135 2136 alg = &t_alg->crypto_alg; 2137 2138 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 2139 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2140 template->driver_name); 2141 alg->cra_module = THIS_MODULE; 2142 alg->cra_init = caam_cra_init; 2143 alg->cra_exit = caam_cra_exit; 2144 alg->cra_priority = CAAM_CRA_PRIORITY; 2145 alg->cra_blocksize = template->blocksize; 2146 alg->cra_alignmask = 0; 2147 alg->cra_ctxsize = sizeof(struct caam_ctx); 2148 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 2149 template->type; 2150 switch (template->type) { 2151 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2152 alg->cra_type = &crypto_ablkcipher_type; 2153 alg->cra_ablkcipher = template->template_ablkcipher; 2154 break; 2155 case CRYPTO_ALG_TYPE_AEAD: 2156 alg->cra_type = &crypto_aead_type; 2157 alg->cra_aead = template->template_aead; 2158 break; 2159 } 2160 2161 t_alg->class1_alg_type = template->class1_alg_type; 2162 t_alg->class2_alg_type = template->class2_alg_type; 2163 t_alg->alg_op = template->alg_op; 2164 2165 return t_alg; 2166 } 2167 2168 static int __init caam_algapi_init(void) 2169 { 2170 int i = 0, err = 0; 2171 2172 INIT_LIST_HEAD(&alg_list); 2173 2174 /* register crypto algorithms the device supports */ 2175 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2176 /* TODO: check if h/w supports alg */ 2177 struct caam_crypto_alg *t_alg; 2178 2179 t_alg = caam_alg_alloc(&driver_algs[i]); 2180 if (IS_ERR(t_alg)) { 2181 err = PTR_ERR(t_alg); 2182 pr_warn("%s alg allocation failed\n", 2183 driver_algs[i].driver_name); 2184 continue; 2185 } 2186 2187 err = crypto_register_alg(&t_alg->crypto_alg); 2188 if (err) { 2189 pr_warn("%s alg registration failed\n", 2190 t_alg->crypto_alg.cra_driver_name); 2191 kfree(t_alg); 2192 } else 2193 list_add_tail(&t_alg->entry, &alg_list); 2194 } 2195 if (!list_empty(&alg_list)) 2196 pr_info("caam algorithms registered in /proc/crypto\n"); 2197 2198 return err; 2199 } 2200 2201 module_init(caam_algapi_init); 2202 module_exit(caam_algapi_exit); 2203 2204 MODULE_LICENSE("GPL"); 2205 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 2206 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 2207