1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * 6 * Based on talitos crypto API driver. 7 * 8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 9 * 10 * --------------- --------------- 11 * | JobDesc #1 |-------------------->| ShareDesc | 12 * | *(packet 1) | | (PDB) | 13 * --------------- |------------->| (hashKey) | 14 * . | | (cipherKey) | 15 * . | |-------->| (operation) | 16 * --------------- | | --------------- 17 * | JobDesc #2 |------| | 18 * | *(packet 2) | | 19 * --------------- | 20 * . | 21 * . | 22 * --------------- | 23 * | JobDesc #3 |------------ 24 * | *(packet 3) | 25 * --------------- 26 * 27 * The SharedDesc never changes for a connection unless rekeyed, but 28 * each packet will likely be in a different place. So all we need 29 * to know to process the packet is where the input is, where the 30 * output goes, and what context we want to process with. Context is 31 * in the SharedDesc, packet references in the JobDesc. 32 * 33 * So, a job desc looks like: 34 * 35 * --------------------- 36 * | Header | 37 * | ShareDesc Pointer | 38 * | SEQ_OUT_PTR | 39 * | (output buffer) | 40 * | (output length) | 41 * | SEQ_IN_PTR | 42 * | (input buffer) | 43 * | (input length) | 44 * --------------------- 45 */ 46 47 #include "compat.h" 48 49 #include "regs.h" 50 #include "intern.h" 51 #include "desc_constr.h" 52 #include "jr.h" 53 #include "error.h" 54 #include "sg_sw_sec4.h" 55 #include "key_gen.h" 56 57 /* 58 * crypto alg 59 */ 60 #define CAAM_CRA_PRIORITY 3000 61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 62 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 63 SHA512_DIGEST_SIZE * 2) 64 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 65 #define CAAM_MAX_IV_LENGTH 16 66 67 /* length of descriptors text */ 68 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) 69 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ) 70 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ) 71 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) 72 73 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) 74 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 75 20 * CAAM_CMD_SZ) 76 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 77 15 * CAAM_CMD_SZ) 78 79 #define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \ 80 CAAM_MAX_KEY_SIZE) 81 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 82 83 #ifdef DEBUG 84 /* for print_hex_dumps with line references */ 85 #define debug(format, arg...) printk(format, arg) 86 #else 87 #define debug(format, arg...) 88 #endif 89 90 /* Set DK bit in class 1 operation if shared */ 91 static inline void append_dec_op1(u32 *desc, u32 type) 92 { 93 u32 *jump_cmd, *uncond_jump_cmd; 94 95 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 96 append_operation(desc, type | OP_ALG_AS_INITFINAL | 97 OP_ALG_DECRYPT); 98 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 99 set_jump_tgt_here(desc, jump_cmd); 100 append_operation(desc, type | OP_ALG_AS_INITFINAL | 101 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 102 set_jump_tgt_here(desc, uncond_jump_cmd); 103 } 104 105 /* 106 * Wait for completion of class 1 key loading before allowing 107 * error propagation 108 */ 109 static inline void append_dec_shr_done(u32 *desc) 110 { 111 u32 *jump_cmd; 112 113 jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL); 114 set_jump_tgt_here(desc, jump_cmd); 115 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 116 } 117 118 /* 119 * For aead functions, read payload and write payload, 120 * both of which are specified in req->src and req->dst 121 */ 122 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 123 { 124 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 125 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 126 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 127 } 128 129 /* 130 * For aead encrypt and decrypt, read iv for both classes 131 */ 132 static inline void aead_append_ld_iv(u32 *desc, int ivsize) 133 { 134 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 135 LDST_CLASS_1_CCB | ivsize); 136 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); 137 } 138 139 /* 140 * For ablkcipher encrypt and decrypt, read from req->src and 141 * write to req->dst 142 */ 143 static inline void ablkcipher_append_src_dst(u32 *desc) 144 { 145 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 146 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 147 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 148 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 149 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 150 } 151 152 /* 153 * If all data, including src (with assoc and iv) or dst (with iv only) are 154 * contiguous 155 */ 156 #define GIV_SRC_CONTIG 1 157 #define GIV_DST_CONTIG (1 << 1) 158 159 /* 160 * per-session context 161 */ 162 struct caam_ctx { 163 struct device *jrdev; 164 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 165 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 166 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 167 dma_addr_t sh_desc_enc_dma; 168 dma_addr_t sh_desc_dec_dma; 169 dma_addr_t sh_desc_givenc_dma; 170 u32 class1_alg_type; 171 u32 class2_alg_type; 172 u32 alg_op; 173 u8 key[CAAM_MAX_KEY_SIZE]; 174 dma_addr_t key_dma; 175 unsigned int enckeylen; 176 unsigned int split_key_len; 177 unsigned int split_key_pad_len; 178 unsigned int authsize; 179 }; 180 181 static void append_key_aead(u32 *desc, struct caam_ctx *ctx, 182 int keys_fit_inline) 183 { 184 if (keys_fit_inline) { 185 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 186 ctx->split_key_len, CLASS_2 | 187 KEY_DEST_MDHA_SPLIT | KEY_ENC); 188 append_key_as_imm(desc, (void *)ctx->key + 189 ctx->split_key_pad_len, ctx->enckeylen, 190 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 191 } else { 192 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 193 KEY_DEST_MDHA_SPLIT | KEY_ENC); 194 append_key(desc, ctx->key_dma + ctx->split_key_pad_len, 195 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 196 } 197 } 198 199 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, 200 int keys_fit_inline) 201 { 202 u32 *key_jump_cmd; 203 204 init_sh_desc(desc, HDR_SHARE_SERIAL); 205 206 /* Skip if already shared */ 207 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 208 JUMP_COND_SHRD); 209 210 append_key_aead(desc, ctx, keys_fit_inline); 211 212 set_jump_tgt_here(desc, key_jump_cmd); 213 214 /* Propagate errors from shared to job descriptor */ 215 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 216 } 217 218 static int aead_set_sh_desc(struct crypto_aead *aead) 219 { 220 struct aead_tfm *tfm = &aead->base.crt_aead; 221 struct caam_ctx *ctx = crypto_aead_ctx(aead); 222 struct device *jrdev = ctx->jrdev; 223 bool keys_fit_inline = false; 224 u32 *key_jump_cmd, *jump_cmd; 225 u32 geniv, moveiv; 226 u32 *desc; 227 228 if (!ctx->enckeylen || !ctx->authsize) 229 return 0; 230 231 /* 232 * Job Descriptor and Shared Descriptors 233 * must all fit into the 64-word Descriptor h/w Buffer 234 */ 235 if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + 236 ctx->split_key_pad_len + ctx->enckeylen <= 237 CAAM_DESC_BYTES_MAX) 238 keys_fit_inline = true; 239 240 /* aead_encrypt shared descriptor */ 241 desc = ctx->sh_desc_enc; 242 243 init_sh_desc_key_aead(desc, ctx, keys_fit_inline); 244 245 /* Class 2 operation */ 246 append_operation(desc, ctx->class2_alg_type | 247 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 248 249 /* cryptlen = seqoutlen - authsize */ 250 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 251 252 /* assoclen + cryptlen = seqinlen - ivsize */ 253 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); 254 255 /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */ 256 append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); 257 258 /* read assoc before reading payload */ 259 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 260 KEY_VLF); 261 aead_append_ld_iv(desc, tfm->ivsize); 262 263 /* Class 1 operation */ 264 append_operation(desc, ctx->class1_alg_type | 265 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 266 267 /* Read and write cryptlen bytes */ 268 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 269 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 270 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 271 272 /* Write ICV */ 273 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 274 LDST_SRCDST_BYTE_CONTEXT); 275 276 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 277 desc_bytes(desc), 278 DMA_TO_DEVICE); 279 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 280 dev_err(jrdev, "unable to map shared descriptor\n"); 281 return -ENOMEM; 282 } 283 #ifdef DEBUG 284 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", 285 DUMP_PREFIX_ADDRESS, 16, 4, desc, 286 desc_bytes(desc), 1); 287 #endif 288 289 /* 290 * Job Descriptor and Shared Descriptors 291 * must all fit into the 64-word Descriptor h/w Buffer 292 */ 293 if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + 294 ctx->split_key_pad_len + ctx->enckeylen <= 295 CAAM_DESC_BYTES_MAX) 296 keys_fit_inline = true; 297 298 desc = ctx->sh_desc_dec; 299 300 /* aead_decrypt shared descriptor */ 301 init_sh_desc(desc, HDR_SHARE_SERIAL); 302 303 /* Skip if already shared */ 304 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 305 JUMP_COND_SHRD); 306 307 append_key_aead(desc, ctx, keys_fit_inline); 308 309 /* Only propagate error immediately if shared */ 310 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 311 set_jump_tgt_here(desc, key_jump_cmd); 312 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 313 set_jump_tgt_here(desc, jump_cmd); 314 315 /* Class 2 operation */ 316 append_operation(desc, ctx->class2_alg_type | 317 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 318 319 /* assoclen + cryptlen = seqinlen - ivsize */ 320 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, 321 ctx->authsize + tfm->ivsize) 322 /* assoclen = (assoclen + cryptlen) - cryptlen */ 323 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 324 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); 325 326 /* read assoc before reading payload */ 327 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 328 KEY_VLF); 329 330 aead_append_ld_iv(desc, tfm->ivsize); 331 332 append_dec_op1(desc, ctx->class1_alg_type); 333 334 /* Read and write cryptlen bytes */ 335 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 336 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 337 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 338 339 /* Load ICV */ 340 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 341 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 342 append_dec_shr_done(desc); 343 344 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 345 desc_bytes(desc), 346 DMA_TO_DEVICE); 347 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 348 dev_err(jrdev, "unable to map shared descriptor\n"); 349 return -ENOMEM; 350 } 351 #ifdef DEBUG 352 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", 353 DUMP_PREFIX_ADDRESS, 16, 4, desc, 354 desc_bytes(desc), 1); 355 #endif 356 357 /* 358 * Job Descriptor and Shared Descriptors 359 * must all fit into the 64-word Descriptor h/w Buffer 360 */ 361 if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + 362 ctx->split_key_pad_len + ctx->enckeylen <= 363 CAAM_DESC_BYTES_MAX) 364 keys_fit_inline = true; 365 366 /* aead_givencrypt shared descriptor */ 367 desc = ctx->sh_desc_givenc; 368 369 init_sh_desc_key_aead(desc, ctx, keys_fit_inline); 370 371 /* Generate IV */ 372 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 373 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 374 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); 375 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 376 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 377 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 378 append_move(desc, MOVE_SRC_INFIFO | 379 MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT)); 380 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 381 382 /* Copy IV to class 1 context */ 383 append_move(desc, MOVE_SRC_CLASS1CTX | 384 MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT)); 385 386 /* Return to encryption */ 387 append_operation(desc, ctx->class2_alg_type | 388 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 389 390 /* ivsize + cryptlen = seqoutlen - authsize */ 391 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 392 393 /* assoclen = seqinlen - (ivsize + cryptlen) */ 394 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 395 396 /* read assoc before reading payload */ 397 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 398 KEY_VLF); 399 400 /* Copy iv from class 1 ctx to class 2 fifo*/ 401 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 402 NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); 403 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 404 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 405 append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB | 406 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 407 408 /* Class 1 operation */ 409 append_operation(desc, ctx->class1_alg_type | 410 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 411 412 /* Will write ivsize + cryptlen */ 413 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 414 415 /* Not need to reload iv */ 416 append_seq_fifo_load(desc, tfm->ivsize, 417 FIFOLD_CLASS_SKIP); 418 419 /* Will read cryptlen */ 420 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 421 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 422 423 /* Write ICV */ 424 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 425 LDST_SRCDST_BYTE_CONTEXT); 426 427 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 428 desc_bytes(desc), 429 DMA_TO_DEVICE); 430 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 431 dev_err(jrdev, "unable to map shared descriptor\n"); 432 return -ENOMEM; 433 } 434 #ifdef DEBUG 435 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", 436 DUMP_PREFIX_ADDRESS, 16, 4, desc, 437 desc_bytes(desc), 1); 438 #endif 439 440 return 0; 441 } 442 443 static int aead_setauthsize(struct crypto_aead *authenc, 444 unsigned int authsize) 445 { 446 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 447 448 ctx->authsize = authsize; 449 aead_set_sh_desc(authenc); 450 451 return 0; 452 } 453 454 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, 455 u32 authkeylen) 456 { 457 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, 458 ctx->split_key_pad_len, key_in, authkeylen, 459 ctx->alg_op); 460 } 461 462 static int aead_setkey(struct crypto_aead *aead, 463 const u8 *key, unsigned int keylen) 464 { 465 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 466 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 467 struct caam_ctx *ctx = crypto_aead_ctx(aead); 468 struct device *jrdev = ctx->jrdev; 469 struct rtattr *rta = (void *)key; 470 struct crypto_authenc_key_param *param; 471 unsigned int authkeylen; 472 unsigned int enckeylen; 473 int ret = 0; 474 475 param = RTA_DATA(rta); 476 enckeylen = be32_to_cpu(param->enckeylen); 477 478 key += RTA_ALIGN(rta->rta_len); 479 keylen -= RTA_ALIGN(rta->rta_len); 480 481 if (keylen < enckeylen) 482 goto badkey; 483 484 authkeylen = keylen - enckeylen; 485 486 if (keylen > CAAM_MAX_KEY_SIZE) 487 goto badkey; 488 489 /* Pick class 2 key length from algorithm submask */ 490 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 491 OP_ALG_ALGSEL_SHIFT] * 2; 492 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 493 494 #ifdef DEBUG 495 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 496 keylen, enckeylen, authkeylen); 497 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 498 ctx->split_key_len, ctx->split_key_pad_len); 499 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 500 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 501 #endif 502 503 ret = gen_split_aead_key(ctx, key, authkeylen); 504 if (ret) { 505 goto badkey; 506 } 507 508 /* postpend encryption key to auth split key */ 509 memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen); 510 511 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 512 enckeylen, DMA_TO_DEVICE); 513 if (dma_mapping_error(jrdev, ctx->key_dma)) { 514 dev_err(jrdev, "unable to map key i/o memory\n"); 515 return -ENOMEM; 516 } 517 #ifdef DEBUG 518 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 519 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 520 ctx->split_key_pad_len + enckeylen, 1); 521 #endif 522 523 ctx->enckeylen = enckeylen; 524 525 ret = aead_set_sh_desc(aead); 526 if (ret) { 527 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + 528 enckeylen, DMA_TO_DEVICE); 529 } 530 531 return ret; 532 badkey: 533 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 534 return -EINVAL; 535 } 536 537 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 538 const u8 *key, unsigned int keylen) 539 { 540 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 541 struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher; 542 struct device *jrdev = ctx->jrdev; 543 int ret = 0; 544 u32 *key_jump_cmd, *jump_cmd; 545 u32 *desc; 546 547 #ifdef DEBUG 548 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 549 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 550 #endif 551 552 memcpy(ctx->key, key, keylen); 553 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 554 DMA_TO_DEVICE); 555 if (dma_mapping_error(jrdev, ctx->key_dma)) { 556 dev_err(jrdev, "unable to map key i/o memory\n"); 557 return -ENOMEM; 558 } 559 ctx->enckeylen = keylen; 560 561 /* ablkcipher_encrypt shared descriptor */ 562 desc = ctx->sh_desc_enc; 563 init_sh_desc(desc, HDR_SHARE_SERIAL); 564 /* Skip if already shared */ 565 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 566 JUMP_COND_SHRD); 567 568 /* Load class1 key only */ 569 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 570 ctx->enckeylen, CLASS_1 | 571 KEY_DEST_CLASS_REG); 572 573 set_jump_tgt_here(desc, key_jump_cmd); 574 575 /* Propagate errors from shared to job descriptor */ 576 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 577 578 /* Load iv */ 579 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 580 LDST_CLASS_1_CCB | tfm->ivsize); 581 582 /* Load operation */ 583 append_operation(desc, ctx->class1_alg_type | 584 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 585 586 /* Perform operation */ 587 ablkcipher_append_src_dst(desc); 588 589 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 590 desc_bytes(desc), 591 DMA_TO_DEVICE); 592 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 593 dev_err(jrdev, "unable to map shared descriptor\n"); 594 return -ENOMEM; 595 } 596 #ifdef DEBUG 597 print_hex_dump(KERN_ERR, 598 "ablkcipher enc shdesc@"__stringify(__LINE__)": ", 599 DUMP_PREFIX_ADDRESS, 16, 4, desc, 600 desc_bytes(desc), 1); 601 #endif 602 /* ablkcipher_decrypt shared descriptor */ 603 desc = ctx->sh_desc_dec; 604 605 init_sh_desc(desc, HDR_SHARE_SERIAL); 606 /* Skip if already shared */ 607 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 608 JUMP_COND_SHRD); 609 610 /* Load class1 key only */ 611 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 612 ctx->enckeylen, CLASS_1 | 613 KEY_DEST_CLASS_REG); 614 615 /* For aead, only propagate error immediately if shared */ 616 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 617 set_jump_tgt_here(desc, key_jump_cmd); 618 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 619 set_jump_tgt_here(desc, jump_cmd); 620 621 /* load IV */ 622 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 623 LDST_CLASS_1_CCB | tfm->ivsize); 624 625 /* Choose operation */ 626 append_dec_op1(desc, ctx->class1_alg_type); 627 628 /* Perform operation */ 629 ablkcipher_append_src_dst(desc); 630 631 /* Wait for key to load before allowing propagating error */ 632 append_dec_shr_done(desc); 633 634 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 635 desc_bytes(desc), 636 DMA_TO_DEVICE); 637 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 638 dev_err(jrdev, "unable to map shared descriptor\n"); 639 return -ENOMEM; 640 } 641 642 #ifdef DEBUG 643 print_hex_dump(KERN_ERR, 644 "ablkcipher dec shdesc@"__stringify(__LINE__)": ", 645 DUMP_PREFIX_ADDRESS, 16, 4, desc, 646 desc_bytes(desc), 1); 647 #endif 648 649 return ret; 650 } 651 652 /* 653 * aead_edesc - s/w-extended aead descriptor 654 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 655 * @assoc_chained: if source is chained 656 * @src_nents: number of segments in input scatterlist 657 * @src_chained: if source is chained 658 * @dst_nents: number of segments in output scatterlist 659 * @dst_chained: if destination is chained 660 * @iv_dma: dma address of iv for checking continuity and link table 661 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 662 * @sec4_sg_bytes: length of dma mapped sec4_sg space 663 * @sec4_sg_dma: bus physical mapped address of h/w link table 664 * @hw_desc: the h/w job descriptor followed by any referenced link tables 665 */ 666 struct aead_edesc { 667 int assoc_nents; 668 bool assoc_chained; 669 int src_nents; 670 bool src_chained; 671 int dst_nents; 672 bool dst_chained; 673 dma_addr_t iv_dma; 674 int sec4_sg_bytes; 675 dma_addr_t sec4_sg_dma; 676 struct sec4_sg_entry *sec4_sg; 677 u32 hw_desc[0]; 678 }; 679 680 /* 681 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 682 * @src_nents: number of segments in input scatterlist 683 * @src_chained: if source is chained 684 * @dst_nents: number of segments in output scatterlist 685 * @dst_chained: if destination is chained 686 * @iv_dma: dma address of iv for checking continuity and link table 687 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 688 * @sec4_sg_bytes: length of dma mapped sec4_sg space 689 * @sec4_sg_dma: bus physical mapped address of h/w link table 690 * @hw_desc: the h/w job descriptor followed by any referenced link tables 691 */ 692 struct ablkcipher_edesc { 693 int src_nents; 694 bool src_chained; 695 int dst_nents; 696 bool dst_chained; 697 dma_addr_t iv_dma; 698 int sec4_sg_bytes; 699 dma_addr_t sec4_sg_dma; 700 struct sec4_sg_entry *sec4_sg; 701 u32 hw_desc[0]; 702 }; 703 704 static void caam_unmap(struct device *dev, struct scatterlist *src, 705 struct scatterlist *dst, int src_nents, 706 bool src_chained, int dst_nents, bool dst_chained, 707 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 708 int sec4_sg_bytes) 709 { 710 if (dst != src) { 711 dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE, 712 src_chained); 713 dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE, 714 dst_chained); 715 } else { 716 dma_unmap_sg_chained(dev, src, src_nents ? : 1, 717 DMA_BIDIRECTIONAL, src_chained); 718 } 719 720 if (iv_dma) 721 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 722 if (sec4_sg_bytes) 723 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 724 DMA_TO_DEVICE); 725 } 726 727 static void aead_unmap(struct device *dev, 728 struct aead_edesc *edesc, 729 struct aead_request *req) 730 { 731 struct crypto_aead *aead = crypto_aead_reqtfm(req); 732 int ivsize = crypto_aead_ivsize(aead); 733 734 dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents, 735 DMA_TO_DEVICE, edesc->assoc_chained); 736 737 caam_unmap(dev, req->src, req->dst, 738 edesc->src_nents, edesc->src_chained, edesc->dst_nents, 739 edesc->dst_chained, edesc->iv_dma, ivsize, 740 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 741 } 742 743 static void ablkcipher_unmap(struct device *dev, 744 struct ablkcipher_edesc *edesc, 745 struct ablkcipher_request *req) 746 { 747 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 748 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 749 750 caam_unmap(dev, req->src, req->dst, 751 edesc->src_nents, edesc->src_chained, edesc->dst_nents, 752 edesc->dst_chained, edesc->iv_dma, ivsize, 753 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 754 } 755 756 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 757 void *context) 758 { 759 struct aead_request *req = context; 760 struct aead_edesc *edesc; 761 #ifdef DEBUG 762 struct crypto_aead *aead = crypto_aead_reqtfm(req); 763 struct caam_ctx *ctx = crypto_aead_ctx(aead); 764 int ivsize = crypto_aead_ivsize(aead); 765 766 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 767 #endif 768 769 edesc = (struct aead_edesc *)((char *)desc - 770 offsetof(struct aead_edesc, hw_desc)); 771 772 if (err) { 773 char tmp[CAAM_ERROR_STR_MAX]; 774 775 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 776 } 777 778 aead_unmap(jrdev, edesc, req); 779 780 #ifdef DEBUG 781 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", 782 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 783 req->assoclen , 1); 784 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 785 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, 786 edesc->src_nents ? 100 : ivsize, 1); 787 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 788 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 789 edesc->src_nents ? 100 : req->cryptlen + 790 ctx->authsize + 4, 1); 791 #endif 792 793 kfree(edesc); 794 795 aead_request_complete(req, err); 796 } 797 798 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 799 void *context) 800 { 801 struct aead_request *req = context; 802 struct aead_edesc *edesc; 803 #ifdef DEBUG 804 struct crypto_aead *aead = crypto_aead_reqtfm(req); 805 struct caam_ctx *ctx = crypto_aead_ctx(aead); 806 int ivsize = crypto_aead_ivsize(aead); 807 808 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 809 #endif 810 811 edesc = (struct aead_edesc *)((char *)desc - 812 offsetof(struct aead_edesc, hw_desc)); 813 814 #ifdef DEBUG 815 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 816 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 817 ivsize, 1); 818 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 819 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst), 820 req->cryptlen, 1); 821 #endif 822 823 if (err) { 824 char tmp[CAAM_ERROR_STR_MAX]; 825 826 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 827 } 828 829 aead_unmap(jrdev, edesc, req); 830 831 /* 832 * verify hw auth check passed else return -EBADMSG 833 */ 834 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 835 err = -EBADMSG; 836 837 #ifdef DEBUG 838 print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ", 839 DUMP_PREFIX_ADDRESS, 16, 4, 840 ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)), 841 sizeof(struct iphdr) + req->assoclen + 842 ((req->cryptlen > 1500) ? 1500 : req->cryptlen) + 843 ctx->authsize + 36, 1); 844 if (!err && edesc->sec4_sg_bytes) { 845 struct scatterlist *sg = sg_last(req->src, edesc->src_nents); 846 print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ", 847 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), 848 sg->length + ctx->authsize + 16, 1); 849 } 850 #endif 851 852 kfree(edesc); 853 854 aead_request_complete(req, err); 855 } 856 857 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 858 void *context) 859 { 860 struct ablkcipher_request *req = context; 861 struct ablkcipher_edesc *edesc; 862 #ifdef DEBUG 863 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 864 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 865 866 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 867 #endif 868 869 edesc = (struct ablkcipher_edesc *)((char *)desc - 870 offsetof(struct ablkcipher_edesc, hw_desc)); 871 872 if (err) { 873 char tmp[CAAM_ERROR_STR_MAX]; 874 875 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 876 } 877 878 #ifdef DEBUG 879 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 880 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 881 edesc->src_nents > 1 ? 100 : ivsize, 1); 882 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 883 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 884 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 885 #endif 886 887 ablkcipher_unmap(jrdev, edesc, req); 888 kfree(edesc); 889 890 ablkcipher_request_complete(req, err); 891 } 892 893 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 894 void *context) 895 { 896 struct ablkcipher_request *req = context; 897 struct ablkcipher_edesc *edesc; 898 #ifdef DEBUG 899 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 900 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 901 902 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 903 #endif 904 905 edesc = (struct ablkcipher_edesc *)((char *)desc - 906 offsetof(struct ablkcipher_edesc, hw_desc)); 907 if (err) { 908 char tmp[CAAM_ERROR_STR_MAX]; 909 910 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 911 } 912 913 #ifdef DEBUG 914 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 915 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 916 ivsize, 1); 917 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 918 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 919 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 920 #endif 921 922 ablkcipher_unmap(jrdev, edesc, req); 923 kfree(edesc); 924 925 ablkcipher_request_complete(req, err); 926 } 927 928 /* 929 * Fill in aead job descriptor 930 */ 931 static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, 932 struct aead_edesc *edesc, 933 struct aead_request *req, 934 bool all_contig, bool encrypt) 935 { 936 struct crypto_aead *aead = crypto_aead_reqtfm(req); 937 struct caam_ctx *ctx = crypto_aead_ctx(aead); 938 int ivsize = crypto_aead_ivsize(aead); 939 int authsize = ctx->authsize; 940 u32 *desc = edesc->hw_desc; 941 u32 out_options = 0, in_options; 942 dma_addr_t dst_dma, src_dma; 943 int len, sec4_sg_index = 0; 944 945 #ifdef DEBUG 946 debug("assoclen %d cryptlen %d authsize %d\n", 947 req->assoclen, req->cryptlen, authsize); 948 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", 949 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 950 req->assoclen , 1); 951 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 952 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 953 edesc->src_nents ? 100 : ivsize, 1); 954 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 955 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 956 edesc->src_nents ? 100 : req->cryptlen, 1); 957 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", 958 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 959 desc_bytes(sh_desc), 1); 960 #endif 961 962 len = desc_len(sh_desc); 963 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 964 965 if (all_contig) { 966 src_dma = sg_dma_address(req->assoc); 967 in_options = 0; 968 } else { 969 src_dma = edesc->sec4_sg_dma; 970 sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 + 971 (edesc->src_nents ? : 1); 972 in_options = LDST_SGF; 973 } 974 if (encrypt) 975 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + 976 req->cryptlen - authsize, in_options); 977 else 978 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + 979 req->cryptlen, in_options); 980 981 if (likely(req->src == req->dst)) { 982 if (all_contig) { 983 dst_dma = sg_dma_address(req->src); 984 } else { 985 dst_dma = src_dma + sizeof(struct sec4_sg_entry) * 986 ((edesc->assoc_nents ? : 1) + 1); 987 out_options = LDST_SGF; 988 } 989 } else { 990 if (!edesc->dst_nents) { 991 dst_dma = sg_dma_address(req->dst); 992 } else { 993 dst_dma = edesc->sec4_sg_dma + 994 sec4_sg_index * 995 sizeof(struct sec4_sg_entry); 996 out_options = LDST_SGF; 997 } 998 } 999 if (encrypt) 1000 append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options); 1001 else 1002 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize, 1003 out_options); 1004 } 1005 1006 /* 1007 * Fill in aead givencrypt job descriptor 1008 */ 1009 static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, 1010 struct aead_edesc *edesc, 1011 struct aead_request *req, 1012 int contig) 1013 { 1014 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1015 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1016 int ivsize = crypto_aead_ivsize(aead); 1017 int authsize = ctx->authsize; 1018 u32 *desc = edesc->hw_desc; 1019 u32 out_options = 0, in_options; 1020 dma_addr_t dst_dma, src_dma; 1021 int len, sec4_sg_index = 0; 1022 1023 #ifdef DEBUG 1024 debug("assoclen %d cryptlen %d authsize %d\n", 1025 req->assoclen, req->cryptlen, authsize); 1026 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", 1027 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 1028 req->assoclen , 1); 1029 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 1030 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); 1031 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 1032 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1033 edesc->src_nents > 1 ? 100 : req->cryptlen, 1); 1034 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", 1035 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 1036 desc_bytes(sh_desc), 1); 1037 #endif 1038 1039 len = desc_len(sh_desc); 1040 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1041 1042 if (contig & GIV_SRC_CONTIG) { 1043 src_dma = sg_dma_address(req->assoc); 1044 in_options = 0; 1045 } else { 1046 src_dma = edesc->sec4_sg_dma; 1047 sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents; 1048 in_options = LDST_SGF; 1049 } 1050 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + 1051 req->cryptlen - authsize, in_options); 1052 1053 if (contig & GIV_DST_CONTIG) { 1054 dst_dma = edesc->iv_dma; 1055 } else { 1056 if (likely(req->src == req->dst)) { 1057 dst_dma = src_dma + sizeof(struct sec4_sg_entry) * 1058 edesc->assoc_nents; 1059 out_options = LDST_SGF; 1060 } else { 1061 dst_dma = edesc->sec4_sg_dma + 1062 sec4_sg_index * 1063 sizeof(struct sec4_sg_entry); 1064 out_options = LDST_SGF; 1065 } 1066 } 1067 1068 append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options); 1069 } 1070 1071 /* 1072 * Fill in ablkcipher job descriptor 1073 */ 1074 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 1075 struct ablkcipher_edesc *edesc, 1076 struct ablkcipher_request *req, 1077 bool iv_contig) 1078 { 1079 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1080 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1081 u32 *desc = edesc->hw_desc; 1082 u32 out_options = 0, in_options; 1083 dma_addr_t dst_dma, src_dma; 1084 int len, sec4_sg_index = 0; 1085 1086 #ifdef DEBUG 1087 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 1088 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1089 ivsize, 1); 1090 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 1091 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1092 edesc->src_nents ? 100 : req->nbytes, 1); 1093 #endif 1094 1095 len = desc_len(sh_desc); 1096 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1097 1098 if (iv_contig) { 1099 src_dma = edesc->iv_dma; 1100 in_options = 0; 1101 } else { 1102 src_dma = edesc->sec4_sg_dma; 1103 sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents; 1104 in_options = LDST_SGF; 1105 } 1106 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 1107 1108 if (likely(req->src == req->dst)) { 1109 if (!edesc->src_nents && iv_contig) { 1110 dst_dma = sg_dma_address(req->src); 1111 } else { 1112 dst_dma = edesc->sec4_sg_dma + 1113 sizeof(struct sec4_sg_entry); 1114 out_options = LDST_SGF; 1115 } 1116 } else { 1117 if (!edesc->dst_nents) { 1118 dst_dma = sg_dma_address(req->dst); 1119 } else { 1120 dst_dma = edesc->sec4_sg_dma + 1121 sec4_sg_index * sizeof(struct sec4_sg_entry); 1122 out_options = LDST_SGF; 1123 } 1124 } 1125 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 1126 } 1127 1128 /* 1129 * allocate and map the aead extended descriptor 1130 */ 1131 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 1132 int desc_bytes, bool *all_contig_ptr) 1133 { 1134 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1135 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1136 struct device *jrdev = ctx->jrdev; 1137 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1138 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1139 int assoc_nents, src_nents, dst_nents = 0; 1140 struct aead_edesc *edesc; 1141 dma_addr_t iv_dma = 0; 1142 int sgc; 1143 bool all_contig = true; 1144 bool assoc_chained = false, src_chained = false, dst_chained = false; 1145 int ivsize = crypto_aead_ivsize(aead); 1146 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 1147 1148 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); 1149 src_nents = sg_count(req->src, req->cryptlen, &src_chained); 1150 1151 if (unlikely(req->dst != req->src)) 1152 dst_nents = sg_count(req->dst, req->cryptlen, &dst_chained); 1153 1154 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, 1155 DMA_TO_DEVICE, assoc_chained); 1156 if (likely(req->src == req->dst)) { 1157 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1158 DMA_BIDIRECTIONAL, src_chained); 1159 } else { 1160 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1161 DMA_TO_DEVICE, src_chained); 1162 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 1163 DMA_FROM_DEVICE, dst_chained); 1164 } 1165 1166 /* Check if data are contiguous */ 1167 iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); 1168 if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != 1169 iv_dma || src_nents || iv_dma + ivsize != 1170 sg_dma_address(req->src)) { 1171 all_contig = false; 1172 assoc_nents = assoc_nents ? : 1; 1173 src_nents = src_nents ? : 1; 1174 sec4_sg_len = assoc_nents + 1 + src_nents; 1175 } 1176 sec4_sg_len += dst_nents; 1177 1178 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1179 1180 /* allocate space for base edesc and hw desc commands, link tables */ 1181 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + 1182 sec4_sg_bytes, GFP_DMA | flags); 1183 if (!edesc) { 1184 dev_err(jrdev, "could not allocate extended descriptor\n"); 1185 return ERR_PTR(-ENOMEM); 1186 } 1187 1188 edesc->assoc_nents = assoc_nents; 1189 edesc->assoc_chained = assoc_chained; 1190 edesc->src_nents = src_nents; 1191 edesc->src_chained = src_chained; 1192 edesc->dst_nents = dst_nents; 1193 edesc->dst_chained = dst_chained; 1194 edesc->iv_dma = iv_dma; 1195 edesc->sec4_sg_bytes = sec4_sg_bytes; 1196 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1197 desc_bytes; 1198 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1199 sec4_sg_bytes, DMA_TO_DEVICE); 1200 *all_contig_ptr = all_contig; 1201 1202 sec4_sg_index = 0; 1203 if (!all_contig) { 1204 sg_to_sec4_sg(req->assoc, 1205 (assoc_nents ? : 1), 1206 edesc->sec4_sg + 1207 sec4_sg_index, 0); 1208 sec4_sg_index += assoc_nents ? : 1; 1209 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 1210 iv_dma, ivsize, 0); 1211 sec4_sg_index += 1; 1212 sg_to_sec4_sg_last(req->src, 1213 (src_nents ? : 1), 1214 edesc->sec4_sg + 1215 sec4_sg_index, 0); 1216 sec4_sg_index += src_nents ? : 1; 1217 } 1218 if (dst_nents) { 1219 sg_to_sec4_sg_last(req->dst, dst_nents, 1220 edesc->sec4_sg + sec4_sg_index, 0); 1221 } 1222 1223 return edesc; 1224 } 1225 1226 static int aead_encrypt(struct aead_request *req) 1227 { 1228 struct aead_edesc *edesc; 1229 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1230 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1231 struct device *jrdev = ctx->jrdev; 1232 bool all_contig; 1233 u32 *desc; 1234 int ret = 0; 1235 1236 req->cryptlen += ctx->authsize; 1237 1238 /* allocate extended descriptor */ 1239 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * 1240 CAAM_CMD_SZ, &all_contig); 1241 if (IS_ERR(edesc)) 1242 return PTR_ERR(edesc); 1243 1244 /* Create and submit job descriptor */ 1245 init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req, 1246 all_contig, true); 1247 #ifdef DEBUG 1248 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1249 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1250 desc_bytes(edesc->hw_desc), 1); 1251 #endif 1252 1253 desc = edesc->hw_desc; 1254 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1255 if (!ret) { 1256 ret = -EINPROGRESS; 1257 } else { 1258 aead_unmap(jrdev, edesc, req); 1259 kfree(edesc); 1260 } 1261 1262 return ret; 1263 } 1264 1265 static int aead_decrypt(struct aead_request *req) 1266 { 1267 struct aead_edesc *edesc; 1268 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1269 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1270 struct device *jrdev = ctx->jrdev; 1271 bool all_contig; 1272 u32 *desc; 1273 int ret = 0; 1274 1275 /* allocate extended descriptor */ 1276 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * 1277 CAAM_CMD_SZ, &all_contig); 1278 if (IS_ERR(edesc)) 1279 return PTR_ERR(edesc); 1280 1281 #ifdef DEBUG 1282 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", 1283 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1284 req->cryptlen, 1); 1285 #endif 1286 1287 /* Create and submit job descriptor*/ 1288 init_aead_job(ctx->sh_desc_dec, 1289 ctx->sh_desc_dec_dma, edesc, req, all_contig, false); 1290 #ifdef DEBUG 1291 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1292 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1293 desc_bytes(edesc->hw_desc), 1); 1294 #endif 1295 1296 desc = edesc->hw_desc; 1297 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 1298 if (!ret) { 1299 ret = -EINPROGRESS; 1300 } else { 1301 aead_unmap(jrdev, edesc, req); 1302 kfree(edesc); 1303 } 1304 1305 return ret; 1306 } 1307 1308 /* 1309 * allocate and map the aead extended descriptor for aead givencrypt 1310 */ 1311 static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request 1312 *greq, int desc_bytes, 1313 u32 *contig_ptr) 1314 { 1315 struct aead_request *req = &greq->areq; 1316 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1317 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1318 struct device *jrdev = ctx->jrdev; 1319 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1320 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1321 int assoc_nents, src_nents, dst_nents = 0; 1322 struct aead_edesc *edesc; 1323 dma_addr_t iv_dma = 0; 1324 int sgc; 1325 u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG; 1326 int ivsize = crypto_aead_ivsize(aead); 1327 bool assoc_chained = false, src_chained = false, dst_chained = false; 1328 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 1329 1330 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); 1331 src_nents = sg_count(req->src, req->cryptlen, &src_chained); 1332 1333 if (unlikely(req->dst != req->src)) 1334 dst_nents = sg_count(req->dst, req->cryptlen, &dst_chained); 1335 1336 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, 1337 DMA_TO_DEVICE, assoc_chained); 1338 if (likely(req->src == req->dst)) { 1339 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1340 DMA_BIDIRECTIONAL, src_chained); 1341 } else { 1342 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1343 DMA_TO_DEVICE, src_chained); 1344 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 1345 DMA_FROM_DEVICE, dst_chained); 1346 } 1347 1348 /* Check if data are contiguous */ 1349 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 1350 if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != 1351 iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src)) 1352 contig &= ~GIV_SRC_CONTIG; 1353 if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst)) 1354 contig &= ~GIV_DST_CONTIG; 1355 if (unlikely(req->src != req->dst)) { 1356 dst_nents = dst_nents ? : 1; 1357 sec4_sg_len += 1; 1358 } 1359 if (!(contig & GIV_SRC_CONTIG)) { 1360 assoc_nents = assoc_nents ? : 1; 1361 src_nents = src_nents ? : 1; 1362 sec4_sg_len += assoc_nents + 1 + src_nents; 1363 if (likely(req->src == req->dst)) 1364 contig &= ~GIV_DST_CONTIG; 1365 } 1366 sec4_sg_len += dst_nents; 1367 1368 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1369 1370 /* allocate space for base edesc and hw desc commands, link tables */ 1371 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + 1372 sec4_sg_bytes, GFP_DMA | flags); 1373 if (!edesc) { 1374 dev_err(jrdev, "could not allocate extended descriptor\n"); 1375 return ERR_PTR(-ENOMEM); 1376 } 1377 1378 edesc->assoc_nents = assoc_nents; 1379 edesc->assoc_chained = assoc_chained; 1380 edesc->src_nents = src_nents; 1381 edesc->src_chained = src_chained; 1382 edesc->dst_nents = dst_nents; 1383 edesc->dst_chained = dst_chained; 1384 edesc->iv_dma = iv_dma; 1385 edesc->sec4_sg_bytes = sec4_sg_bytes; 1386 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1387 desc_bytes; 1388 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1389 sec4_sg_bytes, DMA_TO_DEVICE); 1390 *contig_ptr = contig; 1391 1392 sec4_sg_index = 0; 1393 if (!(contig & GIV_SRC_CONTIG)) { 1394 sg_to_sec4_sg(req->assoc, assoc_nents, 1395 edesc->sec4_sg + 1396 sec4_sg_index, 0); 1397 sec4_sg_index += assoc_nents; 1398 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 1399 iv_dma, ivsize, 0); 1400 sec4_sg_index += 1; 1401 sg_to_sec4_sg_last(req->src, src_nents, 1402 edesc->sec4_sg + 1403 sec4_sg_index, 0); 1404 sec4_sg_index += src_nents; 1405 } 1406 if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) { 1407 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 1408 iv_dma, ivsize, 0); 1409 sec4_sg_index += 1; 1410 sg_to_sec4_sg_last(req->dst, dst_nents, 1411 edesc->sec4_sg + sec4_sg_index, 0); 1412 } 1413 1414 return edesc; 1415 } 1416 1417 static int aead_givencrypt(struct aead_givcrypt_request *areq) 1418 { 1419 struct aead_request *req = &areq->areq; 1420 struct aead_edesc *edesc; 1421 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1422 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1423 struct device *jrdev = ctx->jrdev; 1424 u32 contig; 1425 u32 *desc; 1426 int ret = 0; 1427 1428 req->cryptlen += ctx->authsize; 1429 1430 /* allocate extended descriptor */ 1431 edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN * 1432 CAAM_CMD_SZ, &contig); 1433 1434 if (IS_ERR(edesc)) 1435 return PTR_ERR(edesc); 1436 1437 #ifdef DEBUG 1438 print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ", 1439 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1440 req->cryptlen, 1); 1441 #endif 1442 1443 /* Create and submit job descriptor*/ 1444 init_aead_giv_job(ctx->sh_desc_givenc, 1445 ctx->sh_desc_givenc_dma, edesc, req, contig); 1446 #ifdef DEBUG 1447 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1448 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1449 desc_bytes(edesc->hw_desc), 1); 1450 #endif 1451 1452 desc = edesc->hw_desc; 1453 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1454 if (!ret) { 1455 ret = -EINPROGRESS; 1456 } else { 1457 aead_unmap(jrdev, edesc, req); 1458 kfree(edesc); 1459 } 1460 1461 return ret; 1462 } 1463 1464 /* 1465 * allocate and map the ablkcipher extended descriptor for ablkcipher 1466 */ 1467 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 1468 *req, int desc_bytes, 1469 bool *iv_contig_out) 1470 { 1471 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1472 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1473 struct device *jrdev = ctx->jrdev; 1474 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1475 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 1476 GFP_KERNEL : GFP_ATOMIC; 1477 int src_nents, dst_nents = 0, sec4_sg_bytes; 1478 struct ablkcipher_edesc *edesc; 1479 dma_addr_t iv_dma = 0; 1480 bool iv_contig = false; 1481 int sgc; 1482 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1483 bool src_chained = false, dst_chained = false; 1484 int sec4_sg_index; 1485 1486 src_nents = sg_count(req->src, req->nbytes, &src_chained); 1487 1488 if (req->dst != req->src) 1489 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); 1490 1491 if (likely(req->src == req->dst)) { 1492 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1493 DMA_BIDIRECTIONAL, src_chained); 1494 } else { 1495 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 1496 DMA_TO_DEVICE, src_chained); 1497 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 1498 DMA_FROM_DEVICE, dst_chained); 1499 } 1500 1501 /* 1502 * Check if iv can be contiguous with source and destination. 1503 * If so, include it. If not, create scatterlist. 1504 */ 1505 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 1506 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) 1507 iv_contig = true; 1508 else 1509 src_nents = src_nents ? : 1; 1510 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 1511 sizeof(struct sec4_sg_entry); 1512 1513 /* allocate space for base edesc and hw desc commands, link tables */ 1514 edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes + 1515 sec4_sg_bytes, GFP_DMA | flags); 1516 if (!edesc) { 1517 dev_err(jrdev, "could not allocate extended descriptor\n"); 1518 return ERR_PTR(-ENOMEM); 1519 } 1520 1521 edesc->src_nents = src_nents; 1522 edesc->src_chained = src_chained; 1523 edesc->dst_nents = dst_nents; 1524 edesc->dst_chained = dst_chained; 1525 edesc->sec4_sg_bytes = sec4_sg_bytes; 1526 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 1527 desc_bytes; 1528 1529 sec4_sg_index = 0; 1530 if (!iv_contig) { 1531 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 1532 sg_to_sec4_sg_last(req->src, src_nents, 1533 edesc->sec4_sg + 1, 0); 1534 sec4_sg_index += 1 + src_nents; 1535 } 1536 1537 if (dst_nents) { 1538 sg_to_sec4_sg_last(req->dst, dst_nents, 1539 edesc->sec4_sg + sec4_sg_index, 0); 1540 } 1541 1542 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1543 sec4_sg_bytes, DMA_TO_DEVICE); 1544 edesc->iv_dma = iv_dma; 1545 1546 #ifdef DEBUG 1547 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", 1548 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1549 sec4_sg_bytes, 1); 1550 #endif 1551 1552 *iv_contig_out = iv_contig; 1553 return edesc; 1554 } 1555 1556 static int ablkcipher_encrypt(struct ablkcipher_request *req) 1557 { 1558 struct ablkcipher_edesc *edesc; 1559 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1560 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1561 struct device *jrdev = ctx->jrdev; 1562 bool iv_contig; 1563 u32 *desc; 1564 int ret = 0; 1565 1566 /* allocate extended descriptor */ 1567 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1568 CAAM_CMD_SZ, &iv_contig); 1569 if (IS_ERR(edesc)) 1570 return PTR_ERR(edesc); 1571 1572 /* Create and submit job descriptor*/ 1573 init_ablkcipher_job(ctx->sh_desc_enc, 1574 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 1575 #ifdef DEBUG 1576 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 1577 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1578 desc_bytes(edesc->hw_desc), 1); 1579 #endif 1580 desc = edesc->hw_desc; 1581 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 1582 1583 if (!ret) { 1584 ret = -EINPROGRESS; 1585 } else { 1586 ablkcipher_unmap(jrdev, edesc, req); 1587 kfree(edesc); 1588 } 1589 1590 return ret; 1591 } 1592 1593 static int ablkcipher_decrypt(struct ablkcipher_request *req) 1594 { 1595 struct ablkcipher_edesc *edesc; 1596 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1597 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1598 struct device *jrdev = ctx->jrdev; 1599 bool iv_contig; 1600 u32 *desc; 1601 int ret = 0; 1602 1603 /* allocate extended descriptor */ 1604 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1605 CAAM_CMD_SZ, &iv_contig); 1606 if (IS_ERR(edesc)) 1607 return PTR_ERR(edesc); 1608 1609 /* Create and submit job descriptor*/ 1610 init_ablkcipher_job(ctx->sh_desc_dec, 1611 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 1612 desc = edesc->hw_desc; 1613 #ifdef DEBUG 1614 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 1615 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1616 desc_bytes(edesc->hw_desc), 1); 1617 #endif 1618 1619 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 1620 if (!ret) { 1621 ret = -EINPROGRESS; 1622 } else { 1623 ablkcipher_unmap(jrdev, edesc, req); 1624 kfree(edesc); 1625 } 1626 1627 return ret; 1628 } 1629 1630 #define template_aead template_u.aead 1631 #define template_ablkcipher template_u.ablkcipher 1632 struct caam_alg_template { 1633 char name[CRYPTO_MAX_ALG_NAME]; 1634 char driver_name[CRYPTO_MAX_ALG_NAME]; 1635 unsigned int blocksize; 1636 u32 type; 1637 union { 1638 struct ablkcipher_alg ablkcipher; 1639 struct aead_alg aead; 1640 struct blkcipher_alg blkcipher; 1641 struct cipher_alg cipher; 1642 struct compress_alg compress; 1643 struct rng_alg rng; 1644 } template_u; 1645 u32 class1_alg_type; 1646 u32 class2_alg_type; 1647 u32 alg_op; 1648 }; 1649 1650 static struct caam_alg_template driver_algs[] = { 1651 /* single-pass ipsec_esp descriptor */ 1652 { 1653 .name = "authenc(hmac(md5),cbc(aes))", 1654 .driver_name = "authenc-hmac-md5-cbc-aes-caam", 1655 .blocksize = AES_BLOCK_SIZE, 1656 .type = CRYPTO_ALG_TYPE_AEAD, 1657 .template_aead = { 1658 .setkey = aead_setkey, 1659 .setauthsize = aead_setauthsize, 1660 .encrypt = aead_encrypt, 1661 .decrypt = aead_decrypt, 1662 .givencrypt = aead_givencrypt, 1663 .geniv = "<built-in>", 1664 .ivsize = AES_BLOCK_SIZE, 1665 .maxauthsize = MD5_DIGEST_SIZE, 1666 }, 1667 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1668 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 1669 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1670 }, 1671 { 1672 .name = "authenc(hmac(sha1),cbc(aes))", 1673 .driver_name = "authenc-hmac-sha1-cbc-aes-caam", 1674 .blocksize = AES_BLOCK_SIZE, 1675 .type = CRYPTO_ALG_TYPE_AEAD, 1676 .template_aead = { 1677 .setkey = aead_setkey, 1678 .setauthsize = aead_setauthsize, 1679 .encrypt = aead_encrypt, 1680 .decrypt = aead_decrypt, 1681 .givencrypt = aead_givencrypt, 1682 .geniv = "<built-in>", 1683 .ivsize = AES_BLOCK_SIZE, 1684 .maxauthsize = SHA1_DIGEST_SIZE, 1685 }, 1686 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1687 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 1688 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1689 }, 1690 { 1691 .name = "authenc(hmac(sha224),cbc(aes))", 1692 .driver_name = "authenc-hmac-sha224-cbc-aes-caam", 1693 .blocksize = AES_BLOCK_SIZE, 1694 .type = CRYPTO_ALG_TYPE_AEAD, 1695 .template_aead = { 1696 .setkey = aead_setkey, 1697 .setauthsize = aead_setauthsize, 1698 .encrypt = aead_encrypt, 1699 .decrypt = aead_decrypt, 1700 .givencrypt = aead_givencrypt, 1701 .geniv = "<built-in>", 1702 .ivsize = AES_BLOCK_SIZE, 1703 .maxauthsize = SHA224_DIGEST_SIZE, 1704 }, 1705 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1706 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1707 OP_ALG_AAI_HMAC_PRECOMP, 1708 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 1709 }, 1710 { 1711 .name = "authenc(hmac(sha256),cbc(aes))", 1712 .driver_name = "authenc-hmac-sha256-cbc-aes-caam", 1713 .blocksize = AES_BLOCK_SIZE, 1714 .type = CRYPTO_ALG_TYPE_AEAD, 1715 .template_aead = { 1716 .setkey = aead_setkey, 1717 .setauthsize = aead_setauthsize, 1718 .encrypt = aead_encrypt, 1719 .decrypt = aead_decrypt, 1720 .givencrypt = aead_givencrypt, 1721 .geniv = "<built-in>", 1722 .ivsize = AES_BLOCK_SIZE, 1723 .maxauthsize = SHA256_DIGEST_SIZE, 1724 }, 1725 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1726 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1727 OP_ALG_AAI_HMAC_PRECOMP, 1728 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1729 }, 1730 { 1731 .name = "authenc(hmac(sha384),cbc(aes))", 1732 .driver_name = "authenc-hmac-sha384-cbc-aes-caam", 1733 .blocksize = AES_BLOCK_SIZE, 1734 .type = CRYPTO_ALG_TYPE_AEAD, 1735 .template_aead = { 1736 .setkey = aead_setkey, 1737 .setauthsize = aead_setauthsize, 1738 .encrypt = aead_encrypt, 1739 .decrypt = aead_decrypt, 1740 .givencrypt = aead_givencrypt, 1741 .geniv = "<built-in>", 1742 .ivsize = AES_BLOCK_SIZE, 1743 .maxauthsize = SHA384_DIGEST_SIZE, 1744 }, 1745 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1746 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1747 OP_ALG_AAI_HMAC_PRECOMP, 1748 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 1749 }, 1750 1751 { 1752 .name = "authenc(hmac(sha512),cbc(aes))", 1753 .driver_name = "authenc-hmac-sha512-cbc-aes-caam", 1754 .blocksize = AES_BLOCK_SIZE, 1755 .type = CRYPTO_ALG_TYPE_AEAD, 1756 .template_aead = { 1757 .setkey = aead_setkey, 1758 .setauthsize = aead_setauthsize, 1759 .encrypt = aead_encrypt, 1760 .decrypt = aead_decrypt, 1761 .givencrypt = aead_givencrypt, 1762 .geniv = "<built-in>", 1763 .ivsize = AES_BLOCK_SIZE, 1764 .maxauthsize = SHA512_DIGEST_SIZE, 1765 }, 1766 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1767 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1768 OP_ALG_AAI_HMAC_PRECOMP, 1769 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 1770 }, 1771 { 1772 .name = "authenc(hmac(md5),cbc(des3_ede))", 1773 .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam", 1774 .blocksize = DES3_EDE_BLOCK_SIZE, 1775 .type = CRYPTO_ALG_TYPE_AEAD, 1776 .template_aead = { 1777 .setkey = aead_setkey, 1778 .setauthsize = aead_setauthsize, 1779 .encrypt = aead_encrypt, 1780 .decrypt = aead_decrypt, 1781 .givencrypt = aead_givencrypt, 1782 .geniv = "<built-in>", 1783 .ivsize = DES3_EDE_BLOCK_SIZE, 1784 .maxauthsize = MD5_DIGEST_SIZE, 1785 }, 1786 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1787 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 1788 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1789 }, 1790 { 1791 .name = "authenc(hmac(sha1),cbc(des3_ede))", 1792 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", 1793 .blocksize = DES3_EDE_BLOCK_SIZE, 1794 .type = CRYPTO_ALG_TYPE_AEAD, 1795 .template_aead = { 1796 .setkey = aead_setkey, 1797 .setauthsize = aead_setauthsize, 1798 .encrypt = aead_encrypt, 1799 .decrypt = aead_decrypt, 1800 .givencrypt = aead_givencrypt, 1801 .geniv = "<built-in>", 1802 .ivsize = DES3_EDE_BLOCK_SIZE, 1803 .maxauthsize = SHA1_DIGEST_SIZE, 1804 }, 1805 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1806 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 1807 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1808 }, 1809 { 1810 .name = "authenc(hmac(sha224),cbc(des3_ede))", 1811 .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam", 1812 .blocksize = DES3_EDE_BLOCK_SIZE, 1813 .type = CRYPTO_ALG_TYPE_AEAD, 1814 .template_aead = { 1815 .setkey = aead_setkey, 1816 .setauthsize = aead_setauthsize, 1817 .encrypt = aead_encrypt, 1818 .decrypt = aead_decrypt, 1819 .givencrypt = aead_givencrypt, 1820 .geniv = "<built-in>", 1821 .ivsize = DES3_EDE_BLOCK_SIZE, 1822 .maxauthsize = SHA224_DIGEST_SIZE, 1823 }, 1824 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1825 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1826 OP_ALG_AAI_HMAC_PRECOMP, 1827 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 1828 }, 1829 { 1830 .name = "authenc(hmac(sha256),cbc(des3_ede))", 1831 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", 1832 .blocksize = DES3_EDE_BLOCK_SIZE, 1833 .type = CRYPTO_ALG_TYPE_AEAD, 1834 .template_aead = { 1835 .setkey = aead_setkey, 1836 .setauthsize = aead_setauthsize, 1837 .encrypt = aead_encrypt, 1838 .decrypt = aead_decrypt, 1839 .givencrypt = aead_givencrypt, 1840 .geniv = "<built-in>", 1841 .ivsize = DES3_EDE_BLOCK_SIZE, 1842 .maxauthsize = SHA256_DIGEST_SIZE, 1843 }, 1844 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1845 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1846 OP_ALG_AAI_HMAC_PRECOMP, 1847 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1848 }, 1849 { 1850 .name = "authenc(hmac(sha384),cbc(des3_ede))", 1851 .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam", 1852 .blocksize = DES3_EDE_BLOCK_SIZE, 1853 .type = CRYPTO_ALG_TYPE_AEAD, 1854 .template_aead = { 1855 .setkey = aead_setkey, 1856 .setauthsize = aead_setauthsize, 1857 .encrypt = aead_encrypt, 1858 .decrypt = aead_decrypt, 1859 .givencrypt = aead_givencrypt, 1860 .geniv = "<built-in>", 1861 .ivsize = DES3_EDE_BLOCK_SIZE, 1862 .maxauthsize = SHA384_DIGEST_SIZE, 1863 }, 1864 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1865 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1866 OP_ALG_AAI_HMAC_PRECOMP, 1867 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 1868 }, 1869 { 1870 .name = "authenc(hmac(sha512),cbc(des3_ede))", 1871 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam", 1872 .blocksize = DES3_EDE_BLOCK_SIZE, 1873 .type = CRYPTO_ALG_TYPE_AEAD, 1874 .template_aead = { 1875 .setkey = aead_setkey, 1876 .setauthsize = aead_setauthsize, 1877 .encrypt = aead_encrypt, 1878 .decrypt = aead_decrypt, 1879 .givencrypt = aead_givencrypt, 1880 .geniv = "<built-in>", 1881 .ivsize = DES3_EDE_BLOCK_SIZE, 1882 .maxauthsize = SHA512_DIGEST_SIZE, 1883 }, 1884 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1885 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1886 OP_ALG_AAI_HMAC_PRECOMP, 1887 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 1888 }, 1889 { 1890 .name = "authenc(hmac(md5),cbc(des))", 1891 .driver_name = "authenc-hmac-md5-cbc-des-caam", 1892 .blocksize = DES_BLOCK_SIZE, 1893 .type = CRYPTO_ALG_TYPE_AEAD, 1894 .template_aead = { 1895 .setkey = aead_setkey, 1896 .setauthsize = aead_setauthsize, 1897 .encrypt = aead_encrypt, 1898 .decrypt = aead_decrypt, 1899 .givencrypt = aead_givencrypt, 1900 .geniv = "<built-in>", 1901 .ivsize = DES_BLOCK_SIZE, 1902 .maxauthsize = MD5_DIGEST_SIZE, 1903 }, 1904 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1905 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 1906 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1907 }, 1908 { 1909 .name = "authenc(hmac(sha1),cbc(des))", 1910 .driver_name = "authenc-hmac-sha1-cbc-des-caam", 1911 .blocksize = DES_BLOCK_SIZE, 1912 .type = CRYPTO_ALG_TYPE_AEAD, 1913 .template_aead = { 1914 .setkey = aead_setkey, 1915 .setauthsize = aead_setauthsize, 1916 .encrypt = aead_encrypt, 1917 .decrypt = aead_decrypt, 1918 .givencrypt = aead_givencrypt, 1919 .geniv = "<built-in>", 1920 .ivsize = DES_BLOCK_SIZE, 1921 .maxauthsize = SHA1_DIGEST_SIZE, 1922 }, 1923 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1924 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 1925 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1926 }, 1927 { 1928 .name = "authenc(hmac(sha224),cbc(des))", 1929 .driver_name = "authenc-hmac-sha224-cbc-des-caam", 1930 .blocksize = DES_BLOCK_SIZE, 1931 .type = CRYPTO_ALG_TYPE_AEAD, 1932 .template_aead = { 1933 .setkey = aead_setkey, 1934 .setauthsize = aead_setauthsize, 1935 .encrypt = aead_encrypt, 1936 .decrypt = aead_decrypt, 1937 .givencrypt = aead_givencrypt, 1938 .geniv = "<built-in>", 1939 .ivsize = DES_BLOCK_SIZE, 1940 .maxauthsize = SHA224_DIGEST_SIZE, 1941 }, 1942 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1943 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1944 OP_ALG_AAI_HMAC_PRECOMP, 1945 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 1946 }, 1947 { 1948 .name = "authenc(hmac(sha256),cbc(des))", 1949 .driver_name = "authenc-hmac-sha256-cbc-des-caam", 1950 .blocksize = DES_BLOCK_SIZE, 1951 .type = CRYPTO_ALG_TYPE_AEAD, 1952 .template_aead = { 1953 .setkey = aead_setkey, 1954 .setauthsize = aead_setauthsize, 1955 .encrypt = aead_encrypt, 1956 .decrypt = aead_decrypt, 1957 .givencrypt = aead_givencrypt, 1958 .geniv = "<built-in>", 1959 .ivsize = DES_BLOCK_SIZE, 1960 .maxauthsize = SHA256_DIGEST_SIZE, 1961 }, 1962 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1963 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1964 OP_ALG_AAI_HMAC_PRECOMP, 1965 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1966 }, 1967 { 1968 .name = "authenc(hmac(sha384),cbc(des))", 1969 .driver_name = "authenc-hmac-sha384-cbc-des-caam", 1970 .blocksize = DES_BLOCK_SIZE, 1971 .type = CRYPTO_ALG_TYPE_AEAD, 1972 .template_aead = { 1973 .setkey = aead_setkey, 1974 .setauthsize = aead_setauthsize, 1975 .encrypt = aead_encrypt, 1976 .decrypt = aead_decrypt, 1977 .givencrypt = aead_givencrypt, 1978 .geniv = "<built-in>", 1979 .ivsize = DES_BLOCK_SIZE, 1980 .maxauthsize = SHA384_DIGEST_SIZE, 1981 }, 1982 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1983 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1984 OP_ALG_AAI_HMAC_PRECOMP, 1985 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 1986 }, 1987 { 1988 .name = "authenc(hmac(sha512),cbc(des))", 1989 .driver_name = "authenc-hmac-sha512-cbc-des-caam", 1990 .blocksize = DES_BLOCK_SIZE, 1991 .type = CRYPTO_ALG_TYPE_AEAD, 1992 .template_aead = { 1993 .setkey = aead_setkey, 1994 .setauthsize = aead_setauthsize, 1995 .encrypt = aead_encrypt, 1996 .decrypt = aead_decrypt, 1997 .givencrypt = aead_givencrypt, 1998 .geniv = "<built-in>", 1999 .ivsize = DES_BLOCK_SIZE, 2000 .maxauthsize = SHA512_DIGEST_SIZE, 2001 }, 2002 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2003 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2004 OP_ALG_AAI_HMAC_PRECOMP, 2005 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 2006 }, 2007 /* ablkcipher descriptor */ 2008 { 2009 .name = "cbc(aes)", 2010 .driver_name = "cbc-aes-caam", 2011 .blocksize = AES_BLOCK_SIZE, 2012 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2013 .template_ablkcipher = { 2014 .setkey = ablkcipher_setkey, 2015 .encrypt = ablkcipher_encrypt, 2016 .decrypt = ablkcipher_decrypt, 2017 .geniv = "eseqiv", 2018 .min_keysize = AES_MIN_KEY_SIZE, 2019 .max_keysize = AES_MAX_KEY_SIZE, 2020 .ivsize = AES_BLOCK_SIZE, 2021 }, 2022 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2023 }, 2024 { 2025 .name = "cbc(des3_ede)", 2026 .driver_name = "cbc-3des-caam", 2027 .blocksize = DES3_EDE_BLOCK_SIZE, 2028 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2029 .template_ablkcipher = { 2030 .setkey = ablkcipher_setkey, 2031 .encrypt = ablkcipher_encrypt, 2032 .decrypt = ablkcipher_decrypt, 2033 .geniv = "eseqiv", 2034 .min_keysize = DES3_EDE_KEY_SIZE, 2035 .max_keysize = DES3_EDE_KEY_SIZE, 2036 .ivsize = DES3_EDE_BLOCK_SIZE, 2037 }, 2038 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2039 }, 2040 { 2041 .name = "cbc(des)", 2042 .driver_name = "cbc-des-caam", 2043 .blocksize = DES_BLOCK_SIZE, 2044 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2045 .template_ablkcipher = { 2046 .setkey = ablkcipher_setkey, 2047 .encrypt = ablkcipher_encrypt, 2048 .decrypt = ablkcipher_decrypt, 2049 .geniv = "eseqiv", 2050 .min_keysize = DES_KEY_SIZE, 2051 .max_keysize = DES_KEY_SIZE, 2052 .ivsize = DES_BLOCK_SIZE, 2053 }, 2054 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2055 } 2056 }; 2057 2058 struct caam_crypto_alg { 2059 struct list_head entry; 2060 struct device *ctrldev; 2061 int class1_alg_type; 2062 int class2_alg_type; 2063 int alg_op; 2064 struct crypto_alg crypto_alg; 2065 }; 2066 2067 static int caam_cra_init(struct crypto_tfm *tfm) 2068 { 2069 struct crypto_alg *alg = tfm->__crt_alg; 2070 struct caam_crypto_alg *caam_alg = 2071 container_of(alg, struct caam_crypto_alg, crypto_alg); 2072 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2073 struct caam_drv_private *priv = dev_get_drvdata(caam_alg->ctrldev); 2074 int tgt_jr = atomic_inc_return(&priv->tfm_count); 2075 2076 /* 2077 * distribute tfms across job rings to ensure in-order 2078 * crypto request processing per tfm 2079 */ 2080 ctx->jrdev = priv->jrdev[(tgt_jr / 2) % priv->total_jobrs]; 2081 2082 /* copy descriptor header template value */ 2083 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type; 2084 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type; 2085 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op; 2086 2087 return 0; 2088 } 2089 2090 static void caam_cra_exit(struct crypto_tfm *tfm) 2091 { 2092 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2093 2094 if (ctx->sh_desc_enc_dma && 2095 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) 2096 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, 2097 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 2098 if (ctx->sh_desc_dec_dma && 2099 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) 2100 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, 2101 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); 2102 if (ctx->sh_desc_givenc_dma && 2103 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) 2104 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, 2105 desc_bytes(ctx->sh_desc_givenc), 2106 DMA_TO_DEVICE); 2107 } 2108 2109 static void __exit caam_algapi_exit(void) 2110 { 2111 2112 struct device_node *dev_node; 2113 struct platform_device *pdev; 2114 struct device *ctrldev; 2115 struct caam_drv_private *priv; 2116 struct caam_crypto_alg *t_alg, *n; 2117 2118 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 2119 if (!dev_node) { 2120 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 2121 if (!dev_node) 2122 return; 2123 } 2124 2125 pdev = of_find_device_by_node(dev_node); 2126 if (!pdev) 2127 return; 2128 2129 ctrldev = &pdev->dev; 2130 of_node_put(dev_node); 2131 priv = dev_get_drvdata(ctrldev); 2132 2133 if (!priv->alg_list.next) 2134 return; 2135 2136 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 2137 crypto_unregister_alg(&t_alg->crypto_alg); 2138 list_del(&t_alg->entry); 2139 kfree(t_alg); 2140 } 2141 } 2142 2143 static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev, 2144 struct caam_alg_template 2145 *template) 2146 { 2147 struct caam_crypto_alg *t_alg; 2148 struct crypto_alg *alg; 2149 2150 t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL); 2151 if (!t_alg) { 2152 dev_err(ctrldev, "failed to allocate t_alg\n"); 2153 return ERR_PTR(-ENOMEM); 2154 } 2155 2156 alg = &t_alg->crypto_alg; 2157 2158 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 2159 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2160 template->driver_name); 2161 alg->cra_module = THIS_MODULE; 2162 alg->cra_init = caam_cra_init; 2163 alg->cra_exit = caam_cra_exit; 2164 alg->cra_priority = CAAM_CRA_PRIORITY; 2165 alg->cra_blocksize = template->blocksize; 2166 alg->cra_alignmask = 0; 2167 alg->cra_ctxsize = sizeof(struct caam_ctx); 2168 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 2169 template->type; 2170 switch (template->type) { 2171 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2172 alg->cra_type = &crypto_ablkcipher_type; 2173 alg->cra_ablkcipher = template->template_ablkcipher; 2174 break; 2175 case CRYPTO_ALG_TYPE_AEAD: 2176 alg->cra_type = &crypto_aead_type; 2177 alg->cra_aead = template->template_aead; 2178 break; 2179 } 2180 2181 t_alg->class1_alg_type = template->class1_alg_type; 2182 t_alg->class2_alg_type = template->class2_alg_type; 2183 t_alg->alg_op = template->alg_op; 2184 t_alg->ctrldev = ctrldev; 2185 2186 return t_alg; 2187 } 2188 2189 static int __init caam_algapi_init(void) 2190 { 2191 struct device_node *dev_node; 2192 struct platform_device *pdev; 2193 struct device *ctrldev; 2194 struct caam_drv_private *priv; 2195 int i = 0, err = 0; 2196 2197 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 2198 if (!dev_node) { 2199 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 2200 if (!dev_node) 2201 return -ENODEV; 2202 } 2203 2204 pdev = of_find_device_by_node(dev_node); 2205 if (!pdev) 2206 return -ENODEV; 2207 2208 ctrldev = &pdev->dev; 2209 priv = dev_get_drvdata(ctrldev); 2210 of_node_put(dev_node); 2211 2212 INIT_LIST_HEAD(&priv->alg_list); 2213 2214 atomic_set(&priv->tfm_count, -1); 2215 2216 /* register crypto algorithms the device supports */ 2217 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2218 /* TODO: check if h/w supports alg */ 2219 struct caam_crypto_alg *t_alg; 2220 2221 t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]); 2222 if (IS_ERR(t_alg)) { 2223 err = PTR_ERR(t_alg); 2224 dev_warn(ctrldev, "%s alg allocation failed\n", 2225 driver_algs[i].driver_name); 2226 continue; 2227 } 2228 2229 err = crypto_register_alg(&t_alg->crypto_alg); 2230 if (err) { 2231 dev_warn(ctrldev, "%s alg registration failed\n", 2232 t_alg->crypto_alg.cra_driver_name); 2233 kfree(t_alg); 2234 } else 2235 list_add_tail(&t_alg->entry, &priv->alg_list); 2236 } 2237 if (!list_empty(&priv->alg_list)) 2238 dev_info(ctrldev, "%s algorithms registered in /proc/crypto\n", 2239 (char *)of_get_property(dev_node, "compatible", NULL)); 2240 2241 return err; 2242 } 2243 2244 module_init(caam_algapi_init); 2245 module_exit(caam_algapi_exit); 2246 2247 MODULE_LICENSE("GPL"); 2248 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 2249 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 2250