1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * caam - Freescale FSL CAAM support for ahash functions of crypto API 4 * 5 * Copyright 2011 Freescale Semiconductor, Inc. 6 * Copyright 2018-2019 NXP 7 * 8 * Based on caamalg.c crypto API driver. 9 * 10 * relationship of digest job descriptor or first job descriptor after init to 11 * shared descriptors: 12 * 13 * --------------- --------------- 14 * | JobDesc #1 |-------------------->| ShareDesc | 15 * | *(packet 1) | | (hashKey) | 16 * --------------- | (operation) | 17 * --------------- 18 * 19 * relationship of subsequent job descriptors to shared descriptors: 20 * 21 * --------------- --------------- 22 * | JobDesc #2 |-------------------->| ShareDesc | 23 * | *(packet 2) | |------------->| (hashKey) | 24 * --------------- | |-------->| (operation) | 25 * . | | | (load ctx2) | 26 * . | | --------------- 27 * --------------- | | 28 * | JobDesc #3 |------| | 29 * | *(packet 3) | | 30 * --------------- | 31 * . | 32 * . | 33 * --------------- | 34 * | JobDesc #4 |------------ 35 * | *(packet 4) | 36 * --------------- 37 * 38 * The SharedDesc never changes for a connection unless rekeyed, but 39 * each packet will likely be in a different place. So all we need 40 * to know to process the packet is where the input is, where the 41 * output goes, and what context we want to process with. Context is 42 * in the SharedDesc, packet references in the JobDesc. 43 * 44 * So, a job desc looks like: 45 * 46 * --------------------- 47 * | Header | 48 * | ShareDesc Pointer | 49 * | SEQ_OUT_PTR | 50 * | (output buffer) | 51 * | (output length) | 52 * | SEQ_IN_PTR | 53 * | (input buffer) | 54 * | (input length) | 55 * --------------------- 56 */ 57 58 #include "compat.h" 59 60 #include "regs.h" 61 #include "intern.h" 62 #include "desc_constr.h" 63 #include "jr.h" 64 #include "error.h" 65 #include "sg_sw_sec4.h" 66 #include "key_gen.h" 67 #include "caamhash_desc.h" 68 69 #define CAAM_CRA_PRIORITY 3000 70 71 /* max hash key is max split key size */ 72 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 73 74 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 75 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE 76 77 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ 78 CAAM_MAX_HASH_KEY_SIZE) 79 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) 80 81 /* caam context sizes for hashes: running digest + 8 */ 82 #define HASH_MSG_LEN 8 83 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 84 85 #ifdef DEBUG 86 /* for print_hex_dumps with line references */ 87 #define debug(format, arg...) printk(format, arg) 88 #else 89 #define debug(format, arg...) 90 #endif 91 92 93 static struct list_head hash_list; 94 95 /* ahash per-session context */ 96 struct caam_hash_ctx { 97 u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 98 u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 99 u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 100 u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 101 u8 key[CAAM_MAX_HASH_KEY_SIZE] ____cacheline_aligned; 102 dma_addr_t sh_desc_update_dma ____cacheline_aligned; 103 dma_addr_t sh_desc_update_first_dma; 104 dma_addr_t sh_desc_fin_dma; 105 dma_addr_t sh_desc_digest_dma; 106 dma_addr_t key_dma; 107 enum dma_data_direction dir; 108 struct device *jrdev; 109 int ctx_len; 110 struct alginfo adata; 111 }; 112 113 /* ahash state */ 114 struct caam_hash_state { 115 dma_addr_t buf_dma; 116 dma_addr_t ctx_dma; 117 int ctx_dma_len; 118 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 119 int buflen_0; 120 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 121 int buflen_1; 122 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 123 int (*update)(struct ahash_request *req); 124 int (*final)(struct ahash_request *req); 125 int (*finup)(struct ahash_request *req); 126 int current_buf; 127 }; 128 129 struct caam_export_state { 130 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 131 u8 caam_ctx[MAX_CTX_LEN]; 132 int buflen; 133 int (*update)(struct ahash_request *req); 134 int (*final)(struct ahash_request *req); 135 int (*finup)(struct ahash_request *req); 136 }; 137 138 static inline void switch_buf(struct caam_hash_state *state) 139 { 140 state->current_buf ^= 1; 141 } 142 143 static inline u8 *current_buf(struct caam_hash_state *state) 144 { 145 return state->current_buf ? state->buf_1 : state->buf_0; 146 } 147 148 static inline u8 *alt_buf(struct caam_hash_state *state) 149 { 150 return state->current_buf ? state->buf_0 : state->buf_1; 151 } 152 153 static inline int *current_buflen(struct caam_hash_state *state) 154 { 155 return state->current_buf ? &state->buflen_1 : &state->buflen_0; 156 } 157 158 static inline int *alt_buflen(struct caam_hash_state *state) 159 { 160 return state->current_buf ? &state->buflen_0 : &state->buflen_1; 161 } 162 163 static inline bool is_cmac_aes(u32 algtype) 164 { 165 return (algtype & (OP_ALG_ALGSEL_MASK | OP_ALG_AAI_MASK)) == 166 (OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC); 167 } 168 /* Common job descriptor seq in/out ptr routines */ 169 170 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ 171 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, 172 struct caam_hash_state *state, 173 int ctx_len) 174 { 175 state->ctx_dma_len = ctx_len; 176 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, 177 ctx_len, DMA_FROM_DEVICE); 178 if (dma_mapping_error(jrdev, state->ctx_dma)) { 179 dev_err(jrdev, "unable to map ctx\n"); 180 state->ctx_dma = 0; 181 return -ENOMEM; 182 } 183 184 append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0); 185 186 return 0; 187 } 188 189 /* Map current buffer in state (if length > 0) and put it in link table */ 190 static inline int buf_map_to_sec4_sg(struct device *jrdev, 191 struct sec4_sg_entry *sec4_sg, 192 struct caam_hash_state *state) 193 { 194 int buflen = *current_buflen(state); 195 196 if (!buflen) 197 return 0; 198 199 state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen, 200 DMA_TO_DEVICE); 201 if (dma_mapping_error(jrdev, state->buf_dma)) { 202 dev_err(jrdev, "unable to map buf\n"); 203 state->buf_dma = 0; 204 return -ENOMEM; 205 } 206 207 dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0); 208 209 return 0; 210 } 211 212 /* Map state->caam_ctx, and add it to link table */ 213 static inline int ctx_map_to_sec4_sg(struct device *jrdev, 214 struct caam_hash_state *state, int ctx_len, 215 struct sec4_sg_entry *sec4_sg, u32 flag) 216 { 217 state->ctx_dma_len = ctx_len; 218 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag); 219 if (dma_mapping_error(jrdev, state->ctx_dma)) { 220 dev_err(jrdev, "unable to map ctx\n"); 221 state->ctx_dma = 0; 222 return -ENOMEM; 223 } 224 225 dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0); 226 227 return 0; 228 } 229 230 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 231 { 232 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 233 int digestsize = crypto_ahash_digestsize(ahash); 234 struct device *jrdev = ctx->jrdev; 235 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 236 u32 *desc; 237 238 ctx->adata.key_virt = ctx->key; 239 240 /* ahash_update shared descriptor */ 241 desc = ctx->sh_desc_update; 242 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 243 ctx->ctx_len, true, ctrlpriv->era); 244 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, 245 desc_bytes(desc), ctx->dir); 246 #ifdef DEBUG 247 print_hex_dump(KERN_ERR, 248 "ahash update shdesc@"__stringify(__LINE__)": ", 249 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 250 #endif 251 252 /* ahash_update_first shared descriptor */ 253 desc = ctx->sh_desc_update_first; 254 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 255 ctx->ctx_len, false, ctrlpriv->era); 256 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, 257 desc_bytes(desc), ctx->dir); 258 #ifdef DEBUG 259 print_hex_dump(KERN_ERR, 260 "ahash update first shdesc@"__stringify(__LINE__)": ", 261 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 262 #endif 263 264 /* ahash_final shared descriptor */ 265 desc = ctx->sh_desc_fin; 266 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 267 ctx->ctx_len, true, ctrlpriv->era); 268 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, 269 desc_bytes(desc), ctx->dir); 270 #ifdef DEBUG 271 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", 272 DUMP_PREFIX_ADDRESS, 16, 4, desc, 273 desc_bytes(desc), 1); 274 #endif 275 276 /* ahash_digest shared descriptor */ 277 desc = ctx->sh_desc_digest; 278 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 279 ctx->ctx_len, false, ctrlpriv->era); 280 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, 281 desc_bytes(desc), ctx->dir); 282 #ifdef DEBUG 283 print_hex_dump(KERN_ERR, 284 "ahash digest shdesc@"__stringify(__LINE__)": ", 285 DUMP_PREFIX_ADDRESS, 16, 4, desc, 286 desc_bytes(desc), 1); 287 #endif 288 289 return 0; 290 } 291 292 static int axcbc_set_sh_desc(struct crypto_ahash *ahash) 293 { 294 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 295 int digestsize = crypto_ahash_digestsize(ahash); 296 struct device *jrdev = ctx->jrdev; 297 u32 *desc; 298 299 /* key is loaded from memory for UPDATE and FINALIZE states */ 300 ctx->adata.key_dma = ctx->key_dma; 301 302 /* shared descriptor for ahash_update */ 303 desc = ctx->sh_desc_update; 304 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE, 305 ctx->ctx_len, ctx->ctx_len, 0); 306 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, 307 desc_bytes(desc), ctx->dir); 308 print_hex_dump_debug("axcbc update shdesc@" __stringify(__LINE__)" : ", 309 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 310 1); 311 312 /* shared descriptor for ahash_{final,finup} */ 313 desc = ctx->sh_desc_fin; 314 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, 315 digestsize, ctx->ctx_len, 0); 316 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, 317 desc_bytes(desc), ctx->dir); 318 print_hex_dump_debug("axcbc finup shdesc@" __stringify(__LINE__)" : ", 319 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 320 1); 321 322 /* key is immediate data for INIT and INITFINAL states */ 323 ctx->adata.key_virt = ctx->key; 324 325 /* shared descriptor for first invocation of ahash_update */ 326 desc = ctx->sh_desc_update_first; 327 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 328 ctx->ctx_len, ctx->key_dma); 329 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, 330 desc_bytes(desc), ctx->dir); 331 print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)" : ", 332 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 333 1); 334 335 /* shared descriptor for ahash_digest */ 336 desc = ctx->sh_desc_digest; 337 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, 338 digestsize, ctx->ctx_len, 0); 339 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, 340 desc_bytes(desc), ctx->dir); 341 print_hex_dump_debug("axcbc digest shdesc@" __stringify(__LINE__)" : ", 342 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 343 1); 344 return 0; 345 } 346 347 static int acmac_set_sh_desc(struct crypto_ahash *ahash) 348 { 349 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 350 int digestsize = crypto_ahash_digestsize(ahash); 351 struct device *jrdev = ctx->jrdev; 352 u32 *desc; 353 354 /* shared descriptor for ahash_update */ 355 desc = ctx->sh_desc_update; 356 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE, 357 ctx->ctx_len, ctx->ctx_len, 0); 358 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, 359 desc_bytes(desc), ctx->dir); 360 print_hex_dump_debug("acmac update shdesc@" __stringify(__LINE__)" : ", 361 DUMP_PREFIX_ADDRESS, 16, 4, desc, 362 desc_bytes(desc), 1); 363 364 /* shared descriptor for ahash_{final,finup} */ 365 desc = ctx->sh_desc_fin; 366 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, 367 digestsize, ctx->ctx_len, 0); 368 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, 369 desc_bytes(desc), ctx->dir); 370 print_hex_dump_debug("acmac finup shdesc@" __stringify(__LINE__)" : ", 371 DUMP_PREFIX_ADDRESS, 16, 4, desc, 372 desc_bytes(desc), 1); 373 374 /* shared descriptor for first invocation of ahash_update */ 375 desc = ctx->sh_desc_update_first; 376 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 377 ctx->ctx_len, 0); 378 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, 379 desc_bytes(desc), ctx->dir); 380 print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)" : ", 381 DUMP_PREFIX_ADDRESS, 16, 4, desc, 382 desc_bytes(desc), 1); 383 384 /* shared descriptor for ahash_digest */ 385 desc = ctx->sh_desc_digest; 386 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, 387 digestsize, ctx->ctx_len, 0); 388 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, 389 desc_bytes(desc), ctx->dir); 390 print_hex_dump_debug("acmac digest shdesc@" __stringify(__LINE__)" : ", 391 DUMP_PREFIX_ADDRESS, 16, 4, desc, 392 desc_bytes(desc), 1); 393 394 return 0; 395 } 396 397 /* Digest hash size if it is too large */ 398 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 399 u32 digestsize) 400 { 401 struct device *jrdev = ctx->jrdev; 402 u32 *desc; 403 struct split_key_result result; 404 dma_addr_t key_dma; 405 int ret; 406 407 desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); 408 if (!desc) { 409 dev_err(jrdev, "unable to allocate key input memory\n"); 410 return -ENOMEM; 411 } 412 413 init_job_desc(desc, 0); 414 415 key_dma = dma_map_single(jrdev, key, *keylen, DMA_BIDIRECTIONAL); 416 if (dma_mapping_error(jrdev, key_dma)) { 417 dev_err(jrdev, "unable to map key memory\n"); 418 kfree(desc); 419 return -ENOMEM; 420 } 421 422 /* Job descriptor to perform unkeyed hash on key_in */ 423 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 424 OP_ALG_AS_INITFINAL); 425 append_seq_in_ptr(desc, key_dma, *keylen, 0); 426 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 427 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 428 append_seq_out_ptr(desc, key_dma, digestsize, 0); 429 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 430 LDST_SRCDST_BYTE_CONTEXT); 431 432 #ifdef DEBUG 433 print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ", 434 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 435 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 436 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 437 #endif 438 439 result.err = 0; 440 init_completion(&result.completion); 441 442 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); 443 if (!ret) { 444 /* in progress */ 445 wait_for_completion(&result.completion); 446 ret = result.err; 447 #ifdef DEBUG 448 print_hex_dump(KERN_ERR, 449 "digested key@"__stringify(__LINE__)": ", 450 DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1); 451 #endif 452 } 453 dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL); 454 455 *keylen = digestsize; 456 457 kfree(desc); 458 459 return ret; 460 } 461 462 static int ahash_setkey(struct crypto_ahash *ahash, 463 const u8 *key, unsigned int keylen) 464 { 465 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 466 int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 467 int digestsize = crypto_ahash_digestsize(ahash); 468 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 469 int ret; 470 u8 *hashed_key = NULL; 471 472 #ifdef DEBUG 473 printk(KERN_ERR "keylen %d\n", keylen); 474 #endif 475 476 if (keylen > blocksize) { 477 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); 478 if (!hashed_key) 479 return -ENOMEM; 480 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 481 if (ret) 482 goto bad_free_key; 483 key = hashed_key; 484 } 485 486 /* 487 * If DKP is supported, use it in the shared descriptor to generate 488 * the split key. 489 */ 490 if (ctrlpriv->era >= 6) { 491 ctx->adata.key_inline = true; 492 ctx->adata.keylen = keylen; 493 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 494 OP_ALG_ALGSEL_MASK); 495 496 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 497 goto bad_free_key; 498 499 memcpy(ctx->key, key, keylen); 500 } else { 501 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, 502 keylen, CAAM_MAX_HASH_KEY_SIZE); 503 if (ret) 504 goto bad_free_key; 505 } 506 507 kfree(hashed_key); 508 return ahash_set_sh_desc(ahash); 509 bad_free_key: 510 kfree(hashed_key); 511 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); 512 return -EINVAL; 513 } 514 515 static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key, 516 unsigned int keylen) 517 { 518 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 519 struct device *jrdev = ctx->jrdev; 520 521 memcpy(ctx->key, key, keylen); 522 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); 523 ctx->adata.keylen = keylen; 524 525 print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ", 526 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, keylen, 1); 527 528 return axcbc_set_sh_desc(ahash); 529 } 530 531 static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key, 532 unsigned int keylen) 533 { 534 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 535 536 /* key is immediate data for all cmac shared descriptors */ 537 ctx->adata.key_virt = key; 538 ctx->adata.keylen = keylen; 539 540 print_hex_dump_debug("acmac ctx.key@" __stringify(__LINE__)" : ", 541 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 542 543 return acmac_set_sh_desc(ahash); 544 } 545 546 /* 547 * ahash_edesc - s/w-extended ahash descriptor 548 * @sec4_sg_dma: physical mapped address of h/w link table 549 * @src_nents: number of segments in input scatterlist 550 * @sec4_sg_bytes: length of dma mapped sec4_sg space 551 * @hw_desc: the h/w job descriptor followed by any referenced link tables 552 * @sec4_sg: h/w link table 553 */ 554 struct ahash_edesc { 555 dma_addr_t sec4_sg_dma; 556 int src_nents; 557 int sec4_sg_bytes; 558 u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned; 559 struct sec4_sg_entry sec4_sg[0]; 560 }; 561 562 static inline void ahash_unmap(struct device *dev, 563 struct ahash_edesc *edesc, 564 struct ahash_request *req, int dst_len) 565 { 566 struct caam_hash_state *state = ahash_request_ctx(req); 567 568 if (edesc->src_nents) 569 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 570 571 if (edesc->sec4_sg_bytes) 572 dma_unmap_single(dev, edesc->sec4_sg_dma, 573 edesc->sec4_sg_bytes, DMA_TO_DEVICE); 574 575 if (state->buf_dma) { 576 dma_unmap_single(dev, state->buf_dma, *current_buflen(state), 577 DMA_TO_DEVICE); 578 state->buf_dma = 0; 579 } 580 } 581 582 static inline void ahash_unmap_ctx(struct device *dev, 583 struct ahash_edesc *edesc, 584 struct ahash_request *req, int dst_len, u32 flag) 585 { 586 struct caam_hash_state *state = ahash_request_ctx(req); 587 588 if (state->ctx_dma) { 589 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 590 state->ctx_dma = 0; 591 } 592 ahash_unmap(dev, edesc, req, dst_len); 593 } 594 595 static void ahash_done(struct device *jrdev, u32 *desc, u32 err, 596 void *context) 597 { 598 struct ahash_request *req = context; 599 struct ahash_edesc *edesc; 600 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 601 int digestsize = crypto_ahash_digestsize(ahash); 602 struct caam_hash_state *state = ahash_request_ctx(req); 603 #ifdef DEBUG 604 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 605 606 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 607 #endif 608 609 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 610 if (err) 611 caam_jr_strstatus(jrdev, err); 612 613 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 614 memcpy(req->result, state->caam_ctx, digestsize); 615 kfree(edesc); 616 617 #ifdef DEBUG 618 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 619 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 620 ctx->ctx_len, 1); 621 #endif 622 623 req->base.complete(&req->base, err); 624 } 625 626 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, 627 void *context) 628 { 629 struct ahash_request *req = context; 630 struct ahash_edesc *edesc; 631 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 632 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 633 struct caam_hash_state *state = ahash_request_ctx(req); 634 #ifdef DEBUG 635 int digestsize = crypto_ahash_digestsize(ahash); 636 637 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 638 #endif 639 640 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 641 if (err) 642 caam_jr_strstatus(jrdev, err); 643 644 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 645 switch_buf(state); 646 kfree(edesc); 647 648 #ifdef DEBUG 649 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 650 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 651 ctx->ctx_len, 1); 652 if (req->result) 653 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 654 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 655 digestsize, 1); 656 #endif 657 658 req->base.complete(&req->base, err); 659 } 660 661 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, 662 void *context) 663 { 664 struct ahash_request *req = context; 665 struct ahash_edesc *edesc; 666 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 667 int digestsize = crypto_ahash_digestsize(ahash); 668 struct caam_hash_state *state = ahash_request_ctx(req); 669 #ifdef DEBUG 670 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 671 672 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 673 #endif 674 675 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 676 if (err) 677 caam_jr_strstatus(jrdev, err); 678 679 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL); 680 memcpy(req->result, state->caam_ctx, digestsize); 681 kfree(edesc); 682 683 #ifdef DEBUG 684 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 685 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 686 ctx->ctx_len, 1); 687 #endif 688 689 req->base.complete(&req->base, err); 690 } 691 692 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, 693 void *context) 694 { 695 struct ahash_request *req = context; 696 struct ahash_edesc *edesc; 697 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 698 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 699 struct caam_hash_state *state = ahash_request_ctx(req); 700 #ifdef DEBUG 701 int digestsize = crypto_ahash_digestsize(ahash); 702 703 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 704 #endif 705 706 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 707 if (err) 708 caam_jr_strstatus(jrdev, err); 709 710 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); 711 switch_buf(state); 712 kfree(edesc); 713 714 #ifdef DEBUG 715 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 716 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 717 ctx->ctx_len, 1); 718 if (req->result) 719 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 720 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 721 digestsize, 1); 722 #endif 723 724 req->base.complete(&req->base, err); 725 } 726 727 /* 728 * Allocate an enhanced descriptor, which contains the hardware descriptor 729 * and space for hardware scatter table containing sg_num entries. 730 */ 731 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx, 732 int sg_num, u32 *sh_desc, 733 dma_addr_t sh_desc_dma, 734 gfp_t flags) 735 { 736 struct ahash_edesc *edesc; 737 unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry); 738 739 edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags); 740 if (!edesc) { 741 dev_err(ctx->jrdev, "could not allocate extended descriptor\n"); 742 return NULL; 743 } 744 745 init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc), 746 HDR_SHARE_DEFER | HDR_REVERSE); 747 748 return edesc; 749 } 750 751 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx, 752 struct ahash_edesc *edesc, 753 struct ahash_request *req, int nents, 754 unsigned int first_sg, 755 unsigned int first_bytes, size_t to_hash) 756 { 757 dma_addr_t src_dma; 758 u32 options; 759 760 if (nents > 1 || first_sg) { 761 struct sec4_sg_entry *sg = edesc->sec4_sg; 762 unsigned int sgsize = sizeof(*sg) * (first_sg + nents); 763 764 sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0); 765 766 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE); 767 if (dma_mapping_error(ctx->jrdev, src_dma)) { 768 dev_err(ctx->jrdev, "unable to map S/G table\n"); 769 return -ENOMEM; 770 } 771 772 edesc->sec4_sg_bytes = sgsize; 773 edesc->sec4_sg_dma = src_dma; 774 options = LDST_SGF; 775 } else { 776 src_dma = sg_dma_address(req->src); 777 options = 0; 778 } 779 780 append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash, 781 options); 782 783 return 0; 784 } 785 786 /* submit update job descriptor */ 787 static int ahash_update_ctx(struct ahash_request *req) 788 { 789 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 790 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 791 struct caam_hash_state *state = ahash_request_ctx(req); 792 struct device *jrdev = ctx->jrdev; 793 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 794 GFP_KERNEL : GFP_ATOMIC; 795 u8 *buf = current_buf(state); 796 int *buflen = current_buflen(state); 797 u8 *next_buf = alt_buf(state); 798 int blocksize = crypto_ahash_blocksize(ahash); 799 int *next_buflen = alt_buflen(state), last_buflen; 800 int in_len = *buflen + req->nbytes, to_hash; 801 u32 *desc; 802 int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index; 803 struct ahash_edesc *edesc; 804 int ret = 0; 805 806 last_buflen = *next_buflen; 807 *next_buflen = in_len & (blocksize - 1); 808 to_hash = in_len - *next_buflen; 809 810 /* 811 * For XCBC and CMAC, if to_hash is multiple of block size, 812 * keep last block in internal buffer 813 */ 814 if ((is_xcbc_aes(ctx->adata.algtype) || 815 is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize && 816 (*next_buflen == 0)) { 817 *next_buflen = blocksize; 818 to_hash -= blocksize; 819 } 820 821 if (to_hash) { 822 src_nents = sg_nents_for_len(req->src, 823 req->nbytes - (*next_buflen)); 824 if (src_nents < 0) { 825 dev_err(jrdev, "Invalid number of src SG.\n"); 826 return src_nents; 827 } 828 829 if (src_nents) { 830 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 831 DMA_TO_DEVICE); 832 if (!mapped_nents) { 833 dev_err(jrdev, "unable to DMA map source\n"); 834 return -ENOMEM; 835 } 836 } else { 837 mapped_nents = 0; 838 } 839 840 sec4_sg_src_index = 1 + (*buflen ? 1 : 0); 841 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * 842 sizeof(struct sec4_sg_entry); 843 844 /* 845 * allocate space for base edesc and hw desc commands, 846 * link tables 847 */ 848 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 849 ctx->sh_desc_update, 850 ctx->sh_desc_update_dma, flags); 851 if (!edesc) { 852 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 853 return -ENOMEM; 854 } 855 856 edesc->src_nents = src_nents; 857 edesc->sec4_sg_bytes = sec4_sg_bytes; 858 859 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, 860 edesc->sec4_sg, DMA_BIDIRECTIONAL); 861 if (ret) 862 goto unmap_ctx; 863 864 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); 865 if (ret) 866 goto unmap_ctx; 867 868 if (mapped_nents) 869 sg_to_sec4_sg_last(req->src, mapped_nents, 870 edesc->sec4_sg + sec4_sg_src_index, 871 0); 872 else 873 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 874 1); 875 876 if (*next_buflen) 877 scatterwalk_map_and_copy(next_buf, req->src, 878 to_hash - *buflen, 879 *next_buflen, 0); 880 desc = edesc->hw_desc; 881 882 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 883 sec4_sg_bytes, 884 DMA_TO_DEVICE); 885 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 886 dev_err(jrdev, "unable to map S/G table\n"); 887 ret = -ENOMEM; 888 goto unmap_ctx; 889 } 890 891 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + 892 to_hash, LDST_SGF); 893 894 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); 895 896 #ifdef DEBUG 897 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 898 DUMP_PREFIX_ADDRESS, 16, 4, desc, 899 desc_bytes(desc), 1); 900 #endif 901 902 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req); 903 if (ret) 904 goto unmap_ctx; 905 906 ret = -EINPROGRESS; 907 } else if (*next_buflen) { 908 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 909 req->nbytes, 0); 910 *buflen = *next_buflen; 911 *next_buflen = last_buflen; 912 } 913 #ifdef DEBUG 914 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 915 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 916 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 917 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 918 *next_buflen, 1); 919 #endif 920 921 return ret; 922 unmap_ctx: 923 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 924 kfree(edesc); 925 return ret; 926 } 927 928 static int ahash_final_ctx(struct ahash_request *req) 929 { 930 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 931 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 932 struct caam_hash_state *state = ahash_request_ctx(req); 933 struct device *jrdev = ctx->jrdev; 934 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 935 GFP_KERNEL : GFP_ATOMIC; 936 int buflen = *current_buflen(state); 937 u32 *desc; 938 int sec4_sg_bytes, sec4_sg_src_index; 939 int digestsize = crypto_ahash_digestsize(ahash); 940 struct ahash_edesc *edesc; 941 int ret; 942 943 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 944 sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); 945 946 /* allocate space for base edesc and hw desc commands, link tables */ 947 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index, 948 ctx->sh_desc_fin, ctx->sh_desc_fin_dma, 949 flags); 950 if (!edesc) 951 return -ENOMEM; 952 953 desc = edesc->hw_desc; 954 955 edesc->sec4_sg_bytes = sec4_sg_bytes; 956 957 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, 958 edesc->sec4_sg, DMA_BIDIRECTIONAL); 959 if (ret) 960 goto unmap_ctx; 961 962 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); 963 if (ret) 964 goto unmap_ctx; 965 966 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1); 967 968 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 969 sec4_sg_bytes, DMA_TO_DEVICE); 970 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 971 dev_err(jrdev, "unable to map S/G table\n"); 972 ret = -ENOMEM; 973 goto unmap_ctx; 974 } 975 976 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen, 977 LDST_SGF); 978 append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); 979 980 #ifdef DEBUG 981 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 982 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 983 #endif 984 985 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 986 if (ret) 987 goto unmap_ctx; 988 989 return -EINPROGRESS; 990 unmap_ctx: 991 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL); 992 kfree(edesc); 993 return ret; 994 } 995 996 static int ahash_finup_ctx(struct ahash_request *req) 997 { 998 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 999 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1000 struct caam_hash_state *state = ahash_request_ctx(req); 1001 struct device *jrdev = ctx->jrdev; 1002 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1003 GFP_KERNEL : GFP_ATOMIC; 1004 int buflen = *current_buflen(state); 1005 u32 *desc; 1006 int sec4_sg_src_index; 1007 int src_nents, mapped_nents; 1008 int digestsize = crypto_ahash_digestsize(ahash); 1009 struct ahash_edesc *edesc; 1010 int ret; 1011 1012 src_nents = sg_nents_for_len(req->src, req->nbytes); 1013 if (src_nents < 0) { 1014 dev_err(jrdev, "Invalid number of src SG.\n"); 1015 return src_nents; 1016 } 1017 1018 if (src_nents) { 1019 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1020 DMA_TO_DEVICE); 1021 if (!mapped_nents) { 1022 dev_err(jrdev, "unable to DMA map source\n"); 1023 return -ENOMEM; 1024 } 1025 } else { 1026 mapped_nents = 0; 1027 } 1028 1029 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 1030 1031 /* allocate space for base edesc and hw desc commands, link tables */ 1032 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 1033 ctx->sh_desc_fin, ctx->sh_desc_fin_dma, 1034 flags); 1035 if (!edesc) { 1036 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1037 return -ENOMEM; 1038 } 1039 1040 desc = edesc->hw_desc; 1041 1042 edesc->src_nents = src_nents; 1043 1044 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, 1045 edesc->sec4_sg, DMA_BIDIRECTIONAL); 1046 if (ret) 1047 goto unmap_ctx; 1048 1049 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); 1050 if (ret) 1051 goto unmap_ctx; 1052 1053 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1054 sec4_sg_src_index, ctx->ctx_len + buflen, 1055 req->nbytes); 1056 if (ret) 1057 goto unmap_ctx; 1058 1059 append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); 1060 1061 #ifdef DEBUG 1062 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1063 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1064 #endif 1065 1066 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 1067 if (ret) 1068 goto unmap_ctx; 1069 1070 return -EINPROGRESS; 1071 unmap_ctx: 1072 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL); 1073 kfree(edesc); 1074 return ret; 1075 } 1076 1077 static int ahash_digest(struct ahash_request *req) 1078 { 1079 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1080 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1081 struct caam_hash_state *state = ahash_request_ctx(req); 1082 struct device *jrdev = ctx->jrdev; 1083 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1084 GFP_KERNEL : GFP_ATOMIC; 1085 u32 *desc; 1086 int digestsize = crypto_ahash_digestsize(ahash); 1087 int src_nents, mapped_nents; 1088 struct ahash_edesc *edesc; 1089 int ret; 1090 1091 state->buf_dma = 0; 1092 1093 src_nents = sg_nents_for_len(req->src, req->nbytes); 1094 if (src_nents < 0) { 1095 dev_err(jrdev, "Invalid number of src SG.\n"); 1096 return src_nents; 1097 } 1098 1099 if (src_nents) { 1100 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1101 DMA_TO_DEVICE); 1102 if (!mapped_nents) { 1103 dev_err(jrdev, "unable to map source for DMA\n"); 1104 return -ENOMEM; 1105 } 1106 } else { 1107 mapped_nents = 0; 1108 } 1109 1110 /* allocate space for base edesc and hw desc commands, link tables */ 1111 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0, 1112 ctx->sh_desc_digest, ctx->sh_desc_digest_dma, 1113 flags); 1114 if (!edesc) { 1115 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1116 return -ENOMEM; 1117 } 1118 1119 edesc->src_nents = src_nents; 1120 1121 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, 1122 req->nbytes); 1123 if (ret) { 1124 ahash_unmap(jrdev, edesc, req, digestsize); 1125 kfree(edesc); 1126 return ret; 1127 } 1128 1129 desc = edesc->hw_desc; 1130 1131 ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize); 1132 if (ret) { 1133 ahash_unmap(jrdev, edesc, req, digestsize); 1134 kfree(edesc); 1135 return -ENOMEM; 1136 } 1137 1138 #ifdef DEBUG 1139 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1140 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1141 #endif 1142 1143 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1144 if (!ret) { 1145 ret = -EINPROGRESS; 1146 } else { 1147 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 1148 kfree(edesc); 1149 } 1150 1151 return ret; 1152 } 1153 1154 /* submit ahash final if it the first job descriptor */ 1155 static int ahash_final_no_ctx(struct ahash_request *req) 1156 { 1157 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1158 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1159 struct caam_hash_state *state = ahash_request_ctx(req); 1160 struct device *jrdev = ctx->jrdev; 1161 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1162 GFP_KERNEL : GFP_ATOMIC; 1163 u8 *buf = current_buf(state); 1164 int buflen = *current_buflen(state); 1165 u32 *desc; 1166 int digestsize = crypto_ahash_digestsize(ahash); 1167 struct ahash_edesc *edesc; 1168 int ret; 1169 1170 /* allocate space for base edesc and hw desc commands, link tables */ 1171 edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest, 1172 ctx->sh_desc_digest_dma, flags); 1173 if (!edesc) 1174 return -ENOMEM; 1175 1176 desc = edesc->hw_desc; 1177 1178 if (buflen) { 1179 state->buf_dma = dma_map_single(jrdev, buf, buflen, 1180 DMA_TO_DEVICE); 1181 if (dma_mapping_error(jrdev, state->buf_dma)) { 1182 dev_err(jrdev, "unable to map src\n"); 1183 goto unmap; 1184 } 1185 1186 append_seq_in_ptr(desc, state->buf_dma, buflen, 0); 1187 } 1188 1189 ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize); 1190 if (ret) 1191 goto unmap; 1192 1193 #ifdef DEBUG 1194 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1195 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1196 #endif 1197 1198 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1199 if (!ret) { 1200 ret = -EINPROGRESS; 1201 } else { 1202 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 1203 kfree(edesc); 1204 } 1205 1206 return ret; 1207 unmap: 1208 ahash_unmap(jrdev, edesc, req, digestsize); 1209 kfree(edesc); 1210 return -ENOMEM; 1211 1212 } 1213 1214 /* submit ahash update if it the first job descriptor after update */ 1215 static int ahash_update_no_ctx(struct ahash_request *req) 1216 { 1217 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1218 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1219 struct caam_hash_state *state = ahash_request_ctx(req); 1220 struct device *jrdev = ctx->jrdev; 1221 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1222 GFP_KERNEL : GFP_ATOMIC; 1223 u8 *buf = current_buf(state); 1224 int *buflen = current_buflen(state); 1225 int blocksize = crypto_ahash_blocksize(ahash); 1226 u8 *next_buf = alt_buf(state); 1227 int *next_buflen = alt_buflen(state); 1228 int in_len = *buflen + req->nbytes, to_hash; 1229 int sec4_sg_bytes, src_nents, mapped_nents; 1230 struct ahash_edesc *edesc; 1231 u32 *desc; 1232 int ret = 0; 1233 1234 *next_buflen = in_len & (blocksize - 1); 1235 to_hash = in_len - *next_buflen; 1236 1237 /* 1238 * For XCBC and CMAC, if to_hash is multiple of block size, 1239 * keep last block in internal buffer 1240 */ 1241 if ((is_xcbc_aes(ctx->adata.algtype) || 1242 is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize && 1243 (*next_buflen == 0)) { 1244 *next_buflen = blocksize; 1245 to_hash -= blocksize; 1246 } 1247 1248 if (to_hash) { 1249 src_nents = sg_nents_for_len(req->src, 1250 req->nbytes - *next_buflen); 1251 if (src_nents < 0) { 1252 dev_err(jrdev, "Invalid number of src SG.\n"); 1253 return src_nents; 1254 } 1255 1256 if (src_nents) { 1257 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1258 DMA_TO_DEVICE); 1259 if (!mapped_nents) { 1260 dev_err(jrdev, "unable to DMA map source\n"); 1261 return -ENOMEM; 1262 } 1263 } else { 1264 mapped_nents = 0; 1265 } 1266 1267 sec4_sg_bytes = (1 + mapped_nents) * 1268 sizeof(struct sec4_sg_entry); 1269 1270 /* 1271 * allocate space for base edesc and hw desc commands, 1272 * link tables 1273 */ 1274 edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents, 1275 ctx->sh_desc_update_first, 1276 ctx->sh_desc_update_first_dma, 1277 flags); 1278 if (!edesc) { 1279 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1280 return -ENOMEM; 1281 } 1282 1283 edesc->src_nents = src_nents; 1284 edesc->sec4_sg_bytes = sec4_sg_bytes; 1285 1286 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); 1287 if (ret) 1288 goto unmap_ctx; 1289 1290 sg_to_sec4_sg_last(req->src, mapped_nents, 1291 edesc->sec4_sg + 1, 0); 1292 1293 if (*next_buflen) { 1294 scatterwalk_map_and_copy(next_buf, req->src, 1295 to_hash - *buflen, 1296 *next_buflen, 0); 1297 } 1298 1299 desc = edesc->hw_desc; 1300 1301 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1302 sec4_sg_bytes, 1303 DMA_TO_DEVICE); 1304 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1305 dev_err(jrdev, "unable to map S/G table\n"); 1306 ret = -ENOMEM; 1307 goto unmap_ctx; 1308 } 1309 1310 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF); 1311 1312 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1313 if (ret) 1314 goto unmap_ctx; 1315 1316 #ifdef DEBUG 1317 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1318 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1319 desc_bytes(desc), 1); 1320 #endif 1321 1322 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); 1323 if (ret) 1324 goto unmap_ctx; 1325 1326 ret = -EINPROGRESS; 1327 state->update = ahash_update_ctx; 1328 state->finup = ahash_finup_ctx; 1329 state->final = ahash_final_ctx; 1330 } else if (*next_buflen) { 1331 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 1332 req->nbytes, 0); 1333 *buflen = *next_buflen; 1334 *next_buflen = 0; 1335 } 1336 #ifdef DEBUG 1337 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 1338 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 1339 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1340 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1341 *next_buflen, 1); 1342 #endif 1343 1344 return ret; 1345 unmap_ctx: 1346 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 1347 kfree(edesc); 1348 return ret; 1349 } 1350 1351 /* submit ahash finup if it the first job descriptor after update */ 1352 static int ahash_finup_no_ctx(struct ahash_request *req) 1353 { 1354 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1355 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1356 struct caam_hash_state *state = ahash_request_ctx(req); 1357 struct device *jrdev = ctx->jrdev; 1358 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1359 GFP_KERNEL : GFP_ATOMIC; 1360 int buflen = *current_buflen(state); 1361 u32 *desc; 1362 int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents; 1363 int digestsize = crypto_ahash_digestsize(ahash); 1364 struct ahash_edesc *edesc; 1365 int ret; 1366 1367 src_nents = sg_nents_for_len(req->src, req->nbytes); 1368 if (src_nents < 0) { 1369 dev_err(jrdev, "Invalid number of src SG.\n"); 1370 return src_nents; 1371 } 1372 1373 if (src_nents) { 1374 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1375 DMA_TO_DEVICE); 1376 if (!mapped_nents) { 1377 dev_err(jrdev, "unable to DMA map source\n"); 1378 return -ENOMEM; 1379 } 1380 } else { 1381 mapped_nents = 0; 1382 } 1383 1384 sec4_sg_src_index = 2; 1385 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * 1386 sizeof(struct sec4_sg_entry); 1387 1388 /* allocate space for base edesc and hw desc commands, link tables */ 1389 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 1390 ctx->sh_desc_digest, ctx->sh_desc_digest_dma, 1391 flags); 1392 if (!edesc) { 1393 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1394 return -ENOMEM; 1395 } 1396 1397 desc = edesc->hw_desc; 1398 1399 edesc->src_nents = src_nents; 1400 edesc->sec4_sg_bytes = sec4_sg_bytes; 1401 1402 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); 1403 if (ret) 1404 goto unmap; 1405 1406 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen, 1407 req->nbytes); 1408 if (ret) { 1409 dev_err(jrdev, "unable to map S/G table\n"); 1410 goto unmap; 1411 } 1412 1413 ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize); 1414 if (ret) 1415 goto unmap; 1416 1417 #ifdef DEBUG 1418 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1419 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1420 #endif 1421 1422 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1423 if (!ret) { 1424 ret = -EINPROGRESS; 1425 } else { 1426 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 1427 kfree(edesc); 1428 } 1429 1430 return ret; 1431 unmap: 1432 ahash_unmap(jrdev, edesc, req, digestsize); 1433 kfree(edesc); 1434 return -ENOMEM; 1435 1436 } 1437 1438 /* submit first update job descriptor after init */ 1439 static int ahash_update_first(struct ahash_request *req) 1440 { 1441 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1442 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1443 struct caam_hash_state *state = ahash_request_ctx(req); 1444 struct device *jrdev = ctx->jrdev; 1445 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1446 GFP_KERNEL : GFP_ATOMIC; 1447 u8 *next_buf = alt_buf(state); 1448 int *next_buflen = alt_buflen(state); 1449 int to_hash; 1450 int blocksize = crypto_ahash_blocksize(ahash); 1451 u32 *desc; 1452 int src_nents, mapped_nents; 1453 struct ahash_edesc *edesc; 1454 int ret = 0; 1455 1456 *next_buflen = req->nbytes & (blocksize - 1); 1457 to_hash = req->nbytes - *next_buflen; 1458 1459 /* 1460 * For XCBC and CMAC, if to_hash is multiple of block size, 1461 * keep last block in internal buffer 1462 */ 1463 if ((is_xcbc_aes(ctx->adata.algtype) || 1464 is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize && 1465 (*next_buflen == 0)) { 1466 *next_buflen = blocksize; 1467 to_hash -= blocksize; 1468 } 1469 1470 if (to_hash) { 1471 src_nents = sg_nents_for_len(req->src, 1472 req->nbytes - *next_buflen); 1473 if (src_nents < 0) { 1474 dev_err(jrdev, "Invalid number of src SG.\n"); 1475 return src_nents; 1476 } 1477 1478 if (src_nents) { 1479 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1480 DMA_TO_DEVICE); 1481 if (!mapped_nents) { 1482 dev_err(jrdev, "unable to map source for DMA\n"); 1483 return -ENOMEM; 1484 } 1485 } else { 1486 mapped_nents = 0; 1487 } 1488 1489 /* 1490 * allocate space for base edesc and hw desc commands, 1491 * link tables 1492 */ 1493 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? 1494 mapped_nents : 0, 1495 ctx->sh_desc_update_first, 1496 ctx->sh_desc_update_first_dma, 1497 flags); 1498 if (!edesc) { 1499 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1500 return -ENOMEM; 1501 } 1502 1503 edesc->src_nents = src_nents; 1504 1505 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, 1506 to_hash); 1507 if (ret) 1508 goto unmap_ctx; 1509 1510 if (*next_buflen) 1511 scatterwalk_map_and_copy(next_buf, req->src, to_hash, 1512 *next_buflen, 0); 1513 1514 desc = edesc->hw_desc; 1515 1516 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1517 if (ret) 1518 goto unmap_ctx; 1519 1520 #ifdef DEBUG 1521 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1522 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1523 desc_bytes(desc), 1); 1524 #endif 1525 1526 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); 1527 if (ret) 1528 goto unmap_ctx; 1529 1530 ret = -EINPROGRESS; 1531 state->update = ahash_update_ctx; 1532 state->finup = ahash_finup_ctx; 1533 state->final = ahash_final_ctx; 1534 } else if (*next_buflen) { 1535 state->update = ahash_update_no_ctx; 1536 state->finup = ahash_finup_no_ctx; 1537 state->final = ahash_final_no_ctx; 1538 scatterwalk_map_and_copy(next_buf, req->src, 0, 1539 req->nbytes, 0); 1540 switch_buf(state); 1541 } 1542 #ifdef DEBUG 1543 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1544 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1545 *next_buflen, 1); 1546 #endif 1547 1548 return ret; 1549 unmap_ctx: 1550 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 1551 kfree(edesc); 1552 return ret; 1553 } 1554 1555 static int ahash_finup_first(struct ahash_request *req) 1556 { 1557 return ahash_digest(req); 1558 } 1559 1560 static int ahash_init(struct ahash_request *req) 1561 { 1562 struct caam_hash_state *state = ahash_request_ctx(req); 1563 1564 state->update = ahash_update_first; 1565 state->finup = ahash_finup_first; 1566 state->final = ahash_final_no_ctx; 1567 1568 state->ctx_dma = 0; 1569 state->ctx_dma_len = 0; 1570 state->current_buf = 0; 1571 state->buf_dma = 0; 1572 state->buflen_0 = 0; 1573 state->buflen_1 = 0; 1574 1575 return 0; 1576 } 1577 1578 static int ahash_update(struct ahash_request *req) 1579 { 1580 struct caam_hash_state *state = ahash_request_ctx(req); 1581 1582 return state->update(req); 1583 } 1584 1585 static int ahash_finup(struct ahash_request *req) 1586 { 1587 struct caam_hash_state *state = ahash_request_ctx(req); 1588 1589 return state->finup(req); 1590 } 1591 1592 static int ahash_final(struct ahash_request *req) 1593 { 1594 struct caam_hash_state *state = ahash_request_ctx(req); 1595 1596 return state->final(req); 1597 } 1598 1599 static int ahash_export(struct ahash_request *req, void *out) 1600 { 1601 struct caam_hash_state *state = ahash_request_ctx(req); 1602 struct caam_export_state *export = out; 1603 int len; 1604 u8 *buf; 1605 1606 if (state->current_buf) { 1607 buf = state->buf_1; 1608 len = state->buflen_1; 1609 } else { 1610 buf = state->buf_0; 1611 len = state->buflen_0; 1612 } 1613 1614 memcpy(export->buf, buf, len); 1615 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 1616 export->buflen = len; 1617 export->update = state->update; 1618 export->final = state->final; 1619 export->finup = state->finup; 1620 1621 return 0; 1622 } 1623 1624 static int ahash_import(struct ahash_request *req, const void *in) 1625 { 1626 struct caam_hash_state *state = ahash_request_ctx(req); 1627 const struct caam_export_state *export = in; 1628 1629 memset(state, 0, sizeof(*state)); 1630 memcpy(state->buf_0, export->buf, export->buflen); 1631 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 1632 state->buflen_0 = export->buflen; 1633 state->update = export->update; 1634 state->final = export->final; 1635 state->finup = export->finup; 1636 1637 return 0; 1638 } 1639 1640 struct caam_hash_template { 1641 char name[CRYPTO_MAX_ALG_NAME]; 1642 char driver_name[CRYPTO_MAX_ALG_NAME]; 1643 char hmac_name[CRYPTO_MAX_ALG_NAME]; 1644 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 1645 unsigned int blocksize; 1646 struct ahash_alg template_ahash; 1647 u32 alg_type; 1648 }; 1649 1650 /* ahash descriptors */ 1651 static struct caam_hash_template driver_hash[] = { 1652 { 1653 .name = "sha1", 1654 .driver_name = "sha1-caam", 1655 .hmac_name = "hmac(sha1)", 1656 .hmac_driver_name = "hmac-sha1-caam", 1657 .blocksize = SHA1_BLOCK_SIZE, 1658 .template_ahash = { 1659 .init = ahash_init, 1660 .update = ahash_update, 1661 .final = ahash_final, 1662 .finup = ahash_finup, 1663 .digest = ahash_digest, 1664 .export = ahash_export, 1665 .import = ahash_import, 1666 .setkey = ahash_setkey, 1667 .halg = { 1668 .digestsize = SHA1_DIGEST_SIZE, 1669 .statesize = sizeof(struct caam_export_state), 1670 }, 1671 }, 1672 .alg_type = OP_ALG_ALGSEL_SHA1, 1673 }, { 1674 .name = "sha224", 1675 .driver_name = "sha224-caam", 1676 .hmac_name = "hmac(sha224)", 1677 .hmac_driver_name = "hmac-sha224-caam", 1678 .blocksize = SHA224_BLOCK_SIZE, 1679 .template_ahash = { 1680 .init = ahash_init, 1681 .update = ahash_update, 1682 .final = ahash_final, 1683 .finup = ahash_finup, 1684 .digest = ahash_digest, 1685 .export = ahash_export, 1686 .import = ahash_import, 1687 .setkey = ahash_setkey, 1688 .halg = { 1689 .digestsize = SHA224_DIGEST_SIZE, 1690 .statesize = sizeof(struct caam_export_state), 1691 }, 1692 }, 1693 .alg_type = OP_ALG_ALGSEL_SHA224, 1694 }, { 1695 .name = "sha256", 1696 .driver_name = "sha256-caam", 1697 .hmac_name = "hmac(sha256)", 1698 .hmac_driver_name = "hmac-sha256-caam", 1699 .blocksize = SHA256_BLOCK_SIZE, 1700 .template_ahash = { 1701 .init = ahash_init, 1702 .update = ahash_update, 1703 .final = ahash_final, 1704 .finup = ahash_finup, 1705 .digest = ahash_digest, 1706 .export = ahash_export, 1707 .import = ahash_import, 1708 .setkey = ahash_setkey, 1709 .halg = { 1710 .digestsize = SHA256_DIGEST_SIZE, 1711 .statesize = sizeof(struct caam_export_state), 1712 }, 1713 }, 1714 .alg_type = OP_ALG_ALGSEL_SHA256, 1715 }, { 1716 .name = "sha384", 1717 .driver_name = "sha384-caam", 1718 .hmac_name = "hmac(sha384)", 1719 .hmac_driver_name = "hmac-sha384-caam", 1720 .blocksize = SHA384_BLOCK_SIZE, 1721 .template_ahash = { 1722 .init = ahash_init, 1723 .update = ahash_update, 1724 .final = ahash_final, 1725 .finup = ahash_finup, 1726 .digest = ahash_digest, 1727 .export = ahash_export, 1728 .import = ahash_import, 1729 .setkey = ahash_setkey, 1730 .halg = { 1731 .digestsize = SHA384_DIGEST_SIZE, 1732 .statesize = sizeof(struct caam_export_state), 1733 }, 1734 }, 1735 .alg_type = OP_ALG_ALGSEL_SHA384, 1736 }, { 1737 .name = "sha512", 1738 .driver_name = "sha512-caam", 1739 .hmac_name = "hmac(sha512)", 1740 .hmac_driver_name = "hmac-sha512-caam", 1741 .blocksize = SHA512_BLOCK_SIZE, 1742 .template_ahash = { 1743 .init = ahash_init, 1744 .update = ahash_update, 1745 .final = ahash_final, 1746 .finup = ahash_finup, 1747 .digest = ahash_digest, 1748 .export = ahash_export, 1749 .import = ahash_import, 1750 .setkey = ahash_setkey, 1751 .halg = { 1752 .digestsize = SHA512_DIGEST_SIZE, 1753 .statesize = sizeof(struct caam_export_state), 1754 }, 1755 }, 1756 .alg_type = OP_ALG_ALGSEL_SHA512, 1757 }, { 1758 .name = "md5", 1759 .driver_name = "md5-caam", 1760 .hmac_name = "hmac(md5)", 1761 .hmac_driver_name = "hmac-md5-caam", 1762 .blocksize = MD5_BLOCK_WORDS * 4, 1763 .template_ahash = { 1764 .init = ahash_init, 1765 .update = ahash_update, 1766 .final = ahash_final, 1767 .finup = ahash_finup, 1768 .digest = ahash_digest, 1769 .export = ahash_export, 1770 .import = ahash_import, 1771 .setkey = ahash_setkey, 1772 .halg = { 1773 .digestsize = MD5_DIGEST_SIZE, 1774 .statesize = sizeof(struct caam_export_state), 1775 }, 1776 }, 1777 .alg_type = OP_ALG_ALGSEL_MD5, 1778 }, { 1779 .hmac_name = "xcbc(aes)", 1780 .hmac_driver_name = "xcbc-aes-caam", 1781 .blocksize = AES_BLOCK_SIZE, 1782 .template_ahash = { 1783 .init = ahash_init, 1784 .update = ahash_update, 1785 .final = ahash_final, 1786 .finup = ahash_finup, 1787 .digest = ahash_digest, 1788 .export = ahash_export, 1789 .import = ahash_import, 1790 .setkey = axcbc_setkey, 1791 .halg = { 1792 .digestsize = AES_BLOCK_SIZE, 1793 .statesize = sizeof(struct caam_export_state), 1794 }, 1795 }, 1796 .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XCBC_MAC, 1797 }, { 1798 .hmac_name = "cmac(aes)", 1799 .hmac_driver_name = "cmac-aes-caam", 1800 .blocksize = AES_BLOCK_SIZE, 1801 .template_ahash = { 1802 .init = ahash_init, 1803 .update = ahash_update, 1804 .final = ahash_final, 1805 .finup = ahash_finup, 1806 .digest = ahash_digest, 1807 .export = ahash_export, 1808 .import = ahash_import, 1809 .setkey = acmac_setkey, 1810 .halg = { 1811 .digestsize = AES_BLOCK_SIZE, 1812 .statesize = sizeof(struct caam_export_state), 1813 }, 1814 }, 1815 .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC, 1816 }, 1817 }; 1818 1819 struct caam_hash_alg { 1820 struct list_head entry; 1821 int alg_type; 1822 struct ahash_alg ahash_alg; 1823 }; 1824 1825 static int caam_hash_cra_init(struct crypto_tfm *tfm) 1826 { 1827 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 1828 struct crypto_alg *base = tfm->__crt_alg; 1829 struct hash_alg_common *halg = 1830 container_of(base, struct hash_alg_common, base); 1831 struct ahash_alg *alg = 1832 container_of(halg, struct ahash_alg, halg); 1833 struct caam_hash_alg *caam_hash = 1834 container_of(alg, struct caam_hash_alg, ahash_alg); 1835 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1836 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 1837 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 1838 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 1839 HASH_MSG_LEN + 32, 1840 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 1841 HASH_MSG_LEN + 64, 1842 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 1843 dma_addr_t dma_addr; 1844 struct caam_drv_private *priv; 1845 1846 /* 1847 * Get a Job ring from Job Ring driver to ensure in-order 1848 * crypto request processing per tfm 1849 */ 1850 ctx->jrdev = caam_jr_alloc(); 1851 if (IS_ERR(ctx->jrdev)) { 1852 pr_err("Job Ring Device allocation for transform failed\n"); 1853 return PTR_ERR(ctx->jrdev); 1854 } 1855 1856 priv = dev_get_drvdata(ctx->jrdev->parent); 1857 1858 if (is_xcbc_aes(caam_hash->alg_type)) { 1859 ctx->dir = DMA_TO_DEVICE; 1860 ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type; 1861 ctx->ctx_len = 48; 1862 1863 ctx->key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key, 1864 ARRAY_SIZE(ctx->key), 1865 DMA_BIDIRECTIONAL, 1866 DMA_ATTR_SKIP_CPU_SYNC); 1867 if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { 1868 dev_err(ctx->jrdev, "unable to map key\n"); 1869 caam_jr_free(ctx->jrdev); 1870 return -ENOMEM; 1871 } 1872 } else if (is_cmac_aes(caam_hash->alg_type)) { 1873 ctx->dir = DMA_TO_DEVICE; 1874 ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type; 1875 ctx->ctx_len = 32; 1876 } else { 1877 ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1878 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 1879 ctx->ctx_len = runninglen[(ctx->adata.algtype & 1880 OP_ALG_ALGSEL_SUBMASK) >> 1881 OP_ALG_ALGSEL_SHIFT]; 1882 } 1883 1884 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update, 1885 offsetof(struct caam_hash_ctx, key), 1886 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1887 if (dma_mapping_error(ctx->jrdev, dma_addr)) { 1888 dev_err(ctx->jrdev, "unable to map shared descriptors\n"); 1889 1890 if (is_xcbc_aes(caam_hash->alg_type)) 1891 dma_unmap_single_attrs(ctx->jrdev, ctx->key_dma, 1892 ARRAY_SIZE(ctx->key), 1893 DMA_BIDIRECTIONAL, 1894 DMA_ATTR_SKIP_CPU_SYNC); 1895 1896 caam_jr_free(ctx->jrdev); 1897 return -ENOMEM; 1898 } 1899 1900 ctx->sh_desc_update_dma = dma_addr; 1901 ctx->sh_desc_update_first_dma = dma_addr + 1902 offsetof(struct caam_hash_ctx, 1903 sh_desc_update_first); 1904 ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx, 1905 sh_desc_fin); 1906 ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx, 1907 sh_desc_digest); 1908 1909 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1910 sizeof(struct caam_hash_state)); 1911 1912 /* 1913 * For keyed hash algorithms shared descriptors 1914 * will be created later in setkey() callback 1915 */ 1916 return alg->setkey ? 0 : ahash_set_sh_desc(ahash); 1917 } 1918 1919 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 1920 { 1921 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1922 1923 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma, 1924 offsetof(struct caam_hash_ctx, key), 1925 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1926 if (is_xcbc_aes(ctx->adata.algtype)) 1927 dma_unmap_single_attrs(ctx->jrdev, ctx->key_dma, 1928 ARRAY_SIZE(ctx->key), DMA_BIDIRECTIONAL, 1929 DMA_ATTR_SKIP_CPU_SYNC); 1930 caam_jr_free(ctx->jrdev); 1931 } 1932 1933 static void __exit caam_algapi_hash_exit(void) 1934 { 1935 struct caam_hash_alg *t_alg, *n; 1936 1937 if (!hash_list.next) 1938 return; 1939 1940 list_for_each_entry_safe(t_alg, n, &hash_list, entry) { 1941 crypto_unregister_ahash(&t_alg->ahash_alg); 1942 list_del(&t_alg->entry); 1943 kfree(t_alg); 1944 } 1945 } 1946 1947 static struct caam_hash_alg * 1948 caam_hash_alloc(struct caam_hash_template *template, 1949 bool keyed) 1950 { 1951 struct caam_hash_alg *t_alg; 1952 struct ahash_alg *halg; 1953 struct crypto_alg *alg; 1954 1955 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 1956 if (!t_alg) { 1957 pr_err("failed to allocate t_alg\n"); 1958 return ERR_PTR(-ENOMEM); 1959 } 1960 1961 t_alg->ahash_alg = template->template_ahash; 1962 halg = &t_alg->ahash_alg; 1963 alg = &halg->halg.base; 1964 1965 if (keyed) { 1966 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1967 template->hmac_name); 1968 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1969 template->hmac_driver_name); 1970 } else { 1971 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1972 template->name); 1973 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1974 template->driver_name); 1975 t_alg->ahash_alg.setkey = NULL; 1976 } 1977 alg->cra_module = THIS_MODULE; 1978 alg->cra_init = caam_hash_cra_init; 1979 alg->cra_exit = caam_hash_cra_exit; 1980 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 1981 alg->cra_priority = CAAM_CRA_PRIORITY; 1982 alg->cra_blocksize = template->blocksize; 1983 alg->cra_alignmask = 0; 1984 alg->cra_flags = CRYPTO_ALG_ASYNC; 1985 1986 t_alg->alg_type = template->alg_type; 1987 1988 return t_alg; 1989 } 1990 1991 static int __init caam_algapi_hash_init(void) 1992 { 1993 struct device_node *dev_node; 1994 struct platform_device *pdev; 1995 int i = 0, err = 0; 1996 struct caam_drv_private *priv; 1997 unsigned int md_limit = SHA512_DIGEST_SIZE; 1998 u32 md_inst, md_vid; 1999 2000 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 2001 if (!dev_node) { 2002 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 2003 if (!dev_node) 2004 return -ENODEV; 2005 } 2006 2007 pdev = of_find_device_by_node(dev_node); 2008 if (!pdev) { 2009 of_node_put(dev_node); 2010 return -ENODEV; 2011 } 2012 2013 priv = dev_get_drvdata(&pdev->dev); 2014 of_node_put(dev_node); 2015 2016 /* 2017 * If priv is NULL, it's probably because the caam driver wasn't 2018 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 2019 */ 2020 if (!priv) { 2021 err = -ENODEV; 2022 goto out_put_dev; 2023 } 2024 2025 /* 2026 * Register crypto algorithms the device supports. First, identify 2027 * presence and attributes of MD block. 2028 */ 2029 if (priv->era < 10) { 2030 md_vid = (rd_reg32(&priv->ctrl->perfmon.cha_id_ls) & 2031 CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2032 md_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & 2033 CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2034 } else { 2035 u32 mdha = rd_reg32(&priv->ctrl->vreg.mdha); 2036 2037 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 2038 md_inst = mdha & CHA_VER_NUM_MASK; 2039 } 2040 2041 /* 2042 * Skip registration of any hashing algorithms if MD block 2043 * is not present. 2044 */ 2045 if (!md_inst) { 2046 err = -ENODEV; 2047 goto out_put_dev; 2048 } 2049 2050 /* Limit digest size based on LP256 */ 2051 if (md_vid == CHA_VER_VID_MD_LP256) 2052 md_limit = SHA256_DIGEST_SIZE; 2053 2054 INIT_LIST_HEAD(&hash_list); 2055 2056 /* register crypto algorithms the device supports */ 2057 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 2058 struct caam_hash_alg *t_alg; 2059 struct caam_hash_template *alg = driver_hash + i; 2060 2061 /* If MD size is not supported by device, skip registration */ 2062 if (is_mdha(alg->alg_type) && 2063 alg->template_ahash.halg.digestsize > md_limit) 2064 continue; 2065 2066 /* register hmac version */ 2067 t_alg = caam_hash_alloc(alg, true); 2068 if (IS_ERR(t_alg)) { 2069 err = PTR_ERR(t_alg); 2070 pr_warn("%s alg allocation failed\n", 2071 alg->hmac_driver_name); 2072 continue; 2073 } 2074 2075 err = crypto_register_ahash(&t_alg->ahash_alg); 2076 if (err) { 2077 pr_warn("%s alg registration failed: %d\n", 2078 t_alg->ahash_alg.halg.base.cra_driver_name, 2079 err); 2080 kfree(t_alg); 2081 } else 2082 list_add_tail(&t_alg->entry, &hash_list); 2083 2084 if ((alg->alg_type & OP_ALG_ALGSEL_MASK) == OP_ALG_ALGSEL_AES) 2085 continue; 2086 2087 /* register unkeyed version */ 2088 t_alg = caam_hash_alloc(alg, false); 2089 if (IS_ERR(t_alg)) { 2090 err = PTR_ERR(t_alg); 2091 pr_warn("%s alg allocation failed\n", alg->driver_name); 2092 continue; 2093 } 2094 2095 err = crypto_register_ahash(&t_alg->ahash_alg); 2096 if (err) { 2097 pr_warn("%s alg registration failed: %d\n", 2098 t_alg->ahash_alg.halg.base.cra_driver_name, 2099 err); 2100 kfree(t_alg); 2101 } else 2102 list_add_tail(&t_alg->entry, &hash_list); 2103 } 2104 2105 out_put_dev: 2106 put_device(&pdev->dev); 2107 return err; 2108 } 2109 2110 module_init(caam_algapi_hash_init); 2111 module_exit(caam_algapi_hash_exit); 2112 2113 MODULE_LICENSE("GPL"); 2114 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API"); 2115 MODULE_AUTHOR("Freescale Semiconductor - NMG"); 2116