1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/string.h> 7 #include <crypto/algapi.h> 8 #include <crypto/hash.h> 9 #include <crypto/md5.h> 10 #include <crypto/sm3.h> 11 #include <crypto/internal/hash.h> 12 13 #include "cc_driver.h" 14 #include "cc_request_mgr.h" 15 #include "cc_buffer_mgr.h" 16 #include "cc_hash.h" 17 #include "cc_sram_mgr.h" 18 19 #define CC_MAX_HASH_SEQ_LEN 12 20 #define CC_MAX_OPAD_KEYS_SIZE CC_MAX_HASH_BLCK_SIZE 21 #define CC_SM3_HASH_LEN_SIZE 8 22 23 struct cc_hash_handle { 24 u32 digest_len_sram_addr; /* const value in SRAM*/ 25 u32 larval_digest_sram_addr; /* const value in SRAM */ 26 struct list_head hash_list; 27 }; 28 29 static const u32 cc_digest_len_init[] = { 30 0x00000040, 0x00000000, 0x00000000, 0x00000000 }; 31 static const u32 cc_md5_init[] = { 32 SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 }; 33 static const u32 cc_sha1_init[] = { 34 SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 }; 35 static const u32 cc_sha224_init[] = { 36 SHA224_H7, SHA224_H6, SHA224_H5, SHA224_H4, 37 SHA224_H3, SHA224_H2, SHA224_H1, SHA224_H0 }; 38 static const u32 cc_sha256_init[] = { 39 SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4, 40 SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 }; 41 static const u32 cc_digest_len_sha512_init[] = { 42 0x00000080, 0x00000000, 0x00000000, 0x00000000 }; 43 44 /* 45 * Due to the way the HW works, every double word in the SHA384 and SHA512 46 * larval hashes must be stored in hi/lo order 47 */ 48 #define hilo(x) upper_32_bits(x), lower_32_bits(x) 49 static const u32 cc_sha384_init[] = { 50 hilo(SHA384_H7), hilo(SHA384_H6), hilo(SHA384_H5), hilo(SHA384_H4), 51 hilo(SHA384_H3), hilo(SHA384_H2), hilo(SHA384_H1), hilo(SHA384_H0) }; 52 static const u32 cc_sha512_init[] = { 53 hilo(SHA512_H7), hilo(SHA512_H6), hilo(SHA512_H5), hilo(SHA512_H4), 54 hilo(SHA512_H3), hilo(SHA512_H2), hilo(SHA512_H1), hilo(SHA512_H0) }; 55 56 static const u32 cc_sm3_init[] = { 57 SM3_IVH, SM3_IVG, SM3_IVF, SM3_IVE, 58 SM3_IVD, SM3_IVC, SM3_IVB, SM3_IVA }; 59 60 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[], 61 unsigned int *seq_size); 62 63 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[], 64 unsigned int *seq_size); 65 66 static const void *cc_larval_digest(struct device *dev, u32 mode); 67 68 struct cc_hash_alg { 69 struct list_head entry; 70 int hash_mode; 71 int hw_mode; 72 int inter_digestsize; 73 struct cc_drvdata *drvdata; 74 struct ahash_alg ahash_alg; 75 }; 76 77 struct hash_key_req_ctx { 78 u32 keylen; 79 dma_addr_t key_dma_addr; 80 u8 *key; 81 }; 82 83 /* hash per-session context */ 84 struct cc_hash_ctx { 85 struct cc_drvdata *drvdata; 86 /* holds the origin digest; the digest after "setkey" if HMAC,* 87 * the initial digest if HASH. 88 */ 89 u8 digest_buff[CC_MAX_HASH_DIGEST_SIZE] ____cacheline_aligned; 90 u8 opad_tmp_keys_buff[CC_MAX_OPAD_KEYS_SIZE] ____cacheline_aligned; 91 92 dma_addr_t opad_tmp_keys_dma_addr ____cacheline_aligned; 93 dma_addr_t digest_buff_dma_addr; 94 /* use for hmac with key large then mode block size */ 95 struct hash_key_req_ctx key_params; 96 int hash_mode; 97 int hw_mode; 98 int inter_digestsize; 99 unsigned int hash_len; 100 struct completion setkey_comp; 101 bool is_hmac; 102 }; 103 104 static void cc_set_desc(struct ahash_req_ctx *areq_ctx, struct cc_hash_ctx *ctx, 105 unsigned int flow_mode, struct cc_hw_desc desc[], 106 bool is_not_last_data, unsigned int *seq_size); 107 108 static void cc_set_endianity(u32 mode, struct cc_hw_desc *desc) 109 { 110 if (mode == DRV_HASH_MD5 || mode == DRV_HASH_SHA384 || 111 mode == DRV_HASH_SHA512) { 112 set_bytes_swap(desc, 1); 113 } else { 114 set_cipher_config0(desc, HASH_DIGEST_RESULT_LITTLE_ENDIAN); 115 } 116 } 117 118 static int cc_map_result(struct device *dev, struct ahash_req_ctx *state, 119 unsigned int digestsize) 120 { 121 state->digest_result_dma_addr = 122 dma_map_single(dev, state->digest_result_buff, 123 digestsize, DMA_BIDIRECTIONAL); 124 if (dma_mapping_error(dev, state->digest_result_dma_addr)) { 125 dev_err(dev, "Mapping digest result buffer %u B for DMA failed\n", 126 digestsize); 127 return -ENOMEM; 128 } 129 dev_dbg(dev, "Mapped digest result buffer %u B at va=%p to dma=%pad\n", 130 digestsize, state->digest_result_buff, 131 &state->digest_result_dma_addr); 132 133 return 0; 134 } 135 136 static void cc_init_req(struct device *dev, struct ahash_req_ctx *state, 137 struct cc_hash_ctx *ctx) 138 { 139 bool is_hmac = ctx->is_hmac; 140 141 memset(state, 0, sizeof(*state)); 142 143 if (is_hmac) { 144 if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC && 145 ctx->hw_mode != DRV_CIPHER_CMAC) { 146 dma_sync_single_for_cpu(dev, ctx->digest_buff_dma_addr, 147 ctx->inter_digestsize, 148 DMA_BIDIRECTIONAL); 149 150 memcpy(state->digest_buff, ctx->digest_buff, 151 ctx->inter_digestsize); 152 if (ctx->hash_mode == DRV_HASH_SHA512 || 153 ctx->hash_mode == DRV_HASH_SHA384) 154 memcpy(state->digest_bytes_len, 155 cc_digest_len_sha512_init, 156 ctx->hash_len); 157 else 158 memcpy(state->digest_bytes_len, 159 cc_digest_len_init, 160 ctx->hash_len); 161 } 162 163 if (ctx->hash_mode != DRV_HASH_NULL) { 164 dma_sync_single_for_cpu(dev, 165 ctx->opad_tmp_keys_dma_addr, 166 ctx->inter_digestsize, 167 DMA_BIDIRECTIONAL); 168 memcpy(state->opad_digest_buff, 169 ctx->opad_tmp_keys_buff, ctx->inter_digestsize); 170 } 171 } else { /*hash*/ 172 /* Copy the initial digests if hash flow. */ 173 const void *larval = cc_larval_digest(dev, ctx->hash_mode); 174 175 memcpy(state->digest_buff, larval, ctx->inter_digestsize); 176 } 177 } 178 179 static int cc_map_req(struct device *dev, struct ahash_req_ctx *state, 180 struct cc_hash_ctx *ctx) 181 { 182 bool is_hmac = ctx->is_hmac; 183 184 state->digest_buff_dma_addr = 185 dma_map_single(dev, state->digest_buff, 186 ctx->inter_digestsize, DMA_BIDIRECTIONAL); 187 if (dma_mapping_error(dev, state->digest_buff_dma_addr)) { 188 dev_err(dev, "Mapping digest len %d B at va=%p for DMA failed\n", 189 ctx->inter_digestsize, state->digest_buff); 190 return -EINVAL; 191 } 192 dev_dbg(dev, "Mapped digest %d B at va=%p to dma=%pad\n", 193 ctx->inter_digestsize, state->digest_buff, 194 &state->digest_buff_dma_addr); 195 196 if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC) { 197 state->digest_bytes_len_dma_addr = 198 dma_map_single(dev, state->digest_bytes_len, 199 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); 200 if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) { 201 dev_err(dev, "Mapping digest len %u B at va=%p for DMA failed\n", 202 HASH_MAX_LEN_SIZE, state->digest_bytes_len); 203 goto unmap_digest_buf; 204 } 205 dev_dbg(dev, "Mapped digest len %u B at va=%p to dma=%pad\n", 206 HASH_MAX_LEN_SIZE, state->digest_bytes_len, 207 &state->digest_bytes_len_dma_addr); 208 } 209 210 if (is_hmac && ctx->hash_mode != DRV_HASH_NULL) { 211 state->opad_digest_dma_addr = 212 dma_map_single(dev, state->opad_digest_buff, 213 ctx->inter_digestsize, 214 DMA_BIDIRECTIONAL); 215 if (dma_mapping_error(dev, state->opad_digest_dma_addr)) { 216 dev_err(dev, "Mapping opad digest %d B at va=%p for DMA failed\n", 217 ctx->inter_digestsize, 218 state->opad_digest_buff); 219 goto unmap_digest_len; 220 } 221 dev_dbg(dev, "Mapped opad digest %d B at va=%p to dma=%pad\n", 222 ctx->inter_digestsize, state->opad_digest_buff, 223 &state->opad_digest_dma_addr); 224 } 225 226 return 0; 227 228 unmap_digest_len: 229 if (state->digest_bytes_len_dma_addr) { 230 dma_unmap_single(dev, state->digest_bytes_len_dma_addr, 231 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); 232 state->digest_bytes_len_dma_addr = 0; 233 } 234 unmap_digest_buf: 235 if (state->digest_buff_dma_addr) { 236 dma_unmap_single(dev, state->digest_buff_dma_addr, 237 ctx->inter_digestsize, DMA_BIDIRECTIONAL); 238 state->digest_buff_dma_addr = 0; 239 } 240 241 return -EINVAL; 242 } 243 244 static void cc_unmap_req(struct device *dev, struct ahash_req_ctx *state, 245 struct cc_hash_ctx *ctx) 246 { 247 if (state->digest_buff_dma_addr) { 248 dma_unmap_single(dev, state->digest_buff_dma_addr, 249 ctx->inter_digestsize, DMA_BIDIRECTIONAL); 250 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n", 251 &state->digest_buff_dma_addr); 252 state->digest_buff_dma_addr = 0; 253 } 254 if (state->digest_bytes_len_dma_addr) { 255 dma_unmap_single(dev, state->digest_bytes_len_dma_addr, 256 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL); 257 dev_dbg(dev, "Unmapped digest-bytes-len buffer: digest_bytes_len_dma_addr=%pad\n", 258 &state->digest_bytes_len_dma_addr); 259 state->digest_bytes_len_dma_addr = 0; 260 } 261 if (state->opad_digest_dma_addr) { 262 dma_unmap_single(dev, state->opad_digest_dma_addr, 263 ctx->inter_digestsize, DMA_BIDIRECTIONAL); 264 dev_dbg(dev, "Unmapped opad-digest: opad_digest_dma_addr=%pad\n", 265 &state->opad_digest_dma_addr); 266 state->opad_digest_dma_addr = 0; 267 } 268 } 269 270 static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state, 271 unsigned int digestsize, u8 *result) 272 { 273 if (state->digest_result_dma_addr) { 274 dma_unmap_single(dev, state->digest_result_dma_addr, digestsize, 275 DMA_BIDIRECTIONAL); 276 dev_dbg(dev, "unmpa digest result buffer va (%p) pa (%pad) len %u\n", 277 state->digest_result_buff, 278 &state->digest_result_dma_addr, digestsize); 279 memcpy(result, state->digest_result_buff, digestsize); 280 } 281 state->digest_result_dma_addr = 0; 282 } 283 284 static void cc_update_complete(struct device *dev, void *cc_req, int err) 285 { 286 struct ahash_request *req = (struct ahash_request *)cc_req; 287 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 288 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 289 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 290 291 dev_dbg(dev, "req=%p\n", req); 292 293 if (err != -EINPROGRESS) { 294 /* Not a BACKLOG notification */ 295 cc_unmap_hash_request(dev, state, req->src, false); 296 cc_unmap_req(dev, state, ctx); 297 } 298 299 ahash_request_complete(req, err); 300 } 301 302 static void cc_digest_complete(struct device *dev, void *cc_req, int err) 303 { 304 struct ahash_request *req = (struct ahash_request *)cc_req; 305 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 306 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 307 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 308 u32 digestsize = crypto_ahash_digestsize(tfm); 309 310 dev_dbg(dev, "req=%p\n", req); 311 312 if (err != -EINPROGRESS) { 313 /* Not a BACKLOG notification */ 314 cc_unmap_hash_request(dev, state, req->src, false); 315 cc_unmap_result(dev, state, digestsize, req->result); 316 cc_unmap_req(dev, state, ctx); 317 } 318 319 ahash_request_complete(req, err); 320 } 321 322 static void cc_hash_complete(struct device *dev, void *cc_req, int err) 323 { 324 struct ahash_request *req = (struct ahash_request *)cc_req; 325 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 326 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 327 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 328 u32 digestsize = crypto_ahash_digestsize(tfm); 329 330 dev_dbg(dev, "req=%p\n", req); 331 332 if (err != -EINPROGRESS) { 333 /* Not a BACKLOG notification */ 334 cc_unmap_hash_request(dev, state, req->src, false); 335 cc_unmap_result(dev, state, digestsize, req->result); 336 cc_unmap_req(dev, state, ctx); 337 } 338 339 ahash_request_complete(req, err); 340 } 341 342 static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req, 343 int idx) 344 { 345 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 346 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 347 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 348 u32 digestsize = crypto_ahash_digestsize(tfm); 349 350 /* Get final MAC result */ 351 hw_desc_init(&desc[idx]); 352 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 353 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, digestsize, 354 NS_BIT, 1); 355 set_queue_last_ind(ctx->drvdata, &desc[idx]); 356 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 357 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 358 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 359 cc_set_endianity(ctx->hash_mode, &desc[idx]); 360 idx++; 361 362 return idx; 363 } 364 365 static int cc_fin_hmac(struct cc_hw_desc *desc, struct ahash_request *req, 366 int idx) 367 { 368 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 369 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 370 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 371 u32 digestsize = crypto_ahash_digestsize(tfm); 372 373 /* store the hash digest result in the context */ 374 hw_desc_init(&desc[idx]); 375 set_cipher_mode(&desc[idx], ctx->hw_mode); 376 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, digestsize, 377 NS_BIT, 0); 378 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 379 cc_set_endianity(ctx->hash_mode, &desc[idx]); 380 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 381 idx++; 382 383 /* Loading hash opad xor key state */ 384 hw_desc_init(&desc[idx]); 385 set_cipher_mode(&desc[idx], ctx->hw_mode); 386 set_din_type(&desc[idx], DMA_DLLI, state->opad_digest_dma_addr, 387 ctx->inter_digestsize, NS_BIT); 388 set_flow_mode(&desc[idx], S_DIN_to_HASH); 389 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 390 idx++; 391 392 /* Load the hash current length */ 393 hw_desc_init(&desc[idx]); 394 set_cipher_mode(&desc[idx], ctx->hw_mode); 395 set_din_sram(&desc[idx], 396 cc_digest_len_addr(ctx->drvdata, ctx->hash_mode), 397 ctx->hash_len); 398 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 399 set_flow_mode(&desc[idx], S_DIN_to_HASH); 400 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 401 idx++; 402 403 /* Memory Barrier: wait for IPAD/OPAD axi write to complete */ 404 hw_desc_init(&desc[idx]); 405 set_din_no_dma(&desc[idx], 0, 0xfffff0); 406 set_dout_no_dma(&desc[idx], 0, 0, 1); 407 idx++; 408 409 /* Perform HASH update */ 410 hw_desc_init(&desc[idx]); 411 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 412 digestsize, NS_BIT); 413 set_flow_mode(&desc[idx], DIN_HASH); 414 idx++; 415 416 return idx; 417 } 418 419 static int cc_hash_digest(struct ahash_request *req) 420 { 421 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 422 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 423 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 424 u32 digestsize = crypto_ahash_digestsize(tfm); 425 struct scatterlist *src = req->src; 426 unsigned int nbytes = req->nbytes; 427 u8 *result = req->result; 428 struct device *dev = drvdata_to_dev(ctx->drvdata); 429 bool is_hmac = ctx->is_hmac; 430 struct cc_crypto_req cc_req = {}; 431 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 432 u32 larval_digest_addr; 433 int idx = 0; 434 int rc = 0; 435 gfp_t flags = cc_gfp_flags(&req->base); 436 437 dev_dbg(dev, "===== %s-digest (%d) ====\n", is_hmac ? "hmac" : "hash", 438 nbytes); 439 440 cc_init_req(dev, state, ctx); 441 442 if (cc_map_req(dev, state, ctx)) { 443 dev_err(dev, "map_ahash_source() failed\n"); 444 return -ENOMEM; 445 } 446 447 if (cc_map_result(dev, state, digestsize)) { 448 dev_err(dev, "map_ahash_digest() failed\n"); 449 cc_unmap_req(dev, state, ctx); 450 return -ENOMEM; 451 } 452 453 if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1, 454 flags)) { 455 dev_err(dev, "map_ahash_request_final() failed\n"); 456 cc_unmap_result(dev, state, digestsize, result); 457 cc_unmap_req(dev, state, ctx); 458 return -ENOMEM; 459 } 460 461 /* Setup request structure */ 462 cc_req.user_cb = cc_digest_complete; 463 cc_req.user_arg = req; 464 465 /* If HMAC then load hash IPAD xor key, if HASH then load initial 466 * digest 467 */ 468 hw_desc_init(&desc[idx]); 469 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 470 if (is_hmac) { 471 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 472 ctx->inter_digestsize, NS_BIT); 473 } else { 474 larval_digest_addr = cc_larval_digest_addr(ctx->drvdata, 475 ctx->hash_mode); 476 set_din_sram(&desc[idx], larval_digest_addr, 477 ctx->inter_digestsize); 478 } 479 set_flow_mode(&desc[idx], S_DIN_to_HASH); 480 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 481 idx++; 482 483 /* Load the hash current length */ 484 hw_desc_init(&desc[idx]); 485 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 486 487 if (is_hmac) { 488 set_din_type(&desc[idx], DMA_DLLI, 489 state->digest_bytes_len_dma_addr, 490 ctx->hash_len, NS_BIT); 491 } else { 492 set_din_const(&desc[idx], 0, ctx->hash_len); 493 if (nbytes) 494 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 495 else 496 set_cipher_do(&desc[idx], DO_PAD); 497 } 498 set_flow_mode(&desc[idx], S_DIN_to_HASH); 499 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 500 idx++; 501 502 cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx); 503 504 if (is_hmac) { 505 /* HW last hash block padding (aka. "DO_PAD") */ 506 hw_desc_init(&desc[idx]); 507 set_cipher_mode(&desc[idx], ctx->hw_mode); 508 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, 509 ctx->hash_len, NS_BIT, 0); 510 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 511 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 512 set_cipher_do(&desc[idx], DO_PAD); 513 idx++; 514 515 idx = cc_fin_hmac(desc, req, idx); 516 } 517 518 idx = cc_fin_result(desc, req, idx); 519 520 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 521 if (rc != -EINPROGRESS && rc != -EBUSY) { 522 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 523 cc_unmap_hash_request(dev, state, src, true); 524 cc_unmap_result(dev, state, digestsize, result); 525 cc_unmap_req(dev, state, ctx); 526 } 527 return rc; 528 } 529 530 static int cc_restore_hash(struct cc_hw_desc *desc, struct cc_hash_ctx *ctx, 531 struct ahash_req_ctx *state, unsigned int idx) 532 { 533 /* Restore hash digest */ 534 hw_desc_init(&desc[idx]); 535 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 536 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 537 ctx->inter_digestsize, NS_BIT); 538 set_flow_mode(&desc[idx], S_DIN_to_HASH); 539 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 540 idx++; 541 542 /* Restore hash current length */ 543 hw_desc_init(&desc[idx]); 544 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 545 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 546 set_din_type(&desc[idx], DMA_DLLI, state->digest_bytes_len_dma_addr, 547 ctx->hash_len, NS_BIT); 548 set_flow_mode(&desc[idx], S_DIN_to_HASH); 549 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 550 idx++; 551 552 cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx); 553 554 return idx; 555 } 556 557 static int cc_hash_update(struct ahash_request *req) 558 { 559 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 560 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 561 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 562 unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base); 563 struct scatterlist *src = req->src; 564 unsigned int nbytes = req->nbytes; 565 struct device *dev = drvdata_to_dev(ctx->drvdata); 566 struct cc_crypto_req cc_req = {}; 567 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 568 u32 idx = 0; 569 int rc; 570 gfp_t flags = cc_gfp_flags(&req->base); 571 572 dev_dbg(dev, "===== %s-update (%d) ====\n", ctx->is_hmac ? 573 "hmac" : "hash", nbytes); 574 575 if (nbytes == 0) { 576 /* no real updates required */ 577 return 0; 578 } 579 580 rc = cc_map_hash_request_update(ctx->drvdata, state, src, nbytes, 581 block_size, flags); 582 if (rc) { 583 if (rc == 1) { 584 dev_dbg(dev, " data size not require HW update %x\n", 585 nbytes); 586 /* No hardware updates are required */ 587 return 0; 588 } 589 dev_err(dev, "map_ahash_request_update() failed\n"); 590 return -ENOMEM; 591 } 592 593 if (cc_map_req(dev, state, ctx)) { 594 dev_err(dev, "map_ahash_source() failed\n"); 595 cc_unmap_hash_request(dev, state, src, true); 596 return -EINVAL; 597 } 598 599 /* Setup request structure */ 600 cc_req.user_cb = cc_update_complete; 601 cc_req.user_arg = req; 602 603 idx = cc_restore_hash(desc, ctx, state, idx); 604 605 /* store the hash digest result in context */ 606 hw_desc_init(&desc[idx]); 607 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 608 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, 609 ctx->inter_digestsize, NS_BIT, 0); 610 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 611 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 612 idx++; 613 614 /* store current hash length in context */ 615 hw_desc_init(&desc[idx]); 616 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 617 set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr, 618 ctx->hash_len, NS_BIT, 1); 619 set_queue_last_ind(ctx->drvdata, &desc[idx]); 620 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 621 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 622 idx++; 623 624 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 625 if (rc != -EINPROGRESS && rc != -EBUSY) { 626 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 627 cc_unmap_hash_request(dev, state, src, true); 628 cc_unmap_req(dev, state, ctx); 629 } 630 return rc; 631 } 632 633 static int cc_do_finup(struct ahash_request *req, bool update) 634 { 635 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 636 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 637 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 638 u32 digestsize = crypto_ahash_digestsize(tfm); 639 struct scatterlist *src = req->src; 640 unsigned int nbytes = req->nbytes; 641 u8 *result = req->result; 642 struct device *dev = drvdata_to_dev(ctx->drvdata); 643 bool is_hmac = ctx->is_hmac; 644 struct cc_crypto_req cc_req = {}; 645 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 646 unsigned int idx = 0; 647 int rc; 648 gfp_t flags = cc_gfp_flags(&req->base); 649 650 dev_dbg(dev, "===== %s-%s (%d) ====\n", is_hmac ? "hmac" : "hash", 651 update ? "finup" : "final", nbytes); 652 653 if (cc_map_req(dev, state, ctx)) { 654 dev_err(dev, "map_ahash_source() failed\n"); 655 return -EINVAL; 656 } 657 658 if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, update, 659 flags)) { 660 dev_err(dev, "map_ahash_request_final() failed\n"); 661 cc_unmap_req(dev, state, ctx); 662 return -ENOMEM; 663 } 664 if (cc_map_result(dev, state, digestsize)) { 665 dev_err(dev, "map_ahash_digest() failed\n"); 666 cc_unmap_hash_request(dev, state, src, true); 667 cc_unmap_req(dev, state, ctx); 668 return -ENOMEM; 669 } 670 671 /* Setup request structure */ 672 cc_req.user_cb = cc_hash_complete; 673 cc_req.user_arg = req; 674 675 idx = cc_restore_hash(desc, ctx, state, idx); 676 677 /* Pad the hash */ 678 hw_desc_init(&desc[idx]); 679 set_cipher_do(&desc[idx], DO_PAD); 680 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode); 681 set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr, 682 ctx->hash_len, NS_BIT, 0); 683 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 684 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 685 idx++; 686 687 if (is_hmac) 688 idx = cc_fin_hmac(desc, req, idx); 689 690 idx = cc_fin_result(desc, req, idx); 691 692 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 693 if (rc != -EINPROGRESS && rc != -EBUSY) { 694 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 695 cc_unmap_hash_request(dev, state, src, true); 696 cc_unmap_result(dev, state, digestsize, result); 697 cc_unmap_req(dev, state, ctx); 698 } 699 return rc; 700 } 701 702 static int cc_hash_finup(struct ahash_request *req) 703 { 704 return cc_do_finup(req, true); 705 } 706 707 708 static int cc_hash_final(struct ahash_request *req) 709 { 710 return cc_do_finup(req, false); 711 } 712 713 static int cc_hash_init(struct ahash_request *req) 714 { 715 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 716 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 717 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 718 struct device *dev = drvdata_to_dev(ctx->drvdata); 719 720 dev_dbg(dev, "===== init (%d) ====\n", req->nbytes); 721 722 cc_init_req(dev, state, ctx); 723 724 return 0; 725 } 726 727 static int cc_hash_setkey(struct crypto_ahash *ahash, const u8 *key, 728 unsigned int keylen) 729 { 730 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; 731 struct cc_crypto_req cc_req = {}; 732 struct cc_hash_ctx *ctx = NULL; 733 int blocksize = 0; 734 int digestsize = 0; 735 int i, idx = 0, rc = 0; 736 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 737 u32 larval_addr; 738 struct device *dev; 739 740 ctx = crypto_ahash_ctx_dma(ahash); 741 dev = drvdata_to_dev(ctx->drvdata); 742 dev_dbg(dev, "start keylen: %d", keylen); 743 744 blocksize = crypto_tfm_alg_blocksize(&ahash->base); 745 digestsize = crypto_ahash_digestsize(ahash); 746 747 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->hash_mode); 748 749 /* The keylen value distinguishes HASH in case keylen is ZERO bytes, 750 * any NON-ZERO value utilizes HMAC flow 751 */ 752 ctx->key_params.keylen = keylen; 753 ctx->key_params.key_dma_addr = 0; 754 ctx->is_hmac = true; 755 ctx->key_params.key = NULL; 756 757 if (keylen) { 758 ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL); 759 if (!ctx->key_params.key) 760 return -ENOMEM; 761 762 ctx->key_params.key_dma_addr = 763 dma_map_single(dev, ctx->key_params.key, keylen, 764 DMA_TO_DEVICE); 765 if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) { 766 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 767 ctx->key_params.key, keylen); 768 kfree_sensitive(ctx->key_params.key); 769 return -ENOMEM; 770 } 771 dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n", 772 &ctx->key_params.key_dma_addr, ctx->key_params.keylen); 773 774 if (keylen > blocksize) { 775 /* Load hash initial state */ 776 hw_desc_init(&desc[idx]); 777 set_cipher_mode(&desc[idx], ctx->hw_mode); 778 set_din_sram(&desc[idx], larval_addr, 779 ctx->inter_digestsize); 780 set_flow_mode(&desc[idx], S_DIN_to_HASH); 781 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 782 idx++; 783 784 /* Load the hash current length*/ 785 hw_desc_init(&desc[idx]); 786 set_cipher_mode(&desc[idx], ctx->hw_mode); 787 set_din_const(&desc[idx], 0, ctx->hash_len); 788 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 789 set_flow_mode(&desc[idx], S_DIN_to_HASH); 790 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 791 idx++; 792 793 hw_desc_init(&desc[idx]); 794 set_din_type(&desc[idx], DMA_DLLI, 795 ctx->key_params.key_dma_addr, keylen, 796 NS_BIT); 797 set_flow_mode(&desc[idx], DIN_HASH); 798 idx++; 799 800 /* Get hashed key */ 801 hw_desc_init(&desc[idx]); 802 set_cipher_mode(&desc[idx], ctx->hw_mode); 803 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr, 804 digestsize, NS_BIT, 0); 805 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 806 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 807 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 808 cc_set_endianity(ctx->hash_mode, &desc[idx]); 809 idx++; 810 811 hw_desc_init(&desc[idx]); 812 set_din_const(&desc[idx], 0, (blocksize - digestsize)); 813 set_flow_mode(&desc[idx], BYPASS); 814 set_dout_dlli(&desc[idx], 815 (ctx->opad_tmp_keys_dma_addr + 816 digestsize), 817 (blocksize - digestsize), NS_BIT, 0); 818 idx++; 819 } else { 820 hw_desc_init(&desc[idx]); 821 set_din_type(&desc[idx], DMA_DLLI, 822 ctx->key_params.key_dma_addr, keylen, 823 NS_BIT); 824 set_flow_mode(&desc[idx], BYPASS); 825 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr, 826 keylen, NS_BIT, 0); 827 idx++; 828 829 if ((blocksize - keylen)) { 830 hw_desc_init(&desc[idx]); 831 set_din_const(&desc[idx], 0, 832 (blocksize - keylen)); 833 set_flow_mode(&desc[idx], BYPASS); 834 set_dout_dlli(&desc[idx], 835 (ctx->opad_tmp_keys_dma_addr + 836 keylen), (blocksize - keylen), 837 NS_BIT, 0); 838 idx++; 839 } 840 } 841 } else { 842 hw_desc_init(&desc[idx]); 843 set_din_const(&desc[idx], 0, blocksize); 844 set_flow_mode(&desc[idx], BYPASS); 845 set_dout_dlli(&desc[idx], (ctx->opad_tmp_keys_dma_addr), 846 blocksize, NS_BIT, 0); 847 idx++; 848 } 849 850 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 851 if (rc) { 852 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 853 goto out; 854 } 855 856 /* calc derived HMAC key */ 857 for (idx = 0, i = 0; i < 2; i++) { 858 /* Load hash initial state */ 859 hw_desc_init(&desc[idx]); 860 set_cipher_mode(&desc[idx], ctx->hw_mode); 861 set_din_sram(&desc[idx], larval_addr, ctx->inter_digestsize); 862 set_flow_mode(&desc[idx], S_DIN_to_HASH); 863 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 864 idx++; 865 866 /* Load the hash current length*/ 867 hw_desc_init(&desc[idx]); 868 set_cipher_mode(&desc[idx], ctx->hw_mode); 869 set_din_const(&desc[idx], 0, ctx->hash_len); 870 set_flow_mode(&desc[idx], S_DIN_to_HASH); 871 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 872 idx++; 873 874 /* Prepare ipad key */ 875 hw_desc_init(&desc[idx]); 876 set_xor_val(&desc[idx], hmac_pad_const[i]); 877 set_cipher_mode(&desc[idx], ctx->hw_mode); 878 set_flow_mode(&desc[idx], S_DIN_to_HASH); 879 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 880 idx++; 881 882 /* Perform HASH update */ 883 hw_desc_init(&desc[idx]); 884 set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr, 885 blocksize, NS_BIT); 886 set_cipher_mode(&desc[idx], ctx->hw_mode); 887 set_xor_active(&desc[idx]); 888 set_flow_mode(&desc[idx], DIN_HASH); 889 idx++; 890 891 /* Get the IPAD/OPAD xor key (Note, IPAD is the initial digest 892 * of the first HASH "update" state) 893 */ 894 hw_desc_init(&desc[idx]); 895 set_cipher_mode(&desc[idx], ctx->hw_mode); 896 if (i > 0) /* Not first iteration */ 897 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr, 898 ctx->inter_digestsize, NS_BIT, 0); 899 else /* First iteration */ 900 set_dout_dlli(&desc[idx], ctx->digest_buff_dma_addr, 901 ctx->inter_digestsize, NS_BIT, 0); 902 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 903 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 904 idx++; 905 } 906 907 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 908 909 out: 910 if (ctx->key_params.key_dma_addr) { 911 dma_unmap_single(dev, ctx->key_params.key_dma_addr, 912 ctx->key_params.keylen, DMA_TO_DEVICE); 913 dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n", 914 &ctx->key_params.key_dma_addr, ctx->key_params.keylen); 915 } 916 917 kfree_sensitive(ctx->key_params.key); 918 919 return rc; 920 } 921 922 static int cc_xcbc_setkey(struct crypto_ahash *ahash, 923 const u8 *key, unsigned int keylen) 924 { 925 struct cc_crypto_req cc_req = {}; 926 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 927 struct device *dev = drvdata_to_dev(ctx->drvdata); 928 int rc = 0; 929 unsigned int idx = 0; 930 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 931 932 dev_dbg(dev, "===== setkey (%d) ====\n", keylen); 933 934 switch (keylen) { 935 case AES_KEYSIZE_128: 936 case AES_KEYSIZE_192: 937 case AES_KEYSIZE_256: 938 break; 939 default: 940 return -EINVAL; 941 } 942 943 ctx->key_params.keylen = keylen; 944 945 ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL); 946 if (!ctx->key_params.key) 947 return -ENOMEM; 948 949 ctx->key_params.key_dma_addr = 950 dma_map_single(dev, ctx->key_params.key, keylen, DMA_TO_DEVICE); 951 if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) { 952 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 953 key, keylen); 954 kfree_sensitive(ctx->key_params.key); 955 return -ENOMEM; 956 } 957 dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n", 958 &ctx->key_params.key_dma_addr, ctx->key_params.keylen); 959 960 ctx->is_hmac = true; 961 /* 1. Load the AES key */ 962 hw_desc_init(&desc[idx]); 963 set_din_type(&desc[idx], DMA_DLLI, ctx->key_params.key_dma_addr, 964 keylen, NS_BIT); 965 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 966 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 967 set_key_size_aes(&desc[idx], keylen); 968 set_flow_mode(&desc[idx], S_DIN_to_AES); 969 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 970 idx++; 971 972 hw_desc_init(&desc[idx]); 973 set_din_const(&desc[idx], 0x01010101, CC_AES_128_BIT_KEY_SIZE); 974 set_flow_mode(&desc[idx], DIN_AES_DOUT); 975 set_dout_dlli(&desc[idx], 976 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET), 977 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0); 978 idx++; 979 980 hw_desc_init(&desc[idx]); 981 set_din_const(&desc[idx], 0x02020202, CC_AES_128_BIT_KEY_SIZE); 982 set_flow_mode(&desc[idx], DIN_AES_DOUT); 983 set_dout_dlli(&desc[idx], 984 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET), 985 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0); 986 idx++; 987 988 hw_desc_init(&desc[idx]); 989 set_din_const(&desc[idx], 0x03030303, CC_AES_128_BIT_KEY_SIZE); 990 set_flow_mode(&desc[idx], DIN_AES_DOUT); 991 set_dout_dlli(&desc[idx], 992 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET), 993 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0); 994 idx++; 995 996 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 997 998 dma_unmap_single(dev, ctx->key_params.key_dma_addr, 999 ctx->key_params.keylen, DMA_TO_DEVICE); 1000 dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n", 1001 &ctx->key_params.key_dma_addr, ctx->key_params.keylen); 1002 1003 kfree_sensitive(ctx->key_params.key); 1004 1005 return rc; 1006 } 1007 1008 static int cc_cmac_setkey(struct crypto_ahash *ahash, 1009 const u8 *key, unsigned int keylen) 1010 { 1011 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 1012 struct device *dev = drvdata_to_dev(ctx->drvdata); 1013 1014 dev_dbg(dev, "===== setkey (%d) ====\n", keylen); 1015 1016 ctx->is_hmac = true; 1017 1018 switch (keylen) { 1019 case AES_KEYSIZE_128: 1020 case AES_KEYSIZE_192: 1021 case AES_KEYSIZE_256: 1022 break; 1023 default: 1024 return -EINVAL; 1025 } 1026 1027 ctx->key_params.keylen = keylen; 1028 1029 /* STAT_PHASE_1: Copy key to ctx */ 1030 1031 dma_sync_single_for_cpu(dev, ctx->opad_tmp_keys_dma_addr, 1032 keylen, DMA_TO_DEVICE); 1033 1034 memcpy(ctx->opad_tmp_keys_buff, key, keylen); 1035 if (keylen == 24) { 1036 memset(ctx->opad_tmp_keys_buff + 24, 0, 1037 CC_AES_KEY_SIZE_MAX - 24); 1038 } 1039 1040 dma_sync_single_for_device(dev, ctx->opad_tmp_keys_dma_addr, 1041 keylen, DMA_TO_DEVICE); 1042 1043 ctx->key_params.keylen = keylen; 1044 1045 return 0; 1046 } 1047 1048 static void cc_free_ctx(struct cc_hash_ctx *ctx) 1049 { 1050 struct device *dev = drvdata_to_dev(ctx->drvdata); 1051 1052 if (ctx->digest_buff_dma_addr) { 1053 dma_unmap_single(dev, ctx->digest_buff_dma_addr, 1054 sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL); 1055 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n", 1056 &ctx->digest_buff_dma_addr); 1057 ctx->digest_buff_dma_addr = 0; 1058 } 1059 if (ctx->opad_tmp_keys_dma_addr) { 1060 dma_unmap_single(dev, ctx->opad_tmp_keys_dma_addr, 1061 sizeof(ctx->opad_tmp_keys_buff), 1062 DMA_BIDIRECTIONAL); 1063 dev_dbg(dev, "Unmapped opad-digest: opad_tmp_keys_dma_addr=%pad\n", 1064 &ctx->opad_tmp_keys_dma_addr); 1065 ctx->opad_tmp_keys_dma_addr = 0; 1066 } 1067 1068 ctx->key_params.keylen = 0; 1069 } 1070 1071 static int cc_alloc_ctx(struct cc_hash_ctx *ctx) 1072 { 1073 struct device *dev = drvdata_to_dev(ctx->drvdata); 1074 1075 ctx->key_params.keylen = 0; 1076 1077 ctx->digest_buff_dma_addr = 1078 dma_map_single(dev, ctx->digest_buff, sizeof(ctx->digest_buff), 1079 DMA_BIDIRECTIONAL); 1080 if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) { 1081 dev_err(dev, "Mapping digest len %zu B at va=%p for DMA failed\n", 1082 sizeof(ctx->digest_buff), ctx->digest_buff); 1083 goto fail; 1084 } 1085 dev_dbg(dev, "Mapped digest %zu B at va=%p to dma=%pad\n", 1086 sizeof(ctx->digest_buff), ctx->digest_buff, 1087 &ctx->digest_buff_dma_addr); 1088 1089 ctx->opad_tmp_keys_dma_addr = 1090 dma_map_single(dev, ctx->opad_tmp_keys_buff, 1091 sizeof(ctx->opad_tmp_keys_buff), 1092 DMA_BIDIRECTIONAL); 1093 if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) { 1094 dev_err(dev, "Mapping opad digest %zu B at va=%p for DMA failed\n", 1095 sizeof(ctx->opad_tmp_keys_buff), 1096 ctx->opad_tmp_keys_buff); 1097 goto fail; 1098 } 1099 dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%p to dma=%pad\n", 1100 sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff, 1101 &ctx->opad_tmp_keys_dma_addr); 1102 1103 ctx->is_hmac = false; 1104 return 0; 1105 1106 fail: 1107 cc_free_ctx(ctx); 1108 return -ENOMEM; 1109 } 1110 1111 static int cc_get_hash_len(struct crypto_tfm *tfm) 1112 { 1113 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 1114 1115 if (ctx->hash_mode == DRV_HASH_SM3) 1116 return CC_SM3_HASH_LEN_SIZE; 1117 else 1118 return cc_get_default_hash_len(ctx->drvdata); 1119 } 1120 1121 static int cc_cra_init(struct crypto_tfm *tfm) 1122 { 1123 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 1124 struct hash_alg_common *hash_alg_common = 1125 container_of(tfm->__crt_alg, struct hash_alg_common, base); 1126 struct ahash_alg *ahash_alg = 1127 container_of(hash_alg_common, struct ahash_alg, halg); 1128 struct cc_hash_alg *cc_alg = 1129 container_of(ahash_alg, struct cc_hash_alg, ahash_alg); 1130 1131 crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), 1132 sizeof(struct ahash_req_ctx)); 1133 1134 ctx->hash_mode = cc_alg->hash_mode; 1135 ctx->hw_mode = cc_alg->hw_mode; 1136 ctx->inter_digestsize = cc_alg->inter_digestsize; 1137 ctx->drvdata = cc_alg->drvdata; 1138 ctx->hash_len = cc_get_hash_len(tfm); 1139 return cc_alloc_ctx(ctx); 1140 } 1141 1142 static void cc_cra_exit(struct crypto_tfm *tfm) 1143 { 1144 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 1145 struct device *dev = drvdata_to_dev(ctx->drvdata); 1146 1147 dev_dbg(dev, "cc_cra_exit"); 1148 cc_free_ctx(ctx); 1149 } 1150 1151 static int cc_mac_update(struct ahash_request *req) 1152 { 1153 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1154 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1155 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 1156 struct device *dev = drvdata_to_dev(ctx->drvdata); 1157 unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base); 1158 struct cc_crypto_req cc_req = {}; 1159 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 1160 int rc; 1161 u32 idx = 0; 1162 gfp_t flags = cc_gfp_flags(&req->base); 1163 1164 if (req->nbytes == 0) { 1165 /* no real updates required */ 1166 return 0; 1167 } 1168 1169 state->xcbc_count++; 1170 1171 rc = cc_map_hash_request_update(ctx->drvdata, state, req->src, 1172 req->nbytes, block_size, flags); 1173 if (rc) { 1174 if (rc == 1) { 1175 dev_dbg(dev, " data size not require HW update %x\n", 1176 req->nbytes); 1177 /* No hardware updates are required */ 1178 return 0; 1179 } 1180 dev_err(dev, "map_ahash_request_update() failed\n"); 1181 return -ENOMEM; 1182 } 1183 1184 if (cc_map_req(dev, state, ctx)) { 1185 dev_err(dev, "map_ahash_source() failed\n"); 1186 return -EINVAL; 1187 } 1188 1189 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) 1190 cc_setup_xcbc(req, desc, &idx); 1191 else 1192 cc_setup_cmac(req, desc, &idx); 1193 1194 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, true, &idx); 1195 1196 /* store the hash digest result in context */ 1197 hw_desc_init(&desc[idx]); 1198 set_cipher_mode(&desc[idx], ctx->hw_mode); 1199 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, 1200 ctx->inter_digestsize, NS_BIT, 1); 1201 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1202 set_flow_mode(&desc[idx], S_AES_to_DOUT); 1203 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1204 idx++; 1205 1206 /* Setup request structure */ 1207 cc_req.user_cb = cc_update_complete; 1208 cc_req.user_arg = req; 1209 1210 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 1211 if (rc != -EINPROGRESS && rc != -EBUSY) { 1212 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 1213 cc_unmap_hash_request(dev, state, req->src, true); 1214 cc_unmap_req(dev, state, ctx); 1215 } 1216 return rc; 1217 } 1218 1219 static int cc_mac_final(struct ahash_request *req) 1220 { 1221 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1222 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1223 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 1224 struct device *dev = drvdata_to_dev(ctx->drvdata); 1225 struct cc_crypto_req cc_req = {}; 1226 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 1227 int idx = 0; 1228 int rc = 0; 1229 u32 key_size, key_len; 1230 u32 digestsize = crypto_ahash_digestsize(tfm); 1231 gfp_t flags = cc_gfp_flags(&req->base); 1232 u32 rem_cnt = *cc_hash_buf_cnt(state); 1233 1234 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) { 1235 key_size = CC_AES_128_BIT_KEY_SIZE; 1236 key_len = CC_AES_128_BIT_KEY_SIZE; 1237 } else { 1238 key_size = (ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE : 1239 ctx->key_params.keylen; 1240 key_len = ctx->key_params.keylen; 1241 } 1242 1243 dev_dbg(dev, "===== final xcbc reminder (%d) ====\n", rem_cnt); 1244 1245 if (cc_map_req(dev, state, ctx)) { 1246 dev_err(dev, "map_ahash_source() failed\n"); 1247 return -EINVAL; 1248 } 1249 1250 if (cc_map_hash_request_final(ctx->drvdata, state, req->src, 1251 req->nbytes, 0, flags)) { 1252 dev_err(dev, "map_ahash_request_final() failed\n"); 1253 cc_unmap_req(dev, state, ctx); 1254 return -ENOMEM; 1255 } 1256 1257 if (cc_map_result(dev, state, digestsize)) { 1258 dev_err(dev, "map_ahash_digest() failed\n"); 1259 cc_unmap_hash_request(dev, state, req->src, true); 1260 cc_unmap_req(dev, state, ctx); 1261 return -ENOMEM; 1262 } 1263 1264 /* Setup request structure */ 1265 cc_req.user_cb = cc_hash_complete; 1266 cc_req.user_arg = req; 1267 1268 if (state->xcbc_count && rem_cnt == 0) { 1269 /* Load key for ECB decryption */ 1270 hw_desc_init(&desc[idx]); 1271 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 1272 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_DECRYPT); 1273 set_din_type(&desc[idx], DMA_DLLI, 1274 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET), 1275 key_size, NS_BIT); 1276 set_key_size_aes(&desc[idx], key_len); 1277 set_flow_mode(&desc[idx], S_DIN_to_AES); 1278 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1279 idx++; 1280 1281 /* Initiate decryption of block state to previous 1282 * block_state-XOR-M[n] 1283 */ 1284 hw_desc_init(&desc[idx]); 1285 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 1286 CC_AES_BLOCK_SIZE, NS_BIT); 1287 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, 1288 CC_AES_BLOCK_SIZE, NS_BIT, 0); 1289 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1290 idx++; 1291 1292 /* Memory Barrier: wait for axi write to complete */ 1293 hw_desc_init(&desc[idx]); 1294 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1295 set_dout_no_dma(&desc[idx], 0, 0, 1); 1296 idx++; 1297 } 1298 1299 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) 1300 cc_setup_xcbc(req, desc, &idx); 1301 else 1302 cc_setup_cmac(req, desc, &idx); 1303 1304 if (state->xcbc_count == 0) { 1305 hw_desc_init(&desc[idx]); 1306 set_cipher_mode(&desc[idx], ctx->hw_mode); 1307 set_key_size_aes(&desc[idx], key_len); 1308 set_cmac_size0_mode(&desc[idx]); 1309 set_flow_mode(&desc[idx], S_DIN_to_AES); 1310 idx++; 1311 } else if (rem_cnt > 0) { 1312 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx); 1313 } else { 1314 hw_desc_init(&desc[idx]); 1315 set_din_const(&desc[idx], 0x00, CC_AES_BLOCK_SIZE); 1316 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1317 idx++; 1318 } 1319 1320 /* Get final MAC result */ 1321 hw_desc_init(&desc[idx]); 1322 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, 1323 digestsize, NS_BIT, 1); 1324 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1325 set_flow_mode(&desc[idx], S_AES_to_DOUT); 1326 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1327 set_cipher_mode(&desc[idx], ctx->hw_mode); 1328 idx++; 1329 1330 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 1331 if (rc != -EINPROGRESS && rc != -EBUSY) { 1332 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 1333 cc_unmap_hash_request(dev, state, req->src, true); 1334 cc_unmap_result(dev, state, digestsize, req->result); 1335 cc_unmap_req(dev, state, ctx); 1336 } 1337 return rc; 1338 } 1339 1340 static int cc_mac_finup(struct ahash_request *req) 1341 { 1342 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1343 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1344 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 1345 struct device *dev = drvdata_to_dev(ctx->drvdata); 1346 struct cc_crypto_req cc_req = {}; 1347 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 1348 int idx = 0; 1349 int rc = 0; 1350 u32 key_len = 0; 1351 u32 digestsize = crypto_ahash_digestsize(tfm); 1352 gfp_t flags = cc_gfp_flags(&req->base); 1353 1354 dev_dbg(dev, "===== finup xcbc(%d) ====\n", req->nbytes); 1355 if (state->xcbc_count > 0 && req->nbytes == 0) { 1356 dev_dbg(dev, "No data to update. Call to fdx_mac_final\n"); 1357 return cc_mac_final(req); 1358 } 1359 1360 if (cc_map_req(dev, state, ctx)) { 1361 dev_err(dev, "map_ahash_source() failed\n"); 1362 return -EINVAL; 1363 } 1364 1365 if (cc_map_hash_request_final(ctx->drvdata, state, req->src, 1366 req->nbytes, 1, flags)) { 1367 dev_err(dev, "map_ahash_request_final() failed\n"); 1368 cc_unmap_req(dev, state, ctx); 1369 return -ENOMEM; 1370 } 1371 if (cc_map_result(dev, state, digestsize)) { 1372 dev_err(dev, "map_ahash_digest() failed\n"); 1373 cc_unmap_hash_request(dev, state, req->src, true); 1374 cc_unmap_req(dev, state, ctx); 1375 return -ENOMEM; 1376 } 1377 1378 /* Setup request structure */ 1379 cc_req.user_cb = cc_hash_complete; 1380 cc_req.user_arg = req; 1381 1382 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) { 1383 key_len = CC_AES_128_BIT_KEY_SIZE; 1384 cc_setup_xcbc(req, desc, &idx); 1385 } else { 1386 key_len = ctx->key_params.keylen; 1387 cc_setup_cmac(req, desc, &idx); 1388 } 1389 1390 if (req->nbytes == 0) { 1391 hw_desc_init(&desc[idx]); 1392 set_cipher_mode(&desc[idx], ctx->hw_mode); 1393 set_key_size_aes(&desc[idx], key_len); 1394 set_cmac_size0_mode(&desc[idx]); 1395 set_flow_mode(&desc[idx], S_DIN_to_AES); 1396 idx++; 1397 } else { 1398 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx); 1399 } 1400 1401 /* Get final MAC result */ 1402 hw_desc_init(&desc[idx]); 1403 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, 1404 digestsize, NS_BIT, 1); 1405 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1406 set_flow_mode(&desc[idx], S_AES_to_DOUT); 1407 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1408 set_cipher_mode(&desc[idx], ctx->hw_mode); 1409 idx++; 1410 1411 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 1412 if (rc != -EINPROGRESS && rc != -EBUSY) { 1413 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 1414 cc_unmap_hash_request(dev, state, req->src, true); 1415 cc_unmap_result(dev, state, digestsize, req->result); 1416 cc_unmap_req(dev, state, ctx); 1417 } 1418 return rc; 1419 } 1420 1421 static int cc_mac_digest(struct ahash_request *req) 1422 { 1423 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1424 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1425 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 1426 struct device *dev = drvdata_to_dev(ctx->drvdata); 1427 u32 digestsize = crypto_ahash_digestsize(tfm); 1428 struct cc_crypto_req cc_req = {}; 1429 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN]; 1430 u32 key_len; 1431 unsigned int idx = 0; 1432 int rc; 1433 gfp_t flags = cc_gfp_flags(&req->base); 1434 1435 dev_dbg(dev, "===== -digest mac (%d) ====\n", req->nbytes); 1436 1437 cc_init_req(dev, state, ctx); 1438 1439 if (cc_map_req(dev, state, ctx)) { 1440 dev_err(dev, "map_ahash_source() failed\n"); 1441 return -ENOMEM; 1442 } 1443 if (cc_map_result(dev, state, digestsize)) { 1444 dev_err(dev, "map_ahash_digest() failed\n"); 1445 cc_unmap_req(dev, state, ctx); 1446 return -ENOMEM; 1447 } 1448 1449 if (cc_map_hash_request_final(ctx->drvdata, state, req->src, 1450 req->nbytes, 1, flags)) { 1451 dev_err(dev, "map_ahash_request_final() failed\n"); 1452 cc_unmap_result(dev, state, digestsize, req->result); 1453 cc_unmap_req(dev, state, ctx); 1454 return -ENOMEM; 1455 } 1456 1457 /* Setup request structure */ 1458 cc_req.user_cb = cc_digest_complete; 1459 cc_req.user_arg = req; 1460 1461 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) { 1462 key_len = CC_AES_128_BIT_KEY_SIZE; 1463 cc_setup_xcbc(req, desc, &idx); 1464 } else { 1465 key_len = ctx->key_params.keylen; 1466 cc_setup_cmac(req, desc, &idx); 1467 } 1468 1469 if (req->nbytes == 0) { 1470 hw_desc_init(&desc[idx]); 1471 set_cipher_mode(&desc[idx], ctx->hw_mode); 1472 set_key_size_aes(&desc[idx], key_len); 1473 set_cmac_size0_mode(&desc[idx]); 1474 set_flow_mode(&desc[idx], S_DIN_to_AES); 1475 idx++; 1476 } else { 1477 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx); 1478 } 1479 1480 /* Get final MAC result */ 1481 hw_desc_init(&desc[idx]); 1482 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, 1483 CC_AES_BLOCK_SIZE, NS_BIT, 1); 1484 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1485 set_flow_mode(&desc[idx], S_AES_to_DOUT); 1486 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1487 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1488 set_cipher_mode(&desc[idx], ctx->hw_mode); 1489 idx++; 1490 1491 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base); 1492 if (rc != -EINPROGRESS && rc != -EBUSY) { 1493 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 1494 cc_unmap_hash_request(dev, state, req->src, true); 1495 cc_unmap_result(dev, state, digestsize, req->result); 1496 cc_unmap_req(dev, state, ctx); 1497 } 1498 return rc; 1499 } 1500 1501 static int cc_hash_export(struct ahash_request *req, void *out) 1502 { 1503 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1504 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 1505 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1506 u8 *curr_buff = cc_hash_buf(state); 1507 u32 curr_buff_cnt = *cc_hash_buf_cnt(state); 1508 const u32 tmp = CC_EXPORT_MAGIC; 1509 1510 memcpy(out, &tmp, sizeof(u32)); 1511 out += sizeof(u32); 1512 1513 memcpy(out, state->digest_buff, ctx->inter_digestsize); 1514 out += ctx->inter_digestsize; 1515 1516 memcpy(out, state->digest_bytes_len, ctx->hash_len); 1517 out += ctx->hash_len; 1518 1519 memcpy(out, &curr_buff_cnt, sizeof(u32)); 1520 out += sizeof(u32); 1521 1522 memcpy(out, curr_buff, curr_buff_cnt); 1523 1524 return 0; 1525 } 1526 1527 static int cc_hash_import(struct ahash_request *req, const void *in) 1528 { 1529 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1530 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 1531 struct device *dev = drvdata_to_dev(ctx->drvdata); 1532 struct ahash_req_ctx *state = ahash_request_ctx_dma(req); 1533 u32 tmp; 1534 1535 memcpy(&tmp, in, sizeof(u32)); 1536 if (tmp != CC_EXPORT_MAGIC) 1537 return -EINVAL; 1538 in += sizeof(u32); 1539 1540 cc_init_req(dev, state, ctx); 1541 1542 memcpy(state->digest_buff, in, ctx->inter_digestsize); 1543 in += ctx->inter_digestsize; 1544 1545 memcpy(state->digest_bytes_len, in, ctx->hash_len); 1546 in += ctx->hash_len; 1547 1548 /* Sanity check the data as much as possible */ 1549 memcpy(&tmp, in, sizeof(u32)); 1550 if (tmp > CC_MAX_HASH_BLCK_SIZE) 1551 return -EINVAL; 1552 in += sizeof(u32); 1553 1554 state->buf_cnt[0] = tmp; 1555 memcpy(state->buffers[0], in, tmp); 1556 1557 return 0; 1558 } 1559 1560 struct cc_hash_template { 1561 char name[CRYPTO_MAX_ALG_NAME]; 1562 char driver_name[CRYPTO_MAX_ALG_NAME]; 1563 char mac_name[CRYPTO_MAX_ALG_NAME]; 1564 char mac_driver_name[CRYPTO_MAX_ALG_NAME]; 1565 unsigned int blocksize; 1566 bool is_mac; 1567 bool synchronize; 1568 struct ahash_alg template_ahash; 1569 int hash_mode; 1570 int hw_mode; 1571 int inter_digestsize; 1572 struct cc_drvdata *drvdata; 1573 u32 min_hw_rev; 1574 enum cc_std_body std_body; 1575 }; 1576 1577 #define CC_STATE_SIZE(_x) \ 1578 ((_x) + HASH_MAX_LEN_SIZE + CC_MAX_HASH_BLCK_SIZE + (2 * sizeof(u32))) 1579 1580 /* hash descriptors */ 1581 static struct cc_hash_template driver_hash[] = { 1582 //Asynchronous hash template 1583 { 1584 .name = "sha1", 1585 .driver_name = "sha1-ccree", 1586 .mac_name = "hmac(sha1)", 1587 .mac_driver_name = "hmac-sha1-ccree", 1588 .blocksize = SHA1_BLOCK_SIZE, 1589 .is_mac = true, 1590 .synchronize = false, 1591 .template_ahash = { 1592 .init = cc_hash_init, 1593 .update = cc_hash_update, 1594 .final = cc_hash_final, 1595 .finup = cc_hash_finup, 1596 .digest = cc_hash_digest, 1597 .export = cc_hash_export, 1598 .import = cc_hash_import, 1599 .setkey = cc_hash_setkey, 1600 .halg = { 1601 .digestsize = SHA1_DIGEST_SIZE, 1602 .statesize = CC_STATE_SIZE(SHA1_DIGEST_SIZE), 1603 }, 1604 }, 1605 .hash_mode = DRV_HASH_SHA1, 1606 .hw_mode = DRV_HASH_HW_SHA1, 1607 .inter_digestsize = SHA1_DIGEST_SIZE, 1608 .min_hw_rev = CC_HW_REV_630, 1609 .std_body = CC_STD_NIST, 1610 }, 1611 { 1612 .name = "sha256", 1613 .driver_name = "sha256-ccree", 1614 .mac_name = "hmac(sha256)", 1615 .mac_driver_name = "hmac-sha256-ccree", 1616 .blocksize = SHA256_BLOCK_SIZE, 1617 .is_mac = true, 1618 .template_ahash = { 1619 .init = cc_hash_init, 1620 .update = cc_hash_update, 1621 .final = cc_hash_final, 1622 .finup = cc_hash_finup, 1623 .digest = cc_hash_digest, 1624 .export = cc_hash_export, 1625 .import = cc_hash_import, 1626 .setkey = cc_hash_setkey, 1627 .halg = { 1628 .digestsize = SHA256_DIGEST_SIZE, 1629 .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE) 1630 }, 1631 }, 1632 .hash_mode = DRV_HASH_SHA256, 1633 .hw_mode = DRV_HASH_HW_SHA256, 1634 .inter_digestsize = SHA256_DIGEST_SIZE, 1635 .min_hw_rev = CC_HW_REV_630, 1636 .std_body = CC_STD_NIST, 1637 }, 1638 { 1639 .name = "sha224", 1640 .driver_name = "sha224-ccree", 1641 .mac_name = "hmac(sha224)", 1642 .mac_driver_name = "hmac-sha224-ccree", 1643 .blocksize = SHA224_BLOCK_SIZE, 1644 .is_mac = true, 1645 .template_ahash = { 1646 .init = cc_hash_init, 1647 .update = cc_hash_update, 1648 .final = cc_hash_final, 1649 .finup = cc_hash_finup, 1650 .digest = cc_hash_digest, 1651 .export = cc_hash_export, 1652 .import = cc_hash_import, 1653 .setkey = cc_hash_setkey, 1654 .halg = { 1655 .digestsize = SHA224_DIGEST_SIZE, 1656 .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE), 1657 }, 1658 }, 1659 .hash_mode = DRV_HASH_SHA224, 1660 .hw_mode = DRV_HASH_HW_SHA256, 1661 .inter_digestsize = SHA256_DIGEST_SIZE, 1662 .min_hw_rev = CC_HW_REV_630, 1663 .std_body = CC_STD_NIST, 1664 }, 1665 { 1666 .name = "sha384", 1667 .driver_name = "sha384-ccree", 1668 .mac_name = "hmac(sha384)", 1669 .mac_driver_name = "hmac-sha384-ccree", 1670 .blocksize = SHA384_BLOCK_SIZE, 1671 .is_mac = true, 1672 .template_ahash = { 1673 .init = cc_hash_init, 1674 .update = cc_hash_update, 1675 .final = cc_hash_final, 1676 .finup = cc_hash_finup, 1677 .digest = cc_hash_digest, 1678 .export = cc_hash_export, 1679 .import = cc_hash_import, 1680 .setkey = cc_hash_setkey, 1681 .halg = { 1682 .digestsize = SHA384_DIGEST_SIZE, 1683 .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE), 1684 }, 1685 }, 1686 .hash_mode = DRV_HASH_SHA384, 1687 .hw_mode = DRV_HASH_HW_SHA512, 1688 .inter_digestsize = SHA512_DIGEST_SIZE, 1689 .min_hw_rev = CC_HW_REV_712, 1690 .std_body = CC_STD_NIST, 1691 }, 1692 { 1693 .name = "sha512", 1694 .driver_name = "sha512-ccree", 1695 .mac_name = "hmac(sha512)", 1696 .mac_driver_name = "hmac-sha512-ccree", 1697 .blocksize = SHA512_BLOCK_SIZE, 1698 .is_mac = true, 1699 .template_ahash = { 1700 .init = cc_hash_init, 1701 .update = cc_hash_update, 1702 .final = cc_hash_final, 1703 .finup = cc_hash_finup, 1704 .digest = cc_hash_digest, 1705 .export = cc_hash_export, 1706 .import = cc_hash_import, 1707 .setkey = cc_hash_setkey, 1708 .halg = { 1709 .digestsize = SHA512_DIGEST_SIZE, 1710 .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE), 1711 }, 1712 }, 1713 .hash_mode = DRV_HASH_SHA512, 1714 .hw_mode = DRV_HASH_HW_SHA512, 1715 .inter_digestsize = SHA512_DIGEST_SIZE, 1716 .min_hw_rev = CC_HW_REV_712, 1717 .std_body = CC_STD_NIST, 1718 }, 1719 { 1720 .name = "md5", 1721 .driver_name = "md5-ccree", 1722 .mac_name = "hmac(md5)", 1723 .mac_driver_name = "hmac-md5-ccree", 1724 .blocksize = MD5_HMAC_BLOCK_SIZE, 1725 .is_mac = true, 1726 .template_ahash = { 1727 .init = cc_hash_init, 1728 .update = cc_hash_update, 1729 .final = cc_hash_final, 1730 .finup = cc_hash_finup, 1731 .digest = cc_hash_digest, 1732 .export = cc_hash_export, 1733 .import = cc_hash_import, 1734 .setkey = cc_hash_setkey, 1735 .halg = { 1736 .digestsize = MD5_DIGEST_SIZE, 1737 .statesize = CC_STATE_SIZE(MD5_DIGEST_SIZE), 1738 }, 1739 }, 1740 .hash_mode = DRV_HASH_MD5, 1741 .hw_mode = DRV_HASH_HW_MD5, 1742 .inter_digestsize = MD5_DIGEST_SIZE, 1743 .min_hw_rev = CC_HW_REV_630, 1744 .std_body = CC_STD_NIST, 1745 }, 1746 { 1747 .name = "sm3", 1748 .driver_name = "sm3-ccree", 1749 .blocksize = SM3_BLOCK_SIZE, 1750 .is_mac = false, 1751 .template_ahash = { 1752 .init = cc_hash_init, 1753 .update = cc_hash_update, 1754 .final = cc_hash_final, 1755 .finup = cc_hash_finup, 1756 .digest = cc_hash_digest, 1757 .export = cc_hash_export, 1758 .import = cc_hash_import, 1759 .setkey = cc_hash_setkey, 1760 .halg = { 1761 .digestsize = SM3_DIGEST_SIZE, 1762 .statesize = CC_STATE_SIZE(SM3_DIGEST_SIZE), 1763 }, 1764 }, 1765 .hash_mode = DRV_HASH_SM3, 1766 .hw_mode = DRV_HASH_HW_SM3, 1767 .inter_digestsize = SM3_DIGEST_SIZE, 1768 .min_hw_rev = CC_HW_REV_713, 1769 .std_body = CC_STD_OSCCA, 1770 }, 1771 { 1772 .mac_name = "xcbc(aes)", 1773 .mac_driver_name = "xcbc-aes-ccree", 1774 .blocksize = AES_BLOCK_SIZE, 1775 .is_mac = true, 1776 .template_ahash = { 1777 .init = cc_hash_init, 1778 .update = cc_mac_update, 1779 .final = cc_mac_final, 1780 .finup = cc_mac_finup, 1781 .digest = cc_mac_digest, 1782 .setkey = cc_xcbc_setkey, 1783 .export = cc_hash_export, 1784 .import = cc_hash_import, 1785 .halg = { 1786 .digestsize = AES_BLOCK_SIZE, 1787 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE), 1788 }, 1789 }, 1790 .hash_mode = DRV_HASH_NULL, 1791 .hw_mode = DRV_CIPHER_XCBC_MAC, 1792 .inter_digestsize = AES_BLOCK_SIZE, 1793 .min_hw_rev = CC_HW_REV_630, 1794 .std_body = CC_STD_NIST, 1795 }, 1796 { 1797 .mac_name = "cmac(aes)", 1798 .mac_driver_name = "cmac-aes-ccree", 1799 .blocksize = AES_BLOCK_SIZE, 1800 .is_mac = true, 1801 .template_ahash = { 1802 .init = cc_hash_init, 1803 .update = cc_mac_update, 1804 .final = cc_mac_final, 1805 .finup = cc_mac_finup, 1806 .digest = cc_mac_digest, 1807 .setkey = cc_cmac_setkey, 1808 .export = cc_hash_export, 1809 .import = cc_hash_import, 1810 .halg = { 1811 .digestsize = AES_BLOCK_SIZE, 1812 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE), 1813 }, 1814 }, 1815 .hash_mode = DRV_HASH_NULL, 1816 .hw_mode = DRV_CIPHER_CMAC, 1817 .inter_digestsize = AES_BLOCK_SIZE, 1818 .min_hw_rev = CC_HW_REV_630, 1819 .std_body = CC_STD_NIST, 1820 }, 1821 }; 1822 1823 static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template, 1824 struct device *dev, bool keyed) 1825 { 1826 struct cc_hash_alg *t_crypto_alg; 1827 struct crypto_alg *alg; 1828 struct ahash_alg *halg; 1829 1830 t_crypto_alg = devm_kzalloc(dev, sizeof(*t_crypto_alg), GFP_KERNEL); 1831 if (!t_crypto_alg) 1832 return ERR_PTR(-ENOMEM); 1833 1834 t_crypto_alg->ahash_alg = template->template_ahash; 1835 halg = &t_crypto_alg->ahash_alg; 1836 alg = &halg->halg.base; 1837 1838 if (keyed) { 1839 strscpy(alg->cra_name, template->mac_name); 1840 strscpy(alg->cra_driver_name, template->mac_driver_name); 1841 } else { 1842 halg->setkey = NULL; 1843 strscpy(alg->cra_name, template->name); 1844 strscpy(alg->cra_driver_name, template->driver_name); 1845 } 1846 alg->cra_module = THIS_MODULE; 1847 alg->cra_ctxsize = sizeof(struct cc_hash_ctx) + crypto_dma_padding(); 1848 alg->cra_priority = CC_CRA_PRIO; 1849 alg->cra_blocksize = template->blocksize; 1850 alg->cra_alignmask = 0; 1851 alg->cra_exit = cc_cra_exit; 1852 1853 alg->cra_init = cc_cra_init; 1854 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 1855 1856 t_crypto_alg->hash_mode = template->hash_mode; 1857 t_crypto_alg->hw_mode = template->hw_mode; 1858 t_crypto_alg->inter_digestsize = template->inter_digestsize; 1859 1860 return t_crypto_alg; 1861 } 1862 1863 static int cc_init_copy_sram(struct cc_drvdata *drvdata, const u32 *data, 1864 unsigned int size, u32 *sram_buff_ofs) 1865 { 1866 struct cc_hw_desc larval_seq[CC_DIGEST_SIZE_MAX / sizeof(u32)]; 1867 unsigned int larval_seq_len = 0; 1868 int rc; 1869 1870 cc_set_sram_desc(data, *sram_buff_ofs, size / sizeof(*data), 1871 larval_seq, &larval_seq_len); 1872 rc = send_request_init(drvdata, larval_seq, larval_seq_len); 1873 if (rc) 1874 return rc; 1875 1876 *sram_buff_ofs += size; 1877 return 0; 1878 } 1879 1880 int cc_init_hash_sram(struct cc_drvdata *drvdata) 1881 { 1882 struct cc_hash_handle *hash_handle = drvdata->hash_handle; 1883 u32 sram_buff_ofs = hash_handle->digest_len_sram_addr; 1884 bool large_sha_supported = (drvdata->hw_rev >= CC_HW_REV_712); 1885 bool sm3_supported = (drvdata->hw_rev >= CC_HW_REV_713); 1886 int rc = 0; 1887 1888 /* Copy-to-sram digest-len */ 1889 rc = cc_init_copy_sram(drvdata, cc_digest_len_init, 1890 sizeof(cc_digest_len_init), &sram_buff_ofs); 1891 if (rc) 1892 goto init_digest_const_err; 1893 1894 if (large_sha_supported) { 1895 /* Copy-to-sram digest-len for sha384/512 */ 1896 rc = cc_init_copy_sram(drvdata, cc_digest_len_sha512_init, 1897 sizeof(cc_digest_len_sha512_init), 1898 &sram_buff_ofs); 1899 if (rc) 1900 goto init_digest_const_err; 1901 } 1902 1903 /* The initial digests offset */ 1904 hash_handle->larval_digest_sram_addr = sram_buff_ofs; 1905 1906 /* Copy-to-sram initial SHA* digests */ 1907 rc = cc_init_copy_sram(drvdata, cc_md5_init, sizeof(cc_md5_init), 1908 &sram_buff_ofs); 1909 if (rc) 1910 goto init_digest_const_err; 1911 1912 rc = cc_init_copy_sram(drvdata, cc_sha1_init, sizeof(cc_sha1_init), 1913 &sram_buff_ofs); 1914 if (rc) 1915 goto init_digest_const_err; 1916 1917 rc = cc_init_copy_sram(drvdata, cc_sha224_init, sizeof(cc_sha224_init), 1918 &sram_buff_ofs); 1919 if (rc) 1920 goto init_digest_const_err; 1921 1922 rc = cc_init_copy_sram(drvdata, cc_sha256_init, sizeof(cc_sha256_init), 1923 &sram_buff_ofs); 1924 if (rc) 1925 goto init_digest_const_err; 1926 1927 if (sm3_supported) { 1928 rc = cc_init_copy_sram(drvdata, cc_sm3_init, 1929 sizeof(cc_sm3_init), &sram_buff_ofs); 1930 if (rc) 1931 goto init_digest_const_err; 1932 } 1933 1934 if (large_sha_supported) { 1935 rc = cc_init_copy_sram(drvdata, cc_sha384_init, 1936 sizeof(cc_sha384_init), &sram_buff_ofs); 1937 if (rc) 1938 goto init_digest_const_err; 1939 1940 rc = cc_init_copy_sram(drvdata, cc_sha512_init, 1941 sizeof(cc_sha512_init), &sram_buff_ofs); 1942 if (rc) 1943 goto init_digest_const_err; 1944 } 1945 1946 init_digest_const_err: 1947 return rc; 1948 } 1949 1950 int cc_hash_alloc(struct cc_drvdata *drvdata) 1951 { 1952 struct cc_hash_handle *hash_handle; 1953 u32 sram_buff; 1954 u32 sram_size_to_alloc; 1955 struct device *dev = drvdata_to_dev(drvdata); 1956 int rc = 0; 1957 int alg; 1958 1959 hash_handle = devm_kzalloc(dev, sizeof(*hash_handle), GFP_KERNEL); 1960 if (!hash_handle) 1961 return -ENOMEM; 1962 1963 INIT_LIST_HEAD(&hash_handle->hash_list); 1964 drvdata->hash_handle = hash_handle; 1965 1966 sram_size_to_alloc = sizeof(cc_digest_len_init) + 1967 sizeof(cc_md5_init) + 1968 sizeof(cc_sha1_init) + 1969 sizeof(cc_sha224_init) + 1970 sizeof(cc_sha256_init); 1971 1972 if (drvdata->hw_rev >= CC_HW_REV_713) 1973 sram_size_to_alloc += sizeof(cc_sm3_init); 1974 1975 if (drvdata->hw_rev >= CC_HW_REV_712) 1976 sram_size_to_alloc += sizeof(cc_digest_len_sha512_init) + 1977 sizeof(cc_sha384_init) + sizeof(cc_sha512_init); 1978 1979 sram_buff = cc_sram_alloc(drvdata, sram_size_to_alloc); 1980 if (sram_buff == NULL_SRAM_ADDR) { 1981 rc = -ENOMEM; 1982 goto fail; 1983 } 1984 1985 /* The initial digest-len offset */ 1986 hash_handle->digest_len_sram_addr = sram_buff; 1987 1988 /*must be set before the alg registration as it is being used there*/ 1989 rc = cc_init_hash_sram(drvdata); 1990 if (rc) { 1991 dev_err(dev, "Init digest CONST failed (rc=%d)\n", rc); 1992 goto fail; 1993 } 1994 1995 /* ahash registration */ 1996 for (alg = 0; alg < ARRAY_SIZE(driver_hash); alg++) { 1997 struct cc_hash_alg *t_alg; 1998 int hw_mode = driver_hash[alg].hw_mode; 1999 2000 /* Check that the HW revision and variants are suitable */ 2001 if ((driver_hash[alg].min_hw_rev > drvdata->hw_rev) || 2002 !(drvdata->std_bodies & driver_hash[alg].std_body)) 2003 continue; 2004 2005 if (driver_hash[alg].is_mac) { 2006 /* register hmac version */ 2007 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, true); 2008 if (IS_ERR(t_alg)) { 2009 rc = PTR_ERR(t_alg); 2010 dev_err(dev, "%s alg allocation failed\n", 2011 driver_hash[alg].driver_name); 2012 goto fail; 2013 } 2014 t_alg->drvdata = drvdata; 2015 2016 rc = crypto_register_ahash(&t_alg->ahash_alg); 2017 if (rc) { 2018 dev_err(dev, "%s alg registration failed\n", 2019 driver_hash[alg].driver_name); 2020 goto fail; 2021 } 2022 2023 list_add_tail(&t_alg->entry, &hash_handle->hash_list); 2024 } 2025 if (hw_mode == DRV_CIPHER_XCBC_MAC || 2026 hw_mode == DRV_CIPHER_CMAC) 2027 continue; 2028 2029 /* register hash version */ 2030 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, false); 2031 if (IS_ERR(t_alg)) { 2032 rc = PTR_ERR(t_alg); 2033 dev_err(dev, "%s alg allocation failed\n", 2034 driver_hash[alg].driver_name); 2035 goto fail; 2036 } 2037 t_alg->drvdata = drvdata; 2038 2039 rc = crypto_register_ahash(&t_alg->ahash_alg); 2040 if (rc) { 2041 dev_err(dev, "%s alg registration failed\n", 2042 driver_hash[alg].driver_name); 2043 goto fail; 2044 } 2045 2046 list_add_tail(&t_alg->entry, &hash_handle->hash_list); 2047 } 2048 2049 return 0; 2050 2051 fail: 2052 cc_hash_free(drvdata); 2053 return rc; 2054 } 2055 2056 int cc_hash_free(struct cc_drvdata *drvdata) 2057 { 2058 struct cc_hash_alg *t_hash_alg, *hash_n; 2059 struct cc_hash_handle *hash_handle = drvdata->hash_handle; 2060 2061 list_for_each_entry_safe(t_hash_alg, hash_n, &hash_handle->hash_list, 2062 entry) { 2063 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 2064 list_del(&t_hash_alg->entry); 2065 } 2066 2067 return 0; 2068 } 2069 2070 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[], 2071 unsigned int *seq_size) 2072 { 2073 unsigned int idx = *seq_size; 2074 struct ahash_req_ctx *state = ahash_request_ctx_dma(areq); 2075 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2076 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 2077 2078 /* Setup XCBC MAC K1 */ 2079 hw_desc_init(&desc[idx]); 2080 set_din_type(&desc[idx], DMA_DLLI, (ctx->opad_tmp_keys_dma_addr + 2081 XCBC_MAC_K1_OFFSET), 2082 CC_AES_128_BIT_KEY_SIZE, NS_BIT); 2083 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 2084 set_hash_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC, ctx->hash_mode); 2085 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2086 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 2087 set_flow_mode(&desc[idx], S_DIN_to_AES); 2088 idx++; 2089 2090 /* Setup XCBC MAC K2 */ 2091 hw_desc_init(&desc[idx]); 2092 set_din_type(&desc[idx], DMA_DLLI, 2093 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET), 2094 CC_AES_128_BIT_KEY_SIZE, NS_BIT); 2095 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 2096 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 2097 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2098 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 2099 set_flow_mode(&desc[idx], S_DIN_to_AES); 2100 idx++; 2101 2102 /* Setup XCBC MAC K3 */ 2103 hw_desc_init(&desc[idx]); 2104 set_din_type(&desc[idx], DMA_DLLI, 2105 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET), 2106 CC_AES_128_BIT_KEY_SIZE, NS_BIT); 2107 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); 2108 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 2109 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2110 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 2111 set_flow_mode(&desc[idx], S_DIN_to_AES); 2112 idx++; 2113 2114 /* Loading MAC state */ 2115 hw_desc_init(&desc[idx]); 2116 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 2117 CC_AES_BLOCK_SIZE, NS_BIT); 2118 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 2119 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 2120 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2121 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 2122 set_flow_mode(&desc[idx], S_DIN_to_AES); 2123 idx++; 2124 *seq_size = idx; 2125 } 2126 2127 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[], 2128 unsigned int *seq_size) 2129 { 2130 unsigned int idx = *seq_size; 2131 struct ahash_req_ctx *state = ahash_request_ctx_dma(areq); 2132 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 2133 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm); 2134 2135 /* Setup CMAC Key */ 2136 hw_desc_init(&desc[idx]); 2137 set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr, 2138 ((ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE : 2139 ctx->key_params.keylen), NS_BIT); 2140 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 2141 set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC); 2142 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2143 set_key_size_aes(&desc[idx], ctx->key_params.keylen); 2144 set_flow_mode(&desc[idx], S_DIN_to_AES); 2145 idx++; 2146 2147 /* Load MAC state */ 2148 hw_desc_init(&desc[idx]); 2149 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr, 2150 CC_AES_BLOCK_SIZE, NS_BIT); 2151 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 2152 set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC); 2153 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 2154 set_key_size_aes(&desc[idx], ctx->key_params.keylen); 2155 set_flow_mode(&desc[idx], S_DIN_to_AES); 2156 idx++; 2157 *seq_size = idx; 2158 } 2159 2160 static void cc_set_desc(struct ahash_req_ctx *areq_ctx, 2161 struct cc_hash_ctx *ctx, unsigned int flow_mode, 2162 struct cc_hw_desc desc[], bool is_not_last_data, 2163 unsigned int *seq_size) 2164 { 2165 unsigned int idx = *seq_size; 2166 struct device *dev = drvdata_to_dev(ctx->drvdata); 2167 2168 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_DLLI) { 2169 hw_desc_init(&desc[idx]); 2170 set_din_type(&desc[idx], DMA_DLLI, 2171 sg_dma_address(areq_ctx->curr_sg), 2172 areq_ctx->curr_sg->length, NS_BIT); 2173 set_flow_mode(&desc[idx], flow_mode); 2174 idx++; 2175 } else { 2176 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) { 2177 dev_dbg(dev, " NULL mode\n"); 2178 /* nothing to build */ 2179 return; 2180 } 2181 /* bypass */ 2182 hw_desc_init(&desc[idx]); 2183 set_din_type(&desc[idx], DMA_DLLI, 2184 areq_ctx->mlli_params.mlli_dma_addr, 2185 areq_ctx->mlli_params.mlli_len, NS_BIT); 2186 set_dout_sram(&desc[idx], ctx->drvdata->mlli_sram_addr, 2187 areq_ctx->mlli_params.mlli_len); 2188 set_flow_mode(&desc[idx], BYPASS); 2189 idx++; 2190 /* process */ 2191 hw_desc_init(&desc[idx]); 2192 set_din_type(&desc[idx], DMA_MLLI, 2193 ctx->drvdata->mlli_sram_addr, 2194 areq_ctx->mlli_nents, NS_BIT); 2195 set_flow_mode(&desc[idx], flow_mode); 2196 idx++; 2197 } 2198 if (is_not_last_data) 2199 set_din_not_last_indication(&desc[(idx - 1)]); 2200 /* return updated desc sequence size */ 2201 *seq_size = idx; 2202 } 2203 2204 static const void *cc_larval_digest(struct device *dev, u32 mode) 2205 { 2206 switch (mode) { 2207 case DRV_HASH_MD5: 2208 return cc_md5_init; 2209 case DRV_HASH_SHA1: 2210 return cc_sha1_init; 2211 case DRV_HASH_SHA224: 2212 return cc_sha224_init; 2213 case DRV_HASH_SHA256: 2214 return cc_sha256_init; 2215 case DRV_HASH_SHA384: 2216 return cc_sha384_init; 2217 case DRV_HASH_SHA512: 2218 return cc_sha512_init; 2219 case DRV_HASH_SM3: 2220 return cc_sm3_init; 2221 default: 2222 dev_err(dev, "Invalid hash mode (%d)\n", mode); 2223 return cc_md5_init; 2224 } 2225 } 2226 2227 /** 2228 * cc_larval_digest_addr() - Get the address of the initial digest in SRAM 2229 * according to the given hash mode 2230 * 2231 * @drvdata: Associated device driver context 2232 * @mode: The Hash mode. Supported modes: MD5/SHA1/SHA224/SHA256 2233 * 2234 * Return: 2235 * The address of the initial digest in SRAM 2236 */ 2237 u32 cc_larval_digest_addr(void *drvdata, u32 mode) 2238 { 2239 struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata; 2240 struct cc_hash_handle *hash_handle = _drvdata->hash_handle; 2241 struct device *dev = drvdata_to_dev(_drvdata); 2242 bool sm3_supported = (_drvdata->hw_rev >= CC_HW_REV_713); 2243 u32 addr; 2244 2245 switch (mode) { 2246 case DRV_HASH_NULL: 2247 break; /*Ignore*/ 2248 case DRV_HASH_MD5: 2249 return (hash_handle->larval_digest_sram_addr); 2250 case DRV_HASH_SHA1: 2251 return (hash_handle->larval_digest_sram_addr + 2252 sizeof(cc_md5_init)); 2253 case DRV_HASH_SHA224: 2254 return (hash_handle->larval_digest_sram_addr + 2255 sizeof(cc_md5_init) + 2256 sizeof(cc_sha1_init)); 2257 case DRV_HASH_SHA256: 2258 return (hash_handle->larval_digest_sram_addr + 2259 sizeof(cc_md5_init) + 2260 sizeof(cc_sha1_init) + 2261 sizeof(cc_sha224_init)); 2262 case DRV_HASH_SM3: 2263 return (hash_handle->larval_digest_sram_addr + 2264 sizeof(cc_md5_init) + 2265 sizeof(cc_sha1_init) + 2266 sizeof(cc_sha224_init) + 2267 sizeof(cc_sha256_init)); 2268 case DRV_HASH_SHA384: 2269 addr = (hash_handle->larval_digest_sram_addr + 2270 sizeof(cc_md5_init) + 2271 sizeof(cc_sha1_init) + 2272 sizeof(cc_sha224_init) + 2273 sizeof(cc_sha256_init)); 2274 if (sm3_supported) 2275 addr += sizeof(cc_sm3_init); 2276 return addr; 2277 case DRV_HASH_SHA512: 2278 addr = (hash_handle->larval_digest_sram_addr + 2279 sizeof(cc_md5_init) + 2280 sizeof(cc_sha1_init) + 2281 sizeof(cc_sha224_init) + 2282 sizeof(cc_sha256_init) + 2283 sizeof(cc_sha384_init)); 2284 if (sm3_supported) 2285 addr += sizeof(cc_sm3_init); 2286 return addr; 2287 default: 2288 dev_err(dev, "Invalid hash mode (%d)\n", mode); 2289 } 2290 2291 /*This is valid wrong value to avoid kernel crash*/ 2292 return hash_handle->larval_digest_sram_addr; 2293 } 2294 2295 u32 cc_digest_len_addr(void *drvdata, u32 mode) 2296 { 2297 struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata; 2298 struct cc_hash_handle *hash_handle = _drvdata->hash_handle; 2299 u32 digest_len_addr = hash_handle->digest_len_sram_addr; 2300 2301 switch (mode) { 2302 case DRV_HASH_SHA1: 2303 case DRV_HASH_SHA224: 2304 case DRV_HASH_SHA256: 2305 case DRV_HASH_MD5: 2306 return digest_len_addr; 2307 case DRV_HASH_SHA384: 2308 case DRV_HASH_SHA512: 2309 return digest_len_addr + sizeof(cc_digest_len_init); 2310 default: 2311 return digest_len_addr; /*to avoid kernel crash*/ 2312 } 2313 } 2314