1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <crypto/algapi.h> 7 #include <crypto/internal/aead.h> 8 #include <crypto/authenc.h> 9 #include <crypto/internal/des.h> 10 #include <linux/rtnetlink.h> 11 #include "cc_driver.h" 12 #include "cc_buffer_mgr.h" 13 #include "cc_aead.h" 14 #include "cc_request_mgr.h" 15 #include "cc_hash.h" 16 #include "cc_sram_mgr.h" 17 18 #define template_aead template_u.aead 19 20 #define MAX_AEAD_SETKEY_SEQ 12 21 #define MAX_AEAD_PROCESS_SEQ 23 22 23 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE) 24 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE) 25 26 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE 27 28 struct cc_aead_handle { 29 cc_sram_addr_t sram_workspace_addr; 30 struct list_head aead_list; 31 }; 32 33 struct cc_hmac_s { 34 u8 *padded_authkey; 35 u8 *ipad_opad; /* IPAD, OPAD*/ 36 dma_addr_t padded_authkey_dma_addr; 37 dma_addr_t ipad_opad_dma_addr; 38 }; 39 40 struct cc_xcbc_s { 41 u8 *xcbc_keys; /* K1,K2,K3 */ 42 dma_addr_t xcbc_keys_dma_addr; 43 }; 44 45 struct cc_aead_ctx { 46 struct cc_drvdata *drvdata; 47 u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */ 48 u8 *enckey; 49 dma_addr_t enckey_dma_addr; 50 union { 51 struct cc_hmac_s hmac; 52 struct cc_xcbc_s xcbc; 53 } auth_state; 54 unsigned int enc_keylen; 55 unsigned int auth_keylen; 56 unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */ 57 unsigned int hash_len; 58 enum drv_cipher_mode cipher_mode; 59 enum cc_flow_mode flow_mode; 60 enum drv_hash_mode auth_mode; 61 }; 62 63 static inline bool valid_assoclen(struct aead_request *req) 64 { 65 return ((req->assoclen == 16) || (req->assoclen == 20)); 66 } 67 68 static void cc_aead_exit(struct crypto_aead *tfm) 69 { 70 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 71 struct device *dev = drvdata_to_dev(ctx->drvdata); 72 73 dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm), 74 crypto_tfm_alg_name(&tfm->base)); 75 76 /* Unmap enckey buffer */ 77 if (ctx->enckey) { 78 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey, 79 ctx->enckey_dma_addr); 80 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n", 81 &ctx->enckey_dma_addr); 82 ctx->enckey_dma_addr = 0; 83 ctx->enckey = NULL; 84 } 85 86 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 87 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 88 89 if (xcbc->xcbc_keys) { 90 dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3, 91 xcbc->xcbc_keys, 92 xcbc->xcbc_keys_dma_addr); 93 } 94 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n", 95 &xcbc->xcbc_keys_dma_addr); 96 xcbc->xcbc_keys_dma_addr = 0; 97 xcbc->xcbc_keys = NULL; 98 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */ 99 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 100 101 if (hmac->ipad_opad) { 102 dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE, 103 hmac->ipad_opad, 104 hmac->ipad_opad_dma_addr); 105 dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n", 106 &hmac->ipad_opad_dma_addr); 107 hmac->ipad_opad_dma_addr = 0; 108 hmac->ipad_opad = NULL; 109 } 110 if (hmac->padded_authkey) { 111 dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE, 112 hmac->padded_authkey, 113 hmac->padded_authkey_dma_addr); 114 dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n", 115 &hmac->padded_authkey_dma_addr); 116 hmac->padded_authkey_dma_addr = 0; 117 hmac->padded_authkey = NULL; 118 } 119 } 120 } 121 122 static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm) 123 { 124 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 125 126 return cc_get_default_hash_len(ctx->drvdata); 127 } 128 129 static int cc_aead_init(struct crypto_aead *tfm) 130 { 131 struct aead_alg *alg = crypto_aead_alg(tfm); 132 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 133 struct cc_crypto_alg *cc_alg = 134 container_of(alg, struct cc_crypto_alg, aead_alg); 135 struct device *dev = drvdata_to_dev(cc_alg->drvdata); 136 137 dev_dbg(dev, "Initializing context @%p for %s\n", ctx, 138 crypto_tfm_alg_name(&tfm->base)); 139 140 /* Initialize modes in instance */ 141 ctx->cipher_mode = cc_alg->cipher_mode; 142 ctx->flow_mode = cc_alg->flow_mode; 143 ctx->auth_mode = cc_alg->auth_mode; 144 ctx->drvdata = cc_alg->drvdata; 145 crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx)); 146 147 /* Allocate key buffer, cache line aligned */ 148 ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE, 149 &ctx->enckey_dma_addr, GFP_KERNEL); 150 if (!ctx->enckey) { 151 dev_err(dev, "Failed allocating key buffer\n"); 152 goto init_failed; 153 } 154 dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n", 155 ctx->enckey); 156 157 /* Set default authlen value */ 158 159 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 160 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 161 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3; 162 163 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */ 164 /* (and temporary for user key - up to 256b) */ 165 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size, 166 &xcbc->xcbc_keys_dma_addr, 167 GFP_KERNEL); 168 if (!xcbc->xcbc_keys) { 169 dev_err(dev, "Failed allocating buffer for XCBC keys\n"); 170 goto init_failed; 171 } 172 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */ 173 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 174 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE; 175 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr; 176 177 /* Allocate dma-coherent buffer for IPAD + OPAD */ 178 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size, 179 &hmac->ipad_opad_dma_addr, 180 GFP_KERNEL); 181 182 if (!hmac->ipad_opad) { 183 dev_err(dev, "Failed allocating IPAD/OPAD buffer\n"); 184 goto init_failed; 185 } 186 187 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n", 188 hmac->ipad_opad); 189 190 hmac->padded_authkey = dma_alloc_coherent(dev, 191 MAX_HMAC_BLOCK_SIZE, 192 pkey_dma, 193 GFP_KERNEL); 194 195 if (!hmac->padded_authkey) { 196 dev_err(dev, "failed to allocate padded_authkey\n"); 197 goto init_failed; 198 } 199 } else { 200 ctx->auth_state.hmac.ipad_opad = NULL; 201 ctx->auth_state.hmac.padded_authkey = NULL; 202 } 203 ctx->hash_len = cc_get_aead_hash_len(tfm); 204 205 return 0; 206 207 init_failed: 208 cc_aead_exit(tfm); 209 return -ENOMEM; 210 } 211 212 static void cc_aead_complete(struct device *dev, void *cc_req, int err) 213 { 214 struct aead_request *areq = (struct aead_request *)cc_req; 215 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 216 struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); 217 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 218 219 /* BACKLOG notification */ 220 if (err == -EINPROGRESS) 221 goto done; 222 223 cc_unmap_aead_request(dev, areq); 224 225 /* Restore ordinary iv pointer */ 226 areq->iv = areq_ctx->backup_iv; 227 228 if (err) 229 goto done; 230 231 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 232 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr, 233 ctx->authsize) != 0) { 234 dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n", 235 ctx->authsize, ctx->cipher_mode); 236 /* In case of payload authentication failure, MUST NOT 237 * revealed the decrypted message --> zero its memory. 238 */ 239 sg_zero_buffer(areq->dst, sg_nents(areq->dst), 240 areq->cryptlen, 0); 241 err = -EBADMSG; 242 } 243 /*ENCRYPT*/ 244 } else if (areq_ctx->is_icv_fragmented) { 245 u32 skip = areq->cryptlen + areq_ctx->dst_offset; 246 247 cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl, 248 skip, (skip + ctx->authsize), 249 CC_SG_FROM_BUF); 250 } 251 done: 252 aead_request_complete(areq, err); 253 } 254 255 static unsigned int xcbc_setkey(struct cc_hw_desc *desc, 256 struct cc_aead_ctx *ctx) 257 { 258 /* Load the AES key */ 259 hw_desc_init(&desc[0]); 260 /* We are using for the source/user key the same buffer 261 * as for the output keys, * because after this key loading it 262 * is not needed anymore 263 */ 264 set_din_type(&desc[0], DMA_DLLI, 265 ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen, 266 NS_BIT); 267 set_cipher_mode(&desc[0], DRV_CIPHER_ECB); 268 set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT); 269 set_key_size_aes(&desc[0], ctx->auth_keylen); 270 set_flow_mode(&desc[0], S_DIN_to_AES); 271 set_setup_mode(&desc[0], SETUP_LOAD_KEY0); 272 273 hw_desc_init(&desc[1]); 274 set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE); 275 set_flow_mode(&desc[1], DIN_AES_DOUT); 276 set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr, 277 AES_KEYSIZE_128, NS_BIT, 0); 278 279 hw_desc_init(&desc[2]); 280 set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE); 281 set_flow_mode(&desc[2], DIN_AES_DOUT); 282 set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 283 + AES_KEYSIZE_128), 284 AES_KEYSIZE_128, NS_BIT, 0); 285 286 hw_desc_init(&desc[3]); 287 set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE); 288 set_flow_mode(&desc[3], DIN_AES_DOUT); 289 set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 290 + 2 * AES_KEYSIZE_128), 291 AES_KEYSIZE_128, NS_BIT, 0); 292 293 return 4; 294 } 295 296 static unsigned int hmac_setkey(struct cc_hw_desc *desc, 297 struct cc_aead_ctx *ctx) 298 { 299 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; 300 unsigned int digest_ofs = 0; 301 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 302 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 303 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 304 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 305 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 306 307 unsigned int idx = 0; 308 int i; 309 310 /* calc derived HMAC key */ 311 for (i = 0; i < 2; i++) { 312 /* Load hash initial state */ 313 hw_desc_init(&desc[idx]); 314 set_cipher_mode(&desc[idx], hash_mode); 315 set_din_sram(&desc[idx], 316 cc_larval_digest_addr(ctx->drvdata, 317 ctx->auth_mode), 318 digest_size); 319 set_flow_mode(&desc[idx], S_DIN_to_HASH); 320 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 321 idx++; 322 323 /* Load the hash current length*/ 324 hw_desc_init(&desc[idx]); 325 set_cipher_mode(&desc[idx], hash_mode); 326 set_din_const(&desc[idx], 0, ctx->hash_len); 327 set_flow_mode(&desc[idx], S_DIN_to_HASH); 328 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 329 idx++; 330 331 /* Prepare ipad key */ 332 hw_desc_init(&desc[idx]); 333 set_xor_val(&desc[idx], hmac_pad_const[i]); 334 set_cipher_mode(&desc[idx], hash_mode); 335 set_flow_mode(&desc[idx], S_DIN_to_HASH); 336 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 337 idx++; 338 339 /* Perform HASH update */ 340 hw_desc_init(&desc[idx]); 341 set_din_type(&desc[idx], DMA_DLLI, 342 hmac->padded_authkey_dma_addr, 343 SHA256_BLOCK_SIZE, NS_BIT); 344 set_cipher_mode(&desc[idx], hash_mode); 345 set_xor_active(&desc[idx]); 346 set_flow_mode(&desc[idx], DIN_HASH); 347 idx++; 348 349 /* Get the digset */ 350 hw_desc_init(&desc[idx]); 351 set_cipher_mode(&desc[idx], hash_mode); 352 set_dout_dlli(&desc[idx], 353 (hmac->ipad_opad_dma_addr + digest_ofs), 354 digest_size, NS_BIT, 0); 355 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 356 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 357 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 358 idx++; 359 360 digest_ofs += digest_size; 361 } 362 363 return idx; 364 } 365 366 static int validate_keys_sizes(struct cc_aead_ctx *ctx) 367 { 368 struct device *dev = drvdata_to_dev(ctx->drvdata); 369 370 dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n", 371 ctx->enc_keylen, ctx->auth_keylen); 372 373 switch (ctx->auth_mode) { 374 case DRV_HASH_SHA1: 375 case DRV_HASH_SHA256: 376 break; 377 case DRV_HASH_XCBC_MAC: 378 if (ctx->auth_keylen != AES_KEYSIZE_128 && 379 ctx->auth_keylen != AES_KEYSIZE_192 && 380 ctx->auth_keylen != AES_KEYSIZE_256) 381 return -ENOTSUPP; 382 break; 383 case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */ 384 if (ctx->auth_keylen > 0) 385 return -EINVAL; 386 break; 387 default: 388 dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode); 389 return -EINVAL; 390 } 391 /* Check cipher key size */ 392 if (ctx->flow_mode == S_DIN_to_DES) { 393 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) { 394 dev_err(dev, "Invalid cipher(3DES) key size: %u\n", 395 ctx->enc_keylen); 396 return -EINVAL; 397 } 398 } else { /* Default assumed to be AES ciphers */ 399 if (ctx->enc_keylen != AES_KEYSIZE_128 && 400 ctx->enc_keylen != AES_KEYSIZE_192 && 401 ctx->enc_keylen != AES_KEYSIZE_256) { 402 dev_err(dev, "Invalid cipher(AES) key size: %u\n", 403 ctx->enc_keylen); 404 return -EINVAL; 405 } 406 } 407 408 return 0; /* All tests of keys sizes passed */ 409 } 410 411 /* This function prepers the user key so it can pass to the hmac processing 412 * (copy to intenral buffer or hash in case of key longer than block 413 */ 414 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey, 415 unsigned int keylen) 416 { 417 dma_addr_t key_dma_addr = 0; 418 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 419 struct device *dev = drvdata_to_dev(ctx->drvdata); 420 u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode); 421 struct cc_crypto_req cc_req = {}; 422 unsigned int blocksize; 423 unsigned int digestsize; 424 unsigned int hashmode; 425 unsigned int idx = 0; 426 int rc = 0; 427 u8 *key = NULL; 428 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 429 dma_addr_t padded_authkey_dma_addr = 430 ctx->auth_state.hmac.padded_authkey_dma_addr; 431 432 switch (ctx->auth_mode) { /* auth_key required and >0 */ 433 case DRV_HASH_SHA1: 434 blocksize = SHA1_BLOCK_SIZE; 435 digestsize = SHA1_DIGEST_SIZE; 436 hashmode = DRV_HASH_HW_SHA1; 437 break; 438 case DRV_HASH_SHA256: 439 default: 440 blocksize = SHA256_BLOCK_SIZE; 441 digestsize = SHA256_DIGEST_SIZE; 442 hashmode = DRV_HASH_HW_SHA256; 443 } 444 445 if (keylen != 0) { 446 447 key = kmemdup(authkey, keylen, GFP_KERNEL); 448 if (!key) 449 return -ENOMEM; 450 451 key_dma_addr = dma_map_single(dev, (void *)key, keylen, 452 DMA_TO_DEVICE); 453 if (dma_mapping_error(dev, key_dma_addr)) { 454 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 455 key, keylen); 456 kzfree(key); 457 return -ENOMEM; 458 } 459 if (keylen > blocksize) { 460 /* Load hash initial state */ 461 hw_desc_init(&desc[idx]); 462 set_cipher_mode(&desc[idx], hashmode); 463 set_din_sram(&desc[idx], larval_addr, digestsize); 464 set_flow_mode(&desc[idx], S_DIN_to_HASH); 465 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 466 idx++; 467 468 /* Load the hash current length*/ 469 hw_desc_init(&desc[idx]); 470 set_cipher_mode(&desc[idx], hashmode); 471 set_din_const(&desc[idx], 0, ctx->hash_len); 472 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 473 set_flow_mode(&desc[idx], S_DIN_to_HASH); 474 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 475 idx++; 476 477 hw_desc_init(&desc[idx]); 478 set_din_type(&desc[idx], DMA_DLLI, 479 key_dma_addr, keylen, NS_BIT); 480 set_flow_mode(&desc[idx], DIN_HASH); 481 idx++; 482 483 /* Get hashed key */ 484 hw_desc_init(&desc[idx]); 485 set_cipher_mode(&desc[idx], hashmode); 486 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 487 digestsize, NS_BIT, 0); 488 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 489 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 490 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 491 set_cipher_config0(&desc[idx], 492 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 493 idx++; 494 495 hw_desc_init(&desc[idx]); 496 set_din_const(&desc[idx], 0, (blocksize - digestsize)); 497 set_flow_mode(&desc[idx], BYPASS); 498 set_dout_dlli(&desc[idx], (padded_authkey_dma_addr + 499 digestsize), (blocksize - digestsize), 500 NS_BIT, 0); 501 idx++; 502 } else { 503 hw_desc_init(&desc[idx]); 504 set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, 505 keylen, NS_BIT); 506 set_flow_mode(&desc[idx], BYPASS); 507 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 508 keylen, NS_BIT, 0); 509 idx++; 510 511 if ((blocksize - keylen) != 0) { 512 hw_desc_init(&desc[idx]); 513 set_din_const(&desc[idx], 0, 514 (blocksize - keylen)); 515 set_flow_mode(&desc[idx], BYPASS); 516 set_dout_dlli(&desc[idx], 517 (padded_authkey_dma_addr + 518 keylen), 519 (blocksize - keylen), NS_BIT, 0); 520 idx++; 521 } 522 } 523 } else { 524 hw_desc_init(&desc[idx]); 525 set_din_const(&desc[idx], 0, (blocksize - keylen)); 526 set_flow_mode(&desc[idx], BYPASS); 527 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 528 blocksize, NS_BIT, 0); 529 idx++; 530 } 531 532 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 533 if (rc) 534 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 535 536 if (key_dma_addr) 537 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE); 538 539 kzfree(key); 540 541 return rc; 542 } 543 544 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key, 545 unsigned int keylen) 546 { 547 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 548 struct cc_crypto_req cc_req = {}; 549 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 550 unsigned int seq_len = 0; 551 struct device *dev = drvdata_to_dev(ctx->drvdata); 552 const u8 *enckey, *authkey; 553 int rc; 554 555 dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n", 556 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen); 557 558 /* STAT_PHASE_0: Init and sanity checks */ 559 560 if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */ 561 struct crypto_authenc_keys keys; 562 563 rc = crypto_authenc_extractkeys(&keys, key, keylen); 564 if (rc) 565 goto badkey; 566 enckey = keys.enckey; 567 authkey = keys.authkey; 568 ctx->enc_keylen = keys.enckeylen; 569 ctx->auth_keylen = keys.authkeylen; 570 571 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 572 /* the nonce is stored in bytes at end of key */ 573 rc = -EINVAL; 574 if (ctx->enc_keylen < 575 (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE)) 576 goto badkey; 577 /* Copy nonce from last 4 bytes in CTR key to 578 * first 4 bytes in CTR IV 579 */ 580 memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen - 581 CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); 582 /* Set CTR key size */ 583 ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE; 584 } 585 } else { /* non-authenc - has just one key */ 586 enckey = key; 587 authkey = NULL; 588 ctx->enc_keylen = keylen; 589 ctx->auth_keylen = 0; 590 } 591 592 rc = validate_keys_sizes(ctx); 593 if (rc) 594 goto badkey; 595 596 /* STAT_PHASE_1: Copy key to ctx */ 597 598 /* Get key material */ 599 memcpy(ctx->enckey, enckey, ctx->enc_keylen); 600 if (ctx->enc_keylen == 24) 601 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24); 602 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 603 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey, 604 ctx->auth_keylen); 605 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */ 606 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen); 607 if (rc) 608 goto badkey; 609 } 610 611 /* STAT_PHASE_2: Create sequence */ 612 613 switch (ctx->auth_mode) { 614 case DRV_HASH_SHA1: 615 case DRV_HASH_SHA256: 616 seq_len = hmac_setkey(desc, ctx); 617 break; 618 case DRV_HASH_XCBC_MAC: 619 seq_len = xcbc_setkey(desc, ctx); 620 break; 621 case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */ 622 break; /* No auth. key setup */ 623 default: 624 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 625 rc = -ENOTSUPP; 626 goto badkey; 627 } 628 629 /* STAT_PHASE_3: Submit sequence to HW */ 630 631 if (seq_len > 0) { /* For CCM there is no sequence to setup the key */ 632 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len); 633 if (rc) { 634 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 635 goto setkey_error; 636 } 637 } 638 639 /* Update STAT_PHASE_3 */ 640 return rc; 641 642 badkey: 643 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 644 645 setkey_error: 646 return rc; 647 } 648 649 static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 650 unsigned int keylen) 651 { 652 struct crypto_authenc_keys keys; 653 int err; 654 655 err = crypto_authenc_extractkeys(&keys, key, keylen); 656 if (unlikely(err)) 657 return err; 658 659 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 660 cc_aead_setkey(aead, key, keylen); 661 662 memzero_explicit(&keys, sizeof(keys)); 663 return err; 664 } 665 666 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key, 667 unsigned int keylen) 668 { 669 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 670 671 if (keylen < 3) 672 return -EINVAL; 673 674 keylen -= 3; 675 memcpy(ctx->ctr_nonce, key + keylen, 3); 676 677 return cc_aead_setkey(tfm, key, keylen); 678 } 679 680 static int cc_aead_setauthsize(struct crypto_aead *authenc, 681 unsigned int authsize) 682 { 683 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 684 struct device *dev = drvdata_to_dev(ctx->drvdata); 685 686 /* Unsupported auth. sizes */ 687 if (authsize == 0 || 688 authsize > crypto_aead_maxauthsize(authenc)) { 689 return -ENOTSUPP; 690 } 691 692 ctx->authsize = authsize; 693 dev_dbg(dev, "authlen=%d\n", ctx->authsize); 694 695 return 0; 696 } 697 698 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc, 699 unsigned int authsize) 700 { 701 switch (authsize) { 702 case 8: 703 case 12: 704 case 16: 705 break; 706 default: 707 return -EINVAL; 708 } 709 710 return cc_aead_setauthsize(authenc, authsize); 711 } 712 713 static int cc_ccm_setauthsize(struct crypto_aead *authenc, 714 unsigned int authsize) 715 { 716 switch (authsize) { 717 case 4: 718 case 6: 719 case 8: 720 case 10: 721 case 12: 722 case 14: 723 case 16: 724 break; 725 default: 726 return -EINVAL; 727 } 728 729 return cc_aead_setauthsize(authenc, authsize); 730 } 731 732 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, 733 struct cc_hw_desc desc[], unsigned int *seq_size) 734 { 735 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 736 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 737 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 738 enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type; 739 unsigned int idx = *seq_size; 740 struct device *dev = drvdata_to_dev(ctx->drvdata); 741 742 switch (assoc_dma_type) { 743 case CC_DMA_BUF_DLLI: 744 dev_dbg(dev, "ASSOC buffer type DLLI\n"); 745 hw_desc_init(&desc[idx]); 746 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src), 747 areq_ctx->assoclen, NS_BIT); 748 set_flow_mode(&desc[idx], flow_mode); 749 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 750 areq_ctx->cryptlen > 0) 751 set_din_not_last_indication(&desc[idx]); 752 break; 753 case CC_DMA_BUF_MLLI: 754 dev_dbg(dev, "ASSOC buffer type MLLI\n"); 755 hw_desc_init(&desc[idx]); 756 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr, 757 areq_ctx->assoc.mlli_nents, NS_BIT); 758 set_flow_mode(&desc[idx], flow_mode); 759 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 760 areq_ctx->cryptlen > 0) 761 set_din_not_last_indication(&desc[idx]); 762 break; 763 case CC_DMA_BUF_NULL: 764 default: 765 dev_err(dev, "Invalid ASSOC buffer type\n"); 766 } 767 768 *seq_size = (++idx); 769 } 770 771 static void cc_proc_authen_desc(struct aead_request *areq, 772 unsigned int flow_mode, 773 struct cc_hw_desc desc[], 774 unsigned int *seq_size, int direct) 775 { 776 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 777 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 778 unsigned int idx = *seq_size; 779 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 780 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 781 struct device *dev = drvdata_to_dev(ctx->drvdata); 782 783 switch (data_dma_type) { 784 case CC_DMA_BUF_DLLI: 785 { 786 struct scatterlist *cipher = 787 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 788 areq_ctx->dst_sgl : areq_ctx->src_sgl; 789 790 unsigned int offset = 791 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 792 areq_ctx->dst_offset : areq_ctx->src_offset; 793 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n"); 794 hw_desc_init(&desc[idx]); 795 set_din_type(&desc[idx], DMA_DLLI, 796 (sg_dma_address(cipher) + offset), 797 areq_ctx->cryptlen, NS_BIT); 798 set_flow_mode(&desc[idx], flow_mode); 799 break; 800 } 801 case CC_DMA_BUF_MLLI: 802 { 803 /* DOUBLE-PASS flow (as default) 804 * assoc. + iv + data -compact in one table 805 * if assoclen is ZERO only IV perform 806 */ 807 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr; 808 u32 mlli_nents = areq_ctx->assoc.mlli_nents; 809 810 if (areq_ctx->is_single_pass) { 811 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 812 mlli_addr = areq_ctx->dst.sram_addr; 813 mlli_nents = areq_ctx->dst.mlli_nents; 814 } else { 815 mlli_addr = areq_ctx->src.sram_addr; 816 mlli_nents = areq_ctx->src.mlli_nents; 817 } 818 } 819 820 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n"); 821 hw_desc_init(&desc[idx]); 822 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents, 823 NS_BIT); 824 set_flow_mode(&desc[idx], flow_mode); 825 break; 826 } 827 case CC_DMA_BUF_NULL: 828 default: 829 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n"); 830 } 831 832 *seq_size = (++idx); 833 } 834 835 static void cc_proc_cipher_desc(struct aead_request *areq, 836 unsigned int flow_mode, 837 struct cc_hw_desc desc[], 838 unsigned int *seq_size) 839 { 840 unsigned int idx = *seq_size; 841 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq); 842 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 843 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 844 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 845 struct device *dev = drvdata_to_dev(ctx->drvdata); 846 847 if (areq_ctx->cryptlen == 0) 848 return; /*null processing*/ 849 850 switch (data_dma_type) { 851 case CC_DMA_BUF_DLLI: 852 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n"); 853 hw_desc_init(&desc[idx]); 854 set_din_type(&desc[idx], DMA_DLLI, 855 (sg_dma_address(areq_ctx->src_sgl) + 856 areq_ctx->src_offset), areq_ctx->cryptlen, 857 NS_BIT); 858 set_dout_dlli(&desc[idx], 859 (sg_dma_address(areq_ctx->dst_sgl) + 860 areq_ctx->dst_offset), 861 areq_ctx->cryptlen, NS_BIT, 0); 862 set_flow_mode(&desc[idx], flow_mode); 863 break; 864 case CC_DMA_BUF_MLLI: 865 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n"); 866 hw_desc_init(&desc[idx]); 867 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr, 868 areq_ctx->src.mlli_nents, NS_BIT); 869 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr, 870 areq_ctx->dst.mlli_nents, NS_BIT, 0); 871 set_flow_mode(&desc[idx], flow_mode); 872 break; 873 case CC_DMA_BUF_NULL: 874 default: 875 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n"); 876 } 877 878 *seq_size = (++idx); 879 } 880 881 static void cc_proc_digest_desc(struct aead_request *req, 882 struct cc_hw_desc desc[], 883 unsigned int *seq_size) 884 { 885 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 886 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 887 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 888 unsigned int idx = *seq_size; 889 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 890 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 891 int direct = req_ctx->gen_ctx.op_type; 892 893 /* Get final ICV result */ 894 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 895 hw_desc_init(&desc[idx]); 896 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 897 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 898 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize, 899 NS_BIT, 1); 900 set_queue_last_ind(ctx->drvdata, &desc[idx]); 901 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 902 set_aes_not_hash_mode(&desc[idx]); 903 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 904 } else { 905 set_cipher_config0(&desc[idx], 906 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 907 set_cipher_mode(&desc[idx], hash_mode); 908 } 909 } else { /*Decrypt*/ 910 /* Get ICV out from hardware */ 911 hw_desc_init(&desc[idx]); 912 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 913 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 914 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, 915 ctx->authsize, NS_BIT, 1); 916 set_queue_last_ind(ctx->drvdata, &desc[idx]); 917 set_cipher_config0(&desc[idx], 918 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 919 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 920 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 921 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 922 set_aes_not_hash_mode(&desc[idx]); 923 } else { 924 set_cipher_mode(&desc[idx], hash_mode); 925 } 926 } 927 928 *seq_size = (++idx); 929 } 930 931 static void cc_set_cipher_desc(struct aead_request *req, 932 struct cc_hw_desc desc[], 933 unsigned int *seq_size) 934 { 935 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 936 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 937 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 938 unsigned int hw_iv_size = req_ctx->hw_iv_size; 939 unsigned int idx = *seq_size; 940 int direct = req_ctx->gen_ctx.op_type; 941 942 /* Setup cipher state */ 943 hw_desc_init(&desc[idx]); 944 set_cipher_config0(&desc[idx], direct); 945 set_flow_mode(&desc[idx], ctx->flow_mode); 946 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr, 947 hw_iv_size, NS_BIT); 948 if (ctx->cipher_mode == DRV_CIPHER_CTR) 949 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 950 else 951 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 952 set_cipher_mode(&desc[idx], ctx->cipher_mode); 953 idx++; 954 955 /* Setup enc. key */ 956 hw_desc_init(&desc[idx]); 957 set_cipher_config0(&desc[idx], direct); 958 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 959 set_flow_mode(&desc[idx], ctx->flow_mode); 960 if (ctx->flow_mode == S_DIN_to_AES) { 961 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 962 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 963 ctx->enc_keylen), NS_BIT); 964 set_key_size_aes(&desc[idx], ctx->enc_keylen); 965 } else { 966 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 967 ctx->enc_keylen, NS_BIT); 968 set_key_size_des(&desc[idx], ctx->enc_keylen); 969 } 970 set_cipher_mode(&desc[idx], ctx->cipher_mode); 971 idx++; 972 973 *seq_size = idx; 974 } 975 976 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[], 977 unsigned int *seq_size, unsigned int data_flow_mode) 978 { 979 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 980 int direct = req_ctx->gen_ctx.op_type; 981 unsigned int idx = *seq_size; 982 983 if (req_ctx->cryptlen == 0) 984 return; /*null processing*/ 985 986 cc_set_cipher_desc(req, desc, &idx); 987 cc_proc_cipher_desc(req, data_flow_mode, desc, &idx); 988 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 989 /* We must wait for DMA to write all cipher */ 990 hw_desc_init(&desc[idx]); 991 set_din_no_dma(&desc[idx], 0, 0xfffff0); 992 set_dout_no_dma(&desc[idx], 0, 0, 1); 993 idx++; 994 } 995 996 *seq_size = idx; 997 } 998 999 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[], 1000 unsigned int *seq_size) 1001 { 1002 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1003 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1004 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 1005 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 1006 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1007 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1008 unsigned int idx = *seq_size; 1009 1010 /* Loading hash ipad xor key state */ 1011 hw_desc_init(&desc[idx]); 1012 set_cipher_mode(&desc[idx], hash_mode); 1013 set_din_type(&desc[idx], DMA_DLLI, 1014 ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size, 1015 NS_BIT); 1016 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1017 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1018 idx++; 1019 1020 /* Load init. digest len (64 bytes) */ 1021 hw_desc_init(&desc[idx]); 1022 set_cipher_mode(&desc[idx], hash_mode); 1023 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1024 ctx->hash_len); 1025 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1026 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1027 idx++; 1028 1029 *seq_size = idx; 1030 } 1031 1032 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[], 1033 unsigned int *seq_size) 1034 { 1035 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1036 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1037 unsigned int idx = *seq_size; 1038 1039 /* Loading MAC state */ 1040 hw_desc_init(&desc[idx]); 1041 set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE); 1042 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1043 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1044 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1045 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1046 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1047 set_aes_not_hash_mode(&desc[idx]); 1048 idx++; 1049 1050 /* Setup XCBC MAC K1 */ 1051 hw_desc_init(&desc[idx]); 1052 set_din_type(&desc[idx], DMA_DLLI, 1053 ctx->auth_state.xcbc.xcbc_keys_dma_addr, 1054 AES_KEYSIZE_128, NS_BIT); 1055 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1056 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1057 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1058 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1059 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1060 set_aes_not_hash_mode(&desc[idx]); 1061 idx++; 1062 1063 /* Setup XCBC MAC K2 */ 1064 hw_desc_init(&desc[idx]); 1065 set_din_type(&desc[idx], DMA_DLLI, 1066 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1067 AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1068 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1069 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1070 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1071 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1072 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1073 set_aes_not_hash_mode(&desc[idx]); 1074 idx++; 1075 1076 /* Setup XCBC MAC K3 */ 1077 hw_desc_init(&desc[idx]); 1078 set_din_type(&desc[idx], DMA_DLLI, 1079 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1080 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1081 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); 1082 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1083 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1084 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1085 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1086 set_aes_not_hash_mode(&desc[idx]); 1087 idx++; 1088 1089 *seq_size = idx; 1090 } 1091 1092 static void cc_proc_header_desc(struct aead_request *req, 1093 struct cc_hw_desc desc[], 1094 unsigned int *seq_size) 1095 { 1096 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1097 unsigned int idx = *seq_size; 1098 1099 /* Hash associated data */ 1100 if (areq_ctx->assoclen > 0) 1101 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1102 1103 /* Hash IV */ 1104 *seq_size = idx; 1105 } 1106 1107 static void cc_proc_scheme_desc(struct aead_request *req, 1108 struct cc_hw_desc desc[], 1109 unsigned int *seq_size) 1110 { 1111 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1112 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1113 struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle; 1114 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 1115 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 1116 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1117 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1118 unsigned int idx = *seq_size; 1119 1120 hw_desc_init(&desc[idx]); 1121 set_cipher_mode(&desc[idx], hash_mode); 1122 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1123 ctx->hash_len); 1124 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1125 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 1126 set_cipher_do(&desc[idx], DO_PAD); 1127 idx++; 1128 1129 /* Get final ICV result */ 1130 hw_desc_init(&desc[idx]); 1131 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1132 digest_size); 1133 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1134 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1135 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1136 set_cipher_mode(&desc[idx], hash_mode); 1137 idx++; 1138 1139 /* Loading hash opad xor key state */ 1140 hw_desc_init(&desc[idx]); 1141 set_cipher_mode(&desc[idx], hash_mode); 1142 set_din_type(&desc[idx], DMA_DLLI, 1143 (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size), 1144 digest_size, NS_BIT); 1145 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1146 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1147 idx++; 1148 1149 /* Load init. digest len (64 bytes) */ 1150 hw_desc_init(&desc[idx]); 1151 set_cipher_mode(&desc[idx], hash_mode); 1152 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1153 ctx->hash_len); 1154 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1155 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1156 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1157 idx++; 1158 1159 /* Perform HASH update */ 1160 hw_desc_init(&desc[idx]); 1161 set_din_sram(&desc[idx], aead_handle->sram_workspace_addr, 1162 digest_size); 1163 set_flow_mode(&desc[idx], DIN_HASH); 1164 idx++; 1165 1166 *seq_size = idx; 1167 } 1168 1169 static void cc_mlli_to_sram(struct aead_request *req, 1170 struct cc_hw_desc desc[], unsigned int *seq_size) 1171 { 1172 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1173 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1174 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1175 struct device *dev = drvdata_to_dev(ctx->drvdata); 1176 1177 if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || 1178 req_ctx->data_buff_type == CC_DMA_BUF_MLLI || 1179 !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) { 1180 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n", 1181 (unsigned int)ctx->drvdata->mlli_sram_addr, 1182 req_ctx->mlli_params.mlli_len); 1183 /* Copy MLLI table host-to-sram */ 1184 hw_desc_init(&desc[*seq_size]); 1185 set_din_type(&desc[*seq_size], DMA_DLLI, 1186 req_ctx->mlli_params.mlli_dma_addr, 1187 req_ctx->mlli_params.mlli_len, NS_BIT); 1188 set_dout_sram(&desc[*seq_size], 1189 ctx->drvdata->mlli_sram_addr, 1190 req_ctx->mlli_params.mlli_len); 1191 set_flow_mode(&desc[*seq_size], BYPASS); 1192 (*seq_size)++; 1193 } 1194 } 1195 1196 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct, 1197 enum cc_flow_mode setup_flow_mode, 1198 bool is_single_pass) 1199 { 1200 enum cc_flow_mode data_flow_mode; 1201 1202 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1203 if (setup_flow_mode == S_DIN_to_AES) 1204 data_flow_mode = is_single_pass ? 1205 AES_to_HASH_and_DOUT : DIN_AES_DOUT; 1206 else 1207 data_flow_mode = is_single_pass ? 1208 DES_to_HASH_and_DOUT : DIN_DES_DOUT; 1209 } else { /* Decrypt */ 1210 if (setup_flow_mode == S_DIN_to_AES) 1211 data_flow_mode = is_single_pass ? 1212 AES_and_HASH : DIN_AES_DOUT; 1213 else 1214 data_flow_mode = is_single_pass ? 1215 DES_and_HASH : DIN_DES_DOUT; 1216 } 1217 1218 return data_flow_mode; 1219 } 1220 1221 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1222 unsigned int *seq_size) 1223 { 1224 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1225 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1226 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1227 int direct = req_ctx->gen_ctx.op_type; 1228 unsigned int data_flow_mode = 1229 cc_get_data_flow(direct, ctx->flow_mode, 1230 req_ctx->is_single_pass); 1231 1232 if (req_ctx->is_single_pass) { 1233 /** 1234 * Single-pass flow 1235 */ 1236 cc_set_hmac_desc(req, desc, seq_size); 1237 cc_set_cipher_desc(req, desc, seq_size); 1238 cc_proc_header_desc(req, desc, seq_size); 1239 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1240 cc_proc_scheme_desc(req, desc, seq_size); 1241 cc_proc_digest_desc(req, desc, seq_size); 1242 return; 1243 } 1244 1245 /** 1246 * Double-pass flow 1247 * Fallback for unsupported single-pass modes, 1248 * i.e. using assoc. data of non-word-multiple 1249 */ 1250 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1251 /* encrypt first.. */ 1252 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1253 /* authenc after..*/ 1254 cc_set_hmac_desc(req, desc, seq_size); 1255 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1256 cc_proc_scheme_desc(req, desc, seq_size); 1257 cc_proc_digest_desc(req, desc, seq_size); 1258 1259 } else { /*DECRYPT*/ 1260 /* authenc first..*/ 1261 cc_set_hmac_desc(req, desc, seq_size); 1262 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1263 cc_proc_scheme_desc(req, desc, seq_size); 1264 /* decrypt after.. */ 1265 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1266 /* read the digest result with setting the completion bit 1267 * must be after the cipher operation 1268 */ 1269 cc_proc_digest_desc(req, desc, seq_size); 1270 } 1271 } 1272 1273 static void 1274 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1275 unsigned int *seq_size) 1276 { 1277 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1278 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1279 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1280 int direct = req_ctx->gen_ctx.op_type; 1281 unsigned int data_flow_mode = 1282 cc_get_data_flow(direct, ctx->flow_mode, 1283 req_ctx->is_single_pass); 1284 1285 if (req_ctx->is_single_pass) { 1286 /** 1287 * Single-pass flow 1288 */ 1289 cc_set_xcbc_desc(req, desc, seq_size); 1290 cc_set_cipher_desc(req, desc, seq_size); 1291 cc_proc_header_desc(req, desc, seq_size); 1292 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1293 cc_proc_digest_desc(req, desc, seq_size); 1294 return; 1295 } 1296 1297 /** 1298 * Double-pass flow 1299 * Fallback for unsupported single-pass modes, 1300 * i.e. using assoc. data of non-word-multiple 1301 */ 1302 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1303 /* encrypt first.. */ 1304 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1305 /* authenc after.. */ 1306 cc_set_xcbc_desc(req, desc, seq_size); 1307 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1308 cc_proc_digest_desc(req, desc, seq_size); 1309 } else { /*DECRYPT*/ 1310 /* authenc first.. */ 1311 cc_set_xcbc_desc(req, desc, seq_size); 1312 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1313 /* decrypt after..*/ 1314 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1315 /* read the digest result with setting the completion bit 1316 * must be after the cipher operation 1317 */ 1318 cc_proc_digest_desc(req, desc, seq_size); 1319 } 1320 } 1321 1322 static int validate_data_size(struct cc_aead_ctx *ctx, 1323 enum drv_crypto_direction direct, 1324 struct aead_request *req) 1325 { 1326 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1327 struct device *dev = drvdata_to_dev(ctx->drvdata); 1328 unsigned int assoclen = areq_ctx->assoclen; 1329 unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ? 1330 (req->cryptlen - ctx->authsize) : req->cryptlen; 1331 1332 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT && 1333 req->cryptlen < ctx->authsize) 1334 goto data_size_err; 1335 1336 areq_ctx->is_single_pass = true; /*defaulted to fast flow*/ 1337 1338 switch (ctx->flow_mode) { 1339 case S_DIN_to_AES: 1340 if (ctx->cipher_mode == DRV_CIPHER_CBC && 1341 !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)) 1342 goto data_size_err; 1343 if (ctx->cipher_mode == DRV_CIPHER_CCM) 1344 break; 1345 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1346 if (areq_ctx->plaintext_authenticate_only) 1347 areq_ctx->is_single_pass = false; 1348 break; 1349 } 1350 1351 if (!IS_ALIGNED(assoclen, sizeof(u32))) 1352 areq_ctx->is_single_pass = false; 1353 1354 if (ctx->cipher_mode == DRV_CIPHER_CTR && 1355 !IS_ALIGNED(cipherlen, sizeof(u32))) 1356 areq_ctx->is_single_pass = false; 1357 1358 break; 1359 case S_DIN_to_DES: 1360 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE)) 1361 goto data_size_err; 1362 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE)) 1363 areq_ctx->is_single_pass = false; 1364 break; 1365 default: 1366 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode); 1367 goto data_size_err; 1368 } 1369 1370 return 0; 1371 1372 data_size_err: 1373 return -EINVAL; 1374 } 1375 1376 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size) 1377 { 1378 unsigned int len = 0; 1379 1380 if (header_size == 0) 1381 return 0; 1382 1383 if (header_size < ((1UL << 16) - (1UL << 8))) { 1384 len = 2; 1385 1386 pa0_buff[0] = (header_size >> 8) & 0xFF; 1387 pa0_buff[1] = header_size & 0xFF; 1388 } else { 1389 len = 6; 1390 1391 pa0_buff[0] = 0xFF; 1392 pa0_buff[1] = 0xFE; 1393 pa0_buff[2] = (header_size >> 24) & 0xFF; 1394 pa0_buff[3] = (header_size >> 16) & 0xFF; 1395 pa0_buff[4] = (header_size >> 8) & 0xFF; 1396 pa0_buff[5] = header_size & 0xFF; 1397 } 1398 1399 return len; 1400 } 1401 1402 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize) 1403 { 1404 __be32 data; 1405 1406 memset(block, 0, csize); 1407 block += csize; 1408 1409 if (csize >= 4) 1410 csize = 4; 1411 else if (msglen > (1 << (8 * csize))) 1412 return -EOVERFLOW; 1413 1414 data = cpu_to_be32(msglen); 1415 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 1416 1417 return 0; 1418 } 1419 1420 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], 1421 unsigned int *seq_size) 1422 { 1423 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1424 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1425 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1426 unsigned int idx = *seq_size; 1427 unsigned int cipher_flow_mode; 1428 dma_addr_t mac_result; 1429 1430 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1431 cipher_flow_mode = AES_to_HASH_and_DOUT; 1432 mac_result = req_ctx->mac_buf_dma_addr; 1433 } else { /* Encrypt */ 1434 cipher_flow_mode = AES_and_HASH; 1435 mac_result = req_ctx->icv_dma_addr; 1436 } 1437 1438 /* load key */ 1439 hw_desc_init(&desc[idx]); 1440 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1441 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1442 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1443 ctx->enc_keylen), NS_BIT); 1444 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1445 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1446 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1447 set_flow_mode(&desc[idx], S_DIN_to_AES); 1448 idx++; 1449 1450 /* load ctr state */ 1451 hw_desc_init(&desc[idx]); 1452 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1453 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1454 set_din_type(&desc[idx], DMA_DLLI, 1455 req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT); 1456 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1457 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1458 set_flow_mode(&desc[idx], S_DIN_to_AES); 1459 idx++; 1460 1461 /* load MAC key */ 1462 hw_desc_init(&desc[idx]); 1463 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1464 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1465 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1466 ctx->enc_keylen), NS_BIT); 1467 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1468 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1469 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1470 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1471 set_aes_not_hash_mode(&desc[idx]); 1472 idx++; 1473 1474 /* load MAC state */ 1475 hw_desc_init(&desc[idx]); 1476 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1477 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1478 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1479 AES_BLOCK_SIZE, NS_BIT); 1480 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1481 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1482 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1483 set_aes_not_hash_mode(&desc[idx]); 1484 idx++; 1485 1486 /* process assoc data */ 1487 if (req_ctx->assoclen > 0) { 1488 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1489 } else { 1490 hw_desc_init(&desc[idx]); 1491 set_din_type(&desc[idx], DMA_DLLI, 1492 sg_dma_address(&req_ctx->ccm_adata_sg), 1493 AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT); 1494 set_flow_mode(&desc[idx], DIN_HASH); 1495 idx++; 1496 } 1497 1498 /* process the cipher */ 1499 if (req_ctx->cryptlen) 1500 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx); 1501 1502 /* Read temporal MAC */ 1503 hw_desc_init(&desc[idx]); 1504 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1505 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize, 1506 NS_BIT, 0); 1507 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1508 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1509 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1510 set_aes_not_hash_mode(&desc[idx]); 1511 idx++; 1512 1513 /* load AES-CTR state (for last MAC calculation)*/ 1514 hw_desc_init(&desc[idx]); 1515 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1516 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1517 set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr, 1518 AES_BLOCK_SIZE, NS_BIT); 1519 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1520 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1521 set_flow_mode(&desc[idx], S_DIN_to_AES); 1522 idx++; 1523 1524 hw_desc_init(&desc[idx]); 1525 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1526 set_dout_no_dma(&desc[idx], 0, 0, 1); 1527 idx++; 1528 1529 /* encrypt the "T" value and store MAC in mac_state */ 1530 hw_desc_init(&desc[idx]); 1531 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1532 ctx->authsize, NS_BIT); 1533 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1534 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1535 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1536 idx++; 1537 1538 *seq_size = idx; 1539 return 0; 1540 } 1541 1542 static int config_ccm_adata(struct aead_request *req) 1543 { 1544 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1545 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1546 struct device *dev = drvdata_to_dev(ctx->drvdata); 1547 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1548 //unsigned int size_of_a = 0, rem_a_size = 0; 1549 unsigned int lp = req->iv[0]; 1550 /* Note: The code assume that req->iv[0] already contains the value 1551 * of L' of RFC3610 1552 */ 1553 unsigned int l = lp + 1; /* This is L' of RFC 3610. */ 1554 unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */ 1555 u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET; 1556 u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET; 1557 u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; 1558 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1559 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1560 req->cryptlen : 1561 (req->cryptlen - ctx->authsize); 1562 int rc; 1563 1564 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1565 memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3); 1566 1567 /* taken from crypto/ccm.c */ 1568 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1569 if (l < 2 || l > 8) { 1570 dev_err(dev, "illegal iv value %X\n", req->iv[0]); 1571 return -EINVAL; 1572 } 1573 memcpy(b0, req->iv, AES_BLOCK_SIZE); 1574 1575 /* format control info per RFC 3610 and 1576 * NIST Special Publication 800-38C 1577 */ 1578 *b0 |= (8 * ((m - 2) / 2)); 1579 if (req_ctx->assoclen > 0) 1580 *b0 |= 64; /* Enable bit 6 if Adata exists. */ 1581 1582 rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */ 1583 if (rc) { 1584 dev_err(dev, "message len overflow detected"); 1585 return rc; 1586 } 1587 /* END of "taken from crypto/ccm.c" */ 1588 1589 /* l(a) - size of associated data. */ 1590 req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen); 1591 1592 memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1); 1593 req->iv[15] = 1; 1594 1595 memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE); 1596 ctr_count_0[15] = 0; 1597 1598 return 0; 1599 } 1600 1601 static void cc_proc_rfc4309_ccm(struct aead_request *req) 1602 { 1603 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1604 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1605 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1606 1607 /* L' */ 1608 memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE); 1609 /* For RFC 4309, always use 4 bytes for message length 1610 * (at most 2^32-1 bytes). 1611 */ 1612 areq_ctx->ctr_iv[0] = 3; 1613 1614 /* In RFC 4309 there is an 11-bytes nonce+IV part, 1615 * that we build here. 1616 */ 1617 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce, 1618 CCM_BLOCK_NONCE_SIZE); 1619 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv, 1620 CCM_BLOCK_IV_SIZE); 1621 req->iv = areq_ctx->ctr_iv; 1622 areq_ctx->assoclen -= CCM_BLOCK_IV_SIZE; 1623 } 1624 1625 static void cc_set_ghash_desc(struct aead_request *req, 1626 struct cc_hw_desc desc[], unsigned int *seq_size) 1627 { 1628 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1629 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1630 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1631 unsigned int idx = *seq_size; 1632 1633 /* load key to AES*/ 1634 hw_desc_init(&desc[idx]); 1635 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 1636 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1637 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1638 ctx->enc_keylen, NS_BIT); 1639 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1640 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1641 set_flow_mode(&desc[idx], S_DIN_to_AES); 1642 idx++; 1643 1644 /* process one zero block to generate hkey */ 1645 hw_desc_init(&desc[idx]); 1646 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1647 set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE, 1648 NS_BIT, 0); 1649 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1650 idx++; 1651 1652 /* Memory Barrier */ 1653 hw_desc_init(&desc[idx]); 1654 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1655 set_dout_no_dma(&desc[idx], 0, 0, 1); 1656 idx++; 1657 1658 /* Load GHASH subkey */ 1659 hw_desc_init(&desc[idx]); 1660 set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr, 1661 AES_BLOCK_SIZE, NS_BIT); 1662 set_dout_no_dma(&desc[idx], 0, 0, 1); 1663 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1664 set_aes_not_hash_mode(&desc[idx]); 1665 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1666 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1667 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1668 idx++; 1669 1670 /* Configure Hash Engine to work with GHASH. 1671 * Since it was not possible to extend HASH submodes to add GHASH, 1672 * The following command is necessary in order to 1673 * select GHASH (according to HW designers) 1674 */ 1675 hw_desc_init(&desc[idx]); 1676 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1677 set_dout_no_dma(&desc[idx], 0, 0, 1); 1678 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1679 set_aes_not_hash_mode(&desc[idx]); 1680 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1681 set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK 1682 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1683 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1684 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1685 idx++; 1686 1687 /* Load GHASH initial STATE (which is 0). (for any hash there is an 1688 * initial state) 1689 */ 1690 hw_desc_init(&desc[idx]); 1691 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1692 set_dout_no_dma(&desc[idx], 0, 0, 1); 1693 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1694 set_aes_not_hash_mode(&desc[idx]); 1695 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1696 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1697 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1698 idx++; 1699 1700 *seq_size = idx; 1701 } 1702 1703 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[], 1704 unsigned int *seq_size) 1705 { 1706 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1707 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1708 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1709 unsigned int idx = *seq_size; 1710 1711 /* load key to AES*/ 1712 hw_desc_init(&desc[idx]); 1713 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1714 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1715 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1716 ctx->enc_keylen, NS_BIT); 1717 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1718 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1719 set_flow_mode(&desc[idx], S_DIN_to_AES); 1720 idx++; 1721 1722 if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) { 1723 /* load AES/CTR initial CTR value inc by 2*/ 1724 hw_desc_init(&desc[idx]); 1725 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1726 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1727 set_din_type(&desc[idx], DMA_DLLI, 1728 req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE, 1729 NS_BIT); 1730 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1731 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1732 set_flow_mode(&desc[idx], S_DIN_to_AES); 1733 idx++; 1734 } 1735 1736 *seq_size = idx; 1737 } 1738 1739 static void cc_proc_gcm_result(struct aead_request *req, 1740 struct cc_hw_desc desc[], 1741 unsigned int *seq_size) 1742 { 1743 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1744 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1745 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1746 dma_addr_t mac_result; 1747 unsigned int idx = *seq_size; 1748 1749 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1750 mac_result = req_ctx->mac_buf_dma_addr; 1751 } else { /* Encrypt */ 1752 mac_result = req_ctx->icv_dma_addr; 1753 } 1754 1755 /* process(ghash) gcm_block_len */ 1756 hw_desc_init(&desc[idx]); 1757 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr, 1758 AES_BLOCK_SIZE, NS_BIT); 1759 set_flow_mode(&desc[idx], DIN_HASH); 1760 idx++; 1761 1762 /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */ 1763 hw_desc_init(&desc[idx]); 1764 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1765 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1766 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE, 1767 NS_BIT, 0); 1768 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1769 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1770 set_aes_not_hash_mode(&desc[idx]); 1771 1772 idx++; 1773 1774 /* load AES/CTR initial CTR value inc by 1*/ 1775 hw_desc_init(&desc[idx]); 1776 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1777 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1778 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr, 1779 AES_BLOCK_SIZE, NS_BIT); 1780 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1781 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1782 set_flow_mode(&desc[idx], S_DIN_to_AES); 1783 idx++; 1784 1785 /* Memory Barrier */ 1786 hw_desc_init(&desc[idx]); 1787 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1788 set_dout_no_dma(&desc[idx], 0, 0, 1); 1789 idx++; 1790 1791 /* process GCTR on stored GHASH and store MAC in mac_state*/ 1792 hw_desc_init(&desc[idx]); 1793 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1794 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1795 AES_BLOCK_SIZE, NS_BIT); 1796 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1797 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1798 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1799 idx++; 1800 1801 *seq_size = idx; 1802 } 1803 1804 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[], 1805 unsigned int *seq_size) 1806 { 1807 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1808 unsigned int cipher_flow_mode; 1809 1810 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1811 cipher_flow_mode = AES_and_HASH; 1812 } else { /* Encrypt */ 1813 cipher_flow_mode = AES_to_HASH_and_DOUT; 1814 } 1815 1816 //in RFC4543 no data to encrypt. just copy data from src to dest. 1817 if (req_ctx->plaintext_authenticate_only) { 1818 cc_proc_cipher_desc(req, BYPASS, desc, seq_size); 1819 cc_set_ghash_desc(req, desc, seq_size); 1820 /* process(ghash) assoc data */ 1821 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1822 cc_set_gctr_desc(req, desc, seq_size); 1823 cc_proc_gcm_result(req, desc, seq_size); 1824 return 0; 1825 } 1826 1827 // for gcm and rfc4106. 1828 cc_set_ghash_desc(req, desc, seq_size); 1829 /* process(ghash) assoc data */ 1830 if (req_ctx->assoclen > 0) 1831 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1832 cc_set_gctr_desc(req, desc, seq_size); 1833 /* process(gctr+ghash) */ 1834 if (req_ctx->cryptlen) 1835 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size); 1836 cc_proc_gcm_result(req, desc, seq_size); 1837 1838 return 0; 1839 } 1840 1841 static int config_gcm_context(struct aead_request *req) 1842 { 1843 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1844 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1845 struct aead_req_ctx *req_ctx = aead_request_ctx(req); 1846 struct device *dev = drvdata_to_dev(ctx->drvdata); 1847 1848 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1849 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1850 req->cryptlen : 1851 (req->cryptlen - ctx->authsize); 1852 __be32 counter = cpu_to_be32(2); 1853 1854 dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n", 1855 __func__, cryptlen, req_ctx->assoclen, ctx->authsize); 1856 1857 memset(req_ctx->hkey, 0, AES_BLOCK_SIZE); 1858 1859 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1860 1861 memcpy(req->iv + 12, &counter, 4); 1862 memcpy(req_ctx->gcm_iv_inc2, req->iv, 16); 1863 1864 counter = cpu_to_be32(1); 1865 memcpy(req->iv + 12, &counter, 4); 1866 memcpy(req_ctx->gcm_iv_inc1, req->iv, 16); 1867 1868 if (!req_ctx->plaintext_authenticate_only) { 1869 __be64 temp64; 1870 1871 temp64 = cpu_to_be64(req_ctx->assoclen * 8); 1872 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1873 temp64 = cpu_to_be64(cryptlen * 8); 1874 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1875 } else { 1876 /* rfc4543=> all data(AAD,IV,Plain) are considered additional 1877 * data that is nothing is encrypted. 1878 */ 1879 __be64 temp64; 1880 1881 temp64 = cpu_to_be64((req_ctx->assoclen + 1882 GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8); 1883 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1884 temp64 = 0; 1885 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1886 } 1887 1888 return 0; 1889 } 1890 1891 static void cc_proc_rfc4_gcm(struct aead_request *req) 1892 { 1893 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1894 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1895 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1896 1897 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET, 1898 ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE); 1899 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv, 1900 GCM_BLOCK_RFC4_IV_SIZE); 1901 req->iv = areq_ctx->ctr_iv; 1902 areq_ctx->assoclen -= GCM_BLOCK_RFC4_IV_SIZE; 1903 } 1904 1905 static int cc_proc_aead(struct aead_request *req, 1906 enum drv_crypto_direction direct) 1907 { 1908 int rc = 0; 1909 int seq_len = 0; 1910 struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ]; 1911 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1912 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1913 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 1914 struct device *dev = drvdata_to_dev(ctx->drvdata); 1915 struct cc_crypto_req cc_req = {}; 1916 1917 dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n", 1918 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"), 1919 ctx, req, req->iv, sg_virt(req->src), req->src->offset, 1920 sg_virt(req->dst), req->dst->offset, req->cryptlen); 1921 1922 /* STAT_PHASE_0: Init and sanity checks */ 1923 1924 /* Check data length according to mode */ 1925 if (validate_data_size(ctx, direct, req)) { 1926 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n", 1927 req->cryptlen, areq_ctx->assoclen); 1928 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN); 1929 return -EINVAL; 1930 } 1931 1932 /* Setup request structure */ 1933 cc_req.user_cb = (void *)cc_aead_complete; 1934 cc_req.user_arg = (void *)req; 1935 1936 /* Setup request context */ 1937 areq_ctx->gen_ctx.op_type = direct; 1938 areq_ctx->req_authsize = ctx->authsize; 1939 areq_ctx->cipher_mode = ctx->cipher_mode; 1940 1941 /* STAT_PHASE_1: Map buffers */ 1942 1943 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 1944 /* Build CTR IV - Copy nonce from last 4 bytes in 1945 * CTR key to first 4 bytes in CTR IV 1946 */ 1947 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce, 1948 CTR_RFC3686_NONCE_SIZE); 1949 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv, 1950 CTR_RFC3686_IV_SIZE); 1951 /* Initialize counter portion of counter block */ 1952 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE + 1953 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1954 1955 /* Replace with counter iv */ 1956 req->iv = areq_ctx->ctr_iv; 1957 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE; 1958 } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) || 1959 (ctx->cipher_mode == DRV_CIPHER_GCTR)) { 1960 areq_ctx->hw_iv_size = AES_BLOCK_SIZE; 1961 if (areq_ctx->ctr_iv != req->iv) { 1962 memcpy(areq_ctx->ctr_iv, req->iv, 1963 crypto_aead_ivsize(tfm)); 1964 req->iv = areq_ctx->ctr_iv; 1965 } 1966 } else { 1967 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm); 1968 } 1969 1970 if (ctx->cipher_mode == DRV_CIPHER_CCM) { 1971 rc = config_ccm_adata(req); 1972 if (rc) { 1973 dev_dbg(dev, "config_ccm_adata() returned with a failure %d!", 1974 rc); 1975 goto exit; 1976 } 1977 } else { 1978 areq_ctx->ccm_hdr_size = ccm_header_size_null; 1979 } 1980 1981 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1982 rc = config_gcm_context(req); 1983 if (rc) { 1984 dev_dbg(dev, "config_gcm_context() returned with a failure %d!", 1985 rc); 1986 goto exit; 1987 } 1988 } 1989 1990 rc = cc_map_aead_request(ctx->drvdata, req); 1991 if (rc) { 1992 dev_err(dev, "map_request() failed\n"); 1993 goto exit; 1994 } 1995 1996 /* STAT_PHASE_2: Create sequence */ 1997 1998 /* Load MLLI tables to SRAM if necessary */ 1999 cc_mlli_to_sram(req, desc, &seq_len); 2000 2001 /*TODO: move seq len by reference */ 2002 switch (ctx->auth_mode) { 2003 case DRV_HASH_SHA1: 2004 case DRV_HASH_SHA256: 2005 cc_hmac_authenc(req, desc, &seq_len); 2006 break; 2007 case DRV_HASH_XCBC_MAC: 2008 cc_xcbc_authenc(req, desc, &seq_len); 2009 break; 2010 case DRV_HASH_NULL: 2011 if (ctx->cipher_mode == DRV_CIPHER_CCM) 2012 cc_ccm(req, desc, &seq_len); 2013 if (ctx->cipher_mode == DRV_CIPHER_GCTR) 2014 cc_gcm(req, desc, &seq_len); 2015 break; 2016 default: 2017 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 2018 cc_unmap_aead_request(dev, req); 2019 rc = -ENOTSUPP; 2020 goto exit; 2021 } 2022 2023 /* STAT_PHASE_3: Lock HW and push sequence */ 2024 2025 rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base); 2026 2027 if (rc != -EINPROGRESS && rc != -EBUSY) { 2028 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 2029 cc_unmap_aead_request(dev, req); 2030 } 2031 2032 exit: 2033 return rc; 2034 } 2035 2036 static int cc_aead_encrypt(struct aead_request *req) 2037 { 2038 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2039 int rc; 2040 2041 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2042 2043 /* No generated IV required */ 2044 areq_ctx->backup_iv = req->iv; 2045 areq_ctx->assoclen = req->assoclen; 2046 areq_ctx->is_gcm4543 = false; 2047 2048 areq_ctx->plaintext_authenticate_only = false; 2049 2050 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2051 if (rc != -EINPROGRESS && rc != -EBUSY) 2052 req->iv = areq_ctx->backup_iv; 2053 2054 return rc; 2055 } 2056 2057 static int cc_rfc4309_ccm_encrypt(struct aead_request *req) 2058 { 2059 /* Very similar to cc_aead_encrypt() above. */ 2060 2061 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2062 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2063 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2064 struct device *dev = drvdata_to_dev(ctx->drvdata); 2065 int rc = -EINVAL; 2066 2067 if (!valid_assoclen(req)) { 2068 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2069 goto out; 2070 } 2071 2072 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2073 2074 /* No generated IV required */ 2075 areq_ctx->backup_iv = req->iv; 2076 areq_ctx->assoclen = req->assoclen; 2077 areq_ctx->is_gcm4543 = true; 2078 2079 cc_proc_rfc4309_ccm(req); 2080 2081 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2082 if (rc != -EINPROGRESS && rc != -EBUSY) 2083 req->iv = areq_ctx->backup_iv; 2084 out: 2085 return rc; 2086 } 2087 2088 static int cc_aead_decrypt(struct aead_request *req) 2089 { 2090 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2091 int rc; 2092 2093 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2094 2095 /* No generated IV required */ 2096 areq_ctx->backup_iv = req->iv; 2097 areq_ctx->assoclen = req->assoclen; 2098 areq_ctx->is_gcm4543 = false; 2099 2100 areq_ctx->plaintext_authenticate_only = false; 2101 2102 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2103 if (rc != -EINPROGRESS && rc != -EBUSY) 2104 req->iv = areq_ctx->backup_iv; 2105 2106 return rc; 2107 } 2108 2109 static int cc_rfc4309_ccm_decrypt(struct aead_request *req) 2110 { 2111 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2112 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2113 struct device *dev = drvdata_to_dev(ctx->drvdata); 2114 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2115 int rc = -EINVAL; 2116 2117 if (!valid_assoclen(req)) { 2118 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2119 goto out; 2120 } 2121 2122 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2123 2124 /* No generated IV required */ 2125 areq_ctx->backup_iv = req->iv; 2126 areq_ctx->assoclen = req->assoclen; 2127 2128 areq_ctx->is_gcm4543 = true; 2129 cc_proc_rfc4309_ccm(req); 2130 2131 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2132 if (rc != -EINPROGRESS && rc != -EBUSY) 2133 req->iv = areq_ctx->backup_iv; 2134 2135 out: 2136 return rc; 2137 } 2138 2139 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2140 unsigned int keylen) 2141 { 2142 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2143 struct device *dev = drvdata_to_dev(ctx->drvdata); 2144 2145 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2146 2147 if (keylen < 4) 2148 return -EINVAL; 2149 2150 keylen -= 4; 2151 memcpy(ctx->ctr_nonce, key + keylen, 4); 2152 2153 return cc_aead_setkey(tfm, key, keylen); 2154 } 2155 2156 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2157 unsigned int keylen) 2158 { 2159 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2160 struct device *dev = drvdata_to_dev(ctx->drvdata); 2161 2162 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2163 2164 if (keylen < 4) 2165 return -EINVAL; 2166 2167 keylen -= 4; 2168 memcpy(ctx->ctr_nonce, key + keylen, 4); 2169 2170 return cc_aead_setkey(tfm, key, keylen); 2171 } 2172 2173 static int cc_gcm_setauthsize(struct crypto_aead *authenc, 2174 unsigned int authsize) 2175 { 2176 switch (authsize) { 2177 case 4: 2178 case 8: 2179 case 12: 2180 case 13: 2181 case 14: 2182 case 15: 2183 case 16: 2184 break; 2185 default: 2186 return -EINVAL; 2187 } 2188 2189 return cc_aead_setauthsize(authenc, authsize); 2190 } 2191 2192 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc, 2193 unsigned int authsize) 2194 { 2195 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2196 struct device *dev = drvdata_to_dev(ctx->drvdata); 2197 2198 dev_dbg(dev, "authsize %d\n", authsize); 2199 2200 switch (authsize) { 2201 case 8: 2202 case 12: 2203 case 16: 2204 break; 2205 default: 2206 return -EINVAL; 2207 } 2208 2209 return cc_aead_setauthsize(authenc, authsize); 2210 } 2211 2212 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc, 2213 unsigned int authsize) 2214 { 2215 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2216 struct device *dev = drvdata_to_dev(ctx->drvdata); 2217 2218 dev_dbg(dev, "authsize %d\n", authsize); 2219 2220 if (authsize != 16) 2221 return -EINVAL; 2222 2223 return cc_aead_setauthsize(authenc, authsize); 2224 } 2225 2226 static int cc_rfc4106_gcm_encrypt(struct aead_request *req) 2227 { 2228 /* Very similar to cc_aead_encrypt() above. */ 2229 2230 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2231 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2232 struct device *dev = drvdata_to_dev(ctx->drvdata); 2233 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2234 int rc = -EINVAL; 2235 2236 if (!valid_assoclen(req)) { 2237 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2238 goto out; 2239 } 2240 2241 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2242 2243 /* No generated IV required */ 2244 areq_ctx->backup_iv = req->iv; 2245 areq_ctx->assoclen = req->assoclen; 2246 areq_ctx->plaintext_authenticate_only = false; 2247 2248 cc_proc_rfc4_gcm(req); 2249 areq_ctx->is_gcm4543 = true; 2250 2251 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2252 if (rc != -EINPROGRESS && rc != -EBUSY) 2253 req->iv = areq_ctx->backup_iv; 2254 out: 2255 return rc; 2256 } 2257 2258 static int cc_rfc4543_gcm_encrypt(struct aead_request *req) 2259 { 2260 /* Very similar to cc_aead_encrypt() above. */ 2261 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2262 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2263 struct device *dev = drvdata_to_dev(ctx->drvdata); 2264 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2265 int rc = -EINVAL; 2266 2267 if (!valid_assoclen(req)) { 2268 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2269 goto out; 2270 } 2271 2272 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2273 2274 //plaintext is not encryped with rfc4543 2275 areq_ctx->plaintext_authenticate_only = true; 2276 2277 /* No generated IV required */ 2278 areq_ctx->backup_iv = req->iv; 2279 areq_ctx->assoclen = req->assoclen; 2280 2281 cc_proc_rfc4_gcm(req); 2282 areq_ctx->is_gcm4543 = true; 2283 2284 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2285 if (rc != -EINPROGRESS && rc != -EBUSY) 2286 req->iv = areq_ctx->backup_iv; 2287 out: 2288 return rc; 2289 } 2290 2291 static int cc_rfc4106_gcm_decrypt(struct aead_request *req) 2292 { 2293 /* Very similar to cc_aead_decrypt() above. */ 2294 2295 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2296 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2297 struct device *dev = drvdata_to_dev(ctx->drvdata); 2298 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2299 int rc = -EINVAL; 2300 2301 if (!valid_assoclen(req)) { 2302 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2303 goto out; 2304 } 2305 2306 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2307 2308 /* No generated IV required */ 2309 areq_ctx->backup_iv = req->iv; 2310 areq_ctx->assoclen = req->assoclen; 2311 areq_ctx->plaintext_authenticate_only = false; 2312 2313 cc_proc_rfc4_gcm(req); 2314 areq_ctx->is_gcm4543 = true; 2315 2316 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2317 if (rc != -EINPROGRESS && rc != -EBUSY) 2318 req->iv = areq_ctx->backup_iv; 2319 out: 2320 return rc; 2321 } 2322 2323 static int cc_rfc4543_gcm_decrypt(struct aead_request *req) 2324 { 2325 /* Very similar to cc_aead_decrypt() above. */ 2326 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2327 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2328 struct device *dev = drvdata_to_dev(ctx->drvdata); 2329 struct aead_req_ctx *areq_ctx = aead_request_ctx(req); 2330 int rc = -EINVAL; 2331 2332 if (!valid_assoclen(req)) { 2333 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen); 2334 goto out; 2335 } 2336 2337 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2338 2339 //plaintext is not decryped with rfc4543 2340 areq_ctx->plaintext_authenticate_only = true; 2341 2342 /* No generated IV required */ 2343 areq_ctx->backup_iv = req->iv; 2344 areq_ctx->assoclen = req->assoclen; 2345 2346 cc_proc_rfc4_gcm(req); 2347 areq_ctx->is_gcm4543 = true; 2348 2349 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2350 if (rc != -EINPROGRESS && rc != -EBUSY) 2351 req->iv = areq_ctx->backup_iv; 2352 out: 2353 return rc; 2354 } 2355 2356 /* aead alg */ 2357 static struct cc_alg_template aead_algs[] = { 2358 { 2359 .name = "authenc(hmac(sha1),cbc(aes))", 2360 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree", 2361 .blocksize = AES_BLOCK_SIZE, 2362 .template_aead = { 2363 .setkey = cc_aead_setkey, 2364 .setauthsize = cc_aead_setauthsize, 2365 .encrypt = cc_aead_encrypt, 2366 .decrypt = cc_aead_decrypt, 2367 .init = cc_aead_init, 2368 .exit = cc_aead_exit, 2369 .ivsize = AES_BLOCK_SIZE, 2370 .maxauthsize = SHA1_DIGEST_SIZE, 2371 }, 2372 .cipher_mode = DRV_CIPHER_CBC, 2373 .flow_mode = S_DIN_to_AES, 2374 .auth_mode = DRV_HASH_SHA1, 2375 .min_hw_rev = CC_HW_REV_630, 2376 .std_body = CC_STD_NIST, 2377 }, 2378 { 2379 .name = "authenc(hmac(sha1),cbc(des3_ede))", 2380 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree", 2381 .blocksize = DES3_EDE_BLOCK_SIZE, 2382 .template_aead = { 2383 .setkey = cc_des3_aead_setkey, 2384 .setauthsize = cc_aead_setauthsize, 2385 .encrypt = cc_aead_encrypt, 2386 .decrypt = cc_aead_decrypt, 2387 .init = cc_aead_init, 2388 .exit = cc_aead_exit, 2389 .ivsize = DES3_EDE_BLOCK_SIZE, 2390 .maxauthsize = SHA1_DIGEST_SIZE, 2391 }, 2392 .cipher_mode = DRV_CIPHER_CBC, 2393 .flow_mode = S_DIN_to_DES, 2394 .auth_mode = DRV_HASH_SHA1, 2395 .min_hw_rev = CC_HW_REV_630, 2396 .std_body = CC_STD_NIST, 2397 }, 2398 { 2399 .name = "authenc(hmac(sha256),cbc(aes))", 2400 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree", 2401 .blocksize = AES_BLOCK_SIZE, 2402 .template_aead = { 2403 .setkey = cc_aead_setkey, 2404 .setauthsize = cc_aead_setauthsize, 2405 .encrypt = cc_aead_encrypt, 2406 .decrypt = cc_aead_decrypt, 2407 .init = cc_aead_init, 2408 .exit = cc_aead_exit, 2409 .ivsize = AES_BLOCK_SIZE, 2410 .maxauthsize = SHA256_DIGEST_SIZE, 2411 }, 2412 .cipher_mode = DRV_CIPHER_CBC, 2413 .flow_mode = S_DIN_to_AES, 2414 .auth_mode = DRV_HASH_SHA256, 2415 .min_hw_rev = CC_HW_REV_630, 2416 .std_body = CC_STD_NIST, 2417 }, 2418 { 2419 .name = "authenc(hmac(sha256),cbc(des3_ede))", 2420 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree", 2421 .blocksize = DES3_EDE_BLOCK_SIZE, 2422 .template_aead = { 2423 .setkey = cc_des3_aead_setkey, 2424 .setauthsize = cc_aead_setauthsize, 2425 .encrypt = cc_aead_encrypt, 2426 .decrypt = cc_aead_decrypt, 2427 .init = cc_aead_init, 2428 .exit = cc_aead_exit, 2429 .ivsize = DES3_EDE_BLOCK_SIZE, 2430 .maxauthsize = SHA256_DIGEST_SIZE, 2431 }, 2432 .cipher_mode = DRV_CIPHER_CBC, 2433 .flow_mode = S_DIN_to_DES, 2434 .auth_mode = DRV_HASH_SHA256, 2435 .min_hw_rev = CC_HW_REV_630, 2436 .std_body = CC_STD_NIST, 2437 }, 2438 { 2439 .name = "authenc(xcbc(aes),cbc(aes))", 2440 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree", 2441 .blocksize = AES_BLOCK_SIZE, 2442 .template_aead = { 2443 .setkey = cc_aead_setkey, 2444 .setauthsize = cc_aead_setauthsize, 2445 .encrypt = cc_aead_encrypt, 2446 .decrypt = cc_aead_decrypt, 2447 .init = cc_aead_init, 2448 .exit = cc_aead_exit, 2449 .ivsize = AES_BLOCK_SIZE, 2450 .maxauthsize = AES_BLOCK_SIZE, 2451 }, 2452 .cipher_mode = DRV_CIPHER_CBC, 2453 .flow_mode = S_DIN_to_AES, 2454 .auth_mode = DRV_HASH_XCBC_MAC, 2455 .min_hw_rev = CC_HW_REV_630, 2456 .std_body = CC_STD_NIST, 2457 }, 2458 { 2459 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2460 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree", 2461 .blocksize = 1, 2462 .template_aead = { 2463 .setkey = cc_aead_setkey, 2464 .setauthsize = cc_aead_setauthsize, 2465 .encrypt = cc_aead_encrypt, 2466 .decrypt = cc_aead_decrypt, 2467 .init = cc_aead_init, 2468 .exit = cc_aead_exit, 2469 .ivsize = CTR_RFC3686_IV_SIZE, 2470 .maxauthsize = SHA1_DIGEST_SIZE, 2471 }, 2472 .cipher_mode = DRV_CIPHER_CTR, 2473 .flow_mode = S_DIN_to_AES, 2474 .auth_mode = DRV_HASH_SHA1, 2475 .min_hw_rev = CC_HW_REV_630, 2476 .std_body = CC_STD_NIST, 2477 }, 2478 { 2479 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2480 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree", 2481 .blocksize = 1, 2482 .template_aead = { 2483 .setkey = cc_aead_setkey, 2484 .setauthsize = cc_aead_setauthsize, 2485 .encrypt = cc_aead_encrypt, 2486 .decrypt = cc_aead_decrypt, 2487 .init = cc_aead_init, 2488 .exit = cc_aead_exit, 2489 .ivsize = CTR_RFC3686_IV_SIZE, 2490 .maxauthsize = SHA256_DIGEST_SIZE, 2491 }, 2492 .cipher_mode = DRV_CIPHER_CTR, 2493 .flow_mode = S_DIN_to_AES, 2494 .auth_mode = DRV_HASH_SHA256, 2495 .min_hw_rev = CC_HW_REV_630, 2496 .std_body = CC_STD_NIST, 2497 }, 2498 { 2499 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))", 2500 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree", 2501 .blocksize = 1, 2502 .template_aead = { 2503 .setkey = cc_aead_setkey, 2504 .setauthsize = cc_aead_setauthsize, 2505 .encrypt = cc_aead_encrypt, 2506 .decrypt = cc_aead_decrypt, 2507 .init = cc_aead_init, 2508 .exit = cc_aead_exit, 2509 .ivsize = CTR_RFC3686_IV_SIZE, 2510 .maxauthsize = AES_BLOCK_SIZE, 2511 }, 2512 .cipher_mode = DRV_CIPHER_CTR, 2513 .flow_mode = S_DIN_to_AES, 2514 .auth_mode = DRV_HASH_XCBC_MAC, 2515 .min_hw_rev = CC_HW_REV_630, 2516 .std_body = CC_STD_NIST, 2517 }, 2518 { 2519 .name = "ccm(aes)", 2520 .driver_name = "ccm-aes-ccree", 2521 .blocksize = 1, 2522 .template_aead = { 2523 .setkey = cc_aead_setkey, 2524 .setauthsize = cc_ccm_setauthsize, 2525 .encrypt = cc_aead_encrypt, 2526 .decrypt = cc_aead_decrypt, 2527 .init = cc_aead_init, 2528 .exit = cc_aead_exit, 2529 .ivsize = AES_BLOCK_SIZE, 2530 .maxauthsize = AES_BLOCK_SIZE, 2531 }, 2532 .cipher_mode = DRV_CIPHER_CCM, 2533 .flow_mode = S_DIN_to_AES, 2534 .auth_mode = DRV_HASH_NULL, 2535 .min_hw_rev = CC_HW_REV_630, 2536 .std_body = CC_STD_NIST, 2537 }, 2538 { 2539 .name = "rfc4309(ccm(aes))", 2540 .driver_name = "rfc4309-ccm-aes-ccree", 2541 .blocksize = 1, 2542 .template_aead = { 2543 .setkey = cc_rfc4309_ccm_setkey, 2544 .setauthsize = cc_rfc4309_ccm_setauthsize, 2545 .encrypt = cc_rfc4309_ccm_encrypt, 2546 .decrypt = cc_rfc4309_ccm_decrypt, 2547 .init = cc_aead_init, 2548 .exit = cc_aead_exit, 2549 .ivsize = CCM_BLOCK_IV_SIZE, 2550 .maxauthsize = AES_BLOCK_SIZE, 2551 }, 2552 .cipher_mode = DRV_CIPHER_CCM, 2553 .flow_mode = S_DIN_to_AES, 2554 .auth_mode = DRV_HASH_NULL, 2555 .min_hw_rev = CC_HW_REV_630, 2556 .std_body = CC_STD_NIST, 2557 }, 2558 { 2559 .name = "gcm(aes)", 2560 .driver_name = "gcm-aes-ccree", 2561 .blocksize = 1, 2562 .template_aead = { 2563 .setkey = cc_aead_setkey, 2564 .setauthsize = cc_gcm_setauthsize, 2565 .encrypt = cc_aead_encrypt, 2566 .decrypt = cc_aead_decrypt, 2567 .init = cc_aead_init, 2568 .exit = cc_aead_exit, 2569 .ivsize = 12, 2570 .maxauthsize = AES_BLOCK_SIZE, 2571 }, 2572 .cipher_mode = DRV_CIPHER_GCTR, 2573 .flow_mode = S_DIN_to_AES, 2574 .auth_mode = DRV_HASH_NULL, 2575 .min_hw_rev = CC_HW_REV_630, 2576 .std_body = CC_STD_NIST, 2577 }, 2578 { 2579 .name = "rfc4106(gcm(aes))", 2580 .driver_name = "rfc4106-gcm-aes-ccree", 2581 .blocksize = 1, 2582 .template_aead = { 2583 .setkey = cc_rfc4106_gcm_setkey, 2584 .setauthsize = cc_rfc4106_gcm_setauthsize, 2585 .encrypt = cc_rfc4106_gcm_encrypt, 2586 .decrypt = cc_rfc4106_gcm_decrypt, 2587 .init = cc_aead_init, 2588 .exit = cc_aead_exit, 2589 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2590 .maxauthsize = AES_BLOCK_SIZE, 2591 }, 2592 .cipher_mode = DRV_CIPHER_GCTR, 2593 .flow_mode = S_DIN_to_AES, 2594 .auth_mode = DRV_HASH_NULL, 2595 .min_hw_rev = CC_HW_REV_630, 2596 .std_body = CC_STD_NIST, 2597 }, 2598 { 2599 .name = "rfc4543(gcm(aes))", 2600 .driver_name = "rfc4543-gcm-aes-ccree", 2601 .blocksize = 1, 2602 .template_aead = { 2603 .setkey = cc_rfc4543_gcm_setkey, 2604 .setauthsize = cc_rfc4543_gcm_setauthsize, 2605 .encrypt = cc_rfc4543_gcm_encrypt, 2606 .decrypt = cc_rfc4543_gcm_decrypt, 2607 .init = cc_aead_init, 2608 .exit = cc_aead_exit, 2609 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2610 .maxauthsize = AES_BLOCK_SIZE, 2611 }, 2612 .cipher_mode = DRV_CIPHER_GCTR, 2613 .flow_mode = S_DIN_to_AES, 2614 .auth_mode = DRV_HASH_NULL, 2615 .min_hw_rev = CC_HW_REV_630, 2616 .std_body = CC_STD_NIST, 2617 }, 2618 }; 2619 2620 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, 2621 struct device *dev) 2622 { 2623 struct cc_crypto_alg *t_alg; 2624 struct aead_alg *alg; 2625 2626 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 2627 if (!t_alg) 2628 return ERR_PTR(-ENOMEM); 2629 2630 alg = &tmpl->template_aead; 2631 2632 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 2633 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2634 tmpl->driver_name); 2635 alg->base.cra_module = THIS_MODULE; 2636 alg->base.cra_priority = CC_CRA_PRIO; 2637 2638 alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx); 2639 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2640 alg->init = cc_aead_init; 2641 alg->exit = cc_aead_exit; 2642 2643 t_alg->aead_alg = *alg; 2644 2645 t_alg->cipher_mode = tmpl->cipher_mode; 2646 t_alg->flow_mode = tmpl->flow_mode; 2647 t_alg->auth_mode = tmpl->auth_mode; 2648 2649 return t_alg; 2650 } 2651 2652 int cc_aead_free(struct cc_drvdata *drvdata) 2653 { 2654 struct cc_crypto_alg *t_alg, *n; 2655 struct cc_aead_handle *aead_handle = 2656 (struct cc_aead_handle *)drvdata->aead_handle; 2657 2658 if (aead_handle) { 2659 /* Remove registered algs */ 2660 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list, 2661 entry) { 2662 crypto_unregister_aead(&t_alg->aead_alg); 2663 list_del(&t_alg->entry); 2664 kfree(t_alg); 2665 } 2666 kfree(aead_handle); 2667 drvdata->aead_handle = NULL; 2668 } 2669 2670 return 0; 2671 } 2672 2673 int cc_aead_alloc(struct cc_drvdata *drvdata) 2674 { 2675 struct cc_aead_handle *aead_handle; 2676 struct cc_crypto_alg *t_alg; 2677 int rc = -ENOMEM; 2678 int alg; 2679 struct device *dev = drvdata_to_dev(drvdata); 2680 2681 aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL); 2682 if (!aead_handle) { 2683 rc = -ENOMEM; 2684 goto fail0; 2685 } 2686 2687 INIT_LIST_HEAD(&aead_handle->aead_list); 2688 drvdata->aead_handle = aead_handle; 2689 2690 aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata, 2691 MAX_HMAC_DIGEST_SIZE); 2692 2693 if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) { 2694 dev_err(dev, "SRAM pool exhausted\n"); 2695 rc = -ENOMEM; 2696 goto fail1; 2697 } 2698 2699 /* Linux crypto */ 2700 for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) { 2701 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) || 2702 !(drvdata->std_bodies & aead_algs[alg].std_body)) 2703 continue; 2704 2705 t_alg = cc_create_aead_alg(&aead_algs[alg], dev); 2706 if (IS_ERR(t_alg)) { 2707 rc = PTR_ERR(t_alg); 2708 dev_err(dev, "%s alg allocation failed\n", 2709 aead_algs[alg].driver_name); 2710 goto fail1; 2711 } 2712 t_alg->drvdata = drvdata; 2713 rc = crypto_register_aead(&t_alg->aead_alg); 2714 if (rc) { 2715 dev_err(dev, "%s alg registration failed\n", 2716 t_alg->aead_alg.base.cra_driver_name); 2717 goto fail2; 2718 } else { 2719 list_add_tail(&t_alg->entry, &aead_handle->aead_list); 2720 dev_dbg(dev, "Registered %s\n", 2721 t_alg->aead_alg.base.cra_driver_name); 2722 } 2723 } 2724 2725 return 0; 2726 2727 fail2: 2728 kfree(t_alg); 2729 fail1: 2730 cc_aead_free(drvdata); 2731 fail0: 2732 return rc; 2733 } 2734