1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/rtnetlink.h> 7 #include <linux/string.h> 8 #include <crypto/algapi.h> 9 #include <crypto/internal/aead.h> 10 #include <crypto/authenc.h> 11 #include <crypto/gcm.h> 12 #include <crypto/internal/des.h> 13 #include "cc_driver.h" 14 #include "cc_buffer_mgr.h" 15 #include "cc_aead.h" 16 #include "cc_request_mgr.h" 17 #include "cc_hash.h" 18 #include "cc_sram_mgr.h" 19 20 #define template_aead template_u.aead 21 22 #define MAX_AEAD_SETKEY_SEQ 12 23 #define MAX_AEAD_PROCESS_SEQ 23 24 25 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE) 26 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE) 27 28 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE 29 30 struct cc_aead_handle { 31 u32 sram_workspace_addr; 32 struct list_head aead_list; 33 }; 34 35 struct cc_hmac_s { 36 u8 *padded_authkey; 37 u8 *ipad_opad; /* IPAD, OPAD*/ 38 dma_addr_t padded_authkey_dma_addr; 39 dma_addr_t ipad_opad_dma_addr; 40 }; 41 42 struct cc_xcbc_s { 43 u8 *xcbc_keys; /* K1,K2,K3 */ 44 dma_addr_t xcbc_keys_dma_addr; 45 }; 46 47 struct cc_aead_ctx { 48 struct cc_drvdata *drvdata; 49 u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */ 50 u8 *enckey; 51 dma_addr_t enckey_dma_addr; 52 union { 53 struct cc_hmac_s hmac; 54 struct cc_xcbc_s xcbc; 55 } auth_state; 56 unsigned int enc_keylen; 57 unsigned int auth_keylen; 58 unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */ 59 unsigned int hash_len; 60 enum drv_cipher_mode cipher_mode; 61 enum cc_flow_mode flow_mode; 62 enum drv_hash_mode auth_mode; 63 }; 64 65 static void cc_aead_exit(struct crypto_aead *tfm) 66 { 67 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 68 struct device *dev = drvdata_to_dev(ctx->drvdata); 69 70 dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm), 71 crypto_tfm_alg_name(&tfm->base)); 72 73 /* Unmap enckey buffer */ 74 if (ctx->enckey) { 75 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey, 76 ctx->enckey_dma_addr); 77 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n", 78 &ctx->enckey_dma_addr); 79 ctx->enckey_dma_addr = 0; 80 ctx->enckey = NULL; 81 } 82 83 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 84 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 85 86 if (xcbc->xcbc_keys) { 87 dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3, 88 xcbc->xcbc_keys, 89 xcbc->xcbc_keys_dma_addr); 90 } 91 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n", 92 &xcbc->xcbc_keys_dma_addr); 93 xcbc->xcbc_keys_dma_addr = 0; 94 xcbc->xcbc_keys = NULL; 95 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */ 96 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 97 98 if (hmac->ipad_opad) { 99 dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE, 100 hmac->ipad_opad, 101 hmac->ipad_opad_dma_addr); 102 dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n", 103 &hmac->ipad_opad_dma_addr); 104 hmac->ipad_opad_dma_addr = 0; 105 hmac->ipad_opad = NULL; 106 } 107 if (hmac->padded_authkey) { 108 dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE, 109 hmac->padded_authkey, 110 hmac->padded_authkey_dma_addr); 111 dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n", 112 &hmac->padded_authkey_dma_addr); 113 hmac->padded_authkey_dma_addr = 0; 114 hmac->padded_authkey = NULL; 115 } 116 } 117 } 118 119 static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm) 120 { 121 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 122 123 return cc_get_default_hash_len(ctx->drvdata); 124 } 125 126 static int cc_aead_init(struct crypto_aead *tfm) 127 { 128 struct aead_alg *alg = crypto_aead_alg(tfm); 129 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 130 struct cc_crypto_alg *cc_alg = 131 container_of(alg, struct cc_crypto_alg, aead_alg); 132 struct device *dev = drvdata_to_dev(cc_alg->drvdata); 133 134 dev_dbg(dev, "Initializing context @%p for %s\n", ctx, 135 crypto_tfm_alg_name(&tfm->base)); 136 137 /* Initialize modes in instance */ 138 ctx->cipher_mode = cc_alg->cipher_mode; 139 ctx->flow_mode = cc_alg->flow_mode; 140 ctx->auth_mode = cc_alg->auth_mode; 141 ctx->drvdata = cc_alg->drvdata; 142 crypto_aead_set_reqsize_dma(tfm, sizeof(struct aead_req_ctx)); 143 144 /* Allocate key buffer, cache line aligned */ 145 ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE, 146 &ctx->enckey_dma_addr, GFP_KERNEL); 147 if (!ctx->enckey) { 148 dev_err(dev, "Failed allocating key buffer\n"); 149 goto init_failed; 150 } 151 dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n", 152 ctx->enckey); 153 154 /* Set default authlen value */ 155 156 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */ 157 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc; 158 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3; 159 160 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */ 161 /* (and temporary for user key - up to 256b) */ 162 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size, 163 &xcbc->xcbc_keys_dma_addr, 164 GFP_KERNEL); 165 if (!xcbc->xcbc_keys) { 166 dev_err(dev, "Failed allocating buffer for XCBC keys\n"); 167 goto init_failed; 168 } 169 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */ 170 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 171 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE; 172 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr; 173 174 /* Allocate dma-coherent buffer for IPAD + OPAD */ 175 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size, 176 &hmac->ipad_opad_dma_addr, 177 GFP_KERNEL); 178 179 if (!hmac->ipad_opad) { 180 dev_err(dev, "Failed allocating IPAD/OPAD buffer\n"); 181 goto init_failed; 182 } 183 184 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n", 185 hmac->ipad_opad); 186 187 hmac->padded_authkey = dma_alloc_coherent(dev, 188 MAX_HMAC_BLOCK_SIZE, 189 pkey_dma, 190 GFP_KERNEL); 191 192 if (!hmac->padded_authkey) { 193 dev_err(dev, "failed to allocate padded_authkey\n"); 194 goto init_failed; 195 } 196 } else { 197 ctx->auth_state.hmac.ipad_opad = NULL; 198 ctx->auth_state.hmac.padded_authkey = NULL; 199 } 200 ctx->hash_len = cc_get_aead_hash_len(tfm); 201 202 return 0; 203 204 init_failed: 205 cc_aead_exit(tfm); 206 return -ENOMEM; 207 } 208 209 static void cc_aead_complete(struct device *dev, void *cc_req, int err) 210 { 211 struct aead_request *areq = (struct aead_request *)cc_req; 212 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); 213 struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req); 214 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 215 216 /* BACKLOG notification */ 217 if (err == -EINPROGRESS) 218 goto done; 219 220 cc_unmap_aead_request(dev, areq); 221 222 /* Restore ordinary iv pointer */ 223 areq->iv = areq_ctx->backup_iv; 224 225 if (err) 226 goto done; 227 228 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 229 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr, 230 ctx->authsize) != 0) { 231 dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n", 232 ctx->authsize, ctx->cipher_mode); 233 /* In case of payload authentication failure, MUST NOT 234 * revealed the decrypted message --> zero its memory. 235 */ 236 sg_zero_buffer(areq->dst, sg_nents(areq->dst), 237 areq->cryptlen, areq->assoclen); 238 err = -EBADMSG; 239 } 240 /*ENCRYPT*/ 241 } else if (areq_ctx->is_icv_fragmented) { 242 u32 skip = areq->cryptlen + areq_ctx->dst_offset; 243 244 cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl, 245 skip, (skip + ctx->authsize), 246 CC_SG_FROM_BUF); 247 } 248 done: 249 aead_request_complete(areq, err); 250 } 251 252 static unsigned int xcbc_setkey(struct cc_hw_desc *desc, 253 struct cc_aead_ctx *ctx) 254 { 255 /* Load the AES key */ 256 hw_desc_init(&desc[0]); 257 /* We are using for the source/user key the same buffer 258 * as for the output keys, * because after this key loading it 259 * is not needed anymore 260 */ 261 set_din_type(&desc[0], DMA_DLLI, 262 ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen, 263 NS_BIT); 264 set_cipher_mode(&desc[0], DRV_CIPHER_ECB); 265 set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT); 266 set_key_size_aes(&desc[0], ctx->auth_keylen); 267 set_flow_mode(&desc[0], S_DIN_to_AES); 268 set_setup_mode(&desc[0], SETUP_LOAD_KEY0); 269 270 hw_desc_init(&desc[1]); 271 set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE); 272 set_flow_mode(&desc[1], DIN_AES_DOUT); 273 set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr, 274 AES_KEYSIZE_128, NS_BIT, 0); 275 276 hw_desc_init(&desc[2]); 277 set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE); 278 set_flow_mode(&desc[2], DIN_AES_DOUT); 279 set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 280 + AES_KEYSIZE_128), 281 AES_KEYSIZE_128, NS_BIT, 0); 282 283 hw_desc_init(&desc[3]); 284 set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE); 285 set_flow_mode(&desc[3], DIN_AES_DOUT); 286 set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr 287 + 2 * AES_KEYSIZE_128), 288 AES_KEYSIZE_128, NS_BIT, 0); 289 290 return 4; 291 } 292 293 static unsigned int hmac_setkey(struct cc_hw_desc *desc, 294 struct cc_aead_ctx *ctx) 295 { 296 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST }; 297 unsigned int digest_ofs = 0; 298 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 299 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 300 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 301 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 302 struct cc_hmac_s *hmac = &ctx->auth_state.hmac; 303 304 unsigned int idx = 0; 305 int i; 306 307 /* calc derived HMAC key */ 308 for (i = 0; i < 2; i++) { 309 /* Load hash initial state */ 310 hw_desc_init(&desc[idx]); 311 set_cipher_mode(&desc[idx], hash_mode); 312 set_din_sram(&desc[idx], 313 cc_larval_digest_addr(ctx->drvdata, 314 ctx->auth_mode), 315 digest_size); 316 set_flow_mode(&desc[idx], S_DIN_to_HASH); 317 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 318 idx++; 319 320 /* Load the hash current length*/ 321 hw_desc_init(&desc[idx]); 322 set_cipher_mode(&desc[idx], hash_mode); 323 set_din_const(&desc[idx], 0, ctx->hash_len); 324 set_flow_mode(&desc[idx], S_DIN_to_HASH); 325 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 326 idx++; 327 328 /* Prepare ipad key */ 329 hw_desc_init(&desc[idx]); 330 set_xor_val(&desc[idx], hmac_pad_const[i]); 331 set_cipher_mode(&desc[idx], hash_mode); 332 set_flow_mode(&desc[idx], S_DIN_to_HASH); 333 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 334 idx++; 335 336 /* Perform HASH update */ 337 hw_desc_init(&desc[idx]); 338 set_din_type(&desc[idx], DMA_DLLI, 339 hmac->padded_authkey_dma_addr, 340 SHA256_BLOCK_SIZE, NS_BIT); 341 set_cipher_mode(&desc[idx], hash_mode); 342 set_xor_active(&desc[idx]); 343 set_flow_mode(&desc[idx], DIN_HASH); 344 idx++; 345 346 /* Get the digset */ 347 hw_desc_init(&desc[idx]); 348 set_cipher_mode(&desc[idx], hash_mode); 349 set_dout_dlli(&desc[idx], 350 (hmac->ipad_opad_dma_addr + digest_ofs), 351 digest_size, NS_BIT, 0); 352 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 353 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 354 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 355 idx++; 356 357 digest_ofs += digest_size; 358 } 359 360 return idx; 361 } 362 363 static int validate_keys_sizes(struct cc_aead_ctx *ctx) 364 { 365 struct device *dev = drvdata_to_dev(ctx->drvdata); 366 367 dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n", 368 ctx->enc_keylen, ctx->auth_keylen); 369 370 switch (ctx->auth_mode) { 371 case DRV_HASH_SHA1: 372 case DRV_HASH_SHA256: 373 break; 374 case DRV_HASH_XCBC_MAC: 375 if (ctx->auth_keylen != AES_KEYSIZE_128 && 376 ctx->auth_keylen != AES_KEYSIZE_192 && 377 ctx->auth_keylen != AES_KEYSIZE_256) 378 return -ENOTSUPP; 379 break; 380 case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */ 381 if (ctx->auth_keylen > 0) 382 return -EINVAL; 383 break; 384 default: 385 dev_dbg(dev, "Invalid auth_mode=%d\n", ctx->auth_mode); 386 return -EINVAL; 387 } 388 /* Check cipher key size */ 389 if (ctx->flow_mode == S_DIN_to_DES) { 390 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) { 391 dev_dbg(dev, "Invalid cipher(3DES) key size: %u\n", 392 ctx->enc_keylen); 393 return -EINVAL; 394 } 395 } else { /* Default assumed to be AES ciphers */ 396 if (ctx->enc_keylen != AES_KEYSIZE_128 && 397 ctx->enc_keylen != AES_KEYSIZE_192 && 398 ctx->enc_keylen != AES_KEYSIZE_256) { 399 dev_dbg(dev, "Invalid cipher(AES) key size: %u\n", 400 ctx->enc_keylen); 401 return -EINVAL; 402 } 403 } 404 405 return 0; /* All tests of keys sizes passed */ 406 } 407 408 /* This function prepers the user key so it can pass to the hmac processing 409 * (copy to intenral buffer or hash in case of key longer than block 410 */ 411 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey, 412 unsigned int keylen) 413 { 414 dma_addr_t key_dma_addr = 0; 415 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 416 struct device *dev = drvdata_to_dev(ctx->drvdata); 417 u32 larval_addr; 418 struct cc_crypto_req cc_req = {}; 419 unsigned int blocksize; 420 unsigned int digestsize; 421 unsigned int hashmode; 422 unsigned int idx = 0; 423 int rc = 0; 424 u8 *key = NULL; 425 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 426 dma_addr_t padded_authkey_dma_addr = 427 ctx->auth_state.hmac.padded_authkey_dma_addr; 428 429 switch (ctx->auth_mode) { /* auth_key required and >0 */ 430 case DRV_HASH_SHA1: 431 blocksize = SHA1_BLOCK_SIZE; 432 digestsize = SHA1_DIGEST_SIZE; 433 hashmode = DRV_HASH_HW_SHA1; 434 break; 435 case DRV_HASH_SHA256: 436 default: 437 blocksize = SHA256_BLOCK_SIZE; 438 digestsize = SHA256_DIGEST_SIZE; 439 hashmode = DRV_HASH_HW_SHA256; 440 } 441 442 if (keylen != 0) { 443 444 key = kmemdup(authkey, keylen, GFP_KERNEL); 445 if (!key) 446 return -ENOMEM; 447 448 key_dma_addr = dma_map_single(dev, key, keylen, DMA_TO_DEVICE); 449 if (dma_mapping_error(dev, key_dma_addr)) { 450 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n", 451 key, keylen); 452 kfree_sensitive(key); 453 return -ENOMEM; 454 } 455 if (keylen > blocksize) { 456 /* Load hash initial state */ 457 hw_desc_init(&desc[idx]); 458 set_cipher_mode(&desc[idx], hashmode); 459 larval_addr = cc_larval_digest_addr(ctx->drvdata, 460 ctx->auth_mode); 461 set_din_sram(&desc[idx], larval_addr, digestsize); 462 set_flow_mode(&desc[idx], S_DIN_to_HASH); 463 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 464 idx++; 465 466 /* Load the hash current length*/ 467 hw_desc_init(&desc[idx]); 468 set_cipher_mode(&desc[idx], hashmode); 469 set_din_const(&desc[idx], 0, ctx->hash_len); 470 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 471 set_flow_mode(&desc[idx], S_DIN_to_HASH); 472 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 473 idx++; 474 475 hw_desc_init(&desc[idx]); 476 set_din_type(&desc[idx], DMA_DLLI, 477 key_dma_addr, keylen, NS_BIT); 478 set_flow_mode(&desc[idx], DIN_HASH); 479 idx++; 480 481 /* Get hashed key */ 482 hw_desc_init(&desc[idx]); 483 set_cipher_mode(&desc[idx], hashmode); 484 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 485 digestsize, NS_BIT, 0); 486 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 487 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 488 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 489 set_cipher_config0(&desc[idx], 490 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 491 idx++; 492 493 hw_desc_init(&desc[idx]); 494 set_din_const(&desc[idx], 0, (blocksize - digestsize)); 495 set_flow_mode(&desc[idx], BYPASS); 496 set_dout_dlli(&desc[idx], (padded_authkey_dma_addr + 497 digestsize), (blocksize - digestsize), 498 NS_BIT, 0); 499 idx++; 500 } else { 501 hw_desc_init(&desc[idx]); 502 set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, 503 keylen, NS_BIT); 504 set_flow_mode(&desc[idx], BYPASS); 505 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 506 keylen, NS_BIT, 0); 507 idx++; 508 509 if ((blocksize - keylen) != 0) { 510 hw_desc_init(&desc[idx]); 511 set_din_const(&desc[idx], 0, 512 (blocksize - keylen)); 513 set_flow_mode(&desc[idx], BYPASS); 514 set_dout_dlli(&desc[idx], 515 (padded_authkey_dma_addr + 516 keylen), 517 (blocksize - keylen), NS_BIT, 0); 518 idx++; 519 } 520 } 521 } else { 522 hw_desc_init(&desc[idx]); 523 set_din_const(&desc[idx], 0, (blocksize - keylen)); 524 set_flow_mode(&desc[idx], BYPASS); 525 set_dout_dlli(&desc[idx], padded_authkey_dma_addr, 526 blocksize, NS_BIT, 0); 527 idx++; 528 } 529 530 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx); 531 if (rc) 532 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 533 534 if (key_dma_addr) 535 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE); 536 537 kfree_sensitive(key); 538 539 return rc; 540 } 541 542 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key, 543 unsigned int keylen) 544 { 545 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 546 struct cc_crypto_req cc_req = {}; 547 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ]; 548 unsigned int seq_len = 0; 549 struct device *dev = drvdata_to_dev(ctx->drvdata); 550 const u8 *enckey, *authkey; 551 int rc; 552 553 dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n", 554 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen); 555 556 /* STAT_PHASE_0: Init and sanity checks */ 557 558 if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */ 559 struct crypto_authenc_keys keys; 560 561 rc = crypto_authenc_extractkeys(&keys, key, keylen); 562 if (rc) 563 return rc; 564 enckey = keys.enckey; 565 authkey = keys.authkey; 566 ctx->enc_keylen = keys.enckeylen; 567 ctx->auth_keylen = keys.authkeylen; 568 569 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 570 /* the nonce is stored in bytes at end of key */ 571 if (ctx->enc_keylen < 572 (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE)) 573 return -EINVAL; 574 /* Copy nonce from last 4 bytes in CTR key to 575 * first 4 bytes in CTR IV 576 */ 577 memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen - 578 CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); 579 /* Set CTR key size */ 580 ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE; 581 } 582 } else { /* non-authenc - has just one key */ 583 enckey = key; 584 authkey = NULL; 585 ctx->enc_keylen = keylen; 586 ctx->auth_keylen = 0; 587 } 588 589 rc = validate_keys_sizes(ctx); 590 if (rc) 591 return rc; 592 593 /* STAT_PHASE_1: Copy key to ctx */ 594 595 /* Get key material */ 596 memcpy(ctx->enckey, enckey, ctx->enc_keylen); 597 if (ctx->enc_keylen == 24) 598 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24); 599 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 600 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey, 601 ctx->auth_keylen); 602 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */ 603 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen); 604 if (rc) 605 return rc; 606 } 607 608 /* STAT_PHASE_2: Create sequence */ 609 610 switch (ctx->auth_mode) { 611 case DRV_HASH_SHA1: 612 case DRV_HASH_SHA256: 613 seq_len = hmac_setkey(desc, ctx); 614 break; 615 case DRV_HASH_XCBC_MAC: 616 seq_len = xcbc_setkey(desc, ctx); 617 break; 618 case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */ 619 break; /* No auth. key setup */ 620 default: 621 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 622 return -ENOTSUPP; 623 } 624 625 /* STAT_PHASE_3: Submit sequence to HW */ 626 627 if (seq_len > 0) { /* For CCM there is no sequence to setup the key */ 628 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len); 629 if (rc) { 630 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 631 return rc; 632 } 633 } 634 635 /* Update STAT_PHASE_3 */ 636 return rc; 637 } 638 639 static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 640 unsigned int keylen) 641 { 642 struct crypto_authenc_keys keys; 643 int err; 644 645 err = crypto_authenc_extractkeys(&keys, key, keylen); 646 if (unlikely(err)) 647 return err; 648 649 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 650 cc_aead_setkey(aead, key, keylen); 651 652 memzero_explicit(&keys, sizeof(keys)); 653 return err; 654 } 655 656 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key, 657 unsigned int keylen) 658 { 659 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 660 661 if (keylen < 3) 662 return -EINVAL; 663 664 keylen -= 3; 665 memcpy(ctx->ctr_nonce, key + keylen, 3); 666 667 return cc_aead_setkey(tfm, key, keylen); 668 } 669 670 static int cc_aead_setauthsize(struct crypto_aead *authenc, 671 unsigned int authsize) 672 { 673 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 674 struct device *dev = drvdata_to_dev(ctx->drvdata); 675 676 /* Unsupported auth. sizes */ 677 if (authsize == 0 || 678 authsize > crypto_aead_maxauthsize(authenc)) { 679 return -ENOTSUPP; 680 } 681 682 ctx->authsize = authsize; 683 dev_dbg(dev, "authlen=%d\n", ctx->authsize); 684 685 return 0; 686 } 687 688 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc, 689 unsigned int authsize) 690 { 691 switch (authsize) { 692 case 8: 693 case 12: 694 case 16: 695 break; 696 default: 697 return -EINVAL; 698 } 699 700 return cc_aead_setauthsize(authenc, authsize); 701 } 702 703 static int cc_ccm_setauthsize(struct crypto_aead *authenc, 704 unsigned int authsize) 705 { 706 switch (authsize) { 707 case 4: 708 case 6: 709 case 8: 710 case 10: 711 case 12: 712 case 14: 713 case 16: 714 break; 715 default: 716 return -EINVAL; 717 } 718 719 return cc_aead_setauthsize(authenc, authsize); 720 } 721 722 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode, 723 struct cc_hw_desc desc[], unsigned int *seq_size) 724 { 725 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 726 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 727 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); 728 enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type; 729 unsigned int idx = *seq_size; 730 struct device *dev = drvdata_to_dev(ctx->drvdata); 731 732 switch (assoc_dma_type) { 733 case CC_DMA_BUF_DLLI: 734 dev_dbg(dev, "ASSOC buffer type DLLI\n"); 735 hw_desc_init(&desc[idx]); 736 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src), 737 areq_ctx->assoclen, NS_BIT); 738 set_flow_mode(&desc[idx], flow_mode); 739 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 740 areq_ctx->cryptlen > 0) 741 set_din_not_last_indication(&desc[idx]); 742 break; 743 case CC_DMA_BUF_MLLI: 744 dev_dbg(dev, "ASSOC buffer type MLLI\n"); 745 hw_desc_init(&desc[idx]); 746 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr, 747 areq_ctx->assoc.mlli_nents, NS_BIT); 748 set_flow_mode(&desc[idx], flow_mode); 749 if (ctx->auth_mode == DRV_HASH_XCBC_MAC && 750 areq_ctx->cryptlen > 0) 751 set_din_not_last_indication(&desc[idx]); 752 break; 753 case CC_DMA_BUF_NULL: 754 default: 755 dev_err(dev, "Invalid ASSOC buffer type\n"); 756 } 757 758 *seq_size = (++idx); 759 } 760 761 static void cc_proc_authen_desc(struct aead_request *areq, 762 unsigned int flow_mode, 763 struct cc_hw_desc desc[], 764 unsigned int *seq_size, int direct) 765 { 766 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); 767 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 768 unsigned int idx = *seq_size; 769 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 770 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 771 struct device *dev = drvdata_to_dev(ctx->drvdata); 772 773 switch (data_dma_type) { 774 case CC_DMA_BUF_DLLI: 775 { 776 struct scatterlist *cipher = 777 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 778 areq_ctx->dst_sgl : areq_ctx->src_sgl; 779 780 unsigned int offset = 781 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 782 areq_ctx->dst_offset : areq_ctx->src_offset; 783 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n"); 784 hw_desc_init(&desc[idx]); 785 set_din_type(&desc[idx], DMA_DLLI, 786 (sg_dma_address(cipher) + offset), 787 areq_ctx->cryptlen, NS_BIT); 788 set_flow_mode(&desc[idx], flow_mode); 789 break; 790 } 791 case CC_DMA_BUF_MLLI: 792 { 793 /* DOUBLE-PASS flow (as default) 794 * assoc. + iv + data -compact in one table 795 * if assoclen is ZERO only IV perform 796 */ 797 u32 mlli_addr = areq_ctx->assoc.sram_addr; 798 u32 mlli_nents = areq_ctx->assoc.mlli_nents; 799 800 if (areq_ctx->is_single_pass) { 801 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 802 mlli_addr = areq_ctx->dst.sram_addr; 803 mlli_nents = areq_ctx->dst.mlli_nents; 804 } else { 805 mlli_addr = areq_ctx->src.sram_addr; 806 mlli_nents = areq_ctx->src.mlli_nents; 807 } 808 } 809 810 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n"); 811 hw_desc_init(&desc[idx]); 812 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents, 813 NS_BIT); 814 set_flow_mode(&desc[idx], flow_mode); 815 break; 816 } 817 case CC_DMA_BUF_NULL: 818 default: 819 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n"); 820 } 821 822 *seq_size = (++idx); 823 } 824 825 static void cc_proc_cipher_desc(struct aead_request *areq, 826 unsigned int flow_mode, 827 struct cc_hw_desc desc[], 828 unsigned int *seq_size) 829 { 830 unsigned int idx = *seq_size; 831 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(areq); 832 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type; 833 struct crypto_aead *tfm = crypto_aead_reqtfm(areq); 834 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 835 struct device *dev = drvdata_to_dev(ctx->drvdata); 836 837 if (areq_ctx->cryptlen == 0) 838 return; /*null processing*/ 839 840 switch (data_dma_type) { 841 case CC_DMA_BUF_DLLI: 842 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n"); 843 hw_desc_init(&desc[idx]); 844 set_din_type(&desc[idx], DMA_DLLI, 845 (sg_dma_address(areq_ctx->src_sgl) + 846 areq_ctx->src_offset), areq_ctx->cryptlen, 847 NS_BIT); 848 set_dout_dlli(&desc[idx], 849 (sg_dma_address(areq_ctx->dst_sgl) + 850 areq_ctx->dst_offset), 851 areq_ctx->cryptlen, NS_BIT, 0); 852 set_flow_mode(&desc[idx], flow_mode); 853 break; 854 case CC_DMA_BUF_MLLI: 855 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n"); 856 hw_desc_init(&desc[idx]); 857 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr, 858 areq_ctx->src.mlli_nents, NS_BIT); 859 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr, 860 areq_ctx->dst.mlli_nents, NS_BIT, 0); 861 set_flow_mode(&desc[idx], flow_mode); 862 break; 863 case CC_DMA_BUF_NULL: 864 default: 865 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n"); 866 } 867 868 *seq_size = (++idx); 869 } 870 871 static void cc_proc_digest_desc(struct aead_request *req, 872 struct cc_hw_desc desc[], 873 unsigned int *seq_size) 874 { 875 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 876 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 877 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 878 unsigned int idx = *seq_size; 879 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 880 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 881 int direct = req_ctx->gen_ctx.op_type; 882 883 /* Get final ICV result */ 884 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 885 hw_desc_init(&desc[idx]); 886 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 887 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 888 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize, 889 NS_BIT, 1); 890 set_queue_last_ind(ctx->drvdata, &desc[idx]); 891 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 892 set_aes_not_hash_mode(&desc[idx]); 893 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 894 } else { 895 set_cipher_config0(&desc[idx], 896 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 897 set_cipher_mode(&desc[idx], hash_mode); 898 } 899 } else { /*Decrypt*/ 900 /* Get ICV out from hardware */ 901 hw_desc_init(&desc[idx]); 902 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 903 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 904 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, 905 ctx->authsize, NS_BIT, 1); 906 set_queue_last_ind(ctx->drvdata, &desc[idx]); 907 set_cipher_config0(&desc[idx], 908 HASH_DIGEST_RESULT_LITTLE_ENDIAN); 909 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED); 910 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { 911 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 912 set_aes_not_hash_mode(&desc[idx]); 913 } else { 914 set_cipher_mode(&desc[idx], hash_mode); 915 } 916 } 917 918 *seq_size = (++idx); 919 } 920 921 static void cc_set_cipher_desc(struct aead_request *req, 922 struct cc_hw_desc desc[], 923 unsigned int *seq_size) 924 { 925 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 926 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 927 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 928 unsigned int hw_iv_size = req_ctx->hw_iv_size; 929 unsigned int idx = *seq_size; 930 int direct = req_ctx->gen_ctx.op_type; 931 932 /* Setup cipher state */ 933 hw_desc_init(&desc[idx]); 934 set_cipher_config0(&desc[idx], direct); 935 set_flow_mode(&desc[idx], ctx->flow_mode); 936 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr, 937 hw_iv_size, NS_BIT); 938 if (ctx->cipher_mode == DRV_CIPHER_CTR) 939 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 940 else 941 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 942 set_cipher_mode(&desc[idx], ctx->cipher_mode); 943 idx++; 944 945 /* Setup enc. key */ 946 hw_desc_init(&desc[idx]); 947 set_cipher_config0(&desc[idx], direct); 948 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 949 set_flow_mode(&desc[idx], ctx->flow_mode); 950 if (ctx->flow_mode == S_DIN_to_AES) { 951 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 952 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 953 ctx->enc_keylen), NS_BIT); 954 set_key_size_aes(&desc[idx], ctx->enc_keylen); 955 } else { 956 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 957 ctx->enc_keylen, NS_BIT); 958 set_key_size_des(&desc[idx], ctx->enc_keylen); 959 } 960 set_cipher_mode(&desc[idx], ctx->cipher_mode); 961 idx++; 962 963 *seq_size = idx; 964 } 965 966 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[], 967 unsigned int *seq_size, unsigned int data_flow_mode) 968 { 969 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 970 int direct = req_ctx->gen_ctx.op_type; 971 unsigned int idx = *seq_size; 972 973 if (req_ctx->cryptlen == 0) 974 return; /*null processing*/ 975 976 cc_set_cipher_desc(req, desc, &idx); 977 cc_proc_cipher_desc(req, data_flow_mode, desc, &idx); 978 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 979 /* We must wait for DMA to write all cipher */ 980 hw_desc_init(&desc[idx]); 981 set_din_no_dma(&desc[idx], 0, 0xfffff0); 982 set_dout_no_dma(&desc[idx], 0, 0, 1); 983 idx++; 984 } 985 986 *seq_size = idx; 987 } 988 989 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[], 990 unsigned int *seq_size) 991 { 992 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 993 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 994 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 995 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 996 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 997 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 998 unsigned int idx = *seq_size; 999 1000 /* Loading hash ipad xor key state */ 1001 hw_desc_init(&desc[idx]); 1002 set_cipher_mode(&desc[idx], hash_mode); 1003 set_din_type(&desc[idx], DMA_DLLI, 1004 ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size, 1005 NS_BIT); 1006 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1007 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1008 idx++; 1009 1010 /* Load init. digest len (64 bytes) */ 1011 hw_desc_init(&desc[idx]); 1012 set_cipher_mode(&desc[idx], hash_mode); 1013 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1014 ctx->hash_len); 1015 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1016 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1017 idx++; 1018 1019 *seq_size = idx; 1020 } 1021 1022 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[], 1023 unsigned int *seq_size) 1024 { 1025 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1026 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1027 unsigned int idx = *seq_size; 1028 1029 /* Loading MAC state */ 1030 hw_desc_init(&desc[idx]); 1031 set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE); 1032 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1033 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1034 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1035 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1036 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1037 set_aes_not_hash_mode(&desc[idx]); 1038 idx++; 1039 1040 /* Setup XCBC MAC K1 */ 1041 hw_desc_init(&desc[idx]); 1042 set_din_type(&desc[idx], DMA_DLLI, 1043 ctx->auth_state.xcbc.xcbc_keys_dma_addr, 1044 AES_KEYSIZE_128, NS_BIT); 1045 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1046 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1047 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1048 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1049 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1050 set_aes_not_hash_mode(&desc[idx]); 1051 idx++; 1052 1053 /* Setup XCBC MAC K2 */ 1054 hw_desc_init(&desc[idx]); 1055 set_din_type(&desc[idx], DMA_DLLI, 1056 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1057 AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1058 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1059 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1060 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1061 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1062 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1063 set_aes_not_hash_mode(&desc[idx]); 1064 idx++; 1065 1066 /* Setup XCBC MAC K3 */ 1067 hw_desc_init(&desc[idx]); 1068 set_din_type(&desc[idx], DMA_DLLI, 1069 (ctx->auth_state.xcbc.xcbc_keys_dma_addr + 1070 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT); 1071 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2); 1072 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC); 1073 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1074 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE); 1075 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1076 set_aes_not_hash_mode(&desc[idx]); 1077 idx++; 1078 1079 *seq_size = idx; 1080 } 1081 1082 static void cc_proc_header_desc(struct aead_request *req, 1083 struct cc_hw_desc desc[], 1084 unsigned int *seq_size) 1085 { 1086 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1087 unsigned int idx = *seq_size; 1088 1089 /* Hash associated data */ 1090 if (areq_ctx->assoclen > 0) 1091 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1092 1093 /* Hash IV */ 1094 *seq_size = idx; 1095 } 1096 1097 static void cc_proc_scheme_desc(struct aead_request *req, 1098 struct cc_hw_desc desc[], 1099 unsigned int *seq_size) 1100 { 1101 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1102 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1103 struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle; 1104 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ? 1105 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256; 1106 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ? 1107 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE; 1108 unsigned int idx = *seq_size; 1109 1110 hw_desc_init(&desc[idx]); 1111 set_cipher_mode(&desc[idx], hash_mode); 1112 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1113 ctx->hash_len); 1114 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1115 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1); 1116 set_cipher_do(&desc[idx], DO_PAD); 1117 idx++; 1118 1119 /* Get final ICV result */ 1120 hw_desc_init(&desc[idx]); 1121 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr, 1122 digest_size); 1123 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1124 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1125 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1126 set_cipher_mode(&desc[idx], hash_mode); 1127 idx++; 1128 1129 /* Loading hash opad xor key state */ 1130 hw_desc_init(&desc[idx]); 1131 set_cipher_mode(&desc[idx], hash_mode); 1132 set_din_type(&desc[idx], DMA_DLLI, 1133 (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size), 1134 digest_size, NS_BIT); 1135 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1136 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1137 idx++; 1138 1139 /* Load init. digest len (64 bytes) */ 1140 hw_desc_init(&desc[idx]); 1141 set_cipher_mode(&desc[idx], hash_mode); 1142 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode), 1143 ctx->hash_len); 1144 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1145 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1146 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1147 idx++; 1148 1149 /* Perform HASH update */ 1150 hw_desc_init(&desc[idx]); 1151 set_din_sram(&desc[idx], aead_handle->sram_workspace_addr, 1152 digest_size); 1153 set_flow_mode(&desc[idx], DIN_HASH); 1154 idx++; 1155 1156 *seq_size = idx; 1157 } 1158 1159 static void cc_mlli_to_sram(struct aead_request *req, 1160 struct cc_hw_desc desc[], unsigned int *seq_size) 1161 { 1162 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1163 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1164 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1165 struct device *dev = drvdata_to_dev(ctx->drvdata); 1166 1167 if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || 1168 req_ctx->data_buff_type == CC_DMA_BUF_MLLI || 1169 !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) { 1170 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n", 1171 ctx->drvdata->mlli_sram_addr, 1172 req_ctx->mlli_params.mlli_len); 1173 /* Copy MLLI table host-to-sram */ 1174 hw_desc_init(&desc[*seq_size]); 1175 set_din_type(&desc[*seq_size], DMA_DLLI, 1176 req_ctx->mlli_params.mlli_dma_addr, 1177 req_ctx->mlli_params.mlli_len, NS_BIT); 1178 set_dout_sram(&desc[*seq_size], 1179 ctx->drvdata->mlli_sram_addr, 1180 req_ctx->mlli_params.mlli_len); 1181 set_flow_mode(&desc[*seq_size], BYPASS); 1182 (*seq_size)++; 1183 } 1184 } 1185 1186 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct, 1187 enum cc_flow_mode setup_flow_mode, 1188 bool is_single_pass) 1189 { 1190 enum cc_flow_mode data_flow_mode; 1191 1192 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1193 if (setup_flow_mode == S_DIN_to_AES) 1194 data_flow_mode = is_single_pass ? 1195 AES_to_HASH_and_DOUT : DIN_AES_DOUT; 1196 else 1197 data_flow_mode = is_single_pass ? 1198 DES_to_HASH_and_DOUT : DIN_DES_DOUT; 1199 } else { /* Decrypt */ 1200 if (setup_flow_mode == S_DIN_to_AES) 1201 data_flow_mode = is_single_pass ? 1202 AES_and_HASH : DIN_AES_DOUT; 1203 else 1204 data_flow_mode = is_single_pass ? 1205 DES_and_HASH : DIN_DES_DOUT; 1206 } 1207 1208 return data_flow_mode; 1209 } 1210 1211 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1212 unsigned int *seq_size) 1213 { 1214 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1215 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1216 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1217 int direct = req_ctx->gen_ctx.op_type; 1218 unsigned int data_flow_mode = 1219 cc_get_data_flow(direct, ctx->flow_mode, 1220 req_ctx->is_single_pass); 1221 1222 if (req_ctx->is_single_pass) { 1223 /* 1224 * Single-pass flow 1225 */ 1226 cc_set_hmac_desc(req, desc, seq_size); 1227 cc_set_cipher_desc(req, desc, seq_size); 1228 cc_proc_header_desc(req, desc, seq_size); 1229 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1230 cc_proc_scheme_desc(req, desc, seq_size); 1231 cc_proc_digest_desc(req, desc, seq_size); 1232 return; 1233 } 1234 1235 /* 1236 * Double-pass flow 1237 * Fallback for unsupported single-pass modes, 1238 * i.e. using assoc. data of non-word-multiple 1239 */ 1240 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1241 /* encrypt first.. */ 1242 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1243 /* authenc after..*/ 1244 cc_set_hmac_desc(req, desc, seq_size); 1245 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1246 cc_proc_scheme_desc(req, desc, seq_size); 1247 cc_proc_digest_desc(req, desc, seq_size); 1248 1249 } else { /*DECRYPT*/ 1250 /* authenc first..*/ 1251 cc_set_hmac_desc(req, desc, seq_size); 1252 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1253 cc_proc_scheme_desc(req, desc, seq_size); 1254 /* decrypt after.. */ 1255 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1256 /* read the digest result with setting the completion bit 1257 * must be after the cipher operation 1258 */ 1259 cc_proc_digest_desc(req, desc, seq_size); 1260 } 1261 } 1262 1263 static void 1264 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[], 1265 unsigned int *seq_size) 1266 { 1267 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1268 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1269 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1270 int direct = req_ctx->gen_ctx.op_type; 1271 unsigned int data_flow_mode = 1272 cc_get_data_flow(direct, ctx->flow_mode, 1273 req_ctx->is_single_pass); 1274 1275 if (req_ctx->is_single_pass) { 1276 /* 1277 * Single-pass flow 1278 */ 1279 cc_set_xcbc_desc(req, desc, seq_size); 1280 cc_set_cipher_desc(req, desc, seq_size); 1281 cc_proc_header_desc(req, desc, seq_size); 1282 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size); 1283 cc_proc_digest_desc(req, desc, seq_size); 1284 return; 1285 } 1286 1287 /* 1288 * Double-pass flow 1289 * Fallback for unsupported single-pass modes, 1290 * i.e. using assoc. data of non-word-multiple 1291 */ 1292 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) { 1293 /* encrypt first.. */ 1294 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1295 /* authenc after.. */ 1296 cc_set_xcbc_desc(req, desc, seq_size); 1297 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1298 cc_proc_digest_desc(req, desc, seq_size); 1299 } else { /*DECRYPT*/ 1300 /* authenc first.. */ 1301 cc_set_xcbc_desc(req, desc, seq_size); 1302 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct); 1303 /* decrypt after..*/ 1304 cc_proc_cipher(req, desc, seq_size, data_flow_mode); 1305 /* read the digest result with setting the completion bit 1306 * must be after the cipher operation 1307 */ 1308 cc_proc_digest_desc(req, desc, seq_size); 1309 } 1310 } 1311 1312 static int validate_data_size(struct cc_aead_ctx *ctx, 1313 enum drv_crypto_direction direct, 1314 struct aead_request *req) 1315 { 1316 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1317 struct device *dev = drvdata_to_dev(ctx->drvdata); 1318 unsigned int assoclen = areq_ctx->assoclen; 1319 unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ? 1320 (req->cryptlen - ctx->authsize) : req->cryptlen; 1321 1322 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT && 1323 req->cryptlen < ctx->authsize) 1324 goto data_size_err; 1325 1326 areq_ctx->is_single_pass = true; /*defaulted to fast flow*/ 1327 1328 switch (ctx->flow_mode) { 1329 case S_DIN_to_AES: 1330 if (ctx->cipher_mode == DRV_CIPHER_CBC && 1331 !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)) 1332 goto data_size_err; 1333 if (ctx->cipher_mode == DRV_CIPHER_CCM) 1334 break; 1335 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1336 if (areq_ctx->plaintext_authenticate_only) 1337 areq_ctx->is_single_pass = false; 1338 break; 1339 } 1340 1341 if (!IS_ALIGNED(assoclen, sizeof(u32))) 1342 areq_ctx->is_single_pass = false; 1343 1344 if (ctx->cipher_mode == DRV_CIPHER_CTR && 1345 !IS_ALIGNED(cipherlen, sizeof(u32))) 1346 areq_ctx->is_single_pass = false; 1347 1348 break; 1349 case S_DIN_to_DES: 1350 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE)) 1351 goto data_size_err; 1352 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE)) 1353 areq_ctx->is_single_pass = false; 1354 break; 1355 default: 1356 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode); 1357 goto data_size_err; 1358 } 1359 1360 return 0; 1361 1362 data_size_err: 1363 return -EINVAL; 1364 } 1365 1366 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size) 1367 { 1368 unsigned int len = 0; 1369 1370 if (header_size == 0) 1371 return 0; 1372 1373 if (header_size < ((1UL << 16) - (1UL << 8))) { 1374 len = 2; 1375 1376 pa0_buff[0] = (header_size >> 8) & 0xFF; 1377 pa0_buff[1] = header_size & 0xFF; 1378 } else { 1379 len = 6; 1380 1381 pa0_buff[0] = 0xFF; 1382 pa0_buff[1] = 0xFE; 1383 pa0_buff[2] = (header_size >> 24) & 0xFF; 1384 pa0_buff[3] = (header_size >> 16) & 0xFF; 1385 pa0_buff[4] = (header_size >> 8) & 0xFF; 1386 pa0_buff[5] = header_size & 0xFF; 1387 } 1388 1389 return len; 1390 } 1391 1392 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize) 1393 { 1394 __be32 data; 1395 1396 memset(block, 0, csize); 1397 block += csize; 1398 1399 if (csize >= 4) 1400 csize = 4; 1401 else if (msglen > (1 << (8 * csize))) 1402 return -EOVERFLOW; 1403 1404 data = cpu_to_be32(msglen); 1405 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 1406 1407 return 0; 1408 } 1409 1410 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[], 1411 unsigned int *seq_size) 1412 { 1413 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1414 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1415 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1416 unsigned int idx = *seq_size; 1417 unsigned int cipher_flow_mode; 1418 dma_addr_t mac_result; 1419 1420 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1421 cipher_flow_mode = AES_to_HASH_and_DOUT; 1422 mac_result = req_ctx->mac_buf_dma_addr; 1423 } else { /* Encrypt */ 1424 cipher_flow_mode = AES_and_HASH; 1425 mac_result = req_ctx->icv_dma_addr; 1426 } 1427 1428 /* load key */ 1429 hw_desc_init(&desc[idx]); 1430 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1431 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1432 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1433 ctx->enc_keylen), NS_BIT); 1434 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1435 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1436 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1437 set_flow_mode(&desc[idx], S_DIN_to_AES); 1438 idx++; 1439 1440 /* load ctr state */ 1441 hw_desc_init(&desc[idx]); 1442 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1443 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1444 set_din_type(&desc[idx], DMA_DLLI, 1445 req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT); 1446 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1447 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1448 set_flow_mode(&desc[idx], S_DIN_to_AES); 1449 idx++; 1450 1451 /* load MAC key */ 1452 hw_desc_init(&desc[idx]); 1453 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1454 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1455 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX : 1456 ctx->enc_keylen), NS_BIT); 1457 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1458 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1459 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1460 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1461 set_aes_not_hash_mode(&desc[idx]); 1462 idx++; 1463 1464 /* load MAC state */ 1465 hw_desc_init(&desc[idx]); 1466 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1467 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1468 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1469 AES_BLOCK_SIZE, NS_BIT); 1470 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT); 1471 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1472 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1473 set_aes_not_hash_mode(&desc[idx]); 1474 idx++; 1475 1476 /* process assoc data */ 1477 if (req_ctx->assoclen > 0) { 1478 cc_set_assoc_desc(req, DIN_HASH, desc, &idx); 1479 } else { 1480 hw_desc_init(&desc[idx]); 1481 set_din_type(&desc[idx], DMA_DLLI, 1482 sg_dma_address(&req_ctx->ccm_adata_sg), 1483 AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT); 1484 set_flow_mode(&desc[idx], DIN_HASH); 1485 idx++; 1486 } 1487 1488 /* process the cipher */ 1489 if (req_ctx->cryptlen) 1490 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx); 1491 1492 /* Read temporal MAC */ 1493 hw_desc_init(&desc[idx]); 1494 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC); 1495 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize, 1496 NS_BIT, 0); 1497 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1498 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN); 1499 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1500 set_aes_not_hash_mode(&desc[idx]); 1501 idx++; 1502 1503 /* load AES-CTR state (for last MAC calculation)*/ 1504 hw_desc_init(&desc[idx]); 1505 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR); 1506 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1507 set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr, 1508 AES_BLOCK_SIZE, NS_BIT); 1509 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1510 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1511 set_flow_mode(&desc[idx], S_DIN_to_AES); 1512 idx++; 1513 1514 hw_desc_init(&desc[idx]); 1515 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1516 set_dout_no_dma(&desc[idx], 0, 0, 1); 1517 idx++; 1518 1519 /* encrypt the "T" value and store MAC in mac_state */ 1520 hw_desc_init(&desc[idx]); 1521 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1522 ctx->authsize, NS_BIT); 1523 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1524 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1525 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1526 idx++; 1527 1528 *seq_size = idx; 1529 return 0; 1530 } 1531 1532 static int config_ccm_adata(struct aead_request *req) 1533 { 1534 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1535 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1536 struct device *dev = drvdata_to_dev(ctx->drvdata); 1537 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1538 //unsigned int size_of_a = 0, rem_a_size = 0; 1539 unsigned int lp = req->iv[0]; 1540 /* Note: The code assume that req->iv[0] already contains the value 1541 * of L' of RFC3610 1542 */ 1543 unsigned int l = lp + 1; /* This is L' of RFC 3610. */ 1544 unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */ 1545 u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET; 1546 u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET; 1547 u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; 1548 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1549 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1550 req->cryptlen : 1551 (req->cryptlen - ctx->authsize); 1552 int rc; 1553 1554 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1555 memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3); 1556 1557 /* taken from crypto/ccm.c */ 1558 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1559 if (l < 2 || l > 8) { 1560 dev_dbg(dev, "illegal iv value %X\n", req->iv[0]); 1561 return -EINVAL; 1562 } 1563 memcpy(b0, req->iv, AES_BLOCK_SIZE); 1564 1565 /* format control info per RFC 3610 and 1566 * NIST Special Publication 800-38C 1567 */ 1568 *b0 |= (8 * ((m - 2) / 2)); 1569 if (req_ctx->assoclen > 0) 1570 *b0 |= 64; /* Enable bit 6 if Adata exists. */ 1571 1572 rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */ 1573 if (rc) { 1574 dev_err(dev, "message len overflow detected"); 1575 return rc; 1576 } 1577 /* END of "taken from crypto/ccm.c" */ 1578 1579 /* l(a) - size of associated data. */ 1580 req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen); 1581 1582 memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1); 1583 req->iv[15] = 1; 1584 1585 memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE); 1586 ctr_count_0[15] = 0; 1587 1588 return 0; 1589 } 1590 1591 static void cc_proc_rfc4309_ccm(struct aead_request *req) 1592 { 1593 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1594 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1595 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1596 1597 /* L' */ 1598 memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE); 1599 /* For RFC 4309, always use 4 bytes for message length 1600 * (at most 2^32-1 bytes). 1601 */ 1602 areq_ctx->ctr_iv[0] = 3; 1603 1604 /* In RFC 4309 there is an 11-bytes nonce+IV part, 1605 * that we build here. 1606 */ 1607 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce, 1608 CCM_BLOCK_NONCE_SIZE); 1609 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv, 1610 CCM_BLOCK_IV_SIZE); 1611 req->iv = areq_ctx->ctr_iv; 1612 } 1613 1614 static void cc_set_ghash_desc(struct aead_request *req, 1615 struct cc_hw_desc desc[], unsigned int *seq_size) 1616 { 1617 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1618 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1619 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1620 unsigned int idx = *seq_size; 1621 1622 /* load key to AES*/ 1623 hw_desc_init(&desc[idx]); 1624 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB); 1625 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1626 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1627 ctx->enc_keylen, NS_BIT); 1628 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1629 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1630 set_flow_mode(&desc[idx], S_DIN_to_AES); 1631 idx++; 1632 1633 /* process one zero block to generate hkey */ 1634 hw_desc_init(&desc[idx]); 1635 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1636 set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE, 1637 NS_BIT, 0); 1638 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1639 idx++; 1640 1641 /* Memory Barrier */ 1642 hw_desc_init(&desc[idx]); 1643 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1644 set_dout_no_dma(&desc[idx], 0, 0, 1); 1645 idx++; 1646 1647 /* Load GHASH subkey */ 1648 hw_desc_init(&desc[idx]); 1649 set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr, 1650 AES_BLOCK_SIZE, NS_BIT); 1651 set_dout_no_dma(&desc[idx], 0, 0, 1); 1652 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1653 set_aes_not_hash_mode(&desc[idx]); 1654 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1655 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1656 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1657 idx++; 1658 1659 /* Configure Hash Engine to work with GHASH. 1660 * Since it was not possible to extend HASH submodes to add GHASH, 1661 * The following command is necessary in order to 1662 * select GHASH (according to HW designers) 1663 */ 1664 hw_desc_init(&desc[idx]); 1665 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1666 set_dout_no_dma(&desc[idx], 0, 0, 1); 1667 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1668 set_aes_not_hash_mode(&desc[idx]); 1669 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1670 set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK 1671 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1672 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1673 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1674 idx++; 1675 1676 /* Load GHASH initial STATE (which is 0). (for any hash there is an 1677 * initial state) 1678 */ 1679 hw_desc_init(&desc[idx]); 1680 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE); 1681 set_dout_no_dma(&desc[idx], 0, 0, 1); 1682 set_flow_mode(&desc[idx], S_DIN_to_HASH); 1683 set_aes_not_hash_mode(&desc[idx]); 1684 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1685 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED); 1686 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0); 1687 idx++; 1688 1689 *seq_size = idx; 1690 } 1691 1692 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[], 1693 unsigned int *seq_size) 1694 { 1695 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1696 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1697 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1698 unsigned int idx = *seq_size; 1699 1700 /* load key to AES*/ 1701 hw_desc_init(&desc[idx]); 1702 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1703 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1704 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr, 1705 ctx->enc_keylen, NS_BIT); 1706 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1707 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0); 1708 set_flow_mode(&desc[idx], S_DIN_to_AES); 1709 idx++; 1710 1711 if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) { 1712 /* load AES/CTR initial CTR value inc by 2*/ 1713 hw_desc_init(&desc[idx]); 1714 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1715 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1716 set_din_type(&desc[idx], DMA_DLLI, 1717 req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE, 1718 NS_BIT); 1719 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1720 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1721 set_flow_mode(&desc[idx], S_DIN_to_AES); 1722 idx++; 1723 } 1724 1725 *seq_size = idx; 1726 } 1727 1728 static void cc_proc_gcm_result(struct aead_request *req, 1729 struct cc_hw_desc desc[], 1730 unsigned int *seq_size) 1731 { 1732 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1733 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1734 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1735 dma_addr_t mac_result; 1736 unsigned int idx = *seq_size; 1737 1738 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1739 mac_result = req_ctx->mac_buf_dma_addr; 1740 } else { /* Encrypt */ 1741 mac_result = req_ctx->icv_dma_addr; 1742 } 1743 1744 /* process(ghash) gcm_block_len */ 1745 hw_desc_init(&desc[idx]); 1746 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr, 1747 AES_BLOCK_SIZE, NS_BIT); 1748 set_flow_mode(&desc[idx], DIN_HASH); 1749 idx++; 1750 1751 /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */ 1752 hw_desc_init(&desc[idx]); 1753 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH); 1754 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1755 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE, 1756 NS_BIT, 0); 1757 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0); 1758 set_flow_mode(&desc[idx], S_HASH_to_DOUT); 1759 set_aes_not_hash_mode(&desc[idx]); 1760 1761 idx++; 1762 1763 /* load AES/CTR initial CTR value inc by 1*/ 1764 hw_desc_init(&desc[idx]); 1765 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1766 set_key_size_aes(&desc[idx], ctx->enc_keylen); 1767 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr, 1768 AES_BLOCK_SIZE, NS_BIT); 1769 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT); 1770 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1); 1771 set_flow_mode(&desc[idx], S_DIN_to_AES); 1772 idx++; 1773 1774 /* Memory Barrier */ 1775 hw_desc_init(&desc[idx]); 1776 set_din_no_dma(&desc[idx], 0, 0xfffff0); 1777 set_dout_no_dma(&desc[idx], 0, 0, 1); 1778 idx++; 1779 1780 /* process GCTR on stored GHASH and store MAC in mac_state*/ 1781 hw_desc_init(&desc[idx]); 1782 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR); 1783 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr, 1784 AES_BLOCK_SIZE, NS_BIT); 1785 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1); 1786 set_queue_last_ind(ctx->drvdata, &desc[idx]); 1787 set_flow_mode(&desc[idx], DIN_AES_DOUT); 1788 idx++; 1789 1790 *seq_size = idx; 1791 } 1792 1793 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[], 1794 unsigned int *seq_size) 1795 { 1796 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1797 unsigned int cipher_flow_mode; 1798 1799 //in RFC4543 no data to encrypt. just copy data from src to dest. 1800 if (req_ctx->plaintext_authenticate_only) { 1801 cc_proc_cipher_desc(req, BYPASS, desc, seq_size); 1802 cc_set_ghash_desc(req, desc, seq_size); 1803 /* process(ghash) assoc data */ 1804 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1805 cc_set_gctr_desc(req, desc, seq_size); 1806 cc_proc_gcm_result(req, desc, seq_size); 1807 return 0; 1808 } 1809 1810 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 1811 cipher_flow_mode = AES_and_HASH; 1812 } else { /* Encrypt */ 1813 cipher_flow_mode = AES_to_HASH_and_DOUT; 1814 } 1815 1816 // for gcm and rfc4106. 1817 cc_set_ghash_desc(req, desc, seq_size); 1818 /* process(ghash) assoc data */ 1819 if (req_ctx->assoclen > 0) 1820 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size); 1821 cc_set_gctr_desc(req, desc, seq_size); 1822 /* process(gctr+ghash) */ 1823 if (req_ctx->cryptlen) 1824 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size); 1825 cc_proc_gcm_result(req, desc, seq_size); 1826 1827 return 0; 1828 } 1829 1830 static int config_gcm_context(struct aead_request *req) 1831 { 1832 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1833 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1834 struct aead_req_ctx *req_ctx = aead_request_ctx_dma(req); 1835 struct device *dev = drvdata_to_dev(ctx->drvdata); 1836 1837 unsigned int cryptlen = (req_ctx->gen_ctx.op_type == 1838 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 1839 req->cryptlen : 1840 (req->cryptlen - ctx->authsize); 1841 __be32 counter = cpu_to_be32(2); 1842 1843 dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n", 1844 __func__, cryptlen, req_ctx->assoclen, ctx->authsize); 1845 1846 memset(req_ctx->hkey, 0, AES_BLOCK_SIZE); 1847 1848 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE); 1849 1850 memcpy(req->iv + 12, &counter, 4); 1851 memcpy(req_ctx->gcm_iv_inc2, req->iv, 16); 1852 1853 counter = cpu_to_be32(1); 1854 memcpy(req->iv + 12, &counter, 4); 1855 memcpy(req_ctx->gcm_iv_inc1, req->iv, 16); 1856 1857 if (!req_ctx->plaintext_authenticate_only) { 1858 __be64 temp64; 1859 1860 temp64 = cpu_to_be64(req_ctx->assoclen * 8); 1861 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1862 temp64 = cpu_to_be64(cryptlen * 8); 1863 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1864 } else { 1865 /* rfc4543=> all data(AAD,IV,Plain) are considered additional 1866 * data that is nothing is encrypted. 1867 */ 1868 __be64 temp64; 1869 1870 temp64 = cpu_to_be64((req_ctx->assoclen + cryptlen) * 8); 1871 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64)); 1872 temp64 = 0; 1873 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8); 1874 } 1875 1876 return 0; 1877 } 1878 1879 static void cc_proc_rfc4_gcm(struct aead_request *req) 1880 { 1881 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1882 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1883 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1884 1885 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET, 1886 ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE); 1887 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv, 1888 GCM_BLOCK_RFC4_IV_SIZE); 1889 req->iv = areq_ctx->ctr_iv; 1890 } 1891 1892 static int cc_proc_aead(struct aead_request *req, 1893 enum drv_crypto_direction direct) 1894 { 1895 int rc = 0; 1896 int seq_len = 0; 1897 struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ]; 1898 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1899 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 1900 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 1901 struct device *dev = drvdata_to_dev(ctx->drvdata); 1902 struct cc_crypto_req cc_req = {}; 1903 1904 dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n", 1905 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"), 1906 ctx, req, req->iv, sg_virt(req->src), req->src->offset, 1907 sg_virt(req->dst), req->dst->offset, req->cryptlen); 1908 1909 /* STAT_PHASE_0: Init and sanity checks */ 1910 1911 /* Check data length according to mode */ 1912 if (validate_data_size(ctx, direct, req)) { 1913 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n", 1914 req->cryptlen, areq_ctx->assoclen); 1915 return -EINVAL; 1916 } 1917 1918 /* Setup request structure */ 1919 cc_req.user_cb = cc_aead_complete; 1920 cc_req.user_arg = req; 1921 1922 /* Setup request context */ 1923 areq_ctx->gen_ctx.op_type = direct; 1924 areq_ctx->req_authsize = ctx->authsize; 1925 areq_ctx->cipher_mode = ctx->cipher_mode; 1926 1927 /* STAT_PHASE_1: Map buffers */ 1928 1929 if (ctx->cipher_mode == DRV_CIPHER_CTR) { 1930 /* Build CTR IV - Copy nonce from last 4 bytes in 1931 * CTR key to first 4 bytes in CTR IV 1932 */ 1933 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce, 1934 CTR_RFC3686_NONCE_SIZE); 1935 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv, 1936 CTR_RFC3686_IV_SIZE); 1937 /* Initialize counter portion of counter block */ 1938 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE + 1939 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1940 1941 /* Replace with counter iv */ 1942 req->iv = areq_ctx->ctr_iv; 1943 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE; 1944 } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) || 1945 (ctx->cipher_mode == DRV_CIPHER_GCTR)) { 1946 areq_ctx->hw_iv_size = AES_BLOCK_SIZE; 1947 if (areq_ctx->ctr_iv != req->iv) { 1948 memcpy(areq_ctx->ctr_iv, req->iv, 1949 crypto_aead_ivsize(tfm)); 1950 req->iv = areq_ctx->ctr_iv; 1951 } 1952 } else { 1953 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm); 1954 } 1955 1956 if (ctx->cipher_mode == DRV_CIPHER_CCM) { 1957 rc = config_ccm_adata(req); 1958 if (rc) { 1959 dev_dbg(dev, "config_ccm_adata() returned with a failure %d!", 1960 rc); 1961 goto exit; 1962 } 1963 } else { 1964 areq_ctx->ccm_hdr_size = ccm_header_size_null; 1965 } 1966 1967 if (ctx->cipher_mode == DRV_CIPHER_GCTR) { 1968 rc = config_gcm_context(req); 1969 if (rc) { 1970 dev_dbg(dev, "config_gcm_context() returned with a failure %d!", 1971 rc); 1972 goto exit; 1973 } 1974 } 1975 1976 rc = cc_map_aead_request(ctx->drvdata, req); 1977 if (rc) { 1978 dev_err(dev, "map_request() failed\n"); 1979 goto exit; 1980 } 1981 1982 /* STAT_PHASE_2: Create sequence */ 1983 1984 /* Load MLLI tables to SRAM if necessary */ 1985 cc_mlli_to_sram(req, desc, &seq_len); 1986 1987 switch (ctx->auth_mode) { 1988 case DRV_HASH_SHA1: 1989 case DRV_HASH_SHA256: 1990 cc_hmac_authenc(req, desc, &seq_len); 1991 break; 1992 case DRV_HASH_XCBC_MAC: 1993 cc_xcbc_authenc(req, desc, &seq_len); 1994 break; 1995 case DRV_HASH_NULL: 1996 if (ctx->cipher_mode == DRV_CIPHER_CCM) 1997 cc_ccm(req, desc, &seq_len); 1998 if (ctx->cipher_mode == DRV_CIPHER_GCTR) 1999 cc_gcm(req, desc, &seq_len); 2000 break; 2001 default: 2002 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode); 2003 cc_unmap_aead_request(dev, req); 2004 rc = -ENOTSUPP; 2005 goto exit; 2006 } 2007 2008 /* STAT_PHASE_3: Lock HW and push sequence */ 2009 2010 rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base); 2011 2012 if (rc != -EINPROGRESS && rc != -EBUSY) { 2013 dev_err(dev, "send_request() failed (rc=%d)\n", rc); 2014 cc_unmap_aead_request(dev, req); 2015 } 2016 2017 exit: 2018 return rc; 2019 } 2020 2021 static int cc_aead_encrypt(struct aead_request *req) 2022 { 2023 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2024 int rc; 2025 2026 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2027 2028 /* No generated IV required */ 2029 areq_ctx->backup_iv = req->iv; 2030 areq_ctx->assoclen = req->assoclen; 2031 2032 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2033 if (rc != -EINPROGRESS && rc != -EBUSY) 2034 req->iv = areq_ctx->backup_iv; 2035 2036 return rc; 2037 } 2038 2039 static int cc_rfc4309_ccm_encrypt(struct aead_request *req) 2040 { 2041 /* Very similar to cc_aead_encrypt() above. */ 2042 2043 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2044 int rc; 2045 2046 rc = crypto_ipsec_check_assoclen(req->assoclen); 2047 if (rc) 2048 goto out; 2049 2050 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2051 2052 /* No generated IV required */ 2053 areq_ctx->backup_iv = req->iv; 2054 areq_ctx->assoclen = req->assoclen - CCM_BLOCK_IV_SIZE; 2055 2056 cc_proc_rfc4309_ccm(req); 2057 2058 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2059 if (rc != -EINPROGRESS && rc != -EBUSY) 2060 req->iv = areq_ctx->backup_iv; 2061 out: 2062 return rc; 2063 } 2064 2065 static int cc_aead_decrypt(struct aead_request *req) 2066 { 2067 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2068 int rc; 2069 2070 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2071 2072 /* No generated IV required */ 2073 areq_ctx->backup_iv = req->iv; 2074 areq_ctx->assoclen = req->assoclen; 2075 2076 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2077 if (rc != -EINPROGRESS && rc != -EBUSY) 2078 req->iv = areq_ctx->backup_iv; 2079 2080 return rc; 2081 } 2082 2083 static int cc_rfc4309_ccm_decrypt(struct aead_request *req) 2084 { 2085 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2086 int rc; 2087 2088 rc = crypto_ipsec_check_assoclen(req->assoclen); 2089 if (rc) 2090 goto out; 2091 2092 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2093 2094 /* No generated IV required */ 2095 areq_ctx->backup_iv = req->iv; 2096 areq_ctx->assoclen = req->assoclen - CCM_BLOCK_IV_SIZE; 2097 2098 cc_proc_rfc4309_ccm(req); 2099 2100 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2101 if (rc != -EINPROGRESS && rc != -EBUSY) 2102 req->iv = areq_ctx->backup_iv; 2103 2104 out: 2105 return rc; 2106 } 2107 2108 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2109 unsigned int keylen) 2110 { 2111 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2112 struct device *dev = drvdata_to_dev(ctx->drvdata); 2113 2114 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2115 2116 if (keylen < 4) 2117 return -EINVAL; 2118 2119 keylen -= 4; 2120 memcpy(ctx->ctr_nonce, key + keylen, 4); 2121 2122 return cc_aead_setkey(tfm, key, keylen); 2123 } 2124 2125 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 2126 unsigned int keylen) 2127 { 2128 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm); 2129 struct device *dev = drvdata_to_dev(ctx->drvdata); 2130 2131 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key); 2132 2133 if (keylen < 4) 2134 return -EINVAL; 2135 2136 keylen -= 4; 2137 memcpy(ctx->ctr_nonce, key + keylen, 4); 2138 2139 return cc_aead_setkey(tfm, key, keylen); 2140 } 2141 2142 static int cc_gcm_setauthsize(struct crypto_aead *authenc, 2143 unsigned int authsize) 2144 { 2145 switch (authsize) { 2146 case 4: 2147 case 8: 2148 case 12: 2149 case 13: 2150 case 14: 2151 case 15: 2152 case 16: 2153 break; 2154 default: 2155 return -EINVAL; 2156 } 2157 2158 return cc_aead_setauthsize(authenc, authsize); 2159 } 2160 2161 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc, 2162 unsigned int authsize) 2163 { 2164 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2165 struct device *dev = drvdata_to_dev(ctx->drvdata); 2166 2167 dev_dbg(dev, "authsize %d\n", authsize); 2168 2169 switch (authsize) { 2170 case 8: 2171 case 12: 2172 case 16: 2173 break; 2174 default: 2175 return -EINVAL; 2176 } 2177 2178 return cc_aead_setauthsize(authenc, authsize); 2179 } 2180 2181 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc, 2182 unsigned int authsize) 2183 { 2184 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc); 2185 struct device *dev = drvdata_to_dev(ctx->drvdata); 2186 2187 dev_dbg(dev, "authsize %d\n", authsize); 2188 2189 if (authsize != 16) 2190 return -EINVAL; 2191 2192 return cc_aead_setauthsize(authenc, authsize); 2193 } 2194 2195 static int cc_rfc4106_gcm_encrypt(struct aead_request *req) 2196 { 2197 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2198 int rc; 2199 2200 rc = crypto_ipsec_check_assoclen(req->assoclen); 2201 if (rc) 2202 goto out; 2203 2204 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2205 2206 /* No generated IV required */ 2207 areq_ctx->backup_iv = req->iv; 2208 areq_ctx->assoclen = req->assoclen - GCM_BLOCK_RFC4_IV_SIZE; 2209 2210 cc_proc_rfc4_gcm(req); 2211 2212 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2213 if (rc != -EINPROGRESS && rc != -EBUSY) 2214 req->iv = areq_ctx->backup_iv; 2215 out: 2216 return rc; 2217 } 2218 2219 static int cc_rfc4543_gcm_encrypt(struct aead_request *req) 2220 { 2221 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2222 int rc; 2223 2224 rc = crypto_ipsec_check_assoclen(req->assoclen); 2225 if (rc) 2226 goto out; 2227 2228 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2229 2230 //plaintext is not encrypted with rfc4543 2231 areq_ctx->plaintext_authenticate_only = true; 2232 2233 /* No generated IV required */ 2234 areq_ctx->backup_iv = req->iv; 2235 areq_ctx->assoclen = req->assoclen; 2236 2237 cc_proc_rfc4_gcm(req); 2238 2239 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 2240 if (rc != -EINPROGRESS && rc != -EBUSY) 2241 req->iv = areq_ctx->backup_iv; 2242 out: 2243 return rc; 2244 } 2245 2246 static int cc_rfc4106_gcm_decrypt(struct aead_request *req) 2247 { 2248 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2249 int rc; 2250 2251 rc = crypto_ipsec_check_assoclen(req->assoclen); 2252 if (rc) 2253 goto out; 2254 2255 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2256 2257 /* No generated IV required */ 2258 areq_ctx->backup_iv = req->iv; 2259 areq_ctx->assoclen = req->assoclen - GCM_BLOCK_RFC4_IV_SIZE; 2260 2261 cc_proc_rfc4_gcm(req); 2262 2263 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2264 if (rc != -EINPROGRESS && rc != -EBUSY) 2265 req->iv = areq_ctx->backup_iv; 2266 out: 2267 return rc; 2268 } 2269 2270 static int cc_rfc4543_gcm_decrypt(struct aead_request *req) 2271 { 2272 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 2273 int rc; 2274 2275 rc = crypto_ipsec_check_assoclen(req->assoclen); 2276 if (rc) 2277 goto out; 2278 2279 memset(areq_ctx, 0, sizeof(*areq_ctx)); 2280 2281 //plaintext is not decrypted with rfc4543 2282 areq_ctx->plaintext_authenticate_only = true; 2283 2284 /* No generated IV required */ 2285 areq_ctx->backup_iv = req->iv; 2286 areq_ctx->assoclen = req->assoclen; 2287 2288 cc_proc_rfc4_gcm(req); 2289 2290 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT); 2291 if (rc != -EINPROGRESS && rc != -EBUSY) 2292 req->iv = areq_ctx->backup_iv; 2293 out: 2294 return rc; 2295 } 2296 2297 /* aead alg */ 2298 static struct cc_alg_template aead_algs[] = { 2299 { 2300 .name = "authenc(hmac(sha1),cbc(aes))", 2301 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree", 2302 .blocksize = AES_BLOCK_SIZE, 2303 .template_aead = { 2304 .setkey = cc_aead_setkey, 2305 .setauthsize = cc_aead_setauthsize, 2306 .encrypt = cc_aead_encrypt, 2307 .decrypt = cc_aead_decrypt, 2308 .init = cc_aead_init, 2309 .exit = cc_aead_exit, 2310 .ivsize = AES_BLOCK_SIZE, 2311 .maxauthsize = SHA1_DIGEST_SIZE, 2312 }, 2313 .cipher_mode = DRV_CIPHER_CBC, 2314 .flow_mode = S_DIN_to_AES, 2315 .auth_mode = DRV_HASH_SHA1, 2316 .min_hw_rev = CC_HW_REV_630, 2317 .std_body = CC_STD_NIST, 2318 }, 2319 { 2320 .name = "authenc(hmac(sha1),cbc(des3_ede))", 2321 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree", 2322 .blocksize = DES3_EDE_BLOCK_SIZE, 2323 .template_aead = { 2324 .setkey = cc_des3_aead_setkey, 2325 .setauthsize = cc_aead_setauthsize, 2326 .encrypt = cc_aead_encrypt, 2327 .decrypt = cc_aead_decrypt, 2328 .init = cc_aead_init, 2329 .exit = cc_aead_exit, 2330 .ivsize = DES3_EDE_BLOCK_SIZE, 2331 .maxauthsize = SHA1_DIGEST_SIZE, 2332 }, 2333 .cipher_mode = DRV_CIPHER_CBC, 2334 .flow_mode = S_DIN_to_DES, 2335 .auth_mode = DRV_HASH_SHA1, 2336 .min_hw_rev = CC_HW_REV_630, 2337 .std_body = CC_STD_NIST, 2338 }, 2339 { 2340 .name = "authenc(hmac(sha256),cbc(aes))", 2341 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree", 2342 .blocksize = AES_BLOCK_SIZE, 2343 .template_aead = { 2344 .setkey = cc_aead_setkey, 2345 .setauthsize = cc_aead_setauthsize, 2346 .encrypt = cc_aead_encrypt, 2347 .decrypt = cc_aead_decrypt, 2348 .init = cc_aead_init, 2349 .exit = cc_aead_exit, 2350 .ivsize = AES_BLOCK_SIZE, 2351 .maxauthsize = SHA256_DIGEST_SIZE, 2352 }, 2353 .cipher_mode = DRV_CIPHER_CBC, 2354 .flow_mode = S_DIN_to_AES, 2355 .auth_mode = DRV_HASH_SHA256, 2356 .min_hw_rev = CC_HW_REV_630, 2357 .std_body = CC_STD_NIST, 2358 }, 2359 { 2360 .name = "authenc(hmac(sha256),cbc(des3_ede))", 2361 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree", 2362 .blocksize = DES3_EDE_BLOCK_SIZE, 2363 .template_aead = { 2364 .setkey = cc_des3_aead_setkey, 2365 .setauthsize = cc_aead_setauthsize, 2366 .encrypt = cc_aead_encrypt, 2367 .decrypt = cc_aead_decrypt, 2368 .init = cc_aead_init, 2369 .exit = cc_aead_exit, 2370 .ivsize = DES3_EDE_BLOCK_SIZE, 2371 .maxauthsize = SHA256_DIGEST_SIZE, 2372 }, 2373 .cipher_mode = DRV_CIPHER_CBC, 2374 .flow_mode = S_DIN_to_DES, 2375 .auth_mode = DRV_HASH_SHA256, 2376 .min_hw_rev = CC_HW_REV_630, 2377 .std_body = CC_STD_NIST, 2378 }, 2379 { 2380 .name = "authenc(xcbc(aes),cbc(aes))", 2381 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree", 2382 .blocksize = AES_BLOCK_SIZE, 2383 .template_aead = { 2384 .setkey = cc_aead_setkey, 2385 .setauthsize = cc_aead_setauthsize, 2386 .encrypt = cc_aead_encrypt, 2387 .decrypt = cc_aead_decrypt, 2388 .init = cc_aead_init, 2389 .exit = cc_aead_exit, 2390 .ivsize = AES_BLOCK_SIZE, 2391 .maxauthsize = AES_BLOCK_SIZE, 2392 }, 2393 .cipher_mode = DRV_CIPHER_CBC, 2394 .flow_mode = S_DIN_to_AES, 2395 .auth_mode = DRV_HASH_XCBC_MAC, 2396 .min_hw_rev = CC_HW_REV_630, 2397 .std_body = CC_STD_NIST, 2398 }, 2399 { 2400 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2401 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree", 2402 .blocksize = 1, 2403 .template_aead = { 2404 .setkey = cc_aead_setkey, 2405 .setauthsize = cc_aead_setauthsize, 2406 .encrypt = cc_aead_encrypt, 2407 .decrypt = cc_aead_decrypt, 2408 .init = cc_aead_init, 2409 .exit = cc_aead_exit, 2410 .ivsize = CTR_RFC3686_IV_SIZE, 2411 .maxauthsize = SHA1_DIGEST_SIZE, 2412 }, 2413 .cipher_mode = DRV_CIPHER_CTR, 2414 .flow_mode = S_DIN_to_AES, 2415 .auth_mode = DRV_HASH_SHA1, 2416 .min_hw_rev = CC_HW_REV_630, 2417 .std_body = CC_STD_NIST, 2418 }, 2419 { 2420 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2421 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree", 2422 .blocksize = 1, 2423 .template_aead = { 2424 .setkey = cc_aead_setkey, 2425 .setauthsize = cc_aead_setauthsize, 2426 .encrypt = cc_aead_encrypt, 2427 .decrypt = cc_aead_decrypt, 2428 .init = cc_aead_init, 2429 .exit = cc_aead_exit, 2430 .ivsize = CTR_RFC3686_IV_SIZE, 2431 .maxauthsize = SHA256_DIGEST_SIZE, 2432 }, 2433 .cipher_mode = DRV_CIPHER_CTR, 2434 .flow_mode = S_DIN_to_AES, 2435 .auth_mode = DRV_HASH_SHA256, 2436 .min_hw_rev = CC_HW_REV_630, 2437 .std_body = CC_STD_NIST, 2438 }, 2439 { 2440 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))", 2441 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree", 2442 .blocksize = 1, 2443 .template_aead = { 2444 .setkey = cc_aead_setkey, 2445 .setauthsize = cc_aead_setauthsize, 2446 .encrypt = cc_aead_encrypt, 2447 .decrypt = cc_aead_decrypt, 2448 .init = cc_aead_init, 2449 .exit = cc_aead_exit, 2450 .ivsize = CTR_RFC3686_IV_SIZE, 2451 .maxauthsize = AES_BLOCK_SIZE, 2452 }, 2453 .cipher_mode = DRV_CIPHER_CTR, 2454 .flow_mode = S_DIN_to_AES, 2455 .auth_mode = DRV_HASH_XCBC_MAC, 2456 .min_hw_rev = CC_HW_REV_630, 2457 .std_body = CC_STD_NIST, 2458 }, 2459 { 2460 .name = "ccm(aes)", 2461 .driver_name = "ccm-aes-ccree", 2462 .blocksize = 1, 2463 .template_aead = { 2464 .setkey = cc_aead_setkey, 2465 .setauthsize = cc_ccm_setauthsize, 2466 .encrypt = cc_aead_encrypt, 2467 .decrypt = cc_aead_decrypt, 2468 .init = cc_aead_init, 2469 .exit = cc_aead_exit, 2470 .ivsize = AES_BLOCK_SIZE, 2471 .maxauthsize = AES_BLOCK_SIZE, 2472 }, 2473 .cipher_mode = DRV_CIPHER_CCM, 2474 .flow_mode = S_DIN_to_AES, 2475 .auth_mode = DRV_HASH_NULL, 2476 .min_hw_rev = CC_HW_REV_630, 2477 .std_body = CC_STD_NIST, 2478 }, 2479 { 2480 .name = "rfc4309(ccm(aes))", 2481 .driver_name = "rfc4309-ccm-aes-ccree", 2482 .blocksize = 1, 2483 .template_aead = { 2484 .setkey = cc_rfc4309_ccm_setkey, 2485 .setauthsize = cc_rfc4309_ccm_setauthsize, 2486 .encrypt = cc_rfc4309_ccm_encrypt, 2487 .decrypt = cc_rfc4309_ccm_decrypt, 2488 .init = cc_aead_init, 2489 .exit = cc_aead_exit, 2490 .ivsize = CCM_BLOCK_IV_SIZE, 2491 .maxauthsize = AES_BLOCK_SIZE, 2492 }, 2493 .cipher_mode = DRV_CIPHER_CCM, 2494 .flow_mode = S_DIN_to_AES, 2495 .auth_mode = DRV_HASH_NULL, 2496 .min_hw_rev = CC_HW_REV_630, 2497 .std_body = CC_STD_NIST, 2498 }, 2499 { 2500 .name = "gcm(aes)", 2501 .driver_name = "gcm-aes-ccree", 2502 .blocksize = 1, 2503 .template_aead = { 2504 .setkey = cc_aead_setkey, 2505 .setauthsize = cc_gcm_setauthsize, 2506 .encrypt = cc_aead_encrypt, 2507 .decrypt = cc_aead_decrypt, 2508 .init = cc_aead_init, 2509 .exit = cc_aead_exit, 2510 .ivsize = 12, 2511 .maxauthsize = AES_BLOCK_SIZE, 2512 }, 2513 .cipher_mode = DRV_CIPHER_GCTR, 2514 .flow_mode = S_DIN_to_AES, 2515 .auth_mode = DRV_HASH_NULL, 2516 .min_hw_rev = CC_HW_REV_630, 2517 .std_body = CC_STD_NIST, 2518 }, 2519 { 2520 .name = "rfc4106(gcm(aes))", 2521 .driver_name = "rfc4106-gcm-aes-ccree", 2522 .blocksize = 1, 2523 .template_aead = { 2524 .setkey = cc_rfc4106_gcm_setkey, 2525 .setauthsize = cc_rfc4106_gcm_setauthsize, 2526 .encrypt = cc_rfc4106_gcm_encrypt, 2527 .decrypt = cc_rfc4106_gcm_decrypt, 2528 .init = cc_aead_init, 2529 .exit = cc_aead_exit, 2530 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2531 .maxauthsize = AES_BLOCK_SIZE, 2532 }, 2533 .cipher_mode = DRV_CIPHER_GCTR, 2534 .flow_mode = S_DIN_to_AES, 2535 .auth_mode = DRV_HASH_NULL, 2536 .min_hw_rev = CC_HW_REV_630, 2537 .std_body = CC_STD_NIST, 2538 }, 2539 { 2540 .name = "rfc4543(gcm(aes))", 2541 .driver_name = "rfc4543-gcm-aes-ccree", 2542 .blocksize = 1, 2543 .template_aead = { 2544 .setkey = cc_rfc4543_gcm_setkey, 2545 .setauthsize = cc_rfc4543_gcm_setauthsize, 2546 .encrypt = cc_rfc4543_gcm_encrypt, 2547 .decrypt = cc_rfc4543_gcm_decrypt, 2548 .init = cc_aead_init, 2549 .exit = cc_aead_exit, 2550 .ivsize = GCM_BLOCK_RFC4_IV_SIZE, 2551 .maxauthsize = AES_BLOCK_SIZE, 2552 }, 2553 .cipher_mode = DRV_CIPHER_GCTR, 2554 .flow_mode = S_DIN_to_AES, 2555 .auth_mode = DRV_HASH_NULL, 2556 .min_hw_rev = CC_HW_REV_630, 2557 .std_body = CC_STD_NIST, 2558 }, 2559 }; 2560 2561 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, 2562 struct device *dev) 2563 { 2564 struct cc_crypto_alg *t_alg; 2565 struct aead_alg *alg; 2566 2567 t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL); 2568 if (!t_alg) 2569 return ERR_PTR(-ENOMEM); 2570 2571 alg = &tmpl->template_aead; 2572 2573 if (strscpy(alg->base.cra_name, tmpl->name) < 0) 2574 return ERR_PTR(-EINVAL); 2575 if (strscpy(alg->base.cra_driver_name, tmpl->driver_name) < 0) 2576 return ERR_PTR(-EINVAL); 2577 2578 alg->base.cra_module = THIS_MODULE; 2579 alg->base.cra_priority = CC_CRA_PRIO; 2580 2581 alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx); 2582 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2583 alg->base.cra_blocksize = tmpl->blocksize; 2584 alg->init = cc_aead_init; 2585 alg->exit = cc_aead_exit; 2586 2587 t_alg->aead_alg = *alg; 2588 2589 t_alg->cipher_mode = tmpl->cipher_mode; 2590 t_alg->flow_mode = tmpl->flow_mode; 2591 t_alg->auth_mode = tmpl->auth_mode; 2592 2593 return t_alg; 2594 } 2595 2596 int cc_aead_free(struct cc_drvdata *drvdata) 2597 { 2598 struct cc_crypto_alg *t_alg, *n; 2599 struct cc_aead_handle *aead_handle = drvdata->aead_handle; 2600 2601 /* Remove registered algs */ 2602 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list, entry) { 2603 crypto_unregister_aead(&t_alg->aead_alg); 2604 list_del(&t_alg->entry); 2605 } 2606 2607 return 0; 2608 } 2609 2610 int cc_aead_alloc(struct cc_drvdata *drvdata) 2611 { 2612 struct cc_aead_handle *aead_handle; 2613 struct cc_crypto_alg *t_alg; 2614 int rc = -ENOMEM; 2615 int alg; 2616 struct device *dev = drvdata_to_dev(drvdata); 2617 2618 aead_handle = devm_kmalloc(dev, sizeof(*aead_handle), GFP_KERNEL); 2619 if (!aead_handle) { 2620 rc = -ENOMEM; 2621 goto fail0; 2622 } 2623 2624 INIT_LIST_HEAD(&aead_handle->aead_list); 2625 drvdata->aead_handle = aead_handle; 2626 2627 aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata, 2628 MAX_HMAC_DIGEST_SIZE); 2629 2630 if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) { 2631 rc = -ENOMEM; 2632 goto fail1; 2633 } 2634 2635 /* Linux crypto */ 2636 for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) { 2637 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) || 2638 !(drvdata->std_bodies & aead_algs[alg].std_body)) 2639 continue; 2640 2641 t_alg = cc_create_aead_alg(&aead_algs[alg], dev); 2642 if (IS_ERR(t_alg)) { 2643 rc = PTR_ERR(t_alg); 2644 dev_err(dev, "%s alg allocation failed\n", 2645 aead_algs[alg].driver_name); 2646 goto fail1; 2647 } 2648 t_alg->drvdata = drvdata; 2649 rc = crypto_register_aead(&t_alg->aead_alg); 2650 if (rc) { 2651 dev_err(dev, "%s alg registration failed\n", 2652 t_alg->aead_alg.base.cra_driver_name); 2653 goto fail1; 2654 } 2655 2656 list_add_tail(&t_alg->entry, &aead_handle->aead_list); 2657 dev_dbg(dev, "Registered %s\n", 2658 t_alg->aead_alg.base.cra_driver_name); 2659 } 2660 2661 return 0; 2662 2663 fail1: 2664 cc_aead_free(drvdata); 2665 fail0: 2666 return rc; 2667 } 2668