1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTX CPT driver 3 * 4 * Copyright (C) 2019 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <crypto/aes.h> 12 #include <crypto/authenc.h> 13 #include <crypto/cryptd.h> 14 #include <crypto/des.h> 15 #include <crypto/internal/aead.h> 16 #include <crypto/sha1.h> 17 #include <crypto/sha2.h> 18 #include <crypto/xts.h> 19 #include <crypto/scatterwalk.h> 20 #include <linux/rtnetlink.h> 21 #include <linux/sort.h> 22 #include <linux/module.h> 23 #include "otx_cptvf.h" 24 #include "otx_cptvf_algs.h" 25 #include "otx_cptvf_reqmgr.h" 26 27 #define CPT_MAX_VF_NUM 64 28 /* Size of salt in AES GCM mode */ 29 #define AES_GCM_SALT_SIZE 4 30 /* Size of IV in AES GCM mode */ 31 #define AES_GCM_IV_SIZE 8 32 /* Size of ICV (Integrity Check Value) in AES GCM mode */ 33 #define AES_GCM_ICV_SIZE 16 34 /* Offset of IV in AES GCM mode */ 35 #define AES_GCM_IV_OFFSET 8 36 #define CONTROL_WORD_LEN 8 37 #define KEY2_OFFSET 48 38 #define DMA_MODE_FLAG(dma_mode) \ 39 (((dma_mode) == OTX_CPT_DMA_GATHER_SCATTER) ? (1 << 7) : 0) 40 41 /* Truncated SHA digest size */ 42 #define SHA1_TRUNC_DIGEST_SIZE 12 43 #define SHA256_TRUNC_DIGEST_SIZE 16 44 #define SHA384_TRUNC_DIGEST_SIZE 24 45 #define SHA512_TRUNC_DIGEST_SIZE 32 46 47 static DEFINE_MUTEX(mutex); 48 static int is_crypto_registered; 49 50 struct cpt_device_desc { 51 enum otx_cptpf_type pf_type; 52 struct pci_dev *dev; 53 int num_queues; 54 }; 55 56 struct cpt_device_table { 57 atomic_t count; 58 struct cpt_device_desc desc[CPT_MAX_VF_NUM]; 59 }; 60 61 static struct cpt_device_table se_devices = { 62 .count = ATOMIC_INIT(0) 63 }; 64 65 static struct cpt_device_table ae_devices = { 66 .count = ATOMIC_INIT(0) 67 }; 68 69 static inline int get_se_device(struct pci_dev **pdev, int *cpu_num) 70 { 71 int count, ret = 0; 72 73 count = atomic_read(&se_devices.count); 74 if (count < 1) 75 return -ENODEV; 76 77 *cpu_num = get_cpu(); 78 79 if (se_devices.desc[0].pf_type == OTX_CPT_SE) { 80 /* 81 * On OcteonTX platform there is one CPT instruction queue bound 82 * to each VF. We get maximum performance if one CPT queue 83 * is available for each cpu otherwise CPT queues need to be 84 * shared between cpus. 85 */ 86 if (*cpu_num >= count) 87 *cpu_num %= count; 88 *pdev = se_devices.desc[*cpu_num].dev; 89 } else { 90 pr_err("Unknown PF type %d\n", se_devices.desc[0].pf_type); 91 ret = -EINVAL; 92 } 93 put_cpu(); 94 95 return ret; 96 } 97 98 static inline int validate_hmac_cipher_null(struct otx_cpt_req_info *cpt_req) 99 { 100 struct otx_cpt_req_ctx *rctx; 101 struct aead_request *req; 102 struct crypto_aead *tfm; 103 104 req = container_of(cpt_req->areq, struct aead_request, base); 105 tfm = crypto_aead_reqtfm(req); 106 rctx = aead_request_ctx_dma(req); 107 if (memcmp(rctx->fctx.hmac.s.hmac_calc, 108 rctx->fctx.hmac.s.hmac_recv, 109 crypto_aead_authsize(tfm)) != 0) 110 return -EBADMSG; 111 112 return 0; 113 } 114 115 static void otx_cpt_aead_callback(int status, void *arg1, void *arg2) 116 { 117 struct otx_cpt_info_buffer *cpt_info = arg2; 118 struct crypto_async_request *areq = arg1; 119 struct otx_cpt_req_info *cpt_req; 120 struct pci_dev *pdev; 121 122 if (!cpt_info) 123 goto complete; 124 125 cpt_req = cpt_info->req; 126 if (!status) { 127 /* 128 * When selected cipher is NULL we need to manually 129 * verify whether calculated hmac value matches 130 * received hmac value 131 */ 132 if (cpt_req->req_type == OTX_CPT_AEAD_ENC_DEC_NULL_REQ && 133 !cpt_req->is_enc) 134 status = validate_hmac_cipher_null(cpt_req); 135 } 136 pdev = cpt_info->pdev; 137 do_request_cleanup(pdev, cpt_info); 138 139 complete: 140 if (areq) 141 crypto_request_complete(areq, status); 142 } 143 144 static void output_iv_copyback(struct crypto_async_request *areq) 145 { 146 struct otx_cpt_req_info *req_info; 147 struct skcipher_request *sreq; 148 struct crypto_skcipher *stfm; 149 struct otx_cpt_req_ctx *rctx; 150 struct otx_cpt_enc_ctx *ctx; 151 u32 start, ivsize; 152 153 sreq = container_of(areq, struct skcipher_request, base); 154 stfm = crypto_skcipher_reqtfm(sreq); 155 ctx = crypto_skcipher_ctx(stfm); 156 if (ctx->cipher_type == OTX_CPT_AES_CBC || 157 ctx->cipher_type == OTX_CPT_DES3_CBC) { 158 rctx = skcipher_request_ctx_dma(sreq); 159 req_info = &rctx->cpt_req; 160 ivsize = crypto_skcipher_ivsize(stfm); 161 start = sreq->cryptlen - ivsize; 162 163 if (req_info->is_enc) { 164 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start, 165 ivsize, 0); 166 } else { 167 if (sreq->src != sreq->dst) { 168 scatterwalk_map_and_copy(sreq->iv, sreq->src, 169 start, ivsize, 0); 170 } else { 171 memcpy(sreq->iv, req_info->iv_out, ivsize); 172 kfree(req_info->iv_out); 173 } 174 } 175 } 176 } 177 178 static void otx_cpt_skcipher_callback(int status, void *arg1, void *arg2) 179 { 180 struct otx_cpt_info_buffer *cpt_info = arg2; 181 struct crypto_async_request *areq = arg1; 182 struct pci_dev *pdev; 183 184 if (areq) { 185 if (!status) 186 output_iv_copyback(areq); 187 if (cpt_info) { 188 pdev = cpt_info->pdev; 189 do_request_cleanup(pdev, cpt_info); 190 } 191 crypto_request_complete(areq, status); 192 } 193 } 194 195 static inline void update_input_data(struct otx_cpt_req_info *req_info, 196 struct scatterlist *inp_sg, 197 u32 nbytes, u32 *argcnt) 198 { 199 req_info->req.dlen += nbytes; 200 201 while (nbytes) { 202 u32 len = min(nbytes, inp_sg->length); 203 u8 *ptr = sg_virt(inp_sg); 204 205 req_info->in[*argcnt].vptr = (void *)ptr; 206 req_info->in[*argcnt].size = len; 207 nbytes -= len; 208 ++(*argcnt); 209 inp_sg = sg_next(inp_sg); 210 } 211 } 212 213 static inline void update_output_data(struct otx_cpt_req_info *req_info, 214 struct scatterlist *outp_sg, 215 u32 offset, u32 nbytes, u32 *argcnt) 216 { 217 req_info->rlen += nbytes; 218 219 while (nbytes) { 220 u32 len = min(nbytes, outp_sg->length - offset); 221 u8 *ptr = sg_virt(outp_sg); 222 223 req_info->out[*argcnt].vptr = (void *) (ptr + offset); 224 req_info->out[*argcnt].size = len; 225 nbytes -= len; 226 ++(*argcnt); 227 offset = 0; 228 outp_sg = sg_next(outp_sg); 229 } 230 } 231 232 static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, 233 u32 *argcnt) 234 { 235 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 236 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 237 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 238 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); 239 struct otx_cpt_enc_ctx *ctx = crypto_tfm_ctx(tfm); 240 struct otx_cpt_fc_ctx *fctx = &rctx->fctx; 241 int ivsize = crypto_skcipher_ivsize(stfm); 242 u32 start = req->cryptlen - ivsize; 243 gfp_t flags; 244 245 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 246 GFP_KERNEL : GFP_ATOMIC; 247 req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER; 248 req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ; 249 250 req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC | 251 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER); 252 if (enc) { 253 req_info->req.opcode.s.minor = 2; 254 } else { 255 req_info->req.opcode.s.minor = 3; 256 if ((ctx->cipher_type == OTX_CPT_AES_CBC || 257 ctx->cipher_type == OTX_CPT_DES3_CBC) && 258 req->src == req->dst) { 259 req_info->iv_out = kmalloc(ivsize, flags); 260 if (!req_info->iv_out) 261 return -ENOMEM; 262 263 scatterwalk_map_and_copy(req_info->iv_out, req->src, 264 start, ivsize, 0); 265 } 266 } 267 /* Encryption data length */ 268 req_info->req.param1 = req->cryptlen; 269 /* Authentication data length */ 270 req_info->req.param2 = 0; 271 272 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 273 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 274 fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR; 275 276 if (ctx->cipher_type == OTX_CPT_AES_XTS) 277 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); 278 else 279 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); 280 281 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm)); 282 283 fctx->enc.enc_ctrl.flags = cpu_to_be64(fctx->enc.enc_ctrl.cflags); 284 285 /* 286 * Storing Packet Data Information in offset 287 * Control Word First 8 bytes 288 */ 289 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; 290 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 291 req_info->req.dlen += CONTROL_WORD_LEN; 292 ++(*argcnt); 293 294 req_info->in[*argcnt].vptr = (u8 *)fctx; 295 req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx); 296 req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx); 297 298 ++(*argcnt); 299 300 return 0; 301 } 302 303 static inline u32 create_input_list(struct skcipher_request *req, u32 enc, 304 u32 enc_iv_len) 305 { 306 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 307 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 308 u32 argcnt = 0; 309 int ret; 310 311 ret = create_ctx_hdr(req, enc, &argcnt); 312 if (ret) 313 return ret; 314 315 update_input_data(req_info, req->src, req->cryptlen, &argcnt); 316 req_info->incnt = argcnt; 317 318 return 0; 319 } 320 321 static inline void create_output_list(struct skcipher_request *req, 322 u32 enc_iv_len) 323 { 324 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 325 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 326 u32 argcnt = 0; 327 328 /* 329 * OUTPUT Buffer Processing 330 * AES encryption/decryption output would be 331 * received in the following format 332 * 333 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| 334 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] 335 */ 336 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt); 337 req_info->outcnt = argcnt; 338 } 339 340 static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) 341 { 342 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); 343 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req); 344 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 345 u32 enc_iv_len = crypto_skcipher_ivsize(stfm); 346 struct pci_dev *pdev; 347 int status, cpu_num; 348 349 /* Validate that request doesn't exceed maximum CPT supported size */ 350 if (req->cryptlen > OTX_CPT_MAX_REQ_SIZE) 351 return -E2BIG; 352 353 /* Clear control words */ 354 rctx->ctrl_word.flags = 0; 355 rctx->fctx.enc.enc_ctrl.flags = 0; 356 357 status = create_input_list(req, enc, enc_iv_len); 358 if (status) 359 return status; 360 create_output_list(req, enc_iv_len); 361 362 status = get_se_device(&pdev, &cpu_num); 363 if (status) 364 return status; 365 366 req_info->callback = (void *)otx_cpt_skcipher_callback; 367 req_info->areq = &req->base; 368 req_info->req_type = OTX_CPT_ENC_DEC_REQ; 369 req_info->is_enc = enc; 370 req_info->is_trunc_hmac = false; 371 req_info->ctrl.s.grp = 0; 372 373 /* 374 * We perform an asynchronous send and once 375 * the request is completed the driver would 376 * intimate through registered call back functions 377 */ 378 status = otx_cpt_do_request(pdev, req_info, cpu_num); 379 380 return status; 381 } 382 383 static int otx_cpt_skcipher_encrypt(struct skcipher_request *req) 384 { 385 return cpt_enc_dec(req, true); 386 } 387 388 static int otx_cpt_skcipher_decrypt(struct skcipher_request *req) 389 { 390 return cpt_enc_dec(req, false); 391 } 392 393 static int otx_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm, 394 const u8 *key, u32 keylen) 395 { 396 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 397 const u8 *key2 = key + (keylen / 2); 398 const u8 *key1 = key; 399 int ret; 400 401 ret = xts_verify_key(tfm, key, keylen); 402 if (ret) 403 return ret; 404 ctx->key_len = keylen; 405 memcpy(ctx->enc_key, key1, keylen / 2); 406 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); 407 ctx->cipher_type = OTX_CPT_AES_XTS; 408 switch (ctx->key_len) { 409 case 2 * AES_KEYSIZE_128: 410 ctx->key_type = OTX_CPT_AES_128_BIT; 411 break; 412 case 2 * AES_KEYSIZE_256: 413 ctx->key_type = OTX_CPT_AES_256_BIT; 414 break; 415 default: 416 return -EINVAL; 417 } 418 419 return 0; 420 } 421 422 static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 423 u32 keylen, u8 cipher_type) 424 { 425 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 426 427 if (keylen != DES3_EDE_KEY_SIZE) 428 return -EINVAL; 429 430 ctx->key_len = keylen; 431 ctx->cipher_type = cipher_type; 432 433 memcpy(ctx->enc_key, key, keylen); 434 435 return 0; 436 } 437 438 static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 439 u32 keylen, u8 cipher_type) 440 { 441 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 442 443 switch (keylen) { 444 case AES_KEYSIZE_128: 445 ctx->key_type = OTX_CPT_AES_128_BIT; 446 break; 447 case AES_KEYSIZE_192: 448 ctx->key_type = OTX_CPT_AES_192_BIT; 449 break; 450 case AES_KEYSIZE_256: 451 ctx->key_type = OTX_CPT_AES_256_BIT; 452 break; 453 default: 454 return -EINVAL; 455 } 456 ctx->key_len = keylen; 457 ctx->cipher_type = cipher_type; 458 459 memcpy(ctx->enc_key, key, keylen); 460 461 return 0; 462 } 463 464 static int otx_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm, 465 const u8 *key, u32 keylen) 466 { 467 return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_CBC); 468 } 469 470 static int otx_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm, 471 const u8 *key, u32 keylen) 472 { 473 return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_ECB); 474 } 475 476 static int otx_cpt_skcipher_cfb_aes_setkey(struct crypto_skcipher *tfm, 477 const u8 *key, u32 keylen) 478 { 479 return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_CFB); 480 } 481 482 static int otx_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm, 483 const u8 *key, u32 keylen) 484 { 485 return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_CBC); 486 } 487 488 static int otx_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm, 489 const u8 *key, u32 keylen) 490 { 491 return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_ECB); 492 } 493 494 static int otx_cpt_enc_dec_init(struct crypto_skcipher *tfm) 495 { 496 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 497 498 memset(ctx, 0, sizeof(*ctx)); 499 /* 500 * Additional memory for skcipher_request is 501 * allocated since the cryptd daemon uses 502 * this memory for request_ctx information 503 */ 504 crypto_skcipher_set_reqsize_dma( 505 tfm, sizeof(struct otx_cpt_req_ctx) + 506 sizeof(struct skcipher_request)); 507 508 return 0; 509 } 510 511 static int cpt_aead_init(struct crypto_aead *tfm, u8 cipher_type, u8 mac_type) 512 { 513 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 514 515 ctx->cipher_type = cipher_type; 516 ctx->mac_type = mac_type; 517 518 /* 519 * When selected cipher is NULL we use HMAC opcode instead of 520 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms 521 * for calculating ipad and opad 522 */ 523 if (ctx->cipher_type != OTX_CPT_CIPHER_NULL) { 524 switch (ctx->mac_type) { 525 case OTX_CPT_SHA1: 526 ctx->hashalg = crypto_alloc_shash("sha1", 0, 527 CRYPTO_ALG_ASYNC); 528 if (IS_ERR(ctx->hashalg)) 529 return PTR_ERR(ctx->hashalg); 530 break; 531 532 case OTX_CPT_SHA256: 533 ctx->hashalg = crypto_alloc_shash("sha256", 0, 534 CRYPTO_ALG_ASYNC); 535 if (IS_ERR(ctx->hashalg)) 536 return PTR_ERR(ctx->hashalg); 537 break; 538 539 case OTX_CPT_SHA384: 540 ctx->hashalg = crypto_alloc_shash("sha384", 0, 541 CRYPTO_ALG_ASYNC); 542 if (IS_ERR(ctx->hashalg)) 543 return PTR_ERR(ctx->hashalg); 544 break; 545 546 case OTX_CPT_SHA512: 547 ctx->hashalg = crypto_alloc_shash("sha512", 0, 548 CRYPTO_ALG_ASYNC); 549 if (IS_ERR(ctx->hashalg)) 550 return PTR_ERR(ctx->hashalg); 551 break; 552 } 553 } 554 555 crypto_aead_set_reqsize_dma(tfm, sizeof(struct otx_cpt_req_ctx)); 556 557 return 0; 558 } 559 560 static int otx_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm) 561 { 562 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA1); 563 } 564 565 static int otx_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm) 566 { 567 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA256); 568 } 569 570 static int otx_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm) 571 { 572 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA384); 573 } 574 575 static int otx_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm) 576 { 577 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA512); 578 } 579 580 static int otx_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm) 581 { 582 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA1); 583 } 584 585 static int otx_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm) 586 { 587 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA256); 588 } 589 590 static int otx_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm) 591 { 592 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA384); 593 } 594 595 static int otx_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm) 596 { 597 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA512); 598 } 599 600 static int otx_cpt_aead_gcm_aes_init(struct crypto_aead *tfm) 601 { 602 return cpt_aead_init(tfm, OTX_CPT_AES_GCM, OTX_CPT_MAC_NULL); 603 } 604 605 static void otx_cpt_aead_exit(struct crypto_aead *tfm) 606 { 607 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 608 609 kfree(ctx->ipad); 610 kfree(ctx->opad); 611 if (ctx->hashalg) 612 crypto_free_shash(ctx->hashalg); 613 kfree(ctx->sdesc); 614 } 615 616 /* 617 * This is the Integrity Check Value validation (aka the authentication tag 618 * length) 619 */ 620 static int otx_cpt_aead_set_authsize(struct crypto_aead *tfm, 621 unsigned int authsize) 622 { 623 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 624 625 switch (ctx->mac_type) { 626 case OTX_CPT_SHA1: 627 if (authsize != SHA1_DIGEST_SIZE && 628 authsize != SHA1_TRUNC_DIGEST_SIZE) 629 return -EINVAL; 630 631 if (authsize == SHA1_TRUNC_DIGEST_SIZE) 632 ctx->is_trunc_hmac = true; 633 break; 634 635 case OTX_CPT_SHA256: 636 if (authsize != SHA256_DIGEST_SIZE && 637 authsize != SHA256_TRUNC_DIGEST_SIZE) 638 return -EINVAL; 639 640 if (authsize == SHA256_TRUNC_DIGEST_SIZE) 641 ctx->is_trunc_hmac = true; 642 break; 643 644 case OTX_CPT_SHA384: 645 if (authsize != SHA384_DIGEST_SIZE && 646 authsize != SHA384_TRUNC_DIGEST_SIZE) 647 return -EINVAL; 648 649 if (authsize == SHA384_TRUNC_DIGEST_SIZE) 650 ctx->is_trunc_hmac = true; 651 break; 652 653 case OTX_CPT_SHA512: 654 if (authsize != SHA512_DIGEST_SIZE && 655 authsize != SHA512_TRUNC_DIGEST_SIZE) 656 return -EINVAL; 657 658 if (authsize == SHA512_TRUNC_DIGEST_SIZE) 659 ctx->is_trunc_hmac = true; 660 break; 661 662 case OTX_CPT_MAC_NULL: 663 if (ctx->cipher_type == OTX_CPT_AES_GCM) { 664 if (authsize != AES_GCM_ICV_SIZE) 665 return -EINVAL; 666 } else 667 return -EINVAL; 668 break; 669 670 default: 671 return -EINVAL; 672 } 673 674 tfm->authsize = authsize; 675 return 0; 676 } 677 678 static struct otx_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg) 679 { 680 struct otx_cpt_sdesc *sdesc; 681 int size; 682 683 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); 684 sdesc = kmalloc(size, GFP_KERNEL); 685 if (!sdesc) 686 return NULL; 687 688 sdesc->shash.tfm = alg; 689 690 return sdesc; 691 } 692 693 static inline void swap_data32(void *buf, u32 len) 694 { 695 cpu_to_be32_array(buf, buf, len / 4); 696 } 697 698 static inline void swap_data64(void *buf, u32 len) 699 { 700 __be64 *dst = buf; 701 u64 *src = buf; 702 int i = 0; 703 704 for (i = 0 ; i < len / 8; i++, src++, dst++) 705 *dst = cpu_to_be64p(src); 706 } 707 708 static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad) 709 { 710 struct sha512_state *sha512; 711 struct sha256_state *sha256; 712 struct sha1_state *sha1; 713 714 switch (mac_type) { 715 case OTX_CPT_SHA1: 716 sha1 = (struct sha1_state *) in_pad; 717 swap_data32(sha1->state, SHA1_DIGEST_SIZE); 718 memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE); 719 break; 720 721 case OTX_CPT_SHA256: 722 sha256 = (struct sha256_state *) in_pad; 723 swap_data32(sha256->state, SHA256_DIGEST_SIZE); 724 memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE); 725 break; 726 727 case OTX_CPT_SHA384: 728 case OTX_CPT_SHA512: 729 sha512 = (struct sha512_state *) in_pad; 730 swap_data64(sha512->state, SHA512_DIGEST_SIZE); 731 memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE); 732 break; 733 734 default: 735 return -EINVAL; 736 } 737 738 return 0; 739 } 740 741 static int aead_hmac_init(struct crypto_aead *cipher) 742 { 743 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 744 int state_size = crypto_shash_statesize(ctx->hashalg); 745 int ds = crypto_shash_digestsize(ctx->hashalg); 746 int bs = crypto_shash_blocksize(ctx->hashalg); 747 int authkeylen = ctx->auth_key_len; 748 u8 *ipad = NULL, *opad = NULL; 749 int ret = 0, icount = 0; 750 751 ctx->sdesc = alloc_sdesc(ctx->hashalg); 752 if (!ctx->sdesc) 753 return -ENOMEM; 754 755 ctx->ipad = kzalloc(bs, GFP_KERNEL); 756 if (!ctx->ipad) { 757 ret = -ENOMEM; 758 goto calc_fail; 759 } 760 761 ctx->opad = kzalloc(bs, GFP_KERNEL); 762 if (!ctx->opad) { 763 ret = -ENOMEM; 764 goto calc_fail; 765 } 766 767 ipad = kzalloc(state_size, GFP_KERNEL); 768 if (!ipad) { 769 ret = -ENOMEM; 770 goto calc_fail; 771 } 772 773 opad = kzalloc(state_size, GFP_KERNEL); 774 if (!opad) { 775 ret = -ENOMEM; 776 goto calc_fail; 777 } 778 779 if (authkeylen > bs) { 780 ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key, 781 authkeylen, ipad); 782 if (ret) 783 goto calc_fail; 784 785 authkeylen = ds; 786 } else { 787 memcpy(ipad, ctx->key, authkeylen); 788 } 789 790 memset(ipad + authkeylen, 0, bs - authkeylen); 791 memcpy(opad, ipad, bs); 792 793 for (icount = 0; icount < bs; icount++) { 794 ipad[icount] ^= 0x36; 795 opad[icount] ^= 0x5c; 796 } 797 798 /* 799 * Partial Hash calculated from the software 800 * algorithm is retrieved for IPAD & OPAD 801 */ 802 803 /* IPAD Calculation */ 804 crypto_shash_init(&ctx->sdesc->shash); 805 crypto_shash_update(&ctx->sdesc->shash, ipad, bs); 806 crypto_shash_export(&ctx->sdesc->shash, ipad); 807 ret = copy_pad(ctx->mac_type, ctx->ipad, ipad); 808 if (ret) 809 goto calc_fail; 810 811 /* OPAD Calculation */ 812 crypto_shash_init(&ctx->sdesc->shash); 813 crypto_shash_update(&ctx->sdesc->shash, opad, bs); 814 crypto_shash_export(&ctx->sdesc->shash, opad); 815 ret = copy_pad(ctx->mac_type, ctx->opad, opad); 816 if (ret) 817 goto calc_fail; 818 819 kfree(ipad); 820 kfree(opad); 821 822 return 0; 823 824 calc_fail: 825 kfree(ctx->ipad); 826 ctx->ipad = NULL; 827 kfree(ctx->opad); 828 ctx->opad = NULL; 829 kfree(ipad); 830 kfree(opad); 831 kfree(ctx->sdesc); 832 ctx->sdesc = NULL; 833 834 return ret; 835 } 836 837 static int otx_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, 838 const unsigned char *key, 839 unsigned int keylen) 840 { 841 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 842 struct crypto_authenc_key_param *param; 843 int enckeylen = 0, authkeylen = 0; 844 struct rtattr *rta = (void *)key; 845 int status = -EINVAL; 846 847 if (!RTA_OK(rta, keylen)) 848 goto badkey; 849 850 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) 851 goto badkey; 852 853 if (RTA_PAYLOAD(rta) < sizeof(*param)) 854 goto badkey; 855 856 param = RTA_DATA(rta); 857 enckeylen = be32_to_cpu(param->enckeylen); 858 key += RTA_ALIGN(rta->rta_len); 859 keylen -= RTA_ALIGN(rta->rta_len); 860 if (keylen < enckeylen) 861 goto badkey; 862 863 if (keylen > OTX_CPT_MAX_KEY_SIZE) 864 goto badkey; 865 866 authkeylen = keylen - enckeylen; 867 memcpy(ctx->key, key, keylen); 868 869 switch (enckeylen) { 870 case AES_KEYSIZE_128: 871 ctx->key_type = OTX_CPT_AES_128_BIT; 872 break; 873 case AES_KEYSIZE_192: 874 ctx->key_type = OTX_CPT_AES_192_BIT; 875 break; 876 case AES_KEYSIZE_256: 877 ctx->key_type = OTX_CPT_AES_256_BIT; 878 break; 879 default: 880 /* Invalid key length */ 881 goto badkey; 882 } 883 884 ctx->enc_key_len = enckeylen; 885 ctx->auth_key_len = authkeylen; 886 887 status = aead_hmac_init(cipher); 888 if (status) 889 goto badkey; 890 891 return 0; 892 badkey: 893 return status; 894 } 895 896 static int otx_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, 897 const unsigned char *key, 898 unsigned int keylen) 899 { 900 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 901 struct crypto_authenc_key_param *param; 902 struct rtattr *rta = (void *)key; 903 int enckeylen = 0; 904 905 if (!RTA_OK(rta, keylen)) 906 goto badkey; 907 908 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) 909 goto badkey; 910 911 if (RTA_PAYLOAD(rta) < sizeof(*param)) 912 goto badkey; 913 914 param = RTA_DATA(rta); 915 enckeylen = be32_to_cpu(param->enckeylen); 916 key += RTA_ALIGN(rta->rta_len); 917 keylen -= RTA_ALIGN(rta->rta_len); 918 if (enckeylen != 0) 919 goto badkey; 920 921 if (keylen > OTX_CPT_MAX_KEY_SIZE) 922 goto badkey; 923 924 memcpy(ctx->key, key, keylen); 925 ctx->enc_key_len = enckeylen; 926 ctx->auth_key_len = keylen; 927 return 0; 928 badkey: 929 return -EINVAL; 930 } 931 932 static int otx_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, 933 const unsigned char *key, 934 unsigned int keylen) 935 { 936 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher); 937 938 /* 939 * For aes gcm we expect to get encryption key (16, 24, 32 bytes) 940 * and salt (4 bytes) 941 */ 942 switch (keylen) { 943 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE: 944 ctx->key_type = OTX_CPT_AES_128_BIT; 945 ctx->enc_key_len = AES_KEYSIZE_128; 946 break; 947 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE: 948 ctx->key_type = OTX_CPT_AES_192_BIT; 949 ctx->enc_key_len = AES_KEYSIZE_192; 950 break; 951 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE: 952 ctx->key_type = OTX_CPT_AES_256_BIT; 953 ctx->enc_key_len = AES_KEYSIZE_256; 954 break; 955 default: 956 /* Invalid key and salt length */ 957 return -EINVAL; 958 } 959 960 /* Store encryption key and salt */ 961 memcpy(ctx->key, key, keylen); 962 963 return 0; 964 } 965 966 static inline u32 create_aead_ctx_hdr(struct aead_request *req, u32 enc, 967 u32 *argcnt) 968 { 969 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 970 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 971 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 972 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 973 struct otx_cpt_fc_ctx *fctx = &rctx->fctx; 974 int mac_len = crypto_aead_authsize(tfm); 975 int ds; 976 977 rctx->ctrl_word.e.enc_data_offset = req->assoclen; 978 979 switch (ctx->cipher_type) { 980 case OTX_CPT_AES_CBC: 981 fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR; 982 /* Copy encryption key to context */ 983 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len, 984 ctx->enc_key_len); 985 /* Copy IV to context */ 986 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm)); 987 988 ds = crypto_shash_digestsize(ctx->hashalg); 989 if (ctx->mac_type == OTX_CPT_SHA384) 990 ds = SHA512_DIGEST_SIZE; 991 if (ctx->ipad) 992 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds); 993 if (ctx->opad) 994 memcpy(fctx->hmac.e.opad, ctx->opad, ds); 995 break; 996 997 case OTX_CPT_AES_GCM: 998 fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_DPTR; 999 /* Copy encryption key to context */ 1000 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len); 1001 /* Copy salt to context */ 1002 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len, 1003 AES_GCM_SALT_SIZE); 1004 1005 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET; 1006 break; 1007 1008 default: 1009 /* Unknown cipher type */ 1010 return -EINVAL; 1011 } 1012 rctx->ctrl_word.flags = cpu_to_be64(rctx->ctrl_word.cflags); 1013 1014 req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER; 1015 req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ; 1016 req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC | 1017 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER); 1018 if (enc) { 1019 req_info->req.opcode.s.minor = 2; 1020 req_info->req.param1 = req->cryptlen; 1021 req_info->req.param2 = req->cryptlen + req->assoclen; 1022 } else { 1023 req_info->req.opcode.s.minor = 3; 1024 req_info->req.param1 = req->cryptlen - mac_len; 1025 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len; 1026 } 1027 1028 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 1029 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 1030 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type; 1031 fctx->enc.enc_ctrl.e.mac_len = mac_len; 1032 fctx->enc.enc_ctrl.flags = cpu_to_be64(fctx->enc.enc_ctrl.cflags); 1033 1034 /* 1035 * Storing Packet Data Information in offset 1036 * Control Word First 8 bytes 1037 */ 1038 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; 1039 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 1040 req_info->req.dlen += CONTROL_WORD_LEN; 1041 ++(*argcnt); 1042 1043 req_info->in[*argcnt].vptr = (u8 *)fctx; 1044 req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx); 1045 req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx); 1046 ++(*argcnt); 1047 1048 return 0; 1049 } 1050 1051 static inline u32 create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt, 1052 u32 enc) 1053 { 1054 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1055 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1056 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm); 1057 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 1058 1059 req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER; 1060 req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ; 1061 req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_HMAC | 1062 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER); 1063 req_info->is_trunc_hmac = ctx->is_trunc_hmac; 1064 1065 req_info->req.opcode.s.minor = 0; 1066 req_info->req.param1 = ctx->auth_key_len; 1067 req_info->req.param2 = ctx->mac_type << 8; 1068 1069 /* Add authentication key */ 1070 req_info->in[*argcnt].vptr = ctx->key; 1071 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8); 1072 req_info->req.dlen += round_up(ctx->auth_key_len, 8); 1073 ++(*argcnt); 1074 1075 return 0; 1076 } 1077 1078 static inline u32 create_aead_input_list(struct aead_request *req, u32 enc) 1079 { 1080 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1081 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 1082 u32 inputlen = req->cryptlen + req->assoclen; 1083 u32 status, argcnt = 0; 1084 1085 status = create_aead_ctx_hdr(req, enc, &argcnt); 1086 if (status) 1087 return status; 1088 update_input_data(req_info, req->src, inputlen, &argcnt); 1089 req_info->incnt = argcnt; 1090 1091 return 0; 1092 } 1093 1094 static inline u32 create_aead_output_list(struct aead_request *req, u32 enc, 1095 u32 mac_len) 1096 { 1097 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1098 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 1099 u32 argcnt = 0, outputlen = 0; 1100 1101 if (enc) 1102 outputlen = req->cryptlen + req->assoclen + mac_len; 1103 else 1104 outputlen = req->cryptlen + req->assoclen - mac_len; 1105 1106 update_output_data(req_info, req->dst, 0, outputlen, &argcnt); 1107 req_info->outcnt = argcnt; 1108 1109 return 0; 1110 } 1111 1112 static inline u32 create_aead_null_input_list(struct aead_request *req, 1113 u32 enc, u32 mac_len) 1114 { 1115 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1116 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 1117 u32 inputlen, argcnt = 0; 1118 1119 if (enc) 1120 inputlen = req->cryptlen + req->assoclen; 1121 else 1122 inputlen = req->cryptlen + req->assoclen - mac_len; 1123 1124 create_hmac_ctx_hdr(req, &argcnt, enc); 1125 update_input_data(req_info, req->src, inputlen, &argcnt); 1126 req_info->incnt = argcnt; 1127 1128 return 0; 1129 } 1130 1131 static inline u32 create_aead_null_output_list(struct aead_request *req, 1132 u32 enc, u32 mac_len) 1133 { 1134 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1135 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 1136 struct scatterlist *dst; 1137 u8 *ptr = NULL; 1138 int argcnt = 0, status, offset; 1139 u32 inputlen; 1140 1141 if (enc) 1142 inputlen = req->cryptlen + req->assoclen; 1143 else 1144 inputlen = req->cryptlen + req->assoclen - mac_len; 1145 1146 /* 1147 * If source and destination are different 1148 * then copy payload to destination 1149 */ 1150 if (req->src != req->dst) { 1151 1152 ptr = kmalloc(inputlen, (req_info->areq->flags & 1153 CRYPTO_TFM_REQ_MAY_SLEEP) ? 1154 GFP_KERNEL : GFP_ATOMIC); 1155 if (!ptr) { 1156 status = -ENOMEM; 1157 goto error; 1158 } 1159 1160 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr, 1161 inputlen); 1162 if (status != inputlen) { 1163 status = -EINVAL; 1164 goto error_free; 1165 } 1166 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr, 1167 inputlen); 1168 if (status != inputlen) { 1169 status = -EINVAL; 1170 goto error_free; 1171 } 1172 kfree(ptr); 1173 } 1174 1175 if (enc) { 1176 /* 1177 * In an encryption scenario hmac needs 1178 * to be appended after payload 1179 */ 1180 dst = req->dst; 1181 offset = inputlen; 1182 while (offset >= dst->length) { 1183 offset -= dst->length; 1184 dst = sg_next(dst); 1185 if (!dst) { 1186 status = -ENOENT; 1187 goto error; 1188 } 1189 } 1190 1191 update_output_data(req_info, dst, offset, mac_len, &argcnt); 1192 } else { 1193 /* 1194 * In a decryption scenario calculated hmac for received 1195 * payload needs to be compare with hmac received 1196 */ 1197 status = sg_copy_buffer(req->src, sg_nents(req->src), 1198 rctx->fctx.hmac.s.hmac_recv, mac_len, 1199 inputlen, true); 1200 if (status != mac_len) { 1201 status = -EINVAL; 1202 goto error; 1203 } 1204 1205 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc; 1206 req_info->out[argcnt].size = mac_len; 1207 argcnt++; 1208 } 1209 1210 req_info->outcnt = argcnt; 1211 return 0; 1212 1213 error_free: 1214 kfree(ptr); 1215 error: 1216 return status; 1217 } 1218 1219 static u32 cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) 1220 { 1221 struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req); 1222 struct otx_cpt_req_info *req_info = &rctx->cpt_req; 1223 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1224 struct pci_dev *pdev; 1225 u32 status, cpu_num; 1226 1227 /* Clear control words */ 1228 rctx->ctrl_word.flags = 0; 1229 rctx->fctx.enc.enc_ctrl.flags = 0; 1230 1231 req_info->callback = otx_cpt_aead_callback; 1232 req_info->areq = &req->base; 1233 req_info->req_type = reg_type; 1234 req_info->is_enc = enc; 1235 req_info->is_trunc_hmac = false; 1236 1237 switch (reg_type) { 1238 case OTX_CPT_AEAD_ENC_DEC_REQ: 1239 status = create_aead_input_list(req, enc); 1240 if (status) 1241 return status; 1242 status = create_aead_output_list(req, enc, 1243 crypto_aead_authsize(tfm)); 1244 if (status) 1245 return status; 1246 break; 1247 1248 case OTX_CPT_AEAD_ENC_DEC_NULL_REQ: 1249 status = create_aead_null_input_list(req, enc, 1250 crypto_aead_authsize(tfm)); 1251 if (status) 1252 return status; 1253 status = create_aead_null_output_list(req, enc, 1254 crypto_aead_authsize(tfm)); 1255 if (status) 1256 return status; 1257 break; 1258 1259 default: 1260 return -EINVAL; 1261 } 1262 1263 /* Validate that request doesn't exceed maximum CPT supported size */ 1264 if (req_info->req.param1 > OTX_CPT_MAX_REQ_SIZE || 1265 req_info->req.param2 > OTX_CPT_MAX_REQ_SIZE) 1266 return -E2BIG; 1267 1268 status = get_se_device(&pdev, &cpu_num); 1269 if (status) 1270 return status; 1271 1272 req_info->ctrl.s.grp = 0; 1273 1274 status = otx_cpt_do_request(pdev, req_info, cpu_num); 1275 /* 1276 * We perform an asynchronous send and once 1277 * the request is completed the driver would 1278 * intimate through registered call back functions 1279 */ 1280 return status; 1281 } 1282 1283 static int otx_cpt_aead_encrypt(struct aead_request *req) 1284 { 1285 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, true); 1286 } 1287 1288 static int otx_cpt_aead_decrypt(struct aead_request *req) 1289 { 1290 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, false); 1291 } 1292 1293 static int otx_cpt_aead_null_encrypt(struct aead_request *req) 1294 { 1295 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, true); 1296 } 1297 1298 static int otx_cpt_aead_null_decrypt(struct aead_request *req) 1299 { 1300 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, false); 1301 } 1302 1303 static struct skcipher_alg otx_cpt_skciphers[] = { { 1304 .base.cra_name = "xts(aes)", 1305 .base.cra_driver_name = "cpt_xts_aes", 1306 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1307 .base.cra_blocksize = AES_BLOCK_SIZE, 1308 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx), 1309 .base.cra_alignmask = 7, 1310 .base.cra_priority = 4001, 1311 .base.cra_module = THIS_MODULE, 1312 1313 .init = otx_cpt_enc_dec_init, 1314 .ivsize = AES_BLOCK_SIZE, 1315 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1316 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1317 .setkey = otx_cpt_skcipher_xts_setkey, 1318 .encrypt = otx_cpt_skcipher_encrypt, 1319 .decrypt = otx_cpt_skcipher_decrypt, 1320 }, { 1321 .base.cra_name = "cbc(aes)", 1322 .base.cra_driver_name = "cpt_cbc_aes", 1323 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1324 .base.cra_blocksize = AES_BLOCK_SIZE, 1325 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx), 1326 .base.cra_alignmask = 7, 1327 .base.cra_priority = 4001, 1328 .base.cra_module = THIS_MODULE, 1329 1330 .init = otx_cpt_enc_dec_init, 1331 .ivsize = AES_BLOCK_SIZE, 1332 .min_keysize = AES_MIN_KEY_SIZE, 1333 .max_keysize = AES_MAX_KEY_SIZE, 1334 .setkey = otx_cpt_skcipher_cbc_aes_setkey, 1335 .encrypt = otx_cpt_skcipher_encrypt, 1336 .decrypt = otx_cpt_skcipher_decrypt, 1337 }, { 1338 .base.cra_name = "ecb(aes)", 1339 .base.cra_driver_name = "cpt_ecb_aes", 1340 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1341 .base.cra_blocksize = AES_BLOCK_SIZE, 1342 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx), 1343 .base.cra_alignmask = 7, 1344 .base.cra_priority = 4001, 1345 .base.cra_module = THIS_MODULE, 1346 1347 .init = otx_cpt_enc_dec_init, 1348 .ivsize = 0, 1349 .min_keysize = AES_MIN_KEY_SIZE, 1350 .max_keysize = AES_MAX_KEY_SIZE, 1351 .setkey = otx_cpt_skcipher_ecb_aes_setkey, 1352 .encrypt = otx_cpt_skcipher_encrypt, 1353 .decrypt = otx_cpt_skcipher_decrypt, 1354 }, { 1355 .base.cra_name = "cfb(aes)", 1356 .base.cra_driver_name = "cpt_cfb_aes", 1357 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1358 .base.cra_blocksize = AES_BLOCK_SIZE, 1359 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx), 1360 .base.cra_alignmask = 7, 1361 .base.cra_priority = 4001, 1362 .base.cra_module = THIS_MODULE, 1363 1364 .init = otx_cpt_enc_dec_init, 1365 .ivsize = AES_BLOCK_SIZE, 1366 .min_keysize = AES_MIN_KEY_SIZE, 1367 .max_keysize = AES_MAX_KEY_SIZE, 1368 .setkey = otx_cpt_skcipher_cfb_aes_setkey, 1369 .encrypt = otx_cpt_skcipher_encrypt, 1370 .decrypt = otx_cpt_skcipher_decrypt, 1371 }, { 1372 .base.cra_name = "cbc(des3_ede)", 1373 .base.cra_driver_name = "cpt_cbc_des3_ede", 1374 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1375 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 1376 .base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx), 1377 .base.cra_alignmask = 7, 1378 .base.cra_priority = 4001, 1379 .base.cra_module = THIS_MODULE, 1380 1381 .init = otx_cpt_enc_dec_init, 1382 .min_keysize = DES3_EDE_KEY_SIZE, 1383 .max_keysize = DES3_EDE_KEY_SIZE, 1384 .ivsize = DES_BLOCK_SIZE, 1385 .setkey = otx_cpt_skcipher_cbc_des3_setkey, 1386 .encrypt = otx_cpt_skcipher_encrypt, 1387 .decrypt = otx_cpt_skcipher_decrypt, 1388 }, { 1389 .base.cra_name = "ecb(des3_ede)", 1390 .base.cra_driver_name = "cpt_ecb_des3_ede", 1391 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1392 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 1393 .base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx), 1394 .base.cra_alignmask = 7, 1395 .base.cra_priority = 4001, 1396 .base.cra_module = THIS_MODULE, 1397 1398 .init = otx_cpt_enc_dec_init, 1399 .min_keysize = DES3_EDE_KEY_SIZE, 1400 .max_keysize = DES3_EDE_KEY_SIZE, 1401 .ivsize = 0, 1402 .setkey = otx_cpt_skcipher_ecb_des3_setkey, 1403 .encrypt = otx_cpt_skcipher_encrypt, 1404 .decrypt = otx_cpt_skcipher_decrypt, 1405 } }; 1406 1407 static struct aead_alg otx_cpt_aeads[] = { { 1408 .base = { 1409 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1410 .cra_driver_name = "cpt_hmac_sha1_cbc_aes", 1411 .cra_blocksize = AES_BLOCK_SIZE, 1412 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1413 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1414 .cra_priority = 4001, 1415 .cra_alignmask = 0, 1416 .cra_module = THIS_MODULE, 1417 }, 1418 .init = otx_cpt_aead_cbc_aes_sha1_init, 1419 .exit = otx_cpt_aead_exit, 1420 .setkey = otx_cpt_aead_cbc_aes_sha_setkey, 1421 .setauthsize = otx_cpt_aead_set_authsize, 1422 .encrypt = otx_cpt_aead_encrypt, 1423 .decrypt = otx_cpt_aead_decrypt, 1424 .ivsize = AES_BLOCK_SIZE, 1425 .maxauthsize = SHA1_DIGEST_SIZE, 1426 }, { 1427 .base = { 1428 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1429 .cra_driver_name = "cpt_hmac_sha256_cbc_aes", 1430 .cra_blocksize = AES_BLOCK_SIZE, 1431 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1432 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1433 .cra_priority = 4001, 1434 .cra_alignmask = 0, 1435 .cra_module = THIS_MODULE, 1436 }, 1437 .init = otx_cpt_aead_cbc_aes_sha256_init, 1438 .exit = otx_cpt_aead_exit, 1439 .setkey = otx_cpt_aead_cbc_aes_sha_setkey, 1440 .setauthsize = otx_cpt_aead_set_authsize, 1441 .encrypt = otx_cpt_aead_encrypt, 1442 .decrypt = otx_cpt_aead_decrypt, 1443 .ivsize = AES_BLOCK_SIZE, 1444 .maxauthsize = SHA256_DIGEST_SIZE, 1445 }, { 1446 .base = { 1447 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1448 .cra_driver_name = "cpt_hmac_sha384_cbc_aes", 1449 .cra_blocksize = AES_BLOCK_SIZE, 1450 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1451 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1452 .cra_priority = 4001, 1453 .cra_alignmask = 0, 1454 .cra_module = THIS_MODULE, 1455 }, 1456 .init = otx_cpt_aead_cbc_aes_sha384_init, 1457 .exit = otx_cpt_aead_exit, 1458 .setkey = otx_cpt_aead_cbc_aes_sha_setkey, 1459 .setauthsize = otx_cpt_aead_set_authsize, 1460 .encrypt = otx_cpt_aead_encrypt, 1461 .decrypt = otx_cpt_aead_decrypt, 1462 .ivsize = AES_BLOCK_SIZE, 1463 .maxauthsize = SHA384_DIGEST_SIZE, 1464 }, { 1465 .base = { 1466 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1467 .cra_driver_name = "cpt_hmac_sha512_cbc_aes", 1468 .cra_blocksize = AES_BLOCK_SIZE, 1469 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1470 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1471 .cra_priority = 4001, 1472 .cra_alignmask = 0, 1473 .cra_module = THIS_MODULE, 1474 }, 1475 .init = otx_cpt_aead_cbc_aes_sha512_init, 1476 .exit = otx_cpt_aead_exit, 1477 .setkey = otx_cpt_aead_cbc_aes_sha_setkey, 1478 .setauthsize = otx_cpt_aead_set_authsize, 1479 .encrypt = otx_cpt_aead_encrypt, 1480 .decrypt = otx_cpt_aead_decrypt, 1481 .ivsize = AES_BLOCK_SIZE, 1482 .maxauthsize = SHA512_DIGEST_SIZE, 1483 }, { 1484 .base = { 1485 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))", 1486 .cra_driver_name = "cpt_hmac_sha1_ecb_null", 1487 .cra_blocksize = 1, 1488 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1489 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1490 .cra_priority = 4001, 1491 .cra_alignmask = 0, 1492 .cra_module = THIS_MODULE, 1493 }, 1494 .init = otx_cpt_aead_ecb_null_sha1_init, 1495 .exit = otx_cpt_aead_exit, 1496 .setkey = otx_cpt_aead_ecb_null_sha_setkey, 1497 .setauthsize = otx_cpt_aead_set_authsize, 1498 .encrypt = otx_cpt_aead_null_encrypt, 1499 .decrypt = otx_cpt_aead_null_decrypt, 1500 .ivsize = 0, 1501 .maxauthsize = SHA1_DIGEST_SIZE, 1502 }, { 1503 .base = { 1504 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))", 1505 .cra_driver_name = "cpt_hmac_sha256_ecb_null", 1506 .cra_blocksize = 1, 1507 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1508 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1509 .cra_priority = 4001, 1510 .cra_alignmask = 0, 1511 .cra_module = THIS_MODULE, 1512 }, 1513 .init = otx_cpt_aead_ecb_null_sha256_init, 1514 .exit = otx_cpt_aead_exit, 1515 .setkey = otx_cpt_aead_ecb_null_sha_setkey, 1516 .setauthsize = otx_cpt_aead_set_authsize, 1517 .encrypt = otx_cpt_aead_null_encrypt, 1518 .decrypt = otx_cpt_aead_null_decrypt, 1519 .ivsize = 0, 1520 .maxauthsize = SHA256_DIGEST_SIZE, 1521 }, { 1522 .base = { 1523 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))", 1524 .cra_driver_name = "cpt_hmac_sha384_ecb_null", 1525 .cra_blocksize = 1, 1526 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1527 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1528 .cra_priority = 4001, 1529 .cra_alignmask = 0, 1530 .cra_module = THIS_MODULE, 1531 }, 1532 .init = otx_cpt_aead_ecb_null_sha384_init, 1533 .exit = otx_cpt_aead_exit, 1534 .setkey = otx_cpt_aead_ecb_null_sha_setkey, 1535 .setauthsize = otx_cpt_aead_set_authsize, 1536 .encrypt = otx_cpt_aead_null_encrypt, 1537 .decrypt = otx_cpt_aead_null_decrypt, 1538 .ivsize = 0, 1539 .maxauthsize = SHA384_DIGEST_SIZE, 1540 }, { 1541 .base = { 1542 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))", 1543 .cra_driver_name = "cpt_hmac_sha512_ecb_null", 1544 .cra_blocksize = 1, 1545 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1546 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1547 .cra_priority = 4001, 1548 .cra_alignmask = 0, 1549 .cra_module = THIS_MODULE, 1550 }, 1551 .init = otx_cpt_aead_ecb_null_sha512_init, 1552 .exit = otx_cpt_aead_exit, 1553 .setkey = otx_cpt_aead_ecb_null_sha_setkey, 1554 .setauthsize = otx_cpt_aead_set_authsize, 1555 .encrypt = otx_cpt_aead_null_encrypt, 1556 .decrypt = otx_cpt_aead_null_decrypt, 1557 .ivsize = 0, 1558 .maxauthsize = SHA512_DIGEST_SIZE, 1559 }, { 1560 .base = { 1561 .cra_name = "rfc4106(gcm(aes))", 1562 .cra_driver_name = "cpt_rfc4106_gcm_aes", 1563 .cra_blocksize = 1, 1564 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 1565 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING, 1566 .cra_priority = 4001, 1567 .cra_alignmask = 0, 1568 .cra_module = THIS_MODULE, 1569 }, 1570 .init = otx_cpt_aead_gcm_aes_init, 1571 .exit = otx_cpt_aead_exit, 1572 .setkey = otx_cpt_aead_gcm_aes_setkey, 1573 .setauthsize = otx_cpt_aead_set_authsize, 1574 .encrypt = otx_cpt_aead_encrypt, 1575 .decrypt = otx_cpt_aead_decrypt, 1576 .ivsize = AES_GCM_IV_SIZE, 1577 .maxauthsize = AES_GCM_ICV_SIZE, 1578 } }; 1579 1580 static inline int is_any_alg_used(void) 1581 { 1582 int i; 1583 1584 for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++) 1585 if (refcount_read(&otx_cpt_skciphers[i].base.cra_refcnt) != 1) 1586 return true; 1587 for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++) 1588 if (refcount_read(&otx_cpt_aeads[i].base.cra_refcnt) != 1) 1589 return true; 1590 return false; 1591 } 1592 1593 static inline int cpt_register_algs(void) 1594 { 1595 int i, err = 0; 1596 1597 if (!IS_ENABLED(CONFIG_DM_CRYPT)) { 1598 for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++) 1599 otx_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; 1600 1601 err = crypto_register_skciphers(otx_cpt_skciphers, 1602 ARRAY_SIZE(otx_cpt_skciphers)); 1603 if (err) 1604 return err; 1605 } 1606 1607 for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++) 1608 otx_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; 1609 1610 err = crypto_register_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads)); 1611 if (err) { 1612 crypto_unregister_skciphers(otx_cpt_skciphers, 1613 ARRAY_SIZE(otx_cpt_skciphers)); 1614 return err; 1615 } 1616 1617 return 0; 1618 } 1619 1620 static inline void cpt_unregister_algs(void) 1621 { 1622 crypto_unregister_skciphers(otx_cpt_skciphers, 1623 ARRAY_SIZE(otx_cpt_skciphers)); 1624 crypto_unregister_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads)); 1625 } 1626 1627 static int compare_func(const void *lptr, const void *rptr) 1628 { 1629 struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr; 1630 struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr; 1631 1632 if (ldesc->dev->devfn < rdesc->dev->devfn) 1633 return -1; 1634 if (ldesc->dev->devfn > rdesc->dev->devfn) 1635 return 1; 1636 return 0; 1637 } 1638 1639 static void swap_func(void *lptr, void *rptr, int size) 1640 { 1641 struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr; 1642 struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr; 1643 1644 swap(*ldesc, *rdesc); 1645 } 1646 1647 int otx_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, 1648 enum otx_cptpf_type pf_type, 1649 enum otx_cptvf_type engine_type, 1650 int num_queues, int num_devices) 1651 { 1652 int ret = 0; 1653 int count; 1654 1655 mutex_lock(&mutex); 1656 switch (engine_type) { 1657 case OTX_CPT_SE_TYPES: 1658 count = atomic_read(&se_devices.count); 1659 if (count >= CPT_MAX_VF_NUM) { 1660 dev_err(&pdev->dev, "No space to add a new device\n"); 1661 ret = -ENOSPC; 1662 goto err; 1663 } 1664 se_devices.desc[count].pf_type = pf_type; 1665 se_devices.desc[count].num_queues = num_queues; 1666 se_devices.desc[count++].dev = pdev; 1667 atomic_inc(&se_devices.count); 1668 1669 if (atomic_read(&se_devices.count) == num_devices && 1670 is_crypto_registered == false) { 1671 if (cpt_register_algs()) { 1672 dev_err(&pdev->dev, 1673 "Error in registering crypto algorithms\n"); 1674 ret = -EINVAL; 1675 goto err; 1676 } 1677 try_module_get(mod); 1678 is_crypto_registered = true; 1679 } 1680 sort(se_devices.desc, count, sizeof(struct cpt_device_desc), 1681 compare_func, swap_func); 1682 break; 1683 1684 case OTX_CPT_AE_TYPES: 1685 count = atomic_read(&ae_devices.count); 1686 if (count >= CPT_MAX_VF_NUM) { 1687 dev_err(&pdev->dev, "No space to a add new device\n"); 1688 ret = -ENOSPC; 1689 goto err; 1690 } 1691 ae_devices.desc[count].pf_type = pf_type; 1692 ae_devices.desc[count].num_queues = num_queues; 1693 ae_devices.desc[count++].dev = pdev; 1694 atomic_inc(&ae_devices.count); 1695 sort(ae_devices.desc, count, sizeof(struct cpt_device_desc), 1696 compare_func, swap_func); 1697 break; 1698 1699 default: 1700 dev_err(&pdev->dev, "Unknown VF type %d\n", engine_type); 1701 ret = BAD_OTX_CPTVF_TYPE; 1702 } 1703 err: 1704 mutex_unlock(&mutex); 1705 return ret; 1706 } 1707 1708 void otx_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod, 1709 enum otx_cptvf_type engine_type) 1710 { 1711 struct cpt_device_table *dev_tbl; 1712 bool dev_found = false; 1713 int i, j, count; 1714 1715 mutex_lock(&mutex); 1716 1717 dev_tbl = (engine_type == OTX_CPT_AE_TYPES) ? &ae_devices : &se_devices; 1718 count = atomic_read(&dev_tbl->count); 1719 for (i = 0; i < count; i++) 1720 if (pdev == dev_tbl->desc[i].dev) { 1721 for (j = i; j < count-1; j++) 1722 dev_tbl->desc[j] = dev_tbl->desc[j+1]; 1723 dev_found = true; 1724 break; 1725 } 1726 1727 if (!dev_found) { 1728 dev_err(&pdev->dev, "%s device not found\n", __func__); 1729 goto exit; 1730 } 1731 1732 if (engine_type != OTX_CPT_AE_TYPES) { 1733 if (atomic_dec_and_test(&se_devices.count) && 1734 !is_any_alg_used()) { 1735 cpt_unregister_algs(); 1736 module_put(mod); 1737 is_crypto_registered = false; 1738 } 1739 } else 1740 atomic_dec(&ae_devices.count); 1741 exit: 1742 mutex_unlock(&mutex); 1743 } 1744