1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Copyright (C) 2016 Cavium, Inc. 5 */ 6 7 #include <crypto/aes.h> 8 #include <crypto/algapi.h> 9 #include <crypto/authenc.h> 10 #include <crypto/internal/des.h> 11 #include <crypto/xts.h> 12 #include <linux/crypto.h> 13 #include <linux/err.h> 14 #include <linux/list.h> 15 #include <linux/scatterlist.h> 16 17 #include "cptvf.h" 18 #include "cptvf_algs.h" 19 20 struct cpt_device_handle { 21 void *cdev[MAX_DEVICES]; 22 u32 dev_count; 23 }; 24 25 static struct cpt_device_handle dev_handle; 26 27 static void cvm_callback(u32 status, void *arg) 28 { 29 struct crypto_async_request *req = (struct crypto_async_request *)arg; 30 31 req->complete(req, !status); 32 } 33 34 static inline void update_input_iv(struct cpt_request_info *req_info, 35 u8 *iv, u32 enc_iv_len, 36 u32 *argcnt) 37 { 38 /* Setting the iv information */ 39 req_info->in[*argcnt].vptr = (void *)iv; 40 req_info->in[*argcnt].size = enc_iv_len; 41 req_info->req.dlen += enc_iv_len; 42 43 ++(*argcnt); 44 } 45 46 static inline void update_output_iv(struct cpt_request_info *req_info, 47 u8 *iv, u32 enc_iv_len, 48 u32 *argcnt) 49 { 50 /* Setting the iv information */ 51 req_info->out[*argcnt].vptr = (void *)iv; 52 req_info->out[*argcnt].size = enc_iv_len; 53 req_info->rlen += enc_iv_len; 54 55 ++(*argcnt); 56 } 57 58 static inline void update_input_data(struct cpt_request_info *req_info, 59 struct scatterlist *inp_sg, 60 u32 nbytes, u32 *argcnt) 61 { 62 req_info->req.dlen += nbytes; 63 64 while (nbytes) { 65 u32 len = min(nbytes, inp_sg->length); 66 u8 *ptr = sg_virt(inp_sg); 67 68 req_info->in[*argcnt].vptr = (void *)ptr; 69 req_info->in[*argcnt].size = len; 70 nbytes -= len; 71 72 ++(*argcnt); 73 ++inp_sg; 74 } 75 } 76 77 static inline void update_output_data(struct cpt_request_info *req_info, 78 struct scatterlist *outp_sg, 79 u32 nbytes, u32 *argcnt) 80 { 81 req_info->rlen += nbytes; 82 83 while (nbytes) { 84 u32 len = min(nbytes, outp_sg->length); 85 u8 *ptr = sg_virt(outp_sg); 86 87 req_info->out[*argcnt].vptr = (void *)ptr; 88 req_info->out[*argcnt].size = len; 89 nbytes -= len; 90 ++(*argcnt); 91 ++outp_sg; 92 } 93 } 94 95 static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, 96 u32 *argcnt) 97 { 98 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 99 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 100 struct cvm_req_ctx *rctx = skcipher_request_ctx(req); 101 struct fc_context *fctx = &rctx->fctx; 102 u64 *offset_control = &rctx->control_word; 103 u32 enc_iv_len = crypto_skcipher_ivsize(tfm); 104 struct cpt_request_info *req_info = &rctx->cpt_req; 105 u64 *ctrl_flags = NULL; 106 107 req_info->ctrl.s.grp = 0; 108 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER; 109 req_info->ctrl.s.se_req = SE_CORE_REQ; 110 111 req_info->req.opcode.s.major = MAJOR_OP_FC | 112 DMA_MODE_FLAG(DMA_GATHER_SCATTER); 113 if (enc) 114 req_info->req.opcode.s.minor = 2; 115 else 116 req_info->req.opcode.s.minor = 3; 117 118 req_info->req.param1 = req->cryptlen; /* Encryption Data length */ 119 req_info->req.param2 = 0; /*Auth data length */ 120 121 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 122 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 123 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR; 124 125 if (ctx->cipher_type == AES_XTS) 126 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); 127 else 128 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); 129 ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags; 130 *ctrl_flags = cpu_to_be64(*ctrl_flags); 131 132 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16)); 133 /* Storing Packet Data Information in offset 134 * Control Word First 8 bytes 135 */ 136 req_info->in[*argcnt].vptr = (u8 *)offset_control; 137 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 138 req_info->req.dlen += CONTROL_WORD_LEN; 139 ++(*argcnt); 140 141 req_info->in[*argcnt].vptr = (u8 *)fctx; 142 req_info->in[*argcnt].size = sizeof(struct fc_context); 143 req_info->req.dlen += sizeof(struct fc_context); 144 145 ++(*argcnt); 146 147 return 0; 148 } 149 150 static inline u32 create_input_list(struct skcipher_request *req, u32 enc, 151 u32 enc_iv_len) 152 { 153 struct cvm_req_ctx *rctx = skcipher_request_ctx(req); 154 struct cpt_request_info *req_info = &rctx->cpt_req; 155 u32 argcnt = 0; 156 157 create_ctx_hdr(req, enc, &argcnt); 158 update_input_iv(req_info, req->iv, enc_iv_len, &argcnt); 159 update_input_data(req_info, req->src, req->cryptlen, &argcnt); 160 req_info->incnt = argcnt; 161 162 return 0; 163 } 164 165 static inline void store_cb_info(struct skcipher_request *req, 166 struct cpt_request_info *req_info) 167 { 168 req_info->callback = (void *)cvm_callback; 169 req_info->callback_arg = (void *)&req->base; 170 } 171 172 static inline void create_output_list(struct skcipher_request *req, 173 u32 enc_iv_len) 174 { 175 struct cvm_req_ctx *rctx = skcipher_request_ctx(req); 176 struct cpt_request_info *req_info = &rctx->cpt_req; 177 u32 argcnt = 0; 178 179 /* OUTPUT Buffer Processing 180 * AES encryption/decryption output would be 181 * received in the following format 182 * 183 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| 184 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] 185 */ 186 /* Reading IV information */ 187 update_output_iv(req_info, req->iv, enc_iv_len, &argcnt); 188 update_output_data(req_info, req->dst, req->cryptlen, &argcnt); 189 req_info->outcnt = argcnt; 190 } 191 192 static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc) 193 { 194 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 195 struct cvm_req_ctx *rctx = skcipher_request_ctx(req); 196 u32 enc_iv_len = crypto_skcipher_ivsize(tfm); 197 struct fc_context *fctx = &rctx->fctx; 198 struct cpt_request_info *req_info = &rctx->cpt_req; 199 void *cdev = NULL; 200 int status; 201 202 memset(req_info, 0, sizeof(struct cpt_request_info)); 203 memset(fctx, 0, sizeof(struct fc_context)); 204 create_input_list(req, enc, enc_iv_len); 205 create_output_list(req, enc_iv_len); 206 store_cb_info(req, req_info); 207 cdev = dev_handle.cdev[smp_processor_id()]; 208 status = cptvf_do_request(cdev, req_info); 209 /* We perform an asynchronous send and once 210 * the request is completed the driver would 211 * intimate through registered call back functions 212 */ 213 214 if (status) 215 return status; 216 else 217 return -EINPROGRESS; 218 } 219 220 static int cvm_encrypt(struct skcipher_request *req) 221 { 222 return cvm_enc_dec(req, true); 223 } 224 225 static int cvm_decrypt(struct skcipher_request *req) 226 { 227 return cvm_enc_dec(req, false); 228 } 229 230 static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, 231 u32 keylen) 232 { 233 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 234 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm); 235 int err; 236 const u8 *key1 = key; 237 const u8 *key2 = key + (keylen / 2); 238 239 err = xts_check_key(tfm, key, keylen); 240 if (err) 241 return err; 242 ctx->key_len = keylen; 243 memcpy(ctx->enc_key, key1, keylen / 2); 244 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); 245 ctx->cipher_type = AES_XTS; 246 switch (ctx->key_len) { 247 case 32: 248 ctx->key_type = AES_128_BIT; 249 break; 250 case 64: 251 ctx->key_type = AES_256_BIT; 252 break; 253 default: 254 return -EINVAL; 255 } 256 257 return 0; 258 } 259 260 static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen) 261 { 262 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) { 263 ctx->key_len = keylen; 264 switch (ctx->key_len) { 265 case 16: 266 ctx->key_type = AES_128_BIT; 267 break; 268 case 24: 269 ctx->key_type = AES_192_BIT; 270 break; 271 case 32: 272 ctx->key_type = AES_256_BIT; 273 break; 274 default: 275 return -EINVAL; 276 } 277 278 if (ctx->cipher_type == DES3_CBC) 279 ctx->key_type = 0; 280 281 return 0; 282 } 283 284 return -EINVAL; 285 } 286 287 static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key, 288 u32 keylen, u8 cipher_type) 289 { 290 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 291 struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm); 292 293 ctx->cipher_type = cipher_type; 294 if (!cvm_validate_keylen(ctx, keylen)) { 295 memcpy(ctx->enc_key, key, keylen); 296 return 0; 297 } else { 298 return -EINVAL; 299 } 300 } 301 302 static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 303 u32 keylen) 304 { 305 return cvm_setkey(cipher, key, keylen, AES_CBC); 306 } 307 308 static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 309 u32 keylen) 310 { 311 return cvm_setkey(cipher, key, keylen, AES_ECB); 312 } 313 314 static int cvm_cfb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 315 u32 keylen) 316 { 317 return cvm_setkey(cipher, key, keylen, AES_CFB); 318 } 319 320 static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 321 u32 keylen) 322 { 323 return verify_skcipher_des3_key(cipher, key) ?: 324 cvm_setkey(cipher, key, keylen, DES3_CBC); 325 } 326 327 static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 328 u32 keylen) 329 { 330 return verify_skcipher_des3_key(cipher, key) ?: 331 cvm_setkey(cipher, key, keylen, DES3_ECB); 332 } 333 334 static int cvm_enc_dec_init(struct crypto_skcipher *tfm) 335 { 336 crypto_skcipher_set_reqsize(tfm, sizeof(struct cvm_req_ctx)); 337 338 return 0; 339 } 340 341 static struct skcipher_alg algs[] = { { 342 .base.cra_flags = CRYPTO_ALG_ASYNC, 343 .base.cra_blocksize = AES_BLOCK_SIZE, 344 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 345 .base.cra_alignmask = 7, 346 .base.cra_priority = 4001, 347 .base.cra_name = "xts(aes)", 348 .base.cra_driver_name = "cavium-xts-aes", 349 .base.cra_module = THIS_MODULE, 350 351 .ivsize = AES_BLOCK_SIZE, 352 .min_keysize = 2 * AES_MIN_KEY_SIZE, 353 .max_keysize = 2 * AES_MAX_KEY_SIZE, 354 .setkey = cvm_xts_setkey, 355 .encrypt = cvm_encrypt, 356 .decrypt = cvm_decrypt, 357 .init = cvm_enc_dec_init, 358 }, { 359 .base.cra_flags = CRYPTO_ALG_ASYNC, 360 .base.cra_blocksize = AES_BLOCK_SIZE, 361 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 362 .base.cra_alignmask = 7, 363 .base.cra_priority = 4001, 364 .base.cra_name = "cbc(aes)", 365 .base.cra_driver_name = "cavium-cbc-aes", 366 .base.cra_module = THIS_MODULE, 367 368 .ivsize = AES_BLOCK_SIZE, 369 .min_keysize = AES_MIN_KEY_SIZE, 370 .max_keysize = AES_MAX_KEY_SIZE, 371 .setkey = cvm_cbc_aes_setkey, 372 .encrypt = cvm_encrypt, 373 .decrypt = cvm_decrypt, 374 .init = cvm_enc_dec_init, 375 }, { 376 .base.cra_flags = CRYPTO_ALG_ASYNC, 377 .base.cra_blocksize = AES_BLOCK_SIZE, 378 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 379 .base.cra_alignmask = 7, 380 .base.cra_priority = 4001, 381 .base.cra_name = "ecb(aes)", 382 .base.cra_driver_name = "cavium-ecb-aes", 383 .base.cra_module = THIS_MODULE, 384 385 .min_keysize = AES_MIN_KEY_SIZE, 386 .max_keysize = AES_MAX_KEY_SIZE, 387 .setkey = cvm_ecb_aes_setkey, 388 .encrypt = cvm_encrypt, 389 .decrypt = cvm_decrypt, 390 .init = cvm_enc_dec_init, 391 }, { 392 .base.cra_flags = CRYPTO_ALG_ASYNC, 393 .base.cra_blocksize = AES_BLOCK_SIZE, 394 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 395 .base.cra_alignmask = 7, 396 .base.cra_priority = 4001, 397 .base.cra_name = "cfb(aes)", 398 .base.cra_driver_name = "cavium-cfb-aes", 399 .base.cra_module = THIS_MODULE, 400 401 .ivsize = AES_BLOCK_SIZE, 402 .min_keysize = AES_MIN_KEY_SIZE, 403 .max_keysize = AES_MAX_KEY_SIZE, 404 .setkey = cvm_cfb_aes_setkey, 405 .encrypt = cvm_encrypt, 406 .decrypt = cvm_decrypt, 407 .init = cvm_enc_dec_init, 408 }, { 409 .base.cra_flags = CRYPTO_ALG_ASYNC, 410 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 411 .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), 412 .base.cra_alignmask = 7, 413 .base.cra_priority = 4001, 414 .base.cra_name = "cbc(des3_ede)", 415 .base.cra_driver_name = "cavium-cbc-des3_ede", 416 .base.cra_module = THIS_MODULE, 417 418 .min_keysize = DES3_EDE_KEY_SIZE, 419 .max_keysize = DES3_EDE_KEY_SIZE, 420 .ivsize = DES_BLOCK_SIZE, 421 .setkey = cvm_cbc_des3_setkey, 422 .encrypt = cvm_encrypt, 423 .decrypt = cvm_decrypt, 424 .init = cvm_enc_dec_init, 425 }, { 426 .base.cra_flags = CRYPTO_ALG_ASYNC, 427 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 428 .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), 429 .base.cra_alignmask = 7, 430 .base.cra_priority = 4001, 431 .base.cra_name = "ecb(des3_ede)", 432 .base.cra_driver_name = "cavium-ecb-des3_ede", 433 .base.cra_module = THIS_MODULE, 434 435 .min_keysize = DES3_EDE_KEY_SIZE, 436 .max_keysize = DES3_EDE_KEY_SIZE, 437 .ivsize = DES_BLOCK_SIZE, 438 .setkey = cvm_ecb_des3_setkey, 439 .encrypt = cvm_encrypt, 440 .decrypt = cvm_decrypt, 441 .init = cvm_enc_dec_init, 442 } }; 443 444 static inline int cav_register_algs(void) 445 { 446 int err = 0; 447 448 err = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); 449 if (err) 450 return err; 451 452 return 0; 453 } 454 455 static inline void cav_unregister_algs(void) 456 { 457 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); 458 } 459 460 int cvm_crypto_init(struct cpt_vf *cptvf) 461 { 462 struct pci_dev *pdev = cptvf->pdev; 463 u32 dev_count; 464 465 dev_count = dev_handle.dev_count; 466 dev_handle.cdev[dev_count] = cptvf; 467 dev_handle.dev_count++; 468 469 if (dev_count == 3) { 470 if (cav_register_algs()) { 471 dev_err(&pdev->dev, "Error in registering crypto algorithms\n"); 472 return -EINVAL; 473 } 474 } 475 476 return 0; 477 } 478 479 void cvm_crypto_exit(void) 480 { 481 u32 dev_count; 482 483 dev_count = --dev_handle.dev_count; 484 if (!dev_count) 485 cav_unregister_algs(); 486 } 487