1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Copyright (C) 2016 Cavium, Inc. 5 */ 6 7 #include <crypto/aes.h> 8 #include <crypto/algapi.h> 9 #include <crypto/authenc.h> 10 #include <crypto/internal/des.h> 11 #include <crypto/xts.h> 12 #include <linux/crypto.h> 13 #include <linux/err.h> 14 #include <linux/list.h> 15 #include <linux/scatterlist.h> 16 17 #include "cptvf.h" 18 #include "cptvf_algs.h" 19 20 struct cpt_device_handle { 21 void *cdev[MAX_DEVICES]; 22 u32 dev_count; 23 }; 24 25 static struct cpt_device_handle dev_handle; 26 27 static void cvm_callback(u32 status, void *arg) 28 { 29 struct crypto_async_request *req = (struct crypto_async_request *)arg; 30 31 crypto_request_complete(req, !status); 32 } 33 34 static inline void update_input_iv(struct cpt_request_info *req_info, 35 u8 *iv, u32 enc_iv_len, 36 u32 *argcnt) 37 { 38 /* Setting the iv information */ 39 req_info->in[*argcnt].vptr = (void *)iv; 40 req_info->in[*argcnt].size = enc_iv_len; 41 req_info->req.dlen += enc_iv_len; 42 43 ++(*argcnt); 44 } 45 46 static inline void update_output_iv(struct cpt_request_info *req_info, 47 u8 *iv, u32 enc_iv_len, 48 u32 *argcnt) 49 { 50 /* Setting the iv information */ 51 req_info->out[*argcnt].vptr = (void *)iv; 52 req_info->out[*argcnt].size = enc_iv_len; 53 req_info->rlen += enc_iv_len; 54 55 ++(*argcnt); 56 } 57 58 static inline void update_input_data(struct cpt_request_info *req_info, 59 struct scatterlist *inp_sg, 60 u32 nbytes, u32 *argcnt) 61 { 62 req_info->req.dlen += nbytes; 63 64 while (nbytes) { 65 u32 len = min(nbytes, inp_sg->length); 66 u8 *ptr = sg_virt(inp_sg); 67 68 req_info->in[*argcnt].vptr = (void *)ptr; 69 req_info->in[*argcnt].size = len; 70 nbytes -= len; 71 72 ++(*argcnt); 73 ++inp_sg; 74 } 75 } 76 77 static inline void update_output_data(struct cpt_request_info *req_info, 78 struct scatterlist *outp_sg, 79 u32 nbytes, u32 *argcnt) 80 { 81 req_info->rlen += nbytes; 82 83 while (nbytes) { 84 u32 len = min(nbytes, outp_sg->length); 85 u8 *ptr = sg_virt(outp_sg); 86 87 req_info->out[*argcnt].vptr = (void *)ptr; 88 req_info->out[*argcnt].size = len; 89 nbytes -= len; 90 ++(*argcnt); 91 ++outp_sg; 92 } 93 } 94 95 static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, 96 u32 *argcnt) 97 { 98 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 99 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 100 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); 101 struct fc_context *fctx = &rctx->fctx; 102 u32 enc_iv_len = crypto_skcipher_ivsize(tfm); 103 struct cpt_request_info *req_info = &rctx->cpt_req; 104 __be64 *ctrl_flags = NULL; 105 __be64 *offset_control; 106 107 req_info->ctrl.s.grp = 0; 108 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER; 109 req_info->ctrl.s.se_req = SE_CORE_REQ; 110 111 req_info->req.opcode.s.major = MAJOR_OP_FC | 112 DMA_MODE_FLAG(DMA_GATHER_SCATTER); 113 if (enc) 114 req_info->req.opcode.s.minor = 2; 115 else 116 req_info->req.opcode.s.minor = 3; 117 118 req_info->req.param1 = req->cryptlen; /* Encryption Data length */ 119 req_info->req.param2 = 0; /*Auth data length */ 120 121 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 122 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 123 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR; 124 125 if (ctx->cipher_type == AES_XTS) 126 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); 127 else 128 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); 129 ctrl_flags = (__be64 *)&fctx->enc.enc_ctrl.flags; 130 *ctrl_flags = cpu_to_be64(fctx->enc.enc_ctrl.flags); 131 132 offset_control = (__be64 *)&rctx->control_word; 133 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16)); 134 /* Storing Packet Data Information in offset 135 * Control Word First 8 bytes 136 */ 137 req_info->in[*argcnt].vptr = (u8 *)offset_control; 138 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 139 req_info->req.dlen += CONTROL_WORD_LEN; 140 ++(*argcnt); 141 142 req_info->in[*argcnt].vptr = (u8 *)fctx; 143 req_info->in[*argcnt].size = sizeof(struct fc_context); 144 req_info->req.dlen += sizeof(struct fc_context); 145 146 ++(*argcnt); 147 148 return 0; 149 } 150 151 static inline u32 create_input_list(struct skcipher_request *req, u32 enc, 152 u32 enc_iv_len) 153 { 154 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); 155 struct cpt_request_info *req_info = &rctx->cpt_req; 156 u32 argcnt = 0; 157 158 create_ctx_hdr(req, enc, &argcnt); 159 update_input_iv(req_info, req->iv, enc_iv_len, &argcnt); 160 update_input_data(req_info, req->src, req->cryptlen, &argcnt); 161 req_info->incnt = argcnt; 162 163 return 0; 164 } 165 166 static inline void store_cb_info(struct skcipher_request *req, 167 struct cpt_request_info *req_info) 168 { 169 req_info->callback = (void *)cvm_callback; 170 req_info->callback_arg = (void *)&req->base; 171 } 172 173 static inline void create_output_list(struct skcipher_request *req, 174 u32 enc_iv_len) 175 { 176 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); 177 struct cpt_request_info *req_info = &rctx->cpt_req; 178 u32 argcnt = 0; 179 180 /* OUTPUT Buffer Processing 181 * AES encryption/decryption output would be 182 * received in the following format 183 * 184 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| 185 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] 186 */ 187 /* Reading IV information */ 188 update_output_iv(req_info, req->iv, enc_iv_len, &argcnt); 189 update_output_data(req_info, req->dst, req->cryptlen, &argcnt); 190 req_info->outcnt = argcnt; 191 } 192 193 static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc) 194 { 195 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 196 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); 197 u32 enc_iv_len = crypto_skcipher_ivsize(tfm); 198 struct fc_context *fctx = &rctx->fctx; 199 struct cpt_request_info *req_info = &rctx->cpt_req; 200 void *cdev = NULL; 201 int status; 202 203 memset(req_info, 0, sizeof(struct cpt_request_info)); 204 req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0; 205 memset(fctx, 0, sizeof(struct fc_context)); 206 create_input_list(req, enc, enc_iv_len); 207 create_output_list(req, enc_iv_len); 208 store_cb_info(req, req_info); 209 cdev = dev_handle.cdev[smp_processor_id()]; 210 status = cptvf_do_request(cdev, req_info); 211 /* We perform an asynchronous send and once 212 * the request is completed the driver would 213 * intimate through registered call back functions 214 */ 215 216 if (status) 217 return status; 218 else 219 return -EINPROGRESS; 220 } 221 222 static int cvm_encrypt(struct skcipher_request *req) 223 { 224 return cvm_enc_dec(req, true); 225 } 226 227 static int cvm_decrypt(struct skcipher_request *req) 228 { 229 return cvm_enc_dec(req, false); 230 } 231 232 static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, 233 u32 keylen) 234 { 235 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher); 236 int err; 237 const u8 *key1 = key; 238 const u8 *key2 = key + (keylen / 2); 239 240 err = xts_verify_key(cipher, key, keylen); 241 if (err) 242 return err; 243 ctx->key_len = keylen; 244 memcpy(ctx->enc_key, key1, keylen / 2); 245 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); 246 ctx->cipher_type = AES_XTS; 247 switch (ctx->key_len) { 248 case 32: 249 ctx->key_type = AES_128_BIT; 250 break; 251 case 64: 252 ctx->key_type = AES_256_BIT; 253 break; 254 default: 255 return -EINVAL; 256 } 257 258 return 0; 259 } 260 261 static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen) 262 { 263 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) { 264 ctx->key_len = keylen; 265 switch (ctx->key_len) { 266 case 16: 267 ctx->key_type = AES_128_BIT; 268 break; 269 case 24: 270 ctx->key_type = AES_192_BIT; 271 break; 272 case 32: 273 ctx->key_type = AES_256_BIT; 274 break; 275 default: 276 return -EINVAL; 277 } 278 279 if (ctx->cipher_type == DES3_CBC) 280 ctx->key_type = 0; 281 282 return 0; 283 } 284 285 return -EINVAL; 286 } 287 288 static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key, 289 u32 keylen, u8 cipher_type) 290 { 291 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher); 292 293 ctx->cipher_type = cipher_type; 294 if (!cvm_validate_keylen(ctx, keylen)) { 295 memcpy(ctx->enc_key, key, keylen); 296 return 0; 297 } else { 298 return -EINVAL; 299 } 300 } 301 302 static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 303 u32 keylen) 304 { 305 return cvm_setkey(cipher, key, keylen, AES_CBC); 306 } 307 308 static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 309 u32 keylen) 310 { 311 return cvm_setkey(cipher, key, keylen, AES_ECB); 312 } 313 314 static int cvm_cfb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 315 u32 keylen) 316 { 317 return cvm_setkey(cipher, key, keylen, AES_CFB); 318 } 319 320 static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 321 u32 keylen) 322 { 323 return verify_skcipher_des3_key(cipher, key) ?: 324 cvm_setkey(cipher, key, keylen, DES3_CBC); 325 } 326 327 static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 328 u32 keylen) 329 { 330 return verify_skcipher_des3_key(cipher, key) ?: 331 cvm_setkey(cipher, key, keylen, DES3_ECB); 332 } 333 334 static int cvm_enc_dec_init(struct crypto_skcipher *tfm) 335 { 336 crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct cvm_req_ctx)); 337 338 return 0; 339 } 340 341 static struct skcipher_alg algs[] = { { 342 .base.cra_flags = CRYPTO_ALG_ASYNC | 343 CRYPTO_ALG_ALLOCATES_MEMORY, 344 .base.cra_blocksize = AES_BLOCK_SIZE, 345 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 346 .base.cra_alignmask = 7, 347 .base.cra_priority = 4001, 348 .base.cra_name = "xts(aes)", 349 .base.cra_driver_name = "cavium-xts-aes", 350 .base.cra_module = THIS_MODULE, 351 352 .ivsize = AES_BLOCK_SIZE, 353 .min_keysize = 2 * AES_MIN_KEY_SIZE, 354 .max_keysize = 2 * AES_MAX_KEY_SIZE, 355 .setkey = cvm_xts_setkey, 356 .encrypt = cvm_encrypt, 357 .decrypt = cvm_decrypt, 358 .init = cvm_enc_dec_init, 359 }, { 360 .base.cra_flags = CRYPTO_ALG_ASYNC | 361 CRYPTO_ALG_ALLOCATES_MEMORY, 362 .base.cra_blocksize = AES_BLOCK_SIZE, 363 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 364 .base.cra_alignmask = 7, 365 .base.cra_priority = 4001, 366 .base.cra_name = "cbc(aes)", 367 .base.cra_driver_name = "cavium-cbc-aes", 368 .base.cra_module = THIS_MODULE, 369 370 .ivsize = AES_BLOCK_SIZE, 371 .min_keysize = AES_MIN_KEY_SIZE, 372 .max_keysize = AES_MAX_KEY_SIZE, 373 .setkey = cvm_cbc_aes_setkey, 374 .encrypt = cvm_encrypt, 375 .decrypt = cvm_decrypt, 376 .init = cvm_enc_dec_init, 377 }, { 378 .base.cra_flags = CRYPTO_ALG_ASYNC | 379 CRYPTO_ALG_ALLOCATES_MEMORY, 380 .base.cra_blocksize = AES_BLOCK_SIZE, 381 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 382 .base.cra_alignmask = 7, 383 .base.cra_priority = 4001, 384 .base.cra_name = "ecb(aes)", 385 .base.cra_driver_name = "cavium-ecb-aes", 386 .base.cra_module = THIS_MODULE, 387 388 .min_keysize = AES_MIN_KEY_SIZE, 389 .max_keysize = AES_MAX_KEY_SIZE, 390 .setkey = cvm_ecb_aes_setkey, 391 .encrypt = cvm_encrypt, 392 .decrypt = cvm_decrypt, 393 .init = cvm_enc_dec_init, 394 }, { 395 .base.cra_flags = CRYPTO_ALG_ASYNC | 396 CRYPTO_ALG_ALLOCATES_MEMORY, 397 .base.cra_blocksize = AES_BLOCK_SIZE, 398 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 399 .base.cra_alignmask = 7, 400 .base.cra_priority = 4001, 401 .base.cra_name = "cfb(aes)", 402 .base.cra_driver_name = "cavium-cfb-aes", 403 .base.cra_module = THIS_MODULE, 404 405 .ivsize = AES_BLOCK_SIZE, 406 .min_keysize = AES_MIN_KEY_SIZE, 407 .max_keysize = AES_MAX_KEY_SIZE, 408 .setkey = cvm_cfb_aes_setkey, 409 .encrypt = cvm_encrypt, 410 .decrypt = cvm_decrypt, 411 .init = cvm_enc_dec_init, 412 }, { 413 .base.cra_flags = CRYPTO_ALG_ASYNC | 414 CRYPTO_ALG_ALLOCATES_MEMORY, 415 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 416 .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), 417 .base.cra_alignmask = 7, 418 .base.cra_priority = 4001, 419 .base.cra_name = "cbc(des3_ede)", 420 .base.cra_driver_name = "cavium-cbc-des3_ede", 421 .base.cra_module = THIS_MODULE, 422 423 .min_keysize = DES3_EDE_KEY_SIZE, 424 .max_keysize = DES3_EDE_KEY_SIZE, 425 .ivsize = DES_BLOCK_SIZE, 426 .setkey = cvm_cbc_des3_setkey, 427 .encrypt = cvm_encrypt, 428 .decrypt = cvm_decrypt, 429 .init = cvm_enc_dec_init, 430 }, { 431 .base.cra_flags = CRYPTO_ALG_ASYNC | 432 CRYPTO_ALG_ALLOCATES_MEMORY, 433 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 434 .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), 435 .base.cra_alignmask = 7, 436 .base.cra_priority = 4001, 437 .base.cra_name = "ecb(des3_ede)", 438 .base.cra_driver_name = "cavium-ecb-des3_ede", 439 .base.cra_module = THIS_MODULE, 440 441 .min_keysize = DES3_EDE_KEY_SIZE, 442 .max_keysize = DES3_EDE_KEY_SIZE, 443 .ivsize = DES_BLOCK_SIZE, 444 .setkey = cvm_ecb_des3_setkey, 445 .encrypt = cvm_encrypt, 446 .decrypt = cvm_decrypt, 447 .init = cvm_enc_dec_init, 448 } }; 449 450 static inline int cav_register_algs(void) 451 { 452 return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); 453 } 454 455 static inline void cav_unregister_algs(void) 456 { 457 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); 458 } 459 460 int cvm_crypto_init(struct cpt_vf *cptvf) 461 { 462 struct pci_dev *pdev = cptvf->pdev; 463 u32 dev_count; 464 465 dev_count = dev_handle.dev_count; 466 dev_handle.cdev[dev_count] = cptvf; 467 dev_handle.dev_count++; 468 469 if (dev_count == 3) { 470 if (cav_register_algs()) { 471 dev_err(&pdev->dev, "Error in registering crypto algorithms\n"); 472 return -EINVAL; 473 } 474 } 475 476 return 0; 477 } 478 479 void cvm_crypto_exit(void) 480 { 481 u32 dev_count; 482 483 dev_count = --dev_handle.dev_count; 484 if (!dev_count) 485 cav_unregister_algs(); 486 } 487