1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * Copyright (C) 2016 Cavium, Inc. 5 */ 6 7 #include <crypto/aes.h> 8 #include <crypto/algapi.h> 9 #include <crypto/authenc.h> 10 #include <crypto/internal/des.h> 11 #include <crypto/xts.h> 12 #include <linux/crypto.h> 13 #include <linux/err.h> 14 #include <linux/list.h> 15 #include <linux/scatterlist.h> 16 17 #include "cptvf.h" 18 #include "cptvf_algs.h" 19 20 struct cpt_device_handle { 21 void *cdev[MAX_DEVICES]; 22 u32 dev_count; 23 }; 24 25 static struct cpt_device_handle dev_handle; 26 27 static void cvm_callback(u32 status, void *arg) 28 { 29 struct crypto_async_request *req = (struct crypto_async_request *)arg; 30 31 crypto_request_complete(req, !status); 32 } 33 34 static inline void update_input_iv(struct cpt_request_info *req_info, 35 u8 *iv, u32 enc_iv_len, 36 u32 *argcnt) 37 { 38 /* Setting the iv information */ 39 req_info->in[*argcnt].vptr = (void *)iv; 40 req_info->in[*argcnt].size = enc_iv_len; 41 req_info->req.dlen += enc_iv_len; 42 43 ++(*argcnt); 44 } 45 46 static inline void update_output_iv(struct cpt_request_info *req_info, 47 u8 *iv, u32 enc_iv_len, 48 u32 *argcnt) 49 { 50 /* Setting the iv information */ 51 req_info->out[*argcnt].vptr = (void *)iv; 52 req_info->out[*argcnt].size = enc_iv_len; 53 req_info->rlen += enc_iv_len; 54 55 ++(*argcnt); 56 } 57 58 static inline void update_input_data(struct cpt_request_info *req_info, 59 struct scatterlist *inp_sg, 60 u32 nbytes, u32 *argcnt) 61 { 62 req_info->req.dlen += nbytes; 63 64 while (nbytes) { 65 u32 len = min(nbytes, inp_sg->length); 66 u8 *ptr = sg_virt(inp_sg); 67 68 req_info->in[*argcnt].vptr = (void *)ptr; 69 req_info->in[*argcnt].size = len; 70 nbytes -= len; 71 72 ++(*argcnt); 73 ++inp_sg; 74 } 75 } 76 77 static inline void update_output_data(struct cpt_request_info *req_info, 78 struct scatterlist *outp_sg, 79 u32 nbytes, u32 *argcnt) 80 { 81 req_info->rlen += nbytes; 82 83 while (nbytes) { 84 u32 len = min(nbytes, outp_sg->length); 85 u8 *ptr = sg_virt(outp_sg); 86 87 req_info->out[*argcnt].vptr = (void *)ptr; 88 req_info->out[*argcnt].size = len; 89 nbytes -= len; 90 ++(*argcnt); 91 ++outp_sg; 92 } 93 } 94 95 static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, 96 u32 *argcnt) 97 { 98 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 99 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm); 100 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); 101 struct fc_context *fctx = &rctx->fctx; 102 u32 enc_iv_len = crypto_skcipher_ivsize(tfm); 103 struct cpt_request_info *req_info = &rctx->cpt_req; 104 __be64 *ctrl_flags = NULL; 105 __be64 *offset_control; 106 107 req_info->ctrl.s.grp = 0; 108 req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER; 109 req_info->ctrl.s.se_req = SE_CORE_REQ; 110 111 req_info->req.opcode.s.major = MAJOR_OP_FC | 112 DMA_MODE_FLAG(DMA_GATHER_SCATTER); 113 if (enc) 114 req_info->req.opcode.s.minor = 2; 115 else 116 req_info->req.opcode.s.minor = 3; 117 118 req_info->req.param1 = req->cryptlen; /* Encryption Data length */ 119 req_info->req.param2 = 0; /*Auth data length */ 120 121 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; 122 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; 123 fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR; 124 125 if (ctx->cipher_type == AES_XTS) 126 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); 127 else 128 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); 129 ctrl_flags = (__be64 *)&fctx->enc.enc_ctrl.flags; 130 *ctrl_flags = cpu_to_be64(fctx->enc.enc_ctrl.flags); 131 132 offset_control = (__be64 *)&rctx->control_word; 133 *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16)); 134 /* Storing Packet Data Information in offset 135 * Control Word First 8 bytes 136 */ 137 req_info->in[*argcnt].vptr = (u8 *)offset_control; 138 req_info->in[*argcnt].size = CONTROL_WORD_LEN; 139 req_info->req.dlen += CONTROL_WORD_LEN; 140 ++(*argcnt); 141 142 req_info->in[*argcnt].vptr = (u8 *)fctx; 143 req_info->in[*argcnt].size = sizeof(struct fc_context); 144 req_info->req.dlen += sizeof(struct fc_context); 145 146 ++(*argcnt); 147 148 return 0; 149 } 150 151 static inline u32 create_input_list(struct skcipher_request *req, u32 enc, 152 u32 enc_iv_len) 153 { 154 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); 155 struct cpt_request_info *req_info = &rctx->cpt_req; 156 u32 argcnt = 0; 157 158 create_ctx_hdr(req, enc, &argcnt); 159 update_input_iv(req_info, req->iv, enc_iv_len, &argcnt); 160 update_input_data(req_info, req->src, req->cryptlen, &argcnt); 161 req_info->incnt = argcnt; 162 163 return 0; 164 } 165 166 static inline void store_cb_info(struct skcipher_request *req, 167 struct cpt_request_info *req_info) 168 { 169 req_info->callback = (void *)cvm_callback; 170 req_info->callback_arg = (void *)&req->base; 171 } 172 173 static inline void create_output_list(struct skcipher_request *req, 174 u32 enc_iv_len) 175 { 176 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); 177 struct cpt_request_info *req_info = &rctx->cpt_req; 178 u32 argcnt = 0; 179 180 /* OUTPUT Buffer Processing 181 * AES encryption/decryption output would be 182 * received in the following format 183 * 184 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| 185 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] 186 */ 187 /* Reading IV information */ 188 update_output_iv(req_info, req->iv, enc_iv_len, &argcnt); 189 update_output_data(req_info, req->dst, req->cryptlen, &argcnt); 190 req_info->outcnt = argcnt; 191 } 192 193 static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc) 194 { 195 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 196 struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); 197 u32 enc_iv_len = crypto_skcipher_ivsize(tfm); 198 struct fc_context *fctx = &rctx->fctx; 199 struct cpt_request_info *req_info = &rctx->cpt_req; 200 void *cdev = NULL; 201 int status; 202 203 memset(req_info, 0, sizeof(struct cpt_request_info)); 204 req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0; 205 memset(fctx, 0, sizeof(struct fc_context)); 206 create_input_list(req, enc, enc_iv_len); 207 create_output_list(req, enc_iv_len); 208 store_cb_info(req, req_info); 209 cdev = dev_handle.cdev[smp_processor_id()]; 210 status = cptvf_do_request(cdev, req_info); 211 /* We perform an asynchronous send and once 212 * the request is completed the driver would 213 * intimate through registered call back functions 214 */ 215 216 if (status) 217 return status; 218 else 219 return -EINPROGRESS; 220 } 221 222 static int cvm_encrypt(struct skcipher_request *req) 223 { 224 return cvm_enc_dec(req, true); 225 } 226 227 static int cvm_decrypt(struct skcipher_request *req) 228 { 229 return cvm_enc_dec(req, false); 230 } 231 232 static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, 233 u32 keylen) 234 { 235 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher); 236 int err; 237 const u8 *key1 = key; 238 const u8 *key2 = key + (keylen / 2); 239 240 err = xts_verify_key(cipher, key, keylen); 241 if (err) 242 return err; 243 ctx->key_len = keylen; 244 memcpy(ctx->enc_key, key1, keylen / 2); 245 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); 246 ctx->cipher_type = AES_XTS; 247 switch (ctx->key_len) { 248 case 32: 249 ctx->key_type = AES_128_BIT; 250 break; 251 case 64: 252 ctx->key_type = AES_256_BIT; 253 break; 254 default: 255 return -EINVAL; 256 } 257 258 return 0; 259 } 260 261 static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen) 262 { 263 if ((keylen == 16) || (keylen == 24) || (keylen == 32)) { 264 ctx->key_len = keylen; 265 switch (ctx->key_len) { 266 case 16: 267 ctx->key_type = AES_128_BIT; 268 break; 269 case 24: 270 ctx->key_type = AES_192_BIT; 271 break; 272 case 32: 273 ctx->key_type = AES_256_BIT; 274 break; 275 default: 276 return -EINVAL; 277 } 278 279 if (ctx->cipher_type == DES3_CBC) 280 ctx->key_type = 0; 281 282 return 0; 283 } 284 285 return -EINVAL; 286 } 287 288 static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key, 289 u32 keylen, u8 cipher_type) 290 { 291 struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher); 292 293 ctx->cipher_type = cipher_type; 294 if (!cvm_validate_keylen(ctx, keylen)) { 295 memcpy(ctx->enc_key, key, keylen); 296 return 0; 297 } else { 298 return -EINVAL; 299 } 300 } 301 302 static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 303 u32 keylen) 304 { 305 return cvm_setkey(cipher, key, keylen, AES_CBC); 306 } 307 308 static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 309 u32 keylen) 310 { 311 return cvm_setkey(cipher, key, keylen, AES_ECB); 312 } 313 314 static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 315 u32 keylen) 316 { 317 return verify_skcipher_des3_key(cipher, key) ?: 318 cvm_setkey(cipher, key, keylen, DES3_CBC); 319 } 320 321 static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 322 u32 keylen) 323 { 324 return verify_skcipher_des3_key(cipher, key) ?: 325 cvm_setkey(cipher, key, keylen, DES3_ECB); 326 } 327 328 static int cvm_enc_dec_init(struct crypto_skcipher *tfm) 329 { 330 crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct cvm_req_ctx)); 331 332 return 0; 333 } 334 335 static struct skcipher_alg algs[] = { { 336 .base.cra_flags = CRYPTO_ALG_ASYNC | 337 CRYPTO_ALG_ALLOCATES_MEMORY, 338 .base.cra_blocksize = AES_BLOCK_SIZE, 339 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 340 .base.cra_alignmask = 7, 341 .base.cra_priority = 4001, 342 .base.cra_name = "xts(aes)", 343 .base.cra_driver_name = "cavium-xts-aes", 344 .base.cra_module = THIS_MODULE, 345 346 .ivsize = AES_BLOCK_SIZE, 347 .min_keysize = 2 * AES_MIN_KEY_SIZE, 348 .max_keysize = 2 * AES_MAX_KEY_SIZE, 349 .setkey = cvm_xts_setkey, 350 .encrypt = cvm_encrypt, 351 .decrypt = cvm_decrypt, 352 .init = cvm_enc_dec_init, 353 }, { 354 .base.cra_flags = CRYPTO_ALG_ASYNC | 355 CRYPTO_ALG_ALLOCATES_MEMORY, 356 .base.cra_blocksize = AES_BLOCK_SIZE, 357 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 358 .base.cra_alignmask = 7, 359 .base.cra_priority = 4001, 360 .base.cra_name = "cbc(aes)", 361 .base.cra_driver_name = "cavium-cbc-aes", 362 .base.cra_module = THIS_MODULE, 363 364 .ivsize = AES_BLOCK_SIZE, 365 .min_keysize = AES_MIN_KEY_SIZE, 366 .max_keysize = AES_MAX_KEY_SIZE, 367 .setkey = cvm_cbc_aes_setkey, 368 .encrypt = cvm_encrypt, 369 .decrypt = cvm_decrypt, 370 .init = cvm_enc_dec_init, 371 }, { 372 .base.cra_flags = CRYPTO_ALG_ASYNC | 373 CRYPTO_ALG_ALLOCATES_MEMORY, 374 .base.cra_blocksize = AES_BLOCK_SIZE, 375 .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), 376 .base.cra_alignmask = 7, 377 .base.cra_priority = 4001, 378 .base.cra_name = "ecb(aes)", 379 .base.cra_driver_name = "cavium-ecb-aes", 380 .base.cra_module = THIS_MODULE, 381 382 .min_keysize = AES_MIN_KEY_SIZE, 383 .max_keysize = AES_MAX_KEY_SIZE, 384 .setkey = cvm_ecb_aes_setkey, 385 .encrypt = cvm_encrypt, 386 .decrypt = cvm_decrypt, 387 .init = cvm_enc_dec_init, 388 }, { 389 .base.cra_flags = CRYPTO_ALG_ASYNC | 390 CRYPTO_ALG_ALLOCATES_MEMORY, 391 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 392 .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), 393 .base.cra_alignmask = 7, 394 .base.cra_priority = 4001, 395 .base.cra_name = "cbc(des3_ede)", 396 .base.cra_driver_name = "cavium-cbc-des3_ede", 397 .base.cra_module = THIS_MODULE, 398 399 .min_keysize = DES3_EDE_KEY_SIZE, 400 .max_keysize = DES3_EDE_KEY_SIZE, 401 .ivsize = DES_BLOCK_SIZE, 402 .setkey = cvm_cbc_des3_setkey, 403 .encrypt = cvm_encrypt, 404 .decrypt = cvm_decrypt, 405 .init = cvm_enc_dec_init, 406 }, { 407 .base.cra_flags = CRYPTO_ALG_ASYNC | 408 CRYPTO_ALG_ALLOCATES_MEMORY, 409 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 410 .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), 411 .base.cra_alignmask = 7, 412 .base.cra_priority = 4001, 413 .base.cra_name = "ecb(des3_ede)", 414 .base.cra_driver_name = "cavium-ecb-des3_ede", 415 .base.cra_module = THIS_MODULE, 416 417 .min_keysize = DES3_EDE_KEY_SIZE, 418 .max_keysize = DES3_EDE_KEY_SIZE, 419 .ivsize = DES_BLOCK_SIZE, 420 .setkey = cvm_ecb_des3_setkey, 421 .encrypt = cvm_encrypt, 422 .decrypt = cvm_decrypt, 423 .init = cvm_enc_dec_init, 424 } }; 425 426 static inline int cav_register_algs(void) 427 { 428 return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); 429 } 430 431 static inline void cav_unregister_algs(void) 432 { 433 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); 434 } 435 436 int cvm_crypto_init(struct cpt_vf *cptvf) 437 { 438 struct pci_dev *pdev = cptvf->pdev; 439 u32 dev_count; 440 441 dev_count = dev_handle.dev_count; 442 dev_handle.cdev[dev_count] = cptvf; 443 dev_handle.dev_count++; 444 445 if (dev_count == 3) { 446 if (cav_register_algs()) { 447 dev_err(&pdev->dev, "Error in registering crypto algorithms\n"); 448 return -EINVAL; 449 } 450 } 451 452 return 0; 453 } 454 455 void cvm_crypto_exit(void) 456 { 457 u32 dev_count; 458 459 dev_count = --dev_handle.dev_count; 460 if (!dev_count) 461 cav_unregister_algs(); 462 } 463