1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/crypto.h> 3 #include <linux/kernel.h> 4 #include <linux/module.h> 5 #include <linux/printk.h> 6 7 #include <crypto/aes.h> 8 #include <crypto/skcipher.h> 9 #include <crypto/scatterwalk.h> 10 #include <crypto/ctr.h> 11 #include <crypto/internal/des.h> 12 #include <crypto/xts.h> 13 14 #include "nitrox_dev.h" 15 #include "nitrox_common.h" 16 #include "nitrox_req.h" 17 18 struct nitrox_cipher { 19 const char *name; 20 enum flexi_cipher value; 21 }; 22 23 /** 24 * supported cipher list 25 */ 26 static const struct nitrox_cipher flexi_cipher_table[] = { 27 { "null", CIPHER_NULL }, 28 { "cbc(des3_ede)", CIPHER_3DES_CBC }, 29 { "ecb(des3_ede)", CIPHER_3DES_ECB }, 30 { "cbc(aes)", CIPHER_AES_CBC }, 31 { "ecb(aes)", CIPHER_AES_ECB }, 32 { "cfb(aes)", CIPHER_AES_CFB }, 33 { "rfc3686(ctr(aes))", CIPHER_AES_CTR }, 34 { "xts(aes)", CIPHER_AES_XTS }, 35 { "cts(cbc(aes))", CIPHER_AES_CBC_CTS }, 36 { NULL, CIPHER_INVALID } 37 }; 38 39 static enum flexi_cipher flexi_cipher_type(const char *name) 40 { 41 const struct nitrox_cipher *cipher = flexi_cipher_table; 42 43 while (cipher->name) { 44 if (!strcmp(cipher->name, name)) 45 break; 46 cipher++; 47 } 48 return cipher->value; 49 } 50 51 static void free_src_sglist(struct skcipher_request *skreq) 52 { 53 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 54 55 kfree(nkreq->src); 56 } 57 58 static void free_dst_sglist(struct skcipher_request *skreq) 59 { 60 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 61 62 kfree(nkreq->dst); 63 } 64 65 static void nitrox_skcipher_callback(void *arg, int err) 66 { 67 struct skcipher_request *skreq = arg; 68 69 free_src_sglist(skreq); 70 free_dst_sglist(skreq); 71 if (err) { 72 pr_err_ratelimited("request failed status 0x%0x\n", err); 73 err = -EINVAL; 74 } 75 76 skcipher_request_complete(skreq, err); 77 } 78 79 static void nitrox_cbc_cipher_callback(void *arg, int err) 80 { 81 struct skcipher_request *skreq = arg; 82 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 83 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); 84 int ivsize = crypto_skcipher_ivsize(cipher); 85 unsigned int start = skreq->cryptlen - ivsize; 86 87 if (err) { 88 nitrox_skcipher_callback(arg, err); 89 return; 90 } 91 92 if (nkreq->creq.ctrl.s.arg == ENCRYPT) { 93 scatterwalk_map_and_copy(skreq->iv, skreq->dst, start, ivsize, 94 0); 95 } else { 96 if (skreq->src != skreq->dst) { 97 scatterwalk_map_and_copy(skreq->iv, skreq->src, start, 98 ivsize, 0); 99 } else { 100 memcpy(skreq->iv, nkreq->iv_out, ivsize); 101 kfree(nkreq->iv_out); 102 } 103 } 104 105 nitrox_skcipher_callback(arg, err); 106 } 107 108 static int nitrox_skcipher_init(struct crypto_skcipher *tfm) 109 { 110 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); 111 struct crypto_ctx_hdr *chdr; 112 113 /* get the first device */ 114 nctx->ndev = nitrox_get_first_device(); 115 if (!nctx->ndev) 116 return -ENODEV; 117 118 /* allocate nitrox crypto context */ 119 chdr = crypto_alloc_context(nctx->ndev); 120 if (!chdr) { 121 nitrox_put_device(nctx->ndev); 122 return -ENOMEM; 123 } 124 125 nctx->callback = nitrox_skcipher_callback; 126 nctx->chdr = chdr; 127 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + 128 sizeof(struct ctx_hdr)); 129 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) + 130 sizeof(struct nitrox_kcrypt_request)); 131 return 0; 132 } 133 134 static int nitrox_cbc_init(struct crypto_skcipher *tfm) 135 { 136 int err; 137 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); 138 139 err = nitrox_skcipher_init(tfm); 140 if (err) 141 return err; 142 143 nctx->callback = nitrox_cbc_cipher_callback; 144 return 0; 145 } 146 147 static void nitrox_skcipher_exit(struct crypto_skcipher *tfm) 148 { 149 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); 150 151 /* free the nitrox crypto context */ 152 if (nctx->u.ctx_handle) { 153 struct flexi_crypto_context *fctx = nctx->u.fctx; 154 155 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys)); 156 memzero_explicit(&fctx->auth, sizeof(struct auth_keys)); 157 crypto_free_context((void *)nctx->chdr); 158 } 159 nitrox_put_device(nctx->ndev); 160 161 nctx->u.ctx_handle = 0; 162 nctx->ndev = NULL; 163 } 164 165 static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher, 166 int aes_keylen, const u8 *key, 167 unsigned int keylen) 168 { 169 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 170 struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); 171 struct flexi_crypto_context *fctx; 172 union fc_ctx_flags *flags; 173 enum flexi_cipher cipher_type; 174 const char *name; 175 176 name = crypto_tfm_alg_name(tfm); 177 cipher_type = flexi_cipher_type(name); 178 if (unlikely(cipher_type == CIPHER_INVALID)) { 179 pr_err("unsupported cipher: %s\n", name); 180 return -EINVAL; 181 } 182 183 /* fill crypto context */ 184 fctx = nctx->u.fctx; 185 flags = &fctx->flags; 186 flags->f = 0; 187 flags->w0.cipher_type = cipher_type; 188 flags->w0.aes_keylen = aes_keylen; 189 flags->w0.iv_source = IV_FROM_DPTR; 190 flags->f = cpu_to_be64(*(u64 *)&flags->w0); 191 /* copy the key to context */ 192 memcpy(fctx->crypto.u.key, key, keylen); 193 194 return 0; 195 } 196 197 static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, 198 unsigned int keylen) 199 { 200 int aes_keylen; 201 202 aes_keylen = flexi_aes_keylen(keylen); 203 if (aes_keylen < 0) 204 return -EINVAL; 205 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); 206 } 207 208 static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize) 209 { 210 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 211 int nents = sg_nents(skreq->src) + 1; 212 int ret; 213 214 /* Allocate buffer to hold IV and input scatterlist array */ 215 ret = alloc_src_req_buf(nkreq, nents, ivsize); 216 if (ret) 217 return ret; 218 219 nitrox_creq_copy_iv(nkreq->src, skreq->iv, ivsize); 220 nitrox_creq_set_src_sg(nkreq, nents, ivsize, skreq->src, 221 skreq->cryptlen); 222 223 return 0; 224 } 225 226 static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize) 227 { 228 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 229 int nents = sg_nents(skreq->dst) + 3; 230 int ret; 231 232 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist 233 * array 234 */ 235 ret = alloc_dst_req_buf(nkreq, nents); 236 if (ret) 237 return ret; 238 239 nitrox_creq_set_orh(nkreq); 240 nitrox_creq_set_comp(nkreq); 241 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, skreq->dst, 242 skreq->cryptlen); 243 244 return 0; 245 } 246 247 static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc) 248 { 249 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); 250 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); 251 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 252 int ivsize = crypto_skcipher_ivsize(cipher); 253 struct se_crypto_request *creq; 254 int ret; 255 256 creq = &nkreq->creq; 257 creq->flags = skreq->base.flags; 258 creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 259 GFP_KERNEL : GFP_ATOMIC; 260 261 /* fill the request */ 262 creq->ctrl.value = 0; 263 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; 264 creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT); 265 /* param0: length of the data to be encrypted */ 266 creq->gph.param0 = cpu_to_be16(skreq->cryptlen); 267 creq->gph.param1 = 0; 268 /* param2: encryption data offset */ 269 creq->gph.param2 = cpu_to_be16(ivsize); 270 creq->gph.param3 = 0; 271 272 creq->ctx_handle = nctx->u.ctx_handle; 273 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); 274 275 ret = alloc_src_sglist(skreq, ivsize); 276 if (ret) 277 return ret; 278 279 ret = alloc_dst_sglist(skreq, ivsize); 280 if (ret) { 281 free_src_sglist(skreq); 282 return ret; 283 } 284 285 /* send the crypto request */ 286 return nitrox_process_se_request(nctx->ndev, creq, nctx->callback, 287 skreq); 288 } 289 290 static int nitrox_cbc_decrypt(struct skcipher_request *skreq) 291 { 292 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); 293 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); 294 int ivsize = crypto_skcipher_ivsize(cipher); 295 gfp_t flags = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 296 GFP_KERNEL : GFP_ATOMIC; 297 unsigned int start = skreq->cryptlen - ivsize; 298 299 if (skreq->src != skreq->dst) 300 return nitrox_skcipher_crypt(skreq, false); 301 302 nkreq->iv_out = kmalloc(ivsize, flags); 303 if (!nkreq->iv_out) 304 return -ENOMEM; 305 306 scatterwalk_map_and_copy(nkreq->iv_out, skreq->src, start, ivsize, 0); 307 return nitrox_skcipher_crypt(skreq, false); 308 } 309 310 static int nitrox_aes_encrypt(struct skcipher_request *skreq) 311 { 312 return nitrox_skcipher_crypt(skreq, true); 313 } 314 315 static int nitrox_aes_decrypt(struct skcipher_request *skreq) 316 { 317 return nitrox_skcipher_crypt(skreq, false); 318 } 319 320 static int nitrox_3des_setkey(struct crypto_skcipher *cipher, 321 const u8 *key, unsigned int keylen) 322 { 323 return verify_skcipher_des3_key(cipher, key) ?: 324 nitrox_skcipher_setkey(cipher, 0, key, keylen); 325 } 326 327 static int nitrox_3des_encrypt(struct skcipher_request *skreq) 328 { 329 return nitrox_skcipher_crypt(skreq, true); 330 } 331 332 static int nitrox_3des_decrypt(struct skcipher_request *skreq) 333 { 334 return nitrox_skcipher_crypt(skreq, false); 335 } 336 337 static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher, 338 const u8 *key, unsigned int keylen) 339 { 340 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 341 struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); 342 struct flexi_crypto_context *fctx; 343 int aes_keylen, ret; 344 345 ret = xts_check_key(tfm, key, keylen); 346 if (ret) 347 return ret; 348 349 keylen /= 2; 350 351 aes_keylen = flexi_aes_keylen(keylen); 352 if (aes_keylen < 0) 353 return -EINVAL; 354 355 fctx = nctx->u.fctx; 356 /* copy KEY2 */ 357 memcpy(fctx->auth.u.key2, (key + keylen), keylen); 358 359 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); 360 } 361 362 static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher, 363 const u8 *key, unsigned int keylen) 364 { 365 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); 366 struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); 367 struct flexi_crypto_context *fctx; 368 int aes_keylen; 369 370 if (keylen < CTR_RFC3686_NONCE_SIZE) 371 return -EINVAL; 372 373 fctx = nctx->u.fctx; 374 375 memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE), 376 CTR_RFC3686_NONCE_SIZE); 377 378 keylen -= CTR_RFC3686_NONCE_SIZE; 379 380 aes_keylen = flexi_aes_keylen(keylen); 381 if (aes_keylen < 0) 382 return -EINVAL; 383 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); 384 } 385 386 static struct skcipher_alg nitrox_skciphers[] = { { 387 .base = { 388 .cra_name = "cbc(aes)", 389 .cra_driver_name = "n5_cbc(aes)", 390 .cra_priority = PRIO, 391 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 392 .cra_blocksize = AES_BLOCK_SIZE, 393 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 394 .cra_alignmask = 0, 395 .cra_module = THIS_MODULE, 396 }, 397 .min_keysize = AES_MIN_KEY_SIZE, 398 .max_keysize = AES_MAX_KEY_SIZE, 399 .ivsize = AES_BLOCK_SIZE, 400 .setkey = nitrox_aes_setkey, 401 .encrypt = nitrox_aes_encrypt, 402 .decrypt = nitrox_cbc_decrypt, 403 .init = nitrox_cbc_init, 404 .exit = nitrox_skcipher_exit, 405 }, { 406 .base = { 407 .cra_name = "ecb(aes)", 408 .cra_driver_name = "n5_ecb(aes)", 409 .cra_priority = PRIO, 410 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 411 .cra_blocksize = AES_BLOCK_SIZE, 412 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 413 .cra_alignmask = 0, 414 .cra_module = THIS_MODULE, 415 }, 416 .min_keysize = AES_MIN_KEY_SIZE, 417 .max_keysize = AES_MAX_KEY_SIZE, 418 .ivsize = AES_BLOCK_SIZE, 419 .setkey = nitrox_aes_setkey, 420 .encrypt = nitrox_aes_encrypt, 421 .decrypt = nitrox_aes_decrypt, 422 .init = nitrox_skcipher_init, 423 .exit = nitrox_skcipher_exit, 424 }, { 425 .base = { 426 .cra_name = "cfb(aes)", 427 .cra_driver_name = "n5_cfb(aes)", 428 .cra_priority = PRIO, 429 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 430 .cra_blocksize = AES_BLOCK_SIZE, 431 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 432 .cra_alignmask = 0, 433 .cra_module = THIS_MODULE, 434 }, 435 .min_keysize = AES_MIN_KEY_SIZE, 436 .max_keysize = AES_MAX_KEY_SIZE, 437 .ivsize = AES_BLOCK_SIZE, 438 .setkey = nitrox_aes_setkey, 439 .encrypt = nitrox_aes_encrypt, 440 .decrypt = nitrox_aes_decrypt, 441 .init = nitrox_skcipher_init, 442 .exit = nitrox_skcipher_exit, 443 }, { 444 .base = { 445 .cra_name = "xts(aes)", 446 .cra_driver_name = "n5_xts(aes)", 447 .cra_priority = PRIO, 448 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 449 .cra_blocksize = AES_BLOCK_SIZE, 450 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 451 .cra_alignmask = 0, 452 .cra_module = THIS_MODULE, 453 }, 454 .min_keysize = 2 * AES_MIN_KEY_SIZE, 455 .max_keysize = 2 * AES_MAX_KEY_SIZE, 456 .ivsize = AES_BLOCK_SIZE, 457 .setkey = nitrox_aes_xts_setkey, 458 .encrypt = nitrox_aes_encrypt, 459 .decrypt = nitrox_aes_decrypt, 460 .init = nitrox_skcipher_init, 461 .exit = nitrox_skcipher_exit, 462 }, { 463 .base = { 464 .cra_name = "rfc3686(ctr(aes))", 465 .cra_driver_name = "n5_rfc3686(ctr(aes))", 466 .cra_priority = PRIO, 467 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 468 .cra_blocksize = 1, 469 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 470 .cra_alignmask = 0, 471 .cra_module = THIS_MODULE, 472 }, 473 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 474 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 475 .ivsize = CTR_RFC3686_IV_SIZE, 476 .init = nitrox_skcipher_init, 477 .exit = nitrox_skcipher_exit, 478 .setkey = nitrox_aes_ctr_rfc3686_setkey, 479 .encrypt = nitrox_aes_encrypt, 480 .decrypt = nitrox_aes_decrypt, 481 }, { 482 .base = { 483 .cra_name = "cts(cbc(aes))", 484 .cra_driver_name = "n5_cts(cbc(aes))", 485 .cra_priority = PRIO, 486 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 487 .cra_blocksize = AES_BLOCK_SIZE, 488 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 489 .cra_alignmask = 0, 490 .cra_module = THIS_MODULE, 491 }, 492 .min_keysize = AES_MIN_KEY_SIZE, 493 .max_keysize = AES_MAX_KEY_SIZE, 494 .ivsize = AES_BLOCK_SIZE, 495 .setkey = nitrox_aes_setkey, 496 .encrypt = nitrox_aes_encrypt, 497 .decrypt = nitrox_aes_decrypt, 498 .init = nitrox_skcipher_init, 499 .exit = nitrox_skcipher_exit, 500 }, { 501 .base = { 502 .cra_name = "cbc(des3_ede)", 503 .cra_driver_name = "n5_cbc(des3_ede)", 504 .cra_priority = PRIO, 505 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 506 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 507 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 508 .cra_alignmask = 0, 509 .cra_module = THIS_MODULE, 510 }, 511 .min_keysize = DES3_EDE_KEY_SIZE, 512 .max_keysize = DES3_EDE_KEY_SIZE, 513 .ivsize = DES3_EDE_BLOCK_SIZE, 514 .setkey = nitrox_3des_setkey, 515 .encrypt = nitrox_3des_encrypt, 516 .decrypt = nitrox_cbc_decrypt, 517 .init = nitrox_cbc_init, 518 .exit = nitrox_skcipher_exit, 519 }, { 520 .base = { 521 .cra_name = "ecb(des3_ede)", 522 .cra_driver_name = "n5_ecb(des3_ede)", 523 .cra_priority = PRIO, 524 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 525 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 526 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 527 .cra_alignmask = 0, 528 .cra_module = THIS_MODULE, 529 }, 530 .min_keysize = DES3_EDE_KEY_SIZE, 531 .max_keysize = DES3_EDE_KEY_SIZE, 532 .ivsize = DES3_EDE_BLOCK_SIZE, 533 .setkey = nitrox_3des_setkey, 534 .encrypt = nitrox_3des_encrypt, 535 .decrypt = nitrox_3des_decrypt, 536 .init = nitrox_skcipher_init, 537 .exit = nitrox_skcipher_exit, 538 } 539 540 }; 541 542 int nitrox_register_skciphers(void) 543 { 544 return crypto_register_skciphers(nitrox_skciphers, 545 ARRAY_SIZE(nitrox_skciphers)); 546 } 547 548 void nitrox_unregister_skciphers(void) 549 { 550 crypto_unregister_skciphers(nitrox_skciphers, 551 ARRAY_SIZE(nitrox_skciphers)); 552 } 553