1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/printk.h> 4 #include <linux/crypto.h> 5 #include <linux/rtnetlink.h> 6 7 #include <crypto/aead.h> 8 #include <crypto/authenc.h> 9 #include <crypto/des.h> 10 #include <crypto/internal/aead.h> 11 #include <crypto/scatterwalk.h> 12 #include <crypto/gcm.h> 13 14 #include "nitrox_dev.h" 15 #include "nitrox_common.h" 16 #include "nitrox_req.h" 17 18 #define GCM_AES_SALT_SIZE 4 19 20 union gph_p3 { 21 struct { 22 #ifdef __BIG_ENDIAN_BITFIELD 23 u16 iv_offset : 8; 24 u16 auth_offset : 8; 25 #else 26 u16 auth_offset : 8; 27 u16 iv_offset : 8; 28 #endif 29 }; 30 u16 param; 31 }; 32 33 static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, 34 unsigned int keylen) 35 { 36 int aes_keylen; 37 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 38 struct flexi_crypto_context *fctx; 39 union fc_ctx_flags flags; 40 41 aes_keylen = flexi_aes_keylen(keylen); 42 if (aes_keylen < 0) 43 return -EINVAL; 44 45 /* fill crypto context */ 46 fctx = nctx->u.fctx; 47 flags.fu = be64_to_cpu(fctx->flags.f); 48 flags.w0.aes_keylen = aes_keylen; 49 fctx->flags.f = cpu_to_be64(flags.fu); 50 51 /* copy enc key to context */ 52 memset(&fctx->crypto, 0, sizeof(fctx->crypto)); 53 memcpy(fctx->crypto.u.key, key, keylen); 54 55 return 0; 56 } 57 58 static int nitrox_aead_setauthsize(struct crypto_aead *aead, 59 unsigned int authsize) 60 { 61 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 62 struct flexi_crypto_context *fctx = nctx->u.fctx; 63 union fc_ctx_flags flags; 64 65 flags.fu = be64_to_cpu(fctx->flags.f); 66 flags.w0.mac_len = authsize; 67 fctx->flags.f = cpu_to_be64(flags.fu); 68 69 aead->authsize = authsize; 70 71 return 0; 72 } 73 74 static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead, 75 unsigned int authsize) 76 { 77 switch (authsize) { 78 case 4: 79 case 8: 80 case 12: 81 case 13: 82 case 14: 83 case 15: 84 case 16: 85 break; 86 default: 87 return -EINVAL; 88 } 89 90 return nitrox_aead_setauthsize(aead, authsize); 91 } 92 93 static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq, 94 struct scatterlist *src, char *iv, int ivsize, 95 int buflen) 96 { 97 int nents = sg_nents_for_len(src, buflen); 98 int ret; 99 100 if (nents < 0) 101 return nents; 102 103 /* IV entry */ 104 nents += 1; 105 /* Allocate buffer to hold IV and input scatterlist array */ 106 ret = alloc_src_req_buf(nkreq, nents, ivsize); 107 if (ret) 108 return ret; 109 110 nitrox_creq_copy_iv(nkreq->src, iv, ivsize); 111 nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen); 112 113 return 0; 114 } 115 116 static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq, 117 struct scatterlist *dst, int ivsize, int buflen) 118 { 119 int nents = sg_nents_for_len(dst, buflen); 120 int ret; 121 122 if (nents < 0) 123 return nents; 124 125 /* IV, ORH, COMPLETION entries */ 126 nents += 3; 127 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist 128 * array 129 */ 130 ret = alloc_dst_req_buf(nkreq, nents); 131 if (ret) 132 return ret; 133 134 nitrox_creq_set_orh(nkreq); 135 nitrox_creq_set_comp(nkreq); 136 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen); 137 138 return 0; 139 } 140 141 static void free_src_sglist(struct nitrox_kcrypt_request *nkreq) 142 { 143 kfree(nkreq->src); 144 } 145 146 static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq) 147 { 148 kfree(nkreq->dst); 149 } 150 151 static int nitrox_set_creq(struct nitrox_aead_rctx *rctx) 152 { 153 struct se_crypto_request *creq = &rctx->nkreq.creq; 154 union gph_p3 param3; 155 int ret; 156 157 creq->flags = rctx->flags; 158 creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : 159 GFP_ATOMIC; 160 161 creq->ctrl.value = 0; 162 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; 163 creq->ctrl.s.arg = rctx->ctrl_arg; 164 165 creq->gph.param0 = cpu_to_be16(rctx->cryptlen); 166 creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen); 167 creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen); 168 param3.iv_offset = 0; 169 param3.auth_offset = rctx->ivsize; 170 creq->gph.param3 = cpu_to_be16(param3.param); 171 172 creq->ctx_handle = rctx->ctx_handle; 173 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); 174 175 ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize, 176 rctx->srclen); 177 if (ret) 178 return ret; 179 180 ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize, 181 rctx->dstlen); 182 if (ret) { 183 free_src_sglist(&rctx->nkreq); 184 return ret; 185 } 186 187 return 0; 188 } 189 190 static void nitrox_aead_callback(void *arg, int err) 191 { 192 struct aead_request *areq = arg; 193 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); 194 195 free_src_sglist(&rctx->nkreq); 196 free_dst_sglist(&rctx->nkreq); 197 if (err) { 198 pr_err_ratelimited("request failed status 0x%0x\n", err); 199 err = -EINVAL; 200 } 201 202 aead_request_complete(areq, err); 203 } 204 205 static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen) 206 { 207 if (assoclen <= 512) 208 return true; 209 210 return false; 211 } 212 213 static int nitrox_aes_gcm_enc(struct aead_request *areq) 214 { 215 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 216 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 217 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); 218 struct se_crypto_request *creq = &rctx->nkreq.creq; 219 struct flexi_crypto_context *fctx = nctx->u.fctx; 220 int ret; 221 222 if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen)) 223 return -EINVAL; 224 225 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); 226 227 rctx->cryptlen = areq->cryptlen; 228 rctx->assoclen = areq->assoclen; 229 rctx->srclen = areq->assoclen + areq->cryptlen; 230 rctx->dstlen = rctx->srclen + aead->authsize; 231 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; 232 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; 233 rctx->flags = areq->base.flags; 234 rctx->ctx_handle = nctx->u.ctx_handle; 235 rctx->src = areq->src; 236 rctx->dst = areq->dst; 237 rctx->ctrl_arg = ENCRYPT; 238 ret = nitrox_set_creq(rctx); 239 if (ret) 240 return ret; 241 242 /* send the crypto request */ 243 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, 244 areq); 245 } 246 247 static int nitrox_aes_gcm_dec(struct aead_request *areq) 248 { 249 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 250 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 251 struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); 252 struct se_crypto_request *creq = &rctx->nkreq.creq; 253 struct flexi_crypto_context *fctx = nctx->u.fctx; 254 int ret; 255 256 if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen)) 257 return -EINVAL; 258 259 memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); 260 261 rctx->cryptlen = areq->cryptlen - aead->authsize; 262 rctx->assoclen = areq->assoclen; 263 rctx->srclen = areq->cryptlen + areq->assoclen; 264 rctx->dstlen = rctx->srclen - aead->authsize; 265 rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; 266 rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; 267 rctx->flags = areq->base.flags; 268 rctx->ctx_handle = nctx->u.ctx_handle; 269 rctx->src = areq->src; 270 rctx->dst = areq->dst; 271 rctx->ctrl_arg = DECRYPT; 272 ret = nitrox_set_creq(rctx); 273 if (ret) 274 return ret; 275 276 /* send the crypto request */ 277 return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback, 278 areq); 279 } 280 281 static int nitrox_aead_init(struct crypto_aead *aead) 282 { 283 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 284 struct crypto_ctx_hdr *chdr; 285 286 /* get the first device */ 287 nctx->ndev = nitrox_get_first_device(); 288 if (!nctx->ndev) 289 return -ENODEV; 290 291 /* allocate nitrox crypto context */ 292 chdr = crypto_alloc_context(nctx->ndev); 293 if (!chdr) { 294 nitrox_put_device(nctx->ndev); 295 return -ENOMEM; 296 } 297 nctx->chdr = chdr; 298 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr + 299 sizeof(struct ctx_hdr)); 300 nctx->u.fctx->flags.f = 0; 301 302 return 0; 303 } 304 305 static int nitrox_gcm_common_init(struct crypto_aead *aead) 306 { 307 int ret; 308 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 309 union fc_ctx_flags *flags; 310 311 ret = nitrox_aead_init(aead); 312 if (ret) 313 return ret; 314 315 flags = &nctx->u.fctx->flags; 316 flags->w0.cipher_type = CIPHER_AES_GCM; 317 flags->w0.hash_type = AUTH_NULL; 318 flags->w0.iv_source = IV_FROM_DPTR; 319 /* ask microcode to calculate ipad/opad */ 320 flags->w0.auth_input_type = 1; 321 flags->f = cpu_to_be64(flags->fu); 322 323 return 0; 324 } 325 326 static int nitrox_aes_gcm_init(struct crypto_aead *aead) 327 { 328 int ret; 329 330 ret = nitrox_gcm_common_init(aead); 331 if (ret) 332 return ret; 333 334 crypto_aead_set_reqsize(aead, 335 sizeof(struct aead_request) + 336 sizeof(struct nitrox_aead_rctx)); 337 338 return 0; 339 } 340 341 static void nitrox_aead_exit(struct crypto_aead *aead) 342 { 343 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 344 345 /* free the nitrox crypto context */ 346 if (nctx->u.ctx_handle) { 347 struct flexi_crypto_context *fctx = nctx->u.fctx; 348 349 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys)); 350 memzero_explicit(&fctx->auth, sizeof(struct auth_keys)); 351 crypto_free_context((void *)nctx->chdr); 352 } 353 nitrox_put_device(nctx->ndev); 354 355 nctx->u.ctx_handle = 0; 356 nctx->ndev = NULL; 357 } 358 359 static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key, 360 unsigned int keylen) 361 { 362 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 363 struct flexi_crypto_context *fctx = nctx->u.fctx; 364 int ret; 365 366 if (keylen < GCM_AES_SALT_SIZE) 367 return -EINVAL; 368 369 keylen -= GCM_AES_SALT_SIZE; 370 ret = nitrox_aes_gcm_setkey(aead, key, keylen); 371 if (ret) 372 return ret; 373 374 memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE); 375 return 0; 376 } 377 378 static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead, 379 unsigned int authsize) 380 { 381 switch (authsize) { 382 case 8: 383 case 12: 384 case 16: 385 break; 386 default: 387 return -EINVAL; 388 } 389 390 return nitrox_aead_setauthsize(aead, authsize); 391 } 392 393 static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq) 394 { 395 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq); 396 struct nitrox_aead_rctx *aead_rctx = &rctx->base; 397 unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; 398 struct scatterlist *sg; 399 400 if (areq->assoclen != 16 && areq->assoclen != 20) 401 return -EINVAL; 402 403 scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0); 404 sg_init_table(rctx->src, 3); 405 sg_set_buf(rctx->src, rctx->assoc, assoclen); 406 sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen); 407 if (sg != rctx->src + 1) 408 sg_chain(rctx->src, 2, sg); 409 410 if (areq->src != areq->dst) { 411 sg_init_table(rctx->dst, 3); 412 sg_set_buf(rctx->dst, rctx->assoc, assoclen); 413 sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen); 414 if (sg != rctx->dst + 1) 415 sg_chain(rctx->dst, 2, sg); 416 } 417 418 aead_rctx->src = rctx->src; 419 aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst; 420 421 return 0; 422 } 423 424 static void nitrox_rfc4106_callback(void *arg, int err) 425 { 426 struct aead_request *areq = arg; 427 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq); 428 struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq; 429 430 free_src_sglist(nkreq); 431 free_dst_sglist(nkreq); 432 if (err) { 433 pr_err_ratelimited("request failed status 0x%0x\n", err); 434 err = -EINVAL; 435 } 436 437 aead_request_complete(areq, err); 438 } 439 440 static int nitrox_rfc4106_enc(struct aead_request *areq) 441 { 442 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 443 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); 444 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq); 445 struct nitrox_aead_rctx *aead_rctx = &rctx->base; 446 struct se_crypto_request *creq = &aead_rctx->nkreq.creq; 447 int ret; 448 449 aead_rctx->cryptlen = areq->cryptlen; 450 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; 451 aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen; 452 aead_rctx->dstlen = aead_rctx->srclen + aead->authsize; 453 aead_rctx->iv = areq->iv; 454 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; 455 aead_rctx->flags = areq->base.flags; 456 aead_rctx->ctx_handle = nctx->u.ctx_handle; 457 aead_rctx->ctrl_arg = ENCRYPT; 458 459 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); 460 if (ret) 461 return ret; 462 463 ret = nitrox_set_creq(aead_rctx); 464 if (ret) 465 return ret; 466 467 /* send the crypto request */ 468 return nitrox_process_se_request(nctx->ndev, creq, 469 nitrox_rfc4106_callback, areq); 470 } 471 472 static int nitrox_rfc4106_dec(struct aead_request *areq) 473 { 474 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 475 struct nitrox_crypto_ctx *nctx = crypto_aead_ctx_dma(aead); 476 struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); 477 struct nitrox_aead_rctx *aead_rctx = &rctx->base; 478 struct se_crypto_request *creq = &aead_rctx->nkreq.creq; 479 int ret; 480 481 aead_rctx->cryptlen = areq->cryptlen - aead->authsize; 482 aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; 483 aead_rctx->srclen = 484 areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen; 485 aead_rctx->dstlen = aead_rctx->srclen - aead->authsize; 486 aead_rctx->iv = areq->iv; 487 aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; 488 aead_rctx->flags = areq->base.flags; 489 aead_rctx->ctx_handle = nctx->u.ctx_handle; 490 aead_rctx->ctrl_arg = DECRYPT; 491 492 ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); 493 if (ret) 494 return ret; 495 496 ret = nitrox_set_creq(aead_rctx); 497 if (ret) 498 return ret; 499 500 /* send the crypto request */ 501 return nitrox_process_se_request(nctx->ndev, creq, 502 nitrox_rfc4106_callback, areq); 503 } 504 505 static int nitrox_rfc4106_init(struct crypto_aead *aead) 506 { 507 int ret; 508 509 ret = nitrox_gcm_common_init(aead); 510 if (ret) 511 return ret; 512 513 crypto_aead_set_reqsize_dma(aead, sizeof(struct aead_request) + 514 sizeof(struct nitrox_rfc4106_rctx)); 515 516 return 0; 517 } 518 519 static struct aead_alg nitrox_aeads[] = { { 520 .base = { 521 .cra_name = "gcm(aes)", 522 .cra_driver_name = "n5_aes_gcm", 523 .cra_priority = PRIO, 524 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 525 .cra_blocksize = 1, 526 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 527 .cra_alignmask = 0, 528 .cra_module = THIS_MODULE, 529 }, 530 .setkey = nitrox_aes_gcm_setkey, 531 .setauthsize = nitrox_aes_gcm_setauthsize, 532 .encrypt = nitrox_aes_gcm_enc, 533 .decrypt = nitrox_aes_gcm_dec, 534 .init = nitrox_aes_gcm_init, 535 .exit = nitrox_aead_exit, 536 .ivsize = GCM_AES_IV_SIZE, 537 .maxauthsize = AES_BLOCK_SIZE, 538 }, { 539 .base = { 540 .cra_name = "rfc4106(gcm(aes))", 541 .cra_driver_name = "n5_rfc4106", 542 .cra_priority = PRIO, 543 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, 544 .cra_blocksize = 1, 545 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), 546 .cra_alignmask = 0, 547 .cra_module = THIS_MODULE, 548 }, 549 .setkey = nitrox_rfc4106_setkey, 550 .setauthsize = nitrox_rfc4106_setauthsize, 551 .encrypt = nitrox_rfc4106_enc, 552 .decrypt = nitrox_rfc4106_dec, 553 .init = nitrox_rfc4106_init, 554 .exit = nitrox_aead_exit, 555 .ivsize = GCM_RFC4106_IV_SIZE, 556 .maxauthsize = AES_BLOCK_SIZE, 557 } }; 558 559 int nitrox_register_aeads(void) 560 { 561 return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); 562 } 563 564 void nitrox_unregister_aeads(void) 565 { 566 crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads)); 567 } 568