1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * GCM: Galois/Counter Mode. 4 * 5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> 6 */ 7 8 #include <crypto/gf128mul.h> 9 #include <crypto/internal/aead.h> 10 #include <crypto/internal/skcipher.h> 11 #include <crypto/internal/hash.h> 12 #include <crypto/null.h> 13 #include <crypto/scatterwalk.h> 14 #include <crypto/gcm.h> 15 #include <crypto/hash.h> 16 #include <linux/err.h> 17 #include <linux/init.h> 18 #include <linux/kernel.h> 19 #include <linux/module.h> 20 #include <linux/slab.h> 21 22 struct gcm_instance_ctx { 23 struct crypto_skcipher_spawn ctr; 24 struct crypto_ahash_spawn ghash; 25 }; 26 27 struct crypto_gcm_ctx { 28 struct crypto_skcipher *ctr; 29 struct crypto_ahash *ghash; 30 }; 31 32 struct crypto_rfc4106_ctx { 33 struct crypto_aead *child; 34 u8 nonce[4]; 35 }; 36 37 struct crypto_rfc4106_req_ctx { 38 struct scatterlist src[3]; 39 struct scatterlist dst[3]; 40 struct aead_request subreq; 41 }; 42 43 struct crypto_rfc4543_instance_ctx { 44 struct crypto_aead_spawn aead; 45 }; 46 47 struct crypto_rfc4543_ctx { 48 struct crypto_aead *child; 49 struct crypto_sync_skcipher *null; 50 u8 nonce[4]; 51 }; 52 53 struct crypto_rfc4543_req_ctx { 54 struct aead_request subreq; 55 }; 56 57 struct crypto_gcm_ghash_ctx { 58 unsigned int cryptlen; 59 struct scatterlist *src; 60 int (*complete)(struct aead_request *req, u32 flags); 61 }; 62 63 struct crypto_gcm_req_priv_ctx { 64 u8 iv[16]; 65 u8 auth_tag[16]; 66 u8 iauth_tag[16]; 67 struct scatterlist src[3]; 68 struct scatterlist dst[3]; 69 struct scatterlist sg; 70 struct crypto_gcm_ghash_ctx ghash_ctx; 71 union { 72 struct ahash_request ahreq; 73 struct skcipher_request skreq; 74 } u; 75 }; 76 77 static struct { 78 u8 buf[16]; 79 struct scatterlist sg; 80 } *gcm_zeroes; 81 82 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc); 83 84 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( 85 struct aead_request *req) 86 { 87 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); 88 89 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); 90 } 91 92 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, 93 unsigned int keylen) 94 { 95 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 96 struct crypto_ahash *ghash = ctx->ghash; 97 struct crypto_skcipher *ctr = ctx->ctr; 98 struct { 99 be128 hash; 100 u8 iv[16]; 101 102 struct crypto_wait wait; 103 104 struct scatterlist sg[1]; 105 struct skcipher_request req; 106 } *data; 107 int err; 108 109 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); 110 crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & 111 CRYPTO_TFM_REQ_MASK); 112 err = crypto_skcipher_setkey(ctr, key, keylen); 113 if (err) 114 return err; 115 116 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr), 117 GFP_KERNEL); 118 if (!data) 119 return -ENOMEM; 120 121 crypto_init_wait(&data->wait); 122 sg_init_one(data->sg, &data->hash, sizeof(data->hash)); 123 skcipher_request_set_tfm(&data->req, ctr); 124 skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | 125 CRYPTO_TFM_REQ_MAY_BACKLOG, 126 crypto_req_done, 127 &data->wait); 128 skcipher_request_set_crypt(&data->req, data->sg, data->sg, 129 sizeof(data->hash), data->iv); 130 131 err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), 132 &data->wait); 133 134 if (err) 135 goto out; 136 137 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); 138 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & 139 CRYPTO_TFM_REQ_MASK); 140 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); 141 out: 142 kfree_sensitive(data); 143 return err; 144 } 145 146 static int crypto_gcm_setauthsize(struct crypto_aead *tfm, 147 unsigned int authsize) 148 { 149 return crypto_gcm_check_authsize(authsize); 150 } 151 152 static void crypto_gcm_init_common(struct aead_request *req) 153 { 154 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 155 __be32 counter = cpu_to_be32(1); 156 struct scatterlist *sg; 157 158 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); 159 memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE); 160 memcpy(pctx->iv + GCM_AES_IV_SIZE, &counter, 4); 161 162 sg_init_table(pctx->src, 3); 163 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); 164 sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); 165 if (sg != pctx->src + 1) 166 sg_chain(pctx->src, 2, sg); 167 168 if (req->src != req->dst) { 169 sg_init_table(pctx->dst, 3); 170 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); 171 sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); 172 if (sg != pctx->dst + 1) 173 sg_chain(pctx->dst, 2, sg); 174 } 175 } 176 177 static void crypto_gcm_init_crypt(struct aead_request *req, 178 unsigned int cryptlen) 179 { 180 struct crypto_aead *aead = crypto_aead_reqtfm(req); 181 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 182 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 183 struct skcipher_request *skreq = &pctx->u.skreq; 184 struct scatterlist *dst; 185 186 dst = req->src == req->dst ? pctx->src : pctx->dst; 187 188 skcipher_request_set_tfm(skreq, ctx->ctr); 189 skcipher_request_set_crypt(skreq, pctx->src, dst, 190 cryptlen + sizeof(pctx->auth_tag), 191 pctx->iv); 192 } 193 194 static inline unsigned int gcm_remain(unsigned int len) 195 { 196 len &= 0xfU; 197 return len ? 16 - len : 0; 198 } 199 200 static void gcm_hash_len_done(void *data, int err); 201 202 static int gcm_hash_update(struct aead_request *req, 203 crypto_completion_t compl, 204 struct scatterlist *src, 205 unsigned int len, u32 flags) 206 { 207 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 208 struct ahash_request *ahreq = &pctx->u.ahreq; 209 210 ahash_request_set_callback(ahreq, flags, compl, req); 211 ahash_request_set_crypt(ahreq, src, NULL, len); 212 213 return crypto_ahash_update(ahreq); 214 } 215 216 static int gcm_hash_remain(struct aead_request *req, 217 unsigned int remain, 218 crypto_completion_t compl, u32 flags) 219 { 220 return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); 221 } 222 223 static int gcm_hash_len(struct aead_request *req, u32 flags) 224 { 225 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 226 struct ahash_request *ahreq = &pctx->u.ahreq; 227 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 228 be128 lengths; 229 230 lengths.a = cpu_to_be64(req->assoclen * 8); 231 lengths.b = cpu_to_be64(gctx->cryptlen * 8); 232 memcpy(pctx->iauth_tag, &lengths, 16); 233 sg_init_one(&pctx->sg, pctx->iauth_tag, 16); 234 ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); 235 ahash_request_set_crypt(ahreq, &pctx->sg, 236 pctx->iauth_tag, sizeof(lengths)); 237 238 return crypto_ahash_finup(ahreq); 239 } 240 241 static int gcm_hash_len_continue(struct aead_request *req, u32 flags) 242 { 243 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 244 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 245 246 return gctx->complete(req, flags); 247 } 248 249 static void gcm_hash_len_done(void *data, int err) 250 { 251 struct aead_request *req = data; 252 253 if (err) 254 goto out; 255 256 err = gcm_hash_len_continue(req, 0); 257 if (err == -EINPROGRESS) 258 return; 259 260 out: 261 aead_request_complete(req, err); 262 } 263 264 static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) 265 { 266 return gcm_hash_len(req, flags) ?: 267 gcm_hash_len_continue(req, flags); 268 } 269 270 static void gcm_hash_crypt_remain_done(void *data, int err) 271 { 272 struct aead_request *req = data; 273 274 if (err) 275 goto out; 276 277 err = gcm_hash_crypt_remain_continue(req, 0); 278 if (err == -EINPROGRESS) 279 return; 280 281 out: 282 aead_request_complete(req, err); 283 } 284 285 static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) 286 { 287 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 288 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 289 unsigned int remain; 290 291 remain = gcm_remain(gctx->cryptlen); 292 if (remain) 293 return gcm_hash_remain(req, remain, 294 gcm_hash_crypt_remain_done, flags) ?: 295 gcm_hash_crypt_remain_continue(req, flags); 296 297 return gcm_hash_crypt_remain_continue(req, flags); 298 } 299 300 static void gcm_hash_crypt_done(void *data, int err) 301 { 302 struct aead_request *req = data; 303 304 if (err) 305 goto out; 306 307 err = gcm_hash_crypt_continue(req, 0); 308 if (err == -EINPROGRESS) 309 return; 310 311 out: 312 aead_request_complete(req, err); 313 } 314 315 static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) 316 { 317 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 318 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 319 320 if (gctx->cryptlen) 321 return gcm_hash_update(req, gcm_hash_crypt_done, 322 gctx->src, gctx->cryptlen, flags) ?: 323 gcm_hash_crypt_continue(req, flags); 324 325 return gcm_hash_crypt_remain_continue(req, flags); 326 } 327 328 static void gcm_hash_assoc_remain_done(void *data, int err) 329 { 330 struct aead_request *req = data; 331 332 if (err) 333 goto out; 334 335 err = gcm_hash_assoc_remain_continue(req, 0); 336 if (err == -EINPROGRESS) 337 return; 338 339 out: 340 aead_request_complete(req, err); 341 } 342 343 static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) 344 { 345 unsigned int remain; 346 347 remain = gcm_remain(req->assoclen); 348 if (remain) 349 return gcm_hash_remain(req, remain, 350 gcm_hash_assoc_remain_done, flags) ?: 351 gcm_hash_assoc_remain_continue(req, flags); 352 353 return gcm_hash_assoc_remain_continue(req, flags); 354 } 355 356 static void gcm_hash_assoc_done(void *data, int err) 357 { 358 struct aead_request *req = data; 359 360 if (err) 361 goto out; 362 363 err = gcm_hash_assoc_continue(req, 0); 364 if (err == -EINPROGRESS) 365 return; 366 367 out: 368 aead_request_complete(req, err); 369 } 370 371 static int gcm_hash_init_continue(struct aead_request *req, u32 flags) 372 { 373 if (req->assoclen) 374 return gcm_hash_update(req, gcm_hash_assoc_done, 375 req->src, req->assoclen, flags) ?: 376 gcm_hash_assoc_continue(req, flags); 377 378 return gcm_hash_assoc_remain_continue(req, flags); 379 } 380 381 static void gcm_hash_init_done(void *data, int err) 382 { 383 struct aead_request *req = data; 384 385 if (err) 386 goto out; 387 388 err = gcm_hash_init_continue(req, 0); 389 if (err == -EINPROGRESS) 390 return; 391 392 out: 393 aead_request_complete(req, err); 394 } 395 396 static int gcm_hash(struct aead_request *req, u32 flags) 397 { 398 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 399 struct ahash_request *ahreq = &pctx->u.ahreq; 400 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 401 402 ahash_request_set_tfm(ahreq, ctx->ghash); 403 404 ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); 405 return crypto_ahash_init(ahreq) ?: 406 gcm_hash_init_continue(req, flags); 407 } 408 409 static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) 410 { 411 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 412 struct crypto_aead *aead = crypto_aead_reqtfm(req); 413 u8 *auth_tag = pctx->auth_tag; 414 415 crypto_xor(auth_tag, pctx->iauth_tag, 16); 416 scatterwalk_map_and_copy(auth_tag, req->dst, 417 req->assoclen + req->cryptlen, 418 crypto_aead_authsize(aead), 1); 419 return 0; 420 } 421 422 static int gcm_encrypt_continue(struct aead_request *req, u32 flags) 423 { 424 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 425 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 426 427 gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); 428 gctx->cryptlen = req->cryptlen; 429 gctx->complete = gcm_enc_copy_hash; 430 431 return gcm_hash(req, flags); 432 } 433 434 static void gcm_encrypt_done(void *data, int err) 435 { 436 struct aead_request *req = data; 437 438 if (err) 439 goto out; 440 441 err = gcm_encrypt_continue(req, 0); 442 if (err == -EINPROGRESS) 443 return; 444 445 out: 446 aead_request_complete(req, err); 447 } 448 449 static int crypto_gcm_encrypt(struct aead_request *req) 450 { 451 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 452 struct skcipher_request *skreq = &pctx->u.skreq; 453 u32 flags = aead_request_flags(req); 454 455 crypto_gcm_init_common(req); 456 crypto_gcm_init_crypt(req, req->cryptlen); 457 skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); 458 459 return crypto_skcipher_encrypt(skreq) ?: 460 gcm_encrypt_continue(req, flags); 461 } 462 463 static int crypto_gcm_verify(struct aead_request *req) 464 { 465 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 466 struct crypto_aead *aead = crypto_aead_reqtfm(req); 467 u8 *auth_tag = pctx->auth_tag; 468 u8 *iauth_tag = pctx->iauth_tag; 469 unsigned int authsize = crypto_aead_authsize(aead); 470 unsigned int cryptlen = req->cryptlen - authsize; 471 472 crypto_xor(auth_tag, iauth_tag, 16); 473 scatterwalk_map_and_copy(iauth_tag, req->src, 474 req->assoclen + cryptlen, authsize, 0); 475 return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; 476 } 477 478 static void gcm_decrypt_done(void *data, int err) 479 { 480 struct aead_request *req = data; 481 482 if (!err) 483 err = crypto_gcm_verify(req); 484 485 aead_request_complete(req, err); 486 } 487 488 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) 489 { 490 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 491 struct skcipher_request *skreq = &pctx->u.skreq; 492 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 493 494 crypto_gcm_init_crypt(req, gctx->cryptlen); 495 skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); 496 return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); 497 } 498 499 static int crypto_gcm_decrypt(struct aead_request *req) 500 { 501 struct crypto_aead *aead = crypto_aead_reqtfm(req); 502 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 503 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 504 unsigned int authsize = crypto_aead_authsize(aead); 505 unsigned int cryptlen = req->cryptlen; 506 u32 flags = aead_request_flags(req); 507 508 cryptlen -= authsize; 509 510 crypto_gcm_init_common(req); 511 512 gctx->src = sg_next(pctx->src); 513 gctx->cryptlen = cryptlen; 514 gctx->complete = gcm_dec_hash_continue; 515 516 return gcm_hash(req, flags); 517 } 518 519 static int crypto_gcm_init_tfm(struct crypto_aead *tfm) 520 { 521 struct aead_instance *inst = aead_alg_instance(tfm); 522 struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); 523 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 524 struct crypto_skcipher *ctr; 525 struct crypto_ahash *ghash; 526 unsigned long align; 527 int err; 528 529 ghash = crypto_spawn_ahash(&ictx->ghash); 530 if (IS_ERR(ghash)) 531 return PTR_ERR(ghash); 532 533 ctr = crypto_spawn_skcipher(&ictx->ctr); 534 err = PTR_ERR(ctr); 535 if (IS_ERR(ctr)) 536 goto err_free_hash; 537 538 ctx->ctr = ctr; 539 ctx->ghash = ghash; 540 541 align = crypto_aead_alignmask(tfm); 542 align &= ~(crypto_tfm_ctx_alignment() - 1); 543 crypto_aead_set_reqsize(tfm, 544 align + offsetof(struct crypto_gcm_req_priv_ctx, u) + 545 max(sizeof(struct skcipher_request) + 546 crypto_skcipher_reqsize(ctr), 547 sizeof(struct ahash_request) + 548 crypto_ahash_reqsize(ghash))); 549 550 return 0; 551 552 err_free_hash: 553 crypto_free_ahash(ghash); 554 return err; 555 } 556 557 static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) 558 { 559 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 560 561 crypto_free_ahash(ctx->ghash); 562 crypto_free_skcipher(ctx->ctr); 563 } 564 565 static void crypto_gcm_free(struct aead_instance *inst) 566 { 567 struct gcm_instance_ctx *ctx = aead_instance_ctx(inst); 568 569 crypto_drop_skcipher(&ctx->ctr); 570 crypto_drop_ahash(&ctx->ghash); 571 kfree(inst); 572 } 573 574 static int crypto_gcm_create_common(struct crypto_template *tmpl, 575 struct rtattr **tb, 576 const char *ctr_name, 577 const char *ghash_name) 578 { 579 struct skcipher_alg_common *ctr; 580 u32 mask; 581 struct aead_instance *inst; 582 struct gcm_instance_ctx *ctx; 583 struct hash_alg_common *ghash; 584 int err; 585 586 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 587 if (err) 588 return err; 589 590 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 591 if (!inst) 592 return -ENOMEM; 593 ctx = aead_instance_ctx(inst); 594 595 err = crypto_grab_ahash(&ctx->ghash, aead_crypto_instance(inst), 596 ghash_name, 0, mask); 597 if (err) 598 goto err_free_inst; 599 ghash = crypto_spawn_ahash_alg(&ctx->ghash); 600 601 err = -EINVAL; 602 if (strcmp(ghash->base.cra_name, "ghash") != 0 || 603 ghash->digestsize != 16) 604 goto err_free_inst; 605 606 err = crypto_grab_skcipher(&ctx->ctr, aead_crypto_instance(inst), 607 ctr_name, 0, mask); 608 if (err) 609 goto err_free_inst; 610 ctr = crypto_spawn_skcipher_alg_common(&ctx->ctr); 611 612 /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ 613 err = -EINVAL; 614 if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || 615 ctr->ivsize != 16 || ctr->base.cra_blocksize != 1) 616 goto err_free_inst; 617 618 err = -ENAMETOOLONG; 619 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 620 "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) 621 goto err_free_inst; 622 623 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 624 "gcm_base(%s,%s)", ctr->base.cra_driver_name, 625 ghash->base.cra_driver_name) >= 626 CRYPTO_MAX_ALG_NAME) 627 goto err_free_inst; 628 629 inst->alg.base.cra_priority = (ghash->base.cra_priority + 630 ctr->base.cra_priority) / 2; 631 inst->alg.base.cra_blocksize = 1; 632 inst->alg.base.cra_alignmask = ctr->base.cra_alignmask; 633 inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); 634 inst->alg.ivsize = GCM_AES_IV_SIZE; 635 inst->alg.chunksize = ctr->chunksize; 636 inst->alg.maxauthsize = 16; 637 inst->alg.init = crypto_gcm_init_tfm; 638 inst->alg.exit = crypto_gcm_exit_tfm; 639 inst->alg.setkey = crypto_gcm_setkey; 640 inst->alg.setauthsize = crypto_gcm_setauthsize; 641 inst->alg.encrypt = crypto_gcm_encrypt; 642 inst->alg.decrypt = crypto_gcm_decrypt; 643 644 inst->free = crypto_gcm_free; 645 646 err = aead_register_instance(tmpl, inst); 647 if (err) { 648 err_free_inst: 649 crypto_gcm_free(inst); 650 } 651 return err; 652 } 653 654 static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) 655 { 656 const char *cipher_name; 657 char ctr_name[CRYPTO_MAX_ALG_NAME]; 658 659 cipher_name = crypto_attr_alg_name(tb[1]); 660 if (IS_ERR(cipher_name)) 661 return PTR_ERR(cipher_name); 662 663 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= 664 CRYPTO_MAX_ALG_NAME) 665 return -ENAMETOOLONG; 666 667 return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); 668 } 669 670 static int crypto_gcm_base_create(struct crypto_template *tmpl, 671 struct rtattr **tb) 672 { 673 const char *ctr_name; 674 const char *ghash_name; 675 676 ctr_name = crypto_attr_alg_name(tb[1]); 677 if (IS_ERR(ctr_name)) 678 return PTR_ERR(ctr_name); 679 680 ghash_name = crypto_attr_alg_name(tb[2]); 681 if (IS_ERR(ghash_name)) 682 return PTR_ERR(ghash_name); 683 684 return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); 685 } 686 687 static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, 688 unsigned int keylen) 689 { 690 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 691 struct crypto_aead *child = ctx->child; 692 693 if (keylen < 4) 694 return -EINVAL; 695 696 keylen -= 4; 697 memcpy(ctx->nonce, key + keylen, 4); 698 699 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 700 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 701 CRYPTO_TFM_REQ_MASK); 702 return crypto_aead_setkey(child, key, keylen); 703 } 704 705 static int crypto_rfc4106_setauthsize(struct crypto_aead *parent, 706 unsigned int authsize) 707 { 708 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 709 int err; 710 711 err = crypto_rfc4106_check_authsize(authsize); 712 if (err) 713 return err; 714 715 return crypto_aead_setauthsize(ctx->child, authsize); 716 } 717 718 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) 719 { 720 struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req); 721 struct crypto_aead *aead = crypto_aead_reqtfm(req); 722 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead); 723 struct aead_request *subreq = &rctx->subreq; 724 struct crypto_aead *child = ctx->child; 725 struct scatterlist *sg; 726 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), 727 crypto_aead_alignmask(child) + 1); 728 729 scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0); 730 731 memcpy(iv, ctx->nonce, 4); 732 memcpy(iv + 4, req->iv, 8); 733 734 sg_init_table(rctx->src, 3); 735 sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8); 736 sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen); 737 if (sg != rctx->src + 1) 738 sg_chain(rctx->src, 2, sg); 739 740 if (req->src != req->dst) { 741 sg_init_table(rctx->dst, 3); 742 sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8); 743 sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen); 744 if (sg != rctx->dst + 1) 745 sg_chain(rctx->dst, 2, sg); 746 } 747 748 aead_request_set_tfm(subreq, child); 749 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 750 req->base.data); 751 aead_request_set_crypt(subreq, rctx->src, 752 req->src == req->dst ? rctx->src : rctx->dst, 753 req->cryptlen, iv); 754 aead_request_set_ad(subreq, req->assoclen - 8); 755 756 return subreq; 757 } 758 759 static int crypto_rfc4106_encrypt(struct aead_request *req) 760 { 761 int err; 762 763 err = crypto_ipsec_check_assoclen(req->assoclen); 764 if (err) 765 return err; 766 767 req = crypto_rfc4106_crypt(req); 768 769 return crypto_aead_encrypt(req); 770 } 771 772 static int crypto_rfc4106_decrypt(struct aead_request *req) 773 { 774 int err; 775 776 err = crypto_ipsec_check_assoclen(req->assoclen); 777 if (err) 778 return err; 779 780 req = crypto_rfc4106_crypt(req); 781 782 return crypto_aead_decrypt(req); 783 } 784 785 static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm) 786 { 787 struct aead_instance *inst = aead_alg_instance(tfm); 788 struct crypto_aead_spawn *spawn = aead_instance_ctx(inst); 789 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 790 struct crypto_aead *aead; 791 unsigned long align; 792 793 aead = crypto_spawn_aead(spawn); 794 if (IS_ERR(aead)) 795 return PTR_ERR(aead); 796 797 ctx->child = aead; 798 799 align = crypto_aead_alignmask(aead); 800 align &= ~(crypto_tfm_ctx_alignment() - 1); 801 crypto_aead_set_reqsize( 802 tfm, 803 sizeof(struct crypto_rfc4106_req_ctx) + 804 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 805 align + 24); 806 807 return 0; 808 } 809 810 static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm) 811 { 812 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 813 814 crypto_free_aead(ctx->child); 815 } 816 817 static void crypto_rfc4106_free(struct aead_instance *inst) 818 { 819 crypto_drop_aead(aead_instance_ctx(inst)); 820 kfree(inst); 821 } 822 823 static int crypto_rfc4106_create(struct crypto_template *tmpl, 824 struct rtattr **tb) 825 { 826 u32 mask; 827 struct aead_instance *inst; 828 struct crypto_aead_spawn *spawn; 829 struct aead_alg *alg; 830 int err; 831 832 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 833 if (err) 834 return err; 835 836 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 837 if (!inst) 838 return -ENOMEM; 839 840 spawn = aead_instance_ctx(inst); 841 err = crypto_grab_aead(spawn, aead_crypto_instance(inst), 842 crypto_attr_alg_name(tb[1]), 0, mask); 843 if (err) 844 goto err_free_inst; 845 846 alg = crypto_spawn_aead_alg(spawn); 847 848 err = -EINVAL; 849 850 /* Underlying IV size must be 12. */ 851 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) 852 goto err_free_inst; 853 854 /* Not a stream cipher? */ 855 if (alg->base.cra_blocksize != 1) 856 goto err_free_inst; 857 858 err = -ENAMETOOLONG; 859 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 860 "rfc4106(%s)", alg->base.cra_name) >= 861 CRYPTO_MAX_ALG_NAME || 862 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 863 "rfc4106(%s)", alg->base.cra_driver_name) >= 864 CRYPTO_MAX_ALG_NAME) 865 goto err_free_inst; 866 867 inst->alg.base.cra_priority = alg->base.cra_priority; 868 inst->alg.base.cra_blocksize = 1; 869 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 870 871 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); 872 873 inst->alg.ivsize = GCM_RFC4106_IV_SIZE; 874 inst->alg.chunksize = crypto_aead_alg_chunksize(alg); 875 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 876 877 inst->alg.init = crypto_rfc4106_init_tfm; 878 inst->alg.exit = crypto_rfc4106_exit_tfm; 879 880 inst->alg.setkey = crypto_rfc4106_setkey; 881 inst->alg.setauthsize = crypto_rfc4106_setauthsize; 882 inst->alg.encrypt = crypto_rfc4106_encrypt; 883 inst->alg.decrypt = crypto_rfc4106_decrypt; 884 885 inst->free = crypto_rfc4106_free; 886 887 err = aead_register_instance(tmpl, inst); 888 if (err) { 889 err_free_inst: 890 crypto_rfc4106_free(inst); 891 } 892 return err; 893 } 894 895 static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key, 896 unsigned int keylen) 897 { 898 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 899 struct crypto_aead *child = ctx->child; 900 901 if (keylen < 4) 902 return -EINVAL; 903 904 keylen -= 4; 905 memcpy(ctx->nonce, key + keylen, 4); 906 907 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 908 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 909 CRYPTO_TFM_REQ_MASK); 910 return crypto_aead_setkey(child, key, keylen); 911 } 912 913 static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, 914 unsigned int authsize) 915 { 916 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 917 918 if (authsize != 16) 919 return -EINVAL; 920 921 return crypto_aead_setauthsize(ctx->child, authsize); 922 } 923 924 static int crypto_rfc4543_crypt(struct aead_request *req, bool enc) 925 { 926 struct crypto_aead *aead = crypto_aead_reqtfm(req); 927 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 928 struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req); 929 struct aead_request *subreq = &rctx->subreq; 930 unsigned int authsize = crypto_aead_authsize(aead); 931 u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), 932 crypto_aead_alignmask(ctx->child) + 1); 933 int err; 934 935 if (req->src != req->dst) { 936 err = crypto_rfc4543_copy_src_to_dst(req, enc); 937 if (err) 938 return err; 939 } 940 941 memcpy(iv, ctx->nonce, 4); 942 memcpy(iv + 4, req->iv, 8); 943 944 aead_request_set_tfm(subreq, ctx->child); 945 aead_request_set_callback(subreq, req->base.flags, 946 req->base.complete, req->base.data); 947 aead_request_set_crypt(subreq, req->src, req->dst, 948 enc ? 0 : authsize, iv); 949 aead_request_set_ad(subreq, req->assoclen + req->cryptlen - 950 subreq->cryptlen); 951 952 return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); 953 } 954 955 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc) 956 { 957 struct crypto_aead *aead = crypto_aead_reqtfm(req); 958 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 959 unsigned int authsize = crypto_aead_authsize(aead); 960 unsigned int nbytes = req->assoclen + req->cryptlen - 961 (enc ? 0 : authsize); 962 SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null); 963 964 skcipher_request_set_sync_tfm(nreq, ctx->null); 965 skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL); 966 skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL); 967 968 return crypto_skcipher_encrypt(nreq); 969 } 970 971 static int crypto_rfc4543_encrypt(struct aead_request *req) 972 { 973 return crypto_ipsec_check_assoclen(req->assoclen) ?: 974 crypto_rfc4543_crypt(req, true); 975 } 976 977 static int crypto_rfc4543_decrypt(struct aead_request *req) 978 { 979 return crypto_ipsec_check_assoclen(req->assoclen) ?: 980 crypto_rfc4543_crypt(req, false); 981 } 982 983 static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) 984 { 985 struct aead_instance *inst = aead_alg_instance(tfm); 986 struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst); 987 struct crypto_aead_spawn *spawn = &ictx->aead; 988 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 989 struct crypto_aead *aead; 990 struct crypto_sync_skcipher *null; 991 unsigned long align; 992 int err = 0; 993 994 aead = crypto_spawn_aead(spawn); 995 if (IS_ERR(aead)) 996 return PTR_ERR(aead); 997 998 null = crypto_get_default_null_skcipher(); 999 err = PTR_ERR(null); 1000 if (IS_ERR(null)) 1001 goto err_free_aead; 1002 1003 ctx->child = aead; 1004 ctx->null = null; 1005 1006 align = crypto_aead_alignmask(aead); 1007 align &= ~(crypto_tfm_ctx_alignment() - 1); 1008 crypto_aead_set_reqsize( 1009 tfm, 1010 sizeof(struct crypto_rfc4543_req_ctx) + 1011 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 1012 align + GCM_AES_IV_SIZE); 1013 1014 return 0; 1015 1016 err_free_aead: 1017 crypto_free_aead(aead); 1018 return err; 1019 } 1020 1021 static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) 1022 { 1023 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 1024 1025 crypto_free_aead(ctx->child); 1026 crypto_put_default_null_skcipher(); 1027 } 1028 1029 static void crypto_rfc4543_free(struct aead_instance *inst) 1030 { 1031 struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst); 1032 1033 crypto_drop_aead(&ctx->aead); 1034 1035 kfree(inst); 1036 } 1037 1038 static int crypto_rfc4543_create(struct crypto_template *tmpl, 1039 struct rtattr **tb) 1040 { 1041 u32 mask; 1042 struct aead_instance *inst; 1043 struct aead_alg *alg; 1044 struct crypto_rfc4543_instance_ctx *ctx; 1045 int err; 1046 1047 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 1048 if (err) 1049 return err; 1050 1051 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 1052 if (!inst) 1053 return -ENOMEM; 1054 1055 ctx = aead_instance_ctx(inst); 1056 err = crypto_grab_aead(&ctx->aead, aead_crypto_instance(inst), 1057 crypto_attr_alg_name(tb[1]), 0, mask); 1058 if (err) 1059 goto err_free_inst; 1060 1061 alg = crypto_spawn_aead_alg(&ctx->aead); 1062 1063 err = -EINVAL; 1064 1065 /* Underlying IV size must be 12. */ 1066 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) 1067 goto err_free_inst; 1068 1069 /* Not a stream cipher? */ 1070 if (alg->base.cra_blocksize != 1) 1071 goto err_free_inst; 1072 1073 err = -ENAMETOOLONG; 1074 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 1075 "rfc4543(%s)", alg->base.cra_name) >= 1076 CRYPTO_MAX_ALG_NAME || 1077 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 1078 "rfc4543(%s)", alg->base.cra_driver_name) >= 1079 CRYPTO_MAX_ALG_NAME) 1080 goto err_free_inst; 1081 1082 inst->alg.base.cra_priority = alg->base.cra_priority; 1083 inst->alg.base.cra_blocksize = 1; 1084 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 1085 1086 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); 1087 1088 inst->alg.ivsize = GCM_RFC4543_IV_SIZE; 1089 inst->alg.chunksize = crypto_aead_alg_chunksize(alg); 1090 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 1091 1092 inst->alg.init = crypto_rfc4543_init_tfm; 1093 inst->alg.exit = crypto_rfc4543_exit_tfm; 1094 1095 inst->alg.setkey = crypto_rfc4543_setkey; 1096 inst->alg.setauthsize = crypto_rfc4543_setauthsize; 1097 inst->alg.encrypt = crypto_rfc4543_encrypt; 1098 inst->alg.decrypt = crypto_rfc4543_decrypt; 1099 1100 inst->free = crypto_rfc4543_free; 1101 1102 err = aead_register_instance(tmpl, inst); 1103 if (err) { 1104 err_free_inst: 1105 crypto_rfc4543_free(inst); 1106 } 1107 return err; 1108 } 1109 1110 static struct crypto_template crypto_gcm_tmpls[] = { 1111 { 1112 .name = "gcm_base", 1113 .create = crypto_gcm_base_create, 1114 .module = THIS_MODULE, 1115 }, { 1116 .name = "gcm", 1117 .create = crypto_gcm_create, 1118 .module = THIS_MODULE, 1119 }, { 1120 .name = "rfc4106", 1121 .create = crypto_rfc4106_create, 1122 .module = THIS_MODULE, 1123 }, { 1124 .name = "rfc4543", 1125 .create = crypto_rfc4543_create, 1126 .module = THIS_MODULE, 1127 }, 1128 }; 1129 1130 static int __init crypto_gcm_module_init(void) 1131 { 1132 int err; 1133 1134 gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL); 1135 if (!gcm_zeroes) 1136 return -ENOMEM; 1137 1138 sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf)); 1139 1140 err = crypto_register_templates(crypto_gcm_tmpls, 1141 ARRAY_SIZE(crypto_gcm_tmpls)); 1142 if (err) 1143 kfree(gcm_zeroes); 1144 1145 return err; 1146 } 1147 1148 static void __exit crypto_gcm_module_exit(void) 1149 { 1150 kfree(gcm_zeroes); 1151 crypto_unregister_templates(crypto_gcm_tmpls, 1152 ARRAY_SIZE(crypto_gcm_tmpls)); 1153 } 1154 1155 subsys_initcall(crypto_gcm_module_init); 1156 module_exit(crypto_gcm_module_exit); 1157 1158 MODULE_LICENSE("GPL"); 1159 MODULE_DESCRIPTION("Galois/Counter Mode"); 1160 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); 1161 MODULE_ALIAS_CRYPTO("gcm_base"); 1162 MODULE_ALIAS_CRYPTO("rfc4106"); 1163 MODULE_ALIAS_CRYPTO("rfc4543"); 1164 MODULE_ALIAS_CRYPTO("gcm"); 1165