1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * GCM: Galois/Counter Mode. 4 * 5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> 6 */ 7 8 #include <crypto/gf128mul.h> 9 #include <crypto/internal/aead.h> 10 #include <crypto/internal/skcipher.h> 11 #include <crypto/internal/hash.h> 12 #include <crypto/null.h> 13 #include <crypto/scatterwalk.h> 14 #include <crypto/gcm.h> 15 #include <crypto/hash.h> 16 #include <linux/err.h> 17 #include <linux/init.h> 18 #include <linux/kernel.h> 19 #include <linux/module.h> 20 #include <linux/slab.h> 21 22 struct gcm_instance_ctx { 23 struct crypto_skcipher_spawn ctr; 24 struct crypto_ahash_spawn ghash; 25 }; 26 27 struct crypto_gcm_ctx { 28 struct crypto_skcipher *ctr; 29 struct crypto_ahash *ghash; 30 }; 31 32 struct crypto_rfc4106_ctx { 33 struct crypto_aead *child; 34 u8 nonce[4]; 35 }; 36 37 struct crypto_rfc4106_req_ctx { 38 struct scatterlist src[3]; 39 struct scatterlist dst[3]; 40 struct aead_request subreq; 41 }; 42 43 struct crypto_rfc4543_instance_ctx { 44 struct crypto_aead_spawn aead; 45 }; 46 47 struct crypto_rfc4543_ctx { 48 struct crypto_aead *child; 49 struct crypto_sync_skcipher *null; 50 u8 nonce[4]; 51 }; 52 53 struct crypto_rfc4543_req_ctx { 54 struct aead_request subreq; 55 }; 56 57 struct crypto_gcm_ghash_ctx { 58 unsigned int cryptlen; 59 struct scatterlist *src; 60 int (*complete)(struct aead_request *req, u32 flags); 61 }; 62 63 struct crypto_gcm_req_priv_ctx { 64 u8 iv[16]; 65 u8 auth_tag[16]; 66 u8 iauth_tag[16]; 67 struct scatterlist src[3]; 68 struct scatterlist dst[3]; 69 struct scatterlist sg; 70 struct crypto_gcm_ghash_ctx ghash_ctx; 71 union { 72 struct ahash_request ahreq; 73 struct skcipher_request skreq; 74 } u; 75 }; 76 77 static struct { 78 u8 buf[16]; 79 struct scatterlist sg; 80 } *gcm_zeroes; 81 82 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc); 83 84 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( 85 struct aead_request *req) 86 { 87 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); 88 89 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); 90 } 91 92 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, 93 unsigned int keylen) 94 { 95 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 96 struct crypto_ahash *ghash = ctx->ghash; 97 struct crypto_skcipher *ctr = ctx->ctr; 98 struct { 99 be128 hash; 100 u8 iv[16]; 101 102 struct crypto_wait wait; 103 104 struct scatterlist sg[1]; 105 struct skcipher_request req; 106 } *data; 107 int err; 108 109 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); 110 crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & 111 CRYPTO_TFM_REQ_MASK); 112 err = crypto_skcipher_setkey(ctr, key, keylen); 113 if (err) 114 return err; 115 116 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr), 117 GFP_KERNEL); 118 if (!data) 119 return -ENOMEM; 120 121 crypto_init_wait(&data->wait); 122 sg_init_one(data->sg, &data->hash, sizeof(data->hash)); 123 skcipher_request_set_tfm(&data->req, ctr); 124 skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | 125 CRYPTO_TFM_REQ_MAY_BACKLOG, 126 crypto_req_done, 127 &data->wait); 128 skcipher_request_set_crypt(&data->req, data->sg, data->sg, 129 sizeof(data->hash), data->iv); 130 131 err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), 132 &data->wait); 133 134 if (err) 135 goto out; 136 137 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); 138 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & 139 CRYPTO_TFM_REQ_MASK); 140 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); 141 out: 142 kfree_sensitive(data); 143 return err; 144 } 145 146 static int crypto_gcm_setauthsize(struct crypto_aead *tfm, 147 unsigned int authsize) 148 { 149 return crypto_gcm_check_authsize(authsize); 150 } 151 152 static void crypto_gcm_init_common(struct aead_request *req) 153 { 154 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 155 __be32 counter = cpu_to_be32(1); 156 struct scatterlist *sg; 157 158 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); 159 memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE); 160 memcpy(pctx->iv + GCM_AES_IV_SIZE, &counter, 4); 161 162 sg_init_table(pctx->src, 3); 163 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); 164 sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); 165 if (sg != pctx->src + 1) 166 sg_chain(pctx->src, 2, sg); 167 168 if (req->src != req->dst) { 169 sg_init_table(pctx->dst, 3); 170 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); 171 sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); 172 if (sg != pctx->dst + 1) 173 sg_chain(pctx->dst, 2, sg); 174 } 175 } 176 177 static void crypto_gcm_init_crypt(struct aead_request *req, 178 unsigned int cryptlen) 179 { 180 struct crypto_aead *aead = crypto_aead_reqtfm(req); 181 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 182 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 183 struct skcipher_request *skreq = &pctx->u.skreq; 184 struct scatterlist *dst; 185 186 dst = req->src == req->dst ? pctx->src : pctx->dst; 187 188 skcipher_request_set_tfm(skreq, ctx->ctr); 189 skcipher_request_set_crypt(skreq, pctx->src, dst, 190 cryptlen + sizeof(pctx->auth_tag), 191 pctx->iv); 192 } 193 194 static inline unsigned int gcm_remain(unsigned int len) 195 { 196 len &= 0xfU; 197 return len ? 16 - len : 0; 198 } 199 200 static void gcm_hash_len_done(void *data, int err); 201 202 static int gcm_hash_update(struct aead_request *req, 203 crypto_completion_t compl, 204 struct scatterlist *src, 205 unsigned int len, u32 flags) 206 { 207 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 208 struct ahash_request *ahreq = &pctx->u.ahreq; 209 210 ahash_request_set_callback(ahreq, flags, compl, req); 211 ahash_request_set_crypt(ahreq, src, NULL, len); 212 213 return crypto_ahash_update(ahreq); 214 } 215 216 static int gcm_hash_remain(struct aead_request *req, 217 unsigned int remain, 218 crypto_completion_t compl, u32 flags) 219 { 220 return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); 221 } 222 223 static int gcm_hash_len(struct aead_request *req, u32 flags) 224 { 225 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 226 struct ahash_request *ahreq = &pctx->u.ahreq; 227 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 228 be128 lengths; 229 230 lengths.a = cpu_to_be64(req->assoclen * 8); 231 lengths.b = cpu_to_be64(gctx->cryptlen * 8); 232 memcpy(pctx->iauth_tag, &lengths, 16); 233 sg_init_one(&pctx->sg, pctx->iauth_tag, 16); 234 ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); 235 ahash_request_set_crypt(ahreq, &pctx->sg, 236 pctx->iauth_tag, sizeof(lengths)); 237 238 return crypto_ahash_finup(ahreq); 239 } 240 241 static int gcm_hash_len_continue(struct aead_request *req, u32 flags) 242 { 243 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 244 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 245 246 return gctx->complete(req, flags); 247 } 248 249 static void gcm_hash_len_done(void *data, int err) 250 { 251 struct aead_request *req = data; 252 253 if (err) 254 goto out; 255 256 err = gcm_hash_len_continue(req, 0); 257 if (err == -EINPROGRESS) 258 return; 259 260 out: 261 aead_request_complete(req, err); 262 } 263 264 static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) 265 { 266 return gcm_hash_len(req, flags) ?: 267 gcm_hash_len_continue(req, flags); 268 } 269 270 static void gcm_hash_crypt_remain_done(void *data, int err) 271 { 272 struct aead_request *req = data; 273 274 if (err) 275 goto out; 276 277 err = gcm_hash_crypt_remain_continue(req, 0); 278 if (err == -EINPROGRESS) 279 return; 280 281 out: 282 aead_request_complete(req, err); 283 } 284 285 static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) 286 { 287 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 288 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 289 unsigned int remain; 290 291 remain = gcm_remain(gctx->cryptlen); 292 if (remain) 293 return gcm_hash_remain(req, remain, 294 gcm_hash_crypt_remain_done, flags) ?: 295 gcm_hash_crypt_remain_continue(req, flags); 296 297 return gcm_hash_crypt_remain_continue(req, flags); 298 } 299 300 static void gcm_hash_crypt_done(void *data, int err) 301 { 302 struct aead_request *req = data; 303 304 if (err) 305 goto out; 306 307 err = gcm_hash_crypt_continue(req, 0); 308 if (err == -EINPROGRESS) 309 return; 310 311 out: 312 aead_request_complete(req, err); 313 } 314 315 static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) 316 { 317 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 318 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 319 320 if (gctx->cryptlen) 321 return gcm_hash_update(req, gcm_hash_crypt_done, 322 gctx->src, gctx->cryptlen, flags) ?: 323 gcm_hash_crypt_continue(req, flags); 324 325 return gcm_hash_crypt_remain_continue(req, flags); 326 } 327 328 static void gcm_hash_assoc_remain_done(void *data, int err) 329 { 330 struct aead_request *req = data; 331 332 if (err) 333 goto out; 334 335 err = gcm_hash_assoc_remain_continue(req, 0); 336 if (err == -EINPROGRESS) 337 return; 338 339 out: 340 aead_request_complete(req, err); 341 } 342 343 static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) 344 { 345 unsigned int remain; 346 347 remain = gcm_remain(req->assoclen); 348 if (remain) 349 return gcm_hash_remain(req, remain, 350 gcm_hash_assoc_remain_done, flags) ?: 351 gcm_hash_assoc_remain_continue(req, flags); 352 353 return gcm_hash_assoc_remain_continue(req, flags); 354 } 355 356 static void gcm_hash_assoc_done(void *data, int err) 357 { 358 struct aead_request *req = data; 359 360 if (err) 361 goto out; 362 363 err = gcm_hash_assoc_continue(req, 0); 364 if (err == -EINPROGRESS) 365 return; 366 367 out: 368 aead_request_complete(req, err); 369 } 370 371 static int gcm_hash_init_continue(struct aead_request *req, u32 flags) 372 { 373 if (req->assoclen) 374 return gcm_hash_update(req, gcm_hash_assoc_done, 375 req->src, req->assoclen, flags) ?: 376 gcm_hash_assoc_continue(req, flags); 377 378 return gcm_hash_assoc_remain_continue(req, flags); 379 } 380 381 static void gcm_hash_init_done(void *data, int err) 382 { 383 struct aead_request *req = data; 384 385 if (err) 386 goto out; 387 388 err = gcm_hash_init_continue(req, 0); 389 if (err == -EINPROGRESS) 390 return; 391 392 out: 393 aead_request_complete(req, err); 394 } 395 396 static int gcm_hash(struct aead_request *req, u32 flags) 397 { 398 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 399 struct ahash_request *ahreq = &pctx->u.ahreq; 400 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 401 402 ahash_request_set_tfm(ahreq, ctx->ghash); 403 404 ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); 405 return crypto_ahash_init(ahreq) ?: 406 gcm_hash_init_continue(req, flags); 407 } 408 409 static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) 410 { 411 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 412 struct crypto_aead *aead = crypto_aead_reqtfm(req); 413 u8 *auth_tag = pctx->auth_tag; 414 415 crypto_xor(auth_tag, pctx->iauth_tag, 16); 416 scatterwalk_map_and_copy(auth_tag, req->dst, 417 req->assoclen + req->cryptlen, 418 crypto_aead_authsize(aead), 1); 419 return 0; 420 } 421 422 static int gcm_encrypt_continue(struct aead_request *req, u32 flags) 423 { 424 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 425 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 426 427 gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); 428 gctx->cryptlen = req->cryptlen; 429 gctx->complete = gcm_enc_copy_hash; 430 431 return gcm_hash(req, flags); 432 } 433 434 static void gcm_encrypt_done(void *data, int err) 435 { 436 struct aead_request *req = data; 437 438 if (err) 439 goto out; 440 441 err = gcm_encrypt_continue(req, 0); 442 if (err == -EINPROGRESS) 443 return; 444 445 out: 446 aead_request_complete(req, err); 447 } 448 449 static int crypto_gcm_encrypt(struct aead_request *req) 450 { 451 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 452 struct skcipher_request *skreq = &pctx->u.skreq; 453 u32 flags = aead_request_flags(req); 454 455 crypto_gcm_init_common(req); 456 crypto_gcm_init_crypt(req, req->cryptlen); 457 skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); 458 459 return crypto_skcipher_encrypt(skreq) ?: 460 gcm_encrypt_continue(req, flags); 461 } 462 463 static int crypto_gcm_verify(struct aead_request *req) 464 { 465 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 466 struct crypto_aead *aead = crypto_aead_reqtfm(req); 467 u8 *auth_tag = pctx->auth_tag; 468 u8 *iauth_tag = pctx->iauth_tag; 469 unsigned int authsize = crypto_aead_authsize(aead); 470 unsigned int cryptlen = req->cryptlen - authsize; 471 472 crypto_xor(auth_tag, iauth_tag, 16); 473 scatterwalk_map_and_copy(iauth_tag, req->src, 474 req->assoclen + cryptlen, authsize, 0); 475 return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; 476 } 477 478 static void gcm_decrypt_done(void *data, int err) 479 { 480 struct aead_request *req = data; 481 482 if (!err) 483 err = crypto_gcm_verify(req); 484 485 aead_request_complete(req, err); 486 } 487 488 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) 489 { 490 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 491 struct skcipher_request *skreq = &pctx->u.skreq; 492 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 493 494 crypto_gcm_init_crypt(req, gctx->cryptlen); 495 skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); 496 return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); 497 } 498 499 static int crypto_gcm_decrypt(struct aead_request *req) 500 { 501 struct crypto_aead *aead = crypto_aead_reqtfm(req); 502 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 503 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 504 unsigned int authsize = crypto_aead_authsize(aead); 505 unsigned int cryptlen = req->cryptlen; 506 u32 flags = aead_request_flags(req); 507 508 cryptlen -= authsize; 509 510 crypto_gcm_init_common(req); 511 512 gctx->src = sg_next(pctx->src); 513 gctx->cryptlen = cryptlen; 514 gctx->complete = gcm_dec_hash_continue; 515 516 return gcm_hash(req, flags); 517 } 518 519 static int crypto_gcm_init_tfm(struct crypto_aead *tfm) 520 { 521 struct aead_instance *inst = aead_alg_instance(tfm); 522 struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); 523 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 524 struct crypto_skcipher *ctr; 525 struct crypto_ahash *ghash; 526 unsigned long align; 527 int err; 528 529 ghash = crypto_spawn_ahash(&ictx->ghash); 530 if (IS_ERR(ghash)) 531 return PTR_ERR(ghash); 532 533 ctr = crypto_spawn_skcipher(&ictx->ctr); 534 err = PTR_ERR(ctr); 535 if (IS_ERR(ctr)) 536 goto err_free_hash; 537 538 ctx->ctr = ctr; 539 ctx->ghash = ghash; 540 541 align = crypto_aead_alignmask(tfm); 542 align &= ~(crypto_tfm_ctx_alignment() - 1); 543 crypto_aead_set_reqsize(tfm, 544 align + offsetof(struct crypto_gcm_req_priv_ctx, u) + 545 max(sizeof(struct skcipher_request) + 546 crypto_skcipher_reqsize(ctr), 547 sizeof(struct ahash_request) + 548 crypto_ahash_reqsize(ghash))); 549 550 return 0; 551 552 err_free_hash: 553 crypto_free_ahash(ghash); 554 return err; 555 } 556 557 static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) 558 { 559 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 560 561 crypto_free_ahash(ctx->ghash); 562 crypto_free_skcipher(ctx->ctr); 563 } 564 565 static void crypto_gcm_free(struct aead_instance *inst) 566 { 567 struct gcm_instance_ctx *ctx = aead_instance_ctx(inst); 568 569 crypto_drop_skcipher(&ctx->ctr); 570 crypto_drop_ahash(&ctx->ghash); 571 kfree(inst); 572 } 573 574 static int crypto_gcm_create_common(struct crypto_template *tmpl, 575 struct rtattr **tb, 576 const char *ctr_name, 577 const char *ghash_name) 578 { 579 struct skcipher_alg_common *ctr; 580 u32 mask; 581 struct aead_instance *inst; 582 struct gcm_instance_ctx *ctx; 583 struct hash_alg_common *ghash; 584 int err; 585 586 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 587 if (err) 588 return err; 589 590 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 591 if (!inst) 592 return -ENOMEM; 593 ctx = aead_instance_ctx(inst); 594 595 err = crypto_grab_ahash(&ctx->ghash, aead_crypto_instance(inst), 596 ghash_name, 0, mask); 597 if (err) 598 goto err_free_inst; 599 ghash = crypto_spawn_ahash_alg(&ctx->ghash); 600 601 err = -EINVAL; 602 if (strcmp(ghash->base.cra_name, "ghash") != 0 || 603 ghash->digestsize != 16) 604 goto err_free_inst; 605 606 err = crypto_grab_skcipher(&ctx->ctr, aead_crypto_instance(inst), 607 ctr_name, 0, mask); 608 if (err) 609 goto err_free_inst; 610 ctr = crypto_spawn_skcipher_alg_common(&ctx->ctr); 611 612 /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ 613 err = -EINVAL; 614 if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || 615 ctr->ivsize != 16 || ctr->base.cra_blocksize != 1) 616 goto err_free_inst; 617 618 err = -ENAMETOOLONG; 619 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 620 "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) 621 goto err_free_inst; 622 623 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 624 "gcm_base(%s,%s)", ctr->base.cra_driver_name, 625 ghash->base.cra_driver_name) >= 626 CRYPTO_MAX_ALG_NAME) 627 goto err_free_inst; 628 629 inst->alg.base.cra_priority = (ghash->base.cra_priority + 630 ctr->base.cra_priority) / 2; 631 inst->alg.base.cra_blocksize = 1; 632 inst->alg.base.cra_alignmask = ghash->base.cra_alignmask | 633 ctr->base.cra_alignmask; 634 inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); 635 inst->alg.ivsize = GCM_AES_IV_SIZE; 636 inst->alg.chunksize = ctr->chunksize; 637 inst->alg.maxauthsize = 16; 638 inst->alg.init = crypto_gcm_init_tfm; 639 inst->alg.exit = crypto_gcm_exit_tfm; 640 inst->alg.setkey = crypto_gcm_setkey; 641 inst->alg.setauthsize = crypto_gcm_setauthsize; 642 inst->alg.encrypt = crypto_gcm_encrypt; 643 inst->alg.decrypt = crypto_gcm_decrypt; 644 645 inst->free = crypto_gcm_free; 646 647 err = aead_register_instance(tmpl, inst); 648 if (err) { 649 err_free_inst: 650 crypto_gcm_free(inst); 651 } 652 return err; 653 } 654 655 static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) 656 { 657 const char *cipher_name; 658 char ctr_name[CRYPTO_MAX_ALG_NAME]; 659 660 cipher_name = crypto_attr_alg_name(tb[1]); 661 if (IS_ERR(cipher_name)) 662 return PTR_ERR(cipher_name); 663 664 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= 665 CRYPTO_MAX_ALG_NAME) 666 return -ENAMETOOLONG; 667 668 return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); 669 } 670 671 static int crypto_gcm_base_create(struct crypto_template *tmpl, 672 struct rtattr **tb) 673 { 674 const char *ctr_name; 675 const char *ghash_name; 676 677 ctr_name = crypto_attr_alg_name(tb[1]); 678 if (IS_ERR(ctr_name)) 679 return PTR_ERR(ctr_name); 680 681 ghash_name = crypto_attr_alg_name(tb[2]); 682 if (IS_ERR(ghash_name)) 683 return PTR_ERR(ghash_name); 684 685 return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); 686 } 687 688 static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, 689 unsigned int keylen) 690 { 691 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 692 struct crypto_aead *child = ctx->child; 693 694 if (keylen < 4) 695 return -EINVAL; 696 697 keylen -= 4; 698 memcpy(ctx->nonce, key + keylen, 4); 699 700 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 701 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 702 CRYPTO_TFM_REQ_MASK); 703 return crypto_aead_setkey(child, key, keylen); 704 } 705 706 static int crypto_rfc4106_setauthsize(struct crypto_aead *parent, 707 unsigned int authsize) 708 { 709 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 710 int err; 711 712 err = crypto_rfc4106_check_authsize(authsize); 713 if (err) 714 return err; 715 716 return crypto_aead_setauthsize(ctx->child, authsize); 717 } 718 719 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) 720 { 721 struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req); 722 struct crypto_aead *aead = crypto_aead_reqtfm(req); 723 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead); 724 struct aead_request *subreq = &rctx->subreq; 725 struct crypto_aead *child = ctx->child; 726 struct scatterlist *sg; 727 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), 728 crypto_aead_alignmask(child) + 1); 729 730 scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0); 731 732 memcpy(iv, ctx->nonce, 4); 733 memcpy(iv + 4, req->iv, 8); 734 735 sg_init_table(rctx->src, 3); 736 sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8); 737 sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen); 738 if (sg != rctx->src + 1) 739 sg_chain(rctx->src, 2, sg); 740 741 if (req->src != req->dst) { 742 sg_init_table(rctx->dst, 3); 743 sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8); 744 sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen); 745 if (sg != rctx->dst + 1) 746 sg_chain(rctx->dst, 2, sg); 747 } 748 749 aead_request_set_tfm(subreq, child); 750 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 751 req->base.data); 752 aead_request_set_crypt(subreq, rctx->src, 753 req->src == req->dst ? rctx->src : rctx->dst, 754 req->cryptlen, iv); 755 aead_request_set_ad(subreq, req->assoclen - 8); 756 757 return subreq; 758 } 759 760 static int crypto_rfc4106_encrypt(struct aead_request *req) 761 { 762 int err; 763 764 err = crypto_ipsec_check_assoclen(req->assoclen); 765 if (err) 766 return err; 767 768 req = crypto_rfc4106_crypt(req); 769 770 return crypto_aead_encrypt(req); 771 } 772 773 static int crypto_rfc4106_decrypt(struct aead_request *req) 774 { 775 int err; 776 777 err = crypto_ipsec_check_assoclen(req->assoclen); 778 if (err) 779 return err; 780 781 req = crypto_rfc4106_crypt(req); 782 783 return crypto_aead_decrypt(req); 784 } 785 786 static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm) 787 { 788 struct aead_instance *inst = aead_alg_instance(tfm); 789 struct crypto_aead_spawn *spawn = aead_instance_ctx(inst); 790 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 791 struct crypto_aead *aead; 792 unsigned long align; 793 794 aead = crypto_spawn_aead(spawn); 795 if (IS_ERR(aead)) 796 return PTR_ERR(aead); 797 798 ctx->child = aead; 799 800 align = crypto_aead_alignmask(aead); 801 align &= ~(crypto_tfm_ctx_alignment() - 1); 802 crypto_aead_set_reqsize( 803 tfm, 804 sizeof(struct crypto_rfc4106_req_ctx) + 805 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 806 align + 24); 807 808 return 0; 809 } 810 811 static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm) 812 { 813 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 814 815 crypto_free_aead(ctx->child); 816 } 817 818 static void crypto_rfc4106_free(struct aead_instance *inst) 819 { 820 crypto_drop_aead(aead_instance_ctx(inst)); 821 kfree(inst); 822 } 823 824 static int crypto_rfc4106_create(struct crypto_template *tmpl, 825 struct rtattr **tb) 826 { 827 u32 mask; 828 struct aead_instance *inst; 829 struct crypto_aead_spawn *spawn; 830 struct aead_alg *alg; 831 int err; 832 833 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 834 if (err) 835 return err; 836 837 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 838 if (!inst) 839 return -ENOMEM; 840 841 spawn = aead_instance_ctx(inst); 842 err = crypto_grab_aead(spawn, aead_crypto_instance(inst), 843 crypto_attr_alg_name(tb[1]), 0, mask); 844 if (err) 845 goto err_free_inst; 846 847 alg = crypto_spawn_aead_alg(spawn); 848 849 err = -EINVAL; 850 851 /* Underlying IV size must be 12. */ 852 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) 853 goto err_free_inst; 854 855 /* Not a stream cipher? */ 856 if (alg->base.cra_blocksize != 1) 857 goto err_free_inst; 858 859 err = -ENAMETOOLONG; 860 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 861 "rfc4106(%s)", alg->base.cra_name) >= 862 CRYPTO_MAX_ALG_NAME || 863 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 864 "rfc4106(%s)", alg->base.cra_driver_name) >= 865 CRYPTO_MAX_ALG_NAME) 866 goto err_free_inst; 867 868 inst->alg.base.cra_priority = alg->base.cra_priority; 869 inst->alg.base.cra_blocksize = 1; 870 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 871 872 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); 873 874 inst->alg.ivsize = GCM_RFC4106_IV_SIZE; 875 inst->alg.chunksize = crypto_aead_alg_chunksize(alg); 876 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 877 878 inst->alg.init = crypto_rfc4106_init_tfm; 879 inst->alg.exit = crypto_rfc4106_exit_tfm; 880 881 inst->alg.setkey = crypto_rfc4106_setkey; 882 inst->alg.setauthsize = crypto_rfc4106_setauthsize; 883 inst->alg.encrypt = crypto_rfc4106_encrypt; 884 inst->alg.decrypt = crypto_rfc4106_decrypt; 885 886 inst->free = crypto_rfc4106_free; 887 888 err = aead_register_instance(tmpl, inst); 889 if (err) { 890 err_free_inst: 891 crypto_rfc4106_free(inst); 892 } 893 return err; 894 } 895 896 static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key, 897 unsigned int keylen) 898 { 899 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 900 struct crypto_aead *child = ctx->child; 901 902 if (keylen < 4) 903 return -EINVAL; 904 905 keylen -= 4; 906 memcpy(ctx->nonce, key + keylen, 4); 907 908 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 909 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 910 CRYPTO_TFM_REQ_MASK); 911 return crypto_aead_setkey(child, key, keylen); 912 } 913 914 static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, 915 unsigned int authsize) 916 { 917 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 918 919 if (authsize != 16) 920 return -EINVAL; 921 922 return crypto_aead_setauthsize(ctx->child, authsize); 923 } 924 925 static int crypto_rfc4543_crypt(struct aead_request *req, bool enc) 926 { 927 struct crypto_aead *aead = crypto_aead_reqtfm(req); 928 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 929 struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req); 930 struct aead_request *subreq = &rctx->subreq; 931 unsigned int authsize = crypto_aead_authsize(aead); 932 u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), 933 crypto_aead_alignmask(ctx->child) + 1); 934 int err; 935 936 if (req->src != req->dst) { 937 err = crypto_rfc4543_copy_src_to_dst(req, enc); 938 if (err) 939 return err; 940 } 941 942 memcpy(iv, ctx->nonce, 4); 943 memcpy(iv + 4, req->iv, 8); 944 945 aead_request_set_tfm(subreq, ctx->child); 946 aead_request_set_callback(subreq, req->base.flags, 947 req->base.complete, req->base.data); 948 aead_request_set_crypt(subreq, req->src, req->dst, 949 enc ? 0 : authsize, iv); 950 aead_request_set_ad(subreq, req->assoclen + req->cryptlen - 951 subreq->cryptlen); 952 953 return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); 954 } 955 956 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc) 957 { 958 struct crypto_aead *aead = crypto_aead_reqtfm(req); 959 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 960 unsigned int authsize = crypto_aead_authsize(aead); 961 unsigned int nbytes = req->assoclen + req->cryptlen - 962 (enc ? 0 : authsize); 963 SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null); 964 965 skcipher_request_set_sync_tfm(nreq, ctx->null); 966 skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL); 967 skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL); 968 969 return crypto_skcipher_encrypt(nreq); 970 } 971 972 static int crypto_rfc4543_encrypt(struct aead_request *req) 973 { 974 return crypto_ipsec_check_assoclen(req->assoclen) ?: 975 crypto_rfc4543_crypt(req, true); 976 } 977 978 static int crypto_rfc4543_decrypt(struct aead_request *req) 979 { 980 return crypto_ipsec_check_assoclen(req->assoclen) ?: 981 crypto_rfc4543_crypt(req, false); 982 } 983 984 static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) 985 { 986 struct aead_instance *inst = aead_alg_instance(tfm); 987 struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst); 988 struct crypto_aead_spawn *spawn = &ictx->aead; 989 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 990 struct crypto_aead *aead; 991 struct crypto_sync_skcipher *null; 992 unsigned long align; 993 int err = 0; 994 995 aead = crypto_spawn_aead(spawn); 996 if (IS_ERR(aead)) 997 return PTR_ERR(aead); 998 999 null = crypto_get_default_null_skcipher(); 1000 err = PTR_ERR(null); 1001 if (IS_ERR(null)) 1002 goto err_free_aead; 1003 1004 ctx->child = aead; 1005 ctx->null = null; 1006 1007 align = crypto_aead_alignmask(aead); 1008 align &= ~(crypto_tfm_ctx_alignment() - 1); 1009 crypto_aead_set_reqsize( 1010 tfm, 1011 sizeof(struct crypto_rfc4543_req_ctx) + 1012 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 1013 align + GCM_AES_IV_SIZE); 1014 1015 return 0; 1016 1017 err_free_aead: 1018 crypto_free_aead(aead); 1019 return err; 1020 } 1021 1022 static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) 1023 { 1024 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 1025 1026 crypto_free_aead(ctx->child); 1027 crypto_put_default_null_skcipher(); 1028 } 1029 1030 static void crypto_rfc4543_free(struct aead_instance *inst) 1031 { 1032 struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst); 1033 1034 crypto_drop_aead(&ctx->aead); 1035 1036 kfree(inst); 1037 } 1038 1039 static int crypto_rfc4543_create(struct crypto_template *tmpl, 1040 struct rtattr **tb) 1041 { 1042 u32 mask; 1043 struct aead_instance *inst; 1044 struct aead_alg *alg; 1045 struct crypto_rfc4543_instance_ctx *ctx; 1046 int err; 1047 1048 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); 1049 if (err) 1050 return err; 1051 1052 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 1053 if (!inst) 1054 return -ENOMEM; 1055 1056 ctx = aead_instance_ctx(inst); 1057 err = crypto_grab_aead(&ctx->aead, aead_crypto_instance(inst), 1058 crypto_attr_alg_name(tb[1]), 0, mask); 1059 if (err) 1060 goto err_free_inst; 1061 1062 alg = crypto_spawn_aead_alg(&ctx->aead); 1063 1064 err = -EINVAL; 1065 1066 /* Underlying IV size must be 12. */ 1067 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) 1068 goto err_free_inst; 1069 1070 /* Not a stream cipher? */ 1071 if (alg->base.cra_blocksize != 1) 1072 goto err_free_inst; 1073 1074 err = -ENAMETOOLONG; 1075 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 1076 "rfc4543(%s)", alg->base.cra_name) >= 1077 CRYPTO_MAX_ALG_NAME || 1078 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 1079 "rfc4543(%s)", alg->base.cra_driver_name) >= 1080 CRYPTO_MAX_ALG_NAME) 1081 goto err_free_inst; 1082 1083 inst->alg.base.cra_priority = alg->base.cra_priority; 1084 inst->alg.base.cra_blocksize = 1; 1085 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 1086 1087 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); 1088 1089 inst->alg.ivsize = GCM_RFC4543_IV_SIZE; 1090 inst->alg.chunksize = crypto_aead_alg_chunksize(alg); 1091 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 1092 1093 inst->alg.init = crypto_rfc4543_init_tfm; 1094 inst->alg.exit = crypto_rfc4543_exit_tfm; 1095 1096 inst->alg.setkey = crypto_rfc4543_setkey; 1097 inst->alg.setauthsize = crypto_rfc4543_setauthsize; 1098 inst->alg.encrypt = crypto_rfc4543_encrypt; 1099 inst->alg.decrypt = crypto_rfc4543_decrypt; 1100 1101 inst->free = crypto_rfc4543_free; 1102 1103 err = aead_register_instance(tmpl, inst); 1104 if (err) { 1105 err_free_inst: 1106 crypto_rfc4543_free(inst); 1107 } 1108 return err; 1109 } 1110 1111 static struct crypto_template crypto_gcm_tmpls[] = { 1112 { 1113 .name = "gcm_base", 1114 .create = crypto_gcm_base_create, 1115 .module = THIS_MODULE, 1116 }, { 1117 .name = "gcm", 1118 .create = crypto_gcm_create, 1119 .module = THIS_MODULE, 1120 }, { 1121 .name = "rfc4106", 1122 .create = crypto_rfc4106_create, 1123 .module = THIS_MODULE, 1124 }, { 1125 .name = "rfc4543", 1126 .create = crypto_rfc4543_create, 1127 .module = THIS_MODULE, 1128 }, 1129 }; 1130 1131 static int __init crypto_gcm_module_init(void) 1132 { 1133 int err; 1134 1135 gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL); 1136 if (!gcm_zeroes) 1137 return -ENOMEM; 1138 1139 sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf)); 1140 1141 err = crypto_register_templates(crypto_gcm_tmpls, 1142 ARRAY_SIZE(crypto_gcm_tmpls)); 1143 if (err) 1144 kfree(gcm_zeroes); 1145 1146 return err; 1147 } 1148 1149 static void __exit crypto_gcm_module_exit(void) 1150 { 1151 kfree(gcm_zeroes); 1152 crypto_unregister_templates(crypto_gcm_tmpls, 1153 ARRAY_SIZE(crypto_gcm_tmpls)); 1154 } 1155 1156 subsys_initcall(crypto_gcm_module_init); 1157 module_exit(crypto_gcm_module_exit); 1158 1159 MODULE_LICENSE("GPL"); 1160 MODULE_DESCRIPTION("Galois/Counter Mode"); 1161 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); 1162 MODULE_ALIAS_CRYPTO("gcm_base"); 1163 MODULE_ALIAS_CRYPTO("rfc4106"); 1164 MODULE_ALIAS_CRYPTO("rfc4543"); 1165 MODULE_ALIAS_CRYPTO("gcm"); 1166