1 /* 2 * Copyright (C) 2003 Christophe Saout <christophe@saout.de> 3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> 4 * Copyright (C) 2006 Red Hat, Inc. All rights reserved. 5 * 6 * This file is released under the GPL. 7 */ 8 9 #include <linux/err.h> 10 #include <linux/module.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/bio.h> 14 #include <linux/blkdev.h> 15 #include <linux/mempool.h> 16 #include <linux/slab.h> 17 #include <linux/crypto.h> 18 #include <linux/workqueue.h> 19 #include <linux/backing-dev.h> 20 #include <asm/atomic.h> 21 #include <linux/scatterlist.h> 22 #include <asm/page.h> 23 #include <asm/unaligned.h> 24 25 #include "dm.h" 26 27 #define DM_MSG_PREFIX "crypt" 28 #define MESG_STR(x) x, sizeof(x) 29 30 /* 31 * per bio private data 32 */ 33 struct dm_crypt_io { 34 struct dm_target *target; 35 struct bio *base_bio; 36 struct work_struct work; 37 atomic_t pending; 38 int error; 39 int post_process; 40 }; 41 42 /* 43 * context holding the current state of a multi-part conversion 44 */ 45 struct convert_context { 46 struct bio *bio_in; 47 struct bio *bio_out; 48 unsigned int offset_in; 49 unsigned int offset_out; 50 unsigned int idx_in; 51 unsigned int idx_out; 52 sector_t sector; 53 int write; 54 }; 55 56 struct crypt_config; 57 58 struct crypt_iv_operations { 59 int (*ctr)(struct crypt_config *cc, struct dm_target *ti, 60 const char *opts); 61 void (*dtr)(struct crypt_config *cc); 62 const char *(*status)(struct crypt_config *cc); 63 int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector); 64 }; 65 66 /* 67 * Crypt: maps a linear range of a block device 68 * and encrypts / decrypts at the same time. 69 */ 70 enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; 71 struct crypt_config { 72 struct dm_dev *dev; 73 sector_t start; 74 75 /* 76 * pool for per bio private data and 77 * for encryption buffer pages 78 */ 79 mempool_t *io_pool; 80 mempool_t *page_pool; 81 struct bio_set *bs; 82 83 /* 84 * crypto related data 85 */ 86 struct crypt_iv_operations *iv_gen_ops; 87 char *iv_mode; 88 union { 89 struct crypto_cipher *essiv_tfm; 90 int benbi_shift; 91 } iv_gen_private; 92 sector_t iv_offset; 93 unsigned int iv_size; 94 95 char cipher[CRYPTO_MAX_ALG_NAME]; 96 char chainmode[CRYPTO_MAX_ALG_NAME]; 97 struct crypto_blkcipher *tfm; 98 unsigned long flags; 99 unsigned int key_size; 100 u8 key[0]; 101 }; 102 103 #define MIN_IOS 16 104 #define MIN_POOL_PAGES 32 105 #define MIN_BIO_PAGES 8 106 107 static struct kmem_cache *_crypt_io_pool; 108 109 static void clone_init(struct dm_crypt_io *, struct bio *); 110 111 /* 112 * Different IV generation algorithms: 113 * 114 * plain: the initial vector is the 32-bit little-endian version of the sector 115 * number, padded with zeros if neccessary. 116 * 117 * essiv: "encrypted sector|salt initial vector", the sector number is 118 * encrypted with the bulk cipher using a salt as key. The salt 119 * should be derived from the bulk cipher's key via hashing. 120 * 121 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1 122 * (needed for LRW-32-AES and possible other narrow block modes) 123 * 124 * null: the initial vector is always zero. Provides compatibility with 125 * obsolete loop_fish2 devices. Do not use for new devices. 126 * 127 * plumb: unimplemented, see: 128 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454 129 */ 130 131 static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 132 { 133 memset(iv, 0, cc->iv_size); 134 *(u32 *)iv = cpu_to_le32(sector & 0xffffffff); 135 136 return 0; 137 } 138 139 static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, 140 const char *opts) 141 { 142 struct crypto_cipher *essiv_tfm; 143 struct crypto_hash *hash_tfm; 144 struct hash_desc desc; 145 struct scatterlist sg; 146 unsigned int saltsize; 147 u8 *salt; 148 int err; 149 150 if (opts == NULL) { 151 ti->error = "Digest algorithm missing for ESSIV mode"; 152 return -EINVAL; 153 } 154 155 /* Hash the cipher key with the given hash algorithm */ 156 hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC); 157 if (IS_ERR(hash_tfm)) { 158 ti->error = "Error initializing ESSIV hash"; 159 return PTR_ERR(hash_tfm); 160 } 161 162 saltsize = crypto_hash_digestsize(hash_tfm); 163 salt = kmalloc(saltsize, GFP_KERNEL); 164 if (salt == NULL) { 165 ti->error = "Error kmallocing salt storage in ESSIV"; 166 crypto_free_hash(hash_tfm); 167 return -ENOMEM; 168 } 169 170 sg_set_buf(&sg, cc->key, cc->key_size); 171 desc.tfm = hash_tfm; 172 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 173 err = crypto_hash_digest(&desc, &sg, cc->key_size, salt); 174 crypto_free_hash(hash_tfm); 175 176 if (err) { 177 ti->error = "Error calculating hash in ESSIV"; 178 return err; 179 } 180 181 /* Setup the essiv_tfm with the given salt */ 182 essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 183 if (IS_ERR(essiv_tfm)) { 184 ti->error = "Error allocating crypto tfm for ESSIV"; 185 kfree(salt); 186 return PTR_ERR(essiv_tfm); 187 } 188 if (crypto_cipher_blocksize(essiv_tfm) != 189 crypto_blkcipher_ivsize(cc->tfm)) { 190 ti->error = "Block size of ESSIV cipher does " 191 "not match IV size of block cipher"; 192 crypto_free_cipher(essiv_tfm); 193 kfree(salt); 194 return -EINVAL; 195 } 196 err = crypto_cipher_setkey(essiv_tfm, salt, saltsize); 197 if (err) { 198 ti->error = "Failed to set key for ESSIV cipher"; 199 crypto_free_cipher(essiv_tfm); 200 kfree(salt); 201 return err; 202 } 203 kfree(salt); 204 205 cc->iv_gen_private.essiv_tfm = essiv_tfm; 206 return 0; 207 } 208 209 static void crypt_iv_essiv_dtr(struct crypt_config *cc) 210 { 211 crypto_free_cipher(cc->iv_gen_private.essiv_tfm); 212 cc->iv_gen_private.essiv_tfm = NULL; 213 } 214 215 static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 216 { 217 memset(iv, 0, cc->iv_size); 218 *(u64 *)iv = cpu_to_le64(sector); 219 crypto_cipher_encrypt_one(cc->iv_gen_private.essiv_tfm, iv, iv); 220 return 0; 221 } 222 223 static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti, 224 const char *opts) 225 { 226 unsigned int bs = crypto_blkcipher_blocksize(cc->tfm); 227 int log = ilog2(bs); 228 229 /* we need to calculate how far we must shift the sector count 230 * to get the cipher block count, we use this shift in _gen */ 231 232 if (1 << log != bs) { 233 ti->error = "cypher blocksize is not a power of 2"; 234 return -EINVAL; 235 } 236 237 if (log > 9) { 238 ti->error = "cypher blocksize is > 512"; 239 return -EINVAL; 240 } 241 242 cc->iv_gen_private.benbi_shift = 9 - log; 243 244 return 0; 245 } 246 247 static void crypt_iv_benbi_dtr(struct crypt_config *cc) 248 { 249 } 250 251 static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 252 { 253 __be64 val; 254 255 memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */ 256 257 val = cpu_to_be64(((u64)sector << cc->iv_gen_private.benbi_shift) + 1); 258 put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64))); 259 260 return 0; 261 } 262 263 static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, sector_t sector) 264 { 265 memset(iv, 0, cc->iv_size); 266 267 return 0; 268 } 269 270 static struct crypt_iv_operations crypt_iv_plain_ops = { 271 .generator = crypt_iv_plain_gen 272 }; 273 274 static struct crypt_iv_operations crypt_iv_essiv_ops = { 275 .ctr = crypt_iv_essiv_ctr, 276 .dtr = crypt_iv_essiv_dtr, 277 .generator = crypt_iv_essiv_gen 278 }; 279 280 static struct crypt_iv_operations crypt_iv_benbi_ops = { 281 .ctr = crypt_iv_benbi_ctr, 282 .dtr = crypt_iv_benbi_dtr, 283 .generator = crypt_iv_benbi_gen 284 }; 285 286 static struct crypt_iv_operations crypt_iv_null_ops = { 287 .generator = crypt_iv_null_gen 288 }; 289 290 static int 291 crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out, 292 struct scatterlist *in, unsigned int length, 293 int write, sector_t sector) 294 { 295 u8 iv[cc->iv_size] __attribute__ ((aligned(__alignof__(u64)))); 296 struct blkcipher_desc desc = { 297 .tfm = cc->tfm, 298 .info = iv, 299 .flags = CRYPTO_TFM_REQ_MAY_SLEEP, 300 }; 301 int r; 302 303 if (cc->iv_gen_ops) { 304 r = cc->iv_gen_ops->generator(cc, iv, sector); 305 if (r < 0) 306 return r; 307 308 if (write) 309 r = crypto_blkcipher_encrypt_iv(&desc, out, in, length); 310 else 311 r = crypto_blkcipher_decrypt_iv(&desc, out, in, length); 312 } else { 313 if (write) 314 r = crypto_blkcipher_encrypt(&desc, out, in, length); 315 else 316 r = crypto_blkcipher_decrypt(&desc, out, in, length); 317 } 318 319 return r; 320 } 321 322 static void 323 crypt_convert_init(struct crypt_config *cc, struct convert_context *ctx, 324 struct bio *bio_out, struct bio *bio_in, 325 sector_t sector, int write) 326 { 327 ctx->bio_in = bio_in; 328 ctx->bio_out = bio_out; 329 ctx->offset_in = 0; 330 ctx->offset_out = 0; 331 ctx->idx_in = bio_in ? bio_in->bi_idx : 0; 332 ctx->idx_out = bio_out ? bio_out->bi_idx : 0; 333 ctx->sector = sector + cc->iv_offset; 334 ctx->write = write; 335 } 336 337 /* 338 * Encrypt / decrypt data from one bio to another one (can be the same one) 339 */ 340 static int crypt_convert(struct crypt_config *cc, 341 struct convert_context *ctx) 342 { 343 int r = 0; 344 345 while(ctx->idx_in < ctx->bio_in->bi_vcnt && 346 ctx->idx_out < ctx->bio_out->bi_vcnt) { 347 struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in); 348 struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out); 349 struct scatterlist sg_in = { 350 .page = bv_in->bv_page, 351 .offset = bv_in->bv_offset + ctx->offset_in, 352 .length = 1 << SECTOR_SHIFT 353 }; 354 struct scatterlist sg_out = { 355 .page = bv_out->bv_page, 356 .offset = bv_out->bv_offset + ctx->offset_out, 357 .length = 1 << SECTOR_SHIFT 358 }; 359 360 ctx->offset_in += sg_in.length; 361 if (ctx->offset_in >= bv_in->bv_len) { 362 ctx->offset_in = 0; 363 ctx->idx_in++; 364 } 365 366 ctx->offset_out += sg_out.length; 367 if (ctx->offset_out >= bv_out->bv_len) { 368 ctx->offset_out = 0; 369 ctx->idx_out++; 370 } 371 372 r = crypt_convert_scatterlist(cc, &sg_out, &sg_in, sg_in.length, 373 ctx->write, ctx->sector); 374 if (r < 0) 375 break; 376 377 ctx->sector++; 378 } 379 380 return r; 381 } 382 383 static void dm_crypt_bio_destructor(struct bio *bio) 384 { 385 struct dm_crypt_io *io = bio->bi_private; 386 struct crypt_config *cc = io->target->private; 387 388 bio_free(bio, cc->bs); 389 } 390 391 /* 392 * Generate a new unfragmented bio with the given size 393 * This should never violate the device limitations 394 * May return a smaller bio when running out of pages 395 */ 396 static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size) 397 { 398 struct crypt_config *cc = io->target->private; 399 struct bio *clone; 400 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 401 gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; 402 unsigned int i; 403 404 clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 405 if (!clone) 406 return NULL; 407 408 clone_init(io, clone); 409 410 for (i = 0; i < nr_iovecs; i++) { 411 struct bio_vec *bv = bio_iovec_idx(clone, i); 412 413 bv->bv_page = mempool_alloc(cc->page_pool, gfp_mask); 414 if (!bv->bv_page) 415 break; 416 417 /* 418 * if additional pages cannot be allocated without waiting, 419 * return a partially allocated bio, the caller will then try 420 * to allocate additional bios while submitting this partial bio 421 */ 422 if (i == (MIN_BIO_PAGES - 1)) 423 gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; 424 425 bv->bv_offset = 0; 426 if (size > PAGE_SIZE) 427 bv->bv_len = PAGE_SIZE; 428 else 429 bv->bv_len = size; 430 431 clone->bi_size += bv->bv_len; 432 clone->bi_vcnt++; 433 size -= bv->bv_len; 434 } 435 436 if (!clone->bi_size) { 437 bio_put(clone); 438 return NULL; 439 } 440 441 return clone; 442 } 443 444 static void crypt_free_buffer_pages(struct crypt_config *cc, 445 struct bio *clone, unsigned int bytes) 446 { 447 unsigned int i, start, end; 448 struct bio_vec *bv; 449 450 /* 451 * This is ugly, but Jens Axboe thinks that using bi_idx in the 452 * endio function is too dangerous at the moment, so I calculate the 453 * correct position using bi_vcnt and bi_size. 454 * The bv_offset and bv_len fields might already be modified but we 455 * know that we always allocated whole pages. 456 * A fix to the bi_idx issue in the kernel is in the works, so 457 * we will hopefully be able to revert to the cleaner solution soon. 458 */ 459 i = clone->bi_vcnt - 1; 460 bv = bio_iovec_idx(clone, i); 461 end = (i << PAGE_SHIFT) + (bv->bv_offset + bv->bv_len) - clone->bi_size; 462 start = end - bytes; 463 464 start >>= PAGE_SHIFT; 465 if (!clone->bi_size) 466 end = clone->bi_vcnt; 467 else 468 end >>= PAGE_SHIFT; 469 470 for (i = start; i < end; i++) { 471 bv = bio_iovec_idx(clone, i); 472 BUG_ON(!bv->bv_page); 473 mempool_free(bv->bv_page, cc->page_pool); 474 bv->bv_page = NULL; 475 } 476 } 477 478 /* 479 * One of the bios was finished. Check for completion of 480 * the whole request and correctly clean up the buffer. 481 */ 482 static void dec_pending(struct dm_crypt_io *io, int error) 483 { 484 struct crypt_config *cc = (struct crypt_config *) io->target->private; 485 486 if (error < 0) 487 io->error = error; 488 489 if (!atomic_dec_and_test(&io->pending)) 490 return; 491 492 bio_endio(io->base_bio, io->error); 493 494 mempool_free(io, cc->io_pool); 495 } 496 497 /* 498 * kcryptd: 499 * 500 * Needed because it would be very unwise to do decryption in an 501 * interrupt context. 502 */ 503 static struct workqueue_struct *_kcryptd_workqueue; 504 static void kcryptd_do_work(struct work_struct *work); 505 506 static void kcryptd_queue_io(struct dm_crypt_io *io) 507 { 508 INIT_WORK(&io->work, kcryptd_do_work); 509 queue_work(_kcryptd_workqueue, &io->work); 510 } 511 512 static void crypt_endio(struct bio *clone, int error) 513 { 514 struct dm_crypt_io *io = clone->bi_private; 515 struct crypt_config *cc = io->target->private; 516 unsigned read_io = bio_data_dir(clone) == READ; 517 518 /* 519 * free the processed pages 520 */ 521 if (!read_io) { 522 crypt_free_buffer_pages(cc, clone, clone->bi_size); 523 goto out; 524 } 525 526 if (unlikely(!bio_flagged(clone, BIO_UPTODATE))) { 527 error = -EIO; 528 goto out; 529 } 530 531 bio_put(clone); 532 io->post_process = 1; 533 kcryptd_queue_io(io); 534 return; 535 536 out: 537 bio_put(clone); 538 dec_pending(io, error); 539 } 540 541 static void clone_init(struct dm_crypt_io *io, struct bio *clone) 542 { 543 struct crypt_config *cc = io->target->private; 544 545 clone->bi_private = io; 546 clone->bi_end_io = crypt_endio; 547 clone->bi_bdev = cc->dev->bdev; 548 clone->bi_rw = io->base_bio->bi_rw; 549 clone->bi_destructor = dm_crypt_bio_destructor; 550 } 551 552 static void process_read(struct dm_crypt_io *io) 553 { 554 struct crypt_config *cc = io->target->private; 555 struct bio *base_bio = io->base_bio; 556 struct bio *clone; 557 sector_t sector = base_bio->bi_sector - io->target->begin; 558 559 atomic_inc(&io->pending); 560 561 /* 562 * The block layer might modify the bvec array, so always 563 * copy the required bvecs because we need the original 564 * one in order to decrypt the whole bio data *afterwards*. 565 */ 566 clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs); 567 if (unlikely(!clone)) { 568 dec_pending(io, -ENOMEM); 569 return; 570 } 571 572 clone_init(io, clone); 573 clone->bi_idx = 0; 574 clone->bi_vcnt = bio_segments(base_bio); 575 clone->bi_size = base_bio->bi_size; 576 clone->bi_sector = cc->start + sector; 577 memcpy(clone->bi_io_vec, bio_iovec(base_bio), 578 sizeof(struct bio_vec) * clone->bi_vcnt); 579 580 generic_make_request(clone); 581 } 582 583 static void process_write(struct dm_crypt_io *io) 584 { 585 struct crypt_config *cc = io->target->private; 586 struct bio *base_bio = io->base_bio; 587 struct bio *clone; 588 struct convert_context ctx; 589 unsigned remaining = base_bio->bi_size; 590 sector_t sector = base_bio->bi_sector - io->target->begin; 591 592 atomic_inc(&io->pending); 593 594 crypt_convert_init(cc, &ctx, NULL, base_bio, sector, 1); 595 596 /* 597 * The allocated buffers can be smaller than the whole bio, 598 * so repeat the whole process until all the data can be handled. 599 */ 600 while (remaining) { 601 clone = crypt_alloc_buffer(io, remaining); 602 if (unlikely(!clone)) { 603 dec_pending(io, -ENOMEM); 604 return; 605 } 606 607 ctx.bio_out = clone; 608 ctx.idx_out = 0; 609 610 if (unlikely(crypt_convert(cc, &ctx) < 0)) { 611 crypt_free_buffer_pages(cc, clone, clone->bi_size); 612 bio_put(clone); 613 dec_pending(io, -EIO); 614 return; 615 } 616 617 /* crypt_convert should have filled the clone bio */ 618 BUG_ON(ctx.idx_out < clone->bi_vcnt); 619 620 clone->bi_sector = cc->start + sector; 621 remaining -= clone->bi_size; 622 sector += bio_sectors(clone); 623 624 /* Grab another reference to the io struct 625 * before we kick off the request */ 626 if (remaining) 627 atomic_inc(&io->pending); 628 629 generic_make_request(clone); 630 631 /* Do not reference clone after this - it 632 * may be gone already. */ 633 634 /* out of memory -> run queues */ 635 if (remaining) 636 congestion_wait(WRITE, HZ/100); 637 } 638 } 639 640 static void process_read_endio(struct dm_crypt_io *io) 641 { 642 struct crypt_config *cc = io->target->private; 643 struct convert_context ctx; 644 645 crypt_convert_init(cc, &ctx, io->base_bio, io->base_bio, 646 io->base_bio->bi_sector - io->target->begin, 0); 647 648 dec_pending(io, crypt_convert(cc, &ctx)); 649 } 650 651 static void kcryptd_do_work(struct work_struct *work) 652 { 653 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 654 655 if (io->post_process) 656 process_read_endio(io); 657 else if (bio_data_dir(io->base_bio) == READ) 658 process_read(io); 659 else 660 process_write(io); 661 } 662 663 /* 664 * Decode key from its hex representation 665 */ 666 static int crypt_decode_key(u8 *key, char *hex, unsigned int size) 667 { 668 char buffer[3]; 669 char *endp; 670 unsigned int i; 671 672 buffer[2] = '\0'; 673 674 for (i = 0; i < size; i++) { 675 buffer[0] = *hex++; 676 buffer[1] = *hex++; 677 678 key[i] = (u8)simple_strtoul(buffer, &endp, 16); 679 680 if (endp != &buffer[2]) 681 return -EINVAL; 682 } 683 684 if (*hex != '\0') 685 return -EINVAL; 686 687 return 0; 688 } 689 690 /* 691 * Encode key into its hex representation 692 */ 693 static void crypt_encode_key(char *hex, u8 *key, unsigned int size) 694 { 695 unsigned int i; 696 697 for (i = 0; i < size; i++) { 698 sprintf(hex, "%02x", *key); 699 hex += 2; 700 key++; 701 } 702 } 703 704 static int crypt_set_key(struct crypt_config *cc, char *key) 705 { 706 unsigned key_size = strlen(key) >> 1; 707 708 if (cc->key_size && cc->key_size != key_size) 709 return -EINVAL; 710 711 cc->key_size = key_size; /* initial settings */ 712 713 if ((!key_size && strcmp(key, "-")) || 714 (key_size && crypt_decode_key(cc->key, key, key_size) < 0)) 715 return -EINVAL; 716 717 set_bit(DM_CRYPT_KEY_VALID, &cc->flags); 718 719 return 0; 720 } 721 722 static int crypt_wipe_key(struct crypt_config *cc) 723 { 724 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); 725 memset(&cc->key, 0, cc->key_size * sizeof(u8)); 726 return 0; 727 } 728 729 /* 730 * Construct an encryption mapping: 731 * <cipher> <key> <iv_offset> <dev_path> <start> 732 */ 733 static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) 734 { 735 struct crypt_config *cc; 736 struct crypto_blkcipher *tfm; 737 char *tmp; 738 char *cipher; 739 char *chainmode; 740 char *ivmode; 741 char *ivopts; 742 unsigned int key_size; 743 unsigned long long tmpll; 744 745 if (argc != 5) { 746 ti->error = "Not enough arguments"; 747 return -EINVAL; 748 } 749 750 tmp = argv[0]; 751 cipher = strsep(&tmp, "-"); 752 chainmode = strsep(&tmp, "-"); 753 ivopts = strsep(&tmp, "-"); 754 ivmode = strsep(&ivopts, ":"); 755 756 if (tmp) 757 DMWARN("Unexpected additional cipher options"); 758 759 key_size = strlen(argv[1]) >> 1; 760 761 cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL); 762 if (cc == NULL) { 763 ti->error = 764 "Cannot allocate transparent encryption context"; 765 return -ENOMEM; 766 } 767 768 if (crypt_set_key(cc, argv[1])) { 769 ti->error = "Error decoding key"; 770 goto bad1; 771 } 772 773 /* Compatiblity mode for old dm-crypt cipher strings */ 774 if (!chainmode || (strcmp(chainmode, "plain") == 0 && !ivmode)) { 775 chainmode = "cbc"; 776 ivmode = "plain"; 777 } 778 779 if (strcmp(chainmode, "ecb") && !ivmode) { 780 ti->error = "This chaining mode requires an IV mechanism"; 781 goto bad1; 782 } 783 784 if (snprintf(cc->cipher, CRYPTO_MAX_ALG_NAME, "%s(%s)", chainmode, 785 cipher) >= CRYPTO_MAX_ALG_NAME) { 786 ti->error = "Chain mode + cipher name is too long"; 787 goto bad1; 788 } 789 790 tfm = crypto_alloc_blkcipher(cc->cipher, 0, CRYPTO_ALG_ASYNC); 791 if (IS_ERR(tfm)) { 792 ti->error = "Error allocating crypto tfm"; 793 goto bad1; 794 } 795 796 strcpy(cc->cipher, cipher); 797 strcpy(cc->chainmode, chainmode); 798 cc->tfm = tfm; 799 800 /* 801 * Choose ivmode. Valid modes: "plain", "essiv:<esshash>", "benbi". 802 * See comments at iv code 803 */ 804 805 if (ivmode == NULL) 806 cc->iv_gen_ops = NULL; 807 else if (strcmp(ivmode, "plain") == 0) 808 cc->iv_gen_ops = &crypt_iv_plain_ops; 809 else if (strcmp(ivmode, "essiv") == 0) 810 cc->iv_gen_ops = &crypt_iv_essiv_ops; 811 else if (strcmp(ivmode, "benbi") == 0) 812 cc->iv_gen_ops = &crypt_iv_benbi_ops; 813 else if (strcmp(ivmode, "null") == 0) 814 cc->iv_gen_ops = &crypt_iv_null_ops; 815 else { 816 ti->error = "Invalid IV mode"; 817 goto bad2; 818 } 819 820 if (cc->iv_gen_ops && cc->iv_gen_ops->ctr && 821 cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0) 822 goto bad2; 823 824 cc->iv_size = crypto_blkcipher_ivsize(tfm); 825 if (cc->iv_size) 826 /* at least a 64 bit sector number should fit in our buffer */ 827 cc->iv_size = max(cc->iv_size, 828 (unsigned int)(sizeof(u64) / sizeof(u8))); 829 else { 830 if (cc->iv_gen_ops) { 831 DMWARN("Selected cipher does not support IVs"); 832 if (cc->iv_gen_ops->dtr) 833 cc->iv_gen_ops->dtr(cc); 834 cc->iv_gen_ops = NULL; 835 } 836 } 837 838 cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); 839 if (!cc->io_pool) { 840 ti->error = "Cannot allocate crypt io mempool"; 841 goto bad3; 842 } 843 844 cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); 845 if (!cc->page_pool) { 846 ti->error = "Cannot allocate page mempool"; 847 goto bad4; 848 } 849 850 cc->bs = bioset_create(MIN_IOS, MIN_IOS); 851 if (!cc->bs) { 852 ti->error = "Cannot allocate crypt bioset"; 853 goto bad_bs; 854 } 855 856 if (crypto_blkcipher_setkey(tfm, cc->key, key_size) < 0) { 857 ti->error = "Error setting key"; 858 goto bad5; 859 } 860 861 if (sscanf(argv[2], "%llu", &tmpll) != 1) { 862 ti->error = "Invalid iv_offset sector"; 863 goto bad5; 864 } 865 cc->iv_offset = tmpll; 866 867 if (sscanf(argv[4], "%llu", &tmpll) != 1) { 868 ti->error = "Invalid device sector"; 869 goto bad5; 870 } 871 cc->start = tmpll; 872 873 if (dm_get_device(ti, argv[3], cc->start, ti->len, 874 dm_table_get_mode(ti->table), &cc->dev)) { 875 ti->error = "Device lookup failed"; 876 goto bad5; 877 } 878 879 if (ivmode && cc->iv_gen_ops) { 880 if (ivopts) 881 *(ivopts - 1) = ':'; 882 cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL); 883 if (!cc->iv_mode) { 884 ti->error = "Error kmallocing iv_mode string"; 885 goto bad5; 886 } 887 strcpy(cc->iv_mode, ivmode); 888 } else 889 cc->iv_mode = NULL; 890 891 ti->private = cc; 892 return 0; 893 894 bad5: 895 bioset_free(cc->bs); 896 bad_bs: 897 mempool_destroy(cc->page_pool); 898 bad4: 899 mempool_destroy(cc->io_pool); 900 bad3: 901 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 902 cc->iv_gen_ops->dtr(cc); 903 bad2: 904 crypto_free_blkcipher(tfm); 905 bad1: 906 /* Must zero key material before freeing */ 907 memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8)); 908 kfree(cc); 909 return -EINVAL; 910 } 911 912 static void crypt_dtr(struct dm_target *ti) 913 { 914 struct crypt_config *cc = (struct crypt_config *) ti->private; 915 916 flush_workqueue(_kcryptd_workqueue); 917 918 bioset_free(cc->bs); 919 mempool_destroy(cc->page_pool); 920 mempool_destroy(cc->io_pool); 921 922 kfree(cc->iv_mode); 923 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 924 cc->iv_gen_ops->dtr(cc); 925 crypto_free_blkcipher(cc->tfm); 926 dm_put_device(ti, cc->dev); 927 928 /* Must zero key material before freeing */ 929 memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8)); 930 kfree(cc); 931 } 932 933 static int crypt_map(struct dm_target *ti, struct bio *bio, 934 union map_info *map_context) 935 { 936 struct crypt_config *cc = ti->private; 937 struct dm_crypt_io *io; 938 939 io = mempool_alloc(cc->io_pool, GFP_NOIO); 940 io->target = ti; 941 io->base_bio = bio; 942 io->error = io->post_process = 0; 943 atomic_set(&io->pending, 0); 944 kcryptd_queue_io(io); 945 946 return DM_MAPIO_SUBMITTED; 947 } 948 949 static int crypt_status(struct dm_target *ti, status_type_t type, 950 char *result, unsigned int maxlen) 951 { 952 struct crypt_config *cc = (struct crypt_config *) ti->private; 953 unsigned int sz = 0; 954 955 switch (type) { 956 case STATUSTYPE_INFO: 957 result[0] = '\0'; 958 break; 959 960 case STATUSTYPE_TABLE: 961 if (cc->iv_mode) 962 DMEMIT("%s-%s-%s ", cc->cipher, cc->chainmode, 963 cc->iv_mode); 964 else 965 DMEMIT("%s-%s ", cc->cipher, cc->chainmode); 966 967 if (cc->key_size > 0) { 968 if ((maxlen - sz) < ((cc->key_size << 1) + 1)) 969 return -ENOMEM; 970 971 crypt_encode_key(result + sz, cc->key, cc->key_size); 972 sz += cc->key_size << 1; 973 } else { 974 if (sz >= maxlen) 975 return -ENOMEM; 976 result[sz++] = '-'; 977 } 978 979 DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 980 cc->dev->name, (unsigned long long)cc->start); 981 break; 982 } 983 return 0; 984 } 985 986 static void crypt_postsuspend(struct dm_target *ti) 987 { 988 struct crypt_config *cc = ti->private; 989 990 set_bit(DM_CRYPT_SUSPENDED, &cc->flags); 991 } 992 993 static int crypt_preresume(struct dm_target *ti) 994 { 995 struct crypt_config *cc = ti->private; 996 997 if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) { 998 DMERR("aborting resume - crypt key is not set."); 999 return -EAGAIN; 1000 } 1001 1002 return 0; 1003 } 1004 1005 static void crypt_resume(struct dm_target *ti) 1006 { 1007 struct crypt_config *cc = ti->private; 1008 1009 clear_bit(DM_CRYPT_SUSPENDED, &cc->flags); 1010 } 1011 1012 /* Message interface 1013 * key set <key> 1014 * key wipe 1015 */ 1016 static int crypt_message(struct dm_target *ti, unsigned argc, char **argv) 1017 { 1018 struct crypt_config *cc = ti->private; 1019 1020 if (argc < 2) 1021 goto error; 1022 1023 if (!strnicmp(argv[0], MESG_STR("key"))) { 1024 if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) { 1025 DMWARN("not suspended during key manipulation."); 1026 return -EINVAL; 1027 } 1028 if (argc == 3 && !strnicmp(argv[1], MESG_STR("set"))) 1029 return crypt_set_key(cc, argv[2]); 1030 if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe"))) 1031 return crypt_wipe_key(cc); 1032 } 1033 1034 error: 1035 DMWARN("unrecognised message received."); 1036 return -EINVAL; 1037 } 1038 1039 static struct target_type crypt_target = { 1040 .name = "crypt", 1041 .version= {1, 5, 0}, 1042 .module = THIS_MODULE, 1043 .ctr = crypt_ctr, 1044 .dtr = crypt_dtr, 1045 .map = crypt_map, 1046 .status = crypt_status, 1047 .postsuspend = crypt_postsuspend, 1048 .preresume = crypt_preresume, 1049 .resume = crypt_resume, 1050 .message = crypt_message, 1051 }; 1052 1053 static int __init dm_crypt_init(void) 1054 { 1055 int r; 1056 1057 _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); 1058 if (!_crypt_io_pool) 1059 return -ENOMEM; 1060 1061 _kcryptd_workqueue = create_workqueue("kcryptd"); 1062 if (!_kcryptd_workqueue) { 1063 r = -ENOMEM; 1064 DMERR("couldn't create kcryptd"); 1065 goto bad1; 1066 } 1067 1068 r = dm_register_target(&crypt_target); 1069 if (r < 0) { 1070 DMERR("register failed %d", r); 1071 goto bad2; 1072 } 1073 1074 return 0; 1075 1076 bad2: 1077 destroy_workqueue(_kcryptd_workqueue); 1078 bad1: 1079 kmem_cache_destroy(_crypt_io_pool); 1080 return r; 1081 } 1082 1083 static void __exit dm_crypt_exit(void) 1084 { 1085 int r = dm_unregister_target(&crypt_target); 1086 1087 if (r < 0) 1088 DMERR("unregister failed %d", r); 1089 1090 destroy_workqueue(_kcryptd_workqueue); 1091 kmem_cache_destroy(_crypt_io_pool); 1092 } 1093 1094 module_init(dm_crypt_init); 1095 module_exit(dm_crypt_exit); 1096 1097 MODULE_AUTHOR("Christophe Saout <christophe@saout.de>"); 1098 MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption"); 1099 MODULE_LICENSE("GPL"); 1100