1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2003 Jana Saout <jana@saout.de>
4 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
5 * Copyright (C) 2006-2020 Red Hat, Inc. All rights reserved.
6 * Copyright (C) 2013-2020 Milan Broz <gmazyland@gmail.com>
7 *
8 * This file is released under the GPL.
9 */
10
11 #include <linux/completion.h>
12 #include <linux/err.h>
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/kernel.h>
16 #include <linux/key.h>
17 #include <linux/bio.h>
18 #include <linux/blkdev.h>
19 #include <linux/blk-integrity.h>
20 #include <linux/mempool.h>
21 #include <linux/slab.h>
22 #include <linux/crypto.h>
23 #include <linux/workqueue.h>
24 #include <linux/kthread.h>
25 #include <linux/backing-dev.h>
26 #include <linux/atomic.h>
27 #include <linux/scatterlist.h>
28 #include <linux/rbtree.h>
29 #include <linux/ctype.h>
30 #include <asm/page.h>
31 #include <asm/unaligned.h>
32 #include <crypto/hash.h>
33 #include <crypto/md5.h>
34 #include <crypto/skcipher.h>
35 #include <crypto/aead.h>
36 #include <crypto/authenc.h>
37 #include <crypto/utils.h>
38 #include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */
39 #include <linux/key-type.h>
40 #include <keys/user-type.h>
41 #include <keys/encrypted-type.h>
42 #include <keys/trusted-type.h>
43
44 #include <linux/device-mapper.h>
45
46 #include "dm-audit.h"
47
48 #define DM_MSG_PREFIX "crypt"
49
50 static DEFINE_IDA(workqueue_ida);
51
52 /*
53 * context holding the current state of a multi-part conversion
54 */
55 struct convert_context {
56 struct completion restart;
57 struct bio *bio_in;
58 struct bvec_iter iter_in;
59 struct bio *bio_out;
60 struct bvec_iter iter_out;
61 atomic_t cc_pending;
62 u64 cc_sector;
63 union {
64 struct skcipher_request *req;
65 struct aead_request *req_aead;
66 } r;
67 bool aead_recheck;
68 bool aead_failed;
69
70 };
71
72 /*
73 * per bio private data
74 */
75 struct dm_crypt_io {
76 struct crypt_config *cc;
77 struct bio *base_bio;
78 u8 *integrity_metadata;
79 bool integrity_metadata_from_pool:1;
80
81 struct work_struct work;
82
83 struct convert_context ctx;
84
85 atomic_t io_pending;
86 blk_status_t error;
87 sector_t sector;
88
89 struct bvec_iter saved_bi_iter;
90
91 struct rb_node rb_node;
92 } CRYPTO_MINALIGN_ATTR;
93
94 struct dm_crypt_request {
95 struct convert_context *ctx;
96 struct scatterlist sg_in[4];
97 struct scatterlist sg_out[4];
98 u64 iv_sector;
99 };
100
101 struct crypt_config;
102
103 struct crypt_iv_operations {
104 int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
105 const char *opts);
106 void (*dtr)(struct crypt_config *cc);
107 int (*init)(struct crypt_config *cc);
108 int (*wipe)(struct crypt_config *cc);
109 int (*generator)(struct crypt_config *cc, u8 *iv,
110 struct dm_crypt_request *dmreq);
111 int (*post)(struct crypt_config *cc, u8 *iv,
112 struct dm_crypt_request *dmreq);
113 };
114
115 struct iv_benbi_private {
116 int shift;
117 };
118
119 #define LMK_SEED_SIZE 64 /* hash + 0 */
120 struct iv_lmk_private {
121 struct crypto_shash *hash_tfm;
122 u8 *seed;
123 };
124
125 #define TCW_WHITENING_SIZE 16
126 struct iv_tcw_private {
127 struct crypto_shash *crc32_tfm;
128 u8 *iv_seed;
129 u8 *whitening;
130 };
131
132 #define ELEPHANT_MAX_KEY_SIZE 32
133 struct iv_elephant_private {
134 struct crypto_skcipher *tfm;
135 };
136
137 /*
138 * Crypt: maps a linear range of a block device
139 * and encrypts / decrypts at the same time.
140 */
141 enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
142 DM_CRYPT_SAME_CPU, DM_CRYPT_HIGH_PRIORITY,
143 DM_CRYPT_NO_OFFLOAD, DM_CRYPT_NO_READ_WORKQUEUE,
144 DM_CRYPT_NO_WRITE_WORKQUEUE, DM_CRYPT_WRITE_INLINE };
145
146 enum cipher_flags {
147 CRYPT_MODE_INTEGRITY_AEAD, /* Use authenticated mode for cipher */
148 CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */
149 CRYPT_ENCRYPT_PREPROCESS, /* Must preprocess data for encryption (elephant) */
150 };
151
152 /*
153 * The fields in here must be read only after initialization.
154 */
155 struct crypt_config {
156 struct dm_dev *dev;
157 sector_t start;
158
159 struct percpu_counter n_allocated_pages;
160
161 struct workqueue_struct *io_queue;
162 struct workqueue_struct *crypt_queue;
163
164 spinlock_t write_thread_lock;
165 struct task_struct *write_thread;
166 struct rb_root write_tree;
167
168 char *cipher_string;
169 char *cipher_auth;
170 char *key_string;
171
172 const struct crypt_iv_operations *iv_gen_ops;
173 union {
174 struct iv_benbi_private benbi;
175 struct iv_lmk_private lmk;
176 struct iv_tcw_private tcw;
177 struct iv_elephant_private elephant;
178 } iv_gen_private;
179 u64 iv_offset;
180 unsigned int iv_size;
181 unsigned short sector_size;
182 unsigned char sector_shift;
183
184 union {
185 struct crypto_skcipher **tfms;
186 struct crypto_aead **tfms_aead;
187 } cipher_tfm;
188 unsigned int tfms_count;
189 int workqueue_id;
190 unsigned long cipher_flags;
191
192 /*
193 * Layout of each crypto request:
194 *
195 * struct skcipher_request
196 * context
197 * padding
198 * struct dm_crypt_request
199 * padding
200 * IV
201 *
202 * The padding is added so that dm_crypt_request and the IV are
203 * correctly aligned.
204 */
205 unsigned int dmreq_start;
206
207 unsigned int per_bio_data_size;
208
209 unsigned long flags;
210 unsigned int key_size;
211 unsigned int key_parts; /* independent parts in key buffer */
212 unsigned int key_extra_size; /* additional keys length */
213 unsigned int key_mac_size; /* MAC key size for authenc(...) */
214
215 unsigned int integrity_tag_size;
216 unsigned int integrity_iv_size;
217 unsigned int used_tag_size;
218 unsigned int tuple_size;
219
220 /*
221 * pool for per bio private data, crypto requests,
222 * encryption requeusts/buffer pages and integrity tags
223 */
224 unsigned int tag_pool_max_sectors;
225 mempool_t tag_pool;
226 mempool_t req_pool;
227 mempool_t page_pool;
228
229 struct bio_set bs;
230 struct mutex bio_alloc_lock;
231
232 u8 *authenc_key; /* space for keys in authenc() format (if used) */
233 u8 key[] __counted_by(key_size);
234 };
235
236 #define MIN_IOS 64
237 #define MAX_TAG_SIZE 480
238 #define POOL_ENTRY_SIZE 512
239
240 static DEFINE_SPINLOCK(dm_crypt_clients_lock);
241 static unsigned int dm_crypt_clients_n;
242 static volatile unsigned long dm_crypt_pages_per_client;
243 #define DM_CRYPT_MEMORY_PERCENT 2
244 #define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_VECS * 16)
245 #define DM_CRYPT_DEFAULT_MAX_READ_SIZE 131072
246 #define DM_CRYPT_DEFAULT_MAX_WRITE_SIZE 131072
247
248 static unsigned int max_read_size = 0;
249 module_param(max_read_size, uint, 0644);
250 MODULE_PARM_DESC(max_read_size, "Maximum size of a read request");
251 static unsigned int max_write_size = 0;
252 module_param(max_write_size, uint, 0644);
253 MODULE_PARM_DESC(max_write_size, "Maximum size of a write request");
get_max_request_size(struct crypt_config * cc,bool wrt)254 static unsigned get_max_request_size(struct crypt_config *cc, bool wrt)
255 {
256 unsigned val, sector_align;
257 val = !wrt ? READ_ONCE(max_read_size) : READ_ONCE(max_write_size);
258 if (likely(!val))
259 val = !wrt ? DM_CRYPT_DEFAULT_MAX_READ_SIZE : DM_CRYPT_DEFAULT_MAX_WRITE_SIZE;
260 if (wrt || cc->used_tag_size) {
261 if (unlikely(val > BIO_MAX_VECS << PAGE_SHIFT))
262 val = BIO_MAX_VECS << PAGE_SHIFT;
263 }
264 sector_align = max(bdev_logical_block_size(cc->dev->bdev), (unsigned)cc->sector_size);
265 val = round_down(val, sector_align);
266 if (unlikely(!val))
267 val = sector_align;
268 return val >> SECTOR_SHIFT;
269 }
270
271 static void crypt_endio(struct bio *clone);
272 static void kcryptd_queue_crypt(struct dm_crypt_io *io);
273 static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
274 struct scatterlist *sg);
275
276 static bool crypt_integrity_aead(struct crypt_config *cc);
277
278 /*
279 * Use this to access cipher attributes that are independent of the key.
280 */
any_tfm(struct crypt_config * cc)281 static struct crypto_skcipher *any_tfm(struct crypt_config *cc)
282 {
283 return cc->cipher_tfm.tfms[0];
284 }
285
any_tfm_aead(struct crypt_config * cc)286 static struct crypto_aead *any_tfm_aead(struct crypt_config *cc)
287 {
288 return cc->cipher_tfm.tfms_aead[0];
289 }
290
291 /*
292 * Different IV generation algorithms:
293 *
294 * plain: the initial vector is the 32-bit little-endian version of the sector
295 * number, padded with zeros if necessary.
296 *
297 * plain64: the initial vector is the 64-bit little-endian version of the sector
298 * number, padded with zeros if necessary.
299 *
300 * plain64be: the initial vector is the 64-bit big-endian version of the sector
301 * number, padded with zeros if necessary.
302 *
303 * essiv: "encrypted sector|salt initial vector", the sector number is
304 * encrypted with the bulk cipher using a salt as key. The salt
305 * should be derived from the bulk cipher's key via hashing.
306 *
307 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
308 * (needed for LRW-32-AES and possible other narrow block modes)
309 *
310 * null: the initial vector is always zero. Provides compatibility with
311 * obsolete loop_fish2 devices. Do not use for new devices.
312 *
313 * lmk: Compatible implementation of the block chaining mode used
314 * by the Loop-AES block device encryption system
315 * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
316 * It operates on full 512 byte sectors and uses CBC
317 * with an IV derived from the sector number, the data and
318 * optionally extra IV seed.
319 * This means that after decryption the first block
320 * of sector must be tweaked according to decrypted data.
321 * Loop-AES can use three encryption schemes:
322 * version 1: is plain aes-cbc mode
323 * version 2: uses 64 multikey scheme with lmk IV generator
324 * version 3: the same as version 2 with additional IV seed
325 * (it uses 65 keys, last key is used as IV seed)
326 *
327 * tcw: Compatible implementation of the block chaining mode used
328 * by the TrueCrypt device encryption system (prior to version 4.1).
329 * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat
330 * It operates on full 512 byte sectors and uses CBC
331 * with an IV derived from initial key and the sector number.
332 * In addition, whitening value is applied on every sector, whitening
333 * is calculated from initial key, sector number and mixed using CRC32.
334 * Note that this encryption scheme is vulnerable to watermarking attacks
335 * and should be used for old compatible containers access only.
336 *
337 * eboiv: Encrypted byte-offset IV (used in Bitlocker in CBC mode)
338 * The IV is encrypted little-endian byte-offset (with the same key
339 * and cipher as the volume).
340 *
341 * elephant: The extended version of eboiv with additional Elephant diffuser
342 * used with Bitlocker CBC mode.
343 * This mode was used in older Windows systems
344 * https://download.microsoft.com/download/0/2/3/0238acaf-d3bf-4a6d-b3d6-0a0be4bbb36e/bitlockercipher200608.pdf
345 */
346
crypt_iv_plain_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)347 static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
348 struct dm_crypt_request *dmreq)
349 {
350 memset(iv, 0, cc->iv_size);
351 *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
352
353 return 0;
354 }
355
crypt_iv_plain64_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)356 static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
357 struct dm_crypt_request *dmreq)
358 {
359 memset(iv, 0, cc->iv_size);
360 *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
361
362 return 0;
363 }
364
crypt_iv_plain64be_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)365 static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv,
366 struct dm_crypt_request *dmreq)
367 {
368 memset(iv, 0, cc->iv_size);
369 /* iv_size is at least of size u64; usually it is 16 bytes */
370 *(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector);
371
372 return 0;
373 }
374
crypt_iv_essiv_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)375 static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
376 struct dm_crypt_request *dmreq)
377 {
378 /*
379 * ESSIV encryption of the IV is now handled by the crypto API,
380 * so just pass the plain sector number here.
381 */
382 memset(iv, 0, cc->iv_size);
383 *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
384
385 return 0;
386 }
387
crypt_iv_benbi_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)388 static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
389 const char *opts)
390 {
391 unsigned int bs;
392 int log;
393
394 if (crypt_integrity_aead(cc))
395 bs = crypto_aead_blocksize(any_tfm_aead(cc));
396 else
397 bs = crypto_skcipher_blocksize(any_tfm(cc));
398 log = ilog2(bs);
399
400 /*
401 * We need to calculate how far we must shift the sector count
402 * to get the cipher block count, we use this shift in _gen.
403 */
404 if (1 << log != bs) {
405 ti->error = "cypher blocksize is not a power of 2";
406 return -EINVAL;
407 }
408
409 if (log > 9) {
410 ti->error = "cypher blocksize is > 512";
411 return -EINVAL;
412 }
413
414 cc->iv_gen_private.benbi.shift = 9 - log;
415
416 return 0;
417 }
418
crypt_iv_benbi_dtr(struct crypt_config * cc)419 static void crypt_iv_benbi_dtr(struct crypt_config *cc)
420 {
421 }
422
crypt_iv_benbi_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)423 static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
424 struct dm_crypt_request *dmreq)
425 {
426 __be64 val;
427
428 memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
429
430 val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
431 put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
432
433 return 0;
434 }
435
crypt_iv_null_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)436 static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
437 struct dm_crypt_request *dmreq)
438 {
439 memset(iv, 0, cc->iv_size);
440
441 return 0;
442 }
443
crypt_iv_lmk_dtr(struct crypt_config * cc)444 static void crypt_iv_lmk_dtr(struct crypt_config *cc)
445 {
446 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
447
448 if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm))
449 crypto_free_shash(lmk->hash_tfm);
450 lmk->hash_tfm = NULL;
451
452 kfree_sensitive(lmk->seed);
453 lmk->seed = NULL;
454 }
455
crypt_iv_lmk_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)456 static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
457 const char *opts)
458 {
459 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
460
461 if (cc->sector_size != (1 << SECTOR_SHIFT)) {
462 ti->error = "Unsupported sector size for LMK";
463 return -EINVAL;
464 }
465
466 lmk->hash_tfm = crypto_alloc_shash("md5", 0,
467 CRYPTO_ALG_ALLOCATES_MEMORY);
468 if (IS_ERR(lmk->hash_tfm)) {
469 ti->error = "Error initializing LMK hash";
470 return PTR_ERR(lmk->hash_tfm);
471 }
472
473 /* No seed in LMK version 2 */
474 if (cc->key_parts == cc->tfms_count) {
475 lmk->seed = NULL;
476 return 0;
477 }
478
479 lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
480 if (!lmk->seed) {
481 crypt_iv_lmk_dtr(cc);
482 ti->error = "Error kmallocing seed storage in LMK";
483 return -ENOMEM;
484 }
485
486 return 0;
487 }
488
crypt_iv_lmk_init(struct crypt_config * cc)489 static int crypt_iv_lmk_init(struct crypt_config *cc)
490 {
491 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
492 int subkey_size = cc->key_size / cc->key_parts;
493
494 /* LMK seed is on the position of LMK_KEYS + 1 key */
495 if (lmk->seed)
496 memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
497 crypto_shash_digestsize(lmk->hash_tfm));
498
499 return 0;
500 }
501
crypt_iv_lmk_wipe(struct crypt_config * cc)502 static int crypt_iv_lmk_wipe(struct crypt_config *cc)
503 {
504 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
505
506 if (lmk->seed)
507 memset(lmk->seed, 0, LMK_SEED_SIZE);
508
509 return 0;
510 }
511
crypt_iv_lmk_one(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq,u8 * data)512 static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
513 struct dm_crypt_request *dmreq,
514 u8 *data)
515 {
516 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
517 SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
518 struct md5_state md5state;
519 __le32 buf[4];
520 int i, r;
521
522 desc->tfm = lmk->hash_tfm;
523
524 r = crypto_shash_init(desc);
525 if (r)
526 return r;
527
528 if (lmk->seed) {
529 r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE);
530 if (r)
531 return r;
532 }
533
534 /* Sector is always 512B, block size 16, add data of blocks 1-31 */
535 r = crypto_shash_update(desc, data + 16, 16 * 31);
536 if (r)
537 return r;
538
539 /* Sector is cropped to 56 bits here */
540 buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
541 buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
542 buf[2] = cpu_to_le32(4024);
543 buf[3] = 0;
544 r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf));
545 if (r)
546 return r;
547
548 /* No MD5 padding here */
549 r = crypto_shash_export(desc, &md5state);
550 if (r)
551 return r;
552
553 for (i = 0; i < MD5_HASH_WORDS; i++)
554 __cpu_to_le32s(&md5state.hash[i]);
555 memcpy(iv, &md5state.hash, cc->iv_size);
556
557 return 0;
558 }
559
crypt_iv_lmk_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)560 static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
561 struct dm_crypt_request *dmreq)
562 {
563 struct scatterlist *sg;
564 u8 *src;
565 int r = 0;
566
567 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
568 sg = crypt_get_sg_data(cc, dmreq->sg_in);
569 src = kmap_local_page(sg_page(sg));
570 r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset);
571 kunmap_local(src);
572 } else
573 memset(iv, 0, cc->iv_size);
574
575 return r;
576 }
577
crypt_iv_lmk_post(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)578 static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
579 struct dm_crypt_request *dmreq)
580 {
581 struct scatterlist *sg;
582 u8 *dst;
583 int r;
584
585 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
586 return 0;
587
588 sg = crypt_get_sg_data(cc, dmreq->sg_out);
589 dst = kmap_local_page(sg_page(sg));
590 r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset);
591
592 /* Tweak the first block of plaintext sector */
593 if (!r)
594 crypto_xor(dst + sg->offset, iv, cc->iv_size);
595
596 kunmap_local(dst);
597 return r;
598 }
599
crypt_iv_tcw_dtr(struct crypt_config * cc)600 static void crypt_iv_tcw_dtr(struct crypt_config *cc)
601 {
602 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
603
604 kfree_sensitive(tcw->iv_seed);
605 tcw->iv_seed = NULL;
606 kfree_sensitive(tcw->whitening);
607 tcw->whitening = NULL;
608
609 if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm))
610 crypto_free_shash(tcw->crc32_tfm);
611 tcw->crc32_tfm = NULL;
612 }
613
crypt_iv_tcw_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)614 static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,
615 const char *opts)
616 {
617 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
618
619 if (cc->sector_size != (1 << SECTOR_SHIFT)) {
620 ti->error = "Unsupported sector size for TCW";
621 return -EINVAL;
622 }
623
624 if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {
625 ti->error = "Wrong key size for TCW";
626 return -EINVAL;
627 }
628
629 tcw->crc32_tfm = crypto_alloc_shash("crc32", 0,
630 CRYPTO_ALG_ALLOCATES_MEMORY);
631 if (IS_ERR(tcw->crc32_tfm)) {
632 ti->error = "Error initializing CRC32 in TCW";
633 return PTR_ERR(tcw->crc32_tfm);
634 }
635
636 tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL);
637 tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);
638 if (!tcw->iv_seed || !tcw->whitening) {
639 crypt_iv_tcw_dtr(cc);
640 ti->error = "Error allocating seed storage in TCW";
641 return -ENOMEM;
642 }
643
644 return 0;
645 }
646
crypt_iv_tcw_init(struct crypt_config * cc)647 static int crypt_iv_tcw_init(struct crypt_config *cc)
648 {
649 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
650 int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;
651
652 memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);
653 memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],
654 TCW_WHITENING_SIZE);
655
656 return 0;
657 }
658
crypt_iv_tcw_wipe(struct crypt_config * cc)659 static int crypt_iv_tcw_wipe(struct crypt_config *cc)
660 {
661 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
662
663 memset(tcw->iv_seed, 0, cc->iv_size);
664 memset(tcw->whitening, 0, TCW_WHITENING_SIZE);
665
666 return 0;
667 }
668
crypt_iv_tcw_whitening(struct crypt_config * cc,struct dm_crypt_request * dmreq,u8 * data)669 static int crypt_iv_tcw_whitening(struct crypt_config *cc,
670 struct dm_crypt_request *dmreq,
671 u8 *data)
672 {
673 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
674 __le64 sector = cpu_to_le64(dmreq->iv_sector);
675 u8 buf[TCW_WHITENING_SIZE];
676 SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);
677 int i, r;
678
679 /* xor whitening with sector number */
680 crypto_xor_cpy(buf, tcw->whitening, (u8 *)§or, 8);
681 crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)§or, 8);
682
683 /* calculate crc32 for every 32bit part and xor it */
684 desc->tfm = tcw->crc32_tfm;
685 for (i = 0; i < 4; i++) {
686 r = crypto_shash_digest(desc, &buf[i * 4], 4, &buf[i * 4]);
687 if (r)
688 goto out;
689 }
690 crypto_xor(&buf[0], &buf[12], 4);
691 crypto_xor(&buf[4], &buf[8], 4);
692
693 /* apply whitening (8 bytes) to whole sector */
694 for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
695 crypto_xor(data + i * 8, buf, 8);
696 out:
697 memzero_explicit(buf, sizeof(buf));
698 return r;
699 }
700
crypt_iv_tcw_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)701 static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
702 struct dm_crypt_request *dmreq)
703 {
704 struct scatterlist *sg;
705 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
706 __le64 sector = cpu_to_le64(dmreq->iv_sector);
707 u8 *src;
708 int r = 0;
709
710 /* Remove whitening from ciphertext */
711 if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
712 sg = crypt_get_sg_data(cc, dmreq->sg_in);
713 src = kmap_local_page(sg_page(sg));
714 r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset);
715 kunmap_local(src);
716 }
717
718 /* Calculate IV */
719 crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)§or, 8);
720 if (cc->iv_size > 8)
721 crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)§or,
722 cc->iv_size - 8);
723
724 return r;
725 }
726
crypt_iv_tcw_post(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)727 static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
728 struct dm_crypt_request *dmreq)
729 {
730 struct scatterlist *sg;
731 u8 *dst;
732 int r;
733
734 if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
735 return 0;
736
737 /* Apply whitening on ciphertext */
738 sg = crypt_get_sg_data(cc, dmreq->sg_out);
739 dst = kmap_local_page(sg_page(sg));
740 r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset);
741 kunmap_local(dst);
742
743 return r;
744 }
745
crypt_iv_random_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)746 static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv,
747 struct dm_crypt_request *dmreq)
748 {
749 /* Used only for writes, there must be an additional space to store IV */
750 get_random_bytes(iv, cc->iv_size);
751 return 0;
752 }
753
crypt_iv_eboiv_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)754 static int crypt_iv_eboiv_ctr(struct crypt_config *cc, struct dm_target *ti,
755 const char *opts)
756 {
757 if (crypt_integrity_aead(cc)) {
758 ti->error = "AEAD transforms not supported for EBOIV";
759 return -EINVAL;
760 }
761
762 if (crypto_skcipher_blocksize(any_tfm(cc)) != cc->iv_size) {
763 ti->error = "Block size of EBOIV cipher does not match IV size of block cipher";
764 return -EINVAL;
765 }
766
767 return 0;
768 }
769
crypt_iv_eboiv_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)770 static int crypt_iv_eboiv_gen(struct crypt_config *cc, u8 *iv,
771 struct dm_crypt_request *dmreq)
772 {
773 struct crypto_skcipher *tfm = any_tfm(cc);
774 struct skcipher_request *req;
775 struct scatterlist src, dst;
776 DECLARE_CRYPTO_WAIT(wait);
777 unsigned int reqsize;
778 int err;
779 u8 *buf;
780
781 reqsize = sizeof(*req) + crypto_skcipher_reqsize(tfm);
782 reqsize = ALIGN(reqsize, __alignof__(__le64));
783
784 req = kmalloc(reqsize + cc->iv_size, GFP_NOIO);
785 if (!req)
786 return -ENOMEM;
787
788 skcipher_request_set_tfm(req, tfm);
789
790 buf = (u8 *)req + reqsize;
791 memset(buf, 0, cc->iv_size);
792 *(__le64 *)buf = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
793
794 sg_init_one(&src, page_address(ZERO_PAGE(0)), cc->iv_size);
795 sg_init_one(&dst, iv, cc->iv_size);
796 skcipher_request_set_crypt(req, &src, &dst, cc->iv_size, buf);
797 skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
798 err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
799 kfree_sensitive(req);
800
801 return err;
802 }
803
crypt_iv_elephant_dtr(struct crypt_config * cc)804 static void crypt_iv_elephant_dtr(struct crypt_config *cc)
805 {
806 struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
807
808 crypto_free_skcipher(elephant->tfm);
809 elephant->tfm = NULL;
810 }
811
crypt_iv_elephant_ctr(struct crypt_config * cc,struct dm_target * ti,const char * opts)812 static int crypt_iv_elephant_ctr(struct crypt_config *cc, struct dm_target *ti,
813 const char *opts)
814 {
815 struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
816 int r;
817
818 elephant->tfm = crypto_alloc_skcipher("ecb(aes)", 0,
819 CRYPTO_ALG_ALLOCATES_MEMORY);
820 if (IS_ERR(elephant->tfm)) {
821 r = PTR_ERR(elephant->tfm);
822 elephant->tfm = NULL;
823 return r;
824 }
825
826 r = crypt_iv_eboiv_ctr(cc, ti, NULL);
827 if (r)
828 crypt_iv_elephant_dtr(cc);
829 return r;
830 }
831
diffuser_disk_to_cpu(u32 * d,size_t n)832 static void diffuser_disk_to_cpu(u32 *d, size_t n)
833 {
834 #ifndef __LITTLE_ENDIAN
835 int i;
836
837 for (i = 0; i < n; i++)
838 d[i] = le32_to_cpu((__le32)d[i]);
839 #endif
840 }
841
diffuser_cpu_to_disk(__le32 * d,size_t n)842 static void diffuser_cpu_to_disk(__le32 *d, size_t n)
843 {
844 #ifndef __LITTLE_ENDIAN
845 int i;
846
847 for (i = 0; i < n; i++)
848 d[i] = cpu_to_le32((u32)d[i]);
849 #endif
850 }
851
diffuser_a_decrypt(u32 * d,size_t n)852 static void diffuser_a_decrypt(u32 *d, size_t n)
853 {
854 int i, i1, i2, i3;
855
856 for (i = 0; i < 5; i++) {
857 i1 = 0;
858 i2 = n - 2;
859 i3 = n - 5;
860
861 while (i1 < (n - 1)) {
862 d[i1] += d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
863 i1++; i2++; i3++;
864
865 if (i3 >= n)
866 i3 -= n;
867
868 d[i1] += d[i2] ^ d[i3];
869 i1++; i2++; i3++;
870
871 if (i2 >= n)
872 i2 -= n;
873
874 d[i1] += d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
875 i1++; i2++; i3++;
876
877 d[i1] += d[i2] ^ d[i3];
878 i1++; i2++; i3++;
879 }
880 }
881 }
882
diffuser_a_encrypt(u32 * d,size_t n)883 static void diffuser_a_encrypt(u32 *d, size_t n)
884 {
885 int i, i1, i2, i3;
886
887 for (i = 0; i < 5; i++) {
888 i1 = n - 1;
889 i2 = n - 2 - 1;
890 i3 = n - 5 - 1;
891
892 while (i1 > 0) {
893 d[i1] -= d[i2] ^ d[i3];
894 i1--; i2--; i3--;
895
896 d[i1] -= d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
897 i1--; i2--; i3--;
898
899 if (i2 < 0)
900 i2 += n;
901
902 d[i1] -= d[i2] ^ d[i3];
903 i1--; i2--; i3--;
904
905 if (i3 < 0)
906 i3 += n;
907
908 d[i1] -= d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
909 i1--; i2--; i3--;
910 }
911 }
912 }
913
diffuser_b_decrypt(u32 * d,size_t n)914 static void diffuser_b_decrypt(u32 *d, size_t n)
915 {
916 int i, i1, i2, i3;
917
918 for (i = 0; i < 3; i++) {
919 i1 = 0;
920 i2 = 2;
921 i3 = 5;
922
923 while (i1 < (n - 1)) {
924 d[i1] += d[i2] ^ d[i3];
925 i1++; i2++; i3++;
926
927 d[i1] += d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
928 i1++; i2++; i3++;
929
930 if (i2 >= n)
931 i2 -= n;
932
933 d[i1] += d[i2] ^ d[i3];
934 i1++; i2++; i3++;
935
936 if (i3 >= n)
937 i3 -= n;
938
939 d[i1] += d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
940 i1++; i2++; i3++;
941 }
942 }
943 }
944
diffuser_b_encrypt(u32 * d,size_t n)945 static void diffuser_b_encrypt(u32 *d, size_t n)
946 {
947 int i, i1, i2, i3;
948
949 for (i = 0; i < 3; i++) {
950 i1 = n - 1;
951 i2 = 2 - 1;
952 i3 = 5 - 1;
953
954 while (i1 > 0) {
955 d[i1] -= d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
956 i1--; i2--; i3--;
957
958 if (i3 < 0)
959 i3 += n;
960
961 d[i1] -= d[i2] ^ d[i3];
962 i1--; i2--; i3--;
963
964 if (i2 < 0)
965 i2 += n;
966
967 d[i1] -= d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
968 i1--; i2--; i3--;
969
970 d[i1] -= d[i2] ^ d[i3];
971 i1--; i2--; i3--;
972 }
973 }
974 }
975
crypt_iv_elephant(struct crypt_config * cc,struct dm_crypt_request * dmreq)976 static int crypt_iv_elephant(struct crypt_config *cc, struct dm_crypt_request *dmreq)
977 {
978 struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
979 u8 *es, *ks, *data, *data2, *data_offset;
980 struct skcipher_request *req;
981 struct scatterlist *sg, *sg2, src, dst;
982 DECLARE_CRYPTO_WAIT(wait);
983 int i, r;
984
985 req = skcipher_request_alloc(elephant->tfm, GFP_NOIO);
986 es = kzalloc(16, GFP_NOIO); /* Key for AES */
987 ks = kzalloc(32, GFP_NOIO); /* Elephant sector key */
988
989 if (!req || !es || !ks) {
990 r = -ENOMEM;
991 goto out;
992 }
993
994 *(__le64 *)es = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
995
996 /* E(Ks, e(s)) */
997 sg_init_one(&src, es, 16);
998 sg_init_one(&dst, ks, 16);
999 skcipher_request_set_crypt(req, &src, &dst, 16, NULL);
1000 skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
1001 r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
1002 if (r)
1003 goto out;
1004
1005 /* E(Ks, e'(s)) */
1006 es[15] = 0x80;
1007 sg_init_one(&dst, &ks[16], 16);
1008 r = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
1009 if (r)
1010 goto out;
1011
1012 sg = crypt_get_sg_data(cc, dmreq->sg_out);
1013 data = kmap_local_page(sg_page(sg));
1014 data_offset = data + sg->offset;
1015
1016 /* Cannot modify original bio, copy to sg_out and apply Elephant to it */
1017 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
1018 sg2 = crypt_get_sg_data(cc, dmreq->sg_in);
1019 data2 = kmap_local_page(sg_page(sg2));
1020 memcpy(data_offset, data2 + sg2->offset, cc->sector_size);
1021 kunmap_local(data2);
1022 }
1023
1024 if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
1025 diffuser_disk_to_cpu((u32 *)data_offset, cc->sector_size / sizeof(u32));
1026 diffuser_b_decrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));
1027 diffuser_a_decrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));
1028 diffuser_cpu_to_disk((__le32 *)data_offset, cc->sector_size / sizeof(u32));
1029 }
1030
1031 for (i = 0; i < (cc->sector_size / 32); i++)
1032 crypto_xor(data_offset + i * 32, ks, 32);
1033
1034 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
1035 diffuser_disk_to_cpu((u32 *)data_offset, cc->sector_size / sizeof(u32));
1036 diffuser_a_encrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));
1037 diffuser_b_encrypt((u32 *)data_offset, cc->sector_size / sizeof(u32));
1038 diffuser_cpu_to_disk((__le32 *)data_offset, cc->sector_size / sizeof(u32));
1039 }
1040
1041 kunmap_local(data);
1042 out:
1043 kfree_sensitive(ks);
1044 kfree_sensitive(es);
1045 skcipher_request_free(req);
1046 return r;
1047 }
1048
crypt_iv_elephant_gen(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)1049 static int crypt_iv_elephant_gen(struct crypt_config *cc, u8 *iv,
1050 struct dm_crypt_request *dmreq)
1051 {
1052 int r;
1053
1054 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
1055 r = crypt_iv_elephant(cc, dmreq);
1056 if (r)
1057 return r;
1058 }
1059
1060 return crypt_iv_eboiv_gen(cc, iv, dmreq);
1061 }
1062
crypt_iv_elephant_post(struct crypt_config * cc,u8 * iv,struct dm_crypt_request * dmreq)1063 static int crypt_iv_elephant_post(struct crypt_config *cc, u8 *iv,
1064 struct dm_crypt_request *dmreq)
1065 {
1066 if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
1067 return crypt_iv_elephant(cc, dmreq);
1068
1069 return 0;
1070 }
1071
crypt_iv_elephant_init(struct crypt_config * cc)1072 static int crypt_iv_elephant_init(struct crypt_config *cc)
1073 {
1074 struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
1075 int key_offset = cc->key_size - cc->key_extra_size;
1076
1077 return crypto_skcipher_setkey(elephant->tfm, &cc->key[key_offset], cc->key_extra_size);
1078 }
1079
crypt_iv_elephant_wipe(struct crypt_config * cc)1080 static int crypt_iv_elephant_wipe(struct crypt_config *cc)
1081 {
1082 struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
1083 u8 key[ELEPHANT_MAX_KEY_SIZE];
1084
1085 memset(key, 0, cc->key_extra_size);
1086 return crypto_skcipher_setkey(elephant->tfm, key, cc->key_extra_size);
1087 }
1088
1089 static const struct crypt_iv_operations crypt_iv_plain_ops = {
1090 .generator = crypt_iv_plain_gen
1091 };
1092
1093 static const struct crypt_iv_operations crypt_iv_plain64_ops = {
1094 .generator = crypt_iv_plain64_gen
1095 };
1096
1097 static const struct crypt_iv_operations crypt_iv_plain64be_ops = {
1098 .generator = crypt_iv_plain64be_gen
1099 };
1100
1101 static const struct crypt_iv_operations crypt_iv_essiv_ops = {
1102 .generator = crypt_iv_essiv_gen
1103 };
1104
1105 static const struct crypt_iv_operations crypt_iv_benbi_ops = {
1106 .ctr = crypt_iv_benbi_ctr,
1107 .dtr = crypt_iv_benbi_dtr,
1108 .generator = crypt_iv_benbi_gen
1109 };
1110
1111 static const struct crypt_iv_operations crypt_iv_null_ops = {
1112 .generator = crypt_iv_null_gen
1113 };
1114
1115 static const struct crypt_iv_operations crypt_iv_lmk_ops = {
1116 .ctr = crypt_iv_lmk_ctr,
1117 .dtr = crypt_iv_lmk_dtr,
1118 .init = crypt_iv_lmk_init,
1119 .wipe = crypt_iv_lmk_wipe,
1120 .generator = crypt_iv_lmk_gen,
1121 .post = crypt_iv_lmk_post
1122 };
1123
1124 static const struct crypt_iv_operations crypt_iv_tcw_ops = {
1125 .ctr = crypt_iv_tcw_ctr,
1126 .dtr = crypt_iv_tcw_dtr,
1127 .init = crypt_iv_tcw_init,
1128 .wipe = crypt_iv_tcw_wipe,
1129 .generator = crypt_iv_tcw_gen,
1130 .post = crypt_iv_tcw_post
1131 };
1132
1133 static const struct crypt_iv_operations crypt_iv_random_ops = {
1134 .generator = crypt_iv_random_gen
1135 };
1136
1137 static const struct crypt_iv_operations crypt_iv_eboiv_ops = {
1138 .ctr = crypt_iv_eboiv_ctr,
1139 .generator = crypt_iv_eboiv_gen
1140 };
1141
1142 static const struct crypt_iv_operations crypt_iv_elephant_ops = {
1143 .ctr = crypt_iv_elephant_ctr,
1144 .dtr = crypt_iv_elephant_dtr,
1145 .init = crypt_iv_elephant_init,
1146 .wipe = crypt_iv_elephant_wipe,
1147 .generator = crypt_iv_elephant_gen,
1148 .post = crypt_iv_elephant_post
1149 };
1150
1151 /*
1152 * Integrity extensions
1153 */
crypt_integrity_aead(struct crypt_config * cc)1154 static bool crypt_integrity_aead(struct crypt_config *cc)
1155 {
1156 return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
1157 }
1158
crypt_integrity_hmac(struct crypt_config * cc)1159 static bool crypt_integrity_hmac(struct crypt_config *cc)
1160 {
1161 return crypt_integrity_aead(cc) && cc->key_mac_size;
1162 }
1163
1164 /* Get sg containing data */
crypt_get_sg_data(struct crypt_config * cc,struct scatterlist * sg)1165 static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
1166 struct scatterlist *sg)
1167 {
1168 if (unlikely(crypt_integrity_aead(cc)))
1169 return &sg[2];
1170
1171 return sg;
1172 }
1173
dm_crypt_integrity_io_alloc(struct dm_crypt_io * io,struct bio * bio)1174 static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)
1175 {
1176 struct bio_integrity_payload *bip;
1177 unsigned int tag_len;
1178 int ret;
1179
1180 if (!bio_sectors(bio) || !io->cc->tuple_size)
1181 return 0;
1182
1183 bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
1184 if (IS_ERR(bip))
1185 return PTR_ERR(bip);
1186
1187 tag_len = io->cc->tuple_size * (bio_sectors(bio) >> io->cc->sector_shift);
1188
1189 bip->bip_iter.bi_sector = io->cc->start + io->sector;
1190
1191 ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata),
1192 tag_len, offset_in_page(io->integrity_metadata));
1193 if (unlikely(ret != tag_len))
1194 return -ENOMEM;
1195
1196 return 0;
1197 }
1198
crypt_integrity_ctr(struct crypt_config * cc,struct dm_target * ti)1199 static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti)
1200 {
1201 #ifdef CONFIG_BLK_DEV_INTEGRITY
1202 struct blk_integrity *bi = blk_get_integrity(cc->dev->bdev->bd_disk);
1203 struct mapped_device *md = dm_table_get_md(ti->table);
1204
1205 /* We require an underlying device with non-PI metadata */
1206 if (!bi || bi->csum_type != BLK_INTEGRITY_CSUM_NONE) {
1207 ti->error = "Integrity profile not supported.";
1208 return -EINVAL;
1209 }
1210
1211 if (bi->tuple_size < cc->used_tag_size) {
1212 ti->error = "Integrity profile tag size mismatch.";
1213 return -EINVAL;
1214 }
1215 cc->tuple_size = bi->tuple_size;
1216 if (1 << bi->interval_exp != cc->sector_size) {
1217 ti->error = "Integrity profile sector size mismatch.";
1218 return -EINVAL;
1219 }
1220
1221 if (crypt_integrity_aead(cc)) {
1222 cc->integrity_tag_size = cc->used_tag_size - cc->integrity_iv_size;
1223 DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md),
1224 cc->integrity_tag_size, cc->integrity_iv_size);
1225
1226 if (crypto_aead_setauthsize(any_tfm_aead(cc), cc->integrity_tag_size)) {
1227 ti->error = "Integrity AEAD auth tag size is not supported.";
1228 return -EINVAL;
1229 }
1230 } else if (cc->integrity_iv_size)
1231 DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md),
1232 cc->integrity_iv_size);
1233
1234 if ((cc->integrity_tag_size + cc->integrity_iv_size) > cc->tuple_size) {
1235 ti->error = "Not enough space for integrity tag in the profile.";
1236 return -EINVAL;
1237 }
1238
1239 return 0;
1240 #else
1241 ti->error = "Integrity profile not supported.";
1242 return -EINVAL;
1243 #endif
1244 }
1245
crypt_convert_init(struct crypt_config * cc,struct convert_context * ctx,struct bio * bio_out,struct bio * bio_in,sector_t sector)1246 static void crypt_convert_init(struct crypt_config *cc,
1247 struct convert_context *ctx,
1248 struct bio *bio_out, struct bio *bio_in,
1249 sector_t sector)
1250 {
1251 ctx->bio_in = bio_in;
1252 ctx->bio_out = bio_out;
1253 if (bio_in)
1254 ctx->iter_in = bio_in->bi_iter;
1255 if (bio_out)
1256 ctx->iter_out = bio_out->bi_iter;
1257 ctx->cc_sector = sector + cc->iv_offset;
1258 init_completion(&ctx->restart);
1259 }
1260
dmreq_of_req(struct crypt_config * cc,void * req)1261 static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
1262 void *req)
1263 {
1264 return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
1265 }
1266
req_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1267 static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq)
1268 {
1269 return (void *)((char *)dmreq - cc->dmreq_start);
1270 }
1271
iv_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1272 static u8 *iv_of_dmreq(struct crypt_config *cc,
1273 struct dm_crypt_request *dmreq)
1274 {
1275 if (crypt_integrity_aead(cc))
1276 return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1277 crypto_aead_alignmask(any_tfm_aead(cc)) + 1);
1278 else
1279 return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1280 crypto_skcipher_alignmask(any_tfm(cc)) + 1);
1281 }
1282
org_iv_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1283 static u8 *org_iv_of_dmreq(struct crypt_config *cc,
1284 struct dm_crypt_request *dmreq)
1285 {
1286 return iv_of_dmreq(cc, dmreq) + cc->iv_size;
1287 }
1288
org_sector_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1289 static __le64 *org_sector_of_dmreq(struct crypt_config *cc,
1290 struct dm_crypt_request *dmreq)
1291 {
1292 u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size;
1293
1294 return (__le64 *) ptr;
1295 }
1296
org_tag_of_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1297 static unsigned int *org_tag_of_dmreq(struct crypt_config *cc,
1298 struct dm_crypt_request *dmreq)
1299 {
1300 u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size +
1301 cc->iv_size + sizeof(uint64_t);
1302
1303 return (unsigned int *)ptr;
1304 }
1305
tag_from_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1306 static void *tag_from_dmreq(struct crypt_config *cc,
1307 struct dm_crypt_request *dmreq)
1308 {
1309 struct convert_context *ctx = dmreq->ctx;
1310 struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
1311
1312 return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) *
1313 cc->tuple_size];
1314 }
1315
iv_tag_from_dmreq(struct crypt_config * cc,struct dm_crypt_request * dmreq)1316 static void *iv_tag_from_dmreq(struct crypt_config *cc,
1317 struct dm_crypt_request *dmreq)
1318 {
1319 return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size;
1320 }
1321
crypt_convert_block_aead(struct crypt_config * cc,struct convert_context * ctx,struct aead_request * req,unsigned int tag_offset)1322 static int crypt_convert_block_aead(struct crypt_config *cc,
1323 struct convert_context *ctx,
1324 struct aead_request *req,
1325 unsigned int tag_offset)
1326 {
1327 struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1328 struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
1329 struct dm_crypt_request *dmreq;
1330 u8 *iv, *org_iv, *tag_iv, *tag;
1331 __le64 *sector;
1332 int r = 0;
1333
1334 BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size);
1335
1336 /* Reject unexpected unaligned bio. */
1337 if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
1338 return -EIO;
1339
1340 dmreq = dmreq_of_req(cc, req);
1341 dmreq->iv_sector = ctx->cc_sector;
1342 if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1343 dmreq->iv_sector >>= cc->sector_shift;
1344 dmreq->ctx = ctx;
1345
1346 *org_tag_of_dmreq(cc, dmreq) = tag_offset;
1347
1348 sector = org_sector_of_dmreq(cc, dmreq);
1349 *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1350
1351 iv = iv_of_dmreq(cc, dmreq);
1352 org_iv = org_iv_of_dmreq(cc, dmreq);
1353 tag = tag_from_dmreq(cc, dmreq);
1354 tag_iv = iv_tag_from_dmreq(cc, dmreq);
1355
1356 /* AEAD request:
1357 * |----- AAD -------|------ DATA -------|-- AUTH TAG --|
1358 * | (authenticated) | (auth+encryption) | |
1359 * | sector_LE | IV | sector in/out | tag in/out |
1360 */
1361 sg_init_table(dmreq->sg_in, 4);
1362 sg_set_buf(&dmreq->sg_in[0], sector, sizeof(uint64_t));
1363 sg_set_buf(&dmreq->sg_in[1], org_iv, cc->iv_size);
1364 sg_set_page(&dmreq->sg_in[2], bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
1365 sg_set_buf(&dmreq->sg_in[3], tag, cc->integrity_tag_size);
1366
1367 sg_init_table(dmreq->sg_out, 4);
1368 sg_set_buf(&dmreq->sg_out[0], sector, sizeof(uint64_t));
1369 sg_set_buf(&dmreq->sg_out[1], org_iv, cc->iv_size);
1370 sg_set_page(&dmreq->sg_out[2], bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
1371 sg_set_buf(&dmreq->sg_out[3], tag, cc->integrity_tag_size);
1372
1373 if (cc->iv_gen_ops) {
1374 /* For READs use IV stored in integrity metadata */
1375 if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1376 memcpy(org_iv, tag_iv, cc->iv_size);
1377 } else {
1378 r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
1379 if (r < 0)
1380 return r;
1381 /* Store generated IV in integrity metadata */
1382 if (cc->integrity_iv_size)
1383 memcpy(tag_iv, org_iv, cc->iv_size);
1384 }
1385 /* Working copy of IV, to be modified in crypto API */
1386 memcpy(iv, org_iv, cc->iv_size);
1387 }
1388
1389 aead_request_set_ad(req, sizeof(uint64_t) + cc->iv_size);
1390 if (bio_data_dir(ctx->bio_in) == WRITE) {
1391 aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
1392 cc->sector_size, iv);
1393 r = crypto_aead_encrypt(req);
1394 if (cc->integrity_tag_size + cc->integrity_iv_size != cc->tuple_size)
1395 memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0,
1396 cc->tuple_size - (cc->integrity_tag_size + cc->integrity_iv_size));
1397 } else {
1398 aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
1399 cc->sector_size + cc->integrity_tag_size, iv);
1400 r = crypto_aead_decrypt(req);
1401 }
1402
1403 if (r == -EBADMSG) {
1404 sector_t s = le64_to_cpu(*sector);
1405
1406 ctx->aead_failed = true;
1407 if (ctx->aead_recheck) {
1408 DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
1409 ctx->bio_in->bi_bdev, s);
1410 dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",
1411 ctx->bio_in, s, 0);
1412 }
1413 }
1414
1415 if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1416 r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1417
1418 bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
1419 bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
1420
1421 return r;
1422 }
1423
crypt_convert_block_skcipher(struct crypt_config * cc,struct convert_context * ctx,struct skcipher_request * req,unsigned int tag_offset)1424 static int crypt_convert_block_skcipher(struct crypt_config *cc,
1425 struct convert_context *ctx,
1426 struct skcipher_request *req,
1427 unsigned int tag_offset)
1428 {
1429 struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1430 struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
1431 struct scatterlist *sg_in, *sg_out;
1432 struct dm_crypt_request *dmreq;
1433 u8 *iv, *org_iv, *tag_iv;
1434 __le64 *sector;
1435 int r = 0;
1436
1437 /* Reject unexpected unaligned bio. */
1438 if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
1439 return -EIO;
1440
1441 dmreq = dmreq_of_req(cc, req);
1442 dmreq->iv_sector = ctx->cc_sector;
1443 if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1444 dmreq->iv_sector >>= cc->sector_shift;
1445 dmreq->ctx = ctx;
1446
1447 *org_tag_of_dmreq(cc, dmreq) = tag_offset;
1448
1449 iv = iv_of_dmreq(cc, dmreq);
1450 org_iv = org_iv_of_dmreq(cc, dmreq);
1451 tag_iv = iv_tag_from_dmreq(cc, dmreq);
1452
1453 sector = org_sector_of_dmreq(cc, dmreq);
1454 *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1455
1456 /* For skcipher we use only the first sg item */
1457 sg_in = &dmreq->sg_in[0];
1458 sg_out = &dmreq->sg_out[0];
1459
1460 sg_init_table(sg_in, 1);
1461 sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
1462
1463 sg_init_table(sg_out, 1);
1464 sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
1465
1466 if (cc->iv_gen_ops) {
1467 /* For READs use IV stored in integrity metadata */
1468 if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1469 memcpy(org_iv, tag_iv, cc->integrity_iv_size);
1470 } else {
1471 r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
1472 if (r < 0)
1473 return r;
1474 /* Data can be already preprocessed in generator */
1475 if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags))
1476 sg_in = sg_out;
1477 /* Store generated IV in integrity metadata */
1478 if (cc->integrity_iv_size)
1479 memcpy(tag_iv, org_iv, cc->integrity_iv_size);
1480 }
1481 /* Working copy of IV, to be modified in crypto API */
1482 memcpy(iv, org_iv, cc->iv_size);
1483 }
1484
1485 skcipher_request_set_crypt(req, sg_in, sg_out, cc->sector_size, iv);
1486
1487 if (bio_data_dir(ctx->bio_in) == WRITE)
1488 r = crypto_skcipher_encrypt(req);
1489 else
1490 r = crypto_skcipher_decrypt(req);
1491
1492 if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1493 r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1494
1495 bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
1496 bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
1497
1498 return r;
1499 }
1500
1501 static void kcryptd_async_done(void *async_req, int error);
1502
crypt_alloc_req_skcipher(struct crypt_config * cc,struct convert_context * ctx)1503 static int crypt_alloc_req_skcipher(struct crypt_config *cc,
1504 struct convert_context *ctx)
1505 {
1506 unsigned int key_index = ctx->cc_sector & (cc->tfms_count - 1);
1507
1508 if (!ctx->r.req) {
1509 ctx->r.req = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
1510 if (!ctx->r.req)
1511 return -ENOMEM;
1512 }
1513
1514 skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]);
1515
1516 /*
1517 * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1518 * requests if driver request queue is full.
1519 */
1520 skcipher_request_set_callback(ctx->r.req,
1521 CRYPTO_TFM_REQ_MAY_BACKLOG,
1522 kcryptd_async_done, dmreq_of_req(cc, ctx->r.req));
1523
1524 return 0;
1525 }
1526
crypt_alloc_req_aead(struct crypt_config * cc,struct convert_context * ctx)1527 static int crypt_alloc_req_aead(struct crypt_config *cc,
1528 struct convert_context *ctx)
1529 {
1530 if (!ctx->r.req_aead) {
1531 ctx->r.req_aead = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
1532 if (!ctx->r.req_aead)
1533 return -ENOMEM;
1534 }
1535
1536 aead_request_set_tfm(ctx->r.req_aead, cc->cipher_tfm.tfms_aead[0]);
1537
1538 /*
1539 * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1540 * requests if driver request queue is full.
1541 */
1542 aead_request_set_callback(ctx->r.req_aead,
1543 CRYPTO_TFM_REQ_MAY_BACKLOG,
1544 kcryptd_async_done, dmreq_of_req(cc, ctx->r.req_aead));
1545
1546 return 0;
1547 }
1548
crypt_alloc_req(struct crypt_config * cc,struct convert_context * ctx)1549 static int crypt_alloc_req(struct crypt_config *cc,
1550 struct convert_context *ctx)
1551 {
1552 if (crypt_integrity_aead(cc))
1553 return crypt_alloc_req_aead(cc, ctx);
1554 else
1555 return crypt_alloc_req_skcipher(cc, ctx);
1556 }
1557
crypt_free_req_skcipher(struct crypt_config * cc,struct skcipher_request * req,struct bio * base_bio)1558 static void crypt_free_req_skcipher(struct crypt_config *cc,
1559 struct skcipher_request *req, struct bio *base_bio)
1560 {
1561 struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
1562
1563 if ((struct skcipher_request *)(io + 1) != req)
1564 mempool_free(req, &cc->req_pool);
1565 }
1566
crypt_free_req_aead(struct crypt_config * cc,struct aead_request * req,struct bio * base_bio)1567 static void crypt_free_req_aead(struct crypt_config *cc,
1568 struct aead_request *req, struct bio *base_bio)
1569 {
1570 struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
1571
1572 if ((struct aead_request *)(io + 1) != req)
1573 mempool_free(req, &cc->req_pool);
1574 }
1575
crypt_free_req(struct crypt_config * cc,void * req,struct bio * base_bio)1576 static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio)
1577 {
1578 if (crypt_integrity_aead(cc))
1579 crypt_free_req_aead(cc, req, base_bio);
1580 else
1581 crypt_free_req_skcipher(cc, req, base_bio);
1582 }
1583
1584 /*
1585 * Encrypt / decrypt data from one bio to another one (can be the same one)
1586 */
crypt_convert(struct crypt_config * cc,struct convert_context * ctx,bool atomic,bool reset_pending)1587 static blk_status_t crypt_convert(struct crypt_config *cc,
1588 struct convert_context *ctx, bool atomic, bool reset_pending)
1589 {
1590 unsigned int tag_offset = 0;
1591 unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;
1592 int r;
1593
1594 /*
1595 * if reset_pending is set we are dealing with the bio for the first time,
1596 * else we're continuing to work on the previous bio, so don't mess with
1597 * the cc_pending counter
1598 */
1599 if (reset_pending)
1600 atomic_set(&ctx->cc_pending, 1);
1601
1602 while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
1603
1604 r = crypt_alloc_req(cc, ctx);
1605 if (r) {
1606 complete(&ctx->restart);
1607 return BLK_STS_DEV_RESOURCE;
1608 }
1609
1610 atomic_inc(&ctx->cc_pending);
1611
1612 if (crypt_integrity_aead(cc))
1613 r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, tag_offset);
1614 else
1615 r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, tag_offset);
1616
1617 switch (r) {
1618 /*
1619 * The request was queued by a crypto driver
1620 * but the driver request queue is full, let's wait.
1621 */
1622 case -EBUSY:
1623 if (in_interrupt()) {
1624 if (try_wait_for_completion(&ctx->restart)) {
1625 /*
1626 * we don't have to block to wait for completion,
1627 * so proceed
1628 */
1629 } else {
1630 /*
1631 * we can't wait for completion without blocking
1632 * exit and continue processing in a workqueue
1633 */
1634 ctx->r.req = NULL;
1635 ctx->cc_sector += sector_step;
1636 tag_offset++;
1637 return BLK_STS_DEV_RESOURCE;
1638 }
1639 } else {
1640 wait_for_completion(&ctx->restart);
1641 }
1642 reinit_completion(&ctx->restart);
1643 fallthrough;
1644 /*
1645 * The request is queued and processed asynchronously,
1646 * completion function kcryptd_async_done() will be called.
1647 */
1648 case -EINPROGRESS:
1649 ctx->r.req = NULL;
1650 ctx->cc_sector += sector_step;
1651 tag_offset++;
1652 continue;
1653 /*
1654 * The request was already processed (synchronously).
1655 */
1656 case 0:
1657 atomic_dec(&ctx->cc_pending);
1658 ctx->cc_sector += sector_step;
1659 tag_offset++;
1660 if (!atomic)
1661 cond_resched();
1662 continue;
1663 /*
1664 * There was a data integrity error.
1665 */
1666 case -EBADMSG:
1667 atomic_dec(&ctx->cc_pending);
1668 return BLK_STS_PROTECTION;
1669 /*
1670 * There was an error while processing the request.
1671 */
1672 default:
1673 atomic_dec(&ctx->cc_pending);
1674 return BLK_STS_IOERR;
1675 }
1676 }
1677
1678 return 0;
1679 }
1680
1681 static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
1682
1683 /*
1684 * Generate a new unfragmented bio with the given size
1685 * This should never violate the device limitations (but if it did then block
1686 * core should split the bio as needed).
1687 *
1688 * This function may be called concurrently. If we allocate from the mempool
1689 * concurrently, there is a possibility of deadlock. For example, if we have
1690 * mempool of 256 pages, two processes, each wanting 256, pages allocate from
1691 * the mempool concurrently, it may deadlock in a situation where both processes
1692 * have allocated 128 pages and the mempool is exhausted.
1693 *
1694 * In order to avoid this scenario we allocate the pages under a mutex.
1695 *
1696 * In order to not degrade performance with excessive locking, we try
1697 * non-blocking allocations without a mutex first but on failure we fallback
1698 * to blocking allocations with a mutex.
1699 *
1700 * In order to reduce allocation overhead, we try to allocate compound pages in
1701 * the first pass. If they are not available, we fall back to the mempool.
1702 */
crypt_alloc_buffer(struct dm_crypt_io * io,unsigned int size)1703 static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned int size)
1704 {
1705 struct crypt_config *cc = io->cc;
1706 struct bio *clone;
1707 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1708 gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;
1709 unsigned int remaining_size;
1710 unsigned int order = MAX_PAGE_ORDER;
1711
1712 retry:
1713 if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
1714 mutex_lock(&cc->bio_alloc_lock);
1715
1716 clone = bio_alloc_bioset(cc->dev->bdev, nr_iovecs, io->base_bio->bi_opf,
1717 GFP_NOIO, &cc->bs);
1718 clone->bi_private = io;
1719 clone->bi_end_io = crypt_endio;
1720 clone->bi_ioprio = io->base_bio->bi_ioprio;
1721
1722 remaining_size = size;
1723
1724 while (remaining_size) {
1725 struct page *pages;
1726 unsigned size_to_add;
1727 unsigned remaining_order = __fls((remaining_size + PAGE_SIZE - 1) >> PAGE_SHIFT);
1728 order = min(order, remaining_order);
1729
1730 while (order > 0) {
1731 if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) +
1732 (1 << order) > dm_crypt_pages_per_client))
1733 goto decrease_order;
1734 pages = alloc_pages(gfp_mask
1735 | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | __GFP_COMP,
1736 order);
1737 if (likely(pages != NULL)) {
1738 percpu_counter_add(&cc->n_allocated_pages, 1 << order);
1739 goto have_pages;
1740 }
1741 decrease_order:
1742 order--;
1743 }
1744
1745 pages = mempool_alloc(&cc->page_pool, gfp_mask);
1746 if (!pages) {
1747 crypt_free_buffer_pages(cc, clone);
1748 bio_put(clone);
1749 gfp_mask |= __GFP_DIRECT_RECLAIM;
1750 order = 0;
1751 goto retry;
1752 }
1753
1754 have_pages:
1755 size_to_add = min((unsigned)PAGE_SIZE << order, remaining_size);
1756 __bio_add_page(clone, pages, size_to_add, 0);
1757 remaining_size -= size_to_add;
1758 }
1759
1760 /* Allocate space for integrity tags */
1761 if (dm_crypt_integrity_io_alloc(io, clone)) {
1762 crypt_free_buffer_pages(cc, clone);
1763 bio_put(clone);
1764 clone = NULL;
1765 }
1766
1767 if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
1768 mutex_unlock(&cc->bio_alloc_lock);
1769
1770 return clone;
1771 }
1772
crypt_free_buffer_pages(struct crypt_config * cc,struct bio * clone)1773 static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
1774 {
1775 struct folio_iter fi;
1776
1777 if (clone->bi_vcnt > 0) { /* bio_for_each_folio_all crashes with an empty bio */
1778 bio_for_each_folio_all(fi, clone) {
1779 if (folio_test_large(fi.folio)) {
1780 percpu_counter_sub(&cc->n_allocated_pages,
1781 1 << folio_order(fi.folio));
1782 folio_put(fi.folio);
1783 } else {
1784 mempool_free(&fi.folio->page, &cc->page_pool);
1785 }
1786 }
1787 }
1788 }
1789
crypt_io_init(struct dm_crypt_io * io,struct crypt_config * cc,struct bio * bio,sector_t sector)1790 static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,
1791 struct bio *bio, sector_t sector)
1792 {
1793 io->cc = cc;
1794 io->base_bio = bio;
1795 io->sector = sector;
1796 io->error = 0;
1797 io->ctx.aead_recheck = false;
1798 io->ctx.aead_failed = false;
1799 io->ctx.r.req = NULL;
1800 io->integrity_metadata = NULL;
1801 io->integrity_metadata_from_pool = false;
1802 atomic_set(&io->io_pending, 0);
1803 }
1804
crypt_inc_pending(struct dm_crypt_io * io)1805 static void crypt_inc_pending(struct dm_crypt_io *io)
1806 {
1807 atomic_inc(&io->io_pending);
1808 }
1809
1810 static void kcryptd_queue_read(struct dm_crypt_io *io);
1811
1812 /*
1813 * One of the bios was finished. Check for completion of
1814 * the whole request and correctly clean up the buffer.
1815 */
crypt_dec_pending(struct dm_crypt_io * io)1816 static void crypt_dec_pending(struct dm_crypt_io *io)
1817 {
1818 struct crypt_config *cc = io->cc;
1819 struct bio *base_bio = io->base_bio;
1820 blk_status_t error = io->error;
1821
1822 if (!atomic_dec_and_test(&io->io_pending))
1823 return;
1824
1825 if (likely(!io->ctx.aead_recheck) && unlikely(io->ctx.aead_failed) &&
1826 cc->used_tag_size && bio_data_dir(base_bio) == READ) {
1827 io->ctx.aead_recheck = true;
1828 io->ctx.aead_failed = false;
1829 io->error = 0;
1830 kcryptd_queue_read(io);
1831 return;
1832 }
1833
1834 if (io->ctx.r.req)
1835 crypt_free_req(cc, io->ctx.r.req, base_bio);
1836
1837 if (unlikely(io->integrity_metadata_from_pool))
1838 mempool_free(io->integrity_metadata, &io->cc->tag_pool);
1839 else
1840 kfree(io->integrity_metadata);
1841
1842 base_bio->bi_status = error;
1843
1844 bio_endio(base_bio);
1845 }
1846
1847 /*
1848 * kcryptd/kcryptd_io:
1849 *
1850 * Needed because it would be very unwise to do decryption in an
1851 * interrupt context.
1852 *
1853 * kcryptd performs the actual encryption or decryption.
1854 *
1855 * kcryptd_io performs the IO submission.
1856 *
1857 * They must be separated as otherwise the final stages could be
1858 * starved by new requests which can block in the first stages due
1859 * to memory allocation.
1860 *
1861 * The work is done per CPU global for all dm-crypt instances.
1862 * They should not depend on each other and do not block.
1863 */
crypt_endio(struct bio * clone)1864 static void crypt_endio(struct bio *clone)
1865 {
1866 struct dm_crypt_io *io = clone->bi_private;
1867 struct crypt_config *cc = io->cc;
1868 unsigned int rw = bio_data_dir(clone);
1869 blk_status_t error = clone->bi_status;
1870
1871 if (io->ctx.aead_recheck && !error) {
1872 kcryptd_queue_crypt(io);
1873 return;
1874 }
1875
1876 /*
1877 * free the processed pages
1878 */
1879 if (rw == WRITE || io->ctx.aead_recheck)
1880 crypt_free_buffer_pages(cc, clone);
1881
1882 bio_put(clone);
1883
1884 if (rw == READ && !error) {
1885 kcryptd_queue_crypt(io);
1886 return;
1887 }
1888
1889 if (unlikely(error))
1890 io->error = error;
1891
1892 crypt_dec_pending(io);
1893 }
1894
1895 #define CRYPT_MAP_READ_GFP GFP_NOWAIT
1896
kcryptd_io_read(struct dm_crypt_io * io,gfp_t gfp)1897 static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
1898 {
1899 struct crypt_config *cc = io->cc;
1900 struct bio *clone;
1901
1902 if (io->ctx.aead_recheck) {
1903 if (!(gfp & __GFP_DIRECT_RECLAIM))
1904 return 1;
1905 crypt_inc_pending(io);
1906 clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
1907 if (unlikely(!clone)) {
1908 crypt_dec_pending(io);
1909 return 1;
1910 }
1911 clone->bi_iter.bi_sector = cc->start + io->sector;
1912 crypt_convert_init(cc, &io->ctx, clone, clone, io->sector);
1913 io->saved_bi_iter = clone->bi_iter;
1914 dm_submit_bio_remap(io->base_bio, clone);
1915 return 0;
1916 }
1917
1918 /*
1919 * We need the original biovec array in order to decrypt the whole bio
1920 * data *afterwards* -- thanks to immutable biovecs we don't need to
1921 * worry about the block layer modifying the biovec array; so leverage
1922 * bio_alloc_clone().
1923 */
1924 clone = bio_alloc_clone(cc->dev->bdev, io->base_bio, gfp, &cc->bs);
1925 if (!clone)
1926 return 1;
1927 clone->bi_private = io;
1928 clone->bi_end_io = crypt_endio;
1929
1930 crypt_inc_pending(io);
1931
1932 clone->bi_iter.bi_sector = cc->start + io->sector;
1933
1934 if (dm_crypt_integrity_io_alloc(io, clone)) {
1935 crypt_dec_pending(io);
1936 bio_put(clone);
1937 return 1;
1938 }
1939
1940 dm_submit_bio_remap(io->base_bio, clone);
1941 return 0;
1942 }
1943
kcryptd_io_read_work(struct work_struct * work)1944 static void kcryptd_io_read_work(struct work_struct *work)
1945 {
1946 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1947
1948 crypt_inc_pending(io);
1949 if (kcryptd_io_read(io, GFP_NOIO))
1950 io->error = BLK_STS_RESOURCE;
1951 crypt_dec_pending(io);
1952 }
1953
kcryptd_queue_read(struct dm_crypt_io * io)1954 static void kcryptd_queue_read(struct dm_crypt_io *io)
1955 {
1956 struct crypt_config *cc = io->cc;
1957
1958 INIT_WORK(&io->work, kcryptd_io_read_work);
1959 queue_work(cc->io_queue, &io->work);
1960 }
1961
kcryptd_io_write(struct dm_crypt_io * io)1962 static void kcryptd_io_write(struct dm_crypt_io *io)
1963 {
1964 struct bio *clone = io->ctx.bio_out;
1965
1966 dm_submit_bio_remap(io->base_bio, clone);
1967 }
1968
1969 #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)
1970
dmcrypt_write(void * data)1971 static int dmcrypt_write(void *data)
1972 {
1973 struct crypt_config *cc = data;
1974 struct dm_crypt_io *io;
1975
1976 while (1) {
1977 struct rb_root write_tree;
1978 struct blk_plug plug;
1979
1980 spin_lock_irq(&cc->write_thread_lock);
1981 continue_locked:
1982
1983 if (!RB_EMPTY_ROOT(&cc->write_tree))
1984 goto pop_from_list;
1985
1986 set_current_state(TASK_INTERRUPTIBLE);
1987
1988 spin_unlock_irq(&cc->write_thread_lock);
1989
1990 if (unlikely(kthread_should_stop())) {
1991 set_current_state(TASK_RUNNING);
1992 break;
1993 }
1994
1995 schedule();
1996
1997 spin_lock_irq(&cc->write_thread_lock);
1998 goto continue_locked;
1999
2000 pop_from_list:
2001 write_tree = cc->write_tree;
2002 cc->write_tree = RB_ROOT;
2003 spin_unlock_irq(&cc->write_thread_lock);
2004
2005 BUG_ON(rb_parent(write_tree.rb_node));
2006
2007 /*
2008 * Note: we cannot walk the tree here with rb_next because
2009 * the structures may be freed when kcryptd_io_write is called.
2010 */
2011 blk_start_plug(&plug);
2012 do {
2013 io = crypt_io_from_node(rb_first(&write_tree));
2014 rb_erase(&io->rb_node, &write_tree);
2015 kcryptd_io_write(io);
2016 cond_resched();
2017 } while (!RB_EMPTY_ROOT(&write_tree));
2018 blk_finish_plug(&plug);
2019 }
2020 return 0;
2021 }
2022
kcryptd_crypt_write_io_submit(struct dm_crypt_io * io,int async)2023 static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
2024 {
2025 struct bio *clone = io->ctx.bio_out;
2026 struct crypt_config *cc = io->cc;
2027 unsigned long flags;
2028 sector_t sector;
2029 struct rb_node **rbp, *parent;
2030
2031 if (unlikely(io->error)) {
2032 crypt_free_buffer_pages(cc, clone);
2033 bio_put(clone);
2034 crypt_dec_pending(io);
2035 return;
2036 }
2037
2038 /* crypt_convert should have filled the clone bio */
2039 BUG_ON(io->ctx.iter_out.bi_size);
2040
2041 clone->bi_iter.bi_sector = cc->start + io->sector;
2042
2043 if ((likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) ||
2044 test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) {
2045 dm_submit_bio_remap(io->base_bio, clone);
2046 return;
2047 }
2048
2049 spin_lock_irqsave(&cc->write_thread_lock, flags);
2050 if (RB_EMPTY_ROOT(&cc->write_tree))
2051 wake_up_process(cc->write_thread);
2052 rbp = &cc->write_tree.rb_node;
2053 parent = NULL;
2054 sector = io->sector;
2055 while (*rbp) {
2056 parent = *rbp;
2057 if (sector < crypt_io_from_node(parent)->sector)
2058 rbp = &(*rbp)->rb_left;
2059 else
2060 rbp = &(*rbp)->rb_right;
2061 }
2062 rb_link_node(&io->rb_node, parent, rbp);
2063 rb_insert_color(&io->rb_node, &cc->write_tree);
2064 spin_unlock_irqrestore(&cc->write_thread_lock, flags);
2065 }
2066
kcryptd_crypt_write_inline(struct crypt_config * cc,struct convert_context * ctx)2067 static bool kcryptd_crypt_write_inline(struct crypt_config *cc,
2068 struct convert_context *ctx)
2069
2070 {
2071 if (!test_bit(DM_CRYPT_WRITE_INLINE, &cc->flags))
2072 return false;
2073
2074 /*
2075 * Note: zone append writes (REQ_OP_ZONE_APPEND) do not have ordering
2076 * constraints so they do not need to be issued inline by
2077 * kcryptd_crypt_write_convert().
2078 */
2079 switch (bio_op(ctx->bio_in)) {
2080 case REQ_OP_WRITE:
2081 case REQ_OP_WRITE_ZEROES:
2082 return true;
2083 default:
2084 return false;
2085 }
2086 }
2087
kcryptd_crypt_write_continue(struct work_struct * work)2088 static void kcryptd_crypt_write_continue(struct work_struct *work)
2089 {
2090 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
2091 struct crypt_config *cc = io->cc;
2092 struct convert_context *ctx = &io->ctx;
2093 int crypt_finished;
2094 sector_t sector = io->sector;
2095 blk_status_t r;
2096
2097 wait_for_completion(&ctx->restart);
2098 reinit_completion(&ctx->restart);
2099
2100 r = crypt_convert(cc, &io->ctx, true, false);
2101 if (r)
2102 io->error = r;
2103 crypt_finished = atomic_dec_and_test(&ctx->cc_pending);
2104 if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
2105 /* Wait for completion signaled by kcryptd_async_done() */
2106 wait_for_completion(&ctx->restart);
2107 crypt_finished = 1;
2108 }
2109
2110 /* Encryption was already finished, submit io now */
2111 if (crypt_finished) {
2112 kcryptd_crypt_write_io_submit(io, 0);
2113 io->sector = sector;
2114 }
2115
2116 crypt_dec_pending(io);
2117 }
2118
kcryptd_crypt_write_convert(struct dm_crypt_io * io)2119 static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
2120 {
2121 struct crypt_config *cc = io->cc;
2122 struct convert_context *ctx = &io->ctx;
2123 struct bio *clone;
2124 int crypt_finished;
2125 sector_t sector = io->sector;
2126 blk_status_t r;
2127
2128 /*
2129 * Prevent io from disappearing until this function completes.
2130 */
2131 crypt_inc_pending(io);
2132 crypt_convert_init(cc, ctx, NULL, io->base_bio, sector);
2133
2134 clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
2135 if (unlikely(!clone)) {
2136 io->error = BLK_STS_IOERR;
2137 goto dec;
2138 }
2139
2140 io->ctx.bio_out = clone;
2141 io->ctx.iter_out = clone->bi_iter;
2142
2143 if (crypt_integrity_aead(cc)) {
2144 bio_copy_data(clone, io->base_bio);
2145 io->ctx.bio_in = clone;
2146 io->ctx.iter_in = clone->bi_iter;
2147 }
2148
2149 sector += bio_sectors(clone);
2150
2151 crypt_inc_pending(io);
2152 r = crypt_convert(cc, ctx,
2153 test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags), true);
2154 /*
2155 * Crypto API backlogged the request, because its queue was full
2156 * and we're in softirq context, so continue from a workqueue
2157 * (TODO: is it actually possible to be in softirq in the write path?)
2158 */
2159 if (r == BLK_STS_DEV_RESOURCE) {
2160 INIT_WORK(&io->work, kcryptd_crypt_write_continue);
2161 queue_work(cc->crypt_queue, &io->work);
2162 return;
2163 }
2164 if (r)
2165 io->error = r;
2166 crypt_finished = atomic_dec_and_test(&ctx->cc_pending);
2167 if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
2168 /* Wait for completion signaled by kcryptd_async_done() */
2169 wait_for_completion(&ctx->restart);
2170 crypt_finished = 1;
2171 }
2172
2173 /* Encryption was already finished, submit io now */
2174 if (crypt_finished) {
2175 kcryptd_crypt_write_io_submit(io, 0);
2176 io->sector = sector;
2177 }
2178
2179 dec:
2180 crypt_dec_pending(io);
2181 }
2182
kcryptd_crypt_read_done(struct dm_crypt_io * io)2183 static void kcryptd_crypt_read_done(struct dm_crypt_io *io)
2184 {
2185 if (io->ctx.aead_recheck) {
2186 if (!io->error) {
2187 io->ctx.bio_in->bi_iter = io->saved_bi_iter;
2188 bio_copy_data(io->base_bio, io->ctx.bio_in);
2189 }
2190 crypt_free_buffer_pages(io->cc, io->ctx.bio_in);
2191 bio_put(io->ctx.bio_in);
2192 }
2193 crypt_dec_pending(io);
2194 }
2195
kcryptd_crypt_read_continue(struct work_struct * work)2196 static void kcryptd_crypt_read_continue(struct work_struct *work)
2197 {
2198 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
2199 struct crypt_config *cc = io->cc;
2200 blk_status_t r;
2201
2202 wait_for_completion(&io->ctx.restart);
2203 reinit_completion(&io->ctx.restart);
2204
2205 r = crypt_convert(cc, &io->ctx, true, false);
2206 if (r)
2207 io->error = r;
2208
2209 if (atomic_dec_and_test(&io->ctx.cc_pending))
2210 kcryptd_crypt_read_done(io);
2211
2212 crypt_dec_pending(io);
2213 }
2214
kcryptd_crypt_read_convert(struct dm_crypt_io * io)2215 static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
2216 {
2217 struct crypt_config *cc = io->cc;
2218 blk_status_t r;
2219
2220 crypt_inc_pending(io);
2221
2222 if (io->ctx.aead_recheck) {
2223 io->ctx.cc_sector = io->sector + cc->iv_offset;
2224 r = crypt_convert(cc, &io->ctx,
2225 test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);
2226 } else {
2227 crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
2228 io->sector);
2229
2230 r = crypt_convert(cc, &io->ctx,
2231 test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);
2232 }
2233 /*
2234 * Crypto API backlogged the request, because its queue was full
2235 * and we're in softirq context, so continue from a workqueue
2236 */
2237 if (r == BLK_STS_DEV_RESOURCE) {
2238 INIT_WORK(&io->work, kcryptd_crypt_read_continue);
2239 queue_work(cc->crypt_queue, &io->work);
2240 return;
2241 }
2242 if (r)
2243 io->error = r;
2244
2245 if (atomic_dec_and_test(&io->ctx.cc_pending))
2246 kcryptd_crypt_read_done(io);
2247
2248 crypt_dec_pending(io);
2249 }
2250
kcryptd_async_done(void * data,int error)2251 static void kcryptd_async_done(void *data, int error)
2252 {
2253 struct dm_crypt_request *dmreq = data;
2254 struct convert_context *ctx = dmreq->ctx;
2255 struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
2256 struct crypt_config *cc = io->cc;
2257
2258 /*
2259 * A request from crypto driver backlog is going to be processed now,
2260 * finish the completion and continue in crypt_convert().
2261 * (Callback will be called for the second time for this request.)
2262 */
2263 if (error == -EINPROGRESS) {
2264 complete(&ctx->restart);
2265 return;
2266 }
2267
2268 if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
2269 error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq);
2270
2271 if (error == -EBADMSG) {
2272 sector_t s = le64_to_cpu(*org_sector_of_dmreq(cc, dmreq));
2273
2274 ctx->aead_failed = true;
2275 if (ctx->aead_recheck) {
2276 DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
2277 ctx->bio_in->bi_bdev, s);
2278 dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",
2279 ctx->bio_in, s, 0);
2280 }
2281 io->error = BLK_STS_PROTECTION;
2282 } else if (error < 0)
2283 io->error = BLK_STS_IOERR;
2284
2285 crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio);
2286
2287 if (!atomic_dec_and_test(&ctx->cc_pending))
2288 return;
2289
2290 /*
2291 * The request is fully completed: for inline writes, let
2292 * kcryptd_crypt_write_convert() do the IO submission.
2293 */
2294 if (bio_data_dir(io->base_bio) == READ) {
2295 kcryptd_crypt_read_done(io);
2296 return;
2297 }
2298
2299 if (kcryptd_crypt_write_inline(cc, ctx)) {
2300 complete(&ctx->restart);
2301 return;
2302 }
2303
2304 kcryptd_crypt_write_io_submit(io, 1);
2305 }
2306
kcryptd_crypt(struct work_struct * work)2307 static void kcryptd_crypt(struct work_struct *work)
2308 {
2309 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
2310
2311 if (bio_data_dir(io->base_bio) == READ)
2312 kcryptd_crypt_read_convert(io);
2313 else
2314 kcryptd_crypt_write_convert(io);
2315 }
2316
kcryptd_queue_crypt(struct dm_crypt_io * io)2317 static void kcryptd_queue_crypt(struct dm_crypt_io *io)
2318 {
2319 struct crypt_config *cc = io->cc;
2320
2321 if ((bio_data_dir(io->base_bio) == READ && test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) ||
2322 (bio_data_dir(io->base_bio) == WRITE && test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) {
2323 /*
2324 * in_hardirq(): Crypto API's skcipher_walk_first() refuses to work in hard IRQ context.
2325 * irqs_disabled(): the kernel may run some IO completion from the idle thread, but
2326 * it is being executed with irqs disabled.
2327 */
2328 if (in_hardirq() || irqs_disabled()) {
2329 INIT_WORK(&io->work, kcryptd_crypt);
2330 queue_work(system_bh_wq, &io->work);
2331 return;
2332 } else {
2333 kcryptd_crypt(&io->work);
2334 return;
2335 }
2336 }
2337
2338 INIT_WORK(&io->work, kcryptd_crypt);
2339 queue_work(cc->crypt_queue, &io->work);
2340 }
2341
crypt_free_tfms_aead(struct crypt_config * cc)2342 static void crypt_free_tfms_aead(struct crypt_config *cc)
2343 {
2344 if (!cc->cipher_tfm.tfms_aead)
2345 return;
2346
2347 if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(cc->cipher_tfm.tfms_aead[0])) {
2348 crypto_free_aead(cc->cipher_tfm.tfms_aead[0]);
2349 cc->cipher_tfm.tfms_aead[0] = NULL;
2350 }
2351
2352 kfree(cc->cipher_tfm.tfms_aead);
2353 cc->cipher_tfm.tfms_aead = NULL;
2354 }
2355
crypt_free_tfms_skcipher(struct crypt_config * cc)2356 static void crypt_free_tfms_skcipher(struct crypt_config *cc)
2357 {
2358 unsigned int i;
2359
2360 if (!cc->cipher_tfm.tfms)
2361 return;
2362
2363 for (i = 0; i < cc->tfms_count; i++)
2364 if (cc->cipher_tfm.tfms[i] && !IS_ERR(cc->cipher_tfm.tfms[i])) {
2365 crypto_free_skcipher(cc->cipher_tfm.tfms[i]);
2366 cc->cipher_tfm.tfms[i] = NULL;
2367 }
2368
2369 kfree(cc->cipher_tfm.tfms);
2370 cc->cipher_tfm.tfms = NULL;
2371 }
2372
crypt_free_tfms(struct crypt_config * cc)2373 static void crypt_free_tfms(struct crypt_config *cc)
2374 {
2375 if (crypt_integrity_aead(cc))
2376 crypt_free_tfms_aead(cc);
2377 else
2378 crypt_free_tfms_skcipher(cc);
2379 }
2380
crypt_alloc_tfms_skcipher(struct crypt_config * cc,char * ciphermode)2381 static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode)
2382 {
2383 unsigned int i;
2384 int err;
2385
2386 cc->cipher_tfm.tfms = kcalloc(cc->tfms_count,
2387 sizeof(struct crypto_skcipher *),
2388 GFP_KERNEL);
2389 if (!cc->cipher_tfm.tfms)
2390 return -ENOMEM;
2391
2392 for (i = 0; i < cc->tfms_count; i++) {
2393 cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(ciphermode, 0,
2394 CRYPTO_ALG_ALLOCATES_MEMORY);
2395 if (IS_ERR(cc->cipher_tfm.tfms[i])) {
2396 err = PTR_ERR(cc->cipher_tfm.tfms[i]);
2397 crypt_free_tfms(cc);
2398 return err;
2399 }
2400 }
2401
2402 /*
2403 * dm-crypt performance can vary greatly depending on which crypto
2404 * algorithm implementation is used. Help people debug performance
2405 * problems by logging the ->cra_driver_name.
2406 */
2407 DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
2408 crypto_skcipher_alg(any_tfm(cc))->base.cra_driver_name);
2409 return 0;
2410 }
2411
crypt_alloc_tfms_aead(struct crypt_config * cc,char * ciphermode)2412 static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode)
2413 {
2414 int err;
2415
2416 cc->cipher_tfm.tfms = kmalloc(sizeof(struct crypto_aead *), GFP_KERNEL);
2417 if (!cc->cipher_tfm.tfms)
2418 return -ENOMEM;
2419
2420 cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(ciphermode, 0,
2421 CRYPTO_ALG_ALLOCATES_MEMORY);
2422 if (IS_ERR(cc->cipher_tfm.tfms_aead[0])) {
2423 err = PTR_ERR(cc->cipher_tfm.tfms_aead[0]);
2424 crypt_free_tfms(cc);
2425 return err;
2426 }
2427
2428 DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
2429 crypto_aead_alg(any_tfm_aead(cc))->base.cra_driver_name);
2430 return 0;
2431 }
2432
crypt_alloc_tfms(struct crypt_config * cc,char * ciphermode)2433 static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
2434 {
2435 if (crypt_integrity_aead(cc))
2436 return crypt_alloc_tfms_aead(cc, ciphermode);
2437 else
2438 return crypt_alloc_tfms_skcipher(cc, ciphermode);
2439 }
2440
crypt_subkey_size(struct crypt_config * cc)2441 static unsigned int crypt_subkey_size(struct crypt_config *cc)
2442 {
2443 return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);
2444 }
2445
crypt_authenckey_size(struct crypt_config * cc)2446 static unsigned int crypt_authenckey_size(struct crypt_config *cc)
2447 {
2448 return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param));
2449 }
2450
2451 /*
2452 * If AEAD is composed like authenc(hmac(sha256),xts(aes)),
2453 * the key must be for some reason in special format.
2454 * This funcion converts cc->key to this special format.
2455 */
crypt_copy_authenckey(char * p,const void * key,unsigned int enckeylen,unsigned int authkeylen)2456 static void crypt_copy_authenckey(char *p, const void *key,
2457 unsigned int enckeylen, unsigned int authkeylen)
2458 {
2459 struct crypto_authenc_key_param *param;
2460 struct rtattr *rta;
2461
2462 rta = (struct rtattr *)p;
2463 param = RTA_DATA(rta);
2464 param->enckeylen = cpu_to_be32(enckeylen);
2465 rta->rta_len = RTA_LENGTH(sizeof(*param));
2466 rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
2467 p += RTA_SPACE(sizeof(*param));
2468 memcpy(p, key + enckeylen, authkeylen);
2469 p += authkeylen;
2470 memcpy(p, key, enckeylen);
2471 }
2472
crypt_setkey(struct crypt_config * cc)2473 static int crypt_setkey(struct crypt_config *cc)
2474 {
2475 unsigned int subkey_size;
2476 int err = 0, i, r;
2477
2478 /* Ignore extra keys (which are used for IV etc) */
2479 subkey_size = crypt_subkey_size(cc);
2480
2481 if (crypt_integrity_hmac(cc)) {
2482 if (subkey_size < cc->key_mac_size)
2483 return -EINVAL;
2484
2485 crypt_copy_authenckey(cc->authenc_key, cc->key,
2486 subkey_size - cc->key_mac_size,
2487 cc->key_mac_size);
2488 }
2489
2490 for (i = 0; i < cc->tfms_count; i++) {
2491 if (crypt_integrity_hmac(cc))
2492 r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],
2493 cc->authenc_key, crypt_authenckey_size(cc));
2494 else if (crypt_integrity_aead(cc))
2495 r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],
2496 cc->key + (i * subkey_size),
2497 subkey_size);
2498 else
2499 r = crypto_skcipher_setkey(cc->cipher_tfm.tfms[i],
2500 cc->key + (i * subkey_size),
2501 subkey_size);
2502 if (r)
2503 err = r;
2504 }
2505
2506 if (crypt_integrity_hmac(cc))
2507 memzero_explicit(cc->authenc_key, crypt_authenckey_size(cc));
2508
2509 return err;
2510 }
2511
2512 #ifdef CONFIG_KEYS
2513
contains_whitespace(const char * str)2514 static bool contains_whitespace(const char *str)
2515 {
2516 while (*str)
2517 if (isspace(*str++))
2518 return true;
2519 return false;
2520 }
2521
set_key_user(struct crypt_config * cc,struct key * key)2522 static int set_key_user(struct crypt_config *cc, struct key *key)
2523 {
2524 const struct user_key_payload *ukp;
2525
2526 ukp = user_key_payload_locked(key);
2527 if (!ukp)
2528 return -EKEYREVOKED;
2529
2530 if (cc->key_size != ukp->datalen)
2531 return -EINVAL;
2532
2533 memcpy(cc->key, ukp->data, cc->key_size);
2534
2535 return 0;
2536 }
2537
set_key_encrypted(struct crypt_config * cc,struct key * key)2538 static int set_key_encrypted(struct crypt_config *cc, struct key *key)
2539 {
2540 const struct encrypted_key_payload *ekp;
2541
2542 ekp = key->payload.data[0];
2543 if (!ekp)
2544 return -EKEYREVOKED;
2545
2546 if (cc->key_size != ekp->decrypted_datalen)
2547 return -EINVAL;
2548
2549 memcpy(cc->key, ekp->decrypted_data, cc->key_size);
2550
2551 return 0;
2552 }
2553
set_key_trusted(struct crypt_config * cc,struct key * key)2554 static int set_key_trusted(struct crypt_config *cc, struct key *key)
2555 {
2556 const struct trusted_key_payload *tkp;
2557
2558 tkp = key->payload.data[0];
2559 if (!tkp)
2560 return -EKEYREVOKED;
2561
2562 if (cc->key_size != tkp->key_len)
2563 return -EINVAL;
2564
2565 memcpy(cc->key, tkp->key, cc->key_size);
2566
2567 return 0;
2568 }
2569
crypt_set_keyring_key(struct crypt_config * cc,const char * key_string)2570 static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2571 {
2572 char *new_key_string, *key_desc;
2573 int ret;
2574 struct key_type *type;
2575 struct key *key;
2576 int (*set_key)(struct crypt_config *cc, struct key *key);
2577
2578 /*
2579 * Reject key_string with whitespace. dm core currently lacks code for
2580 * proper whitespace escaping in arguments on DM_TABLE_STATUS path.
2581 */
2582 if (contains_whitespace(key_string)) {
2583 DMERR("whitespace chars not allowed in key string");
2584 return -EINVAL;
2585 }
2586
2587 /* look for next ':' separating key_type from key_description */
2588 key_desc = strchr(key_string, ':');
2589 if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
2590 return -EINVAL;
2591
2592 if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) {
2593 type = &key_type_logon;
2594 set_key = set_key_user;
2595 } else if (!strncmp(key_string, "user:", key_desc - key_string + 1)) {
2596 type = &key_type_user;
2597 set_key = set_key_user;
2598 } else if (IS_ENABLED(CONFIG_ENCRYPTED_KEYS) &&
2599 !strncmp(key_string, "encrypted:", key_desc - key_string + 1)) {
2600 type = &key_type_encrypted;
2601 set_key = set_key_encrypted;
2602 } else if (IS_ENABLED(CONFIG_TRUSTED_KEYS) &&
2603 !strncmp(key_string, "trusted:", key_desc - key_string + 1)) {
2604 type = &key_type_trusted;
2605 set_key = set_key_trusted;
2606 } else {
2607 return -EINVAL;
2608 }
2609
2610 new_key_string = kstrdup(key_string, GFP_KERNEL);
2611 if (!new_key_string)
2612 return -ENOMEM;
2613
2614 key = request_key(type, key_desc + 1, NULL);
2615 if (IS_ERR(key)) {
2616 kfree_sensitive(new_key_string);
2617 return PTR_ERR(key);
2618 }
2619
2620 down_read(&key->sem);
2621
2622 ret = set_key(cc, key);
2623 if (ret < 0) {
2624 up_read(&key->sem);
2625 key_put(key);
2626 kfree_sensitive(new_key_string);
2627 return ret;
2628 }
2629
2630 up_read(&key->sem);
2631 key_put(key);
2632
2633 /* clear the flag since following operations may invalidate previously valid key */
2634 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2635
2636 ret = crypt_setkey(cc);
2637
2638 if (!ret) {
2639 set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2640 kfree_sensitive(cc->key_string);
2641 cc->key_string = new_key_string;
2642 } else
2643 kfree_sensitive(new_key_string);
2644
2645 return ret;
2646 }
2647
get_key_size(char ** key_string)2648 static int get_key_size(char **key_string)
2649 {
2650 char *colon, dummy;
2651 int ret;
2652
2653 if (*key_string[0] != ':')
2654 return strlen(*key_string) >> 1;
2655
2656 /* look for next ':' in key string */
2657 colon = strpbrk(*key_string + 1, ":");
2658 if (!colon)
2659 return -EINVAL;
2660
2661 if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')
2662 return -EINVAL;
2663
2664 *key_string = colon;
2665
2666 /* remaining key string should be :<logon|user>:<key_desc> */
2667
2668 return ret;
2669 }
2670
2671 #else
2672
crypt_set_keyring_key(struct crypt_config * cc,const char * key_string)2673 static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2674 {
2675 return -EINVAL;
2676 }
2677
get_key_size(char ** key_string)2678 static int get_key_size(char **key_string)
2679 {
2680 return (*key_string[0] == ':') ? -EINVAL : (int)(strlen(*key_string) >> 1);
2681 }
2682
2683 #endif /* CONFIG_KEYS */
2684
crypt_set_key(struct crypt_config * cc,char * key)2685 static int crypt_set_key(struct crypt_config *cc, char *key)
2686 {
2687 int r = -EINVAL;
2688 int key_string_len = strlen(key);
2689
2690 /* Hyphen (which gives a key_size of zero) means there is no key. */
2691 if (!cc->key_size && strcmp(key, "-"))
2692 goto out;
2693
2694 /* ':' means the key is in kernel keyring, short-circuit normal key processing */
2695 if (key[0] == ':') {
2696 r = crypt_set_keyring_key(cc, key + 1);
2697 goto out;
2698 }
2699
2700 /* clear the flag since following operations may invalidate previously valid key */
2701 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2702
2703 /* wipe references to any kernel keyring key */
2704 kfree_sensitive(cc->key_string);
2705 cc->key_string = NULL;
2706
2707 /* Decode key from its hex representation. */
2708 if (cc->key_size && hex2bin(cc->key, key, cc->key_size) < 0)
2709 goto out;
2710
2711 r = crypt_setkey(cc);
2712 if (!r)
2713 set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2714
2715 out:
2716 /* Hex key string not needed after here, so wipe it. */
2717 memset(key, '0', key_string_len);
2718
2719 return r;
2720 }
2721
crypt_wipe_key(struct crypt_config * cc)2722 static int crypt_wipe_key(struct crypt_config *cc)
2723 {
2724 int r;
2725
2726 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2727 get_random_bytes(&cc->key, cc->key_size);
2728
2729 /* Wipe IV private keys */
2730 if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
2731 r = cc->iv_gen_ops->wipe(cc);
2732 if (r)
2733 return r;
2734 }
2735
2736 kfree_sensitive(cc->key_string);
2737 cc->key_string = NULL;
2738 r = crypt_setkey(cc);
2739 memset(&cc->key, 0, cc->key_size * sizeof(u8));
2740
2741 return r;
2742 }
2743
crypt_calculate_pages_per_client(void)2744 static void crypt_calculate_pages_per_client(void)
2745 {
2746 unsigned long pages = (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT / 100;
2747
2748 if (!dm_crypt_clients_n)
2749 return;
2750
2751 pages /= dm_crypt_clients_n;
2752 if (pages < DM_CRYPT_MIN_PAGES_PER_CLIENT)
2753 pages = DM_CRYPT_MIN_PAGES_PER_CLIENT;
2754 dm_crypt_pages_per_client = pages;
2755 }
2756
crypt_page_alloc(gfp_t gfp_mask,void * pool_data)2757 static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data)
2758 {
2759 struct crypt_config *cc = pool_data;
2760 struct page *page;
2761
2762 /*
2763 * Note, percpu_counter_read_positive() may over (and under) estimate
2764 * the current usage by at most (batch - 1) * num_online_cpus() pages,
2765 * but avoids potential spinlock contention of an exact result.
2766 */
2767 if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) >= dm_crypt_pages_per_client) &&
2768 likely(gfp_mask & __GFP_NORETRY))
2769 return NULL;
2770
2771 page = alloc_page(gfp_mask);
2772 if (likely(page != NULL))
2773 percpu_counter_add(&cc->n_allocated_pages, 1);
2774
2775 return page;
2776 }
2777
crypt_page_free(void * page,void * pool_data)2778 static void crypt_page_free(void *page, void *pool_data)
2779 {
2780 struct crypt_config *cc = pool_data;
2781
2782 __free_page(page);
2783 percpu_counter_sub(&cc->n_allocated_pages, 1);
2784 }
2785
crypt_dtr(struct dm_target * ti)2786 static void crypt_dtr(struct dm_target *ti)
2787 {
2788 struct crypt_config *cc = ti->private;
2789
2790 ti->private = NULL;
2791
2792 if (!cc)
2793 return;
2794
2795 if (cc->write_thread)
2796 kthread_stop(cc->write_thread);
2797
2798 if (cc->io_queue)
2799 destroy_workqueue(cc->io_queue);
2800 if (cc->crypt_queue)
2801 destroy_workqueue(cc->crypt_queue);
2802
2803 if (cc->workqueue_id)
2804 ida_free(&workqueue_ida, cc->workqueue_id);
2805
2806 crypt_free_tfms(cc);
2807
2808 bioset_exit(&cc->bs);
2809
2810 mempool_exit(&cc->page_pool);
2811 mempool_exit(&cc->req_pool);
2812 mempool_exit(&cc->tag_pool);
2813
2814 WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0);
2815 percpu_counter_destroy(&cc->n_allocated_pages);
2816
2817 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
2818 cc->iv_gen_ops->dtr(cc);
2819
2820 if (cc->dev)
2821 dm_put_device(ti, cc->dev);
2822
2823 kfree_sensitive(cc->cipher_string);
2824 kfree_sensitive(cc->key_string);
2825 kfree_sensitive(cc->cipher_auth);
2826 kfree_sensitive(cc->authenc_key);
2827
2828 mutex_destroy(&cc->bio_alloc_lock);
2829
2830 /* Must zero key material before freeing */
2831 kfree_sensitive(cc);
2832
2833 spin_lock(&dm_crypt_clients_lock);
2834 WARN_ON(!dm_crypt_clients_n);
2835 dm_crypt_clients_n--;
2836 crypt_calculate_pages_per_client();
2837 spin_unlock(&dm_crypt_clients_lock);
2838
2839 dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);
2840 }
2841
crypt_ctr_ivmode(struct dm_target * ti,const char * ivmode)2842 static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
2843 {
2844 struct crypt_config *cc = ti->private;
2845
2846 if (crypt_integrity_aead(cc))
2847 cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
2848 else
2849 cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
2850
2851 if (cc->iv_size)
2852 /* at least a 64 bit sector number should fit in our buffer */
2853 cc->iv_size = max(cc->iv_size,
2854 (unsigned int)(sizeof(u64) / sizeof(u8)));
2855 else if (ivmode) {
2856 DMWARN("Selected cipher does not support IVs");
2857 ivmode = NULL;
2858 }
2859
2860 /* Choose ivmode, see comments at iv code. */
2861 if (ivmode == NULL)
2862 cc->iv_gen_ops = NULL;
2863 else if (strcmp(ivmode, "plain") == 0)
2864 cc->iv_gen_ops = &crypt_iv_plain_ops;
2865 else if (strcmp(ivmode, "plain64") == 0)
2866 cc->iv_gen_ops = &crypt_iv_plain64_ops;
2867 else if (strcmp(ivmode, "plain64be") == 0)
2868 cc->iv_gen_ops = &crypt_iv_plain64be_ops;
2869 else if (strcmp(ivmode, "essiv") == 0)
2870 cc->iv_gen_ops = &crypt_iv_essiv_ops;
2871 else if (strcmp(ivmode, "benbi") == 0)
2872 cc->iv_gen_ops = &crypt_iv_benbi_ops;
2873 else if (strcmp(ivmode, "null") == 0)
2874 cc->iv_gen_ops = &crypt_iv_null_ops;
2875 else if (strcmp(ivmode, "eboiv") == 0)
2876 cc->iv_gen_ops = &crypt_iv_eboiv_ops;
2877 else if (strcmp(ivmode, "elephant") == 0) {
2878 cc->iv_gen_ops = &crypt_iv_elephant_ops;
2879 cc->key_parts = 2;
2880 cc->key_extra_size = cc->key_size / 2;
2881 if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE)
2882 return -EINVAL;
2883 set_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags);
2884 } else if (strcmp(ivmode, "lmk") == 0) {
2885 cc->iv_gen_ops = &crypt_iv_lmk_ops;
2886 /*
2887 * Version 2 and 3 is recognised according
2888 * to length of provided multi-key string.
2889 * If present (version 3), last key is used as IV seed.
2890 * All keys (including IV seed) are always the same size.
2891 */
2892 if (cc->key_size % cc->key_parts) {
2893 cc->key_parts++;
2894 cc->key_extra_size = cc->key_size / cc->key_parts;
2895 }
2896 } else if (strcmp(ivmode, "tcw") == 0) {
2897 cc->iv_gen_ops = &crypt_iv_tcw_ops;
2898 cc->key_parts += 2; /* IV + whitening */
2899 cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
2900 } else if (strcmp(ivmode, "random") == 0) {
2901 cc->iv_gen_ops = &crypt_iv_random_ops;
2902 /* Need storage space in integrity fields. */
2903 cc->integrity_iv_size = cc->iv_size;
2904 } else {
2905 ti->error = "Invalid IV mode";
2906 return -EINVAL;
2907 }
2908
2909 return 0;
2910 }
2911
2912 /*
2913 * Workaround to parse HMAC algorithm from AEAD crypto API spec.
2914 * The HMAC is needed to calculate tag size (HMAC digest size).
2915 * This should be probably done by crypto-api calls (once available...)
2916 */
crypt_ctr_auth_cipher(struct crypt_config * cc,char * cipher_api)2917 static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api)
2918 {
2919 char *start, *end, *mac_alg = NULL;
2920 struct crypto_ahash *mac;
2921
2922 if (!strstarts(cipher_api, "authenc("))
2923 return 0;
2924
2925 start = strchr(cipher_api, '(');
2926 end = strchr(cipher_api, ',');
2927 if (!start || !end || ++start > end)
2928 return -EINVAL;
2929
2930 mac_alg = kmemdup_nul(start, end - start, GFP_KERNEL);
2931 if (!mac_alg)
2932 return -ENOMEM;
2933
2934 mac = crypto_alloc_ahash(mac_alg, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
2935 kfree(mac_alg);
2936
2937 if (IS_ERR(mac))
2938 return PTR_ERR(mac);
2939
2940 cc->key_mac_size = crypto_ahash_digestsize(mac);
2941 crypto_free_ahash(mac);
2942
2943 cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL);
2944 if (!cc->authenc_key)
2945 return -ENOMEM;
2946
2947 return 0;
2948 }
2949
crypt_ctr_cipher_new(struct dm_target * ti,char * cipher_in,char * key,char ** ivmode,char ** ivopts)2950 static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key,
2951 char **ivmode, char **ivopts)
2952 {
2953 struct crypt_config *cc = ti->private;
2954 char *tmp, *cipher_api, buf[CRYPTO_MAX_ALG_NAME];
2955 int ret = -EINVAL;
2956
2957 cc->tfms_count = 1;
2958
2959 /*
2960 * New format (capi: prefix)
2961 * capi:cipher_api_spec-iv:ivopts
2962 */
2963 tmp = &cipher_in[strlen("capi:")];
2964
2965 /* Separate IV options if present, it can contain another '-' in hash name */
2966 *ivopts = strrchr(tmp, ':');
2967 if (*ivopts) {
2968 **ivopts = '\0';
2969 (*ivopts)++;
2970 }
2971 /* Parse IV mode */
2972 *ivmode = strrchr(tmp, '-');
2973 if (*ivmode) {
2974 **ivmode = '\0';
2975 (*ivmode)++;
2976 }
2977 /* The rest is crypto API spec */
2978 cipher_api = tmp;
2979
2980 /* Alloc AEAD, can be used only in new format. */
2981 if (crypt_integrity_aead(cc)) {
2982 ret = crypt_ctr_auth_cipher(cc, cipher_api);
2983 if (ret < 0) {
2984 ti->error = "Invalid AEAD cipher spec";
2985 return ret;
2986 }
2987 }
2988
2989 if (*ivmode && !strcmp(*ivmode, "lmk"))
2990 cc->tfms_count = 64;
2991
2992 if (*ivmode && !strcmp(*ivmode, "essiv")) {
2993 if (!*ivopts) {
2994 ti->error = "Digest algorithm missing for ESSIV mode";
2995 return -EINVAL;
2996 }
2997 ret = snprintf(buf, CRYPTO_MAX_ALG_NAME, "essiv(%s,%s)",
2998 cipher_api, *ivopts);
2999 if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
3000 ti->error = "Cannot allocate cipher string";
3001 return -ENOMEM;
3002 }
3003 cipher_api = buf;
3004 }
3005
3006 cc->key_parts = cc->tfms_count;
3007
3008 /* Allocate cipher */
3009 ret = crypt_alloc_tfms(cc, cipher_api);
3010 if (ret < 0) {
3011 ti->error = "Error allocating crypto tfm";
3012 return ret;
3013 }
3014
3015 if (crypt_integrity_aead(cc))
3016 cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
3017 else
3018 cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
3019
3020 return 0;
3021 }
3022
crypt_ctr_cipher_old(struct dm_target * ti,char * cipher_in,char * key,char ** ivmode,char ** ivopts)3023 static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key,
3024 char **ivmode, char **ivopts)
3025 {
3026 struct crypt_config *cc = ti->private;
3027 char *tmp, *cipher, *chainmode, *keycount;
3028 char *cipher_api = NULL;
3029 int ret = -EINVAL;
3030 char dummy;
3031
3032 if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) {
3033 ti->error = "Bad cipher specification";
3034 return -EINVAL;
3035 }
3036
3037 /*
3038 * Legacy dm-crypt cipher specification
3039 * cipher[:keycount]-mode-iv:ivopts
3040 */
3041 tmp = cipher_in;
3042 keycount = strsep(&tmp, "-");
3043 cipher = strsep(&keycount, ":");
3044
3045 if (!keycount)
3046 cc->tfms_count = 1;
3047 else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 ||
3048 !is_power_of_2(cc->tfms_count)) {
3049 ti->error = "Bad cipher key count specification";
3050 return -EINVAL;
3051 }
3052 cc->key_parts = cc->tfms_count;
3053
3054 chainmode = strsep(&tmp, "-");
3055 *ivmode = strsep(&tmp, ":");
3056 *ivopts = tmp;
3057
3058 /*
3059 * For compatibility with the original dm-crypt mapping format, if
3060 * only the cipher name is supplied, use cbc-plain.
3061 */
3062 if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) {
3063 chainmode = "cbc";
3064 *ivmode = "plain";
3065 }
3066
3067 if (strcmp(chainmode, "ecb") && !*ivmode) {
3068 ti->error = "IV mechanism required";
3069 return -EINVAL;
3070 }
3071
3072 cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
3073 if (!cipher_api)
3074 goto bad_mem;
3075
3076 if (*ivmode && !strcmp(*ivmode, "essiv")) {
3077 if (!*ivopts) {
3078 ti->error = "Digest algorithm missing for ESSIV mode";
3079 kfree(cipher_api);
3080 return -EINVAL;
3081 }
3082 ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
3083 "essiv(%s(%s),%s)", chainmode, cipher, *ivopts);
3084 } else {
3085 ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
3086 "%s(%s)", chainmode, cipher);
3087 }
3088 if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
3089 kfree(cipher_api);
3090 goto bad_mem;
3091 }
3092
3093 /* Allocate cipher */
3094 ret = crypt_alloc_tfms(cc, cipher_api);
3095 if (ret < 0) {
3096 ti->error = "Error allocating crypto tfm";
3097 kfree(cipher_api);
3098 return ret;
3099 }
3100 kfree(cipher_api);
3101
3102 return 0;
3103 bad_mem:
3104 ti->error = "Cannot allocate cipher strings";
3105 return -ENOMEM;
3106 }
3107
crypt_ctr_cipher(struct dm_target * ti,char * cipher_in,char * key)3108 static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)
3109 {
3110 struct crypt_config *cc = ti->private;
3111 char *ivmode = NULL, *ivopts = NULL;
3112 int ret;
3113
3114 cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL);
3115 if (!cc->cipher_string) {
3116 ti->error = "Cannot allocate cipher strings";
3117 return -ENOMEM;
3118 }
3119
3120 if (strstarts(cipher_in, "capi:"))
3121 ret = crypt_ctr_cipher_new(ti, cipher_in, key, &ivmode, &ivopts);
3122 else
3123 ret = crypt_ctr_cipher_old(ti, cipher_in, key, &ivmode, &ivopts);
3124 if (ret)
3125 return ret;
3126
3127 /* Initialize IV */
3128 ret = crypt_ctr_ivmode(ti, ivmode);
3129 if (ret < 0)
3130 return ret;
3131
3132 /* Initialize and set key */
3133 ret = crypt_set_key(cc, key);
3134 if (ret < 0) {
3135 ti->error = "Error decoding and setting key";
3136 return ret;
3137 }
3138
3139 /* Allocate IV */
3140 if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
3141 ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
3142 if (ret < 0) {
3143 ti->error = "Error creating IV";
3144 return ret;
3145 }
3146 }
3147
3148 /* Initialize IV (set keys for ESSIV etc) */
3149 if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
3150 ret = cc->iv_gen_ops->init(cc);
3151 if (ret < 0) {
3152 ti->error = "Error initialising IV";
3153 return ret;
3154 }
3155 }
3156
3157 /* wipe the kernel key payload copy */
3158 if (cc->key_string)
3159 memset(cc->key, 0, cc->key_size * sizeof(u8));
3160
3161 return ret;
3162 }
3163
crypt_ctr_optional(struct dm_target * ti,unsigned int argc,char ** argv)3164 static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv)
3165 {
3166 struct crypt_config *cc = ti->private;
3167 struct dm_arg_set as;
3168 static const struct dm_arg _args[] = {
3169 {0, 9, "Invalid number of feature args"},
3170 };
3171 unsigned int opt_params, val;
3172 const char *opt_string, *sval;
3173 char dummy;
3174 int ret;
3175
3176 /* Optional parameters */
3177 as.argc = argc;
3178 as.argv = argv;
3179
3180 ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
3181 if (ret)
3182 return ret;
3183
3184 while (opt_params--) {
3185 opt_string = dm_shift_arg(&as);
3186 if (!opt_string) {
3187 ti->error = "Not enough feature arguments";
3188 return -EINVAL;
3189 }
3190
3191 if (!strcasecmp(opt_string, "allow_discards"))
3192 ti->num_discard_bios = 1;
3193
3194 else if (!strcasecmp(opt_string, "same_cpu_crypt"))
3195 set_bit(DM_CRYPT_SAME_CPU, &cc->flags);
3196 else if (!strcasecmp(opt_string, "high_priority"))
3197 set_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags);
3198
3199 else if (!strcasecmp(opt_string, "submit_from_crypt_cpus"))
3200 set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
3201 else if (!strcasecmp(opt_string, "no_read_workqueue"))
3202 set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);
3203 else if (!strcasecmp(opt_string, "no_write_workqueue"))
3204 set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
3205 else if (sscanf(opt_string, "integrity:%u:", &val) == 1) {
3206 if (val == 0 || val > MAX_TAG_SIZE) {
3207 ti->error = "Invalid integrity arguments";
3208 return -EINVAL;
3209 }
3210 cc->used_tag_size = val;
3211 sval = strchr(opt_string + strlen("integrity:"), ':') + 1;
3212 if (!strcasecmp(sval, "aead")) {
3213 set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
3214 } else if (strcasecmp(sval, "none")) {
3215 ti->error = "Unknown integrity profile";
3216 return -EINVAL;
3217 }
3218
3219 cc->cipher_auth = kstrdup(sval, GFP_KERNEL);
3220 if (!cc->cipher_auth)
3221 return -ENOMEM;
3222 } else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) {
3223 if (cc->sector_size < (1 << SECTOR_SHIFT) ||
3224 cc->sector_size > 4096 ||
3225 (cc->sector_size & (cc->sector_size - 1))) {
3226 ti->error = "Invalid feature value for sector_size";
3227 return -EINVAL;
3228 }
3229 if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) {
3230 ti->error = "Device size is not multiple of sector_size feature";
3231 return -EINVAL;
3232 }
3233 cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT;
3234 } else if (!strcasecmp(opt_string, "iv_large_sectors"))
3235 set_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
3236 else {
3237 ti->error = "Invalid feature arguments";
3238 return -EINVAL;
3239 }
3240 }
3241
3242 return 0;
3243 }
3244
3245 #ifdef CONFIG_BLK_DEV_ZONED
crypt_report_zones(struct dm_target * ti,struct dm_report_zones_args * args,unsigned int nr_zones)3246 static int crypt_report_zones(struct dm_target *ti,
3247 struct dm_report_zones_args *args, unsigned int nr_zones)
3248 {
3249 struct crypt_config *cc = ti->private;
3250
3251 return dm_report_zones(cc->dev->bdev, cc->start,
3252 cc->start + dm_target_offset(ti, args->next_sector),
3253 args, nr_zones);
3254 }
3255 #else
3256 #define crypt_report_zones NULL
3257 #endif
3258
3259 /*
3260 * Construct an encryption mapping:
3261 * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start>
3262 */
crypt_ctr(struct dm_target * ti,unsigned int argc,char ** argv)3263 static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
3264 {
3265 struct crypt_config *cc;
3266 const char *devname = dm_table_device_name(ti->table);
3267 int key_size, wq_id;
3268 unsigned int align_mask;
3269 unsigned int common_wq_flags;
3270 unsigned long long tmpll;
3271 int ret;
3272 size_t iv_size_padding, additional_req_size;
3273 char dummy;
3274
3275 if (argc < 5) {
3276 ti->error = "Not enough arguments";
3277 return -EINVAL;
3278 }
3279
3280 key_size = get_key_size(&argv[1]);
3281 if (key_size < 0) {
3282 ti->error = "Cannot parse key size";
3283 return -EINVAL;
3284 }
3285
3286 cc = kzalloc(struct_size(cc, key, key_size), GFP_KERNEL);
3287 if (!cc) {
3288 ti->error = "Cannot allocate encryption context";
3289 return -ENOMEM;
3290 }
3291 cc->key_size = key_size;
3292 cc->sector_size = (1 << SECTOR_SHIFT);
3293 cc->sector_shift = 0;
3294
3295 ti->private = cc;
3296
3297 spin_lock(&dm_crypt_clients_lock);
3298 dm_crypt_clients_n++;
3299 crypt_calculate_pages_per_client();
3300 spin_unlock(&dm_crypt_clients_lock);
3301
3302 ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL);
3303 if (ret < 0)
3304 goto bad;
3305
3306 /* Optional parameters need to be read before cipher constructor */
3307 if (argc > 5) {
3308 ret = crypt_ctr_optional(ti, argc - 5, &argv[5]);
3309 if (ret)
3310 goto bad;
3311 }
3312
3313 ret = crypt_ctr_cipher(ti, argv[0], argv[1]);
3314 if (ret < 0)
3315 goto bad;
3316
3317 if (crypt_integrity_aead(cc)) {
3318 cc->dmreq_start = sizeof(struct aead_request);
3319 cc->dmreq_start += crypto_aead_reqsize(any_tfm_aead(cc));
3320 align_mask = crypto_aead_alignmask(any_tfm_aead(cc));
3321 } else {
3322 cc->dmreq_start = sizeof(struct skcipher_request);
3323 cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc));
3324 align_mask = crypto_skcipher_alignmask(any_tfm(cc));
3325 }
3326 cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request));
3327
3328 if (align_mask < CRYPTO_MINALIGN) {
3329 /* Allocate the padding exactly */
3330 iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request))
3331 & align_mask;
3332 } else {
3333 /*
3334 * If the cipher requires greater alignment than kmalloc
3335 * alignment, we don't know the exact position of the
3336 * initialization vector. We must assume worst case.
3337 */
3338 iv_size_padding = align_mask;
3339 }
3340
3341 /* ...| IV + padding | original IV | original sec. number | bio tag offset | */
3342 additional_req_size = sizeof(struct dm_crypt_request) +
3343 iv_size_padding + cc->iv_size +
3344 cc->iv_size +
3345 sizeof(uint64_t) +
3346 sizeof(unsigned int);
3347
3348 ret = mempool_init_kmalloc_pool(&cc->req_pool, MIN_IOS, cc->dmreq_start + additional_req_size);
3349 if (ret) {
3350 ti->error = "Cannot allocate crypt request mempool";
3351 goto bad;
3352 }
3353
3354 cc->per_bio_data_size = ti->per_io_data_size =
3355 ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size,
3356 ARCH_DMA_MINALIGN);
3357
3358 ret = mempool_init(&cc->page_pool, BIO_MAX_VECS, crypt_page_alloc, crypt_page_free, cc);
3359 if (ret) {
3360 ti->error = "Cannot allocate page mempool";
3361 goto bad;
3362 }
3363
3364 ret = bioset_init(&cc->bs, MIN_IOS, 0, BIOSET_NEED_BVECS);
3365 if (ret) {
3366 ti->error = "Cannot allocate crypt bioset";
3367 goto bad;
3368 }
3369
3370 mutex_init(&cc->bio_alloc_lock);
3371
3372 ret = -EINVAL;
3373 if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) ||
3374 (tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) {
3375 ti->error = "Invalid iv_offset sector";
3376 goto bad;
3377 }
3378 cc->iv_offset = tmpll;
3379
3380 ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev);
3381 if (ret) {
3382 ti->error = "Device lookup failed";
3383 goto bad;
3384 }
3385
3386 ret = -EINVAL;
3387 if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || tmpll != (sector_t)tmpll) {
3388 ti->error = "Invalid device sector";
3389 goto bad;
3390 }
3391 cc->start = tmpll;
3392
3393 if (bdev_is_zoned(cc->dev->bdev)) {
3394 /*
3395 * For zoned block devices, we need to preserve the issuer write
3396 * ordering. To do so, disable write workqueues and force inline
3397 * encryption completion.
3398 */
3399 set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
3400 set_bit(DM_CRYPT_WRITE_INLINE, &cc->flags);
3401
3402 /*
3403 * All zone append writes to a zone of a zoned block device will
3404 * have the same BIO sector, the start of the zone. When the
3405 * cypher IV mode uses sector values, all data targeting a
3406 * zone will be encrypted using the first sector numbers of the
3407 * zone. This will not result in write errors but will
3408 * cause most reads to fail as reads will use the sector values
3409 * for the actual data locations, resulting in IV mismatch.
3410 * To avoid this problem, ask DM core to emulate zone append
3411 * operations with regular writes.
3412 */
3413 DMDEBUG("Zone append operations will be emulated");
3414 ti->emulate_zone_append = true;
3415 }
3416
3417 if (crypt_integrity_aead(cc) || cc->integrity_iv_size) {
3418 ret = crypt_integrity_ctr(cc, ti);
3419 if (ret)
3420 goto bad;
3421
3422 cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->tuple_size;
3423 if (!cc->tag_pool_max_sectors)
3424 cc->tag_pool_max_sectors = 1;
3425
3426 ret = mempool_init_kmalloc_pool(&cc->tag_pool, MIN_IOS,
3427 cc->tag_pool_max_sectors * cc->tuple_size);
3428 if (ret) {
3429 ti->error = "Cannot allocate integrity tags mempool";
3430 goto bad;
3431 }
3432
3433 cc->tag_pool_max_sectors <<= cc->sector_shift;
3434 }
3435
3436 wq_id = ida_alloc_min(&workqueue_ida, 1, GFP_KERNEL);
3437 if (wq_id < 0) {
3438 ti->error = "Couldn't get workqueue id";
3439 ret = wq_id;
3440 goto bad;
3441 }
3442 cc->workqueue_id = wq_id;
3443
3444 ret = -ENOMEM;
3445 common_wq_flags = WQ_MEM_RECLAIM | WQ_SYSFS;
3446 if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags))
3447 common_wq_flags |= WQ_HIGHPRI;
3448
3449 cc->io_queue = alloc_workqueue("kcryptd_io-%s-%d", common_wq_flags, 1, devname, wq_id);
3450 if (!cc->io_queue) {
3451 ti->error = "Couldn't create kcryptd io queue";
3452 goto bad;
3453 }
3454
3455 if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) {
3456 cc->crypt_queue = alloc_workqueue("kcryptd-%s-%d",
3457 common_wq_flags | WQ_CPU_INTENSIVE,
3458 1, devname, wq_id);
3459 } else {
3460 /*
3461 * While crypt_queue is certainly CPU intensive, the use of
3462 * WQ_CPU_INTENSIVE is meaningless with WQ_UNBOUND.
3463 */
3464 cc->crypt_queue = alloc_workqueue("kcryptd-%s-%d",
3465 common_wq_flags | WQ_UNBOUND,
3466 num_online_cpus(), devname, wq_id);
3467 }
3468 if (!cc->crypt_queue) {
3469 ti->error = "Couldn't create kcryptd queue";
3470 goto bad;
3471 }
3472
3473 spin_lock_init(&cc->write_thread_lock);
3474 cc->write_tree = RB_ROOT;
3475
3476 cc->write_thread = kthread_run(dmcrypt_write, cc, "dmcrypt_write/%s", devname);
3477 if (IS_ERR(cc->write_thread)) {
3478 ret = PTR_ERR(cc->write_thread);
3479 cc->write_thread = NULL;
3480 ti->error = "Couldn't spawn write thread";
3481 goto bad;
3482 }
3483 if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags))
3484 set_user_nice(cc->write_thread, MIN_NICE);
3485
3486 ti->num_flush_bios = 1;
3487 ti->limit_swap_bios = true;
3488 ti->accounts_remapped_io = true;
3489
3490 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1);
3491 return 0;
3492
3493 bad:
3494 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0);
3495 crypt_dtr(ti);
3496 return ret;
3497 }
3498
crypt_map(struct dm_target * ti,struct bio * bio)3499 static int crypt_map(struct dm_target *ti, struct bio *bio)
3500 {
3501 struct dm_crypt_io *io;
3502 struct crypt_config *cc = ti->private;
3503 unsigned max_sectors;
3504
3505 /*
3506 * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.
3507 * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight
3508 * - for REQ_OP_DISCARD caller must use flush if IO ordering matters
3509 */
3510 if (unlikely(bio->bi_opf & REQ_PREFLUSH ||
3511 bio_op(bio) == REQ_OP_DISCARD)) {
3512 bio_set_dev(bio, cc->dev->bdev);
3513 if (bio_sectors(bio))
3514 bio->bi_iter.bi_sector = cc->start +
3515 dm_target_offset(ti, bio->bi_iter.bi_sector);
3516 return DM_MAPIO_REMAPPED;
3517 }
3518
3519 /*
3520 * Check if bio is too large, split as needed.
3521 */
3522 max_sectors = get_max_request_size(cc, bio_data_dir(bio) == WRITE);
3523 if (unlikely(bio_sectors(bio) > max_sectors))
3524 dm_accept_partial_bio(bio, max_sectors);
3525
3526 /*
3527 * Ensure that bio is a multiple of internal sector encryption size
3528 * and is aligned to this size as defined in IO hints.
3529 */
3530 if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0))
3531 return DM_MAPIO_KILL;
3532
3533 if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1)))
3534 return DM_MAPIO_KILL;
3535
3536 io = dm_per_bio_data(bio, cc->per_bio_data_size);
3537 crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
3538
3539 if (cc->tuple_size) {
3540 unsigned int tag_len = cc->tuple_size * (bio_sectors(bio) >> cc->sector_shift);
3541
3542 if (unlikely(tag_len > KMALLOC_MAX_SIZE))
3543 io->integrity_metadata = NULL;
3544 else
3545 io->integrity_metadata = kmalloc(tag_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
3546
3547 if (unlikely(!io->integrity_metadata)) {
3548 if (bio_sectors(bio) > cc->tag_pool_max_sectors)
3549 dm_accept_partial_bio(bio, cc->tag_pool_max_sectors);
3550 io->integrity_metadata = mempool_alloc(&cc->tag_pool, GFP_NOIO);
3551 io->integrity_metadata_from_pool = true;
3552 }
3553 }
3554
3555 if (crypt_integrity_aead(cc))
3556 io->ctx.r.req_aead = (struct aead_request *)(io + 1);
3557 else
3558 io->ctx.r.req = (struct skcipher_request *)(io + 1);
3559
3560 if (bio_data_dir(io->base_bio) == READ) {
3561 if (kcryptd_io_read(io, CRYPT_MAP_READ_GFP))
3562 kcryptd_queue_read(io);
3563 } else
3564 kcryptd_queue_crypt(io);
3565
3566 return DM_MAPIO_SUBMITTED;
3567 }
3568
hex2asc(unsigned char c)3569 static char hex2asc(unsigned char c)
3570 {
3571 return c + '0' + ((unsigned int)(9 - c) >> 4 & 0x27);
3572 }
3573
crypt_status(struct dm_target * ti,status_type_t type,unsigned int status_flags,char * result,unsigned int maxlen)3574 static void crypt_status(struct dm_target *ti, status_type_t type,
3575 unsigned int status_flags, char *result, unsigned int maxlen)
3576 {
3577 struct crypt_config *cc = ti->private;
3578 unsigned int i, sz = 0;
3579 int num_feature_args = 0;
3580
3581 switch (type) {
3582 case STATUSTYPE_INFO:
3583 result[0] = '\0';
3584 break;
3585
3586 case STATUSTYPE_TABLE:
3587 DMEMIT("%s ", cc->cipher_string);
3588
3589 if (cc->key_size > 0) {
3590 if (cc->key_string)
3591 DMEMIT(":%u:%s", cc->key_size, cc->key_string);
3592 else {
3593 for (i = 0; i < cc->key_size; i++) {
3594 DMEMIT("%c%c", hex2asc(cc->key[i] >> 4),
3595 hex2asc(cc->key[i] & 0xf));
3596 }
3597 }
3598 } else
3599 DMEMIT("-");
3600
3601 DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
3602 cc->dev->name, (unsigned long long)cc->start);
3603
3604 num_feature_args += !!ti->num_discard_bios;
3605 num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags);
3606 num_feature_args += test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags);
3607 num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
3608 num_feature_args += test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);
3609 num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
3610 num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT);
3611 num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
3612 if (cc->used_tag_size)
3613 num_feature_args++;
3614 if (num_feature_args) {
3615 DMEMIT(" %d", num_feature_args);
3616 if (ti->num_discard_bios)
3617 DMEMIT(" allow_discards");
3618 if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
3619 DMEMIT(" same_cpu_crypt");
3620 if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags))
3621 DMEMIT(" high_priority");
3622 if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags))
3623 DMEMIT(" submit_from_crypt_cpus");
3624 if (test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags))
3625 DMEMIT(" no_read_workqueue");
3626 if (test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))
3627 DMEMIT(" no_write_workqueue");
3628 if (cc->used_tag_size)
3629 DMEMIT(" integrity:%u:%s", cc->used_tag_size, cc->cipher_auth);
3630 if (cc->sector_size != (1 << SECTOR_SHIFT))
3631 DMEMIT(" sector_size:%d", cc->sector_size);
3632 if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
3633 DMEMIT(" iv_large_sectors");
3634 }
3635 break;
3636
3637 case STATUSTYPE_IMA:
3638 DMEMIT_TARGET_NAME_VERSION(ti->type);
3639 DMEMIT(",allow_discards=%c", ti->num_discard_bios ? 'y' : 'n');
3640 DMEMIT(",same_cpu_crypt=%c", test_bit(DM_CRYPT_SAME_CPU, &cc->flags) ? 'y' : 'n');
3641 DMEMIT(",high_priority=%c", test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags) ? 'y' : 'n');
3642 DMEMIT(",submit_from_crypt_cpus=%c", test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags) ?
3643 'y' : 'n');
3644 DMEMIT(",no_read_workqueue=%c", test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags) ?
3645 'y' : 'n');
3646 DMEMIT(",no_write_workqueue=%c", test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags) ?
3647 'y' : 'n');
3648 DMEMIT(",iv_large_sectors=%c", test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags) ?
3649 'y' : 'n');
3650
3651 if (cc->used_tag_size)
3652 DMEMIT(",integrity_tag_size=%u,cipher_auth=%s",
3653 cc->used_tag_size, cc->cipher_auth);
3654 if (cc->sector_size != (1 << SECTOR_SHIFT))
3655 DMEMIT(",sector_size=%d", cc->sector_size);
3656 if (cc->cipher_string)
3657 DMEMIT(",cipher_string=%s", cc->cipher_string);
3658
3659 DMEMIT(",key_size=%u", cc->key_size);
3660 DMEMIT(",key_parts=%u", cc->key_parts);
3661 DMEMIT(",key_extra_size=%u", cc->key_extra_size);
3662 DMEMIT(",key_mac_size=%u", cc->key_mac_size);
3663 DMEMIT(";");
3664 break;
3665 }
3666 }
3667
crypt_postsuspend(struct dm_target * ti)3668 static void crypt_postsuspend(struct dm_target *ti)
3669 {
3670 struct crypt_config *cc = ti->private;
3671
3672 set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
3673 }
3674
crypt_preresume(struct dm_target * ti)3675 static int crypt_preresume(struct dm_target *ti)
3676 {
3677 struct crypt_config *cc = ti->private;
3678
3679 if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
3680 DMERR("aborting resume - crypt key is not set.");
3681 return -EAGAIN;
3682 }
3683
3684 return 0;
3685 }
3686
crypt_resume(struct dm_target * ti)3687 static void crypt_resume(struct dm_target *ti)
3688 {
3689 struct crypt_config *cc = ti->private;
3690
3691 clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
3692 }
3693
3694 /* Message interface
3695 * key set <key>
3696 * key wipe
3697 */
crypt_message(struct dm_target * ti,unsigned int argc,char ** argv,char * result,unsigned int maxlen)3698 static int crypt_message(struct dm_target *ti, unsigned int argc, char **argv,
3699 char *result, unsigned int maxlen)
3700 {
3701 struct crypt_config *cc = ti->private;
3702 int key_size, ret = -EINVAL;
3703
3704 if (argc < 2)
3705 goto error;
3706
3707 if (!strcasecmp(argv[0], "key")) {
3708 if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
3709 DMWARN("not suspended during key manipulation.");
3710 return -EINVAL;
3711 }
3712 if (argc == 3 && !strcasecmp(argv[1], "set")) {
3713 /* The key size may not be changed. */
3714 key_size = get_key_size(&argv[2]);
3715 if (key_size < 0 || cc->key_size != key_size) {
3716 memset(argv[2], '0', strlen(argv[2]));
3717 return -EINVAL;
3718 }
3719
3720 ret = crypt_set_key(cc, argv[2]);
3721 if (ret)
3722 return ret;
3723 if (cc->iv_gen_ops && cc->iv_gen_ops->init)
3724 ret = cc->iv_gen_ops->init(cc);
3725 /* wipe the kernel key payload copy */
3726 if (cc->key_string)
3727 memset(cc->key, 0, cc->key_size * sizeof(u8));
3728 return ret;
3729 }
3730 if (argc == 2 && !strcasecmp(argv[1], "wipe"))
3731 return crypt_wipe_key(cc);
3732 }
3733
3734 error:
3735 DMWARN("unrecognised message received.");
3736 return -EINVAL;
3737 }
3738
crypt_iterate_devices(struct dm_target * ti,iterate_devices_callout_fn fn,void * data)3739 static int crypt_iterate_devices(struct dm_target *ti,
3740 iterate_devices_callout_fn fn, void *data)
3741 {
3742 struct crypt_config *cc = ti->private;
3743
3744 return fn(ti, cc->dev, cc->start, ti->len, data);
3745 }
3746
crypt_io_hints(struct dm_target * ti,struct queue_limits * limits)3747 static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
3748 {
3749 struct crypt_config *cc = ti->private;
3750
3751 limits->logical_block_size =
3752 max_t(unsigned int, limits->logical_block_size, cc->sector_size);
3753 limits->physical_block_size =
3754 max_t(unsigned int, limits->physical_block_size, cc->sector_size);
3755 limits->io_min = max_t(unsigned int, limits->io_min, cc->sector_size);
3756 limits->dma_alignment = limits->logical_block_size - 1;
3757 }
3758
3759 static struct target_type crypt_target = {
3760 .name = "crypt",
3761 .version = {1, 27, 0},
3762 .module = THIS_MODULE,
3763 .ctr = crypt_ctr,
3764 .dtr = crypt_dtr,
3765 .features = DM_TARGET_ZONED_HM,
3766 .report_zones = crypt_report_zones,
3767 .map = crypt_map,
3768 .status = crypt_status,
3769 .postsuspend = crypt_postsuspend,
3770 .preresume = crypt_preresume,
3771 .resume = crypt_resume,
3772 .message = crypt_message,
3773 .iterate_devices = crypt_iterate_devices,
3774 .io_hints = crypt_io_hints,
3775 };
3776 module_dm(crypt);
3777
3778 MODULE_AUTHOR("Jana Saout <jana@saout.de>");
3779 MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
3780 MODULE_LICENSE("GPL");
3781