xref: /linux/drivers/md/dm-integrity.c (revision 3fd6c59042dbba50391e30862beac979491145fe)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved.
4  * Copyright (C) 2016-2017 Milan Broz
5  * Copyright (C) 2016-2017 Mikulas Patocka
6  *
7  * This file is released under the GPL.
8  */
9 
10 #include "dm-bio-record.h"
11 
12 #include <linux/compiler.h>
13 #include <linux/module.h>
14 #include <linux/device-mapper.h>
15 #include <linux/dm-io.h>
16 #include <linux/vmalloc.h>
17 #include <linux/sort.h>
18 #include <linux/rbtree.h>
19 #include <linux/delay.h>
20 #include <linux/random.h>
21 #include <linux/reboot.h>
22 #include <crypto/hash.h>
23 #include <crypto/skcipher.h>
24 #include <linux/async_tx.h>
25 #include <linux/dm-bufio.h>
26 
27 #include "dm-audit.h"
28 
29 #define DM_MSG_PREFIX "integrity"
30 
31 #define DEFAULT_INTERLEAVE_SECTORS	32768
32 #define DEFAULT_JOURNAL_SIZE_FACTOR	7
33 #define DEFAULT_SECTORS_PER_BITMAP_BIT	32768
34 #define DEFAULT_BUFFER_SECTORS		128
35 #define DEFAULT_JOURNAL_WATERMARK	50
36 #define DEFAULT_SYNC_MSEC		10000
37 #define DEFAULT_MAX_JOURNAL_SECTORS	(IS_ENABLED(CONFIG_64BIT) ? 131072 : 8192)
38 #define MIN_LOG2_INTERLEAVE_SECTORS	3
39 #define MAX_LOG2_INTERLEAVE_SECTORS	31
40 #define METADATA_WORKQUEUE_MAX_ACTIVE	16
41 #define RECALC_SECTORS			(IS_ENABLED(CONFIG_64BIT) ? 32768 : 2048)
42 #define RECALC_WRITE_SUPER		16
43 #define BITMAP_BLOCK_SIZE		4096	/* don't change it */
44 #define BITMAP_FLUSH_INTERVAL		(10 * HZ)
45 #define DISCARD_FILLER			0xf6
46 #define SALT_SIZE			16
47 #define RECHECK_POOL_SIZE		256
48 
49 /*
50  * Warning - DEBUG_PRINT prints security-sensitive data to the log,
51  * so it should not be enabled in the official kernel
52  */
53 //#define DEBUG_PRINT
54 //#define INTERNAL_VERIFY
55 
56 /*
57  * On disk structures
58  */
59 
60 #define SB_MAGIC			"integrt"
61 #define SB_VERSION_1			1
62 #define SB_VERSION_2			2
63 #define SB_VERSION_3			3
64 #define SB_VERSION_4			4
65 #define SB_VERSION_5			5
66 #define SB_VERSION_6			6
67 #define SB_SECTORS			8
68 #define MAX_SECTORS_PER_BLOCK		8
69 
70 struct superblock {
71 	__u8 magic[8];
72 	__u8 version;
73 	__u8 log2_interleave_sectors;
74 	__le16 integrity_tag_size;
75 	__le32 journal_sections;
76 	__le64 provided_data_sectors;	/* userspace uses this value */
77 	__le32 flags;
78 	__u8 log2_sectors_per_block;
79 	__u8 log2_blocks_per_bitmap_bit;
80 	__u8 pad[2];
81 	__le64 recalc_sector;
82 	__u8 pad2[8];
83 	__u8 salt[SALT_SIZE];
84 };
85 
86 #define SB_FLAG_HAVE_JOURNAL_MAC	0x1
87 #define SB_FLAG_RECALCULATING		0x2
88 #define SB_FLAG_DIRTY_BITMAP		0x4
89 #define SB_FLAG_FIXED_PADDING		0x8
90 #define SB_FLAG_FIXED_HMAC		0x10
91 #define SB_FLAG_INLINE			0x20
92 
93 #define	JOURNAL_ENTRY_ROUNDUP		8
94 
95 typedef __le64 commit_id_t;
96 #define JOURNAL_MAC_PER_SECTOR		8
97 
98 struct journal_entry {
99 	union {
100 		struct {
101 			__le32 sector_lo;
102 			__le32 sector_hi;
103 		} s;
104 		__le64 sector;
105 	} u;
106 	commit_id_t last_bytes[];
107 	/* __u8 tag[0]; */
108 };
109 
110 #define journal_entry_tag(ic, je)		((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block])
111 
112 #if BITS_PER_LONG == 64
113 #define journal_entry_set_sector(je, x)		do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0)
114 #else
115 #define journal_entry_set_sector(je, x)		do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32((x) >> 32)); } while (0)
116 #endif
117 #define journal_entry_get_sector(je)		le64_to_cpu((je)->u.sector)
118 #define journal_entry_is_unused(je)		((je)->u.s.sector_hi == cpu_to_le32(-1))
119 #define journal_entry_set_unused(je)		((je)->u.s.sector_hi = cpu_to_le32(-1))
120 #define journal_entry_is_inprogress(je)		((je)->u.s.sector_hi == cpu_to_le32(-2))
121 #define journal_entry_set_inprogress(je)	((je)->u.s.sector_hi = cpu_to_le32(-2))
122 
123 #define JOURNAL_BLOCK_SECTORS		8
124 #define JOURNAL_SECTOR_DATA		((1 << SECTOR_SHIFT) - sizeof(commit_id_t))
125 #define JOURNAL_MAC_SIZE		(JOURNAL_MAC_PER_SECTOR * JOURNAL_BLOCK_SECTORS)
126 
127 struct journal_sector {
128 	struct_group(sectors,
129 		__u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR];
130 		__u8 mac[JOURNAL_MAC_PER_SECTOR];
131 	);
132 	commit_id_t commit_id;
133 };
134 
135 #define MAX_TAG_SIZE			(JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK]))
136 
137 #define METADATA_PADDING_SECTORS	8
138 
139 #define N_COMMIT_IDS			4
140 
prev_commit_seq(unsigned char seq)141 static unsigned char prev_commit_seq(unsigned char seq)
142 {
143 	return (seq + N_COMMIT_IDS - 1) % N_COMMIT_IDS;
144 }
145 
next_commit_seq(unsigned char seq)146 static unsigned char next_commit_seq(unsigned char seq)
147 {
148 	return (seq + 1) % N_COMMIT_IDS;
149 }
150 
151 /*
152  * In-memory structures
153  */
154 
155 struct journal_node {
156 	struct rb_node node;
157 	sector_t sector;
158 };
159 
160 struct alg_spec {
161 	char *alg_string;
162 	char *key_string;
163 	__u8 *key;
164 	unsigned int key_size;
165 };
166 
167 struct dm_integrity_c {
168 	struct dm_dev *dev;
169 	struct dm_dev *meta_dev;
170 	unsigned int tag_size;
171 	__s8 log2_tag_size;
172 	unsigned int tuple_size;
173 	sector_t start;
174 	mempool_t journal_io_mempool;
175 	struct dm_io_client *io;
176 	struct dm_bufio_client *bufio;
177 	struct workqueue_struct *metadata_wq;
178 	struct superblock *sb;
179 	unsigned int journal_pages;
180 	unsigned int n_bitmap_blocks;
181 
182 	struct page_list *journal;
183 	struct page_list *journal_io;
184 	struct page_list *journal_xor;
185 	struct page_list *recalc_bitmap;
186 	struct page_list *may_write_bitmap;
187 	struct bitmap_block_status *bbs;
188 	unsigned int bitmap_flush_interval;
189 	int synchronous_mode;
190 	struct bio_list synchronous_bios;
191 	struct delayed_work bitmap_flush_work;
192 
193 	struct crypto_skcipher *journal_crypt;
194 	struct scatterlist **journal_scatterlist;
195 	struct scatterlist **journal_io_scatterlist;
196 	struct skcipher_request **sk_requests;
197 
198 	struct crypto_shash *journal_mac;
199 
200 	struct journal_node *journal_tree;
201 	struct rb_root journal_tree_root;
202 
203 	sector_t provided_data_sectors;
204 
205 	unsigned short journal_entry_size;
206 	unsigned char journal_entries_per_sector;
207 	unsigned char journal_section_entries;
208 	unsigned short journal_section_sectors;
209 	unsigned int journal_sections;
210 	unsigned int journal_entries;
211 	sector_t data_device_sectors;
212 	sector_t meta_device_sectors;
213 	unsigned int initial_sectors;
214 	unsigned int metadata_run;
215 	__s8 log2_metadata_run;
216 	__u8 log2_buffer_sectors;
217 	__u8 sectors_per_block;
218 	__u8 log2_blocks_per_bitmap_bit;
219 
220 	unsigned char mode;
221 
222 	int failed;
223 
224 	struct crypto_shash *internal_hash;
225 
226 	struct dm_target *ti;
227 
228 	/* these variables are locked with endio_wait.lock */
229 	struct rb_root in_progress;
230 	struct list_head wait_list;
231 	wait_queue_head_t endio_wait;
232 	struct workqueue_struct *wait_wq;
233 	struct workqueue_struct *offload_wq;
234 
235 	unsigned char commit_seq;
236 	commit_id_t commit_ids[N_COMMIT_IDS];
237 
238 	unsigned int committed_section;
239 	unsigned int n_committed_sections;
240 
241 	unsigned int uncommitted_section;
242 	unsigned int n_uncommitted_sections;
243 
244 	unsigned int free_section;
245 	unsigned char free_section_entry;
246 	unsigned int free_sectors;
247 
248 	unsigned int free_sectors_threshold;
249 
250 	struct workqueue_struct *commit_wq;
251 	struct work_struct commit_work;
252 
253 	struct workqueue_struct *writer_wq;
254 	struct work_struct writer_work;
255 
256 	struct workqueue_struct *recalc_wq;
257 	struct work_struct recalc_work;
258 
259 	struct bio_list flush_bio_list;
260 
261 	unsigned long autocommit_jiffies;
262 	struct timer_list autocommit_timer;
263 	unsigned int autocommit_msec;
264 
265 	wait_queue_head_t copy_to_journal_wait;
266 
267 	struct completion crypto_backoff;
268 
269 	bool wrote_to_journal;
270 	bool journal_uptodate;
271 	bool just_formatted;
272 	bool recalculate_flag;
273 	bool reset_recalculate_flag;
274 	bool discard;
275 	bool fix_padding;
276 	bool fix_hmac;
277 	bool legacy_recalculate;
278 
279 	struct alg_spec internal_hash_alg;
280 	struct alg_spec journal_crypt_alg;
281 	struct alg_spec journal_mac_alg;
282 
283 	atomic64_t number_of_mismatches;
284 
285 	mempool_t recheck_pool;
286 	struct bio_set recheck_bios;
287 	struct bio_set recalc_bios;
288 
289 	struct notifier_block reboot_notifier;
290 };
291 
292 struct dm_integrity_range {
293 	sector_t logical_sector;
294 	sector_t n_sectors;
295 	bool waiting;
296 	union {
297 		struct rb_node node;
298 		struct {
299 			struct task_struct *task;
300 			struct list_head wait_entry;
301 		};
302 	};
303 };
304 
305 struct dm_integrity_io {
306 	struct work_struct work;
307 
308 	struct dm_integrity_c *ic;
309 	enum req_op op;
310 	bool fua;
311 
312 	struct dm_integrity_range range;
313 
314 	sector_t metadata_block;
315 	unsigned int metadata_offset;
316 
317 	atomic_t in_flight;
318 	blk_status_t bi_status;
319 
320 	struct completion *completion;
321 
322 	struct dm_bio_details bio_details;
323 
324 	char *integrity_payload;
325 	unsigned payload_len;
326 	bool integrity_payload_from_mempool;
327 	bool integrity_range_locked;
328 };
329 
330 struct journal_completion {
331 	struct dm_integrity_c *ic;
332 	atomic_t in_flight;
333 	struct completion comp;
334 };
335 
336 struct journal_io {
337 	struct dm_integrity_range range;
338 	struct journal_completion *comp;
339 };
340 
341 struct bitmap_block_status {
342 	struct work_struct work;
343 	struct dm_integrity_c *ic;
344 	unsigned int idx;
345 	unsigned long *bitmap;
346 	struct bio_list bio_queue;
347 	spinlock_t bio_queue_lock;
348 
349 };
350 
351 static struct kmem_cache *journal_io_cache;
352 
353 #define JOURNAL_IO_MEMPOOL	32
354 
355 #ifdef DEBUG_PRINT
356 #define DEBUG_print(x, ...)			printk(KERN_DEBUG x, ##__VA_ARGS__)
357 #define DEBUG_bytes(bytes, len, msg, ...)	printk(KERN_DEBUG msg "%s%*ph\n", ##__VA_ARGS__, \
358 						       len ? ": " : "", len, bytes)
359 #else
360 #define DEBUG_print(x, ...)			do { } while (0)
361 #define DEBUG_bytes(bytes, len, msg, ...)	do { } while (0)
362 #endif
363 
364 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
365 static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map);
366 static void integrity_bio_wait(struct work_struct *w);
367 static void dm_integrity_dtr(struct dm_target *ti);
368 
dm_integrity_io_error(struct dm_integrity_c * ic,const char * msg,int err)369 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err)
370 {
371 	if (err == -EILSEQ)
372 		atomic64_inc(&ic->number_of_mismatches);
373 	if (!cmpxchg(&ic->failed, 0, err))
374 		DMERR("Error on %s: %d", msg, err);
375 }
376 
dm_integrity_failed(struct dm_integrity_c * ic)377 static int dm_integrity_failed(struct dm_integrity_c *ic)
378 {
379 	return READ_ONCE(ic->failed);
380 }
381 
dm_integrity_disable_recalculate(struct dm_integrity_c * ic)382 static bool dm_integrity_disable_recalculate(struct dm_integrity_c *ic)
383 {
384 	if (ic->legacy_recalculate)
385 		return false;
386 	if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) ?
387 	    ic->internal_hash_alg.key || ic->journal_mac_alg.key :
388 	    ic->internal_hash_alg.key && !ic->journal_mac_alg.key)
389 		return true;
390 	return false;
391 }
392 
dm_integrity_commit_id(struct dm_integrity_c * ic,unsigned int i,unsigned int j,unsigned char seq)393 static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned int i,
394 					  unsigned int j, unsigned char seq)
395 {
396 	/*
397 	 * Xor the number with section and sector, so that if a piece of
398 	 * journal is written at wrong place, it is detected.
399 	 */
400 	return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j);
401 }
402 
get_area_and_offset(struct dm_integrity_c * ic,sector_t data_sector,sector_t * area,sector_t * offset)403 static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector,
404 				sector_t *area, sector_t *offset)
405 {
406 	if (!ic->meta_dev) {
407 		__u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors;
408 		*area = data_sector >> log2_interleave_sectors;
409 		*offset = (unsigned int)data_sector & ((1U << log2_interleave_sectors) - 1);
410 	} else {
411 		*area = 0;
412 		*offset = data_sector;
413 	}
414 }
415 
416 #define sector_to_block(ic, n)						\
417 do {									\
418 	BUG_ON((n) & (unsigned int)((ic)->sectors_per_block - 1));		\
419 	(n) >>= (ic)->sb->log2_sectors_per_block;			\
420 } while (0)
421 
get_metadata_sector_and_offset(struct dm_integrity_c * ic,sector_t area,sector_t offset,unsigned int * metadata_offset)422 static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area,
423 					    sector_t offset, unsigned int *metadata_offset)
424 {
425 	__u64 ms;
426 	unsigned int mo;
427 
428 	ms = area << ic->sb->log2_interleave_sectors;
429 	if (likely(ic->log2_metadata_run >= 0))
430 		ms += area << ic->log2_metadata_run;
431 	else
432 		ms += area * ic->metadata_run;
433 	ms >>= ic->log2_buffer_sectors;
434 
435 	sector_to_block(ic, offset);
436 
437 	if (likely(ic->log2_tag_size >= 0)) {
438 		ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size);
439 		mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
440 	} else {
441 		ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors);
442 		mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
443 	}
444 	*metadata_offset = mo;
445 	return ms;
446 }
447 
get_data_sector(struct dm_integrity_c * ic,sector_t area,sector_t offset)448 static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset)
449 {
450 	sector_t result;
451 
452 	if (ic->meta_dev)
453 		return offset;
454 
455 	result = area << ic->sb->log2_interleave_sectors;
456 	if (likely(ic->log2_metadata_run >= 0))
457 		result += (area + 1) << ic->log2_metadata_run;
458 	else
459 		result += (area + 1) * ic->metadata_run;
460 
461 	result += (sector_t)ic->initial_sectors + offset;
462 	result += ic->start;
463 
464 	return result;
465 }
466 
wraparound_section(struct dm_integrity_c * ic,unsigned int * sec_ptr)467 static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr)
468 {
469 	if (unlikely(*sec_ptr >= ic->journal_sections))
470 		*sec_ptr -= ic->journal_sections;
471 }
472 
sb_set_version(struct dm_integrity_c * ic)473 static void sb_set_version(struct dm_integrity_c *ic)
474 {
475 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE))
476 		ic->sb->version = SB_VERSION_6;
477 	else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC))
478 		ic->sb->version = SB_VERSION_5;
479 	else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING))
480 		ic->sb->version = SB_VERSION_4;
481 	else if (ic->mode == 'B' || ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP))
482 		ic->sb->version = SB_VERSION_3;
483 	else if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
484 		ic->sb->version = SB_VERSION_2;
485 	else
486 		ic->sb->version = SB_VERSION_1;
487 }
488 
sb_mac(struct dm_integrity_c * ic,bool wr)489 static int sb_mac(struct dm_integrity_c *ic, bool wr)
490 {
491 	SHASH_DESC_ON_STACK(desc, ic->journal_mac);
492 	int r;
493 	unsigned int mac_size = crypto_shash_digestsize(ic->journal_mac);
494 	__u8 *sb = (__u8 *)ic->sb;
495 	__u8 *mac = sb + (1 << SECTOR_SHIFT) - mac_size;
496 
497 	if (sizeof(struct superblock) + mac_size > 1 << SECTOR_SHIFT ||
498 	    mac_size > HASH_MAX_DIGESTSIZE) {
499 		dm_integrity_io_error(ic, "digest is too long", -EINVAL);
500 		return -EINVAL;
501 	}
502 
503 	desc->tfm = ic->journal_mac;
504 
505 	if (likely(wr)) {
506 		r = crypto_shash_digest(desc, sb, mac - sb, mac);
507 		if (unlikely(r < 0)) {
508 			dm_integrity_io_error(ic, "crypto_shash_digest", r);
509 			return r;
510 		}
511 	} else {
512 		__u8 actual_mac[HASH_MAX_DIGESTSIZE];
513 
514 		r = crypto_shash_digest(desc, sb, mac - sb, actual_mac);
515 		if (unlikely(r < 0)) {
516 			dm_integrity_io_error(ic, "crypto_shash_digest", r);
517 			return r;
518 		}
519 		if (memcmp(mac, actual_mac, mac_size)) {
520 			dm_integrity_io_error(ic, "superblock mac", -EILSEQ);
521 			dm_audit_log_target(DM_MSG_PREFIX, "mac-superblock", ic->ti, 0);
522 			return -EILSEQ;
523 		}
524 	}
525 
526 	return 0;
527 }
528 
sync_rw_sb(struct dm_integrity_c * ic,blk_opf_t opf)529 static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf)
530 {
531 	struct dm_io_request io_req;
532 	struct dm_io_region io_loc;
533 	const enum req_op op = opf & REQ_OP_MASK;
534 	int r;
535 
536 	io_req.bi_opf = opf;
537 	io_req.mem.type = DM_IO_KMEM;
538 	io_req.mem.ptr.addr = ic->sb;
539 	io_req.notify.fn = NULL;
540 	io_req.client = ic->io;
541 	io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
542 	io_loc.sector = ic->start;
543 	io_loc.count = SB_SECTORS;
544 
545 	if (op == REQ_OP_WRITE) {
546 		sb_set_version(ic);
547 		if (ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
548 			r = sb_mac(ic, true);
549 			if (unlikely(r))
550 				return r;
551 		}
552 	}
553 
554 	r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
555 	if (unlikely(r))
556 		return r;
557 
558 	if (op == REQ_OP_READ) {
559 		if (ic->mode != 'R' && ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
560 			r = sb_mac(ic, false);
561 			if (unlikely(r))
562 				return r;
563 		}
564 	}
565 
566 	return 0;
567 }
568 
569 #define BITMAP_OP_TEST_ALL_SET		0
570 #define BITMAP_OP_TEST_ALL_CLEAR	1
571 #define BITMAP_OP_SET			2
572 #define BITMAP_OP_CLEAR			3
573 
block_bitmap_op(struct dm_integrity_c * ic,struct page_list * bitmap,sector_t sector,sector_t n_sectors,int mode)574 static bool block_bitmap_op(struct dm_integrity_c *ic, struct page_list *bitmap,
575 			    sector_t sector, sector_t n_sectors, int mode)
576 {
577 	unsigned long bit, end_bit, this_end_bit, page, end_page;
578 	unsigned long *data;
579 
580 	if (unlikely(((sector | n_sectors) & ((1 << ic->sb->log2_sectors_per_block) - 1)) != 0)) {
581 		DMCRIT("invalid bitmap access (%llx,%llx,%d,%d,%d)",
582 			sector,
583 			n_sectors,
584 			ic->sb->log2_sectors_per_block,
585 			ic->log2_blocks_per_bitmap_bit,
586 			mode);
587 		BUG();
588 	}
589 
590 	if (unlikely(!n_sectors))
591 		return true;
592 
593 	bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
594 	end_bit = (sector + n_sectors - 1) >>
595 		(ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
596 
597 	page = bit / (PAGE_SIZE * 8);
598 	bit %= PAGE_SIZE * 8;
599 
600 	end_page = end_bit / (PAGE_SIZE * 8);
601 	end_bit %= PAGE_SIZE * 8;
602 
603 repeat:
604 	if (page < end_page)
605 		this_end_bit = PAGE_SIZE * 8 - 1;
606 	else
607 		this_end_bit = end_bit;
608 
609 	data = lowmem_page_address(bitmap[page].page);
610 
611 	if (mode == BITMAP_OP_TEST_ALL_SET) {
612 		while (bit <= this_end_bit) {
613 			if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
614 				do {
615 					if (data[bit / BITS_PER_LONG] != -1)
616 						return false;
617 					bit += BITS_PER_LONG;
618 				} while (this_end_bit >= bit + BITS_PER_LONG - 1);
619 				continue;
620 			}
621 			if (!test_bit(bit, data))
622 				return false;
623 			bit++;
624 		}
625 	} else if (mode == BITMAP_OP_TEST_ALL_CLEAR) {
626 		while (bit <= this_end_bit) {
627 			if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
628 				do {
629 					if (data[bit / BITS_PER_LONG] != 0)
630 						return false;
631 					bit += BITS_PER_LONG;
632 				} while (this_end_bit >= bit + BITS_PER_LONG - 1);
633 				continue;
634 			}
635 			if (test_bit(bit, data))
636 				return false;
637 			bit++;
638 		}
639 	} else if (mode == BITMAP_OP_SET) {
640 		while (bit <= this_end_bit) {
641 			if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
642 				do {
643 					data[bit / BITS_PER_LONG] = -1;
644 					bit += BITS_PER_LONG;
645 				} while (this_end_bit >= bit + BITS_PER_LONG - 1);
646 				continue;
647 			}
648 			__set_bit(bit, data);
649 			bit++;
650 		}
651 	} else if (mode == BITMAP_OP_CLEAR) {
652 		if (!bit && this_end_bit == PAGE_SIZE * 8 - 1)
653 			clear_page(data);
654 		else {
655 			while (bit <= this_end_bit) {
656 				if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
657 					do {
658 						data[bit / BITS_PER_LONG] = 0;
659 						bit += BITS_PER_LONG;
660 					} while (this_end_bit >= bit + BITS_PER_LONG - 1);
661 					continue;
662 				}
663 				__clear_bit(bit, data);
664 				bit++;
665 			}
666 		}
667 	} else {
668 		BUG();
669 	}
670 
671 	if (unlikely(page < end_page)) {
672 		bit = 0;
673 		page++;
674 		goto repeat;
675 	}
676 
677 	return true;
678 }
679 
block_bitmap_copy(struct dm_integrity_c * ic,struct page_list * dst,struct page_list * src)680 static void block_bitmap_copy(struct dm_integrity_c *ic, struct page_list *dst, struct page_list *src)
681 {
682 	unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE);
683 	unsigned int i;
684 
685 	for (i = 0; i < n_bitmap_pages; i++) {
686 		unsigned long *dst_data = lowmem_page_address(dst[i].page);
687 		unsigned long *src_data = lowmem_page_address(src[i].page);
688 
689 		copy_page(dst_data, src_data);
690 	}
691 }
692 
sector_to_bitmap_block(struct dm_integrity_c * ic,sector_t sector)693 static struct bitmap_block_status *sector_to_bitmap_block(struct dm_integrity_c *ic, sector_t sector)
694 {
695 	unsigned int bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
696 	unsigned int bitmap_block = bit / (BITMAP_BLOCK_SIZE * 8);
697 
698 	BUG_ON(bitmap_block >= ic->n_bitmap_blocks);
699 	return &ic->bbs[bitmap_block];
700 }
701 
access_journal_check(struct dm_integrity_c * ic,unsigned int section,unsigned int offset,bool e,const char * function)702 static void access_journal_check(struct dm_integrity_c *ic, unsigned int section, unsigned int offset,
703 				 bool e, const char *function)
704 {
705 #if defined(CONFIG_DM_DEBUG) || defined(INTERNAL_VERIFY)
706 	unsigned int limit = e ? ic->journal_section_entries : ic->journal_section_sectors;
707 
708 	if (unlikely(section >= ic->journal_sections) ||
709 	    unlikely(offset >= limit)) {
710 		DMCRIT("%s: invalid access at (%u,%u), limit (%u,%u)",
711 		       function, section, offset, ic->journal_sections, limit);
712 		BUG();
713 	}
714 #endif
715 }
716 
page_list_location(struct dm_integrity_c * ic,unsigned int section,unsigned int offset,unsigned int * pl_index,unsigned int * pl_offset)717 static void page_list_location(struct dm_integrity_c *ic, unsigned int section, unsigned int offset,
718 			       unsigned int *pl_index, unsigned int *pl_offset)
719 {
720 	unsigned int sector;
721 
722 	access_journal_check(ic, section, offset, false, "page_list_location");
723 
724 	sector = section * ic->journal_section_sectors + offset;
725 
726 	*pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
727 	*pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
728 }
729 
access_page_list(struct dm_integrity_c * ic,struct page_list * pl,unsigned int section,unsigned int offset,unsigned int * n_sectors)730 static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl,
731 					       unsigned int section, unsigned int offset, unsigned int *n_sectors)
732 {
733 	unsigned int pl_index, pl_offset;
734 	char *va;
735 
736 	page_list_location(ic, section, offset, &pl_index, &pl_offset);
737 
738 	if (n_sectors)
739 		*n_sectors = (PAGE_SIZE - pl_offset) >> SECTOR_SHIFT;
740 
741 	va = lowmem_page_address(pl[pl_index].page);
742 
743 	return (struct journal_sector *)(va + pl_offset);
744 }
745 
access_journal(struct dm_integrity_c * ic,unsigned int section,unsigned int offset)746 static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset)
747 {
748 	return access_page_list(ic, ic->journal, section, offset, NULL);
749 }
750 
access_journal_entry(struct dm_integrity_c * ic,unsigned int section,unsigned int n)751 static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned int section, unsigned int n)
752 {
753 	unsigned int rel_sector, offset;
754 	struct journal_sector *js;
755 
756 	access_journal_check(ic, section, n, true, "access_journal_entry");
757 
758 	rel_sector = n % JOURNAL_BLOCK_SECTORS;
759 	offset = n / JOURNAL_BLOCK_SECTORS;
760 
761 	js = access_journal(ic, section, rel_sector);
762 	return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size);
763 }
764 
access_journal_data(struct dm_integrity_c * ic,unsigned int section,unsigned int n)765 static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned int section, unsigned int n)
766 {
767 	n <<= ic->sb->log2_sectors_per_block;
768 
769 	n += JOURNAL_BLOCK_SECTORS;
770 
771 	access_journal_check(ic, section, n, false, "access_journal_data");
772 
773 	return access_journal(ic, section, n);
774 }
775 
section_mac(struct dm_integrity_c * ic,unsigned int section,__u8 result[JOURNAL_MAC_SIZE])776 static void section_mac(struct dm_integrity_c *ic, unsigned int section, __u8 result[JOURNAL_MAC_SIZE])
777 {
778 	SHASH_DESC_ON_STACK(desc, ic->journal_mac);
779 	int r;
780 	unsigned int j, size;
781 
782 	desc->tfm = ic->journal_mac;
783 
784 	r = crypto_shash_init(desc);
785 	if (unlikely(r < 0)) {
786 		dm_integrity_io_error(ic, "crypto_shash_init", r);
787 		goto err;
788 	}
789 
790 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
791 		__le64 section_le;
792 
793 		r = crypto_shash_update(desc, (__u8 *)&ic->sb->salt, SALT_SIZE);
794 		if (unlikely(r < 0)) {
795 			dm_integrity_io_error(ic, "crypto_shash_update", r);
796 			goto err;
797 		}
798 
799 		section_le = cpu_to_le64(section);
800 		r = crypto_shash_update(desc, (__u8 *)&section_le, sizeof(section_le));
801 		if (unlikely(r < 0)) {
802 			dm_integrity_io_error(ic, "crypto_shash_update", r);
803 			goto err;
804 		}
805 	}
806 
807 	for (j = 0; j < ic->journal_section_entries; j++) {
808 		struct journal_entry *je = access_journal_entry(ic, section, j);
809 
810 		r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof(je->u.sector));
811 		if (unlikely(r < 0)) {
812 			dm_integrity_io_error(ic, "crypto_shash_update", r);
813 			goto err;
814 		}
815 	}
816 
817 	size = crypto_shash_digestsize(ic->journal_mac);
818 
819 	if (likely(size <= JOURNAL_MAC_SIZE)) {
820 		r = crypto_shash_final(desc, result);
821 		if (unlikely(r < 0)) {
822 			dm_integrity_io_error(ic, "crypto_shash_final", r);
823 			goto err;
824 		}
825 		memset(result + size, 0, JOURNAL_MAC_SIZE - size);
826 	} else {
827 		__u8 digest[HASH_MAX_DIGESTSIZE];
828 
829 		if (WARN_ON(size > sizeof(digest))) {
830 			dm_integrity_io_error(ic, "digest_size", -EINVAL);
831 			goto err;
832 		}
833 		r = crypto_shash_final(desc, digest);
834 		if (unlikely(r < 0)) {
835 			dm_integrity_io_error(ic, "crypto_shash_final", r);
836 			goto err;
837 		}
838 		memcpy(result, digest, JOURNAL_MAC_SIZE);
839 	}
840 
841 	return;
842 err:
843 	memset(result, 0, JOURNAL_MAC_SIZE);
844 }
845 
rw_section_mac(struct dm_integrity_c * ic,unsigned int section,bool wr)846 static void rw_section_mac(struct dm_integrity_c *ic, unsigned int section, bool wr)
847 {
848 	__u8 result[JOURNAL_MAC_SIZE];
849 	unsigned int j;
850 
851 	if (!ic->journal_mac)
852 		return;
853 
854 	section_mac(ic, section, result);
855 
856 	for (j = 0; j < JOURNAL_BLOCK_SECTORS; j++) {
857 		struct journal_sector *js = access_journal(ic, section, j);
858 
859 		if (likely(wr))
860 			memcpy(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR);
861 		else {
862 			if (memcmp(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR)) {
863 				dm_integrity_io_error(ic, "journal mac", -EILSEQ);
864 				dm_audit_log_target(DM_MSG_PREFIX, "mac-journal", ic->ti, 0);
865 			}
866 		}
867 	}
868 }
869 
complete_journal_op(void * context)870 static void complete_journal_op(void *context)
871 {
872 	struct journal_completion *comp = context;
873 
874 	BUG_ON(!atomic_read(&comp->in_flight));
875 	if (likely(atomic_dec_and_test(&comp->in_flight)))
876 		complete(&comp->comp);
877 }
878 
xor_journal(struct dm_integrity_c * ic,bool encrypt,unsigned int section,unsigned int n_sections,struct journal_completion * comp)879 static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section,
880 			unsigned int n_sections, struct journal_completion *comp)
881 {
882 	struct async_submit_ctl submit;
883 	size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT;
884 	unsigned int pl_index, pl_offset, section_index;
885 	struct page_list *source_pl, *target_pl;
886 
887 	if (likely(encrypt)) {
888 		source_pl = ic->journal;
889 		target_pl = ic->journal_io;
890 	} else {
891 		source_pl = ic->journal_io;
892 		target_pl = ic->journal;
893 	}
894 
895 	page_list_location(ic, section, 0, &pl_index, &pl_offset);
896 
897 	atomic_add(roundup(pl_offset + n_bytes, PAGE_SIZE) >> PAGE_SHIFT, &comp->in_flight);
898 
899 	init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, complete_journal_op, comp, NULL);
900 
901 	section_index = pl_index;
902 
903 	do {
904 		size_t this_step;
905 		struct page *src_pages[2];
906 		struct page *dst_page;
907 
908 		while (unlikely(pl_index == section_index)) {
909 			unsigned int dummy;
910 
911 			if (likely(encrypt))
912 				rw_section_mac(ic, section, true);
913 			section++;
914 			n_sections--;
915 			if (!n_sections)
916 				break;
917 			page_list_location(ic, section, 0, &section_index, &dummy);
918 		}
919 
920 		this_step = min(n_bytes, (size_t)PAGE_SIZE - pl_offset);
921 		dst_page = target_pl[pl_index].page;
922 		src_pages[0] = source_pl[pl_index].page;
923 		src_pages[1] = ic->journal_xor[pl_index].page;
924 
925 		async_xor(dst_page, src_pages, pl_offset, 2, this_step, &submit);
926 
927 		pl_index++;
928 		pl_offset = 0;
929 		n_bytes -= this_step;
930 	} while (n_bytes);
931 
932 	BUG_ON(n_sections);
933 
934 	async_tx_issue_pending_all();
935 }
936 
complete_journal_encrypt(void * data,int err)937 static void complete_journal_encrypt(void *data, int err)
938 {
939 	struct journal_completion *comp = data;
940 
941 	if (unlikely(err)) {
942 		if (likely(err == -EINPROGRESS)) {
943 			complete(&comp->ic->crypto_backoff);
944 			return;
945 		}
946 		dm_integrity_io_error(comp->ic, "asynchronous encrypt", err);
947 	}
948 	complete_journal_op(comp);
949 }
950 
do_crypt(bool encrypt,struct skcipher_request * req,struct journal_completion * comp)951 static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp)
952 {
953 	int r;
954 
955 	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
956 				      complete_journal_encrypt, comp);
957 	if (likely(encrypt))
958 		r = crypto_skcipher_encrypt(req);
959 	else
960 		r = crypto_skcipher_decrypt(req);
961 	if (likely(!r))
962 		return false;
963 	if (likely(r == -EINPROGRESS))
964 		return true;
965 	if (likely(r == -EBUSY)) {
966 		wait_for_completion(&comp->ic->crypto_backoff);
967 		reinit_completion(&comp->ic->crypto_backoff);
968 		return true;
969 	}
970 	dm_integrity_io_error(comp->ic, "encrypt", r);
971 	return false;
972 }
973 
crypt_journal(struct dm_integrity_c * ic,bool encrypt,unsigned int section,unsigned int n_sections,struct journal_completion * comp)974 static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section,
975 			  unsigned int n_sections, struct journal_completion *comp)
976 {
977 	struct scatterlist **source_sg;
978 	struct scatterlist **target_sg;
979 
980 	atomic_add(2, &comp->in_flight);
981 
982 	if (likely(encrypt)) {
983 		source_sg = ic->journal_scatterlist;
984 		target_sg = ic->journal_io_scatterlist;
985 	} else {
986 		source_sg = ic->journal_io_scatterlist;
987 		target_sg = ic->journal_scatterlist;
988 	}
989 
990 	do {
991 		struct skcipher_request *req;
992 		unsigned int ivsize;
993 		char *iv;
994 
995 		if (likely(encrypt))
996 			rw_section_mac(ic, section, true);
997 
998 		req = ic->sk_requests[section];
999 		ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
1000 		iv = req->iv;
1001 
1002 		memcpy(iv, iv + ivsize, ivsize);
1003 
1004 		req->src = source_sg[section];
1005 		req->dst = target_sg[section];
1006 
1007 		if (unlikely(do_crypt(encrypt, req, comp)))
1008 			atomic_inc(&comp->in_flight);
1009 
1010 		section++;
1011 		n_sections--;
1012 	} while (n_sections);
1013 
1014 	atomic_dec(&comp->in_flight);
1015 	complete_journal_op(comp);
1016 }
1017 
encrypt_journal(struct dm_integrity_c * ic,bool encrypt,unsigned int section,unsigned int n_sections,struct journal_completion * comp)1018 static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section,
1019 			    unsigned int n_sections, struct journal_completion *comp)
1020 {
1021 	if (ic->journal_xor)
1022 		return xor_journal(ic, encrypt, section, n_sections, comp);
1023 	else
1024 		return crypt_journal(ic, encrypt, section, n_sections, comp);
1025 }
1026 
complete_journal_io(unsigned long error,void * context)1027 static void complete_journal_io(unsigned long error, void *context)
1028 {
1029 	struct journal_completion *comp = context;
1030 
1031 	if (unlikely(error != 0))
1032 		dm_integrity_io_error(comp->ic, "writing journal", -EIO);
1033 	complete_journal_op(comp);
1034 }
1035 
rw_journal_sectors(struct dm_integrity_c * ic,blk_opf_t opf,unsigned int sector,unsigned int n_sectors,struct journal_completion * comp)1036 static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf,
1037 			       unsigned int sector, unsigned int n_sectors,
1038 			       struct journal_completion *comp)
1039 {
1040 	struct dm_io_request io_req;
1041 	struct dm_io_region io_loc;
1042 	unsigned int pl_index, pl_offset;
1043 	int r;
1044 
1045 	if (unlikely(dm_integrity_failed(ic))) {
1046 		if (comp)
1047 			complete_journal_io(-1UL, comp);
1048 		return;
1049 	}
1050 
1051 	pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
1052 	pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
1053 
1054 	io_req.bi_opf = opf;
1055 	io_req.mem.type = DM_IO_PAGE_LIST;
1056 	if (ic->journal_io)
1057 		io_req.mem.ptr.pl = &ic->journal_io[pl_index];
1058 	else
1059 		io_req.mem.ptr.pl = &ic->journal[pl_index];
1060 	io_req.mem.offset = pl_offset;
1061 	if (likely(comp != NULL)) {
1062 		io_req.notify.fn = complete_journal_io;
1063 		io_req.notify.context = comp;
1064 	} else {
1065 		io_req.notify.fn = NULL;
1066 	}
1067 	io_req.client = ic->io;
1068 	io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
1069 	io_loc.sector = ic->start + SB_SECTORS + sector;
1070 	io_loc.count = n_sectors;
1071 
1072 	r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
1073 	if (unlikely(r)) {
1074 		dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ?
1075 				      "reading journal" : "writing journal", r);
1076 		if (comp) {
1077 			WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
1078 			complete_journal_io(-1UL, comp);
1079 		}
1080 	}
1081 }
1082 
rw_journal(struct dm_integrity_c * ic,blk_opf_t opf,unsigned int section,unsigned int n_sections,struct journal_completion * comp)1083 static void rw_journal(struct dm_integrity_c *ic, blk_opf_t opf,
1084 		       unsigned int section, unsigned int n_sections,
1085 		       struct journal_completion *comp)
1086 {
1087 	unsigned int sector, n_sectors;
1088 
1089 	sector = section * ic->journal_section_sectors;
1090 	n_sectors = n_sections * ic->journal_section_sectors;
1091 
1092 	rw_journal_sectors(ic, opf, sector, n_sectors, comp);
1093 }
1094 
write_journal(struct dm_integrity_c * ic,unsigned int commit_start,unsigned int commit_sections)1095 static void write_journal(struct dm_integrity_c *ic, unsigned int commit_start, unsigned int commit_sections)
1096 {
1097 	struct journal_completion io_comp;
1098 	struct journal_completion crypt_comp_1;
1099 	struct journal_completion crypt_comp_2;
1100 	unsigned int i;
1101 
1102 	io_comp.ic = ic;
1103 	init_completion(&io_comp.comp);
1104 
1105 	if (commit_start + commit_sections <= ic->journal_sections) {
1106 		io_comp.in_flight = (atomic_t)ATOMIC_INIT(1);
1107 		if (ic->journal_io) {
1108 			crypt_comp_1.ic = ic;
1109 			init_completion(&crypt_comp_1.comp);
1110 			crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
1111 			encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1);
1112 			wait_for_completion_io(&crypt_comp_1.comp);
1113 		} else {
1114 			for (i = 0; i < commit_sections; i++)
1115 				rw_section_mac(ic, commit_start + i, true);
1116 		}
1117 		rw_journal(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, commit_start,
1118 			   commit_sections, &io_comp);
1119 	} else {
1120 		unsigned int to_end;
1121 
1122 		io_comp.in_flight = (atomic_t)ATOMIC_INIT(2);
1123 		to_end = ic->journal_sections - commit_start;
1124 		if (ic->journal_io) {
1125 			crypt_comp_1.ic = ic;
1126 			init_completion(&crypt_comp_1.comp);
1127 			crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
1128 			encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1);
1129 			if (try_wait_for_completion(&crypt_comp_1.comp)) {
1130 				rw_journal(ic, REQ_OP_WRITE | REQ_FUA,
1131 					   commit_start, to_end, &io_comp);
1132 				reinit_completion(&crypt_comp_1.comp);
1133 				crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
1134 				encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1);
1135 				wait_for_completion_io(&crypt_comp_1.comp);
1136 			} else {
1137 				crypt_comp_2.ic = ic;
1138 				init_completion(&crypt_comp_2.comp);
1139 				crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0);
1140 				encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2);
1141 				wait_for_completion_io(&crypt_comp_1.comp);
1142 				rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp);
1143 				wait_for_completion_io(&crypt_comp_2.comp);
1144 			}
1145 		} else {
1146 			for (i = 0; i < to_end; i++)
1147 				rw_section_mac(ic, commit_start + i, true);
1148 			rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp);
1149 			for (i = 0; i < commit_sections - to_end; i++)
1150 				rw_section_mac(ic, i, true);
1151 		}
1152 		rw_journal(ic, REQ_OP_WRITE | REQ_FUA, 0, commit_sections - to_end, &io_comp);
1153 	}
1154 
1155 	wait_for_completion_io(&io_comp.comp);
1156 }
1157 
copy_from_journal(struct dm_integrity_c * ic,unsigned int section,unsigned int offset,unsigned int n_sectors,sector_t target,io_notify_fn fn,void * data)1158 static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset,
1159 			      unsigned int n_sectors, sector_t target, io_notify_fn fn, void *data)
1160 {
1161 	struct dm_io_request io_req;
1162 	struct dm_io_region io_loc;
1163 	int r;
1164 	unsigned int sector, pl_index, pl_offset;
1165 
1166 	BUG_ON((target | n_sectors | offset) & (unsigned int)(ic->sectors_per_block - 1));
1167 
1168 	if (unlikely(dm_integrity_failed(ic))) {
1169 		fn(-1UL, data);
1170 		return;
1171 	}
1172 
1173 	sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset;
1174 
1175 	pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
1176 	pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
1177 
1178 	io_req.bi_opf = REQ_OP_WRITE;
1179 	io_req.mem.type = DM_IO_PAGE_LIST;
1180 	io_req.mem.ptr.pl = &ic->journal[pl_index];
1181 	io_req.mem.offset = pl_offset;
1182 	io_req.notify.fn = fn;
1183 	io_req.notify.context = data;
1184 	io_req.client = ic->io;
1185 	io_loc.bdev = ic->dev->bdev;
1186 	io_loc.sector = target;
1187 	io_loc.count = n_sectors;
1188 
1189 	r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
1190 	if (unlikely(r)) {
1191 		WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
1192 		fn(-1UL, data);
1193 	}
1194 }
1195 
ranges_overlap(struct dm_integrity_range * range1,struct dm_integrity_range * range2)1196 static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2)
1197 {
1198 	return range1->logical_sector < range2->logical_sector + range2->n_sectors &&
1199 	       range1->logical_sector + range1->n_sectors > range2->logical_sector;
1200 }
1201 
add_new_range(struct dm_integrity_c * ic,struct dm_integrity_range * new_range,bool check_waiting)1202 static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting)
1203 {
1204 	struct rb_node **n = &ic->in_progress.rb_node;
1205 	struct rb_node *parent;
1206 
1207 	BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned int)(ic->sectors_per_block - 1));
1208 
1209 	if (likely(check_waiting)) {
1210 		struct dm_integrity_range *range;
1211 
1212 		list_for_each_entry(range, &ic->wait_list, wait_entry) {
1213 			if (unlikely(ranges_overlap(range, new_range)))
1214 				return false;
1215 		}
1216 	}
1217 
1218 	parent = NULL;
1219 
1220 	while (*n) {
1221 		struct dm_integrity_range *range = container_of(*n, struct dm_integrity_range, node);
1222 
1223 		parent = *n;
1224 		if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector)
1225 			n = &range->node.rb_left;
1226 		else if (new_range->logical_sector >= range->logical_sector + range->n_sectors)
1227 			n = &range->node.rb_right;
1228 		else
1229 			return false;
1230 	}
1231 
1232 	rb_link_node(&new_range->node, parent, n);
1233 	rb_insert_color(&new_range->node, &ic->in_progress);
1234 
1235 	return true;
1236 }
1237 
remove_range_unlocked(struct dm_integrity_c * ic,struct dm_integrity_range * range)1238 static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range)
1239 {
1240 	rb_erase(&range->node, &ic->in_progress);
1241 	while (unlikely(!list_empty(&ic->wait_list))) {
1242 		struct dm_integrity_range *last_range =
1243 			list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry);
1244 		struct task_struct *last_range_task;
1245 
1246 		last_range_task = last_range->task;
1247 		list_del(&last_range->wait_entry);
1248 		if (!add_new_range(ic, last_range, false)) {
1249 			last_range->task = last_range_task;
1250 			list_add(&last_range->wait_entry, &ic->wait_list);
1251 			break;
1252 		}
1253 		last_range->waiting = false;
1254 		wake_up_process(last_range_task);
1255 	}
1256 }
1257 
remove_range(struct dm_integrity_c * ic,struct dm_integrity_range * range)1258 static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range)
1259 {
1260 	unsigned long flags;
1261 
1262 	spin_lock_irqsave(&ic->endio_wait.lock, flags);
1263 	remove_range_unlocked(ic, range);
1264 	spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1265 }
1266 
wait_and_add_new_range(struct dm_integrity_c * ic,struct dm_integrity_range * new_range)1267 static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
1268 {
1269 	new_range->waiting = true;
1270 	list_add_tail(&new_range->wait_entry, &ic->wait_list);
1271 	new_range->task = current;
1272 	do {
1273 		__set_current_state(TASK_UNINTERRUPTIBLE);
1274 		spin_unlock_irq(&ic->endio_wait.lock);
1275 		io_schedule();
1276 		spin_lock_irq(&ic->endio_wait.lock);
1277 	} while (unlikely(new_range->waiting));
1278 }
1279 
add_new_range_and_wait(struct dm_integrity_c * ic,struct dm_integrity_range * new_range)1280 static void add_new_range_and_wait(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
1281 {
1282 	if (unlikely(!add_new_range(ic, new_range, true)))
1283 		wait_and_add_new_range(ic, new_range);
1284 }
1285 
init_journal_node(struct journal_node * node)1286 static void init_journal_node(struct journal_node *node)
1287 {
1288 	RB_CLEAR_NODE(&node->node);
1289 	node->sector = (sector_t)-1;
1290 }
1291 
add_journal_node(struct dm_integrity_c * ic,struct journal_node * node,sector_t sector)1292 static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector)
1293 {
1294 	struct rb_node **link;
1295 	struct rb_node *parent;
1296 
1297 	node->sector = sector;
1298 	BUG_ON(!RB_EMPTY_NODE(&node->node));
1299 
1300 	link = &ic->journal_tree_root.rb_node;
1301 	parent = NULL;
1302 
1303 	while (*link) {
1304 		struct journal_node *j;
1305 
1306 		parent = *link;
1307 		j = container_of(parent, struct journal_node, node);
1308 		if (sector < j->sector)
1309 			link = &j->node.rb_left;
1310 		else
1311 			link = &j->node.rb_right;
1312 	}
1313 
1314 	rb_link_node(&node->node, parent, link);
1315 	rb_insert_color(&node->node, &ic->journal_tree_root);
1316 }
1317 
remove_journal_node(struct dm_integrity_c * ic,struct journal_node * node)1318 static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node)
1319 {
1320 	BUG_ON(RB_EMPTY_NODE(&node->node));
1321 	rb_erase(&node->node, &ic->journal_tree_root);
1322 	init_journal_node(node);
1323 }
1324 
1325 #define NOT_FOUND	(-1U)
1326 
find_journal_node(struct dm_integrity_c * ic,sector_t sector,sector_t * next_sector)1327 static unsigned int find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector)
1328 {
1329 	struct rb_node *n = ic->journal_tree_root.rb_node;
1330 	unsigned int found = NOT_FOUND;
1331 
1332 	*next_sector = (sector_t)-1;
1333 	while (n) {
1334 		struct journal_node *j = container_of(n, struct journal_node, node);
1335 
1336 		if (sector == j->sector)
1337 			found = j - ic->journal_tree;
1338 
1339 		if (sector < j->sector) {
1340 			*next_sector = j->sector;
1341 			n = j->node.rb_left;
1342 		} else
1343 			n = j->node.rb_right;
1344 	}
1345 
1346 	return found;
1347 }
1348 
test_journal_node(struct dm_integrity_c * ic,unsigned int pos,sector_t sector)1349 static bool test_journal_node(struct dm_integrity_c *ic, unsigned int pos, sector_t sector)
1350 {
1351 	struct journal_node *node, *next_node;
1352 	struct rb_node *next;
1353 
1354 	if (unlikely(pos >= ic->journal_entries))
1355 		return false;
1356 	node = &ic->journal_tree[pos];
1357 	if (unlikely(RB_EMPTY_NODE(&node->node)))
1358 		return false;
1359 	if (unlikely(node->sector != sector))
1360 		return false;
1361 
1362 	next = rb_next(&node->node);
1363 	if (unlikely(!next))
1364 		return true;
1365 
1366 	next_node = container_of(next, struct journal_node, node);
1367 	return next_node->sector != sector;
1368 }
1369 
find_newer_committed_node(struct dm_integrity_c * ic,struct journal_node * node)1370 static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node)
1371 {
1372 	struct rb_node *next;
1373 	struct journal_node *next_node;
1374 	unsigned int next_section;
1375 
1376 	BUG_ON(RB_EMPTY_NODE(&node->node));
1377 
1378 	next = rb_next(&node->node);
1379 	if (unlikely(!next))
1380 		return false;
1381 
1382 	next_node = container_of(next, struct journal_node, node);
1383 
1384 	if (next_node->sector != node->sector)
1385 		return false;
1386 
1387 	next_section = (unsigned int)(next_node - ic->journal_tree) / ic->journal_section_entries;
1388 	if (next_section >= ic->committed_section &&
1389 	    next_section < ic->committed_section + ic->n_committed_sections)
1390 		return true;
1391 	if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections)
1392 		return true;
1393 
1394 	return false;
1395 }
1396 
1397 #define TAG_READ	0
1398 #define TAG_WRITE	1
1399 #define TAG_CMP		2
1400 
dm_integrity_rw_tag(struct dm_integrity_c * ic,unsigned char * tag,sector_t * metadata_block,unsigned int * metadata_offset,unsigned int total_size,int op)1401 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block,
1402 			       unsigned int *metadata_offset, unsigned int total_size, int op)
1403 {
1404 #define MAY_BE_FILLER		1
1405 #define MAY_BE_HASH		2
1406 	unsigned int hash_offset = 0;
1407 	unsigned int may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0);
1408 
1409 	do {
1410 		unsigned char *data, *dp;
1411 		struct dm_buffer *b;
1412 		unsigned int to_copy;
1413 		int r;
1414 
1415 		r = dm_integrity_failed(ic);
1416 		if (unlikely(r))
1417 			return r;
1418 
1419 		data = dm_bufio_read(ic->bufio, *metadata_block, &b);
1420 		if (IS_ERR(data))
1421 			return PTR_ERR(data);
1422 
1423 		to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size);
1424 		dp = data + *metadata_offset;
1425 		if (op == TAG_READ) {
1426 			memcpy(tag, dp, to_copy);
1427 		} else if (op == TAG_WRITE) {
1428 			if (memcmp(dp, tag, to_copy)) {
1429 				memcpy(dp, tag, to_copy);
1430 				dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy);
1431 			}
1432 		} else {
1433 			/* e.g.: op == TAG_CMP */
1434 
1435 			if (likely(is_power_of_2(ic->tag_size))) {
1436 				if (unlikely(memcmp(dp, tag, to_copy)))
1437 					if (unlikely(!ic->discard) ||
1438 					    unlikely(memchr_inv(dp, DISCARD_FILLER, to_copy) != NULL)) {
1439 						goto thorough_test;
1440 				}
1441 			} else {
1442 				unsigned int i, ts;
1443 thorough_test:
1444 				ts = total_size;
1445 
1446 				for (i = 0; i < to_copy; i++, ts--) {
1447 					if (unlikely(dp[i] != tag[i]))
1448 						may_be &= ~MAY_BE_HASH;
1449 					if (likely(dp[i] != DISCARD_FILLER))
1450 						may_be &= ~MAY_BE_FILLER;
1451 					hash_offset++;
1452 					if (unlikely(hash_offset == ic->tag_size)) {
1453 						if (unlikely(!may_be)) {
1454 							dm_bufio_release(b);
1455 							return ts;
1456 						}
1457 						hash_offset = 0;
1458 						may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0);
1459 					}
1460 				}
1461 			}
1462 		}
1463 		dm_bufio_release(b);
1464 
1465 		tag += to_copy;
1466 		*metadata_offset += to_copy;
1467 		if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) {
1468 			(*metadata_block)++;
1469 			*metadata_offset = 0;
1470 		}
1471 
1472 		if (unlikely(!is_power_of_2(ic->tag_size)))
1473 			hash_offset = (hash_offset + to_copy) % ic->tag_size;
1474 
1475 		total_size -= to_copy;
1476 	} while (unlikely(total_size));
1477 
1478 	return 0;
1479 #undef MAY_BE_FILLER
1480 #undef MAY_BE_HASH
1481 }
1482 
1483 struct flush_request {
1484 	struct dm_io_request io_req;
1485 	struct dm_io_region io_reg;
1486 	struct dm_integrity_c *ic;
1487 	struct completion comp;
1488 };
1489 
flush_notify(unsigned long error,void * fr_)1490 static void flush_notify(unsigned long error, void *fr_)
1491 {
1492 	struct flush_request *fr = fr_;
1493 
1494 	if (unlikely(error != 0))
1495 		dm_integrity_io_error(fr->ic, "flushing disk cache", -EIO);
1496 	complete(&fr->comp);
1497 }
1498 
dm_integrity_flush_buffers(struct dm_integrity_c * ic,bool flush_data)1499 static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_data)
1500 {
1501 	int r;
1502 	struct flush_request fr;
1503 
1504 	if (!ic->meta_dev)
1505 		flush_data = false;
1506 	if (flush_data) {
1507 		fr.io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1508 		fr.io_req.mem.type = DM_IO_KMEM;
1509 		fr.io_req.mem.ptr.addr = NULL;
1510 		fr.io_req.notify.fn = flush_notify;
1511 		fr.io_req.notify.context = &fr;
1512 		fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio);
1513 		fr.io_reg.bdev = ic->dev->bdev;
1514 		fr.io_reg.sector = 0;
1515 		fr.io_reg.count = 0;
1516 		fr.ic = ic;
1517 		init_completion(&fr.comp);
1518 		r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT);
1519 		BUG_ON(r);
1520 	}
1521 
1522 	r = dm_bufio_write_dirty_buffers(ic->bufio);
1523 	if (unlikely(r))
1524 		dm_integrity_io_error(ic, "writing tags", r);
1525 
1526 	if (flush_data)
1527 		wait_for_completion(&fr.comp);
1528 }
1529 
sleep_on_endio_wait(struct dm_integrity_c * ic)1530 static void sleep_on_endio_wait(struct dm_integrity_c *ic)
1531 {
1532 	DECLARE_WAITQUEUE(wait, current);
1533 
1534 	__add_wait_queue(&ic->endio_wait, &wait);
1535 	__set_current_state(TASK_UNINTERRUPTIBLE);
1536 	spin_unlock_irq(&ic->endio_wait.lock);
1537 	io_schedule();
1538 	spin_lock_irq(&ic->endio_wait.lock);
1539 	__remove_wait_queue(&ic->endio_wait, &wait);
1540 }
1541 
autocommit_fn(struct timer_list * t)1542 static void autocommit_fn(struct timer_list *t)
1543 {
1544 	struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer);
1545 
1546 	if (likely(!dm_integrity_failed(ic)))
1547 		queue_work(ic->commit_wq, &ic->commit_work);
1548 }
1549 
schedule_autocommit(struct dm_integrity_c * ic)1550 static void schedule_autocommit(struct dm_integrity_c *ic)
1551 {
1552 	if (!timer_pending(&ic->autocommit_timer))
1553 		mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies);
1554 }
1555 
submit_flush_bio(struct dm_integrity_c * ic,struct dm_integrity_io * dio)1556 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1557 {
1558 	struct bio *bio;
1559 	unsigned long flags;
1560 
1561 	spin_lock_irqsave(&ic->endio_wait.lock, flags);
1562 	bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1563 	bio_list_add(&ic->flush_bio_list, bio);
1564 	spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1565 
1566 	queue_work(ic->commit_wq, &ic->commit_work);
1567 }
1568 
do_endio(struct dm_integrity_c * ic,struct bio * bio)1569 static void do_endio(struct dm_integrity_c *ic, struct bio *bio)
1570 {
1571 	int r;
1572 
1573 	r = dm_integrity_failed(ic);
1574 	if (unlikely(r) && !bio->bi_status)
1575 		bio->bi_status = errno_to_blk_status(r);
1576 	if (unlikely(ic->synchronous_mode) && bio_op(bio) == REQ_OP_WRITE) {
1577 		unsigned long flags;
1578 
1579 		spin_lock_irqsave(&ic->endio_wait.lock, flags);
1580 		bio_list_add(&ic->synchronous_bios, bio);
1581 		queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
1582 		spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1583 		return;
1584 	}
1585 	bio_endio(bio);
1586 }
1587 
do_endio_flush(struct dm_integrity_c * ic,struct dm_integrity_io * dio)1588 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1589 {
1590 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1591 
1592 	if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))
1593 		submit_flush_bio(ic, dio);
1594 	else
1595 		do_endio(ic, bio);
1596 }
1597 
dec_in_flight(struct dm_integrity_io * dio)1598 static void dec_in_flight(struct dm_integrity_io *dio)
1599 {
1600 	if (atomic_dec_and_test(&dio->in_flight)) {
1601 		struct dm_integrity_c *ic = dio->ic;
1602 		struct bio *bio;
1603 
1604 		remove_range(ic, &dio->range);
1605 
1606 		if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))
1607 			schedule_autocommit(ic);
1608 
1609 		bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1610 		if (unlikely(dio->bi_status) && !bio->bi_status)
1611 			bio->bi_status = dio->bi_status;
1612 		if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {
1613 			dio->range.logical_sector += dio->range.n_sectors;
1614 			bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);
1615 			INIT_WORK(&dio->work, integrity_bio_wait);
1616 			queue_work(ic->offload_wq, &dio->work);
1617 			return;
1618 		}
1619 		do_endio_flush(ic, dio);
1620 	}
1621 }
1622 
integrity_end_io(struct bio * bio)1623 static void integrity_end_io(struct bio *bio)
1624 {
1625 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1626 
1627 	dm_bio_restore(&dio->bio_details, bio);
1628 	if (bio->bi_integrity)
1629 		bio->bi_opf |= REQ_INTEGRITY;
1630 
1631 	if (dio->completion)
1632 		complete(dio->completion);
1633 
1634 	dec_in_flight(dio);
1635 }
1636 
integrity_sector_checksum(struct dm_integrity_c * ic,sector_t sector,const char * data,char * result)1637 static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector,
1638 				      const char *data, char *result)
1639 {
1640 	__le64 sector_le = cpu_to_le64(sector);
1641 	SHASH_DESC_ON_STACK(req, ic->internal_hash);
1642 	int r;
1643 	unsigned int digest_size;
1644 
1645 	req->tfm = ic->internal_hash;
1646 
1647 	r = crypto_shash_init(req);
1648 	if (unlikely(r < 0)) {
1649 		dm_integrity_io_error(ic, "crypto_shash_init", r);
1650 		goto failed;
1651 	}
1652 
1653 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
1654 		r = crypto_shash_update(req, (__u8 *)&ic->sb->salt, SALT_SIZE);
1655 		if (unlikely(r < 0)) {
1656 			dm_integrity_io_error(ic, "crypto_shash_update", r);
1657 			goto failed;
1658 		}
1659 	}
1660 
1661 	r = crypto_shash_update(req, (const __u8 *)&sector_le, sizeof(sector_le));
1662 	if (unlikely(r < 0)) {
1663 		dm_integrity_io_error(ic, "crypto_shash_update", r);
1664 		goto failed;
1665 	}
1666 
1667 	r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT);
1668 	if (unlikely(r < 0)) {
1669 		dm_integrity_io_error(ic, "crypto_shash_update", r);
1670 		goto failed;
1671 	}
1672 
1673 	r = crypto_shash_final(req, result);
1674 	if (unlikely(r < 0)) {
1675 		dm_integrity_io_error(ic, "crypto_shash_final", r);
1676 		goto failed;
1677 	}
1678 
1679 	digest_size = crypto_shash_digestsize(ic->internal_hash);
1680 	if (unlikely(digest_size < ic->tag_size))
1681 		memset(result + digest_size, 0, ic->tag_size - digest_size);
1682 
1683 	return;
1684 
1685 failed:
1686 	/* this shouldn't happen anyway, the hash functions have no reason to fail */
1687 	get_random_bytes(result, ic->tag_size);
1688 }
1689 
integrity_recheck(struct dm_integrity_io * dio,char * checksum)1690 static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum)
1691 {
1692 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1693 	struct dm_integrity_c *ic = dio->ic;
1694 	struct bvec_iter iter;
1695 	struct bio_vec bv;
1696 	sector_t sector, logical_sector, area, offset;
1697 	struct page *page;
1698 
1699 	get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1700 	dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset,
1701 							     &dio->metadata_offset);
1702 	sector = get_data_sector(ic, area, offset);
1703 	logical_sector = dio->range.logical_sector;
1704 
1705 	page = mempool_alloc(&ic->recheck_pool, GFP_NOIO);
1706 
1707 	__bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {
1708 		unsigned pos = 0;
1709 
1710 		do {
1711 			sector_t alignment;
1712 			char *mem;
1713 			char *buffer = page_to_virt(page);
1714 			int r;
1715 			struct dm_io_request io_req;
1716 			struct dm_io_region io_loc;
1717 			io_req.bi_opf = REQ_OP_READ;
1718 			io_req.mem.type = DM_IO_KMEM;
1719 			io_req.mem.ptr.addr = buffer;
1720 			io_req.notify.fn = NULL;
1721 			io_req.client = ic->io;
1722 			io_loc.bdev = ic->dev->bdev;
1723 			io_loc.sector = sector;
1724 			io_loc.count = ic->sectors_per_block;
1725 
1726 			/* Align the bio to logical block size */
1727 			alignment = dio->range.logical_sector | bio_sectors(bio) | (PAGE_SIZE >> SECTOR_SHIFT);
1728 			alignment &= -alignment;
1729 			io_loc.sector = round_down(io_loc.sector, alignment);
1730 			io_loc.count += sector - io_loc.sector;
1731 			buffer += (sector - io_loc.sector) << SECTOR_SHIFT;
1732 			io_loc.count = round_up(io_loc.count, alignment);
1733 
1734 			r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
1735 			if (unlikely(r)) {
1736 				dio->bi_status = errno_to_blk_status(r);
1737 				goto free_ret;
1738 			}
1739 
1740 			integrity_sector_checksum(ic, logical_sector, buffer, checksum);
1741 			r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block,
1742 						&dio->metadata_offset, ic->tag_size, TAG_CMP);
1743 			if (r) {
1744 				if (r > 0) {
1745 					DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
1746 						    bio->bi_bdev, logical_sector);
1747 					atomic64_inc(&ic->number_of_mismatches);
1748 					dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum",
1749 							 bio, logical_sector, 0);
1750 					r = -EILSEQ;
1751 				}
1752 				dio->bi_status = errno_to_blk_status(r);
1753 				goto free_ret;
1754 			}
1755 
1756 			mem = bvec_kmap_local(&bv);
1757 			memcpy(mem + pos, buffer, ic->sectors_per_block << SECTOR_SHIFT);
1758 			kunmap_local(mem);
1759 
1760 			pos += ic->sectors_per_block << SECTOR_SHIFT;
1761 			sector += ic->sectors_per_block;
1762 			logical_sector += ic->sectors_per_block;
1763 		} while (pos < bv.bv_len);
1764 	}
1765 free_ret:
1766 	mempool_free(page, &ic->recheck_pool);
1767 }
1768 
integrity_metadata(struct work_struct * w)1769 static void integrity_metadata(struct work_struct *w)
1770 {
1771 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1772 	struct dm_integrity_c *ic = dio->ic;
1773 
1774 	int r;
1775 
1776 	if (ic->internal_hash) {
1777 		struct bvec_iter iter;
1778 		struct bio_vec bv;
1779 		unsigned int digest_size = crypto_shash_digestsize(ic->internal_hash);
1780 		struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1781 		char *checksums;
1782 		unsigned int extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
1783 		char checksums_onstack[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
1784 		sector_t sector;
1785 		unsigned int sectors_to_process;
1786 
1787 		if (unlikely(ic->mode == 'R'))
1788 			goto skip_io;
1789 
1790 		if (likely(dio->op != REQ_OP_DISCARD))
1791 			checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space,
1792 					    GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1793 		else
1794 			checksums = kmalloc(PAGE_SIZE, GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1795 		if (!checksums) {
1796 			checksums = checksums_onstack;
1797 			if (WARN_ON(extra_space &&
1798 				    digest_size > sizeof(checksums_onstack))) {
1799 				r = -EINVAL;
1800 				goto error;
1801 			}
1802 		}
1803 
1804 		if (unlikely(dio->op == REQ_OP_DISCARD)) {
1805 			unsigned int bi_size = dio->bio_details.bi_iter.bi_size;
1806 			unsigned int max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE;
1807 			unsigned int max_blocks = max_size / ic->tag_size;
1808 
1809 			memset(checksums, DISCARD_FILLER, max_size);
1810 
1811 			while (bi_size) {
1812 				unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block);
1813 
1814 				this_step_blocks = min(this_step_blocks, max_blocks);
1815 				r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1816 							this_step_blocks * ic->tag_size, TAG_WRITE);
1817 				if (unlikely(r)) {
1818 					if (likely(checksums != checksums_onstack))
1819 						kfree(checksums);
1820 					goto error;
1821 				}
1822 
1823 				bi_size -= this_step_blocks << (SECTOR_SHIFT + ic->sb->log2_sectors_per_block);
1824 			}
1825 
1826 			if (likely(checksums != checksums_onstack))
1827 				kfree(checksums);
1828 			goto skip_io;
1829 		}
1830 
1831 		sector = dio->range.logical_sector;
1832 		sectors_to_process = dio->range.n_sectors;
1833 
1834 		__bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {
1835 			struct bio_vec bv_copy = bv;
1836 			unsigned int pos;
1837 			char *mem, *checksums_ptr;
1838 
1839 again:
1840 			mem = bvec_kmap_local(&bv_copy);
1841 			pos = 0;
1842 			checksums_ptr = checksums;
1843 			do {
1844 				integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr);
1845 				checksums_ptr += ic->tag_size;
1846 				sectors_to_process -= ic->sectors_per_block;
1847 				pos += ic->sectors_per_block << SECTOR_SHIFT;
1848 				sector += ic->sectors_per_block;
1849 			} while (pos < bv_copy.bv_len && sectors_to_process && checksums != checksums_onstack);
1850 			kunmap_local(mem);
1851 
1852 			r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1853 						checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);
1854 			if (unlikely(r)) {
1855 				if (likely(checksums != checksums_onstack))
1856 					kfree(checksums);
1857 				if (r > 0) {
1858 					integrity_recheck(dio, checksums_onstack);
1859 					goto skip_io;
1860 				}
1861 				goto error;
1862 			}
1863 
1864 			if (!sectors_to_process)
1865 				break;
1866 
1867 			if (unlikely(pos < bv_copy.bv_len)) {
1868 				bv_copy.bv_offset += pos;
1869 				bv_copy.bv_len -= pos;
1870 				goto again;
1871 			}
1872 		}
1873 
1874 		if (likely(checksums != checksums_onstack))
1875 			kfree(checksums);
1876 	} else {
1877 		struct bio_integrity_payload *bip = dio->bio_details.bi_integrity;
1878 
1879 		if (bip) {
1880 			struct bio_vec biv;
1881 			struct bvec_iter iter;
1882 			unsigned int data_to_process = dio->range.n_sectors;
1883 
1884 			sector_to_block(ic, data_to_process);
1885 			data_to_process *= ic->tag_size;
1886 
1887 			bip_for_each_vec(biv, bip, iter) {
1888 				unsigned char *tag;
1889 				unsigned int this_len;
1890 
1891 				BUG_ON(PageHighMem(biv.bv_page));
1892 				tag = bvec_virt(&biv);
1893 				this_len = min(biv.bv_len, data_to_process);
1894 				r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,
1895 							this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE);
1896 				if (unlikely(r))
1897 					goto error;
1898 				data_to_process -= this_len;
1899 				if (!data_to_process)
1900 					break;
1901 			}
1902 		}
1903 	}
1904 skip_io:
1905 	dec_in_flight(dio);
1906 	return;
1907 error:
1908 	dio->bi_status = errno_to_blk_status(r);
1909 	dec_in_flight(dio);
1910 }
1911 
dm_integrity_check_limits(struct dm_integrity_c * ic,sector_t logical_sector,struct bio * bio)1912 static inline bool dm_integrity_check_limits(struct dm_integrity_c *ic, sector_t logical_sector, struct bio *bio)
1913 {
1914 	if (unlikely(logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {
1915 		DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx",
1916 		      logical_sector, bio_sectors(bio),
1917 		      ic->provided_data_sectors);
1918 		return false;
1919 	}
1920 	if (unlikely((logical_sector | bio_sectors(bio)) & (unsigned int)(ic->sectors_per_block - 1))) {
1921 		DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x",
1922 		      ic->sectors_per_block,
1923 		      logical_sector, bio_sectors(bio));
1924 		return false;
1925 	}
1926 	if (ic->sectors_per_block > 1 && likely(bio_op(bio) != REQ_OP_DISCARD)) {
1927 		struct bvec_iter iter;
1928 		struct bio_vec bv;
1929 
1930 		bio_for_each_segment(bv, bio, iter) {
1931 			if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) {
1932 				DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary",
1933 					bv.bv_offset, bv.bv_len, ic->sectors_per_block);
1934 				return false;
1935 			}
1936 		}
1937 	}
1938 	return true;
1939 }
1940 
dm_integrity_map(struct dm_target * ti,struct bio * bio)1941 static int dm_integrity_map(struct dm_target *ti, struct bio *bio)
1942 {
1943 	struct dm_integrity_c *ic = ti->private;
1944 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1945 	struct bio_integrity_payload *bip;
1946 
1947 	sector_t area, offset;
1948 
1949 	dio->ic = ic;
1950 	dio->bi_status = 0;
1951 	dio->op = bio_op(bio);
1952 
1953 	if (ic->mode == 'I') {
1954 		bio->bi_iter.bi_sector = dm_target_offset(ic->ti, bio->bi_iter.bi_sector);
1955 		dio->integrity_payload = NULL;
1956 		dio->integrity_payload_from_mempool = false;
1957 		dio->integrity_range_locked = false;
1958 		return dm_integrity_map_inline(dio, true);
1959 	}
1960 
1961 	if (unlikely(dio->op == REQ_OP_DISCARD)) {
1962 		if (ti->max_io_len) {
1963 			sector_t sec = dm_target_offset(ti, bio->bi_iter.bi_sector);
1964 			unsigned int log2_max_io_len = __fls(ti->max_io_len);
1965 			sector_t start_boundary = sec >> log2_max_io_len;
1966 			sector_t end_boundary = (sec + bio_sectors(bio) - 1) >> log2_max_io_len;
1967 
1968 			if (start_boundary < end_boundary) {
1969 				sector_t len = ti->max_io_len - (sec & (ti->max_io_len - 1));
1970 
1971 				dm_accept_partial_bio(bio, len);
1972 			}
1973 		}
1974 	}
1975 
1976 	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1977 		submit_flush_bio(ic, dio);
1978 		return DM_MAPIO_SUBMITTED;
1979 	}
1980 
1981 	dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1982 	dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA;
1983 	if (unlikely(dio->fua)) {
1984 		/*
1985 		 * Don't pass down the FUA flag because we have to flush
1986 		 * disk cache anyway.
1987 		 */
1988 		bio->bi_opf &= ~REQ_FUA;
1989 	}
1990 	if (unlikely(!dm_integrity_check_limits(ic, dio->range.logical_sector, bio)))
1991 		return DM_MAPIO_KILL;
1992 
1993 	bip = bio_integrity(bio);
1994 	if (!ic->internal_hash) {
1995 		if (bip) {
1996 			unsigned int wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block;
1997 
1998 			if (ic->log2_tag_size >= 0)
1999 				wanted_tag_size <<= ic->log2_tag_size;
2000 			else
2001 				wanted_tag_size *= ic->tag_size;
2002 			if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) {
2003 				DMERR("Invalid integrity data size %u, expected %u",
2004 				      bip->bip_iter.bi_size, wanted_tag_size);
2005 				return DM_MAPIO_KILL;
2006 			}
2007 		}
2008 	} else {
2009 		if (unlikely(bip != NULL)) {
2010 			DMERR("Unexpected integrity data when using internal hash");
2011 			return DM_MAPIO_KILL;
2012 		}
2013 	}
2014 
2015 	if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ))
2016 		return DM_MAPIO_KILL;
2017 
2018 	get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
2019 	dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
2020 	bio->bi_iter.bi_sector = get_data_sector(ic, area, offset);
2021 
2022 	dm_integrity_map_continue(dio, true);
2023 	return DM_MAPIO_SUBMITTED;
2024 }
2025 
__journal_read_write(struct dm_integrity_io * dio,struct bio * bio,unsigned int journal_section,unsigned int journal_entry)2026 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,
2027 				 unsigned int journal_section, unsigned int journal_entry)
2028 {
2029 	struct dm_integrity_c *ic = dio->ic;
2030 	sector_t logical_sector;
2031 	unsigned int n_sectors;
2032 
2033 	logical_sector = dio->range.logical_sector;
2034 	n_sectors = dio->range.n_sectors;
2035 	do {
2036 		struct bio_vec bv = bio_iovec(bio);
2037 		char *mem;
2038 
2039 		if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors))
2040 			bv.bv_len = n_sectors << SECTOR_SHIFT;
2041 		n_sectors -= bv.bv_len >> SECTOR_SHIFT;
2042 		bio_advance_iter(bio, &bio->bi_iter, bv.bv_len);
2043 retry_kmap:
2044 		mem = kmap_local_page(bv.bv_page);
2045 		if (likely(dio->op == REQ_OP_WRITE))
2046 			flush_dcache_page(bv.bv_page);
2047 
2048 		do {
2049 			struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry);
2050 
2051 			if (unlikely(dio->op == REQ_OP_READ)) {
2052 				struct journal_sector *js;
2053 				char *mem_ptr;
2054 				unsigned int s;
2055 
2056 				if (unlikely(journal_entry_is_inprogress(je))) {
2057 					flush_dcache_page(bv.bv_page);
2058 					kunmap_local(mem);
2059 
2060 					__io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
2061 					goto retry_kmap;
2062 				}
2063 				smp_rmb();
2064 				BUG_ON(journal_entry_get_sector(je) != logical_sector);
2065 				js = access_journal_data(ic, journal_section, journal_entry);
2066 				mem_ptr = mem + bv.bv_offset;
2067 				s = 0;
2068 				do {
2069 					memcpy(mem_ptr, js, JOURNAL_SECTOR_DATA);
2070 					*(commit_id_t *)(mem_ptr + JOURNAL_SECTOR_DATA) = je->last_bytes[s];
2071 					js++;
2072 					mem_ptr += 1 << SECTOR_SHIFT;
2073 				} while (++s < ic->sectors_per_block);
2074 #ifdef INTERNAL_VERIFY
2075 				if (ic->internal_hash) {
2076 					char checksums_onstack[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
2077 
2078 					integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack);
2079 					if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) {
2080 						DMERR_LIMIT("Checksum failed when reading from journal, at sector 0x%llx",
2081 							    logical_sector);
2082 						dm_audit_log_bio(DM_MSG_PREFIX, "journal-checksum",
2083 								 bio, logical_sector, 0);
2084 					}
2085 				}
2086 #endif
2087 			}
2088 
2089 			if (!ic->internal_hash) {
2090 				struct bio_integrity_payload *bip = bio_integrity(bio);
2091 				unsigned int tag_todo = ic->tag_size;
2092 				char *tag_ptr = journal_entry_tag(ic, je);
2093 
2094 				if (bip) {
2095 					do {
2096 						struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter);
2097 						unsigned int tag_now = min(biv.bv_len, tag_todo);
2098 						char *tag_addr;
2099 
2100 						BUG_ON(PageHighMem(biv.bv_page));
2101 						tag_addr = bvec_virt(&biv);
2102 						if (likely(dio->op == REQ_OP_WRITE))
2103 							memcpy(tag_ptr, tag_addr, tag_now);
2104 						else
2105 							memcpy(tag_addr, tag_ptr, tag_now);
2106 						bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now);
2107 						tag_ptr += tag_now;
2108 						tag_todo -= tag_now;
2109 					} while (unlikely(tag_todo));
2110 				} else if (likely(dio->op == REQ_OP_WRITE))
2111 					memset(tag_ptr, 0, tag_todo);
2112 			}
2113 
2114 			if (likely(dio->op == REQ_OP_WRITE)) {
2115 				struct journal_sector *js;
2116 				unsigned int s;
2117 
2118 				js = access_journal_data(ic, journal_section, journal_entry);
2119 				memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT);
2120 
2121 				s = 0;
2122 				do {
2123 					je->last_bytes[s] = js[s].commit_id;
2124 				} while (++s < ic->sectors_per_block);
2125 
2126 				if (ic->internal_hash) {
2127 					unsigned int digest_size = crypto_shash_digestsize(ic->internal_hash);
2128 
2129 					if (unlikely(digest_size > ic->tag_size)) {
2130 						char checksums_onstack[HASH_MAX_DIGESTSIZE];
2131 
2132 						integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack);
2133 						memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size);
2134 					} else
2135 						integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je));
2136 				}
2137 
2138 				journal_entry_set_sector(je, logical_sector);
2139 			}
2140 			logical_sector += ic->sectors_per_block;
2141 
2142 			journal_entry++;
2143 			if (unlikely(journal_entry == ic->journal_section_entries)) {
2144 				journal_entry = 0;
2145 				journal_section++;
2146 				wraparound_section(ic, &journal_section);
2147 			}
2148 
2149 			bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT;
2150 		} while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT);
2151 
2152 		if (unlikely(dio->op == REQ_OP_READ))
2153 			flush_dcache_page(bv.bv_page);
2154 		kunmap_local(mem);
2155 	} while (n_sectors);
2156 
2157 	if (likely(dio->op == REQ_OP_WRITE)) {
2158 		smp_mb();
2159 		if (unlikely(waitqueue_active(&ic->copy_to_journal_wait)))
2160 			wake_up(&ic->copy_to_journal_wait);
2161 		if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold)
2162 			queue_work(ic->commit_wq, &ic->commit_work);
2163 		else
2164 			schedule_autocommit(ic);
2165 	} else
2166 		remove_range(ic, &dio->range);
2167 
2168 	if (unlikely(bio->bi_iter.bi_size)) {
2169 		sector_t area, offset;
2170 
2171 		dio->range.logical_sector = logical_sector;
2172 		get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
2173 		dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
2174 		return true;
2175 	}
2176 
2177 	return false;
2178 }
2179 
dm_integrity_map_continue(struct dm_integrity_io * dio,bool from_map)2180 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)
2181 {
2182 	struct dm_integrity_c *ic = dio->ic;
2183 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2184 	unsigned int journal_section, journal_entry;
2185 	unsigned int journal_read_pos;
2186 	sector_t recalc_sector;
2187 	struct completion read_comp;
2188 	bool discard_retried = false;
2189 	bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;
2190 
2191 	if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')
2192 		need_sync_io = true;
2193 
2194 	if (need_sync_io && from_map) {
2195 		INIT_WORK(&dio->work, integrity_bio_wait);
2196 		queue_work(ic->offload_wq, &dio->work);
2197 		return;
2198 	}
2199 
2200 lock_retry:
2201 	spin_lock_irq(&ic->endio_wait.lock);
2202 retry:
2203 	if (unlikely(dm_integrity_failed(ic))) {
2204 		spin_unlock_irq(&ic->endio_wait.lock);
2205 		do_endio(ic, bio);
2206 		return;
2207 	}
2208 	dio->range.n_sectors = bio_sectors(bio);
2209 	journal_read_pos = NOT_FOUND;
2210 	if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {
2211 		if (dio->op == REQ_OP_WRITE) {
2212 			unsigned int next_entry, i, pos;
2213 			unsigned int ws, we, range_sectors;
2214 
2215 			dio->range.n_sectors = min(dio->range.n_sectors,
2216 						   (sector_t)ic->free_sectors << ic->sb->log2_sectors_per_block);
2217 			if (unlikely(!dio->range.n_sectors)) {
2218 				if (from_map)
2219 					goto offload_to_thread;
2220 				sleep_on_endio_wait(ic);
2221 				goto retry;
2222 			}
2223 			range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;
2224 			ic->free_sectors -= range_sectors;
2225 			journal_section = ic->free_section;
2226 			journal_entry = ic->free_section_entry;
2227 
2228 			next_entry = ic->free_section_entry + range_sectors;
2229 			ic->free_section_entry = next_entry % ic->journal_section_entries;
2230 			ic->free_section += next_entry / ic->journal_section_entries;
2231 			ic->n_uncommitted_sections += next_entry / ic->journal_section_entries;
2232 			wraparound_section(ic, &ic->free_section);
2233 
2234 			pos = journal_section * ic->journal_section_entries + journal_entry;
2235 			ws = journal_section;
2236 			we = journal_entry;
2237 			i = 0;
2238 			do {
2239 				struct journal_entry *je;
2240 
2241 				add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);
2242 				pos++;
2243 				if (unlikely(pos >= ic->journal_entries))
2244 					pos = 0;
2245 
2246 				je = access_journal_entry(ic, ws, we);
2247 				BUG_ON(!journal_entry_is_unused(je));
2248 				journal_entry_set_inprogress(je);
2249 				we++;
2250 				if (unlikely(we == ic->journal_section_entries)) {
2251 					we = 0;
2252 					ws++;
2253 					wraparound_section(ic, &ws);
2254 				}
2255 			} while ((i += ic->sectors_per_block) < dio->range.n_sectors);
2256 
2257 			spin_unlock_irq(&ic->endio_wait.lock);
2258 			goto journal_read_write;
2259 		} else {
2260 			sector_t next_sector;
2261 
2262 			journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2263 			if (likely(journal_read_pos == NOT_FOUND)) {
2264 				if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))
2265 					dio->range.n_sectors = next_sector - dio->range.logical_sector;
2266 			} else {
2267 				unsigned int i;
2268 				unsigned int jp = journal_read_pos + 1;
2269 
2270 				for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {
2271 					if (!test_journal_node(ic, jp, dio->range.logical_sector + i))
2272 						break;
2273 				}
2274 				dio->range.n_sectors = i;
2275 			}
2276 		}
2277 	}
2278 	if (unlikely(!add_new_range(ic, &dio->range, true))) {
2279 		/*
2280 		 * We must not sleep in the request routine because it could
2281 		 * stall bios on current->bio_list.
2282 		 * So, we offload the bio to a workqueue if we have to sleep.
2283 		 */
2284 		if (from_map) {
2285 offload_to_thread:
2286 			spin_unlock_irq(&ic->endio_wait.lock);
2287 			INIT_WORK(&dio->work, integrity_bio_wait);
2288 			queue_work(ic->wait_wq, &dio->work);
2289 			return;
2290 		}
2291 		if (journal_read_pos != NOT_FOUND)
2292 			dio->range.n_sectors = ic->sectors_per_block;
2293 		wait_and_add_new_range(ic, &dio->range);
2294 		/*
2295 		 * wait_and_add_new_range drops the spinlock, so the journal
2296 		 * may have been changed arbitrarily. We need to recheck.
2297 		 * To simplify the code, we restrict I/O size to just one block.
2298 		 */
2299 		if (journal_read_pos != NOT_FOUND) {
2300 			sector_t next_sector;
2301 			unsigned int new_pos;
2302 
2303 			new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2304 			if (unlikely(new_pos != journal_read_pos)) {
2305 				remove_range_unlocked(ic, &dio->range);
2306 				goto retry;
2307 			}
2308 		}
2309 	}
2310 	if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {
2311 		sector_t next_sector;
2312 		unsigned int new_pos;
2313 
2314 		new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2315 		if (unlikely(new_pos != NOT_FOUND) ||
2316 		    unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) {
2317 			remove_range_unlocked(ic, &dio->range);
2318 			spin_unlock_irq(&ic->endio_wait.lock);
2319 			queue_work(ic->commit_wq, &ic->commit_work);
2320 			flush_workqueue(ic->commit_wq);
2321 			queue_work(ic->writer_wq, &ic->writer_work);
2322 			flush_workqueue(ic->writer_wq);
2323 			discard_retried = true;
2324 			goto lock_retry;
2325 		}
2326 	}
2327 	recalc_sector = le64_to_cpu(ic->sb->recalc_sector);
2328 	spin_unlock_irq(&ic->endio_wait.lock);
2329 
2330 	if (unlikely(journal_read_pos != NOT_FOUND)) {
2331 		journal_section = journal_read_pos / ic->journal_section_entries;
2332 		journal_entry = journal_read_pos % ic->journal_section_entries;
2333 		goto journal_read_write;
2334 	}
2335 
2336 	if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {
2337 		if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
2338 				     dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {
2339 			struct bitmap_block_status *bbs;
2340 
2341 			bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);
2342 			spin_lock(&bbs->bio_queue_lock);
2343 			bio_list_add(&bbs->bio_queue, bio);
2344 			spin_unlock(&bbs->bio_queue_lock);
2345 			queue_work(ic->writer_wq, &bbs->work);
2346 			return;
2347 		}
2348 	}
2349 
2350 	dio->in_flight = (atomic_t)ATOMIC_INIT(2);
2351 
2352 	if (need_sync_io) {
2353 		init_completion(&read_comp);
2354 		dio->completion = &read_comp;
2355 	} else
2356 		dio->completion = NULL;
2357 
2358 	dm_bio_record(&dio->bio_details, bio);
2359 	bio_set_dev(bio, ic->dev->bdev);
2360 	bio->bi_integrity = NULL;
2361 	bio->bi_opf &= ~REQ_INTEGRITY;
2362 	bio->bi_end_io = integrity_end_io;
2363 	bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;
2364 
2365 	if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {
2366 		integrity_metadata(&dio->work);
2367 		dm_integrity_flush_buffers(ic, false);
2368 
2369 		dio->in_flight = (atomic_t)ATOMIC_INIT(1);
2370 		dio->completion = NULL;
2371 
2372 		submit_bio_noacct(bio);
2373 
2374 		return;
2375 	}
2376 
2377 	submit_bio_noacct(bio);
2378 
2379 	if (need_sync_io) {
2380 		wait_for_completion_io(&read_comp);
2381 		if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
2382 		    dio->range.logical_sector + dio->range.n_sectors > recalc_sector)
2383 			goto skip_check;
2384 		if (ic->mode == 'B') {
2385 			if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,
2386 					     dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))
2387 				goto skip_check;
2388 		}
2389 
2390 		if (likely(!bio->bi_status))
2391 			integrity_metadata(&dio->work);
2392 		else
2393 skip_check:
2394 			dec_in_flight(dio);
2395 	} else {
2396 		INIT_WORK(&dio->work, integrity_metadata);
2397 		queue_work(ic->metadata_wq, &dio->work);
2398 	}
2399 
2400 	return;
2401 
2402 journal_read_write:
2403 	if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))
2404 		goto lock_retry;
2405 
2406 	do_endio_flush(ic, dio);
2407 }
2408 
dm_integrity_map_inline(struct dm_integrity_io * dio,bool from_map)2409 static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map)
2410 {
2411 	struct dm_integrity_c *ic = dio->ic;
2412 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2413 	struct bio_integrity_payload *bip;
2414 	unsigned ret;
2415 	sector_t recalc_sector;
2416 
2417 	if (unlikely(bio_integrity(bio))) {
2418 		bio->bi_status = BLK_STS_NOTSUPP;
2419 		bio_endio(bio);
2420 		return DM_MAPIO_SUBMITTED;
2421 	}
2422 
2423 	bio_set_dev(bio, ic->dev->bdev);
2424 	if (unlikely((bio->bi_opf & REQ_PREFLUSH) != 0))
2425 		return DM_MAPIO_REMAPPED;
2426 
2427 retry:
2428 	if (!dio->integrity_payload) {
2429 		unsigned digest_size, extra_size;
2430 		dio->payload_len = ic->tuple_size * (bio_sectors(bio) >> ic->sb->log2_sectors_per_block);
2431 		digest_size = crypto_shash_digestsize(ic->internal_hash);
2432 		extra_size = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
2433 		dio->payload_len += extra_size;
2434 		dio->integrity_payload = kmalloc(dio->payload_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
2435 		if (unlikely(!dio->integrity_payload)) {
2436 			const unsigned x_size = PAGE_SIZE << 1;
2437 			if (dio->payload_len > x_size) {
2438 				unsigned sectors = ((x_size - extra_size) / ic->tuple_size) << ic->sb->log2_sectors_per_block;
2439 				if (WARN_ON(!sectors || sectors >= bio_sectors(bio))) {
2440 					bio->bi_status = BLK_STS_NOTSUPP;
2441 					bio_endio(bio);
2442 					return DM_MAPIO_SUBMITTED;
2443 				}
2444 				dm_accept_partial_bio(bio, sectors);
2445 				goto retry;
2446 			}
2447 		}
2448 	}
2449 
2450 	dio->range.logical_sector = bio->bi_iter.bi_sector;
2451 	dio->range.n_sectors = bio_sectors(bio);
2452 
2453 	if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)))
2454 		goto skip_spinlock;
2455 #ifdef CONFIG_64BIT
2456 	/*
2457 	 * On 64-bit CPUs we can optimize the lock away (so that it won't cause
2458 	 * cache line bouncing) and use acquire/release barriers instead.
2459 	 *
2460 	 * Paired with smp_store_release in integrity_recalc_inline.
2461 	 */
2462 	recalc_sector = le64_to_cpu(smp_load_acquire(&ic->sb->recalc_sector));
2463 	if (likely(dio->range.logical_sector + dio->range.n_sectors <= recalc_sector))
2464 		goto skip_spinlock;
2465 #endif
2466 	spin_lock_irq(&ic->endio_wait.lock);
2467 	recalc_sector = le64_to_cpu(ic->sb->recalc_sector);
2468 	if (dio->range.logical_sector + dio->range.n_sectors <= recalc_sector)
2469 		goto skip_unlock;
2470 	if (unlikely(!add_new_range(ic, &dio->range, true))) {
2471 		if (from_map) {
2472 			spin_unlock_irq(&ic->endio_wait.lock);
2473 			INIT_WORK(&dio->work, integrity_bio_wait);
2474 			queue_work(ic->wait_wq, &dio->work);
2475 			return DM_MAPIO_SUBMITTED;
2476 		}
2477 		wait_and_add_new_range(ic, &dio->range);
2478 	}
2479 	dio->integrity_range_locked = true;
2480 skip_unlock:
2481 	spin_unlock_irq(&ic->endio_wait.lock);
2482 skip_spinlock:
2483 
2484 	if (unlikely(!dio->integrity_payload)) {
2485 		dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO));
2486 		dio->integrity_payload_from_mempool = true;
2487 	}
2488 
2489 	dio->bio_details.bi_iter = bio->bi_iter;
2490 
2491 	if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) {
2492 		return DM_MAPIO_KILL;
2493 	}
2494 
2495 	bio->bi_iter.bi_sector += ic->start + SB_SECTORS;
2496 
2497 	bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
2498 	if (IS_ERR(bip)) {
2499 		bio->bi_status = errno_to_blk_status(PTR_ERR(bip));
2500 		bio_endio(bio);
2501 		return DM_MAPIO_SUBMITTED;
2502 	}
2503 
2504 	if (dio->op == REQ_OP_WRITE) {
2505 		unsigned pos = 0;
2506 		while (dio->bio_details.bi_iter.bi_size) {
2507 			struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);
2508 			const char *mem = bvec_kmap_local(&bv);
2509 			if (ic->tag_size < ic->tuple_size)
2510 				memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tuple_size);
2511 			integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, dio->integrity_payload + pos);
2512 			kunmap_local(mem);
2513 			pos += ic->tuple_size;
2514 			bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);
2515 		}
2516 	}
2517 
2518 	ret = bio_integrity_add_page(bio, virt_to_page(dio->integrity_payload),
2519 					dio->payload_len, offset_in_page(dio->integrity_payload));
2520 	if (unlikely(ret != dio->payload_len)) {
2521 		bio->bi_status = BLK_STS_RESOURCE;
2522 		bio_endio(bio);
2523 		return DM_MAPIO_SUBMITTED;
2524 	}
2525 
2526 	return DM_MAPIO_REMAPPED;
2527 }
2528 
dm_integrity_free_payload(struct dm_integrity_io * dio)2529 static inline void dm_integrity_free_payload(struct dm_integrity_io *dio)
2530 {
2531 	struct dm_integrity_c *ic = dio->ic;
2532 	if (unlikely(dio->integrity_payload_from_mempool))
2533 		mempool_free(virt_to_page(dio->integrity_payload), &ic->recheck_pool);
2534 	else
2535 		kfree(dio->integrity_payload);
2536 	dio->integrity_payload = NULL;
2537 	dio->integrity_payload_from_mempool = false;
2538 }
2539 
dm_integrity_inline_recheck(struct work_struct * w)2540 static void dm_integrity_inline_recheck(struct work_struct *w)
2541 {
2542 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
2543 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2544 	struct dm_integrity_c *ic = dio->ic;
2545 	struct bio *outgoing_bio;
2546 	void *outgoing_data;
2547 
2548 	dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO));
2549 	dio->integrity_payload_from_mempool = true;
2550 
2551 	outgoing_data = dio->integrity_payload + PAGE_SIZE;
2552 
2553 	while (dio->bio_details.bi_iter.bi_size) {
2554 		char digest[HASH_MAX_DIGESTSIZE];
2555 		int r;
2556 		struct bio_integrity_payload *bip;
2557 		struct bio_vec bv;
2558 		char *mem;
2559 
2560 		outgoing_bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_READ, GFP_NOIO, &ic->recheck_bios);
2561 
2562 		r = bio_add_page(outgoing_bio, virt_to_page(outgoing_data), ic->sectors_per_block << SECTOR_SHIFT, 0);
2563 		if (unlikely(r != (ic->sectors_per_block << SECTOR_SHIFT))) {
2564 			bio_put(outgoing_bio);
2565 			bio->bi_status = BLK_STS_RESOURCE;
2566 			bio_endio(bio);
2567 			return;
2568 		}
2569 
2570 		bip = bio_integrity_alloc(outgoing_bio, GFP_NOIO, 1);
2571 		if (IS_ERR(bip)) {
2572 			bio_put(outgoing_bio);
2573 			bio->bi_status = errno_to_blk_status(PTR_ERR(bip));
2574 			bio_endio(bio);
2575 			return;
2576 		}
2577 
2578 		r = bio_integrity_add_page(outgoing_bio, virt_to_page(dio->integrity_payload), ic->tuple_size, 0);
2579 		if (unlikely(r != ic->tuple_size)) {
2580 			bio_put(outgoing_bio);
2581 			bio->bi_status = BLK_STS_RESOURCE;
2582 			bio_endio(bio);
2583 			return;
2584 		}
2585 
2586 		outgoing_bio->bi_iter.bi_sector = dio->bio_details.bi_iter.bi_sector + ic->start + SB_SECTORS;
2587 
2588 		r = submit_bio_wait(outgoing_bio);
2589 		if (unlikely(r != 0)) {
2590 			bio_put(outgoing_bio);
2591 			bio->bi_status = errno_to_blk_status(r);
2592 			bio_endio(bio);
2593 			return;
2594 		}
2595 		bio_put(outgoing_bio);
2596 
2597 		integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, outgoing_data, digest);
2598 		if (unlikely(memcmp(digest, dio->integrity_payload, min(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)))) {
2599 			DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
2600 				ic->dev->bdev, dio->bio_details.bi_iter.bi_sector);
2601 			atomic64_inc(&ic->number_of_mismatches);
2602 			dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum",
2603 				bio, dio->bio_details.bi_iter.bi_sector, 0);
2604 
2605 			bio->bi_status = BLK_STS_PROTECTION;
2606 			bio_endio(bio);
2607 			return;
2608 		}
2609 
2610 		bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);
2611 		mem = bvec_kmap_local(&bv);
2612 		memcpy(mem, outgoing_data, ic->sectors_per_block << SECTOR_SHIFT);
2613 		kunmap_local(mem);
2614 
2615 		bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);
2616 	}
2617 
2618 	bio_endio(bio);
2619 }
2620 
dm_integrity_end_io(struct dm_target * ti,struct bio * bio,blk_status_t * status)2621 static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
2622 {
2623 	struct dm_integrity_c *ic = ti->private;
2624 	if (ic->mode == 'I') {
2625 		struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
2626 		if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK)) {
2627 			unsigned pos = 0;
2628 			if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
2629 			    unlikely(dio->integrity_range_locked))
2630 				goto skip_check;
2631 			while (dio->bio_details.bi_iter.bi_size) {
2632 				char digest[HASH_MAX_DIGESTSIZE];
2633 				struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);
2634 				char *mem = bvec_kmap_local(&bv);
2635 				//memset(mem, 0xff, ic->sectors_per_block << SECTOR_SHIFT);
2636 				integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, digest);
2637 				if (unlikely(memcmp(digest, dio->integrity_payload + pos,
2638 						min(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)))) {
2639 					kunmap_local(mem);
2640 					dm_integrity_free_payload(dio);
2641 					INIT_WORK(&dio->work, dm_integrity_inline_recheck);
2642 					queue_work(ic->offload_wq, &dio->work);
2643 					return DM_ENDIO_INCOMPLETE;
2644 				}
2645 				kunmap_local(mem);
2646 				pos += ic->tuple_size;
2647 				bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);
2648 			}
2649 		}
2650 skip_check:
2651 		dm_integrity_free_payload(dio);
2652 		if (unlikely(dio->integrity_range_locked))
2653 			remove_range(ic, &dio->range);
2654 	}
2655 	return DM_ENDIO_DONE;
2656 }
2657 
integrity_bio_wait(struct work_struct * w)2658 static void integrity_bio_wait(struct work_struct *w)
2659 {
2660 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
2661 	struct dm_integrity_c *ic = dio->ic;
2662 
2663 	if (ic->mode == 'I') {
2664 		struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2665 		int r = dm_integrity_map_inline(dio, false);
2666 		switch (r) {
2667 			case DM_MAPIO_KILL:
2668 				bio->bi_status = BLK_STS_IOERR;
2669 				fallthrough;
2670 			case DM_MAPIO_REMAPPED:
2671 				submit_bio_noacct(bio);
2672 				fallthrough;
2673 			case DM_MAPIO_SUBMITTED:
2674 				return;
2675 			default:
2676 				BUG();
2677 		}
2678 	} else {
2679 		dm_integrity_map_continue(dio, false);
2680 	}
2681 }
2682 
pad_uncommitted(struct dm_integrity_c * ic)2683 static void pad_uncommitted(struct dm_integrity_c *ic)
2684 {
2685 	if (ic->free_section_entry) {
2686 		ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry;
2687 		ic->free_section_entry = 0;
2688 		ic->free_section++;
2689 		wraparound_section(ic, &ic->free_section);
2690 		ic->n_uncommitted_sections++;
2691 	}
2692 	if (WARN_ON(ic->journal_sections * ic->journal_section_entries !=
2693 		    (ic->n_uncommitted_sections + ic->n_committed_sections) *
2694 		    ic->journal_section_entries + ic->free_sectors)) {
2695 		DMCRIT("journal_sections %u, journal_section_entries %u, "
2696 		       "n_uncommitted_sections %u, n_committed_sections %u, "
2697 		       "journal_section_entries %u, free_sectors %u",
2698 		       ic->journal_sections, ic->journal_section_entries,
2699 		       ic->n_uncommitted_sections, ic->n_committed_sections,
2700 		       ic->journal_section_entries, ic->free_sectors);
2701 	}
2702 }
2703 
integrity_commit(struct work_struct * w)2704 static void integrity_commit(struct work_struct *w)
2705 {
2706 	struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work);
2707 	unsigned int commit_start, commit_sections;
2708 	unsigned int i, j, n;
2709 	struct bio *flushes;
2710 
2711 	del_timer(&ic->autocommit_timer);
2712 
2713 	if (ic->mode == 'I')
2714 		return;
2715 
2716 	spin_lock_irq(&ic->endio_wait.lock);
2717 	flushes = bio_list_get(&ic->flush_bio_list);
2718 	if (unlikely(ic->mode != 'J')) {
2719 		spin_unlock_irq(&ic->endio_wait.lock);
2720 		dm_integrity_flush_buffers(ic, true);
2721 		goto release_flush_bios;
2722 	}
2723 
2724 	pad_uncommitted(ic);
2725 	commit_start = ic->uncommitted_section;
2726 	commit_sections = ic->n_uncommitted_sections;
2727 	spin_unlock_irq(&ic->endio_wait.lock);
2728 
2729 	if (!commit_sections)
2730 		goto release_flush_bios;
2731 
2732 	ic->wrote_to_journal = true;
2733 
2734 	i = commit_start;
2735 	for (n = 0; n < commit_sections; n++) {
2736 		for (j = 0; j < ic->journal_section_entries; j++) {
2737 			struct journal_entry *je;
2738 
2739 			je = access_journal_entry(ic, i, j);
2740 			io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
2741 		}
2742 		for (j = 0; j < ic->journal_section_sectors; j++) {
2743 			struct journal_sector *js;
2744 
2745 			js = access_journal(ic, i, j);
2746 			js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq);
2747 		}
2748 		i++;
2749 		if (unlikely(i >= ic->journal_sections))
2750 			ic->commit_seq = next_commit_seq(ic->commit_seq);
2751 		wraparound_section(ic, &i);
2752 	}
2753 	smp_rmb();
2754 
2755 	write_journal(ic, commit_start, commit_sections);
2756 
2757 	spin_lock_irq(&ic->endio_wait.lock);
2758 	ic->uncommitted_section += commit_sections;
2759 	wraparound_section(ic, &ic->uncommitted_section);
2760 	ic->n_uncommitted_sections -= commit_sections;
2761 	ic->n_committed_sections += commit_sections;
2762 	spin_unlock_irq(&ic->endio_wait.lock);
2763 
2764 	if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold)
2765 		queue_work(ic->writer_wq, &ic->writer_work);
2766 
2767 release_flush_bios:
2768 	while (flushes) {
2769 		struct bio *next = flushes->bi_next;
2770 
2771 		flushes->bi_next = NULL;
2772 		do_endio(ic, flushes);
2773 		flushes = next;
2774 	}
2775 }
2776 
complete_copy_from_journal(unsigned long error,void * context)2777 static void complete_copy_from_journal(unsigned long error, void *context)
2778 {
2779 	struct journal_io *io = context;
2780 	struct journal_completion *comp = io->comp;
2781 	struct dm_integrity_c *ic = comp->ic;
2782 
2783 	remove_range(ic, &io->range);
2784 	mempool_free(io, &ic->journal_io_mempool);
2785 	if (unlikely(error != 0))
2786 		dm_integrity_io_error(ic, "copying from journal", -EIO);
2787 	complete_journal_op(comp);
2788 }
2789 
restore_last_bytes(struct dm_integrity_c * ic,struct journal_sector * js,struct journal_entry * je)2790 static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js,
2791 			       struct journal_entry *je)
2792 {
2793 	unsigned int s = 0;
2794 
2795 	do {
2796 		js->commit_id = je->last_bytes[s];
2797 		js++;
2798 	} while (++s < ic->sectors_per_block);
2799 }
2800 
do_journal_write(struct dm_integrity_c * ic,unsigned int write_start,unsigned int write_sections,bool from_replay)2801 static void do_journal_write(struct dm_integrity_c *ic, unsigned int write_start,
2802 			     unsigned int write_sections, bool from_replay)
2803 {
2804 	unsigned int i, j, n;
2805 	struct journal_completion comp;
2806 	struct blk_plug plug;
2807 
2808 	blk_start_plug(&plug);
2809 
2810 	comp.ic = ic;
2811 	comp.in_flight = (atomic_t)ATOMIC_INIT(1);
2812 	init_completion(&comp.comp);
2813 
2814 	i = write_start;
2815 	for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) {
2816 #ifndef INTERNAL_VERIFY
2817 		if (unlikely(from_replay))
2818 #endif
2819 			rw_section_mac(ic, i, false);
2820 		for (j = 0; j < ic->journal_section_entries; j++) {
2821 			struct journal_entry *je = access_journal_entry(ic, i, j);
2822 			sector_t sec, area, offset;
2823 			unsigned int k, l, next_loop;
2824 			sector_t metadata_block;
2825 			unsigned int metadata_offset;
2826 			struct journal_io *io;
2827 
2828 			if (journal_entry_is_unused(je))
2829 				continue;
2830 			BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay);
2831 			sec = journal_entry_get_sector(je);
2832 			if (unlikely(from_replay)) {
2833 				if (unlikely(sec & (unsigned int)(ic->sectors_per_block - 1))) {
2834 					dm_integrity_io_error(ic, "invalid sector in journal", -EIO);
2835 					sec &= ~(sector_t)(ic->sectors_per_block - 1);
2836 				}
2837 				if (unlikely(sec >= ic->provided_data_sectors)) {
2838 					journal_entry_set_unused(je);
2839 					continue;
2840 				}
2841 			}
2842 			get_area_and_offset(ic, sec, &area, &offset);
2843 			restore_last_bytes(ic, access_journal_data(ic, i, j), je);
2844 			for (k = j + 1; k < ic->journal_section_entries; k++) {
2845 				struct journal_entry *je2 = access_journal_entry(ic, i, k);
2846 				sector_t sec2, area2, offset2;
2847 
2848 				if (journal_entry_is_unused(je2))
2849 					break;
2850 				BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay);
2851 				sec2 = journal_entry_get_sector(je2);
2852 				if (unlikely(sec2 >= ic->provided_data_sectors))
2853 					break;
2854 				get_area_and_offset(ic, sec2, &area2, &offset2);
2855 				if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block))
2856 					break;
2857 				restore_last_bytes(ic, access_journal_data(ic, i, k), je2);
2858 			}
2859 			next_loop = k - 1;
2860 
2861 			io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO);
2862 			io->comp = &comp;
2863 			io->range.logical_sector = sec;
2864 			io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block;
2865 
2866 			spin_lock_irq(&ic->endio_wait.lock);
2867 			add_new_range_and_wait(ic, &io->range);
2868 
2869 			if (likely(!from_replay)) {
2870 				struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries];
2871 
2872 				/* don't write if there is newer committed sector */
2873 				while (j < k && find_newer_committed_node(ic, &section_node[j])) {
2874 					struct journal_entry *je2 = access_journal_entry(ic, i, j);
2875 
2876 					journal_entry_set_unused(je2);
2877 					remove_journal_node(ic, &section_node[j]);
2878 					j++;
2879 					sec += ic->sectors_per_block;
2880 					offset += ic->sectors_per_block;
2881 				}
2882 				while (j < k && find_newer_committed_node(ic, &section_node[k - 1])) {
2883 					struct journal_entry *je2 = access_journal_entry(ic, i, k - 1);
2884 
2885 					journal_entry_set_unused(je2);
2886 					remove_journal_node(ic, &section_node[k - 1]);
2887 					k--;
2888 				}
2889 				if (j == k) {
2890 					remove_range_unlocked(ic, &io->range);
2891 					spin_unlock_irq(&ic->endio_wait.lock);
2892 					mempool_free(io, &ic->journal_io_mempool);
2893 					goto skip_io;
2894 				}
2895 				for (l = j; l < k; l++)
2896 					remove_journal_node(ic, &section_node[l]);
2897 			}
2898 			spin_unlock_irq(&ic->endio_wait.lock);
2899 
2900 			metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
2901 			for (l = j; l < k; l++) {
2902 				int r;
2903 				struct journal_entry *je2 = access_journal_entry(ic, i, l);
2904 
2905 				if (
2906 #ifndef INTERNAL_VERIFY
2907 				    unlikely(from_replay) &&
2908 #endif
2909 				    ic->internal_hash) {
2910 					char test_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
2911 
2912 					integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block),
2913 								  (char *)access_journal_data(ic, i, l), test_tag);
2914 					if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) {
2915 						dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ);
2916 						dm_audit_log_target(DM_MSG_PREFIX, "integrity-replay-journal", ic->ti, 0);
2917 					}
2918 				}
2919 
2920 				journal_entry_set_unused(je2);
2921 				r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset,
2922 							ic->tag_size, TAG_WRITE);
2923 				if (unlikely(r))
2924 					dm_integrity_io_error(ic, "reading tags", r);
2925 			}
2926 
2927 			atomic_inc(&comp.in_flight);
2928 			copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block,
2929 					  (k - j) << ic->sb->log2_sectors_per_block,
2930 					  get_data_sector(ic, area, offset),
2931 					  complete_copy_from_journal, io);
2932 skip_io:
2933 			j = next_loop;
2934 		}
2935 	}
2936 
2937 	dm_bufio_write_dirty_buffers_async(ic->bufio);
2938 
2939 	blk_finish_plug(&plug);
2940 
2941 	complete_journal_op(&comp);
2942 	wait_for_completion_io(&comp.comp);
2943 
2944 	dm_integrity_flush_buffers(ic, true);
2945 }
2946 
integrity_writer(struct work_struct * w)2947 static void integrity_writer(struct work_struct *w)
2948 {
2949 	struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work);
2950 	unsigned int write_start, write_sections;
2951 	unsigned int prev_free_sectors;
2952 
2953 	spin_lock_irq(&ic->endio_wait.lock);
2954 	write_start = ic->committed_section;
2955 	write_sections = ic->n_committed_sections;
2956 	spin_unlock_irq(&ic->endio_wait.lock);
2957 
2958 	if (!write_sections)
2959 		return;
2960 
2961 	do_journal_write(ic, write_start, write_sections, false);
2962 
2963 	spin_lock_irq(&ic->endio_wait.lock);
2964 
2965 	ic->committed_section += write_sections;
2966 	wraparound_section(ic, &ic->committed_section);
2967 	ic->n_committed_sections -= write_sections;
2968 
2969 	prev_free_sectors = ic->free_sectors;
2970 	ic->free_sectors += write_sections * ic->journal_section_entries;
2971 	if (unlikely(!prev_free_sectors))
2972 		wake_up_locked(&ic->endio_wait);
2973 
2974 	spin_unlock_irq(&ic->endio_wait.lock);
2975 }
2976 
recalc_write_super(struct dm_integrity_c * ic)2977 static void recalc_write_super(struct dm_integrity_c *ic)
2978 {
2979 	int r;
2980 
2981 	dm_integrity_flush_buffers(ic, false);
2982 	if (dm_integrity_failed(ic))
2983 		return;
2984 
2985 	r = sync_rw_sb(ic, REQ_OP_WRITE);
2986 	if (unlikely(r))
2987 		dm_integrity_io_error(ic, "writing superblock", r);
2988 }
2989 
integrity_recalc(struct work_struct * w)2990 static void integrity_recalc(struct work_struct *w)
2991 {
2992 	struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work);
2993 	size_t recalc_tags_size;
2994 	u8 *recalc_buffer = NULL;
2995 	u8 *recalc_tags = NULL;
2996 	struct dm_integrity_range range;
2997 	struct dm_io_request io_req;
2998 	struct dm_io_region io_loc;
2999 	sector_t area, offset;
3000 	sector_t metadata_block;
3001 	unsigned int metadata_offset;
3002 	sector_t logical_sector, n_sectors;
3003 	__u8 *t;
3004 	unsigned int i;
3005 	int r;
3006 	unsigned int super_counter = 0;
3007 	unsigned recalc_sectors = RECALC_SECTORS;
3008 
3009 retry:
3010 	recalc_buffer = __vmalloc(recalc_sectors << SECTOR_SHIFT, GFP_NOIO);
3011 	if (!recalc_buffer) {
3012 oom:
3013 		recalc_sectors >>= 1;
3014 		if (recalc_sectors >= 1U << ic->sb->log2_sectors_per_block)
3015 			goto retry;
3016 		DMCRIT("out of memory for recalculate buffer - recalculation disabled");
3017 		goto free_ret;
3018 	}
3019 	recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size;
3020 	if (crypto_shash_digestsize(ic->internal_hash) > ic->tag_size)
3021 		recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tag_size;
3022 	recalc_tags = kvmalloc(recalc_tags_size, GFP_NOIO);
3023 	if (!recalc_tags) {
3024 		vfree(recalc_buffer);
3025 		recalc_buffer = NULL;
3026 		goto oom;
3027 	}
3028 
3029 	DEBUG_print("start recalculation... (position %llx)\n", le64_to_cpu(ic->sb->recalc_sector));
3030 
3031 	spin_lock_irq(&ic->endio_wait.lock);
3032 
3033 next_chunk:
3034 
3035 	if (unlikely(dm_post_suspending(ic->ti)))
3036 		goto unlock_ret;
3037 
3038 	range.logical_sector = le64_to_cpu(ic->sb->recalc_sector);
3039 	if (unlikely(range.logical_sector >= ic->provided_data_sectors)) {
3040 		if (ic->mode == 'B') {
3041 			block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3042 			DEBUG_print("queue_delayed_work: bitmap_flush_work\n");
3043 			queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
3044 		}
3045 		goto unlock_ret;
3046 	}
3047 
3048 	get_area_and_offset(ic, range.logical_sector, &area, &offset);
3049 	range.n_sectors = min((sector_t)recalc_sectors, ic->provided_data_sectors - range.logical_sector);
3050 	if (!ic->meta_dev)
3051 		range.n_sectors = min(range.n_sectors, ((sector_t)1U << ic->sb->log2_interleave_sectors) - (unsigned int)offset);
3052 
3053 	add_new_range_and_wait(ic, &range);
3054 	spin_unlock_irq(&ic->endio_wait.lock);
3055 	logical_sector = range.logical_sector;
3056 	n_sectors = range.n_sectors;
3057 
3058 	if (ic->mode == 'B') {
3059 		if (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, n_sectors, BITMAP_OP_TEST_ALL_CLEAR))
3060 			goto advance_and_next;
3061 
3062 		while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector,
3063 				       ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) {
3064 			logical_sector += ic->sectors_per_block;
3065 			n_sectors -= ic->sectors_per_block;
3066 			cond_resched();
3067 		}
3068 		while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector + n_sectors - ic->sectors_per_block,
3069 				       ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) {
3070 			n_sectors -= ic->sectors_per_block;
3071 			cond_resched();
3072 		}
3073 		get_area_and_offset(ic, logical_sector, &area, &offset);
3074 	}
3075 
3076 	DEBUG_print("recalculating: %llx, %llx\n", logical_sector, n_sectors);
3077 
3078 	if (unlikely(++super_counter == RECALC_WRITE_SUPER)) {
3079 		recalc_write_super(ic);
3080 		if (ic->mode == 'B')
3081 			queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval);
3082 
3083 		super_counter = 0;
3084 	}
3085 
3086 	if (unlikely(dm_integrity_failed(ic)))
3087 		goto err;
3088 
3089 	io_req.bi_opf = REQ_OP_READ;
3090 	io_req.mem.type = DM_IO_VMA;
3091 	io_req.mem.ptr.addr = recalc_buffer;
3092 	io_req.notify.fn = NULL;
3093 	io_req.client = ic->io;
3094 	io_loc.bdev = ic->dev->bdev;
3095 	io_loc.sector = get_data_sector(ic, area, offset);
3096 	io_loc.count = n_sectors;
3097 
3098 	r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
3099 	if (unlikely(r)) {
3100 		dm_integrity_io_error(ic, "reading data", r);
3101 		goto err;
3102 	}
3103 
3104 	t = recalc_tags;
3105 	for (i = 0; i < n_sectors; i += ic->sectors_per_block) {
3106 		integrity_sector_checksum(ic, logical_sector + i, recalc_buffer + (i << SECTOR_SHIFT), t);
3107 		t += ic->tag_size;
3108 	}
3109 
3110 	metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
3111 
3112 	r = dm_integrity_rw_tag(ic, recalc_tags, &metadata_block, &metadata_offset, t - recalc_tags, TAG_WRITE);
3113 	if (unlikely(r)) {
3114 		dm_integrity_io_error(ic, "writing tags", r);
3115 		goto err;
3116 	}
3117 
3118 	if (ic->mode == 'B') {
3119 		sector_t start, end;
3120 
3121 		start = (range.logical_sector >>
3122 			 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) <<
3123 			(ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
3124 		end = ((range.logical_sector + range.n_sectors) >>
3125 		       (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) <<
3126 			(ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
3127 		block_bitmap_op(ic, ic->recalc_bitmap, start, end - start, BITMAP_OP_CLEAR);
3128 	}
3129 
3130 advance_and_next:
3131 	cond_resched();
3132 
3133 	spin_lock_irq(&ic->endio_wait.lock);
3134 	remove_range_unlocked(ic, &range);
3135 	ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors);
3136 	goto next_chunk;
3137 
3138 err:
3139 	remove_range(ic, &range);
3140 	goto free_ret;
3141 
3142 unlock_ret:
3143 	spin_unlock_irq(&ic->endio_wait.lock);
3144 
3145 	recalc_write_super(ic);
3146 
3147 free_ret:
3148 	vfree(recalc_buffer);
3149 	kvfree(recalc_tags);
3150 }
3151 
integrity_recalc_inline(struct work_struct * w)3152 static void integrity_recalc_inline(struct work_struct *w)
3153 {
3154 	struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work);
3155 	size_t recalc_tags_size;
3156 	u8 *recalc_buffer = NULL;
3157 	u8 *recalc_tags = NULL;
3158 	struct dm_integrity_range range;
3159 	struct bio *bio;
3160 	struct bio_integrity_payload *bip;
3161 	__u8 *t;
3162 	unsigned int i;
3163 	int r;
3164 	unsigned ret;
3165 	unsigned int super_counter = 0;
3166 	unsigned recalc_sectors = RECALC_SECTORS;
3167 
3168 retry:
3169 	recalc_buffer = kmalloc(recalc_sectors << SECTOR_SHIFT, GFP_NOIO | __GFP_NOWARN);
3170 	if (!recalc_buffer) {
3171 oom:
3172 		recalc_sectors >>= 1;
3173 		if (recalc_sectors >= 1U << ic->sb->log2_sectors_per_block)
3174 			goto retry;
3175 		DMCRIT("out of memory for recalculate buffer - recalculation disabled");
3176 		goto free_ret;
3177 	}
3178 
3179 	recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tuple_size;
3180 	if (crypto_shash_digestsize(ic->internal_hash) > ic->tuple_size)
3181 		recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tuple_size;
3182 	recalc_tags = kmalloc(recalc_tags_size, GFP_NOIO | __GFP_NOWARN);
3183 	if (!recalc_tags) {
3184 		kfree(recalc_buffer);
3185 		recalc_buffer = NULL;
3186 		goto oom;
3187 	}
3188 
3189 	spin_lock_irq(&ic->endio_wait.lock);
3190 
3191 next_chunk:
3192 	if (unlikely(dm_post_suspending(ic->ti)))
3193 		goto unlock_ret;
3194 
3195 	range.logical_sector = le64_to_cpu(ic->sb->recalc_sector);
3196 	if (unlikely(range.logical_sector >= ic->provided_data_sectors))
3197 		goto unlock_ret;
3198 	range.n_sectors = min((sector_t)recalc_sectors, ic->provided_data_sectors - range.logical_sector);
3199 
3200 	add_new_range_and_wait(ic, &range);
3201 	spin_unlock_irq(&ic->endio_wait.lock);
3202 
3203 	if (unlikely(++super_counter == RECALC_WRITE_SUPER)) {
3204 		recalc_write_super(ic);
3205 		super_counter = 0;
3206 	}
3207 
3208 	if (unlikely(dm_integrity_failed(ic)))
3209 		goto err;
3210 
3211 	DEBUG_print("recalculating: %llx - %llx\n", range.logical_sector, range.n_sectors);
3212 
3213 	bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_READ, GFP_NOIO, &ic->recalc_bios);
3214 	bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector;
3215 	__bio_add_page(bio, virt_to_page(recalc_buffer), range.n_sectors << SECTOR_SHIFT, offset_in_page(recalc_buffer));
3216 	r = submit_bio_wait(bio);
3217 	bio_put(bio);
3218 	if (unlikely(r)) {
3219 		dm_integrity_io_error(ic, "reading data", r);
3220 		goto err;
3221 	}
3222 
3223 	t = recalc_tags;
3224 	for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) {
3225 		memset(t, 0, ic->tuple_size);
3226 		integrity_sector_checksum(ic, range.logical_sector + i, recalc_buffer + (i << SECTOR_SHIFT), t);
3227 		t += ic->tuple_size;
3228 	}
3229 
3230 	bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_WRITE, GFP_NOIO, &ic->recalc_bios);
3231 	bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector;
3232 	__bio_add_page(bio, virt_to_page(recalc_buffer), range.n_sectors << SECTOR_SHIFT, offset_in_page(recalc_buffer));
3233 
3234 	bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
3235 	if (unlikely(IS_ERR(bip))) {
3236 		bio_put(bio);
3237 		DMCRIT("out of memory for bio integrity payload - recalculation disabled");
3238 		goto err;
3239 	}
3240 	ret = bio_integrity_add_page(bio, virt_to_page(recalc_tags), t - recalc_tags, offset_in_page(recalc_tags));
3241 	if (unlikely(ret != t - recalc_tags)) {
3242 		bio_put(bio);
3243 		dm_integrity_io_error(ic, "attaching integrity tags", -ENOMEM);
3244 		goto err;
3245 	}
3246 
3247 	r = submit_bio_wait(bio);
3248 	bio_put(bio);
3249 	if (unlikely(r)) {
3250 		dm_integrity_io_error(ic, "writing data", r);
3251 		goto err;
3252 	}
3253 
3254 	cond_resched();
3255 	spin_lock_irq(&ic->endio_wait.lock);
3256 	remove_range_unlocked(ic, &range);
3257 #ifdef CONFIG_64BIT
3258 	/* Paired with smp_load_acquire in dm_integrity_map_inline. */
3259 	smp_store_release(&ic->sb->recalc_sector, cpu_to_le64(range.logical_sector + range.n_sectors));
3260 #else
3261 	ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors);
3262 #endif
3263 	goto next_chunk;
3264 
3265 err:
3266 	remove_range(ic, &range);
3267 	goto free_ret;
3268 
3269 unlock_ret:
3270 	spin_unlock_irq(&ic->endio_wait.lock);
3271 
3272 	recalc_write_super(ic);
3273 
3274 free_ret:
3275 	kfree(recalc_buffer);
3276 	kfree(recalc_tags);
3277 }
3278 
bitmap_block_work(struct work_struct * w)3279 static void bitmap_block_work(struct work_struct *w)
3280 {
3281 	struct bitmap_block_status *bbs = container_of(w, struct bitmap_block_status, work);
3282 	struct dm_integrity_c *ic = bbs->ic;
3283 	struct bio *bio;
3284 	struct bio_list bio_queue;
3285 	struct bio_list waiting;
3286 
3287 	bio_list_init(&waiting);
3288 
3289 	spin_lock(&bbs->bio_queue_lock);
3290 	bio_queue = bbs->bio_queue;
3291 	bio_list_init(&bbs->bio_queue);
3292 	spin_unlock(&bbs->bio_queue_lock);
3293 
3294 	while ((bio = bio_list_pop(&bio_queue))) {
3295 		struct dm_integrity_io *dio;
3296 
3297 		dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
3298 
3299 		if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
3300 				    dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {
3301 			remove_range(ic, &dio->range);
3302 			INIT_WORK(&dio->work, integrity_bio_wait);
3303 			queue_work(ic->offload_wq, &dio->work);
3304 		} else {
3305 			block_bitmap_op(ic, ic->journal, dio->range.logical_sector,
3306 					dio->range.n_sectors, BITMAP_OP_SET);
3307 			bio_list_add(&waiting, bio);
3308 		}
3309 	}
3310 
3311 	if (bio_list_empty(&waiting))
3312 		return;
3313 
3314 	rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC,
3315 			   bbs->idx * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT),
3316 			   BITMAP_BLOCK_SIZE >> SECTOR_SHIFT, NULL);
3317 
3318 	while ((bio = bio_list_pop(&waiting))) {
3319 		struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
3320 
3321 		block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
3322 				dio->range.n_sectors, BITMAP_OP_SET);
3323 
3324 		remove_range(ic, &dio->range);
3325 		INIT_WORK(&dio->work, integrity_bio_wait);
3326 		queue_work(ic->offload_wq, &dio->work);
3327 	}
3328 
3329 	queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval);
3330 }
3331 
bitmap_flush_work(struct work_struct * work)3332 static void bitmap_flush_work(struct work_struct *work)
3333 {
3334 	struct dm_integrity_c *ic = container_of(work, struct dm_integrity_c, bitmap_flush_work.work);
3335 	struct dm_integrity_range range;
3336 	unsigned long limit;
3337 	struct bio *bio;
3338 
3339 	dm_integrity_flush_buffers(ic, false);
3340 
3341 	range.logical_sector = 0;
3342 	range.n_sectors = ic->provided_data_sectors;
3343 
3344 	spin_lock_irq(&ic->endio_wait.lock);
3345 	add_new_range_and_wait(ic, &range);
3346 	spin_unlock_irq(&ic->endio_wait.lock);
3347 
3348 	dm_integrity_flush_buffers(ic, true);
3349 
3350 	limit = ic->provided_data_sectors;
3351 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
3352 		limit = le64_to_cpu(ic->sb->recalc_sector)
3353 			>> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)
3354 			<< (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
3355 	}
3356 	/*DEBUG_print("zeroing journal\n");*/
3357 	block_bitmap_op(ic, ic->journal, 0, limit, BITMAP_OP_CLEAR);
3358 	block_bitmap_op(ic, ic->may_write_bitmap, 0, limit, BITMAP_OP_CLEAR);
3359 
3360 	rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3361 			   ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3362 
3363 	spin_lock_irq(&ic->endio_wait.lock);
3364 	remove_range_unlocked(ic, &range);
3365 	while (unlikely((bio = bio_list_pop(&ic->synchronous_bios)) != NULL)) {
3366 		bio_endio(bio);
3367 		spin_unlock_irq(&ic->endio_wait.lock);
3368 		spin_lock_irq(&ic->endio_wait.lock);
3369 	}
3370 	spin_unlock_irq(&ic->endio_wait.lock);
3371 }
3372 
3373 
init_journal(struct dm_integrity_c * ic,unsigned int start_section,unsigned int n_sections,unsigned char commit_seq)3374 static void init_journal(struct dm_integrity_c *ic, unsigned int start_section,
3375 			 unsigned int n_sections, unsigned char commit_seq)
3376 {
3377 	unsigned int i, j, n;
3378 
3379 	if (!n_sections)
3380 		return;
3381 
3382 	for (n = 0; n < n_sections; n++) {
3383 		i = start_section + n;
3384 		wraparound_section(ic, &i);
3385 		for (j = 0; j < ic->journal_section_sectors; j++) {
3386 			struct journal_sector *js = access_journal(ic, i, j);
3387 
3388 			BUILD_BUG_ON(sizeof(js->sectors) != JOURNAL_SECTOR_DATA);
3389 			memset(&js->sectors, 0, sizeof(js->sectors));
3390 			js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq);
3391 		}
3392 		for (j = 0; j < ic->journal_section_entries; j++) {
3393 			struct journal_entry *je = access_journal_entry(ic, i, j);
3394 
3395 			journal_entry_set_unused(je);
3396 		}
3397 	}
3398 
3399 	write_journal(ic, start_section, n_sections);
3400 }
3401 
find_commit_seq(struct dm_integrity_c * ic,unsigned int i,unsigned int j,commit_id_t id)3402 static int find_commit_seq(struct dm_integrity_c *ic, unsigned int i, unsigned int j, commit_id_t id)
3403 {
3404 	unsigned char k;
3405 
3406 	for (k = 0; k < N_COMMIT_IDS; k++) {
3407 		if (dm_integrity_commit_id(ic, i, j, k) == id)
3408 			return k;
3409 	}
3410 	dm_integrity_io_error(ic, "journal commit id", -EIO);
3411 	return -EIO;
3412 }
3413 
replay_journal(struct dm_integrity_c * ic)3414 static void replay_journal(struct dm_integrity_c *ic)
3415 {
3416 	unsigned int i, j;
3417 	bool used_commit_ids[N_COMMIT_IDS];
3418 	unsigned int max_commit_id_sections[N_COMMIT_IDS];
3419 	unsigned int write_start, write_sections;
3420 	unsigned int continue_section;
3421 	bool journal_empty;
3422 	unsigned char unused, last_used, want_commit_seq;
3423 
3424 	if (ic->mode == 'R')
3425 		return;
3426 
3427 	if (ic->journal_uptodate)
3428 		return;
3429 
3430 	last_used = 0;
3431 	write_start = 0;
3432 
3433 	if (!ic->just_formatted) {
3434 		DEBUG_print("reading journal\n");
3435 		rw_journal(ic, REQ_OP_READ, 0, ic->journal_sections, NULL);
3436 		if (ic->journal_io)
3437 			DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal");
3438 		if (ic->journal_io) {
3439 			struct journal_completion crypt_comp;
3440 
3441 			crypt_comp.ic = ic;
3442 			init_completion(&crypt_comp.comp);
3443 			crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0);
3444 			encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp);
3445 			wait_for_completion(&crypt_comp.comp);
3446 		}
3447 		DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal");
3448 	}
3449 
3450 	if (dm_integrity_failed(ic))
3451 		goto clear_journal;
3452 
3453 	journal_empty = true;
3454 	memset(used_commit_ids, 0, sizeof(used_commit_ids));
3455 	memset(max_commit_id_sections, 0, sizeof(max_commit_id_sections));
3456 	for (i = 0; i < ic->journal_sections; i++) {
3457 		for (j = 0; j < ic->journal_section_sectors; j++) {
3458 			int k;
3459 			struct journal_sector *js = access_journal(ic, i, j);
3460 
3461 			k = find_commit_seq(ic, i, j, js->commit_id);
3462 			if (k < 0)
3463 				goto clear_journal;
3464 			used_commit_ids[k] = true;
3465 			max_commit_id_sections[k] = i;
3466 		}
3467 		if (journal_empty) {
3468 			for (j = 0; j < ic->journal_section_entries; j++) {
3469 				struct journal_entry *je = access_journal_entry(ic, i, j);
3470 
3471 				if (!journal_entry_is_unused(je)) {
3472 					journal_empty = false;
3473 					break;
3474 				}
3475 			}
3476 		}
3477 	}
3478 
3479 	if (!used_commit_ids[N_COMMIT_IDS - 1]) {
3480 		unused = N_COMMIT_IDS - 1;
3481 		while (unused && !used_commit_ids[unused - 1])
3482 			unused--;
3483 	} else {
3484 		for (unused = 0; unused < N_COMMIT_IDS; unused++)
3485 			if (!used_commit_ids[unused])
3486 				break;
3487 		if (unused == N_COMMIT_IDS) {
3488 			dm_integrity_io_error(ic, "journal commit ids", -EIO);
3489 			goto clear_journal;
3490 		}
3491 	}
3492 	DEBUG_print("first unused commit seq %d [%d,%d,%d,%d]\n",
3493 		    unused, used_commit_ids[0], used_commit_ids[1],
3494 		    used_commit_ids[2], used_commit_ids[3]);
3495 
3496 	last_used = prev_commit_seq(unused);
3497 	want_commit_seq = prev_commit_seq(last_used);
3498 
3499 	if (!used_commit_ids[want_commit_seq] && used_commit_ids[prev_commit_seq(want_commit_seq)])
3500 		journal_empty = true;
3501 
3502 	write_start = max_commit_id_sections[last_used] + 1;
3503 	if (unlikely(write_start >= ic->journal_sections))
3504 		want_commit_seq = next_commit_seq(want_commit_seq);
3505 	wraparound_section(ic, &write_start);
3506 
3507 	i = write_start;
3508 	for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) {
3509 		for (j = 0; j < ic->journal_section_sectors; j++) {
3510 			struct journal_sector *js = access_journal(ic, i, j);
3511 
3512 			if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) {
3513 				/*
3514 				 * This could be caused by crash during writing.
3515 				 * We won't replay the inconsistent part of the
3516 				 * journal.
3517 				 */
3518 				DEBUG_print("commit id mismatch at position (%u, %u): %d != %d\n",
3519 					    i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq);
3520 				goto brk;
3521 			}
3522 		}
3523 		i++;
3524 		if (unlikely(i >= ic->journal_sections))
3525 			want_commit_seq = next_commit_seq(want_commit_seq);
3526 		wraparound_section(ic, &i);
3527 	}
3528 brk:
3529 
3530 	if (!journal_empty) {
3531 		DEBUG_print("replaying %u sections, starting at %u, commit seq %d\n",
3532 			    write_sections, write_start, want_commit_seq);
3533 		do_journal_write(ic, write_start, write_sections, true);
3534 	}
3535 
3536 	if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) {
3537 		continue_section = write_start;
3538 		ic->commit_seq = want_commit_seq;
3539 		DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq);
3540 	} else {
3541 		unsigned int s;
3542 		unsigned char erase_seq;
3543 
3544 clear_journal:
3545 		DEBUG_print("clearing journal\n");
3546 
3547 		erase_seq = prev_commit_seq(prev_commit_seq(last_used));
3548 		s = write_start;
3549 		init_journal(ic, s, 1, erase_seq);
3550 		s++;
3551 		wraparound_section(ic, &s);
3552 		if (ic->journal_sections >= 2) {
3553 			init_journal(ic, s, ic->journal_sections - 2, erase_seq);
3554 			s += ic->journal_sections - 2;
3555 			wraparound_section(ic, &s);
3556 			init_journal(ic, s, 1, erase_seq);
3557 		}
3558 
3559 		continue_section = 0;
3560 		ic->commit_seq = next_commit_seq(erase_seq);
3561 	}
3562 
3563 	ic->committed_section = continue_section;
3564 	ic->n_committed_sections = 0;
3565 
3566 	ic->uncommitted_section = continue_section;
3567 	ic->n_uncommitted_sections = 0;
3568 
3569 	ic->free_section = continue_section;
3570 	ic->free_section_entry = 0;
3571 	ic->free_sectors = ic->journal_entries;
3572 
3573 	ic->journal_tree_root = RB_ROOT;
3574 	for (i = 0; i < ic->journal_entries; i++)
3575 		init_journal_node(&ic->journal_tree[i]);
3576 }
3577 
dm_integrity_enter_synchronous_mode(struct dm_integrity_c * ic)3578 static void dm_integrity_enter_synchronous_mode(struct dm_integrity_c *ic)
3579 {
3580 	DEBUG_print("%s\n", __func__);
3581 
3582 	if (ic->mode == 'B') {
3583 		ic->bitmap_flush_interval = msecs_to_jiffies(10) + 1;
3584 		ic->synchronous_mode = 1;
3585 
3586 		cancel_delayed_work_sync(&ic->bitmap_flush_work);
3587 		queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
3588 		flush_workqueue(ic->commit_wq);
3589 	}
3590 }
3591 
dm_integrity_reboot(struct notifier_block * n,unsigned long code,void * x)3592 static int dm_integrity_reboot(struct notifier_block *n, unsigned long code, void *x)
3593 {
3594 	struct dm_integrity_c *ic = container_of(n, struct dm_integrity_c, reboot_notifier);
3595 
3596 	DEBUG_print("%s\n", __func__);
3597 
3598 	dm_integrity_enter_synchronous_mode(ic);
3599 
3600 	return NOTIFY_DONE;
3601 }
3602 
dm_integrity_postsuspend(struct dm_target * ti)3603 static void dm_integrity_postsuspend(struct dm_target *ti)
3604 {
3605 	struct dm_integrity_c *ic = ti->private;
3606 	int r;
3607 
3608 	WARN_ON(unregister_reboot_notifier(&ic->reboot_notifier));
3609 
3610 	del_timer_sync(&ic->autocommit_timer);
3611 
3612 	if (ic->recalc_wq)
3613 		drain_workqueue(ic->recalc_wq);
3614 
3615 	if (ic->mode == 'B')
3616 		cancel_delayed_work_sync(&ic->bitmap_flush_work);
3617 
3618 	queue_work(ic->commit_wq, &ic->commit_work);
3619 	drain_workqueue(ic->commit_wq);
3620 
3621 	if (ic->mode == 'J') {
3622 		queue_work(ic->writer_wq, &ic->writer_work);
3623 		drain_workqueue(ic->writer_wq);
3624 		dm_integrity_flush_buffers(ic, true);
3625 		if (ic->wrote_to_journal) {
3626 			init_journal(ic, ic->free_section,
3627 				     ic->journal_sections - ic->free_section, ic->commit_seq);
3628 			if (ic->free_section) {
3629 				init_journal(ic, 0, ic->free_section,
3630 					     next_commit_seq(ic->commit_seq));
3631 			}
3632 		}
3633 	}
3634 
3635 	if (ic->mode == 'B') {
3636 		dm_integrity_flush_buffers(ic, true);
3637 #if 1
3638 		/* set to 0 to test bitmap replay code */
3639 		init_journal(ic, 0, ic->journal_sections, 0);
3640 		ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3641 		r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3642 		if (unlikely(r))
3643 			dm_integrity_io_error(ic, "writing superblock", r);
3644 #endif
3645 	}
3646 
3647 	BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
3648 
3649 	ic->journal_uptodate = true;
3650 }
3651 
dm_integrity_resume(struct dm_target * ti)3652 static void dm_integrity_resume(struct dm_target *ti)
3653 {
3654 	struct dm_integrity_c *ic = ti->private;
3655 	__u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors);
3656 	int r;
3657 
3658 	DEBUG_print("resume\n");
3659 
3660 	ic->wrote_to_journal = false;
3661 
3662 	if (ic->provided_data_sectors != old_provided_data_sectors) {
3663 		if (ic->provided_data_sectors > old_provided_data_sectors &&
3664 		    ic->mode == 'B' &&
3665 		    ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) {
3666 			rw_journal_sectors(ic, REQ_OP_READ, 0,
3667 					   ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3668 			block_bitmap_op(ic, ic->journal, old_provided_data_sectors,
3669 					ic->provided_data_sectors - old_provided_data_sectors, BITMAP_OP_SET);
3670 			rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3671 					   ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3672 		}
3673 
3674 		ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
3675 		r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3676 		if (unlikely(r))
3677 			dm_integrity_io_error(ic, "writing superblock", r);
3678 	}
3679 
3680 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) {
3681 		DEBUG_print("resume dirty_bitmap\n");
3682 		rw_journal_sectors(ic, REQ_OP_READ, 0,
3683 				   ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3684 		if (ic->mode == 'B') {
3685 			if (ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit &&
3686 			    !ic->reset_recalculate_flag) {
3687 				block_bitmap_copy(ic, ic->recalc_bitmap, ic->journal);
3688 				block_bitmap_copy(ic, ic->may_write_bitmap, ic->journal);
3689 				if (!block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors,
3690 						     BITMAP_OP_TEST_ALL_CLEAR)) {
3691 					ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3692 					ic->sb->recalc_sector = cpu_to_le64(0);
3693 				}
3694 			} else {
3695 				DEBUG_print("non-matching blocks_per_bitmap_bit: %u, %u\n",
3696 					    ic->sb->log2_blocks_per_bitmap_bit, ic->log2_blocks_per_bitmap_bit);
3697 				ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit;
3698 				block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3699 				block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3700 				block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3701 				rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3702 						   ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3703 				ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3704 				ic->sb->recalc_sector = cpu_to_le64(0);
3705 			}
3706 		} else {
3707 			if (!(ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit &&
3708 			      block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_TEST_ALL_CLEAR)) ||
3709 			    ic->reset_recalculate_flag) {
3710 				ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3711 				ic->sb->recalc_sector = cpu_to_le64(0);
3712 			}
3713 			init_journal(ic, 0, ic->journal_sections, 0);
3714 			replay_journal(ic);
3715 			ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3716 		}
3717 		r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3718 		if (unlikely(r))
3719 			dm_integrity_io_error(ic, "writing superblock", r);
3720 	} else {
3721 		replay_journal(ic);
3722 		if (ic->reset_recalculate_flag) {
3723 			ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3724 			ic->sb->recalc_sector = cpu_to_le64(0);
3725 		}
3726 		if (ic->mode == 'B') {
3727 			ic->sb->flags |= cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3728 			ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit;
3729 			r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3730 			if (unlikely(r))
3731 				dm_integrity_io_error(ic, "writing superblock", r);
3732 
3733 			block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3734 			block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3735 			block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3736 			if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
3737 			    le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors) {
3738 				block_bitmap_op(ic, ic->journal, le64_to_cpu(ic->sb->recalc_sector),
3739 						ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
3740 				block_bitmap_op(ic, ic->recalc_bitmap, le64_to_cpu(ic->sb->recalc_sector),
3741 						ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
3742 				block_bitmap_op(ic, ic->may_write_bitmap, le64_to_cpu(ic->sb->recalc_sector),
3743 						ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
3744 			}
3745 			rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3746 					   ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3747 		}
3748 	}
3749 
3750 	DEBUG_print("testing recalc: %x\n", ic->sb->flags);
3751 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
3752 		__u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector);
3753 
3754 		DEBUG_print("recalc pos: %llx / %llx\n", recalc_pos, ic->provided_data_sectors);
3755 		if (recalc_pos < ic->provided_data_sectors) {
3756 			queue_work(ic->recalc_wq, &ic->recalc_work);
3757 		} else if (recalc_pos > ic->provided_data_sectors) {
3758 			ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors);
3759 			recalc_write_super(ic);
3760 		}
3761 	}
3762 
3763 	ic->reboot_notifier.notifier_call = dm_integrity_reboot;
3764 	ic->reboot_notifier.next = NULL;
3765 	ic->reboot_notifier.priority = INT_MAX - 1;	/* be notified after md and before hardware drivers */
3766 	WARN_ON(register_reboot_notifier(&ic->reboot_notifier));
3767 
3768 #if 0
3769 	/* set to 1 to stress test synchronous mode */
3770 	dm_integrity_enter_synchronous_mode(ic);
3771 #endif
3772 }
3773 
dm_integrity_status(struct dm_target * ti,status_type_t type,unsigned int status_flags,char * result,unsigned int maxlen)3774 static void dm_integrity_status(struct dm_target *ti, status_type_t type,
3775 				unsigned int status_flags, char *result, unsigned int maxlen)
3776 {
3777 	struct dm_integrity_c *ic = ti->private;
3778 	unsigned int arg_count;
3779 	size_t sz = 0;
3780 
3781 	switch (type) {
3782 	case STATUSTYPE_INFO:
3783 		DMEMIT("%llu %llu",
3784 			(unsigned long long)atomic64_read(&ic->number_of_mismatches),
3785 			ic->provided_data_sectors);
3786 		if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
3787 			DMEMIT(" %llu", le64_to_cpu(ic->sb->recalc_sector));
3788 		else
3789 			DMEMIT(" -");
3790 		break;
3791 
3792 	case STATUSTYPE_TABLE: {
3793 		__u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100;
3794 
3795 		watermark_percentage += ic->journal_entries / 2;
3796 		do_div(watermark_percentage, ic->journal_entries);
3797 		arg_count = 3;
3798 		arg_count += !!ic->meta_dev;
3799 		arg_count += ic->sectors_per_block != 1;
3800 		arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING));
3801 		arg_count += ic->reset_recalculate_flag;
3802 		arg_count += ic->discard;
3803 		arg_count += ic->mode == 'J';
3804 		arg_count += ic->mode == 'J';
3805 		arg_count += ic->mode == 'B';
3806 		arg_count += ic->mode == 'B';
3807 		arg_count += !!ic->internal_hash_alg.alg_string;
3808 		arg_count += !!ic->journal_crypt_alg.alg_string;
3809 		arg_count += !!ic->journal_mac_alg.alg_string;
3810 		arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0;
3811 		arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0;
3812 		arg_count += ic->legacy_recalculate;
3813 		DMEMIT("%s %llu %u %c %u", ic->dev->name, ic->start,
3814 		       ic->tag_size, ic->mode, arg_count);
3815 		if (ic->meta_dev)
3816 			DMEMIT(" meta_device:%s", ic->meta_dev->name);
3817 		if (ic->sectors_per_block != 1)
3818 			DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT);
3819 		if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
3820 			DMEMIT(" recalculate");
3821 		if (ic->reset_recalculate_flag)
3822 			DMEMIT(" reset_recalculate");
3823 		if (ic->discard)
3824 			DMEMIT(" allow_discards");
3825 		DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS);
3826 		DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors);
3827 		DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors);
3828 		if (ic->mode == 'J') {
3829 			DMEMIT(" journal_watermark:%u", (unsigned int)watermark_percentage);
3830 			DMEMIT(" commit_time:%u", ic->autocommit_msec);
3831 		}
3832 		if (ic->mode == 'B') {
3833 			DMEMIT(" sectors_per_bit:%llu", (sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit);
3834 			DMEMIT(" bitmap_flush_interval:%u", jiffies_to_msecs(ic->bitmap_flush_interval));
3835 		}
3836 		if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0)
3837 			DMEMIT(" fix_padding");
3838 		if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0)
3839 			DMEMIT(" fix_hmac");
3840 		if (ic->legacy_recalculate)
3841 			DMEMIT(" legacy_recalculate");
3842 
3843 #define EMIT_ALG(a, n)							\
3844 		do {							\
3845 			if (ic->a.alg_string) {				\
3846 				DMEMIT(" %s:%s", n, ic->a.alg_string);	\
3847 				if (ic->a.key_string)			\
3848 					DMEMIT(":%s", ic->a.key_string);\
3849 			}						\
3850 		} while (0)
3851 		EMIT_ALG(internal_hash_alg, "internal_hash");
3852 		EMIT_ALG(journal_crypt_alg, "journal_crypt");
3853 		EMIT_ALG(journal_mac_alg, "journal_mac");
3854 		break;
3855 	}
3856 	case STATUSTYPE_IMA:
3857 		DMEMIT_TARGET_NAME_VERSION(ti->type);
3858 		DMEMIT(",dev_name=%s,start=%llu,tag_size=%u,mode=%c",
3859 			ic->dev->name, ic->start, ic->tag_size, ic->mode);
3860 
3861 		if (ic->meta_dev)
3862 			DMEMIT(",meta_device=%s", ic->meta_dev->name);
3863 		if (ic->sectors_per_block != 1)
3864 			DMEMIT(",block_size=%u", ic->sectors_per_block << SECTOR_SHIFT);
3865 
3866 		DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ?
3867 		       'y' : 'n');
3868 		DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n');
3869 		DMEMIT(",fix_padding=%c",
3870 		       ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n');
3871 		DMEMIT(",fix_hmac=%c",
3872 		       ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) ? 'y' : 'n');
3873 		DMEMIT(",legacy_recalculate=%c", ic->legacy_recalculate ? 'y' : 'n');
3874 
3875 		DMEMIT(",journal_sectors=%u", ic->initial_sectors - SB_SECTORS);
3876 		DMEMIT(",interleave_sectors=%u", 1U << ic->sb->log2_interleave_sectors);
3877 		DMEMIT(",buffer_sectors=%u", 1U << ic->log2_buffer_sectors);
3878 		DMEMIT(";");
3879 		break;
3880 	}
3881 }
3882 
dm_integrity_iterate_devices(struct dm_target * ti,iterate_devices_callout_fn fn,void * data)3883 static int dm_integrity_iterate_devices(struct dm_target *ti,
3884 					iterate_devices_callout_fn fn, void *data)
3885 {
3886 	struct dm_integrity_c *ic = ti->private;
3887 
3888 	if (!ic->meta_dev)
3889 		return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data);
3890 	else
3891 		return fn(ti, ic->dev, 0, ti->len, data);
3892 }
3893 
dm_integrity_io_hints(struct dm_target * ti,struct queue_limits * limits)3894 static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits)
3895 {
3896 	struct dm_integrity_c *ic = ti->private;
3897 
3898 	if (ic->sectors_per_block > 1) {
3899 		limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
3900 		limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
3901 		limits->io_min = ic->sectors_per_block << SECTOR_SHIFT;
3902 		limits->dma_alignment = limits->logical_block_size - 1;
3903 		limits->discard_granularity = ic->sectors_per_block << SECTOR_SHIFT;
3904 	}
3905 
3906 	if (!ic->internal_hash) {
3907 		struct blk_integrity *bi = &limits->integrity;
3908 
3909 		memset(bi, 0, sizeof(*bi));
3910 		bi->tuple_size = ic->tag_size;
3911 		bi->tag_size = bi->tuple_size;
3912 		bi->interval_exp =
3913 			ic->sb->log2_sectors_per_block + SECTOR_SHIFT;
3914 	}
3915 
3916 	limits->max_integrity_segments = USHRT_MAX;
3917 }
3918 
calculate_journal_section_size(struct dm_integrity_c * ic)3919 static void calculate_journal_section_size(struct dm_integrity_c *ic)
3920 {
3921 	unsigned int sector_space = JOURNAL_SECTOR_DATA;
3922 
3923 	ic->journal_sections = le32_to_cpu(ic->sb->journal_sections);
3924 	ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block]) + ic->tag_size,
3925 					 JOURNAL_ENTRY_ROUNDUP);
3926 
3927 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC))
3928 		sector_space -= JOURNAL_MAC_PER_SECTOR;
3929 	ic->journal_entries_per_sector = sector_space / ic->journal_entry_size;
3930 	ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS;
3931 	ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JOURNAL_BLOCK_SECTORS;
3932 	ic->journal_entries = ic->journal_section_entries * ic->journal_sections;
3933 }
3934 
calculate_device_limits(struct dm_integrity_c * ic)3935 static int calculate_device_limits(struct dm_integrity_c *ic)
3936 {
3937 	__u64 initial_sectors;
3938 
3939 	calculate_journal_section_size(ic);
3940 	initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections;
3941 	if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UINT_MAX)
3942 		return -EINVAL;
3943 	ic->initial_sectors = initial_sectors;
3944 
3945 	if (ic->mode == 'I') {
3946 		if (ic->initial_sectors + ic->provided_data_sectors > ic->meta_device_sectors)
3947 			return -EINVAL;
3948 	} else if (!ic->meta_dev) {
3949 		sector_t last_sector, last_area, last_offset;
3950 
3951 		/* we have to maintain excessive padding for compatibility with existing volumes */
3952 		__u64 metadata_run_padding =
3953 			ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING) ?
3954 			(__u64)(METADATA_PADDING_SECTORS << SECTOR_SHIFT) :
3955 			(__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS);
3956 
3957 		ic->metadata_run = round_up((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block),
3958 					    metadata_run_padding) >> SECTOR_SHIFT;
3959 		if (!(ic->metadata_run & (ic->metadata_run - 1)))
3960 			ic->log2_metadata_run = __ffs(ic->metadata_run);
3961 		else
3962 			ic->log2_metadata_run = -1;
3963 
3964 		get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset);
3965 		last_sector = get_data_sector(ic, last_area, last_offset);
3966 		if (last_sector < ic->start || last_sector >= ic->meta_device_sectors)
3967 			return -EINVAL;
3968 	} else {
3969 		__u64 meta_size = (ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size;
3970 
3971 		meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1))
3972 				>> (ic->log2_buffer_sectors + SECTOR_SHIFT);
3973 		meta_size <<= ic->log2_buffer_sectors;
3974 		if (ic->initial_sectors + meta_size < ic->initial_sectors ||
3975 		    ic->initial_sectors + meta_size > ic->meta_device_sectors)
3976 			return -EINVAL;
3977 		ic->metadata_run = 1;
3978 		ic->log2_metadata_run = 0;
3979 	}
3980 
3981 	return 0;
3982 }
3983 
get_provided_data_sectors(struct dm_integrity_c * ic)3984 static void get_provided_data_sectors(struct dm_integrity_c *ic)
3985 {
3986 	if (!ic->meta_dev) {
3987 		int test_bit;
3988 
3989 		ic->provided_data_sectors = 0;
3990 		for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) {
3991 			__u64 prev_data_sectors = ic->provided_data_sectors;
3992 
3993 			ic->provided_data_sectors |= (sector_t)1 << test_bit;
3994 			if (calculate_device_limits(ic))
3995 				ic->provided_data_sectors = prev_data_sectors;
3996 		}
3997 	} else {
3998 		ic->provided_data_sectors = ic->data_device_sectors;
3999 		ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1);
4000 	}
4001 }
4002 
initialize_superblock(struct dm_integrity_c * ic,unsigned int journal_sectors,unsigned int interleave_sectors)4003 static int initialize_superblock(struct dm_integrity_c *ic,
4004 				 unsigned int journal_sectors, unsigned int interleave_sectors)
4005 {
4006 	unsigned int journal_sections;
4007 	int test_bit;
4008 
4009 	memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT);
4010 	memcpy(ic->sb->magic, SB_MAGIC, 8);
4011 	if (ic->mode == 'I')
4012 		ic->sb->flags |= cpu_to_le32(SB_FLAG_INLINE);
4013 	ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size);
4014 	ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block);
4015 	if (ic->journal_mac_alg.alg_string)
4016 		ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC);
4017 
4018 	calculate_journal_section_size(ic);
4019 	journal_sections = journal_sectors / ic->journal_section_sectors;
4020 	if (!journal_sections)
4021 		journal_sections = 1;
4022 	if (ic->mode == 'I')
4023 		journal_sections = 0;
4024 
4025 	if (ic->fix_hmac && (ic->internal_hash_alg.alg_string || ic->journal_mac_alg.alg_string)) {
4026 		ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_HMAC);
4027 		get_random_bytes(ic->sb->salt, SALT_SIZE);
4028 	}
4029 
4030 	if (!ic->meta_dev) {
4031 		if (ic->fix_padding)
4032 			ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING);
4033 		ic->sb->journal_sections = cpu_to_le32(journal_sections);
4034 		if (!interleave_sectors)
4035 			interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
4036 		ic->sb->log2_interleave_sectors = __fls(interleave_sectors);
4037 		ic->sb->log2_interleave_sectors = max_t(__u8, MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
4038 		ic->sb->log2_interleave_sectors = min_t(__u8, MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
4039 
4040 		get_provided_data_sectors(ic);
4041 		if (!ic->provided_data_sectors)
4042 			return -EINVAL;
4043 	} else {
4044 		ic->sb->log2_interleave_sectors = 0;
4045 
4046 		get_provided_data_sectors(ic);
4047 		if (!ic->provided_data_sectors)
4048 			return -EINVAL;
4049 
4050 try_smaller_buffer:
4051 		ic->sb->journal_sections = cpu_to_le32(0);
4052 		for (test_bit = fls(journal_sections) - 1; test_bit >= 0; test_bit--) {
4053 			__u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections);
4054 			__u32 test_journal_sections = prev_journal_sections | (1U << test_bit);
4055 
4056 			if (test_journal_sections > journal_sections)
4057 				continue;
4058 			ic->sb->journal_sections = cpu_to_le32(test_journal_sections);
4059 			if (calculate_device_limits(ic))
4060 				ic->sb->journal_sections = cpu_to_le32(prev_journal_sections);
4061 
4062 		}
4063 		if (!le32_to_cpu(ic->sb->journal_sections)) {
4064 			if (ic->log2_buffer_sectors > 3) {
4065 				ic->log2_buffer_sectors--;
4066 				goto try_smaller_buffer;
4067 			}
4068 			return -EINVAL;
4069 		}
4070 	}
4071 
4072 	ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
4073 
4074 	sb_set_version(ic);
4075 
4076 	return 0;
4077 }
4078 
dm_integrity_free_page_list(struct page_list * pl)4079 static void dm_integrity_free_page_list(struct page_list *pl)
4080 {
4081 	unsigned int i;
4082 
4083 	if (!pl)
4084 		return;
4085 	for (i = 0; pl[i].page; i++)
4086 		__free_page(pl[i].page);
4087 	kvfree(pl);
4088 }
4089 
dm_integrity_alloc_page_list(unsigned int n_pages)4090 static struct page_list *dm_integrity_alloc_page_list(unsigned int n_pages)
4091 {
4092 	struct page_list *pl;
4093 	unsigned int i;
4094 
4095 	pl = kvmalloc_array(n_pages + 1, sizeof(struct page_list), GFP_KERNEL | __GFP_ZERO);
4096 	if (!pl)
4097 		return NULL;
4098 
4099 	for (i = 0; i < n_pages; i++) {
4100 		pl[i].page = alloc_page(GFP_KERNEL);
4101 		if (!pl[i].page) {
4102 			dm_integrity_free_page_list(pl);
4103 			return NULL;
4104 		}
4105 		if (i)
4106 			pl[i - 1].next = &pl[i];
4107 	}
4108 	pl[i].page = NULL;
4109 	pl[i].next = NULL;
4110 
4111 	return pl;
4112 }
4113 
dm_integrity_free_journal_scatterlist(struct dm_integrity_c * ic,struct scatterlist ** sl)4114 static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl)
4115 {
4116 	unsigned int i;
4117 
4118 	for (i = 0; i < ic->journal_sections; i++)
4119 		kvfree(sl[i]);
4120 	kvfree(sl);
4121 }
4122 
dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c * ic,struct page_list * pl)4123 static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic,
4124 								   struct page_list *pl)
4125 {
4126 	struct scatterlist **sl;
4127 	unsigned int i;
4128 
4129 	sl = kvmalloc_array(ic->journal_sections,
4130 			    sizeof(struct scatterlist *),
4131 			    GFP_KERNEL | __GFP_ZERO);
4132 	if (!sl)
4133 		return NULL;
4134 
4135 	for (i = 0; i < ic->journal_sections; i++) {
4136 		struct scatterlist *s;
4137 		unsigned int start_index, start_offset;
4138 		unsigned int end_index, end_offset;
4139 		unsigned int n_pages;
4140 		unsigned int idx;
4141 
4142 		page_list_location(ic, i, 0, &start_index, &start_offset);
4143 		page_list_location(ic, i, ic->journal_section_sectors - 1,
4144 				   &end_index, &end_offset);
4145 
4146 		n_pages = (end_index - start_index + 1);
4147 
4148 		s = kvmalloc_array(n_pages, sizeof(struct scatterlist),
4149 				   GFP_KERNEL);
4150 		if (!s) {
4151 			dm_integrity_free_journal_scatterlist(ic, sl);
4152 			return NULL;
4153 		}
4154 
4155 		sg_init_table(s, n_pages);
4156 		for (idx = start_index; idx <= end_index; idx++) {
4157 			char *va = lowmem_page_address(pl[idx].page);
4158 			unsigned int start = 0, end = PAGE_SIZE;
4159 
4160 			if (idx == start_index)
4161 				start = start_offset;
4162 			if (idx == end_index)
4163 				end = end_offset + (1 << SECTOR_SHIFT);
4164 			sg_set_buf(&s[idx - start_index], va + start, end - start);
4165 		}
4166 
4167 		sl[i] = s;
4168 	}
4169 
4170 	return sl;
4171 }
4172 
free_alg(struct alg_spec * a)4173 static void free_alg(struct alg_spec *a)
4174 {
4175 	kfree_sensitive(a->alg_string);
4176 	kfree_sensitive(a->key);
4177 	memset(a, 0, sizeof(*a));
4178 }
4179 
get_alg_and_key(const char * arg,struct alg_spec * a,char ** error,char * error_inval)4180 static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval)
4181 {
4182 	char *k;
4183 
4184 	free_alg(a);
4185 
4186 	a->alg_string = kstrdup(strchr(arg, ':') + 1, GFP_KERNEL);
4187 	if (!a->alg_string)
4188 		goto nomem;
4189 
4190 	k = strchr(a->alg_string, ':');
4191 	if (k) {
4192 		*k = 0;
4193 		a->key_string = k + 1;
4194 		if (strlen(a->key_string) & 1)
4195 			goto inval;
4196 
4197 		a->key_size = strlen(a->key_string) / 2;
4198 		a->key = kmalloc(a->key_size, GFP_KERNEL);
4199 		if (!a->key)
4200 			goto nomem;
4201 		if (hex2bin(a->key, a->key_string, a->key_size))
4202 			goto inval;
4203 	}
4204 
4205 	return 0;
4206 inval:
4207 	*error = error_inval;
4208 	return -EINVAL;
4209 nomem:
4210 	*error = "Out of memory for an argument";
4211 	return -ENOMEM;
4212 }
4213 
get_mac(struct crypto_shash ** hash,struct alg_spec * a,char ** error,char * error_alg,char * error_key)4214 static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error,
4215 		   char *error_alg, char *error_key)
4216 {
4217 	int r;
4218 
4219 	if (a->alg_string) {
4220 		*hash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
4221 		if (IS_ERR(*hash)) {
4222 			*error = error_alg;
4223 			r = PTR_ERR(*hash);
4224 			*hash = NULL;
4225 			return r;
4226 		}
4227 
4228 		if (a->key) {
4229 			r = crypto_shash_setkey(*hash, a->key, a->key_size);
4230 			if (r) {
4231 				*error = error_key;
4232 				return r;
4233 			}
4234 		} else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) {
4235 			*error = error_key;
4236 			return -ENOKEY;
4237 		}
4238 	}
4239 
4240 	return 0;
4241 }
4242 
create_journal(struct dm_integrity_c * ic,char ** error)4243 static int create_journal(struct dm_integrity_c *ic, char **error)
4244 {
4245 	int r = 0;
4246 	unsigned int i;
4247 	__u64 journal_pages, journal_desc_size, journal_tree_size;
4248 	unsigned char *crypt_data = NULL, *crypt_iv = NULL;
4249 	struct skcipher_request *req = NULL;
4250 
4251 	ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL);
4252 	ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL);
4253 	ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL);
4254 	ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL);
4255 
4256 	journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors,
4257 				PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT);
4258 	journal_desc_size = journal_pages * sizeof(struct page_list);
4259 	if (journal_pages >= totalram_pages() - totalhigh_pages() || journal_desc_size > ULONG_MAX) {
4260 		*error = "Journal doesn't fit into memory";
4261 		r = -ENOMEM;
4262 		goto bad;
4263 	}
4264 	ic->journal_pages = journal_pages;
4265 
4266 	ic->journal = dm_integrity_alloc_page_list(ic->journal_pages);
4267 	if (!ic->journal) {
4268 		*error = "Could not allocate memory for journal";
4269 		r = -ENOMEM;
4270 		goto bad;
4271 	}
4272 	if (ic->journal_crypt_alg.alg_string) {
4273 		unsigned int ivsize, blocksize;
4274 		struct journal_completion comp;
4275 
4276 		comp.ic = ic;
4277 		ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
4278 		if (IS_ERR(ic->journal_crypt)) {
4279 			*error = "Invalid journal cipher";
4280 			r = PTR_ERR(ic->journal_crypt);
4281 			ic->journal_crypt = NULL;
4282 			goto bad;
4283 		}
4284 		ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
4285 		blocksize = crypto_skcipher_blocksize(ic->journal_crypt);
4286 
4287 		if (ic->journal_crypt_alg.key) {
4288 			r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key,
4289 						   ic->journal_crypt_alg.key_size);
4290 			if (r) {
4291 				*error = "Error setting encryption key";
4292 				goto bad;
4293 			}
4294 		}
4295 		DEBUG_print("cipher %s, block size %u iv size %u\n",
4296 			    ic->journal_crypt_alg.alg_string, blocksize, ivsize);
4297 
4298 		ic->journal_io = dm_integrity_alloc_page_list(ic->journal_pages);
4299 		if (!ic->journal_io) {
4300 			*error = "Could not allocate memory for journal io";
4301 			r = -ENOMEM;
4302 			goto bad;
4303 		}
4304 
4305 		if (blocksize == 1) {
4306 			struct scatterlist *sg;
4307 
4308 			req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
4309 			if (!req) {
4310 				*error = "Could not allocate crypt request";
4311 				r = -ENOMEM;
4312 				goto bad;
4313 			}
4314 
4315 			crypt_iv = kzalloc(ivsize, GFP_KERNEL);
4316 			if (!crypt_iv) {
4317 				*error = "Could not allocate iv";
4318 				r = -ENOMEM;
4319 				goto bad;
4320 			}
4321 
4322 			ic->journal_xor = dm_integrity_alloc_page_list(ic->journal_pages);
4323 			if (!ic->journal_xor) {
4324 				*error = "Could not allocate memory for journal xor";
4325 				r = -ENOMEM;
4326 				goto bad;
4327 			}
4328 
4329 			sg = kvmalloc_array(ic->journal_pages + 1,
4330 					    sizeof(struct scatterlist),
4331 					    GFP_KERNEL);
4332 			if (!sg) {
4333 				*error = "Unable to allocate sg list";
4334 				r = -ENOMEM;
4335 				goto bad;
4336 			}
4337 			sg_init_table(sg, ic->journal_pages + 1);
4338 			for (i = 0; i < ic->journal_pages; i++) {
4339 				char *va = lowmem_page_address(ic->journal_xor[i].page);
4340 
4341 				clear_page(va);
4342 				sg_set_buf(&sg[i], va, PAGE_SIZE);
4343 			}
4344 			sg_set_buf(&sg[i], &ic->commit_ids, sizeof(ic->commit_ids));
4345 
4346 			skcipher_request_set_crypt(req, sg, sg,
4347 						   PAGE_SIZE * ic->journal_pages + sizeof(ic->commit_ids), crypt_iv);
4348 			init_completion(&comp.comp);
4349 			comp.in_flight = (atomic_t)ATOMIC_INIT(1);
4350 			if (do_crypt(true, req, &comp))
4351 				wait_for_completion(&comp.comp);
4352 			kvfree(sg);
4353 			r = dm_integrity_failed(ic);
4354 			if (r) {
4355 				*error = "Unable to encrypt journal";
4356 				goto bad;
4357 			}
4358 			DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data");
4359 
4360 			crypto_free_skcipher(ic->journal_crypt);
4361 			ic->journal_crypt = NULL;
4362 		} else {
4363 			unsigned int crypt_len = roundup(ivsize, blocksize);
4364 
4365 			req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
4366 			if (!req) {
4367 				*error = "Could not allocate crypt request";
4368 				r = -ENOMEM;
4369 				goto bad;
4370 			}
4371 
4372 			crypt_iv = kmalloc(ivsize, GFP_KERNEL);
4373 			if (!crypt_iv) {
4374 				*error = "Could not allocate iv";
4375 				r = -ENOMEM;
4376 				goto bad;
4377 			}
4378 
4379 			crypt_data = kmalloc(crypt_len, GFP_KERNEL);
4380 			if (!crypt_data) {
4381 				*error = "Unable to allocate crypt data";
4382 				r = -ENOMEM;
4383 				goto bad;
4384 			}
4385 
4386 			ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal);
4387 			if (!ic->journal_scatterlist) {
4388 				*error = "Unable to allocate sg list";
4389 				r = -ENOMEM;
4390 				goto bad;
4391 			}
4392 			ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io);
4393 			if (!ic->journal_io_scatterlist) {
4394 				*error = "Unable to allocate sg list";
4395 				r = -ENOMEM;
4396 				goto bad;
4397 			}
4398 			ic->sk_requests = kvmalloc_array(ic->journal_sections,
4399 							 sizeof(struct skcipher_request *),
4400 							 GFP_KERNEL | __GFP_ZERO);
4401 			if (!ic->sk_requests) {
4402 				*error = "Unable to allocate sk requests";
4403 				r = -ENOMEM;
4404 				goto bad;
4405 			}
4406 			for (i = 0; i < ic->journal_sections; i++) {
4407 				struct scatterlist sg;
4408 				struct skcipher_request *section_req;
4409 				__le32 section_le = cpu_to_le32(i);
4410 
4411 				memset(crypt_iv, 0x00, ivsize);
4412 				memset(crypt_data, 0x00, crypt_len);
4413 				memcpy(crypt_data, &section_le, min_t(size_t, crypt_len, sizeof(section_le)));
4414 
4415 				sg_init_one(&sg, crypt_data, crypt_len);
4416 				skcipher_request_set_crypt(req, &sg, &sg, crypt_len, crypt_iv);
4417 				init_completion(&comp.comp);
4418 				comp.in_flight = (atomic_t)ATOMIC_INIT(1);
4419 				if (do_crypt(true, req, &comp))
4420 					wait_for_completion(&comp.comp);
4421 
4422 				r = dm_integrity_failed(ic);
4423 				if (r) {
4424 					*error = "Unable to generate iv";
4425 					goto bad;
4426 				}
4427 
4428 				section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
4429 				if (!section_req) {
4430 					*error = "Unable to allocate crypt request";
4431 					r = -ENOMEM;
4432 					goto bad;
4433 				}
4434 				section_req->iv = kmalloc_array(ivsize, 2,
4435 								GFP_KERNEL);
4436 				if (!section_req->iv) {
4437 					skcipher_request_free(section_req);
4438 					*error = "Unable to allocate iv";
4439 					r = -ENOMEM;
4440 					goto bad;
4441 				}
4442 				memcpy(section_req->iv + ivsize, crypt_data, ivsize);
4443 				section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT;
4444 				ic->sk_requests[i] = section_req;
4445 				DEBUG_bytes(crypt_data, ivsize, "iv(%u)", i);
4446 			}
4447 		}
4448 	}
4449 
4450 	for (i = 0; i < N_COMMIT_IDS; i++) {
4451 		unsigned int j;
4452 
4453 retest_commit_id:
4454 		for (j = 0; j < i; j++) {
4455 			if (ic->commit_ids[j] == ic->commit_ids[i]) {
4456 				ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1);
4457 				goto retest_commit_id;
4458 			}
4459 		}
4460 		DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]);
4461 	}
4462 
4463 	journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node);
4464 	if (journal_tree_size > ULONG_MAX) {
4465 		*error = "Journal doesn't fit into memory";
4466 		r = -ENOMEM;
4467 		goto bad;
4468 	}
4469 	ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL);
4470 	if (!ic->journal_tree) {
4471 		*error = "Could not allocate memory for journal tree";
4472 		r = -ENOMEM;
4473 	}
4474 bad:
4475 	kfree(crypt_data);
4476 	kfree(crypt_iv);
4477 	skcipher_request_free(req);
4478 
4479 	return r;
4480 }
4481 
4482 /*
4483  * Construct a integrity mapping
4484  *
4485  * Arguments:
4486  *	device
4487  *	offset from the start of the device
4488  *	tag size
4489  *	D - direct writes, J - journal writes, B - bitmap mode, R - recovery mode
4490  *	number of optional arguments
4491  *	optional arguments:
4492  *		journal_sectors
4493  *		interleave_sectors
4494  *		buffer_sectors
4495  *		journal_watermark
4496  *		commit_time
4497  *		meta_device
4498  *		block_size
4499  *		sectors_per_bit
4500  *		bitmap_flush_interval
4501  *		internal_hash
4502  *		journal_crypt
4503  *		journal_mac
4504  *		recalculate
4505  */
dm_integrity_ctr(struct dm_target * ti,unsigned int argc,char ** argv)4506 static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
4507 {
4508 	struct dm_integrity_c *ic;
4509 	char dummy;
4510 	int r;
4511 	unsigned int extra_args;
4512 	struct dm_arg_set as;
4513 	static const struct dm_arg _args[] = {
4514 		{0, 18, "Invalid number of feature args"},
4515 	};
4516 	unsigned int journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec;
4517 	bool should_write_sb;
4518 	__u64 threshold;
4519 	unsigned long long start;
4520 	__s8 log2_sectors_per_bitmap_bit = -1;
4521 	__s8 log2_blocks_per_bitmap_bit;
4522 	__u64 bits_in_journal;
4523 	__u64 n_bitmap_bits;
4524 
4525 #define DIRECT_ARGUMENTS	4
4526 
4527 	if (argc <= DIRECT_ARGUMENTS) {
4528 		ti->error = "Invalid argument count";
4529 		return -EINVAL;
4530 	}
4531 
4532 	ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL);
4533 	if (!ic) {
4534 		ti->error = "Cannot allocate integrity context";
4535 		return -ENOMEM;
4536 	}
4537 	ti->private = ic;
4538 	ti->per_io_data_size = sizeof(struct dm_integrity_io);
4539 	ic->ti = ti;
4540 
4541 	ic->in_progress = RB_ROOT;
4542 	INIT_LIST_HEAD(&ic->wait_list);
4543 	init_waitqueue_head(&ic->endio_wait);
4544 	bio_list_init(&ic->flush_bio_list);
4545 	init_waitqueue_head(&ic->copy_to_journal_wait);
4546 	init_completion(&ic->crypto_backoff);
4547 	atomic64_set(&ic->number_of_mismatches, 0);
4548 	ic->bitmap_flush_interval = BITMAP_FLUSH_INTERVAL;
4549 
4550 	r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev);
4551 	if (r) {
4552 		ti->error = "Device lookup failed";
4553 		goto bad;
4554 	}
4555 
4556 	if (sscanf(argv[1], "%llu%c", &start, &dummy) != 1 || start != (sector_t)start) {
4557 		ti->error = "Invalid starting offset";
4558 		r = -EINVAL;
4559 		goto bad;
4560 	}
4561 	ic->start = start;
4562 
4563 	if (strcmp(argv[2], "-")) {
4564 		if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) {
4565 			ti->error = "Invalid tag size";
4566 			r = -EINVAL;
4567 			goto bad;
4568 		}
4569 	}
4570 
4571 	if (!strcmp(argv[3], "J") || !strcmp(argv[3], "B") ||
4572 	    !strcmp(argv[3], "D") || !strcmp(argv[3], "R") ||
4573 	    !strcmp(argv[3], "I")) {
4574 		ic->mode = argv[3][0];
4575 	} else {
4576 		ti->error = "Invalid mode (expecting J, B, D, R, I)";
4577 		r = -EINVAL;
4578 		goto bad;
4579 	}
4580 
4581 	journal_sectors = 0;
4582 	interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
4583 	buffer_sectors = DEFAULT_BUFFER_SECTORS;
4584 	journal_watermark = DEFAULT_JOURNAL_WATERMARK;
4585 	sync_msec = DEFAULT_SYNC_MSEC;
4586 	ic->sectors_per_block = 1;
4587 
4588 	as.argc = argc - DIRECT_ARGUMENTS;
4589 	as.argv = argv + DIRECT_ARGUMENTS;
4590 	r = dm_read_arg_group(_args, &as, &extra_args, &ti->error);
4591 	if (r)
4592 		goto bad;
4593 
4594 	while (extra_args--) {
4595 		const char *opt_string;
4596 		unsigned int val;
4597 		unsigned long long llval;
4598 
4599 		opt_string = dm_shift_arg(&as);
4600 		if (!opt_string) {
4601 			r = -EINVAL;
4602 			ti->error = "Not enough feature arguments";
4603 			goto bad;
4604 		}
4605 		if (sscanf(opt_string, "journal_sectors:%u%c", &val, &dummy) == 1)
4606 			journal_sectors = val ? val : 1;
4607 		else if (sscanf(opt_string, "interleave_sectors:%u%c", &val, &dummy) == 1)
4608 			interleave_sectors = val;
4609 		else if (sscanf(opt_string, "buffer_sectors:%u%c", &val, &dummy) == 1)
4610 			buffer_sectors = val;
4611 		else if (sscanf(opt_string, "journal_watermark:%u%c", &val, &dummy) == 1 && val <= 100)
4612 			journal_watermark = val;
4613 		else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1)
4614 			sync_msec = val;
4615 		else if (!strncmp(opt_string, "meta_device:", strlen("meta_device:"))) {
4616 			if (ic->meta_dev) {
4617 				dm_put_device(ti, ic->meta_dev);
4618 				ic->meta_dev = NULL;
4619 			}
4620 			r = dm_get_device(ti, strchr(opt_string, ':') + 1,
4621 					  dm_table_get_mode(ti->table), &ic->meta_dev);
4622 			if (r) {
4623 				ti->error = "Device lookup failed";
4624 				goto bad;
4625 			}
4626 		} else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) {
4627 			if (val < 1 << SECTOR_SHIFT ||
4628 			    val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT ||
4629 			    (val & (val - 1))) {
4630 				r = -EINVAL;
4631 				ti->error = "Invalid block_size argument";
4632 				goto bad;
4633 			}
4634 			ic->sectors_per_block = val >> SECTOR_SHIFT;
4635 		} else if (sscanf(opt_string, "sectors_per_bit:%llu%c", &llval, &dummy) == 1) {
4636 			log2_sectors_per_bitmap_bit = !llval ? 0 : __ilog2_u64(llval);
4637 		} else if (sscanf(opt_string, "bitmap_flush_interval:%u%c", &val, &dummy) == 1) {
4638 			if ((uint64_t)val >= (uint64_t)UINT_MAX * 1000 / HZ) {
4639 				r = -EINVAL;
4640 				ti->error = "Invalid bitmap_flush_interval argument";
4641 				goto bad;
4642 			}
4643 			ic->bitmap_flush_interval = msecs_to_jiffies(val);
4644 		} else if (!strncmp(opt_string, "internal_hash:", strlen("internal_hash:"))) {
4645 			r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error,
4646 					    "Invalid internal_hash argument");
4647 			if (r)
4648 				goto bad;
4649 		} else if (!strncmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) {
4650 			r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error,
4651 					    "Invalid journal_crypt argument");
4652 			if (r)
4653 				goto bad;
4654 		} else if (!strncmp(opt_string, "journal_mac:", strlen("journal_mac:"))) {
4655 			r = get_alg_and_key(opt_string, &ic->journal_mac_alg, &ti->error,
4656 					    "Invalid journal_mac argument");
4657 			if (r)
4658 				goto bad;
4659 		} else if (!strcmp(opt_string, "recalculate")) {
4660 			ic->recalculate_flag = true;
4661 		} else if (!strcmp(opt_string, "reset_recalculate")) {
4662 			ic->recalculate_flag = true;
4663 			ic->reset_recalculate_flag = true;
4664 		} else if (!strcmp(opt_string, "allow_discards")) {
4665 			ic->discard = true;
4666 		} else if (!strcmp(opt_string, "fix_padding")) {
4667 			ic->fix_padding = true;
4668 		} else if (!strcmp(opt_string, "fix_hmac")) {
4669 			ic->fix_hmac = true;
4670 		} else if (!strcmp(opt_string, "legacy_recalculate")) {
4671 			ic->legacy_recalculate = true;
4672 		} else {
4673 			r = -EINVAL;
4674 			ti->error = "Invalid argument";
4675 			goto bad;
4676 		}
4677 	}
4678 
4679 	ic->data_device_sectors = bdev_nr_sectors(ic->dev->bdev);
4680 	if (!ic->meta_dev)
4681 		ic->meta_device_sectors = ic->data_device_sectors;
4682 	else
4683 		ic->meta_device_sectors = bdev_nr_sectors(ic->meta_dev->bdev);
4684 
4685 	if (!journal_sectors) {
4686 		journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS,
4687 				      ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR);
4688 	}
4689 
4690 	if (!buffer_sectors)
4691 		buffer_sectors = 1;
4692 	ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT);
4693 
4694 	r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error,
4695 		    "Invalid internal hash", "Error setting internal hash key");
4696 	if (r)
4697 		goto bad;
4698 
4699 	r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error,
4700 		    "Invalid journal mac", "Error setting journal mac key");
4701 	if (r)
4702 		goto bad;
4703 
4704 	if (!ic->tag_size) {
4705 		if (!ic->internal_hash) {
4706 			ti->error = "Unknown tag size";
4707 			r = -EINVAL;
4708 			goto bad;
4709 		}
4710 		ic->tag_size = crypto_shash_digestsize(ic->internal_hash);
4711 	}
4712 	if (ic->tag_size > MAX_TAG_SIZE) {
4713 		ti->error = "Too big tag size";
4714 		r = -EINVAL;
4715 		goto bad;
4716 	}
4717 	if (!(ic->tag_size & (ic->tag_size - 1)))
4718 		ic->log2_tag_size = __ffs(ic->tag_size);
4719 	else
4720 		ic->log2_tag_size = -1;
4721 
4722 	if (ic->mode == 'I') {
4723 		struct blk_integrity *bi;
4724 		if (ic->meta_dev) {
4725 			r = -EINVAL;
4726 			ti->error = "Metadata device not supported in inline mode";
4727 			goto bad;
4728 		}
4729 		if (!ic->internal_hash_alg.alg_string) {
4730 			r = -EINVAL;
4731 			ti->error = "Internal hash not set in inline mode";
4732 			goto bad;
4733 		}
4734 		if (ic->journal_crypt_alg.alg_string || ic->journal_mac_alg.alg_string) {
4735 			r = -EINVAL;
4736 			ti->error = "Journal crypt not supported in inline mode";
4737 			goto bad;
4738 		}
4739 		if (ic->discard) {
4740 			r = -EINVAL;
4741 			ti->error = "Discards not supported in inline mode";
4742 			goto bad;
4743 		}
4744 		bi = blk_get_integrity(ic->dev->bdev->bd_disk);
4745 		if (!bi || bi->csum_type != BLK_INTEGRITY_CSUM_NONE) {
4746 			r = -EINVAL;
4747 			ti->error = "Integrity profile not supported";
4748 			goto bad;
4749 		}
4750 		/*printk("tag_size: %u, tuple_size: %u\n", bi->tag_size, bi->tuple_size);*/
4751 		if (bi->tuple_size < ic->tag_size) {
4752 			r = -EINVAL;
4753 			ti->error = "The integrity profile is smaller than tag size";
4754 			goto bad;
4755 		}
4756 		if ((unsigned long)bi->tuple_size > PAGE_SIZE / 2) {
4757 			r = -EINVAL;
4758 			ti->error = "Too big tuple size";
4759 			goto bad;
4760 		}
4761 		ic->tuple_size = bi->tuple_size;
4762 		if (1 << bi->interval_exp != ic->sectors_per_block << SECTOR_SHIFT) {
4763 			r = -EINVAL;
4764 			ti->error = "Integrity profile sector size mismatch";
4765 			goto bad;
4766 		}
4767 	}
4768 
4769 	if (ic->mode == 'B' && !ic->internal_hash) {
4770 		r = -EINVAL;
4771 		ti->error = "Bitmap mode can be only used with internal hash";
4772 		goto bad;
4773 	}
4774 
4775 	if (ic->discard && !ic->internal_hash) {
4776 		r = -EINVAL;
4777 		ti->error = "Discard can be only used with internal hash";
4778 		goto bad;
4779 	}
4780 
4781 	ic->autocommit_jiffies = msecs_to_jiffies(sync_msec);
4782 	ic->autocommit_msec = sync_msec;
4783 	timer_setup(&ic->autocommit_timer, autocommit_fn, 0);
4784 
4785 	ic->io = dm_io_client_create();
4786 	if (IS_ERR(ic->io)) {
4787 		r = PTR_ERR(ic->io);
4788 		ic->io = NULL;
4789 		ti->error = "Cannot allocate dm io";
4790 		goto bad;
4791 	}
4792 
4793 	r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache);
4794 	if (r) {
4795 		ti->error = "Cannot allocate mempool";
4796 		goto bad;
4797 	}
4798 
4799 	r = mempool_init_page_pool(&ic->recheck_pool, 1, ic->mode == 'I' ? 1 : 0);
4800 	if (r) {
4801 		ti->error = "Cannot allocate mempool";
4802 		goto bad;
4803 	}
4804 
4805 	if (ic->mode == 'I') {
4806 		r = bioset_init(&ic->recheck_bios, RECHECK_POOL_SIZE, 0, BIOSET_NEED_BVECS);
4807 		if (r) {
4808 			ti->error = "Cannot allocate bio set";
4809 			goto bad;
4810 		}
4811 		r = bioset_integrity_create(&ic->recheck_bios, RECHECK_POOL_SIZE);
4812 		if (r) {
4813 			ti->error = "Cannot allocate bio integrity set";
4814 			r = -ENOMEM;
4815 			goto bad;
4816 		}
4817 		r = bioset_init(&ic->recalc_bios, 1, 0, BIOSET_NEED_BVECS);
4818 		if (r) {
4819 			ti->error = "Cannot allocate bio set";
4820 			goto bad;
4821 		}
4822 		r = bioset_integrity_create(&ic->recalc_bios, 1);
4823 		if (r) {
4824 			ti->error = "Cannot allocate bio integrity set";
4825 			r = -ENOMEM;
4826 			goto bad;
4827 		}
4828 	}
4829 
4830 	ic->metadata_wq = alloc_workqueue("dm-integrity-metadata",
4831 					  WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE);
4832 	if (!ic->metadata_wq) {
4833 		ti->error = "Cannot allocate workqueue";
4834 		r = -ENOMEM;
4835 		goto bad;
4836 	}
4837 
4838 	/*
4839 	 * If this workqueue weren't ordered, it would cause bio reordering
4840 	 * and reduced performance.
4841 	 */
4842 	ic->wait_wq = alloc_ordered_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM);
4843 	if (!ic->wait_wq) {
4844 		ti->error = "Cannot allocate workqueue";
4845 		r = -ENOMEM;
4846 		goto bad;
4847 	}
4848 
4849 	ic->offload_wq = alloc_workqueue("dm-integrity-offload", WQ_MEM_RECLAIM,
4850 					  METADATA_WORKQUEUE_MAX_ACTIVE);
4851 	if (!ic->offload_wq) {
4852 		ti->error = "Cannot allocate workqueue";
4853 		r = -ENOMEM;
4854 		goto bad;
4855 	}
4856 
4857 	ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1);
4858 	if (!ic->commit_wq) {
4859 		ti->error = "Cannot allocate workqueue";
4860 		r = -ENOMEM;
4861 		goto bad;
4862 	}
4863 	INIT_WORK(&ic->commit_work, integrity_commit);
4864 
4865 	if (ic->mode == 'J' || ic->mode == 'B') {
4866 		ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1);
4867 		if (!ic->writer_wq) {
4868 			ti->error = "Cannot allocate workqueue";
4869 			r = -ENOMEM;
4870 			goto bad;
4871 		}
4872 		INIT_WORK(&ic->writer_work, integrity_writer);
4873 	}
4874 
4875 	ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL);
4876 	if (!ic->sb) {
4877 		r = -ENOMEM;
4878 		ti->error = "Cannot allocate superblock area";
4879 		goto bad;
4880 	}
4881 
4882 	r = sync_rw_sb(ic, REQ_OP_READ);
4883 	if (r) {
4884 		ti->error = "Error reading superblock";
4885 		goto bad;
4886 	}
4887 	should_write_sb = false;
4888 	if (memcmp(ic->sb->magic, SB_MAGIC, 8)) {
4889 		if (ic->mode != 'R') {
4890 			if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) {
4891 				r = -EINVAL;
4892 				ti->error = "The device is not initialized";
4893 				goto bad;
4894 			}
4895 		}
4896 
4897 		r = initialize_superblock(ic, journal_sectors, interleave_sectors);
4898 		if (r) {
4899 			ti->error = "Could not initialize superblock";
4900 			goto bad;
4901 		}
4902 		if (ic->mode != 'R')
4903 			should_write_sb = true;
4904 	}
4905 
4906 	if (!ic->sb->version || ic->sb->version > SB_VERSION_6) {
4907 		r = -EINVAL;
4908 		ti->error = "Unknown version";
4909 		goto bad;
4910 	}
4911 	if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) != (ic->mode == 'I')) {
4912 		r = -EINVAL;
4913 		ti->error = "Inline flag mismatch";
4914 		goto bad;
4915 	}
4916 	if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) {
4917 		r = -EINVAL;
4918 		ti->error = "Tag size doesn't match the information in superblock";
4919 		goto bad;
4920 	}
4921 	if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) {
4922 		r = -EINVAL;
4923 		ti->error = "Block size doesn't match the information in superblock";
4924 		goto bad;
4925 	}
4926 	if (ic->mode != 'I') {
4927 		if (!le32_to_cpu(ic->sb->journal_sections)) {
4928 			r = -EINVAL;
4929 			ti->error = "Corrupted superblock, journal_sections is 0";
4930 			goto bad;
4931 		}
4932 	} else {
4933 		if (le32_to_cpu(ic->sb->journal_sections)) {
4934 			r = -EINVAL;
4935 			ti->error = "Corrupted superblock, journal_sections is not 0";
4936 			goto bad;
4937 		}
4938 	}
4939 	/* make sure that ti->max_io_len doesn't overflow */
4940 	if (!ic->meta_dev) {
4941 		if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS ||
4942 		    ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) {
4943 			r = -EINVAL;
4944 			ti->error = "Invalid interleave_sectors in the superblock";
4945 			goto bad;
4946 		}
4947 	} else {
4948 		if (ic->sb->log2_interleave_sectors) {
4949 			r = -EINVAL;
4950 			ti->error = "Invalid interleave_sectors in the superblock";
4951 			goto bad;
4952 		}
4953 	}
4954 	if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) {
4955 		r = -EINVAL;
4956 		ti->error = "Journal mac mismatch";
4957 		goto bad;
4958 	}
4959 
4960 	get_provided_data_sectors(ic);
4961 	if (!ic->provided_data_sectors) {
4962 		r = -EINVAL;
4963 		ti->error = "The device is too small";
4964 		goto bad;
4965 	}
4966 
4967 try_smaller_buffer:
4968 	r = calculate_device_limits(ic);
4969 	if (r) {
4970 		if (ic->meta_dev) {
4971 			if (ic->log2_buffer_sectors > 3) {
4972 				ic->log2_buffer_sectors--;
4973 				goto try_smaller_buffer;
4974 			}
4975 		}
4976 		ti->error = "The device is too small";
4977 		goto bad;
4978 	}
4979 
4980 	if (log2_sectors_per_bitmap_bit < 0)
4981 		log2_sectors_per_bitmap_bit = __fls(DEFAULT_SECTORS_PER_BITMAP_BIT);
4982 	if (log2_sectors_per_bitmap_bit < ic->sb->log2_sectors_per_block)
4983 		log2_sectors_per_bitmap_bit = ic->sb->log2_sectors_per_block;
4984 
4985 	bits_in_journal = ((__u64)ic->journal_section_sectors * ic->journal_sections) << (SECTOR_SHIFT + 3);
4986 	if (bits_in_journal > UINT_MAX)
4987 		bits_in_journal = UINT_MAX;
4988 	if (bits_in_journal)
4989 		while (bits_in_journal < (ic->provided_data_sectors + ((sector_t)1 << log2_sectors_per_bitmap_bit) - 1) >> log2_sectors_per_bitmap_bit)
4990 			log2_sectors_per_bitmap_bit++;
4991 
4992 	log2_blocks_per_bitmap_bit = log2_sectors_per_bitmap_bit - ic->sb->log2_sectors_per_block;
4993 	ic->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit;
4994 	if (should_write_sb)
4995 		ic->sb->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit;
4996 
4997 	n_bitmap_bits = ((ic->provided_data_sectors >> ic->sb->log2_sectors_per_block)
4998 				+ (((sector_t)1 << log2_blocks_per_bitmap_bit) - 1)) >> log2_blocks_per_bitmap_bit;
4999 	ic->n_bitmap_blocks = DIV_ROUND_UP(n_bitmap_bits, BITMAP_BLOCK_SIZE * 8);
5000 
5001 	if (!ic->meta_dev)
5002 		ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run));
5003 
5004 	if (ti->len > ic->provided_data_sectors) {
5005 		r = -EINVAL;
5006 		ti->error = "Not enough provided sectors for requested mapping size";
5007 		goto bad;
5008 	}
5009 
5010 	threshold = (__u64)ic->journal_entries * (100 - journal_watermark);
5011 	threshold += 50;
5012 	do_div(threshold, 100);
5013 	ic->free_sectors_threshold = threshold;
5014 
5015 	DEBUG_print("initialized:\n");
5016 	DEBUG_print("	integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size));
5017 	DEBUG_print("	journal_entry_size %u\n", ic->journal_entry_size);
5018 	DEBUG_print("	journal_entries_per_sector %u\n", ic->journal_entries_per_sector);
5019 	DEBUG_print("	journal_section_entries %u\n", ic->journal_section_entries);
5020 	DEBUG_print("	journal_section_sectors %u\n", ic->journal_section_sectors);
5021 	DEBUG_print("	journal_sections %u\n", (unsigned int)le32_to_cpu(ic->sb->journal_sections));
5022 	DEBUG_print("	journal_entries %u\n", ic->journal_entries);
5023 	DEBUG_print("	log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors);
5024 	DEBUG_print("	data_device_sectors 0x%llx\n", bdev_nr_sectors(ic->dev->bdev));
5025 	DEBUG_print("	initial_sectors 0x%x\n", ic->initial_sectors);
5026 	DEBUG_print("	metadata_run 0x%x\n", ic->metadata_run);
5027 	DEBUG_print("	log2_metadata_run %d\n", ic->log2_metadata_run);
5028 	DEBUG_print("	provided_data_sectors 0x%llx (%llu)\n", ic->provided_data_sectors, ic->provided_data_sectors);
5029 	DEBUG_print("	log2_buffer_sectors %u\n", ic->log2_buffer_sectors);
5030 	DEBUG_print("	bits_in_journal %llu\n", bits_in_journal);
5031 
5032 	if (ic->recalculate_flag && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) {
5033 		ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
5034 		ic->sb->recalc_sector = cpu_to_le64(0);
5035 	}
5036 
5037 	if (ic->internal_hash) {
5038 		ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1);
5039 		if (!ic->recalc_wq) {
5040 			ti->error = "Cannot allocate workqueue";
5041 			r = -ENOMEM;
5042 			goto bad;
5043 		}
5044 		INIT_WORK(&ic->recalc_work, ic->mode == 'I' ? integrity_recalc_inline : integrity_recalc);
5045 	} else {
5046 		if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
5047 			ti->error = "Recalculate can only be specified with internal_hash";
5048 			r = -EINVAL;
5049 			goto bad;
5050 		}
5051 	}
5052 
5053 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
5054 	    le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors &&
5055 	    dm_integrity_disable_recalculate(ic)) {
5056 		ti->error = "Recalculating with HMAC is disabled for security reasons - if you really need it, use the argument \"legacy_recalculate\"";
5057 		r = -EOPNOTSUPP;
5058 		goto bad;
5059 	}
5060 
5061 	ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev,
5062 			1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL, 0);
5063 	if (IS_ERR(ic->bufio)) {
5064 		r = PTR_ERR(ic->bufio);
5065 		ti->error = "Cannot initialize dm-bufio";
5066 		ic->bufio = NULL;
5067 		goto bad;
5068 	}
5069 	dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors);
5070 
5071 	if (ic->mode != 'R' && ic->mode != 'I') {
5072 		r = create_journal(ic, &ti->error);
5073 		if (r)
5074 			goto bad;
5075 
5076 	}
5077 
5078 	if (ic->mode == 'B') {
5079 		unsigned int i;
5080 		unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE);
5081 
5082 		ic->recalc_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages);
5083 		if (!ic->recalc_bitmap) {
5084 			r = -ENOMEM;
5085 			goto bad;
5086 		}
5087 		ic->may_write_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages);
5088 		if (!ic->may_write_bitmap) {
5089 			r = -ENOMEM;
5090 			goto bad;
5091 		}
5092 		ic->bbs = kvmalloc_array(ic->n_bitmap_blocks, sizeof(struct bitmap_block_status), GFP_KERNEL);
5093 		if (!ic->bbs) {
5094 			r = -ENOMEM;
5095 			goto bad;
5096 		}
5097 		INIT_DELAYED_WORK(&ic->bitmap_flush_work, bitmap_flush_work);
5098 		for (i = 0; i < ic->n_bitmap_blocks; i++) {
5099 			struct bitmap_block_status *bbs = &ic->bbs[i];
5100 			unsigned int sector, pl_index, pl_offset;
5101 
5102 			INIT_WORK(&bbs->work, bitmap_block_work);
5103 			bbs->ic = ic;
5104 			bbs->idx = i;
5105 			bio_list_init(&bbs->bio_queue);
5106 			spin_lock_init(&bbs->bio_queue_lock);
5107 
5108 			sector = i * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT);
5109 			pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
5110 			pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
5111 
5112 			bbs->bitmap = lowmem_page_address(ic->journal[pl_index].page) + pl_offset;
5113 		}
5114 	}
5115 
5116 	if (should_write_sb) {
5117 		init_journal(ic, 0, ic->journal_sections, 0);
5118 		r = dm_integrity_failed(ic);
5119 		if (unlikely(r)) {
5120 			ti->error = "Error initializing journal";
5121 			goto bad;
5122 		}
5123 		r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
5124 		if (r) {
5125 			ti->error = "Error initializing superblock";
5126 			goto bad;
5127 		}
5128 		ic->just_formatted = true;
5129 	}
5130 
5131 	if (!ic->meta_dev && ic->mode != 'I') {
5132 		r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors);
5133 		if (r)
5134 			goto bad;
5135 	}
5136 	if (ic->mode == 'B') {
5137 		unsigned int max_io_len;
5138 
5139 		max_io_len = ((sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit) * (BITMAP_BLOCK_SIZE * 8);
5140 		if (!max_io_len)
5141 			max_io_len = 1U << 31;
5142 		DEBUG_print("max_io_len: old %u, new %u\n", ti->max_io_len, max_io_len);
5143 		if (!ti->max_io_len || ti->max_io_len > max_io_len) {
5144 			r = dm_set_target_max_io_len(ti, max_io_len);
5145 			if (r)
5146 				goto bad;
5147 		}
5148 	}
5149 
5150 	ti->num_flush_bios = 1;
5151 	ti->flush_supported = true;
5152 	if (ic->discard)
5153 		ti->num_discard_bios = 1;
5154 
5155 	if (ic->mode == 'I')
5156 		ti->mempool_needs_integrity = true;
5157 
5158 	dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1);
5159 	return 0;
5160 
5161 bad:
5162 	dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0);
5163 	dm_integrity_dtr(ti);
5164 	return r;
5165 }
5166 
dm_integrity_dtr(struct dm_target * ti)5167 static void dm_integrity_dtr(struct dm_target *ti)
5168 {
5169 	struct dm_integrity_c *ic = ti->private;
5170 
5171 	BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
5172 	BUG_ON(!list_empty(&ic->wait_list));
5173 
5174 	if (ic->mode == 'B')
5175 		cancel_delayed_work_sync(&ic->bitmap_flush_work);
5176 	if (ic->metadata_wq)
5177 		destroy_workqueue(ic->metadata_wq);
5178 	if (ic->wait_wq)
5179 		destroy_workqueue(ic->wait_wq);
5180 	if (ic->offload_wq)
5181 		destroy_workqueue(ic->offload_wq);
5182 	if (ic->commit_wq)
5183 		destroy_workqueue(ic->commit_wq);
5184 	if (ic->writer_wq)
5185 		destroy_workqueue(ic->writer_wq);
5186 	if (ic->recalc_wq)
5187 		destroy_workqueue(ic->recalc_wq);
5188 	kvfree(ic->bbs);
5189 	if (ic->bufio)
5190 		dm_bufio_client_destroy(ic->bufio);
5191 	bioset_exit(&ic->recalc_bios);
5192 	bioset_exit(&ic->recheck_bios);
5193 	mempool_exit(&ic->recheck_pool);
5194 	mempool_exit(&ic->journal_io_mempool);
5195 	if (ic->io)
5196 		dm_io_client_destroy(ic->io);
5197 	if (ic->dev)
5198 		dm_put_device(ti, ic->dev);
5199 	if (ic->meta_dev)
5200 		dm_put_device(ti, ic->meta_dev);
5201 	dm_integrity_free_page_list(ic->journal);
5202 	dm_integrity_free_page_list(ic->journal_io);
5203 	dm_integrity_free_page_list(ic->journal_xor);
5204 	dm_integrity_free_page_list(ic->recalc_bitmap);
5205 	dm_integrity_free_page_list(ic->may_write_bitmap);
5206 	if (ic->journal_scatterlist)
5207 		dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist);
5208 	if (ic->journal_io_scatterlist)
5209 		dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist);
5210 	if (ic->sk_requests) {
5211 		unsigned int i;
5212 
5213 		for (i = 0; i < ic->journal_sections; i++) {
5214 			struct skcipher_request *req;
5215 
5216 			req = ic->sk_requests[i];
5217 			if (req) {
5218 				kfree_sensitive(req->iv);
5219 				skcipher_request_free(req);
5220 			}
5221 		}
5222 		kvfree(ic->sk_requests);
5223 	}
5224 	kvfree(ic->journal_tree);
5225 	if (ic->sb)
5226 		free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT);
5227 
5228 	if (ic->internal_hash)
5229 		crypto_free_shash(ic->internal_hash);
5230 	free_alg(&ic->internal_hash_alg);
5231 
5232 	if (ic->journal_crypt)
5233 		crypto_free_skcipher(ic->journal_crypt);
5234 	free_alg(&ic->journal_crypt_alg);
5235 
5236 	if (ic->journal_mac)
5237 		crypto_free_shash(ic->journal_mac);
5238 	free_alg(&ic->journal_mac_alg);
5239 
5240 	kfree(ic);
5241 	dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);
5242 }
5243 
5244 static struct target_type integrity_target = {
5245 	.name			= "integrity",
5246 	.version		= {1, 13, 0},
5247 	.module			= THIS_MODULE,
5248 	.features		= DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
5249 	.ctr			= dm_integrity_ctr,
5250 	.dtr			= dm_integrity_dtr,
5251 	.map			= dm_integrity_map,
5252 	.end_io			= dm_integrity_end_io,
5253 	.postsuspend		= dm_integrity_postsuspend,
5254 	.resume			= dm_integrity_resume,
5255 	.status			= dm_integrity_status,
5256 	.iterate_devices	= dm_integrity_iterate_devices,
5257 	.io_hints		= dm_integrity_io_hints,
5258 };
5259 
dm_integrity_init(void)5260 static int __init dm_integrity_init(void)
5261 {
5262 	int r;
5263 
5264 	journal_io_cache = kmem_cache_create("integrity_journal_io",
5265 					     sizeof(struct journal_io), 0, 0, NULL);
5266 	if (!journal_io_cache) {
5267 		DMERR("can't allocate journal io cache");
5268 		return -ENOMEM;
5269 	}
5270 
5271 	r = dm_register_target(&integrity_target);
5272 	if (r < 0) {
5273 		kmem_cache_destroy(journal_io_cache);
5274 		return r;
5275 	}
5276 
5277 	return 0;
5278 }
5279 
dm_integrity_exit(void)5280 static void __exit dm_integrity_exit(void)
5281 {
5282 	dm_unregister_target(&integrity_target);
5283 	kmem_cache_destroy(journal_io_cache);
5284 }
5285 
5286 module_init(dm_integrity_init);
5287 module_exit(dm_integrity_exit);
5288 
5289 MODULE_AUTHOR("Milan Broz");
5290 MODULE_AUTHOR("Mikulas Patocka");
5291 MODULE_DESCRIPTION(DM_NAME " target for integrity tags extension");
5292 MODULE_LICENSE("GPL");
5293