xref: /linux/drivers/md/dm-writecache.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2018 Red Hat. All rights reserved.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include <linux/device-mapper.h>
9 #include <linux/module.h>
10 #include <linux/init.h>
11 #include <linux/vmalloc.h>
12 #include <linux/kthread.h>
13 #include <linux/dm-io.h>
14 #include <linux/dm-kcopyd.h>
15 #include <linux/dax.h>
16 #include <linux/libnvdimm.h>
17 #include <linux/delay.h>
18 #include "dm-io-tracker.h"
19 
20 #define DM_MSG_PREFIX "writecache"
21 
22 #define HIGH_WATERMARK			50
23 #define LOW_WATERMARK			45
24 #define MAX_WRITEBACK_JOBS		min(0x10000000 / PAGE_SIZE, totalram_pages() / 16)
25 #define ENDIO_LATENCY			16
26 #define WRITEBACK_LATENCY		64
27 #define AUTOCOMMIT_BLOCKS_SSD		65536
28 #define AUTOCOMMIT_BLOCKS_PMEM		64
29 #define AUTOCOMMIT_MSEC			1000
30 #define MAX_AGE_DIV			16
31 #define MAX_AGE_UNSPECIFIED		-1UL
32 #define PAUSE_WRITEBACK			(HZ * 3)
33 
34 #define BITMAP_GRANULARITY	65536
35 #if BITMAP_GRANULARITY < PAGE_SIZE
36 #undef BITMAP_GRANULARITY
37 #define BITMAP_GRANULARITY	PAGE_SIZE
38 #endif
39 
40 #if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_FS_DAX)
41 #define DM_WRITECACHE_HAS_PMEM
42 #endif
43 
44 #ifdef DM_WRITECACHE_HAS_PMEM
45 #define pmem_assign(dest, src)					\
46 do {								\
47 	typeof(dest) uniq = (src);				\
48 	memcpy_flushcache(&(dest), &uniq, sizeof(dest));	\
49 } while (0)
50 #else
51 #define pmem_assign(dest, src)	((dest) = (src))
52 #endif
53 
54 #if IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC) && defined(DM_WRITECACHE_HAS_PMEM)
55 #define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
56 #endif
57 
58 #define MEMORY_SUPERBLOCK_MAGIC		0x23489321
59 #define MEMORY_SUPERBLOCK_VERSION	1
60 
61 struct wc_memory_entry {
62 	__le64 original_sector;
63 	__le64 seq_count;
64 };
65 
66 struct wc_memory_superblock {
67 	union {
68 		struct {
69 			__le32 magic;
70 			__le32 version;
71 			__le32 block_size;
72 			__le32 pad;
73 			__le64 n_blocks;
74 			__le64 seq_count;
75 		};
76 		__le64 padding[8];
77 	};
78 	struct wc_memory_entry entries[];
79 };
80 
81 struct wc_entry {
82 	struct rb_node rb_node;
83 	struct list_head lru;
84 	u32 age; // jiffies
85 	unsigned short wc_list_contiguous;
86 	bool write_in_progress;
87 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
88 	uint64_t original_sector;
89 	uint64_t seq_count;
90 #endif
91 };
92 
93 #ifdef DM_WRITECACHE_HAS_PMEM
94 #define WC_MODE_PMEM(wc)			((wc)->pmem_mode)
95 #define WC_MODE_FUA(wc)				((wc)->writeback_fua)
96 #else
97 #define WC_MODE_PMEM(wc)			false
98 #define WC_MODE_FUA(wc)				false
99 #endif
100 #define WC_MODE_SORT_FREELIST(wc)		(!WC_MODE_PMEM(wc))
101 
102 struct dm_writecache {
103 	struct mutex lock;
104 	struct list_head lru;
105 	union {
106 		struct list_head freelist;
107 		struct {
108 			struct rb_root freetree;
109 			struct wc_entry *current_free;
110 		};
111 	};
112 	struct rb_root tree;
113 
114 	size_t freelist_size;
115 	size_t writeback_size;
116 	size_t freelist_high_watermark;
117 	size_t freelist_low_watermark;
118 	unsigned long max_age;
119 	unsigned long pause;
120 
121 	unsigned int uncommitted_blocks;
122 	unsigned int autocommit_blocks;
123 	unsigned int max_writeback_jobs;
124 
125 	int error;
126 
127 	unsigned long autocommit_jiffies;
128 	struct timer_list autocommit_timer;
129 	struct wait_queue_head freelist_wait;
130 
131 	struct timer_list max_age_timer;
132 
133 	atomic_t bio_in_progress[2];
134 	struct wait_queue_head bio_in_progress_wait[2];
135 
136 	struct dm_target *ti;
137 	struct dm_dev *dev;
138 	struct dm_dev *ssd_dev;
139 	sector_t start_sector;
140 	void *memory_map;
141 	uint64_t memory_map_size;
142 	size_t metadata_sectors;
143 	size_t n_blocks;
144 	uint64_t seq_count;
145 	sector_t data_device_sectors;
146 	void *block_start;
147 	struct wc_entry *entries;
148 	unsigned int block_size;
149 	unsigned char block_size_bits;
150 
151 	bool pmem_mode:1;
152 	bool writeback_fua:1;
153 
154 	bool overwrote_committed:1;
155 	bool memory_vmapped:1;
156 
157 	bool start_sector_set:1;
158 	bool high_wm_percent_set:1;
159 	bool low_wm_percent_set:1;
160 	bool max_writeback_jobs_set:1;
161 	bool autocommit_blocks_set:1;
162 	bool autocommit_time_set:1;
163 	bool max_age_set:1;
164 	bool writeback_fua_set:1;
165 	bool flush_on_suspend:1;
166 	bool cleaner:1;
167 	bool cleaner_set:1;
168 	bool metadata_only:1;
169 	bool pause_set:1;
170 
171 	unsigned int high_wm_percent_value;
172 	unsigned int low_wm_percent_value;
173 	unsigned int autocommit_time_value;
174 	unsigned int max_age_value;
175 	unsigned int pause_value;
176 
177 	unsigned int writeback_all;
178 	struct workqueue_struct *writeback_wq;
179 	struct work_struct writeback_work;
180 	struct work_struct flush_work;
181 
182 	struct dm_io_tracker iot;
183 
184 	struct dm_io_client *dm_io;
185 
186 	raw_spinlock_t endio_list_lock;
187 	struct list_head endio_list;
188 	struct task_struct *endio_thread;
189 
190 	struct task_struct *flush_thread;
191 	spinlock_t flush_list_lock;
192 	struct bio_list flush_list;
193 
194 	struct dm_kcopyd_client *dm_kcopyd;
195 	unsigned long *dirty_bitmap;
196 	unsigned int dirty_bitmap_size;
197 
198 	struct bio_set bio_set;
199 	mempool_t copy_pool;
200 
201 	struct {
202 		unsigned long long reads;
203 		unsigned long long read_hits;
204 		unsigned long long writes;
205 		unsigned long long write_hits_uncommitted;
206 		unsigned long long write_hits_committed;
207 		unsigned long long writes_around;
208 		unsigned long long writes_allocate;
209 		unsigned long long writes_blocked_on_freelist;
210 		unsigned long long flushes;
211 		unsigned long long discards;
212 	} stats;
213 };
214 
215 #define WB_LIST_INLINE		16
216 
217 struct writeback_struct {
218 	struct list_head endio_entry;
219 	struct dm_writecache *wc;
220 	struct wc_entry **wc_list;
221 	unsigned int wc_list_n;
222 	struct wc_entry *wc_list_inline[WB_LIST_INLINE];
223 	struct bio bio;
224 };
225 
226 struct copy_struct {
227 	struct list_head endio_entry;
228 	struct dm_writecache *wc;
229 	struct wc_entry *e;
230 	unsigned int n_entries;
231 	int error;
232 };
233 
234 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle,
235 					    "A percentage of time allocated for data copying");
236 
237 static void wc_lock(struct dm_writecache *wc)
238 {
239 	mutex_lock(&wc->lock);
240 }
241 
242 static void wc_unlock(struct dm_writecache *wc)
243 {
244 	mutex_unlock(&wc->lock);
245 }
246 
247 #ifdef DM_WRITECACHE_HAS_PMEM
248 static int persistent_memory_claim(struct dm_writecache *wc)
249 {
250 	int r;
251 	loff_t s;
252 	long p, da;
253 	unsigned long pfn;
254 	int id;
255 	struct page **pages;
256 	sector_t offset;
257 
258 	wc->memory_vmapped = false;
259 
260 	s = wc->memory_map_size;
261 	p = s >> PAGE_SHIFT;
262 	if (!p) {
263 		r = -EINVAL;
264 		goto err1;
265 	}
266 	if (p != s >> PAGE_SHIFT) {
267 		r = -EOVERFLOW;
268 		goto err1;
269 	}
270 
271 	offset = get_start_sect(wc->ssd_dev->bdev);
272 	if (offset & (PAGE_SIZE / 512 - 1)) {
273 		r = -EINVAL;
274 		goto err1;
275 	}
276 	offset >>= PAGE_SHIFT - 9;
277 
278 	id = dax_read_lock();
279 
280 	da = dax_direct_access(wc->ssd_dev->dax_dev, offset, p, DAX_ACCESS,
281 			&wc->memory_map, &pfn);
282 	if (da < 0) {
283 		wc->memory_map = NULL;
284 		r = da;
285 		goto err2;
286 	}
287 	if (!pfn_valid(pfn)) {
288 		wc->memory_map = NULL;
289 		r = -EOPNOTSUPP;
290 		goto err2;
291 	}
292 	if (da != p) {
293 		long i;
294 
295 		wc->memory_map = NULL;
296 		pages = vmalloc_array(p, sizeof(struct page *));
297 		if (!pages) {
298 			r = -ENOMEM;
299 			goto err2;
300 		}
301 		i = 0;
302 		do {
303 			long daa;
304 
305 			daa = dax_direct_access(wc->ssd_dev->dax_dev, offset + i,
306 					p - i, DAX_ACCESS, NULL, &pfn);
307 			if (daa <= 0) {
308 				r = daa ? daa : -EINVAL;
309 				goto err3;
310 			}
311 			if (!pfn_valid(pfn)) {
312 				r = -EOPNOTSUPP;
313 				goto err3;
314 			}
315 			while (daa-- && i < p) {
316 				pages[i++] = pfn_to_page(pfn);
317 				pfn++;
318 				if (!(i & 15))
319 					cond_resched();
320 			}
321 		} while (i < p);
322 		wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL);
323 		if (!wc->memory_map) {
324 			r = -ENOMEM;
325 			goto err3;
326 		}
327 		vfree(pages);
328 		wc->memory_vmapped = true;
329 	}
330 
331 	dax_read_unlock(id);
332 
333 	wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT;
334 	wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT;
335 
336 	return 0;
337 err3:
338 	vfree(pages);
339 err2:
340 	dax_read_unlock(id);
341 err1:
342 	return r;
343 }
344 #else
345 static int persistent_memory_claim(struct dm_writecache *wc)
346 {
347 	return -EOPNOTSUPP;
348 }
349 #endif
350 
351 static void persistent_memory_release(struct dm_writecache *wc)
352 {
353 	if (wc->memory_vmapped)
354 		vunmap(wc->memory_map - ((size_t)wc->start_sector << SECTOR_SHIFT));
355 }
356 
357 static struct page *persistent_memory_page(void *addr)
358 {
359 	if (is_vmalloc_addr(addr))
360 		return vmalloc_to_page(addr);
361 	else
362 		return virt_to_page(addr);
363 }
364 
365 static unsigned int persistent_memory_page_offset(void *addr)
366 {
367 	return (unsigned long)addr & (PAGE_SIZE - 1);
368 }
369 
370 static void persistent_memory_flush_cache(void *ptr, size_t size)
371 {
372 	if (is_vmalloc_addr(ptr))
373 		flush_kernel_vmap_range(ptr, size);
374 }
375 
376 static void persistent_memory_invalidate_cache(void *ptr, size_t size)
377 {
378 	if (is_vmalloc_addr(ptr))
379 		invalidate_kernel_vmap_range(ptr, size);
380 }
381 
382 static struct wc_memory_superblock *sb(struct dm_writecache *wc)
383 {
384 	return wc->memory_map;
385 }
386 
387 static inline unsigned long wc_entry_index(const struct wc_entry *base, const struct wc_entry *entry)
388 {
389 	return entry - base;
390 }
391 
392 static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e)
393 {
394 	const unsigned long index = wc_entry_index(wc->entries, e);
395 	return &sb(wc)->entries[index];
396 }
397 
398 static void *memory_data(struct dm_writecache *wc, struct wc_entry *e)
399 {
400 	const unsigned long index = wc_entry_index(wc->entries, e);
401 	return (char *)wc->block_start + (index << wc->block_size_bits);
402 }
403 
404 static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e)
405 {
406 	const unsigned long index = wc_entry_index(wc->entries, e);
407 	return wc->start_sector + wc->metadata_sectors +
408 		((sector_t)index << (wc->block_size_bits - SECTOR_SHIFT));
409 }
410 
411 static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e)
412 {
413 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
414 	return e->original_sector;
415 #else
416 	return le64_to_cpu(memory_entry(wc, e)->original_sector);
417 #endif
418 }
419 
420 static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e)
421 {
422 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
423 	return e->seq_count;
424 #else
425 	return le64_to_cpu(memory_entry(wc, e)->seq_count);
426 #endif
427 }
428 
429 static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e)
430 {
431 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
432 	e->seq_count = -1;
433 #endif
434 	pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1));
435 }
436 
437 static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e,
438 					    uint64_t original_sector, uint64_t seq_count)
439 {
440 	struct wc_memory_entry me;
441 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
442 	e->original_sector = original_sector;
443 	e->seq_count = seq_count;
444 #endif
445 	me.original_sector = cpu_to_le64(original_sector);
446 	me.seq_count = cpu_to_le64(seq_count);
447 	pmem_assign(*memory_entry(wc, e), me);
448 }
449 
450 #define writecache_error(wc, err, msg, arg...)				\
451 do {									\
452 	if (!cmpxchg(&(wc)->error, 0, err))				\
453 		DMERR(msg, ##arg);					\
454 	wake_up(&(wc)->freelist_wait);					\
455 } while (0)
456 
457 #define writecache_has_error(wc)	(unlikely(READ_ONCE((wc)->error)))
458 
459 static void writecache_flush_all_metadata(struct dm_writecache *wc)
460 {
461 	if (!WC_MODE_PMEM(wc))
462 		memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size);
463 }
464 
465 static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size)
466 {
467 	if (!WC_MODE_PMEM(wc))
468 		__set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY,
469 			  wc->dirty_bitmap);
470 }
471 
472 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev);
473 
474 struct io_notify {
475 	struct dm_writecache *wc;
476 	struct completion c;
477 	atomic_t count;
478 };
479 
480 static void writecache_notify_io(unsigned long error, unsigned long unsup, void *context)
481 {
482 	struct io_notify *endio = context;
483 
484 	if (unlikely(error != 0))
485 		writecache_error(endio->wc, -EIO, "error writing metadata");
486 	else if (unlikely(unsup != 0))
487 		writecache_error(endio->wc, -EOPNOTSUPP, "error writing metadata");
488 	BUG_ON(atomic_read(&endio->count) <= 0);
489 	if (atomic_dec_and_test(&endio->count))
490 		complete(&endio->c);
491 }
492 
493 static void writecache_wait_for_ios(struct dm_writecache *wc, int direction)
494 {
495 	wait_event(wc->bio_in_progress_wait[direction],
496 		   !atomic_read(&wc->bio_in_progress[direction]));
497 }
498 
499 static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
500 {
501 	struct dm_io_region region;
502 	struct dm_io_request req;
503 	struct io_notify endio = {
504 		wc,
505 		COMPLETION_INITIALIZER_ONSTACK(endio.c),
506 		ATOMIC_INIT(1),
507 	};
508 	unsigned int bitmap_bits = wc->dirty_bitmap_size * 8;
509 	unsigned int i = 0;
510 
511 	while (1) {
512 		unsigned int j;
513 
514 		i = find_next_bit(wc->dirty_bitmap, bitmap_bits, i);
515 		if (unlikely(i == bitmap_bits))
516 			break;
517 		j = find_next_zero_bit(wc->dirty_bitmap, bitmap_bits, i);
518 
519 		region.bdev = wc->ssd_dev->bdev;
520 		region.sector = (sector_t)i * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
521 		region.count = (sector_t)(j - i) * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
522 
523 		if (unlikely(region.sector >= wc->metadata_sectors))
524 			break;
525 		if (unlikely(region.sector + region.count > wc->metadata_sectors))
526 			region.count = wc->metadata_sectors - region.sector;
527 
528 		region.sector += wc->start_sector;
529 		atomic_inc(&endio.count);
530 		req.bi_opf = REQ_OP_WRITE | REQ_SYNC;
531 		req.mem.type = DM_IO_VMA;
532 		req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY;
533 		req.client = wc->dm_io;
534 		req.notify.fn = writecache_notify_io;
535 		req.notify.context = &endio;
536 
537 		/* writing via async dm-io (implied by notify.fn above) won't return an error */
538 		(void) dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
539 		i = j;
540 	}
541 
542 	writecache_notify_io(0, 0, &endio);
543 	wait_for_completion_io(&endio.c);
544 
545 	if (wait_for_ios)
546 		writecache_wait_for_ios(wc, WRITE);
547 
548 	writecache_disk_flush(wc, wc->ssd_dev);
549 
550 	memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size);
551 }
552 
553 static void ssd_commit_superblock(struct dm_writecache *wc)
554 {
555 	int r;
556 	struct dm_io_region region;
557 	struct dm_io_request req;
558 
559 	region.bdev = wc->ssd_dev->bdev;
560 	region.sector = 0;
561 	region.count = max(4096U, wc->block_size) >> SECTOR_SHIFT;
562 
563 	if (unlikely(region.sector + region.count > wc->metadata_sectors))
564 		region.count = wc->metadata_sectors - region.sector;
565 
566 	region.sector += wc->start_sector;
567 
568 	req.bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_FUA;
569 	req.mem.type = DM_IO_VMA;
570 	req.mem.ptr.vma = (char *)wc->memory_map;
571 	req.client = wc->dm_io;
572 	req.notify.fn = NULL;
573 	req.notify.context = NULL;
574 
575 	r = dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
576 	if (unlikely(r))
577 		writecache_error(wc, r, "error writing superblock");
578 }
579 
580 static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
581 {
582 	if (WC_MODE_PMEM(wc))
583 		pmem_wmb();
584 	else
585 		ssd_commit_flushed(wc, wait_for_ios);
586 }
587 
588 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
589 {
590 	int r;
591 	struct dm_io_region region;
592 	struct dm_io_request req;
593 
594 	region.bdev = dev->bdev;
595 	region.sector = 0;
596 	region.count = 0;
597 	req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
598 	req.mem.type = DM_IO_KMEM;
599 	req.mem.ptr.addr = NULL;
600 	req.client = wc->dm_io;
601 	req.notify.fn = NULL;
602 
603 	r = dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
604 	if (unlikely(r))
605 		writecache_error(wc, r, "error flushing metadata: %d", r);
606 }
607 
608 #define WFE_RETURN_FOLLOWING	1
609 #define WFE_LOWEST_SEQ		2
610 
611 static struct wc_entry *writecache_find_entry(struct dm_writecache *wc,
612 					      uint64_t block, int flags)
613 {
614 	struct wc_entry *e;
615 	struct rb_node *node = wc->tree.rb_node;
616 
617 	if (unlikely(!node))
618 		return NULL;
619 
620 	while (1) {
621 		e = container_of(node, struct wc_entry, rb_node);
622 		if (read_original_sector(wc, e) == block)
623 			break;
624 
625 		node = (read_original_sector(wc, e) >= block ?
626 			e->rb_node.rb_left : e->rb_node.rb_right);
627 		if (unlikely(!node)) {
628 			if (!(flags & WFE_RETURN_FOLLOWING))
629 				return NULL;
630 			if (read_original_sector(wc, e) >= block)
631 				return e;
632 
633 			node = rb_next(&e->rb_node);
634 			if (unlikely(!node))
635 				return NULL;
636 
637 			e = container_of(node, struct wc_entry, rb_node);
638 			return e;
639 		}
640 	}
641 
642 	while (1) {
643 		struct wc_entry *e2;
644 
645 		if (flags & WFE_LOWEST_SEQ)
646 			node = rb_prev(&e->rb_node);
647 		else
648 			node = rb_next(&e->rb_node);
649 		if (unlikely(!node))
650 			return e;
651 		e2 = container_of(node, struct wc_entry, rb_node);
652 		if (read_original_sector(wc, e2) != block)
653 			return e;
654 		e = e2;
655 	}
656 }
657 
658 static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *ins)
659 {
660 	struct wc_entry *e;
661 	struct rb_node **node = &wc->tree.rb_node, *parent = NULL;
662 
663 	while (*node) {
664 		e = container_of(*node, struct wc_entry, rb_node);
665 		parent = &e->rb_node;
666 		if (read_original_sector(wc, e) > read_original_sector(wc, ins))
667 			node = &parent->rb_left;
668 		else
669 			node = &parent->rb_right;
670 	}
671 	rb_link_node(&ins->rb_node, parent, node);
672 	rb_insert_color(&ins->rb_node, &wc->tree);
673 	list_add(&ins->lru, &wc->lru);
674 	ins->age = (u32)jiffies;
675 }
676 
677 static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
678 {
679 	list_del(&e->lru);
680 	rb_erase(&e->rb_node, &wc->tree);
681 }
682 
683 static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e)
684 {
685 	if (WC_MODE_SORT_FREELIST(wc)) {
686 		struct rb_node **node = &wc->freetree.rb_node, *parent = NULL;
687 
688 		if (unlikely(!*node))
689 			wc->current_free = e;
690 		while (*node) {
691 			parent = *node;
692 			if (&e->rb_node < *node)
693 				node = &parent->rb_left;
694 			else
695 				node = &parent->rb_right;
696 		}
697 		rb_link_node(&e->rb_node, parent, node);
698 		rb_insert_color(&e->rb_node, &wc->freetree);
699 	} else {
700 		list_add_tail(&e->lru, &wc->freelist);
701 	}
702 	wc->freelist_size++;
703 }
704 
705 static inline void writecache_verify_watermark(struct dm_writecache *wc)
706 {
707 	if (unlikely(wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark))
708 		queue_work(wc->writeback_wq, &wc->writeback_work);
709 }
710 
711 static void writecache_max_age_timer(struct timer_list *t)
712 {
713 	struct dm_writecache *wc = timer_container_of(wc, t, max_age_timer);
714 
715 	if (!dm_suspended(wc->ti) && !writecache_has_error(wc)) {
716 		queue_work(wc->writeback_wq, &wc->writeback_work);
717 		mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
718 	}
719 }
720 
721 static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector)
722 {
723 	struct wc_entry *e;
724 
725 	if (WC_MODE_SORT_FREELIST(wc)) {
726 		struct rb_node *next;
727 
728 		if (unlikely(!wc->current_free))
729 			return NULL;
730 		e = wc->current_free;
731 		if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
732 			return NULL;
733 		next = rb_next(&e->rb_node);
734 		rb_erase(&e->rb_node, &wc->freetree);
735 		if (unlikely(!next))
736 			next = rb_first(&wc->freetree);
737 		wc->current_free = next ? container_of(next, struct wc_entry, rb_node) : NULL;
738 	} else {
739 		if (unlikely(list_empty(&wc->freelist)))
740 			return NULL;
741 		e = container_of(wc->freelist.next, struct wc_entry, lru);
742 		if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
743 			return NULL;
744 		list_del(&e->lru);
745 	}
746 	wc->freelist_size--;
747 
748 	writecache_verify_watermark(wc);
749 
750 	return e;
751 }
752 
753 static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e)
754 {
755 	writecache_unlink(wc, e);
756 	writecache_add_to_freelist(wc, e);
757 	clear_seq_count(wc, e);
758 	writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
759 	if (unlikely(waitqueue_active(&wc->freelist_wait)))
760 		wake_up(&wc->freelist_wait);
761 }
762 
763 static void writecache_wait_on_freelist(struct dm_writecache *wc)
764 {
765 	DEFINE_WAIT(wait);
766 
767 	prepare_to_wait(&wc->freelist_wait, &wait, TASK_UNINTERRUPTIBLE);
768 	wc_unlock(wc);
769 	io_schedule();
770 	finish_wait(&wc->freelist_wait, &wait);
771 	wc_lock(wc);
772 }
773 
774 static void writecache_poison_lists(struct dm_writecache *wc)
775 {
776 	/*
777 	 * Catch incorrect access to these values while the device is suspended.
778 	 */
779 	memset(&wc->tree, -1, sizeof(wc->tree));
780 	wc->lru.next = LIST_POISON1;
781 	wc->lru.prev = LIST_POISON2;
782 	wc->freelist.next = LIST_POISON1;
783 	wc->freelist.prev = LIST_POISON2;
784 }
785 
786 static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e)
787 {
788 	writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
789 	if (WC_MODE_PMEM(wc))
790 		writecache_flush_region(wc, memory_data(wc, e), wc->block_size);
791 }
792 
793 static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e)
794 {
795 	return read_seq_count(wc, e) < wc->seq_count;
796 }
797 
798 static void writecache_flush(struct dm_writecache *wc)
799 {
800 	struct wc_entry *e, *e2;
801 	bool need_flush_after_free;
802 
803 	wc->uncommitted_blocks = 0;
804 	timer_delete(&wc->autocommit_timer);
805 
806 	if (list_empty(&wc->lru))
807 		return;
808 
809 	e = container_of(wc->lru.next, struct wc_entry, lru);
810 	if (writecache_entry_is_committed(wc, e)) {
811 		if (wc->overwrote_committed) {
812 			writecache_wait_for_ios(wc, WRITE);
813 			writecache_disk_flush(wc, wc->ssd_dev);
814 			wc->overwrote_committed = false;
815 		}
816 		return;
817 	}
818 	while (1) {
819 		writecache_flush_entry(wc, e);
820 		if (unlikely(e->lru.next == &wc->lru))
821 			break;
822 		e2 = container_of(e->lru.next, struct wc_entry, lru);
823 		if (writecache_entry_is_committed(wc, e2))
824 			break;
825 		e = e2;
826 		cond_resched();
827 	}
828 	writecache_commit_flushed(wc, true);
829 
830 	wc->seq_count++;
831 	pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count));
832 	if (WC_MODE_PMEM(wc))
833 		writecache_commit_flushed(wc, false);
834 	else
835 		ssd_commit_superblock(wc);
836 
837 	wc->overwrote_committed = false;
838 
839 	need_flush_after_free = false;
840 	while (1) {
841 		/* Free another committed entry with lower seq-count */
842 		struct rb_node *rb_node = rb_prev(&e->rb_node);
843 
844 		if (rb_node) {
845 			e2 = container_of(rb_node, struct wc_entry, rb_node);
846 			if (read_original_sector(wc, e2) == read_original_sector(wc, e) &&
847 			    likely(!e2->write_in_progress)) {
848 				writecache_free_entry(wc, e2);
849 				need_flush_after_free = true;
850 			}
851 		}
852 		if (unlikely(e->lru.prev == &wc->lru))
853 			break;
854 		e = container_of(e->lru.prev, struct wc_entry, lru);
855 		cond_resched();
856 	}
857 
858 	if (need_flush_after_free)
859 		writecache_commit_flushed(wc, false);
860 }
861 
862 static void writecache_flush_work(struct work_struct *work)
863 {
864 	struct dm_writecache *wc = container_of(work, struct dm_writecache, flush_work);
865 
866 	wc_lock(wc);
867 	writecache_flush(wc);
868 	wc_unlock(wc);
869 }
870 
871 static void writecache_autocommit_timer(struct timer_list *t)
872 {
873 	struct dm_writecache *wc = timer_container_of(wc, t, autocommit_timer);
874 
875 	if (!writecache_has_error(wc))
876 		queue_work(wc->writeback_wq, &wc->flush_work);
877 }
878 
879 static void writecache_schedule_autocommit(struct dm_writecache *wc)
880 {
881 	if (!timer_pending(&wc->autocommit_timer))
882 		mod_timer(&wc->autocommit_timer, jiffies + wc->autocommit_jiffies);
883 }
884 
885 static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_t end)
886 {
887 	struct wc_entry *e;
888 	bool discarded_something = false;
889 
890 	e = writecache_find_entry(wc, start, WFE_RETURN_FOLLOWING | WFE_LOWEST_SEQ);
891 	if (unlikely(!e))
892 		return;
893 
894 	while (read_original_sector(wc, e) < end) {
895 		struct rb_node *node = rb_next(&e->rb_node);
896 
897 		if (likely(!e->write_in_progress)) {
898 			if (!discarded_something) {
899 				if (!WC_MODE_PMEM(wc)) {
900 					writecache_wait_for_ios(wc, READ);
901 					writecache_wait_for_ios(wc, WRITE);
902 				}
903 				discarded_something = true;
904 			}
905 			writecache_free_entry(wc, e);
906 		}
907 
908 		if (unlikely(!node))
909 			break;
910 
911 		e = container_of(node, struct wc_entry, rb_node);
912 	}
913 
914 	if (discarded_something)
915 		writecache_commit_flushed(wc, false);
916 }
917 
918 static bool writecache_wait_for_writeback(struct dm_writecache *wc)
919 {
920 	if (wc->writeback_size) {
921 		writecache_wait_on_freelist(wc);
922 		return true;
923 	}
924 	return false;
925 }
926 
927 static void writecache_suspend(struct dm_target *ti)
928 {
929 	struct dm_writecache *wc = ti->private;
930 	bool flush_on_suspend;
931 
932 	timer_delete_sync(&wc->autocommit_timer);
933 	timer_delete_sync(&wc->max_age_timer);
934 
935 	wc_lock(wc);
936 	writecache_flush(wc);
937 	flush_on_suspend = wc->flush_on_suspend;
938 	if (flush_on_suspend) {
939 		wc->flush_on_suspend = false;
940 		wc->writeback_all++;
941 		queue_work(wc->writeback_wq, &wc->writeback_work);
942 	}
943 	wc_unlock(wc);
944 
945 	drain_workqueue(wc->writeback_wq);
946 
947 	wc_lock(wc);
948 	if (flush_on_suspend)
949 		wc->writeback_all--;
950 	while (writecache_wait_for_writeback(wc))
951 		;
952 
953 	if (WC_MODE_PMEM(wc))
954 		persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
955 
956 	writecache_poison_lists(wc);
957 
958 	wc_unlock(wc);
959 }
960 
961 static int writecache_alloc_entries(struct dm_writecache *wc)
962 {
963 	size_t b;
964 
965 	if (wc->entries)
966 		return 0;
967 	wc->entries = vmalloc_array(wc->n_blocks, sizeof(struct wc_entry));
968 	if (!wc->entries)
969 		return -ENOMEM;
970 	for (b = 0; b < wc->n_blocks; b++) {
971 		struct wc_entry *e = &wc->entries[b];
972 
973 		e->write_in_progress = false;
974 		cond_resched();
975 	}
976 
977 	return 0;
978 }
979 
980 static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors)
981 {
982 	struct dm_io_region region;
983 	struct dm_io_request req;
984 
985 	region.bdev = wc->ssd_dev->bdev;
986 	region.sector = wc->start_sector;
987 	region.count = n_sectors;
988 	req.bi_opf = REQ_OP_READ | REQ_SYNC;
989 	req.mem.type = DM_IO_VMA;
990 	req.mem.ptr.vma = (char *)wc->memory_map;
991 	req.client = wc->dm_io;
992 	req.notify.fn = NULL;
993 
994 	return dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
995 }
996 
997 static void writecache_resume(struct dm_target *ti)
998 {
999 	struct dm_writecache *wc = ti->private;
1000 	size_t b;
1001 	bool need_flush = false;
1002 	__le64 sb_seq_count;
1003 	int r;
1004 
1005 	wc_lock(wc);
1006 
1007 	wc->data_device_sectors = bdev_nr_sectors(wc->dev->bdev);
1008 
1009 	if (WC_MODE_PMEM(wc)) {
1010 		persistent_memory_invalidate_cache(wc->memory_map, wc->memory_map_size);
1011 	} else {
1012 		r = writecache_read_metadata(wc, wc->metadata_sectors);
1013 		if (r) {
1014 			size_t sb_entries_offset;
1015 
1016 			writecache_error(wc, r, "unable to read metadata: %d", r);
1017 			sb_entries_offset = offsetof(struct wc_memory_superblock, entries);
1018 			memset((char *)wc->memory_map + sb_entries_offset, -1,
1019 			       (wc->metadata_sectors << SECTOR_SHIFT) - sb_entries_offset);
1020 		}
1021 	}
1022 
1023 	wc->tree = RB_ROOT;
1024 	INIT_LIST_HEAD(&wc->lru);
1025 	if (WC_MODE_SORT_FREELIST(wc)) {
1026 		wc->freetree = RB_ROOT;
1027 		wc->current_free = NULL;
1028 	} else {
1029 		INIT_LIST_HEAD(&wc->freelist);
1030 	}
1031 	wc->freelist_size = 0;
1032 
1033 	r = copy_mc_to_kernel(&sb_seq_count, &sb(wc)->seq_count,
1034 			      sizeof(uint64_t));
1035 	if (r) {
1036 		writecache_error(wc, r, "hardware memory error when reading superblock: %d", r);
1037 		sb_seq_count = cpu_to_le64(0);
1038 	}
1039 	wc->seq_count = le64_to_cpu(sb_seq_count);
1040 
1041 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
1042 	for (b = 0; b < wc->n_blocks; b++) {
1043 		struct wc_entry *e = &wc->entries[b];
1044 		struct wc_memory_entry wme;
1045 
1046 		if (writecache_has_error(wc)) {
1047 			e->original_sector = -1;
1048 			e->seq_count = -1;
1049 			continue;
1050 		}
1051 		r = copy_mc_to_kernel(&wme, memory_entry(wc, e),
1052 				      sizeof(struct wc_memory_entry));
1053 		if (r) {
1054 			writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d",
1055 					 (unsigned long)b, r);
1056 			e->original_sector = -1;
1057 			e->seq_count = -1;
1058 		} else {
1059 			e->original_sector = le64_to_cpu(wme.original_sector);
1060 			e->seq_count = le64_to_cpu(wme.seq_count);
1061 		}
1062 		cond_resched();
1063 	}
1064 #endif
1065 	for (b = 0; b < wc->n_blocks; b++) {
1066 		struct wc_entry *e = &wc->entries[b];
1067 
1068 		if (!writecache_entry_is_committed(wc, e)) {
1069 			if (read_seq_count(wc, e) != -1) {
1070 erase_this:
1071 				clear_seq_count(wc, e);
1072 				need_flush = true;
1073 			}
1074 			writecache_add_to_freelist(wc, e);
1075 		} else {
1076 			struct wc_entry *old;
1077 
1078 			old = writecache_find_entry(wc, read_original_sector(wc, e), 0);
1079 			if (!old) {
1080 				writecache_insert_entry(wc, e);
1081 			} else {
1082 				if (read_seq_count(wc, old) == read_seq_count(wc, e)) {
1083 					writecache_error(wc, -EINVAL,
1084 						 "two identical entries, position %llu, sector %llu, sequence %llu",
1085 						 (unsigned long long)b, (unsigned long long)read_original_sector(wc, e),
1086 						 (unsigned long long)read_seq_count(wc, e));
1087 				}
1088 				if (read_seq_count(wc, old) > read_seq_count(wc, e)) {
1089 					goto erase_this;
1090 				} else {
1091 					writecache_free_entry(wc, old);
1092 					writecache_insert_entry(wc, e);
1093 					need_flush = true;
1094 				}
1095 			}
1096 		}
1097 		cond_resched();
1098 	}
1099 
1100 	if (need_flush) {
1101 		writecache_flush_all_metadata(wc);
1102 		writecache_commit_flushed(wc, false);
1103 	}
1104 
1105 	writecache_verify_watermark(wc);
1106 
1107 	if (wc->max_age != MAX_AGE_UNSPECIFIED)
1108 		mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
1109 
1110 	wc_unlock(wc);
1111 }
1112 
1113 static int process_flush_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
1114 {
1115 	if (argc != 1)
1116 		return -EINVAL;
1117 
1118 	wc_lock(wc);
1119 	if (dm_suspended(wc->ti)) {
1120 		wc_unlock(wc);
1121 		return -EBUSY;
1122 	}
1123 	if (writecache_has_error(wc)) {
1124 		wc_unlock(wc);
1125 		return -EIO;
1126 	}
1127 
1128 	writecache_flush(wc);
1129 	wc->writeback_all++;
1130 	queue_work(wc->writeback_wq, &wc->writeback_work);
1131 	wc_unlock(wc);
1132 
1133 	flush_workqueue(wc->writeback_wq);
1134 
1135 	wc_lock(wc);
1136 	wc->writeback_all--;
1137 	if (writecache_has_error(wc)) {
1138 		wc_unlock(wc);
1139 		return -EIO;
1140 	}
1141 	wc_unlock(wc);
1142 
1143 	return 0;
1144 }
1145 
1146 static int process_flush_on_suspend_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
1147 {
1148 	if (argc != 1)
1149 		return -EINVAL;
1150 
1151 	wc_lock(wc);
1152 	wc->flush_on_suspend = true;
1153 	wc_unlock(wc);
1154 
1155 	return 0;
1156 }
1157 
1158 static void activate_cleaner(struct dm_writecache *wc)
1159 {
1160 	wc->flush_on_suspend = true;
1161 	wc->cleaner = true;
1162 	wc->freelist_high_watermark = wc->n_blocks;
1163 	wc->freelist_low_watermark = wc->n_blocks;
1164 }
1165 
1166 static int process_cleaner_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
1167 {
1168 	if (argc != 1)
1169 		return -EINVAL;
1170 
1171 	wc_lock(wc);
1172 	activate_cleaner(wc);
1173 	if (!dm_suspended(wc->ti))
1174 		writecache_verify_watermark(wc);
1175 	wc_unlock(wc);
1176 
1177 	return 0;
1178 }
1179 
1180 static int process_clear_stats_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
1181 {
1182 	if (argc != 1)
1183 		return -EINVAL;
1184 
1185 	wc_lock(wc);
1186 	memset(&wc->stats, 0, sizeof(wc->stats));
1187 	wc_unlock(wc);
1188 
1189 	return 0;
1190 }
1191 
1192 static int writecache_message(struct dm_target *ti, unsigned int argc, char **argv,
1193 			      char *result, unsigned int maxlen)
1194 {
1195 	int r = -EINVAL;
1196 	struct dm_writecache *wc = ti->private;
1197 
1198 	if (!strcasecmp(argv[0], "flush"))
1199 		r = process_flush_mesg(argc, argv, wc);
1200 	else if (!strcasecmp(argv[0], "flush_on_suspend"))
1201 		r = process_flush_on_suspend_mesg(argc, argv, wc);
1202 	else if (!strcasecmp(argv[0], "cleaner"))
1203 		r = process_cleaner_mesg(argc, argv, wc);
1204 	else if (!strcasecmp(argv[0], "clear_stats"))
1205 		r = process_clear_stats_mesg(argc, argv, wc);
1206 	else
1207 		DMERR("unrecognised message received: %s", argv[0]);
1208 
1209 	return r;
1210 }
1211 
1212 static void memcpy_flushcache_optimized(void *dest, void *source, size_t size)
1213 {
1214 	/*
1215 	 * clflushopt performs better with block size 1024, 2048, 4096
1216 	 * non-temporal stores perform better with block size 512
1217 	 *
1218 	 * block size   512             1024            2048            4096
1219 	 * movnti       496 MB/s        642 MB/s        725 MB/s        744 MB/s
1220 	 * clflushopt   373 MB/s        688 MB/s        1.1 GB/s        1.2 GB/s
1221 	 *
1222 	 * We see that movnti performs better for 512-byte blocks, and
1223 	 * clflushopt performs better for 1024-byte and larger blocks. So, we
1224 	 * prefer clflushopt for sizes >= 768.
1225 	 *
1226 	 * NOTE: this happens to be the case now (with dm-writecache's single
1227 	 * threaded model) but re-evaluate this once memcpy_flushcache() is
1228 	 * enabled to use movdir64b which might invalidate this performance
1229 	 * advantage seen with cache-allocating-writes plus flushing.
1230 	 */
1231 #ifdef CONFIG_X86
1232 	if (cpu_feature_enabled(X86_FEATURE_CLFLUSHOPT) &&
1233 	    likely(boot_cpu_data.x86_clflush_size == 64) &&
1234 	    likely(size >= 768)) {
1235 		do {
1236 			memcpy((void *)dest, (void *)source, 64);
1237 			clflushopt((void *)dest);
1238 			dest += 64;
1239 			source += 64;
1240 			size -= 64;
1241 		} while (size >= 64);
1242 		return;
1243 	}
1244 #endif
1245 	memcpy_flushcache(dest, source, size);
1246 }
1247 
1248 static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
1249 {
1250 	void *buf;
1251 	unsigned int size;
1252 	int rw = bio_data_dir(bio);
1253 	unsigned int remaining_size = wc->block_size;
1254 
1255 	do {
1256 		struct bio_vec bv = bio_iter_iovec(bio, bio->bi_iter);
1257 
1258 		buf = bvec_kmap_local(&bv);
1259 		size = bv.bv_len;
1260 		if (unlikely(size > remaining_size))
1261 			size = remaining_size;
1262 
1263 		if (rw == READ) {
1264 			int r;
1265 
1266 			r = copy_mc_to_kernel(buf, data, size);
1267 			flush_dcache_page(bio_page(bio));
1268 			if (unlikely(r)) {
1269 				writecache_error(wc, r, "hardware memory error when reading data: %d", r);
1270 				bio->bi_status = BLK_STS_IOERR;
1271 			}
1272 		} else {
1273 			flush_dcache_page(bio_page(bio));
1274 			memcpy_flushcache_optimized(data, buf, size);
1275 		}
1276 
1277 		kunmap_local(buf);
1278 
1279 		data = (char *)data + size;
1280 		remaining_size -= size;
1281 		bio_advance(bio, size);
1282 	} while (unlikely(remaining_size));
1283 }
1284 
1285 static int writecache_flush_thread(void *data)
1286 {
1287 	struct dm_writecache *wc = data;
1288 
1289 	while (1) {
1290 		struct bio *bio;
1291 
1292 		wc_lock(wc);
1293 		spin_lock_irq(&wc->flush_list_lock);
1294 		bio = bio_list_pop(&wc->flush_list);
1295 		if (!bio) {
1296 			set_current_state(TASK_INTERRUPTIBLE);
1297 			spin_unlock_irq(&wc->flush_list_lock);
1298 			wc_unlock(wc);
1299 
1300 			if (unlikely(kthread_should_stop())) {
1301 				set_current_state(TASK_RUNNING);
1302 				break;
1303 			}
1304 
1305 			schedule();
1306 			continue;
1307 		}
1308 		spin_unlock_irq(&wc->flush_list_lock);
1309 
1310 		if (bio_op(bio) == REQ_OP_DISCARD) {
1311 			writecache_discard(wc, bio->bi_iter.bi_sector,
1312 					   bio_end_sector(bio));
1313 			wc_unlock(wc);
1314 			bio_set_dev(bio, wc->dev->bdev);
1315 			submit_bio_noacct(bio);
1316 		} else {
1317 			writecache_flush(wc);
1318 			wc_unlock(wc);
1319 			if (writecache_has_error(wc))
1320 				bio->bi_status = BLK_STS_IOERR;
1321 			bio_endio(bio);
1322 		}
1323 	}
1324 
1325 	return 0;
1326 }
1327 
1328 static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio)
1329 {
1330 	unsigned long flags;
1331 	spin_lock_irqsave(&wc->flush_list_lock, flags);
1332 	if (bio_list_empty(&wc->flush_list))
1333 		wake_up_process(wc->flush_thread);
1334 	bio_list_add(&wc->flush_list, bio);
1335 	spin_unlock_irqrestore(&wc->flush_list_lock, flags);
1336 }
1337 
1338 enum wc_map_op {
1339 	WC_MAP_SUBMIT,
1340 	WC_MAP_REMAP,
1341 	WC_MAP_REMAP_ORIGIN,
1342 	WC_MAP_RETURN,
1343 	WC_MAP_ERROR,
1344 };
1345 
1346 static void writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio,
1347 					struct wc_entry *e)
1348 {
1349 	if (e) {
1350 		sector_t next_boundary =
1351 			read_original_sector(wc, e) - bio->bi_iter.bi_sector;
1352 		if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT)
1353 			dm_accept_partial_bio(bio, next_boundary);
1354 	}
1355 }
1356 
1357 static enum wc_map_op writecache_map_read(struct dm_writecache *wc, struct bio *bio)
1358 {
1359 	enum wc_map_op map_op;
1360 	struct wc_entry *e;
1361 
1362 read_next_block:
1363 	wc->stats.reads++;
1364 	e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
1365 	if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) {
1366 		wc->stats.read_hits++;
1367 		if (WC_MODE_PMEM(wc)) {
1368 			bio_copy_block(wc, bio, memory_data(wc, e));
1369 			if (bio->bi_iter.bi_size)
1370 				goto read_next_block;
1371 			map_op = WC_MAP_SUBMIT;
1372 		} else {
1373 			dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
1374 			bio_set_dev(bio, wc->ssd_dev->bdev);
1375 			bio->bi_iter.bi_sector = cache_sector(wc, e);
1376 			if (!writecache_entry_is_committed(wc, e))
1377 				writecache_wait_for_ios(wc, WRITE);
1378 			map_op = WC_MAP_REMAP;
1379 		}
1380 	} else {
1381 		writecache_map_remap_origin(wc, bio, e);
1382 		wc->stats.reads += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
1383 		map_op = WC_MAP_REMAP_ORIGIN;
1384 	}
1385 
1386 	return map_op;
1387 }
1388 
1389 static void writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio,
1390 				    struct wc_entry *e, bool search_used)
1391 {
1392 	unsigned int bio_size = wc->block_size;
1393 	sector_t start_cache_sec = cache_sector(wc, e);
1394 	sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT);
1395 
1396 	while (bio_size < bio->bi_iter.bi_size) {
1397 		if (!search_used) {
1398 			struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec);
1399 
1400 			if (!f)
1401 				break;
1402 			write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector +
1403 							(bio_size >> SECTOR_SHIFT), wc->seq_count);
1404 			writecache_insert_entry(wc, f);
1405 			wc->uncommitted_blocks++;
1406 		} else {
1407 			struct wc_entry *f;
1408 			struct rb_node *next = rb_next(&e->rb_node);
1409 
1410 			if (!next)
1411 				break;
1412 			f = container_of(next, struct wc_entry, rb_node);
1413 			if (f != e + 1)
1414 				break;
1415 			if (read_original_sector(wc, f) !=
1416 			    read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
1417 				break;
1418 			if (unlikely(f->write_in_progress))
1419 				break;
1420 			if (writecache_entry_is_committed(wc, f))
1421 				wc->overwrote_committed = true;
1422 			e = f;
1423 		}
1424 		bio_size += wc->block_size;
1425 		current_cache_sec += wc->block_size >> SECTOR_SHIFT;
1426 	}
1427 
1428 	bio_set_dev(bio, wc->ssd_dev->bdev);
1429 	bio->bi_iter.bi_sector = start_cache_sec;
1430 	dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
1431 
1432 	wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1433 	if (!search_used)
1434 		wc->stats.writes_allocate += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
1435 
1436 	if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
1437 		wc->uncommitted_blocks = 0;
1438 		queue_work(wc->writeback_wq, &wc->flush_work);
1439 	} else {
1440 		writecache_schedule_autocommit(wc);
1441 	}
1442 }
1443 
1444 static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio *bio)
1445 {
1446 	struct wc_entry *e;
1447 	enum wc_map_op ret = WC_MAP_SUBMIT;
1448 	bool need_flush = false;
1449 
1450 	do {
1451 		bool found_entry = false;
1452 		bool search_used = false;
1453 
1454 		if (writecache_has_error(wc)) {
1455 			wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1456 			return WC_MAP_ERROR;
1457 		}
1458 		e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
1459 		if (e) {
1460 			if (!writecache_entry_is_committed(wc, e)) {
1461 				wc->stats.write_hits_uncommitted++;
1462 				search_used = true;
1463 				goto bio_copy;
1464 			}
1465 			wc->stats.write_hits_committed++;
1466 			if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
1467 				wc->overwrote_committed = true;
1468 				search_used = true;
1469 				goto bio_copy;
1470 			}
1471 			found_entry = true;
1472 		} else {
1473 			if (unlikely(wc->cleaner) ||
1474 			    (wc->metadata_only && !(bio->bi_opf & REQ_META)))
1475 				goto direct_write;
1476 		}
1477 		e = writecache_pop_from_freelist(wc, (sector_t)-1);
1478 		if (unlikely(!e)) {
1479 			if (!WC_MODE_PMEM(wc) && !found_entry) {
1480 direct_write:
1481 				e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
1482 				writecache_map_remap_origin(wc, bio, e);
1483 				wc->stats.writes_around += bio->bi_iter.bi_size >> wc->block_size_bits;
1484 				wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1485 				ret = WC_MAP_REMAP_ORIGIN;
1486 				goto flush_pmem_and_ret;
1487 			}
1488 			wc->stats.writes_blocked_on_freelist++;
1489 			writecache_wait_on_freelist(wc);
1490 			continue;
1491 		}
1492 		write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
1493 		writecache_insert_entry(wc, e);
1494 		wc->uncommitted_blocks++;
1495 		wc->stats.writes_allocate++;
1496 bio_copy:
1497 		if (WC_MODE_PMEM(wc)) {
1498 			bio_copy_block(wc, bio, memory_data(wc, e));
1499 			wc->stats.writes++;
1500 			need_flush = true;
1501 		} else {
1502 			writecache_bio_copy_ssd(wc, bio, e, search_used);
1503 			return WC_MAP_REMAP;
1504 		}
1505 	} while (bio->bi_iter.bi_size);
1506 
1507 flush_pmem_and_ret:
1508 	if (need_flush) {
1509 		if (unlikely(bio->bi_opf & REQ_FUA || wc->uncommitted_blocks >= wc->autocommit_blocks)) {
1510 			writecache_flush(wc);
1511 			if (writecache_has_error(wc))
1512 				ret = WC_MAP_ERROR;
1513 		} else {
1514 			writecache_schedule_autocommit(wc);
1515 		}
1516 	}
1517 
1518 	return ret;
1519 }
1520 
1521 static enum wc_map_op writecache_map_flush(struct dm_writecache *wc, struct bio *bio)
1522 {
1523 	if (writecache_has_error(wc))
1524 		return WC_MAP_ERROR;
1525 
1526 	if (WC_MODE_PMEM(wc)) {
1527 		wc->stats.flushes++;
1528 		writecache_flush(wc);
1529 		if (writecache_has_error(wc))
1530 			return WC_MAP_ERROR;
1531 		else if (unlikely(wc->cleaner) || unlikely(wc->metadata_only))
1532 			return WC_MAP_REMAP_ORIGIN;
1533 		return WC_MAP_SUBMIT;
1534 	}
1535 	/* SSD: */
1536 	if (dm_bio_get_target_bio_nr(bio))
1537 		return WC_MAP_REMAP_ORIGIN;
1538 	wc->stats.flushes++;
1539 	writecache_offload_bio(wc, bio);
1540 	return WC_MAP_RETURN;
1541 }
1542 
1543 static enum wc_map_op writecache_map_discard(struct dm_writecache *wc, struct bio *bio)
1544 {
1545 	wc->stats.discards += bio->bi_iter.bi_size >> wc->block_size_bits;
1546 
1547 	if (writecache_has_error(wc))
1548 		return WC_MAP_ERROR;
1549 
1550 	if (WC_MODE_PMEM(wc)) {
1551 		writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio));
1552 		return WC_MAP_REMAP_ORIGIN;
1553 	}
1554 	/* SSD: */
1555 	writecache_offload_bio(wc, bio);
1556 	return WC_MAP_RETURN;
1557 }
1558 
1559 static int writecache_map(struct dm_target *ti, struct bio *bio)
1560 {
1561 	struct dm_writecache *wc = ti->private;
1562 	enum wc_map_op map_op;
1563 
1564 	bio->bi_private = NULL;
1565 
1566 	wc_lock(wc);
1567 
1568 	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1569 		map_op = writecache_map_flush(wc, bio);
1570 		goto done;
1571 	}
1572 
1573 	bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1574 
1575 	if (unlikely((((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) &
1576 				(wc->block_size / 512 - 1)) != 0)) {
1577 		DMERR("I/O is not aligned, sector %llu, size %u, block size %u",
1578 		      (unsigned long long)bio->bi_iter.bi_sector,
1579 		      bio->bi_iter.bi_size, wc->block_size);
1580 		map_op = WC_MAP_ERROR;
1581 		goto done;
1582 	}
1583 
1584 	if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
1585 		map_op = writecache_map_discard(wc, bio);
1586 		goto done;
1587 	}
1588 
1589 	if (bio_data_dir(bio) == READ)
1590 		map_op = writecache_map_read(wc, bio);
1591 	else
1592 		map_op = writecache_map_write(wc, bio);
1593 done:
1594 	switch (map_op) {
1595 	case WC_MAP_REMAP_ORIGIN:
1596 		if (likely(wc->pause != 0)) {
1597 			if (bio_op(bio) == REQ_OP_WRITE) {
1598 				dm_iot_io_begin(&wc->iot, 1);
1599 				bio->bi_private = (void *)2;
1600 			}
1601 		}
1602 		bio_set_dev(bio, wc->dev->bdev);
1603 		wc_unlock(wc);
1604 		return DM_MAPIO_REMAPPED;
1605 
1606 	case WC_MAP_REMAP:
1607 		/* make sure that writecache_end_io decrements bio_in_progress: */
1608 		if (unlikely(bio->bi_opf & REQ_FUA) && bio_op(bio) == REQ_OP_WRITE)
1609 			bio->bi_private = (void *)3;
1610 		else
1611 			bio->bi_private = (void *)1;
1612 		atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]);
1613 		wc_unlock(wc);
1614 		return DM_MAPIO_REMAPPED;
1615 
1616 	case WC_MAP_SUBMIT:
1617 		wc_unlock(wc);
1618 		bio_endio(bio);
1619 		return DM_MAPIO_SUBMITTED;
1620 
1621 	case WC_MAP_RETURN:
1622 		wc_unlock(wc);
1623 		return DM_MAPIO_SUBMITTED;
1624 
1625 	case WC_MAP_ERROR:
1626 		wc_unlock(wc);
1627 		bio_io_error(bio);
1628 		return DM_MAPIO_SUBMITTED;
1629 
1630 	default:
1631 		BUG();
1632 		wc_unlock(wc);
1633 		return DM_MAPIO_KILL;
1634 	}
1635 }
1636 
1637 static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
1638 {
1639 	struct dm_writecache *wc = ti->private;
1640 
1641 	if (bio->bi_private == (void *)1 || bio->bi_private == (void *)3) {
1642 		int dir = bio_data_dir(bio);
1643 		bool fua = bio->bi_private == (void *)3;
1644 
1645 		if (atomic_dec_and_test(&wc->bio_in_progress[dir]))
1646 			if (unlikely(waitqueue_active(&wc->bio_in_progress_wait[dir])))
1647 				wake_up(&wc->bio_in_progress_wait[dir]);
1648 
1649 		if (fua && likely(*status == BLK_STS_OK)) {
1650 			bio->bi_private = NULL;
1651 			writecache_offload_bio(wc, bio);
1652 			return DM_ENDIO_INCOMPLETE;
1653 		}
1654 	} else if (bio->bi_private == (void *)2) {
1655 		dm_iot_io_end(&wc->iot, 1);
1656 	}
1657 	return 0;
1658 }
1659 
1660 static int writecache_iterate_devices(struct dm_target *ti,
1661 				      iterate_devices_callout_fn fn, void *data)
1662 {
1663 	struct dm_writecache *wc = ti->private;
1664 
1665 	return fn(ti, wc->dev, 0, ti->len, data);
1666 }
1667 
1668 static void writecache_io_hints(struct dm_target *ti, struct queue_limits *limits)
1669 {
1670 	struct dm_writecache *wc = ti->private;
1671 
1672 	dm_stack_bs_limits(limits, wc->block_size);
1673 }
1674 
1675 static void writecache_writeback_endio(struct bio *bio)
1676 {
1677 	struct writeback_struct *wb = container_of(bio, struct writeback_struct, bio);
1678 	struct dm_writecache *wc = wb->wc;
1679 	unsigned long flags;
1680 
1681 	raw_spin_lock_irqsave(&wc->endio_list_lock, flags);
1682 	if (unlikely(list_empty(&wc->endio_list)))
1683 		wake_up_process(wc->endio_thread);
1684 	list_add_tail(&wb->endio_entry, &wc->endio_list);
1685 	raw_spin_unlock_irqrestore(&wc->endio_list_lock, flags);
1686 }
1687 
1688 static void writecache_copy_endio(int read_err, unsigned long write_err, void *ptr)
1689 {
1690 	struct copy_struct *c = ptr;
1691 	struct dm_writecache *wc = c->wc;
1692 
1693 	c->error = likely(!(read_err | write_err)) ? 0 : -EIO;
1694 
1695 	raw_spin_lock_irq(&wc->endio_list_lock);
1696 	if (unlikely(list_empty(&wc->endio_list)))
1697 		wake_up_process(wc->endio_thread);
1698 	list_add_tail(&c->endio_entry, &wc->endio_list);
1699 	raw_spin_unlock_irq(&wc->endio_list_lock);
1700 }
1701 
1702 static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *list)
1703 {
1704 	unsigned int i;
1705 	struct writeback_struct *wb;
1706 	struct wc_entry *e;
1707 	unsigned long n_walked = 0;
1708 
1709 	do {
1710 		wb = list_entry(list->next, struct writeback_struct, endio_entry);
1711 		list_del(&wb->endio_entry);
1712 
1713 		if (unlikely(wb->bio.bi_status != BLK_STS_OK))
1714 			writecache_error(wc, blk_status_to_errno(wb->bio.bi_status),
1715 					"write error %d", wb->bio.bi_status);
1716 		i = 0;
1717 		do {
1718 			e = wb->wc_list[i];
1719 			BUG_ON(!e->write_in_progress);
1720 			e->write_in_progress = false;
1721 			INIT_LIST_HEAD(&e->lru);
1722 			if (!writecache_has_error(wc))
1723 				writecache_free_entry(wc, e);
1724 			BUG_ON(!wc->writeback_size);
1725 			wc->writeback_size--;
1726 			n_walked++;
1727 			if (unlikely(n_walked >= ENDIO_LATENCY)) {
1728 				writecache_commit_flushed(wc, false);
1729 				wc_unlock(wc);
1730 				wc_lock(wc);
1731 				n_walked = 0;
1732 			}
1733 		} while (++i < wb->wc_list_n);
1734 
1735 		if (wb->wc_list != wb->wc_list_inline)
1736 			kfree(wb->wc_list);
1737 		bio_put(&wb->bio);
1738 	} while (!list_empty(list));
1739 }
1740 
1741 static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *list)
1742 {
1743 	struct copy_struct *c;
1744 	struct wc_entry *e;
1745 
1746 	do {
1747 		c = list_entry(list->next, struct copy_struct, endio_entry);
1748 		list_del(&c->endio_entry);
1749 
1750 		if (unlikely(c->error))
1751 			writecache_error(wc, c->error, "copy error");
1752 
1753 		e = c->e;
1754 		do {
1755 			BUG_ON(!e->write_in_progress);
1756 			e->write_in_progress = false;
1757 			INIT_LIST_HEAD(&e->lru);
1758 			if (!writecache_has_error(wc))
1759 				writecache_free_entry(wc, e);
1760 
1761 			BUG_ON(!wc->writeback_size);
1762 			wc->writeback_size--;
1763 			e++;
1764 		} while (--c->n_entries);
1765 		mempool_free(c, &wc->copy_pool);
1766 	} while (!list_empty(list));
1767 }
1768 
1769 static int writecache_endio_thread(void *data)
1770 {
1771 	struct dm_writecache *wc = data;
1772 
1773 	while (1) {
1774 		struct list_head list;
1775 
1776 		raw_spin_lock_irq(&wc->endio_list_lock);
1777 		if (!list_empty(&wc->endio_list))
1778 			goto pop_from_list;
1779 		set_current_state(TASK_INTERRUPTIBLE);
1780 		raw_spin_unlock_irq(&wc->endio_list_lock);
1781 
1782 		if (unlikely(kthread_should_stop())) {
1783 			set_current_state(TASK_RUNNING);
1784 			break;
1785 		}
1786 
1787 		schedule();
1788 
1789 		continue;
1790 
1791 pop_from_list:
1792 		list = wc->endio_list;
1793 		list.next->prev = list.prev->next = &list;
1794 		INIT_LIST_HEAD(&wc->endio_list);
1795 		raw_spin_unlock_irq(&wc->endio_list_lock);
1796 
1797 		if (!WC_MODE_FUA(wc))
1798 			writecache_disk_flush(wc, wc->dev);
1799 
1800 		wc_lock(wc);
1801 
1802 		if (WC_MODE_PMEM(wc)) {
1803 			__writecache_endio_pmem(wc, &list);
1804 		} else {
1805 			__writecache_endio_ssd(wc, &list);
1806 			writecache_wait_for_ios(wc, READ);
1807 		}
1808 
1809 		writecache_commit_flushed(wc, false);
1810 
1811 		wc_unlock(wc);
1812 	}
1813 
1814 	return 0;
1815 }
1816 
1817 static bool wc_add_block(struct writeback_struct *wb, struct wc_entry *e)
1818 {
1819 	struct dm_writecache *wc = wb->wc;
1820 	unsigned int block_size = wc->block_size;
1821 	void *address = memory_data(wc, e);
1822 
1823 	persistent_memory_flush_cache(address, block_size);
1824 
1825 	if (unlikely(bio_end_sector(&wb->bio) >= wc->data_device_sectors))
1826 		return true;
1827 
1828 	return bio_add_page(&wb->bio, persistent_memory_page(address),
1829 			    block_size, persistent_memory_page_offset(address)) != 0;
1830 }
1831 
1832 struct writeback_list {
1833 	struct list_head list;
1834 	size_t size;
1835 };
1836 
1837 static void __writeback_throttle(struct dm_writecache *wc, struct writeback_list *wbl)
1838 {
1839 	if (unlikely(wc->max_writeback_jobs)) {
1840 		if (READ_ONCE(wc->writeback_size) - wbl->size >= wc->max_writeback_jobs) {
1841 			wc_lock(wc);
1842 			while (wc->writeback_size - wbl->size >= wc->max_writeback_jobs)
1843 				writecache_wait_on_freelist(wc);
1844 			wc_unlock(wc);
1845 		}
1846 	}
1847 	cond_resched();
1848 }
1849 
1850 static void __writecache_writeback_pmem(struct dm_writecache *wc, struct writeback_list *wbl)
1851 {
1852 	struct wc_entry *e, *f;
1853 	struct bio *bio;
1854 	struct writeback_struct *wb;
1855 	unsigned int max_pages;
1856 
1857 	while (wbl->size) {
1858 		wbl->size--;
1859 		e = container_of(wbl->list.prev, struct wc_entry, lru);
1860 		list_del(&e->lru);
1861 
1862 		max_pages = e->wc_list_contiguous;
1863 
1864 		bio = bio_alloc_bioset(wc->dev->bdev, max_pages, REQ_OP_WRITE,
1865 				       GFP_NOIO, &wc->bio_set);
1866 		wb = container_of(bio, struct writeback_struct, bio);
1867 		wb->wc = wc;
1868 		bio->bi_end_io = writecache_writeback_endio;
1869 		bio->bi_iter.bi_sector = read_original_sector(wc, e);
1870 
1871 		if (unlikely(max_pages > WB_LIST_INLINE))
1872 			wb->wc_list = kmalloc_objs(struct wc_entry *, max_pages,
1873 						   GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
1874 
1875 		if (likely(max_pages <= WB_LIST_INLINE) || unlikely(!wb->wc_list)) {
1876 			wb->wc_list = wb->wc_list_inline;
1877 			max_pages = WB_LIST_INLINE;
1878 		}
1879 
1880 		BUG_ON(!wc_add_block(wb, e));
1881 
1882 		wb->wc_list[0] = e;
1883 		wb->wc_list_n = 1;
1884 
1885 		while (wbl->size && wb->wc_list_n < max_pages) {
1886 			f = container_of(wbl->list.prev, struct wc_entry, lru);
1887 			if (read_original_sector(wc, f) !=
1888 			    read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
1889 				break;
1890 			if (!wc_add_block(wb, f))
1891 				break;
1892 			wbl->size--;
1893 			list_del(&f->lru);
1894 			wb->wc_list[wb->wc_list_n++] = f;
1895 			e = f;
1896 		}
1897 		if (WC_MODE_FUA(wc))
1898 			bio->bi_opf |= REQ_FUA;
1899 		if (writecache_has_error(wc)) {
1900 			bio->bi_status = BLK_STS_IOERR;
1901 			bio_endio(bio);
1902 		} else if (unlikely(!bio_sectors(bio))) {
1903 			bio->bi_status = BLK_STS_OK;
1904 			bio_endio(bio);
1905 		} else {
1906 			submit_bio(bio);
1907 		}
1908 
1909 		__writeback_throttle(wc, wbl);
1910 	}
1911 }
1912 
1913 static void __writecache_writeback_ssd(struct dm_writecache *wc, struct writeback_list *wbl)
1914 {
1915 	struct wc_entry *e, *f;
1916 	struct dm_io_region from, to;
1917 	struct copy_struct *c;
1918 
1919 	while (wbl->size) {
1920 		unsigned int n_sectors;
1921 
1922 		wbl->size--;
1923 		e = container_of(wbl->list.prev, struct wc_entry, lru);
1924 		list_del(&e->lru);
1925 
1926 		n_sectors = e->wc_list_contiguous << (wc->block_size_bits - SECTOR_SHIFT);
1927 
1928 		from.bdev = wc->ssd_dev->bdev;
1929 		from.sector = cache_sector(wc, e);
1930 		from.count = n_sectors;
1931 		to.bdev = wc->dev->bdev;
1932 		to.sector = read_original_sector(wc, e);
1933 		to.count = n_sectors;
1934 
1935 		c = mempool_alloc(&wc->copy_pool, GFP_NOIO);
1936 		c->wc = wc;
1937 		c->e = e;
1938 		c->n_entries = e->wc_list_contiguous;
1939 
1940 		while ((n_sectors -= wc->block_size >> SECTOR_SHIFT)) {
1941 			wbl->size--;
1942 			f = container_of(wbl->list.prev, struct wc_entry, lru);
1943 			BUG_ON(f != e + 1);
1944 			list_del(&f->lru);
1945 			e = f;
1946 		}
1947 
1948 		if (unlikely(to.sector + to.count > wc->data_device_sectors)) {
1949 			if (to.sector >= wc->data_device_sectors) {
1950 				writecache_copy_endio(0, 0, c);
1951 				continue;
1952 			}
1953 			from.count = to.count = wc->data_device_sectors - to.sector;
1954 		}
1955 
1956 		dm_kcopyd_copy(wc->dm_kcopyd, &from, 1, &to, 0, writecache_copy_endio, c);
1957 
1958 		__writeback_throttle(wc, wbl);
1959 	}
1960 }
1961 
1962 static void writecache_writeback(struct work_struct *work)
1963 {
1964 	struct dm_writecache *wc = container_of(work, struct dm_writecache, writeback_work);
1965 	struct blk_plug plug;
1966 	struct wc_entry *f, *g, *e = NULL;
1967 	struct rb_node *node, *next_node;
1968 	struct list_head skipped;
1969 	struct writeback_list wbl;
1970 	unsigned long n_walked;
1971 
1972 	if (!WC_MODE_PMEM(wc)) {
1973 		/* Wait for any active kcopyd work on behalf of ssd writeback */
1974 		dm_kcopyd_client_flush(wc->dm_kcopyd);
1975 	}
1976 
1977 	if (likely(wc->pause != 0)) {
1978 		while (1) {
1979 			unsigned long idle;
1980 
1981 			if (unlikely(wc->cleaner) || unlikely(wc->writeback_all) ||
1982 			    unlikely(dm_suspended(wc->ti)))
1983 				break;
1984 			idle = dm_iot_idle_time(&wc->iot);
1985 			if (idle >= wc->pause)
1986 				break;
1987 			idle = wc->pause - idle;
1988 			if (idle > HZ)
1989 				idle = HZ;
1990 			schedule_timeout_idle(idle);
1991 		}
1992 	}
1993 
1994 	wc_lock(wc);
1995 restart:
1996 	if (writecache_has_error(wc)) {
1997 		wc_unlock(wc);
1998 		return;
1999 	}
2000 
2001 	if (unlikely(wc->writeback_all)) {
2002 		if (writecache_wait_for_writeback(wc))
2003 			goto restart;
2004 	}
2005 
2006 	if (wc->overwrote_committed)
2007 		writecache_wait_for_ios(wc, WRITE);
2008 
2009 	n_walked = 0;
2010 	INIT_LIST_HEAD(&skipped);
2011 	INIT_LIST_HEAD(&wbl.list);
2012 	wbl.size = 0;
2013 	while (!list_empty(&wc->lru) &&
2014 	       (wc->writeback_all ||
2015 		wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark ||
2016 		((u32)(jiffies - container_of(wc->lru.prev, struct wc_entry, lru)->age) >=
2017 		 wc->max_age - wc->max_age / MAX_AGE_DIV))) {
2018 
2019 		n_walked++;
2020 		if (unlikely(n_walked > WRITEBACK_LATENCY) &&
2021 		    likely(!wc->writeback_all)) {
2022 			if (likely(!dm_suspended(wc->ti)))
2023 				queue_work(wc->writeback_wq, &wc->writeback_work);
2024 			break;
2025 		}
2026 
2027 		if (unlikely(wc->writeback_all)) {
2028 			if (unlikely(!e)) {
2029 				writecache_flush(wc);
2030 				e = container_of(rb_first(&wc->tree), struct wc_entry, rb_node);
2031 			} else
2032 				e = g;
2033 		} else
2034 			e = container_of(wc->lru.prev, struct wc_entry, lru);
2035 		BUG_ON(e->write_in_progress);
2036 		if (unlikely(!writecache_entry_is_committed(wc, e)))
2037 			writecache_flush(wc);
2038 
2039 		node = rb_prev(&e->rb_node);
2040 		if (node) {
2041 			f = container_of(node, struct wc_entry, rb_node);
2042 			if (unlikely(read_original_sector(wc, f) ==
2043 				     read_original_sector(wc, e))) {
2044 				BUG_ON(!f->write_in_progress);
2045 				list_move(&e->lru, &skipped);
2046 				cond_resched();
2047 				continue;
2048 			}
2049 		}
2050 		wc->writeback_size++;
2051 		list_move(&e->lru, &wbl.list);
2052 		wbl.size++;
2053 		e->write_in_progress = true;
2054 		e->wc_list_contiguous = 1;
2055 
2056 		f = e;
2057 
2058 		while (1) {
2059 			next_node = rb_next(&f->rb_node);
2060 			if (unlikely(!next_node))
2061 				break;
2062 			g = container_of(next_node, struct wc_entry, rb_node);
2063 			if (unlikely(read_original_sector(wc, g) ==
2064 			    read_original_sector(wc, f))) {
2065 				f = g;
2066 				continue;
2067 			}
2068 			if (read_original_sector(wc, g) !=
2069 			    read_original_sector(wc, f) + (wc->block_size >> SECTOR_SHIFT))
2070 				break;
2071 			if (unlikely(g->write_in_progress))
2072 				break;
2073 			if (unlikely(!writecache_entry_is_committed(wc, g)))
2074 				break;
2075 
2076 			if (!WC_MODE_PMEM(wc)) {
2077 				if (g != f + 1)
2078 					break;
2079 			}
2080 
2081 			n_walked++;
2082 			//if (unlikely(n_walked > WRITEBACK_LATENCY) && likely(!wc->writeback_all))
2083 			//	break;
2084 
2085 			wc->writeback_size++;
2086 			list_move(&g->lru, &wbl.list);
2087 			wbl.size++;
2088 			g->write_in_progress = true;
2089 			g->wc_list_contiguous = BIO_MAX_VECS;
2090 			f = g;
2091 			e->wc_list_contiguous++;
2092 			if (unlikely(e->wc_list_contiguous == BIO_MAX_VECS)) {
2093 				if (unlikely(wc->writeback_all)) {
2094 					next_node = rb_next(&f->rb_node);
2095 					if (likely(next_node))
2096 						g = container_of(next_node, struct wc_entry, rb_node);
2097 				}
2098 				break;
2099 			}
2100 		}
2101 		cond_resched();
2102 	}
2103 
2104 	if (!list_empty(&skipped)) {
2105 		list_splice_tail(&skipped, &wc->lru);
2106 		/*
2107 		 * If we didn't do any progress, we must wait until some
2108 		 * writeback finishes to avoid burning CPU in a loop
2109 		 */
2110 		if (unlikely(!wbl.size))
2111 			writecache_wait_for_writeback(wc);
2112 	}
2113 
2114 	wc_unlock(wc);
2115 
2116 	blk_start_plug(&plug);
2117 
2118 	if (WC_MODE_PMEM(wc))
2119 		__writecache_writeback_pmem(wc, &wbl);
2120 	else
2121 		__writecache_writeback_ssd(wc, &wbl);
2122 
2123 	blk_finish_plug(&plug);
2124 
2125 	if (unlikely(wc->writeback_all)) {
2126 		wc_lock(wc);
2127 		while (writecache_wait_for_writeback(wc))
2128 			;
2129 		wc_unlock(wc);
2130 	}
2131 }
2132 
2133 static int calculate_memory_size(uint64_t device_size, unsigned int block_size,
2134 				 size_t *n_blocks_p, size_t *n_metadata_blocks_p)
2135 {
2136 	uint64_t n_blocks, offset;
2137 
2138 	n_blocks = device_size;
2139 	do_div(n_blocks, block_size + sizeof(struct wc_memory_entry));
2140 
2141 	while (1) {
2142 		if (!n_blocks)
2143 			return -ENOSPC;
2144 		/* Verify the following entries[n_blocks] won't overflow */
2145 		if (n_blocks >= ((size_t)-sizeof(struct wc_memory_superblock) /
2146 				 sizeof(struct wc_memory_entry)))
2147 			return -EFBIG;
2148 		offset = offsetof(struct wc_memory_superblock, entries[n_blocks]);
2149 		offset = (offset + block_size - 1) & ~(uint64_t)(block_size - 1);
2150 		if (offset + n_blocks * block_size <= device_size)
2151 			break;
2152 		n_blocks--;
2153 	}
2154 
2155 	if (n_blocks_p)
2156 		*n_blocks_p = n_blocks;
2157 	if (n_metadata_blocks_p)
2158 		*n_metadata_blocks_p = offset >> __ffs(block_size);
2159 	return 0;
2160 }
2161 
2162 static int init_memory(struct dm_writecache *wc)
2163 {
2164 	size_t b;
2165 	int r;
2166 
2167 	r = calculate_memory_size(wc->memory_map_size, wc->block_size, &wc->n_blocks, NULL);
2168 	if (r)
2169 		return r;
2170 
2171 	r = writecache_alloc_entries(wc);
2172 	if (r)
2173 		return r;
2174 
2175 	for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++)
2176 		pmem_assign(sb(wc)->padding[b], cpu_to_le64(0));
2177 	pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION));
2178 	pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size));
2179 	pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks));
2180 	pmem_assign(sb(wc)->seq_count, cpu_to_le64(0));
2181 
2182 	for (b = 0; b < wc->n_blocks; b++) {
2183 		write_original_sector_seq_count(wc, &wc->entries[b], -1, -1);
2184 		cond_resched();
2185 	}
2186 
2187 	writecache_flush_all_metadata(wc);
2188 	writecache_commit_flushed(wc, false);
2189 	pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC));
2190 	writecache_flush_region(wc, &sb(wc)->magic, sizeof(sb(wc)->magic));
2191 	writecache_commit_flushed(wc, false);
2192 
2193 	return 0;
2194 }
2195 
2196 static void writecache_dtr(struct dm_target *ti)
2197 {
2198 	struct dm_writecache *wc = ti->private;
2199 
2200 	if (!wc)
2201 		return;
2202 
2203 	if (wc->endio_thread)
2204 		kthread_stop(wc->endio_thread);
2205 
2206 	if (wc->flush_thread)
2207 		kthread_stop(wc->flush_thread);
2208 
2209 	bioset_exit(&wc->bio_set);
2210 
2211 	mempool_exit(&wc->copy_pool);
2212 
2213 	if (wc->writeback_wq)
2214 		destroy_workqueue(wc->writeback_wq);
2215 
2216 	if (wc->dev)
2217 		dm_put_device(ti, wc->dev);
2218 
2219 	if (wc->ssd_dev)
2220 		dm_put_device(ti, wc->ssd_dev);
2221 
2222 	vfree(wc->entries);
2223 
2224 	if (wc->memory_map) {
2225 		if (WC_MODE_PMEM(wc))
2226 			persistent_memory_release(wc);
2227 		else
2228 			vfree(wc->memory_map);
2229 	}
2230 
2231 	if (wc->dm_kcopyd)
2232 		dm_kcopyd_client_destroy(wc->dm_kcopyd);
2233 
2234 	if (wc->dm_io)
2235 		dm_io_client_destroy(wc->dm_io);
2236 
2237 	vfree(wc->dirty_bitmap);
2238 
2239 	kfree(wc);
2240 }
2241 
2242 static int writecache_ctr(struct dm_target *ti, unsigned int argc, char **argv)
2243 {
2244 	struct dm_writecache *wc;
2245 	struct dm_arg_set as;
2246 	const char *string;
2247 	unsigned int opt_params;
2248 	size_t offset, data_size;
2249 	int i, r;
2250 	char dummy;
2251 	int high_wm_percent = HIGH_WATERMARK;
2252 	int low_wm_percent = LOW_WATERMARK;
2253 	uint64_t x;
2254 	struct wc_memory_superblock s;
2255 
2256 	static struct dm_arg _args[] = {
2257 		{0, 18, "Invalid number of feature args"},
2258 	};
2259 
2260 	as.argc = argc;
2261 	as.argv = argv;
2262 
2263 	wc = kzalloc_obj(struct dm_writecache);
2264 	if (!wc) {
2265 		ti->error = "Cannot allocate writecache structure";
2266 		r = -ENOMEM;
2267 		goto bad;
2268 	}
2269 	ti->private = wc;
2270 	wc->ti = ti;
2271 
2272 	mutex_init(&wc->lock);
2273 	wc->max_age = MAX_AGE_UNSPECIFIED;
2274 	writecache_poison_lists(wc);
2275 	init_waitqueue_head(&wc->freelist_wait);
2276 	timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0);
2277 	timer_setup(&wc->max_age_timer, writecache_max_age_timer, 0);
2278 
2279 	for (i = 0; i < 2; i++) {
2280 		atomic_set(&wc->bio_in_progress[i], 0);
2281 		init_waitqueue_head(&wc->bio_in_progress_wait[i]);
2282 	}
2283 
2284 	wc->dm_io = dm_io_client_create();
2285 	if (IS_ERR(wc->dm_io)) {
2286 		r = PTR_ERR(wc->dm_io);
2287 		ti->error = "Unable to allocate dm-io client";
2288 		wc->dm_io = NULL;
2289 		goto bad;
2290 	}
2291 
2292 	wc->writeback_wq = alloc_workqueue("writecache-writeback",
2293 					   WQ_MEM_RECLAIM | WQ_PERCPU, 1);
2294 	if (!wc->writeback_wq) {
2295 		r = -ENOMEM;
2296 		ti->error = "Could not allocate writeback workqueue";
2297 		goto bad;
2298 	}
2299 	INIT_WORK(&wc->writeback_work, writecache_writeback);
2300 	INIT_WORK(&wc->flush_work, writecache_flush_work);
2301 
2302 	dm_iot_init(&wc->iot);
2303 
2304 	raw_spin_lock_init(&wc->endio_list_lock);
2305 	INIT_LIST_HEAD(&wc->endio_list);
2306 	wc->endio_thread = kthread_run(writecache_endio_thread, wc, "writecache_endio");
2307 	if (IS_ERR(wc->endio_thread)) {
2308 		r = PTR_ERR(wc->endio_thread);
2309 		wc->endio_thread = NULL;
2310 		ti->error = "Couldn't spawn endio thread";
2311 		goto bad;
2312 	}
2313 
2314 	/*
2315 	 * Parse the mode (pmem or ssd)
2316 	 */
2317 	string = dm_shift_arg(&as);
2318 	if (!string)
2319 		goto bad_arguments;
2320 
2321 	if (!strcasecmp(string, "s")) {
2322 		wc->pmem_mode = false;
2323 	} else if (!strcasecmp(string, "p")) {
2324 #ifdef DM_WRITECACHE_HAS_PMEM
2325 		wc->pmem_mode = true;
2326 		wc->writeback_fua = true;
2327 #else
2328 		/*
2329 		 * If the architecture doesn't support persistent memory or
2330 		 * the kernel doesn't support any DAX drivers, this driver can
2331 		 * only be used in SSD-only mode.
2332 		 */
2333 		r = -EOPNOTSUPP;
2334 		ti->error = "Persistent memory or DAX not supported on this system";
2335 		goto bad;
2336 #endif
2337 	} else {
2338 		goto bad_arguments;
2339 	}
2340 
2341 	if (WC_MODE_PMEM(wc)) {
2342 		r = bioset_init(&wc->bio_set, BIO_POOL_SIZE,
2343 				offsetof(struct writeback_struct, bio),
2344 				BIOSET_NEED_BVECS);
2345 		if (r) {
2346 			ti->error = "Could not allocate bio set";
2347 			goto bad;
2348 		}
2349 	} else {
2350 		wc->pause = PAUSE_WRITEBACK;
2351 		r = mempool_init_kmalloc_pool(&wc->copy_pool, 1, sizeof(struct copy_struct));
2352 		if (r) {
2353 			ti->error = "Could not allocate mempool";
2354 			goto bad;
2355 		}
2356 	}
2357 
2358 	/*
2359 	 * Parse the origin data device
2360 	 */
2361 	string = dm_shift_arg(&as);
2362 	if (!string)
2363 		goto bad_arguments;
2364 	r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->dev);
2365 	if (r) {
2366 		ti->error = "Origin data device lookup failed";
2367 		goto bad;
2368 	}
2369 
2370 	/*
2371 	 * Parse cache data device (be it pmem or ssd)
2372 	 */
2373 	string = dm_shift_arg(&as);
2374 	if (!string)
2375 		goto bad_arguments;
2376 
2377 	r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->ssd_dev);
2378 	if (r) {
2379 		ti->error = "Cache data device lookup failed";
2380 		goto bad;
2381 	}
2382 	wc->memory_map_size = bdev_nr_bytes(wc->ssd_dev->bdev);
2383 
2384 	/*
2385 	 * Parse the cache block size
2386 	 */
2387 	string = dm_shift_arg(&as);
2388 	if (!string)
2389 		goto bad_arguments;
2390 	if (sscanf(string, "%u%c", &wc->block_size, &dummy) != 1 ||
2391 	    wc->block_size < 512 || wc->block_size > PAGE_SIZE ||
2392 	    (wc->block_size & (wc->block_size - 1))) {
2393 		r = -EINVAL;
2394 		ti->error = "Invalid block size";
2395 		goto bad;
2396 	}
2397 	if (wc->block_size < bdev_logical_block_size(wc->dev->bdev) ||
2398 	    wc->block_size < bdev_logical_block_size(wc->ssd_dev->bdev)) {
2399 		r = -EINVAL;
2400 		ti->error = "Block size is smaller than device logical block size";
2401 		goto bad;
2402 	}
2403 	wc->block_size_bits = __ffs(wc->block_size);
2404 
2405 	wc->max_writeback_jobs = MAX_WRITEBACK_JOBS;
2406 	wc->autocommit_blocks = !WC_MODE_PMEM(wc) ? AUTOCOMMIT_BLOCKS_SSD : AUTOCOMMIT_BLOCKS_PMEM;
2407 	wc->autocommit_jiffies = msecs_to_jiffies(AUTOCOMMIT_MSEC);
2408 
2409 	/*
2410 	 * Parse optional arguments
2411 	 */
2412 	r = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
2413 	if (r)
2414 		goto bad;
2415 
2416 	while (opt_params) {
2417 		string = dm_shift_arg(&as), opt_params--;
2418 		if (!strcasecmp(string, "start_sector") && opt_params >= 1) {
2419 			unsigned long long start_sector;
2420 
2421 			string = dm_shift_arg(&as), opt_params--;
2422 			if (sscanf(string, "%llu%c", &start_sector, &dummy) != 1)
2423 				goto invalid_optional;
2424 			wc->start_sector = start_sector;
2425 			wc->start_sector_set = true;
2426 			if (wc->start_sector != start_sector ||
2427 			    wc->start_sector >= wc->memory_map_size >> SECTOR_SHIFT)
2428 				goto invalid_optional;
2429 		} else if (!strcasecmp(string, "high_watermark") && opt_params >= 1) {
2430 			string = dm_shift_arg(&as), opt_params--;
2431 			if (sscanf(string, "%d%c", &high_wm_percent, &dummy) != 1)
2432 				goto invalid_optional;
2433 			if (high_wm_percent < 0 || high_wm_percent > 100)
2434 				goto invalid_optional;
2435 			wc->high_wm_percent_value = high_wm_percent;
2436 			wc->high_wm_percent_set = true;
2437 		} else if (!strcasecmp(string, "low_watermark") && opt_params >= 1) {
2438 			string = dm_shift_arg(&as), opt_params--;
2439 			if (sscanf(string, "%d%c", &low_wm_percent, &dummy) != 1)
2440 				goto invalid_optional;
2441 			if (low_wm_percent < 0 || low_wm_percent > 100)
2442 				goto invalid_optional;
2443 			wc->low_wm_percent_value = low_wm_percent;
2444 			wc->low_wm_percent_set = true;
2445 		} else if (!strcasecmp(string, "writeback_jobs") && opt_params >= 1) {
2446 			string = dm_shift_arg(&as), opt_params--;
2447 			if (sscanf(string, "%u%c", &wc->max_writeback_jobs, &dummy) != 1)
2448 				goto invalid_optional;
2449 			wc->max_writeback_jobs_set = true;
2450 		} else if (!strcasecmp(string, "autocommit_blocks") && opt_params >= 1) {
2451 			string = dm_shift_arg(&as), opt_params--;
2452 			if (sscanf(string, "%u%c", &wc->autocommit_blocks, &dummy) != 1)
2453 				goto invalid_optional;
2454 			wc->autocommit_blocks_set = true;
2455 		} else if (!strcasecmp(string, "autocommit_time") && opt_params >= 1) {
2456 			unsigned int autocommit_msecs;
2457 
2458 			string = dm_shift_arg(&as), opt_params--;
2459 			if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1)
2460 				goto invalid_optional;
2461 			if (autocommit_msecs > 3600000)
2462 				goto invalid_optional;
2463 			wc->autocommit_jiffies = msecs_to_jiffies(autocommit_msecs);
2464 			wc->autocommit_time_value = autocommit_msecs;
2465 			wc->autocommit_time_set = true;
2466 		} else if (!strcasecmp(string, "max_age") && opt_params >= 1) {
2467 			unsigned int max_age_msecs;
2468 			unsigned long max_age_jiffies;
2469 
2470 			string = dm_shift_arg(&as), opt_params--;
2471 			if (sscanf(string, "%u%c", &max_age_msecs, &dummy) != 1)
2472 				goto invalid_optional;
2473 			max_age_jiffies = msecs_to_jiffies(max_age_msecs);
2474 			if (max_age_jiffies >= min(MAX_JIFFY_OFFSET, (7 << 28))) // 7/8ths of 1 << 31
2475 				goto invalid_optional;
2476 			wc->max_age = max_age_jiffies;
2477 			wc->max_age_set = true;
2478 			wc->max_age_value = max_age_msecs;
2479 		} else if (!strcasecmp(string, "cleaner")) {
2480 			wc->cleaner_set = true;
2481 			wc->cleaner = true;
2482 		} else if (!strcasecmp(string, "fua")) {
2483 			if (WC_MODE_PMEM(wc)) {
2484 				wc->writeback_fua = true;
2485 				wc->writeback_fua_set = true;
2486 			} else
2487 				goto invalid_optional;
2488 		} else if (!strcasecmp(string, "nofua")) {
2489 			if (WC_MODE_PMEM(wc)) {
2490 				wc->writeback_fua = false;
2491 				wc->writeback_fua_set = true;
2492 			} else
2493 				goto invalid_optional;
2494 		} else if (!strcasecmp(string, "metadata_only")) {
2495 			wc->metadata_only = true;
2496 		} else if (!strcasecmp(string, "pause_writeback") && opt_params >= 1) {
2497 			unsigned int pause_msecs;
2498 
2499 			if (WC_MODE_PMEM(wc))
2500 				goto invalid_optional;
2501 			string = dm_shift_arg(&as), opt_params--;
2502 			if (sscanf(string, "%u%c", &pause_msecs, &dummy) != 1)
2503 				goto invalid_optional;
2504 			if (pause_msecs > 60000)
2505 				goto invalid_optional;
2506 			wc->pause = msecs_to_jiffies(pause_msecs);
2507 			wc->pause_set = true;
2508 			wc->pause_value = pause_msecs;
2509 		} else {
2510 invalid_optional:
2511 			r = -EINVAL;
2512 			ti->error = "Invalid optional argument";
2513 			goto bad;
2514 		}
2515 	}
2516 
2517 	if (high_wm_percent < low_wm_percent) {
2518 		r = -EINVAL;
2519 		ti->error = "High watermark must be greater than or equal to low watermark";
2520 		goto bad;
2521 	}
2522 
2523 	if (WC_MODE_PMEM(wc)) {
2524 		if (!wc->ssd_dev->dax_dev) {
2525 			r = -EOPNOTSUPP;
2526 			ti->error = "Cache device is not a DAX device";
2527 			goto bad;
2528 		}
2529 		if (!dax_synchronous(wc->ssd_dev->dax_dev)) {
2530 			r = -EOPNOTSUPP;
2531 			ti->error = "Asynchronous persistent memory not supported as pmem cache";
2532 			goto bad;
2533 		}
2534 
2535 		r = persistent_memory_claim(wc);
2536 		if (r) {
2537 			ti->error = "Unable to map persistent memory for cache";
2538 			goto bad;
2539 		}
2540 	} else {
2541 		size_t n_blocks, n_metadata_blocks;
2542 		uint64_t n_bitmap_bits;
2543 
2544 		wc->memory_map_size -= (uint64_t)wc->start_sector << SECTOR_SHIFT;
2545 
2546 		spin_lock_init(&wc->flush_list_lock);
2547 		bio_list_init(&wc->flush_list);
2548 		wc->flush_thread = kthread_run(writecache_flush_thread, wc, "dm_writecache_flush");
2549 		if (IS_ERR(wc->flush_thread)) {
2550 			r = PTR_ERR(wc->flush_thread);
2551 			wc->flush_thread = NULL;
2552 			ti->error = "Couldn't spawn flush thread";
2553 			goto bad;
2554 		}
2555 
2556 		r = calculate_memory_size(wc->memory_map_size, wc->block_size,
2557 					  &n_blocks, &n_metadata_blocks);
2558 		if (r) {
2559 			ti->error = "Invalid device size";
2560 			goto bad;
2561 		}
2562 
2563 		n_bitmap_bits = (((uint64_t)n_metadata_blocks << wc->block_size_bits) +
2564 				 BITMAP_GRANULARITY - 1) / BITMAP_GRANULARITY;
2565 		/* this is limitation of test_bit functions */
2566 		if (n_bitmap_bits > 1U << 31) {
2567 			r = -EFBIG;
2568 			ti->error = "Invalid device size";
2569 			goto bad;
2570 		}
2571 
2572 		wc->memory_map = vmalloc(n_metadata_blocks << wc->block_size_bits);
2573 		if (!wc->memory_map) {
2574 			r = -ENOMEM;
2575 			ti->error = "Unable to allocate memory for metadata";
2576 			goto bad;
2577 		}
2578 
2579 		wc->dm_kcopyd = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2580 		if (IS_ERR(wc->dm_kcopyd)) {
2581 			r = PTR_ERR(wc->dm_kcopyd);
2582 			ti->error = "Unable to allocate dm-kcopyd client";
2583 			wc->dm_kcopyd = NULL;
2584 			goto bad;
2585 		}
2586 
2587 		wc->metadata_sectors = n_metadata_blocks << (wc->block_size_bits - SECTOR_SHIFT);
2588 		wc->dirty_bitmap_size = (n_bitmap_bits + BITS_PER_LONG - 1) /
2589 			BITS_PER_LONG * sizeof(unsigned long);
2590 		wc->dirty_bitmap = vzalloc(wc->dirty_bitmap_size);
2591 		if (!wc->dirty_bitmap) {
2592 			r = -ENOMEM;
2593 			ti->error = "Unable to allocate dirty bitmap";
2594 			goto bad;
2595 		}
2596 
2597 		r = writecache_read_metadata(wc, wc->block_size >> SECTOR_SHIFT);
2598 		if (r) {
2599 			ti->error = "Unable to read first block of metadata";
2600 			goto bad;
2601 		}
2602 	}
2603 
2604 	r = copy_mc_to_kernel(&s, sb(wc), sizeof(struct wc_memory_superblock));
2605 	if (r) {
2606 		ti->error = "Hardware memory error when reading superblock";
2607 		goto bad;
2608 	}
2609 	if (!le32_to_cpu(s.magic) && !le32_to_cpu(s.version)) {
2610 		r = init_memory(wc);
2611 		if (r) {
2612 			ti->error = "Unable to initialize device";
2613 			goto bad;
2614 		}
2615 		r = copy_mc_to_kernel(&s, sb(wc),
2616 				      sizeof(struct wc_memory_superblock));
2617 		if (r) {
2618 			ti->error = "Hardware memory error when reading superblock";
2619 			goto bad;
2620 		}
2621 	}
2622 
2623 	if (le32_to_cpu(s.magic) != MEMORY_SUPERBLOCK_MAGIC) {
2624 		ti->error = "Invalid magic in the superblock";
2625 		r = -EINVAL;
2626 		goto bad;
2627 	}
2628 
2629 	if (le32_to_cpu(s.version) != MEMORY_SUPERBLOCK_VERSION) {
2630 		ti->error = "Invalid version in the superblock";
2631 		r = -EINVAL;
2632 		goto bad;
2633 	}
2634 
2635 	if (le32_to_cpu(s.block_size) != wc->block_size) {
2636 		ti->error = "Block size does not match superblock";
2637 		r = -EINVAL;
2638 		goto bad;
2639 	}
2640 
2641 	wc->n_blocks = le64_to_cpu(s.n_blocks);
2642 
2643 	offset = wc->n_blocks * sizeof(struct wc_memory_entry);
2644 	if (offset / sizeof(struct wc_memory_entry) != le64_to_cpu(sb(wc)->n_blocks)) {
2645 overflow:
2646 		ti->error = "Overflow in size calculation";
2647 		r = -EINVAL;
2648 		goto bad;
2649 	}
2650 	offset += sizeof(struct wc_memory_superblock);
2651 	if (offset < sizeof(struct wc_memory_superblock))
2652 		goto overflow;
2653 	offset = (offset + wc->block_size - 1) & ~(size_t)(wc->block_size - 1);
2654 	data_size = wc->n_blocks * (size_t)wc->block_size;
2655 	if (!offset || (data_size / wc->block_size != wc->n_blocks) ||
2656 	    (offset + data_size < offset))
2657 		goto overflow;
2658 	if (offset + data_size > wc->memory_map_size) {
2659 		ti->error = "Memory area is too small";
2660 		r = -EINVAL;
2661 		goto bad;
2662 	}
2663 
2664 	wc->metadata_sectors = offset >> SECTOR_SHIFT;
2665 	wc->block_start = (char *)sb(wc) + offset;
2666 
2667 	x = (uint64_t)wc->n_blocks * (100 - high_wm_percent);
2668 	x += 50;
2669 	do_div(x, 100);
2670 	wc->freelist_high_watermark = x;
2671 	x = (uint64_t)wc->n_blocks * (100 - low_wm_percent);
2672 	x += 50;
2673 	do_div(x, 100);
2674 	wc->freelist_low_watermark = x;
2675 
2676 	if (wc->cleaner)
2677 		activate_cleaner(wc);
2678 
2679 	r = writecache_alloc_entries(wc);
2680 	if (r) {
2681 		ti->error = "Cannot allocate memory";
2682 		goto bad;
2683 	}
2684 
2685 	ti->num_flush_bios = WC_MODE_PMEM(wc) ? 1 : 2;
2686 	ti->flush_supported = true;
2687 	ti->num_discard_bios = 1;
2688 
2689 	if (WC_MODE_PMEM(wc))
2690 		persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
2691 
2692 	return 0;
2693 
2694 bad_arguments:
2695 	r = -EINVAL;
2696 	ti->error = "Bad arguments";
2697 bad:
2698 	writecache_dtr(ti);
2699 	return r;
2700 }
2701 
2702 static void writecache_status(struct dm_target *ti, status_type_t type,
2703 			      unsigned int status_flags, char *result, unsigned int maxlen)
2704 {
2705 	struct dm_writecache *wc = ti->private;
2706 	unsigned int extra_args;
2707 	unsigned int sz = 0;
2708 
2709 	switch (type) {
2710 	case STATUSTYPE_INFO:
2711 		DMEMIT("%ld %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu",
2712 		       writecache_has_error(wc),
2713 		       (unsigned long long)wc->n_blocks, (unsigned long long)wc->freelist_size,
2714 		       (unsigned long long)wc->writeback_size,
2715 		       wc->stats.reads,
2716 		       wc->stats.read_hits,
2717 		       wc->stats.writes,
2718 		       wc->stats.write_hits_uncommitted,
2719 		       wc->stats.write_hits_committed,
2720 		       wc->stats.writes_around,
2721 		       wc->stats.writes_allocate,
2722 		       wc->stats.writes_blocked_on_freelist,
2723 		       wc->stats.flushes,
2724 		       wc->stats.discards);
2725 		break;
2726 	case STATUSTYPE_TABLE:
2727 		DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's',
2728 				wc->dev->name, wc->ssd_dev->name, wc->block_size);
2729 		extra_args = 0;
2730 		if (wc->start_sector_set)
2731 			extra_args += 2;
2732 		if (wc->high_wm_percent_set)
2733 			extra_args += 2;
2734 		if (wc->low_wm_percent_set)
2735 			extra_args += 2;
2736 		if (wc->max_writeback_jobs_set)
2737 			extra_args += 2;
2738 		if (wc->autocommit_blocks_set)
2739 			extra_args += 2;
2740 		if (wc->autocommit_time_set)
2741 			extra_args += 2;
2742 		if (wc->max_age_set)
2743 			extra_args += 2;
2744 		if (wc->cleaner_set)
2745 			extra_args++;
2746 		if (wc->writeback_fua_set)
2747 			extra_args++;
2748 		if (wc->metadata_only)
2749 			extra_args++;
2750 		if (wc->pause_set)
2751 			extra_args += 2;
2752 
2753 		DMEMIT("%u", extra_args);
2754 		if (wc->start_sector_set)
2755 			DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector);
2756 		if (wc->high_wm_percent_set)
2757 			DMEMIT(" high_watermark %u", wc->high_wm_percent_value);
2758 		if (wc->low_wm_percent_set)
2759 			DMEMIT(" low_watermark %u", wc->low_wm_percent_value);
2760 		if (wc->max_writeback_jobs_set)
2761 			DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs);
2762 		if (wc->autocommit_blocks_set)
2763 			DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks);
2764 		if (wc->autocommit_time_set)
2765 			DMEMIT(" autocommit_time %u", wc->autocommit_time_value);
2766 		if (wc->max_age_set)
2767 			DMEMIT(" max_age %u", wc->max_age_value);
2768 		if (wc->cleaner_set)
2769 			DMEMIT(" cleaner");
2770 		if (wc->writeback_fua_set)
2771 			DMEMIT(" %sfua", wc->writeback_fua ? "" : "no");
2772 		if (wc->metadata_only)
2773 			DMEMIT(" metadata_only");
2774 		if (wc->pause_set)
2775 			DMEMIT(" pause_writeback %u", wc->pause_value);
2776 		break;
2777 	case STATUSTYPE_IMA:
2778 		*result = '\0';
2779 		break;
2780 	}
2781 }
2782 
2783 static struct target_type writecache_target = {
2784 	.name			= "writecache",
2785 	.version		= {1, 6, 0},
2786 	.module			= THIS_MODULE,
2787 	.ctr			= writecache_ctr,
2788 	.dtr			= writecache_dtr,
2789 	.status			= writecache_status,
2790 	.postsuspend		= writecache_suspend,
2791 	.resume			= writecache_resume,
2792 	.message		= writecache_message,
2793 	.map			= writecache_map,
2794 	.end_io			= writecache_end_io,
2795 	.iterate_devices	= writecache_iterate_devices,
2796 	.io_hints		= writecache_io_hints,
2797 };
2798 module_dm(writecache);
2799 
2800 MODULE_DESCRIPTION(DM_NAME " writecache target");
2801 MODULE_AUTHOR("Mikulas Patocka <dm-devel@lists.linux.dev>");
2802 MODULE_LICENSE("GPL");
2803