dm-writecache.c (6d7e0a34206d4a5d0c619c0608eae57f4c557064) dm-writecache.c (1edaa447d958bec24c6a79685a5790d98976fd16)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2018 Red Hat. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include <linux/device-mapper.h>

--- 12 unchanged lines hidden (view full) ---

21#define HIGH_WATERMARK 50
22#define LOW_WATERMARK 45
23#define MAX_WRITEBACK_JOBS 0
24#define ENDIO_LATENCY 16
25#define WRITEBACK_LATENCY 64
26#define AUTOCOMMIT_BLOCKS_SSD 65536
27#define AUTOCOMMIT_BLOCKS_PMEM 64
28#define AUTOCOMMIT_MSEC 1000
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2018 Red Hat. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include <linux/device-mapper.h>

--- 12 unchanged lines hidden (view full) ---

21#define HIGH_WATERMARK 50
22#define LOW_WATERMARK 45
23#define MAX_WRITEBACK_JOBS 0
24#define ENDIO_LATENCY 16
25#define WRITEBACK_LATENCY 64
26#define AUTOCOMMIT_BLOCKS_SSD 65536
27#define AUTOCOMMIT_BLOCKS_PMEM 64
28#define AUTOCOMMIT_MSEC 1000
29#define MAX_AGE_DIV 16
30#define MAX_AGE_UNSPECIFIED -1UL
29
30#define BITMAP_GRANULARITY 65536
31#if BITMAP_GRANULARITY < PAGE_SIZE
32#undef BITMAP_GRANULARITY
33#define BITMAP_GRANULARITY PAGE_SIZE
34#endif
35
36#if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_DAX_DRIVER)

--- 46 unchanged lines hidden (view full) ---

83 :1
84#endif
85 ;
86 unsigned long index
87#if BITS_PER_LONG == 64
88 :47
89#endif
90 ;
31
32#define BITMAP_GRANULARITY 65536
33#if BITMAP_GRANULARITY < PAGE_SIZE
34#undef BITMAP_GRANULARITY
35#define BITMAP_GRANULARITY PAGE_SIZE
36#endif
37
38#if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_DAX_DRIVER)

--- 46 unchanged lines hidden (view full) ---

85 :1
86#endif
87 ;
88 unsigned long index
89#if BITS_PER_LONG == 64
90 :47
91#endif
92 ;
93 unsigned long age;
91#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
92 uint64_t original_sector;
93 uint64_t seq_count;
94#endif
95};
96
97#ifdef DM_WRITECACHE_HAS_PMEM
98#define WC_MODE_PMEM(wc) ((wc)->pmem_mode)

--- 15 unchanged lines hidden (view full) ---

114 };
115 };
116 struct rb_root tree;
117
118 size_t freelist_size;
119 size_t writeback_size;
120 size_t freelist_high_watermark;
121 size_t freelist_low_watermark;
94#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
95 uint64_t original_sector;
96 uint64_t seq_count;
97#endif
98};
99
100#ifdef DM_WRITECACHE_HAS_PMEM
101#define WC_MODE_PMEM(wc) ((wc)->pmem_mode)

--- 15 unchanged lines hidden (view full) ---

117 };
118 };
119 struct rb_root tree;
120
121 size_t freelist_size;
122 size_t writeback_size;
123 size_t freelist_high_watermark;
124 size_t freelist_low_watermark;
125 unsigned long max_age;
122
123 unsigned uncommitted_blocks;
124 unsigned autocommit_blocks;
125 unsigned max_writeback_jobs;
126
127 int error;
128
129 unsigned long autocommit_jiffies;
130 struct timer_list autocommit_timer;
131 struct wait_queue_head freelist_wait;
132
126
127 unsigned uncommitted_blocks;
128 unsigned autocommit_blocks;
129 unsigned max_writeback_jobs;
130
131 int error;
132
133 unsigned long autocommit_jiffies;
134 struct timer_list autocommit_timer;
135 struct wait_queue_head freelist_wait;
136
137 struct timer_list max_age_timer;
138
133 atomic_t bio_in_progress[2];
134 struct wait_queue_head bio_in_progress_wait[2];
135
136 struct dm_target *ti;
137 struct dm_dev *dev;
138 struct dm_dev *ssd_dev;
139 sector_t start_sector;
140 void *memory_map;

--- 14 unchanged lines hidden (view full) ---

155
156 bool high_wm_percent_set:1;
157 bool low_wm_percent_set:1;
158 bool max_writeback_jobs_set:1;
159 bool autocommit_blocks_set:1;
160 bool autocommit_time_set:1;
161 bool writeback_fua_set:1;
162 bool flush_on_suspend:1;
139 atomic_t bio_in_progress[2];
140 struct wait_queue_head bio_in_progress_wait[2];
141
142 struct dm_target *ti;
143 struct dm_dev *dev;
144 struct dm_dev *ssd_dev;
145 sector_t start_sector;
146 void *memory_map;

--- 14 unchanged lines hidden (view full) ---

161
162 bool high_wm_percent_set:1;
163 bool low_wm_percent_set:1;
164 bool max_writeback_jobs_set:1;
165 bool autocommit_blocks_set:1;
166 bool autocommit_time_set:1;
167 bool writeback_fua_set:1;
168 bool flush_on_suspend:1;
169 bool cleaner:1;
163
164 unsigned writeback_all;
165 struct workqueue_struct *writeback_wq;
166 struct work_struct writeback_work;
167 struct work_struct flush_work;
168
169 struct dm_io_client *dm_io;
170

--- 326 unchanged lines hidden (view full) ---

497 if (wait_for_ios)
498 writecache_wait_for_ios(wc, WRITE);
499
500 writecache_disk_flush(wc, wc->ssd_dev);
501
502 memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size);
503}
504
170
171 unsigned writeback_all;
172 struct workqueue_struct *writeback_wq;
173 struct work_struct writeback_work;
174 struct work_struct flush_work;
175
176 struct dm_io_client *dm_io;
177

--- 326 unchanged lines hidden (view full) ---

504 if (wait_for_ios)
505 writecache_wait_for_ios(wc, WRITE);
506
507 writecache_disk_flush(wc, wc->ssd_dev);
508
509 memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size);
510}
511
512static void ssd_commit_superblock(struct dm_writecache *wc)
513{
514 int r;
515 struct dm_io_region region;
516 struct dm_io_request req;
517
518 region.bdev = wc->ssd_dev->bdev;
519 region.sector = 0;
520 region.count = PAGE_SIZE;
521
522 if (unlikely(region.sector + region.count > wc->metadata_sectors))
523 region.count = wc->metadata_sectors - region.sector;
524
525 region.sector += wc->start_sector;
526
527 req.bi_op = REQ_OP_WRITE;
528 req.bi_op_flags = REQ_SYNC | REQ_FUA;
529 req.mem.type = DM_IO_VMA;
530 req.mem.ptr.vma = (char *)wc->memory_map;
531 req.client = wc->dm_io;
532 req.notify.fn = NULL;
533 req.notify.context = NULL;
534
535 r = dm_io(&req, 1, &region, NULL);
536 if (unlikely(r))
537 writecache_error(wc, r, "error writing superblock");
538}
539
505static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
506{
507 if (WC_MODE_PMEM(wc))
508 wmb();
509 else
510 ssd_commit_flushed(wc, wait_for_ios);
511}
512

--- 78 unchanged lines hidden (view full) ---

591 if (read_original_sector(wc, e) > read_original_sector(wc, ins))
592 node = &parent->rb_left;
593 else
594 node = &parent->rb_right;
595 }
596 rb_link_node(&ins->rb_node, parent, node);
597 rb_insert_color(&ins->rb_node, &wc->tree);
598 list_add(&ins->lru, &wc->lru);
540static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
541{
542 if (WC_MODE_PMEM(wc))
543 wmb();
544 else
545 ssd_commit_flushed(wc, wait_for_ios);
546}
547

--- 78 unchanged lines hidden (view full) ---

626 if (read_original_sector(wc, e) > read_original_sector(wc, ins))
627 node = &parent->rb_left;
628 else
629 node = &parent->rb_right;
630 }
631 rb_link_node(&ins->rb_node, parent, node);
632 rb_insert_color(&ins->rb_node, &wc->tree);
633 list_add(&ins->lru, &wc->lru);
634 ins->age = jiffies;
599}
600
601static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
602{
603 list_del(&e->lru);
604 rb_erase(&e->rb_node, &wc->tree);
605}
606

--- 19 unchanged lines hidden (view full) ---

626}
627
628static inline void writecache_verify_watermark(struct dm_writecache *wc)
629{
630 if (unlikely(wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark))
631 queue_work(wc->writeback_wq, &wc->writeback_work);
632}
633
635}
636
637static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
638{
639 list_del(&e->lru);
640 rb_erase(&e->rb_node, &wc->tree);
641}
642

--- 19 unchanged lines hidden (view full) ---

662}
663
664static inline void writecache_verify_watermark(struct dm_writecache *wc)
665{
666 if (unlikely(wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark))
667 queue_work(wc->writeback_wq, &wc->writeback_work);
668}
669
670static void writecache_max_age_timer(struct timer_list *t)
671{
672 struct dm_writecache *wc = from_timer(wc, t, max_age_timer);
673
674 if (!dm_suspended(wc->ti) && !writecache_has_error(wc)) {
675 queue_work(wc->writeback_wq, &wc->writeback_work);
676 mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
677 }
678}
679
634static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector)
635{
636 struct wc_entry *e;
637
638 if (WC_MODE_SORT_FREELIST(wc)) {
639 struct rb_node *next;
640 if (unlikely(!wc->current_free))
641 return NULL;

--- 94 unchanged lines hidden (view full) ---

736 break;
737 e = e2;
738 cond_resched();
739 }
740 writecache_commit_flushed(wc, true);
741
742 wc->seq_count++;
743 pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count));
680static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector)
681{
682 struct wc_entry *e;
683
684 if (WC_MODE_SORT_FREELIST(wc)) {
685 struct rb_node *next;
686 if (unlikely(!wc->current_free))
687 return NULL;

--- 94 unchanged lines hidden (view full) ---

782 break;
783 e = e2;
784 cond_resched();
785 }
786 writecache_commit_flushed(wc, true);
787
788 wc->seq_count++;
789 pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count));
744 writecache_flush_region(wc, &sb(wc)->seq_count, sizeof sb(wc)->seq_count);
745 writecache_commit_flushed(wc, false);
790 if (WC_MODE_PMEM(wc))
791 writecache_commit_flushed(wc, false);
792 else
793 ssd_commit_superblock(wc);
746
747 wc->overwrote_committed = false;
748
749 need_flush_after_free = false;
750 while (1) {
751 /* Free another committed entry with lower seq-count */
752 struct rb_node *rb_node = rb_prev(&e->rb_node);
753

--- 78 unchanged lines hidden (view full) ---

832}
833
834static void writecache_suspend(struct dm_target *ti)
835{
836 struct dm_writecache *wc = ti->private;
837 bool flush_on_suspend;
838
839 del_timer_sync(&wc->autocommit_timer);
794
795 wc->overwrote_committed = false;
796
797 need_flush_after_free = false;
798 while (1) {
799 /* Free another committed entry with lower seq-count */
800 struct rb_node *rb_node = rb_prev(&e->rb_node);
801

--- 78 unchanged lines hidden (view full) ---

880}
881
882static void writecache_suspend(struct dm_target *ti)
883{
884 struct dm_writecache *wc = ti->private;
885 bool flush_on_suspend;
886
887 del_timer_sync(&wc->autocommit_timer);
888 del_timer_sync(&wc->max_age_timer);
840
841 wc_lock(wc);
842 writecache_flush(wc);
843 flush_on_suspend = wc->flush_on_suspend;
844 if (flush_on_suspend) {
845 wc->flush_on_suspend = false;
846 wc->writeback_all++;
847 queue_work(wc->writeback_wq, &wc->writeback_work);

--- 23 unchanged lines hidden (view full) ---

871 return 0;
872 wc->entries = vmalloc(array_size(sizeof(struct wc_entry), wc->n_blocks));
873 if (!wc->entries)
874 return -ENOMEM;
875 for (b = 0; b < wc->n_blocks; b++) {
876 struct wc_entry *e = &wc->entries[b];
877 e->index = b;
878 e->write_in_progress = false;
889
890 wc_lock(wc);
891 writecache_flush(wc);
892 flush_on_suspend = wc->flush_on_suspend;
893 if (flush_on_suspend) {
894 wc->flush_on_suspend = false;
895 wc->writeback_all++;
896 queue_work(wc->writeback_wq, &wc->writeback_work);

--- 23 unchanged lines hidden (view full) ---

920 return 0;
921 wc->entries = vmalloc(array_size(sizeof(struct wc_entry), wc->n_blocks));
922 if (!wc->entries)
923 return -ENOMEM;
924 for (b = 0; b < wc->n_blocks; b++) {
925 struct wc_entry *e = &wc->entries[b];
926 e->index = b;
927 e->write_in_progress = false;
928 cond_resched();
879 }
880
881 return 0;
882}
883
884static void writecache_resume(struct dm_target *ti)
885{
886 struct dm_writecache *wc = ti->private;

--- 38 unchanged lines hidden (view full) ---

925 writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d",
926 (unsigned long)b, r);
927 e->original_sector = -1;
928 e->seq_count = -1;
929 } else {
930 e->original_sector = le64_to_cpu(wme.original_sector);
931 e->seq_count = le64_to_cpu(wme.seq_count);
932 }
929 }
930
931 return 0;
932}
933
934static void writecache_resume(struct dm_target *ti)
935{
936 struct dm_writecache *wc = ti->private;

--- 38 unchanged lines hidden (view full) ---

975 writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d",
976 (unsigned long)b, r);
977 e->original_sector = -1;
978 e->seq_count = -1;
979 } else {
980 e->original_sector = le64_to_cpu(wme.original_sector);
981 e->seq_count = le64_to_cpu(wme.seq_count);
982 }
983 cond_resched();
933 }
934#endif
935 for (b = 0; b < wc->n_blocks; b++) {
936 struct wc_entry *e = &wc->entries[b];
937 if (!writecache_entry_is_committed(wc, e)) {
938 if (read_seq_count(wc, e) != -1) {
939erase_this:
940 clear_seq_count(wc, e);

--- 27 unchanged lines hidden (view full) ---

968
969 if (need_flush) {
970 writecache_flush_all_metadata(wc);
971 writecache_commit_flushed(wc, false);
972 }
973
974 writecache_verify_watermark(wc);
975
984 }
985#endif
986 for (b = 0; b < wc->n_blocks; b++) {
987 struct wc_entry *e = &wc->entries[b];
988 if (!writecache_entry_is_committed(wc, e)) {
989 if (read_seq_count(wc, e) != -1) {
990erase_this:
991 clear_seq_count(wc, e);

--- 27 unchanged lines hidden (view full) ---

1019
1020 if (need_flush) {
1021 writecache_flush_all_metadata(wc);
1022 writecache_commit_flushed(wc, false);
1023 }
1024
1025 writecache_verify_watermark(wc);
1026
1027 if (wc->max_age != MAX_AGE_UNSPECIFIED)
1028 mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
1029
976 wc_unlock(wc);
977}
978
979static int process_flush_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
980{
981 if (argc != 1)
982 return -EINVAL;
983

--- 32 unchanged lines hidden (view full) ---

1016
1017 wc_lock(wc);
1018 wc->flush_on_suspend = true;
1019 wc_unlock(wc);
1020
1021 return 0;
1022}
1023
1030 wc_unlock(wc);
1031}
1032
1033static int process_flush_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
1034{
1035 if (argc != 1)
1036 return -EINVAL;
1037

--- 32 unchanged lines hidden (view full) ---

1070
1071 wc_lock(wc);
1072 wc->flush_on_suspend = true;
1073 wc_unlock(wc);
1074
1075 return 0;
1076}
1077
1078static void activate_cleaner(struct dm_writecache *wc)
1079{
1080 wc->flush_on_suspend = true;
1081 wc->cleaner = true;
1082 wc->freelist_high_watermark = wc->n_blocks;
1083 wc->freelist_low_watermark = wc->n_blocks;
1084}
1085
1086static int process_cleaner_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
1087{
1088 if (argc != 1)
1089 return -EINVAL;
1090
1091 wc_lock(wc);
1092 activate_cleaner(wc);
1093 if (!dm_suspended(wc->ti))
1094 writecache_verify_watermark(wc);
1095 wc_unlock(wc);
1096
1097 return 0;
1098}
1099
1024static int writecache_message(struct dm_target *ti, unsigned argc, char **argv,
1025 char *result, unsigned maxlen)
1026{
1027 int r = -EINVAL;
1028 struct dm_writecache *wc = ti->private;
1029
1030 if (!strcasecmp(argv[0], "flush"))
1031 r = process_flush_mesg(argc, argv, wc);
1032 else if (!strcasecmp(argv[0], "flush_on_suspend"))
1033 r = process_flush_on_suspend_mesg(argc, argv, wc);
1100static int writecache_message(struct dm_target *ti, unsigned argc, char **argv,
1101 char *result, unsigned maxlen)
1102{
1103 int r = -EINVAL;
1104 struct dm_writecache *wc = ti->private;
1105
1106 if (!strcasecmp(argv[0], "flush"))
1107 r = process_flush_mesg(argc, argv, wc);
1108 else if (!strcasecmp(argv[0], "flush_on_suspend"))
1109 r = process_flush_on_suspend_mesg(argc, argv, wc);
1110 else if (!strcasecmp(argv[0], "cleaner"))
1111 r = process_cleaner_mesg(argc, argv, wc);
1034 else
1035 DMERR("unrecognised message received: %s", argv[0]);
1036
1037 return r;
1038}
1039
1040static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
1041{

--- 147 unchanged lines hidden (view full) ---

1189 if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) {
1190 dm_accept_partial_bio(bio, next_boundary);
1191 }
1192 }
1193 goto unlock_remap_origin;
1194 }
1195 } else {
1196 do {
1112 else
1113 DMERR("unrecognised message received: %s", argv[0]);
1114
1115 return r;
1116}
1117
1118static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
1119{

--- 147 unchanged lines hidden (view full) ---

1267 if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) {
1268 dm_accept_partial_bio(bio, next_boundary);
1269 }
1270 }
1271 goto unlock_remap_origin;
1272 }
1273 } else {
1274 do {
1275 bool found_entry = false;
1197 if (writecache_has_error(wc))
1198 goto unlock_error;
1199 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
1200 if (e) {
1201 if (!writecache_entry_is_committed(wc, e))
1202 goto bio_copy;
1203 if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
1204 wc->overwrote_committed = true;
1205 goto bio_copy;
1206 }
1276 if (writecache_has_error(wc))
1277 goto unlock_error;
1278 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
1279 if (e) {
1280 if (!writecache_entry_is_committed(wc, e))
1281 goto bio_copy;
1282 if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
1283 wc->overwrote_committed = true;
1284 goto bio_copy;
1285 }
1286 found_entry = true;
1287 } else {
1288 if (unlikely(wc->cleaner))
1289 goto direct_write;
1207 }
1208 e = writecache_pop_from_freelist(wc, (sector_t)-1);
1209 if (unlikely(!e)) {
1290 }
1291 e = writecache_pop_from_freelist(wc, (sector_t)-1);
1292 if (unlikely(!e)) {
1293 if (!found_entry) {
1294direct_write:
1295 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
1296 if (e) {
1297 sector_t next_boundary = read_original_sector(wc, e) - bio->bi_iter.bi_sector;
1298 BUG_ON(!next_boundary);
1299 if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) {
1300 dm_accept_partial_bio(bio, next_boundary);
1301 }
1302 }
1303 goto unlock_remap_origin;
1304 }
1210 writecache_wait_on_freelist(wc);
1211 continue;
1212 }
1213 write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
1214 writecache_insert_entry(wc, e);
1215 wc->uncommitted_blocks++;
1216bio_copy:
1217 if (WC_MODE_PMEM(wc)) {

--- 396 unchanged lines hidden (view full) ---

1614 }
1615
1616 n_walked = 0;
1617 INIT_LIST_HEAD(&skipped);
1618 INIT_LIST_HEAD(&wbl.list);
1619 wbl.size = 0;
1620 while (!list_empty(&wc->lru) &&
1621 (wc->writeback_all ||
1305 writecache_wait_on_freelist(wc);
1306 continue;
1307 }
1308 write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
1309 writecache_insert_entry(wc, e);
1310 wc->uncommitted_blocks++;
1311bio_copy:
1312 if (WC_MODE_PMEM(wc)) {

--- 396 unchanged lines hidden (view full) ---

1709 }
1710
1711 n_walked = 0;
1712 INIT_LIST_HEAD(&skipped);
1713 INIT_LIST_HEAD(&wbl.list);
1714 wbl.size = 0;
1715 while (!list_empty(&wc->lru) &&
1716 (wc->writeback_all ||
1622 wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark)) {
1717 wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark ||
1718 (jiffies - container_of(wc->lru.prev, struct wc_entry, lru)->age >=
1719 wc->max_age - wc->max_age / MAX_AGE_DIV))) {
1623
1624 n_walked++;
1625 if (unlikely(n_walked > WRITEBACK_LATENCY) &&
1626 likely(!wc->writeback_all) && likely(!dm_suspended(wc->ti))) {
1627 queue_work(wc->writeback_wq, &wc->writeback_work);
1628 break;
1629 }
1630

--- 155 unchanged lines hidden (view full) ---

1786
1787 for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++)
1788 pmem_assign(sb(wc)->padding[b], cpu_to_le64(0));
1789 pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION));
1790 pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size));
1791 pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks));
1792 pmem_assign(sb(wc)->seq_count, cpu_to_le64(0));
1793
1720
1721 n_walked++;
1722 if (unlikely(n_walked > WRITEBACK_LATENCY) &&
1723 likely(!wc->writeback_all) && likely(!dm_suspended(wc->ti))) {
1724 queue_work(wc->writeback_wq, &wc->writeback_work);
1725 break;
1726 }
1727

--- 155 unchanged lines hidden (view full) ---

1883
1884 for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++)
1885 pmem_assign(sb(wc)->padding[b], cpu_to_le64(0));
1886 pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION));
1887 pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size));
1888 pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks));
1889 pmem_assign(sb(wc)->seq_count, cpu_to_le64(0));
1890
1794 for (b = 0; b < wc->n_blocks; b++)
1891 for (b = 0; b < wc->n_blocks; b++) {
1795 write_original_sector_seq_count(wc, &wc->entries[b], -1, -1);
1892 write_original_sector_seq_count(wc, &wc->entries[b], -1, -1);
1893 cond_resched();
1894 }
1796
1797 writecache_flush_all_metadata(wc);
1798 writecache_commit_flushed(wc, false);
1799 pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC));
1800 writecache_flush_region(wc, &sb(wc)->magic, sizeof sb(wc)->magic);
1801 writecache_commit_flushed(wc, false);
1802
1803 return 0;

--- 73 unchanged lines hidden (view full) ---

1877 ti->error = "Cannot allocate writecache structure";
1878 r = -ENOMEM;
1879 goto bad;
1880 }
1881 ti->private = wc;
1882 wc->ti = ti;
1883
1884 mutex_init(&wc->lock);
1895
1896 writecache_flush_all_metadata(wc);
1897 writecache_commit_flushed(wc, false);
1898 pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC));
1899 writecache_flush_region(wc, &sb(wc)->magic, sizeof sb(wc)->magic);
1900 writecache_commit_flushed(wc, false);
1901
1902 return 0;

--- 73 unchanged lines hidden (view full) ---

1976 ti->error = "Cannot allocate writecache structure";
1977 r = -ENOMEM;
1978 goto bad;
1979 }
1980 ti->private = wc;
1981 wc->ti = ti;
1982
1983 mutex_init(&wc->lock);
1984 wc->max_age = MAX_AGE_UNSPECIFIED;
1885 writecache_poison_lists(wc);
1886 init_waitqueue_head(&wc->freelist_wait);
1887 timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0);
1985 writecache_poison_lists(wc);
1986 init_waitqueue_head(&wc->freelist_wait);
1987 timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0);
1988 timer_setup(&wc->max_age_timer, writecache_max_age_timer, 0);
1888
1889 for (i = 0; i < 2; i++) {
1890 atomic_set(&wc->bio_in_progress[i], 0);
1891 init_waitqueue_head(&wc->bio_in_progress_wait[i]);
1892 }
1893
1894 wc->dm_io = dm_io_client_create();
1895 if (IS_ERR(wc->dm_io)) {

--- 157 unchanged lines hidden (view full) ---

2053 unsigned autocommit_msecs;
2054 string = dm_shift_arg(&as), opt_params--;
2055 if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1)
2056 goto invalid_optional;
2057 if (autocommit_msecs > 3600000)
2058 goto invalid_optional;
2059 wc->autocommit_jiffies = msecs_to_jiffies(autocommit_msecs);
2060 wc->autocommit_time_set = true;
1989
1990 for (i = 0; i < 2; i++) {
1991 atomic_set(&wc->bio_in_progress[i], 0);
1992 init_waitqueue_head(&wc->bio_in_progress_wait[i]);
1993 }
1994
1995 wc->dm_io = dm_io_client_create();
1996 if (IS_ERR(wc->dm_io)) {

--- 157 unchanged lines hidden (view full) ---

2154 unsigned autocommit_msecs;
2155 string = dm_shift_arg(&as), opt_params--;
2156 if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1)
2157 goto invalid_optional;
2158 if (autocommit_msecs > 3600000)
2159 goto invalid_optional;
2160 wc->autocommit_jiffies = msecs_to_jiffies(autocommit_msecs);
2161 wc->autocommit_time_set = true;
2162 } else if (!strcasecmp(string, "max_age") && opt_params >= 1) {
2163 unsigned max_age_msecs;
2164 string = dm_shift_arg(&as), opt_params--;
2165 if (sscanf(string, "%u%c", &max_age_msecs, &dummy) != 1)
2166 goto invalid_optional;
2167 if (max_age_msecs > 86400000)
2168 goto invalid_optional;
2169 wc->max_age = msecs_to_jiffies(max_age_msecs);
2170 } else if (!strcasecmp(string, "cleaner")) {
2171 wc->cleaner = true;
2061 } else if (!strcasecmp(string, "fua")) {
2062 if (WC_MODE_PMEM(wc)) {
2063 wc->writeback_fua = true;
2064 wc->writeback_fua_set = true;
2065 } else goto invalid_optional;
2066 } else if (!strcasecmp(string, "nofua")) {
2067 if (WC_MODE_PMEM(wc)) {
2068 wc->writeback_fua = false;

--- 161 unchanged lines hidden (view full) ---

2230 x += 50;
2231 do_div(x, 100);
2232 wc->freelist_high_watermark = x;
2233 x = (uint64_t)wc->n_blocks * (100 - low_wm_percent);
2234 x += 50;
2235 do_div(x, 100);
2236 wc->freelist_low_watermark = x;
2237
2172 } else if (!strcasecmp(string, "fua")) {
2173 if (WC_MODE_PMEM(wc)) {
2174 wc->writeback_fua = true;
2175 wc->writeback_fua_set = true;
2176 } else goto invalid_optional;
2177 } else if (!strcasecmp(string, "nofua")) {
2178 if (WC_MODE_PMEM(wc)) {
2179 wc->writeback_fua = false;

--- 161 unchanged lines hidden (view full) ---

2341 x += 50;
2342 do_div(x, 100);
2343 wc->freelist_high_watermark = x;
2344 x = (uint64_t)wc->n_blocks * (100 - low_wm_percent);
2345 x += 50;
2346 do_div(x, 100);
2347 wc->freelist_low_watermark = x;
2348
2349 if (wc->cleaner)
2350 activate_cleaner(wc);
2351
2238 r = writecache_alloc_entries(wc);
2239 if (r) {
2240 ti->error = "Cannot allocate memory";
2241 goto bad;
2242 }
2243
2244 ti->num_flush_bios = 1;
2245 ti->flush_supported = true;

--- 27 unchanged lines hidden (view full) ---

2273 (unsigned long long)wc->writeback_size);
2274 break;
2275 case STATUSTYPE_TABLE:
2276 DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's',
2277 wc->dev->name, wc->ssd_dev->name, wc->block_size);
2278 extra_args = 0;
2279 if (wc->start_sector)
2280 extra_args += 2;
2352 r = writecache_alloc_entries(wc);
2353 if (r) {
2354 ti->error = "Cannot allocate memory";
2355 goto bad;
2356 }
2357
2358 ti->num_flush_bios = 1;
2359 ti->flush_supported = true;

--- 27 unchanged lines hidden (view full) ---

2387 (unsigned long long)wc->writeback_size);
2388 break;
2389 case STATUSTYPE_TABLE:
2390 DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's',
2391 wc->dev->name, wc->ssd_dev->name, wc->block_size);
2392 extra_args = 0;
2393 if (wc->start_sector)
2394 extra_args += 2;
2281 if (wc->high_wm_percent_set)
2395 if (wc->high_wm_percent_set && !wc->cleaner)
2282 extra_args += 2;
2396 extra_args += 2;
2283 if (wc->low_wm_percent_set)
2397 if (wc->low_wm_percent_set && !wc->cleaner)
2284 extra_args += 2;
2285 if (wc->max_writeback_jobs_set)
2286 extra_args += 2;
2287 if (wc->autocommit_blocks_set)
2288 extra_args += 2;
2289 if (wc->autocommit_time_set)
2290 extra_args += 2;
2398 extra_args += 2;
2399 if (wc->max_writeback_jobs_set)
2400 extra_args += 2;
2401 if (wc->autocommit_blocks_set)
2402 extra_args += 2;
2403 if (wc->autocommit_time_set)
2404 extra_args += 2;
2405 if (wc->cleaner)
2406 extra_args++;
2291 if (wc->writeback_fua_set)
2292 extra_args++;
2293
2294 DMEMIT("%u", extra_args);
2295 if (wc->start_sector)
2296 DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector);
2407 if (wc->writeback_fua_set)
2408 extra_args++;
2409
2410 DMEMIT("%u", extra_args);
2411 if (wc->start_sector)
2412 DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector);
2297 if (wc->high_wm_percent_set) {
2413 if (wc->high_wm_percent_set && !wc->cleaner) {
2298 x = (uint64_t)wc->freelist_high_watermark * 100;
2299 x += wc->n_blocks / 2;
2300 do_div(x, (size_t)wc->n_blocks);
2301 DMEMIT(" high_watermark %u", 100 - (unsigned)x);
2302 }
2414 x = (uint64_t)wc->freelist_high_watermark * 100;
2415 x += wc->n_blocks / 2;
2416 do_div(x, (size_t)wc->n_blocks);
2417 DMEMIT(" high_watermark %u", 100 - (unsigned)x);
2418 }
2303 if (wc->low_wm_percent_set) {
2419 if (wc->low_wm_percent_set && !wc->cleaner) {
2304 x = (uint64_t)wc->freelist_low_watermark * 100;
2305 x += wc->n_blocks / 2;
2306 do_div(x, (size_t)wc->n_blocks);
2307 DMEMIT(" low_watermark %u", 100 - (unsigned)x);
2308 }
2309 if (wc->max_writeback_jobs_set)
2310 DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs);
2311 if (wc->autocommit_blocks_set)
2312 DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks);
2313 if (wc->autocommit_time_set)
2314 DMEMIT(" autocommit_time %u", jiffies_to_msecs(wc->autocommit_jiffies));
2420 x = (uint64_t)wc->freelist_low_watermark * 100;
2421 x += wc->n_blocks / 2;
2422 do_div(x, (size_t)wc->n_blocks);
2423 DMEMIT(" low_watermark %u", 100 - (unsigned)x);
2424 }
2425 if (wc->max_writeback_jobs_set)
2426 DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs);
2427 if (wc->autocommit_blocks_set)
2428 DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks);
2429 if (wc->autocommit_time_set)
2430 DMEMIT(" autocommit_time %u", jiffies_to_msecs(wc->autocommit_jiffies));
2431 if (wc->max_age != MAX_AGE_UNSPECIFIED)
2432 DMEMIT(" max_age %u", jiffies_to_msecs(wc->max_age));
2433 if (wc->cleaner)
2434 DMEMIT(" cleaner");
2315 if (wc->writeback_fua_set)
2316 DMEMIT(" %sfua", wc->writeback_fua ? "" : "no");
2317 break;
2318 }
2319}
2320
2321static struct target_type writecache_target = {
2322 .name = "writecache",
2435 if (wc->writeback_fua_set)
2436 DMEMIT(" %sfua", wc->writeback_fua ? "" : "no");
2437 break;
2438 }
2439}
2440
2441static struct target_type writecache_target = {
2442 .name = "writecache",
2323 .version = {1, 2, 0},
2443 .version = {1, 3, 0},
2324 .module = THIS_MODULE,
2325 .ctr = writecache_ctr,
2326 .dtr = writecache_dtr,
2327 .status = writecache_status,
2328 .postsuspend = writecache_suspend,
2329 .resume = writecache_resume,
2330 .message = writecache_message,
2331 .map = writecache_map,

--- 29 unchanged lines hidden --- 		2444 .module = THIS_MODULE,
2445 .ctr = writecache_ctr,
2446 .dtr = writecache_dtr,
2447 .status = writecache_status,
2448 .postsuspend = writecache_suspend,
2449 .resume = writecache_resume,
2450 .message = writecache_message,
2451 .map = writecache_map,

--- 29 unchanged lines hidden ---