1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018 Red Hat. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include <linux/device-mapper.h> 9 #include <linux/module.h> 10 #include <linux/init.h> 11 #include <linux/vmalloc.h> 12 #include <linux/kthread.h> 13 #include <linux/dm-io.h> 14 #include <linux/dm-kcopyd.h> 15 #include <linux/dax.h> 16 #include <linux/libnvdimm.h> 17 #include <linux/delay.h> 18 #include "dm-io-tracker.h" 19 20 #define DM_MSG_PREFIX "writecache" 21 22 #define HIGH_WATERMARK 50 23 #define LOW_WATERMARK 45 24 #define MAX_WRITEBACK_JOBS min(0x10000000 / PAGE_SIZE, totalram_pages() / 16) 25 #define ENDIO_LATENCY 16 26 #define WRITEBACK_LATENCY 64 27 #define AUTOCOMMIT_BLOCKS_SSD 65536 28 #define AUTOCOMMIT_BLOCKS_PMEM 64 29 #define AUTOCOMMIT_MSEC 1000 30 #define MAX_AGE_DIV 16 31 #define MAX_AGE_UNSPECIFIED -1UL 32 #define PAUSE_WRITEBACK (HZ * 3) 33 34 #define BITMAP_GRANULARITY 65536 35 #if BITMAP_GRANULARITY < PAGE_SIZE 36 #undef BITMAP_GRANULARITY 37 #define BITMAP_GRANULARITY PAGE_SIZE 38 #endif 39 40 #if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_FS_DAX) 41 #define DM_WRITECACHE_HAS_PMEM 42 #endif 43 44 #ifdef DM_WRITECACHE_HAS_PMEM 45 #define pmem_assign(dest, src) \ 46 do { \ 47 typeof(dest) uniq = (src); \ 48 memcpy_flushcache(&(dest), &uniq, sizeof(dest)); \ 49 } while (0) 50 #else 51 #define pmem_assign(dest, src) ((dest) = (src)) 52 #endif 53 54 #if IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC) && defined(DM_WRITECACHE_HAS_PMEM) 55 #define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 56 #endif 57 58 #define MEMORY_SUPERBLOCK_MAGIC 0x23489321 59 #define MEMORY_SUPERBLOCK_VERSION 1 60 61 struct wc_memory_entry { 62 __le64 original_sector; 63 __le64 seq_count; 64 }; 65 66 struct wc_memory_superblock { 67 union { 68 struct { 69 __le32 magic; 70 __le32 version; 71 __le32 block_size; 72 __le32 pad; 73 __le64 n_blocks; 74 __le64 seq_count; 75 }; 76 __le64 padding[8]; 77 }; 78 struct wc_memory_entry entries[]; 79 }; 80 81 struct wc_entry { 82 struct rb_node rb_node; 83 struct list_head lru; 84 u32 age; // jiffies 85 unsigned short wc_list_contiguous; 86 bool write_in_progress; 87 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 88 uint64_t original_sector; 89 uint64_t seq_count; 90 #endif 91 }; 92 93 #ifdef DM_WRITECACHE_HAS_PMEM 94 #define WC_MODE_PMEM(wc) ((wc)->pmem_mode) 95 #define WC_MODE_FUA(wc) ((wc)->writeback_fua) 96 #else 97 #define WC_MODE_PMEM(wc) false 98 #define WC_MODE_FUA(wc) false 99 #endif 100 #define WC_MODE_SORT_FREELIST(wc) (!WC_MODE_PMEM(wc)) 101 102 struct dm_writecache { 103 struct mutex lock; 104 struct list_head lru; 105 union { 106 struct list_head freelist; 107 struct { 108 struct rb_root freetree; 109 struct wc_entry *current_free; 110 }; 111 }; 112 struct rb_root tree; 113 114 size_t freelist_size; 115 size_t writeback_size; 116 size_t freelist_high_watermark; 117 size_t freelist_low_watermark; 118 unsigned long max_age; 119 unsigned long pause; 120 121 unsigned int uncommitted_blocks; 122 unsigned int autocommit_blocks; 123 unsigned int max_writeback_jobs; 124 125 int error; 126 127 unsigned long autocommit_jiffies; 128 struct timer_list autocommit_timer; 129 struct wait_queue_head freelist_wait; 130 131 struct timer_list max_age_timer; 132 133 atomic_t bio_in_progress[2]; 134 struct wait_queue_head bio_in_progress_wait[2]; 135 136 struct dm_target *ti; 137 struct dm_dev *dev; 138 struct dm_dev *ssd_dev; 139 sector_t start_sector; 140 void *memory_map; 141 uint64_t memory_map_size; 142 size_t metadata_sectors; 143 size_t n_blocks; 144 uint64_t seq_count; 145 sector_t data_device_sectors; 146 void *block_start; 147 struct wc_entry *entries; 148 unsigned int block_size; 149 unsigned char block_size_bits; 150 151 bool pmem_mode:1; 152 bool writeback_fua:1; 153 154 bool overwrote_committed:1; 155 bool memory_vmapped:1; 156 157 bool start_sector_set:1; 158 bool high_wm_percent_set:1; 159 bool low_wm_percent_set:1; 160 bool max_writeback_jobs_set:1; 161 bool autocommit_blocks_set:1; 162 bool autocommit_time_set:1; 163 bool max_age_set:1; 164 bool writeback_fua_set:1; 165 bool flush_on_suspend:1; 166 bool cleaner:1; 167 bool cleaner_set:1; 168 bool metadata_only:1; 169 bool pause_set:1; 170 171 unsigned int high_wm_percent_value; 172 unsigned int low_wm_percent_value; 173 unsigned int autocommit_time_value; 174 unsigned int max_age_value; 175 unsigned int pause_value; 176 177 unsigned int writeback_all; 178 struct workqueue_struct *writeback_wq; 179 struct work_struct writeback_work; 180 struct work_struct flush_work; 181 182 struct dm_io_tracker iot; 183 184 struct dm_io_client *dm_io; 185 186 raw_spinlock_t endio_list_lock; 187 struct list_head endio_list; 188 struct task_struct *endio_thread; 189 190 struct task_struct *flush_thread; 191 spinlock_t flush_list_lock; 192 struct bio_list flush_list; 193 194 struct dm_kcopyd_client *dm_kcopyd; 195 unsigned long *dirty_bitmap; 196 unsigned int dirty_bitmap_size; 197 198 struct bio_set bio_set; 199 mempool_t copy_pool; 200 201 struct { 202 unsigned long long reads; 203 unsigned long long read_hits; 204 unsigned long long writes; 205 unsigned long long write_hits_uncommitted; 206 unsigned long long write_hits_committed; 207 unsigned long long writes_around; 208 unsigned long long writes_allocate; 209 unsigned long long writes_blocked_on_freelist; 210 unsigned long long flushes; 211 unsigned long long discards; 212 } stats; 213 }; 214 215 #define WB_LIST_INLINE 16 216 217 struct writeback_struct { 218 struct list_head endio_entry; 219 struct dm_writecache *wc; 220 struct wc_entry **wc_list; 221 unsigned int wc_list_n; 222 struct wc_entry *wc_list_inline[WB_LIST_INLINE]; 223 struct bio bio; 224 }; 225 226 struct copy_struct { 227 struct list_head endio_entry; 228 struct dm_writecache *wc; 229 struct wc_entry *e; 230 unsigned int n_entries; 231 int error; 232 }; 233 234 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle, 235 "A percentage of time allocated for data copying"); 236 237 static void wc_lock(struct dm_writecache *wc) 238 { 239 mutex_lock(&wc->lock); 240 } 241 242 static void wc_unlock(struct dm_writecache *wc) 243 { 244 mutex_unlock(&wc->lock); 245 } 246 247 #ifdef DM_WRITECACHE_HAS_PMEM 248 static int persistent_memory_claim(struct dm_writecache *wc) 249 { 250 int r; 251 loff_t s; 252 long p, da; 253 unsigned long pfn; 254 int id; 255 struct page **pages; 256 sector_t offset; 257 258 wc->memory_vmapped = false; 259 260 s = wc->memory_map_size; 261 p = s >> PAGE_SHIFT; 262 if (!p) { 263 r = -EINVAL; 264 goto err1; 265 } 266 if (p != s >> PAGE_SHIFT) { 267 r = -EOVERFLOW; 268 goto err1; 269 } 270 271 offset = get_start_sect(wc->ssd_dev->bdev); 272 if (offset & (PAGE_SIZE / 512 - 1)) { 273 r = -EINVAL; 274 goto err1; 275 } 276 offset >>= PAGE_SHIFT - 9; 277 278 id = dax_read_lock(); 279 280 da = dax_direct_access(wc->ssd_dev->dax_dev, offset, p, DAX_ACCESS, 281 &wc->memory_map, &pfn); 282 if (da < 0) { 283 wc->memory_map = NULL; 284 r = da; 285 goto err2; 286 } 287 if (!pfn_valid(pfn)) { 288 wc->memory_map = NULL; 289 r = -EOPNOTSUPP; 290 goto err2; 291 } 292 if (da != p) { 293 long i; 294 295 wc->memory_map = NULL; 296 pages = vmalloc_array(p, sizeof(struct page *)); 297 if (!pages) { 298 r = -ENOMEM; 299 goto err2; 300 } 301 i = 0; 302 do { 303 long daa; 304 305 daa = dax_direct_access(wc->ssd_dev->dax_dev, offset + i, 306 p - i, DAX_ACCESS, NULL, &pfn); 307 if (daa <= 0) { 308 r = daa ? daa : -EINVAL; 309 goto err3; 310 } 311 if (!pfn_valid(pfn)) { 312 r = -EOPNOTSUPP; 313 goto err3; 314 } 315 while (daa-- && i < p) { 316 pages[i++] = pfn_to_page(pfn); 317 pfn++; 318 if (!(i & 15)) 319 cond_resched(); 320 } 321 } while (i < p); 322 wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL); 323 if (!wc->memory_map) { 324 r = -ENOMEM; 325 goto err3; 326 } 327 vfree(pages); 328 wc->memory_vmapped = true; 329 } 330 331 dax_read_unlock(id); 332 333 wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT; 334 wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT; 335 336 return 0; 337 err3: 338 vfree(pages); 339 err2: 340 dax_read_unlock(id); 341 err1: 342 return r; 343 } 344 #else 345 static int persistent_memory_claim(struct dm_writecache *wc) 346 { 347 return -EOPNOTSUPP; 348 } 349 #endif 350 351 static void persistent_memory_release(struct dm_writecache *wc) 352 { 353 if (wc->memory_vmapped) 354 vunmap(wc->memory_map - ((size_t)wc->start_sector << SECTOR_SHIFT)); 355 } 356 357 static struct page *persistent_memory_page(void *addr) 358 { 359 if (is_vmalloc_addr(addr)) 360 return vmalloc_to_page(addr); 361 else 362 return virt_to_page(addr); 363 } 364 365 static unsigned int persistent_memory_page_offset(void *addr) 366 { 367 return (unsigned long)addr & (PAGE_SIZE - 1); 368 } 369 370 static void persistent_memory_flush_cache(void *ptr, size_t size) 371 { 372 if (is_vmalloc_addr(ptr)) 373 flush_kernel_vmap_range(ptr, size); 374 } 375 376 static void persistent_memory_invalidate_cache(void *ptr, size_t size) 377 { 378 if (is_vmalloc_addr(ptr)) 379 invalidate_kernel_vmap_range(ptr, size); 380 } 381 382 static struct wc_memory_superblock *sb(struct dm_writecache *wc) 383 { 384 return wc->memory_map; 385 } 386 387 static inline unsigned long wc_entry_index(const struct wc_entry *base, const struct wc_entry *entry) 388 { 389 return entry - base; 390 } 391 392 static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e) 393 { 394 const unsigned long index = wc_entry_index(wc->entries, e); 395 return &sb(wc)->entries[index]; 396 } 397 398 static void *memory_data(struct dm_writecache *wc, struct wc_entry *e) 399 { 400 const unsigned long index = wc_entry_index(wc->entries, e); 401 return (char *)wc->block_start + (index << wc->block_size_bits); 402 } 403 404 static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e) 405 { 406 const unsigned long index = wc_entry_index(wc->entries, e); 407 return wc->start_sector + wc->metadata_sectors + 408 ((sector_t)index << (wc->block_size_bits - SECTOR_SHIFT)); 409 } 410 411 static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e) 412 { 413 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 414 return e->original_sector; 415 #else 416 return le64_to_cpu(memory_entry(wc, e)->original_sector); 417 #endif 418 } 419 420 static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e) 421 { 422 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 423 return e->seq_count; 424 #else 425 return le64_to_cpu(memory_entry(wc, e)->seq_count); 426 #endif 427 } 428 429 static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e) 430 { 431 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 432 e->seq_count = -1; 433 #endif 434 pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1)); 435 } 436 437 static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e, 438 uint64_t original_sector, uint64_t seq_count) 439 { 440 struct wc_memory_entry me; 441 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 442 e->original_sector = original_sector; 443 e->seq_count = seq_count; 444 #endif 445 me.original_sector = cpu_to_le64(original_sector); 446 me.seq_count = cpu_to_le64(seq_count); 447 pmem_assign(*memory_entry(wc, e), me); 448 } 449 450 #define writecache_error(wc, err, msg, arg...) \ 451 do { \ 452 if (!cmpxchg(&(wc)->error, 0, err)) \ 453 DMERR(msg, ##arg); \ 454 wake_up(&(wc)->freelist_wait); \ 455 } while (0) 456 457 #define writecache_has_error(wc) (unlikely(READ_ONCE((wc)->error))) 458 459 static void writecache_flush_all_metadata(struct dm_writecache *wc) 460 { 461 if (!WC_MODE_PMEM(wc)) 462 memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size); 463 } 464 465 static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size) 466 { 467 if (!WC_MODE_PMEM(wc)) 468 __set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY, 469 wc->dirty_bitmap); 470 } 471 472 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev); 473 474 struct io_notify { 475 struct dm_writecache *wc; 476 struct completion c; 477 atomic_t count; 478 }; 479 480 static void writecache_notify_io(unsigned long error, unsigned long unsup, void *context) 481 { 482 struct io_notify *endio = context; 483 484 if (unlikely(error != 0)) 485 writecache_error(endio->wc, -EIO, "error writing metadata"); 486 else if (unlikely(unsup != 0)) 487 writecache_error(endio->wc, -EOPNOTSUPP, "error writing metadata"); 488 BUG_ON(atomic_read(&endio->count) <= 0); 489 if (atomic_dec_and_test(&endio->count)) 490 complete(&endio->c); 491 } 492 493 static void writecache_wait_for_ios(struct dm_writecache *wc, int direction) 494 { 495 wait_event(wc->bio_in_progress_wait[direction], 496 !atomic_read(&wc->bio_in_progress[direction])); 497 } 498 499 static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) 500 { 501 struct dm_io_region region; 502 struct dm_io_request req; 503 struct io_notify endio = { 504 wc, 505 COMPLETION_INITIALIZER_ONSTACK(endio.c), 506 ATOMIC_INIT(1), 507 }; 508 unsigned int bitmap_bits = wc->dirty_bitmap_size * 8; 509 unsigned int i = 0; 510 511 while (1) { 512 unsigned int j; 513 514 i = find_next_bit(wc->dirty_bitmap, bitmap_bits, i); 515 if (unlikely(i == bitmap_bits)) 516 break; 517 j = find_next_zero_bit(wc->dirty_bitmap, bitmap_bits, i); 518 519 region.bdev = wc->ssd_dev->bdev; 520 region.sector = (sector_t)i * (BITMAP_GRANULARITY >> SECTOR_SHIFT); 521 region.count = (sector_t)(j - i) * (BITMAP_GRANULARITY >> SECTOR_SHIFT); 522 523 if (unlikely(region.sector >= wc->metadata_sectors)) 524 break; 525 if (unlikely(region.sector + region.count > wc->metadata_sectors)) 526 region.count = wc->metadata_sectors - region.sector; 527 528 region.sector += wc->start_sector; 529 atomic_inc(&endio.count); 530 req.bi_opf = REQ_OP_WRITE | REQ_SYNC; 531 req.mem.type = DM_IO_VMA; 532 req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY; 533 req.client = wc->dm_io; 534 req.notify.fn = writecache_notify_io; 535 req.notify.context = &endio; 536 537 /* writing via async dm-io (implied by notify.fn above) won't return an error */ 538 (void) dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); 539 i = j; 540 } 541 542 writecache_notify_io(0, 0, &endio); 543 wait_for_completion_io(&endio.c); 544 545 if (wait_for_ios) 546 writecache_wait_for_ios(wc, WRITE); 547 548 writecache_disk_flush(wc, wc->ssd_dev); 549 550 memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size); 551 } 552 553 static void ssd_commit_superblock(struct dm_writecache *wc) 554 { 555 int r; 556 struct dm_io_region region; 557 struct dm_io_request req; 558 559 region.bdev = wc->ssd_dev->bdev; 560 region.sector = 0; 561 region.count = max(4096U, wc->block_size) >> SECTOR_SHIFT; 562 563 if (unlikely(region.sector + region.count > wc->metadata_sectors)) 564 region.count = wc->metadata_sectors - region.sector; 565 566 region.sector += wc->start_sector; 567 568 req.bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_FUA; 569 req.mem.type = DM_IO_VMA; 570 req.mem.ptr.vma = (char *)wc->memory_map; 571 req.client = wc->dm_io; 572 req.notify.fn = NULL; 573 req.notify.context = NULL; 574 575 r = dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); 576 if (unlikely(r)) 577 writecache_error(wc, r, "error writing superblock"); 578 } 579 580 static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) 581 { 582 if (WC_MODE_PMEM(wc)) 583 pmem_wmb(); 584 else 585 ssd_commit_flushed(wc, wait_for_ios); 586 } 587 588 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev) 589 { 590 int r; 591 struct dm_io_region region; 592 struct dm_io_request req; 593 594 region.bdev = dev->bdev; 595 region.sector = 0; 596 region.count = 0; 597 req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; 598 req.mem.type = DM_IO_KMEM; 599 req.mem.ptr.addr = NULL; 600 req.client = wc->dm_io; 601 req.notify.fn = NULL; 602 603 r = dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); 604 if (unlikely(r)) 605 writecache_error(wc, r, "error flushing metadata: %d", r); 606 } 607 608 #define WFE_RETURN_FOLLOWING 1 609 #define WFE_LOWEST_SEQ 2 610 611 static struct wc_entry *writecache_find_entry(struct dm_writecache *wc, 612 uint64_t block, int flags) 613 { 614 struct wc_entry *e; 615 struct rb_node *node = wc->tree.rb_node; 616 617 if (unlikely(!node)) 618 return NULL; 619 620 while (1) { 621 e = container_of(node, struct wc_entry, rb_node); 622 if (read_original_sector(wc, e) == block) 623 break; 624 625 node = (read_original_sector(wc, e) >= block ? 626 e->rb_node.rb_left : e->rb_node.rb_right); 627 if (unlikely(!node)) { 628 if (!(flags & WFE_RETURN_FOLLOWING)) 629 return NULL; 630 if (read_original_sector(wc, e) >= block) 631 return e; 632 633 node = rb_next(&e->rb_node); 634 if (unlikely(!node)) 635 return NULL; 636 637 e = container_of(node, struct wc_entry, rb_node); 638 return e; 639 } 640 } 641 642 while (1) { 643 struct wc_entry *e2; 644 645 if (flags & WFE_LOWEST_SEQ) 646 node = rb_prev(&e->rb_node); 647 else 648 node = rb_next(&e->rb_node); 649 if (unlikely(!node)) 650 return e; 651 e2 = container_of(node, struct wc_entry, rb_node); 652 if (read_original_sector(wc, e2) != block) 653 return e; 654 e = e2; 655 } 656 } 657 658 static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *ins) 659 { 660 struct wc_entry *e; 661 struct rb_node **node = &wc->tree.rb_node, *parent = NULL; 662 663 while (*node) { 664 e = container_of(*node, struct wc_entry, rb_node); 665 parent = &e->rb_node; 666 if (read_original_sector(wc, e) > read_original_sector(wc, ins)) 667 node = &parent->rb_left; 668 else 669 node = &parent->rb_right; 670 } 671 rb_link_node(&ins->rb_node, parent, node); 672 rb_insert_color(&ins->rb_node, &wc->tree); 673 list_add(&ins->lru, &wc->lru); 674 ins->age = (u32)jiffies; 675 } 676 677 static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e) 678 { 679 list_del(&e->lru); 680 rb_erase(&e->rb_node, &wc->tree); 681 } 682 683 static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e) 684 { 685 if (WC_MODE_SORT_FREELIST(wc)) { 686 struct rb_node **node = &wc->freetree.rb_node, *parent = NULL; 687 688 if (unlikely(!*node)) 689 wc->current_free = e; 690 while (*node) { 691 parent = *node; 692 if (&e->rb_node < *node) 693 node = &parent->rb_left; 694 else 695 node = &parent->rb_right; 696 } 697 rb_link_node(&e->rb_node, parent, node); 698 rb_insert_color(&e->rb_node, &wc->freetree); 699 } else { 700 list_add_tail(&e->lru, &wc->freelist); 701 } 702 wc->freelist_size++; 703 } 704 705 static inline void writecache_verify_watermark(struct dm_writecache *wc) 706 { 707 if (unlikely(wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark)) 708 queue_work(wc->writeback_wq, &wc->writeback_work); 709 } 710 711 static void writecache_max_age_timer(struct timer_list *t) 712 { 713 struct dm_writecache *wc = timer_container_of(wc, t, max_age_timer); 714 715 if (!dm_suspended(wc->ti) && !writecache_has_error(wc)) { 716 queue_work(wc->writeback_wq, &wc->writeback_work); 717 mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV); 718 } 719 } 720 721 static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector) 722 { 723 struct wc_entry *e; 724 725 if (WC_MODE_SORT_FREELIST(wc)) { 726 struct rb_node *next; 727 728 if (unlikely(!wc->current_free)) 729 return NULL; 730 e = wc->current_free; 731 if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector)) 732 return NULL; 733 next = rb_next(&e->rb_node); 734 rb_erase(&e->rb_node, &wc->freetree); 735 if (unlikely(!next)) 736 next = rb_first(&wc->freetree); 737 wc->current_free = next ? container_of(next, struct wc_entry, rb_node) : NULL; 738 } else { 739 if (unlikely(list_empty(&wc->freelist))) 740 return NULL; 741 e = container_of(wc->freelist.next, struct wc_entry, lru); 742 if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector)) 743 return NULL; 744 list_del(&e->lru); 745 } 746 wc->freelist_size--; 747 748 writecache_verify_watermark(wc); 749 750 return e; 751 } 752 753 static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e) 754 { 755 writecache_unlink(wc, e); 756 writecache_add_to_freelist(wc, e); 757 clear_seq_count(wc, e); 758 writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry)); 759 if (unlikely(waitqueue_active(&wc->freelist_wait))) 760 wake_up(&wc->freelist_wait); 761 } 762 763 static void writecache_wait_on_freelist(struct dm_writecache *wc) 764 { 765 DEFINE_WAIT(wait); 766 767 prepare_to_wait(&wc->freelist_wait, &wait, TASK_UNINTERRUPTIBLE); 768 wc_unlock(wc); 769 io_schedule(); 770 finish_wait(&wc->freelist_wait, &wait); 771 wc_lock(wc); 772 } 773 774 static void writecache_poison_lists(struct dm_writecache *wc) 775 { 776 /* 777 * Catch incorrect access to these values while the device is suspended. 778 */ 779 memset(&wc->tree, -1, sizeof(wc->tree)); 780 wc->lru.next = LIST_POISON1; 781 wc->lru.prev = LIST_POISON2; 782 wc->freelist.next = LIST_POISON1; 783 wc->freelist.prev = LIST_POISON2; 784 } 785 786 static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e) 787 { 788 writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry)); 789 if (WC_MODE_PMEM(wc)) 790 writecache_flush_region(wc, memory_data(wc, e), wc->block_size); 791 } 792 793 static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e) 794 { 795 return read_seq_count(wc, e) < wc->seq_count; 796 } 797 798 static void writecache_flush(struct dm_writecache *wc) 799 { 800 struct wc_entry *e, *e2; 801 bool need_flush_after_free; 802 803 wc->uncommitted_blocks = 0; 804 timer_delete(&wc->autocommit_timer); 805 806 if (list_empty(&wc->lru)) 807 return; 808 809 e = container_of(wc->lru.next, struct wc_entry, lru); 810 if (writecache_entry_is_committed(wc, e)) { 811 if (wc->overwrote_committed) { 812 writecache_wait_for_ios(wc, WRITE); 813 writecache_disk_flush(wc, wc->ssd_dev); 814 wc->overwrote_committed = false; 815 } 816 return; 817 } 818 while (1) { 819 writecache_flush_entry(wc, e); 820 if (unlikely(e->lru.next == &wc->lru)) 821 break; 822 e2 = container_of(e->lru.next, struct wc_entry, lru); 823 if (writecache_entry_is_committed(wc, e2)) 824 break; 825 e = e2; 826 cond_resched(); 827 } 828 writecache_commit_flushed(wc, true); 829 830 wc->seq_count++; 831 pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count)); 832 if (WC_MODE_PMEM(wc)) 833 writecache_commit_flushed(wc, false); 834 else 835 ssd_commit_superblock(wc); 836 837 wc->overwrote_committed = false; 838 839 need_flush_after_free = false; 840 while (1) { 841 /* Free another committed entry with lower seq-count */ 842 struct rb_node *rb_node = rb_prev(&e->rb_node); 843 844 if (rb_node) { 845 e2 = container_of(rb_node, struct wc_entry, rb_node); 846 if (read_original_sector(wc, e2) == read_original_sector(wc, e) && 847 likely(!e2->write_in_progress)) { 848 writecache_free_entry(wc, e2); 849 need_flush_after_free = true; 850 } 851 } 852 if (unlikely(e->lru.prev == &wc->lru)) 853 break; 854 e = container_of(e->lru.prev, struct wc_entry, lru); 855 cond_resched(); 856 } 857 858 if (need_flush_after_free) 859 writecache_commit_flushed(wc, false); 860 } 861 862 static void writecache_flush_work(struct work_struct *work) 863 { 864 struct dm_writecache *wc = container_of(work, struct dm_writecache, flush_work); 865 866 wc_lock(wc); 867 writecache_flush(wc); 868 wc_unlock(wc); 869 } 870 871 static void writecache_autocommit_timer(struct timer_list *t) 872 { 873 struct dm_writecache *wc = timer_container_of(wc, t, autocommit_timer); 874 875 if (!writecache_has_error(wc)) 876 queue_work(wc->writeback_wq, &wc->flush_work); 877 } 878 879 static void writecache_schedule_autocommit(struct dm_writecache *wc) 880 { 881 if (!timer_pending(&wc->autocommit_timer)) 882 mod_timer(&wc->autocommit_timer, jiffies + wc->autocommit_jiffies); 883 } 884 885 static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_t end) 886 { 887 struct wc_entry *e; 888 bool discarded_something = false; 889 890 e = writecache_find_entry(wc, start, WFE_RETURN_FOLLOWING | WFE_LOWEST_SEQ); 891 if (unlikely(!e)) 892 return; 893 894 while (read_original_sector(wc, e) < end) { 895 struct rb_node *node = rb_next(&e->rb_node); 896 897 if (likely(!e->write_in_progress)) { 898 if (!discarded_something) { 899 if (!WC_MODE_PMEM(wc)) { 900 writecache_wait_for_ios(wc, READ); 901 writecache_wait_for_ios(wc, WRITE); 902 } 903 discarded_something = true; 904 } 905 writecache_free_entry(wc, e); 906 } 907 908 if (unlikely(!node)) 909 break; 910 911 e = container_of(node, struct wc_entry, rb_node); 912 } 913 914 if (discarded_something) 915 writecache_commit_flushed(wc, false); 916 } 917 918 static bool writecache_wait_for_writeback(struct dm_writecache *wc) 919 { 920 if (wc->writeback_size) { 921 writecache_wait_on_freelist(wc); 922 return true; 923 } 924 return false; 925 } 926 927 static void writecache_suspend(struct dm_target *ti) 928 { 929 struct dm_writecache *wc = ti->private; 930 bool flush_on_suspend; 931 932 timer_delete_sync(&wc->autocommit_timer); 933 timer_delete_sync(&wc->max_age_timer); 934 935 wc_lock(wc); 936 writecache_flush(wc); 937 flush_on_suspend = wc->flush_on_suspend; 938 if (flush_on_suspend) { 939 wc->flush_on_suspend = false; 940 wc->writeback_all++; 941 queue_work(wc->writeback_wq, &wc->writeback_work); 942 } 943 wc_unlock(wc); 944 945 drain_workqueue(wc->writeback_wq); 946 947 wc_lock(wc); 948 if (flush_on_suspend) 949 wc->writeback_all--; 950 while (writecache_wait_for_writeback(wc)) 951 ; 952 953 if (WC_MODE_PMEM(wc)) 954 persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size); 955 956 writecache_poison_lists(wc); 957 958 wc_unlock(wc); 959 } 960 961 static int writecache_alloc_entries(struct dm_writecache *wc) 962 { 963 size_t b; 964 965 if (wc->entries) 966 return 0; 967 wc->entries = vmalloc_array(wc->n_blocks, sizeof(struct wc_entry)); 968 if (!wc->entries) 969 return -ENOMEM; 970 for (b = 0; b < wc->n_blocks; b++) { 971 struct wc_entry *e = &wc->entries[b]; 972 973 e->write_in_progress = false; 974 cond_resched(); 975 } 976 977 return 0; 978 } 979 980 static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors) 981 { 982 struct dm_io_region region; 983 struct dm_io_request req; 984 985 region.bdev = wc->ssd_dev->bdev; 986 region.sector = wc->start_sector; 987 region.count = n_sectors; 988 req.bi_opf = REQ_OP_READ | REQ_SYNC; 989 req.mem.type = DM_IO_VMA; 990 req.mem.ptr.vma = (char *)wc->memory_map; 991 req.client = wc->dm_io; 992 req.notify.fn = NULL; 993 994 return dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); 995 } 996 997 static void writecache_resume(struct dm_target *ti) 998 { 999 struct dm_writecache *wc = ti->private; 1000 size_t b; 1001 bool need_flush = false; 1002 __le64 sb_seq_count; 1003 int r; 1004 1005 wc_lock(wc); 1006 1007 wc->data_device_sectors = bdev_nr_sectors(wc->dev->bdev); 1008 1009 if (WC_MODE_PMEM(wc)) { 1010 persistent_memory_invalidate_cache(wc->memory_map, wc->memory_map_size); 1011 } else { 1012 r = writecache_read_metadata(wc, wc->metadata_sectors); 1013 if (r) { 1014 size_t sb_entries_offset; 1015 1016 writecache_error(wc, r, "unable to read metadata: %d", r); 1017 sb_entries_offset = offsetof(struct wc_memory_superblock, entries); 1018 memset((char *)wc->memory_map + sb_entries_offset, -1, 1019 (wc->metadata_sectors << SECTOR_SHIFT) - sb_entries_offset); 1020 } 1021 } 1022 1023 wc->tree = RB_ROOT; 1024 INIT_LIST_HEAD(&wc->lru); 1025 if (WC_MODE_SORT_FREELIST(wc)) { 1026 wc->freetree = RB_ROOT; 1027 wc->current_free = NULL; 1028 } else { 1029 INIT_LIST_HEAD(&wc->freelist); 1030 } 1031 wc->freelist_size = 0; 1032 1033 r = copy_mc_to_kernel(&sb_seq_count, &sb(wc)->seq_count, 1034 sizeof(uint64_t)); 1035 if (r) { 1036 writecache_error(wc, r, "hardware memory error when reading superblock: %d", r); 1037 sb_seq_count = cpu_to_le64(0); 1038 } 1039 wc->seq_count = le64_to_cpu(sb_seq_count); 1040 1041 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 1042 for (b = 0; b < wc->n_blocks; b++) { 1043 struct wc_entry *e = &wc->entries[b]; 1044 struct wc_memory_entry wme; 1045 1046 if (writecache_has_error(wc)) { 1047 e->original_sector = -1; 1048 e->seq_count = -1; 1049 continue; 1050 } 1051 r = copy_mc_to_kernel(&wme, memory_entry(wc, e), 1052 sizeof(struct wc_memory_entry)); 1053 if (r) { 1054 writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d", 1055 (unsigned long)b, r); 1056 e->original_sector = -1; 1057 e->seq_count = -1; 1058 } else { 1059 e->original_sector = le64_to_cpu(wme.original_sector); 1060 e->seq_count = le64_to_cpu(wme.seq_count); 1061 } 1062 cond_resched(); 1063 } 1064 #endif 1065 for (b = 0; b < wc->n_blocks; b++) { 1066 struct wc_entry *e = &wc->entries[b]; 1067 1068 if (!writecache_entry_is_committed(wc, e)) { 1069 if (read_seq_count(wc, e) != -1) { 1070 erase_this: 1071 clear_seq_count(wc, e); 1072 need_flush = true; 1073 } 1074 writecache_add_to_freelist(wc, e); 1075 } else { 1076 struct wc_entry *old; 1077 1078 old = writecache_find_entry(wc, read_original_sector(wc, e), 0); 1079 if (!old) { 1080 writecache_insert_entry(wc, e); 1081 } else { 1082 if (read_seq_count(wc, old) == read_seq_count(wc, e)) { 1083 writecache_error(wc, -EINVAL, 1084 "two identical entries, position %llu, sector %llu, sequence %llu", 1085 (unsigned long long)b, (unsigned long long)read_original_sector(wc, e), 1086 (unsigned long long)read_seq_count(wc, e)); 1087 } 1088 if (read_seq_count(wc, old) > read_seq_count(wc, e)) { 1089 goto erase_this; 1090 } else { 1091 writecache_free_entry(wc, old); 1092 writecache_insert_entry(wc, e); 1093 need_flush = true; 1094 } 1095 } 1096 } 1097 cond_resched(); 1098 } 1099 1100 if (need_flush) { 1101 writecache_flush_all_metadata(wc); 1102 writecache_commit_flushed(wc, false); 1103 } 1104 1105 writecache_verify_watermark(wc); 1106 1107 if (wc->max_age != MAX_AGE_UNSPECIFIED) 1108 mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV); 1109 1110 wc_unlock(wc); 1111 } 1112 1113 static int process_flush_mesg(unsigned int argc, char **argv, struct dm_writecache *wc) 1114 { 1115 if (argc != 1) 1116 return -EINVAL; 1117 1118 wc_lock(wc); 1119 if (dm_suspended(wc->ti)) { 1120 wc_unlock(wc); 1121 return -EBUSY; 1122 } 1123 if (writecache_has_error(wc)) { 1124 wc_unlock(wc); 1125 return -EIO; 1126 } 1127 1128 writecache_flush(wc); 1129 wc->writeback_all++; 1130 queue_work(wc->writeback_wq, &wc->writeback_work); 1131 wc_unlock(wc); 1132 1133 flush_workqueue(wc->writeback_wq); 1134 1135 wc_lock(wc); 1136 wc->writeback_all--; 1137 if (writecache_has_error(wc)) { 1138 wc_unlock(wc); 1139 return -EIO; 1140 } 1141 wc_unlock(wc); 1142 1143 return 0; 1144 } 1145 1146 static int process_flush_on_suspend_mesg(unsigned int argc, char **argv, struct dm_writecache *wc) 1147 { 1148 if (argc != 1) 1149 return -EINVAL; 1150 1151 wc_lock(wc); 1152 wc->flush_on_suspend = true; 1153 wc_unlock(wc); 1154 1155 return 0; 1156 } 1157 1158 static void activate_cleaner(struct dm_writecache *wc) 1159 { 1160 wc->flush_on_suspend = true; 1161 wc->cleaner = true; 1162 wc->freelist_high_watermark = wc->n_blocks; 1163 wc->freelist_low_watermark = wc->n_blocks; 1164 } 1165 1166 static int process_cleaner_mesg(unsigned int argc, char **argv, struct dm_writecache *wc) 1167 { 1168 if (argc != 1) 1169 return -EINVAL; 1170 1171 wc_lock(wc); 1172 activate_cleaner(wc); 1173 if (!dm_suspended(wc->ti)) 1174 writecache_verify_watermark(wc); 1175 wc_unlock(wc); 1176 1177 return 0; 1178 } 1179 1180 static int process_clear_stats_mesg(unsigned int argc, char **argv, struct dm_writecache *wc) 1181 { 1182 if (argc != 1) 1183 return -EINVAL; 1184 1185 wc_lock(wc); 1186 memset(&wc->stats, 0, sizeof(wc->stats)); 1187 wc_unlock(wc); 1188 1189 return 0; 1190 } 1191 1192 static int writecache_message(struct dm_target *ti, unsigned int argc, char **argv, 1193 char *result, unsigned int maxlen) 1194 { 1195 int r = -EINVAL; 1196 struct dm_writecache *wc = ti->private; 1197 1198 if (!strcasecmp(argv[0], "flush")) 1199 r = process_flush_mesg(argc, argv, wc); 1200 else if (!strcasecmp(argv[0], "flush_on_suspend")) 1201 r = process_flush_on_suspend_mesg(argc, argv, wc); 1202 else if (!strcasecmp(argv[0], "cleaner")) 1203 r = process_cleaner_mesg(argc, argv, wc); 1204 else if (!strcasecmp(argv[0], "clear_stats")) 1205 r = process_clear_stats_mesg(argc, argv, wc); 1206 else 1207 DMERR("unrecognised message received: %s", argv[0]); 1208 1209 return r; 1210 } 1211 1212 static void memcpy_flushcache_optimized(void *dest, void *source, size_t size) 1213 { 1214 /* 1215 * clflushopt performs better with block size 1024, 2048, 4096 1216 * non-temporal stores perform better with block size 512 1217 * 1218 * block size 512 1024 2048 4096 1219 * movnti 496 MB/s 642 MB/s 725 MB/s 744 MB/s 1220 * clflushopt 373 MB/s 688 MB/s 1.1 GB/s 1.2 GB/s 1221 * 1222 * We see that movnti performs better for 512-byte blocks, and 1223 * clflushopt performs better for 1024-byte and larger blocks. So, we 1224 * prefer clflushopt for sizes >= 768. 1225 * 1226 * NOTE: this happens to be the case now (with dm-writecache's single 1227 * threaded model) but re-evaluate this once memcpy_flushcache() is 1228 * enabled to use movdir64b which might invalidate this performance 1229 * advantage seen with cache-allocating-writes plus flushing. 1230 */ 1231 #ifdef CONFIG_X86 1232 if (cpu_feature_enabled(X86_FEATURE_CLFLUSHOPT) && 1233 likely(boot_cpu_data.x86_clflush_size == 64) && 1234 likely(size >= 768)) { 1235 do { 1236 memcpy((void *)dest, (void *)source, 64); 1237 clflushopt((void *)dest); 1238 dest += 64; 1239 source += 64; 1240 size -= 64; 1241 } while (size >= 64); 1242 return; 1243 } 1244 #endif 1245 memcpy_flushcache(dest, source, size); 1246 } 1247 1248 static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data) 1249 { 1250 void *buf; 1251 unsigned int size; 1252 int rw = bio_data_dir(bio); 1253 unsigned int remaining_size = wc->block_size; 1254 1255 do { 1256 struct bio_vec bv = bio_iter_iovec(bio, bio->bi_iter); 1257 1258 buf = bvec_kmap_local(&bv); 1259 size = bv.bv_len; 1260 if (unlikely(size > remaining_size)) 1261 size = remaining_size; 1262 1263 if (rw == READ) { 1264 int r; 1265 1266 r = copy_mc_to_kernel(buf, data, size); 1267 flush_dcache_page(bio_page(bio)); 1268 if (unlikely(r)) { 1269 writecache_error(wc, r, "hardware memory error when reading data: %d", r); 1270 bio->bi_status = BLK_STS_IOERR; 1271 } 1272 } else { 1273 flush_dcache_page(bio_page(bio)); 1274 memcpy_flushcache_optimized(data, buf, size); 1275 } 1276 1277 kunmap_local(buf); 1278 1279 data = (char *)data + size; 1280 remaining_size -= size; 1281 bio_advance(bio, size); 1282 } while (unlikely(remaining_size)); 1283 } 1284 1285 static int writecache_flush_thread(void *data) 1286 { 1287 struct dm_writecache *wc = data; 1288 1289 while (1) { 1290 struct bio *bio; 1291 1292 wc_lock(wc); 1293 spin_lock_irq(&wc->flush_list_lock); 1294 bio = bio_list_pop(&wc->flush_list); 1295 if (!bio) { 1296 set_current_state(TASK_INTERRUPTIBLE); 1297 spin_unlock_irq(&wc->flush_list_lock); 1298 wc_unlock(wc); 1299 1300 if (unlikely(kthread_should_stop())) { 1301 set_current_state(TASK_RUNNING); 1302 break; 1303 } 1304 1305 schedule(); 1306 continue; 1307 } 1308 spin_unlock_irq(&wc->flush_list_lock); 1309 1310 if (bio_op(bio) == REQ_OP_DISCARD) { 1311 writecache_discard(wc, bio->bi_iter.bi_sector, 1312 bio_end_sector(bio)); 1313 wc_unlock(wc); 1314 bio_set_dev(bio, wc->dev->bdev); 1315 submit_bio_noacct(bio); 1316 } else { 1317 writecache_flush(wc); 1318 wc_unlock(wc); 1319 if (writecache_has_error(wc)) 1320 bio->bi_status = BLK_STS_IOERR; 1321 bio_endio(bio); 1322 } 1323 } 1324 1325 return 0; 1326 } 1327 1328 static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio) 1329 { 1330 unsigned long flags; 1331 spin_lock_irqsave(&wc->flush_list_lock, flags); 1332 if (bio_list_empty(&wc->flush_list)) 1333 wake_up_process(wc->flush_thread); 1334 bio_list_add(&wc->flush_list, bio); 1335 spin_unlock_irqrestore(&wc->flush_list_lock, flags); 1336 } 1337 1338 enum wc_map_op { 1339 WC_MAP_SUBMIT, 1340 WC_MAP_REMAP, 1341 WC_MAP_REMAP_ORIGIN, 1342 WC_MAP_RETURN, 1343 WC_MAP_ERROR, 1344 }; 1345 1346 static void writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio, 1347 struct wc_entry *e) 1348 { 1349 if (e) { 1350 sector_t next_boundary = 1351 read_original_sector(wc, e) - bio->bi_iter.bi_sector; 1352 if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) 1353 dm_accept_partial_bio(bio, next_boundary); 1354 } 1355 } 1356 1357 static enum wc_map_op writecache_map_read(struct dm_writecache *wc, struct bio *bio) 1358 { 1359 enum wc_map_op map_op; 1360 struct wc_entry *e; 1361 1362 read_next_block: 1363 wc->stats.reads++; 1364 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING); 1365 if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) { 1366 wc->stats.read_hits++; 1367 if (WC_MODE_PMEM(wc)) { 1368 bio_copy_block(wc, bio, memory_data(wc, e)); 1369 if (bio->bi_iter.bi_size) 1370 goto read_next_block; 1371 map_op = WC_MAP_SUBMIT; 1372 } else { 1373 dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT); 1374 bio_set_dev(bio, wc->ssd_dev->bdev); 1375 bio->bi_iter.bi_sector = cache_sector(wc, e); 1376 if (!writecache_entry_is_committed(wc, e)) 1377 writecache_wait_for_ios(wc, WRITE); 1378 map_op = WC_MAP_REMAP; 1379 } 1380 } else { 1381 writecache_map_remap_origin(wc, bio, e); 1382 wc->stats.reads += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits; 1383 map_op = WC_MAP_REMAP_ORIGIN; 1384 } 1385 1386 return map_op; 1387 } 1388 1389 static void writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio, 1390 struct wc_entry *e, bool search_used) 1391 { 1392 unsigned int bio_size = wc->block_size; 1393 sector_t start_cache_sec = cache_sector(wc, e); 1394 sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT); 1395 1396 while (bio_size < bio->bi_iter.bi_size) { 1397 if (!search_used) { 1398 struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec); 1399 1400 if (!f) 1401 break; 1402 write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector + 1403 (bio_size >> SECTOR_SHIFT), wc->seq_count); 1404 writecache_insert_entry(wc, f); 1405 wc->uncommitted_blocks++; 1406 } else { 1407 struct wc_entry *f; 1408 struct rb_node *next = rb_next(&e->rb_node); 1409 1410 if (!next) 1411 break; 1412 f = container_of(next, struct wc_entry, rb_node); 1413 if (f != e + 1) 1414 break; 1415 if (read_original_sector(wc, f) != 1416 read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT)) 1417 break; 1418 if (unlikely(f->write_in_progress)) 1419 break; 1420 if (writecache_entry_is_committed(wc, f)) 1421 wc->overwrote_committed = true; 1422 e = f; 1423 } 1424 bio_size += wc->block_size; 1425 current_cache_sec += wc->block_size >> SECTOR_SHIFT; 1426 } 1427 1428 bio_set_dev(bio, wc->ssd_dev->bdev); 1429 bio->bi_iter.bi_sector = start_cache_sec; 1430 dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT); 1431 1432 wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits; 1433 if (!search_used) 1434 wc->stats.writes_allocate += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits; 1435 1436 if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) { 1437 wc->uncommitted_blocks = 0; 1438 queue_work(wc->writeback_wq, &wc->flush_work); 1439 } else { 1440 writecache_schedule_autocommit(wc); 1441 } 1442 } 1443 1444 static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio *bio) 1445 { 1446 struct wc_entry *e; 1447 enum wc_map_op ret = WC_MAP_SUBMIT; 1448 bool need_flush = false; 1449 1450 do { 1451 bool found_entry = false; 1452 bool search_used = false; 1453 1454 if (writecache_has_error(wc)) { 1455 wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits; 1456 return WC_MAP_ERROR; 1457 } 1458 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0); 1459 if (e) { 1460 if (!writecache_entry_is_committed(wc, e)) { 1461 wc->stats.write_hits_uncommitted++; 1462 search_used = true; 1463 goto bio_copy; 1464 } 1465 wc->stats.write_hits_committed++; 1466 if (!WC_MODE_PMEM(wc) && !e->write_in_progress) { 1467 wc->overwrote_committed = true; 1468 search_used = true; 1469 goto bio_copy; 1470 } 1471 found_entry = true; 1472 } else { 1473 if (unlikely(wc->cleaner) || 1474 (wc->metadata_only && !(bio->bi_opf & REQ_META))) 1475 goto direct_write; 1476 } 1477 e = writecache_pop_from_freelist(wc, (sector_t)-1); 1478 if (unlikely(!e)) { 1479 if (!WC_MODE_PMEM(wc) && !found_entry) { 1480 direct_write: 1481 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING); 1482 writecache_map_remap_origin(wc, bio, e); 1483 wc->stats.writes_around += bio->bi_iter.bi_size >> wc->block_size_bits; 1484 wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits; 1485 ret = WC_MAP_REMAP_ORIGIN; 1486 goto flush_pmem_and_ret; 1487 } 1488 wc->stats.writes_blocked_on_freelist++; 1489 writecache_wait_on_freelist(wc); 1490 continue; 1491 } 1492 write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count); 1493 writecache_insert_entry(wc, e); 1494 wc->uncommitted_blocks++; 1495 wc->stats.writes_allocate++; 1496 bio_copy: 1497 if (WC_MODE_PMEM(wc)) { 1498 bio_copy_block(wc, bio, memory_data(wc, e)); 1499 wc->stats.writes++; 1500 need_flush = true; 1501 } else { 1502 writecache_bio_copy_ssd(wc, bio, e, search_used); 1503 return WC_MAP_REMAP; 1504 } 1505 } while (bio->bi_iter.bi_size); 1506 1507 flush_pmem_and_ret: 1508 if (need_flush) { 1509 if (unlikely(bio->bi_opf & REQ_FUA || wc->uncommitted_blocks >= wc->autocommit_blocks)) { 1510 writecache_flush(wc); 1511 if (writecache_has_error(wc)) 1512 ret = WC_MAP_ERROR; 1513 } else { 1514 writecache_schedule_autocommit(wc); 1515 } 1516 } 1517 1518 return ret; 1519 } 1520 1521 static enum wc_map_op writecache_map_flush(struct dm_writecache *wc, struct bio *bio) 1522 { 1523 if (writecache_has_error(wc)) 1524 return WC_MAP_ERROR; 1525 1526 if (WC_MODE_PMEM(wc)) { 1527 wc->stats.flushes++; 1528 writecache_flush(wc); 1529 if (writecache_has_error(wc)) 1530 return WC_MAP_ERROR; 1531 else if (unlikely(wc->cleaner) || unlikely(wc->metadata_only)) 1532 return WC_MAP_REMAP_ORIGIN; 1533 return WC_MAP_SUBMIT; 1534 } 1535 /* SSD: */ 1536 if (dm_bio_get_target_bio_nr(bio)) 1537 return WC_MAP_REMAP_ORIGIN; 1538 wc->stats.flushes++; 1539 writecache_offload_bio(wc, bio); 1540 return WC_MAP_RETURN; 1541 } 1542 1543 static enum wc_map_op writecache_map_discard(struct dm_writecache *wc, struct bio *bio) 1544 { 1545 wc->stats.discards += bio->bi_iter.bi_size >> wc->block_size_bits; 1546 1547 if (writecache_has_error(wc)) 1548 return WC_MAP_ERROR; 1549 1550 if (WC_MODE_PMEM(wc)) { 1551 writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio)); 1552 return WC_MAP_REMAP_ORIGIN; 1553 } 1554 /* SSD: */ 1555 writecache_offload_bio(wc, bio); 1556 return WC_MAP_RETURN; 1557 } 1558 1559 static int writecache_map(struct dm_target *ti, struct bio *bio) 1560 { 1561 struct dm_writecache *wc = ti->private; 1562 enum wc_map_op map_op; 1563 1564 bio->bi_private = NULL; 1565 1566 wc_lock(wc); 1567 1568 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { 1569 map_op = writecache_map_flush(wc, bio); 1570 goto done; 1571 } 1572 1573 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); 1574 1575 if (unlikely((((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) & 1576 (wc->block_size / 512 - 1)) != 0)) { 1577 DMERR("I/O is not aligned, sector %llu, size %u, block size %u", 1578 (unsigned long long)bio->bi_iter.bi_sector, 1579 bio->bi_iter.bi_size, wc->block_size); 1580 map_op = WC_MAP_ERROR; 1581 goto done; 1582 } 1583 1584 if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { 1585 map_op = writecache_map_discard(wc, bio); 1586 goto done; 1587 } 1588 1589 if (bio_data_dir(bio) == READ) 1590 map_op = writecache_map_read(wc, bio); 1591 else 1592 map_op = writecache_map_write(wc, bio); 1593 done: 1594 switch (map_op) { 1595 case WC_MAP_REMAP_ORIGIN: 1596 if (likely(wc->pause != 0)) { 1597 if (bio_op(bio) == REQ_OP_WRITE) { 1598 dm_iot_io_begin(&wc->iot, 1); 1599 bio->bi_private = (void *)2; 1600 } 1601 } 1602 bio_set_dev(bio, wc->dev->bdev); 1603 wc_unlock(wc); 1604 return DM_MAPIO_REMAPPED; 1605 1606 case WC_MAP_REMAP: 1607 /* make sure that writecache_end_io decrements bio_in_progress: */ 1608 if (unlikely(bio->bi_opf & REQ_FUA) && bio_op(bio) == REQ_OP_WRITE) 1609 bio->bi_private = (void *)3; 1610 else 1611 bio->bi_private = (void *)1; 1612 atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]); 1613 wc_unlock(wc); 1614 return DM_MAPIO_REMAPPED; 1615 1616 case WC_MAP_SUBMIT: 1617 wc_unlock(wc); 1618 bio_endio(bio); 1619 return DM_MAPIO_SUBMITTED; 1620 1621 case WC_MAP_RETURN: 1622 wc_unlock(wc); 1623 return DM_MAPIO_SUBMITTED; 1624 1625 case WC_MAP_ERROR: 1626 wc_unlock(wc); 1627 bio_io_error(bio); 1628 return DM_MAPIO_SUBMITTED; 1629 1630 default: 1631 BUG(); 1632 wc_unlock(wc); 1633 return DM_MAPIO_KILL; 1634 } 1635 } 1636 1637 static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status) 1638 { 1639 struct dm_writecache *wc = ti->private; 1640 1641 if (bio->bi_private == (void *)1 || bio->bi_private == (void *)3) { 1642 int dir = bio_data_dir(bio); 1643 bool fua = bio->bi_private == (void *)3; 1644 1645 if (atomic_dec_and_test(&wc->bio_in_progress[dir])) 1646 if (unlikely(waitqueue_active(&wc->bio_in_progress_wait[dir]))) 1647 wake_up(&wc->bio_in_progress_wait[dir]); 1648 1649 if (fua && likely(*status == BLK_STS_OK)) { 1650 bio->bi_private = NULL; 1651 writecache_offload_bio(wc, bio); 1652 return DM_ENDIO_INCOMPLETE; 1653 } 1654 } else if (bio->bi_private == (void *)2) { 1655 dm_iot_io_end(&wc->iot, 1); 1656 } 1657 return 0; 1658 } 1659 1660 static int writecache_iterate_devices(struct dm_target *ti, 1661 iterate_devices_callout_fn fn, void *data) 1662 { 1663 struct dm_writecache *wc = ti->private; 1664 1665 return fn(ti, wc->dev, 0, ti->len, data); 1666 } 1667 1668 static void writecache_io_hints(struct dm_target *ti, struct queue_limits *limits) 1669 { 1670 struct dm_writecache *wc = ti->private; 1671 1672 dm_stack_bs_limits(limits, wc->block_size); 1673 } 1674 1675 static void writecache_writeback_endio(struct bio *bio) 1676 { 1677 struct writeback_struct *wb = container_of(bio, struct writeback_struct, bio); 1678 struct dm_writecache *wc = wb->wc; 1679 unsigned long flags; 1680 1681 raw_spin_lock_irqsave(&wc->endio_list_lock, flags); 1682 if (unlikely(list_empty(&wc->endio_list))) 1683 wake_up_process(wc->endio_thread); 1684 list_add_tail(&wb->endio_entry, &wc->endio_list); 1685 raw_spin_unlock_irqrestore(&wc->endio_list_lock, flags); 1686 } 1687 1688 static void writecache_copy_endio(int read_err, unsigned long write_err, void *ptr) 1689 { 1690 struct copy_struct *c = ptr; 1691 struct dm_writecache *wc = c->wc; 1692 1693 c->error = likely(!(read_err | write_err)) ? 0 : -EIO; 1694 1695 raw_spin_lock_irq(&wc->endio_list_lock); 1696 if (unlikely(list_empty(&wc->endio_list))) 1697 wake_up_process(wc->endio_thread); 1698 list_add_tail(&c->endio_entry, &wc->endio_list); 1699 raw_spin_unlock_irq(&wc->endio_list_lock); 1700 } 1701 1702 static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *list) 1703 { 1704 unsigned int i; 1705 struct writeback_struct *wb; 1706 struct wc_entry *e; 1707 unsigned long n_walked = 0; 1708 1709 do { 1710 wb = list_entry(list->next, struct writeback_struct, endio_entry); 1711 list_del(&wb->endio_entry); 1712 1713 if (unlikely(wb->bio.bi_status != BLK_STS_OK)) 1714 writecache_error(wc, blk_status_to_errno(wb->bio.bi_status), 1715 "write error %d", wb->bio.bi_status); 1716 i = 0; 1717 do { 1718 e = wb->wc_list[i]; 1719 BUG_ON(!e->write_in_progress); 1720 e->write_in_progress = false; 1721 INIT_LIST_HEAD(&e->lru); 1722 if (!writecache_has_error(wc)) 1723 writecache_free_entry(wc, e); 1724 BUG_ON(!wc->writeback_size); 1725 wc->writeback_size--; 1726 n_walked++; 1727 if (unlikely(n_walked >= ENDIO_LATENCY)) { 1728 writecache_commit_flushed(wc, false); 1729 wc_unlock(wc); 1730 wc_lock(wc); 1731 n_walked = 0; 1732 } 1733 } while (++i < wb->wc_list_n); 1734 1735 if (wb->wc_list != wb->wc_list_inline) 1736 kfree(wb->wc_list); 1737 bio_put(&wb->bio); 1738 } while (!list_empty(list)); 1739 } 1740 1741 static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *list) 1742 { 1743 struct copy_struct *c; 1744 struct wc_entry *e; 1745 1746 do { 1747 c = list_entry(list->next, struct copy_struct, endio_entry); 1748 list_del(&c->endio_entry); 1749 1750 if (unlikely(c->error)) 1751 writecache_error(wc, c->error, "copy error"); 1752 1753 e = c->e; 1754 do { 1755 BUG_ON(!e->write_in_progress); 1756 e->write_in_progress = false; 1757 INIT_LIST_HEAD(&e->lru); 1758 if (!writecache_has_error(wc)) 1759 writecache_free_entry(wc, e); 1760 1761 BUG_ON(!wc->writeback_size); 1762 wc->writeback_size--; 1763 e++; 1764 } while (--c->n_entries); 1765 mempool_free(c, &wc->copy_pool); 1766 } while (!list_empty(list)); 1767 } 1768 1769 static int writecache_endio_thread(void *data) 1770 { 1771 struct dm_writecache *wc = data; 1772 1773 while (1) { 1774 struct list_head list; 1775 1776 raw_spin_lock_irq(&wc->endio_list_lock); 1777 if (!list_empty(&wc->endio_list)) 1778 goto pop_from_list; 1779 set_current_state(TASK_INTERRUPTIBLE); 1780 raw_spin_unlock_irq(&wc->endio_list_lock); 1781 1782 if (unlikely(kthread_should_stop())) { 1783 set_current_state(TASK_RUNNING); 1784 break; 1785 } 1786 1787 schedule(); 1788 1789 continue; 1790 1791 pop_from_list: 1792 list = wc->endio_list; 1793 list.next->prev = list.prev->next = &list; 1794 INIT_LIST_HEAD(&wc->endio_list); 1795 raw_spin_unlock_irq(&wc->endio_list_lock); 1796 1797 if (!WC_MODE_FUA(wc)) 1798 writecache_disk_flush(wc, wc->dev); 1799 1800 wc_lock(wc); 1801 1802 if (WC_MODE_PMEM(wc)) { 1803 __writecache_endio_pmem(wc, &list); 1804 } else { 1805 __writecache_endio_ssd(wc, &list); 1806 writecache_wait_for_ios(wc, READ); 1807 } 1808 1809 writecache_commit_flushed(wc, false); 1810 1811 wc_unlock(wc); 1812 } 1813 1814 return 0; 1815 } 1816 1817 static bool wc_add_block(struct writeback_struct *wb, struct wc_entry *e) 1818 { 1819 struct dm_writecache *wc = wb->wc; 1820 unsigned int block_size = wc->block_size; 1821 void *address = memory_data(wc, e); 1822 1823 persistent_memory_flush_cache(address, block_size); 1824 1825 if (unlikely(bio_end_sector(&wb->bio) >= wc->data_device_sectors)) 1826 return true; 1827 1828 return bio_add_page(&wb->bio, persistent_memory_page(address), 1829 block_size, persistent_memory_page_offset(address)) != 0; 1830 } 1831 1832 struct writeback_list { 1833 struct list_head list; 1834 size_t size; 1835 }; 1836 1837 static void __writeback_throttle(struct dm_writecache *wc, struct writeback_list *wbl) 1838 { 1839 if (unlikely(wc->max_writeback_jobs)) { 1840 if (READ_ONCE(wc->writeback_size) - wbl->size >= wc->max_writeback_jobs) { 1841 wc_lock(wc); 1842 while (wc->writeback_size - wbl->size >= wc->max_writeback_jobs) 1843 writecache_wait_on_freelist(wc); 1844 wc_unlock(wc); 1845 } 1846 } 1847 cond_resched(); 1848 } 1849 1850 static void __writecache_writeback_pmem(struct dm_writecache *wc, struct writeback_list *wbl) 1851 { 1852 struct wc_entry *e, *f; 1853 struct bio *bio; 1854 struct writeback_struct *wb; 1855 unsigned int max_pages; 1856 1857 while (wbl->size) { 1858 wbl->size--; 1859 e = container_of(wbl->list.prev, struct wc_entry, lru); 1860 list_del(&e->lru); 1861 1862 max_pages = e->wc_list_contiguous; 1863 1864 bio = bio_alloc_bioset(wc->dev->bdev, max_pages, REQ_OP_WRITE, 1865 GFP_NOIO, &wc->bio_set); 1866 wb = container_of(bio, struct writeback_struct, bio); 1867 wb->wc = wc; 1868 bio->bi_end_io = writecache_writeback_endio; 1869 bio->bi_iter.bi_sector = read_original_sector(wc, e); 1870 1871 if (unlikely(max_pages > WB_LIST_INLINE)) 1872 wb->wc_list = kmalloc_objs(struct wc_entry *, max_pages, 1873 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 1874 1875 if (likely(max_pages <= WB_LIST_INLINE) || unlikely(!wb->wc_list)) { 1876 wb->wc_list = wb->wc_list_inline; 1877 max_pages = WB_LIST_INLINE; 1878 } 1879 1880 BUG_ON(!wc_add_block(wb, e)); 1881 1882 wb->wc_list[0] = e; 1883 wb->wc_list_n = 1; 1884 1885 while (wbl->size && wb->wc_list_n < max_pages) { 1886 f = container_of(wbl->list.prev, struct wc_entry, lru); 1887 if (read_original_sector(wc, f) != 1888 read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT)) 1889 break; 1890 if (!wc_add_block(wb, f)) 1891 break; 1892 wbl->size--; 1893 list_del(&f->lru); 1894 wb->wc_list[wb->wc_list_n++] = f; 1895 e = f; 1896 } 1897 if (WC_MODE_FUA(wc)) 1898 bio->bi_opf |= REQ_FUA; 1899 if (writecache_has_error(wc)) { 1900 bio->bi_status = BLK_STS_IOERR; 1901 bio_endio(bio); 1902 } else if (unlikely(!bio_sectors(bio))) { 1903 bio->bi_status = BLK_STS_OK; 1904 bio_endio(bio); 1905 } else { 1906 submit_bio(bio); 1907 } 1908 1909 __writeback_throttle(wc, wbl); 1910 } 1911 } 1912 1913 static void __writecache_writeback_ssd(struct dm_writecache *wc, struct writeback_list *wbl) 1914 { 1915 struct wc_entry *e, *f; 1916 struct dm_io_region from, to; 1917 struct copy_struct *c; 1918 1919 while (wbl->size) { 1920 unsigned int n_sectors; 1921 1922 wbl->size--; 1923 e = container_of(wbl->list.prev, struct wc_entry, lru); 1924 list_del(&e->lru); 1925 1926 n_sectors = e->wc_list_contiguous << (wc->block_size_bits - SECTOR_SHIFT); 1927 1928 from.bdev = wc->ssd_dev->bdev; 1929 from.sector = cache_sector(wc, e); 1930 from.count = n_sectors; 1931 to.bdev = wc->dev->bdev; 1932 to.sector = read_original_sector(wc, e); 1933 to.count = n_sectors; 1934 1935 c = mempool_alloc(&wc->copy_pool, GFP_NOIO); 1936 c->wc = wc; 1937 c->e = e; 1938 c->n_entries = e->wc_list_contiguous; 1939 1940 while ((n_sectors -= wc->block_size >> SECTOR_SHIFT)) { 1941 wbl->size--; 1942 f = container_of(wbl->list.prev, struct wc_entry, lru); 1943 BUG_ON(f != e + 1); 1944 list_del(&f->lru); 1945 e = f; 1946 } 1947 1948 if (unlikely(to.sector + to.count > wc->data_device_sectors)) { 1949 if (to.sector >= wc->data_device_sectors) { 1950 writecache_copy_endio(0, 0, c); 1951 continue; 1952 } 1953 from.count = to.count = wc->data_device_sectors - to.sector; 1954 } 1955 1956 dm_kcopyd_copy(wc->dm_kcopyd, &from, 1, &to, 0, writecache_copy_endio, c); 1957 1958 __writeback_throttle(wc, wbl); 1959 } 1960 } 1961 1962 static void writecache_writeback(struct work_struct *work) 1963 { 1964 struct dm_writecache *wc = container_of(work, struct dm_writecache, writeback_work); 1965 struct blk_plug plug; 1966 struct wc_entry *f, *g, *e = NULL; 1967 struct rb_node *node, *next_node; 1968 struct list_head skipped; 1969 struct writeback_list wbl; 1970 unsigned long n_walked; 1971 1972 if (!WC_MODE_PMEM(wc)) { 1973 /* Wait for any active kcopyd work on behalf of ssd writeback */ 1974 dm_kcopyd_client_flush(wc->dm_kcopyd); 1975 } 1976 1977 if (likely(wc->pause != 0)) { 1978 while (1) { 1979 unsigned long idle; 1980 1981 if (unlikely(wc->cleaner) || unlikely(wc->writeback_all) || 1982 unlikely(dm_suspended(wc->ti))) 1983 break; 1984 idle = dm_iot_idle_time(&wc->iot); 1985 if (idle >= wc->pause) 1986 break; 1987 idle = wc->pause - idle; 1988 if (idle > HZ) 1989 idle = HZ; 1990 schedule_timeout_idle(idle); 1991 } 1992 } 1993 1994 wc_lock(wc); 1995 restart: 1996 if (writecache_has_error(wc)) { 1997 wc_unlock(wc); 1998 return; 1999 } 2000 2001 if (unlikely(wc->writeback_all)) { 2002 if (writecache_wait_for_writeback(wc)) 2003 goto restart; 2004 } 2005 2006 if (wc->overwrote_committed) 2007 writecache_wait_for_ios(wc, WRITE); 2008 2009 n_walked = 0; 2010 INIT_LIST_HEAD(&skipped); 2011 INIT_LIST_HEAD(&wbl.list); 2012 wbl.size = 0; 2013 while (!list_empty(&wc->lru) && 2014 (wc->writeback_all || 2015 wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark || 2016 ((u32)(jiffies - container_of(wc->lru.prev, struct wc_entry, lru)->age) >= 2017 wc->max_age - wc->max_age / MAX_AGE_DIV))) { 2018 2019 n_walked++; 2020 if (unlikely(n_walked > WRITEBACK_LATENCY) && 2021 likely(!wc->writeback_all)) { 2022 if (likely(!dm_suspended(wc->ti))) 2023 queue_work(wc->writeback_wq, &wc->writeback_work); 2024 break; 2025 } 2026 2027 if (unlikely(wc->writeback_all)) { 2028 if (unlikely(!e)) { 2029 writecache_flush(wc); 2030 e = container_of(rb_first(&wc->tree), struct wc_entry, rb_node); 2031 } else 2032 e = g; 2033 } else 2034 e = container_of(wc->lru.prev, struct wc_entry, lru); 2035 BUG_ON(e->write_in_progress); 2036 if (unlikely(!writecache_entry_is_committed(wc, e))) 2037 writecache_flush(wc); 2038 2039 node = rb_prev(&e->rb_node); 2040 if (node) { 2041 f = container_of(node, struct wc_entry, rb_node); 2042 if (unlikely(read_original_sector(wc, f) == 2043 read_original_sector(wc, e))) { 2044 BUG_ON(!f->write_in_progress); 2045 list_move(&e->lru, &skipped); 2046 cond_resched(); 2047 continue; 2048 } 2049 } 2050 wc->writeback_size++; 2051 list_move(&e->lru, &wbl.list); 2052 wbl.size++; 2053 e->write_in_progress = true; 2054 e->wc_list_contiguous = 1; 2055 2056 f = e; 2057 2058 while (1) { 2059 next_node = rb_next(&f->rb_node); 2060 if (unlikely(!next_node)) 2061 break; 2062 g = container_of(next_node, struct wc_entry, rb_node); 2063 if (unlikely(read_original_sector(wc, g) == 2064 read_original_sector(wc, f))) { 2065 f = g; 2066 continue; 2067 } 2068 if (read_original_sector(wc, g) != 2069 read_original_sector(wc, f) + (wc->block_size >> SECTOR_SHIFT)) 2070 break; 2071 if (unlikely(g->write_in_progress)) 2072 break; 2073 if (unlikely(!writecache_entry_is_committed(wc, g))) 2074 break; 2075 2076 if (!WC_MODE_PMEM(wc)) { 2077 if (g != f + 1) 2078 break; 2079 } 2080 2081 n_walked++; 2082 //if (unlikely(n_walked > WRITEBACK_LATENCY) && likely(!wc->writeback_all)) 2083 // break; 2084 2085 wc->writeback_size++; 2086 list_move(&g->lru, &wbl.list); 2087 wbl.size++; 2088 g->write_in_progress = true; 2089 g->wc_list_contiguous = BIO_MAX_VECS; 2090 f = g; 2091 e->wc_list_contiguous++; 2092 if (unlikely(e->wc_list_contiguous == BIO_MAX_VECS)) { 2093 if (unlikely(wc->writeback_all)) { 2094 next_node = rb_next(&f->rb_node); 2095 if (likely(next_node)) 2096 g = container_of(next_node, struct wc_entry, rb_node); 2097 } 2098 break; 2099 } 2100 } 2101 cond_resched(); 2102 } 2103 2104 if (!list_empty(&skipped)) { 2105 list_splice_tail(&skipped, &wc->lru); 2106 /* 2107 * If we didn't do any progress, we must wait until some 2108 * writeback finishes to avoid burning CPU in a loop 2109 */ 2110 if (unlikely(!wbl.size)) 2111 writecache_wait_for_writeback(wc); 2112 } 2113 2114 wc_unlock(wc); 2115 2116 blk_start_plug(&plug); 2117 2118 if (WC_MODE_PMEM(wc)) 2119 __writecache_writeback_pmem(wc, &wbl); 2120 else 2121 __writecache_writeback_ssd(wc, &wbl); 2122 2123 blk_finish_plug(&plug); 2124 2125 if (unlikely(wc->writeback_all)) { 2126 wc_lock(wc); 2127 while (writecache_wait_for_writeback(wc)) 2128 ; 2129 wc_unlock(wc); 2130 } 2131 } 2132 2133 static int calculate_memory_size(uint64_t device_size, unsigned int block_size, 2134 size_t *n_blocks_p, size_t *n_metadata_blocks_p) 2135 { 2136 uint64_t n_blocks, offset; 2137 2138 n_blocks = device_size; 2139 do_div(n_blocks, block_size + sizeof(struct wc_memory_entry)); 2140 2141 while (1) { 2142 if (!n_blocks) 2143 return -ENOSPC; 2144 /* Verify the following entries[n_blocks] won't overflow */ 2145 if (n_blocks >= ((size_t)-sizeof(struct wc_memory_superblock) / 2146 sizeof(struct wc_memory_entry))) 2147 return -EFBIG; 2148 offset = offsetof(struct wc_memory_superblock, entries[n_blocks]); 2149 offset = (offset + block_size - 1) & ~(uint64_t)(block_size - 1); 2150 if (offset + n_blocks * block_size <= device_size) 2151 break; 2152 n_blocks--; 2153 } 2154 2155 if (n_blocks_p) 2156 *n_blocks_p = n_blocks; 2157 if (n_metadata_blocks_p) 2158 *n_metadata_blocks_p = offset >> __ffs(block_size); 2159 return 0; 2160 } 2161 2162 static int init_memory(struct dm_writecache *wc) 2163 { 2164 size_t b; 2165 int r; 2166 2167 r = calculate_memory_size(wc->memory_map_size, wc->block_size, &wc->n_blocks, NULL); 2168 if (r) 2169 return r; 2170 2171 r = writecache_alloc_entries(wc); 2172 if (r) 2173 return r; 2174 2175 for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++) 2176 pmem_assign(sb(wc)->padding[b], cpu_to_le64(0)); 2177 pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION)); 2178 pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size)); 2179 pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks)); 2180 pmem_assign(sb(wc)->seq_count, cpu_to_le64(0)); 2181 2182 for (b = 0; b < wc->n_blocks; b++) { 2183 write_original_sector_seq_count(wc, &wc->entries[b], -1, -1); 2184 cond_resched(); 2185 } 2186 2187 writecache_flush_all_metadata(wc); 2188 writecache_commit_flushed(wc, false); 2189 pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC)); 2190 writecache_flush_region(wc, &sb(wc)->magic, sizeof(sb(wc)->magic)); 2191 writecache_commit_flushed(wc, false); 2192 2193 return 0; 2194 } 2195 2196 static void writecache_dtr(struct dm_target *ti) 2197 { 2198 struct dm_writecache *wc = ti->private; 2199 2200 if (!wc) 2201 return; 2202 2203 if (wc->endio_thread) 2204 kthread_stop(wc->endio_thread); 2205 2206 if (wc->flush_thread) 2207 kthread_stop(wc->flush_thread); 2208 2209 bioset_exit(&wc->bio_set); 2210 2211 mempool_exit(&wc->copy_pool); 2212 2213 if (wc->writeback_wq) 2214 destroy_workqueue(wc->writeback_wq); 2215 2216 if (wc->dev) 2217 dm_put_device(ti, wc->dev); 2218 2219 if (wc->ssd_dev) 2220 dm_put_device(ti, wc->ssd_dev); 2221 2222 vfree(wc->entries); 2223 2224 if (wc->memory_map) { 2225 if (WC_MODE_PMEM(wc)) 2226 persistent_memory_release(wc); 2227 else 2228 vfree(wc->memory_map); 2229 } 2230 2231 if (wc->dm_kcopyd) 2232 dm_kcopyd_client_destroy(wc->dm_kcopyd); 2233 2234 if (wc->dm_io) 2235 dm_io_client_destroy(wc->dm_io); 2236 2237 vfree(wc->dirty_bitmap); 2238 2239 kfree(wc); 2240 } 2241 2242 static int writecache_ctr(struct dm_target *ti, unsigned int argc, char **argv) 2243 { 2244 struct dm_writecache *wc; 2245 struct dm_arg_set as; 2246 const char *string; 2247 unsigned int opt_params; 2248 size_t offset, data_size; 2249 int i, r; 2250 char dummy; 2251 int high_wm_percent = HIGH_WATERMARK; 2252 int low_wm_percent = LOW_WATERMARK; 2253 uint64_t x; 2254 struct wc_memory_superblock s; 2255 2256 static struct dm_arg _args[] = { 2257 {0, 18, "Invalid number of feature args"}, 2258 }; 2259 2260 as.argc = argc; 2261 as.argv = argv; 2262 2263 wc = kzalloc_obj(struct dm_writecache); 2264 if (!wc) { 2265 ti->error = "Cannot allocate writecache structure"; 2266 r = -ENOMEM; 2267 goto bad; 2268 } 2269 ti->private = wc; 2270 wc->ti = ti; 2271 2272 mutex_init(&wc->lock); 2273 wc->max_age = MAX_AGE_UNSPECIFIED; 2274 writecache_poison_lists(wc); 2275 init_waitqueue_head(&wc->freelist_wait); 2276 timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0); 2277 timer_setup(&wc->max_age_timer, writecache_max_age_timer, 0); 2278 2279 for (i = 0; i < 2; i++) { 2280 atomic_set(&wc->bio_in_progress[i], 0); 2281 init_waitqueue_head(&wc->bio_in_progress_wait[i]); 2282 } 2283 2284 wc->dm_io = dm_io_client_create(); 2285 if (IS_ERR(wc->dm_io)) { 2286 r = PTR_ERR(wc->dm_io); 2287 ti->error = "Unable to allocate dm-io client"; 2288 wc->dm_io = NULL; 2289 goto bad; 2290 } 2291 2292 wc->writeback_wq = alloc_workqueue("writecache-writeback", 2293 WQ_MEM_RECLAIM | WQ_PERCPU, 1); 2294 if (!wc->writeback_wq) { 2295 r = -ENOMEM; 2296 ti->error = "Could not allocate writeback workqueue"; 2297 goto bad; 2298 } 2299 INIT_WORK(&wc->writeback_work, writecache_writeback); 2300 INIT_WORK(&wc->flush_work, writecache_flush_work); 2301 2302 dm_iot_init(&wc->iot); 2303 2304 raw_spin_lock_init(&wc->endio_list_lock); 2305 INIT_LIST_HEAD(&wc->endio_list); 2306 wc->endio_thread = kthread_run(writecache_endio_thread, wc, "writecache_endio"); 2307 if (IS_ERR(wc->endio_thread)) { 2308 r = PTR_ERR(wc->endio_thread); 2309 wc->endio_thread = NULL; 2310 ti->error = "Couldn't spawn endio thread"; 2311 goto bad; 2312 } 2313 2314 /* 2315 * Parse the mode (pmem or ssd) 2316 */ 2317 string = dm_shift_arg(&as); 2318 if (!string) 2319 goto bad_arguments; 2320 2321 if (!strcasecmp(string, "s")) { 2322 wc->pmem_mode = false; 2323 } else if (!strcasecmp(string, "p")) { 2324 #ifdef DM_WRITECACHE_HAS_PMEM 2325 wc->pmem_mode = true; 2326 wc->writeback_fua = true; 2327 #else 2328 /* 2329 * If the architecture doesn't support persistent memory or 2330 * the kernel doesn't support any DAX drivers, this driver can 2331 * only be used in SSD-only mode. 2332 */ 2333 r = -EOPNOTSUPP; 2334 ti->error = "Persistent memory or DAX not supported on this system"; 2335 goto bad; 2336 #endif 2337 } else { 2338 goto bad_arguments; 2339 } 2340 2341 if (WC_MODE_PMEM(wc)) { 2342 r = bioset_init(&wc->bio_set, BIO_POOL_SIZE, 2343 offsetof(struct writeback_struct, bio), 2344 BIOSET_NEED_BVECS); 2345 if (r) { 2346 ti->error = "Could not allocate bio set"; 2347 goto bad; 2348 } 2349 } else { 2350 wc->pause = PAUSE_WRITEBACK; 2351 r = mempool_init_kmalloc_pool(&wc->copy_pool, 1, sizeof(struct copy_struct)); 2352 if (r) { 2353 ti->error = "Could not allocate mempool"; 2354 goto bad; 2355 } 2356 } 2357 2358 /* 2359 * Parse the origin data device 2360 */ 2361 string = dm_shift_arg(&as); 2362 if (!string) 2363 goto bad_arguments; 2364 r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->dev); 2365 if (r) { 2366 ti->error = "Origin data device lookup failed"; 2367 goto bad; 2368 } 2369 2370 /* 2371 * Parse cache data device (be it pmem or ssd) 2372 */ 2373 string = dm_shift_arg(&as); 2374 if (!string) 2375 goto bad_arguments; 2376 2377 r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->ssd_dev); 2378 if (r) { 2379 ti->error = "Cache data device lookup failed"; 2380 goto bad; 2381 } 2382 wc->memory_map_size = bdev_nr_bytes(wc->ssd_dev->bdev); 2383 2384 /* 2385 * Parse the cache block size 2386 */ 2387 string = dm_shift_arg(&as); 2388 if (!string) 2389 goto bad_arguments; 2390 if (sscanf(string, "%u%c", &wc->block_size, &dummy) != 1 || 2391 wc->block_size < 512 || wc->block_size > PAGE_SIZE || 2392 (wc->block_size & (wc->block_size - 1))) { 2393 r = -EINVAL; 2394 ti->error = "Invalid block size"; 2395 goto bad; 2396 } 2397 if (wc->block_size < bdev_logical_block_size(wc->dev->bdev) || 2398 wc->block_size < bdev_logical_block_size(wc->ssd_dev->bdev)) { 2399 r = -EINVAL; 2400 ti->error = "Block size is smaller than device logical block size"; 2401 goto bad; 2402 } 2403 wc->block_size_bits = __ffs(wc->block_size); 2404 2405 wc->max_writeback_jobs = MAX_WRITEBACK_JOBS; 2406 wc->autocommit_blocks = !WC_MODE_PMEM(wc) ? AUTOCOMMIT_BLOCKS_SSD : AUTOCOMMIT_BLOCKS_PMEM; 2407 wc->autocommit_jiffies = msecs_to_jiffies(AUTOCOMMIT_MSEC); 2408 2409 /* 2410 * Parse optional arguments 2411 */ 2412 r = dm_read_arg_group(_args, &as, &opt_params, &ti->error); 2413 if (r) 2414 goto bad; 2415 2416 while (opt_params) { 2417 string = dm_shift_arg(&as), opt_params--; 2418 if (!strcasecmp(string, "start_sector") && opt_params >= 1) { 2419 unsigned long long start_sector; 2420 2421 string = dm_shift_arg(&as), opt_params--; 2422 if (sscanf(string, "%llu%c", &start_sector, &dummy) != 1) 2423 goto invalid_optional; 2424 wc->start_sector = start_sector; 2425 wc->start_sector_set = true; 2426 if (wc->start_sector != start_sector || 2427 wc->start_sector >= wc->memory_map_size >> SECTOR_SHIFT) 2428 goto invalid_optional; 2429 } else if (!strcasecmp(string, "high_watermark") && opt_params >= 1) { 2430 string = dm_shift_arg(&as), opt_params--; 2431 if (sscanf(string, "%d%c", &high_wm_percent, &dummy) != 1) 2432 goto invalid_optional; 2433 if (high_wm_percent < 0 || high_wm_percent > 100) 2434 goto invalid_optional; 2435 wc->high_wm_percent_value = high_wm_percent; 2436 wc->high_wm_percent_set = true; 2437 } else if (!strcasecmp(string, "low_watermark") && opt_params >= 1) { 2438 string = dm_shift_arg(&as), opt_params--; 2439 if (sscanf(string, "%d%c", &low_wm_percent, &dummy) != 1) 2440 goto invalid_optional; 2441 if (low_wm_percent < 0 || low_wm_percent > 100) 2442 goto invalid_optional; 2443 wc->low_wm_percent_value = low_wm_percent; 2444 wc->low_wm_percent_set = true; 2445 } else if (!strcasecmp(string, "writeback_jobs") && opt_params >= 1) { 2446 string = dm_shift_arg(&as), opt_params--; 2447 if (sscanf(string, "%u%c", &wc->max_writeback_jobs, &dummy) != 1) 2448 goto invalid_optional; 2449 wc->max_writeback_jobs_set = true; 2450 } else if (!strcasecmp(string, "autocommit_blocks") && opt_params >= 1) { 2451 string = dm_shift_arg(&as), opt_params--; 2452 if (sscanf(string, "%u%c", &wc->autocommit_blocks, &dummy) != 1) 2453 goto invalid_optional; 2454 wc->autocommit_blocks_set = true; 2455 } else if (!strcasecmp(string, "autocommit_time") && opt_params >= 1) { 2456 unsigned int autocommit_msecs; 2457 2458 string = dm_shift_arg(&as), opt_params--; 2459 if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1) 2460 goto invalid_optional; 2461 if (autocommit_msecs > 3600000) 2462 goto invalid_optional; 2463 wc->autocommit_jiffies = msecs_to_jiffies(autocommit_msecs); 2464 wc->autocommit_time_value = autocommit_msecs; 2465 wc->autocommit_time_set = true; 2466 } else if (!strcasecmp(string, "max_age") && opt_params >= 1) { 2467 unsigned int max_age_msecs; 2468 unsigned long max_age_jiffies; 2469 2470 string = dm_shift_arg(&as), opt_params--; 2471 if (sscanf(string, "%u%c", &max_age_msecs, &dummy) != 1) 2472 goto invalid_optional; 2473 max_age_jiffies = msecs_to_jiffies(max_age_msecs); 2474 if (max_age_jiffies >= min(MAX_JIFFY_OFFSET, (7 << 28))) // 7/8ths of 1 << 31 2475 goto invalid_optional; 2476 wc->max_age = max_age_jiffies; 2477 wc->max_age_set = true; 2478 wc->max_age_value = max_age_msecs; 2479 } else if (!strcasecmp(string, "cleaner")) { 2480 wc->cleaner_set = true; 2481 wc->cleaner = true; 2482 } else if (!strcasecmp(string, "fua")) { 2483 if (WC_MODE_PMEM(wc)) { 2484 wc->writeback_fua = true; 2485 wc->writeback_fua_set = true; 2486 } else 2487 goto invalid_optional; 2488 } else if (!strcasecmp(string, "nofua")) { 2489 if (WC_MODE_PMEM(wc)) { 2490 wc->writeback_fua = false; 2491 wc->writeback_fua_set = true; 2492 } else 2493 goto invalid_optional; 2494 } else if (!strcasecmp(string, "metadata_only")) { 2495 wc->metadata_only = true; 2496 } else if (!strcasecmp(string, "pause_writeback") && opt_params >= 1) { 2497 unsigned int pause_msecs; 2498 2499 if (WC_MODE_PMEM(wc)) 2500 goto invalid_optional; 2501 string = dm_shift_arg(&as), opt_params--; 2502 if (sscanf(string, "%u%c", &pause_msecs, &dummy) != 1) 2503 goto invalid_optional; 2504 if (pause_msecs > 60000) 2505 goto invalid_optional; 2506 wc->pause = msecs_to_jiffies(pause_msecs); 2507 wc->pause_set = true; 2508 wc->pause_value = pause_msecs; 2509 } else { 2510 invalid_optional: 2511 r = -EINVAL; 2512 ti->error = "Invalid optional argument"; 2513 goto bad; 2514 } 2515 } 2516 2517 if (high_wm_percent < low_wm_percent) { 2518 r = -EINVAL; 2519 ti->error = "High watermark must be greater than or equal to low watermark"; 2520 goto bad; 2521 } 2522 2523 if (WC_MODE_PMEM(wc)) { 2524 if (!wc->ssd_dev->dax_dev) { 2525 r = -EOPNOTSUPP; 2526 ti->error = "Cache device is not a DAX device"; 2527 goto bad; 2528 } 2529 if (!dax_synchronous(wc->ssd_dev->dax_dev)) { 2530 r = -EOPNOTSUPP; 2531 ti->error = "Asynchronous persistent memory not supported as pmem cache"; 2532 goto bad; 2533 } 2534 2535 r = persistent_memory_claim(wc); 2536 if (r) { 2537 ti->error = "Unable to map persistent memory for cache"; 2538 goto bad; 2539 } 2540 } else { 2541 size_t n_blocks, n_metadata_blocks; 2542 uint64_t n_bitmap_bits; 2543 2544 wc->memory_map_size -= (uint64_t)wc->start_sector << SECTOR_SHIFT; 2545 2546 spin_lock_init(&wc->flush_list_lock); 2547 bio_list_init(&wc->flush_list); 2548 wc->flush_thread = kthread_run(writecache_flush_thread, wc, "dm_writecache_flush"); 2549 if (IS_ERR(wc->flush_thread)) { 2550 r = PTR_ERR(wc->flush_thread); 2551 wc->flush_thread = NULL; 2552 ti->error = "Couldn't spawn flush thread"; 2553 goto bad; 2554 } 2555 2556 r = calculate_memory_size(wc->memory_map_size, wc->block_size, 2557 &n_blocks, &n_metadata_blocks); 2558 if (r) { 2559 ti->error = "Invalid device size"; 2560 goto bad; 2561 } 2562 2563 n_bitmap_bits = (((uint64_t)n_metadata_blocks << wc->block_size_bits) + 2564 BITMAP_GRANULARITY - 1) / BITMAP_GRANULARITY; 2565 /* this is limitation of test_bit functions */ 2566 if (n_bitmap_bits > 1U << 31) { 2567 r = -EFBIG; 2568 ti->error = "Invalid device size"; 2569 goto bad; 2570 } 2571 2572 wc->memory_map = vmalloc(n_metadata_blocks << wc->block_size_bits); 2573 if (!wc->memory_map) { 2574 r = -ENOMEM; 2575 ti->error = "Unable to allocate memory for metadata"; 2576 goto bad; 2577 } 2578 2579 wc->dm_kcopyd = dm_kcopyd_client_create(&dm_kcopyd_throttle); 2580 if (IS_ERR(wc->dm_kcopyd)) { 2581 r = PTR_ERR(wc->dm_kcopyd); 2582 ti->error = "Unable to allocate dm-kcopyd client"; 2583 wc->dm_kcopyd = NULL; 2584 goto bad; 2585 } 2586 2587 wc->metadata_sectors = n_metadata_blocks << (wc->block_size_bits - SECTOR_SHIFT); 2588 wc->dirty_bitmap_size = (n_bitmap_bits + BITS_PER_LONG - 1) / 2589 BITS_PER_LONG * sizeof(unsigned long); 2590 wc->dirty_bitmap = vzalloc(wc->dirty_bitmap_size); 2591 if (!wc->dirty_bitmap) { 2592 r = -ENOMEM; 2593 ti->error = "Unable to allocate dirty bitmap"; 2594 goto bad; 2595 } 2596 2597 r = writecache_read_metadata(wc, wc->block_size >> SECTOR_SHIFT); 2598 if (r) { 2599 ti->error = "Unable to read first block of metadata"; 2600 goto bad; 2601 } 2602 } 2603 2604 r = copy_mc_to_kernel(&s, sb(wc), sizeof(struct wc_memory_superblock)); 2605 if (r) { 2606 ti->error = "Hardware memory error when reading superblock"; 2607 goto bad; 2608 } 2609 if (!le32_to_cpu(s.magic) && !le32_to_cpu(s.version)) { 2610 r = init_memory(wc); 2611 if (r) { 2612 ti->error = "Unable to initialize device"; 2613 goto bad; 2614 } 2615 r = copy_mc_to_kernel(&s, sb(wc), 2616 sizeof(struct wc_memory_superblock)); 2617 if (r) { 2618 ti->error = "Hardware memory error when reading superblock"; 2619 goto bad; 2620 } 2621 } 2622 2623 if (le32_to_cpu(s.magic) != MEMORY_SUPERBLOCK_MAGIC) { 2624 ti->error = "Invalid magic in the superblock"; 2625 r = -EINVAL; 2626 goto bad; 2627 } 2628 2629 if (le32_to_cpu(s.version) != MEMORY_SUPERBLOCK_VERSION) { 2630 ti->error = "Invalid version in the superblock"; 2631 r = -EINVAL; 2632 goto bad; 2633 } 2634 2635 if (le32_to_cpu(s.block_size) != wc->block_size) { 2636 ti->error = "Block size does not match superblock"; 2637 r = -EINVAL; 2638 goto bad; 2639 } 2640 2641 wc->n_blocks = le64_to_cpu(s.n_blocks); 2642 2643 offset = wc->n_blocks * sizeof(struct wc_memory_entry); 2644 if (offset / sizeof(struct wc_memory_entry) != le64_to_cpu(sb(wc)->n_blocks)) { 2645 overflow: 2646 ti->error = "Overflow in size calculation"; 2647 r = -EINVAL; 2648 goto bad; 2649 } 2650 offset += sizeof(struct wc_memory_superblock); 2651 if (offset < sizeof(struct wc_memory_superblock)) 2652 goto overflow; 2653 offset = (offset + wc->block_size - 1) & ~(size_t)(wc->block_size - 1); 2654 data_size = wc->n_blocks * (size_t)wc->block_size; 2655 if (!offset || (data_size / wc->block_size != wc->n_blocks) || 2656 (offset + data_size < offset)) 2657 goto overflow; 2658 if (offset + data_size > wc->memory_map_size) { 2659 ti->error = "Memory area is too small"; 2660 r = -EINVAL; 2661 goto bad; 2662 } 2663 2664 wc->metadata_sectors = offset >> SECTOR_SHIFT; 2665 wc->block_start = (char *)sb(wc) + offset; 2666 2667 x = (uint64_t)wc->n_blocks * (100 - high_wm_percent); 2668 x += 50; 2669 do_div(x, 100); 2670 wc->freelist_high_watermark = x; 2671 x = (uint64_t)wc->n_blocks * (100 - low_wm_percent); 2672 x += 50; 2673 do_div(x, 100); 2674 wc->freelist_low_watermark = x; 2675 2676 if (wc->cleaner) 2677 activate_cleaner(wc); 2678 2679 r = writecache_alloc_entries(wc); 2680 if (r) { 2681 ti->error = "Cannot allocate memory"; 2682 goto bad; 2683 } 2684 2685 ti->num_flush_bios = WC_MODE_PMEM(wc) ? 1 : 2; 2686 ti->flush_supported = true; 2687 ti->num_discard_bios = 1; 2688 2689 if (WC_MODE_PMEM(wc)) 2690 persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size); 2691 2692 return 0; 2693 2694 bad_arguments: 2695 r = -EINVAL; 2696 ti->error = "Bad arguments"; 2697 bad: 2698 writecache_dtr(ti); 2699 return r; 2700 } 2701 2702 static void writecache_status(struct dm_target *ti, status_type_t type, 2703 unsigned int status_flags, char *result, unsigned int maxlen) 2704 { 2705 struct dm_writecache *wc = ti->private; 2706 unsigned int extra_args; 2707 unsigned int sz = 0; 2708 2709 switch (type) { 2710 case STATUSTYPE_INFO: 2711 DMEMIT("%ld %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu", 2712 writecache_has_error(wc), 2713 (unsigned long long)wc->n_blocks, (unsigned long long)wc->freelist_size, 2714 (unsigned long long)wc->writeback_size, 2715 wc->stats.reads, 2716 wc->stats.read_hits, 2717 wc->stats.writes, 2718 wc->stats.write_hits_uncommitted, 2719 wc->stats.write_hits_committed, 2720 wc->stats.writes_around, 2721 wc->stats.writes_allocate, 2722 wc->stats.writes_blocked_on_freelist, 2723 wc->stats.flushes, 2724 wc->stats.discards); 2725 break; 2726 case STATUSTYPE_TABLE: 2727 DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's', 2728 wc->dev->name, wc->ssd_dev->name, wc->block_size); 2729 extra_args = 0; 2730 if (wc->start_sector_set) 2731 extra_args += 2; 2732 if (wc->high_wm_percent_set) 2733 extra_args += 2; 2734 if (wc->low_wm_percent_set) 2735 extra_args += 2; 2736 if (wc->max_writeback_jobs_set) 2737 extra_args += 2; 2738 if (wc->autocommit_blocks_set) 2739 extra_args += 2; 2740 if (wc->autocommit_time_set) 2741 extra_args += 2; 2742 if (wc->max_age_set) 2743 extra_args += 2; 2744 if (wc->cleaner_set) 2745 extra_args++; 2746 if (wc->writeback_fua_set) 2747 extra_args++; 2748 if (wc->metadata_only) 2749 extra_args++; 2750 if (wc->pause_set) 2751 extra_args += 2; 2752 2753 DMEMIT("%u", extra_args); 2754 if (wc->start_sector_set) 2755 DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector); 2756 if (wc->high_wm_percent_set) 2757 DMEMIT(" high_watermark %u", wc->high_wm_percent_value); 2758 if (wc->low_wm_percent_set) 2759 DMEMIT(" low_watermark %u", wc->low_wm_percent_value); 2760 if (wc->max_writeback_jobs_set) 2761 DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs); 2762 if (wc->autocommit_blocks_set) 2763 DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks); 2764 if (wc->autocommit_time_set) 2765 DMEMIT(" autocommit_time %u", wc->autocommit_time_value); 2766 if (wc->max_age_set) 2767 DMEMIT(" max_age %u", wc->max_age_value); 2768 if (wc->cleaner_set) 2769 DMEMIT(" cleaner"); 2770 if (wc->writeback_fua_set) 2771 DMEMIT(" %sfua", wc->writeback_fua ? "" : "no"); 2772 if (wc->metadata_only) 2773 DMEMIT(" metadata_only"); 2774 if (wc->pause_set) 2775 DMEMIT(" pause_writeback %u", wc->pause_value); 2776 break; 2777 case STATUSTYPE_IMA: 2778 *result = '\0'; 2779 break; 2780 } 2781 } 2782 2783 static struct target_type writecache_target = { 2784 .name = "writecache", 2785 .version = {1, 6, 0}, 2786 .module = THIS_MODULE, 2787 .ctr = writecache_ctr, 2788 .dtr = writecache_dtr, 2789 .status = writecache_status, 2790 .postsuspend = writecache_suspend, 2791 .resume = writecache_resume, 2792 .message = writecache_message, 2793 .map = writecache_map, 2794 .end_io = writecache_end_io, 2795 .iterate_devices = writecache_iterate_devices, 2796 .io_hints = writecache_io_hints, 2797 }; 2798 module_dm(writecache); 2799 2800 MODULE_DESCRIPTION(DM_NAME " writecache target"); 2801 MODULE_AUTHOR("Mikulas Patocka <dm-devel@lists.linux.dev>"); 2802 MODULE_LICENSE("GPL"); 2803