1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2018 Red Hat. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include <linux/device-mapper.h> 9 #include <linux/module.h> 10 #include <linux/init.h> 11 #include <linux/vmalloc.h> 12 #include <linux/kthread.h> 13 #include <linux/dm-io.h> 14 #include <linux/dm-kcopyd.h> 15 #include <linux/dax.h> 16 #include <linux/pfn_t.h> 17 #include <linux/libnvdimm.h> 18 19 #define DM_MSG_PREFIX "writecache" 20 21 #define HIGH_WATERMARK 50 22 #define LOW_WATERMARK 45 23 #define MAX_WRITEBACK_JOBS 0 24 #define ENDIO_LATENCY 16 25 #define WRITEBACK_LATENCY 64 26 #define AUTOCOMMIT_BLOCKS_SSD 65536 27 #define AUTOCOMMIT_BLOCKS_PMEM 64 28 #define AUTOCOMMIT_MSEC 1000 29 #define MAX_AGE_DIV 16 30 #define MAX_AGE_UNSPECIFIED -1UL 31 32 #define BITMAP_GRANULARITY 65536 33 #if BITMAP_GRANULARITY < PAGE_SIZE 34 #undef BITMAP_GRANULARITY 35 #define BITMAP_GRANULARITY PAGE_SIZE 36 #endif 37 38 #if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_DAX_DRIVER) 39 #define DM_WRITECACHE_HAS_PMEM 40 #endif 41 42 #ifdef DM_WRITECACHE_HAS_PMEM 43 #define pmem_assign(dest, src) \ 44 do { \ 45 typeof(dest) uniq = (src); \ 46 memcpy_flushcache(&(dest), &uniq, sizeof(dest)); \ 47 } while (0) 48 #else 49 #define pmem_assign(dest, src) ((dest) = (src)) 50 #endif 51 52 #if IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC) && defined(DM_WRITECACHE_HAS_PMEM) 53 #define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 54 #endif 55 56 #define MEMORY_SUPERBLOCK_MAGIC 0x23489321 57 #define MEMORY_SUPERBLOCK_VERSION 1 58 59 struct wc_memory_entry { 60 __le64 original_sector; 61 __le64 seq_count; 62 }; 63 64 struct wc_memory_superblock { 65 union { 66 struct { 67 __le32 magic; 68 __le32 version; 69 __le32 block_size; 70 __le32 pad; 71 __le64 n_blocks; 72 __le64 seq_count; 73 }; 74 __le64 padding[8]; 75 }; 76 struct wc_memory_entry entries[0]; 77 }; 78 79 struct wc_entry { 80 struct rb_node rb_node; 81 struct list_head lru; 82 unsigned short wc_list_contiguous; 83 bool write_in_progress 84 #if BITS_PER_LONG == 64 85 :1 86 #endif 87 ; 88 unsigned long index 89 #if BITS_PER_LONG == 64 90 :47 91 #endif 92 ; 93 unsigned long age; 94 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 95 uint64_t original_sector; 96 uint64_t seq_count; 97 #endif 98 }; 99 100 #ifdef DM_WRITECACHE_HAS_PMEM 101 #define WC_MODE_PMEM(wc) ((wc)->pmem_mode) 102 #define WC_MODE_FUA(wc) ((wc)->writeback_fua) 103 #else 104 #define WC_MODE_PMEM(wc) false 105 #define WC_MODE_FUA(wc) false 106 #endif 107 #define WC_MODE_SORT_FREELIST(wc) (!WC_MODE_PMEM(wc)) 108 109 struct dm_writecache { 110 struct mutex lock; 111 struct list_head lru; 112 union { 113 struct list_head freelist; 114 struct { 115 struct rb_root freetree; 116 struct wc_entry *current_free; 117 }; 118 }; 119 struct rb_root tree; 120 121 size_t freelist_size; 122 size_t writeback_size; 123 size_t freelist_high_watermark; 124 size_t freelist_low_watermark; 125 unsigned long max_age; 126 127 unsigned uncommitted_blocks; 128 unsigned autocommit_blocks; 129 unsigned max_writeback_jobs; 130 131 int error; 132 133 unsigned long autocommit_jiffies; 134 struct timer_list autocommit_timer; 135 struct wait_queue_head freelist_wait; 136 137 struct timer_list max_age_timer; 138 139 atomic_t bio_in_progress[2]; 140 struct wait_queue_head bio_in_progress_wait[2]; 141 142 struct dm_target *ti; 143 struct dm_dev *dev; 144 struct dm_dev *ssd_dev; 145 sector_t start_sector; 146 void *memory_map; 147 uint64_t memory_map_size; 148 size_t metadata_sectors; 149 size_t n_blocks; 150 uint64_t seq_count; 151 void *block_start; 152 struct wc_entry *entries; 153 unsigned block_size; 154 unsigned char block_size_bits; 155 156 bool pmem_mode:1; 157 bool writeback_fua:1; 158 159 bool overwrote_committed:1; 160 bool memory_vmapped:1; 161 162 bool high_wm_percent_set:1; 163 bool low_wm_percent_set:1; 164 bool max_writeback_jobs_set:1; 165 bool autocommit_blocks_set:1; 166 bool autocommit_time_set:1; 167 bool writeback_fua_set:1; 168 bool flush_on_suspend:1; 169 bool cleaner:1; 170 171 unsigned writeback_all; 172 struct workqueue_struct *writeback_wq; 173 struct work_struct writeback_work; 174 struct work_struct flush_work; 175 176 struct dm_io_client *dm_io; 177 178 raw_spinlock_t endio_list_lock; 179 struct list_head endio_list; 180 struct task_struct *endio_thread; 181 182 struct task_struct *flush_thread; 183 struct bio_list flush_list; 184 185 struct dm_kcopyd_client *dm_kcopyd; 186 unsigned long *dirty_bitmap; 187 unsigned dirty_bitmap_size; 188 189 struct bio_set bio_set; 190 mempool_t copy_pool; 191 }; 192 193 #define WB_LIST_INLINE 16 194 195 struct writeback_struct { 196 struct list_head endio_entry; 197 struct dm_writecache *wc; 198 struct wc_entry **wc_list; 199 unsigned wc_list_n; 200 struct wc_entry *wc_list_inline[WB_LIST_INLINE]; 201 struct bio bio; 202 }; 203 204 struct copy_struct { 205 struct list_head endio_entry; 206 struct dm_writecache *wc; 207 struct wc_entry *e; 208 unsigned n_entries; 209 int error; 210 }; 211 212 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle, 213 "A percentage of time allocated for data copying"); 214 215 static void wc_lock(struct dm_writecache *wc) 216 { 217 mutex_lock(&wc->lock); 218 } 219 220 static void wc_unlock(struct dm_writecache *wc) 221 { 222 mutex_unlock(&wc->lock); 223 } 224 225 #ifdef DM_WRITECACHE_HAS_PMEM 226 static int persistent_memory_claim(struct dm_writecache *wc) 227 { 228 int r; 229 loff_t s; 230 long p, da; 231 pfn_t pfn; 232 int id; 233 struct page **pages; 234 235 wc->memory_vmapped = false; 236 237 s = wc->memory_map_size; 238 p = s >> PAGE_SHIFT; 239 if (!p) { 240 r = -EINVAL; 241 goto err1; 242 } 243 if (p != s >> PAGE_SHIFT) { 244 r = -EOVERFLOW; 245 goto err1; 246 } 247 248 id = dax_read_lock(); 249 250 da = dax_direct_access(wc->ssd_dev->dax_dev, 0, p, &wc->memory_map, &pfn); 251 if (da < 0) { 252 wc->memory_map = NULL; 253 r = da; 254 goto err2; 255 } 256 if (!pfn_t_has_page(pfn)) { 257 wc->memory_map = NULL; 258 r = -EOPNOTSUPP; 259 goto err2; 260 } 261 if (da != p) { 262 long i; 263 wc->memory_map = NULL; 264 pages = kvmalloc_array(p, sizeof(struct page *), GFP_KERNEL); 265 if (!pages) { 266 r = -ENOMEM; 267 goto err2; 268 } 269 i = 0; 270 do { 271 long daa; 272 daa = dax_direct_access(wc->ssd_dev->dax_dev, i, p - i, 273 NULL, &pfn); 274 if (daa <= 0) { 275 r = daa ? daa : -EINVAL; 276 goto err3; 277 } 278 if (!pfn_t_has_page(pfn)) { 279 r = -EOPNOTSUPP; 280 goto err3; 281 } 282 while (daa-- && i < p) { 283 pages[i++] = pfn_t_to_page(pfn); 284 pfn.val++; 285 if (!(i & 15)) 286 cond_resched(); 287 } 288 } while (i < p); 289 wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL); 290 if (!wc->memory_map) { 291 r = -ENOMEM; 292 goto err3; 293 } 294 kvfree(pages); 295 wc->memory_vmapped = true; 296 } 297 298 dax_read_unlock(id); 299 300 wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT; 301 wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT; 302 303 return 0; 304 err3: 305 kvfree(pages); 306 err2: 307 dax_read_unlock(id); 308 err1: 309 return r; 310 } 311 #else 312 static int persistent_memory_claim(struct dm_writecache *wc) 313 { 314 BUG(); 315 } 316 #endif 317 318 static void persistent_memory_release(struct dm_writecache *wc) 319 { 320 if (wc->memory_vmapped) 321 vunmap(wc->memory_map - ((size_t)wc->start_sector << SECTOR_SHIFT)); 322 } 323 324 static struct page *persistent_memory_page(void *addr) 325 { 326 if (is_vmalloc_addr(addr)) 327 return vmalloc_to_page(addr); 328 else 329 return virt_to_page(addr); 330 } 331 332 static unsigned persistent_memory_page_offset(void *addr) 333 { 334 return (unsigned long)addr & (PAGE_SIZE - 1); 335 } 336 337 static void persistent_memory_flush_cache(void *ptr, size_t size) 338 { 339 if (is_vmalloc_addr(ptr)) 340 flush_kernel_vmap_range(ptr, size); 341 } 342 343 static void persistent_memory_invalidate_cache(void *ptr, size_t size) 344 { 345 if (is_vmalloc_addr(ptr)) 346 invalidate_kernel_vmap_range(ptr, size); 347 } 348 349 static struct wc_memory_superblock *sb(struct dm_writecache *wc) 350 { 351 return wc->memory_map; 352 } 353 354 static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e) 355 { 356 return &sb(wc)->entries[e->index]; 357 } 358 359 static void *memory_data(struct dm_writecache *wc, struct wc_entry *e) 360 { 361 return (char *)wc->block_start + (e->index << wc->block_size_bits); 362 } 363 364 static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e) 365 { 366 return wc->start_sector + wc->metadata_sectors + 367 ((sector_t)e->index << (wc->block_size_bits - SECTOR_SHIFT)); 368 } 369 370 static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e) 371 { 372 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 373 return e->original_sector; 374 #else 375 return le64_to_cpu(memory_entry(wc, e)->original_sector); 376 #endif 377 } 378 379 static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e) 380 { 381 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 382 return e->seq_count; 383 #else 384 return le64_to_cpu(memory_entry(wc, e)->seq_count); 385 #endif 386 } 387 388 static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e) 389 { 390 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 391 e->seq_count = -1; 392 #endif 393 pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1)); 394 } 395 396 static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e, 397 uint64_t original_sector, uint64_t seq_count) 398 { 399 struct wc_memory_entry me; 400 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 401 e->original_sector = original_sector; 402 e->seq_count = seq_count; 403 #endif 404 me.original_sector = cpu_to_le64(original_sector); 405 me.seq_count = cpu_to_le64(seq_count); 406 pmem_assign(*memory_entry(wc, e), me); 407 } 408 409 #define writecache_error(wc, err, msg, arg...) \ 410 do { \ 411 if (!cmpxchg(&(wc)->error, 0, err)) \ 412 DMERR(msg, ##arg); \ 413 wake_up(&(wc)->freelist_wait); \ 414 } while (0) 415 416 #define writecache_has_error(wc) (unlikely(READ_ONCE((wc)->error))) 417 418 static void writecache_flush_all_metadata(struct dm_writecache *wc) 419 { 420 if (!WC_MODE_PMEM(wc)) 421 memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size); 422 } 423 424 static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size) 425 { 426 if (!WC_MODE_PMEM(wc)) 427 __set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY, 428 wc->dirty_bitmap); 429 } 430 431 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev); 432 433 struct io_notify { 434 struct dm_writecache *wc; 435 struct completion c; 436 atomic_t count; 437 }; 438 439 static void writecache_notify_io(unsigned long error, void *context) 440 { 441 struct io_notify *endio = context; 442 443 if (unlikely(error != 0)) 444 writecache_error(endio->wc, -EIO, "error writing metadata"); 445 BUG_ON(atomic_read(&endio->count) <= 0); 446 if (atomic_dec_and_test(&endio->count)) 447 complete(&endio->c); 448 } 449 450 static void writecache_wait_for_ios(struct dm_writecache *wc, int direction) 451 { 452 wait_event(wc->bio_in_progress_wait[direction], 453 !atomic_read(&wc->bio_in_progress[direction])); 454 } 455 456 static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) 457 { 458 struct dm_io_region region; 459 struct dm_io_request req; 460 struct io_notify endio = { 461 wc, 462 COMPLETION_INITIALIZER_ONSTACK(endio.c), 463 ATOMIC_INIT(1), 464 }; 465 unsigned bitmap_bits = wc->dirty_bitmap_size * 8; 466 unsigned i = 0; 467 468 while (1) { 469 unsigned j; 470 i = find_next_bit(wc->dirty_bitmap, bitmap_bits, i); 471 if (unlikely(i == bitmap_bits)) 472 break; 473 j = find_next_zero_bit(wc->dirty_bitmap, bitmap_bits, i); 474 475 region.bdev = wc->ssd_dev->bdev; 476 region.sector = (sector_t)i * (BITMAP_GRANULARITY >> SECTOR_SHIFT); 477 region.count = (sector_t)(j - i) * (BITMAP_GRANULARITY >> SECTOR_SHIFT); 478 479 if (unlikely(region.sector >= wc->metadata_sectors)) 480 break; 481 if (unlikely(region.sector + region.count > wc->metadata_sectors)) 482 region.count = wc->metadata_sectors - region.sector; 483 484 region.sector += wc->start_sector; 485 atomic_inc(&endio.count); 486 req.bi_op = REQ_OP_WRITE; 487 req.bi_op_flags = REQ_SYNC; 488 req.mem.type = DM_IO_VMA; 489 req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY; 490 req.client = wc->dm_io; 491 req.notify.fn = writecache_notify_io; 492 req.notify.context = &endio; 493 494 /* writing via async dm-io (implied by notify.fn above) won't return an error */ 495 (void) dm_io(&req, 1, ®ion, NULL); 496 i = j; 497 } 498 499 writecache_notify_io(0, &endio); 500 wait_for_completion_io(&endio.c); 501 502 if (wait_for_ios) 503 writecache_wait_for_ios(wc, WRITE); 504 505 writecache_disk_flush(wc, wc->ssd_dev); 506 507 memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size); 508 } 509 510 static void ssd_commit_superblock(struct dm_writecache *wc) 511 { 512 int r; 513 struct dm_io_region region; 514 struct dm_io_request req; 515 516 region.bdev = wc->ssd_dev->bdev; 517 region.sector = 0; 518 region.count = PAGE_SIZE; 519 520 if (unlikely(region.sector + region.count > wc->metadata_sectors)) 521 region.count = wc->metadata_sectors - region.sector; 522 523 region.sector += wc->start_sector; 524 525 req.bi_op = REQ_OP_WRITE; 526 req.bi_op_flags = REQ_SYNC | REQ_FUA; 527 req.mem.type = DM_IO_VMA; 528 req.mem.ptr.vma = (char *)wc->memory_map; 529 req.client = wc->dm_io; 530 req.notify.fn = NULL; 531 req.notify.context = NULL; 532 533 r = dm_io(&req, 1, ®ion, NULL); 534 if (unlikely(r)) 535 writecache_error(wc, r, "error writing superblock"); 536 } 537 538 static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) 539 { 540 if (WC_MODE_PMEM(wc)) 541 pmem_wmb(); 542 else 543 ssd_commit_flushed(wc, wait_for_ios); 544 } 545 546 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev) 547 { 548 int r; 549 struct dm_io_region region; 550 struct dm_io_request req; 551 552 region.bdev = dev->bdev; 553 region.sector = 0; 554 region.count = 0; 555 req.bi_op = REQ_OP_WRITE; 556 req.bi_op_flags = REQ_PREFLUSH; 557 req.mem.type = DM_IO_KMEM; 558 req.mem.ptr.addr = NULL; 559 req.client = wc->dm_io; 560 req.notify.fn = NULL; 561 562 r = dm_io(&req, 1, ®ion, NULL); 563 if (unlikely(r)) 564 writecache_error(wc, r, "error flushing metadata: %d", r); 565 } 566 567 #define WFE_RETURN_FOLLOWING 1 568 #define WFE_LOWEST_SEQ 2 569 570 static struct wc_entry *writecache_find_entry(struct dm_writecache *wc, 571 uint64_t block, int flags) 572 { 573 struct wc_entry *e; 574 struct rb_node *node = wc->tree.rb_node; 575 576 if (unlikely(!node)) 577 return NULL; 578 579 while (1) { 580 e = container_of(node, struct wc_entry, rb_node); 581 if (read_original_sector(wc, e) == block) 582 break; 583 584 node = (read_original_sector(wc, e) >= block ? 585 e->rb_node.rb_left : e->rb_node.rb_right); 586 if (unlikely(!node)) { 587 if (!(flags & WFE_RETURN_FOLLOWING)) 588 return NULL; 589 if (read_original_sector(wc, e) >= block) { 590 return e; 591 } else { 592 node = rb_next(&e->rb_node); 593 if (unlikely(!node)) 594 return NULL; 595 e = container_of(node, struct wc_entry, rb_node); 596 return e; 597 } 598 } 599 } 600 601 while (1) { 602 struct wc_entry *e2; 603 if (flags & WFE_LOWEST_SEQ) 604 node = rb_prev(&e->rb_node); 605 else 606 node = rb_next(&e->rb_node); 607 if (unlikely(!node)) 608 return e; 609 e2 = container_of(node, struct wc_entry, rb_node); 610 if (read_original_sector(wc, e2) != block) 611 return e; 612 e = e2; 613 } 614 } 615 616 static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *ins) 617 { 618 struct wc_entry *e; 619 struct rb_node **node = &wc->tree.rb_node, *parent = NULL; 620 621 while (*node) { 622 e = container_of(*node, struct wc_entry, rb_node); 623 parent = &e->rb_node; 624 if (read_original_sector(wc, e) > read_original_sector(wc, ins)) 625 node = &parent->rb_left; 626 else 627 node = &parent->rb_right; 628 } 629 rb_link_node(&ins->rb_node, parent, node); 630 rb_insert_color(&ins->rb_node, &wc->tree); 631 list_add(&ins->lru, &wc->lru); 632 ins->age = jiffies; 633 } 634 635 static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e) 636 { 637 list_del(&e->lru); 638 rb_erase(&e->rb_node, &wc->tree); 639 } 640 641 static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e) 642 { 643 if (WC_MODE_SORT_FREELIST(wc)) { 644 struct rb_node **node = &wc->freetree.rb_node, *parent = NULL; 645 if (unlikely(!*node)) 646 wc->current_free = e; 647 while (*node) { 648 parent = *node; 649 if (&e->rb_node < *node) 650 node = &parent->rb_left; 651 else 652 node = &parent->rb_right; 653 } 654 rb_link_node(&e->rb_node, parent, node); 655 rb_insert_color(&e->rb_node, &wc->freetree); 656 } else { 657 list_add_tail(&e->lru, &wc->freelist); 658 } 659 wc->freelist_size++; 660 } 661 662 static inline void writecache_verify_watermark(struct dm_writecache *wc) 663 { 664 if (unlikely(wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark)) 665 queue_work(wc->writeback_wq, &wc->writeback_work); 666 } 667 668 static void writecache_max_age_timer(struct timer_list *t) 669 { 670 struct dm_writecache *wc = from_timer(wc, t, max_age_timer); 671 672 if (!dm_suspended(wc->ti) && !writecache_has_error(wc)) { 673 queue_work(wc->writeback_wq, &wc->writeback_work); 674 mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV); 675 } 676 } 677 678 static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector) 679 { 680 struct wc_entry *e; 681 682 if (WC_MODE_SORT_FREELIST(wc)) { 683 struct rb_node *next; 684 if (unlikely(!wc->current_free)) 685 return NULL; 686 e = wc->current_free; 687 if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector)) 688 return NULL; 689 next = rb_next(&e->rb_node); 690 rb_erase(&e->rb_node, &wc->freetree); 691 if (unlikely(!next)) 692 next = rb_first(&wc->freetree); 693 wc->current_free = next ? container_of(next, struct wc_entry, rb_node) : NULL; 694 } else { 695 if (unlikely(list_empty(&wc->freelist))) 696 return NULL; 697 e = container_of(wc->freelist.next, struct wc_entry, lru); 698 if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector)) 699 return NULL; 700 list_del(&e->lru); 701 } 702 wc->freelist_size--; 703 704 writecache_verify_watermark(wc); 705 706 return e; 707 } 708 709 static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e) 710 { 711 writecache_unlink(wc, e); 712 writecache_add_to_freelist(wc, e); 713 clear_seq_count(wc, e); 714 writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry)); 715 if (unlikely(waitqueue_active(&wc->freelist_wait))) 716 wake_up(&wc->freelist_wait); 717 } 718 719 static void writecache_wait_on_freelist(struct dm_writecache *wc) 720 { 721 DEFINE_WAIT(wait); 722 723 prepare_to_wait(&wc->freelist_wait, &wait, TASK_UNINTERRUPTIBLE); 724 wc_unlock(wc); 725 io_schedule(); 726 finish_wait(&wc->freelist_wait, &wait); 727 wc_lock(wc); 728 } 729 730 static void writecache_poison_lists(struct dm_writecache *wc) 731 { 732 /* 733 * Catch incorrect access to these values while the device is suspended. 734 */ 735 memset(&wc->tree, -1, sizeof wc->tree); 736 wc->lru.next = LIST_POISON1; 737 wc->lru.prev = LIST_POISON2; 738 wc->freelist.next = LIST_POISON1; 739 wc->freelist.prev = LIST_POISON2; 740 } 741 742 static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e) 743 { 744 writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry)); 745 if (WC_MODE_PMEM(wc)) 746 writecache_flush_region(wc, memory_data(wc, e), wc->block_size); 747 } 748 749 static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e) 750 { 751 return read_seq_count(wc, e) < wc->seq_count; 752 } 753 754 static void writecache_flush(struct dm_writecache *wc) 755 { 756 struct wc_entry *e, *e2; 757 bool need_flush_after_free; 758 759 wc->uncommitted_blocks = 0; 760 del_timer(&wc->autocommit_timer); 761 762 if (list_empty(&wc->lru)) 763 return; 764 765 e = container_of(wc->lru.next, struct wc_entry, lru); 766 if (writecache_entry_is_committed(wc, e)) { 767 if (wc->overwrote_committed) { 768 writecache_wait_for_ios(wc, WRITE); 769 writecache_disk_flush(wc, wc->ssd_dev); 770 wc->overwrote_committed = false; 771 } 772 return; 773 } 774 while (1) { 775 writecache_flush_entry(wc, e); 776 if (unlikely(e->lru.next == &wc->lru)) 777 break; 778 e2 = container_of(e->lru.next, struct wc_entry, lru); 779 if (writecache_entry_is_committed(wc, e2)) 780 break; 781 e = e2; 782 cond_resched(); 783 } 784 writecache_commit_flushed(wc, true); 785 786 wc->seq_count++; 787 pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count)); 788 if (WC_MODE_PMEM(wc)) 789 writecache_commit_flushed(wc, false); 790 else 791 ssd_commit_superblock(wc); 792 793 wc->overwrote_committed = false; 794 795 need_flush_after_free = false; 796 while (1) { 797 /* Free another committed entry with lower seq-count */ 798 struct rb_node *rb_node = rb_prev(&e->rb_node); 799 800 if (rb_node) { 801 e2 = container_of(rb_node, struct wc_entry, rb_node); 802 if (read_original_sector(wc, e2) == read_original_sector(wc, e) && 803 likely(!e2->write_in_progress)) { 804 writecache_free_entry(wc, e2); 805 need_flush_after_free = true; 806 } 807 } 808 if (unlikely(e->lru.prev == &wc->lru)) 809 break; 810 e = container_of(e->lru.prev, struct wc_entry, lru); 811 cond_resched(); 812 } 813 814 if (need_flush_after_free) 815 writecache_commit_flushed(wc, false); 816 } 817 818 static void writecache_flush_work(struct work_struct *work) 819 { 820 struct dm_writecache *wc = container_of(work, struct dm_writecache, flush_work); 821 822 wc_lock(wc); 823 writecache_flush(wc); 824 wc_unlock(wc); 825 } 826 827 static void writecache_autocommit_timer(struct timer_list *t) 828 { 829 struct dm_writecache *wc = from_timer(wc, t, autocommit_timer); 830 if (!writecache_has_error(wc)) 831 queue_work(wc->writeback_wq, &wc->flush_work); 832 } 833 834 static void writecache_schedule_autocommit(struct dm_writecache *wc) 835 { 836 if (!timer_pending(&wc->autocommit_timer)) 837 mod_timer(&wc->autocommit_timer, jiffies + wc->autocommit_jiffies); 838 } 839 840 static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_t end) 841 { 842 struct wc_entry *e; 843 bool discarded_something = false; 844 845 e = writecache_find_entry(wc, start, WFE_RETURN_FOLLOWING | WFE_LOWEST_SEQ); 846 if (unlikely(!e)) 847 return; 848 849 while (read_original_sector(wc, e) < end) { 850 struct rb_node *node = rb_next(&e->rb_node); 851 852 if (likely(!e->write_in_progress)) { 853 if (!discarded_something) { 854 if (!WC_MODE_PMEM(wc)) { 855 writecache_wait_for_ios(wc, READ); 856 writecache_wait_for_ios(wc, WRITE); 857 } 858 discarded_something = true; 859 } 860 if (!writecache_entry_is_committed(wc, e)) 861 wc->uncommitted_blocks--; 862 writecache_free_entry(wc, e); 863 } 864 865 if (unlikely(!node)) 866 break; 867 868 e = container_of(node, struct wc_entry, rb_node); 869 } 870 871 if (discarded_something) 872 writecache_commit_flushed(wc, false); 873 } 874 875 static bool writecache_wait_for_writeback(struct dm_writecache *wc) 876 { 877 if (wc->writeback_size) { 878 writecache_wait_on_freelist(wc); 879 return true; 880 } 881 return false; 882 } 883 884 static void writecache_suspend(struct dm_target *ti) 885 { 886 struct dm_writecache *wc = ti->private; 887 bool flush_on_suspend; 888 889 del_timer_sync(&wc->autocommit_timer); 890 del_timer_sync(&wc->max_age_timer); 891 892 wc_lock(wc); 893 writecache_flush(wc); 894 flush_on_suspend = wc->flush_on_suspend; 895 if (flush_on_suspend) { 896 wc->flush_on_suspend = false; 897 wc->writeback_all++; 898 queue_work(wc->writeback_wq, &wc->writeback_work); 899 } 900 wc_unlock(wc); 901 902 drain_workqueue(wc->writeback_wq); 903 904 wc_lock(wc); 905 if (flush_on_suspend) 906 wc->writeback_all--; 907 while (writecache_wait_for_writeback(wc)); 908 909 if (WC_MODE_PMEM(wc)) 910 persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size); 911 912 writecache_poison_lists(wc); 913 914 wc_unlock(wc); 915 } 916 917 static int writecache_alloc_entries(struct dm_writecache *wc) 918 { 919 size_t b; 920 921 if (wc->entries) 922 return 0; 923 wc->entries = vmalloc(array_size(sizeof(struct wc_entry), wc->n_blocks)); 924 if (!wc->entries) 925 return -ENOMEM; 926 for (b = 0; b < wc->n_blocks; b++) { 927 struct wc_entry *e = &wc->entries[b]; 928 e->index = b; 929 e->write_in_progress = false; 930 cond_resched(); 931 } 932 933 return 0; 934 } 935 936 static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors) 937 { 938 struct dm_io_region region; 939 struct dm_io_request req; 940 941 region.bdev = wc->ssd_dev->bdev; 942 region.sector = wc->start_sector; 943 region.count = n_sectors; 944 req.bi_op = REQ_OP_READ; 945 req.bi_op_flags = REQ_SYNC; 946 req.mem.type = DM_IO_VMA; 947 req.mem.ptr.vma = (char *)wc->memory_map; 948 req.client = wc->dm_io; 949 req.notify.fn = NULL; 950 951 return dm_io(&req, 1, ®ion, NULL); 952 } 953 954 static void writecache_resume(struct dm_target *ti) 955 { 956 struct dm_writecache *wc = ti->private; 957 size_t b; 958 bool need_flush = false; 959 __le64 sb_seq_count; 960 int r; 961 962 wc_lock(wc); 963 964 if (WC_MODE_PMEM(wc)) { 965 persistent_memory_invalidate_cache(wc->memory_map, wc->memory_map_size); 966 } else { 967 r = writecache_read_metadata(wc, wc->metadata_sectors); 968 if (r) { 969 size_t sb_entries_offset; 970 writecache_error(wc, r, "unable to read metadata: %d", r); 971 sb_entries_offset = offsetof(struct wc_memory_superblock, entries); 972 memset((char *)wc->memory_map + sb_entries_offset, -1, 973 (wc->metadata_sectors << SECTOR_SHIFT) - sb_entries_offset); 974 } 975 } 976 977 wc->tree = RB_ROOT; 978 INIT_LIST_HEAD(&wc->lru); 979 if (WC_MODE_SORT_FREELIST(wc)) { 980 wc->freetree = RB_ROOT; 981 wc->current_free = NULL; 982 } else { 983 INIT_LIST_HEAD(&wc->freelist); 984 } 985 wc->freelist_size = 0; 986 987 r = copy_mc_to_kernel(&sb_seq_count, &sb(wc)->seq_count, 988 sizeof(uint64_t)); 989 if (r) { 990 writecache_error(wc, r, "hardware memory error when reading superblock: %d", r); 991 sb_seq_count = cpu_to_le64(0); 992 } 993 wc->seq_count = le64_to_cpu(sb_seq_count); 994 995 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS 996 for (b = 0; b < wc->n_blocks; b++) { 997 struct wc_entry *e = &wc->entries[b]; 998 struct wc_memory_entry wme; 999 if (writecache_has_error(wc)) { 1000 e->original_sector = -1; 1001 e->seq_count = -1; 1002 continue; 1003 } 1004 r = copy_mc_to_kernel(&wme, memory_entry(wc, e), 1005 sizeof(struct wc_memory_entry)); 1006 if (r) { 1007 writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d", 1008 (unsigned long)b, r); 1009 e->original_sector = -1; 1010 e->seq_count = -1; 1011 } else { 1012 e->original_sector = le64_to_cpu(wme.original_sector); 1013 e->seq_count = le64_to_cpu(wme.seq_count); 1014 } 1015 cond_resched(); 1016 } 1017 #endif 1018 for (b = 0; b < wc->n_blocks; b++) { 1019 struct wc_entry *e = &wc->entries[b]; 1020 if (!writecache_entry_is_committed(wc, e)) { 1021 if (read_seq_count(wc, e) != -1) { 1022 erase_this: 1023 clear_seq_count(wc, e); 1024 need_flush = true; 1025 } 1026 writecache_add_to_freelist(wc, e); 1027 } else { 1028 struct wc_entry *old; 1029 1030 old = writecache_find_entry(wc, read_original_sector(wc, e), 0); 1031 if (!old) { 1032 writecache_insert_entry(wc, e); 1033 } else { 1034 if (read_seq_count(wc, old) == read_seq_count(wc, e)) { 1035 writecache_error(wc, -EINVAL, 1036 "two identical entries, position %llu, sector %llu, sequence %llu", 1037 (unsigned long long)b, (unsigned long long)read_original_sector(wc, e), 1038 (unsigned long long)read_seq_count(wc, e)); 1039 } 1040 if (read_seq_count(wc, old) > read_seq_count(wc, e)) { 1041 goto erase_this; 1042 } else { 1043 writecache_free_entry(wc, old); 1044 writecache_insert_entry(wc, e); 1045 need_flush = true; 1046 } 1047 } 1048 } 1049 cond_resched(); 1050 } 1051 1052 if (need_flush) { 1053 writecache_flush_all_metadata(wc); 1054 writecache_commit_flushed(wc, false); 1055 } 1056 1057 writecache_verify_watermark(wc); 1058 1059 if (wc->max_age != MAX_AGE_UNSPECIFIED) 1060 mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV); 1061 1062 wc_unlock(wc); 1063 } 1064 1065 static int process_flush_mesg(unsigned argc, char **argv, struct dm_writecache *wc) 1066 { 1067 if (argc != 1) 1068 return -EINVAL; 1069 1070 wc_lock(wc); 1071 if (dm_suspended(wc->ti)) { 1072 wc_unlock(wc); 1073 return -EBUSY; 1074 } 1075 if (writecache_has_error(wc)) { 1076 wc_unlock(wc); 1077 return -EIO; 1078 } 1079 1080 writecache_flush(wc); 1081 wc->writeback_all++; 1082 queue_work(wc->writeback_wq, &wc->writeback_work); 1083 wc_unlock(wc); 1084 1085 flush_workqueue(wc->writeback_wq); 1086 1087 wc_lock(wc); 1088 wc->writeback_all--; 1089 if (writecache_has_error(wc)) { 1090 wc_unlock(wc); 1091 return -EIO; 1092 } 1093 wc_unlock(wc); 1094 1095 return 0; 1096 } 1097 1098 static int process_flush_on_suspend_mesg(unsigned argc, char **argv, struct dm_writecache *wc) 1099 { 1100 if (argc != 1) 1101 return -EINVAL; 1102 1103 wc_lock(wc); 1104 wc->flush_on_suspend = true; 1105 wc_unlock(wc); 1106 1107 return 0; 1108 } 1109 1110 static void activate_cleaner(struct dm_writecache *wc) 1111 { 1112 wc->flush_on_suspend = true; 1113 wc->cleaner = true; 1114 wc->freelist_high_watermark = wc->n_blocks; 1115 wc->freelist_low_watermark = wc->n_blocks; 1116 } 1117 1118 static int process_cleaner_mesg(unsigned argc, char **argv, struct dm_writecache *wc) 1119 { 1120 if (argc != 1) 1121 return -EINVAL; 1122 1123 wc_lock(wc); 1124 activate_cleaner(wc); 1125 if (!dm_suspended(wc->ti)) 1126 writecache_verify_watermark(wc); 1127 wc_unlock(wc); 1128 1129 return 0; 1130 } 1131 1132 static int writecache_message(struct dm_target *ti, unsigned argc, char **argv, 1133 char *result, unsigned maxlen) 1134 { 1135 int r = -EINVAL; 1136 struct dm_writecache *wc = ti->private; 1137 1138 if (!strcasecmp(argv[0], "flush")) 1139 r = process_flush_mesg(argc, argv, wc); 1140 else if (!strcasecmp(argv[0], "flush_on_suspend")) 1141 r = process_flush_on_suspend_mesg(argc, argv, wc); 1142 else if (!strcasecmp(argv[0], "cleaner")) 1143 r = process_cleaner_mesg(argc, argv, wc); 1144 else 1145 DMERR("unrecognised message received: %s", argv[0]); 1146 1147 return r; 1148 } 1149 1150 static void memcpy_flushcache_optimized(void *dest, void *source, size_t size) 1151 { 1152 /* 1153 * clflushopt performs better with block size 1024, 2048, 4096 1154 * non-temporal stores perform better with block size 512 1155 * 1156 * block size 512 1024 2048 4096 1157 * movnti 496 MB/s 642 MB/s 725 MB/s 744 MB/s 1158 * clflushopt 373 MB/s 688 MB/s 1.1 GB/s 1.2 GB/s 1159 * 1160 * We see that movnti performs better for 512-byte blocks, and 1161 * clflushopt performs better for 1024-byte and larger blocks. So, we 1162 * prefer clflushopt for sizes >= 768. 1163 * 1164 * NOTE: this happens to be the case now (with dm-writecache's single 1165 * threaded model) but re-evaluate this once memcpy_flushcache() is 1166 * enabled to use movdir64b which might invalidate this performance 1167 * advantage seen with cache-allocating-writes plus flushing. 1168 */ 1169 #ifdef CONFIG_X86 1170 if (static_cpu_has(X86_FEATURE_CLFLUSHOPT) && 1171 likely(boot_cpu_data.x86_clflush_size == 64) && 1172 likely(size >= 768)) { 1173 do { 1174 memcpy((void *)dest, (void *)source, 64); 1175 clflushopt((void *)dest); 1176 dest += 64; 1177 source += 64; 1178 size -= 64; 1179 } while (size >= 64); 1180 return; 1181 } 1182 #endif 1183 memcpy_flushcache(dest, source, size); 1184 } 1185 1186 static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data) 1187 { 1188 void *buf; 1189 unsigned long flags; 1190 unsigned size; 1191 int rw = bio_data_dir(bio); 1192 unsigned remaining_size = wc->block_size; 1193 1194 do { 1195 struct bio_vec bv = bio_iter_iovec(bio, bio->bi_iter); 1196 buf = bvec_kmap_irq(&bv, &flags); 1197 size = bv.bv_len; 1198 if (unlikely(size > remaining_size)) 1199 size = remaining_size; 1200 1201 if (rw == READ) { 1202 int r; 1203 r = copy_mc_to_kernel(buf, data, size); 1204 flush_dcache_page(bio_page(bio)); 1205 if (unlikely(r)) { 1206 writecache_error(wc, r, "hardware memory error when reading data: %d", r); 1207 bio->bi_status = BLK_STS_IOERR; 1208 } 1209 } else { 1210 flush_dcache_page(bio_page(bio)); 1211 memcpy_flushcache_optimized(data, buf, size); 1212 } 1213 1214 bvec_kunmap_irq(buf, &flags); 1215 1216 data = (char *)data + size; 1217 remaining_size -= size; 1218 bio_advance(bio, size); 1219 } while (unlikely(remaining_size)); 1220 } 1221 1222 static int writecache_flush_thread(void *data) 1223 { 1224 struct dm_writecache *wc = data; 1225 1226 while (1) { 1227 struct bio *bio; 1228 1229 wc_lock(wc); 1230 bio = bio_list_pop(&wc->flush_list); 1231 if (!bio) { 1232 set_current_state(TASK_INTERRUPTIBLE); 1233 wc_unlock(wc); 1234 1235 if (unlikely(kthread_should_stop())) { 1236 set_current_state(TASK_RUNNING); 1237 break; 1238 } 1239 1240 schedule(); 1241 continue; 1242 } 1243 1244 if (bio_op(bio) == REQ_OP_DISCARD) { 1245 writecache_discard(wc, bio->bi_iter.bi_sector, 1246 bio_end_sector(bio)); 1247 wc_unlock(wc); 1248 bio_set_dev(bio, wc->dev->bdev); 1249 submit_bio_noacct(bio); 1250 } else { 1251 writecache_flush(wc); 1252 wc_unlock(wc); 1253 if (writecache_has_error(wc)) 1254 bio->bi_status = BLK_STS_IOERR; 1255 bio_endio(bio); 1256 } 1257 } 1258 1259 return 0; 1260 } 1261 1262 static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio) 1263 { 1264 if (bio_list_empty(&wc->flush_list)) 1265 wake_up_process(wc->flush_thread); 1266 bio_list_add(&wc->flush_list, bio); 1267 } 1268 1269 static int writecache_map(struct dm_target *ti, struct bio *bio) 1270 { 1271 struct wc_entry *e; 1272 struct dm_writecache *wc = ti->private; 1273 1274 bio->bi_private = NULL; 1275 1276 wc_lock(wc); 1277 1278 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { 1279 if (writecache_has_error(wc)) 1280 goto unlock_error; 1281 if (WC_MODE_PMEM(wc)) { 1282 writecache_flush(wc); 1283 if (writecache_has_error(wc)) 1284 goto unlock_error; 1285 goto unlock_submit; 1286 } else { 1287 writecache_offload_bio(wc, bio); 1288 goto unlock_return; 1289 } 1290 } 1291 1292 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); 1293 1294 if (unlikely((((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) & 1295 (wc->block_size / 512 - 1)) != 0)) { 1296 DMERR("I/O is not aligned, sector %llu, size %u, block size %u", 1297 (unsigned long long)bio->bi_iter.bi_sector, 1298 bio->bi_iter.bi_size, wc->block_size); 1299 goto unlock_error; 1300 } 1301 1302 if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { 1303 if (writecache_has_error(wc)) 1304 goto unlock_error; 1305 if (WC_MODE_PMEM(wc)) { 1306 writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio)); 1307 goto unlock_remap_origin; 1308 } else { 1309 writecache_offload_bio(wc, bio); 1310 goto unlock_return; 1311 } 1312 } 1313 1314 if (bio_data_dir(bio) == READ) { 1315 read_next_block: 1316 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING); 1317 if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) { 1318 if (WC_MODE_PMEM(wc)) { 1319 bio_copy_block(wc, bio, memory_data(wc, e)); 1320 if (bio->bi_iter.bi_size) 1321 goto read_next_block; 1322 goto unlock_submit; 1323 } else { 1324 dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT); 1325 bio_set_dev(bio, wc->ssd_dev->bdev); 1326 bio->bi_iter.bi_sector = cache_sector(wc, e); 1327 if (!writecache_entry_is_committed(wc, e)) 1328 writecache_wait_for_ios(wc, WRITE); 1329 goto unlock_remap; 1330 } 1331 } else { 1332 if (e) { 1333 sector_t next_boundary = 1334 read_original_sector(wc, e) - bio->bi_iter.bi_sector; 1335 if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) { 1336 dm_accept_partial_bio(bio, next_boundary); 1337 } 1338 } 1339 goto unlock_remap_origin; 1340 } 1341 } else { 1342 do { 1343 bool found_entry = false; 1344 if (writecache_has_error(wc)) 1345 goto unlock_error; 1346 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0); 1347 if (e) { 1348 if (!writecache_entry_is_committed(wc, e)) 1349 goto bio_copy; 1350 if (!WC_MODE_PMEM(wc) && !e->write_in_progress) { 1351 wc->overwrote_committed = true; 1352 goto bio_copy; 1353 } 1354 found_entry = true; 1355 } else { 1356 if (unlikely(wc->cleaner)) 1357 goto direct_write; 1358 } 1359 e = writecache_pop_from_freelist(wc, (sector_t)-1); 1360 if (unlikely(!e)) { 1361 if (!found_entry) { 1362 direct_write: 1363 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING); 1364 if (e) { 1365 sector_t next_boundary = read_original_sector(wc, e) - bio->bi_iter.bi_sector; 1366 BUG_ON(!next_boundary); 1367 if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) { 1368 dm_accept_partial_bio(bio, next_boundary); 1369 } 1370 } 1371 goto unlock_remap_origin; 1372 } 1373 writecache_wait_on_freelist(wc); 1374 continue; 1375 } 1376 write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count); 1377 writecache_insert_entry(wc, e); 1378 wc->uncommitted_blocks++; 1379 bio_copy: 1380 if (WC_MODE_PMEM(wc)) { 1381 bio_copy_block(wc, bio, memory_data(wc, e)); 1382 } else { 1383 unsigned bio_size = wc->block_size; 1384 sector_t start_cache_sec = cache_sector(wc, e); 1385 sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT); 1386 1387 while (bio_size < bio->bi_iter.bi_size) { 1388 struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec); 1389 if (!f) 1390 break; 1391 write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector + 1392 (bio_size >> SECTOR_SHIFT), wc->seq_count); 1393 writecache_insert_entry(wc, f); 1394 wc->uncommitted_blocks++; 1395 bio_size += wc->block_size; 1396 current_cache_sec += wc->block_size >> SECTOR_SHIFT; 1397 } 1398 1399 bio_set_dev(bio, wc->ssd_dev->bdev); 1400 bio->bi_iter.bi_sector = start_cache_sec; 1401 dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT); 1402 1403 if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) { 1404 wc->uncommitted_blocks = 0; 1405 queue_work(wc->writeback_wq, &wc->flush_work); 1406 } else { 1407 writecache_schedule_autocommit(wc); 1408 } 1409 goto unlock_remap; 1410 } 1411 } while (bio->bi_iter.bi_size); 1412 1413 if (unlikely(bio->bi_opf & REQ_FUA || 1414 wc->uncommitted_blocks >= wc->autocommit_blocks)) 1415 writecache_flush(wc); 1416 else 1417 writecache_schedule_autocommit(wc); 1418 goto unlock_submit; 1419 } 1420 1421 unlock_remap_origin: 1422 bio_set_dev(bio, wc->dev->bdev); 1423 wc_unlock(wc); 1424 return DM_MAPIO_REMAPPED; 1425 1426 unlock_remap: 1427 /* make sure that writecache_end_io decrements bio_in_progress: */ 1428 bio->bi_private = (void *)1; 1429 atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]); 1430 wc_unlock(wc); 1431 return DM_MAPIO_REMAPPED; 1432 1433 unlock_submit: 1434 wc_unlock(wc); 1435 bio_endio(bio); 1436 return DM_MAPIO_SUBMITTED; 1437 1438 unlock_return: 1439 wc_unlock(wc); 1440 return DM_MAPIO_SUBMITTED; 1441 1442 unlock_error: 1443 wc_unlock(wc); 1444 bio_io_error(bio); 1445 return DM_MAPIO_SUBMITTED; 1446 } 1447 1448 static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status) 1449 { 1450 struct dm_writecache *wc = ti->private; 1451 1452 if (bio->bi_private != NULL) { 1453 int dir = bio_data_dir(bio); 1454 if (atomic_dec_and_test(&wc->bio_in_progress[dir])) 1455 if (unlikely(waitqueue_active(&wc->bio_in_progress_wait[dir]))) 1456 wake_up(&wc->bio_in_progress_wait[dir]); 1457 } 1458 return 0; 1459 } 1460 1461 static int writecache_iterate_devices(struct dm_target *ti, 1462 iterate_devices_callout_fn fn, void *data) 1463 { 1464 struct dm_writecache *wc = ti->private; 1465 1466 return fn(ti, wc->dev, 0, ti->len, data); 1467 } 1468 1469 static void writecache_io_hints(struct dm_target *ti, struct queue_limits *limits) 1470 { 1471 struct dm_writecache *wc = ti->private; 1472 1473 if (limits->logical_block_size < wc->block_size) 1474 limits->logical_block_size = wc->block_size; 1475 1476 if (limits->physical_block_size < wc->block_size) 1477 limits->physical_block_size = wc->block_size; 1478 1479 if (limits->io_min < wc->block_size) 1480 limits->io_min = wc->block_size; 1481 } 1482 1483 1484 static void writecache_writeback_endio(struct bio *bio) 1485 { 1486 struct writeback_struct *wb = container_of(bio, struct writeback_struct, bio); 1487 struct dm_writecache *wc = wb->wc; 1488 unsigned long flags; 1489 1490 raw_spin_lock_irqsave(&wc->endio_list_lock, flags); 1491 if (unlikely(list_empty(&wc->endio_list))) 1492 wake_up_process(wc->endio_thread); 1493 list_add_tail(&wb->endio_entry, &wc->endio_list); 1494 raw_spin_unlock_irqrestore(&wc->endio_list_lock, flags); 1495 } 1496 1497 static void writecache_copy_endio(int read_err, unsigned long write_err, void *ptr) 1498 { 1499 struct copy_struct *c = ptr; 1500 struct dm_writecache *wc = c->wc; 1501 1502 c->error = likely(!(read_err | write_err)) ? 0 : -EIO; 1503 1504 raw_spin_lock_irq(&wc->endio_list_lock); 1505 if (unlikely(list_empty(&wc->endio_list))) 1506 wake_up_process(wc->endio_thread); 1507 list_add_tail(&c->endio_entry, &wc->endio_list); 1508 raw_spin_unlock_irq(&wc->endio_list_lock); 1509 } 1510 1511 static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *list) 1512 { 1513 unsigned i; 1514 struct writeback_struct *wb; 1515 struct wc_entry *e; 1516 unsigned long n_walked = 0; 1517 1518 do { 1519 wb = list_entry(list->next, struct writeback_struct, endio_entry); 1520 list_del(&wb->endio_entry); 1521 1522 if (unlikely(wb->bio.bi_status != BLK_STS_OK)) 1523 writecache_error(wc, blk_status_to_errno(wb->bio.bi_status), 1524 "write error %d", wb->bio.bi_status); 1525 i = 0; 1526 do { 1527 e = wb->wc_list[i]; 1528 BUG_ON(!e->write_in_progress); 1529 e->write_in_progress = false; 1530 INIT_LIST_HEAD(&e->lru); 1531 if (!writecache_has_error(wc)) 1532 writecache_free_entry(wc, e); 1533 BUG_ON(!wc->writeback_size); 1534 wc->writeback_size--; 1535 n_walked++; 1536 if (unlikely(n_walked >= ENDIO_LATENCY)) { 1537 writecache_commit_flushed(wc, false); 1538 wc_unlock(wc); 1539 wc_lock(wc); 1540 n_walked = 0; 1541 } 1542 } while (++i < wb->wc_list_n); 1543 1544 if (wb->wc_list != wb->wc_list_inline) 1545 kfree(wb->wc_list); 1546 bio_put(&wb->bio); 1547 } while (!list_empty(list)); 1548 } 1549 1550 static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *list) 1551 { 1552 struct copy_struct *c; 1553 struct wc_entry *e; 1554 1555 do { 1556 c = list_entry(list->next, struct copy_struct, endio_entry); 1557 list_del(&c->endio_entry); 1558 1559 if (unlikely(c->error)) 1560 writecache_error(wc, c->error, "copy error"); 1561 1562 e = c->e; 1563 do { 1564 BUG_ON(!e->write_in_progress); 1565 e->write_in_progress = false; 1566 INIT_LIST_HEAD(&e->lru); 1567 if (!writecache_has_error(wc)) 1568 writecache_free_entry(wc, e); 1569 1570 BUG_ON(!wc->writeback_size); 1571 wc->writeback_size--; 1572 e++; 1573 } while (--c->n_entries); 1574 mempool_free(c, &wc->copy_pool); 1575 } while (!list_empty(list)); 1576 } 1577 1578 static int writecache_endio_thread(void *data) 1579 { 1580 struct dm_writecache *wc = data; 1581 1582 while (1) { 1583 struct list_head list; 1584 1585 raw_spin_lock_irq(&wc->endio_list_lock); 1586 if (!list_empty(&wc->endio_list)) 1587 goto pop_from_list; 1588 set_current_state(TASK_INTERRUPTIBLE); 1589 raw_spin_unlock_irq(&wc->endio_list_lock); 1590 1591 if (unlikely(kthread_should_stop())) { 1592 set_current_state(TASK_RUNNING); 1593 break; 1594 } 1595 1596 schedule(); 1597 1598 continue; 1599 1600 pop_from_list: 1601 list = wc->endio_list; 1602 list.next->prev = list.prev->next = &list; 1603 INIT_LIST_HEAD(&wc->endio_list); 1604 raw_spin_unlock_irq(&wc->endio_list_lock); 1605 1606 if (!WC_MODE_FUA(wc)) 1607 writecache_disk_flush(wc, wc->dev); 1608 1609 wc_lock(wc); 1610 1611 if (WC_MODE_PMEM(wc)) { 1612 __writecache_endio_pmem(wc, &list); 1613 } else { 1614 __writecache_endio_ssd(wc, &list); 1615 writecache_wait_for_ios(wc, READ); 1616 } 1617 1618 writecache_commit_flushed(wc, false); 1619 1620 wc_unlock(wc); 1621 } 1622 1623 return 0; 1624 } 1625 1626 static bool wc_add_block(struct writeback_struct *wb, struct wc_entry *e, gfp_t gfp) 1627 { 1628 struct dm_writecache *wc = wb->wc; 1629 unsigned block_size = wc->block_size; 1630 void *address = memory_data(wc, e); 1631 1632 persistent_memory_flush_cache(address, block_size); 1633 return bio_add_page(&wb->bio, persistent_memory_page(address), 1634 block_size, persistent_memory_page_offset(address)) != 0; 1635 } 1636 1637 struct writeback_list { 1638 struct list_head list; 1639 size_t size; 1640 }; 1641 1642 static void __writeback_throttle(struct dm_writecache *wc, struct writeback_list *wbl) 1643 { 1644 if (unlikely(wc->max_writeback_jobs)) { 1645 if (READ_ONCE(wc->writeback_size) - wbl->size >= wc->max_writeback_jobs) { 1646 wc_lock(wc); 1647 while (wc->writeback_size - wbl->size >= wc->max_writeback_jobs) 1648 writecache_wait_on_freelist(wc); 1649 wc_unlock(wc); 1650 } 1651 } 1652 cond_resched(); 1653 } 1654 1655 static void __writecache_writeback_pmem(struct dm_writecache *wc, struct writeback_list *wbl) 1656 { 1657 struct wc_entry *e, *f; 1658 struct bio *bio; 1659 struct writeback_struct *wb; 1660 unsigned max_pages; 1661 1662 while (wbl->size) { 1663 wbl->size--; 1664 e = container_of(wbl->list.prev, struct wc_entry, lru); 1665 list_del(&e->lru); 1666 1667 max_pages = e->wc_list_contiguous; 1668 1669 bio = bio_alloc_bioset(GFP_NOIO, max_pages, &wc->bio_set); 1670 wb = container_of(bio, struct writeback_struct, bio); 1671 wb->wc = wc; 1672 bio->bi_end_io = writecache_writeback_endio; 1673 bio_set_dev(bio, wc->dev->bdev); 1674 bio->bi_iter.bi_sector = read_original_sector(wc, e); 1675 if (max_pages <= WB_LIST_INLINE || 1676 unlikely(!(wb->wc_list = kmalloc_array(max_pages, sizeof(struct wc_entry *), 1677 GFP_NOIO | __GFP_NORETRY | 1678 __GFP_NOMEMALLOC | __GFP_NOWARN)))) { 1679 wb->wc_list = wb->wc_list_inline; 1680 max_pages = WB_LIST_INLINE; 1681 } 1682 1683 BUG_ON(!wc_add_block(wb, e, GFP_NOIO)); 1684 1685 wb->wc_list[0] = e; 1686 wb->wc_list_n = 1; 1687 1688 while (wbl->size && wb->wc_list_n < max_pages) { 1689 f = container_of(wbl->list.prev, struct wc_entry, lru); 1690 if (read_original_sector(wc, f) != 1691 read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT)) 1692 break; 1693 if (!wc_add_block(wb, f, GFP_NOWAIT | __GFP_NOWARN)) 1694 break; 1695 wbl->size--; 1696 list_del(&f->lru); 1697 wb->wc_list[wb->wc_list_n++] = f; 1698 e = f; 1699 } 1700 bio_set_op_attrs(bio, REQ_OP_WRITE, WC_MODE_FUA(wc) * REQ_FUA); 1701 if (writecache_has_error(wc)) { 1702 bio->bi_status = BLK_STS_IOERR; 1703 bio_endio(bio); 1704 } else { 1705 submit_bio(bio); 1706 } 1707 1708 __writeback_throttle(wc, wbl); 1709 } 1710 } 1711 1712 static void __writecache_writeback_ssd(struct dm_writecache *wc, struct writeback_list *wbl) 1713 { 1714 struct wc_entry *e, *f; 1715 struct dm_io_region from, to; 1716 struct copy_struct *c; 1717 1718 while (wbl->size) { 1719 unsigned n_sectors; 1720 1721 wbl->size--; 1722 e = container_of(wbl->list.prev, struct wc_entry, lru); 1723 list_del(&e->lru); 1724 1725 n_sectors = e->wc_list_contiguous << (wc->block_size_bits - SECTOR_SHIFT); 1726 1727 from.bdev = wc->ssd_dev->bdev; 1728 from.sector = cache_sector(wc, e); 1729 from.count = n_sectors; 1730 to.bdev = wc->dev->bdev; 1731 to.sector = read_original_sector(wc, e); 1732 to.count = n_sectors; 1733 1734 c = mempool_alloc(&wc->copy_pool, GFP_NOIO); 1735 c->wc = wc; 1736 c->e = e; 1737 c->n_entries = e->wc_list_contiguous; 1738 1739 while ((n_sectors -= wc->block_size >> SECTOR_SHIFT)) { 1740 wbl->size--; 1741 f = container_of(wbl->list.prev, struct wc_entry, lru); 1742 BUG_ON(f != e + 1); 1743 list_del(&f->lru); 1744 e = f; 1745 } 1746 1747 dm_kcopyd_copy(wc->dm_kcopyd, &from, 1, &to, 0, writecache_copy_endio, c); 1748 1749 __writeback_throttle(wc, wbl); 1750 } 1751 } 1752 1753 static void writecache_writeback(struct work_struct *work) 1754 { 1755 struct dm_writecache *wc = container_of(work, struct dm_writecache, writeback_work); 1756 struct blk_plug plug; 1757 struct wc_entry *f, *g, *e = NULL; 1758 struct rb_node *node, *next_node; 1759 struct list_head skipped; 1760 struct writeback_list wbl; 1761 unsigned long n_walked; 1762 1763 wc_lock(wc); 1764 restart: 1765 if (writecache_has_error(wc)) { 1766 wc_unlock(wc); 1767 return; 1768 } 1769 1770 if (unlikely(wc->writeback_all)) { 1771 if (writecache_wait_for_writeback(wc)) 1772 goto restart; 1773 } 1774 1775 if (wc->overwrote_committed) { 1776 writecache_wait_for_ios(wc, WRITE); 1777 } 1778 1779 n_walked = 0; 1780 INIT_LIST_HEAD(&skipped); 1781 INIT_LIST_HEAD(&wbl.list); 1782 wbl.size = 0; 1783 while (!list_empty(&wc->lru) && 1784 (wc->writeback_all || 1785 wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark || 1786 (jiffies - container_of(wc->lru.prev, struct wc_entry, lru)->age >= 1787 wc->max_age - wc->max_age / MAX_AGE_DIV))) { 1788 1789 n_walked++; 1790 if (unlikely(n_walked > WRITEBACK_LATENCY) && 1791 likely(!wc->writeback_all) && likely(!dm_suspended(wc->ti))) { 1792 queue_work(wc->writeback_wq, &wc->writeback_work); 1793 break; 1794 } 1795 1796 if (unlikely(wc->writeback_all)) { 1797 if (unlikely(!e)) { 1798 writecache_flush(wc); 1799 e = container_of(rb_first(&wc->tree), struct wc_entry, rb_node); 1800 } else 1801 e = g; 1802 } else 1803 e = container_of(wc->lru.prev, struct wc_entry, lru); 1804 BUG_ON(e->write_in_progress); 1805 if (unlikely(!writecache_entry_is_committed(wc, e))) { 1806 writecache_flush(wc); 1807 } 1808 node = rb_prev(&e->rb_node); 1809 if (node) { 1810 f = container_of(node, struct wc_entry, rb_node); 1811 if (unlikely(read_original_sector(wc, f) == 1812 read_original_sector(wc, e))) { 1813 BUG_ON(!f->write_in_progress); 1814 list_del(&e->lru); 1815 list_add(&e->lru, &skipped); 1816 cond_resched(); 1817 continue; 1818 } 1819 } 1820 wc->writeback_size++; 1821 list_del(&e->lru); 1822 list_add(&e->lru, &wbl.list); 1823 wbl.size++; 1824 e->write_in_progress = true; 1825 e->wc_list_contiguous = 1; 1826 1827 f = e; 1828 1829 while (1) { 1830 next_node = rb_next(&f->rb_node); 1831 if (unlikely(!next_node)) 1832 break; 1833 g = container_of(next_node, struct wc_entry, rb_node); 1834 if (unlikely(read_original_sector(wc, g) == 1835 read_original_sector(wc, f))) { 1836 f = g; 1837 continue; 1838 } 1839 if (read_original_sector(wc, g) != 1840 read_original_sector(wc, f) + (wc->block_size >> SECTOR_SHIFT)) 1841 break; 1842 if (unlikely(g->write_in_progress)) 1843 break; 1844 if (unlikely(!writecache_entry_is_committed(wc, g))) 1845 break; 1846 1847 if (!WC_MODE_PMEM(wc)) { 1848 if (g != f + 1) 1849 break; 1850 } 1851 1852 n_walked++; 1853 //if (unlikely(n_walked > WRITEBACK_LATENCY) && likely(!wc->writeback_all)) 1854 // break; 1855 1856 wc->writeback_size++; 1857 list_del(&g->lru); 1858 list_add(&g->lru, &wbl.list); 1859 wbl.size++; 1860 g->write_in_progress = true; 1861 g->wc_list_contiguous = BIO_MAX_PAGES; 1862 f = g; 1863 e->wc_list_contiguous++; 1864 if (unlikely(e->wc_list_contiguous == BIO_MAX_PAGES)) { 1865 if (unlikely(wc->writeback_all)) { 1866 next_node = rb_next(&f->rb_node); 1867 if (likely(next_node)) 1868 g = container_of(next_node, struct wc_entry, rb_node); 1869 } 1870 break; 1871 } 1872 } 1873 cond_resched(); 1874 } 1875 1876 if (!list_empty(&skipped)) { 1877 list_splice_tail(&skipped, &wc->lru); 1878 /* 1879 * If we didn't do any progress, we must wait until some 1880 * writeback finishes to avoid burning CPU in a loop 1881 */ 1882 if (unlikely(!wbl.size)) 1883 writecache_wait_for_writeback(wc); 1884 } 1885 1886 wc_unlock(wc); 1887 1888 blk_start_plug(&plug); 1889 1890 if (WC_MODE_PMEM(wc)) 1891 __writecache_writeback_pmem(wc, &wbl); 1892 else 1893 __writecache_writeback_ssd(wc, &wbl); 1894 1895 blk_finish_plug(&plug); 1896 1897 if (unlikely(wc->writeback_all)) { 1898 wc_lock(wc); 1899 while (writecache_wait_for_writeback(wc)); 1900 wc_unlock(wc); 1901 } 1902 } 1903 1904 static int calculate_memory_size(uint64_t device_size, unsigned block_size, 1905 size_t *n_blocks_p, size_t *n_metadata_blocks_p) 1906 { 1907 uint64_t n_blocks, offset; 1908 struct wc_entry e; 1909 1910 n_blocks = device_size; 1911 do_div(n_blocks, block_size + sizeof(struct wc_memory_entry)); 1912 1913 while (1) { 1914 if (!n_blocks) 1915 return -ENOSPC; 1916 /* Verify the following entries[n_blocks] won't overflow */ 1917 if (n_blocks >= ((size_t)-sizeof(struct wc_memory_superblock) / 1918 sizeof(struct wc_memory_entry))) 1919 return -EFBIG; 1920 offset = offsetof(struct wc_memory_superblock, entries[n_blocks]); 1921 offset = (offset + block_size - 1) & ~(uint64_t)(block_size - 1); 1922 if (offset + n_blocks * block_size <= device_size) 1923 break; 1924 n_blocks--; 1925 } 1926 1927 /* check if the bit field overflows */ 1928 e.index = n_blocks; 1929 if (e.index != n_blocks) 1930 return -EFBIG; 1931 1932 if (n_blocks_p) 1933 *n_blocks_p = n_blocks; 1934 if (n_metadata_blocks_p) 1935 *n_metadata_blocks_p = offset >> __ffs(block_size); 1936 return 0; 1937 } 1938 1939 static int init_memory(struct dm_writecache *wc) 1940 { 1941 size_t b; 1942 int r; 1943 1944 r = calculate_memory_size(wc->memory_map_size, wc->block_size, &wc->n_blocks, NULL); 1945 if (r) 1946 return r; 1947 1948 r = writecache_alloc_entries(wc); 1949 if (r) 1950 return r; 1951 1952 for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++) 1953 pmem_assign(sb(wc)->padding[b], cpu_to_le64(0)); 1954 pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION)); 1955 pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size)); 1956 pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks)); 1957 pmem_assign(sb(wc)->seq_count, cpu_to_le64(0)); 1958 1959 for (b = 0; b < wc->n_blocks; b++) { 1960 write_original_sector_seq_count(wc, &wc->entries[b], -1, -1); 1961 cond_resched(); 1962 } 1963 1964 writecache_flush_all_metadata(wc); 1965 writecache_commit_flushed(wc, false); 1966 pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC)); 1967 writecache_flush_region(wc, &sb(wc)->magic, sizeof sb(wc)->magic); 1968 writecache_commit_flushed(wc, false); 1969 1970 return 0; 1971 } 1972 1973 static void writecache_dtr(struct dm_target *ti) 1974 { 1975 struct dm_writecache *wc = ti->private; 1976 1977 if (!wc) 1978 return; 1979 1980 if (wc->endio_thread) 1981 kthread_stop(wc->endio_thread); 1982 1983 if (wc->flush_thread) 1984 kthread_stop(wc->flush_thread); 1985 1986 bioset_exit(&wc->bio_set); 1987 1988 mempool_exit(&wc->copy_pool); 1989 1990 if (wc->writeback_wq) 1991 destroy_workqueue(wc->writeback_wq); 1992 1993 if (wc->dev) 1994 dm_put_device(ti, wc->dev); 1995 1996 if (wc->ssd_dev) 1997 dm_put_device(ti, wc->ssd_dev); 1998 1999 if (wc->entries) 2000 vfree(wc->entries); 2001 2002 if (wc->memory_map) { 2003 if (WC_MODE_PMEM(wc)) 2004 persistent_memory_release(wc); 2005 else 2006 vfree(wc->memory_map); 2007 } 2008 2009 if (wc->dm_kcopyd) 2010 dm_kcopyd_client_destroy(wc->dm_kcopyd); 2011 2012 if (wc->dm_io) 2013 dm_io_client_destroy(wc->dm_io); 2014 2015 if (wc->dirty_bitmap) 2016 vfree(wc->dirty_bitmap); 2017 2018 kfree(wc); 2019 } 2020 2021 static int writecache_ctr(struct dm_target *ti, unsigned argc, char **argv) 2022 { 2023 struct dm_writecache *wc; 2024 struct dm_arg_set as; 2025 const char *string; 2026 unsigned opt_params; 2027 size_t offset, data_size; 2028 int i, r; 2029 char dummy; 2030 int high_wm_percent = HIGH_WATERMARK; 2031 int low_wm_percent = LOW_WATERMARK; 2032 uint64_t x; 2033 struct wc_memory_superblock s; 2034 2035 static struct dm_arg _args[] = { 2036 {0, 10, "Invalid number of feature args"}, 2037 }; 2038 2039 as.argc = argc; 2040 as.argv = argv; 2041 2042 wc = kzalloc(sizeof(struct dm_writecache), GFP_KERNEL); 2043 if (!wc) { 2044 ti->error = "Cannot allocate writecache structure"; 2045 r = -ENOMEM; 2046 goto bad; 2047 } 2048 ti->private = wc; 2049 wc->ti = ti; 2050 2051 mutex_init(&wc->lock); 2052 wc->max_age = MAX_AGE_UNSPECIFIED; 2053 writecache_poison_lists(wc); 2054 init_waitqueue_head(&wc->freelist_wait); 2055 timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0); 2056 timer_setup(&wc->max_age_timer, writecache_max_age_timer, 0); 2057 2058 for (i = 0; i < 2; i++) { 2059 atomic_set(&wc->bio_in_progress[i], 0); 2060 init_waitqueue_head(&wc->bio_in_progress_wait[i]); 2061 } 2062 2063 wc->dm_io = dm_io_client_create(); 2064 if (IS_ERR(wc->dm_io)) { 2065 r = PTR_ERR(wc->dm_io); 2066 ti->error = "Unable to allocate dm-io client"; 2067 wc->dm_io = NULL; 2068 goto bad; 2069 } 2070 2071 wc->writeback_wq = alloc_workqueue("writecache-writeback", WQ_MEM_RECLAIM, 1); 2072 if (!wc->writeback_wq) { 2073 r = -ENOMEM; 2074 ti->error = "Could not allocate writeback workqueue"; 2075 goto bad; 2076 } 2077 INIT_WORK(&wc->writeback_work, writecache_writeback); 2078 INIT_WORK(&wc->flush_work, writecache_flush_work); 2079 2080 raw_spin_lock_init(&wc->endio_list_lock); 2081 INIT_LIST_HEAD(&wc->endio_list); 2082 wc->endio_thread = kthread_create(writecache_endio_thread, wc, "writecache_endio"); 2083 if (IS_ERR(wc->endio_thread)) { 2084 r = PTR_ERR(wc->endio_thread); 2085 wc->endio_thread = NULL; 2086 ti->error = "Couldn't spawn endio thread"; 2087 goto bad; 2088 } 2089 wake_up_process(wc->endio_thread); 2090 2091 /* 2092 * Parse the mode (pmem or ssd) 2093 */ 2094 string = dm_shift_arg(&as); 2095 if (!string) 2096 goto bad_arguments; 2097 2098 if (!strcasecmp(string, "s")) { 2099 wc->pmem_mode = false; 2100 } else if (!strcasecmp(string, "p")) { 2101 #ifdef DM_WRITECACHE_HAS_PMEM 2102 wc->pmem_mode = true; 2103 wc->writeback_fua = true; 2104 #else 2105 /* 2106 * If the architecture doesn't support persistent memory or 2107 * the kernel doesn't support any DAX drivers, this driver can 2108 * only be used in SSD-only mode. 2109 */ 2110 r = -EOPNOTSUPP; 2111 ti->error = "Persistent memory or DAX not supported on this system"; 2112 goto bad; 2113 #endif 2114 } else { 2115 goto bad_arguments; 2116 } 2117 2118 if (WC_MODE_PMEM(wc)) { 2119 r = bioset_init(&wc->bio_set, BIO_POOL_SIZE, 2120 offsetof(struct writeback_struct, bio), 2121 BIOSET_NEED_BVECS); 2122 if (r) { 2123 ti->error = "Could not allocate bio set"; 2124 goto bad; 2125 } 2126 } else { 2127 r = mempool_init_kmalloc_pool(&wc->copy_pool, 1, sizeof(struct copy_struct)); 2128 if (r) { 2129 ti->error = "Could not allocate mempool"; 2130 goto bad; 2131 } 2132 } 2133 2134 /* 2135 * Parse the origin data device 2136 */ 2137 string = dm_shift_arg(&as); 2138 if (!string) 2139 goto bad_arguments; 2140 r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->dev); 2141 if (r) { 2142 ti->error = "Origin data device lookup failed"; 2143 goto bad; 2144 } 2145 2146 /* 2147 * Parse cache data device (be it pmem or ssd) 2148 */ 2149 string = dm_shift_arg(&as); 2150 if (!string) 2151 goto bad_arguments; 2152 2153 r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->ssd_dev); 2154 if (r) { 2155 ti->error = "Cache data device lookup failed"; 2156 goto bad; 2157 } 2158 wc->memory_map_size = i_size_read(wc->ssd_dev->bdev->bd_inode); 2159 2160 /* 2161 * Parse the cache block size 2162 */ 2163 string = dm_shift_arg(&as); 2164 if (!string) 2165 goto bad_arguments; 2166 if (sscanf(string, "%u%c", &wc->block_size, &dummy) != 1 || 2167 wc->block_size < 512 || wc->block_size > PAGE_SIZE || 2168 (wc->block_size & (wc->block_size - 1))) { 2169 r = -EINVAL; 2170 ti->error = "Invalid block size"; 2171 goto bad; 2172 } 2173 if (wc->block_size < bdev_logical_block_size(wc->dev->bdev) || 2174 wc->block_size < bdev_logical_block_size(wc->ssd_dev->bdev)) { 2175 r = -EINVAL; 2176 ti->error = "Block size is smaller than device logical block size"; 2177 goto bad; 2178 } 2179 wc->block_size_bits = __ffs(wc->block_size); 2180 2181 wc->max_writeback_jobs = MAX_WRITEBACK_JOBS; 2182 wc->autocommit_blocks = !WC_MODE_PMEM(wc) ? AUTOCOMMIT_BLOCKS_SSD : AUTOCOMMIT_BLOCKS_PMEM; 2183 wc->autocommit_jiffies = msecs_to_jiffies(AUTOCOMMIT_MSEC); 2184 2185 /* 2186 * Parse optional arguments 2187 */ 2188 r = dm_read_arg_group(_args, &as, &opt_params, &ti->error); 2189 if (r) 2190 goto bad; 2191 2192 while (opt_params) { 2193 string = dm_shift_arg(&as), opt_params--; 2194 if (!strcasecmp(string, "start_sector") && opt_params >= 1) { 2195 unsigned long long start_sector; 2196 string = dm_shift_arg(&as), opt_params--; 2197 if (sscanf(string, "%llu%c", &start_sector, &dummy) != 1) 2198 goto invalid_optional; 2199 wc->start_sector = start_sector; 2200 if (wc->start_sector != start_sector || 2201 wc->start_sector >= wc->memory_map_size >> SECTOR_SHIFT) 2202 goto invalid_optional; 2203 } else if (!strcasecmp(string, "high_watermark") && opt_params >= 1) { 2204 string = dm_shift_arg(&as), opt_params--; 2205 if (sscanf(string, "%d%c", &high_wm_percent, &dummy) != 1) 2206 goto invalid_optional; 2207 if (high_wm_percent < 0 || high_wm_percent > 100) 2208 goto invalid_optional; 2209 wc->high_wm_percent_set = true; 2210 } else if (!strcasecmp(string, "low_watermark") && opt_params >= 1) { 2211 string = dm_shift_arg(&as), opt_params--; 2212 if (sscanf(string, "%d%c", &low_wm_percent, &dummy) != 1) 2213 goto invalid_optional; 2214 if (low_wm_percent < 0 || low_wm_percent > 100) 2215 goto invalid_optional; 2216 wc->low_wm_percent_set = true; 2217 } else if (!strcasecmp(string, "writeback_jobs") && opt_params >= 1) { 2218 string = dm_shift_arg(&as), opt_params--; 2219 if (sscanf(string, "%u%c", &wc->max_writeback_jobs, &dummy) != 1) 2220 goto invalid_optional; 2221 wc->max_writeback_jobs_set = true; 2222 } else if (!strcasecmp(string, "autocommit_blocks") && opt_params >= 1) { 2223 string = dm_shift_arg(&as), opt_params--; 2224 if (sscanf(string, "%u%c", &wc->autocommit_blocks, &dummy) != 1) 2225 goto invalid_optional; 2226 wc->autocommit_blocks_set = true; 2227 } else if (!strcasecmp(string, "autocommit_time") && opt_params >= 1) { 2228 unsigned autocommit_msecs; 2229 string = dm_shift_arg(&as), opt_params--; 2230 if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1) 2231 goto invalid_optional; 2232 if (autocommit_msecs > 3600000) 2233 goto invalid_optional; 2234 wc->autocommit_jiffies = msecs_to_jiffies(autocommit_msecs); 2235 wc->autocommit_time_set = true; 2236 } else if (!strcasecmp(string, "max_age") && opt_params >= 1) { 2237 unsigned max_age_msecs; 2238 string = dm_shift_arg(&as), opt_params--; 2239 if (sscanf(string, "%u%c", &max_age_msecs, &dummy) != 1) 2240 goto invalid_optional; 2241 if (max_age_msecs > 86400000) 2242 goto invalid_optional; 2243 wc->max_age = msecs_to_jiffies(max_age_msecs); 2244 } else if (!strcasecmp(string, "cleaner")) { 2245 wc->cleaner = true; 2246 } else if (!strcasecmp(string, "fua")) { 2247 if (WC_MODE_PMEM(wc)) { 2248 wc->writeback_fua = true; 2249 wc->writeback_fua_set = true; 2250 } else goto invalid_optional; 2251 } else if (!strcasecmp(string, "nofua")) { 2252 if (WC_MODE_PMEM(wc)) { 2253 wc->writeback_fua = false; 2254 wc->writeback_fua_set = true; 2255 } else goto invalid_optional; 2256 } else { 2257 invalid_optional: 2258 r = -EINVAL; 2259 ti->error = "Invalid optional argument"; 2260 goto bad; 2261 } 2262 } 2263 2264 if (high_wm_percent < low_wm_percent) { 2265 r = -EINVAL; 2266 ti->error = "High watermark must be greater than or equal to low watermark"; 2267 goto bad; 2268 } 2269 2270 if (WC_MODE_PMEM(wc)) { 2271 if (!dax_synchronous(wc->ssd_dev->dax_dev)) { 2272 r = -EOPNOTSUPP; 2273 ti->error = "Asynchronous persistent memory not supported as pmem cache"; 2274 goto bad; 2275 } 2276 2277 r = persistent_memory_claim(wc); 2278 if (r) { 2279 ti->error = "Unable to map persistent memory for cache"; 2280 goto bad; 2281 } 2282 } else { 2283 size_t n_blocks, n_metadata_blocks; 2284 uint64_t n_bitmap_bits; 2285 2286 wc->memory_map_size -= (uint64_t)wc->start_sector << SECTOR_SHIFT; 2287 2288 bio_list_init(&wc->flush_list); 2289 wc->flush_thread = kthread_create(writecache_flush_thread, wc, "dm_writecache_flush"); 2290 if (IS_ERR(wc->flush_thread)) { 2291 r = PTR_ERR(wc->flush_thread); 2292 wc->flush_thread = NULL; 2293 ti->error = "Couldn't spawn flush thread"; 2294 goto bad; 2295 } 2296 wake_up_process(wc->flush_thread); 2297 2298 r = calculate_memory_size(wc->memory_map_size, wc->block_size, 2299 &n_blocks, &n_metadata_blocks); 2300 if (r) { 2301 ti->error = "Invalid device size"; 2302 goto bad; 2303 } 2304 2305 n_bitmap_bits = (((uint64_t)n_metadata_blocks << wc->block_size_bits) + 2306 BITMAP_GRANULARITY - 1) / BITMAP_GRANULARITY; 2307 /* this is limitation of test_bit functions */ 2308 if (n_bitmap_bits > 1U << 31) { 2309 r = -EFBIG; 2310 ti->error = "Invalid device size"; 2311 goto bad; 2312 } 2313 2314 wc->memory_map = vmalloc(n_metadata_blocks << wc->block_size_bits); 2315 if (!wc->memory_map) { 2316 r = -ENOMEM; 2317 ti->error = "Unable to allocate memory for metadata"; 2318 goto bad; 2319 } 2320 2321 wc->dm_kcopyd = dm_kcopyd_client_create(&dm_kcopyd_throttle); 2322 if (IS_ERR(wc->dm_kcopyd)) { 2323 r = PTR_ERR(wc->dm_kcopyd); 2324 ti->error = "Unable to allocate dm-kcopyd client"; 2325 wc->dm_kcopyd = NULL; 2326 goto bad; 2327 } 2328 2329 wc->metadata_sectors = n_metadata_blocks << (wc->block_size_bits - SECTOR_SHIFT); 2330 wc->dirty_bitmap_size = (n_bitmap_bits + BITS_PER_LONG - 1) / 2331 BITS_PER_LONG * sizeof(unsigned long); 2332 wc->dirty_bitmap = vzalloc(wc->dirty_bitmap_size); 2333 if (!wc->dirty_bitmap) { 2334 r = -ENOMEM; 2335 ti->error = "Unable to allocate dirty bitmap"; 2336 goto bad; 2337 } 2338 2339 r = writecache_read_metadata(wc, wc->block_size >> SECTOR_SHIFT); 2340 if (r) { 2341 ti->error = "Unable to read first block of metadata"; 2342 goto bad; 2343 } 2344 } 2345 2346 r = copy_mc_to_kernel(&s, sb(wc), sizeof(struct wc_memory_superblock)); 2347 if (r) { 2348 ti->error = "Hardware memory error when reading superblock"; 2349 goto bad; 2350 } 2351 if (!le32_to_cpu(s.magic) && !le32_to_cpu(s.version)) { 2352 r = init_memory(wc); 2353 if (r) { 2354 ti->error = "Unable to initialize device"; 2355 goto bad; 2356 } 2357 r = copy_mc_to_kernel(&s, sb(wc), 2358 sizeof(struct wc_memory_superblock)); 2359 if (r) { 2360 ti->error = "Hardware memory error when reading superblock"; 2361 goto bad; 2362 } 2363 } 2364 2365 if (le32_to_cpu(s.magic) != MEMORY_SUPERBLOCK_MAGIC) { 2366 ti->error = "Invalid magic in the superblock"; 2367 r = -EINVAL; 2368 goto bad; 2369 } 2370 2371 if (le32_to_cpu(s.version) != MEMORY_SUPERBLOCK_VERSION) { 2372 ti->error = "Invalid version in the superblock"; 2373 r = -EINVAL; 2374 goto bad; 2375 } 2376 2377 if (le32_to_cpu(s.block_size) != wc->block_size) { 2378 ti->error = "Block size does not match superblock"; 2379 r = -EINVAL; 2380 goto bad; 2381 } 2382 2383 wc->n_blocks = le64_to_cpu(s.n_blocks); 2384 2385 offset = wc->n_blocks * sizeof(struct wc_memory_entry); 2386 if (offset / sizeof(struct wc_memory_entry) != le64_to_cpu(sb(wc)->n_blocks)) { 2387 overflow: 2388 ti->error = "Overflow in size calculation"; 2389 r = -EINVAL; 2390 goto bad; 2391 } 2392 offset += sizeof(struct wc_memory_superblock); 2393 if (offset < sizeof(struct wc_memory_superblock)) 2394 goto overflow; 2395 offset = (offset + wc->block_size - 1) & ~(size_t)(wc->block_size - 1); 2396 data_size = wc->n_blocks * (size_t)wc->block_size; 2397 if (!offset || (data_size / wc->block_size != wc->n_blocks) || 2398 (offset + data_size < offset)) 2399 goto overflow; 2400 if (offset + data_size > wc->memory_map_size) { 2401 ti->error = "Memory area is too small"; 2402 r = -EINVAL; 2403 goto bad; 2404 } 2405 2406 wc->metadata_sectors = offset >> SECTOR_SHIFT; 2407 wc->block_start = (char *)sb(wc) + offset; 2408 2409 x = (uint64_t)wc->n_blocks * (100 - high_wm_percent); 2410 x += 50; 2411 do_div(x, 100); 2412 wc->freelist_high_watermark = x; 2413 x = (uint64_t)wc->n_blocks * (100 - low_wm_percent); 2414 x += 50; 2415 do_div(x, 100); 2416 wc->freelist_low_watermark = x; 2417 2418 if (wc->cleaner) 2419 activate_cleaner(wc); 2420 2421 r = writecache_alloc_entries(wc); 2422 if (r) { 2423 ti->error = "Cannot allocate memory"; 2424 goto bad; 2425 } 2426 2427 ti->num_flush_bios = 1; 2428 ti->flush_supported = true; 2429 ti->num_discard_bios = 1; 2430 2431 if (WC_MODE_PMEM(wc)) 2432 persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size); 2433 2434 return 0; 2435 2436 bad_arguments: 2437 r = -EINVAL; 2438 ti->error = "Bad arguments"; 2439 bad: 2440 writecache_dtr(ti); 2441 return r; 2442 } 2443 2444 static void writecache_status(struct dm_target *ti, status_type_t type, 2445 unsigned status_flags, char *result, unsigned maxlen) 2446 { 2447 struct dm_writecache *wc = ti->private; 2448 unsigned extra_args; 2449 unsigned sz = 0; 2450 uint64_t x; 2451 2452 switch (type) { 2453 case STATUSTYPE_INFO: 2454 DMEMIT("%ld %llu %llu %llu", writecache_has_error(wc), 2455 (unsigned long long)wc->n_blocks, (unsigned long long)wc->freelist_size, 2456 (unsigned long long)wc->writeback_size); 2457 break; 2458 case STATUSTYPE_TABLE: 2459 DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's', 2460 wc->dev->name, wc->ssd_dev->name, wc->block_size); 2461 extra_args = 0; 2462 if (wc->start_sector) 2463 extra_args += 2; 2464 if (wc->high_wm_percent_set && !wc->cleaner) 2465 extra_args += 2; 2466 if (wc->low_wm_percent_set && !wc->cleaner) 2467 extra_args += 2; 2468 if (wc->max_writeback_jobs_set) 2469 extra_args += 2; 2470 if (wc->autocommit_blocks_set) 2471 extra_args += 2; 2472 if (wc->autocommit_time_set) 2473 extra_args += 2; 2474 if (wc->cleaner) 2475 extra_args++; 2476 if (wc->writeback_fua_set) 2477 extra_args++; 2478 2479 DMEMIT("%u", extra_args); 2480 if (wc->start_sector) 2481 DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector); 2482 if (wc->high_wm_percent_set && !wc->cleaner) { 2483 x = (uint64_t)wc->freelist_high_watermark * 100; 2484 x += wc->n_blocks / 2; 2485 do_div(x, (size_t)wc->n_blocks); 2486 DMEMIT(" high_watermark %u", 100 - (unsigned)x); 2487 } 2488 if (wc->low_wm_percent_set && !wc->cleaner) { 2489 x = (uint64_t)wc->freelist_low_watermark * 100; 2490 x += wc->n_blocks / 2; 2491 do_div(x, (size_t)wc->n_blocks); 2492 DMEMIT(" low_watermark %u", 100 - (unsigned)x); 2493 } 2494 if (wc->max_writeback_jobs_set) 2495 DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs); 2496 if (wc->autocommit_blocks_set) 2497 DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks); 2498 if (wc->autocommit_time_set) 2499 DMEMIT(" autocommit_time %u", jiffies_to_msecs(wc->autocommit_jiffies)); 2500 if (wc->max_age != MAX_AGE_UNSPECIFIED) 2501 DMEMIT(" max_age %u", jiffies_to_msecs(wc->max_age)); 2502 if (wc->cleaner) 2503 DMEMIT(" cleaner"); 2504 if (wc->writeback_fua_set) 2505 DMEMIT(" %sfua", wc->writeback_fua ? "" : "no"); 2506 break; 2507 } 2508 } 2509 2510 static struct target_type writecache_target = { 2511 .name = "writecache", 2512 .version = {1, 3, 0}, 2513 .module = THIS_MODULE, 2514 .ctr = writecache_ctr, 2515 .dtr = writecache_dtr, 2516 .status = writecache_status, 2517 .postsuspend = writecache_suspend, 2518 .resume = writecache_resume, 2519 .message = writecache_message, 2520 .map = writecache_map, 2521 .end_io = writecache_end_io, 2522 .iterate_devices = writecache_iterate_devices, 2523 .io_hints = writecache_io_hints, 2524 }; 2525 2526 static int __init dm_writecache_init(void) 2527 { 2528 int r; 2529 2530 r = dm_register_target(&writecache_target); 2531 if (r < 0) { 2532 DMERR("register failed %d", r); 2533 return r; 2534 } 2535 2536 return 0; 2537 } 2538 2539 static void __exit dm_writecache_exit(void) 2540 { 2541 dm_unregister_target(&writecache_target); 2542 } 2543 2544 module_init(dm_writecache_init); 2545 module_exit(dm_writecache_exit); 2546 2547 MODULE_DESCRIPTION(DM_NAME " writecache target"); 2548 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>"); 2549 MODULE_LICENSE("GPL"); 2550