1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/blkdev.h> 4 #include <linux/module.h> 5 #include <linux/errno.h> 6 #include <linux/slab.h> 7 #include <linux/init.h> 8 #include <linux/timer.h> 9 #include <linux/sched.h> 10 #include <linux/list.h> 11 #include <linux/file.h> 12 #include <linux/math64.h> 13 #include <linux/seq_file.h> 14 #include <trace/events/block.h> 15 16 #include "md.h" 17 #include "md-bitmap.h" 18 19 /* 20 * #### Background 21 * 22 * Redundant data is used to enhance data fault tolerance, and the storage 23 * methods for redundant data vary depending on the RAID levels. And it's 24 * important to maintain the consistency of redundant data. 25 * 26 * Bitmap is used to record which data blocks have been synchronized and which 27 * ones need to be resynchronized or recovered. Each bit in the bitmap 28 * represents a segment of data in the array. When a bit is set, it indicates 29 * that the multiple redundant copies of that data segment may not be 30 * consistent. Data synchronization can be performed based on the bitmap after 31 * power failure or readding a disk. If there is no bitmap, a full disk 32 * synchronization is required. 33 * 34 * #### Key Features 35 * 36 * - IO fastpath is lockless, if user issues lots of write IO to the same 37 * bitmap bit in a short time, only the first write has additional overhead 38 * to update bitmap bit, no additional overhead for the following writes; 39 * - support only resync or recover written data, means in the case creating 40 * new array or replacing with a new disk, there is no need to do a full disk 41 * resync/recovery; 42 * 43 * #### Key Concept 44 * 45 * ##### State Machine 46 * 47 * Each bit is one byte, contain 6 different states, see llbitmap_state. And 48 * there are total 8 different actions, see llbitmap_action, can change state: 49 * 50 * llbitmap state machine: transitions between states 51 * 52 * | | Startwrite | Startsync | Endsync | Abortsync| 53 * | --------- | ---------- | --------- | ------- | ------- | 54 * | Unwritten | Dirty | x | x | x | 55 * | Clean | Dirty | x | x | x | 56 * | Dirty | x | x | x | x | 57 * | NeedSync | x | Syncing | x | x | 58 * | Syncing | x | Syncing | Dirty | NeedSync | 59 * 60 * | | Reload | Daemon | Discard | Stale | 61 * | --------- | -------- | ------ | --------- | --------- | 62 * | Unwritten | x | x | x | x | 63 * | Clean | x | x | Unwritten | NeedSync | 64 * | Dirty | NeedSync | Clean | Unwritten | NeedSync | 65 * | NeedSync | x | x | Unwritten | x | 66 * | Syncing | NeedSync | x | Unwritten | NeedSync | 67 * 68 * Typical scenarios: 69 * 70 * 1) Create new array 71 * All bits will be set to Unwritten by default, if --assume-clean is set, 72 * all bits will be set to Clean instead. 73 * 74 * 2) write data, raid1/raid10 have full copy of data, while raid456 doesn't and 75 * rely on xor data 76 * 77 * 2.1) write new data to raid1/raid10: 78 * Unwritten --StartWrite--> Dirty 79 * 80 * 2.2) write new data to raid456: 81 * Unwritten --StartWrite--> NeedSync 82 * 83 * Because the initial recover for raid456 is skipped, the xor data is not built 84 * yet, the bit must be set to NeedSync first and after lazy initial recover is 85 * finished, the bit will finally set to Dirty(see 5.1 and 5.4); 86 * 87 * 2.3) cover write 88 * Clean --StartWrite--> Dirty 89 * 90 * 3) daemon, if the array is not degraded: 91 * Dirty --Daemon--> Clean 92 * 93 * 4) discard 94 * {Clean, Dirty, NeedSync, Syncing} --Discard--> Unwritten 95 * 96 * 5) resync and recover 97 * 98 * 5.1) common process 99 * NeedSync --Startsync--> Syncing --Endsync--> Dirty --Daemon--> Clean 100 * 101 * 5.2) resync after power failure 102 * Dirty --Reload--> NeedSync 103 * 104 * 5.3) recover while replacing with a new disk 105 * By default, the old bitmap framework will recover all data, and llbitmap 106 * implements this by a new helper, see llbitmap_skip_sync_blocks: 107 * 108 * skip recover for bits other than dirty or clean; 109 * 110 * 5.4) lazy initial recover for raid5: 111 * By default, the old bitmap framework will only allow new recover when there 112 * are spares(new disk), a new recovery flag MD_RECOVERY_LAZY_RECOVER is added 113 * to perform raid456 lazy recover for set bits(from 2.2). 114 * 115 * 6. special handling for degraded array: 116 * 117 * - Dirty bits will never be cleared, daemon will just do nothing, so that if 118 * a disk is readded, Clean bits can be skipped with recovery; 119 * - Dirty bits will convert to Syncing from start write, to do data recovery 120 * for new added disks; 121 * - New write will convert bits to NeedSync directly; 122 * 123 * ##### Bitmap IO 124 * 125 * ##### Chunksize 126 * 127 * The default bitmap size is 128k, incluing 1k bitmap super block, and 128 * the default size of segment of data in the array each bit(chunksize) is 64k, 129 * and chunksize will adjust to twice the old size each time if the total number 130 * bits is not less than 127k.(see llbitmap_init) 131 * 132 * ##### READ 133 * 134 * While creating bitmap, all pages will be allocated and read for llbitmap, 135 * there won't be read afterwards 136 * 137 * ##### WRITE 138 * 139 * WRITE IO is divided into logical_block_size of the array, the dirty state 140 * of each block is tracked independently, for example: 141 * 142 * each page is 4k, contain 8 blocks; each block is 512 bytes contain 512 bit; 143 * 144 * | page0 | page1 | ... | page 31 | 145 * | | 146 * | \-----------------------\ 147 * | | 148 * | block0 | block1 | ... | block 8| 149 * | | 150 * | \-----------------\ 151 * | | 152 * | bit0 | bit1 | ... | bit511 | 153 * 154 * From IO path, if one bit is changed to Dirty or NeedSync, the corresponding 155 * subpage will be marked dirty, such block must write first before the IO is 156 * issued. This behaviour will affect IO performance, to reduce the impact, if 157 * multiple bits are changed in the same block in a short time, all bits in this 158 * block will be changed to Dirty/NeedSync, so that there won't be any overhead 159 * until daemon clears dirty bits. 160 * 161 * ##### Dirty Bits synchronization 162 * 163 * IO fast path will set bits to dirty, and those dirty bits will be cleared 164 * by daemon after IO is done. llbitmap_page_ctl is used to synchronize between 165 * IO path and daemon; 166 * 167 * IO path: 168 * 1) try to grab a reference, if succeed, set expire time after 5s and return; 169 * 2) if failed to grab a reference, wait for daemon to finish clearing dirty 170 * bits; 171 * 172 * Daemon (Daemon will be woken up every daemon_sleep seconds): 173 * For each page: 174 * 1) check if page expired, if not skip this page; for expired page: 175 * 2) suspend the page and wait for inflight write IO to be done; 176 * 3) change dirty page to clean; 177 * 4) resume the page; 178 */ 179 180 #define BITMAP_DATA_OFFSET 1024 181 182 /* 64k is the max IO size of sync IO for raid1/raid10 */ 183 #define MIN_CHUNK_SIZE (64 * 2) 184 185 /* By default, daemon will be woken up every 30s */ 186 #define DEFAULT_DAEMON_SLEEP 30 187 188 /* 189 * Dirtied bits that have not been accessed for more than 5s will be cleared 190 * by daemon. 191 */ 192 #define DEFAULT_BARRIER_IDLE 5 193 194 enum llbitmap_state { 195 /* No valid data, init state after assemble the array */ 196 BitUnwritten = 0, 197 /* data is consistent */ 198 BitClean, 199 /* data will be consistent after IO is done, set directly for writes */ 200 BitDirty, 201 /* 202 * data need to be resynchronized: 203 * 1) set directly for writes if array is degraded, prevent full disk 204 * synchronization after readding a disk; 205 * 2) reassemble the array after power failure, and dirty bits are 206 * found after reloading the bitmap; 207 * 3) set for first write for raid5, to build initial xor data lazily 208 */ 209 BitNeedSync, 210 /* data is synchronizing */ 211 BitSyncing, 212 /* 213 * Proactive sync requested for unwritten region (raid456 only). 214 * Triggered via sysfs when user wants to pre-build XOR parity 215 * for regions that have never been written. 216 */ 217 BitNeedSyncUnwritten, 218 /* Proactive sync in progress for unwritten region */ 219 BitSyncingUnwritten, 220 /* 221 * XOR parity has been pre-built for a region that has never had 222 * user data written. When user writes to this region, it transitions 223 * to BitDirty. 224 */ 225 BitCleanUnwritten, 226 BitStateCount, 227 BitNone = 0xff, 228 }; 229 230 enum llbitmap_action { 231 /* User write new data, this is the only action from IO fast path */ 232 BitmapActionStartwrite = 0, 233 /* Start recovery */ 234 BitmapActionStartsync, 235 /* Finish recovery */ 236 BitmapActionEndsync, 237 /* Failed recovery */ 238 BitmapActionAbortsync, 239 /* Reassemble the array */ 240 BitmapActionReload, 241 /* Daemon thread is trying to clear dirty bits */ 242 BitmapActionDaemon, 243 /* Data is deleted */ 244 BitmapActionDiscard, 245 /* 246 * Bitmap is stale, mark all bits in addition to BitUnwritten to 247 * BitNeedSync. 248 */ 249 BitmapActionStale, 250 /* 251 * Proactive sync trigger for raid456 - builds XOR parity for 252 * Unwritten regions without requiring user data write first. 253 */ 254 BitmapActionProactiveSync, 255 BitmapActionClearUnwritten, 256 BitmapActionCount, 257 /* Init state is BitUnwritten */ 258 BitmapActionInit, 259 }; 260 261 enum llbitmap_page_state { 262 LLPageFlush = 0, 263 LLPageDirty, 264 }; 265 266 struct llbitmap_page_ctl { 267 char *state; 268 struct page *page; 269 unsigned long expire; 270 unsigned long flags; 271 wait_queue_head_t wait; 272 struct percpu_ref active; 273 /* Per block size dirty state, maximum 64k page / 1 sector = 128 */ 274 unsigned long dirty[]; 275 }; 276 277 struct llbitmap { 278 struct mddev *mddev; 279 struct llbitmap_page_ctl **pctl; 280 281 unsigned int nr_pages; 282 unsigned int io_size; 283 unsigned int blocks_per_page; 284 285 /* shift of one chunk */ 286 unsigned long chunkshift; 287 /* size of one chunk in sector */ 288 unsigned long chunksize; 289 /* total number of chunks */ 290 unsigned long chunks; 291 /* total number of sectors tracked by current bitmap geometry */ 292 sector_t sync_size; 293 unsigned long reshape_chunksize; 294 unsigned long reshape_chunks; 295 sector_t reshape_sync_size; 296 unsigned long last_end_sync; 297 /* 298 * time in seconds that dirty bits will be cleared if the page is not 299 * accessed. 300 */ 301 unsigned long barrier_idle; 302 /* fires on first BitDirty state */ 303 struct timer_list pending_timer; 304 struct work_struct daemon_work; 305 /* 306 * Serialize reshape checkpoint remapping against normal I/O bitmap 307 * updates without blocking concurrent I/O updates on each other. 308 */ 309 rwlock_t reshape_lock; 310 311 unsigned long flags; 312 __u64 events_cleared; 313 314 /* for slow disks */ 315 atomic_t behind_writes; 316 wait_queue_head_t behind_wait; 317 }; 318 319 struct llbitmap_unplug_work { 320 struct work_struct work; 321 struct llbitmap *llbitmap; 322 struct completion *done; 323 }; 324 325 static struct workqueue_struct *md_llbitmap_io_wq; 326 static struct workqueue_struct *md_llbitmap_unplug_wq; 327 328 static char state_machine[BitStateCount][BitmapActionCount] = { 329 [BitUnwritten] = { 330 [BitmapActionStartwrite] = BitDirty, 331 [BitmapActionStartsync] = BitNone, 332 [BitmapActionEndsync] = BitNone, 333 [BitmapActionAbortsync] = BitNone, 334 [BitmapActionReload] = BitNone, 335 [BitmapActionDaemon] = BitNone, 336 [BitmapActionDiscard] = BitNone, 337 [BitmapActionStale] = BitNone, 338 [BitmapActionProactiveSync] = BitNeedSyncUnwritten, 339 [BitmapActionClearUnwritten] = BitNone, 340 }, 341 [BitClean] = { 342 [BitmapActionStartwrite] = BitDirty, 343 [BitmapActionStartsync] = BitNone, 344 [BitmapActionEndsync] = BitNone, 345 [BitmapActionAbortsync] = BitNone, 346 [BitmapActionReload] = BitNone, 347 [BitmapActionDaemon] = BitNone, 348 [BitmapActionDiscard] = BitUnwritten, 349 [BitmapActionStale] = BitNeedSync, 350 [BitmapActionProactiveSync] = BitNone, 351 [BitmapActionClearUnwritten] = BitNone, 352 }, 353 [BitDirty] = { 354 [BitmapActionStartwrite] = BitNone, 355 [BitmapActionStartsync] = BitNone, 356 [BitmapActionEndsync] = BitNone, 357 [BitmapActionAbortsync] = BitNone, 358 [BitmapActionReload] = BitNeedSync, 359 [BitmapActionDaemon] = BitClean, 360 [BitmapActionDiscard] = BitUnwritten, 361 [BitmapActionStale] = BitNeedSync, 362 [BitmapActionProactiveSync] = BitNone, 363 [BitmapActionClearUnwritten] = BitNone, 364 }, 365 [BitNeedSync] = { 366 [BitmapActionStartwrite] = BitNone, 367 [BitmapActionStartsync] = BitSyncing, 368 [BitmapActionEndsync] = BitNone, 369 [BitmapActionAbortsync] = BitNone, 370 [BitmapActionReload] = BitNone, 371 [BitmapActionDaemon] = BitNone, 372 [BitmapActionDiscard] = BitUnwritten, 373 [BitmapActionStale] = BitNone, 374 [BitmapActionProactiveSync] = BitNone, 375 [BitmapActionClearUnwritten] = BitNone, 376 }, 377 [BitSyncing] = { 378 [BitmapActionStartwrite] = BitNone, 379 [BitmapActionStartsync] = BitSyncing, 380 [BitmapActionEndsync] = BitDirty, 381 [BitmapActionAbortsync] = BitNeedSync, 382 [BitmapActionReload] = BitNeedSync, 383 [BitmapActionDaemon] = BitNone, 384 [BitmapActionDiscard] = BitUnwritten, 385 [BitmapActionStale] = BitNeedSync, 386 [BitmapActionProactiveSync] = BitNone, 387 [BitmapActionClearUnwritten] = BitNone, 388 }, 389 [BitNeedSyncUnwritten] = { 390 [BitmapActionStartwrite] = BitNeedSync, 391 [BitmapActionStartsync] = BitSyncingUnwritten, 392 [BitmapActionEndsync] = BitNone, 393 [BitmapActionAbortsync] = BitUnwritten, 394 [BitmapActionReload] = BitUnwritten, 395 [BitmapActionDaemon] = BitNone, 396 [BitmapActionDiscard] = BitUnwritten, 397 [BitmapActionStale] = BitUnwritten, 398 [BitmapActionProactiveSync] = BitNone, 399 [BitmapActionClearUnwritten] = BitUnwritten, 400 }, 401 [BitSyncingUnwritten] = { 402 [BitmapActionStartwrite] = BitSyncing, 403 [BitmapActionStartsync] = BitSyncingUnwritten, 404 [BitmapActionEndsync] = BitCleanUnwritten, 405 [BitmapActionAbortsync] = BitUnwritten, 406 [BitmapActionReload] = BitUnwritten, 407 [BitmapActionDaemon] = BitNone, 408 [BitmapActionDiscard] = BitUnwritten, 409 [BitmapActionStale] = BitUnwritten, 410 [BitmapActionProactiveSync] = BitNone, 411 [BitmapActionClearUnwritten] = BitUnwritten, 412 }, 413 [BitCleanUnwritten] = { 414 [BitmapActionStartwrite] = BitDirty, 415 [BitmapActionStartsync] = BitNone, 416 [BitmapActionEndsync] = BitNone, 417 [BitmapActionAbortsync] = BitNone, 418 [BitmapActionReload] = BitNone, 419 [BitmapActionDaemon] = BitNone, 420 [BitmapActionDiscard] = BitUnwritten, 421 [BitmapActionStale] = BitUnwritten, 422 [BitmapActionProactiveSync] = BitNone, 423 [BitmapActionClearUnwritten] = BitUnwritten, 424 }, 425 }; 426 427 static void __llbitmap_flush(struct mddev *mddev); 428 static void llbitmap_flush(struct mddev *mddev); 429 static void llbitmap_update_sb(void *data); 430 431 static void llbitmap_calculate_chunks(struct mddev *mddev, sector_t blocks, 432 unsigned long *chunksize, 433 unsigned long *chunks) 434 { 435 *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize); 436 while (*chunks > mddev->bitmap_info.space << SECTOR_SHIFT) { 437 *chunksize = *chunksize << 1; 438 *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize); 439 } 440 } 441 442 static sector_t llbitmap_personality_sync_size(struct llbitmap *llbitmap, 443 bool previous) 444 { 445 struct mddev *mddev = llbitmap->mddev; 446 447 if (READ_ONCE(mddev->reshape_position) == MaxSector || 448 !mddev->private || !mddev->pers || 449 !mddev->pers->bitmap_sync_size) 450 return llbitmap->sync_size; 451 return mddev->pers->bitmap_sync_size(mddev, previous); 452 } 453 454 static sector_t llbitmap_logical_size(struct llbitmap *llbitmap, bool previous) 455 { 456 struct mddev *mddev = llbitmap->mddev; 457 458 if (!mddev->private || !mddev->pers || 459 !mddev->pers->bitmap_array_sectors) 460 return llbitmap_personality_sync_size(llbitmap, previous); 461 return mddev->pers->bitmap_array_sectors(mddev, previous); 462 } 463 464 static void llbitmap_refresh_reshape(struct llbitmap *llbitmap) 465 { 466 unsigned long old_chunks = DIV_ROUND_UP_SECTOR_T(llbitmap->sync_size, 467 llbitmap->chunksize); 468 sector_t blocks = llbitmap_personality_sync_size(llbitmap, false); 469 unsigned long chunksize = llbitmap->chunksize; 470 unsigned long chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize); 471 472 llbitmap->reshape_sync_size = blocks; 473 llbitmap->reshape_chunksize = chunksize; 474 llbitmap->reshape_chunks = chunks; 475 llbitmap_calculate_chunks(llbitmap->mddev, blocks, 476 &llbitmap->reshape_chunksize, 477 &llbitmap->reshape_chunks); 478 llbitmap->chunks = max(old_chunks, llbitmap->reshape_chunks); 479 } 480 481 static void llbitmap_map_layout(struct llbitmap *llbitmap, sector_t *offset, 482 unsigned long *sectors, bool previous) 483 { 484 sector_t limit = llbitmap_logical_size(llbitmap, previous); 485 sector_t start = *offset; 486 sector_t end = start + *sectors; 487 488 if (start >= limit) { 489 *sectors = 0; 490 return; 491 } 492 if (end > limit) 493 end = limit; 494 495 *offset = start; 496 *sectors = end - start; 497 if (!*sectors) 498 return; 499 500 if (llbitmap->mddev->pers->bitmap_sector_map) 501 llbitmap->mddev->pers->bitmap_sector_map(llbitmap->mddev, offset, 502 sectors, previous); 503 else if (!previous && llbitmap->mddev->pers->bitmap_sector) 504 llbitmap->mddev->pers->bitmap_sector(llbitmap->mddev, offset, 505 sectors); 506 507 limit = llbitmap_personality_sync_size(llbitmap, previous); 508 start = *offset; 509 end = start + *sectors; 510 if (start >= limit) 511 *sectors = 0; 512 else if (end > limit) 513 *sectors = limit - start; 514 } 515 516 static void llbitmap_encode_range(struct llbitmap *llbitmap, sector_t *offset, 517 unsigned long *sectors, bool previous) 518 { 519 unsigned long chunksize = previous ? llbitmap->chunksize : 520 llbitmap->reshape_chunksize; 521 u64 start; 522 u64 end; 523 524 if (!*sectors) { 525 *offset = 0; 526 return; 527 } 528 529 start = div64_u64(*offset, chunksize); 530 end = div64_u64(*offset + *sectors - 1, chunksize); 531 *offset = (sector_t)start << llbitmap->chunkshift; 532 *sectors = (end - start + 1) << llbitmap->chunkshift; 533 } 534 535 static void llbitmap_encode_discard_range(struct llbitmap *llbitmap, 536 sector_t *offset, 537 unsigned long *sectors, 538 bool previous) 539 { 540 unsigned long chunksize = previous ? llbitmap->chunksize : 541 llbitmap->reshape_chunksize; 542 sector_t end = *offset + *sectors; 543 u64 start; 544 u64 last; 545 546 if (!*sectors) { 547 *offset = 0; 548 return; 549 } 550 551 start = DIV_ROUND_UP_SECTOR_T(*offset, chunksize); 552 last = div64_u64(end, chunksize); 553 if (start >= last) { 554 *offset = 0; 555 *sectors = 0; 556 return; 557 } 558 559 *offset = (sector_t)start << llbitmap->chunkshift; 560 *sectors = (last - start) << llbitmap->chunkshift; 561 } 562 563 static enum llbitmap_state llbitmap_read(struct llbitmap *llbitmap, loff_t pos) 564 { 565 unsigned int idx; 566 unsigned int offset; 567 568 pos += BITMAP_DATA_OFFSET; 569 idx = pos >> PAGE_SHIFT; 570 offset = offset_in_page(pos); 571 572 return llbitmap->pctl[idx]->state[offset]; 573 } 574 575 /* set all the bits in the subpage as dirty */ 576 static void llbitmap_infect_dirty_bits(struct llbitmap *llbitmap, 577 struct llbitmap_page_ctl *pctl, 578 unsigned int block) 579 { 580 bool level_456 = raid_is_456(llbitmap->mddev); 581 unsigned int io_size = llbitmap->io_size; 582 int pos; 583 584 for (pos = block * io_size; pos < (block + 1) * io_size; pos++) { 585 switch (pctl->state[pos]) { 586 case BitUnwritten: 587 pctl->state[pos] = level_456 ? BitNeedSync : BitDirty; 588 break; 589 case BitClean: 590 case BitCleanUnwritten: 591 pctl->state[pos] = BitDirty; 592 break; 593 } 594 } 595 } 596 597 static void llbitmap_set_page_dirty(struct llbitmap *llbitmap, int idx, 598 int offset, bool infect) 599 { 600 struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx]; 601 unsigned int io_size = llbitmap->io_size; 602 int block = offset / io_size; 603 int pos; 604 605 if (!test_bit(LLPageDirty, &pctl->flags)) 606 set_bit(LLPageDirty, &pctl->flags); 607 608 /* 609 * For degraded array, dirty bits will never be cleared, and we must 610 * resync all the dirty bits, hence skip infect new dirty bits to 611 * prevent resync unnecessary data. 612 */ 613 if (llbitmap->mddev->degraded || !infect) { 614 set_bit(block, pctl->dirty); 615 return; 616 } 617 618 /* 619 * The subpage usually contains a total of 512 bits. If any single bit 620 * within the subpage is marked as dirty, the entire sector will be 621 * written. To avoid impacting write performance, when multiple bits 622 * within the same sector are modified within llbitmap->barrier_idle, 623 * all bits in the sector will be collectively marked as dirty at once. 624 */ 625 if (test_and_set_bit(block, pctl->dirty)) { 626 llbitmap_infect_dirty_bits(llbitmap, pctl, block); 627 return; 628 } 629 630 for (pos = block * io_size; pos < (block + 1) * io_size; pos++) { 631 if (pos == offset) 632 continue; 633 if (pctl->state[pos] == BitDirty || 634 pctl->state[pos] == BitNeedSync) { 635 llbitmap_infect_dirty_bits(llbitmap, pctl, block); 636 return; 637 } 638 } 639 } 640 641 static void llbitmap_write(struct llbitmap *llbitmap, enum llbitmap_state state, 642 loff_t pos) 643 { 644 unsigned int idx; 645 unsigned int bit; 646 647 pos += BITMAP_DATA_OFFSET; 648 idx = pos >> PAGE_SHIFT; 649 bit = offset_in_page(pos); 650 651 llbitmap->pctl[idx]->state[bit] = state; 652 if (state == BitDirty || state == BitNeedSync) 653 llbitmap_set_page_dirty(llbitmap, idx, bit, true); 654 else if (state == BitNeedSyncUnwritten) 655 llbitmap_set_page_dirty(llbitmap, idx, bit, false); 656 } 657 658 static unsigned int llbitmap_used_pages(struct llbitmap *llbitmap, 659 unsigned long chunks) 660 { 661 return DIV_ROUND_UP(chunks + BITMAP_DATA_OFFSET, PAGE_SIZE); 662 } 663 664 static struct page *llbitmap_read_page(struct llbitmap *llbitmap, int idx) 665 { 666 struct mddev *mddev = llbitmap->mddev; 667 struct page *page = NULL; 668 struct md_rdev *rdev; 669 670 if (llbitmap->pctl && idx < llbitmap->nr_pages && llbitmap->pctl[idx]) 671 page = llbitmap->pctl[idx]->page; 672 if (page) 673 return page; 674 675 page = alloc_page(GFP_NOIO | __GFP_ZERO); 676 if (!page) 677 return ERR_PTR(-ENOMEM); 678 if (idx >= llbitmap_used_pages(llbitmap, llbitmap->chunks)) 679 return page; 680 681 rdev_for_each(rdev, mddev) { 682 sector_t sector; 683 684 if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags) || 685 !test_bit(In_sync, &rdev->flags)) 686 continue; 687 688 sector = mddev->bitmap_info.offset + 689 (idx << PAGE_SECTORS_SHIFT); 690 691 if (sync_page_io(rdev, sector, PAGE_SIZE, page, REQ_OP_READ, 692 true)) 693 return page; 694 695 md_error(mddev, rdev); 696 } 697 698 __free_page(page); 699 return ERR_PTR(-EIO); 700 } 701 702 static void llbitmap_write_page(struct llbitmap *llbitmap, int idx) 703 { 704 struct page *page = llbitmap->pctl[idx]->page; 705 struct mddev *mddev = llbitmap->mddev; 706 struct md_rdev *rdev; 707 int block; 708 709 for (block = 0; block < llbitmap->blocks_per_page; block++) { 710 struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx]; 711 712 if (!test_and_clear_bit(block, pctl->dirty)) 713 continue; 714 715 rdev_for_each(rdev, mddev) { 716 sector_t sector; 717 sector_t bit_sector = llbitmap->io_size >> SECTOR_SHIFT; 718 719 if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags)) 720 continue; 721 722 sector = mddev->bitmap_info.offset + rdev->sb_start + 723 (idx << PAGE_SECTORS_SHIFT) + 724 block * bit_sector; 725 md_write_metadata(mddev, rdev, sector, 726 llbitmap->io_size, page, 727 block * llbitmap->io_size); 728 } 729 } 730 } 731 732 static void active_release(struct percpu_ref *ref) 733 { 734 struct llbitmap_page_ctl *pctl = 735 container_of(ref, struct llbitmap_page_ctl, active); 736 737 wake_up(&pctl->wait); 738 } 739 740 static void llbitmap_free_pages(struct llbitmap *llbitmap) 741 { 742 int i; 743 744 if (!llbitmap->pctl) 745 return; 746 747 for (i = 0; i < llbitmap->nr_pages; i++) { 748 struct llbitmap_page_ctl *pctl = llbitmap->pctl[i]; 749 750 if (!pctl) 751 continue; 752 if (pctl->page) 753 __free_page(pctl->page); 754 percpu_ref_exit(&pctl->active); 755 kfree(pctl); 756 } 757 758 kfree(llbitmap->pctl); 759 llbitmap->pctl = NULL; 760 } 761 762 static struct llbitmap_page_ctl * 763 llbitmap_alloc_page_ctl(struct llbitmap *llbitmap, int idx) 764 { 765 struct llbitmap_page_ctl *pctl; 766 struct page *page; 767 unsigned int size = struct_size(pctl, dirty, BITS_TO_LONGS( 768 llbitmap->blocks_per_page)); 769 770 size = round_up(size, cache_line_size()); 771 pctl = kzalloc(size, GFP_NOIO); 772 if (!pctl) 773 return ERR_PTR(-ENOMEM); 774 775 page = llbitmap_read_page(llbitmap, idx); 776 777 if (IS_ERR(page)) { 778 kfree(pctl); 779 return ERR_CAST(page); 780 } 781 782 if (percpu_ref_init(&pctl->active, active_release, 783 PERCPU_REF_ALLOW_REINIT, GFP_NOIO)) { 784 __free_page(page); 785 kfree(pctl); 786 return ERR_PTR(-ENOMEM); 787 } 788 789 pctl->page = page; 790 pctl->state = page_address(page); 791 init_waitqueue_head(&pctl->wait); 792 return pctl; 793 } 794 795 static unsigned int llbitmap_reserved_pages(struct llbitmap *llbitmap) 796 { 797 return DIV_ROUND_UP(llbitmap->mddev->bitmap_info.space << SECTOR_SHIFT, 798 PAGE_SIZE); 799 } 800 801 static int llbitmap_expand_pages(struct llbitmap *llbitmap, 802 unsigned long chunks) 803 { 804 struct llbitmap_page_ctl **pctl; 805 unsigned int old_nr_pages = llbitmap->nr_pages; 806 unsigned int nr_pages = llbitmap_used_pages(llbitmap, chunks); 807 unsigned int i; 808 int ret; 809 810 if (nr_pages <= old_nr_pages) 811 return 0; 812 813 pctl = kcalloc(nr_pages, sizeof(*pctl), GFP_NOIO); 814 if (!pctl) 815 return -ENOMEM; 816 817 if (llbitmap->pctl) 818 memcpy(pctl, llbitmap->pctl, 819 array_size(old_nr_pages, sizeof(*pctl))); 820 821 for (i = old_nr_pages; i < nr_pages; i++) { 822 pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i); 823 if (IS_ERR(pctl[i])) 824 goto err_alloc_ptr; 825 } 826 827 kfree(llbitmap->pctl); 828 llbitmap->pctl = pctl; 829 llbitmap->nr_pages = nr_pages; 830 return 0; 831 832 err_alloc_ptr: 833 ret = PTR_ERR(pctl[i]); 834 while (i-- > old_nr_pages) { 835 __free_page(pctl[i]->page); 836 percpu_ref_exit(&pctl[i]->active); 837 kfree(pctl[i]); 838 } 839 kfree(pctl); 840 return ret; 841 } 842 843 static int llbitmap_alloc_pages(struct llbitmap *llbitmap) 844 { 845 unsigned int used_pages = llbitmap_used_pages(llbitmap, llbitmap->chunks); 846 unsigned int nr_pages = max(used_pages, llbitmap_reserved_pages(llbitmap)); 847 int i; 848 849 llbitmap->pctl = kcalloc(nr_pages, sizeof(*llbitmap->pctl), GFP_NOIO); 850 if (!llbitmap->pctl) 851 return -ENOMEM; 852 853 llbitmap->nr_pages = nr_pages; 854 855 for (i = 0; i < nr_pages; i++) { 856 llbitmap->pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i); 857 if (IS_ERR(llbitmap->pctl[i])) { 858 int ret = PTR_ERR(llbitmap->pctl[i]); 859 860 llbitmap->pctl[i] = NULL; 861 llbitmap_free_pages(llbitmap); 862 return ret; 863 } 864 } 865 866 return 0; 867 } 868 869 /* 870 * Check if all underlying disks support write_zeroes with unmap. 871 */ 872 static bool llbitmap_all_disks_support_wzeroes_unmap(struct llbitmap *llbitmap) 873 { 874 struct mddev *mddev = llbitmap->mddev; 875 struct md_rdev *rdev; 876 877 rdev_for_each(rdev, mddev) { 878 if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags)) 879 continue; 880 881 if (bdev_write_zeroes_unmap_sectors(rdev->bdev) == 0) 882 return false; 883 } 884 885 return true; 886 } 887 888 /* 889 * Issue write_zeroes to all underlying disks to zero their data regions. 890 * This ensures parity consistency for RAID-456 (0 XOR 0 = 0). 891 * Returns true if all disks were successfully zeroed. 892 */ 893 static bool llbitmap_zero_all_disks(struct llbitmap *llbitmap) 894 { 895 struct mddev *mddev = llbitmap->mddev; 896 struct md_rdev *rdev; 897 sector_t dev_sectors = mddev->dev_sectors; 898 int ret; 899 900 rdev_for_each(rdev, mddev) { 901 if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags)) 902 continue; 903 904 ret = blkdev_issue_zeroout(rdev->bdev, 905 rdev->data_offset, 906 dev_sectors, 907 GFP_KERNEL, 0); 908 if (ret) { 909 pr_warn("md/llbitmap: failed to zero disk %pg: %d\n", 910 rdev->bdev, ret); 911 return false; 912 } 913 } 914 915 return true; 916 } 917 918 static void llbitmap_mark_range(struct llbitmap *llbitmap, 919 unsigned long start, 920 unsigned long end, 921 enum llbitmap_state state) 922 { 923 while (start <= end) { 924 llbitmap_write(llbitmap, state, start); 925 start++; 926 } 927 } 928 929 static int llbitmap_prepare_resize(struct llbitmap *llbitmap, 930 unsigned long old_chunks, 931 unsigned long new_chunks, 932 unsigned long cache_chunks) 933 { 934 int ret; 935 936 llbitmap_flush(llbitmap->mddev); 937 ret = llbitmap_expand_pages(llbitmap, cache_chunks); 938 if (ret) 939 return ret; 940 if (new_chunks > old_chunks) 941 llbitmap_mark_range(llbitmap, old_chunks, new_chunks - 1, 942 BitUnwritten); 943 return 0; 944 } 945 946 static enum llbitmap_state 947 llbitmap_rmerge_state(struct llbitmap *llbitmap, 948 enum llbitmap_state dst, 949 enum llbitmap_state src) 950 { 951 bool level_456 = raid_is_456(llbitmap->mddev); 952 953 if (dst == BitNeedSync || dst == BitSyncing || 954 src == BitNeedSync || src == BitSyncing) 955 return BitNeedSync; 956 957 if (dst == BitDirty || src == BitDirty) 958 return BitDirty; 959 960 /* 961 * Reshape generates valid target parity/data for both already-written 962 * and not-yet-written regions in the checkpointed range, so a mix of 963 * clean and unwritten still results in a clean destination bit. 964 */ 965 if (level_456 && ((dst == BitClean && src == BitUnwritten) || 966 (src == BitClean && dst == BitUnwritten))) 967 return BitClean; 968 if (dst == BitClean || src == BitClean) 969 return BitClean; 970 return BitUnwritten; 971 } 972 973 static void llbitmap_init_state(struct llbitmap *llbitmap) 974 { 975 struct mddev *mddev = llbitmap->mddev; 976 enum llbitmap_state state = BitUnwritten; 977 unsigned long i; 978 979 if (test_and_clear_bit(BITMAP_CLEAN, &llbitmap->flags)) { 980 state = BitClean; 981 } else if (raid_is_456(mddev) && 982 llbitmap_all_disks_support_wzeroes_unmap(llbitmap)) { 983 /* 984 * All disks support write_zeroes with unmap. Zero all disks 985 * to ensure parity consistency, then set BitCleanUnwritten 986 * to skip initial sync. 987 */ 988 if (llbitmap_zero_all_disks(llbitmap)) 989 state = BitCleanUnwritten; 990 } 991 992 for (i = 0; i < llbitmap->chunks; i++) 993 llbitmap_write(llbitmap, state, i); 994 } 995 996 /* The return value is only used from resync, where @start == @end. */ 997 static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap, 998 unsigned long start, 999 unsigned long end, 1000 enum llbitmap_action action) 1001 { 1002 struct mddev *mddev = llbitmap->mddev; 1003 enum llbitmap_state state = BitNone; 1004 bool level_456 = raid_is_456(llbitmap->mddev); 1005 bool need_resync = false; 1006 bool need_recovery = false; 1007 1008 if (test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags)) 1009 return BitNone; 1010 1011 if (action == BitmapActionInit) { 1012 llbitmap_init_state(llbitmap); 1013 return BitNone; 1014 } 1015 if (start >= llbitmap->chunks) 1016 return BitNone; 1017 if (end >= llbitmap->chunks) 1018 end = llbitmap->chunks - 1; 1019 while (start <= end) { 1020 enum llbitmap_state c = llbitmap_read(llbitmap, start); 1021 1022 if (c < 0 || c >= BitStateCount) { 1023 pr_err("%s: invalid bit %lu state %d action %d, forcing resync\n", 1024 __func__, start, c, action); 1025 state = BitNeedSync; 1026 goto write_bitmap; 1027 } 1028 1029 if (c == BitNeedSync || c == BitNeedSyncUnwritten) 1030 need_resync = !mddev->degraded; 1031 1032 state = state_machine[c][action]; 1033 write_bitmap: 1034 if (unlikely(mddev->degraded)) { 1035 /* For degraded array, mark new data as need sync. */ 1036 if (state == BitDirty && 1037 action == BitmapActionStartwrite) 1038 state = BitNeedSync; 1039 /* 1040 * For degraded array, resync dirty data as well, noted 1041 * if array is still degraded after resync is done, all 1042 * new data will still be dirty until array is clean. 1043 */ 1044 else if (c == BitDirty && 1045 action == BitmapActionStartsync) 1046 state = BitSyncing; 1047 } else if (c == BitUnwritten && state == BitDirty && 1048 action == BitmapActionStartwrite && level_456) { 1049 /* Delay raid456 initial recovery to first write. */ 1050 state = BitNeedSync; 1051 } 1052 1053 if (state == BitNone) { 1054 start++; 1055 continue; 1056 } 1057 1058 llbitmap_write(llbitmap, state, start); 1059 if (state == BitNeedSync || state == BitNeedSyncUnwritten) 1060 need_resync = !mddev->degraded; 1061 else if (state == BitDirty && 1062 !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags) && 1063 !timer_pending(&llbitmap->pending_timer)) 1064 mod_timer(&llbitmap->pending_timer, 1065 jiffies + mddev->bitmap_info.daemon_sleep * HZ); 1066 1067 start++; 1068 } 1069 1070 if (need_resync && level_456) 1071 need_recovery = true; 1072 1073 if (need_recovery) { 1074 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 1075 set_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery); 1076 md_wakeup_thread(mddev->thread); 1077 } else if (need_resync) { 1078 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 1079 set_bit(MD_RECOVERY_SYNC, &mddev->recovery); 1080 md_wakeup_thread(mddev->thread); 1081 } 1082 1083 return state; 1084 } 1085 1086 static void llbitmap_raise_barrier(struct llbitmap *llbitmap, int page_idx) 1087 { 1088 struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx]; 1089 1090 retry: 1091 if (likely(percpu_ref_tryget_live(&pctl->active))) { 1092 WRITE_ONCE(pctl->expire, jiffies + llbitmap->barrier_idle * HZ); 1093 return; 1094 } 1095 1096 wait_event(pctl->wait, !percpu_ref_is_dying(&pctl->active)); 1097 goto retry; 1098 } 1099 1100 static void llbitmap_release_barrier(struct llbitmap *llbitmap, int page_idx) 1101 { 1102 struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx]; 1103 1104 percpu_ref_put(&pctl->active); 1105 } 1106 1107 static int llbitmap_suspend_timeout(struct llbitmap *llbitmap, int page_idx) 1108 { 1109 struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx]; 1110 1111 percpu_ref_kill(&pctl->active); 1112 1113 if (!wait_event_timeout(pctl->wait, percpu_ref_is_zero(&pctl->active), 1114 llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) { 1115 percpu_ref_resurrect(&pctl->active); 1116 return -ETIMEDOUT; 1117 } 1118 1119 return 0; 1120 } 1121 1122 static void llbitmap_resume(struct llbitmap *llbitmap, int page_idx) 1123 { 1124 struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx]; 1125 1126 pctl->expire = LONG_MAX; 1127 percpu_ref_resurrect(&pctl->active); 1128 wake_up(&pctl->wait); 1129 } 1130 1131 static int llbitmap_check_support(struct mddev *mddev) 1132 { 1133 if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) { 1134 pr_notice("md/llbitmap: %s: array with journal cannot have bitmap\n", 1135 mdname(mddev)); 1136 return -EBUSY; 1137 } 1138 1139 if (mddev->bitmap_info.space == 0) { 1140 if (mddev->bitmap_info.default_space == 0) { 1141 pr_notice("md/llbitmap: %s: no space for bitmap\n", 1142 mdname(mddev)); 1143 return -ENOSPC; 1144 } 1145 } 1146 1147 if (!mddev->persistent) { 1148 pr_notice("md/llbitmap: %s: array must be persistent\n", 1149 mdname(mddev)); 1150 return -EOPNOTSUPP; 1151 } 1152 1153 if (mddev->bitmap_info.file) { 1154 pr_notice("md/llbitmap: %s: doesn't support bitmap file\n", 1155 mdname(mddev)); 1156 return -EOPNOTSUPP; 1157 } 1158 1159 if (mddev->bitmap_info.external) { 1160 pr_notice("md/llbitmap: %s: doesn't support external metadata\n", 1161 mdname(mddev)); 1162 return -EOPNOTSUPP; 1163 } 1164 1165 if (mddev_is_dm(mddev)) { 1166 pr_notice("md/llbitmap: %s: doesn't support dm-raid\n", 1167 mdname(mddev)); 1168 return -EOPNOTSUPP; 1169 } 1170 1171 return 0; 1172 } 1173 1174 static int llbitmap_init(struct llbitmap *llbitmap) 1175 { 1176 struct mddev *mddev = llbitmap->mddev; 1177 sector_t blocks = mddev->resync_max_sectors; 1178 unsigned long chunksize = MIN_CHUNK_SIZE; 1179 unsigned long chunks = DIV_ROUND_UP(blocks, chunksize); 1180 unsigned long space = mddev->bitmap_info.space << SECTOR_SHIFT; 1181 int ret; 1182 1183 while (chunks > space) { 1184 chunksize = chunksize << 1; 1185 chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize); 1186 } 1187 1188 llbitmap->barrier_idle = DEFAULT_BARRIER_IDLE; 1189 llbitmap->chunkshift = ffz(~chunksize); 1190 llbitmap->chunksize = chunksize; 1191 llbitmap->chunks = chunks; 1192 llbitmap->sync_size = blocks; 1193 llbitmap_refresh_reshape(llbitmap); 1194 mddev->bitmap_info.daemon_sleep = DEFAULT_DAEMON_SLEEP; 1195 1196 ret = llbitmap_alloc_pages(llbitmap); 1197 if (ret) 1198 return ret; 1199 1200 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, 1201 BitmapActionInit); 1202 /* flush initial llbitmap to disk */ 1203 __llbitmap_flush(mddev); 1204 1205 return 0; 1206 } 1207 1208 static int llbitmap_read_sb(struct llbitmap *llbitmap) 1209 { 1210 struct mddev *mddev = llbitmap->mddev; 1211 unsigned long daemon_sleep; 1212 unsigned long chunksize; 1213 unsigned long events; 1214 sector_t sync_size; 1215 struct page *sb_page; 1216 bitmap_super_t *sb; 1217 int ret = -EINVAL; 1218 1219 if (!mddev->bitmap_info.offset) { 1220 pr_err("md/llbitmap: %s: no super block found", mdname(mddev)); 1221 return -EINVAL; 1222 } 1223 1224 sb_page = llbitmap_read_page(llbitmap, 0); 1225 if (IS_ERR(sb_page)) { 1226 pr_err("md/llbitmap: %s: read super block failed", 1227 mdname(mddev)); 1228 return -EIO; 1229 } 1230 1231 sb = kmap_local_page(sb_page); 1232 if (sb->magic != cpu_to_le32(BITMAP_MAGIC)) { 1233 pr_err("md/llbitmap: %s: invalid super block magic number", 1234 mdname(mddev)); 1235 goto out_put_page; 1236 } 1237 1238 if (sb->version != cpu_to_le32(BITMAP_MAJOR_LOCKLESS)) { 1239 pr_err("md/llbitmap: %s: invalid super block version", 1240 mdname(mddev)); 1241 goto out_put_page; 1242 } 1243 1244 if (memcmp(sb->uuid, mddev->uuid, 16)) { 1245 pr_err("md/llbitmap: %s: bitmap superblock UUID mismatch\n", 1246 mdname(mddev)); 1247 goto out_put_page; 1248 } 1249 1250 if (mddev->bitmap_info.space == 0) { 1251 int room = le32_to_cpu(sb->sectors_reserved); 1252 1253 if (room) 1254 mddev->bitmap_info.space = room; 1255 else 1256 mddev->bitmap_info.space = mddev->bitmap_info.default_space; 1257 } 1258 llbitmap->flags = le32_to_cpu(sb->state) & ~BIT(BITMAP_SHUTDOWN); 1259 if (test_and_clear_bit(BITMAP_FIRST_USE, &llbitmap->flags)) { 1260 ret = llbitmap_init(llbitmap); 1261 goto out_put_page; 1262 } 1263 1264 sync_size = le64_to_cpu(sb->sync_size); 1265 if (!sync_size) 1266 sync_size = mddev->resync_max_sectors; 1267 if (sync_size > mddev->resync_max_sectors) { 1268 pr_err("md/llbitmap: %s: sync_size %llu exceeds array sync size %llu", 1269 mdname(mddev), sync_size, mddev->resync_max_sectors); 1270 goto out_put_page; 1271 } 1272 chunksize = le32_to_cpu(sb->chunksize); 1273 if (!is_power_of_2(chunksize)) { 1274 pr_err("md/llbitmap: %s: chunksize not a power of 2", 1275 mdname(mddev)); 1276 goto out_put_page; 1277 } 1278 1279 if (chunksize < DIV_ROUND_UP_SECTOR_T(sync_size, 1280 mddev->bitmap_info.space << SECTOR_SHIFT)) { 1281 pr_err("md/llbitmap: %s: chunksize too small %lu < %llu / %lu", 1282 mdname(mddev), chunksize, sync_size, 1283 mddev->bitmap_info.space); 1284 goto out_put_page; 1285 } 1286 1287 daemon_sleep = le32_to_cpu(sb->daemon_sleep); 1288 if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT / HZ) { 1289 pr_err("md/llbitmap: %s: daemon sleep %lu period out of range", 1290 mdname(mddev), daemon_sleep); 1291 goto out_put_page; 1292 } 1293 1294 events = le64_to_cpu(sb->events); 1295 if (events < mddev->events) { 1296 pr_warn("md/llbitmap :%s: bitmap file is out of date (%lu < %llu) -- forcing full recovery", 1297 mdname(mddev), events, mddev->events); 1298 set_bit(BITMAP_STALE, &llbitmap->flags); 1299 } 1300 1301 sb->sync_size = cpu_to_le64(mddev->resync_max_sectors); 1302 mddev->bitmap_info.chunksize = chunksize; 1303 mddev->bitmap_info.daemon_sleep = daemon_sleep; 1304 1305 llbitmap->barrier_idle = DEFAULT_BARRIER_IDLE; 1306 llbitmap->chunksize = chunksize; 1307 llbitmap->chunks = DIV_ROUND_UP_SECTOR_T(sync_size, chunksize); 1308 llbitmap->chunkshift = ffz(~chunksize); 1309 llbitmap->sync_size = sync_size; 1310 llbitmap_refresh_reshape(llbitmap); 1311 ret = llbitmap_alloc_pages(llbitmap); 1312 1313 out_put_page: 1314 __free_page(sb_page); 1315 kunmap_local(sb); 1316 return ret; 1317 } 1318 1319 static void llbitmap_pending_timer_fn(struct timer_list *pending_timer) 1320 { 1321 struct llbitmap *llbitmap = 1322 container_of(pending_timer, struct llbitmap, pending_timer); 1323 1324 if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) 1325 return; 1326 1327 if (work_busy(&llbitmap->daemon_work)) { 1328 pr_warn("md/llbitmap: %s daemon_work not finished in %lu seconds\n", 1329 mdname(llbitmap->mddev), 1330 llbitmap->mddev->bitmap_info.daemon_sleep); 1331 set_bit(BITMAP_DAEMON_BUSY, &llbitmap->flags); 1332 return; 1333 } 1334 1335 queue_work(md_llbitmap_io_wq, &llbitmap->daemon_work); 1336 } 1337 1338 static void md_llbitmap_daemon_fn(struct work_struct *work) 1339 { 1340 struct llbitmap *llbitmap = 1341 container_of(work, struct llbitmap, daemon_work); 1342 unsigned long start; 1343 unsigned long end; 1344 bool restart; 1345 int idx; 1346 1347 if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) 1348 return; 1349 1350 if (llbitmap->mddev->degraded) 1351 return; 1352 1353 retry: 1354 start = 0; 1355 end = min(llbitmap->chunks, PAGE_SIZE - BITMAP_DATA_OFFSET) - 1; 1356 restart = false; 1357 1358 for (idx = 0; idx < llbitmap->nr_pages; idx++) { 1359 struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx]; 1360 bool flush = test_and_clear_bit(LLPageFlush, &pctl->flags); 1361 1362 if (idx > 0) { 1363 start = end + 1; 1364 end = min(end + PAGE_SIZE, llbitmap->chunks - 1); 1365 } 1366 1367 if (!flush && time_before(jiffies, pctl->expire)) { 1368 restart = true; 1369 continue; 1370 } 1371 1372 if (llbitmap_suspend_timeout(llbitmap, idx) < 0) { 1373 pr_warn("md/llbitmap: %s: %s waiting for page %d timeout\n", 1374 mdname(llbitmap->mddev), __func__, idx); 1375 continue; 1376 } 1377 1378 llbitmap_state_machine(llbitmap, start, end, BitmapActionDaemon); 1379 llbitmap_resume(llbitmap, idx); 1380 } 1381 1382 /* 1383 * If the daemon took a long time to finish, retry to prevent missing 1384 * clearing dirty bits. 1385 */ 1386 if (test_and_clear_bit(BITMAP_DAEMON_BUSY, &llbitmap->flags)) 1387 goto retry; 1388 1389 /* If some page is dirty but not expired, setup timer again */ 1390 if (restart && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) 1391 mod_timer(&llbitmap->pending_timer, 1392 jiffies + llbitmap->mddev->bitmap_info.daemon_sleep * HZ); 1393 } 1394 1395 static int llbitmap_create(struct mddev *mddev) 1396 { 1397 struct llbitmap *llbitmap; 1398 int ret; 1399 1400 ret = llbitmap_check_support(mddev); 1401 if (ret) 1402 return ret; 1403 1404 llbitmap = kzalloc_obj(*llbitmap, GFP_NOIO); 1405 if (!llbitmap) 1406 return -ENOMEM; 1407 1408 llbitmap->mddev = mddev; 1409 llbitmap->io_size = bdev_logical_block_size(mddev->gendisk->part0); 1410 llbitmap->blocks_per_page = PAGE_SIZE / llbitmap->io_size; 1411 1412 timer_setup(&llbitmap->pending_timer, llbitmap_pending_timer_fn, 0); 1413 INIT_WORK(&llbitmap->daemon_work, md_llbitmap_daemon_fn); 1414 rwlock_init(&llbitmap->reshape_lock); 1415 atomic_set(&llbitmap->behind_writes, 0); 1416 init_waitqueue_head(&llbitmap->behind_wait); 1417 1418 mutex_lock(&mddev->bitmap_info.mutex); 1419 mddev->bitmap = llbitmap; 1420 ret = llbitmap_read_sb(llbitmap); 1421 if (ret) 1422 mddev->bitmap = NULL; 1423 mutex_unlock(&mddev->bitmap_info.mutex); 1424 if (ret) { 1425 kfree(llbitmap); 1426 } 1427 1428 return ret; 1429 } 1430 1431 static int llbitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize) 1432 { 1433 struct llbitmap *llbitmap = mddev->bitmap; 1434 sector_t old_blocks = llbitmap->sync_size; 1435 unsigned long old_chunks = llbitmap->chunks; 1436 unsigned long chunks; 1437 unsigned long cache_chunks; 1438 int ret = 0; 1439 unsigned long bitmap_chunksize; 1440 bool reshape; 1441 bool quiesced = false; 1442 1443 if (chunksize == 0) 1444 chunksize = llbitmap->chunksize; 1445 1446 bitmap_chunksize = chunksize; 1447 llbitmap_calculate_chunks(mddev, blocks, &bitmap_chunksize, &chunks); 1448 1449 reshape = mddev->delta_disks || mddev->new_level != mddev->level || 1450 mddev->new_layout != mddev->layout || 1451 mddev->new_chunk_sectors != mddev->chunk_sectors; 1452 if (!reshape && bitmap_chunksize != llbitmap->chunksize) 1453 return -EOPNOTSUPP; 1454 if (blocks == old_blocks && chunks == llbitmap->chunks) 1455 return 0; 1456 1457 if (mddev->pers->quiesce) { 1458 mddev->pers->quiesce(mddev, 1); 1459 quiesced = true; 1460 } 1461 1462 mutex_lock(&mddev->bitmap_info.mutex); 1463 cache_chunks = reshape ? max(old_chunks, chunks) : chunks; 1464 ret = llbitmap_prepare_resize(llbitmap, old_chunks, chunks, cache_chunks); 1465 if (ret) 1466 goto out; 1467 1468 if (reshape) { 1469 llbitmap->reshape_sync_size = blocks; 1470 llbitmap->reshape_chunksize = bitmap_chunksize; 1471 llbitmap->reshape_chunks = chunks; 1472 llbitmap->chunks = max(old_chunks, chunks); 1473 } else { 1474 if (blocks < old_blocks && chunks < old_chunks) 1475 llbitmap_mark_range(llbitmap, chunks, old_chunks - 1, 1476 BitUnwritten); 1477 mddev->bitmap_info.chunksize = bitmap_chunksize; 1478 llbitmap->chunks = chunks; 1479 llbitmap->sync_size = blocks; 1480 llbitmap_refresh_reshape(llbitmap); 1481 llbitmap_update_sb(llbitmap); 1482 } 1483 __llbitmap_flush(mddev); 1484 mutex_unlock(&mddev->bitmap_info.mutex); 1485 if (quiesced) 1486 mddev->pers->quiesce(mddev, 0); 1487 return 0; 1488 1489 out: 1490 mutex_unlock(&mddev->bitmap_info.mutex); 1491 if (quiesced) 1492 mddev->pers->quiesce(mddev, 0); 1493 return ret; 1494 } 1495 1496 static int llbitmap_load(struct mddev *mddev) 1497 { 1498 enum llbitmap_action action = BitmapActionReload; 1499 struct llbitmap *llbitmap = mddev->bitmap; 1500 int ret; 1501 1502 if (test_and_clear_bit(BITMAP_STALE, &llbitmap->flags)) 1503 action = BitmapActionStale; 1504 1505 mutex_lock(&mddev->bitmap_info.mutex); 1506 llbitmap_refresh_reshape(llbitmap); 1507 ret = llbitmap_expand_pages(llbitmap, llbitmap->chunks); 1508 if (ret) { 1509 mutex_unlock(&mddev->bitmap_info.mutex); 1510 return ret; 1511 } 1512 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, action); 1513 mutex_unlock(&mddev->bitmap_info.mutex); 1514 return 0; 1515 } 1516 1517 static void llbitmap_destroy(struct mddev *mddev) 1518 { 1519 struct llbitmap *llbitmap = mddev->bitmap; 1520 1521 if (!llbitmap) 1522 return; 1523 1524 mutex_lock(&mddev->bitmap_info.mutex); 1525 1526 set_bit(BITMAP_SHUTDOWN, &llbitmap->flags); 1527 timer_shutdown_sync(&llbitmap->pending_timer); 1528 cancel_work_sync(&llbitmap->daemon_work); 1529 flush_workqueue(md_llbitmap_io_wq); 1530 flush_workqueue(md_llbitmap_unplug_wq); 1531 1532 mddev->bitmap = NULL; 1533 llbitmap_free_pages(llbitmap); 1534 kfree(llbitmap); 1535 mutex_unlock(&mddev->bitmap_info.mutex); 1536 } 1537 1538 static bool llbitmap_map_previous(struct llbitmap *llbitmap, sector_t offset, 1539 unsigned long sectors) 1540 { 1541 struct mddev *mddev = llbitmap->mddev; 1542 sector_t boundary = READ_ONCE(mddev->reshape_position); 1543 1544 if (boundary == MaxSector) 1545 return false; 1546 1547 WARN_ON_ONCE(sectors && offset < boundary && offset + sectors > boundary); 1548 1549 return mddev->reshape_backwards ? offset < boundary : offset >= boundary; 1550 } 1551 1552 static void llbitmap_prepare_range(struct mddev *mddev, sector_t *offset, 1553 unsigned long *sectors, bool discard) 1554 { 1555 struct llbitmap *llbitmap = mddev->bitmap; 1556 bool previous; 1557 1558 if (!llbitmap) 1559 return; 1560 1561 previous = llbitmap_map_previous(llbitmap, *offset, *sectors); 1562 llbitmap_map_layout(llbitmap, offset, sectors, previous); 1563 if (discard) 1564 llbitmap_encode_discard_range(llbitmap, offset, sectors, previous); 1565 else 1566 llbitmap_encode_range(llbitmap, offset, sectors, previous); 1567 } 1568 1569 static void llbitmap_start_write(struct mddev *mddev, sector_t offset, 1570 unsigned long sectors) 1571 { 1572 struct llbitmap *llbitmap = mddev->bitmap; 1573 unsigned long start = offset >> llbitmap->chunkshift; 1574 unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift; 1575 int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; 1576 int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; 1577 1578 while (page_start <= page_end) { 1579 llbitmap_raise_barrier(llbitmap, page_start); 1580 page_start++; 1581 } 1582 1583 read_lock(&llbitmap->reshape_lock); 1584 llbitmap_state_machine(llbitmap, start, end, BitmapActionStartwrite); 1585 read_unlock(&llbitmap->reshape_lock); 1586 } 1587 1588 static void llbitmap_end_write(struct mddev *mddev, sector_t offset, 1589 unsigned long sectors) 1590 { 1591 struct llbitmap *llbitmap = mddev->bitmap; 1592 unsigned long start = offset >> llbitmap->chunkshift; 1593 unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift; 1594 int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; 1595 int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; 1596 1597 while (page_start <= page_end) { 1598 llbitmap_release_barrier(llbitmap, page_start); 1599 page_start++; 1600 } 1601 } 1602 1603 static void llbitmap_start_discard(struct mddev *mddev, sector_t offset, 1604 unsigned long sectors) 1605 { 1606 struct llbitmap *llbitmap = mddev->bitmap; 1607 unsigned long start = DIV_ROUND_UP_SECTOR_T(offset, llbitmap->chunksize); 1608 unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift; 1609 int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; 1610 int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; 1611 1612 while (page_start <= page_end) { 1613 llbitmap_raise_barrier(llbitmap, page_start); 1614 page_start++; 1615 } 1616 1617 read_lock(&llbitmap->reshape_lock); 1618 llbitmap_state_machine(llbitmap, start, end, BitmapActionDiscard); 1619 read_unlock(&llbitmap->reshape_lock); 1620 } 1621 1622 static void llbitmap_end_discard(struct mddev *mddev, sector_t offset, 1623 unsigned long sectors) 1624 { 1625 struct llbitmap *llbitmap = mddev->bitmap; 1626 unsigned long start = DIV_ROUND_UP_SECTOR_T(offset, llbitmap->chunksize); 1627 unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift; 1628 int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; 1629 int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; 1630 1631 while (page_start <= page_end) { 1632 llbitmap_release_barrier(llbitmap, page_start); 1633 page_start++; 1634 } 1635 } 1636 1637 static void llbitmap_unplug_fn(struct work_struct *work) 1638 { 1639 struct llbitmap_unplug_work *unplug_work = 1640 container_of(work, struct llbitmap_unplug_work, work); 1641 struct llbitmap *llbitmap = unplug_work->llbitmap; 1642 struct blk_plug plug; 1643 int i; 1644 1645 blk_start_plug(&plug); 1646 1647 for (i = 0; i < llbitmap->nr_pages; i++) { 1648 if (!test_bit(LLPageDirty, &llbitmap->pctl[i]->flags) || 1649 !test_and_clear_bit(LLPageDirty, &llbitmap->pctl[i]->flags)) 1650 continue; 1651 1652 llbitmap_write_page(llbitmap, i); 1653 } 1654 1655 blk_finish_plug(&plug); 1656 md_super_wait(llbitmap->mddev); 1657 complete(unplug_work->done); 1658 } 1659 1660 static bool llbitmap_dirty(struct llbitmap *llbitmap) 1661 { 1662 int i; 1663 1664 for (i = 0; i < llbitmap->nr_pages; i++) 1665 if (test_bit(LLPageDirty, &llbitmap->pctl[i]->flags)) 1666 return true; 1667 1668 return false; 1669 } 1670 1671 static void llbitmap_unplug(struct mddev *mddev, bool sync) 1672 { 1673 DECLARE_COMPLETION_ONSTACK(done); 1674 struct llbitmap *llbitmap = mddev->bitmap; 1675 struct llbitmap_unplug_work unplug_work = { 1676 .llbitmap = llbitmap, 1677 .done = &done, 1678 }; 1679 1680 if (!llbitmap_dirty(llbitmap)) 1681 return; 1682 1683 /* 1684 * Issue new bitmap IO under submit_bio() context will deadlock: 1685 * - the bio will wait for bitmap bio to be done, before it can be 1686 * issued; 1687 * - bitmap bio will be added to current->bio_list and wait for this 1688 * bio to be issued; 1689 */ 1690 INIT_WORK_ONSTACK(&unplug_work.work, llbitmap_unplug_fn); 1691 queue_work(md_llbitmap_unplug_wq, &unplug_work.work); 1692 wait_for_completion(&done); 1693 destroy_work_on_stack(&unplug_work.work); 1694 } 1695 1696 /* 1697 * Force to write all bitmap pages to disk, called when stopping the array, or 1698 * every daemon_sleep seconds when sync_thread is running. 1699 */ 1700 static void __llbitmap_flush(struct mddev *mddev) 1701 { 1702 struct llbitmap *llbitmap = mddev->bitmap; 1703 struct blk_plug plug; 1704 int i; 1705 1706 blk_start_plug(&plug); 1707 for (i = 0; i < llbitmap->nr_pages; i++) { 1708 struct llbitmap_page_ctl *pctl = llbitmap->pctl[i]; 1709 1710 /* mark all blocks as dirty */ 1711 set_bit(LLPageDirty, &pctl->flags); 1712 bitmap_fill(pctl->dirty, llbitmap->blocks_per_page); 1713 llbitmap_write_page(llbitmap, i); 1714 } 1715 blk_finish_plug(&plug); 1716 md_super_wait(llbitmap->mddev); 1717 } 1718 1719 static void llbitmap_flush(struct mddev *mddev) 1720 { 1721 struct llbitmap *llbitmap = mddev->bitmap; 1722 int i; 1723 1724 for (i = 0; i < llbitmap->nr_pages; i++) 1725 set_bit(LLPageFlush, &llbitmap->pctl[i]->flags); 1726 1727 timer_delete_sync(&llbitmap->pending_timer); 1728 queue_work(md_llbitmap_io_wq, &llbitmap->daemon_work); 1729 flush_work(&llbitmap->daemon_work); 1730 1731 __llbitmap_flush(mddev); 1732 } 1733 1734 /* This is used for raid5 lazy initial recovery */ 1735 static bool llbitmap_blocks_synced(struct mddev *mddev, sector_t offset) 1736 { 1737 struct llbitmap *llbitmap = mddev->bitmap; 1738 unsigned long p = offset >> llbitmap->chunkshift; 1739 enum llbitmap_state c; 1740 1741 if (p >= llbitmap->chunks) 1742 return false; 1743 c = llbitmap_read(llbitmap, p); 1744 1745 return c == BitClean || c == BitDirty || c == BitCleanUnwritten; 1746 } 1747 1748 static sector_t llbitmap_skip_sync_blocks(struct mddev *mddev, sector_t offset) 1749 { 1750 struct llbitmap *llbitmap = mddev->bitmap; 1751 unsigned long p = offset >> llbitmap->chunkshift; 1752 int blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); 1753 enum llbitmap_state c; 1754 1755 if (p >= llbitmap->chunks) 1756 return 0; 1757 c = llbitmap_read(llbitmap, p); 1758 1759 /* 1760 * Reshape progress is tracked by array metadata rather than llbitmap. 1761 * Skipping reshape ranges from stale bitmap state can lose data after a 1762 * restart before the corresponding bits are checkpointed to disk. 1763 */ 1764 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 1765 return 0; 1766 1767 /* always skip unwritten blocks */ 1768 if (c == BitUnwritten) 1769 return blocks; 1770 1771 /* Skip CleanUnwritten - no user data, will be reset after recovery */ 1772 if (c == BitCleanUnwritten) 1773 return blocks; 1774 1775 /* For degraded array, don't skip */ 1776 if (mddev->degraded) 1777 return 0; 1778 1779 /* For resync also skip clean/dirty blocks */ 1780 if ((c == BitClean || c == BitDirty) && 1781 test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && 1782 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) 1783 return blocks; 1784 1785 return 0; 1786 } 1787 1788 static bool llbitmap_start_sync(struct mddev *mddev, sector_t offset, 1789 sector_t *blocks, bool degraded) 1790 { 1791 struct llbitmap *llbitmap = mddev->bitmap; 1792 unsigned long p = offset >> llbitmap->chunkshift; 1793 enum llbitmap_state state; 1794 1795 /* 1796 * Before recovery starts, convert CleanUnwritten to Unwritten. 1797 * This ensures the new disk won't have stale parity data. 1798 */ 1799 if (offset == 0 && test_bit(MD_RECOVERY_RECOVER, &mddev->recovery) && 1800 !test_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery)) 1801 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, 1802 BitmapActionClearUnwritten); 1803 1804 1805 /* 1806 * Handle one bit at a time, this is much simpler. And it doesn't matter 1807 * if md_do_sync() loop more times. 1808 */ 1809 *blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); 1810 if (p >= llbitmap->chunks) 1811 return false; 1812 state = llbitmap_state_machine(llbitmap, p, p, BitmapActionStartsync); 1813 return state == BitSyncing || state == BitSyncingUnwritten; 1814 } 1815 1816 /* Something is wrong, sync_thread stop at @offset */ 1817 static void llbitmap_end_sync(struct mddev *mddev, sector_t offset, 1818 sector_t *blocks) 1819 { 1820 struct llbitmap *llbitmap = mddev->bitmap; 1821 unsigned long p = offset >> llbitmap->chunkshift; 1822 1823 *blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); 1824 llbitmap_state_machine(llbitmap, p, llbitmap->chunks - 1, 1825 BitmapActionAbortsync); 1826 } 1827 1828 /* A full sync_thread is finished */ 1829 static void llbitmap_close_sync(struct mddev *mddev) 1830 { 1831 struct llbitmap *llbitmap = mddev->bitmap; 1832 int i; 1833 1834 for (i = 0; i < llbitmap->nr_pages; i++) { 1835 struct llbitmap_page_ctl *pctl = llbitmap->pctl[i]; 1836 1837 /* let daemon_fn clear dirty bits immediately */ 1838 WRITE_ONCE(pctl->expire, jiffies); 1839 } 1840 1841 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, 1842 BitmapActionEndsync); 1843 } 1844 1845 /* 1846 * sync_thread have reached @sector, update metadata every daemon_sleep seconds, 1847 * just in case sync_thread have to restart after power failure. 1848 */ 1849 static void llbitmap_cond_end_sync(struct mddev *mddev, sector_t sector, 1850 bool force) 1851 { 1852 struct llbitmap *llbitmap = mddev->bitmap; 1853 sector_t complete; 1854 1855 if (sector == 0) { 1856 llbitmap->last_end_sync = jiffies; 1857 return; 1858 } 1859 1860 if (!force && time_before(jiffies, llbitmap->last_end_sync + 1861 HZ * mddev->bitmap_info.daemon_sleep)) 1862 return; 1863 1864 wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active)); 1865 1866 mddev->curr_resync_completed = sector; 1867 set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags); 1868 1869 complete = round_down(sector, llbitmap->chunksize); 1870 if (complete) 1871 llbitmap_state_machine(llbitmap, 0, 1872 (complete >> llbitmap->chunkshift) - 1, 1873 BitmapActionEndsync); 1874 __llbitmap_flush(mddev); 1875 1876 llbitmap->last_end_sync = jiffies; 1877 sysfs_notify_dirent_safe(mddev->sysfs_completed); 1878 } 1879 1880 static bool llbitmap_enabled(void *data, bool flush) 1881 { 1882 struct llbitmap *llbitmap = data; 1883 1884 return llbitmap && !test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags); 1885 } 1886 1887 static void llbitmap_dirty_bits(struct mddev *mddev, unsigned long s, 1888 unsigned long e) 1889 { 1890 llbitmap_state_machine(mddev->bitmap, s, e, BitmapActionStartwrite); 1891 } 1892 1893 static int llbitmap_reshape_can_start(struct mddev *mddev) 1894 { 1895 struct llbitmap *llbitmap = mddev->bitmap; 1896 unsigned long chunk; 1897 int ret = 0; 1898 1899 if (!llbitmap) 1900 return 0; 1901 1902 mutex_lock(&mddev->bitmap_info.mutex); 1903 for (chunk = 0; chunk < llbitmap->chunks; chunk++) { 1904 enum llbitmap_state state = llbitmap_read(llbitmap, chunk); 1905 1906 if (state == BitNeedSync || state == BitSyncing) { 1907 ret = -EBUSY; 1908 break; 1909 } 1910 } 1911 mutex_unlock(&mddev->bitmap_info.mutex); 1912 1913 return ret; 1914 } 1915 1916 struct llbitmap_reshape_range { 1917 sector_t offset; 1918 unsigned long sectors; 1919 sector_t start; 1920 sector_t end; 1921 }; 1922 1923 static enum llbitmap_state 1924 llbitmap_reshape_init_dst(struct llbitmap *llbitmap, unsigned long dst, 1925 const struct llbitmap_reshape_range *new) 1926 { 1927 u64 bit_start = (u64)dst * llbitmap->reshape_chunksize; 1928 u64 bit_end = bit_start + llbitmap->reshape_chunksize; 1929 1930 if (!llbitmap->mddev->reshape_backwards) 1931 return bit_start < new->offset ? llbitmap_read(llbitmap, dst) : 1932 BitUnwritten; 1933 return bit_end > new->end ? llbitmap_read(llbitmap, dst) : BitUnwritten; 1934 } 1935 1936 static void llbitmap_reshape_dst_range(struct llbitmap *llbitmap, 1937 unsigned long dst, 1938 const struct llbitmap_reshape_range *new, 1939 struct llbitmap_reshape_range *dst_range) 1940 { 1941 sector_t dst_bit_start = (sector_t)dst * llbitmap->reshape_chunksize; 1942 1943 dst_range->start = max(dst_bit_start, new->offset); 1944 dst_range->end = min(dst_bit_start + llbitmap->reshape_chunksize, 1945 new->end); 1946 dst_range->offset = dst_range->start; 1947 dst_range->sectors = dst_range->end - dst_range->start; 1948 } 1949 1950 static void llbitmap_reshape_map_range(struct llbitmap *llbitmap, 1951 sector_t lo, sector_t hi, 1952 bool previous, 1953 struct llbitmap_reshape_range *range) 1954 { 1955 range->offset = lo; 1956 range->sectors = hi - lo; 1957 llbitmap_map_layout(llbitmap, &range->offset, &range->sectors, previous); 1958 range->start = range->offset; 1959 range->end = range->offset + range->sectors; 1960 } 1961 1962 static bool llbitmap_reshape_src_range(const struct llbitmap_reshape_range *old, 1963 const struct llbitmap_reshape_range *new, 1964 const struct llbitmap_reshape_range *dst, 1965 struct llbitmap_reshape_range *src) 1966 { 1967 if (!old->sectors) 1968 return false; 1969 1970 src->start = old->offset + 1971 mul_u64_u64_div_u64(dst->start - new->offset, 1972 old->sectors, new->sectors); 1973 src->end = old->offset + 1974 mul_u64_u64_div_u64_roundup(dst->end - new->offset, 1975 old->sectors, new->sectors); 1976 if (src->end > old->end) 1977 src->end = old->end; 1978 src->offset = src->start; 1979 src->sectors = src->end - src->start; 1980 1981 return src->sectors; 1982 } 1983 1984 static enum llbitmap_state llbitmap_rmerge_src(struct llbitmap *llbitmap, 1985 enum llbitmap_state state, 1986 const struct llbitmap_reshape_range *src) 1987 { 1988 unsigned long bit = div64_u64(src->start, llbitmap->chunksize); 1989 unsigned long end = div64_u64(src->end - 1, llbitmap->chunksize); 1990 1991 while (bit <= end) { 1992 enum llbitmap_state src_state = llbitmap_read(llbitmap, bit); 1993 1994 state = llbitmap_rmerge_state(llbitmap, state, src_state); 1995 bit++; 1996 } 1997 1998 return state; 1999 } 2000 2001 static void llbitmap_reshape_merge(struct llbitmap *llbitmap, 2002 const struct llbitmap_reshape_range *old, 2003 const struct llbitmap_reshape_range *new) 2004 { 2005 unsigned long dst_start; 2006 unsigned long dst_end; 2007 unsigned long dst; 2008 bool backwards = false; 2009 2010 if (!new->sectors) 2011 return; 2012 2013 dst_start = div64_u64(new->offset, llbitmap->reshape_chunksize); 2014 dst_end = div64_u64(new->end - 1, llbitmap->reshape_chunksize); 2015 if (old->sectors) { 2016 unsigned long src_start = div64_u64(old->offset, 2017 llbitmap->chunksize); 2018 unsigned long src_end = div64_u64(old->end - 1, 2019 llbitmap->chunksize); 2020 2021 backwards = src_start < dst_start && src_end >= dst_start; 2022 } 2023 2024 dst = backwards ? dst_end : dst_start; 2025 while (true) { 2026 struct llbitmap_reshape_range dst_range; 2027 struct llbitmap_reshape_range src; 2028 enum llbitmap_state state; 2029 2030 llbitmap_reshape_dst_range(llbitmap, dst, new, &dst_range); 2031 state = llbitmap_reshape_init_dst(llbitmap, dst, new); 2032 if (llbitmap_reshape_src_range(old, new, &dst_range, &src)) 2033 state = llbitmap_rmerge_src(llbitmap, state, &src); 2034 else 2035 state = llbitmap_rmerge_state(llbitmap, state, BitUnwritten); 2036 llbitmap_write(llbitmap, state, dst); 2037 if (dst == (backwards ? dst_start : dst_end)) 2038 break; 2039 if (backwards) 2040 dst--; 2041 else 2042 dst++; 2043 } 2044 } 2045 2046 static void llbitmap_reshape_finish(struct mddev *mddev) 2047 { 2048 struct llbitmap *llbitmap = mddev->bitmap; 2049 2050 if (mddev->pers->quiesce) 2051 mddev->pers->quiesce(mddev, 1); 2052 2053 mutex_lock(&mddev->bitmap_info.mutex); 2054 llbitmap_flush(mddev); 2055 2056 llbitmap->chunksize = llbitmap->reshape_chunksize; 2057 llbitmap->chunkshift = ffz(~llbitmap->chunksize); 2058 llbitmap->chunks = llbitmap->reshape_chunks; 2059 llbitmap->sync_size = llbitmap->reshape_sync_size; 2060 llbitmap_refresh_reshape(llbitmap); 2061 mddev->bitmap_info.chunksize = llbitmap->chunksize; 2062 llbitmap_update_sb(llbitmap); 2063 __llbitmap_flush(mddev); 2064 mutex_unlock(&mddev->bitmap_info.mutex); 2065 2066 if (mddev->pers->quiesce) 2067 mddev->pers->quiesce(mddev, 0); 2068 } 2069 2070 static void llbitmap_reshape_mark(struct mddev *mddev, sector_t old_pos, 2071 sector_t new_pos) 2072 { 2073 struct llbitmap *llbitmap = mddev->bitmap; 2074 sector_t lo; 2075 sector_t hi; 2076 struct llbitmap_reshape_range old; 2077 struct llbitmap_reshape_range new; 2078 2079 if (!llbitmap || old_pos == new_pos) 2080 return; 2081 2082 lo = min(old_pos, new_pos); 2083 hi = max(old_pos, new_pos); 2084 if (!hi) 2085 return; 2086 2087 llbitmap_reshape_map_range(llbitmap, lo, hi, true, &old); 2088 llbitmap_reshape_map_range(llbitmap, lo, hi, false, &new); 2089 if (!new.sectors) 2090 return; 2091 2092 write_lock(&llbitmap->reshape_lock); 2093 llbitmap_reshape_merge(llbitmap, &old, &new); 2094 write_unlock(&llbitmap->reshape_lock); 2095 } 2096 2097 static void llbitmap_write_sb(struct llbitmap *llbitmap) 2098 { 2099 int nr_blocks = DIV_ROUND_UP(BITMAP_DATA_OFFSET, llbitmap->io_size); 2100 2101 bitmap_fill(llbitmap->pctl[0]->dirty, nr_blocks); 2102 llbitmap_write_page(llbitmap, 0); 2103 md_super_wait(llbitmap->mddev); 2104 } 2105 2106 static void llbitmap_update_sb(void *data) 2107 { 2108 struct llbitmap *llbitmap = data; 2109 struct mddev *mddev = llbitmap->mddev; 2110 struct page *sb_page; 2111 bitmap_super_t *sb; 2112 2113 if (test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags)) 2114 return; 2115 2116 sb_page = llbitmap_read_page(llbitmap, 0); 2117 if (IS_ERR(sb_page)) { 2118 pr_err("%s: %s: read super block failed", __func__, 2119 mdname(mddev)); 2120 set_bit(BITMAP_WRITE_ERROR, &llbitmap->flags); 2121 return; 2122 } 2123 2124 if (mddev->events < llbitmap->events_cleared) 2125 llbitmap->events_cleared = mddev->events; 2126 2127 sb = kmap_local_page(sb_page); 2128 sb->events = cpu_to_le64(mddev->events); 2129 sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN)); 2130 sb->chunksize = cpu_to_le32(llbitmap->chunksize); 2131 sb->sync_size = cpu_to_le64(llbitmap->sync_size); 2132 sb->events_cleared = cpu_to_le64(llbitmap->events_cleared); 2133 sb->sectors_reserved = cpu_to_le32(mddev->bitmap_info.space); 2134 sb->daemon_sleep = cpu_to_le32(mddev->bitmap_info.daemon_sleep); 2135 2136 kunmap_local(sb); 2137 llbitmap_write_sb(llbitmap); 2138 } 2139 2140 static int llbitmap_get_stats(void *data, struct md_bitmap_stats *stats) 2141 { 2142 struct llbitmap *llbitmap = data; 2143 2144 memset(stats, 0, sizeof(*stats)); 2145 2146 stats->missing_pages = 0; 2147 stats->pages = llbitmap->nr_pages; 2148 stats->file_pages = llbitmap->nr_pages; 2149 stats->sync_size = llbitmap->sync_size; 2150 2151 stats->behind_writes = atomic_read(&llbitmap->behind_writes); 2152 stats->behind_wait = wq_has_sleeper(&llbitmap->behind_wait); 2153 stats->events_cleared = llbitmap->events_cleared; 2154 2155 return 0; 2156 } 2157 2158 /* just flag all pages as needing to be written */ 2159 static void llbitmap_write_all(struct mddev *mddev) 2160 { 2161 int i; 2162 struct llbitmap *llbitmap = mddev->bitmap; 2163 2164 for (i = 0; i < llbitmap->nr_pages; i++) { 2165 struct llbitmap_page_ctl *pctl = llbitmap->pctl[i]; 2166 2167 set_bit(LLPageDirty, &pctl->flags); 2168 bitmap_fill(pctl->dirty, llbitmap->blocks_per_page); 2169 } 2170 } 2171 2172 static void llbitmap_start_behind_write(struct mddev *mddev) 2173 { 2174 struct llbitmap *llbitmap = mddev->bitmap; 2175 2176 atomic_inc(&llbitmap->behind_writes); 2177 } 2178 2179 static void llbitmap_end_behind_write(struct mddev *mddev) 2180 { 2181 struct llbitmap *llbitmap = mddev->bitmap; 2182 2183 if (atomic_dec_and_test(&llbitmap->behind_writes)) 2184 wake_up(&llbitmap->behind_wait); 2185 } 2186 2187 static void llbitmap_wait_behind_writes(struct mddev *mddev) 2188 { 2189 struct llbitmap *llbitmap = mddev->bitmap; 2190 2191 if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) 2192 wait_event(llbitmap->behind_wait, 2193 atomic_read(&llbitmap->behind_writes) == 0); 2194 } 2195 2196 static ssize_t bits_show(struct mddev *mddev, char *page) 2197 { 2198 struct llbitmap *llbitmap; 2199 int bits[BitStateCount] = {0}; 2200 loff_t start = 0; 2201 2202 mutex_lock(&mddev->bitmap_info.mutex); 2203 llbitmap = mddev->bitmap; 2204 if (!llbitmap || !llbitmap->pctl) { 2205 mutex_unlock(&mddev->bitmap_info.mutex); 2206 return sprintf(page, "no bitmap\n"); 2207 } 2208 2209 if (test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags)) { 2210 mutex_unlock(&mddev->bitmap_info.mutex); 2211 return sprintf(page, "bitmap io error\n"); 2212 } 2213 2214 while (start < llbitmap->chunks) { 2215 enum llbitmap_state c = llbitmap_read(llbitmap, start); 2216 2217 if (c < 0 || c >= BitStateCount) 2218 pr_err("%s: invalid bit %llu state %d\n", 2219 __func__, start, c); 2220 else 2221 bits[c]++; 2222 start++; 2223 } 2224 2225 mutex_unlock(&mddev->bitmap_info.mutex); 2226 return sprintf(page, 2227 "unwritten %d\nclean %d\ndirty %d\n" 2228 "need sync %d\nsyncing %d\n" 2229 "need sync unwritten %d\nsyncing unwritten %d\n" 2230 "clean unwritten %d\n", 2231 bits[BitUnwritten], bits[BitClean], bits[BitDirty], 2232 bits[BitNeedSync], bits[BitSyncing], 2233 bits[BitNeedSyncUnwritten], bits[BitSyncingUnwritten], 2234 bits[BitCleanUnwritten]); 2235 } 2236 2237 static struct md_sysfs_entry llbitmap_bits = __ATTR_RO(bits); 2238 2239 static ssize_t metadata_show(struct mddev *mddev, char *page) 2240 { 2241 struct llbitmap *llbitmap; 2242 ssize_t ret; 2243 2244 mutex_lock(&mddev->bitmap_info.mutex); 2245 llbitmap = mddev->bitmap; 2246 if (!llbitmap) { 2247 mutex_unlock(&mddev->bitmap_info.mutex); 2248 return sprintf(page, "no bitmap\n"); 2249 } 2250 2251 ret = sprintf(page, "chunksize %lu\nchunkshift %lu\nchunks %lu\noffset %llu\ndaemon_sleep %lu\n", 2252 llbitmap->chunksize, llbitmap->chunkshift, 2253 llbitmap->chunks, mddev->bitmap_info.offset, 2254 llbitmap->mddev->bitmap_info.daemon_sleep); 2255 mutex_unlock(&mddev->bitmap_info.mutex); 2256 2257 return ret; 2258 } 2259 2260 static struct md_sysfs_entry llbitmap_metadata = __ATTR_RO(metadata); 2261 2262 static ssize_t 2263 daemon_sleep_show(struct mddev *mddev, char *page) 2264 { 2265 return sprintf(page, "%lu\n", mddev->bitmap_info.daemon_sleep); 2266 } 2267 2268 static ssize_t 2269 daemon_sleep_store(struct mddev *mddev, const char *buf, size_t len) 2270 { 2271 unsigned long timeout; 2272 int rv = kstrtoul(buf, 10, &timeout); 2273 2274 if (rv) 2275 return rv; 2276 2277 mddev->bitmap_info.daemon_sleep = timeout; 2278 return len; 2279 } 2280 2281 static struct md_sysfs_entry llbitmap_daemon_sleep = __ATTR_RW(daemon_sleep); 2282 2283 static ssize_t 2284 barrier_idle_show(struct mddev *mddev, char *page) 2285 { 2286 struct llbitmap *llbitmap = mddev->bitmap; 2287 2288 return sprintf(page, "%lu\n", llbitmap->barrier_idle); 2289 } 2290 2291 static ssize_t 2292 barrier_idle_store(struct mddev *mddev, const char *buf, size_t len) 2293 { 2294 struct llbitmap *llbitmap = mddev->bitmap; 2295 unsigned long timeout; 2296 int rv = kstrtoul(buf, 10, &timeout); 2297 2298 if (rv) 2299 return rv; 2300 2301 llbitmap->barrier_idle = timeout; 2302 return len; 2303 } 2304 2305 static struct md_sysfs_entry llbitmap_barrier_idle = __ATTR_RW(barrier_idle); 2306 2307 static ssize_t 2308 proactive_sync_store(struct mddev *mddev, const char *buf, size_t len) 2309 { 2310 struct llbitmap *llbitmap; 2311 2312 /* Only for RAID-456 */ 2313 if (!raid_is_456(mddev)) 2314 return -EINVAL; 2315 2316 mutex_lock(&mddev->bitmap_info.mutex); 2317 llbitmap = mddev->bitmap; 2318 if (!llbitmap || !llbitmap->pctl) { 2319 mutex_unlock(&mddev->bitmap_info.mutex); 2320 return -ENODEV; 2321 } 2322 2323 /* Trigger proactive sync on all Unwritten regions */ 2324 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, 2325 BitmapActionProactiveSync); 2326 2327 mutex_unlock(&mddev->bitmap_info.mutex); 2328 return len; 2329 } 2330 2331 static struct md_sysfs_entry llbitmap_proactive_sync = 2332 __ATTR(proactive_sync, 0200, NULL, proactive_sync_store); 2333 2334 static struct attribute *md_llbitmap_attrs[] = { 2335 &llbitmap_bits.attr, 2336 &llbitmap_metadata.attr, 2337 &llbitmap_daemon_sleep.attr, 2338 &llbitmap_barrier_idle.attr, 2339 &llbitmap_proactive_sync.attr, 2340 NULL 2341 }; 2342 2343 static struct attribute_group md_llbitmap_group = { 2344 .name = "llbitmap", 2345 .attrs = md_llbitmap_attrs, 2346 }; 2347 2348 static const struct attribute_group *md_llbitmap_groups[] = { 2349 &md_llbitmap_group, 2350 NULL, 2351 }; 2352 2353 static struct bitmap_operations llbitmap_ops = { 2354 .head = { 2355 .type = MD_BITMAP, 2356 .id = ID_LLBITMAP, 2357 .name = "llbitmap", 2358 }, 2359 2360 .enabled = llbitmap_enabled, 2361 .create = llbitmap_create, 2362 .resize = llbitmap_resize, 2363 .load = llbitmap_load, 2364 .destroy = llbitmap_destroy, 2365 2366 .start_write = llbitmap_start_write, 2367 .end_write = llbitmap_end_write, 2368 .start_discard = llbitmap_start_discard, 2369 .end_discard = llbitmap_end_discard, 2370 .unplug = llbitmap_unplug, 2371 .flush = llbitmap_flush, 2372 2373 .start_behind_write = llbitmap_start_behind_write, 2374 .end_behind_write = llbitmap_end_behind_write, 2375 .wait_behind_writes = llbitmap_wait_behind_writes, 2376 2377 .blocks_synced = llbitmap_blocks_synced, 2378 .skip_sync_blocks = llbitmap_skip_sync_blocks, 2379 .start_sync = llbitmap_start_sync, 2380 .end_sync = llbitmap_end_sync, 2381 .close_sync = llbitmap_close_sync, 2382 .cond_end_sync = llbitmap_cond_end_sync, 2383 2384 .update_sb = llbitmap_update_sb, 2385 .get_stats = llbitmap_get_stats, 2386 .dirty_bits = llbitmap_dirty_bits, 2387 .prepare_range = llbitmap_prepare_range, 2388 .reshape_finish = llbitmap_reshape_finish, 2389 .reshape_can_start = llbitmap_reshape_can_start, 2390 .reshape_mark = llbitmap_reshape_mark, 2391 .write_all = llbitmap_write_all, 2392 2393 .groups = md_llbitmap_groups, 2394 }; 2395 2396 int md_llbitmap_init(void) 2397 { 2398 md_llbitmap_io_wq = alloc_workqueue("md_llbitmap_io", 2399 WQ_MEM_RECLAIM | WQ_UNBOUND, 0); 2400 if (!md_llbitmap_io_wq) 2401 return -ENOMEM; 2402 2403 md_llbitmap_unplug_wq = alloc_workqueue("md_llbitmap_unplug", 2404 WQ_MEM_RECLAIM | WQ_UNBOUND, 0); 2405 if (!md_llbitmap_unplug_wq) { 2406 destroy_workqueue(md_llbitmap_io_wq); 2407 md_llbitmap_io_wq = NULL; 2408 return -ENOMEM; 2409 } 2410 2411 return register_md_submodule(&llbitmap_ops.head); 2412 } 2413 2414 void md_llbitmap_exit(void) 2415 { 2416 destroy_workqueue(md_llbitmap_io_wq); 2417 md_llbitmap_io_wq = NULL; 2418 destroy_workqueue(md_llbitmap_unplug_wq); 2419 md_llbitmap_unplug_wq = NULL; 2420 unregister_md_submodule(&llbitmap_ops.head); 2421 } 2422