1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/f2fs/segment.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/sched/mm.h> 13 #include <linux/prefetch.h> 14 #include <linux/kthread.h> 15 #include <linux/swap.h> 16 #include <linux/timer.h> 17 #include <linux/freezer.h> 18 #include <linux/sched/signal.h> 19 #include <linux/random.h> 20 #include <linux/fserror.h> 21 22 #include "f2fs.h" 23 #include "segment.h" 24 #include "node.h" 25 #include "gc.h" 26 #include "iostat.h" 27 #include <trace/events/f2fs.h> 28 29 #define __reverse_ffz(x) __reverse_ffs(~(x)) 30 31 static struct kmem_cache *discard_entry_slab; 32 static struct kmem_cache *discard_cmd_slab; 33 static struct kmem_cache *sit_entry_set_slab; 34 static struct kmem_cache *revoke_entry_slab; 35 36 static unsigned long __reverse_ulong(unsigned char *str) 37 { 38 unsigned long tmp = 0; 39 int shift = 24, idx = 0; 40 41 #if BITS_PER_LONG == 64 42 shift = 56; 43 #endif 44 while (shift >= 0) { 45 tmp |= (unsigned long)str[idx++] << shift; 46 shift -= BITS_PER_BYTE; 47 } 48 return tmp; 49 } 50 51 /* 52 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since 53 * MSB and LSB are reversed in a byte by f2fs_set_bit. 54 */ 55 static inline unsigned long __reverse_ffs(unsigned long word) 56 { 57 int num = 0; 58 59 #if BITS_PER_LONG == 64 60 if ((word & 0xffffffff00000000UL) == 0) 61 num += 32; 62 else 63 word >>= 32; 64 #endif 65 if ((word & 0xffff0000) == 0) 66 num += 16; 67 else 68 word >>= 16; 69 70 if ((word & 0xff00) == 0) 71 num += 8; 72 else 73 word >>= 8; 74 75 if ((word & 0xf0) == 0) 76 num += 4; 77 else 78 word >>= 4; 79 80 if ((word & 0xc) == 0) 81 num += 2; 82 else 83 word >>= 2; 84 85 if ((word & 0x2) == 0) 86 num += 1; 87 return num; 88 } 89 90 /* 91 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because 92 * f2fs_set_bit makes MSB and LSB reversed in a byte. 93 * @size must be integral times of unsigned long. 94 * Example: 95 * MSB <--> LSB 96 * f2fs_set_bit(0, bitmap) => 1000 0000 97 * f2fs_set_bit(7, bitmap) => 0000 0001 98 */ 99 static unsigned long __find_rev_next_bit(const unsigned long *addr, 100 unsigned long size, unsigned long offset) 101 { 102 const unsigned long *p = addr + BIT_WORD(offset); 103 unsigned long result = size; 104 unsigned long tmp; 105 106 if (offset >= size) 107 return size; 108 109 size -= (offset & ~(BITS_PER_LONG - 1)); 110 offset %= BITS_PER_LONG; 111 112 while (1) { 113 if (*p == 0) 114 goto pass; 115 116 tmp = __reverse_ulong((unsigned char *)p); 117 118 tmp &= ~0UL >> offset; 119 if (size < BITS_PER_LONG) 120 tmp &= (~0UL << (BITS_PER_LONG - size)); 121 if (tmp) 122 goto found; 123 pass: 124 if (size <= BITS_PER_LONG) 125 break; 126 size -= BITS_PER_LONG; 127 offset = 0; 128 p++; 129 } 130 return result; 131 found: 132 return result - size + __reverse_ffs(tmp); 133 } 134 135 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, 136 unsigned long size, unsigned long offset) 137 { 138 const unsigned long *p = addr + BIT_WORD(offset); 139 unsigned long result = size; 140 unsigned long tmp; 141 142 if (offset >= size) 143 return size; 144 145 size -= (offset & ~(BITS_PER_LONG - 1)); 146 offset %= BITS_PER_LONG; 147 148 while (1) { 149 if (*p == ~0UL) 150 goto pass; 151 152 tmp = __reverse_ulong((unsigned char *)p); 153 154 if (offset) 155 tmp |= ~0UL << (BITS_PER_LONG - offset); 156 if (size < BITS_PER_LONG) 157 tmp |= ~0UL >> size; 158 if (tmp != ~0UL) 159 goto found; 160 pass: 161 if (size <= BITS_PER_LONG) 162 break; 163 size -= BITS_PER_LONG; 164 offset = 0; 165 p++; 166 } 167 return result; 168 found: 169 return result - size + __reverse_ffz(tmp); 170 } 171 172 bool f2fs_need_SSR(struct f2fs_sb_info *sbi) 173 { 174 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); 175 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); 176 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); 177 178 if (f2fs_lfs_mode(sbi)) 179 return false; 180 if (sbi->gc_mode == GC_URGENT_HIGH) 181 return true; 182 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 183 return true; 184 185 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + 186 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); 187 } 188 189 void f2fs_abort_atomic_write(struct inode *inode, bool clean) 190 { 191 struct f2fs_inode_info *fi = F2FS_I(inode); 192 193 if (!f2fs_is_atomic_file(inode)) 194 return; 195 196 if (clean) 197 truncate_inode_pages_final(inode->i_mapping); 198 199 release_atomic_write_cnt(inode); 200 clear_inode_flag(inode, FI_ATOMIC_COMMITTED); 201 clear_inode_flag(inode, FI_ATOMIC_REPLACE); 202 clear_inode_flag(inode, FI_ATOMIC_FILE); 203 if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) { 204 clear_inode_flag(inode, FI_ATOMIC_DIRTIED); 205 /* 206 * The vfs inode keeps clean during commit, but the f2fs inode 207 * doesn't. So clear the dirty state after commit and let 208 * f2fs_mark_inode_dirty_sync ensure a consistent dirty state. 209 */ 210 f2fs_inode_synced(inode); 211 f2fs_mark_inode_dirty_sync(inode, true); 212 } 213 stat_dec_atomic_inode(inode); 214 215 F2FS_I(inode)->atomic_write_task = NULL; 216 217 if (clean) { 218 f2fs_i_size_write(inode, fi->original_i_size); 219 fi->original_i_size = 0; 220 } 221 /* avoid stale dirty inode during eviction */ 222 sync_inode_metadata(inode, 0); 223 } 224 225 static int __replace_atomic_write_block(struct inode *inode, pgoff_t index, 226 block_t new_addr, block_t *old_addr, bool recover) 227 { 228 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 229 struct dnode_of_data dn; 230 struct node_info ni; 231 int err; 232 233 retry: 234 set_new_dnode(&dn, inode, NULL, NULL, 0); 235 err = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE); 236 if (err) { 237 if (err == -ENOMEM) { 238 memalloc_retry_wait(GFP_NOFS); 239 goto retry; 240 } 241 return err; 242 } 243 244 err = f2fs_get_node_info(sbi, dn.nid, &ni, false); 245 if (err) { 246 f2fs_put_dnode(&dn); 247 return err; 248 } 249 250 if (recover) { 251 /* dn.data_blkaddr is always valid */ 252 if (!__is_valid_data_blkaddr(new_addr)) { 253 if (new_addr == NULL_ADDR) 254 dec_valid_block_count(sbi, inode, 1); 255 f2fs_invalidate_blocks(sbi, dn.data_blkaddr, 1); 256 f2fs_update_data_blkaddr(&dn, new_addr); 257 } else { 258 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, 259 new_addr, ni.version, true, true); 260 } 261 } else { 262 blkcnt_t count = 1; 263 264 err = inc_valid_block_count(sbi, inode, &count, true, false); 265 if (err) { 266 f2fs_put_dnode(&dn); 267 return err; 268 } 269 270 *old_addr = dn.data_blkaddr; 271 f2fs_truncate_data_blocks_range(&dn, 1); 272 dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count); 273 274 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr, 275 ni.version, true, false); 276 } 277 278 f2fs_put_dnode(&dn); 279 280 trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode, 281 index, old_addr ? *old_addr : 0, new_addr, recover); 282 return 0; 283 } 284 285 static void __complete_revoke_list(struct inode *inode, struct list_head *head, 286 bool revoke) 287 { 288 struct revoke_entry *cur, *tmp; 289 pgoff_t start_index = 0; 290 bool truncate = is_inode_flag_set(inode, FI_ATOMIC_REPLACE); 291 292 list_for_each_entry_safe(cur, tmp, head, list) { 293 if (revoke) { 294 __replace_atomic_write_block(inode, cur->index, 295 cur->old_addr, NULL, true); 296 } else if (truncate) { 297 f2fs_truncate_hole(inode, start_index, cur->index); 298 start_index = cur->index + 1; 299 } 300 301 list_del(&cur->list); 302 kmem_cache_free(revoke_entry_slab, cur); 303 } 304 305 if (!revoke && truncate) 306 f2fs_do_truncate_blocks(inode, start_index * PAGE_SIZE, false); 307 } 308 309 static int __f2fs_commit_atomic_write(struct inode *inode) 310 { 311 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 312 struct f2fs_inode_info *fi = F2FS_I(inode); 313 struct inode *cow_inode = fi->cow_inode; 314 struct revoke_entry *new; 315 struct list_head revoke_list; 316 block_t blkaddr; 317 struct dnode_of_data dn; 318 pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 319 pgoff_t off = 0, blen, index; 320 int ret = 0, i; 321 322 INIT_LIST_HEAD(&revoke_list); 323 324 while (len) { 325 blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len); 326 327 set_new_dnode(&dn, cow_inode, NULL, NULL, 0); 328 ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA); 329 if (ret && ret != -ENOENT) { 330 goto out; 331 } else if (ret == -ENOENT) { 332 ret = 0; 333 if (dn.max_level == 0) 334 goto out; 335 goto next; 336 } 337 338 blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_folio, cow_inode), 339 len); 340 index = off; 341 for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) { 342 blkaddr = f2fs_data_blkaddr(&dn); 343 344 if (!__is_valid_data_blkaddr(blkaddr)) { 345 continue; 346 } else if (!f2fs_is_valid_blkaddr(sbi, blkaddr, 347 DATA_GENERIC_ENHANCE)) { 348 f2fs_put_dnode(&dn); 349 ret = -EFSCORRUPTED; 350 goto out; 351 } 352 353 new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS, 354 true, NULL); 355 356 ret = __replace_atomic_write_block(inode, index, blkaddr, 357 &new->old_addr, false); 358 if (ret) { 359 f2fs_put_dnode(&dn); 360 kmem_cache_free(revoke_entry_slab, new); 361 goto out; 362 } 363 364 f2fs_update_data_blkaddr(&dn, NULL_ADDR); 365 new->index = index; 366 list_add_tail(&new->list, &revoke_list); 367 } 368 f2fs_put_dnode(&dn); 369 next: 370 off += blen; 371 len -= blen; 372 } 373 374 out: 375 if (time_to_inject(sbi, FAULT_ATOMIC_TIMEOUT)) 376 f2fs_schedule_timeout_killable(DEFAULT_FAULT_TIMEOUT, true); 377 378 if (ret) { 379 sbi->revoked_atomic_block += fi->atomic_write_cnt; 380 } else { 381 sbi->committed_atomic_block += fi->atomic_write_cnt; 382 set_inode_flag(inode, FI_ATOMIC_COMMITTED); 383 384 /* 385 * inode may has no FI_ATOMIC_DIRTIED flag due to no write 386 * before commit. 387 */ 388 if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) { 389 /* clear atomic dirty status and set vfs dirty status */ 390 clear_inode_flag(inode, FI_ATOMIC_DIRTIED); 391 f2fs_mark_inode_dirty_sync(inode, true); 392 } 393 } 394 395 __complete_revoke_list(inode, &revoke_list, ret ? true : false); 396 397 return ret; 398 } 399 400 int f2fs_commit_atomic_write(struct inode *inode) 401 { 402 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 403 struct f2fs_inode_info *fi = F2FS_I(inode); 404 struct f2fs_lock_context lc; 405 int err; 406 407 err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); 408 if (err) 409 return err; 410 411 f2fs_down_write(&fi->i_gc_rwsem[WRITE]); 412 f2fs_lock_op(sbi, &lc); 413 414 err = __f2fs_commit_atomic_write(inode); 415 416 f2fs_unlock_op(sbi, &lc); 417 f2fs_up_write(&fi->i_gc_rwsem[WRITE]); 418 419 return err; 420 } 421 422 /* 423 * This function balances dirty node and dentry pages. 424 * In addition, it controls garbage collection. 425 */ 426 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) 427 { 428 if (f2fs_cp_error(sbi)) 429 return; 430 431 if (time_to_inject(sbi, FAULT_CHECKPOINT)) 432 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT); 433 434 /* balance_fs_bg is able to be pending */ 435 if (need && excess_cached_nats(sbi)) 436 f2fs_balance_fs_bg(sbi, false); 437 438 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 439 return; 440 441 /* 442 * We should do GC or end up with checkpoint, if there are so many dirty 443 * dir/node pages without enough free segments. 444 */ 445 if (has_enough_free_secs(sbi, 0, 0)) 446 return; 447 448 /* 449 * Submit all cached OPU/IPU DATA bios before triggering 450 * foreground GC to avoid potential deadlocks. 451 */ 452 f2fs_submit_merged_write(sbi, DATA); 453 f2fs_submit_all_merged_ipu_writes(sbi); 454 455 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread.f2fs_gc_task) { 456 DEFINE_WAIT(wait); 457 458 prepare_to_wait(&sbi->gc_thread.fggc_wq, &wait, 459 TASK_UNINTERRUPTIBLE); 460 wake_up(&sbi->gc_thread.gc_wait_queue_head); 461 io_schedule(); 462 finish_wait(&sbi->gc_thread.fggc_wq, &wait); 463 } else { 464 struct f2fs_gc_control gc_control = { 465 .victim_segno = NULL_SEGNO, 466 .init_gc_type = f2fs_sb_has_blkzoned(sbi) ? 467 FG_GC : BG_GC, 468 .no_bg_gc = true, 469 .should_migrate_blocks = false, 470 .err_gc_skipped = false, 471 .nr_free_secs = 1 }; 472 473 f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); 474 stat_inc_gc_call_count(sbi, FOREGROUND); 475 f2fs_gc(sbi, &gc_control); 476 } 477 } 478 479 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi) 480 { 481 int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2; 482 unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS); 483 unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA); 484 unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES); 485 unsigned int meta = get_pages(sbi, F2FS_DIRTY_META); 486 unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA); 487 unsigned int threshold = 488 SEGS_TO_BLKS(sbi, (factor * DEFAULT_DIRTY_THRESHOLD)); 489 unsigned int global_threshold = threshold * 3 / 2; 490 491 if (dents >= threshold || qdata >= threshold || 492 nodes >= threshold || meta >= threshold || 493 imeta >= threshold) 494 return true; 495 return dents + qdata + nodes + meta + imeta > global_threshold; 496 } 497 498 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg) 499 { 500 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 501 return; 502 503 /* try to shrink extent cache when there is no enough memory */ 504 if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE)) 505 f2fs_shrink_read_extent_tree(sbi, 506 READ_EXTENT_CACHE_SHRINK_NUMBER); 507 508 /* try to shrink age extent cache when there is no enough memory */ 509 if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE)) 510 f2fs_shrink_age_extent_tree(sbi, 511 AGE_EXTENT_CACHE_SHRINK_NUMBER); 512 513 /* check the # of cached NAT entries */ 514 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) 515 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); 516 517 if (!f2fs_available_free_memory(sbi, FREE_NIDS)) 518 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS); 519 else 520 f2fs_build_free_nids(sbi, false, false); 521 522 if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) || 523 excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi)) 524 goto do_sync; 525 526 /* there is background inflight IO or foreground operation recently */ 527 if (is_inflight_io(sbi, REQ_TIME) || 528 (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem))) 529 return; 530 531 /* exceed periodical checkpoint timeout threshold */ 532 if (f2fs_time_over(sbi, CP_TIME)) 533 goto do_sync; 534 535 /* checkpoint is the only way to shrink partial cached entries */ 536 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) && 537 f2fs_available_free_memory(sbi, INO_ENTRIES)) 538 return; 539 540 do_sync: 541 if (test_opt(sbi, DATA_FLUSH) && from_bg) { 542 struct blk_plug plug; 543 544 mutex_lock(&sbi->flush_lock); 545 546 blk_start_plug(&plug); 547 f2fs_sync_dirty_inodes(sbi, FILE_INODE, false); 548 blk_finish_plug(&plug); 549 550 mutex_unlock(&sbi->flush_lock); 551 } 552 stat_inc_cp_call_count(sbi, BACKGROUND); 553 f2fs_sync_fs(sbi->sb, 1); 554 } 555 556 static int __submit_flush_wait(struct f2fs_sb_info *sbi, 557 struct block_device *bdev) 558 { 559 int ret = blkdev_issue_flush(bdev); 560 561 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), 562 test_opt(sbi, FLUSH_MERGE), ret); 563 if (!ret) 564 f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0); 565 return ret; 566 } 567 568 static void f2fs_flush_end_io(struct bio *bio) 569 { 570 complete(bio->bi_private); 571 } 572 573 struct f2fs_flush_bio { 574 struct bio bio; 575 struct completion wait; 576 }; 577 578 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) 579 { 580 struct f2fs_flush_bio *flush_bio; 581 unsigned long devices = 0; 582 int ret = 0; 583 int i; 584 585 if (!f2fs_is_multi_device(sbi)) 586 return __submit_flush_wait(sbi, sbi->sb->s_bdev); 587 588 flush_bio = kmalloc(array_size(sbi->s_ndevs, sizeof(*flush_bio)), 589 GFP_NOFS | __GFP_NOFAIL); 590 591 for (i = 0; i < sbi->s_ndevs; i++) { 592 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO)) 593 continue; 594 595 bio_init(&flush_bio[i].bio, FDEV(i).bdev, NULL, 0, 596 REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH); 597 init_completion(&flush_bio[i].wait); 598 flush_bio[i].bio.bi_private = &flush_bio[i].wait; 599 flush_bio[i].bio.bi_end_io = f2fs_flush_end_io; 600 devices |= BIT(i); 601 submit_bio(&flush_bio[i].bio); 602 } 603 604 for (i = 0; i < sbi->s_ndevs; i++) { 605 int err; 606 607 if (!(devices & BIT(i))) 608 continue; 609 610 wait_for_completion(&flush_bio[i].wait); 611 err = blk_status_to_errno(flush_bio[i].bio.bi_status); 612 trace_f2fs_issue_flush(FDEV(i).bdev, test_opt(sbi, NOBARRIER), 613 test_opt(sbi, FLUSH_MERGE), err); 614 if (!err) 615 f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0); 616 else if (!ret) 617 ret = err; 618 bio_uninit(&flush_bio[i].bio); 619 } 620 kfree(flush_bio); 621 return ret; 622 } 623 624 static int issue_flush_thread(void *data) 625 { 626 struct f2fs_sb_info *sbi = data; 627 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 628 wait_queue_head_t *q = &fcc->flush_wait_queue; 629 repeat: 630 if (kthread_should_stop()) 631 return 0; 632 633 if (!llist_empty(&fcc->issue_list)) { 634 struct flush_cmd *cmd, *next; 635 int ret; 636 637 fcc->dispatch_list = llist_del_all(&fcc->issue_list); 638 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list); 639 640 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode); 641 642 ret = submit_flush_wait(sbi, cmd->ino); 643 atomic_inc(&fcc->issued_flush); 644 645 llist_for_each_entry_safe(cmd, next, 646 fcc->dispatch_list, llnode) { 647 cmd->ret = ret; 648 complete(&cmd->wait); 649 } 650 fcc->dispatch_list = NULL; 651 } 652 653 wait_event_interruptible(*q, 654 kthread_should_stop() || !llist_empty(&fcc->issue_list)); 655 goto repeat; 656 } 657 658 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) 659 { 660 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 661 struct flush_cmd cmd; 662 int ret; 663 664 if (test_opt(sbi, NOBARRIER)) 665 return 0; 666 667 if (!test_opt(sbi, FLUSH_MERGE)) { 668 atomic_inc(&fcc->queued_flush); 669 ret = submit_flush_wait(sbi, ino); 670 atomic_dec(&fcc->queued_flush); 671 atomic_inc(&fcc->issued_flush); 672 return ret; 673 } 674 675 if (atomic_inc_return(&fcc->queued_flush) == 1 || 676 f2fs_is_multi_device(sbi)) { 677 ret = submit_flush_wait(sbi, ino); 678 atomic_dec(&fcc->queued_flush); 679 680 atomic_inc(&fcc->issued_flush); 681 return ret; 682 } 683 684 cmd.ino = ino; 685 init_completion(&cmd.wait); 686 687 llist_add(&cmd.llnode, &fcc->issue_list); 688 689 /* 690 * update issue_list before we wake up issue_flush thread, this 691 * smp_mb() pairs with another barrier in ___wait_event(), see 692 * more details in comments of waitqueue_active(). 693 */ 694 smp_mb(); 695 696 if (waitqueue_active(&fcc->flush_wait_queue)) 697 wake_up(&fcc->flush_wait_queue); 698 699 if (fcc->f2fs_issue_flush) { 700 wait_for_completion(&cmd.wait); 701 atomic_dec(&fcc->queued_flush); 702 } else { 703 struct llist_node *list; 704 705 list = llist_del_all(&fcc->issue_list); 706 if (!list) { 707 wait_for_completion(&cmd.wait); 708 atomic_dec(&fcc->queued_flush); 709 } else { 710 struct flush_cmd *tmp, *next; 711 712 ret = submit_flush_wait(sbi, ino); 713 714 llist_for_each_entry_safe(tmp, next, list, llnode) { 715 if (tmp == &cmd) { 716 cmd.ret = ret; 717 atomic_dec(&fcc->queued_flush); 718 continue; 719 } 720 tmp->ret = ret; 721 complete(&tmp->wait); 722 } 723 } 724 } 725 726 return cmd.ret; 727 } 728 729 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi) 730 { 731 dev_t dev = sbi->sb->s_bdev->bd_dev; 732 struct flush_cmd_control *fcc; 733 734 if (SM_I(sbi)->fcc_info) { 735 fcc = SM_I(sbi)->fcc_info; 736 if (fcc->f2fs_issue_flush) 737 return 0; 738 goto init_thread; 739 } 740 741 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); 742 if (!fcc) 743 return -ENOMEM; 744 atomic_set(&fcc->issued_flush, 0); 745 atomic_set(&fcc->queued_flush, 0); 746 init_waitqueue_head(&fcc->flush_wait_queue); 747 init_llist_head(&fcc->issue_list); 748 SM_I(sbi)->fcc_info = fcc; 749 if (!test_opt(sbi, FLUSH_MERGE)) 750 return 0; 751 752 init_thread: 753 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, 754 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); 755 if (IS_ERR(fcc->f2fs_issue_flush)) { 756 int err = PTR_ERR(fcc->f2fs_issue_flush); 757 758 fcc->f2fs_issue_flush = NULL; 759 return err; 760 } 761 762 return 0; 763 } 764 765 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) 766 { 767 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 768 769 if (fcc && fcc->f2fs_issue_flush) { 770 struct task_struct *flush_thread = fcc->f2fs_issue_flush; 771 772 fcc->f2fs_issue_flush = NULL; 773 kthread_stop(flush_thread); 774 } 775 if (free) { 776 kfree(fcc); 777 SM_I(sbi)->fcc_info = NULL; 778 } 779 } 780 781 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) 782 { 783 int ret = 0, i; 784 785 if (!f2fs_is_multi_device(sbi)) 786 return 0; 787 788 if (test_opt(sbi, NOBARRIER)) 789 return 0; 790 791 for (i = 1; i < sbi->s_ndevs; i++) { 792 int count = DEFAULT_RETRY_IO_COUNT; 793 794 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) 795 continue; 796 797 do { 798 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 799 if (ret) 800 f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); 801 } while (ret && --count); 802 803 if (ret) { 804 f2fs_stop_checkpoint(sbi, false, 805 STOP_CP_REASON_FLUSH_FAIL); 806 break; 807 } 808 809 spin_lock(&sbi->dev_lock); 810 f2fs_clear_bit(i, (char *)&sbi->dirty_device); 811 spin_unlock(&sbi->dev_lock); 812 } 813 814 return ret; 815 } 816 817 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 818 enum dirty_type dirty_type) 819 { 820 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 821 822 /* need not be added */ 823 if (is_curseg(sbi, segno)) 824 return; 825 826 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) 827 dirty_i->nr_dirty[dirty_type]++; 828 829 if (dirty_type == DIRTY) { 830 struct seg_entry *sentry = get_seg_entry(sbi, segno); 831 enum dirty_type t = sentry->type; 832 833 if (unlikely(t >= DIRTY)) { 834 f2fs_bug_on(sbi, 1); 835 return; 836 } 837 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) 838 dirty_i->nr_dirty[t]++; 839 840 if (__is_large_section(sbi)) { 841 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); 842 block_t valid_blocks = 843 get_valid_blocks(sbi, segno, true); 844 845 f2fs_bug_on(sbi, 846 (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 847 !valid_blocks) || 848 valid_blocks == CAP_BLKS_PER_SEC(sbi)); 849 850 if (!is_cursec(sbi, secno)) 851 set_bit(secno, dirty_i->dirty_secmap); 852 } 853 } 854 } 855 856 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 857 enum dirty_type dirty_type) 858 { 859 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 860 block_t valid_blocks; 861 862 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) 863 dirty_i->nr_dirty[dirty_type]--; 864 865 if (dirty_type == DIRTY) { 866 struct seg_entry *sentry = get_seg_entry(sbi, segno); 867 enum dirty_type t = sentry->type; 868 869 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) 870 dirty_i->nr_dirty[t]--; 871 872 valid_blocks = get_valid_blocks(sbi, segno, true); 873 if (valid_blocks == 0) { 874 clear_bit(GET_SEC_FROM_SEG(sbi, segno), 875 dirty_i->victim_secmap); 876 #ifdef CONFIG_F2FS_CHECK_FS 877 clear_bit(segno, SIT_I(sbi)->invalid_segmap); 878 #endif 879 } 880 if (__is_large_section(sbi)) { 881 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); 882 883 if (!valid_blocks || 884 valid_blocks == CAP_BLKS_PER_SEC(sbi)) { 885 clear_bit(secno, dirty_i->dirty_secmap); 886 return; 887 } 888 889 if (!is_cursec(sbi, secno)) 890 set_bit(secno, dirty_i->dirty_secmap); 891 } 892 } 893 } 894 895 /* 896 * Should not occur error such as -ENOMEM. 897 * Adding dirty entry into seglist is not critical operation. 898 * If a given segment is one of current working segments, it won't be added. 899 */ 900 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 901 { 902 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 903 unsigned short valid_blocks, ckpt_valid_blocks; 904 unsigned int usable_blocks; 905 906 if (segno == NULL_SEGNO || is_curseg(sbi, segno)) 907 return; 908 909 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno); 910 mutex_lock(&dirty_i->seglist_lock); 911 912 valid_blocks = get_valid_blocks(sbi, segno, false); 913 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false); 914 915 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) || 916 ckpt_valid_blocks == usable_blocks)) { 917 __locate_dirty_segment(sbi, segno, PRE); 918 __remove_dirty_segment(sbi, segno, DIRTY); 919 } else if (valid_blocks < usable_blocks) { 920 __locate_dirty_segment(sbi, segno, DIRTY); 921 } else { 922 /* Recovery routine with SSR needs this */ 923 __remove_dirty_segment(sbi, segno, DIRTY); 924 } 925 926 mutex_unlock(&dirty_i->seglist_lock); 927 } 928 929 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */ 930 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi) 931 { 932 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 933 unsigned int segno; 934 935 mutex_lock(&dirty_i->seglist_lock); 936 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 937 if (get_valid_blocks(sbi, segno, false)) 938 continue; 939 if (is_curseg(sbi, segno)) 940 continue; 941 __locate_dirty_segment(sbi, segno, PRE); 942 __remove_dirty_segment(sbi, segno, DIRTY); 943 } 944 mutex_unlock(&dirty_i->seglist_lock); 945 } 946 947 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi) 948 { 949 int ovp_hole_segs = 950 (overprovision_segments(sbi) - reserved_segments(sbi)); 951 block_t ovp_holes = SEGS_TO_BLKS(sbi, ovp_hole_segs); 952 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 953 block_t holes[2] = {0, 0}; /* DATA and NODE */ 954 block_t unusable; 955 struct seg_entry *se; 956 unsigned int segno; 957 958 mutex_lock(&dirty_i->seglist_lock); 959 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 960 se = get_seg_entry(sbi, segno); 961 if (IS_NODESEG(se->type)) 962 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) - 963 se->valid_blocks; 964 else 965 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) - 966 se->valid_blocks; 967 } 968 mutex_unlock(&dirty_i->seglist_lock); 969 970 unusable = max(holes[DATA], holes[NODE]); 971 if (unusable > ovp_holes) 972 return unusable - ovp_holes; 973 return 0; 974 } 975 976 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable) 977 { 978 int ovp_hole_segs = 979 (overprovision_segments(sbi) - reserved_segments(sbi)); 980 981 if (F2FS_OPTION(sbi).unusable_cap_perc == 100) 982 return 0; 983 if (unusable > F2FS_OPTION(sbi).unusable_cap) 984 return -EAGAIN; 985 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) && 986 dirty_segments(sbi) > ovp_hole_segs) 987 return -EAGAIN; 988 if (has_not_enough_free_secs(sbi, 0, 0)) 989 return -EAGAIN; 990 return 0; 991 } 992 993 /* This is only used by SBI_CP_DISABLED */ 994 static unsigned int get_free_segment(struct f2fs_sb_info *sbi) 995 { 996 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 997 unsigned int segno = 0; 998 999 mutex_lock(&dirty_i->seglist_lock); 1000 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 1001 if (get_valid_blocks(sbi, segno, false)) 1002 continue; 1003 if (get_ckpt_valid_blocks(sbi, segno, false)) 1004 continue; 1005 mutex_unlock(&dirty_i->seglist_lock); 1006 return segno; 1007 } 1008 mutex_unlock(&dirty_i->seglist_lock); 1009 return NULL_SEGNO; 1010 } 1011 1012 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, 1013 struct block_device *bdev, block_t lstart, 1014 block_t start, block_t len) 1015 { 1016 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1017 struct list_head *pend_list; 1018 struct discard_cmd *dc; 1019 1020 f2fs_bug_on(sbi, !len); 1021 1022 pend_list = &dcc->pend_list[plist_idx(len)]; 1023 1024 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL); 1025 INIT_LIST_HEAD(&dc->list); 1026 dc->bdev = bdev; 1027 dc->di.lstart = lstart; 1028 dc->di.start = start; 1029 dc->di.len = len; 1030 dc->ref = 0; 1031 dc->state = D_PREP; 1032 dc->queued = 0; 1033 dc->error = 0; 1034 init_completion(&dc->wait); 1035 list_add_tail(&dc->list, pend_list); 1036 spin_lock_init(&dc->lock); 1037 dc->bio_ref = 0; 1038 atomic_inc(&dcc->discard_cmd_cnt); 1039 dcc->undiscard_blks += len; 1040 1041 return dc; 1042 } 1043 1044 static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi) 1045 { 1046 #ifdef CONFIG_F2FS_CHECK_FS 1047 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1048 struct rb_node *cur = rb_first_cached(&dcc->root), *next; 1049 struct discard_cmd *cur_dc, *next_dc; 1050 1051 while (cur) { 1052 next = rb_next(cur); 1053 if (!next) 1054 return true; 1055 1056 cur_dc = rb_entry(cur, struct discard_cmd, rb_node); 1057 next_dc = rb_entry(next, struct discard_cmd, rb_node); 1058 1059 if (cur_dc->di.lstart + cur_dc->di.len > next_dc->di.lstart) { 1060 f2fs_info(sbi, "broken discard_rbtree, " 1061 "cur(%u, %u) next(%u, %u)", 1062 cur_dc->di.lstart, cur_dc->di.len, 1063 next_dc->di.lstart, next_dc->di.len); 1064 return false; 1065 } 1066 cur = next; 1067 } 1068 #endif 1069 return true; 1070 } 1071 1072 static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi, 1073 block_t blkaddr) 1074 { 1075 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1076 struct rb_node *node = dcc->root.rb_root.rb_node; 1077 struct discard_cmd *dc; 1078 1079 while (node) { 1080 dc = rb_entry(node, struct discard_cmd, rb_node); 1081 1082 if (blkaddr < dc->di.lstart) 1083 node = node->rb_left; 1084 else if (blkaddr >= dc->di.lstart + dc->di.len) 1085 node = node->rb_right; 1086 else 1087 return dc; 1088 } 1089 return NULL; 1090 } 1091 1092 static struct discard_cmd *__lookup_discard_cmd_ret(struct rb_root_cached *root, 1093 block_t blkaddr, 1094 struct discard_cmd **prev_entry, 1095 struct discard_cmd **next_entry, 1096 struct rb_node ***insert_p, 1097 struct rb_node **insert_parent) 1098 { 1099 struct rb_node **pnode = &root->rb_root.rb_node; 1100 struct rb_node *parent = NULL, *tmp_node; 1101 struct discard_cmd *dc; 1102 1103 *insert_p = NULL; 1104 *insert_parent = NULL; 1105 *prev_entry = NULL; 1106 *next_entry = NULL; 1107 1108 if (RB_EMPTY_ROOT(&root->rb_root)) 1109 return NULL; 1110 1111 while (*pnode) { 1112 parent = *pnode; 1113 dc = rb_entry(*pnode, struct discard_cmd, rb_node); 1114 1115 if (blkaddr < dc->di.lstart) 1116 pnode = &(*pnode)->rb_left; 1117 else if (blkaddr >= dc->di.lstart + dc->di.len) 1118 pnode = &(*pnode)->rb_right; 1119 else 1120 goto lookup_neighbors; 1121 } 1122 1123 *insert_p = pnode; 1124 *insert_parent = parent; 1125 1126 dc = rb_entry(parent, struct discard_cmd, rb_node); 1127 tmp_node = parent; 1128 if (parent && blkaddr > dc->di.lstart) 1129 tmp_node = rb_next(parent); 1130 *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); 1131 1132 tmp_node = parent; 1133 if (parent && blkaddr < dc->di.lstart) 1134 tmp_node = rb_prev(parent); 1135 *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); 1136 return NULL; 1137 1138 lookup_neighbors: 1139 /* lookup prev node for merging backward later */ 1140 tmp_node = rb_prev(&dc->rb_node); 1141 *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); 1142 1143 /* lookup next node for merging frontward later */ 1144 tmp_node = rb_next(&dc->rb_node); 1145 *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); 1146 return dc; 1147 } 1148 1149 static void __detach_discard_cmd(struct discard_cmd_control *dcc, 1150 struct discard_cmd *dc) 1151 { 1152 if (dc->state == D_DONE) 1153 atomic_sub(dc->queued, &dcc->queued_discard); 1154 1155 list_del(&dc->list); 1156 rb_erase_cached(&dc->rb_node, &dcc->root); 1157 dcc->undiscard_blks -= dc->di.len; 1158 1159 kmem_cache_free(discard_cmd_slab, dc); 1160 1161 atomic_dec(&dcc->discard_cmd_cnt); 1162 } 1163 1164 static void __remove_discard_cmd(struct f2fs_sb_info *sbi, 1165 struct discard_cmd *dc) 1166 { 1167 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1168 unsigned long flags; 1169 1170 trace_f2fs_remove_discard(dc->bdev, dc->di.start, dc->di.len); 1171 1172 spin_lock_irqsave(&dc->lock, flags); 1173 if (dc->bio_ref) { 1174 spin_unlock_irqrestore(&dc->lock, flags); 1175 return; 1176 } 1177 spin_unlock_irqrestore(&dc->lock, flags); 1178 1179 f2fs_bug_on(sbi, dc->ref); 1180 1181 if (dc->error == -EOPNOTSUPP) 1182 dc->error = 0; 1183 1184 if (dc->error) 1185 f2fs_info_ratelimited(sbi, 1186 "Issue discard(%u, %u, %u) failed, ret: %d", 1187 dc->di.lstart, dc->di.start, dc->di.len, dc->error); 1188 __detach_discard_cmd(dcc, dc); 1189 } 1190 1191 static void f2fs_submit_discard_endio(struct bio *bio) 1192 { 1193 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private; 1194 unsigned long flags; 1195 1196 spin_lock_irqsave(&dc->lock, flags); 1197 if (!dc->error) 1198 dc->error = blk_status_to_errno(bio->bi_status); 1199 dc->bio_ref--; 1200 if (!dc->bio_ref && dc->state == D_SUBMIT) { 1201 dc->state = D_DONE; 1202 complete_all(&dc->wait); 1203 } 1204 spin_unlock_irqrestore(&dc->lock, flags); 1205 bio_put(bio); 1206 } 1207 1208 static void __check_sit_bitmap(struct f2fs_sb_info *sbi, 1209 block_t start, block_t end) 1210 { 1211 #ifdef CONFIG_F2FS_CHECK_FS 1212 struct seg_entry *sentry; 1213 unsigned int segno; 1214 block_t blk = start; 1215 unsigned long offset, size, *map; 1216 1217 while (blk < end) { 1218 segno = GET_SEGNO(sbi, blk); 1219 sentry = get_seg_entry(sbi, segno); 1220 offset = GET_BLKOFF_FROM_SEG0(sbi, blk); 1221 1222 if (end < START_BLOCK(sbi, segno + 1)) 1223 size = GET_BLKOFF_FROM_SEG0(sbi, end); 1224 else 1225 size = BLKS_PER_SEG(sbi); 1226 map = (unsigned long *)(sentry->cur_valid_map); 1227 offset = __find_rev_next_bit(map, size, offset); 1228 f2fs_bug_on(sbi, offset != size); 1229 blk = START_BLOCK(sbi, segno + 1); 1230 } 1231 #endif 1232 } 1233 1234 static void __init_discard_policy(struct f2fs_sb_info *sbi, 1235 struct discard_policy *dpolicy, 1236 int discard_type, unsigned int granularity) 1237 { 1238 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1239 1240 /* common policy */ 1241 dpolicy->type = discard_type; 1242 dpolicy->sync = true; 1243 dpolicy->ordered = false; 1244 dpolicy->granularity = granularity; 1245 1246 dpolicy->max_requests = dcc->max_discard_request; 1247 dpolicy->io_aware_gran = dcc->discard_io_aware_gran; 1248 dpolicy->timeout = false; 1249 1250 if (discard_type == DPOLICY_BG) { 1251 dpolicy->min_interval = dcc->min_discard_issue_time; 1252 dpolicy->mid_interval = dcc->mid_discard_issue_time; 1253 dpolicy->max_interval = dcc->max_discard_issue_time; 1254 if (dcc->discard_io_aware == DPOLICY_IO_AWARE_ENABLE) 1255 dpolicy->io_aware = true; 1256 else if (dcc->discard_io_aware == DPOLICY_IO_AWARE_DISABLE) 1257 dpolicy->io_aware = false; 1258 dpolicy->sync = false; 1259 dpolicy->ordered = true; 1260 if (utilization(sbi) > dcc->discard_urgent_util) { 1261 dpolicy->granularity = MIN_DISCARD_GRANULARITY; 1262 if (atomic_read(&dcc->discard_cmd_cnt)) 1263 dpolicy->max_interval = 1264 dcc->min_discard_issue_time; 1265 } 1266 } else if (discard_type == DPOLICY_FORCE) { 1267 dpolicy->min_interval = dcc->min_discard_issue_time; 1268 dpolicy->mid_interval = dcc->mid_discard_issue_time; 1269 dpolicy->max_interval = dcc->max_discard_issue_time; 1270 dpolicy->io_aware = false; 1271 } else if (discard_type == DPOLICY_FSTRIM) { 1272 dpolicy->io_aware = false; 1273 } else if (discard_type == DPOLICY_UMOUNT) { 1274 dpolicy->io_aware = false; 1275 /* we need to issue all to keep CP_TRIMMED_FLAG */ 1276 dpolicy->granularity = MIN_DISCARD_GRANULARITY; 1277 dpolicy->timeout = true; 1278 } 1279 } 1280 1281 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1282 struct block_device *bdev, block_t lstart, 1283 block_t start, block_t len); 1284 1285 #ifdef CONFIG_BLK_DEV_ZONED 1286 static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi, 1287 struct discard_cmd *dc, blk_opf_t flag, 1288 struct list_head *wait_list, 1289 unsigned int *issued) 1290 { 1291 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1292 struct block_device *bdev = dc->bdev; 1293 struct bio *bio = bio_alloc(bdev, 0, REQ_OP_ZONE_RESET | flag, GFP_NOFS); 1294 unsigned long flags; 1295 1296 trace_f2fs_issue_reset_zone(bdev, dc->di.start); 1297 1298 spin_lock_irqsave(&dc->lock, flags); 1299 dc->state = D_SUBMIT; 1300 dc->bio_ref++; 1301 spin_unlock_irqrestore(&dc->lock, flags); 1302 1303 if (issued) 1304 (*issued)++; 1305 1306 atomic_inc(&dcc->queued_discard); 1307 dc->queued++; 1308 list_move_tail(&dc->list, wait_list); 1309 1310 /* sanity check on discard range */ 1311 __check_sit_bitmap(sbi, dc->di.lstart, dc->di.lstart + dc->di.len); 1312 1313 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(dc->di.start); 1314 bio->bi_private = dc; 1315 bio->bi_end_io = f2fs_submit_discard_endio; 1316 submit_bio(bio); 1317 1318 atomic_inc(&dcc->issued_discard); 1319 f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, dc->di.len * F2FS_BLKSIZE); 1320 } 1321 #endif 1322 1323 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */ 1324 static int __submit_discard_cmd(struct f2fs_sb_info *sbi, 1325 struct discard_policy *dpolicy, 1326 struct discard_cmd *dc, int *issued) 1327 { 1328 struct block_device *bdev = dc->bdev; 1329 unsigned int max_discard_blocks = 1330 SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); 1331 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1332 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1333 &(dcc->fstrim_list) : &(dcc->wait_list); 1334 blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0; 1335 block_t lstart, start, len, total_len; 1336 1337 if (dc->state != D_PREP) 1338 return 0; 1339 1340 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) 1341 return 0; 1342 1343 #ifdef CONFIG_BLK_DEV_ZONED 1344 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) { 1345 int devi = f2fs_bdev_index(sbi, bdev); 1346 1347 if (devi < 0) 1348 return -EINVAL; 1349 1350 if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) { 1351 __submit_zone_reset_cmd(sbi, dc, flag, 1352 wait_list, issued); 1353 return 0; 1354 } 1355 } 1356 #endif 1357 1358 /* 1359 * stop issuing discard for any of below cases: 1360 * 1. device is conventional zone, but it doesn't support discard. 1361 * 2. device is regulare device, after snapshot it doesn't support 1362 * discard. 1363 */ 1364 if (!bdev_max_discard_sectors(bdev)) 1365 return -EOPNOTSUPP; 1366 1367 trace_f2fs_issue_discard(bdev, dc->di.start, dc->di.len); 1368 1369 lstart = dc->di.lstart; 1370 start = dc->di.start; 1371 len = dc->di.len; 1372 total_len = len; 1373 1374 dc->di.len = 0; 1375 1376 while (total_len && *issued < dpolicy->max_requests) { 1377 struct bio *bio = NULL; 1378 unsigned long flags; 1379 bool last = true; 1380 1381 if (len > max_discard_blocks) { 1382 len = max_discard_blocks; 1383 last = false; 1384 } 1385 1386 (*issued)++; 1387 if (*issued == dpolicy->max_requests) 1388 last = true; 1389 1390 dc->di.len += len; 1391 1392 __blkdev_issue_discard(bdev, SECTOR_FROM_BLOCK(start), 1393 SECTOR_FROM_BLOCK(len), GFP_NOFS, &bio); 1394 f2fs_bug_on(sbi, !bio); 1395 1396 /* 1397 * should keep before submission to avoid D_DONE 1398 * right away 1399 */ 1400 spin_lock_irqsave(&dc->lock, flags); 1401 if (last) 1402 dc->state = D_SUBMIT; 1403 else 1404 dc->state = D_PARTIAL; 1405 dc->bio_ref++; 1406 spin_unlock_irqrestore(&dc->lock, flags); 1407 1408 atomic_inc(&dcc->queued_discard); 1409 dc->queued++; 1410 list_move_tail(&dc->list, wait_list); 1411 1412 /* sanity check on discard range */ 1413 __check_sit_bitmap(sbi, lstart, lstart + len); 1414 1415 bio->bi_private = dc; 1416 bio->bi_end_io = f2fs_submit_discard_endio; 1417 bio->bi_opf |= flag; 1418 submit_bio(bio); 1419 1420 atomic_inc(&dcc->issued_discard); 1421 1422 f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE); 1423 1424 lstart += len; 1425 start += len; 1426 total_len -= len; 1427 len = total_len; 1428 } 1429 1430 if (len) { 1431 dcc->undiscard_blks -= len; 1432 __update_discard_tree_range(sbi, bdev, lstart, start, len); 1433 } 1434 return 0; 1435 } 1436 1437 static void __insert_discard_cmd(struct f2fs_sb_info *sbi, 1438 struct block_device *bdev, block_t lstart, 1439 block_t start, block_t len) 1440 { 1441 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1442 struct rb_node **p = &dcc->root.rb_root.rb_node; 1443 struct rb_node *parent = NULL; 1444 struct discard_cmd *dc; 1445 bool leftmost = true; 1446 1447 /* look up rb tree to find parent node */ 1448 while (*p) { 1449 parent = *p; 1450 dc = rb_entry(parent, struct discard_cmd, rb_node); 1451 1452 if (lstart < dc->di.lstart) { 1453 p = &(*p)->rb_left; 1454 } else if (lstart >= dc->di.lstart + dc->di.len) { 1455 p = &(*p)->rb_right; 1456 leftmost = false; 1457 } else { 1458 /* Let's skip to add, if exists */ 1459 return; 1460 } 1461 } 1462 1463 dc = __create_discard_cmd(sbi, bdev, lstart, start, len); 1464 1465 rb_link_node(&dc->rb_node, parent, p); 1466 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost); 1467 } 1468 1469 static void __relocate_discard_cmd(struct discard_cmd_control *dcc, 1470 struct discard_cmd *dc) 1471 { 1472 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->di.len)]); 1473 } 1474 1475 static void __punch_discard_cmd(struct f2fs_sb_info *sbi, 1476 struct discard_cmd *dc, block_t blkaddr) 1477 { 1478 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1479 struct discard_info di = dc->di; 1480 bool modified = false; 1481 1482 if (dc->state == D_DONE || dc->di.len == 1) { 1483 __remove_discard_cmd(sbi, dc); 1484 return; 1485 } 1486 1487 dcc->undiscard_blks -= di.len; 1488 1489 if (blkaddr > di.lstart) { 1490 dc->di.len = blkaddr - dc->di.lstart; 1491 dcc->undiscard_blks += dc->di.len; 1492 __relocate_discard_cmd(dcc, dc); 1493 modified = true; 1494 } 1495 1496 if (blkaddr < di.lstart + di.len - 1) { 1497 if (modified) { 1498 __insert_discard_cmd(sbi, dc->bdev, blkaddr + 1, 1499 di.start + blkaddr + 1 - di.lstart, 1500 di.lstart + di.len - 1 - blkaddr); 1501 } else { 1502 dc->di.lstart++; 1503 dc->di.len--; 1504 dc->di.start++; 1505 dcc->undiscard_blks += dc->di.len; 1506 __relocate_discard_cmd(dcc, dc); 1507 } 1508 } 1509 } 1510 1511 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1512 struct block_device *bdev, block_t lstart, 1513 block_t start, block_t len) 1514 { 1515 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1516 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1517 struct discard_cmd *dc; 1518 struct discard_info di = {0}; 1519 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1520 unsigned int max_discard_blocks = 1521 SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); 1522 block_t end = lstart + len; 1523 1524 dc = __lookup_discard_cmd_ret(&dcc->root, lstart, 1525 &prev_dc, &next_dc, &insert_p, &insert_parent); 1526 if (dc) 1527 prev_dc = dc; 1528 1529 if (!prev_dc) { 1530 di.lstart = lstart; 1531 di.len = next_dc ? next_dc->di.lstart - lstart : len; 1532 di.len = min(di.len, len); 1533 di.start = start; 1534 } 1535 1536 while (1) { 1537 struct rb_node *node; 1538 bool merged = false; 1539 struct discard_cmd *tdc = NULL; 1540 1541 if (prev_dc) { 1542 di.lstart = prev_dc->di.lstart + prev_dc->di.len; 1543 if (di.lstart < lstart) 1544 di.lstart = lstart; 1545 if (di.lstart >= end) 1546 break; 1547 1548 if (!next_dc || next_dc->di.lstart > end) 1549 di.len = end - di.lstart; 1550 else 1551 di.len = next_dc->di.lstart - di.lstart; 1552 di.start = start + di.lstart - lstart; 1553 } 1554 1555 if (!di.len) 1556 goto next; 1557 1558 if (prev_dc && prev_dc->state == D_PREP && 1559 prev_dc->bdev == bdev && 1560 __is_discard_back_mergeable(&di, &prev_dc->di, 1561 max_discard_blocks)) { 1562 prev_dc->di.len += di.len; 1563 dcc->undiscard_blks += di.len; 1564 __relocate_discard_cmd(dcc, prev_dc); 1565 di = prev_dc->di; 1566 tdc = prev_dc; 1567 merged = true; 1568 } 1569 1570 if (next_dc && next_dc->state == D_PREP && 1571 next_dc->bdev == bdev && 1572 __is_discard_front_mergeable(&di, &next_dc->di, 1573 max_discard_blocks)) { 1574 next_dc->di.lstart = di.lstart; 1575 next_dc->di.len += di.len; 1576 next_dc->di.start = di.start; 1577 dcc->undiscard_blks += di.len; 1578 __relocate_discard_cmd(dcc, next_dc); 1579 if (tdc) 1580 __remove_discard_cmd(sbi, tdc); 1581 merged = true; 1582 } 1583 1584 if (!merged) 1585 __insert_discard_cmd(sbi, bdev, 1586 di.lstart, di.start, di.len); 1587 next: 1588 prev_dc = next_dc; 1589 if (!prev_dc) 1590 break; 1591 1592 node = rb_next(&prev_dc->rb_node); 1593 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1594 } 1595 } 1596 1597 #ifdef CONFIG_BLK_DEV_ZONED 1598 static void __queue_zone_reset_cmd(struct f2fs_sb_info *sbi, 1599 struct block_device *bdev, block_t blkstart, block_t lblkstart, 1600 block_t blklen) 1601 { 1602 trace_f2fs_queue_reset_zone(bdev, blkstart); 1603 1604 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); 1605 __insert_discard_cmd(sbi, bdev, lblkstart, blkstart, blklen); 1606 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); 1607 } 1608 #endif 1609 1610 static void __queue_discard_cmd(struct f2fs_sb_info *sbi, 1611 struct block_device *bdev, block_t blkstart, block_t blklen) 1612 { 1613 block_t lblkstart = blkstart; 1614 1615 if (!f2fs_bdev_support_discard(bdev)) 1616 return; 1617 1618 trace_f2fs_queue_discard(bdev, blkstart, blklen); 1619 1620 if (f2fs_is_multi_device(sbi)) { 1621 int devi = f2fs_target_device_index(sbi, blkstart); 1622 1623 blkstart -= FDEV(devi).start_blk; 1624 } 1625 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); 1626 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); 1627 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); 1628 } 1629 1630 static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, 1631 struct discard_policy *dpolicy, int *issued) 1632 { 1633 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1634 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1635 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1636 struct discard_cmd *dc; 1637 struct blk_plug plug; 1638 bool io_interrupted = false; 1639 1640 mutex_lock(&dcc->cmd_lock); 1641 dc = __lookup_discard_cmd_ret(&dcc->root, dcc->next_pos, 1642 &prev_dc, &next_dc, &insert_p, &insert_parent); 1643 if (!dc) 1644 dc = next_dc; 1645 1646 blk_start_plug(&plug); 1647 1648 while (dc) { 1649 struct rb_node *node; 1650 int err = 0; 1651 1652 if (dc->state != D_PREP) 1653 goto next; 1654 1655 if (*issued > 0 && unlikely(freezing(current))) 1656 break; 1657 1658 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { 1659 io_interrupted = true; 1660 break; 1661 } 1662 1663 dcc->next_pos = dc->di.lstart + dc->di.len; 1664 err = __submit_discard_cmd(sbi, dpolicy, dc, issued); 1665 1666 if (*issued >= dpolicy->max_requests) 1667 break; 1668 next: 1669 node = rb_next(&dc->rb_node); 1670 if (err) 1671 __remove_discard_cmd(sbi, dc); 1672 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1673 } 1674 1675 blk_finish_plug(&plug); 1676 1677 if (!dc) 1678 dcc->next_pos = 0; 1679 1680 mutex_unlock(&dcc->cmd_lock); 1681 1682 if (!(*issued) && io_interrupted) 1683 *issued = -1; 1684 } 1685 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1686 struct discard_policy *dpolicy); 1687 1688 static int __issue_discard_cmd(struct f2fs_sb_info *sbi, 1689 struct discard_policy *dpolicy) 1690 { 1691 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1692 struct list_head *pend_list; 1693 struct discard_cmd *dc, *tmp; 1694 struct blk_plug plug; 1695 int i, issued; 1696 bool io_interrupted = false; 1697 bool suspended = false; 1698 1699 if (dpolicy->timeout) 1700 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT); 1701 1702 retry: 1703 issued = 0; 1704 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1705 if (dpolicy->timeout && 1706 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) 1707 break; 1708 1709 if (i + 1 < dpolicy->granularity) 1710 break; 1711 1712 if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) { 1713 __issue_discard_cmd_orderly(sbi, dpolicy, &issued); 1714 return issued; 1715 } 1716 1717 pend_list = &dcc->pend_list[i]; 1718 1719 mutex_lock(&dcc->cmd_lock); 1720 if (list_empty(pend_list)) 1721 goto next; 1722 if (unlikely(dcc->rbtree_check)) 1723 f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi)); 1724 blk_start_plug(&plug); 1725 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1726 f2fs_bug_on(sbi, dc->state != D_PREP); 1727 1728 if (issued > 0 && unlikely(freezing(current))) { 1729 suspended = true; 1730 break; 1731 } 1732 1733 if (dpolicy->timeout && 1734 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) 1735 break; 1736 1737 if (dpolicy->io_aware && i < dpolicy->io_aware_gran && 1738 !is_idle(sbi, DISCARD_TIME)) { 1739 io_interrupted = true; 1740 break; 1741 } 1742 1743 __submit_discard_cmd(sbi, dpolicy, dc, &issued); 1744 1745 if (issued >= dpolicy->max_requests) 1746 break; 1747 } 1748 blk_finish_plug(&plug); 1749 next: 1750 mutex_unlock(&dcc->cmd_lock); 1751 1752 if (issued >= dpolicy->max_requests || io_interrupted || 1753 suspended) 1754 break; 1755 } 1756 1757 if (dpolicy->type == DPOLICY_UMOUNT && issued) { 1758 __wait_all_discard_cmd(sbi, dpolicy); 1759 goto retry; 1760 } 1761 1762 if (!issued && io_interrupted) 1763 issued = -1; 1764 1765 return issued; 1766 } 1767 1768 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) 1769 { 1770 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1771 struct list_head *pend_list; 1772 struct discard_cmd *dc, *tmp; 1773 int i; 1774 bool dropped = false; 1775 1776 mutex_lock(&dcc->cmd_lock); 1777 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1778 pend_list = &dcc->pend_list[i]; 1779 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1780 f2fs_bug_on(sbi, dc->state != D_PREP); 1781 __remove_discard_cmd(sbi, dc); 1782 dropped = true; 1783 } 1784 } 1785 mutex_unlock(&dcc->cmd_lock); 1786 1787 return dropped; 1788 } 1789 1790 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) 1791 { 1792 __drop_discard_cmd(sbi); 1793 } 1794 1795 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, 1796 struct discard_cmd *dc) 1797 { 1798 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1799 unsigned int len = 0; 1800 1801 wait_for_completion_io(&dc->wait); 1802 mutex_lock(&dcc->cmd_lock); 1803 f2fs_bug_on(sbi, dc->state != D_DONE); 1804 dc->ref--; 1805 if (!dc->ref) { 1806 if (!dc->error) 1807 len = dc->di.len; 1808 __remove_discard_cmd(sbi, dc); 1809 } 1810 mutex_unlock(&dcc->cmd_lock); 1811 1812 return len; 1813 } 1814 1815 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, 1816 struct discard_policy *dpolicy, 1817 block_t start, block_t end) 1818 { 1819 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1820 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1821 &(dcc->fstrim_list) : &(dcc->wait_list); 1822 struct discard_cmd *dc = NULL, *iter, *tmp; 1823 unsigned int trimmed = 0; 1824 1825 next: 1826 dc = NULL; 1827 1828 mutex_lock(&dcc->cmd_lock); 1829 list_for_each_entry_safe(iter, tmp, wait_list, list) { 1830 if (iter->di.lstart + iter->di.len <= start || 1831 end <= iter->di.lstart) 1832 continue; 1833 if (iter->di.len < dpolicy->granularity) 1834 continue; 1835 if (iter->state == D_DONE && !iter->ref) { 1836 wait_for_completion_io(&iter->wait); 1837 if (!iter->error) 1838 trimmed += iter->di.len; 1839 __remove_discard_cmd(sbi, iter); 1840 } else { 1841 iter->ref++; 1842 dc = iter; 1843 break; 1844 } 1845 } 1846 mutex_unlock(&dcc->cmd_lock); 1847 1848 if (dc) { 1849 trimmed += __wait_one_discard_bio(sbi, dc); 1850 goto next; 1851 } 1852 1853 return trimmed; 1854 } 1855 1856 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1857 struct discard_policy *dpolicy) 1858 { 1859 struct discard_policy dp; 1860 unsigned int discard_blks; 1861 1862 if (dpolicy) 1863 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); 1864 1865 /* wait all */ 1866 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY); 1867 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1868 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY); 1869 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1870 1871 return discard_blks; 1872 } 1873 1874 /* This should be covered by global mutex, &sit_i->sentry_lock */ 1875 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) 1876 { 1877 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1878 struct discard_cmd *dc; 1879 bool need_wait = false; 1880 1881 mutex_lock(&dcc->cmd_lock); 1882 dc = __lookup_discard_cmd(sbi, blkaddr); 1883 #ifdef CONFIG_BLK_DEV_ZONED 1884 if (dc && f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(dc->bdev)) { 1885 int devi = f2fs_bdev_index(sbi, dc->bdev); 1886 1887 if (devi < 0) { 1888 mutex_unlock(&dcc->cmd_lock); 1889 return; 1890 } 1891 1892 if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) { 1893 /* force submit zone reset */ 1894 if (dc->state == D_PREP) 1895 __submit_zone_reset_cmd(sbi, dc, REQ_SYNC, 1896 &dcc->wait_list, NULL); 1897 dc->ref++; 1898 mutex_unlock(&dcc->cmd_lock); 1899 /* wait zone reset */ 1900 __wait_one_discard_bio(sbi, dc); 1901 return; 1902 } 1903 } 1904 #endif 1905 if (dc) { 1906 if (dc->state == D_PREP) { 1907 __punch_discard_cmd(sbi, dc, blkaddr); 1908 } else { 1909 dc->ref++; 1910 need_wait = true; 1911 } 1912 } 1913 mutex_unlock(&dcc->cmd_lock); 1914 1915 if (need_wait) 1916 __wait_one_discard_bio(sbi, dc); 1917 } 1918 1919 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) 1920 { 1921 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1922 1923 if (dcc && dcc->f2fs_issue_discard) { 1924 struct task_struct *discard_thread = dcc->f2fs_issue_discard; 1925 1926 dcc->f2fs_issue_discard = NULL; 1927 kthread_stop(discard_thread); 1928 } 1929 } 1930 1931 /** 1932 * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT 1933 * @sbi: the f2fs_sb_info data for discard cmd to issue 1934 * 1935 * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped 1936 * 1937 * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false. 1938 */ 1939 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check) 1940 { 1941 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1942 struct discard_policy dpolicy; 1943 bool dropped; 1944 1945 if (!atomic_read(&dcc->discard_cmd_cnt)) 1946 return true; 1947 1948 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, 1949 dcc->discard_granularity); 1950 __issue_discard_cmd(sbi, &dpolicy); 1951 dropped = __drop_discard_cmd(sbi); 1952 1953 /* just to make sure there is no pending discard commands */ 1954 __wait_all_discard_cmd(sbi, NULL); 1955 1956 f2fs_bug_on(sbi, need_check && atomic_read(&dcc->discard_cmd_cnt)); 1957 return !dropped; 1958 } 1959 1960 static int issue_discard_thread(void *data) 1961 { 1962 struct f2fs_sb_info *sbi = data; 1963 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1964 wait_queue_head_t *q = &dcc->discard_wait_queue; 1965 struct discard_policy dpolicy; 1966 unsigned int wait_ms = dcc->min_discard_issue_time; 1967 int issued; 1968 1969 set_freezable(); 1970 1971 do { 1972 wait_event_freezable_timeout(*q, 1973 kthread_should_stop() || dcc->discard_wake, 1974 msecs_to_jiffies(wait_ms)); 1975 1976 if (sbi->gc_mode == GC_URGENT_HIGH || 1977 !f2fs_available_free_memory(sbi, DISCARD_CACHE)) 1978 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1979 MIN_DISCARD_GRANULARITY); 1980 else 1981 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, 1982 dcc->discard_granularity); 1983 1984 if (dcc->discard_wake) 1985 dcc->discard_wake = false; 1986 1987 /* clean up pending candidates before going to sleep */ 1988 if (atomic_read(&dcc->queued_discard)) 1989 __wait_all_discard_cmd(sbi, NULL); 1990 1991 if (f2fs_readonly(sbi->sb)) 1992 continue; 1993 if (kthread_should_stop()) 1994 return 0; 1995 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) || 1996 !atomic_read(&dcc->discard_cmd_cnt)) { 1997 wait_ms = dpolicy.max_interval; 1998 continue; 1999 } 2000 2001 sb_start_intwrite(sbi->sb); 2002 2003 issued = __issue_discard_cmd(sbi, &dpolicy); 2004 if (issued > 0) { 2005 __wait_all_discard_cmd(sbi, &dpolicy); 2006 wait_ms = dpolicy.min_interval; 2007 } else if (issued == -1) { 2008 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); 2009 if (!wait_ms) 2010 wait_ms = dpolicy.mid_interval; 2011 } else { 2012 wait_ms = dpolicy.max_interval; 2013 } 2014 if (!atomic_read(&dcc->discard_cmd_cnt)) 2015 wait_ms = dpolicy.max_interval; 2016 2017 sb_end_intwrite(sbi->sb); 2018 2019 } while (!kthread_should_stop()); 2020 return 0; 2021 } 2022 2023 #ifdef CONFIG_BLK_DEV_ZONED 2024 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, 2025 struct block_device *bdev, block_t blkstart, block_t blklen) 2026 { 2027 sector_t sector, nr_sects; 2028 block_t lblkstart = blkstart; 2029 int devi = 0; 2030 u64 remainder = 0; 2031 2032 if (f2fs_is_multi_device(sbi)) { 2033 devi = f2fs_target_device_index(sbi, blkstart); 2034 if (blkstart < FDEV(devi).start_blk || 2035 blkstart > FDEV(devi).end_blk) { 2036 f2fs_err(sbi, "Invalid block %x", blkstart); 2037 return -EIO; 2038 } 2039 blkstart -= FDEV(devi).start_blk; 2040 } 2041 2042 /* For sequential zones, reset the zone write pointer */ 2043 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { 2044 sector = SECTOR_FROM_BLOCK(blkstart); 2045 nr_sects = SECTOR_FROM_BLOCK(blklen); 2046 div64_u64_rem(sector, bdev_zone_sectors(bdev), &remainder); 2047 2048 if (remainder || nr_sects != bdev_zone_sectors(bdev)) { 2049 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)", 2050 devi, sbi->s_ndevs ? FDEV(devi).path : "", 2051 blkstart, blklen); 2052 return -EIO; 2053 } 2054 2055 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) { 2056 unsigned int nofs_flags; 2057 int ret; 2058 2059 trace_f2fs_issue_reset_zone(bdev, blkstart); 2060 nofs_flags = memalloc_nofs_save(); 2061 ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, 2062 sector, nr_sects); 2063 memalloc_nofs_restore(nofs_flags); 2064 return ret; 2065 } 2066 2067 __queue_zone_reset_cmd(sbi, bdev, blkstart, lblkstart, blklen); 2068 return 0; 2069 } 2070 2071 /* For conventional zones, use regular discard if supported */ 2072 __queue_discard_cmd(sbi, bdev, lblkstart, blklen); 2073 return 0; 2074 } 2075 #endif 2076 2077 static int __issue_discard_async(struct f2fs_sb_info *sbi, 2078 struct block_device *bdev, block_t blkstart, block_t blklen) 2079 { 2080 #ifdef CONFIG_BLK_DEV_ZONED 2081 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) 2082 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); 2083 #endif 2084 __queue_discard_cmd(sbi, bdev, blkstart, blklen); 2085 return 0; 2086 } 2087 2088 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, 2089 block_t blkstart, block_t blklen) 2090 { 2091 sector_t start = blkstart, len = 0; 2092 struct block_device *bdev; 2093 struct seg_entry *se; 2094 unsigned int offset; 2095 block_t i; 2096 int err = 0; 2097 2098 bdev = f2fs_target_device(sbi, blkstart, NULL); 2099 2100 for (i = blkstart; i < blkstart + blklen; i++, len++) { 2101 if (i != start) { 2102 struct block_device *bdev2 = 2103 f2fs_target_device(sbi, i, NULL); 2104 2105 if (bdev2 != bdev) { 2106 err = __issue_discard_async(sbi, bdev, 2107 start, len); 2108 if (err) 2109 return err; 2110 bdev = bdev2; 2111 start = i; 2112 len = 0; 2113 } 2114 } 2115 2116 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); 2117 offset = GET_BLKOFF_FROM_SEG0(sbi, i); 2118 2119 if (f2fs_block_unit_discard(sbi) && 2120 !f2fs_test_and_set_bit(offset, se->discard_map)) 2121 sbi->discard_blks--; 2122 } 2123 2124 if (len) 2125 err = __issue_discard_async(sbi, bdev, start, len); 2126 return err; 2127 } 2128 2129 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, 2130 bool check_only) 2131 { 2132 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 2133 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); 2134 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 2135 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 2136 unsigned long *discard_map = (unsigned long *)se->discard_map; 2137 unsigned long *dmap = SIT_I(sbi)->tmp_map; 2138 unsigned int start = 0, end = -1; 2139 bool force = (cpc->reason & CP_DISCARD); 2140 struct discard_entry *de = NULL; 2141 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; 2142 int i; 2143 2144 if (se->valid_blocks == BLKS_PER_SEG(sbi) || 2145 !f2fs_hw_support_discard(sbi) || 2146 !f2fs_block_unit_discard(sbi)) 2147 return false; 2148 2149 if (!force) { 2150 if (!f2fs_realtime_discard_enable(sbi) || 2151 (!se->valid_blocks && 2152 !is_curseg(sbi, cpc->trim_start)) || 2153 SM_I(sbi)->dcc_info->nr_discards >= 2154 SM_I(sbi)->dcc_info->max_discards) 2155 return false; 2156 } 2157 2158 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ 2159 for (i = 0; i < entries; i++) 2160 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : 2161 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; 2162 2163 while (force || SM_I(sbi)->dcc_info->nr_discards <= 2164 SM_I(sbi)->dcc_info->max_discards) { 2165 start = __find_rev_next_bit(dmap, BLKS_PER_SEG(sbi), end + 1); 2166 if (start >= BLKS_PER_SEG(sbi)) 2167 break; 2168 2169 end = __find_rev_next_zero_bit(dmap, 2170 BLKS_PER_SEG(sbi), start + 1); 2171 if (force && start && end != BLKS_PER_SEG(sbi) && 2172 (end - start) < cpc->trim_minlen) 2173 continue; 2174 2175 if (check_only) 2176 return true; 2177 2178 if (!de) { 2179 de = f2fs_kmem_cache_alloc(discard_entry_slab, 2180 GFP_F2FS_ZERO, true, NULL); 2181 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); 2182 list_add_tail(&de->list, head); 2183 } 2184 2185 for (i = start; i < end; i++) 2186 __set_bit_le(i, (void *)de->discard_map); 2187 2188 SM_I(sbi)->dcc_info->nr_discards += end - start; 2189 } 2190 return false; 2191 } 2192 2193 static void release_discard_addr(struct discard_entry *entry) 2194 { 2195 list_del(&entry->list); 2196 kmem_cache_free(discard_entry_slab, entry); 2197 } 2198 2199 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) 2200 { 2201 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); 2202 struct discard_entry *entry, *this; 2203 2204 /* drop caches */ 2205 list_for_each_entry_safe(entry, this, head, list) 2206 release_discard_addr(entry); 2207 } 2208 2209 /* 2210 * Should call f2fs_clear_prefree_segments after checkpoint is done. 2211 */ 2212 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) 2213 { 2214 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 2215 unsigned int segno; 2216 2217 mutex_lock(&dirty_i->seglist_lock); 2218 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) 2219 __set_test_and_free(sbi, segno, false); 2220 mutex_unlock(&dirty_i->seglist_lock); 2221 } 2222 2223 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, 2224 struct cp_control *cpc) 2225 { 2226 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2227 struct list_head *head = &dcc->entry_list; 2228 struct discard_entry *entry, *this; 2229 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 2230 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; 2231 unsigned int start = 0, end = -1; 2232 unsigned int secno, start_segno; 2233 bool force = (cpc->reason & CP_DISCARD); 2234 bool section_alignment = F2FS_OPTION(sbi).discard_unit == 2235 DISCARD_UNIT_SECTION; 2236 2237 if (f2fs_lfs_mode(sbi) && __is_large_section(sbi)) 2238 section_alignment = true; 2239 2240 mutex_lock(&dirty_i->seglist_lock); 2241 2242 while (1) { 2243 int i; 2244 2245 if (section_alignment && end != -1) 2246 end--; 2247 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); 2248 if (start >= MAIN_SEGS(sbi)) 2249 break; 2250 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), 2251 start + 1); 2252 2253 if (section_alignment) { 2254 start = rounddown(start, SEGS_PER_SEC(sbi)); 2255 end = roundup(end, SEGS_PER_SEC(sbi)); 2256 } 2257 2258 for (i = start; i < end; i++) { 2259 if (test_and_clear_bit(i, prefree_map)) 2260 dirty_i->nr_dirty[PRE]--; 2261 } 2262 2263 if (!f2fs_realtime_discard_enable(sbi)) 2264 continue; 2265 2266 if (force && start >= cpc->trim_start && 2267 (end - 1) <= cpc->trim_end) 2268 continue; 2269 2270 /* Should cover 2MB zoned device for zone-based reset */ 2271 if (!f2fs_sb_has_blkzoned(sbi) && 2272 (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi))) { 2273 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), 2274 SEGS_TO_BLKS(sbi, end - start)); 2275 continue; 2276 } 2277 next: 2278 secno = GET_SEC_FROM_SEG(sbi, start); 2279 start_segno = GET_SEG_FROM_SEC(sbi, secno); 2280 if (!is_cursec(sbi, secno) && 2281 !get_valid_blocks(sbi, start, true)) 2282 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), 2283 BLKS_PER_SEC(sbi)); 2284 2285 start = start_segno + SEGS_PER_SEC(sbi); 2286 if (start < end) 2287 goto next; 2288 else 2289 end = start - 1; 2290 } 2291 mutex_unlock(&dirty_i->seglist_lock); 2292 2293 if (!f2fs_block_unit_discard(sbi)) 2294 goto wakeup; 2295 2296 /* send small discards */ 2297 list_for_each_entry_safe(entry, this, head, list) { 2298 unsigned int cur_pos = 0, next_pos, len, total_len = 0; 2299 bool is_valid = test_bit_le(0, entry->discard_map); 2300 2301 find_next: 2302 if (is_valid) { 2303 next_pos = find_next_zero_bit_le(entry->discard_map, 2304 BLKS_PER_SEG(sbi), cur_pos); 2305 len = next_pos - cur_pos; 2306 2307 if (f2fs_sb_has_blkzoned(sbi) || 2308 (force && len < cpc->trim_minlen)) 2309 goto skip; 2310 2311 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, 2312 len); 2313 total_len += len; 2314 } else { 2315 next_pos = find_next_bit_le(entry->discard_map, 2316 BLKS_PER_SEG(sbi), cur_pos); 2317 } 2318 skip: 2319 cur_pos = next_pos; 2320 is_valid = !is_valid; 2321 2322 if (cur_pos < BLKS_PER_SEG(sbi)) 2323 goto find_next; 2324 2325 release_discard_addr(entry); 2326 dcc->nr_discards -= total_len; 2327 } 2328 2329 wakeup: 2330 wake_up_discard_thread(sbi, false); 2331 } 2332 2333 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi) 2334 { 2335 dev_t dev = sbi->sb->s_bdev->bd_dev; 2336 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2337 int err = 0; 2338 2339 if (f2fs_sb_has_readonly(sbi)) { 2340 f2fs_info(sbi, 2341 "Skip to start discard thread for readonly image"); 2342 return 0; 2343 } 2344 2345 if (!f2fs_realtime_discard_enable(sbi)) 2346 return 0; 2347 2348 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, 2349 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev)); 2350 if (IS_ERR(dcc->f2fs_issue_discard)) { 2351 err = PTR_ERR(dcc->f2fs_issue_discard); 2352 dcc->f2fs_issue_discard = NULL; 2353 } 2354 2355 return err; 2356 } 2357 2358 static int create_discard_cmd_control(struct f2fs_sb_info *sbi) 2359 { 2360 struct discard_cmd_control *dcc; 2361 int err = 0, i; 2362 2363 if (SM_I(sbi)->dcc_info) { 2364 dcc = SM_I(sbi)->dcc_info; 2365 goto init_thread; 2366 } 2367 2368 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); 2369 if (!dcc) 2370 return -ENOMEM; 2371 2372 dcc->discard_io_aware_gran = MAX_PLIST_NUM; 2373 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; 2374 dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY; 2375 dcc->discard_io_aware = DPOLICY_IO_AWARE_ENABLE; 2376 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT || 2377 F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) 2378 dcc->discard_granularity = BLKS_PER_SEG(sbi); 2379 2380 INIT_LIST_HEAD(&dcc->entry_list); 2381 for (i = 0; i < MAX_PLIST_NUM; i++) 2382 INIT_LIST_HEAD(&dcc->pend_list[i]); 2383 INIT_LIST_HEAD(&dcc->wait_list); 2384 INIT_LIST_HEAD(&dcc->fstrim_list); 2385 mutex_init(&dcc->cmd_lock); 2386 atomic_set(&dcc->issued_discard, 0); 2387 atomic_set(&dcc->queued_discard, 0); 2388 atomic_set(&dcc->discard_cmd_cnt, 0); 2389 dcc->nr_discards = 0; 2390 dcc->max_discards = SEGS_TO_BLKS(sbi, MAIN_SEGS(sbi)); 2391 dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST; 2392 dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME; 2393 dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME; 2394 dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME; 2395 dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL; 2396 dcc->undiscard_blks = 0; 2397 dcc->next_pos = 0; 2398 dcc->root = RB_ROOT_CACHED; 2399 dcc->rbtree_check = false; 2400 2401 init_waitqueue_head(&dcc->discard_wait_queue); 2402 SM_I(sbi)->dcc_info = dcc; 2403 init_thread: 2404 err = f2fs_start_discard_thread(sbi); 2405 if (err) { 2406 kfree(dcc); 2407 SM_I(sbi)->dcc_info = NULL; 2408 } 2409 2410 return err; 2411 } 2412 2413 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) 2414 { 2415 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2416 2417 if (!dcc) 2418 return; 2419 2420 f2fs_stop_discard_thread(sbi); 2421 2422 /* 2423 * Recovery can cache discard commands, so in error path of 2424 * fill_super(), it needs to give a chance to handle them. 2425 */ 2426 f2fs_issue_discard_timeout(sbi, true); 2427 2428 kfree(dcc); 2429 SM_I(sbi)->dcc_info = NULL; 2430 } 2431 2432 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) 2433 { 2434 struct sit_info *sit_i = SIT_I(sbi); 2435 2436 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) { 2437 sit_i->dirty_sentries++; 2438 return false; 2439 } 2440 2441 return true; 2442 } 2443 2444 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, 2445 unsigned int segno, int modified) 2446 { 2447 struct seg_entry *se = get_seg_entry(sbi, segno); 2448 2449 se->type = type; 2450 if (modified) 2451 __mark_sit_entry_dirty(sbi, segno); 2452 } 2453 2454 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi, 2455 block_t blkaddr) 2456 { 2457 unsigned int segno = GET_SEGNO(sbi, blkaddr); 2458 2459 if (segno == NULL_SEGNO) 2460 return 0; 2461 return get_seg_entry(sbi, segno)->mtime; 2462 } 2463 2464 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr, 2465 unsigned long long old_mtime) 2466 { 2467 struct seg_entry *se; 2468 unsigned int segno = GET_SEGNO(sbi, blkaddr); 2469 unsigned long long ctime = get_mtime(sbi, false); 2470 unsigned long long mtime = old_mtime ? old_mtime : ctime; 2471 2472 if (segno == NULL_SEGNO) 2473 return; 2474 2475 se = get_seg_entry(sbi, segno); 2476 2477 if (!se->mtime) 2478 se->mtime = mtime; 2479 else 2480 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime, 2481 se->valid_blocks + 1); 2482 2483 if (ctime > SIT_I(sbi)->max_mtime) 2484 SIT_I(sbi)->max_mtime = ctime; 2485 } 2486 2487 /* 2488 * NOTE: when updating multiple blocks at the same time, please ensure 2489 * that the consecutive input blocks belong to the same segment. 2490 */ 2491 static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_entry *se, 2492 unsigned int segno, block_t blkaddr, unsigned int offset, int del) 2493 { 2494 bool exist; 2495 int i; 2496 int del_count = -del; 2497 2498 f2fs_bug_on(sbi, GET_SEGNO(sbi, blkaddr) != GET_SEGNO(sbi, blkaddr + del_count - 1)); 2499 2500 for (i = 0; i < del_count; i++) { 2501 exist = f2fs_test_and_clear_bit(offset + i, se->cur_valid_map); 2502 if (unlikely(!exist)) { 2503 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", blkaddr + i); 2504 f2fs_bug_on(sbi, 1); 2505 se->valid_blocks++; 2506 del += 1; 2507 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 2508 /* 2509 * If checkpoints are off, we must not reuse data that 2510 * was used in the previous checkpoint. If it was used 2511 * before, we must track that to know how much space we 2512 * really have. 2513 */ 2514 if (f2fs_test_bit(offset + i, se->ckpt_valid_map)) { 2515 spin_lock(&sbi->stat_lock); 2516 sbi->unusable_block_count++; 2517 spin_unlock(&sbi->stat_lock); 2518 } 2519 } 2520 2521 if (f2fs_block_unit_discard(sbi) && 2522 f2fs_test_and_clear_bit(offset + i, se->discard_map)) 2523 sbi->discard_blks++; 2524 2525 if (!f2fs_test_bit(offset + i, se->ckpt_valid_map)) { 2526 se->ckpt_valid_blocks -= 1; 2527 if (__is_large_section(sbi)) 2528 get_sec_entry(sbi, segno)->ckpt_valid_blocks -= 1; 2529 } 2530 } 2531 2532 if (__is_large_section(sbi)) 2533 sanity_check_valid_blocks(sbi, segno); 2534 2535 return del; 2536 } 2537 2538 static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry *se, 2539 unsigned int segno, block_t blkaddr, unsigned int offset, int del) 2540 { 2541 bool exist; 2542 int del_count = del; 2543 int i; 2544 2545 f2fs_bug_on(sbi, GET_SEGNO(sbi, blkaddr) != GET_SEGNO(sbi, blkaddr + del_count - 1)); 2546 2547 for (i = 0; i < del_count; i++) { 2548 exist = f2fs_test_and_set_bit(offset + i, se->cur_valid_map); 2549 if (unlikely(exist)) { 2550 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", blkaddr + i); 2551 f2fs_bug_on(sbi, 1); 2552 se->valid_blocks--; 2553 del -= 1; 2554 continue; 2555 } 2556 2557 if (f2fs_block_unit_discard(sbi) && 2558 !f2fs_test_and_set_bit(offset + i, se->discard_map)) 2559 sbi->discard_blks--; 2560 2561 /* 2562 * SSR should never reuse block which is checkpointed 2563 * or newly invalidated. 2564 */ 2565 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { 2566 if (!f2fs_test_and_set_bit(offset + i, se->ckpt_valid_map)) { 2567 se->ckpt_valid_blocks++; 2568 if (__is_large_section(sbi)) 2569 get_sec_entry(sbi, segno)->ckpt_valid_blocks++; 2570 } 2571 } 2572 2573 if (!f2fs_test_bit(offset + i, se->ckpt_valid_map)) { 2574 se->ckpt_valid_blocks += 1; 2575 if (__is_large_section(sbi)) 2576 get_sec_entry(sbi, segno)->ckpt_valid_blocks += 1; 2577 } 2578 } 2579 2580 if (__is_large_section(sbi)) 2581 sanity_check_valid_blocks(sbi, segno); 2582 2583 return del; 2584 } 2585 2586 /* 2587 * If releasing blocks, this function supports updating multiple consecutive blocks 2588 * at one time, but please note that these consecutive blocks need to belong to the 2589 * same segment. 2590 */ 2591 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) 2592 { 2593 struct seg_entry *se; 2594 unsigned int segno, offset; 2595 long int new_vblocks; 2596 2597 segno = GET_SEGNO(sbi, blkaddr); 2598 if (segno == NULL_SEGNO) 2599 return; 2600 2601 se = get_seg_entry(sbi, segno); 2602 new_vblocks = se->valid_blocks + del; 2603 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2604 2605 f2fs_bug_on(sbi, (new_vblocks < 0 || 2606 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno)))); 2607 2608 se->valid_blocks = new_vblocks; 2609 2610 /* Update valid block bitmap */ 2611 if (del > 0) { 2612 del = update_sit_entry_for_alloc(sbi, se, segno, blkaddr, offset, del); 2613 } else { 2614 del = update_sit_entry_for_release(sbi, se, segno, blkaddr, offset, del); 2615 } 2616 2617 __mark_sit_entry_dirty(sbi, segno); 2618 2619 /* update total number of valid blocks to be written in ckpt area */ 2620 SIT_I(sbi)->written_valid_blocks += del; 2621 2622 if (__is_large_section(sbi)) 2623 get_sec_entry(sbi, segno)->valid_blocks += del; 2624 } 2625 2626 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr, 2627 unsigned int len) 2628 { 2629 unsigned int segno = GET_SEGNO(sbi, addr); 2630 struct sit_info *sit_i = SIT_I(sbi); 2631 block_t addr_start = addr, addr_end = addr + len - 1; 2632 unsigned int seg_num; 2633 unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt; 2634 2635 if (len == 0) 2636 return; 2637 2638 seg_num = GET_SEGNO(sbi, addr_end) - segno + 1; 2639 2640 f2fs_bug_on(sbi, addr == NULL_ADDR); 2641 if (addr == NEW_ADDR || addr == COMPRESS_ADDR) 2642 return; 2643 2644 f2fs_invalidate_internal_cache(sbi, addr, len); 2645 2646 /* add it into sit main buffer */ 2647 down_write(&sit_i->sentry_lock); 2648 2649 if (seg_num == 1) 2650 cnt = len; 2651 else 2652 cnt = max_blocks - GET_BLKOFF_FROM_SEG0(sbi, addr); 2653 2654 do { 2655 update_segment_mtime(sbi, addr_start, 0); 2656 update_sit_entry(sbi, addr_start, -cnt); 2657 2658 /* add it into dirty seglist */ 2659 locate_dirty_segment(sbi, segno); 2660 2661 /* update @addr_start and @cnt and @segno */ 2662 addr_start = START_BLOCK(sbi, ++segno); 2663 if (++i == seg_num) 2664 cnt = GET_BLKOFF_FROM_SEG0(sbi, addr_end) + 1; 2665 else 2666 cnt = max_blocks; 2667 } while (i <= seg_num); 2668 2669 up_write(&sit_i->sentry_lock); 2670 } 2671 2672 void f2fs_reserve_device_alias(struct f2fs_sb_info *sbi, block_t addr, 2673 unsigned int len) 2674 { 2675 unsigned int segno = GET_SEGNO(sbi, addr); 2676 struct sit_info *sit_i = SIT_I(sbi); 2677 block_t addr_start = addr, addr_end = addr + len - 1; 2678 unsigned int seg_num; 2679 unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt; 2680 2681 if (len == 0) 2682 return; 2683 2684 seg_num = GET_SEGNO(sbi, addr_end) - segno + 1; 2685 2686 down_write(&sit_i->sentry_lock); 2687 2688 if (seg_num == 1) 2689 cnt = len; 2690 else 2691 cnt = max_blocks - GET_BLKOFF_FROM_SEG0(sbi, addr); 2692 2693 do { 2694 update_segment_mtime(sbi, addr_start, 0); 2695 update_sit_entry(sbi, addr_start, cnt); 2696 __set_test_and_inuse(sbi, segno); 2697 2698 /* Remove the segment from PRE (prefree) to prevent checkpoint from freeing it! */ 2699 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 2700 if (test_and_clear_bit(segno, DIRTY_I(sbi)->dirty_segmap[PRE])) 2701 DIRTY_I(sbi)->nr_dirty[PRE]--; 2702 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 2703 2704 /* add it into dirty seglist */ 2705 locate_dirty_segment(sbi, segno); 2706 2707 /* update @addr_start and @cnt and @segno */ 2708 addr_start = START_BLOCK(sbi, ++segno); 2709 if (++i == seg_num) 2710 cnt = GET_BLKOFF_FROM_SEG0(sbi, addr_end) + 1; 2711 else 2712 cnt = max_blocks; 2713 } while (i <= seg_num); 2714 2715 up_write(&sit_i->sentry_lock); 2716 } 2717 2718 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) 2719 { 2720 struct sit_info *sit_i = SIT_I(sbi); 2721 unsigned int segno, offset; 2722 struct seg_entry *se; 2723 bool is_cp = false; 2724 2725 if (!__is_valid_data_blkaddr(blkaddr)) 2726 return true; 2727 2728 down_read(&sit_i->sentry_lock); 2729 2730 segno = GET_SEGNO(sbi, blkaddr); 2731 se = get_seg_entry(sbi, segno); 2732 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2733 2734 if (f2fs_test_bit(offset, se->ckpt_valid_map)) 2735 is_cp = true; 2736 2737 up_read(&sit_i->sentry_lock); 2738 2739 return is_cp; 2740 } 2741 2742 static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type) 2743 { 2744 struct curseg_info *curseg = CURSEG_I(sbi, type); 2745 2746 if (sbi->ckpt->alloc_type[type] == SSR) 2747 return BLKS_PER_SEG(sbi); 2748 return curseg->next_blkoff; 2749 } 2750 2751 /* 2752 * Calculate the number of current summary pages for writing 2753 */ 2754 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) 2755 { 2756 int valid_sum_count = 0; 2757 int i, sum_in_page; 2758 2759 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 2760 if (sbi->ckpt->alloc_type[i] != SSR && for_ra) 2761 valid_sum_count += 2762 le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]); 2763 else 2764 valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); 2765 } 2766 2767 sum_in_page = (sbi->blocksize - 2 * sbi->sum_journal_size - 2768 SUM_FOOTER_SIZE) / SUMMARY_SIZE; 2769 if (valid_sum_count <= sum_in_page) 2770 return 1; 2771 else if ((valid_sum_count - sum_in_page) <= 2772 (sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE) 2773 return 2; 2774 return 3; 2775 } 2776 2777 /* 2778 * Caller should put this summary folio 2779 */ 2780 struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno) 2781 { 2782 if (unlikely(f2fs_cp_error(sbi))) 2783 return ERR_PTR(-EIO); 2784 return f2fs_get_meta_folio_retry(sbi, GET_SUM_BLOCK(sbi, segno)); 2785 } 2786 2787 void f2fs_update_meta_page(struct f2fs_sb_info *sbi, 2788 void *src, block_t blk_addr) 2789 { 2790 struct folio *folio; 2791 2792 if (!f2fs_sb_has_packed_ssa(sbi)) 2793 folio = f2fs_grab_meta_folio(sbi, blk_addr); 2794 else 2795 folio = f2fs_get_meta_folio_retry(sbi, blk_addr); 2796 2797 if (IS_ERR(folio)) 2798 return; 2799 2800 memcpy(folio_address(folio), src, PAGE_SIZE); 2801 folio_mark_dirty(folio); 2802 f2fs_folio_put(folio, true); 2803 } 2804 2805 static void write_sum_page(struct f2fs_sb_info *sbi, 2806 struct f2fs_summary_block *sum_blk, unsigned int segno) 2807 { 2808 struct folio *folio; 2809 2810 if (!f2fs_sb_has_packed_ssa(sbi)) 2811 return f2fs_update_meta_page(sbi, (void *)sum_blk, 2812 GET_SUM_BLOCK(sbi, segno)); 2813 2814 folio = f2fs_get_sum_folio(sbi, segno); 2815 if (IS_ERR(folio)) 2816 return; 2817 2818 memcpy(SUM_BLK_PAGE_ADDR(sbi, folio, segno), sum_blk, 2819 sbi->sum_blocksize); 2820 folio_mark_dirty(folio); 2821 f2fs_folio_put(folio, true); 2822 } 2823 2824 static void write_current_sum_page(struct f2fs_sb_info *sbi, 2825 int type, block_t blk_addr) 2826 { 2827 struct curseg_info *curseg = CURSEG_I(sbi, type); 2828 struct folio *folio = f2fs_grab_meta_folio(sbi, blk_addr); 2829 struct f2fs_summary_block *src = curseg->sum_blk; 2830 struct f2fs_summary_block *dst; 2831 2832 dst = folio_address(folio); 2833 memset(dst, 0, PAGE_SIZE); 2834 2835 mutex_lock(&curseg->curseg_mutex); 2836 2837 down_read(&curseg->journal_rwsem); 2838 memcpy(sum_journal(sbi, dst), curseg->journal, sbi->sum_journal_size); 2839 up_read(&curseg->journal_rwsem); 2840 2841 memcpy(sum_entries(dst), sum_entries(src), sbi->sum_entry_size); 2842 memcpy(sum_footer(sbi, dst), sum_footer(sbi, src), SUM_FOOTER_SIZE); 2843 2844 mutex_unlock(&curseg->curseg_mutex); 2845 2846 folio_mark_dirty(folio); 2847 f2fs_folio_put(folio, true); 2848 } 2849 2850 static int is_next_segment_free(struct f2fs_sb_info *sbi, 2851 struct curseg_info *curseg) 2852 { 2853 unsigned int segno = curseg->segno + 1; 2854 struct free_segmap_info *free_i = FREE_I(sbi); 2855 2856 if (segno < MAIN_SEGS(sbi) && segno % SEGS_PER_SEC(sbi)) { 2857 int devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); 2858 2859 if (f2fs_dev_is_reserving(sbi, devi)) 2860 return 0; 2861 return !test_bit(segno, free_i->free_segmap); 2862 } 2863 return 0; 2864 } 2865 2866 /* 2867 * Find a new segment from the free segments bitmap to right order 2868 * This function should be returned with success, otherwise BUG 2869 */ 2870 static int get_new_segment(struct f2fs_sb_info *sbi, 2871 unsigned int *newseg, bool new_sec, bool pinning) 2872 { 2873 struct free_segmap_info *free_i = FREE_I(sbi); 2874 unsigned int segno, secno, zoneno; 2875 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; 2876 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); 2877 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); 2878 unsigned int alloc_policy = sbi->allocate_section_policy; 2879 unsigned int alloc_hint = sbi->allocate_section_hint; 2880 unsigned int max_secno = MAIN_SECS(sbi); 2881 bool init = true; 2882 bool looped = false; 2883 int i, devi; 2884 int ret = 0; 2885 2886 spin_lock(&free_i->segmap_lock); 2887 2888 if (time_to_inject(sbi, FAULT_NO_SEGMENT)) { 2889 ret = -ENOSPC; 2890 goto out_unlock; 2891 } 2892 2893 if (!new_sec && ((*newseg + 1) % SEGS_PER_SEC(sbi))) { 2894 segno = find_next_zero_bit(free_i->free_segmap, 2895 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); 2896 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) { 2897 devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); 2898 2899 if (f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) 2900 goto find_other_zone; 2901 goto got_it; 2902 } 2903 } 2904 2905 #ifdef CONFIG_BLK_DEV_ZONED 2906 /* 2907 * If we format f2fs on zoned storage, let's try to get pinned sections 2908 * from beginning of the storage, which should be a conventional one. 2909 */ 2910 if (f2fs_sb_has_blkzoned(sbi)) { 2911 /* Prioritize writing to conventional zones */ 2912 if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_PRIOR_CONV) 2913 segno = 0; 2914 else 2915 segno = max(sbi->first_seq_zone_segno, *newseg); 2916 hint = GET_SEC_FROM_SEG(sbi, segno); 2917 } 2918 #endif 2919 2920 /* 2921 * Prevent allocate_section_hint from exceeding MAIN_SECS() 2922 * due to desynchronization. 2923 */ 2924 if (alloc_policy != ALLOCATE_FORWARD_NOHINT && 2925 alloc_hint > MAIN_SECS(sbi)) 2926 alloc_hint = MAIN_SECS(sbi); 2927 2928 if (pinning) { 2929 max_secno = sbi->pinned_area_max_secno; 2930 hint = 0; 2931 } else if (alloc_policy == ALLOCATE_FORWARD_FROM_HINT && 2932 hint < alloc_hint) { 2933 hint = alloc_hint; 2934 } else if (alloc_policy == ALLOCATE_FORWARD_WITHIN_HINT && 2935 hint >= alloc_hint) { 2936 hint = 0; 2937 } 2938 2939 find_other_zone: 2940 secno = find_next_zero_bit(free_i->free_secmap, max_secno, hint); 2941 2942 if (secno >= max_secno) { 2943 if (looped) { 2944 ret = (pinning && has_unpinned_area(sbi)) ? 2945 -EAGAIN : -ENOSPC; 2946 f2fs_bug_on(sbi, !pinning); 2947 goto out_unlock; 2948 } 2949 hint = 0; 2950 #ifdef CONFIG_BLK_DEV_ZONED 2951 /* Write only to sequential zones */ 2952 if (f2fs_sb_has_blkzoned(sbi) && 2953 sbi->blkzone_alloc_policy == BLKZONE_ALLOC_ONLY_SEQ) 2954 hint = GET_SEC_FROM_SEG(sbi, sbi->first_seq_zone_segno); 2955 #endif 2956 looped = true; 2957 goto find_other_zone; 2958 } 2959 2960 segno = GET_SEG_FROM_SEC(sbi, secno); 2961 2962 devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); 2963 2964 if (f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) { 2965 while (devi < sbi->s_ndevs && 2966 f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) { 2967 unsigned int end_segno = GET_SEGNO(sbi, FDEV(devi).end_blk); 2968 2969 hint = GET_SEC_FROM_SEG(sbi, end_segno) + 1; 2970 devi++; 2971 } 2972 goto find_other_zone; 2973 } 2974 2975 if (sec_usage_check(sbi, secno)) { 2976 hint = secno + 1; 2977 goto find_other_zone; 2978 } 2979 zoneno = GET_ZONE_FROM_SEC(sbi, secno); 2980 2981 /* give up on finding another zone */ 2982 if (!init) 2983 goto got_it; 2984 if (sbi->secs_per_zone == 1) 2985 goto got_it; 2986 if (zoneno == old_zoneno) 2987 goto got_it; 2988 for (i = 0; i < NR_CURSEG_TYPE; i++) 2989 if (CURSEG_I(sbi, i)->zone == zoneno) 2990 break; 2991 2992 if (i < NR_CURSEG_TYPE) { 2993 /* zone is in user, try another */ 2994 if (zoneno + 1 >= total_zones) 2995 hint = 0; 2996 else 2997 hint = (zoneno + 1) * sbi->secs_per_zone; 2998 init = false; 2999 goto find_other_zone; 3000 } 3001 got_it: 3002 /* set it as dirty segment in free segmap */ 3003 if (test_bit(segno, free_i->free_segmap)) { 3004 ret = -EFSCORRUPTED; 3005 fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); 3006 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_FREE_BITMAP); 3007 goto out_unlock; 3008 } 3009 3010 __set_inuse(sbi, segno); 3011 *newseg = segno; 3012 out_unlock: 3013 spin_unlock(&free_i->segmap_lock); 3014 3015 if (ret == -ENOSPC && !pinning) 3016 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_NO_SEGMENT); 3017 return ret; 3018 } 3019 3020 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) 3021 { 3022 struct curseg_info *curseg = CURSEG_I(sbi, type); 3023 struct summary_footer *sum_footer; 3024 unsigned short seg_type = curseg->seg_type; 3025 3026 /* only happen when get_new_segment() fails */ 3027 if (curseg->next_segno == NULL_SEGNO) 3028 return; 3029 3030 curseg->inited = true; 3031 curseg->segno = curseg->next_segno; 3032 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); 3033 curseg->next_blkoff = 0; 3034 curseg->next_segno = NULL_SEGNO; 3035 3036 sum_footer = sum_footer(sbi, curseg->sum_blk); 3037 memset(sum_footer, 0, sizeof(struct summary_footer)); 3038 3039 sanity_check_seg_type(sbi, seg_type); 3040 3041 if (IS_DATASEG(seg_type)) 3042 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); 3043 if (IS_NODESEG(seg_type)) 3044 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); 3045 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified); 3046 } 3047 3048 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) 3049 { 3050 struct curseg_info *curseg = CURSEG_I(sbi, type); 3051 unsigned short seg_type = curseg->seg_type; 3052 3053 sanity_check_seg_type(sbi, seg_type); 3054 if (__is_large_section(sbi)) { 3055 if (f2fs_need_rand_seg_blk(sbi, type)) { 3056 unsigned int hint = GET_SEC_FROM_SEG(sbi, curseg->segno); 3057 3058 if (GET_SEC_FROM_SEG(sbi, curseg->segno + 1) != hint) 3059 return curseg->segno; 3060 return get_random_u32_inclusive(curseg->segno + 1, 3061 GET_SEG_FROM_SEC(sbi, hint + 1) - 1); 3062 } 3063 return curseg->segno; 3064 } else if (f2fs_need_rand_seg_blk(sbi, type)) { 3065 return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); 3066 } 3067 3068 /* inmem log may not locate on any segment after mount */ 3069 if (!curseg->inited) 3070 return 0; 3071 3072 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 3073 return 0; 3074 3075 if (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)) 3076 return 0; 3077 3078 if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) 3079 return SIT_I(sbi)->last_victim[ALLOC_NEXT]; 3080 3081 /* find segments from 0 to reuse freed segments */ 3082 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) 3083 return 0; 3084 3085 return curseg->segno; 3086 } 3087 3088 static void reset_curseg_fields(struct curseg_info *curseg) 3089 { 3090 curseg->inited = false; 3091 curseg->segno = NULL_SEGNO; 3092 curseg->next_segno = 0; 3093 } 3094 3095 /* 3096 * Allocate a current working segment. 3097 * This function always allocates a free segment in LFS manner. 3098 */ 3099 static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) 3100 { 3101 struct curseg_info *curseg = CURSEG_I(sbi, type); 3102 unsigned int segno = curseg->segno; 3103 bool pinning = type == CURSEG_COLD_DATA_PINNED; 3104 int ret; 3105 3106 if (curseg->inited) 3107 write_sum_page(sbi, curseg->sum_blk, segno); 3108 3109 segno = __get_next_segno(sbi, type); 3110 ret = get_new_segment(sbi, &segno, new_sec, pinning); 3111 if (ret) { 3112 if (ret == -ENOSPC) 3113 reset_curseg_fields(curseg); 3114 return ret; 3115 } 3116 3117 curseg->next_segno = segno; 3118 reset_curseg(sbi, type, 1); 3119 curseg->alloc_type = LFS; 3120 if (f2fs_need_rand_blk(sbi, type)) 3121 curseg->fragment_remained_chunk = 3122 get_random_u32_inclusive(1, sbi->max_fragment_chunk); 3123 return 0; 3124 } 3125 3126 static int __next_free_blkoff(struct f2fs_sb_info *sbi, 3127 int segno, block_t start) 3128 { 3129 struct seg_entry *se = get_seg_entry(sbi, segno); 3130 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 3131 unsigned long *target_map = SIT_I(sbi)->tmp_map; 3132 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 3133 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 3134 int i; 3135 3136 for (i = 0; i < entries; i++) 3137 target_map[i] = ckpt_map[i] | cur_map[i]; 3138 3139 return __find_rev_next_zero_bit(target_map, BLKS_PER_SEG(sbi), start); 3140 } 3141 3142 static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi, 3143 struct curseg_info *seg) 3144 { 3145 return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1); 3146 } 3147 3148 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno) 3149 { 3150 return __next_free_blkoff(sbi, segno, 0) < BLKS_PER_SEG(sbi); 3151 } 3152 3153 /* 3154 * This function always allocates a used segment(from dirty seglist) by SSR 3155 * manner, so it should recover the existing segment information of valid blocks 3156 */ 3157 static int change_curseg(struct f2fs_sb_info *sbi, int type) 3158 { 3159 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 3160 struct curseg_info *curseg = CURSEG_I(sbi, type); 3161 unsigned int new_segno = curseg->next_segno; 3162 struct f2fs_summary_block *sum_node; 3163 struct folio *sum_folio; 3164 3165 if (curseg->inited) 3166 write_sum_page(sbi, curseg->sum_blk, curseg->segno); 3167 3168 __set_test_and_inuse(sbi, new_segno); 3169 3170 mutex_lock(&dirty_i->seglist_lock); 3171 __remove_dirty_segment(sbi, new_segno, PRE); 3172 __remove_dirty_segment(sbi, new_segno, DIRTY); 3173 mutex_unlock(&dirty_i->seglist_lock); 3174 3175 reset_curseg(sbi, type, 1); 3176 curseg->alloc_type = SSR; 3177 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0); 3178 3179 sum_folio = f2fs_get_sum_folio(sbi, new_segno); 3180 if (IS_ERR(sum_folio)) { 3181 /* GC won't be able to use stale summary pages by cp_error */ 3182 memset(curseg->sum_blk, 0, sbi->sum_entry_size); 3183 return PTR_ERR(sum_folio); 3184 } 3185 sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, new_segno); 3186 memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size); 3187 f2fs_folio_put(sum_folio, true); 3188 return 0; 3189 } 3190 3191 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, 3192 int alloc_mode, unsigned long long age); 3193 3194 static int get_atssr_segment(struct f2fs_sb_info *sbi, int type, 3195 int target_type, int alloc_mode, 3196 unsigned long long age) 3197 { 3198 struct curseg_info *curseg = CURSEG_I(sbi, type); 3199 int ret = 0; 3200 3201 curseg->seg_type = target_type; 3202 3203 if (get_ssr_segment(sbi, type, alloc_mode, age)) { 3204 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno); 3205 3206 curseg->seg_type = se->type; 3207 ret = change_curseg(sbi, type); 3208 } else { 3209 /* allocate cold segment by default */ 3210 curseg->seg_type = CURSEG_COLD_DATA; 3211 ret = new_curseg(sbi, type, true); 3212 } 3213 stat_inc_seg_type(sbi, curseg); 3214 return ret; 3215 } 3216 3217 static int __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi, bool force) 3218 { 3219 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC); 3220 int ret = 0; 3221 3222 if (!sbi->am.atgc_enabled && !force) 3223 return 0; 3224 3225 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3226 3227 mutex_lock(&curseg->curseg_mutex); 3228 down_write(&SIT_I(sbi)->sentry_lock); 3229 3230 ret = get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, 3231 CURSEG_COLD_DATA, SSR, 0); 3232 3233 up_write(&SIT_I(sbi)->sentry_lock); 3234 mutex_unlock(&curseg->curseg_mutex); 3235 3236 f2fs_up_read(&SM_I(sbi)->curseg_lock); 3237 return ret; 3238 } 3239 3240 int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi) 3241 { 3242 return __f2fs_init_atgc_curseg(sbi, false); 3243 } 3244 3245 int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi) 3246 { 3247 int ret; 3248 3249 if (!test_opt(sbi, ATGC)) 3250 return 0; 3251 if (sbi->am.atgc_enabled) 3252 return 0; 3253 if (le64_to_cpu(F2FS_CKPT(sbi)->elapsed_time) < 3254 sbi->am.age_threshold) 3255 return 0; 3256 3257 ret = __f2fs_init_atgc_curseg(sbi, true); 3258 if (!ret) { 3259 sbi->am.atgc_enabled = true; 3260 f2fs_info(sbi, "reenabled age threshold GC"); 3261 } 3262 return ret; 3263 } 3264 3265 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type) 3266 { 3267 struct curseg_info *curseg = CURSEG_I(sbi, type); 3268 3269 mutex_lock(&curseg->curseg_mutex); 3270 if (!curseg->inited) 3271 goto out; 3272 3273 if (get_valid_blocks(sbi, curseg->segno, false)) { 3274 write_sum_page(sbi, curseg->sum_blk, curseg->segno); 3275 } else { 3276 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 3277 __set_test_and_free(sbi, curseg->segno, true); 3278 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 3279 } 3280 out: 3281 mutex_unlock(&curseg->curseg_mutex); 3282 } 3283 3284 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi) 3285 { 3286 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); 3287 3288 if (sbi->am.atgc_enabled) 3289 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); 3290 } 3291 3292 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type) 3293 { 3294 struct curseg_info *curseg = CURSEG_I(sbi, type); 3295 3296 mutex_lock(&curseg->curseg_mutex); 3297 if (!curseg->inited) 3298 goto out; 3299 if (get_valid_blocks(sbi, curseg->segno, false)) 3300 goto out; 3301 3302 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 3303 __set_test_and_inuse(sbi, curseg->segno); 3304 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 3305 out: 3306 mutex_unlock(&curseg->curseg_mutex); 3307 } 3308 3309 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi) 3310 { 3311 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); 3312 3313 if (sbi->am.atgc_enabled) 3314 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); 3315 } 3316 3317 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, 3318 int alloc_mode, unsigned long long age) 3319 { 3320 struct curseg_info *curseg = CURSEG_I(sbi, type); 3321 unsigned segno = NULL_SEGNO; 3322 unsigned short seg_type = curseg->seg_type; 3323 int i, cnt; 3324 bool reversed = false; 3325 3326 sanity_check_seg_type(sbi, seg_type); 3327 3328 /* f2fs_need_SSR() already forces to do this */ 3329 if (!f2fs_get_victim(sbi, &segno, BG_GC, seg_type, 3330 alloc_mode, age, false)) { 3331 curseg->next_segno = segno; 3332 return 1; 3333 } 3334 3335 /* For node segments, let's do SSR more intensively */ 3336 if (IS_NODESEG(seg_type)) { 3337 if (seg_type >= CURSEG_WARM_NODE) { 3338 reversed = true; 3339 i = CURSEG_COLD_NODE; 3340 } else { 3341 i = CURSEG_HOT_NODE; 3342 } 3343 cnt = NR_CURSEG_NODE_TYPE; 3344 } else { 3345 if (seg_type >= CURSEG_WARM_DATA) { 3346 reversed = true; 3347 i = CURSEG_COLD_DATA; 3348 } else { 3349 i = CURSEG_HOT_DATA; 3350 } 3351 cnt = NR_CURSEG_DATA_TYPE; 3352 } 3353 3354 for (; cnt-- > 0; reversed ? i-- : i++) { 3355 if (i == seg_type) 3356 continue; 3357 if (!f2fs_get_victim(sbi, &segno, BG_GC, i, 3358 alloc_mode, age, false)) { 3359 curseg->next_segno = segno; 3360 return 1; 3361 } 3362 } 3363 3364 /* find valid_blocks=0 in dirty list */ 3365 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 3366 segno = get_free_segment(sbi); 3367 if (segno != NULL_SEGNO) { 3368 curseg->next_segno = segno; 3369 return 1; 3370 } 3371 } 3372 return 0; 3373 } 3374 3375 static bool need_new_seg(struct f2fs_sb_info *sbi, int type) 3376 { 3377 struct curseg_info *curseg = CURSEG_I(sbi, type); 3378 3379 if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && 3380 curseg->seg_type == CURSEG_WARM_NODE) 3381 return true; 3382 if (curseg->alloc_type == LFS && is_next_segment_free(sbi, curseg) && 3383 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 3384 return true; 3385 if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0)) 3386 return true; 3387 return false; 3388 } 3389 3390 int f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, 3391 unsigned int start, unsigned int end) 3392 { 3393 struct curseg_info *curseg = CURSEG_I(sbi, type); 3394 unsigned int segno; 3395 int ret = 0; 3396 3397 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3398 mutex_lock(&curseg->curseg_mutex); 3399 down_write(&SIT_I(sbi)->sentry_lock); 3400 3401 segno = CURSEG_I(sbi, type)->segno; 3402 if (segno < start || segno > end) 3403 goto unlock; 3404 3405 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0)) 3406 ret = change_curseg(sbi, type); 3407 else 3408 ret = new_curseg(sbi, type, true); 3409 3410 stat_inc_seg_type(sbi, curseg); 3411 3412 locate_dirty_segment(sbi, segno); 3413 unlock: 3414 up_write(&SIT_I(sbi)->sentry_lock); 3415 3416 if (segno != curseg->segno) 3417 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u", 3418 type, segno, curseg->segno); 3419 3420 mutex_unlock(&curseg->curseg_mutex); 3421 f2fs_up_read(&SM_I(sbi)->curseg_lock); 3422 return ret; 3423 } 3424 3425 static int __allocate_new_segment(struct f2fs_sb_info *sbi, int type, 3426 bool new_sec, bool force) 3427 { 3428 struct curseg_info *curseg = CURSEG_I(sbi, type); 3429 unsigned int old_segno; 3430 int err = 0; 3431 3432 if (type == CURSEG_COLD_DATA_PINNED && !curseg->inited) 3433 goto allocate; 3434 3435 if (!force && curseg->inited && 3436 !curseg->next_blkoff && 3437 !get_valid_blocks(sbi, curseg->segno, new_sec) && 3438 !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec)) 3439 return 0; 3440 3441 allocate: 3442 old_segno = curseg->segno; 3443 err = new_curseg(sbi, type, true); 3444 if (err) 3445 return err; 3446 stat_inc_seg_type(sbi, curseg); 3447 locate_dirty_segment(sbi, old_segno); 3448 return 0; 3449 } 3450 3451 int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force) 3452 { 3453 int ret; 3454 3455 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3456 down_write(&SIT_I(sbi)->sentry_lock); 3457 ret = __allocate_new_segment(sbi, type, true, force); 3458 up_write(&SIT_I(sbi)->sentry_lock); 3459 f2fs_up_read(&SM_I(sbi)->curseg_lock); 3460 3461 return ret; 3462 } 3463 3464 int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi) 3465 { 3466 struct f2fs_lock_context lc; 3467 int err; 3468 bool gc_required = true; 3469 3470 retry: 3471 f2fs_lock_op(sbi, &lc); 3472 err = f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false); 3473 f2fs_unlock_op(sbi, &lc); 3474 3475 if (has_unpinned_area(sbi) && err == -EAGAIN && gc_required) { 3476 err = f2fs_gc_range(sbi, 0, 3477 sbi->pinned_area_max_secno * SEGS_PER_SEC(sbi) - 1, 3478 true, ZONED_PIN_SEC_REQUIRED_COUNT, true); 3479 if (err) 3480 return err; 3481 err = f2fs_sync_fs(sbi->sb, 1); 3482 if (!err) { 3483 gc_required = false; 3484 goto retry; 3485 } 3486 } 3487 3488 return err; 3489 } 3490 3491 int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) 3492 { 3493 int i; 3494 int err = 0; 3495 3496 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3497 down_write(&SIT_I(sbi)->sentry_lock); 3498 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) 3499 err += __allocate_new_segment(sbi, i, false, false); 3500 up_write(&SIT_I(sbi)->sentry_lock); 3501 f2fs_up_read(&SM_I(sbi)->curseg_lock); 3502 3503 return err; 3504 } 3505 3506 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, 3507 struct cp_control *cpc) 3508 { 3509 __u64 trim_start = cpc->trim_start; 3510 bool has_candidate = false; 3511 3512 down_write(&SIT_I(sbi)->sentry_lock); 3513 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) { 3514 if (add_discard_addrs(sbi, cpc, true)) { 3515 has_candidate = true; 3516 break; 3517 } 3518 } 3519 up_write(&SIT_I(sbi)->sentry_lock); 3520 3521 cpc->trim_start = trim_start; 3522 return has_candidate; 3523 } 3524 3525 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, 3526 struct discard_policy *dpolicy, 3527 unsigned int start, unsigned int end) 3528 { 3529 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 3530 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 3531 struct rb_node **insert_p = NULL, *insert_parent = NULL; 3532 struct discard_cmd *dc; 3533 struct blk_plug plug; 3534 int issued; 3535 unsigned int trimmed = 0; 3536 3537 next: 3538 issued = 0; 3539 3540 mutex_lock(&dcc->cmd_lock); 3541 if (unlikely(dcc->rbtree_check)) 3542 f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi)); 3543 3544 dc = __lookup_discard_cmd_ret(&dcc->root, start, 3545 &prev_dc, &next_dc, &insert_p, &insert_parent); 3546 if (!dc) 3547 dc = next_dc; 3548 3549 blk_start_plug(&plug); 3550 3551 while (dc && dc->di.lstart <= end) { 3552 struct rb_node *node; 3553 int err = 0; 3554 3555 if (dc->di.len < dpolicy->granularity) 3556 goto skip; 3557 3558 if (dc->state != D_PREP) { 3559 list_move_tail(&dc->list, &dcc->fstrim_list); 3560 goto skip; 3561 } 3562 3563 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); 3564 3565 if (issued >= dpolicy->max_requests) { 3566 start = dc->di.lstart + dc->di.len; 3567 3568 if (err) 3569 __remove_discard_cmd(sbi, dc); 3570 3571 blk_finish_plug(&plug); 3572 mutex_unlock(&dcc->cmd_lock); 3573 trimmed += __wait_all_discard_cmd(sbi, NULL); 3574 f2fs_schedule_timeout(DEFAULT_DISCARD_INTERVAL); 3575 goto next; 3576 } 3577 skip: 3578 node = rb_next(&dc->rb_node); 3579 if (err) 3580 __remove_discard_cmd(sbi, dc); 3581 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 3582 3583 if (fatal_signal_pending(current)) 3584 break; 3585 } 3586 3587 blk_finish_plug(&plug); 3588 mutex_unlock(&dcc->cmd_lock); 3589 3590 return trimmed; 3591 } 3592 3593 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) 3594 { 3595 __u64 start = F2FS_BYTES_TO_BLK(range->start); 3596 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; 3597 unsigned int start_segno, end_segno; 3598 block_t start_block, end_block; 3599 struct cp_control cpc; 3600 struct discard_policy dpolicy; 3601 struct f2fs_lock_context lc; 3602 unsigned long long trimmed = 0; 3603 int err = 0; 3604 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); 3605 3606 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) 3607 return -EINVAL; 3608 3609 if (end < MAIN_BLKADDR(sbi)) 3610 goto out; 3611 3612 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { 3613 f2fs_warn(sbi, "Found FS corruption, run fsck to fix."); 3614 return -EFSCORRUPTED; 3615 } 3616 3617 /* start/end segment number in main_area */ 3618 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); 3619 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : 3620 GET_SEGNO(sbi, end); 3621 if (need_align) { 3622 start_segno = rounddown(start_segno, SEGS_PER_SEC(sbi)); 3623 end_segno = roundup(end_segno + 1, SEGS_PER_SEC(sbi)) - 1; 3624 } 3625 3626 cpc.reason = CP_DISCARD; 3627 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); 3628 cpc.trim_start = start_segno; 3629 cpc.trim_end = end_segno; 3630 3631 if (sbi->discard_blks == 0) 3632 goto out; 3633 3634 f2fs_down_write_trace(&sbi->gc_lock, &lc); 3635 stat_inc_cp_call_count(sbi, TOTAL_CALL); 3636 err = f2fs_write_checkpoint(sbi, &cpc); 3637 f2fs_up_write_trace(&sbi->gc_lock, &lc); 3638 if (err) 3639 goto out; 3640 3641 /* 3642 * We filed discard candidates, but actually we don't need to wait for 3643 * all of them, since they'll be issued in idle time along with runtime 3644 * discard option. User configuration looks like using runtime discard 3645 * or periodic fstrim instead of it. 3646 */ 3647 if (f2fs_realtime_discard_enable(sbi)) 3648 goto out; 3649 3650 start_block = START_BLOCK(sbi, start_segno); 3651 end_block = START_BLOCK(sbi, end_segno + 1); 3652 3653 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); 3654 trimmed = __issue_discard_cmd_range(sbi, &dpolicy, 3655 start_block, end_block); 3656 3657 trimmed += __wait_discard_cmd_range(sbi, &dpolicy, 3658 start_block, end_block); 3659 out: 3660 if (!err) 3661 range->len = F2FS_BLK_TO_BYTES(trimmed); 3662 return err; 3663 } 3664 3665 int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint) 3666 { 3667 if (F2FS_OPTION(sbi).active_logs == 2) 3668 return CURSEG_HOT_DATA; 3669 else if (F2FS_OPTION(sbi).active_logs == 4) 3670 return CURSEG_COLD_DATA; 3671 3672 /* active_log == 6 */ 3673 switch (hint) { 3674 case WRITE_LIFE_SHORT: 3675 return CURSEG_HOT_DATA; 3676 case WRITE_LIFE_EXTREME: 3677 return CURSEG_COLD_DATA; 3678 default: 3679 return CURSEG_WARM_DATA; 3680 } 3681 } 3682 3683 /* 3684 * This returns write hints for each segment type. This hints will be 3685 * passed down to block layer as below by default. 3686 * 3687 * User F2FS Block 3688 * ---- ---- ----- 3689 * META WRITE_LIFE_NONE|REQ_META 3690 * HOT_NODE WRITE_LIFE_NONE 3691 * WARM_NODE WRITE_LIFE_MEDIUM 3692 * COLD_NODE WRITE_LIFE_LONG 3693 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME 3694 * extension list " " 3695 * 3696 * -- buffered io 3697 * COLD_DATA WRITE_LIFE_EXTREME 3698 * HOT_DATA WRITE_LIFE_SHORT 3699 * WARM_DATA WRITE_LIFE_NOT_SET 3700 * 3701 * -- direct io 3702 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 3703 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 3704 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 3705 * WRITE_LIFE_NONE " WRITE_LIFE_NONE 3706 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM 3707 * WRITE_LIFE_LONG " WRITE_LIFE_LONG 3708 */ 3709 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, 3710 enum page_type type, enum temp_type temp) 3711 { 3712 switch (type) { 3713 case DATA: 3714 switch (temp) { 3715 case WARM: 3716 return WRITE_LIFE_NOT_SET; 3717 case HOT: 3718 return WRITE_LIFE_SHORT; 3719 case COLD: 3720 return WRITE_LIFE_EXTREME; 3721 default: 3722 return WRITE_LIFE_NONE; 3723 } 3724 case NODE: 3725 switch (temp) { 3726 case WARM: 3727 return WRITE_LIFE_MEDIUM; 3728 case HOT: 3729 return WRITE_LIFE_NONE; 3730 case COLD: 3731 return WRITE_LIFE_LONG; 3732 default: 3733 return WRITE_LIFE_NONE; 3734 } 3735 case META: 3736 return WRITE_LIFE_NONE; 3737 default: 3738 return WRITE_LIFE_NONE; 3739 } 3740 } 3741 3742 u8 f2fs_io_type_to_write_stream(struct block_device *bdev, 3743 enum page_type type, enum temp_type temp) 3744 { 3745 unsigned short nr = bdev_max_write_streams(bdev); 3746 3747 if (type != DATA || !nr) 3748 return 0; 3749 if (nr < NR_TEMP_TYPE) 3750 return temp == COLD ? nr : HOT + 1; 3751 3752 return temp + 1; 3753 } 3754 3755 static int __get_segment_type_2(struct f2fs_io_info *fio) 3756 { 3757 if (fio->type == DATA) 3758 return CURSEG_HOT_DATA; 3759 else 3760 return CURSEG_HOT_NODE; 3761 } 3762 3763 static int __get_segment_type_4(struct f2fs_io_info *fio) 3764 { 3765 if (fio->type == DATA) { 3766 struct inode *inode = fio_inode(fio); 3767 3768 if (S_ISDIR(inode->i_mode)) 3769 return CURSEG_HOT_DATA; 3770 else 3771 return CURSEG_COLD_DATA; 3772 } else { 3773 if (IS_DNODE(fio->folio) && is_cold_node(fio->folio)) 3774 return CURSEG_WARM_NODE; 3775 else 3776 return CURSEG_COLD_NODE; 3777 } 3778 } 3779 3780 static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs) 3781 { 3782 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3783 struct extent_info ei = {}; 3784 3785 if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) { 3786 if (!ei.age) 3787 return NO_CHECK_TYPE; 3788 if (ei.age <= sbi->hot_data_age_threshold) 3789 return CURSEG_HOT_DATA; 3790 if (ei.age <= sbi->warm_data_age_threshold) 3791 return CURSEG_WARM_DATA; 3792 return CURSEG_COLD_DATA; 3793 } 3794 return NO_CHECK_TYPE; 3795 } 3796 3797 static int __get_segment_type_6(struct f2fs_io_info *fio) 3798 { 3799 if (fio->type == DATA) { 3800 struct inode *inode = fio_inode(fio); 3801 int type; 3802 3803 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) 3804 return CURSEG_COLD_DATA_PINNED; 3805 3806 if (page_private_gcing(fio->page)) { 3807 if (fio->sbi->am.atgc_enabled && 3808 (fio->io_type == FS_DATA_IO) && 3809 (fio->sbi->gc_mode != GC_URGENT_HIGH) && 3810 __is_valid_data_blkaddr(fio->old_blkaddr) && 3811 !is_inode_flag_set(inode, FI_OPU_WRITE)) 3812 return CURSEG_ALL_DATA_ATGC; 3813 else 3814 return CURSEG_COLD_DATA; 3815 } 3816 if (file_is_cold(inode) || f2fs_need_compress_data(inode)) 3817 return CURSEG_COLD_DATA; 3818 3819 type = __get_age_segment_type(inode, fio->folio->index); 3820 if (type != NO_CHECK_TYPE) 3821 return type; 3822 3823 if (file_is_hot(inode) || 3824 is_inode_flag_set(inode, FI_HOT_DATA) || 3825 f2fs_is_cow_file(inode) || 3826 is_inode_flag_set(inode, FI_NEED_IPU)) 3827 return CURSEG_HOT_DATA; 3828 return f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), 3829 inode->i_write_hint); 3830 } else { 3831 if (IS_DNODE(fio->folio)) 3832 return is_cold_node(fio->folio) ? CURSEG_WARM_NODE : 3833 CURSEG_HOT_NODE; 3834 return CURSEG_COLD_NODE; 3835 } 3836 } 3837 3838 enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi, 3839 enum log_type type) 3840 { 3841 struct curseg_info *curseg = CURSEG_I(sbi, type); 3842 enum temp_type temp = COLD; 3843 3844 switch (curseg->seg_type) { 3845 case CURSEG_HOT_NODE: 3846 case CURSEG_HOT_DATA: 3847 temp = HOT; 3848 break; 3849 case CURSEG_WARM_NODE: 3850 case CURSEG_WARM_DATA: 3851 temp = WARM; 3852 break; 3853 case CURSEG_COLD_NODE: 3854 case CURSEG_COLD_DATA: 3855 temp = COLD; 3856 break; 3857 default: 3858 f2fs_bug_on(sbi, 1); 3859 } 3860 3861 return temp; 3862 } 3863 3864 static int __get_segment_type(struct f2fs_io_info *fio) 3865 { 3866 enum log_type type = CURSEG_HOT_DATA; 3867 3868 switch (F2FS_OPTION(fio->sbi).active_logs) { 3869 case 2: 3870 type = __get_segment_type_2(fio); 3871 break; 3872 case 4: 3873 type = __get_segment_type_4(fio); 3874 break; 3875 case 6: 3876 type = __get_segment_type_6(fio); 3877 break; 3878 default: 3879 f2fs_bug_on(fio->sbi, true); 3880 } 3881 3882 fio->temp = f2fs_get_segment_temp(fio->sbi, type); 3883 3884 return type; 3885 } 3886 3887 static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi, 3888 struct curseg_info *seg) 3889 { 3890 /* To allocate block chunks in different sizes, use random number */ 3891 if (--seg->fragment_remained_chunk > 0) 3892 return; 3893 3894 seg->fragment_remained_chunk = 3895 get_random_u32_inclusive(1, sbi->max_fragment_chunk); 3896 seg->next_blkoff += 3897 get_random_u32_inclusive(1, sbi->max_fragment_hole); 3898 } 3899 3900 int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, 3901 block_t old_blkaddr, block_t *new_blkaddr, 3902 struct f2fs_summary *sum, int type, 3903 struct f2fs_io_info *fio) 3904 { 3905 struct sit_info *sit_i = SIT_I(sbi); 3906 struct curseg_info *curseg = CURSEG_I(sbi, type); 3907 unsigned long long old_mtime; 3908 bool from_gc = (type == CURSEG_ALL_DATA_ATGC); 3909 struct seg_entry *se = NULL; 3910 bool segment_full = false; 3911 int ret = 0; 3912 3913 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3914 3915 mutex_lock(&curseg->curseg_mutex); 3916 down_write(&sit_i->sentry_lock); 3917 3918 if (curseg->segno == NULL_SEGNO) { 3919 ret = -ENOSPC; 3920 goto out_err; 3921 } 3922 3923 if (from_gc) { 3924 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO); 3925 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr)); 3926 sanity_check_seg_type(sbi, se->type); 3927 f2fs_bug_on(sbi, IS_NODESEG(se->type)); 3928 } 3929 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 3930 3931 f2fs_bug_on(sbi, curseg->next_blkoff >= BLKS_PER_SEG(sbi)); 3932 3933 f2fs_wait_discard_bio(sbi, *new_blkaddr); 3934 3935 sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; 3936 if (curseg->alloc_type == SSR) { 3937 curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); 3938 } else { 3939 curseg->next_blkoff++; 3940 if (f2fs_need_rand_blk(sbi, type)) 3941 f2fs_randomize_chunk(sbi, curseg); 3942 } 3943 if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno)) 3944 segment_full = true; 3945 stat_inc_block_count(sbi, curseg); 3946 3947 if (from_gc) { 3948 old_mtime = get_segment_mtime(sbi, old_blkaddr); 3949 } else { 3950 update_segment_mtime(sbi, old_blkaddr, 0); 3951 old_mtime = 0; 3952 } 3953 update_segment_mtime(sbi, *new_blkaddr, old_mtime); 3954 3955 /* 3956 * SIT information should be updated before segment allocation, 3957 * since SSR needs latest valid block information. 3958 */ 3959 update_sit_entry(sbi, *new_blkaddr, 1); 3960 update_sit_entry(sbi, old_blkaddr, -1); 3961 3962 /* 3963 * If the current segment is full, flush it out and replace it with a 3964 * new segment. 3965 */ 3966 if (segment_full) { 3967 if (type == CURSEG_COLD_DATA_PINNED && 3968 !((curseg->segno + 1) % sbi->segs_per_sec)) { 3969 write_sum_page(sbi, curseg->sum_blk, curseg->segno); 3970 reset_curseg_fields(curseg); 3971 goto skip_new_segment; 3972 } 3973 3974 if (from_gc) { 3975 ret = get_atssr_segment(sbi, type, se->type, 3976 AT_SSR, se->mtime); 3977 } else { 3978 if (need_new_seg(sbi, type)) 3979 ret = new_curseg(sbi, type, false); 3980 else 3981 ret = change_curseg(sbi, type); 3982 stat_inc_seg_type(sbi, curseg); 3983 } 3984 3985 if (ret) 3986 goto out_err; 3987 } 3988 3989 skip_new_segment: 3990 /* 3991 * segment dirty status should be updated after segment allocation, 3992 * so we just need to update status only one time after previous 3993 * segment being closed. 3994 */ 3995 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 3996 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); 3997 3998 if (IS_DATASEG(curseg->seg_type)) { 3999 unsigned long long new_val; 4000 4001 new_val = atomic64_inc_return(&sbi->allocated_data_blocks); 4002 if (unlikely(new_val == ULLONG_MAX)) 4003 atomic64_set(&sbi->allocated_data_blocks, 0); 4004 } 4005 4006 up_write(&sit_i->sentry_lock); 4007 4008 if (folio && IS_NODESEG(curseg->seg_type)) { 4009 fill_node_footer_blkaddr(folio, NEXT_FREE_BLKADDR(sbi, curseg)); 4010 4011 f2fs_inode_chksum_set(sbi, folio); 4012 } 4013 4014 if (fio) { 4015 struct f2fs_bio_info *io; 4016 4017 INIT_LIST_HEAD(&fio->list); 4018 fio->in_list = 1; 4019 io = sbi->write_io[fio->type] + fio->temp; 4020 spin_lock(&io->io_lock); 4021 list_add_tail(&fio->list, &io->io_list); 4022 spin_unlock(&io->io_lock); 4023 } 4024 4025 mutex_unlock(&curseg->curseg_mutex); 4026 f2fs_up_read(&SM_I(sbi)->curseg_lock); 4027 return 0; 4028 4029 out_err: 4030 *new_blkaddr = NULL_ADDR; 4031 up_write(&sit_i->sentry_lock); 4032 mutex_unlock(&curseg->curseg_mutex); 4033 f2fs_up_read(&SM_I(sbi)->curseg_lock); 4034 return ret; 4035 } 4036 4037 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino, 4038 block_t blkaddr, unsigned int blkcnt) 4039 { 4040 if (!f2fs_is_multi_device(sbi)) 4041 return; 4042 4043 while (1) { 4044 unsigned int devidx = f2fs_target_device_index(sbi, blkaddr); 4045 unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1; 4046 4047 /* update device state for fsync */ 4048 f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO); 4049 4050 /* update device state for checkpoint */ 4051 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { 4052 spin_lock(&sbi->dev_lock); 4053 f2fs_set_bit(devidx, (char *)&sbi->dirty_device); 4054 spin_unlock(&sbi->dev_lock); 4055 } 4056 4057 if (blkcnt <= blks) 4058 break; 4059 blkcnt -= blks; 4060 blkaddr += blks; 4061 } 4062 } 4063 4064 static int log_type_to_seg_type(enum log_type type) 4065 { 4066 int seg_type = CURSEG_COLD_DATA; 4067 4068 switch (type) { 4069 case CURSEG_HOT_DATA: 4070 case CURSEG_WARM_DATA: 4071 case CURSEG_COLD_DATA: 4072 case CURSEG_HOT_NODE: 4073 case CURSEG_WARM_NODE: 4074 case CURSEG_COLD_NODE: 4075 seg_type = (int)type; 4076 break; 4077 case CURSEG_COLD_DATA_PINNED: 4078 case CURSEG_ALL_DATA_ATGC: 4079 seg_type = CURSEG_COLD_DATA; 4080 break; 4081 default: 4082 break; 4083 } 4084 return seg_type; 4085 } 4086 4087 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) 4088 { 4089 struct folio *folio = fio->folio; 4090 enum log_type type = __get_segment_type(fio); 4091 int seg_type = log_type_to_seg_type(type); 4092 bool keep_order = (f2fs_lfs_mode(fio->sbi) && 4093 seg_type == CURSEG_COLD_DATA); 4094 int err; 4095 4096 if (keep_order) 4097 f2fs_down_read(&fio->sbi->io_order_lock); 4098 4099 err = f2fs_allocate_data_block(fio->sbi, folio, fio->old_blkaddr, 4100 &fio->new_blkaddr, sum, type, fio); 4101 if (unlikely(err)) { 4102 f2fs_err_ratelimited(fio->sbi, 4103 "%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d", 4104 __func__, fio->ino, folio->index, type, 4105 fio->old_blkaddr, fio->new_blkaddr, err); 4106 folio_end_writeback(folio); 4107 if (f2fs_in_warm_node_list(folio)) 4108 f2fs_del_fsync_node_entry(fio->sbi, folio); 4109 f2fs_bug_on(fio->sbi, !is_set_ckpt_flags(fio->sbi, 4110 CP_ERROR_FLAG)); 4111 goto out; 4112 } 4113 4114 f2fs_bug_on(fio->sbi, !f2fs_is_valid_blkaddr_raw(fio->sbi, 4115 fio->new_blkaddr, DATA_GENERIC_ENHANCE)); 4116 4117 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) 4118 f2fs_invalidate_internal_cache(fio->sbi, fio->old_blkaddr, 1); 4119 4120 /* writeout dirty page into bdev */ 4121 f2fs_submit_page_write(fio); 4122 4123 f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1); 4124 out: 4125 if (keep_order) 4126 f2fs_up_read(&fio->sbi->io_order_lock); 4127 } 4128 4129 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio, 4130 enum iostat_type io_type) 4131 { 4132 struct f2fs_io_info fio = { 4133 .sbi = sbi, 4134 .type = META, 4135 .temp = HOT, 4136 .op = REQ_OP_WRITE, 4137 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO, 4138 .old_blkaddr = folio->index, 4139 .new_blkaddr = folio->index, 4140 .folio = folio, 4141 .encrypted_page = NULL, 4142 .in_list = 0, 4143 }; 4144 4145 if (unlikely(folio->index >= MAIN_BLKADDR(sbi))) 4146 fio.op_flags &= ~REQ_META; 4147 4148 folio_start_writeback(folio); 4149 f2fs_submit_page_write(&fio); 4150 4151 stat_inc_meta_count(sbi, folio->index); 4152 f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE); 4153 } 4154 4155 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio) 4156 { 4157 struct f2fs_summary sum; 4158 4159 set_summary(&sum, nid, 0, 0); 4160 do_write_page(&sum, fio); 4161 4162 f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE); 4163 } 4164 4165 void f2fs_outplace_write_data(struct dnode_of_data *dn, 4166 struct f2fs_io_info *fio) 4167 { 4168 struct f2fs_sb_info *sbi = fio->sbi; 4169 struct f2fs_summary sum; 4170 4171 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); 4172 if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO) 4173 f2fs_update_age_extent_cache(dn); 4174 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version); 4175 do_write_page(&sum, fio); 4176 f2fs_update_data_blkaddr(dn, fio->new_blkaddr); 4177 4178 f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE); 4179 } 4180 4181 int f2fs_inplace_write_data(struct f2fs_io_info *fio) 4182 { 4183 int err; 4184 struct f2fs_sb_info *sbi = fio->sbi; 4185 unsigned int segno; 4186 4187 fio->new_blkaddr = fio->old_blkaddr; 4188 /* i/o temperature is needed for passing down write hints */ 4189 __get_segment_type(fio); 4190 4191 segno = GET_SEGNO(sbi, fio->new_blkaddr); 4192 4193 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) { 4194 set_sbi_flag(sbi, SBI_NEED_FSCK); 4195 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.", 4196 __func__, segno); 4197 err = -EFSCORRUPTED; 4198 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); 4199 goto drop_bio; 4200 } 4201 4202 if (f2fs_cp_error(sbi)) { 4203 err = -EIO; 4204 goto drop_bio; 4205 } 4206 4207 if (fio->meta_gc) 4208 f2fs_truncate_meta_inode_pages(sbi, fio->new_blkaddr, 1); 4209 4210 stat_inc_inplace_blocks(fio->sbi); 4211 4212 if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi)) 4213 err = f2fs_merge_page_bio(fio); 4214 else 4215 err = f2fs_submit_page_bio(fio); 4216 if (!err) { 4217 f2fs_update_device_state(fio->sbi, fio->ino, 4218 fio->new_blkaddr, 1); 4219 f2fs_update_iostat(fio->sbi, fio_inode(fio), 4220 fio->io_type, F2FS_BLKSIZE); 4221 } 4222 4223 return err; 4224 drop_bio: 4225 if (fio->bio && *(fio->bio)) { 4226 struct bio *bio = *(fio->bio); 4227 4228 bio->bi_status = BLK_STS_IOERR; 4229 bio_endio(bio); 4230 *(fio->bio) = NULL; 4231 } 4232 return err; 4233 } 4234 4235 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, 4236 unsigned int segno) 4237 { 4238 int i; 4239 4240 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { 4241 if (CURSEG_I(sbi, i)->segno == segno) 4242 break; 4243 } 4244 return i; 4245 } 4246 4247 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, 4248 block_t old_blkaddr, block_t new_blkaddr, 4249 bool recover_curseg, bool recover_newaddr, 4250 bool from_gc) 4251 { 4252 struct sit_info *sit_i = SIT_I(sbi); 4253 struct curseg_info *curseg; 4254 unsigned int segno, old_cursegno; 4255 struct seg_entry *se; 4256 int type; 4257 unsigned short old_blkoff; 4258 unsigned char old_alloc_type; 4259 4260 segno = GET_SEGNO(sbi, new_blkaddr); 4261 se = get_seg_entry(sbi, segno); 4262 type = se->type; 4263 4264 f2fs_down_write(&SM_I(sbi)->curseg_lock); 4265 4266 if (!recover_curseg) { 4267 /* for recovery flow */ 4268 if (se->valid_blocks == 0 && !is_curseg(sbi, segno)) { 4269 if (old_blkaddr == NULL_ADDR) 4270 type = CURSEG_COLD_DATA; 4271 else 4272 type = CURSEG_WARM_DATA; 4273 } 4274 } else { 4275 if (is_curseg(sbi, segno)) { 4276 /* se->type is volatile as SSR allocation */ 4277 type = __f2fs_get_curseg(sbi, segno); 4278 f2fs_bug_on(sbi, type == NO_CHECK_TYPE); 4279 } else { 4280 type = CURSEG_WARM_DATA; 4281 } 4282 } 4283 4284 curseg = CURSEG_I(sbi, type); 4285 f2fs_bug_on(sbi, !IS_DATASEG(curseg->seg_type)); 4286 4287 mutex_lock(&curseg->curseg_mutex); 4288 down_write(&sit_i->sentry_lock); 4289 4290 old_cursegno = curseg->segno; 4291 old_blkoff = curseg->next_blkoff; 4292 old_alloc_type = curseg->alloc_type; 4293 4294 /* change the current segment */ 4295 if (segno != curseg->segno) { 4296 curseg->next_segno = segno; 4297 if (change_curseg(sbi, type)) 4298 goto out_unlock; 4299 } 4300 4301 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); 4302 sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; 4303 4304 if (!recover_curseg || recover_newaddr) { 4305 if (!from_gc) 4306 update_segment_mtime(sbi, new_blkaddr, 0); 4307 update_sit_entry(sbi, new_blkaddr, 1); 4308 } 4309 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) { 4310 f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1); 4311 if (!from_gc) 4312 update_segment_mtime(sbi, old_blkaddr, 0); 4313 update_sit_entry(sbi, old_blkaddr, -1); 4314 } 4315 4316 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 4317 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); 4318 4319 locate_dirty_segment(sbi, old_cursegno); 4320 4321 if (recover_curseg) { 4322 if (old_cursegno != curseg->segno) { 4323 curseg->next_segno = old_cursegno; 4324 if (change_curseg(sbi, type)) 4325 goto out_unlock; 4326 } 4327 curseg->next_blkoff = old_blkoff; 4328 curseg->alloc_type = old_alloc_type; 4329 } 4330 4331 out_unlock: 4332 up_write(&sit_i->sentry_lock); 4333 mutex_unlock(&curseg->curseg_mutex); 4334 f2fs_up_write(&SM_I(sbi)->curseg_lock); 4335 } 4336 4337 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, 4338 block_t old_addr, block_t new_addr, 4339 unsigned char version, bool recover_curseg, 4340 bool recover_newaddr) 4341 { 4342 struct f2fs_summary sum; 4343 4344 set_summary(&sum, dn->nid, dn->ofs_in_node, version); 4345 4346 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr, 4347 recover_curseg, recover_newaddr, false); 4348 4349 f2fs_update_data_blkaddr(dn, new_addr); 4350 } 4351 4352 void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type, 4353 bool ordered, bool locked) 4354 { 4355 if (folio_test_writeback(folio)) { 4356 struct f2fs_sb_info *sbi = F2FS_F_SB(folio); 4357 4358 /* submit cached LFS IO */ 4359 f2fs_submit_merged_write_folio(sbi, folio, type); 4360 /* submit cached IPU IO */ 4361 f2fs_submit_merged_ipu_write(sbi, NULL, folio); 4362 if (ordered) { 4363 folio_wait_writeback(folio); 4364 f2fs_bug_on(sbi, locked && folio_test_writeback(folio)); 4365 } else { 4366 folio_wait_stable(folio); 4367 } 4368 } 4369 } 4370 4371 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr) 4372 { 4373 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 4374 struct folio *cfolio; 4375 4376 if (!f2fs_meta_inode_gc_required(inode)) 4377 return; 4378 4379 if (!__is_valid_data_blkaddr(blkaddr)) 4380 return; 4381 4382 cfolio = filemap_lock_folio(META_MAPPING(sbi), blkaddr); 4383 if (!IS_ERR(cfolio)) { 4384 f2fs_folio_wait_writeback(cfolio, DATA, true, true); 4385 f2fs_folio_put(cfolio, true); 4386 } 4387 } 4388 4389 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, 4390 block_t len) 4391 { 4392 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 4393 block_t i; 4394 4395 if (!f2fs_meta_inode_gc_required(inode)) 4396 return; 4397 4398 for (i = 0; i < len; i++) 4399 f2fs_wait_on_block_writeback(inode, blkaddr + i); 4400 4401 f2fs_truncate_meta_inode_pages(sbi, blkaddr, len); 4402 } 4403 4404 static int read_compacted_summaries(struct f2fs_sb_info *sbi) 4405 { 4406 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 4407 struct curseg_info *seg_i; 4408 unsigned char *kaddr; 4409 struct folio *folio; 4410 block_t start; 4411 int i, j, offset; 4412 4413 start = start_sum_block(sbi); 4414 4415 folio = f2fs_get_meta_folio(sbi, start++); 4416 if (IS_ERR(folio)) 4417 return PTR_ERR(folio); 4418 kaddr = folio_address(folio); 4419 4420 /* Step 1: restore nat cache */ 4421 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 4422 memcpy(seg_i->journal, kaddr, sbi->sum_journal_size); 4423 4424 /* Step 2: restore sit cache */ 4425 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 4426 memcpy(seg_i->journal, kaddr + sbi->sum_journal_size, sbi->sum_journal_size); 4427 offset = 2 * sbi->sum_journal_size; 4428 4429 /* Step 3: restore summary entries */ 4430 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 4431 unsigned short blk_off; 4432 unsigned int segno; 4433 4434 seg_i = CURSEG_I(sbi, i); 4435 segno = le32_to_cpu(ckpt->cur_data_segno[i]); 4436 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); 4437 seg_i->next_segno = segno; 4438 reset_curseg(sbi, i, 0); 4439 seg_i->alloc_type = ckpt->alloc_type[i]; 4440 seg_i->next_blkoff = blk_off; 4441 4442 if (seg_i->alloc_type == SSR) 4443 blk_off = BLKS_PER_SEG(sbi); 4444 4445 for (j = 0; j < blk_off; j++) { 4446 struct f2fs_summary *s; 4447 4448 s = (struct f2fs_summary *)(kaddr + offset); 4449 sum_entries(seg_i->sum_blk)[j] = *s; 4450 offset += SUMMARY_SIZE; 4451 if (offset + SUMMARY_SIZE <= sbi->blocksize - 4452 SUM_FOOTER_SIZE) 4453 continue; 4454 4455 f2fs_folio_put(folio, true); 4456 4457 folio = f2fs_get_meta_folio(sbi, start++); 4458 if (IS_ERR(folio)) 4459 return PTR_ERR(folio); 4460 kaddr = folio_address(folio); 4461 offset = 0; 4462 } 4463 } 4464 f2fs_folio_put(folio, true); 4465 return 0; 4466 } 4467 4468 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) 4469 { 4470 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 4471 struct f2fs_summary_block *sum; 4472 struct curseg_info *curseg; 4473 struct folio *new; 4474 unsigned short blk_off; 4475 unsigned int segno = 0; 4476 block_t blk_addr = 0; 4477 int err = 0; 4478 4479 /* get segment number and block addr */ 4480 if (IS_DATASEG(type)) { 4481 segno = le32_to_cpu(ckpt->cur_data_segno[type]); 4482 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - 4483 CURSEG_HOT_DATA]); 4484 if (__exist_node_summaries(sbi)) 4485 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type); 4486 else 4487 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); 4488 } else { 4489 segno = le32_to_cpu(ckpt->cur_node_segno[type - 4490 CURSEG_HOT_NODE]); 4491 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - 4492 CURSEG_HOT_NODE]); 4493 if (__exist_node_summaries(sbi)) 4494 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, 4495 type - CURSEG_HOT_NODE); 4496 else 4497 blk_addr = GET_SUM_BLOCK(sbi, segno); 4498 } 4499 4500 new = f2fs_get_meta_folio(sbi, blk_addr); 4501 if (IS_ERR(new)) 4502 return PTR_ERR(new); 4503 sum = folio_address(new); 4504 4505 if (IS_NODESEG(type)) { 4506 if (__exist_node_summaries(sbi)) { 4507 struct f2fs_summary *ns = sum_entries(sum); 4508 int i; 4509 4510 for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) { 4511 ns->version = 0; 4512 ns->ofs_in_node = 0; 4513 } 4514 } else { 4515 err = f2fs_restore_node_summary(sbi, segno, sum); 4516 if (err) 4517 goto out; 4518 } 4519 } 4520 4521 /* set uncompleted segment to curseg */ 4522 curseg = CURSEG_I(sbi, type); 4523 mutex_lock(&curseg->curseg_mutex); 4524 4525 /* update journal info */ 4526 down_write(&curseg->journal_rwsem); 4527 memcpy(curseg->journal, sum_journal(sbi, sum), sbi->sum_journal_size); 4528 up_write(&curseg->journal_rwsem); 4529 4530 memcpy(sum_entries(curseg->sum_blk), sum_entries(sum), 4531 sbi->sum_entry_size); 4532 memcpy(sum_footer(sbi, curseg->sum_blk), sum_footer(sbi, sum), 4533 SUM_FOOTER_SIZE); 4534 curseg->next_segno = segno; 4535 reset_curseg(sbi, type, 0); 4536 curseg->alloc_type = ckpt->alloc_type[type]; 4537 curseg->next_blkoff = blk_off; 4538 mutex_unlock(&curseg->curseg_mutex); 4539 out: 4540 f2fs_folio_put(new, true); 4541 return err; 4542 } 4543 4544 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) 4545 { 4546 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; 4547 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; 4548 int type = CURSEG_HOT_DATA; 4549 int err; 4550 4551 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { 4552 int npages = f2fs_npages_for_summary_flush(sbi, true); 4553 4554 if (npages >= 2) 4555 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages, 4556 META_CP, true); 4557 4558 /* restore for compacted data summary */ 4559 err = read_compacted_summaries(sbi); 4560 if (err) 4561 return err; 4562 type = CURSEG_HOT_NODE; 4563 } 4564 4565 if (__exist_node_summaries(sbi)) 4566 f2fs_ra_meta_pages(sbi, 4567 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type), 4568 NR_CURSEG_PERSIST_TYPE - type, META_CP, true); 4569 4570 for (; type <= CURSEG_COLD_NODE; type++) { 4571 err = read_normal_summaries(sbi, type); 4572 if (err) 4573 return err; 4574 } 4575 4576 /* sanity check for summary blocks */ 4577 if (nats_in_cursum(nat_j) > sbi->nat_journal_entries || 4578 sits_in_cursum(sit_j) > sbi->sit_journal_entries) { 4579 f2fs_err(sbi, "invalid journal entries nats %u sits %u", 4580 nats_in_cursum(nat_j), sits_in_cursum(sit_j)); 4581 return -EINVAL; 4582 } 4583 4584 return 0; 4585 } 4586 4587 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) 4588 { 4589 struct folio *folio; 4590 unsigned char *kaddr; 4591 struct f2fs_summary *summary; 4592 struct curseg_info *seg_i; 4593 int written_size = 0; 4594 int i, j; 4595 4596 folio = f2fs_grab_meta_folio(sbi, blkaddr++); 4597 kaddr = folio_address(folio); 4598 memset(kaddr, 0, PAGE_SIZE); 4599 4600 /* Step 1: write nat cache */ 4601 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 4602 memcpy(kaddr, seg_i->journal, sbi->sum_journal_size); 4603 written_size += sbi->sum_journal_size; 4604 4605 /* Step 2: write sit cache */ 4606 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 4607 memcpy(kaddr + written_size, seg_i->journal, sbi->sum_journal_size); 4608 written_size += sbi->sum_journal_size; 4609 4610 /* Step 3: write summary entries */ 4611 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 4612 seg_i = CURSEG_I(sbi, i); 4613 for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) { 4614 if (!folio) { 4615 folio = f2fs_grab_meta_folio(sbi, blkaddr++); 4616 kaddr = folio_address(folio); 4617 memset(kaddr, 0, PAGE_SIZE); 4618 written_size = 0; 4619 } 4620 summary = (struct f2fs_summary *)(kaddr + written_size); 4621 *summary = sum_entries(seg_i->sum_blk)[j]; 4622 written_size += SUMMARY_SIZE; 4623 4624 if (written_size + SUMMARY_SIZE <= sbi->blocksize - 4625 SUM_FOOTER_SIZE) 4626 continue; 4627 4628 folio_mark_dirty(folio); 4629 f2fs_folio_put(folio, true); 4630 folio = NULL; 4631 } 4632 } 4633 if (folio) { 4634 folio_mark_dirty(folio); 4635 f2fs_folio_put(folio, true); 4636 } 4637 } 4638 4639 static void write_normal_summaries(struct f2fs_sb_info *sbi, 4640 block_t blkaddr, int type) 4641 { 4642 int i, end; 4643 4644 if (IS_DATASEG(type)) 4645 end = type + NR_CURSEG_DATA_TYPE; 4646 else 4647 end = type + NR_CURSEG_NODE_TYPE; 4648 4649 for (i = type; i < end; i++) 4650 write_current_sum_page(sbi, i, blkaddr + (i - type)); 4651 } 4652 4653 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 4654 { 4655 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) 4656 write_compacted_summaries(sbi, start_blk); 4657 else 4658 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); 4659 } 4660 4661 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 4662 { 4663 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); 4664 } 4665 4666 int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, 4667 struct f2fs_journal *journal, int type, 4668 unsigned int val, int alloc) 4669 { 4670 int i; 4671 4672 if (type == NAT_JOURNAL) { 4673 for (i = 0; i < nats_in_cursum(journal); i++) { 4674 if (le32_to_cpu(nid_in_journal(journal, i)) == val) 4675 return i; 4676 } 4677 if (alloc && __has_cursum_space(sbi, journal, 1, NAT_JOURNAL)) 4678 return update_nats_in_cursum(journal, 1); 4679 } else if (type == SIT_JOURNAL) { 4680 for (i = 0; i < sits_in_cursum(journal); i++) 4681 if (le32_to_cpu(segno_in_journal(journal, i)) == val) 4682 return i; 4683 if (alloc && __has_cursum_space(sbi, journal, 1, SIT_JOURNAL)) 4684 return update_sits_in_cursum(journal, 1); 4685 } 4686 return -1; 4687 } 4688 4689 static struct folio *get_current_sit_folio(struct f2fs_sb_info *sbi, 4690 unsigned int segno) 4691 { 4692 return f2fs_get_meta_folio(sbi, current_sit_addr(sbi, segno)); 4693 } 4694 4695 static struct folio *get_next_sit_folio(struct f2fs_sb_info *sbi, 4696 unsigned int start) 4697 { 4698 struct sit_info *sit_i = SIT_I(sbi); 4699 struct folio *folio; 4700 pgoff_t src_off, dst_off; 4701 4702 src_off = current_sit_addr(sbi, start); 4703 dst_off = next_sit_addr(sbi, src_off); 4704 4705 folio = f2fs_grab_meta_folio(sbi, dst_off); 4706 seg_info_to_sit_folio(sbi, folio, start); 4707 4708 folio_mark_dirty(folio); 4709 set_to_next_sit(sit_i, start); 4710 4711 return folio; 4712 } 4713 4714 static struct sit_entry_set *grab_sit_entry_set(void) 4715 { 4716 struct sit_entry_set *ses = 4717 f2fs_kmem_cache_alloc(sit_entry_set_slab, 4718 GFP_NOFS, true, NULL); 4719 4720 ses->entry_cnt = 0; 4721 INIT_LIST_HEAD(&ses->set_list); 4722 return ses; 4723 } 4724 4725 static void release_sit_entry_set(struct sit_entry_set *ses) 4726 { 4727 list_del(&ses->set_list); 4728 kmem_cache_free(sit_entry_set_slab, ses); 4729 } 4730 4731 static void adjust_sit_entry_set(struct sit_entry_set *ses, 4732 struct list_head *head) 4733 { 4734 struct sit_entry_set *next = ses; 4735 4736 if (list_is_last(&ses->set_list, head)) 4737 return; 4738 4739 list_for_each_entry_continue(next, head, set_list) 4740 if (ses->entry_cnt <= next->entry_cnt) { 4741 list_move_tail(&ses->set_list, &next->set_list); 4742 return; 4743 } 4744 4745 list_move_tail(&ses->set_list, head); 4746 } 4747 4748 static void add_sit_entry(unsigned int segno, struct list_head *head) 4749 { 4750 struct sit_entry_set *ses; 4751 unsigned int start_segno = START_SEGNO(segno); 4752 4753 list_for_each_entry(ses, head, set_list) { 4754 if (ses->start_segno == start_segno) { 4755 ses->entry_cnt++; 4756 adjust_sit_entry_set(ses, head); 4757 return; 4758 } 4759 } 4760 4761 ses = grab_sit_entry_set(); 4762 4763 ses->start_segno = start_segno; 4764 ses->entry_cnt++; 4765 list_add(&ses->set_list, head); 4766 } 4767 4768 static void add_sits_in_set(struct f2fs_sb_info *sbi) 4769 { 4770 struct f2fs_sm_info *sm_info = SM_I(sbi); 4771 struct list_head *set_list = &sm_info->sit_entry_set; 4772 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; 4773 unsigned int segno; 4774 4775 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) 4776 add_sit_entry(segno, set_list); 4777 } 4778 4779 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) 4780 { 4781 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4782 struct f2fs_journal *journal = curseg->journal; 4783 int i; 4784 4785 down_write(&curseg->journal_rwsem); 4786 for (i = 0; i < sits_in_cursum(journal); i++) { 4787 unsigned int segno; 4788 bool dirtied; 4789 4790 segno = le32_to_cpu(segno_in_journal(journal, i)); 4791 dirtied = __mark_sit_entry_dirty(sbi, segno); 4792 4793 if (!dirtied) 4794 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); 4795 } 4796 update_sits_in_cursum(journal, -i); 4797 up_write(&curseg->journal_rwsem); 4798 } 4799 4800 /* 4801 * CP calls this function, which flushes SIT entries including sit_journal, 4802 * and moves prefree segs to free segs. 4803 */ 4804 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) 4805 { 4806 struct sit_info *sit_i = SIT_I(sbi); 4807 unsigned long *bitmap = sit_i->dirty_sentries_bitmap; 4808 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4809 struct f2fs_journal *journal = curseg->journal; 4810 struct sit_entry_set *ses, *tmp; 4811 struct list_head *head = &SM_I(sbi)->sit_entry_set; 4812 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS); 4813 struct seg_entry *se; 4814 4815 down_write(&sit_i->sentry_lock); 4816 4817 if (!sit_i->dirty_sentries) 4818 goto out; 4819 4820 /* 4821 * add and account sit entries of dirty bitmap in sit entry 4822 * set temporarily 4823 */ 4824 add_sits_in_set(sbi); 4825 4826 /* 4827 * if there are no enough space in journal to store dirty sit 4828 * entries, remove all entries from journal and add and account 4829 * them in sit entry set. 4830 */ 4831 if (!__has_cursum_space(sbi, journal, 4832 sit_i->dirty_sentries, SIT_JOURNAL) || !to_journal) 4833 remove_sits_in_journal(sbi); 4834 4835 /* 4836 * there are two steps to flush sit entries: 4837 * #1, flush sit entries to journal in current cold data summary block. 4838 * #2, flush sit entries to sit page. 4839 */ 4840 list_for_each_entry_safe(ses, tmp, head, set_list) { 4841 struct folio *folio = NULL; 4842 struct f2fs_sit_block *raw_sit = NULL; 4843 unsigned int start_segno = ses->start_segno; 4844 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, 4845 (unsigned long)MAIN_SEGS(sbi)); 4846 unsigned int segno = start_segno; 4847 4848 if (to_journal && 4849 !__has_cursum_space(sbi, journal, ses->entry_cnt, 4850 SIT_JOURNAL)) 4851 to_journal = false; 4852 4853 if (to_journal) { 4854 down_write(&curseg->journal_rwsem); 4855 } else { 4856 folio = get_next_sit_folio(sbi, start_segno); 4857 raw_sit = folio_address(folio); 4858 } 4859 4860 /* flush dirty sit entries in region of current sit set */ 4861 for_each_set_bit_from(segno, bitmap, end) { 4862 int offset, sit_offset; 4863 4864 se = get_seg_entry(sbi, segno); 4865 4866 /* add discard candidates */ 4867 if (!(cpc->reason & CP_DISCARD)) { 4868 cpc->trim_start = segno; 4869 add_discard_addrs(sbi, cpc, false); 4870 } 4871 4872 if (to_journal) { 4873 offset = f2fs_lookup_journal_in_cursum(sbi, journal, 4874 SIT_JOURNAL, segno, 1); 4875 f2fs_bug_on(sbi, offset < 0); 4876 segno_in_journal(journal, offset) = 4877 cpu_to_le32(segno); 4878 seg_info_to_raw_sit(se, 4879 &sit_in_journal(journal, offset)); 4880 check_block_count(sbi, segno, 4881 &sit_in_journal(journal, offset)); 4882 } else { 4883 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); 4884 seg_info_to_raw_sit(se, 4885 &raw_sit->entries[sit_offset]); 4886 check_block_count(sbi, segno, 4887 &raw_sit->entries[sit_offset]); 4888 } 4889 4890 /* update ckpt_valid_block */ 4891 if (__is_large_section(sbi)) 4892 set_ckpt_valid_blocks(sbi, segno); 4893 4894 __clear_bit(segno, bitmap); 4895 sit_i->dirty_sentries--; 4896 ses->entry_cnt--; 4897 } 4898 4899 if (to_journal) 4900 up_write(&curseg->journal_rwsem); 4901 else 4902 f2fs_folio_put(folio, true); 4903 4904 f2fs_bug_on(sbi, ses->entry_cnt); 4905 release_sit_entry_set(ses); 4906 } 4907 4908 f2fs_bug_on(sbi, !list_empty(head)); 4909 f2fs_bug_on(sbi, sit_i->dirty_sentries); 4910 out: 4911 if (cpc->reason & CP_DISCARD) { 4912 __u64 trim_start = cpc->trim_start; 4913 4914 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) 4915 add_discard_addrs(sbi, cpc, false); 4916 4917 cpc->trim_start = trim_start; 4918 } 4919 up_write(&sit_i->sentry_lock); 4920 4921 set_prefree_as_free_segments(sbi); 4922 } 4923 4924 static int build_sit_info(struct f2fs_sb_info *sbi) 4925 { 4926 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 4927 struct sit_info *sit_i; 4928 unsigned int sit_segs, start; 4929 char *src_bitmap, *bitmap; 4930 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size; 4931 unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0; 4932 4933 /* allocate memory for SIT information */ 4934 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); 4935 if (!sit_i) 4936 return -ENOMEM; 4937 4938 SM_I(sbi)->sit_info = sit_i; 4939 4940 sit_i->sentries = 4941 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry), 4942 MAIN_SEGS(sbi)), 4943 GFP_KERNEL); 4944 if (!sit_i->sentries) 4945 return -ENOMEM; 4946 4947 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 4948 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size, 4949 GFP_KERNEL); 4950 if (!sit_i->dirty_sentries_bitmap) 4951 return -ENOMEM; 4952 4953 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map); 4954 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 4955 if (!sit_i->bitmap) 4956 return -ENOMEM; 4957 4958 bitmap = sit_i->bitmap; 4959 4960 for (start = 0; start < MAIN_SEGS(sbi); start++) { 4961 sit_i->sentries[start].cur_valid_map = bitmap; 4962 bitmap += SIT_VBLOCK_MAP_SIZE; 4963 4964 sit_i->sentries[start].ckpt_valid_map = bitmap; 4965 bitmap += SIT_VBLOCK_MAP_SIZE; 4966 4967 if (discard_map) { 4968 sit_i->sentries[start].discard_map = bitmap; 4969 bitmap += SIT_VBLOCK_MAP_SIZE; 4970 } 4971 } 4972 4973 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 4974 if (!sit_i->tmp_map) 4975 return -ENOMEM; 4976 4977 if (__is_large_section(sbi)) { 4978 sit_i->sec_entries = 4979 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry), 4980 MAIN_SECS(sbi)), 4981 GFP_KERNEL); 4982 if (!sit_i->sec_entries) 4983 return -ENOMEM; 4984 } 4985 4986 /* get information related with SIT */ 4987 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; 4988 4989 /* setup SIT bitmap from ckeckpoint pack */ 4990 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP); 4991 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); 4992 4993 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL); 4994 if (!sit_i->sit_bitmap) 4995 return -ENOMEM; 4996 4997 #ifdef CONFIG_F2FS_CHECK_FS 4998 sit_i->invalid_segmap = f2fs_kvzalloc(sbi, 4999 main_bitmap_size, GFP_KERNEL); 5000 if (!sit_i->invalid_segmap) 5001 return -ENOMEM; 5002 #endif 5003 5004 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); 5005 sit_i->sit_blocks = SEGS_TO_BLKS(sbi, sit_segs); 5006 sit_i->written_valid_blocks = 0; 5007 sit_i->bitmap_size = sit_bitmap_size; 5008 sit_i->dirty_sentries = 0; 5009 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; 5010 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); 5011 sit_i->mounted_time = ktime_get_boottime_seconds(); 5012 init_rwsem(&sit_i->sentry_lock); 5013 return 0; 5014 } 5015 5016 static int build_free_segmap(struct f2fs_sb_info *sbi) 5017 { 5018 struct free_segmap_info *free_i; 5019 unsigned int bitmap_size, sec_bitmap_size; 5020 5021 /* allocate memory for free segmap information */ 5022 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); 5023 if (!free_i) 5024 return -ENOMEM; 5025 5026 SM_I(sbi)->free_info = free_i; 5027 5028 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 5029 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); 5030 if (!free_i->free_segmap) 5031 return -ENOMEM; 5032 5033 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 5034 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); 5035 if (!free_i->free_secmap) 5036 return -ENOMEM; 5037 5038 /* set all segments as dirty temporarily */ 5039 memset(free_i->free_segmap, 0xff, bitmap_size); 5040 memset(free_i->free_secmap, 0xff, sec_bitmap_size); 5041 5042 /* init free segmap information */ 5043 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); 5044 free_i->free_segments = 0; 5045 free_i->free_sections = 0; 5046 spin_lock_init(&free_i->segmap_lock); 5047 return 0; 5048 } 5049 5050 static int build_curseg(struct f2fs_sb_info *sbi) 5051 { 5052 struct curseg_info *array; 5053 int i; 5054 5055 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, 5056 sizeof(*array)), GFP_KERNEL); 5057 if (!array) 5058 return -ENOMEM; 5059 5060 SM_I(sbi)->curseg_array = array; 5061 5062 for (i = 0; i < NO_CHECK_TYPE; i++) { 5063 mutex_init(&array[i].curseg_mutex); 5064 array[i].sum_blk = f2fs_kzalloc(sbi, sbi->sum_blocksize, 5065 GFP_KERNEL); 5066 if (!array[i].sum_blk) 5067 return -ENOMEM; 5068 init_rwsem(&array[i].journal_rwsem); 5069 array[i].journal = f2fs_kzalloc(sbi, 5070 sbi->sum_journal_size, GFP_KERNEL); 5071 if (!array[i].journal) 5072 return -ENOMEM; 5073 array[i].seg_type = log_type_to_seg_type(i); 5074 reset_curseg_fields(&array[i]); 5075 } 5076 return restore_curseg_summaries(sbi); 5077 } 5078 5079 static int build_sit_entries(struct f2fs_sb_info *sbi) 5080 { 5081 struct sit_info *sit_i = SIT_I(sbi); 5082 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 5083 struct f2fs_journal *journal = curseg->journal; 5084 struct seg_entry *se; 5085 struct f2fs_sit_entry sit; 5086 int sit_blk_cnt = SIT_BLK_CNT(sbi); 5087 unsigned int i, start, end; 5088 unsigned int readed, start_blk = 0; 5089 int err = 0; 5090 block_t sit_valid_blocks[2] = {0, 0}; 5091 5092 do { 5093 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS, 5094 META_SIT, true); 5095 5096 start = start_blk * sit_i->sents_per_block; 5097 end = (start_blk + readed) * sit_i->sents_per_block; 5098 5099 for (; start < end && start < MAIN_SEGS(sbi); start++) { 5100 struct f2fs_sit_block *sit_blk; 5101 struct folio *folio; 5102 5103 se = &sit_i->sentries[start]; 5104 folio = get_current_sit_folio(sbi, start); 5105 if (IS_ERR(folio)) 5106 return PTR_ERR(folio); 5107 sit_blk = folio_address(folio); 5108 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; 5109 f2fs_folio_put(folio, true); 5110 5111 err = check_block_count(sbi, start, &sit); 5112 if (err) 5113 return err; 5114 seg_info_from_raw_sit(se, &sit); 5115 5116 if (se->type >= NR_PERSISTENT_LOG) { 5117 f2fs_err(sbi, "Invalid segment type: %u, segno: %u", 5118 se->type, start); 5119 f2fs_handle_error(sbi, 5120 ERROR_INCONSISTENT_SUM_TYPE); 5121 return -EFSCORRUPTED; 5122 } 5123 5124 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks; 5125 5126 if (!f2fs_block_unit_discard(sbi)) 5127 goto init_discard_map_done; 5128 5129 /* build discard map only one time */ 5130 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 5131 memset(se->discard_map, 0xff, 5132 SIT_VBLOCK_MAP_SIZE); 5133 goto init_discard_map_done; 5134 } 5135 memcpy(se->discard_map, se->cur_valid_map, 5136 SIT_VBLOCK_MAP_SIZE); 5137 sbi->discard_blks += BLKS_PER_SEG(sbi) - 5138 se->valid_blocks; 5139 init_discard_map_done: 5140 if (__is_large_section(sbi)) 5141 get_sec_entry(sbi, start)->valid_blocks += 5142 se->valid_blocks; 5143 } 5144 start_blk += readed; 5145 } while (start_blk < sit_blk_cnt); 5146 5147 down_read(&curseg->journal_rwsem); 5148 for (i = 0; i < sits_in_cursum(journal); i++) { 5149 unsigned int old_valid_blocks; 5150 5151 start = le32_to_cpu(segno_in_journal(journal, i)); 5152 if (start >= MAIN_SEGS(sbi)) { 5153 f2fs_err(sbi, "Wrong journal entry on segno %u", 5154 start); 5155 err = -EFSCORRUPTED; 5156 f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL); 5157 break; 5158 } 5159 5160 se = &sit_i->sentries[start]; 5161 sit = sit_in_journal(journal, i); 5162 5163 old_valid_blocks = se->valid_blocks; 5164 5165 sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks; 5166 5167 err = check_block_count(sbi, start, &sit); 5168 if (err) 5169 break; 5170 seg_info_from_raw_sit(se, &sit); 5171 5172 if (se->type >= NR_PERSISTENT_LOG) { 5173 f2fs_err(sbi, "Invalid segment type: %u, segno: %u", 5174 se->type, start); 5175 err = -EFSCORRUPTED; 5176 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); 5177 break; 5178 } 5179 5180 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks; 5181 5182 if (f2fs_block_unit_discard(sbi)) { 5183 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 5184 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE); 5185 } else { 5186 memcpy(se->discard_map, se->cur_valid_map, 5187 SIT_VBLOCK_MAP_SIZE); 5188 sbi->discard_blks += old_valid_blocks; 5189 sbi->discard_blks -= se->valid_blocks; 5190 } 5191 } 5192 5193 if (__is_large_section(sbi)) { 5194 get_sec_entry(sbi, start)->valid_blocks += 5195 se->valid_blocks; 5196 get_sec_entry(sbi, start)->valid_blocks -= 5197 old_valid_blocks; 5198 } 5199 } 5200 up_read(&curseg->journal_rwsem); 5201 5202 /* update ckpt_valid_block */ 5203 if (__is_large_section(sbi)) { 5204 unsigned int segno; 5205 5206 for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) 5207 set_ckpt_valid_blocks(sbi, segno); 5208 } 5209 5210 if (err) 5211 return err; 5212 5213 if (sit_valid_blocks[NODE] != valid_node_count(sbi)) { 5214 f2fs_err(sbi, "SIT is corrupted node# %u vs %u", 5215 sit_valid_blocks[NODE], valid_node_count(sbi)); 5216 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT); 5217 return -EFSCORRUPTED; 5218 } 5219 5220 if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] > 5221 valid_user_blocks(sbi)) { 5222 f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u", 5223 sit_valid_blocks[DATA], sit_valid_blocks[NODE], 5224 valid_user_blocks(sbi)); 5225 f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT); 5226 return -EFSCORRUPTED; 5227 } 5228 5229 return 0; 5230 } 5231 5232 static void init_free_segmap(struct f2fs_sb_info *sbi) 5233 { 5234 unsigned int start; 5235 int type; 5236 struct seg_entry *sentry; 5237 5238 for (start = 0; start < MAIN_SEGS(sbi); start++) { 5239 if (f2fs_usable_blks_in_seg(sbi, start) == 0) 5240 continue; 5241 sentry = get_seg_entry(sbi, start); 5242 if (!sentry->valid_blocks) 5243 __set_free(sbi, start); 5244 else 5245 SIT_I(sbi)->written_valid_blocks += 5246 sentry->valid_blocks; 5247 } 5248 5249 /* set use the current segments */ 5250 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { 5251 struct curseg_info *curseg_t = CURSEG_I(sbi, type); 5252 5253 __set_test_and_inuse(sbi, curseg_t->segno); 5254 } 5255 } 5256 5257 static void init_dirty_segmap(struct f2fs_sb_info *sbi) 5258 { 5259 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5260 struct free_segmap_info *free_i = FREE_I(sbi); 5261 unsigned int segno = 0, offset = 0, secno; 5262 block_t valid_blocks, usable_blks_in_seg; 5263 5264 while (1) { 5265 /* find dirty segment based on free segmap */ 5266 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); 5267 if (segno >= MAIN_SEGS(sbi)) 5268 break; 5269 offset = segno + 1; 5270 valid_blocks = get_valid_blocks(sbi, segno, false); 5271 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno); 5272 if (valid_blocks == usable_blks_in_seg || !valid_blocks) 5273 continue; 5274 if (valid_blocks > usable_blks_in_seg) { 5275 f2fs_bug_on(sbi, 1); 5276 continue; 5277 } 5278 mutex_lock(&dirty_i->seglist_lock); 5279 __locate_dirty_segment(sbi, segno, DIRTY); 5280 mutex_unlock(&dirty_i->seglist_lock); 5281 } 5282 5283 if (!__is_large_section(sbi)) 5284 return; 5285 5286 mutex_lock(&dirty_i->seglist_lock); 5287 for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { 5288 valid_blocks = get_valid_blocks(sbi, segno, true); 5289 secno = GET_SEC_FROM_SEG(sbi, segno); 5290 5291 if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi)) 5292 continue; 5293 if (is_cursec(sbi, secno)) 5294 continue; 5295 set_bit(secno, dirty_i->dirty_secmap); 5296 } 5297 mutex_unlock(&dirty_i->seglist_lock); 5298 } 5299 5300 static int init_victim_secmap(struct f2fs_sb_info *sbi) 5301 { 5302 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5303 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 5304 5305 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 5306 if (!dirty_i->victim_secmap) 5307 return -ENOMEM; 5308 5309 dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 5310 if (!dirty_i->pinned_secmap) 5311 return -ENOMEM; 5312 5313 dirty_i->pinned_secmap_cnt = 0; 5314 dirty_i->enable_pin_section = true; 5315 return 0; 5316 } 5317 5318 static int build_dirty_segmap(struct f2fs_sb_info *sbi) 5319 { 5320 struct dirty_seglist_info *dirty_i; 5321 unsigned int bitmap_size, i; 5322 5323 /* allocate memory for dirty segments list information */ 5324 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), 5325 GFP_KERNEL); 5326 if (!dirty_i) 5327 return -ENOMEM; 5328 5329 SM_I(sbi)->dirty_info = dirty_i; 5330 mutex_init(&dirty_i->seglist_lock); 5331 5332 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 5333 5334 for (i = 0; i < NR_DIRTY_TYPE; i++) { 5335 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, 5336 GFP_KERNEL); 5337 if (!dirty_i->dirty_segmap[i]) 5338 return -ENOMEM; 5339 } 5340 5341 if (__is_large_section(sbi)) { 5342 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 5343 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi, 5344 bitmap_size, GFP_KERNEL); 5345 if (!dirty_i->dirty_secmap) 5346 return -ENOMEM; 5347 } 5348 5349 init_dirty_segmap(sbi); 5350 return init_victim_secmap(sbi); 5351 } 5352 5353 static int sanity_check_curseg(struct f2fs_sb_info *sbi) 5354 { 5355 int i; 5356 5357 /* 5358 * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr; 5359 * In LFS curseg, all blkaddr after .next_blkoff should be unused. 5360 */ 5361 for (i = 0; i < NR_PERSISTENT_LOG; i++) { 5362 struct curseg_info *curseg = CURSEG_I(sbi, i); 5363 struct seg_entry *se = get_seg_entry(sbi, curseg->segno); 5364 unsigned int blkofs = curseg->next_blkoff; 5365 5366 if (f2fs_sb_has_readonly(sbi) && 5367 i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE) 5368 continue; 5369 5370 sanity_check_seg_type(sbi, curseg->seg_type); 5371 5372 if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) { 5373 f2fs_err(sbi, 5374 "Current segment has invalid alloc_type:%d", 5375 curseg->alloc_type); 5376 f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); 5377 return -EFSCORRUPTED; 5378 } 5379 5380 if (f2fs_test_bit(blkofs, se->cur_valid_map)) 5381 goto out; 5382 5383 if (curseg->alloc_type == SSR) 5384 continue; 5385 5386 for (blkofs += 1; blkofs < BLKS_PER_SEG(sbi); blkofs++) { 5387 if (!f2fs_test_bit(blkofs, se->cur_valid_map)) 5388 continue; 5389 out: 5390 f2fs_err(sbi, 5391 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u", 5392 i, curseg->segno, curseg->alloc_type, 5393 curseg->next_blkoff, blkofs); 5394 f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); 5395 return -EFSCORRUPTED; 5396 } 5397 } 5398 return 0; 5399 } 5400 5401 #ifdef CONFIG_BLK_DEV_ZONED 5402 static int check_zone_write_pointer(struct f2fs_sb_info *sbi, 5403 struct f2fs_dev_info *fdev, 5404 struct blk_zone *zone) 5405 { 5406 unsigned int zone_segno; 5407 block_t zone_block, valid_block_cnt; 5408 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; 5409 int ret; 5410 unsigned int nofs_flags; 5411 5412 if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ) 5413 return 0; 5414 5415 zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block); 5416 zone_segno = GET_SEGNO(sbi, zone_block); 5417 5418 /* 5419 * Skip check of zones cursegs point to, since 5420 * fix_curseg_write_pointer() checks them. 5421 */ 5422 if (zone_segno >= MAIN_SEGS(sbi)) 5423 return 0; 5424 5425 /* 5426 * Get # of valid block of the zone. 5427 */ 5428 valid_block_cnt = get_valid_blocks(sbi, zone_segno, true); 5429 if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, zone_segno))) { 5430 f2fs_notice(sbi, "Open zones: valid block[0x%x,0x%x] cond[%s]", 5431 zone_segno, valid_block_cnt, 5432 blk_zone_cond_str(zone->cond)); 5433 return 0; 5434 } 5435 5436 if ((!valid_block_cnt && zone->cond == BLK_ZONE_COND_EMPTY) || 5437 (valid_block_cnt && zone->cond == BLK_ZONE_COND_FULL)) 5438 return 0; 5439 5440 if (!valid_block_cnt) { 5441 f2fs_notice(sbi, "Zone without valid block has non-zero write " 5442 "pointer. Reset the write pointer: cond[%s]", 5443 blk_zone_cond_str(zone->cond)); 5444 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block, 5445 zone->len >> log_sectors_per_block); 5446 if (ret) 5447 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", 5448 fdev->path, ret); 5449 return ret; 5450 } 5451 5452 /* 5453 * If there are valid blocks and the write pointer doesn't match 5454 * with them, we need to report the inconsistency and fill 5455 * the zone till the end to close the zone. This inconsistency 5456 * does not cause write error because the zone will not be 5457 * selected for write operation until it get discarded. 5458 */ 5459 f2fs_notice(sbi, "Valid blocks are not aligned with write " 5460 "pointer: valid block[0x%x,0x%x] cond[%s]", 5461 zone_segno, valid_block_cnt, blk_zone_cond_str(zone->cond)); 5462 5463 nofs_flags = memalloc_nofs_save(); 5464 ret = blkdev_zone_mgmt(fdev->bdev, REQ_OP_ZONE_FINISH, 5465 zone->start, zone->len); 5466 memalloc_nofs_restore(nofs_flags); 5467 if (ret == -EOPNOTSUPP) { 5468 ret = blkdev_issue_zeroout(fdev->bdev, zone->wp, 5469 zone->len - (zone->wp - zone->start), 5470 GFP_NOFS, 0); 5471 if (ret) 5472 f2fs_err(sbi, "Fill up zone failed: %s (errno=%d)", 5473 fdev->path, ret); 5474 } else if (ret) { 5475 f2fs_err(sbi, "Finishing zone failed: %s (errno=%d)", 5476 fdev->path, ret); 5477 } 5478 5479 return ret; 5480 } 5481 5482 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi, 5483 block_t zone_blkaddr) 5484 { 5485 int i; 5486 5487 for (i = 0; i < sbi->s_ndevs; i++) { 5488 if (!bdev_is_zoned(FDEV(i).bdev)) 5489 continue; 5490 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr && 5491 zone_blkaddr <= FDEV(i).end_blk)) 5492 return &FDEV(i); 5493 } 5494 5495 return NULL; 5496 } 5497 5498 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx, 5499 void *data) 5500 { 5501 memcpy(data, zone, sizeof(struct blk_zone)); 5502 return 0; 5503 } 5504 5505 static int do_fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type) 5506 { 5507 struct curseg_info *cs = CURSEG_I(sbi, type); 5508 struct f2fs_dev_info *zbd; 5509 struct blk_zone zone; 5510 unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off; 5511 block_t cs_zone_block, wp_block; 5512 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; 5513 sector_t zone_sector; 5514 int err; 5515 5516 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); 5517 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); 5518 5519 zbd = get_target_zoned_dev(sbi, cs_zone_block); 5520 if (!zbd) 5521 return 0; 5522 5523 /* report zone for the sector the curseg points to */ 5524 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) 5525 << log_sectors_per_block; 5526 err = blkdev_report_zones(zbd->bdev, zone_sector, 1, 5527 report_one_zone_cb, &zone); 5528 if (err != 1) { 5529 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", 5530 zbd->path, err); 5531 return err; 5532 } 5533 5534 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) 5535 return 0; 5536 5537 /* 5538 * When safely unmounted in the previous mount, we could use current 5539 * segments. Otherwise, allocate new sections. 5540 */ 5541 if (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) { 5542 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block); 5543 wp_segno = GET_SEGNO(sbi, wp_block); 5544 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno); 5545 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0); 5546 5547 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff && 5548 wp_sector_off == 0) 5549 return 0; 5550 5551 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: " 5552 "curseg[0x%x,0x%x] wp[0x%x,0x%x]", type, cs->segno, 5553 cs->next_blkoff, wp_segno, wp_blkoff); 5554 } 5555 5556 /* Allocate a new section if it's not new. */ 5557 if (cs->next_blkoff || 5558 cs->segno != GET_SEG_FROM_SEC(sbi, GET_ZONE_FROM_SEC(sbi, cs_section))) { 5559 unsigned int old_segno = cs->segno, old_blkoff = cs->next_blkoff; 5560 5561 f2fs_allocate_new_section(sbi, type, true); 5562 f2fs_notice(sbi, "Assign new section to curseg[%d]: " 5563 "[0x%x,0x%x] -> [0x%x,0x%x]", 5564 type, old_segno, old_blkoff, 5565 cs->segno, cs->next_blkoff); 5566 } 5567 5568 /* check consistency of the zone curseg pointed to */ 5569 if (check_zone_write_pointer(sbi, zbd, &zone)) 5570 return -EIO; 5571 5572 /* check newly assigned zone */ 5573 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); 5574 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); 5575 5576 zbd = get_target_zoned_dev(sbi, cs_zone_block); 5577 if (!zbd) 5578 return 0; 5579 5580 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) 5581 << log_sectors_per_block; 5582 err = blkdev_report_zones(zbd->bdev, zone_sector, 1, 5583 report_one_zone_cb, &zone); 5584 if (err != 1) { 5585 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", 5586 zbd->path, err); 5587 return err; 5588 } 5589 5590 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) 5591 return 0; 5592 5593 if (zone.wp != zone.start) { 5594 f2fs_notice(sbi, 5595 "New zone for curseg[%d] is not yet discarded. " 5596 "Reset the zone: curseg[0x%x,0x%x]", 5597 type, cs->segno, cs->next_blkoff); 5598 err = __f2fs_issue_discard_zone(sbi, zbd->bdev, cs_zone_block, 5599 zone.len >> log_sectors_per_block); 5600 if (err) { 5601 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", 5602 zbd->path, err); 5603 return err; 5604 } 5605 } 5606 5607 return 0; 5608 } 5609 5610 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi) 5611 { 5612 int i, ret; 5613 5614 for (i = 0; i < NR_PERSISTENT_LOG; i++) { 5615 ret = do_fix_curseg_write_pointer(sbi, i); 5616 if (ret) 5617 return ret; 5618 } 5619 5620 return 0; 5621 } 5622 5623 struct check_zone_write_pointer_args { 5624 struct f2fs_sb_info *sbi; 5625 struct f2fs_dev_info *fdev; 5626 }; 5627 5628 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx, 5629 void *data) 5630 { 5631 struct check_zone_write_pointer_args *args; 5632 5633 args = (struct check_zone_write_pointer_args *)data; 5634 5635 return check_zone_write_pointer(args->sbi, args->fdev, zone); 5636 } 5637 5638 static int check_write_pointer(struct f2fs_sb_info *sbi) 5639 { 5640 int i, ret; 5641 struct check_zone_write_pointer_args args; 5642 5643 for (i = 0; i < sbi->s_ndevs; i++) { 5644 if (!bdev_is_zoned(FDEV(i).bdev)) 5645 continue; 5646 5647 args.sbi = sbi; 5648 args.fdev = &FDEV(i); 5649 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES, 5650 check_zone_write_pointer_cb, &args); 5651 if (ret < 0) 5652 return ret; 5653 } 5654 5655 return 0; 5656 } 5657 5658 int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi) 5659 { 5660 int ret; 5661 5662 if (!f2fs_sb_has_blkzoned(sbi) || f2fs_readonly(sbi->sb) || 5663 f2fs_hw_is_readonly(sbi)) 5664 return 0; 5665 5666 f2fs_notice(sbi, "Checking entire write pointers"); 5667 ret = fix_curseg_write_pointer(sbi); 5668 if (!ret) 5669 ret = check_write_pointer(sbi); 5670 return ret; 5671 } 5672 5673 /* 5674 * Return the number of usable blocks in a segment. The number of blocks 5675 * returned is always equal to the number of blocks in a segment for 5676 * segments fully contained within a sequential zone capacity or a 5677 * conventional zone. For segments partially contained in a sequential 5678 * zone capacity, the number of usable blocks up to the zone capacity 5679 * is returned. 0 is returned in all other cases. 5680 */ 5681 static inline unsigned int f2fs_usable_zone_blks_in_seg( 5682 struct f2fs_sb_info *sbi, unsigned int segno) 5683 { 5684 block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr; 5685 unsigned int secno; 5686 5687 if (!sbi->unusable_blocks_per_sec) 5688 return BLKS_PER_SEG(sbi); 5689 5690 secno = GET_SEC_FROM_SEG(sbi, segno); 5691 seg_start = START_BLOCK(sbi, segno); 5692 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); 5693 sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi); 5694 5695 /* 5696 * If segment starts before zone capacity and spans beyond 5697 * zone capacity, then usable blocks are from seg start to 5698 * zone capacity. If the segment starts after the zone capacity, 5699 * then there are no usable blocks. 5700 */ 5701 if (seg_start >= sec_cap_blkaddr) 5702 return 0; 5703 if (seg_start + BLKS_PER_SEG(sbi) > sec_cap_blkaddr) 5704 return sec_cap_blkaddr - seg_start; 5705 5706 return BLKS_PER_SEG(sbi); 5707 } 5708 #else 5709 int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi) 5710 { 5711 return 0; 5712 } 5713 5714 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi, 5715 unsigned int segno) 5716 { 5717 return 0; 5718 } 5719 5720 #endif 5721 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, 5722 unsigned int segno) 5723 { 5724 if (f2fs_sb_has_blkzoned(sbi)) 5725 return f2fs_usable_zone_blks_in_seg(sbi, segno); 5726 5727 return BLKS_PER_SEG(sbi); 5728 } 5729 5730 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi) 5731 { 5732 if (f2fs_sb_has_blkzoned(sbi)) 5733 return CAP_SEGS_PER_SEC(sbi); 5734 5735 return SEGS_PER_SEC(sbi); 5736 } 5737 5738 unsigned long long f2fs_get_section_mtime(struct f2fs_sb_info *sbi, 5739 unsigned int segno) 5740 { 5741 unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi); 5742 unsigned int secno = 0, start = 0; 5743 unsigned int total_valid_blocks = 0; 5744 unsigned long long mtime = 0; 5745 unsigned int i = 0; 5746 5747 secno = GET_SEC_FROM_SEG(sbi, segno); 5748 start = GET_SEG_FROM_SEC(sbi, secno); 5749 5750 if (!__is_large_section(sbi)) { 5751 mtime = get_seg_entry(sbi, start + i)->mtime; 5752 goto out; 5753 } 5754 5755 for (i = 0; i < usable_segs_per_sec; i++) { 5756 /* for large section, only check the mtime of valid segments */ 5757 struct seg_entry *se = get_seg_entry(sbi, start+i); 5758 5759 mtime += se->mtime * se->valid_blocks; 5760 total_valid_blocks += se->valid_blocks; 5761 } 5762 5763 if (total_valid_blocks == 0) 5764 return INVALID_MTIME; 5765 5766 mtime = div_u64(mtime, total_valid_blocks); 5767 out: 5768 if (unlikely(mtime == INVALID_MTIME)) 5769 mtime -= 1; 5770 return mtime; 5771 } 5772 5773 /* 5774 * Update min, max modified time for cost-benefit GC algorithm 5775 */ 5776 static void init_min_max_mtime(struct f2fs_sb_info *sbi) 5777 { 5778 struct sit_info *sit_i = SIT_I(sbi); 5779 unsigned int segno; 5780 5781 down_write(&sit_i->sentry_lock); 5782 5783 sit_i->min_mtime = ULLONG_MAX; 5784 5785 for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { 5786 unsigned long long mtime = 0; 5787 5788 mtime = f2fs_get_section_mtime(sbi, segno); 5789 5790 if (sit_i->min_mtime > mtime) 5791 sit_i->min_mtime = mtime; 5792 } 5793 sit_i->max_mtime = get_mtime(sbi, false); 5794 sit_i->dirty_max_mtime = 0; 5795 up_write(&sit_i->sentry_lock); 5796 } 5797 5798 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi) 5799 { 5800 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 5801 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 5802 struct f2fs_sm_info *sm_info; 5803 int err; 5804 5805 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); 5806 if (!sm_info) 5807 return -ENOMEM; 5808 5809 /* init sm info */ 5810 sbi->sm_info = sm_info; 5811 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); 5812 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); 5813 sm_info->segment_count = le32_to_cpu(raw_super->segment_count); 5814 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); 5815 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); 5816 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); 5817 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); 5818 sm_info->rec_prefree_segments = sm_info->main_segments * 5819 DEF_RECLAIM_PREFREE_SEGMENTS / 100; 5820 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS) 5821 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS; 5822 5823 if (!f2fs_lfs_mode(sbi)) 5824 sm_info->ipu_policy = BIT(F2FS_IPU_FSYNC); 5825 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; 5826 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS; 5827 sm_info->min_seq_blocks = BLKS_PER_SEG(sbi); 5828 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS; 5829 sm_info->min_ssr_sections = reserved_sections(sbi); 5830 5831 INIT_LIST_HEAD(&sm_info->sit_entry_set); 5832 5833 init_f2fs_rwsem(&sm_info->curseg_lock); 5834 5835 err = f2fs_create_flush_cmd_control(sbi); 5836 if (err) 5837 return err; 5838 5839 err = create_discard_cmd_control(sbi); 5840 if (err) 5841 return err; 5842 5843 err = build_sit_info(sbi); 5844 if (err) 5845 return err; 5846 err = build_free_segmap(sbi); 5847 if (err) 5848 return err; 5849 err = build_curseg(sbi); 5850 if (err) 5851 return err; 5852 5853 /* reinit free segmap based on SIT */ 5854 err = build_sit_entries(sbi); 5855 if (err) 5856 return err; 5857 5858 init_free_segmap(sbi); 5859 err = build_dirty_segmap(sbi); 5860 if (err) 5861 return err; 5862 5863 err = sanity_check_curseg(sbi); 5864 if (err) 5865 return err; 5866 5867 init_min_max_mtime(sbi); 5868 return 0; 5869 } 5870 5871 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, 5872 enum dirty_type dirty_type) 5873 { 5874 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5875 5876 mutex_lock(&dirty_i->seglist_lock); 5877 kvfree(dirty_i->dirty_segmap[dirty_type]); 5878 dirty_i->nr_dirty[dirty_type] = 0; 5879 mutex_unlock(&dirty_i->seglist_lock); 5880 } 5881 5882 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) 5883 { 5884 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5885 5886 kvfree(dirty_i->pinned_secmap); 5887 kvfree(dirty_i->victim_secmap); 5888 } 5889 5890 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) 5891 { 5892 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5893 int i; 5894 5895 if (!dirty_i) 5896 return; 5897 5898 /* discard pre-free/dirty segments list */ 5899 for (i = 0; i < NR_DIRTY_TYPE; i++) 5900 discard_dirty_segmap(sbi, i); 5901 5902 if (__is_large_section(sbi)) { 5903 mutex_lock(&dirty_i->seglist_lock); 5904 kvfree(dirty_i->dirty_secmap); 5905 mutex_unlock(&dirty_i->seglist_lock); 5906 } 5907 5908 destroy_victim_secmap(sbi); 5909 SM_I(sbi)->dirty_info = NULL; 5910 kfree(dirty_i); 5911 } 5912 5913 static void destroy_curseg(struct f2fs_sb_info *sbi) 5914 { 5915 struct curseg_info *array = SM_I(sbi)->curseg_array; 5916 int i; 5917 5918 if (!array) 5919 return; 5920 SM_I(sbi)->curseg_array = NULL; 5921 for (i = 0; i < NR_CURSEG_TYPE; i++) { 5922 kfree(array[i].sum_blk); 5923 kfree(array[i].journal); 5924 } 5925 kfree(array); 5926 } 5927 5928 static void destroy_free_segmap(struct f2fs_sb_info *sbi) 5929 { 5930 struct free_segmap_info *free_i = SM_I(sbi)->free_info; 5931 5932 if (!free_i) 5933 return; 5934 SM_I(sbi)->free_info = NULL; 5935 kvfree(free_i->free_segmap); 5936 kvfree(free_i->free_secmap); 5937 kfree(free_i); 5938 } 5939 5940 static void destroy_sit_info(struct f2fs_sb_info *sbi) 5941 { 5942 struct sit_info *sit_i = SIT_I(sbi); 5943 5944 if (!sit_i) 5945 return; 5946 5947 if (sit_i->sentries) 5948 kvfree(sit_i->bitmap); 5949 kfree(sit_i->tmp_map); 5950 5951 kvfree(sit_i->sentries); 5952 kvfree(sit_i->sec_entries); 5953 kvfree(sit_i->dirty_sentries_bitmap); 5954 5955 SM_I(sbi)->sit_info = NULL; 5956 kfree(sit_i->sit_bitmap); 5957 #ifdef CONFIG_F2FS_CHECK_FS 5958 kvfree(sit_i->invalid_segmap); 5959 #endif 5960 kfree(sit_i); 5961 } 5962 5963 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi) 5964 { 5965 struct f2fs_sm_info *sm_info = SM_I(sbi); 5966 5967 if (!sm_info) 5968 return; 5969 f2fs_destroy_flush_cmd_control(sbi, true); 5970 destroy_discard_cmd_control(sbi); 5971 destroy_dirty_segmap(sbi); 5972 destroy_curseg(sbi); 5973 destroy_free_segmap(sbi); 5974 destroy_sit_info(sbi); 5975 sbi->sm_info = NULL; 5976 kfree(sm_info); 5977 } 5978 5979 int __init f2fs_create_segment_manager_caches(void) 5980 { 5981 discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry", 5982 sizeof(struct discard_entry)); 5983 if (!discard_entry_slab) 5984 goto fail; 5985 5986 discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd", 5987 sizeof(struct discard_cmd)); 5988 if (!discard_cmd_slab) 5989 goto destroy_discard_entry; 5990 5991 sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set", 5992 sizeof(struct sit_entry_set)); 5993 if (!sit_entry_set_slab) 5994 goto destroy_discard_cmd; 5995 5996 revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry", 5997 sizeof(struct revoke_entry)); 5998 if (!revoke_entry_slab) 5999 goto destroy_sit_entry_set; 6000 return 0; 6001 6002 destroy_sit_entry_set: 6003 kmem_cache_destroy(sit_entry_set_slab); 6004 destroy_discard_cmd: 6005 kmem_cache_destroy(discard_cmd_slab); 6006 destroy_discard_entry: 6007 kmem_cache_destroy(discard_entry_slab); 6008 fail: 6009 return -ENOMEM; 6010 } 6011 6012 void f2fs_destroy_segment_manager_caches(void) 6013 { 6014 kmem_cache_destroy(sit_entry_set_slab); 6015 kmem_cache_destroy(discard_cmd_slab); 6016 kmem_cache_destroy(discard_entry_slab); 6017 kmem_cache_destroy(revoke_entry_slab); 6018 } 6019