1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/f2fs/segment.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/bio.h> 11 #include <linux/blkdev.h> 12 #include <linux/sched/mm.h> 13 #include <linux/prefetch.h> 14 #include <linux/kthread.h> 15 #include <linux/swap.h> 16 #include <linux/timer.h> 17 #include <linux/freezer.h> 18 #include <linux/sched/signal.h> 19 #include <linux/random.h> 20 #include <linux/fserror.h> 21 22 #include "f2fs.h" 23 #include "segment.h" 24 #include "node.h" 25 #include "gc.h" 26 #include "iostat.h" 27 #include <trace/events/f2fs.h> 28 29 #define __reverse_ffz(x) __reverse_ffs(~(x)) 30 31 static struct kmem_cache *discard_entry_slab; 32 static struct kmem_cache *discard_cmd_slab; 33 static struct kmem_cache *sit_entry_set_slab; 34 static struct kmem_cache *revoke_entry_slab; 35 36 static unsigned long __reverse_ulong(unsigned char *str) 37 { 38 unsigned long tmp = 0; 39 int shift = 24, idx = 0; 40 41 #if BITS_PER_LONG == 64 42 shift = 56; 43 #endif 44 while (shift >= 0) { 45 tmp |= (unsigned long)str[idx++] << shift; 46 shift -= BITS_PER_BYTE; 47 } 48 return tmp; 49 } 50 51 /* 52 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since 53 * MSB and LSB are reversed in a byte by f2fs_set_bit. 54 */ 55 static inline unsigned long __reverse_ffs(unsigned long word) 56 { 57 int num = 0; 58 59 #if BITS_PER_LONG == 64 60 if ((word & 0xffffffff00000000UL) == 0) 61 num += 32; 62 else 63 word >>= 32; 64 #endif 65 if ((word & 0xffff0000) == 0) 66 num += 16; 67 else 68 word >>= 16; 69 70 if ((word & 0xff00) == 0) 71 num += 8; 72 else 73 word >>= 8; 74 75 if ((word & 0xf0) == 0) 76 num += 4; 77 else 78 word >>= 4; 79 80 if ((word & 0xc) == 0) 81 num += 2; 82 else 83 word >>= 2; 84 85 if ((word & 0x2) == 0) 86 num += 1; 87 return num; 88 } 89 90 /* 91 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because 92 * f2fs_set_bit makes MSB and LSB reversed in a byte. 93 * @size must be integral times of unsigned long. 94 * Example: 95 * MSB <--> LSB 96 * f2fs_set_bit(0, bitmap) => 1000 0000 97 * f2fs_set_bit(7, bitmap) => 0000 0001 98 */ 99 static unsigned long __find_rev_next_bit(const unsigned long *addr, 100 unsigned long size, unsigned long offset) 101 { 102 const unsigned long *p = addr + BIT_WORD(offset); 103 unsigned long result = size; 104 unsigned long tmp; 105 106 if (offset >= size) 107 return size; 108 109 size -= (offset & ~(BITS_PER_LONG - 1)); 110 offset %= BITS_PER_LONG; 111 112 while (1) { 113 if (*p == 0) 114 goto pass; 115 116 tmp = __reverse_ulong((unsigned char *)p); 117 118 tmp &= ~0UL >> offset; 119 if (size < BITS_PER_LONG) 120 tmp &= (~0UL << (BITS_PER_LONG - size)); 121 if (tmp) 122 goto found; 123 pass: 124 if (size <= BITS_PER_LONG) 125 break; 126 size -= BITS_PER_LONG; 127 offset = 0; 128 p++; 129 } 130 return result; 131 found: 132 return result - size + __reverse_ffs(tmp); 133 } 134 135 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, 136 unsigned long size, unsigned long offset) 137 { 138 const unsigned long *p = addr + BIT_WORD(offset); 139 unsigned long result = size; 140 unsigned long tmp; 141 142 if (offset >= size) 143 return size; 144 145 size -= (offset & ~(BITS_PER_LONG - 1)); 146 offset %= BITS_PER_LONG; 147 148 while (1) { 149 if (*p == ~0UL) 150 goto pass; 151 152 tmp = __reverse_ulong((unsigned char *)p); 153 154 if (offset) 155 tmp |= ~0UL << (BITS_PER_LONG - offset); 156 if (size < BITS_PER_LONG) 157 tmp |= ~0UL >> size; 158 if (tmp != ~0UL) 159 goto found; 160 pass: 161 if (size <= BITS_PER_LONG) 162 break; 163 size -= BITS_PER_LONG; 164 offset = 0; 165 p++; 166 } 167 return result; 168 found: 169 return result - size + __reverse_ffz(tmp); 170 } 171 172 bool f2fs_need_SSR(struct f2fs_sb_info *sbi) 173 { 174 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); 175 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); 176 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); 177 178 if (f2fs_lfs_mode(sbi)) 179 return false; 180 if (sbi->gc_mode == GC_URGENT_HIGH) 181 return true; 182 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 183 return true; 184 185 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + 186 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); 187 } 188 189 void f2fs_abort_atomic_write(struct inode *inode, bool clean) 190 { 191 struct f2fs_inode_info *fi = F2FS_I(inode); 192 193 if (!f2fs_is_atomic_file(inode)) 194 return; 195 196 if (clean) 197 truncate_inode_pages(inode->i_mapping, 0); 198 199 release_atomic_write_cnt(inode); 200 clear_inode_flag(inode, FI_ATOMIC_COMMITTED); 201 clear_inode_flag(inode, FI_ATOMIC_REPLACE); 202 clear_inode_flag(inode, FI_ATOMIC_FILE); 203 if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) { 204 clear_inode_flag(inode, FI_ATOMIC_DIRTIED); 205 /* 206 * The vfs inode keeps clean during commit, but the f2fs inode 207 * doesn't. So clear the dirty state after commit and let 208 * f2fs_mark_inode_dirty_sync ensure a consistent dirty state. 209 */ 210 f2fs_inode_synced(inode); 211 f2fs_mark_inode_dirty_sync(inode, true); 212 } 213 stat_dec_atomic_inode(inode); 214 215 F2FS_I(inode)->atomic_write_task = NULL; 216 217 if (clean) { 218 f2fs_i_size_write(inode, fi->original_i_size); 219 fi->original_i_size = 0; 220 } 221 /* avoid stale dirty inode during eviction */ 222 sync_inode_metadata(inode, 0); 223 } 224 225 static int __replace_atomic_write_block(struct inode *inode, pgoff_t index, 226 block_t new_addr, block_t *old_addr, bool recover) 227 { 228 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 229 struct dnode_of_data dn; 230 struct node_info ni; 231 int err; 232 233 retry: 234 set_new_dnode(&dn, inode, NULL, NULL, 0); 235 err = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE); 236 if (err) { 237 if (err == -ENOMEM) { 238 memalloc_retry_wait(GFP_NOFS); 239 goto retry; 240 } 241 return err; 242 } 243 244 err = f2fs_get_node_info(sbi, dn.nid, &ni, false); 245 if (err) { 246 f2fs_put_dnode(&dn); 247 return err; 248 } 249 250 if (recover) { 251 /* dn.data_blkaddr is always valid */ 252 if (!__is_valid_data_blkaddr(new_addr)) { 253 if (new_addr == NULL_ADDR) 254 dec_valid_block_count(sbi, inode, 1); 255 f2fs_invalidate_blocks(sbi, dn.data_blkaddr, 1); 256 f2fs_update_data_blkaddr(&dn, new_addr); 257 } else { 258 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, 259 new_addr, ni.version, true, true); 260 } 261 } else { 262 blkcnt_t count = 1; 263 264 err = inc_valid_block_count(sbi, inode, &count, true, false); 265 if (err) { 266 f2fs_put_dnode(&dn); 267 return err; 268 } 269 270 *old_addr = dn.data_blkaddr; 271 f2fs_truncate_data_blocks_range(&dn, 1); 272 dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count); 273 274 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr, 275 ni.version, true, false); 276 } 277 278 f2fs_put_dnode(&dn); 279 280 trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode, 281 index, old_addr ? *old_addr : 0, new_addr, recover); 282 return 0; 283 } 284 285 static void __complete_revoke_list(struct inode *inode, struct list_head *head, 286 bool revoke) 287 { 288 struct revoke_entry *cur, *tmp; 289 pgoff_t start_index = 0; 290 bool truncate = is_inode_flag_set(inode, FI_ATOMIC_REPLACE); 291 292 list_for_each_entry_safe(cur, tmp, head, list) { 293 if (revoke) { 294 __replace_atomic_write_block(inode, cur->index, 295 cur->old_addr, NULL, true); 296 } else if (truncate) { 297 f2fs_truncate_hole(inode, start_index, cur->index); 298 start_index = cur->index + 1; 299 } 300 301 list_del(&cur->list); 302 kmem_cache_free(revoke_entry_slab, cur); 303 } 304 305 if (!revoke && truncate) 306 f2fs_do_truncate_blocks(inode, start_index * PAGE_SIZE, false); 307 } 308 309 static int __f2fs_commit_atomic_write(struct inode *inode) 310 { 311 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 312 struct f2fs_inode_info *fi = F2FS_I(inode); 313 struct inode *cow_inode = fi->cow_inode; 314 struct revoke_entry *new; 315 struct list_head revoke_list; 316 block_t blkaddr; 317 struct dnode_of_data dn; 318 pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 319 pgoff_t off = 0, blen, index; 320 int ret = 0, i; 321 322 INIT_LIST_HEAD(&revoke_list); 323 324 while (len) { 325 blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len); 326 327 set_new_dnode(&dn, cow_inode, NULL, NULL, 0); 328 ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA); 329 if (ret && ret != -ENOENT) { 330 goto out; 331 } else if (ret == -ENOENT) { 332 ret = 0; 333 if (dn.max_level == 0) 334 goto out; 335 goto next; 336 } 337 338 blen = min((pgoff_t)addrs_per_page(cow_inode, 339 IS_INODE(F2FS_I_SB(cow_inode), dn.node_entry)), len); 340 index = off; 341 for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) { 342 blkaddr = f2fs_data_blkaddr(&dn); 343 344 if (!__is_valid_data_blkaddr(blkaddr)) { 345 continue; 346 } else if (!f2fs_is_valid_blkaddr(sbi, blkaddr, 347 DATA_GENERIC_ENHANCE)) { 348 f2fs_put_dnode(&dn); 349 ret = -EFSCORRUPTED; 350 goto out; 351 } 352 353 new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS, 354 true, NULL); 355 356 ret = __replace_atomic_write_block(inode, index, blkaddr, 357 &new->old_addr, false); 358 if (ret) { 359 f2fs_put_dnode(&dn); 360 kmem_cache_free(revoke_entry_slab, new); 361 goto out; 362 } 363 364 f2fs_update_data_blkaddr(&dn, NULL_ADDR); 365 new->index = index; 366 list_add_tail(&new->list, &revoke_list); 367 } 368 f2fs_put_dnode(&dn); 369 next: 370 off += blen; 371 len -= blen; 372 } 373 374 out: 375 if (time_to_inject(sbi, FAULT_ATOMIC_TIMEOUT)) 376 f2fs_schedule_timeout_killable(DEFAULT_FAULT_TIMEOUT, true); 377 378 if (ret) { 379 sbi->revoked_atomic_block += fi->atomic_write_cnt; 380 } else { 381 sbi->committed_atomic_block += fi->atomic_write_cnt; 382 set_inode_flag(inode, FI_ATOMIC_COMMITTED); 383 384 /* 385 * inode may has no FI_ATOMIC_DIRTIED flag due to no write 386 * before commit. 387 */ 388 if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) { 389 /* clear atomic dirty status and set vfs dirty status */ 390 clear_inode_flag(inode, FI_ATOMIC_DIRTIED); 391 f2fs_mark_inode_dirty_sync(inode, true); 392 } 393 } 394 395 __complete_revoke_list(inode, &revoke_list, ret ? true : false); 396 397 return ret; 398 } 399 400 int f2fs_commit_atomic_write(struct inode *inode) 401 { 402 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 403 struct f2fs_inode_info *fi = F2FS_I(inode); 404 struct f2fs_lock_context lc; 405 int err; 406 407 err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); 408 if (err) 409 return err; 410 411 f2fs_down_write(&fi->i_gc_rwsem[WRITE]); 412 f2fs_lock_op(sbi, &lc); 413 414 err = __f2fs_commit_atomic_write(inode); 415 416 f2fs_unlock_op(sbi, &lc); 417 f2fs_up_write(&fi->i_gc_rwsem[WRITE]); 418 419 return err; 420 } 421 422 /* 423 * This function balances dirty node and dentry pages. 424 * In addition, it controls garbage collection. 425 */ 426 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) 427 { 428 if (f2fs_cp_error(sbi)) 429 return; 430 431 if (time_to_inject(sbi, FAULT_CHECKPOINT)) 432 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT); 433 434 /* balance_fs_bg is able to be pending */ 435 if (need && excess_cached_nats(sbi)) 436 f2fs_balance_fs_bg(sbi, false); 437 438 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 439 return; 440 441 /* 442 * We should do GC or end up with checkpoint, if there are so many dirty 443 * dir/node pages without enough free segments. 444 */ 445 if (has_enough_free_secs(sbi, 0, 0)) 446 return; 447 448 /* 449 * Submit all cached OPU/IPU DATA bios before triggering 450 * foreground GC to avoid potential deadlocks. 451 */ 452 f2fs_submit_merged_write(sbi, DATA); 453 f2fs_submit_all_merged_ipu_writes(sbi); 454 455 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread.f2fs_gc_task) { 456 DEFINE_WAIT(wait); 457 458 prepare_to_wait(&sbi->gc_thread.fggc_wq, &wait, 459 TASK_UNINTERRUPTIBLE); 460 wake_up(&sbi->gc_thread.gc_wait_queue_head); 461 io_schedule(); 462 finish_wait(&sbi->gc_thread.fggc_wq, &wait); 463 } else { 464 struct f2fs_gc_control gc_control = { 465 .victim_segno = NULL_SEGNO, 466 .init_gc_type = f2fs_sb_has_blkzoned(sbi) ? 467 FG_GC : BG_GC, 468 .no_bg_gc = true, 469 .should_migrate_blocks = false, 470 .err_gc_skipped = false, 471 .nr_free_secs = 1 }; 472 473 f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); 474 stat_inc_gc_call_count(sbi, FOREGROUND); 475 f2fs_gc(sbi, &gc_control); 476 } 477 } 478 479 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi) 480 { 481 int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2; 482 unsigned int dents = get_nr_caches(sbi, F2FS_DIRTY_DENTS); 483 unsigned int qdata = get_nr_caches(sbi, F2FS_DIRTY_QDATA); 484 unsigned int nodes = get_nr_caches(sbi, F2FS_DIRTY_NODES); 485 unsigned int meta = get_nr_caches(sbi, F2FS_DIRTY_META); 486 unsigned int imeta = get_nr_caches(sbi, F2FS_DIRTY_IMETA); 487 unsigned int threshold = 488 SEGS_TO_BLKS(sbi, (factor * DEFAULT_DIRTY_THRESHOLD)); 489 unsigned int global_threshold = threshold * 3 / 2; 490 491 if (dents >= threshold || qdata >= threshold || 492 nodes >= threshold || meta >= threshold || 493 imeta >= threshold) 494 return true; 495 return dents + qdata + nodes + meta + imeta > global_threshold; 496 } 497 498 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg) 499 { 500 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 501 return; 502 503 /* try to shrink extent cache when there is no enough memory */ 504 if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE)) 505 f2fs_shrink_read_extent_tree(sbi, 506 READ_EXTENT_CACHE_SHRINK_NUMBER); 507 508 /* try to shrink age extent cache when there is no enough memory */ 509 if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE)) 510 f2fs_shrink_age_extent_tree(sbi, 511 AGE_EXTENT_CACHE_SHRINK_NUMBER); 512 513 /* try to shrink meta cache when there is no enough memory */ 514 if (!f2fs_available_free_memory(sbi, META_BLOCK)) 515 f2fs_shrink_meta_cache(sbi, METADATA_CACHE_SHRINK_NUMBER); 516 517 /* try to shrink node cache when there is no enough memory */ 518 if (!f2fs_available_free_memory(sbi, NODE_BLOCK)) 519 f2fs_shrink_node_cache(sbi, METADATA_CACHE_SHRINK_NUMBER); 520 521 /* check the # of cached NAT entries */ 522 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) 523 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK(sbi)); 524 525 if (!f2fs_available_free_memory(sbi, FREE_NIDS)) 526 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS(sbi)); 527 else 528 f2fs_build_free_nids(sbi, false, false); 529 530 if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) || 531 excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi)) 532 goto do_sync; 533 534 /* there is background inflight IO or foreground operation recently */ 535 if (is_inflight_io(sbi, REQ_TIME) || 536 (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem))) 537 return; 538 539 /* exceed periodical checkpoint timeout threshold */ 540 if (f2fs_time_over(sbi, CP_TIME)) 541 goto do_sync; 542 543 /* checkpoint is the only way to shrink partial cached entries */ 544 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) && 545 f2fs_available_free_memory(sbi, INO_ENTRIES)) 546 return; 547 548 do_sync: 549 if (test_opt(sbi, DATA_FLUSH) && from_bg) { 550 struct blk_plug plug; 551 552 mutex_lock(&sbi->flush_lock); 553 554 blk_start_plug(&plug); 555 f2fs_sync_dirty_inodes(sbi, FILE_INODE, false); 556 blk_finish_plug(&plug); 557 558 mutex_unlock(&sbi->flush_lock); 559 } 560 stat_inc_cp_call_count(sbi, BACKGROUND); 561 f2fs_sync_fs(sbi->sb, 1); 562 } 563 564 static int __submit_flush_wait(struct f2fs_sb_info *sbi, 565 struct block_device *bdev) 566 { 567 int ret = blkdev_issue_flush(bdev); 568 569 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), 570 test_opt(sbi, FLUSH_MERGE), ret); 571 if (!ret) 572 f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0); 573 return ret; 574 } 575 576 static void f2fs_flush_end_io(struct bio *bio) 577 { 578 complete(bio->bi_private); 579 } 580 581 struct f2fs_flush_bio { 582 struct bio bio; 583 struct completion wait; 584 }; 585 586 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) 587 { 588 struct f2fs_flush_bio *flush_bio; 589 unsigned long devices = 0; 590 int ret = 0; 591 int i; 592 593 if (!f2fs_is_multi_device(sbi)) 594 return __submit_flush_wait(sbi, sbi->sb->s_bdev); 595 596 flush_bio = kmalloc(array_size(sbi->s_ndevs, sizeof(*flush_bio)), 597 GFP_NOFS | __GFP_NOFAIL); 598 599 for (i = 0; i < sbi->s_ndevs; i++) { 600 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO)) 601 continue; 602 603 bio_init(&flush_bio[i].bio, FDEV(i).bdev, NULL, 0, 604 REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH); 605 init_completion(&flush_bio[i].wait); 606 flush_bio[i].bio.bi_private = &flush_bio[i].wait; 607 flush_bio[i].bio.bi_end_io = f2fs_flush_end_io; 608 devices |= BIT(i); 609 submit_bio(&flush_bio[i].bio); 610 } 611 612 for (i = 0; i < sbi->s_ndevs; i++) { 613 int err; 614 615 if (!(devices & BIT(i))) 616 continue; 617 618 wait_for_completion(&flush_bio[i].wait); 619 err = blk_status_to_errno(flush_bio[i].bio.bi_status); 620 trace_f2fs_issue_flush(FDEV(i).bdev, test_opt(sbi, NOBARRIER), 621 test_opt(sbi, FLUSH_MERGE), err); 622 if (!err) 623 f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0); 624 else if (!ret) 625 ret = err; 626 bio_uninit(&flush_bio[i].bio); 627 } 628 kfree(flush_bio); 629 return ret; 630 } 631 632 static int issue_flush_thread(void *data) 633 { 634 struct f2fs_sb_info *sbi = data; 635 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 636 wait_queue_head_t *q = &fcc->flush_wait_queue; 637 repeat: 638 if (kthread_should_stop()) 639 return 0; 640 641 if (!llist_empty(&fcc->issue_list)) { 642 struct flush_cmd *cmd, *next; 643 int ret; 644 645 fcc->dispatch_list = llist_del_all(&fcc->issue_list); 646 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list); 647 648 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode); 649 650 ret = submit_flush_wait(sbi, cmd->ino); 651 atomic_inc(&fcc->issued_flush); 652 653 llist_for_each_entry_safe(cmd, next, 654 fcc->dispatch_list, llnode) { 655 cmd->ret = ret; 656 complete(&cmd->wait); 657 } 658 fcc->dispatch_list = NULL; 659 } 660 661 wait_event_interruptible(*q, 662 kthread_should_stop() || !llist_empty(&fcc->issue_list)); 663 goto repeat; 664 } 665 666 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) 667 { 668 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 669 struct flush_cmd cmd; 670 int ret; 671 672 if (test_opt(sbi, NOBARRIER)) 673 return 0; 674 675 if (!test_opt(sbi, FLUSH_MERGE)) { 676 atomic_inc(&fcc->queued_flush); 677 ret = submit_flush_wait(sbi, ino); 678 atomic_dec(&fcc->queued_flush); 679 atomic_inc(&fcc->issued_flush); 680 return ret; 681 } 682 683 if (atomic_inc_return(&fcc->queued_flush) == 1 || 684 f2fs_is_multi_device(sbi)) { 685 ret = submit_flush_wait(sbi, ino); 686 atomic_dec(&fcc->queued_flush); 687 688 atomic_inc(&fcc->issued_flush); 689 return ret; 690 } 691 692 cmd.ino = ino; 693 init_completion(&cmd.wait); 694 695 llist_add(&cmd.llnode, &fcc->issue_list); 696 697 /* 698 * update issue_list before we wake up issue_flush thread, this 699 * smp_mb() pairs with another barrier in ___wait_event(), see 700 * more details in comments of waitqueue_active(). 701 */ 702 smp_mb(); 703 704 if (waitqueue_active(&fcc->flush_wait_queue)) 705 wake_up(&fcc->flush_wait_queue); 706 707 if (fcc->f2fs_issue_flush) { 708 wait_for_completion(&cmd.wait); 709 atomic_dec(&fcc->queued_flush); 710 } else { 711 struct llist_node *list; 712 713 list = llist_del_all(&fcc->issue_list); 714 if (!list) { 715 wait_for_completion(&cmd.wait); 716 atomic_dec(&fcc->queued_flush); 717 } else { 718 struct flush_cmd *tmp, *next; 719 720 ret = submit_flush_wait(sbi, ino); 721 722 llist_for_each_entry_safe(tmp, next, list, llnode) { 723 if (tmp == &cmd) { 724 cmd.ret = ret; 725 atomic_dec(&fcc->queued_flush); 726 continue; 727 } 728 tmp->ret = ret; 729 complete(&tmp->wait); 730 } 731 } 732 } 733 734 return cmd.ret; 735 } 736 737 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi) 738 { 739 dev_t dev = sbi->sb->s_bdev->bd_dev; 740 struct flush_cmd_control *fcc; 741 742 if (SM_I(sbi)->fcc_info) { 743 fcc = SM_I(sbi)->fcc_info; 744 if (fcc->f2fs_issue_flush) 745 return 0; 746 goto init_thread; 747 } 748 749 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); 750 if (!fcc) 751 return -ENOMEM; 752 atomic_set(&fcc->issued_flush, 0); 753 atomic_set(&fcc->queued_flush, 0); 754 init_waitqueue_head(&fcc->flush_wait_queue); 755 init_llist_head(&fcc->issue_list); 756 SM_I(sbi)->fcc_info = fcc; 757 if (!test_opt(sbi, FLUSH_MERGE)) 758 return 0; 759 760 init_thread: 761 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, 762 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); 763 if (IS_ERR(fcc->f2fs_issue_flush)) { 764 int err = PTR_ERR(fcc->f2fs_issue_flush); 765 766 fcc->f2fs_issue_flush = NULL; 767 return err; 768 } 769 770 return 0; 771 } 772 773 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) 774 { 775 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; 776 777 if (fcc && fcc->f2fs_issue_flush) { 778 struct task_struct *flush_thread = fcc->f2fs_issue_flush; 779 780 fcc->f2fs_issue_flush = NULL; 781 kthread_stop(flush_thread); 782 } 783 if (free) { 784 kfree(fcc); 785 SM_I(sbi)->fcc_info = NULL; 786 } 787 } 788 789 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) 790 { 791 int ret = 0, i; 792 793 if (!f2fs_is_multi_device(sbi)) 794 return 0; 795 796 if (test_opt(sbi, NOBARRIER)) 797 return 0; 798 799 for (i = 1; i < sbi->s_ndevs; i++) { 800 int count = DEFAULT_RETRY_IO_COUNT; 801 802 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) 803 continue; 804 805 do { 806 ret = __submit_flush_wait(sbi, FDEV(i).bdev); 807 if (ret) 808 f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); 809 } while (ret && --count); 810 811 if (ret) { 812 f2fs_stop_checkpoint(sbi, false, 813 STOP_CP_REASON_FLUSH_FAIL); 814 break; 815 } 816 817 spin_lock(&sbi->dev_lock); 818 f2fs_clear_bit(i, (char *)&sbi->dirty_device); 819 spin_unlock(&sbi->dev_lock); 820 } 821 822 return ret; 823 } 824 825 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 826 enum dirty_type dirty_type) 827 { 828 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 829 830 /* need not be added */ 831 if (is_curseg(sbi, segno)) 832 return; 833 834 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) 835 dirty_i->nr_dirty[dirty_type]++; 836 837 if (dirty_type == DIRTY) { 838 struct seg_entry *sentry = get_seg_entry(sbi, segno); 839 enum dirty_type t = sentry->type; 840 841 if (unlikely(t >= DIRTY)) { 842 f2fs_bug_on(sbi, 1); 843 return; 844 } 845 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) 846 dirty_i->nr_dirty[t]++; 847 848 if (__is_large_section(sbi)) { 849 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); 850 block_t valid_blocks = 851 get_valid_blocks(sbi, segno, true); 852 853 f2fs_bug_on(sbi, 854 (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 855 !valid_blocks) || 856 valid_blocks == CAP_BLKS_PER_SEC(sbi)); 857 858 if (!is_cursec(sbi, secno)) 859 set_bit(secno, dirty_i->dirty_secmap); 860 } 861 } 862 } 863 864 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 865 enum dirty_type dirty_type) 866 { 867 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 868 block_t valid_blocks; 869 870 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) 871 dirty_i->nr_dirty[dirty_type]--; 872 873 if (dirty_type == DIRTY) { 874 struct seg_entry *sentry = get_seg_entry(sbi, segno); 875 enum dirty_type t = sentry->type; 876 877 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) 878 dirty_i->nr_dirty[t]--; 879 880 valid_blocks = get_valid_blocks(sbi, segno, true); 881 if (valid_blocks == 0) { 882 clear_bit(GET_SEC_FROM_SEG(sbi, segno), 883 dirty_i->victim_secmap); 884 #ifdef CONFIG_F2FS_CHECK_FS 885 clear_bit(segno, SIT_I(sbi)->invalid_segmap); 886 #endif 887 } 888 if (__is_large_section(sbi)) { 889 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); 890 891 if (!valid_blocks || 892 valid_blocks == CAP_BLKS_PER_SEC(sbi)) { 893 clear_bit(secno, dirty_i->dirty_secmap); 894 return; 895 } 896 897 if (!is_cursec(sbi, secno)) 898 set_bit(secno, dirty_i->dirty_secmap); 899 } 900 } 901 } 902 903 /* 904 * Should not occur error such as -ENOMEM. 905 * Adding dirty entry into seglist is not critical operation. 906 * If a given segment is one of current working segments, it won't be added. 907 */ 908 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 909 { 910 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 911 unsigned short valid_blocks, ckpt_valid_blocks; 912 unsigned int usable_blocks; 913 914 if (segno == NULL_SEGNO || is_curseg(sbi, segno)) 915 return; 916 917 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno); 918 mutex_lock(&dirty_i->seglist_lock); 919 920 valid_blocks = get_valid_blocks(sbi, segno, false); 921 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false); 922 923 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) || 924 ckpt_valid_blocks == usable_blocks)) { 925 __locate_dirty_segment(sbi, segno, PRE); 926 __remove_dirty_segment(sbi, segno, DIRTY); 927 } else if (valid_blocks < usable_blocks) { 928 __locate_dirty_segment(sbi, segno, DIRTY); 929 } else { 930 /* Recovery routine with SSR needs this */ 931 __remove_dirty_segment(sbi, segno, DIRTY); 932 } 933 934 mutex_unlock(&dirty_i->seglist_lock); 935 } 936 937 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */ 938 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi) 939 { 940 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 941 unsigned int segno; 942 943 mutex_lock(&dirty_i->seglist_lock); 944 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 945 if (get_valid_blocks(sbi, segno, false)) 946 continue; 947 if (is_curseg(sbi, segno)) 948 continue; 949 __locate_dirty_segment(sbi, segno, PRE); 950 __remove_dirty_segment(sbi, segno, DIRTY); 951 } 952 mutex_unlock(&dirty_i->seglist_lock); 953 } 954 955 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi) 956 { 957 int ovp_hole_segs = 958 (overprovision_segments(sbi) - reserved_segments(sbi)); 959 block_t ovp_holes = SEGS_TO_BLKS(sbi, ovp_hole_segs); 960 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 961 block_t holes[2] = {0, 0}; /* DATA and NODE */ 962 block_t unusable; 963 struct seg_entry *se; 964 unsigned int segno; 965 966 mutex_lock(&dirty_i->seglist_lock); 967 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 968 se = get_seg_entry(sbi, segno); 969 if (IS_NODESEG(se->type)) 970 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) - 971 se->valid_blocks; 972 else 973 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) - 974 se->valid_blocks; 975 } 976 mutex_unlock(&dirty_i->seglist_lock); 977 978 unusable = max(holes[DATA], holes[NODE]); 979 if (unusable > ovp_holes) 980 return unusable - ovp_holes; 981 return 0; 982 } 983 984 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable) 985 { 986 int ovp_hole_segs = 987 (overprovision_segments(sbi) - reserved_segments(sbi)); 988 989 if (F2FS_OPTION(sbi).unusable_cap_perc == 100) 990 return 0; 991 if (unusable > F2FS_OPTION(sbi).unusable_cap) 992 return -EAGAIN; 993 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) && 994 dirty_segments(sbi) > ovp_hole_segs) 995 return -EAGAIN; 996 if (has_not_enough_free_secs(sbi, 0, 0)) 997 return -EAGAIN; 998 return 0; 999 } 1000 1001 /* This is only used by SBI_CP_DISABLED */ 1002 static unsigned int get_free_segment(struct f2fs_sb_info *sbi) 1003 { 1004 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1005 unsigned int segno = 0; 1006 1007 mutex_lock(&dirty_i->seglist_lock); 1008 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { 1009 if (get_valid_blocks(sbi, segno, false)) 1010 continue; 1011 if (get_ckpt_valid_blocks(sbi, segno, false)) 1012 continue; 1013 mutex_unlock(&dirty_i->seglist_lock); 1014 return segno; 1015 } 1016 mutex_unlock(&dirty_i->seglist_lock); 1017 return NULL_SEGNO; 1018 } 1019 1020 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, 1021 struct block_device *bdev, block_t lstart, 1022 block_t start, block_t len) 1023 { 1024 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1025 struct list_head *pend_list; 1026 struct discard_cmd *dc; 1027 1028 f2fs_bug_on(sbi, !len); 1029 1030 pend_list = &dcc->pend_list[plist_idx(len)]; 1031 1032 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL); 1033 INIT_LIST_HEAD(&dc->list); 1034 dc->bdev = bdev; 1035 dc->di.lstart = lstart; 1036 dc->di.start = start; 1037 dc->di.len = len; 1038 dc->ref = 0; 1039 dc->state = D_PREP; 1040 dc->queued = 0; 1041 dc->error = 0; 1042 init_completion(&dc->wait); 1043 list_add_tail(&dc->list, pend_list); 1044 spin_lock_init(&dc->lock); 1045 dc->bio_ref = 0; 1046 atomic_inc(&dcc->discard_cmd_cnt); 1047 dcc->undiscard_blks += len; 1048 1049 return dc; 1050 } 1051 1052 static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi) 1053 { 1054 #ifdef CONFIG_F2FS_CHECK_FS 1055 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1056 struct rb_node *cur = rb_first_cached(&dcc->root), *next; 1057 struct discard_cmd *cur_dc, *next_dc; 1058 1059 while (cur) { 1060 next = rb_next(cur); 1061 if (!next) 1062 return true; 1063 1064 cur_dc = rb_entry(cur, struct discard_cmd, rb_node); 1065 next_dc = rb_entry(next, struct discard_cmd, rb_node); 1066 1067 if (cur_dc->di.lstart + cur_dc->di.len > next_dc->di.lstart) { 1068 f2fs_info(sbi, "broken discard_rbtree, " 1069 "cur(%u, %u) next(%u, %u)", 1070 cur_dc->di.lstart, cur_dc->di.len, 1071 next_dc->di.lstart, next_dc->di.len); 1072 return false; 1073 } 1074 cur = next; 1075 } 1076 #endif 1077 return true; 1078 } 1079 1080 static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi, 1081 block_t blkaddr) 1082 { 1083 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1084 struct rb_node *node = dcc->root.rb_root.rb_node; 1085 struct discard_cmd *dc; 1086 1087 while (node) { 1088 dc = rb_entry(node, struct discard_cmd, rb_node); 1089 1090 if (blkaddr < dc->di.lstart) 1091 node = node->rb_left; 1092 else if (blkaddr >= dc->di.lstart + dc->di.len) 1093 node = node->rb_right; 1094 else 1095 return dc; 1096 } 1097 return NULL; 1098 } 1099 1100 static struct discard_cmd *__lookup_discard_cmd_ret(struct rb_root_cached *root, 1101 block_t blkaddr, 1102 struct discard_cmd **prev_entry, 1103 struct discard_cmd **next_entry, 1104 struct rb_node ***insert_p, 1105 struct rb_node **insert_parent) 1106 { 1107 struct rb_node **pnode = &root->rb_root.rb_node; 1108 struct rb_node *parent = NULL, *tmp_node; 1109 struct discard_cmd *dc; 1110 1111 *insert_p = NULL; 1112 *insert_parent = NULL; 1113 *prev_entry = NULL; 1114 *next_entry = NULL; 1115 1116 if (RB_EMPTY_ROOT(&root->rb_root)) 1117 return NULL; 1118 1119 while (*pnode) { 1120 parent = *pnode; 1121 dc = rb_entry(*pnode, struct discard_cmd, rb_node); 1122 1123 if (blkaddr < dc->di.lstart) 1124 pnode = &(*pnode)->rb_left; 1125 else if (blkaddr >= dc->di.lstart + dc->di.len) 1126 pnode = &(*pnode)->rb_right; 1127 else 1128 goto lookup_neighbors; 1129 } 1130 1131 *insert_p = pnode; 1132 *insert_parent = parent; 1133 1134 dc = rb_entry(parent, struct discard_cmd, rb_node); 1135 tmp_node = parent; 1136 if (parent && blkaddr > dc->di.lstart) 1137 tmp_node = rb_next(parent); 1138 *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); 1139 1140 tmp_node = parent; 1141 if (parent && blkaddr < dc->di.lstart) 1142 tmp_node = rb_prev(parent); 1143 *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); 1144 return NULL; 1145 1146 lookup_neighbors: 1147 /* lookup prev node for merging backward later */ 1148 tmp_node = rb_prev(&dc->rb_node); 1149 *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); 1150 1151 /* lookup next node for merging frontward later */ 1152 tmp_node = rb_next(&dc->rb_node); 1153 *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); 1154 return dc; 1155 } 1156 1157 static void __detach_discard_cmd(struct discard_cmd_control *dcc, 1158 struct discard_cmd *dc) 1159 { 1160 if (dc->state == D_DONE) 1161 atomic_sub(dc->queued, &dcc->queued_discard); 1162 1163 list_del(&dc->list); 1164 rb_erase_cached(&dc->rb_node, &dcc->root); 1165 dcc->undiscard_blks -= dc->di.len; 1166 1167 kmem_cache_free(discard_cmd_slab, dc); 1168 1169 atomic_dec(&dcc->discard_cmd_cnt); 1170 } 1171 1172 static void __remove_discard_cmd(struct f2fs_sb_info *sbi, 1173 struct discard_cmd *dc) 1174 { 1175 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1176 unsigned long flags; 1177 1178 trace_f2fs_remove_discard(dc->bdev, dc->di.start, dc->di.len); 1179 1180 spin_lock_irqsave(&dc->lock, flags); 1181 if (dc->bio_ref) { 1182 spin_unlock_irqrestore(&dc->lock, flags); 1183 return; 1184 } 1185 spin_unlock_irqrestore(&dc->lock, flags); 1186 1187 f2fs_bug_on(sbi, dc->ref); 1188 1189 if (dc->error == -EOPNOTSUPP) 1190 dc->error = 0; 1191 1192 if (dc->error) 1193 f2fs_info_ratelimited(sbi, 1194 "Issue discard(%u, %u, %u) failed, ret: %d", 1195 dc->di.lstart, dc->di.start, dc->di.len, dc->error); 1196 __detach_discard_cmd(dcc, dc); 1197 } 1198 1199 static void f2fs_submit_discard_endio(struct bio *bio) 1200 { 1201 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private; 1202 unsigned long flags; 1203 1204 spin_lock_irqsave(&dc->lock, flags); 1205 if (!dc->error) 1206 dc->error = blk_status_to_errno(bio->bi_status); 1207 dc->bio_ref--; 1208 if (!dc->bio_ref && dc->state == D_SUBMIT) { 1209 dc->state = D_DONE; 1210 complete_all(&dc->wait); 1211 } 1212 spin_unlock_irqrestore(&dc->lock, flags); 1213 bio_put(bio); 1214 } 1215 1216 static void __check_sit_bitmap(struct f2fs_sb_info *sbi, 1217 block_t start, block_t end) 1218 { 1219 #ifdef CONFIG_F2FS_CHECK_FS 1220 struct seg_entry *sentry; 1221 unsigned int segno; 1222 block_t blk = start; 1223 unsigned long offset, size, *map; 1224 1225 while (blk < end) { 1226 segno = GET_SEGNO(sbi, blk); 1227 sentry = get_seg_entry(sbi, segno); 1228 offset = GET_BLKOFF_FROM_SEG0(sbi, blk); 1229 1230 if (end < START_BLOCK(sbi, segno + 1)) 1231 size = GET_BLKOFF_FROM_SEG0(sbi, end); 1232 else 1233 size = BLKS_PER_SEG(sbi); 1234 map = (unsigned long *)(sentry->cur_valid_map); 1235 offset = __find_rev_next_bit(map, size, offset); 1236 f2fs_bug_on(sbi, offset != size); 1237 blk = START_BLOCK(sbi, segno + 1); 1238 } 1239 #endif 1240 } 1241 1242 static void __init_discard_policy(struct f2fs_sb_info *sbi, 1243 struct discard_policy *dpolicy, 1244 int discard_type, unsigned int granularity) 1245 { 1246 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1247 1248 /* common policy */ 1249 dpolicy->type = discard_type; 1250 dpolicy->sync = true; 1251 dpolicy->ordered = false; 1252 dpolicy->granularity = granularity; 1253 1254 dpolicy->max_requests = dcc->max_discard_request; 1255 dpolicy->io_aware_gran = dcc->discard_io_aware_gran; 1256 dpolicy->timeout = false; 1257 1258 if (discard_type == DPOLICY_BG) { 1259 dpolicy->min_interval = dcc->min_discard_issue_time; 1260 dpolicy->mid_interval = dcc->mid_discard_issue_time; 1261 dpolicy->max_interval = dcc->max_discard_issue_time; 1262 if (dcc->discard_io_aware == DPOLICY_IO_AWARE_ENABLE) 1263 dpolicy->io_aware = true; 1264 else if (dcc->discard_io_aware == DPOLICY_IO_AWARE_DISABLE) 1265 dpolicy->io_aware = false; 1266 dpolicy->sync = false; 1267 dpolicy->ordered = true; 1268 if (utilization(sbi) > dcc->discard_urgent_util) { 1269 dpolicy->granularity = MIN_DISCARD_GRANULARITY; 1270 if (atomic_read(&dcc->discard_cmd_cnt)) 1271 dpolicy->max_interval = 1272 dcc->min_discard_issue_time; 1273 } 1274 } else if (discard_type == DPOLICY_FORCE) { 1275 dpolicy->min_interval = dcc->min_discard_issue_time; 1276 dpolicy->mid_interval = dcc->mid_discard_issue_time; 1277 dpolicy->max_interval = dcc->max_discard_issue_time; 1278 dpolicy->io_aware = false; 1279 } else if (discard_type == DPOLICY_FSTRIM) { 1280 dpolicy->io_aware = false; 1281 } else if (discard_type == DPOLICY_UMOUNT) { 1282 dpolicy->io_aware = false; 1283 /* we need to issue all to keep CP_TRIMMED_FLAG */ 1284 dpolicy->granularity = MIN_DISCARD_GRANULARITY; 1285 dpolicy->timeout = true; 1286 } 1287 } 1288 1289 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1290 struct block_device *bdev, block_t lstart, 1291 block_t start, block_t len); 1292 1293 #ifdef CONFIG_BLK_DEV_ZONED 1294 static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi, 1295 struct discard_cmd *dc, blk_opf_t flag, 1296 struct list_head *wait_list, 1297 unsigned int *issued) 1298 { 1299 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1300 struct block_device *bdev = dc->bdev; 1301 struct bio *bio = bio_alloc(bdev, 0, REQ_OP_ZONE_RESET | flag, GFP_NOFS); 1302 unsigned long flags; 1303 1304 trace_f2fs_issue_reset_zone(bdev, dc->di.start); 1305 1306 spin_lock_irqsave(&dc->lock, flags); 1307 dc->state = D_SUBMIT; 1308 dc->bio_ref++; 1309 spin_unlock_irqrestore(&dc->lock, flags); 1310 1311 if (issued) 1312 (*issued)++; 1313 1314 atomic_inc(&dcc->queued_discard); 1315 dc->queued++; 1316 list_move_tail(&dc->list, wait_list); 1317 1318 /* sanity check on discard range */ 1319 __check_sit_bitmap(sbi, dc->di.lstart, dc->di.lstart + dc->di.len); 1320 1321 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(sbi, dc->di.start); 1322 bio->bi_private = dc; 1323 bio->bi_end_io = f2fs_submit_discard_endio; 1324 submit_bio(bio); 1325 1326 atomic_inc(&dcc->issued_discard); 1327 f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, 1328 dc->di.len * F2FS_BLKSIZE(sbi)); 1329 } 1330 #endif 1331 1332 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */ 1333 static int __submit_discard_cmd(struct f2fs_sb_info *sbi, 1334 struct discard_policy *dpolicy, 1335 struct discard_cmd *dc, int *issued) 1336 { 1337 struct block_device *bdev = dc->bdev; 1338 unsigned int max_discard_blocks = 1339 SECTOR_TO_BLOCK(sbi, bdev_max_discard_sectors(bdev)); 1340 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1341 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1342 &(dcc->fstrim_list) : &(dcc->wait_list); 1343 blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0; 1344 block_t lstart, start, len, total_len; 1345 1346 if (dc->state != D_PREP) 1347 return 0; 1348 1349 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) 1350 return 0; 1351 1352 #ifdef CONFIG_BLK_DEV_ZONED 1353 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) { 1354 int devi = f2fs_bdev_index(sbi, bdev); 1355 1356 if (devi < 0) 1357 return -EINVAL; 1358 1359 if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) { 1360 __submit_zone_reset_cmd(sbi, dc, flag, 1361 wait_list, issued); 1362 return 0; 1363 } 1364 } 1365 #endif 1366 1367 /* 1368 * stop issuing discard for any of below cases: 1369 * 1. device is conventional zone, but it doesn't support discard. 1370 * 2. device is regulare device, after snapshot it doesn't support 1371 * discard. 1372 */ 1373 if (!bdev_max_discard_sectors(bdev)) 1374 return -EOPNOTSUPP; 1375 1376 trace_f2fs_issue_discard(bdev, dc->di.start, dc->di.len); 1377 1378 lstart = dc->di.lstart; 1379 start = dc->di.start; 1380 len = dc->di.len; 1381 total_len = len; 1382 1383 dc->di.len = 0; 1384 1385 while (total_len && *issued < dpolicy->max_requests) { 1386 struct bio *bio = NULL; 1387 unsigned long flags; 1388 bool last = true; 1389 1390 if (len > max_discard_blocks) { 1391 len = max_discard_blocks; 1392 last = false; 1393 } 1394 1395 (*issued)++; 1396 if (*issued == dpolicy->max_requests) 1397 last = true; 1398 1399 dc->di.len += len; 1400 1401 __blkdev_issue_discard(bdev, SECTOR_FROM_BLOCK(sbi, start), 1402 SECTOR_FROM_BLOCK(sbi, len), GFP_NOFS, &bio); 1403 f2fs_bug_on(sbi, !bio); 1404 1405 /* 1406 * should keep before submission to avoid D_DONE 1407 * right away 1408 */ 1409 spin_lock_irqsave(&dc->lock, flags); 1410 if (last) 1411 dc->state = D_SUBMIT; 1412 else 1413 dc->state = D_PARTIAL; 1414 dc->bio_ref++; 1415 spin_unlock_irqrestore(&dc->lock, flags); 1416 1417 atomic_inc(&dcc->queued_discard); 1418 dc->queued++; 1419 list_move_tail(&dc->list, wait_list); 1420 1421 /* sanity check on discard range */ 1422 __check_sit_bitmap(sbi, lstart, lstart + len); 1423 1424 bio->bi_private = dc; 1425 bio->bi_end_io = f2fs_submit_discard_endio; 1426 bio->bi_opf |= flag; 1427 submit_bio(bio); 1428 1429 atomic_inc(&dcc->issued_discard); 1430 1431 f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, 1432 len * F2FS_BLKSIZE(sbi)); 1433 1434 lstart += len; 1435 start += len; 1436 total_len -= len; 1437 len = total_len; 1438 } 1439 1440 if (len) { 1441 dcc->undiscard_blks -= len; 1442 __update_discard_tree_range(sbi, bdev, lstart, start, len); 1443 } 1444 return 0; 1445 } 1446 1447 static void __insert_discard_cmd(struct f2fs_sb_info *sbi, 1448 struct block_device *bdev, block_t lstart, 1449 block_t start, block_t len) 1450 { 1451 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1452 struct rb_node **p = &dcc->root.rb_root.rb_node; 1453 struct rb_node *parent = NULL; 1454 struct discard_cmd *dc; 1455 bool leftmost = true; 1456 1457 /* look up rb tree to find parent node */ 1458 while (*p) { 1459 parent = *p; 1460 dc = rb_entry(parent, struct discard_cmd, rb_node); 1461 1462 if (lstart < dc->di.lstart) { 1463 p = &(*p)->rb_left; 1464 } else if (lstart >= dc->di.lstart + dc->di.len) { 1465 p = &(*p)->rb_right; 1466 leftmost = false; 1467 } else { 1468 /* Let's skip to add, if exists */ 1469 return; 1470 } 1471 } 1472 1473 dc = __create_discard_cmd(sbi, bdev, lstart, start, len); 1474 1475 rb_link_node(&dc->rb_node, parent, p); 1476 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost); 1477 } 1478 1479 static void __relocate_discard_cmd(struct discard_cmd_control *dcc, 1480 struct discard_cmd *dc) 1481 { 1482 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->di.len)]); 1483 } 1484 1485 static void __punch_discard_cmd(struct f2fs_sb_info *sbi, 1486 struct discard_cmd *dc, block_t blkaddr) 1487 { 1488 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1489 struct discard_info di = dc->di; 1490 bool modified = false; 1491 1492 if (dc->state == D_DONE || dc->di.len == 1) { 1493 __remove_discard_cmd(sbi, dc); 1494 return; 1495 } 1496 1497 dcc->undiscard_blks -= di.len; 1498 1499 if (blkaddr > di.lstart) { 1500 dc->di.len = blkaddr - dc->di.lstart; 1501 dcc->undiscard_blks += dc->di.len; 1502 __relocate_discard_cmd(dcc, dc); 1503 modified = true; 1504 } 1505 1506 if (blkaddr < di.lstart + di.len - 1) { 1507 if (modified) { 1508 __insert_discard_cmd(sbi, dc->bdev, blkaddr + 1, 1509 di.start + blkaddr + 1 - di.lstart, 1510 di.lstart + di.len - 1 - blkaddr); 1511 } else { 1512 dc->di.lstart++; 1513 dc->di.len--; 1514 dc->di.start++; 1515 dcc->undiscard_blks += dc->di.len; 1516 __relocate_discard_cmd(dcc, dc); 1517 } 1518 } 1519 } 1520 1521 static void __update_discard_tree_range(struct f2fs_sb_info *sbi, 1522 struct block_device *bdev, block_t lstart, 1523 block_t start, block_t len) 1524 { 1525 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1526 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1527 struct discard_cmd *dc; 1528 struct discard_info di = {0}; 1529 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1530 unsigned int max_discard_blocks = 1531 SECTOR_TO_BLOCK(sbi, bdev_max_discard_sectors(bdev)); 1532 block_t end = lstart + len; 1533 1534 dc = __lookup_discard_cmd_ret(&dcc->root, lstart, 1535 &prev_dc, &next_dc, &insert_p, &insert_parent); 1536 if (dc) 1537 prev_dc = dc; 1538 1539 if (!prev_dc) { 1540 di.lstart = lstart; 1541 di.len = next_dc ? next_dc->di.lstart - lstart : len; 1542 di.len = min(di.len, len); 1543 di.start = start; 1544 } 1545 1546 while (1) { 1547 struct rb_node *node; 1548 bool merged = false; 1549 struct discard_cmd *tdc = NULL; 1550 1551 if (prev_dc) { 1552 di.lstart = prev_dc->di.lstart + prev_dc->di.len; 1553 if (di.lstart < lstart) 1554 di.lstart = lstart; 1555 if (di.lstart >= end) 1556 break; 1557 1558 if (!next_dc || next_dc->di.lstart > end) 1559 di.len = end - di.lstart; 1560 else 1561 di.len = next_dc->di.lstart - di.lstart; 1562 di.start = start + di.lstart - lstart; 1563 } 1564 1565 if (!di.len) 1566 goto next; 1567 1568 if (prev_dc && prev_dc->state == D_PREP && 1569 prev_dc->bdev == bdev && 1570 __is_discard_back_mergeable(&di, &prev_dc->di, 1571 max_discard_blocks)) { 1572 prev_dc->di.len += di.len; 1573 dcc->undiscard_blks += di.len; 1574 __relocate_discard_cmd(dcc, prev_dc); 1575 di = prev_dc->di; 1576 tdc = prev_dc; 1577 merged = true; 1578 } 1579 1580 if (next_dc && next_dc->state == D_PREP && 1581 next_dc->bdev == bdev && 1582 __is_discard_front_mergeable(&di, &next_dc->di, 1583 max_discard_blocks)) { 1584 next_dc->di.lstart = di.lstart; 1585 next_dc->di.len += di.len; 1586 next_dc->di.start = di.start; 1587 dcc->undiscard_blks += di.len; 1588 __relocate_discard_cmd(dcc, next_dc); 1589 if (tdc) 1590 __remove_discard_cmd(sbi, tdc); 1591 merged = true; 1592 } 1593 1594 if (!merged) 1595 __insert_discard_cmd(sbi, bdev, 1596 di.lstart, di.start, di.len); 1597 next: 1598 prev_dc = next_dc; 1599 if (!prev_dc) 1600 break; 1601 1602 node = rb_next(&prev_dc->rb_node); 1603 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1604 } 1605 } 1606 1607 #ifdef CONFIG_BLK_DEV_ZONED 1608 static void __queue_zone_reset_cmd(struct f2fs_sb_info *sbi, 1609 struct block_device *bdev, block_t blkstart, block_t lblkstart, 1610 block_t blklen) 1611 { 1612 trace_f2fs_queue_reset_zone(bdev, blkstart); 1613 1614 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); 1615 __insert_discard_cmd(sbi, bdev, lblkstart, blkstart, blklen); 1616 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); 1617 } 1618 #endif 1619 1620 static void __queue_discard_cmd(struct f2fs_sb_info *sbi, 1621 struct block_device *bdev, block_t blkstart, block_t blklen) 1622 { 1623 block_t lblkstart = blkstart; 1624 1625 if (!f2fs_bdev_support_discard(bdev)) 1626 return; 1627 1628 trace_f2fs_queue_discard(bdev, blkstart, blklen); 1629 1630 if (f2fs_is_multi_device(sbi)) { 1631 int devi = f2fs_target_device_index(sbi, blkstart); 1632 1633 blkstart -= FDEV(devi).start_blk; 1634 } 1635 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); 1636 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); 1637 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); 1638 } 1639 1640 static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, 1641 struct discard_policy *dpolicy, int *issued) 1642 { 1643 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1644 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1645 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1646 struct discard_cmd *dc; 1647 struct blk_plug plug; 1648 bool io_interrupted = false; 1649 1650 mutex_lock(&dcc->cmd_lock); 1651 dc = __lookup_discard_cmd_ret(&dcc->root, dcc->next_pos, 1652 &prev_dc, &next_dc, &insert_p, &insert_parent); 1653 if (!dc) 1654 dc = next_dc; 1655 1656 blk_start_plug(&plug); 1657 1658 while (dc) { 1659 struct rb_node *node; 1660 int err = 0; 1661 1662 if (dc->state != D_PREP) 1663 goto next; 1664 1665 if (*issued > 0 && unlikely(freezing(current))) 1666 break; 1667 1668 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { 1669 io_interrupted = true; 1670 break; 1671 } 1672 1673 dcc->next_pos = dc->di.lstart + dc->di.len; 1674 err = __submit_discard_cmd(sbi, dpolicy, dc, issued); 1675 1676 if (*issued >= dpolicy->max_requests) 1677 break; 1678 next: 1679 node = rb_next(&dc->rb_node); 1680 if (err) 1681 __remove_discard_cmd(sbi, dc); 1682 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 1683 } 1684 1685 blk_finish_plug(&plug); 1686 1687 if (!dc) 1688 dcc->next_pos = 0; 1689 1690 mutex_unlock(&dcc->cmd_lock); 1691 1692 if (!(*issued) && io_interrupted) 1693 *issued = -1; 1694 } 1695 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1696 struct discard_policy *dpolicy); 1697 1698 static int __issue_discard_cmd(struct f2fs_sb_info *sbi, 1699 struct discard_policy *dpolicy) 1700 { 1701 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1702 struct list_head *pend_list; 1703 struct discard_cmd *dc, *tmp; 1704 struct blk_plug plug; 1705 int i, issued; 1706 bool io_interrupted = false; 1707 bool suspended = false; 1708 1709 if (dpolicy->timeout) 1710 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT); 1711 1712 retry: 1713 issued = 0; 1714 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1715 if (dpolicy->timeout && 1716 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) 1717 break; 1718 1719 if (i + 1 < dpolicy->granularity) 1720 break; 1721 1722 if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) { 1723 __issue_discard_cmd_orderly(sbi, dpolicy, &issued); 1724 return issued; 1725 } 1726 1727 pend_list = &dcc->pend_list[i]; 1728 1729 mutex_lock(&dcc->cmd_lock); 1730 if (list_empty(pend_list)) 1731 goto next; 1732 if (unlikely(dcc->rbtree_check)) 1733 f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi)); 1734 blk_start_plug(&plug); 1735 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1736 f2fs_bug_on(sbi, dc->state != D_PREP); 1737 1738 if (issued > 0 && unlikely(freezing(current))) { 1739 suspended = true; 1740 break; 1741 } 1742 1743 if (dpolicy->timeout && 1744 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) 1745 break; 1746 1747 if (dpolicy->io_aware && i < dpolicy->io_aware_gran && 1748 !is_idle(sbi, DISCARD_TIME)) { 1749 io_interrupted = true; 1750 break; 1751 } 1752 1753 __submit_discard_cmd(sbi, dpolicy, dc, &issued); 1754 1755 if (issued >= dpolicy->max_requests) 1756 break; 1757 } 1758 blk_finish_plug(&plug); 1759 next: 1760 mutex_unlock(&dcc->cmd_lock); 1761 1762 if (issued >= dpolicy->max_requests || io_interrupted || 1763 suspended) 1764 break; 1765 } 1766 1767 if (dpolicy->type == DPOLICY_UMOUNT && issued) { 1768 __wait_all_discard_cmd(sbi, dpolicy); 1769 goto retry; 1770 } 1771 1772 if (!issued && io_interrupted) 1773 issued = -1; 1774 1775 return issued; 1776 } 1777 1778 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) 1779 { 1780 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1781 struct list_head *pend_list; 1782 struct discard_cmd *dc, *tmp; 1783 int i; 1784 bool dropped = false; 1785 1786 mutex_lock(&dcc->cmd_lock); 1787 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { 1788 pend_list = &dcc->pend_list[i]; 1789 list_for_each_entry_safe(dc, tmp, pend_list, list) { 1790 f2fs_bug_on(sbi, dc->state != D_PREP); 1791 __remove_discard_cmd(sbi, dc); 1792 dropped = true; 1793 } 1794 } 1795 mutex_unlock(&dcc->cmd_lock); 1796 1797 return dropped; 1798 } 1799 1800 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) 1801 { 1802 __drop_discard_cmd(sbi); 1803 } 1804 1805 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, 1806 struct discard_cmd *dc) 1807 { 1808 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1809 unsigned int len = 0; 1810 1811 wait_for_completion_io(&dc->wait); 1812 mutex_lock(&dcc->cmd_lock); 1813 f2fs_bug_on(sbi, dc->state != D_DONE); 1814 dc->ref--; 1815 if (!dc->ref) { 1816 if (!dc->error) 1817 len = dc->di.len; 1818 __remove_discard_cmd(sbi, dc); 1819 } 1820 mutex_unlock(&dcc->cmd_lock); 1821 1822 return len; 1823 } 1824 1825 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, 1826 struct discard_policy *dpolicy, 1827 block_t start, block_t end) 1828 { 1829 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1830 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? 1831 &(dcc->fstrim_list) : &(dcc->wait_list); 1832 struct discard_cmd *dc = NULL, *iter, *tmp; 1833 unsigned int trimmed = 0; 1834 1835 next: 1836 dc = NULL; 1837 1838 mutex_lock(&dcc->cmd_lock); 1839 list_for_each_entry_safe(iter, tmp, wait_list, list) { 1840 if (iter->di.lstart + iter->di.len <= start || 1841 end <= iter->di.lstart) 1842 continue; 1843 if (iter->di.len < dpolicy->granularity) 1844 continue; 1845 if (iter->state == D_DONE && !iter->ref) { 1846 wait_for_completion_io(&iter->wait); 1847 if (!iter->error) 1848 trimmed += iter->di.len; 1849 __remove_discard_cmd(sbi, iter); 1850 } else { 1851 iter->ref++; 1852 dc = iter; 1853 break; 1854 } 1855 } 1856 mutex_unlock(&dcc->cmd_lock); 1857 1858 if (dc) { 1859 trimmed += __wait_one_discard_bio(sbi, dc); 1860 goto next; 1861 } 1862 1863 return trimmed; 1864 } 1865 1866 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, 1867 struct discard_policy *dpolicy) 1868 { 1869 struct discard_policy dp; 1870 unsigned int discard_blks; 1871 1872 if (dpolicy) 1873 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); 1874 1875 /* wait all */ 1876 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY); 1877 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1878 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY); 1879 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); 1880 1881 return discard_blks; 1882 } 1883 1884 void f2fs_drop_discard_cmd_range(struct f2fs_sb_info *sbi, 1885 block_t start, block_t len) 1886 { 1887 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1888 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 1889 struct rb_node **insert_p = NULL, *insert_parent = NULL; 1890 struct discard_cmd *dc, *wait_dc; 1891 u64 cur = start; 1892 u64 end = (u64)start + len; 1893 int count; 1894 1895 if (!f2fs_realtime_discard_enable(sbi)) 1896 return; 1897 1898 next: 1899 count = 0; 1900 wait_dc = NULL; 1901 1902 mutex_lock(&dcc->cmd_lock); 1903 while (cur < end) { 1904 dc = __lookup_discard_cmd_ret(&dcc->root, cur, 1905 &prev_dc, &next_dc, &insert_p, &insert_parent); 1906 if (!dc) 1907 dc = next_dc; 1908 1909 if (!dc || (u64)dc->di.lstart >= end) 1910 break; 1911 1912 if (dc->state == D_PREP) { 1913 cur = (u64)dc->di.lstart + dc->di.len; 1914 __remove_discard_cmd(sbi, dc); 1915 if (++count >= MAX_DISCARD_DROP_COUNT) 1916 break; 1917 continue; 1918 } 1919 1920 dc->ref++; 1921 cur = (u64)dc->di.lstart + dc->di.len; 1922 wait_dc = dc; 1923 break; 1924 } 1925 mutex_unlock(&dcc->cmd_lock); 1926 1927 if (wait_dc) { 1928 __wait_one_discard_bio(sbi, wait_dc); 1929 goto next; 1930 } 1931 1932 if (count >= MAX_DISCARD_DROP_COUNT) { 1933 cond_resched(); 1934 goto next; 1935 } 1936 } 1937 1938 /* This should be covered by global mutex, &sit_i->sentry_lock */ 1939 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) 1940 { 1941 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1942 struct discard_cmd *dc; 1943 bool need_wait = false; 1944 1945 mutex_lock(&dcc->cmd_lock); 1946 dc = __lookup_discard_cmd(sbi, blkaddr); 1947 #ifdef CONFIG_BLK_DEV_ZONED 1948 if (dc && f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(dc->bdev)) { 1949 int devi = f2fs_bdev_index(sbi, dc->bdev); 1950 1951 if (devi < 0) { 1952 mutex_unlock(&dcc->cmd_lock); 1953 return; 1954 } 1955 1956 if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) { 1957 /* force submit zone reset */ 1958 if (dc->state == D_PREP) 1959 __submit_zone_reset_cmd(sbi, dc, REQ_SYNC, 1960 &dcc->wait_list, NULL); 1961 dc->ref++; 1962 mutex_unlock(&dcc->cmd_lock); 1963 /* wait zone reset */ 1964 __wait_one_discard_bio(sbi, dc); 1965 return; 1966 } 1967 } 1968 #endif 1969 if (dc) { 1970 if (dc->state == D_PREP) { 1971 __punch_discard_cmd(sbi, dc, blkaddr); 1972 } else { 1973 dc->ref++; 1974 need_wait = true; 1975 } 1976 } 1977 mutex_unlock(&dcc->cmd_lock); 1978 1979 if (need_wait) 1980 __wait_one_discard_bio(sbi, dc); 1981 } 1982 1983 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) 1984 { 1985 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 1986 1987 if (dcc && dcc->f2fs_issue_discard) { 1988 struct task_struct *discard_thread = dcc->f2fs_issue_discard; 1989 1990 dcc->f2fs_issue_discard = NULL; 1991 kthread_stop(discard_thread); 1992 } 1993 } 1994 1995 /** 1996 * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT 1997 * @sbi: the f2fs_sb_info data for discard cmd to issue 1998 * 1999 * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped 2000 * 2001 * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false. 2002 */ 2003 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check) 2004 { 2005 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2006 struct discard_policy dpolicy; 2007 bool dropped; 2008 2009 if (!atomic_read(&dcc->discard_cmd_cnt)) 2010 return true; 2011 2012 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, 2013 dcc->discard_granularity); 2014 __issue_discard_cmd(sbi, &dpolicy); 2015 dropped = __drop_discard_cmd(sbi); 2016 2017 /* just to make sure there is no pending discard commands */ 2018 __wait_all_discard_cmd(sbi, NULL); 2019 2020 f2fs_bug_on(sbi, need_check && atomic_read(&dcc->discard_cmd_cnt)); 2021 return !dropped; 2022 } 2023 2024 static int issue_discard_thread(void *data) 2025 { 2026 struct f2fs_sb_info *sbi = data; 2027 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2028 wait_queue_head_t *q = &dcc->discard_wait_queue; 2029 struct discard_policy dpolicy; 2030 unsigned int wait_ms = dcc->min_discard_issue_time; 2031 int issued; 2032 2033 set_freezable(); 2034 2035 do { 2036 wait_event_freezable_timeout(*q, 2037 kthread_should_stop() || dcc->discard_wake, 2038 msecs_to_jiffies(wait_ms)); 2039 2040 if (sbi->gc_mode == GC_URGENT_HIGH || 2041 !f2fs_available_free_memory(sbi, DISCARD_CACHE)) 2042 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 2043 MIN_DISCARD_GRANULARITY); 2044 else 2045 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, 2046 dcc->discard_granularity); 2047 2048 if (dcc->discard_wake) 2049 dcc->discard_wake = false; 2050 2051 /* clean up pending candidates before going to sleep */ 2052 if (atomic_read(&dcc->queued_discard)) 2053 __wait_all_discard_cmd(sbi, NULL); 2054 2055 if (f2fs_readonly(sbi->sb)) 2056 continue; 2057 if (kthread_should_stop()) 2058 return 0; 2059 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) || 2060 !atomic_read(&dcc->discard_cmd_cnt)) { 2061 wait_ms = dpolicy.max_interval; 2062 continue; 2063 } 2064 2065 sb_start_intwrite(sbi->sb); 2066 2067 issued = __issue_discard_cmd(sbi, &dpolicy); 2068 if (issued > 0) { 2069 __wait_all_discard_cmd(sbi, &dpolicy); 2070 wait_ms = dpolicy.min_interval; 2071 } else if (issued == -1) { 2072 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); 2073 if (!wait_ms) 2074 wait_ms = dpolicy.mid_interval; 2075 } else { 2076 wait_ms = dpolicy.max_interval; 2077 } 2078 if (!atomic_read(&dcc->discard_cmd_cnt)) 2079 wait_ms = dpolicy.max_interval; 2080 2081 sb_end_intwrite(sbi->sb); 2082 2083 } while (!kthread_should_stop()); 2084 return 0; 2085 } 2086 2087 #ifdef CONFIG_BLK_DEV_ZONED 2088 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, 2089 struct block_device *bdev, block_t blkstart, block_t blklen) 2090 { 2091 sector_t sector, nr_sects; 2092 block_t lblkstart = blkstart; 2093 int devi = 0; 2094 u64 remainder = 0; 2095 2096 if (f2fs_is_multi_device(sbi)) { 2097 devi = f2fs_target_device_index(sbi, blkstart); 2098 if (blkstart < FDEV(devi).start_blk || 2099 blkstart > FDEV(devi).end_blk) { 2100 f2fs_err(sbi, "Invalid block %x", blkstart); 2101 return -EIO; 2102 } 2103 blkstart -= FDEV(devi).start_blk; 2104 } 2105 2106 /* For sequential zones, reset the zone write pointer */ 2107 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { 2108 sector = SECTOR_FROM_BLOCK(sbi, blkstart); 2109 nr_sects = SECTOR_FROM_BLOCK(sbi, blklen); 2110 div64_u64_rem(sector, bdev_zone_sectors(bdev), &remainder); 2111 2112 if (remainder || nr_sects != bdev_zone_sectors(bdev)) { 2113 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)", 2114 devi, sbi->s_ndevs ? FDEV(devi).path : "", 2115 blkstart, blklen); 2116 return -EIO; 2117 } 2118 2119 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) { 2120 unsigned int nofs_flags; 2121 int ret; 2122 2123 trace_f2fs_issue_reset_zone(bdev, blkstart); 2124 nofs_flags = memalloc_nofs_save(); 2125 ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, 2126 sector, nr_sects); 2127 memalloc_nofs_restore(nofs_flags); 2128 return ret; 2129 } 2130 2131 __queue_zone_reset_cmd(sbi, bdev, blkstart, lblkstart, blklen); 2132 return 0; 2133 } 2134 2135 /* For conventional zones, use regular discard if supported */ 2136 __queue_discard_cmd(sbi, bdev, lblkstart, blklen); 2137 return 0; 2138 } 2139 #endif 2140 2141 static int __issue_discard_async(struct f2fs_sb_info *sbi, 2142 struct block_device *bdev, block_t blkstart, block_t blklen) 2143 { 2144 #ifdef CONFIG_BLK_DEV_ZONED 2145 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) 2146 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); 2147 #endif 2148 __queue_discard_cmd(sbi, bdev, blkstart, blklen); 2149 return 0; 2150 } 2151 2152 static int f2fs_issue_discard(struct f2fs_sb_info *sbi, 2153 block_t blkstart, block_t blklen) 2154 { 2155 sector_t start = blkstart, len = 0; 2156 struct block_device *bdev; 2157 struct seg_entry *se; 2158 unsigned int offset; 2159 block_t i; 2160 int err = 0; 2161 2162 bdev = f2fs_target_device(sbi, blkstart, NULL); 2163 2164 for (i = blkstart; i < blkstart + blklen; i++, len++) { 2165 if (i != start) { 2166 struct block_device *bdev2 = 2167 f2fs_target_device(sbi, i, NULL); 2168 2169 if (bdev2 != bdev) { 2170 err = __issue_discard_async(sbi, bdev, 2171 start, len); 2172 if (err) 2173 return err; 2174 bdev = bdev2; 2175 start = i; 2176 len = 0; 2177 } 2178 } 2179 2180 se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); 2181 offset = GET_BLKOFF_FROM_SEG0(sbi, i); 2182 2183 if (f2fs_block_unit_discard(sbi) && 2184 !f2fs_test_and_set_bit(offset, se->discard_map)) 2185 sbi->discard_blks--; 2186 } 2187 2188 if (len) 2189 err = __issue_discard_async(sbi, bdev, start, len); 2190 return err; 2191 } 2192 2193 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, 2194 bool check_only) 2195 { 2196 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 2197 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); 2198 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 2199 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 2200 unsigned long *discard_map = (unsigned long *)se->discard_map; 2201 unsigned long *dmap = SIT_I(sbi)->tmp_map; 2202 unsigned int start = 0, end = -1; 2203 bool force = (cpc->reason & CP_DISCARD); 2204 struct discard_entry *de = NULL; 2205 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; 2206 int i; 2207 2208 if (se->valid_blocks == BLKS_PER_SEG(sbi) || 2209 !f2fs_hw_support_discard(sbi) || 2210 !f2fs_block_unit_discard(sbi)) 2211 return false; 2212 2213 if (!force) { 2214 if (!f2fs_realtime_discard_enable(sbi) || 2215 (!se->valid_blocks && 2216 !is_curseg(sbi, cpc->trim_start)) || 2217 SM_I(sbi)->dcc_info->nr_discards >= 2218 SM_I(sbi)->dcc_info->max_discards) 2219 return false; 2220 } 2221 2222 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ 2223 for (i = 0; i < entries; i++) 2224 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : 2225 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; 2226 2227 while (force || SM_I(sbi)->dcc_info->nr_discards <= 2228 SM_I(sbi)->dcc_info->max_discards) { 2229 start = __find_rev_next_bit(dmap, BLKS_PER_SEG(sbi), end + 1); 2230 if (start >= BLKS_PER_SEG(sbi)) 2231 break; 2232 2233 end = __find_rev_next_zero_bit(dmap, 2234 BLKS_PER_SEG(sbi), start + 1); 2235 if (force && start && end != BLKS_PER_SEG(sbi) && 2236 (end - start) < cpc->trim_minlen) 2237 continue; 2238 2239 if (check_only) 2240 return true; 2241 2242 if (!de) { 2243 de = f2fs_kmem_cache_alloc(discard_entry_slab, 2244 GFP_F2FS_ZERO, true, NULL); 2245 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); 2246 list_add_tail(&de->list, head); 2247 } 2248 2249 for (i = start; i < end; i++) 2250 __set_bit_le(i, (void *)de->discard_map); 2251 2252 SM_I(sbi)->dcc_info->nr_discards += end - start; 2253 } 2254 return false; 2255 } 2256 2257 static void release_discard_addr(struct discard_entry *entry) 2258 { 2259 list_del(&entry->list); 2260 kmem_cache_free(discard_entry_slab, entry); 2261 } 2262 2263 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) 2264 { 2265 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); 2266 struct discard_entry *entry, *this; 2267 2268 /* drop caches */ 2269 list_for_each_entry_safe(entry, this, head, list) 2270 release_discard_addr(entry); 2271 } 2272 2273 /* 2274 * Should call f2fs_clear_prefree_segments after checkpoint is done. 2275 */ 2276 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) 2277 { 2278 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 2279 unsigned int segno; 2280 2281 mutex_lock(&dirty_i->seglist_lock); 2282 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) 2283 __set_test_and_free(sbi, segno, false); 2284 mutex_unlock(&dirty_i->seglist_lock); 2285 } 2286 2287 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, 2288 struct cp_control *cpc) 2289 { 2290 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2291 struct list_head *head = &dcc->entry_list; 2292 struct discard_entry *entry, *this; 2293 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 2294 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; 2295 unsigned int start = 0, end = -1; 2296 unsigned int secno, start_segno; 2297 bool force = (cpc->reason & CP_DISCARD); 2298 bool section_alignment = F2FS_OPTION(sbi).discard_unit == 2299 DISCARD_UNIT_SECTION; 2300 2301 if (f2fs_lfs_mode(sbi) && __is_large_section(sbi)) 2302 section_alignment = true; 2303 2304 mutex_lock(&dirty_i->seglist_lock); 2305 2306 while (1) { 2307 int i; 2308 2309 if (section_alignment && end != -1) 2310 end--; 2311 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); 2312 if (start >= MAIN_SEGS(sbi)) 2313 break; 2314 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), 2315 start + 1); 2316 2317 if (section_alignment) { 2318 start = rounddown(start, SEGS_PER_SEC(sbi)); 2319 end = roundup(end, SEGS_PER_SEC(sbi)); 2320 } 2321 2322 for (i = start; i < end; i++) { 2323 if (test_and_clear_bit(i, prefree_map)) 2324 dirty_i->nr_dirty[PRE]--; 2325 } 2326 2327 if (!f2fs_realtime_discard_enable(sbi)) 2328 continue; 2329 2330 if (force && start >= cpc->trim_start && 2331 (end - 1) <= cpc->trim_end) 2332 continue; 2333 2334 /* Should cover 2MB zoned device for zone-based reset */ 2335 if (!f2fs_sb_has_blkzoned(sbi) && 2336 (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi))) { 2337 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), 2338 SEGS_TO_BLKS(sbi, end - start)); 2339 continue; 2340 } 2341 next: 2342 secno = GET_SEC_FROM_SEG(sbi, start); 2343 start_segno = GET_SEG_FROM_SEC(sbi, secno); 2344 if (!is_cursec(sbi, secno) && 2345 !get_valid_blocks(sbi, start, true)) 2346 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), 2347 BLKS_PER_SEC(sbi)); 2348 2349 start = start_segno + SEGS_PER_SEC(sbi); 2350 if (start < end) 2351 goto next; 2352 else 2353 end = start - 1; 2354 } 2355 mutex_unlock(&dirty_i->seglist_lock); 2356 2357 if (!f2fs_block_unit_discard(sbi)) 2358 goto wakeup; 2359 2360 /* send small discards */ 2361 list_for_each_entry_safe(entry, this, head, list) { 2362 unsigned int cur_pos = 0, next_pos, len, total_len = 0; 2363 bool is_valid = test_bit_le(0, entry->discard_map); 2364 2365 find_next: 2366 if (is_valid) { 2367 next_pos = find_next_zero_bit_le(entry->discard_map, 2368 BLKS_PER_SEG(sbi), cur_pos); 2369 len = next_pos - cur_pos; 2370 2371 if (f2fs_sb_has_blkzoned(sbi) || 2372 (force && len < cpc->trim_minlen)) 2373 goto skip; 2374 2375 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, 2376 len); 2377 total_len += len; 2378 } else { 2379 next_pos = find_next_bit_le(entry->discard_map, 2380 BLKS_PER_SEG(sbi), cur_pos); 2381 } 2382 skip: 2383 cur_pos = next_pos; 2384 is_valid = !is_valid; 2385 2386 if (cur_pos < BLKS_PER_SEG(sbi)) 2387 goto find_next; 2388 2389 release_discard_addr(entry); 2390 dcc->nr_discards -= total_len; 2391 } 2392 2393 wakeup: 2394 wake_up_discard_thread(sbi, false); 2395 } 2396 2397 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi) 2398 { 2399 dev_t dev = sbi->sb->s_bdev->bd_dev; 2400 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2401 int err = 0; 2402 2403 if (f2fs_sb_has_readonly(sbi)) { 2404 f2fs_info(sbi, 2405 "Skip to start discard thread for readonly image"); 2406 return 0; 2407 } 2408 2409 if (!f2fs_realtime_discard_enable(sbi)) 2410 return 0; 2411 2412 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, 2413 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev)); 2414 if (IS_ERR(dcc->f2fs_issue_discard)) { 2415 err = PTR_ERR(dcc->f2fs_issue_discard); 2416 dcc->f2fs_issue_discard = NULL; 2417 } 2418 2419 return err; 2420 } 2421 2422 static int create_discard_cmd_control(struct f2fs_sb_info *sbi) 2423 { 2424 struct discard_cmd_control *dcc; 2425 int err = 0, i; 2426 2427 if (SM_I(sbi)->dcc_info) { 2428 dcc = SM_I(sbi)->dcc_info; 2429 goto init_thread; 2430 } 2431 2432 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); 2433 if (!dcc) 2434 return -ENOMEM; 2435 2436 dcc->discard_io_aware_gran = MAX_PLIST_NUM; 2437 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; 2438 dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY; 2439 dcc->discard_io_aware = DPOLICY_IO_AWARE_ENABLE; 2440 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT || 2441 F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) 2442 dcc->discard_granularity = BLKS_PER_SEG(sbi); 2443 2444 INIT_LIST_HEAD(&dcc->entry_list); 2445 for (i = 0; i < MAX_PLIST_NUM; i++) 2446 INIT_LIST_HEAD(&dcc->pend_list[i]); 2447 INIT_LIST_HEAD(&dcc->wait_list); 2448 INIT_LIST_HEAD(&dcc->fstrim_list); 2449 mutex_init(&dcc->cmd_lock); 2450 atomic_set(&dcc->issued_discard, 0); 2451 atomic_set(&dcc->queued_discard, 0); 2452 atomic_set(&dcc->discard_cmd_cnt, 0); 2453 dcc->nr_discards = 0; 2454 dcc->max_discards = SEGS_TO_BLKS(sbi, MAIN_SEGS(sbi)); 2455 dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST; 2456 dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME; 2457 dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME; 2458 dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME; 2459 dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL; 2460 dcc->undiscard_blks = 0; 2461 dcc->next_pos = 0; 2462 dcc->root = RB_ROOT_CACHED; 2463 dcc->rbtree_check = false; 2464 2465 init_waitqueue_head(&dcc->discard_wait_queue); 2466 SM_I(sbi)->dcc_info = dcc; 2467 init_thread: 2468 err = f2fs_start_discard_thread(sbi); 2469 if (err) { 2470 kfree(dcc); 2471 SM_I(sbi)->dcc_info = NULL; 2472 } 2473 2474 return err; 2475 } 2476 2477 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) 2478 { 2479 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 2480 2481 if (!dcc) 2482 return; 2483 2484 f2fs_stop_discard_thread(sbi); 2485 2486 /* 2487 * Recovery can cache discard commands, so in error path of 2488 * fill_super(), it needs to give a chance to handle them. 2489 */ 2490 f2fs_issue_discard_timeout(sbi, true); 2491 2492 kfree(dcc); 2493 SM_I(sbi)->dcc_info = NULL; 2494 } 2495 2496 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) 2497 { 2498 struct sit_info *sit_i = SIT_I(sbi); 2499 2500 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) { 2501 sit_i->dirty_sentries++; 2502 return false; 2503 } 2504 2505 return true; 2506 } 2507 2508 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, 2509 unsigned int segno, int modified) 2510 { 2511 struct seg_entry *se = get_seg_entry(sbi, segno); 2512 2513 se->type = type; 2514 if (modified) 2515 __mark_sit_entry_dirty(sbi, segno); 2516 } 2517 2518 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi, 2519 block_t blkaddr) 2520 { 2521 unsigned int segno = GET_SEGNO(sbi, blkaddr); 2522 2523 if (segno == NULL_SEGNO) 2524 return 0; 2525 return get_seg_entry(sbi, segno)->mtime; 2526 } 2527 2528 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr, 2529 unsigned long long old_mtime) 2530 { 2531 struct seg_entry *se; 2532 unsigned int segno = GET_SEGNO(sbi, blkaddr); 2533 unsigned long long ctime = get_mtime(sbi, false); 2534 unsigned long long mtime = old_mtime ? old_mtime : ctime; 2535 2536 if (segno == NULL_SEGNO) 2537 return; 2538 2539 se = get_seg_entry(sbi, segno); 2540 2541 if (!se->mtime) 2542 se->mtime = mtime; 2543 else 2544 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime, 2545 se->valid_blocks + 1); 2546 2547 if (ctime > SIT_I(sbi)->max_mtime) 2548 SIT_I(sbi)->max_mtime = ctime; 2549 } 2550 2551 /* 2552 * NOTE: when updating multiple blocks at the same time, please ensure 2553 * that the consecutive input blocks belong to the same segment. 2554 */ 2555 static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_entry *se, 2556 unsigned int segno, block_t blkaddr, unsigned int offset, int del) 2557 { 2558 bool exist; 2559 int i; 2560 int del_count = -del; 2561 2562 f2fs_bug_on(sbi, GET_SEGNO(sbi, blkaddr) != GET_SEGNO(sbi, blkaddr + del_count - 1)); 2563 2564 for (i = 0; i < del_count; i++) { 2565 exist = f2fs_test_and_clear_bit(offset + i, se->cur_valid_map); 2566 if (unlikely(!exist)) { 2567 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", blkaddr + i); 2568 f2fs_bug_on(sbi, 1); 2569 se->valid_blocks++; 2570 del += 1; 2571 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 2572 /* 2573 * If checkpoints are off, we must not reuse data that 2574 * was used in the previous checkpoint. If it was used 2575 * before, we must track that to know how much space we 2576 * really have. 2577 */ 2578 if (f2fs_test_bit(offset + i, se->ckpt_valid_map)) { 2579 spin_lock(&sbi->stat_lock); 2580 sbi->unusable_block_count++; 2581 spin_unlock(&sbi->stat_lock); 2582 } 2583 } 2584 2585 if (f2fs_block_unit_discard(sbi) && 2586 f2fs_test_and_clear_bit(offset + i, se->discard_map)) 2587 sbi->discard_blks++; 2588 2589 if (!f2fs_test_bit(offset + i, se->ckpt_valid_map)) { 2590 se->ckpt_valid_blocks -= 1; 2591 if (__is_large_section(sbi)) 2592 get_sec_entry(sbi, segno)->ckpt_valid_blocks -= 1; 2593 } 2594 } 2595 2596 if (__is_large_section(sbi)) 2597 sanity_check_valid_blocks(sbi, segno); 2598 2599 return del; 2600 } 2601 2602 static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry *se, 2603 unsigned int segno, block_t blkaddr, unsigned int offset, int del) 2604 { 2605 bool exist; 2606 int del_count = del; 2607 int i; 2608 2609 f2fs_bug_on(sbi, GET_SEGNO(sbi, blkaddr) != GET_SEGNO(sbi, blkaddr + del_count - 1)); 2610 2611 for (i = 0; i < del_count; i++) { 2612 exist = f2fs_test_and_set_bit(offset + i, se->cur_valid_map); 2613 if (unlikely(exist)) { 2614 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", blkaddr + i); 2615 f2fs_bug_on(sbi, 1); 2616 se->valid_blocks--; 2617 del -= 1; 2618 continue; 2619 } 2620 2621 if (f2fs_block_unit_discard(sbi) && 2622 !f2fs_test_and_set_bit(offset + i, se->discard_map)) 2623 sbi->discard_blks--; 2624 2625 /* 2626 * SSR should never reuse block which is checkpointed 2627 * or newly invalidated. 2628 */ 2629 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { 2630 if (!f2fs_test_and_set_bit(offset + i, se->ckpt_valid_map)) { 2631 se->ckpt_valid_blocks++; 2632 if (__is_large_section(sbi)) 2633 get_sec_entry(sbi, segno)->ckpt_valid_blocks++; 2634 } 2635 } 2636 2637 if (!f2fs_test_bit(offset + i, se->ckpt_valid_map)) { 2638 se->ckpt_valid_blocks += 1; 2639 if (__is_large_section(sbi)) 2640 get_sec_entry(sbi, segno)->ckpt_valid_blocks += 1; 2641 } 2642 } 2643 2644 if (__is_large_section(sbi)) 2645 sanity_check_valid_blocks(sbi, segno); 2646 2647 return del; 2648 } 2649 2650 /* 2651 * If releasing blocks, this function supports updating multiple consecutive blocks 2652 * at one time, but please note that these consecutive blocks need to belong to the 2653 * same segment. 2654 */ 2655 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) 2656 { 2657 struct seg_entry *se; 2658 unsigned int segno, offset; 2659 long int new_vblocks; 2660 2661 segno = GET_SEGNO(sbi, blkaddr); 2662 if (segno == NULL_SEGNO) 2663 return; 2664 2665 se = get_seg_entry(sbi, segno); 2666 new_vblocks = se->valid_blocks + del; 2667 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2668 2669 f2fs_bug_on(sbi, (new_vblocks < 0 || 2670 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno)))); 2671 2672 se->valid_blocks = new_vblocks; 2673 2674 /* Update valid block bitmap */ 2675 if (del > 0) { 2676 del = update_sit_entry_for_alloc(sbi, se, segno, blkaddr, offset, del); 2677 } else { 2678 del = update_sit_entry_for_release(sbi, se, segno, blkaddr, offset, del); 2679 } 2680 2681 __mark_sit_entry_dirty(sbi, segno); 2682 2683 /* update total number of valid blocks to be written in ckpt area */ 2684 SIT_I(sbi)->written_valid_blocks += del; 2685 2686 if (__is_large_section(sbi)) 2687 get_sec_entry(sbi, segno)->valid_blocks += del; 2688 } 2689 2690 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr, 2691 unsigned int len) 2692 { 2693 unsigned int segno = GET_SEGNO(sbi, addr); 2694 struct sit_info *sit_i = SIT_I(sbi); 2695 block_t addr_start = addr, addr_end = addr + len - 1; 2696 unsigned int seg_num; 2697 unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt; 2698 2699 if (len == 0) 2700 return; 2701 2702 seg_num = GET_SEGNO(sbi, addr_end) - segno + 1; 2703 2704 f2fs_bug_on(sbi, addr == NULL_ADDR); 2705 if (addr == NEW_ADDR || addr == COMPRESS_ADDR) 2706 return; 2707 2708 f2fs_invalidate_internal_cache(sbi, addr, len); 2709 2710 /* add it into sit main buffer */ 2711 down_write(&sit_i->sentry_lock); 2712 2713 if (seg_num == 1) 2714 cnt = len; 2715 else 2716 cnt = max_blocks - GET_BLKOFF_FROM_SEG0(sbi, addr); 2717 2718 do { 2719 update_segment_mtime(sbi, addr_start, 0); 2720 update_sit_entry(sbi, addr_start, -cnt); 2721 2722 /* add it into dirty seglist */ 2723 locate_dirty_segment(sbi, segno); 2724 2725 /* update @addr_start and @cnt and @segno */ 2726 addr_start = START_BLOCK(sbi, ++segno); 2727 if (++i == seg_num) 2728 cnt = GET_BLKOFF_FROM_SEG0(sbi, addr_end) + 1; 2729 else 2730 cnt = max_blocks; 2731 } while (i <= seg_num); 2732 2733 up_write(&sit_i->sentry_lock); 2734 } 2735 2736 void f2fs_reserve_device_alias(struct f2fs_sb_info *sbi, block_t addr, 2737 unsigned int len) 2738 { 2739 unsigned int segno = GET_SEGNO(sbi, addr); 2740 struct sit_info *sit_i = SIT_I(sbi); 2741 block_t addr_start = addr, addr_end = addr + len - 1; 2742 unsigned int seg_num; 2743 unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt; 2744 2745 if (len == 0) 2746 return; 2747 2748 seg_num = GET_SEGNO(sbi, addr_end) - segno + 1; 2749 2750 down_write(&sit_i->sentry_lock); 2751 2752 if (seg_num == 1) 2753 cnt = len; 2754 else 2755 cnt = max_blocks - GET_BLKOFF_FROM_SEG0(sbi, addr); 2756 2757 do { 2758 update_segment_mtime(sbi, addr_start, 0); 2759 update_sit_entry(sbi, addr_start, cnt); 2760 __set_test_and_inuse(sbi, segno); 2761 2762 /* Remove the segment from PRE (prefree) to prevent checkpoint from freeing it! */ 2763 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 2764 if (test_and_clear_bit(segno, DIRTY_I(sbi)->dirty_segmap[PRE])) 2765 DIRTY_I(sbi)->nr_dirty[PRE]--; 2766 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 2767 2768 /* add it into dirty seglist */ 2769 locate_dirty_segment(sbi, segno); 2770 2771 /* update @addr_start and @cnt and @segno */ 2772 addr_start = START_BLOCK(sbi, ++segno); 2773 if (++i == seg_num) 2774 cnt = GET_BLKOFF_FROM_SEG0(sbi, addr_end) + 1; 2775 else 2776 cnt = max_blocks; 2777 } while (i <= seg_num); 2778 2779 up_write(&sit_i->sentry_lock); 2780 } 2781 2782 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) 2783 { 2784 struct sit_info *sit_i = SIT_I(sbi); 2785 unsigned int segno, offset; 2786 struct seg_entry *se; 2787 bool is_cp = false; 2788 2789 if (!__is_valid_data_blkaddr(blkaddr)) 2790 return true; 2791 2792 down_read(&sit_i->sentry_lock); 2793 2794 segno = GET_SEGNO(sbi, blkaddr); 2795 se = get_seg_entry(sbi, segno); 2796 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 2797 2798 if (f2fs_test_bit(offset, se->ckpt_valid_map)) 2799 is_cp = true; 2800 2801 up_read(&sit_i->sentry_lock); 2802 2803 return is_cp; 2804 } 2805 2806 static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type) 2807 { 2808 struct curseg_info *curseg = CURSEG_I(sbi, type); 2809 2810 if (sbi->ckpt->alloc_type[type] == SSR) 2811 return BLKS_PER_SEG(sbi); 2812 return curseg->next_blkoff; 2813 } 2814 2815 /* 2816 * Calculate the number of current summary blocks for writing 2817 */ 2818 int f2fs_nblocks_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) 2819 { 2820 int valid_sum_count = 0; 2821 int i, sum_in_block; 2822 2823 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 2824 if (sbi->ckpt->alloc_type[i] != SSR && for_ra) 2825 valid_sum_count += 2826 le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]); 2827 else 2828 valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); 2829 } 2830 2831 sum_in_block = (sbi->blocksize - 2 * sbi->sum_journal_size - 2832 SUM_FOOTER_SIZE) / SUMMARY_SIZE; 2833 if (valid_sum_count <= sum_in_block) 2834 return 1; 2835 else if ((valid_sum_count - sum_in_block) <= 2836 (sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE) 2837 return 2; 2838 return 3; 2839 } 2840 2841 2842 struct f2fs_cached_block *f2fs_get_sum_cache(struct f2fs_sb_info *sbi, 2843 unsigned int segno) 2844 { 2845 if (unlikely(f2fs_cp_error(sbi))) 2846 return ERR_PTR(-EIO); 2847 return f2fs_get_meta_cache_retry(sbi, GET_SUM_BLOCK(sbi, segno)); 2848 } 2849 2850 void f2fs_update_meta_block(struct f2fs_sb_info *sbi, 2851 void *src, block_t blk_addr) 2852 { 2853 struct f2fs_cached_block *entry; 2854 2855 entry = f2fs_grab_meta_cache(sbi, blk_addr); 2856 2857 memcpy(cache_address(entry), src, F2FS_BLKSIZE(sbi)); 2858 f2fs_mark_cache_dirty(entry); 2859 f2fs_put_cache(entry, true); 2860 } 2861 2862 static void write_sum_block(struct f2fs_sb_info *sbi, 2863 struct f2fs_summary_block *sum_blk, unsigned int segno) 2864 { 2865 struct f2fs_cached_block *entry; 2866 2867 if (!f2fs_sb_has_packed_ssa(sbi)) 2868 return f2fs_update_meta_block(sbi, (void *)sum_blk, 2869 GET_SUM_BLOCK(sbi, segno)); 2870 2871 entry = f2fs_get_sum_cache(sbi, segno); 2872 if (IS_ERR(entry)) 2873 return; 2874 2875 memcpy(SUM_BLK_ENTRY_ADDR(sbi, entry, segno), sum_blk, 2876 sbi->sum_blocksize); 2877 f2fs_mark_cache_dirty(entry); 2878 f2fs_put_cache(entry, true); 2879 } 2880 2881 static void write_current_sum_block(struct f2fs_sb_info *sbi, 2882 int type, block_t blk_addr) 2883 { 2884 struct curseg_info *curseg = CURSEG_I(sbi, type); 2885 struct f2fs_cached_block *entry = f2fs_grab_meta_cache(sbi, blk_addr); 2886 struct f2fs_summary_block *src = curseg->sum_blk; 2887 struct f2fs_summary_block *dst; 2888 2889 dst = cache_address(entry); 2890 memset(dst, 0, sbi->blocksize); 2891 2892 mutex_lock(&curseg->curseg_mutex); 2893 2894 down_read(&curseg->journal_rwsem); 2895 memcpy(sum_journal(sbi, dst), curseg->journal, sbi->sum_journal_size); 2896 up_read(&curseg->journal_rwsem); 2897 2898 memcpy(sum_entries(dst), sum_entries(src), sbi->sum_entry_size); 2899 memcpy(sum_footer(sbi, dst), sum_footer(sbi, src), SUM_FOOTER_SIZE); 2900 2901 mutex_unlock(&curseg->curseg_mutex); 2902 2903 f2fs_mark_cache_dirty(entry); 2904 f2fs_put_cache(entry, true); 2905 } 2906 2907 static int is_next_segment_free(struct f2fs_sb_info *sbi, 2908 struct curseg_info *curseg) 2909 { 2910 unsigned int segno = curseg->segno + 1; 2911 struct free_segmap_info *free_i = FREE_I(sbi); 2912 2913 if (segno < MAIN_SEGS(sbi) && segno % SEGS_PER_SEC(sbi)) { 2914 int devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); 2915 2916 if (f2fs_dev_is_reserving(sbi, devi)) 2917 return 0; 2918 return !test_bit(segno, free_i->free_segmap); 2919 } 2920 return 0; 2921 } 2922 2923 /* 2924 * Find a new segment from the free segments bitmap to right order 2925 * This function should be returned with success, otherwise BUG 2926 */ 2927 static int get_new_segment(struct f2fs_sb_info *sbi, 2928 unsigned int *newseg, bool new_sec, bool pinning) 2929 { 2930 struct free_segmap_info *free_i = FREE_I(sbi); 2931 unsigned int segno, secno, zoneno; 2932 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; 2933 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); 2934 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); 2935 unsigned int alloc_policy = sbi->allocate_section_policy; 2936 unsigned int alloc_hint = sbi->allocate_section_hint; 2937 unsigned int max_secno = MAIN_SECS(sbi); 2938 bool init = true; 2939 bool looped = false; 2940 int i, devi; 2941 int ret = 0; 2942 2943 spin_lock(&free_i->segmap_lock); 2944 2945 if (time_to_inject(sbi, FAULT_NO_SEGMENT)) { 2946 ret = -ENOSPC; 2947 goto out_unlock; 2948 } 2949 2950 if (!new_sec && ((*newseg + 1) % SEGS_PER_SEC(sbi))) { 2951 segno = find_next_zero_bit(free_i->free_segmap, 2952 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); 2953 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) { 2954 devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); 2955 2956 if (f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) 2957 goto find_other_zone; 2958 goto got_it; 2959 } 2960 } 2961 2962 #ifdef CONFIG_BLK_DEV_ZONED 2963 /* 2964 * If we format f2fs on zoned storage, let's try to get pinned sections 2965 * from beginning of the storage, which should be a conventional one. 2966 */ 2967 if (f2fs_sb_has_blkzoned(sbi)) { 2968 /* Prioritize writing to conventional zones */ 2969 if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_PRIOR_CONV) 2970 segno = 0; 2971 else 2972 segno = max(sbi->first_seq_zone_segno, *newseg); 2973 hint = GET_SEC_FROM_SEG(sbi, segno); 2974 } 2975 #endif 2976 2977 /* 2978 * Prevent allocate_section_hint from exceeding MAIN_SECS() 2979 * due to desynchronization. 2980 */ 2981 if (alloc_policy != ALLOCATE_FORWARD_NOHINT && 2982 alloc_hint > MAIN_SECS(sbi)) 2983 alloc_hint = MAIN_SECS(sbi); 2984 2985 if (pinning) { 2986 max_secno = min(sbi->pinned_area_max_secno, MAIN_SECS(sbi)); 2987 hint = 0; 2988 } else if (alloc_policy == ALLOCATE_FORWARD_FROM_HINT && 2989 hint < alloc_hint) { 2990 hint = alloc_hint; 2991 } else if (alloc_policy == ALLOCATE_FORWARD_WITHIN_HINT && 2992 hint >= alloc_hint) { 2993 hint = 0; 2994 } 2995 2996 find_other_zone: 2997 secno = find_next_zero_bit(free_i->free_secmap, max_secno, hint); 2998 2999 if (secno >= max_secno) { 3000 if (looped) { 3001 ret = (pinning && has_unpinned_area(sbi)) ? 3002 -EAGAIN : -ENOSPC; 3003 f2fs_bug_on(sbi, !pinning); 3004 goto out_unlock; 3005 } 3006 hint = 0; 3007 #ifdef CONFIG_BLK_DEV_ZONED 3008 /* Write only to sequential zones */ 3009 if (f2fs_sb_has_blkzoned(sbi) && 3010 sbi->blkzone_alloc_policy == BLKZONE_ALLOC_ONLY_SEQ) 3011 hint = GET_SEC_FROM_SEG(sbi, sbi->first_seq_zone_segno); 3012 #endif 3013 looped = true; 3014 goto find_other_zone; 3015 } 3016 3017 segno = GET_SEG_FROM_SEC(sbi, secno); 3018 3019 devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); 3020 3021 if (f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) { 3022 while (devi < sbi->s_ndevs && 3023 f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) { 3024 unsigned int end_segno = GET_SEGNO(sbi, FDEV(devi).end_blk); 3025 3026 hint = GET_SEC_FROM_SEG(sbi, end_segno) + 1; 3027 devi++; 3028 } 3029 goto find_other_zone; 3030 } 3031 3032 if (sec_usage_check(sbi, secno)) { 3033 hint = secno + 1; 3034 goto find_other_zone; 3035 } 3036 zoneno = GET_ZONE_FROM_SEC(sbi, secno); 3037 3038 /* give up on finding another zone */ 3039 if (!init) 3040 goto got_it; 3041 if (sbi->secs_per_zone == 1) 3042 goto got_it; 3043 if (zoneno == old_zoneno) 3044 goto got_it; 3045 for (i = 0; i < NR_CURSEG_TYPE; i++) 3046 if (CURSEG_I(sbi, i)->zone == zoneno) 3047 break; 3048 3049 if (i < NR_CURSEG_TYPE) { 3050 /* zone is in user, try another */ 3051 if (zoneno + 1 >= total_zones) 3052 hint = 0; 3053 else 3054 hint = (zoneno + 1) * sbi->secs_per_zone; 3055 init = false; 3056 goto find_other_zone; 3057 } 3058 got_it: 3059 /* set it as dirty segment in free segmap */ 3060 if (test_bit(segno, free_i->free_segmap)) { 3061 ret = -EFSCORRUPTED; 3062 fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); 3063 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_FREE_BITMAP); 3064 goto out_unlock; 3065 } 3066 3067 __set_inuse(sbi, segno); 3068 *newseg = segno; 3069 out_unlock: 3070 spin_unlock(&free_i->segmap_lock); 3071 3072 if (ret == -ENOSPC && !pinning) 3073 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_NO_SEGMENT); 3074 return ret; 3075 } 3076 3077 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) 3078 { 3079 struct curseg_info *curseg = CURSEG_I(sbi, type); 3080 struct summary_footer *sum_footer; 3081 unsigned short seg_type = curseg->seg_type; 3082 3083 /* only happen when get_new_segment() fails */ 3084 if (curseg->next_segno == NULL_SEGNO) 3085 return; 3086 3087 curseg->inited = true; 3088 curseg->segno = curseg->next_segno; 3089 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); 3090 curseg->next_blkoff = 0; 3091 curseg->next_segno = NULL_SEGNO; 3092 3093 sum_footer = sum_footer(sbi, curseg->sum_blk); 3094 memset(sum_footer, 0, sizeof(struct summary_footer)); 3095 3096 sanity_check_seg_type(sbi, seg_type); 3097 3098 if (IS_DATASEG(seg_type)) 3099 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); 3100 if (IS_NODESEG(seg_type)) 3101 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); 3102 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified); 3103 } 3104 3105 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) 3106 { 3107 struct curseg_info *curseg = CURSEG_I(sbi, type); 3108 unsigned short seg_type = curseg->seg_type; 3109 3110 sanity_check_seg_type(sbi, seg_type); 3111 if (__is_large_section(sbi)) { 3112 if (f2fs_need_rand_seg_blk(sbi, type)) { 3113 unsigned int hint = GET_SEC_FROM_SEG(sbi, curseg->segno); 3114 3115 if (GET_SEC_FROM_SEG(sbi, curseg->segno + 1) != hint) 3116 return curseg->segno; 3117 return get_random_u32_inclusive(curseg->segno + 1, 3118 GET_SEG_FROM_SEC(sbi, hint + 1) - 1); 3119 } 3120 return curseg->segno; 3121 } else if (f2fs_need_rand_seg_blk(sbi, type)) { 3122 return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); 3123 } 3124 3125 /* inmem log may not locate on any segment after mount */ 3126 if (!curseg->inited) 3127 return 0; 3128 3129 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 3130 return 0; 3131 3132 if (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)) 3133 return 0; 3134 3135 if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) 3136 return SIT_I(sbi)->last_victim[ALLOC_NEXT]; 3137 3138 /* find segments from 0 to reuse freed segments */ 3139 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) 3140 return 0; 3141 3142 return curseg->segno; 3143 } 3144 3145 static void reset_curseg_fields(struct curseg_info *curseg) 3146 { 3147 curseg->inited = false; 3148 curseg->segno = NULL_SEGNO; 3149 curseg->next_segno = 0; 3150 } 3151 3152 /* 3153 * Allocate a current working segment. 3154 * This function always allocates a free segment in LFS manner. 3155 */ 3156 static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) 3157 { 3158 struct curseg_info *curseg = CURSEG_I(sbi, type); 3159 unsigned int segno = curseg->segno; 3160 bool pinning = type == CURSEG_COLD_DATA_PINNED; 3161 int ret; 3162 3163 if (curseg->inited) 3164 write_sum_block(sbi, curseg->sum_blk, segno); 3165 3166 segno = __get_next_segno(sbi, type); 3167 ret = get_new_segment(sbi, &segno, new_sec, pinning); 3168 if (ret) { 3169 if (ret == -ENOSPC) 3170 reset_curseg_fields(curseg); 3171 return ret; 3172 } 3173 3174 curseg->next_segno = segno; 3175 reset_curseg(sbi, type, 1); 3176 curseg->alloc_type = LFS; 3177 if (f2fs_need_rand_blk(sbi, type)) 3178 curseg->fragment_remained_chunk = 3179 get_random_u32_inclusive(1, sbi->max_fragment_chunk); 3180 return 0; 3181 } 3182 3183 static int __next_free_blkoff(struct f2fs_sb_info *sbi, 3184 int segno, block_t start) 3185 { 3186 struct seg_entry *se = get_seg_entry(sbi, segno); 3187 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 3188 unsigned long *target_map = SIT_I(sbi)->tmp_map; 3189 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 3190 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 3191 int i; 3192 3193 for (i = 0; i < entries; i++) 3194 target_map[i] = ckpt_map[i] | cur_map[i]; 3195 3196 return __find_rev_next_zero_bit(target_map, BLKS_PER_SEG(sbi), start); 3197 } 3198 3199 static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi, 3200 struct curseg_info *seg) 3201 { 3202 return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1); 3203 } 3204 3205 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno) 3206 { 3207 return __next_free_blkoff(sbi, segno, 0) < BLKS_PER_SEG(sbi); 3208 } 3209 3210 /* 3211 * This function always allocates a used segment(from dirty seglist) by SSR 3212 * manner, so it should recover the existing segment information of valid blocks 3213 */ 3214 static int change_curseg(struct f2fs_sb_info *sbi, int type) 3215 { 3216 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 3217 struct curseg_info *curseg = CURSEG_I(sbi, type); 3218 unsigned int new_segno = curseg->next_segno; 3219 struct f2fs_summary_block *sum_node; 3220 struct f2fs_cached_block *entry = NULL; 3221 3222 if (curseg->inited) 3223 write_sum_block(sbi, curseg->sum_blk, curseg->segno); 3224 3225 __set_test_and_inuse(sbi, new_segno); 3226 3227 mutex_lock(&dirty_i->seglist_lock); 3228 __remove_dirty_segment(sbi, new_segno, PRE); 3229 __remove_dirty_segment(sbi, new_segno, DIRTY); 3230 mutex_unlock(&dirty_i->seglist_lock); 3231 3232 reset_curseg(sbi, type, 1); 3233 curseg->alloc_type = SSR; 3234 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0); 3235 3236 entry = f2fs_get_sum_cache(sbi, new_segno); 3237 if (IS_ERR(entry)) { 3238 /* GC won't be able to use stale summary blocks by cp_error */ 3239 memset(curseg->sum_blk, 0, sbi->sum_entry_size); 3240 reset_curseg_fields(curseg); 3241 return PTR_ERR(entry); 3242 } 3243 sum_node = SUM_BLK_ENTRY_ADDR(sbi, entry, new_segno); 3244 memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size); 3245 f2fs_put_cache(entry, true); 3246 return 0; 3247 } 3248 3249 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, 3250 int alloc_mode, unsigned long long age); 3251 3252 static int get_atssr_segment(struct f2fs_sb_info *sbi, int type, 3253 int target_type, int alloc_mode, 3254 unsigned long long age) 3255 { 3256 struct curseg_info *curseg = CURSEG_I(sbi, type); 3257 int ret = 0; 3258 3259 curseg->seg_type = target_type; 3260 3261 if (get_ssr_segment(sbi, type, alloc_mode, age)) { 3262 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno); 3263 3264 curseg->seg_type = se->type; 3265 ret = change_curseg(sbi, type); 3266 } else { 3267 /* allocate cold segment by default */ 3268 curseg->seg_type = CURSEG_COLD_DATA; 3269 ret = new_curseg(sbi, type, true); 3270 } 3271 stat_inc_seg_type(sbi, curseg); 3272 return ret; 3273 } 3274 3275 static int __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi, bool force) 3276 { 3277 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC); 3278 int ret = 0; 3279 3280 if (!sbi->am.atgc_enabled && !force) 3281 return 0; 3282 3283 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3284 3285 mutex_lock(&curseg->curseg_mutex); 3286 down_write(&SIT_I(sbi)->sentry_lock); 3287 3288 ret = get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, 3289 CURSEG_COLD_DATA, SSR, 0); 3290 3291 up_write(&SIT_I(sbi)->sentry_lock); 3292 mutex_unlock(&curseg->curseg_mutex); 3293 3294 f2fs_up_read(&SM_I(sbi)->curseg_lock); 3295 return ret; 3296 } 3297 3298 int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi) 3299 { 3300 return __f2fs_init_atgc_curseg(sbi, false); 3301 } 3302 3303 int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi) 3304 { 3305 int ret; 3306 3307 if (!test_opt(sbi, ATGC)) 3308 return 0; 3309 if (sbi->am.atgc_enabled) 3310 return 0; 3311 if (le64_to_cpu(F2FS_CKPT(sbi)->elapsed_time) < 3312 sbi->am.age_threshold) 3313 return 0; 3314 3315 ret = __f2fs_init_atgc_curseg(sbi, true); 3316 if (!ret) { 3317 sbi->am.atgc_enabled = true; 3318 f2fs_info(sbi, "reenabled age threshold GC"); 3319 } 3320 return ret; 3321 } 3322 3323 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type) 3324 { 3325 struct curseg_info *curseg = CURSEG_I(sbi, type); 3326 3327 mutex_lock(&curseg->curseg_mutex); 3328 if (!curseg->inited) 3329 goto out; 3330 3331 if (get_valid_blocks(sbi, curseg->segno, false)) { 3332 write_sum_block(sbi, curseg->sum_blk, curseg->segno); 3333 } else { 3334 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 3335 __set_test_and_free(sbi, curseg->segno, true); 3336 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 3337 } 3338 out: 3339 mutex_unlock(&curseg->curseg_mutex); 3340 } 3341 3342 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi) 3343 { 3344 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); 3345 3346 if (sbi->am.atgc_enabled) 3347 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); 3348 } 3349 3350 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type) 3351 { 3352 struct curseg_info *curseg = CURSEG_I(sbi, type); 3353 3354 mutex_lock(&curseg->curseg_mutex); 3355 if (!curseg->inited) 3356 goto out; 3357 if (get_valid_blocks(sbi, curseg->segno, false)) 3358 goto out; 3359 3360 mutex_lock(&DIRTY_I(sbi)->seglist_lock); 3361 __set_test_and_inuse(sbi, curseg->segno); 3362 mutex_unlock(&DIRTY_I(sbi)->seglist_lock); 3363 out: 3364 mutex_unlock(&curseg->curseg_mutex); 3365 } 3366 3367 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi) 3368 { 3369 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); 3370 3371 if (sbi->am.atgc_enabled) 3372 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); 3373 } 3374 3375 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, 3376 int alloc_mode, unsigned long long age) 3377 { 3378 struct curseg_info *curseg = CURSEG_I(sbi, type); 3379 unsigned segno = NULL_SEGNO; 3380 unsigned short seg_type = curseg->seg_type; 3381 int i, cnt; 3382 bool reversed = false; 3383 3384 sanity_check_seg_type(sbi, seg_type); 3385 3386 /* f2fs_need_SSR() already forces to do this */ 3387 if (!f2fs_get_victim(sbi, &segno, BG_GC, seg_type, 3388 alloc_mode, age, false)) { 3389 curseg->next_segno = segno; 3390 return 1; 3391 } 3392 3393 /* For node segments, let's do SSR more intensively */ 3394 if (IS_NODESEG(seg_type)) { 3395 if (seg_type >= CURSEG_WARM_NODE) { 3396 reversed = true; 3397 i = CURSEG_COLD_NODE; 3398 } else { 3399 i = CURSEG_HOT_NODE; 3400 } 3401 cnt = NR_CURSEG_NODE_TYPE; 3402 } else { 3403 if (seg_type >= CURSEG_WARM_DATA) { 3404 reversed = true; 3405 i = CURSEG_COLD_DATA; 3406 } else { 3407 i = CURSEG_HOT_DATA; 3408 } 3409 cnt = NR_CURSEG_DATA_TYPE; 3410 } 3411 3412 for (; cnt-- > 0; reversed ? i-- : i++) { 3413 if (i == seg_type) 3414 continue; 3415 if (!f2fs_get_victim(sbi, &segno, BG_GC, i, 3416 alloc_mode, age, false)) { 3417 curseg->next_segno = segno; 3418 return 1; 3419 } 3420 } 3421 3422 /* find valid_blocks=0 in dirty list */ 3423 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { 3424 segno = get_free_segment(sbi); 3425 if (segno != NULL_SEGNO) { 3426 curseg->next_segno = segno; 3427 return 1; 3428 } 3429 } 3430 return 0; 3431 } 3432 3433 static bool need_new_seg(struct f2fs_sb_info *sbi, int type) 3434 { 3435 struct curseg_info *curseg = CURSEG_I(sbi, type); 3436 3437 if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && 3438 curseg->seg_type == CURSEG_WARM_NODE) 3439 return true; 3440 if (curseg->alloc_type == LFS && is_next_segment_free(sbi, curseg) && 3441 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) 3442 return true; 3443 if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0)) 3444 return true; 3445 return false; 3446 } 3447 3448 int f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, 3449 unsigned int start, unsigned int end) 3450 { 3451 struct curseg_info *curseg = CURSEG_I(sbi, type); 3452 unsigned int segno; 3453 int ret = 0; 3454 3455 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3456 mutex_lock(&curseg->curseg_mutex); 3457 down_write(&SIT_I(sbi)->sentry_lock); 3458 3459 segno = CURSEG_I(sbi, type)->segno; 3460 if (segno < start || segno > end) 3461 goto unlock; 3462 3463 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0)) 3464 ret = change_curseg(sbi, type); 3465 else 3466 ret = new_curseg(sbi, type, true); 3467 3468 stat_inc_seg_type(sbi, curseg); 3469 3470 locate_dirty_segment(sbi, segno); 3471 unlock: 3472 up_write(&SIT_I(sbi)->sentry_lock); 3473 3474 if (segno != curseg->segno) 3475 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u", 3476 type, segno, curseg->segno); 3477 3478 mutex_unlock(&curseg->curseg_mutex); 3479 f2fs_up_read(&SM_I(sbi)->curseg_lock); 3480 return ret; 3481 } 3482 3483 static int __allocate_new_segment(struct f2fs_sb_info *sbi, int type, 3484 bool new_sec, bool force) 3485 { 3486 struct curseg_info *curseg = CURSEG_I(sbi, type); 3487 unsigned int old_segno; 3488 int err = 0; 3489 3490 if (type == CURSEG_COLD_DATA_PINNED && !curseg->inited) 3491 goto allocate; 3492 3493 if (!force && curseg->inited && 3494 !curseg->next_blkoff && 3495 !get_valid_blocks(sbi, curseg->segno, new_sec) && 3496 !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec)) 3497 return 0; 3498 3499 allocate: 3500 old_segno = curseg->segno; 3501 err = new_curseg(sbi, type, true); 3502 if (err) 3503 return err; 3504 stat_inc_seg_type(sbi, curseg); 3505 locate_dirty_segment(sbi, old_segno); 3506 return 0; 3507 } 3508 3509 int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force) 3510 { 3511 int ret; 3512 3513 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3514 down_write(&SIT_I(sbi)->sentry_lock); 3515 ret = __allocate_new_segment(sbi, type, true, force); 3516 up_write(&SIT_I(sbi)->sentry_lock); 3517 f2fs_up_read(&SM_I(sbi)->curseg_lock); 3518 3519 return ret; 3520 } 3521 3522 int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi) 3523 { 3524 struct f2fs_lock_context lc; 3525 int err; 3526 bool gc_required = true; 3527 3528 retry: 3529 f2fs_lock_op(sbi, &lc); 3530 err = f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false); 3531 f2fs_unlock_op(sbi, &lc); 3532 3533 if (has_unpinned_area(sbi) && err == -EAGAIN && gc_required) { 3534 err = f2fs_gc_range(sbi, 0, 3535 sbi->pinned_area_max_secno * SEGS_PER_SEC(sbi) - 1, 3536 true, ZONED_PIN_SEC_REQUIRED_COUNT, true); 3537 if (err) 3538 return err; 3539 err = f2fs_sync_fs(sbi->sb, 1); 3540 if (!err) { 3541 gc_required = false; 3542 goto retry; 3543 } 3544 } 3545 3546 return err; 3547 } 3548 3549 int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) 3550 { 3551 int i; 3552 int err = 0; 3553 3554 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3555 down_write(&SIT_I(sbi)->sentry_lock); 3556 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) 3557 err += __allocate_new_segment(sbi, i, false, false); 3558 up_write(&SIT_I(sbi)->sentry_lock); 3559 f2fs_up_read(&SM_I(sbi)->curseg_lock); 3560 3561 return err; 3562 } 3563 3564 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, 3565 struct cp_control *cpc) 3566 { 3567 __u64 trim_start = cpc->trim_start; 3568 bool has_candidate = false; 3569 3570 down_write(&SIT_I(sbi)->sentry_lock); 3571 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) { 3572 if (add_discard_addrs(sbi, cpc, true)) { 3573 has_candidate = true; 3574 break; 3575 } 3576 } 3577 up_write(&SIT_I(sbi)->sentry_lock); 3578 3579 cpc->trim_start = trim_start; 3580 return has_candidate; 3581 } 3582 3583 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, 3584 struct discard_policy *dpolicy, 3585 unsigned int start, unsigned int end) 3586 { 3587 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; 3588 struct discard_cmd *prev_dc = NULL, *next_dc = NULL; 3589 struct rb_node **insert_p = NULL, *insert_parent = NULL; 3590 struct discard_cmd *dc; 3591 struct blk_plug plug; 3592 int issued; 3593 unsigned int trimmed = 0; 3594 3595 next: 3596 issued = 0; 3597 3598 mutex_lock(&dcc->cmd_lock); 3599 if (unlikely(dcc->rbtree_check)) 3600 f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi)); 3601 3602 dc = __lookup_discard_cmd_ret(&dcc->root, start, 3603 &prev_dc, &next_dc, &insert_p, &insert_parent); 3604 if (!dc) 3605 dc = next_dc; 3606 3607 blk_start_plug(&plug); 3608 3609 while (dc && dc->di.lstart <= end) { 3610 struct rb_node *node; 3611 int err = 0; 3612 3613 if (dc->di.len < dpolicy->granularity) 3614 goto skip; 3615 3616 if (dc->state != D_PREP) { 3617 list_move_tail(&dc->list, &dcc->fstrim_list); 3618 goto skip; 3619 } 3620 3621 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); 3622 3623 if (issued >= dpolicy->max_requests) { 3624 start = dc->di.lstart + dc->di.len; 3625 3626 if (err) 3627 __remove_discard_cmd(sbi, dc); 3628 3629 blk_finish_plug(&plug); 3630 mutex_unlock(&dcc->cmd_lock); 3631 trimmed += __wait_all_discard_cmd(sbi, NULL); 3632 f2fs_schedule_timeout(DEFAULT_DISCARD_INTERVAL); 3633 goto next; 3634 } 3635 skip: 3636 node = rb_next(&dc->rb_node); 3637 if (err) 3638 __remove_discard_cmd(sbi, dc); 3639 dc = rb_entry_safe(node, struct discard_cmd, rb_node); 3640 3641 if (fatal_signal_pending(current)) 3642 break; 3643 } 3644 3645 blk_finish_plug(&plug); 3646 mutex_unlock(&dcc->cmd_lock); 3647 3648 return trimmed; 3649 } 3650 3651 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) 3652 { 3653 __u64 start = F2FS_BYTES_TO_BLK(sbi, range->start); 3654 __u64 end = start + F2FS_BYTES_TO_BLK(sbi, range->len) - 1; 3655 unsigned int start_segno, end_segno; 3656 block_t start_block, end_block; 3657 struct cp_control cpc; 3658 struct discard_policy dpolicy; 3659 struct f2fs_lock_context lc; 3660 unsigned long long trimmed = 0; 3661 int err = 0; 3662 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); 3663 3664 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) 3665 return -EINVAL; 3666 3667 if (end < MAIN_BLKADDR(sbi)) 3668 goto out; 3669 3670 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { 3671 f2fs_warn(sbi, "Found FS corruption, run fsck to fix."); 3672 return -EFSCORRUPTED; 3673 } 3674 3675 /* start/end segment number in main_area */ 3676 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); 3677 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : 3678 GET_SEGNO(sbi, end); 3679 if (need_align) { 3680 start_segno = rounddown(start_segno, SEGS_PER_SEC(sbi)); 3681 end_segno = roundup(end_segno + 1, SEGS_PER_SEC(sbi)) - 1; 3682 } 3683 3684 cpc.reason = CP_DISCARD; 3685 cpc.trim_minlen = max_t(__u64, 1, 3686 F2FS_BYTES_TO_BLK(sbi, range->minlen)); 3687 cpc.trim_start = start_segno; 3688 cpc.trim_end = end_segno; 3689 3690 if (sbi->discard_blks == 0) 3691 goto out; 3692 3693 f2fs_down_write_trace(&sbi->gc_lock, &lc); 3694 stat_inc_cp_call_count(sbi, TOTAL_CALL); 3695 err = f2fs_write_checkpoint(sbi, &cpc); 3696 f2fs_up_write_trace(&sbi->gc_lock, &lc); 3697 if (err) 3698 goto out; 3699 3700 /* 3701 * We filed discard candidates, but actually we don't need to wait for 3702 * all of them, since they'll be issued in idle time along with runtime 3703 * discard option. User configuration looks like using runtime discard 3704 * or periodic fstrim instead of it. 3705 */ 3706 if (f2fs_realtime_discard_enable(sbi)) 3707 goto out; 3708 3709 start_block = START_BLOCK(sbi, start_segno); 3710 end_block = START_BLOCK(sbi, end_segno + 1); 3711 3712 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); 3713 trimmed = __issue_discard_cmd_range(sbi, &dpolicy, 3714 start_block, end_block); 3715 3716 trimmed += __wait_discard_cmd_range(sbi, &dpolicy, 3717 start_block, end_block); 3718 out: 3719 if (!err) 3720 range->len = F2FS_BLK_TO_BYTES(sbi, trimmed); 3721 return err; 3722 } 3723 3724 int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint) 3725 { 3726 if (F2FS_OPTION(sbi).active_logs == 2) 3727 return CURSEG_HOT_DATA; 3728 else if (F2FS_OPTION(sbi).active_logs == 4) 3729 return CURSEG_COLD_DATA; 3730 3731 /* active_log == 6 */ 3732 switch (hint) { 3733 case WRITE_LIFE_SHORT: 3734 return CURSEG_HOT_DATA; 3735 case WRITE_LIFE_EXTREME: 3736 return CURSEG_COLD_DATA; 3737 default: 3738 return CURSEG_WARM_DATA; 3739 } 3740 } 3741 3742 /* 3743 * This returns write hints for each segment type. This hints will be 3744 * passed down to block layer as below by default. 3745 * 3746 * User F2FS Block 3747 * ---- ---- ----- 3748 * META WRITE_LIFE_NONE|REQ_META 3749 * HOT_NODE WRITE_LIFE_NONE 3750 * WARM_NODE WRITE_LIFE_MEDIUM 3751 * COLD_NODE WRITE_LIFE_LONG 3752 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME 3753 * extension list " " 3754 * 3755 * -- buffered io 3756 * COLD_DATA WRITE_LIFE_EXTREME 3757 * HOT_DATA WRITE_LIFE_SHORT 3758 * WARM_DATA WRITE_LIFE_NOT_SET 3759 * 3760 * -- direct io 3761 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME 3762 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT 3763 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET 3764 * WRITE_LIFE_NONE " WRITE_LIFE_NONE 3765 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM 3766 * WRITE_LIFE_LONG " WRITE_LIFE_LONG 3767 */ 3768 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, 3769 enum page_type type, enum temp_type temp) 3770 { 3771 switch (type) { 3772 case DATA: 3773 switch (temp) { 3774 case WARM: 3775 return WRITE_LIFE_NOT_SET; 3776 case HOT: 3777 return WRITE_LIFE_SHORT; 3778 case COLD: 3779 return WRITE_LIFE_EXTREME; 3780 default: 3781 return WRITE_LIFE_NONE; 3782 } 3783 case NODE: 3784 switch (temp) { 3785 case WARM: 3786 return WRITE_LIFE_MEDIUM; 3787 case HOT: 3788 return WRITE_LIFE_NONE; 3789 case COLD: 3790 return WRITE_LIFE_LONG; 3791 default: 3792 return WRITE_LIFE_NONE; 3793 } 3794 case META: 3795 return WRITE_LIFE_NONE; 3796 default: 3797 return WRITE_LIFE_NONE; 3798 } 3799 } 3800 3801 u8 f2fs_io_type_to_write_stream(struct block_device *bdev, 3802 enum page_type type, enum temp_type temp) 3803 { 3804 unsigned short nr = bdev_max_write_streams(bdev); 3805 3806 if (type != DATA || !nr) 3807 return 0; 3808 if (nr < NR_TEMP_TYPE) 3809 return temp == COLD ? nr : HOT + 1; 3810 3811 return temp + 1; 3812 } 3813 3814 static int __get_segment_type_2(struct f2fs_io_info *fio) 3815 { 3816 if (fio->type == DATA) 3817 return CURSEG_HOT_DATA; 3818 else 3819 return CURSEG_HOT_NODE; 3820 } 3821 3822 static int __get_segment_type_4(struct f2fs_io_info *fio) 3823 { 3824 if (fio->type == DATA) { 3825 struct inode *inode = fio_inode(fio); 3826 3827 if (S_ISDIR(inode->i_mode)) 3828 return CURSEG_HOT_DATA; 3829 else 3830 return CURSEG_COLD_DATA; 3831 } else { 3832 f2fs_bug_on(fio->sbi, !fio->is_cache); 3833 if (IS_DNODE(fio->sbi, fio->cache_entry) && is_cold_node(fio->sbi, fio->cache_entry)) 3834 return CURSEG_WARM_NODE; 3835 else 3836 return CURSEG_COLD_NODE; 3837 } 3838 } 3839 3840 static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs) 3841 { 3842 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3843 struct extent_info ei = {}; 3844 3845 if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) { 3846 if (!ei.age) 3847 return NO_CHECK_TYPE; 3848 if (ei.age <= sbi->hot_data_age_threshold) 3849 return CURSEG_HOT_DATA; 3850 if (ei.age <= sbi->warm_data_age_threshold) 3851 return CURSEG_WARM_DATA; 3852 return CURSEG_COLD_DATA; 3853 } 3854 return NO_CHECK_TYPE; 3855 } 3856 3857 static int __get_segment_type_6(struct f2fs_io_info *fio) 3858 { 3859 if (fio->type == DATA) { 3860 struct inode *inode = fio_inode(fio); 3861 int type; 3862 3863 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) 3864 return CURSEG_COLD_DATA_PINNED; 3865 3866 if (folio_test_f2fs_gcing(fio->folio)) { 3867 if (fio->sbi->am.atgc_enabled && 3868 (fio->io_type == FS_DATA_IO) && 3869 (fio->sbi->gc_mode != GC_URGENT_HIGH) && 3870 __is_valid_data_blkaddr(fio->old_blkaddr) && 3871 !is_inode_flag_set(inode, FI_OPU_WRITE)) 3872 return CURSEG_ALL_DATA_ATGC; 3873 else 3874 return CURSEG_COLD_DATA; 3875 } 3876 if (file_is_cold(inode) || f2fs_need_compress_data(inode)) 3877 return CURSEG_COLD_DATA; 3878 3879 type = __get_age_segment_type(inode, fio->folio->index); 3880 if (type != NO_CHECK_TYPE) 3881 return type; 3882 3883 if (file_is_hot(inode) || 3884 is_inode_flag_set(inode, FI_HOT_DATA) || 3885 f2fs_is_cow_file(inode) || 3886 is_inode_flag_set(inode, FI_NEED_IPU)) 3887 return CURSEG_HOT_DATA; 3888 return f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), 3889 inode->i_write_hint); 3890 } else { 3891 f2fs_bug_on(fio->sbi, !fio->is_cache); 3892 if (IS_DNODE(fio->sbi, fio->cache_entry)) 3893 return is_cold_node(fio->sbi, fio->cache_entry) ? CURSEG_WARM_NODE : CURSEG_HOT_NODE; 3894 return CURSEG_COLD_NODE; 3895 } 3896 } 3897 3898 enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi, 3899 enum log_type type) 3900 { 3901 struct curseg_info *curseg = CURSEG_I(sbi, type); 3902 enum temp_type temp = COLD; 3903 3904 switch (curseg->seg_type) { 3905 case CURSEG_HOT_NODE: 3906 case CURSEG_HOT_DATA: 3907 temp = HOT; 3908 break; 3909 case CURSEG_WARM_NODE: 3910 case CURSEG_WARM_DATA: 3911 temp = WARM; 3912 break; 3913 case CURSEG_COLD_NODE: 3914 case CURSEG_COLD_DATA: 3915 temp = COLD; 3916 break; 3917 default: 3918 f2fs_bug_on(sbi, 1); 3919 } 3920 3921 return temp; 3922 } 3923 3924 static int __get_segment_type(struct f2fs_io_info *fio) 3925 { 3926 enum log_type type = CURSEG_HOT_DATA; 3927 3928 switch (F2FS_OPTION(fio->sbi).active_logs) { 3929 case 2: 3930 type = __get_segment_type_2(fio); 3931 break; 3932 case 4: 3933 type = __get_segment_type_4(fio); 3934 break; 3935 case 6: 3936 type = __get_segment_type_6(fio); 3937 break; 3938 default: 3939 f2fs_bug_on(fio->sbi, true); 3940 } 3941 3942 fio->temp = f2fs_get_segment_temp(fio->sbi, type); 3943 3944 return type; 3945 } 3946 3947 static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi, 3948 struct curseg_info *seg) 3949 { 3950 /* To allocate block chunks in different sizes, use random number */ 3951 if (--seg->fragment_remained_chunk > 0) 3952 return; 3953 3954 seg->fragment_remained_chunk = 3955 get_random_u32_inclusive(1, sbi->max_fragment_chunk); 3956 seg->next_blkoff += 3957 get_random_u32_inclusive(1, sbi->max_fragment_hole); 3958 } 3959 3960 int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, 3961 block_t old_blkaddr, block_t *new_blkaddr, 3962 struct f2fs_summary *sum, int type, 3963 struct f2fs_io_info *fio) 3964 { 3965 struct sit_info *sit_i = SIT_I(sbi); 3966 struct curseg_info *curseg = CURSEG_I(sbi, type); 3967 unsigned long long old_mtime; 3968 bool from_gc = (type == CURSEG_ALL_DATA_ATGC); 3969 struct seg_entry *se = NULL; 3970 bool segment_full = false; 3971 int ret = 0; 3972 3973 f2fs_down_read(&SM_I(sbi)->curseg_lock); 3974 3975 mutex_lock(&curseg->curseg_mutex); 3976 down_write(&sit_i->sentry_lock); 3977 3978 if (curseg->segno == NULL_SEGNO) { 3979 ret = -ENOSPC; 3980 goto out_err; 3981 } 3982 3983 if (from_gc) { 3984 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO); 3985 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr)); 3986 sanity_check_seg_type(sbi, se->type); 3987 f2fs_bug_on(sbi, IS_NODESEG(se->type)); 3988 } 3989 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 3990 3991 f2fs_bug_on(sbi, curseg->next_blkoff >= BLKS_PER_SEG(sbi)); 3992 3993 f2fs_wait_discard_bio(sbi, *new_blkaddr); 3994 3995 sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; 3996 if (curseg->alloc_type == SSR) { 3997 curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); 3998 } else { 3999 curseg->next_blkoff++; 4000 if (f2fs_need_rand_blk(sbi, type)) 4001 f2fs_randomize_chunk(sbi, curseg); 4002 } 4003 if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno)) 4004 segment_full = true; 4005 stat_inc_block_count(sbi, curseg); 4006 4007 if (from_gc) { 4008 old_mtime = get_segment_mtime(sbi, old_blkaddr); 4009 } else { 4010 update_segment_mtime(sbi, old_blkaddr, 0); 4011 old_mtime = 0; 4012 } 4013 update_segment_mtime(sbi, *new_blkaddr, old_mtime); 4014 4015 /* 4016 * SIT information should be updated before segment allocation, 4017 * since SSR needs latest valid block information. 4018 */ 4019 update_sit_entry(sbi, *new_blkaddr, 1); 4020 update_sit_entry(sbi, old_blkaddr, -1); 4021 4022 /* 4023 * If the current segment is full, flush it out and replace it with a 4024 * new segment. 4025 */ 4026 if (segment_full) { 4027 if (type == CURSEG_COLD_DATA_PINNED && 4028 !((curseg->segno + 1) % sbi->segs_per_sec)) { 4029 write_sum_block(sbi, curseg->sum_blk, curseg->segno); 4030 reset_curseg_fields(curseg); 4031 goto skip_new_segment; 4032 } 4033 4034 if (from_gc) { 4035 ret = get_atssr_segment(sbi, type, se->type, 4036 AT_SSR, se->mtime); 4037 } else { 4038 if (need_new_seg(sbi, type)) 4039 ret = new_curseg(sbi, type, false); 4040 else 4041 ret = change_curseg(sbi, type); 4042 stat_inc_seg_type(sbi, curseg); 4043 } 4044 4045 if (ret) 4046 goto out_err; 4047 } 4048 4049 skip_new_segment: 4050 /* 4051 * segment dirty status should be updated after segment allocation, 4052 * so we just need to update status only one time after previous 4053 * segment being closed. 4054 */ 4055 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 4056 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); 4057 4058 if (IS_DATASEG(curseg->seg_type)) { 4059 unsigned long long new_val; 4060 4061 new_val = atomic64_inc_return(&sbi->allocated_data_blocks); 4062 if (unlikely(new_val == ULLONG_MAX)) 4063 atomic64_set(&sbi->allocated_data_blocks, 0); 4064 } 4065 4066 up_write(&sit_i->sentry_lock); 4067 4068 if (fio && fio->is_cache && IS_NODESEG(curseg->seg_type)) { 4069 fill_node_footer_blkaddr(sbi, fio->cache_entry, 4070 NEXT_FREE_BLKADDR(sbi, curseg)); 4071 f2fs_inode_chksum_set(sbi, fio->cache_entry); 4072 } 4073 4074 if (fio) { 4075 struct f2fs_bio_info *io; 4076 4077 INIT_LIST_HEAD(&fio->list); 4078 fio->in_list = 1; 4079 io = sbi->write_io[fio->type] + fio->temp; 4080 spin_lock(&io->io_lock); 4081 list_add_tail(&fio->list, &io->io_list); 4082 spin_unlock(&io->io_lock); 4083 } 4084 4085 mutex_unlock(&curseg->curseg_mutex); 4086 f2fs_up_read(&SM_I(sbi)->curseg_lock); 4087 return 0; 4088 4089 out_err: 4090 *new_blkaddr = NULL_ADDR; 4091 up_write(&sit_i->sentry_lock); 4092 mutex_unlock(&curseg->curseg_mutex); 4093 f2fs_up_read(&SM_I(sbi)->curseg_lock); 4094 return ret; 4095 } 4096 4097 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino, 4098 block_t blkaddr, unsigned int blkcnt) 4099 { 4100 if (!f2fs_is_multi_device(sbi)) 4101 return; 4102 4103 while (1) { 4104 unsigned int devidx = f2fs_target_device_index(sbi, blkaddr); 4105 unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1; 4106 4107 /* update device state for fsync */ 4108 f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO); 4109 4110 /* update device state for checkpoint */ 4111 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { 4112 spin_lock(&sbi->dev_lock); 4113 f2fs_set_bit(devidx, (char *)&sbi->dirty_device); 4114 spin_unlock(&sbi->dev_lock); 4115 } 4116 4117 if (blkcnt <= blks) 4118 break; 4119 blkcnt -= blks; 4120 blkaddr += blks; 4121 } 4122 } 4123 4124 static int log_type_to_seg_type(enum log_type type) 4125 { 4126 int seg_type = CURSEG_COLD_DATA; 4127 4128 switch (type) { 4129 case CURSEG_HOT_DATA: 4130 case CURSEG_WARM_DATA: 4131 case CURSEG_COLD_DATA: 4132 case CURSEG_HOT_NODE: 4133 case CURSEG_WARM_NODE: 4134 case CURSEG_COLD_NODE: 4135 seg_type = (int)type; 4136 break; 4137 case CURSEG_COLD_DATA_PINNED: 4138 case CURSEG_ALL_DATA_ATGC: 4139 seg_type = CURSEG_COLD_DATA; 4140 break; 4141 default: 4142 break; 4143 } 4144 return seg_type; 4145 } 4146 4147 static void do_write_block(struct f2fs_summary *sum, struct f2fs_io_info *fio) 4148 { 4149 struct folio *folio = fio->folio; 4150 enum log_type type = __get_segment_type(fio); 4151 int seg_type = log_type_to_seg_type(type); 4152 bool keep_order = (f2fs_lfs_mode(fio->sbi) && 4153 seg_type == CURSEG_COLD_DATA); 4154 int err; 4155 4156 if (keep_order) 4157 f2fs_down_read(&fio->sbi->io_order_lock); 4158 4159 err = f2fs_allocate_data_block(fio->sbi, fio->old_blkaddr, 4160 &fio->new_blkaddr, sum, type, fio); 4161 if (unlikely(err)) { 4162 if (fio->is_cache) { 4163 f2fs_err_ratelimited(fio->sbi, 4164 "%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d", 4165 __func__, fio->ino, fio->cache_entry->index, type, 4166 fio->old_blkaddr, fio->new_blkaddr, err); 4167 if (f2fs_in_warm_node_list(fio->sbi, fio->cache_entry)) 4168 f2fs_del_fsync_node_entry(fio->sbi, fio->cache_entry); 4169 f2fs_end_cache_writeback(fio->cache_entry); 4170 } else { 4171 f2fs_err_ratelimited(fio->sbi, 4172 "%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d", 4173 __func__, fio->ino, folio->index, type, 4174 fio->old_blkaddr, fio->new_blkaddr, err); 4175 folio_end_writeback(folio); 4176 } 4177 f2fs_bug_on(fio->sbi, !is_set_ckpt_flags(fio->sbi, 4178 CP_ERROR_FLAG)); 4179 goto out; 4180 } 4181 4182 f2fs_bug_on(fio->sbi, !f2fs_is_valid_blkaddr_raw(fio->sbi, 4183 fio->new_blkaddr, DATA_GENERIC_ENHANCE)); 4184 4185 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) 4186 f2fs_invalidate_internal_cache(fio->sbi, fio->old_blkaddr, 1); 4187 4188 /* writeout dirty page into bdev */ 4189 if (fio->is_cache) 4190 f2fs_submit_cache_write(fio); 4191 else 4192 f2fs_submit_page_write(fio); 4193 4194 f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1); 4195 out: 4196 if (keep_order) 4197 f2fs_up_read(&fio->sbi->io_order_lock); 4198 } 4199 4200 void f2fs_do_write_meta_cache(struct f2fs_sb_info *sbi, 4201 struct f2fs_cached_block *entry, 4202 enum iostat_type io_type) 4203 { 4204 struct f2fs_io_info fio = { 4205 .sbi = sbi, 4206 .type = META, 4207 .temp = HOT, 4208 .op = REQ_OP_WRITE, 4209 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO, 4210 .old_blkaddr = entry->index, 4211 .new_blkaddr = entry->index, 4212 .encrypted_page = NULL, 4213 .in_list = 0, 4214 .cache_entry = entry, 4215 .is_cache = 1, 4216 }; 4217 4218 if (unlikely(entry->index >= MAIN_BLKADDR(sbi))) 4219 fio.op_flags &= ~REQ_META; 4220 4221 f2fs_start_cache_writeback(entry); 4222 f2fs_submit_cache_write(&fio); 4223 4224 stat_inc_meta_count(sbi, entry->index); 4225 f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE(sbi)); 4226 } 4227 4228 void f2fs_do_write_node_cache(unsigned int nid, struct f2fs_io_info *fio) 4229 { 4230 struct f2fs_summary sum; 4231 4232 if (fio->is_cache) 4233 trace_f2fs_write_cache(fio->cache_entry, NODE); 4234 4235 set_summary(&sum, nid, 0, 0); 4236 do_write_block(&sum, fio); 4237 4238 f2fs_update_iostat(fio->sbi, NULL, fio->io_type, 4239 F2FS_BLKSIZE(fio->sbi)); 4240 } 4241 4242 void f2fs_outplace_write_data(struct dnode_of_data *dn, 4243 struct f2fs_io_info *fio) 4244 { 4245 struct f2fs_sb_info *sbi = fio->sbi; 4246 struct f2fs_summary sum; 4247 4248 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); 4249 if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO) 4250 f2fs_update_age_extent_cache(dn); 4251 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version); 4252 do_write_block(&sum, fio); 4253 f2fs_update_data_blkaddr(dn, fio->new_blkaddr); 4254 4255 f2fs_update_iostat(sbi, dn->inode, fio->io_type, 4256 F2FS_BLKSIZE(sbi)); 4257 } 4258 4259 int f2fs_inplace_write_data(struct f2fs_io_info *fio) 4260 { 4261 int err; 4262 struct f2fs_sb_info *sbi = fio->sbi; 4263 unsigned int segno; 4264 4265 fio->new_blkaddr = fio->old_blkaddr; 4266 /* i/o temperature is needed for passing down write hints */ 4267 __get_segment_type(fio); 4268 4269 segno = GET_SEGNO(sbi, fio->new_blkaddr); 4270 4271 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) { 4272 set_sbi_flag(sbi, SBI_NEED_FSCK); 4273 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.", 4274 __func__, segno); 4275 err = -EFSCORRUPTED; 4276 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); 4277 goto drop_bio; 4278 } 4279 4280 if (f2fs_cp_error(sbi)) { 4281 err = -EIO; 4282 goto drop_bio; 4283 } 4284 4285 if (fio->meta_gc) 4286 f2fs_truncate_meta_caches(sbi, fio->new_blkaddr, 1); 4287 4288 stat_inc_inplace_blocks(fio->sbi); 4289 4290 if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi)) 4291 err = f2fs_merge_page_bio(fio); 4292 else 4293 err = f2fs_submit_page_bio(fio); 4294 if (!err) { 4295 fio->submitted = 1; 4296 f2fs_update_device_state(fio->sbi, fio->ino, 4297 fio->new_blkaddr, 1); 4298 f2fs_update_iostat(fio->sbi, fio_inode(fio), 4299 fio->io_type, F2FS_BLKSIZE(fio->sbi)); 4300 } 4301 4302 return err; 4303 drop_bio: 4304 if (fio->bio && *(fio->bio)) { 4305 struct bio *bio = *(fio->bio); 4306 4307 bio->bi_status = BLK_STS_IOERR; 4308 bio_endio(bio); 4309 *(fio->bio) = NULL; 4310 } 4311 return err; 4312 } 4313 4314 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, 4315 unsigned int segno) 4316 { 4317 int i; 4318 4319 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { 4320 if (CURSEG_I(sbi, i)->segno == segno) 4321 break; 4322 } 4323 return i; 4324 } 4325 4326 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, 4327 block_t old_blkaddr, block_t new_blkaddr, 4328 bool recover_curseg, bool recover_newaddr, 4329 bool from_gc) 4330 { 4331 struct sit_info *sit_i = SIT_I(sbi); 4332 struct curseg_info *curseg; 4333 unsigned int segno, old_cursegno; 4334 struct seg_entry *se; 4335 int type; 4336 unsigned short old_blkoff; 4337 unsigned char old_alloc_type; 4338 4339 segno = GET_SEGNO(sbi, new_blkaddr); 4340 se = get_seg_entry(sbi, segno); 4341 type = se->type; 4342 4343 f2fs_down_write(&SM_I(sbi)->curseg_lock); 4344 4345 if (!recover_curseg) { 4346 /* for recovery flow */ 4347 if (se->valid_blocks == 0 && !is_curseg(sbi, segno)) { 4348 if (old_blkaddr == NULL_ADDR) 4349 type = CURSEG_COLD_DATA; 4350 else 4351 type = CURSEG_WARM_DATA; 4352 } 4353 } else { 4354 if (is_curseg(sbi, segno)) { 4355 /* se->type is volatile as SSR allocation */ 4356 type = __f2fs_get_curseg(sbi, segno); 4357 f2fs_bug_on(sbi, type == NO_CHECK_TYPE); 4358 } else { 4359 type = CURSEG_WARM_DATA; 4360 } 4361 } 4362 4363 curseg = CURSEG_I(sbi, type); 4364 f2fs_bug_on(sbi, !IS_DATASEG(curseg->seg_type)); 4365 4366 mutex_lock(&curseg->curseg_mutex); 4367 down_write(&sit_i->sentry_lock); 4368 4369 old_cursegno = curseg->segno; 4370 old_blkoff = curseg->next_blkoff; 4371 old_alloc_type = curseg->alloc_type; 4372 4373 /* change the current segment */ 4374 if (segno != curseg->segno) { 4375 curseg->next_segno = segno; 4376 if (change_curseg(sbi, type)) 4377 goto out_unlock; 4378 } 4379 4380 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); 4381 sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; 4382 4383 if (!recover_curseg || recover_newaddr) { 4384 if (!from_gc) 4385 update_segment_mtime(sbi, new_blkaddr, 0); 4386 update_sit_entry(sbi, new_blkaddr, 1); 4387 } 4388 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) { 4389 f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1); 4390 if (!from_gc) 4391 update_segment_mtime(sbi, old_blkaddr, 0); 4392 update_sit_entry(sbi, old_blkaddr, -1); 4393 } 4394 4395 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 4396 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); 4397 4398 locate_dirty_segment(sbi, old_cursegno); 4399 4400 if (recover_curseg) { 4401 if (old_cursegno != curseg->segno) { 4402 curseg->next_segno = old_cursegno; 4403 if (change_curseg(sbi, type)) 4404 goto out_unlock; 4405 } 4406 curseg->next_blkoff = old_blkoff; 4407 curseg->alloc_type = old_alloc_type; 4408 } 4409 4410 out_unlock: 4411 up_write(&sit_i->sentry_lock); 4412 mutex_unlock(&curseg->curseg_mutex); 4413 f2fs_up_write(&SM_I(sbi)->curseg_lock); 4414 } 4415 4416 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, 4417 block_t old_addr, block_t new_addr, 4418 unsigned char version, bool recover_curseg, 4419 bool recover_newaddr) 4420 { 4421 struct f2fs_summary sum; 4422 4423 set_summary(&sum, dn->nid, dn->ofs_in_node, version); 4424 4425 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr, 4426 recover_curseg, recover_newaddr, false); 4427 4428 f2fs_update_data_blkaddr(dn, new_addr); 4429 } 4430 4431 void f2fs_folio_wait_writeback(struct folio *folio, bool ordered, bool locked) 4432 { 4433 if (folio_test_writeback(folio)) { 4434 struct f2fs_sb_info *sbi = F2FS_F_SB(folio); 4435 4436 /* submit cached LFS IO */ 4437 f2fs_submit_merged_write_folio(sbi, folio); 4438 /* submit cached IPU IO */ 4439 f2fs_submit_merged_ipu_write(sbi, NULL, folio); 4440 if (ordered) { 4441 folio_wait_writeback(folio); 4442 f2fs_bug_on(sbi, locked && folio_test_writeback(folio)); 4443 } else { 4444 folio_wait_stable(folio); 4445 } 4446 } 4447 } 4448 4449 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr) 4450 { 4451 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 4452 struct f2fs_cached_block *entry; 4453 4454 if (!f2fs_meta_inode_gc_required(inode)) 4455 return; 4456 4457 if (!__is_valid_data_blkaddr(blkaddr)) 4458 return; 4459 4460 entry = f2fs_find_cache(META_CACHE(sbi), blkaddr, 0); 4461 if (IS_ERR(entry)) 4462 return; 4463 f2fs_lock_cache(entry); 4464 f2fs_cache_wait_writeback_cond(entry, DATA); 4465 f2fs_put_cache(entry, true); 4466 } 4467 4468 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, 4469 block_t len) 4470 { 4471 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 4472 block_t i; 4473 4474 if (!f2fs_meta_inode_gc_required(inode)) 4475 return; 4476 4477 for (i = 0; i < len; i++) 4478 f2fs_wait_on_block_writeback(inode, blkaddr + i); 4479 4480 f2fs_truncate_meta_caches(sbi, blkaddr, len); 4481 } 4482 4483 static int read_compacted_summaries(struct f2fs_sb_info *sbi) 4484 { 4485 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 4486 struct curseg_info *seg_i; 4487 unsigned char *kaddr; 4488 struct f2fs_cached_block *entry; 4489 block_t start; 4490 int i, j, offset; 4491 4492 start = start_sum_block(sbi); 4493 4494 entry = f2fs_get_meta_cache(sbi, start++); 4495 if (IS_ERR(entry)) 4496 return PTR_ERR(entry); 4497 kaddr = cache_address(entry); 4498 4499 /* Step 1: restore nat cache */ 4500 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 4501 memcpy(seg_i->journal, kaddr, sbi->sum_journal_size); 4502 4503 /* Step 2: restore sit cache */ 4504 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 4505 memcpy(seg_i->journal, kaddr + sbi->sum_journal_size, sbi->sum_journal_size); 4506 offset = 2 * sbi->sum_journal_size; 4507 4508 /* Step 3: restore summary entries */ 4509 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 4510 unsigned short blk_off; 4511 unsigned int segno; 4512 4513 seg_i = CURSEG_I(sbi, i); 4514 segno = le32_to_cpu(ckpt->cur_data_segno[i]); 4515 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); 4516 seg_i->next_segno = segno; 4517 reset_curseg(sbi, i, 0); 4518 seg_i->alloc_type = ckpt->alloc_type[i]; 4519 seg_i->next_blkoff = blk_off; 4520 4521 if (seg_i->alloc_type == SSR) 4522 blk_off = BLKS_PER_SEG(sbi); 4523 4524 for (j = 0; j < blk_off; j++) { 4525 struct f2fs_summary *s; 4526 4527 s = (struct f2fs_summary *)(kaddr + offset); 4528 sum_entries(seg_i->sum_blk)[j] = *s; 4529 offset += SUMMARY_SIZE; 4530 if (offset + SUMMARY_SIZE <= sbi->blocksize - 4531 SUM_FOOTER_SIZE) 4532 continue; 4533 4534 f2fs_put_cache(entry, true); 4535 4536 entry = f2fs_get_meta_cache(sbi, start++); 4537 if (IS_ERR(entry)) 4538 return PTR_ERR(entry); 4539 kaddr = cache_address(entry); 4540 offset = 0; 4541 } 4542 } 4543 f2fs_put_cache(entry, true); 4544 return 0; 4545 } 4546 4547 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) 4548 { 4549 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 4550 struct f2fs_summary_block *sum; 4551 struct curseg_info *curseg; 4552 struct f2fs_cached_block *entry; 4553 unsigned short blk_off; 4554 unsigned int segno = 0; 4555 block_t blk_addr = 0; 4556 int err = 0; 4557 4558 /* get segment number and block addr */ 4559 if (IS_DATASEG(type)) { 4560 segno = le32_to_cpu(ckpt->cur_data_segno[type]); 4561 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - 4562 CURSEG_HOT_DATA]); 4563 if (__exist_node_summaries(sbi)) 4564 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type); 4565 else 4566 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); 4567 } else { 4568 segno = le32_to_cpu(ckpt->cur_node_segno[type - 4569 CURSEG_HOT_NODE]); 4570 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - 4571 CURSEG_HOT_NODE]); 4572 if (__exist_node_summaries(sbi)) 4573 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, 4574 type - CURSEG_HOT_NODE); 4575 else 4576 blk_addr = GET_SUM_BLOCK(sbi, segno); 4577 } 4578 4579 entry = f2fs_get_meta_cache(sbi, blk_addr); 4580 if (IS_ERR(entry)) 4581 return PTR_ERR(entry); 4582 sum = cache_address(entry); 4583 4584 if (IS_NODESEG(type)) { 4585 if (__exist_node_summaries(sbi)) { 4586 struct f2fs_summary *ns = sum_entries(sum); 4587 int i; 4588 4589 for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) { 4590 ns->version = 0; 4591 ns->ofs_in_node = 0; 4592 } 4593 } else { 4594 err = f2fs_restore_node_summary(sbi, segno, sum); 4595 if (err) 4596 goto out; 4597 } 4598 } 4599 4600 /* set uncompleted segment to curseg */ 4601 curseg = CURSEG_I(sbi, type); 4602 mutex_lock(&curseg->curseg_mutex); 4603 4604 /* update journal info */ 4605 down_write(&curseg->journal_rwsem); 4606 memcpy(curseg->journal, sum_journal(sbi, sum), sbi->sum_journal_size); 4607 up_write(&curseg->journal_rwsem); 4608 4609 memcpy(sum_entries(curseg->sum_blk), sum_entries(sum), 4610 sbi->sum_entry_size); 4611 memcpy(sum_footer(sbi, curseg->sum_blk), sum_footer(sbi, sum), 4612 SUM_FOOTER_SIZE); 4613 curseg->next_segno = segno; 4614 reset_curseg(sbi, type, 0); 4615 curseg->alloc_type = ckpt->alloc_type[type]; 4616 curseg->next_blkoff = blk_off; 4617 mutex_unlock(&curseg->curseg_mutex); 4618 out: 4619 f2fs_put_cache(entry, true); 4620 return err; 4621 } 4622 4623 static int restore_curseg_summaries(struct f2fs_sb_info *sbi) 4624 { 4625 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; 4626 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; 4627 int type = CURSEG_HOT_DATA; 4628 int err; 4629 4630 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { 4631 int nblocks = f2fs_nblocks_for_summary_flush(sbi, true); 4632 4633 if (nblocks >= 2) 4634 f2fs_ra_meta_caches(sbi, start_sum_block(sbi), 4635 nblocks, META_CP, true); 4636 4637 /* restore for compacted data summary */ 4638 err = read_compacted_summaries(sbi); 4639 if (err) 4640 return err; 4641 type = CURSEG_HOT_NODE; 4642 } 4643 4644 if (__exist_node_summaries(sbi)) 4645 f2fs_ra_meta_caches(sbi, 4646 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type), 4647 NR_CURSEG_PERSIST_TYPE - type, META_CP, true); 4648 4649 for (; type <= CURSEG_COLD_NODE; type++) { 4650 err = read_normal_summaries(sbi, type); 4651 if (err) 4652 return err; 4653 } 4654 4655 /* sanity check for summary blocks */ 4656 if (nats_in_cursum(nat_j) > sbi->nat_journal_entries || 4657 sits_in_cursum(sit_j) > sbi->sit_journal_entries) { 4658 f2fs_err(sbi, "invalid journal entries nats %u sits %u", 4659 nats_in_cursum(nat_j), sits_in_cursum(sit_j)); 4660 return -EINVAL; 4661 } 4662 4663 return 0; 4664 } 4665 4666 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) 4667 { 4668 struct f2fs_cached_block *entry = NULL; 4669 unsigned char *kaddr; 4670 struct f2fs_summary *summary; 4671 struct curseg_info *seg_i; 4672 int written_size = 0; 4673 int i, j; 4674 4675 entry = f2fs_grab_meta_cache(sbi, blkaddr++); 4676 kaddr = cache_address(entry); 4677 memset(kaddr, 0, sbi->blocksize); 4678 4679 /* Step 1: write nat cache */ 4680 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 4681 memcpy(kaddr, seg_i->journal, sbi->sum_journal_size); 4682 written_size += sbi->sum_journal_size; 4683 4684 /* Step 2: write sit cache */ 4685 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 4686 memcpy(kaddr + written_size, seg_i->journal, sbi->sum_journal_size); 4687 written_size += sbi->sum_journal_size; 4688 4689 /* Step 3: write summary entries */ 4690 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 4691 seg_i = CURSEG_I(sbi, i); 4692 for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) { 4693 if (!entry) { 4694 entry = f2fs_grab_meta_cache(sbi, blkaddr++); 4695 kaddr = cache_address(entry); 4696 memset(kaddr, 0, sbi->blocksize); 4697 written_size = 0; 4698 } 4699 summary = (struct f2fs_summary *)(kaddr + written_size); 4700 *summary = sum_entries(seg_i->sum_blk)[j]; 4701 written_size += SUMMARY_SIZE; 4702 4703 if (written_size + SUMMARY_SIZE <= sbi->blocksize - 4704 SUM_FOOTER_SIZE) 4705 continue; 4706 4707 f2fs_mark_cache_dirty(entry); 4708 f2fs_put_cache(entry, true); 4709 entry = NULL; 4710 } 4711 } 4712 if (entry) { 4713 f2fs_mark_cache_dirty(entry); 4714 f2fs_put_cache(entry, true); 4715 } 4716 } 4717 4718 static void write_normal_summaries(struct f2fs_sb_info *sbi, 4719 block_t blkaddr, int type) 4720 { 4721 int i, end; 4722 4723 if (IS_DATASEG(type)) 4724 end = type + NR_CURSEG_DATA_TYPE; 4725 else 4726 end = type + NR_CURSEG_NODE_TYPE; 4727 4728 for (i = type; i < end; i++) 4729 write_current_sum_block(sbi, i, blkaddr + (i - type)); 4730 } 4731 4732 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 4733 { 4734 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) 4735 write_compacted_summaries(sbi, start_blk); 4736 else 4737 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); 4738 } 4739 4740 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 4741 { 4742 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); 4743 } 4744 4745 int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, 4746 struct f2fs_journal *journal, int type, 4747 unsigned int val, int alloc) 4748 { 4749 int i; 4750 4751 if (type == NAT_JOURNAL) { 4752 for (i = 0; i < nats_in_cursum(journal); i++) { 4753 if (le32_to_cpu(nid_in_journal(journal, i)) == val) 4754 return i; 4755 } 4756 if (alloc && __has_cursum_space(sbi, journal, 1, NAT_JOURNAL)) 4757 return update_nats_in_cursum(journal, 1); 4758 } else if (type == SIT_JOURNAL) { 4759 for (i = 0; i < sits_in_cursum(journal); i++) 4760 if (le32_to_cpu(segno_in_journal(journal, i)) == val) 4761 return i; 4762 if (alloc && __has_cursum_space(sbi, journal, 1, SIT_JOURNAL)) 4763 return update_sits_in_cursum(journal, 1); 4764 } 4765 return -1; 4766 } 4767 4768 static struct f2fs_cached_block *get_current_sit_cache(struct f2fs_sb_info *sbi, 4769 unsigned int segno) 4770 { 4771 return f2fs_get_meta_cache(sbi, current_sit_addr(sbi, segno)); 4772 } 4773 4774 static struct f2fs_cached_block *get_next_sit_cache(struct f2fs_sb_info *sbi, 4775 unsigned int start) 4776 { 4777 struct sit_info *sit_i = SIT_I(sbi); 4778 struct f2fs_cached_block *entry; 4779 pgoff_t src_off, dst_off; 4780 4781 src_off = current_sit_addr(sbi, start); 4782 dst_off = next_sit_addr(sbi, src_off); 4783 4784 entry = f2fs_grab_meta_cache(sbi, dst_off); 4785 seg_info_to_sit_block(sbi, entry, start); 4786 4787 f2fs_mark_cache_dirty(entry); 4788 set_to_next_sit(sbi, sit_i, start); 4789 4790 return entry; 4791 } 4792 4793 static struct sit_entry_set *grab_sit_entry_set(void) 4794 { 4795 struct sit_entry_set *ses = 4796 f2fs_kmem_cache_alloc(sit_entry_set_slab, 4797 GFP_NOFS, true, NULL); 4798 4799 ses->entry_cnt = 0; 4800 INIT_LIST_HEAD(&ses->set_list); 4801 return ses; 4802 } 4803 4804 static void release_sit_entry_set(struct sit_entry_set *ses) 4805 { 4806 list_del(&ses->set_list); 4807 kmem_cache_free(sit_entry_set_slab, ses); 4808 } 4809 4810 static void adjust_sit_entry_set(struct sit_entry_set *ses, 4811 struct list_head *head) 4812 { 4813 struct sit_entry_set *next = ses; 4814 4815 if (list_is_last(&ses->set_list, head)) 4816 return; 4817 4818 list_for_each_entry_continue(next, head, set_list) 4819 if (ses->entry_cnt <= next->entry_cnt) { 4820 list_move_tail(&ses->set_list, &next->set_list); 4821 return; 4822 } 4823 4824 list_move_tail(&ses->set_list, head); 4825 } 4826 4827 static void add_sit_entry(struct f2fs_sb_info *sbi, unsigned int segno, 4828 struct list_head *head) 4829 { 4830 struct sit_entry_set *ses; 4831 unsigned int start_segno = f2fs_start_segno(sbi, segno); 4832 4833 list_for_each_entry(ses, head, set_list) { 4834 if (ses->start_segno == start_segno) { 4835 ses->entry_cnt++; 4836 adjust_sit_entry_set(ses, head); 4837 return; 4838 } 4839 } 4840 4841 ses = grab_sit_entry_set(); 4842 4843 ses->start_segno = start_segno; 4844 ses->entry_cnt++; 4845 list_add(&ses->set_list, head); 4846 } 4847 4848 static void add_sits_in_set(struct f2fs_sb_info *sbi) 4849 { 4850 struct f2fs_sm_info *sm_info = SM_I(sbi); 4851 struct list_head *set_list = &sm_info->sit_entry_set; 4852 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; 4853 unsigned int segno; 4854 4855 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) 4856 add_sit_entry(sbi, segno, set_list); 4857 } 4858 4859 static void remove_sits_in_journal(struct f2fs_sb_info *sbi) 4860 { 4861 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4862 struct f2fs_journal *journal = curseg->journal; 4863 int i; 4864 4865 down_write(&curseg->journal_rwsem); 4866 for (i = 0; i < sits_in_cursum(journal); i++) { 4867 unsigned int segno; 4868 bool dirtied; 4869 4870 segno = le32_to_cpu(segno_in_journal(journal, i)); 4871 dirtied = __mark_sit_entry_dirty(sbi, segno); 4872 4873 if (!dirtied) 4874 add_sit_entry(sbi, segno, &SM_I(sbi)->sit_entry_set); 4875 } 4876 update_sits_in_cursum(journal, -i); 4877 up_write(&curseg->journal_rwsem); 4878 } 4879 4880 /* 4881 * CP calls this function, which flushes SIT entries including sit_journal, 4882 * and moves prefree segs to free segs. 4883 */ 4884 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) 4885 { 4886 struct sit_info *sit_i = SIT_I(sbi); 4887 unsigned long *bitmap = sit_i->dirty_sentries_bitmap; 4888 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 4889 struct f2fs_journal *journal = curseg->journal; 4890 struct sit_entry_set *ses, *tmp; 4891 struct list_head *head = &SM_I(sbi)->sit_entry_set; 4892 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS); 4893 struct seg_entry *se; 4894 4895 down_write(&sit_i->sentry_lock); 4896 4897 if (!sit_i->dirty_sentries && to_journal) 4898 goto out; 4899 4900 /* 4901 * add and account sit entries of dirty bitmap in sit entry 4902 * set temporarily 4903 */ 4904 add_sits_in_set(sbi); 4905 4906 /* 4907 * if there are no enough space in journal to store dirty sit 4908 * entries, remove all entries from journal and add and account 4909 * them in sit entry set. 4910 */ 4911 if (!__has_cursum_space(sbi, journal, 4912 sit_i->dirty_sentries, SIT_JOURNAL) || !to_journal) 4913 remove_sits_in_journal(sbi); 4914 4915 /* 4916 * there are two steps to flush sit entries: 4917 * #1, flush sit entries to journal in current cold data summary block. 4918 * #2, flush sit entries to sit page. 4919 */ 4920 list_for_each_entry_safe(ses, tmp, head, set_list) { 4921 struct f2fs_cached_block *entry = NULL; 4922 struct f2fs_sit_block *raw_sit = NULL; 4923 4924 unsigned int start_segno = ses->start_segno; 4925 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK(sbi), 4926 (unsigned long)MAIN_SEGS(sbi)); 4927 unsigned int segno = start_segno; 4928 4929 if (to_journal && 4930 !__has_cursum_space(sbi, journal, ses->entry_cnt, 4931 SIT_JOURNAL)) 4932 to_journal = false; 4933 4934 if (to_journal) { 4935 down_write(&curseg->journal_rwsem); 4936 } else { 4937 entry = get_next_sit_cache(sbi, start_segno); 4938 raw_sit = cache_address(entry); 4939 } 4940 4941 /* flush dirty sit entries in region of current sit set */ 4942 for_each_set_bit_from(segno, bitmap, end) { 4943 int offset, sit_offset; 4944 4945 se = get_seg_entry(sbi, segno); 4946 4947 /* add discard candidates */ 4948 if (!(cpc->reason & CP_DISCARD)) { 4949 cpc->trim_start = segno; 4950 add_discard_addrs(sbi, cpc, false); 4951 } 4952 4953 if (to_journal) { 4954 offset = f2fs_lookup_journal_in_cursum(sbi, journal, 4955 SIT_JOURNAL, segno, 1); 4956 f2fs_bug_on(sbi, offset < 0); 4957 segno_in_journal(journal, offset) = 4958 cpu_to_le32(segno); 4959 seg_info_to_raw_sit(se, 4960 &sit_in_journal(journal, offset)); 4961 check_block_count(sbi, segno, 4962 &sit_in_journal(journal, offset)); 4963 } else { 4964 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); 4965 seg_info_to_raw_sit(se, 4966 &raw_sit->entries[sit_offset]); 4967 check_block_count(sbi, segno, 4968 &raw_sit->entries[sit_offset]); 4969 } 4970 4971 /* update ckpt_valid_block */ 4972 if (__is_large_section(sbi)) 4973 set_ckpt_valid_blocks(sbi, segno); 4974 4975 __clear_bit(segno, bitmap); 4976 sit_i->dirty_sentries--; 4977 ses->entry_cnt--; 4978 } 4979 4980 if (to_journal) 4981 up_write(&curseg->journal_rwsem); 4982 else 4983 f2fs_put_cache(entry, true); 4984 4985 f2fs_bug_on(sbi, ses->entry_cnt); 4986 release_sit_entry_set(ses); 4987 } 4988 4989 f2fs_bug_on(sbi, !list_empty(head)); 4990 f2fs_bug_on(sbi, sit_i->dirty_sentries); 4991 out: 4992 if (cpc->reason & CP_DISCARD) { 4993 __u64 trim_start = cpc->trim_start; 4994 4995 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) 4996 add_discard_addrs(sbi, cpc, false); 4997 4998 cpc->trim_start = trim_start; 4999 } 5000 up_write(&sit_i->sentry_lock); 5001 5002 set_prefree_as_free_segments(sbi); 5003 } 5004 5005 static int build_sit_info(struct f2fs_sb_info *sbi) 5006 { 5007 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 5008 struct sit_info *sit_i; 5009 unsigned int sit_segs, start; 5010 char *src_bitmap, *bitmap; 5011 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size; 5012 unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0; 5013 5014 /* allocate memory for SIT information */ 5015 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); 5016 if (!sit_i) 5017 return -ENOMEM; 5018 5019 SM_I(sbi)->sit_info = sit_i; 5020 5021 sit_i->sentries = 5022 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry), 5023 MAIN_SEGS(sbi)), 5024 GFP_KERNEL); 5025 if (!sit_i->sentries) 5026 return -ENOMEM; 5027 5028 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 5029 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size, 5030 GFP_KERNEL); 5031 if (!sit_i->dirty_sentries_bitmap) 5032 return -ENOMEM; 5033 5034 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map); 5035 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 5036 if (!sit_i->bitmap) 5037 return -ENOMEM; 5038 5039 bitmap = sit_i->bitmap; 5040 5041 for (start = 0; start < MAIN_SEGS(sbi); start++) { 5042 sit_i->sentries[start].cur_valid_map = bitmap; 5043 bitmap += SIT_VBLOCK_MAP_SIZE; 5044 5045 sit_i->sentries[start].ckpt_valid_map = bitmap; 5046 bitmap += SIT_VBLOCK_MAP_SIZE; 5047 5048 if (discard_map) { 5049 sit_i->sentries[start].discard_map = bitmap; 5050 bitmap += SIT_VBLOCK_MAP_SIZE; 5051 } 5052 } 5053 5054 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 5055 if (!sit_i->tmp_map) 5056 return -ENOMEM; 5057 5058 if (__is_large_section(sbi)) { 5059 sit_i->sec_entries = 5060 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry), 5061 MAIN_SECS(sbi)), 5062 GFP_KERNEL); 5063 if (!sit_i->sec_entries) 5064 return -ENOMEM; 5065 } 5066 5067 /* get information related with SIT */ 5068 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; 5069 5070 /* setup SIT bitmap from ckeckpoint pack */ 5071 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP); 5072 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); 5073 5074 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL); 5075 if (!sit_i->sit_bitmap) 5076 return -ENOMEM; 5077 5078 #ifdef CONFIG_F2FS_CHECK_FS 5079 sit_i->invalid_segmap = f2fs_kvzalloc(sbi, 5080 main_bitmap_size, GFP_KERNEL); 5081 if (!sit_i->invalid_segmap) 5082 return -ENOMEM; 5083 #endif 5084 5085 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); 5086 sit_i->sit_blocks = SEGS_TO_BLKS(sbi, sit_segs); 5087 sit_i->written_valid_blocks = 0; 5088 sit_i->bitmap_size = sit_bitmap_size; 5089 sit_i->dirty_sentries = 0; 5090 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK(sbi); 5091 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); 5092 sit_i->mounted_time = ktime_get_boottime_seconds(); 5093 init_rwsem(&sit_i->sentry_lock); 5094 return 0; 5095 } 5096 5097 static int build_free_segmap(struct f2fs_sb_info *sbi) 5098 { 5099 struct free_segmap_info *free_i; 5100 unsigned int bitmap_size, sec_bitmap_size; 5101 5102 /* allocate memory for free segmap information */ 5103 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); 5104 if (!free_i) 5105 return -ENOMEM; 5106 5107 SM_I(sbi)->free_info = free_i; 5108 5109 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 5110 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); 5111 if (!free_i->free_segmap) 5112 return -ENOMEM; 5113 5114 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 5115 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); 5116 if (!free_i->free_secmap) 5117 return -ENOMEM; 5118 5119 /* set all segments as dirty temporarily */ 5120 memset(free_i->free_segmap, 0xff, bitmap_size); 5121 memset(free_i->free_secmap, 0xff, sec_bitmap_size); 5122 5123 /* init free segmap information */ 5124 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); 5125 free_i->free_segments = 0; 5126 free_i->free_sections = 0; 5127 spin_lock_init(&free_i->segmap_lock); 5128 return 0; 5129 } 5130 5131 static int build_curseg(struct f2fs_sb_info *sbi) 5132 { 5133 struct curseg_info *array; 5134 int i; 5135 5136 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, 5137 sizeof(*array)), GFP_KERNEL); 5138 if (!array) 5139 return -ENOMEM; 5140 5141 SM_I(sbi)->curseg_array = array; 5142 5143 for (i = 0; i < NO_CHECK_TYPE; i++) { 5144 mutex_init(&array[i].curseg_mutex); 5145 array[i].sum_blk = f2fs_kzalloc(sbi, sbi->sum_blocksize, 5146 GFP_KERNEL); 5147 if (!array[i].sum_blk) 5148 return -ENOMEM; 5149 init_rwsem(&array[i].journal_rwsem); 5150 array[i].journal = f2fs_kzalloc(sbi, 5151 sbi->sum_journal_size, GFP_KERNEL); 5152 if (!array[i].journal) 5153 return -ENOMEM; 5154 array[i].seg_type = log_type_to_seg_type(i); 5155 reset_curseg_fields(&array[i]); 5156 } 5157 return restore_curseg_summaries(sbi); 5158 } 5159 5160 static int build_sit_entries(struct f2fs_sb_info *sbi) 5161 { 5162 struct sit_info *sit_i = SIT_I(sbi); 5163 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 5164 struct f2fs_journal *journal = curseg->journal; 5165 struct seg_entry *se; 5166 struct f2fs_sit_entry sit; 5167 int sit_blk_cnt = SIT_BLK_CNT(sbi); 5168 unsigned int i, start, end; 5169 unsigned int readed, start_blk = 0; 5170 int err = 0; 5171 block_t sit_valid_blocks[2] = {0, 0}; 5172 5173 do { 5174 readed = f2fs_ra_meta_caches(sbi, start_blk, BIO_MAX_VECS, 5175 META_SIT, true); 5176 5177 start = start_blk * sit_i->sents_per_block; 5178 end = (start_blk + readed) * sit_i->sents_per_block; 5179 5180 for (; start < end && start < MAIN_SEGS(sbi); start++) { 5181 struct f2fs_sit_block *sit_blk; 5182 struct f2fs_cached_block *entry; 5183 5184 se = &sit_i->sentries[start]; 5185 entry = get_current_sit_cache(sbi, start); 5186 if (IS_ERR(entry)) 5187 return PTR_ERR(entry); 5188 sit_blk = cache_address(entry); 5189 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; 5190 f2fs_put_cache(entry, true); 5191 5192 err = check_block_count(sbi, start, &sit); 5193 if (err) 5194 return err; 5195 seg_info_from_raw_sit(se, &sit); 5196 5197 if (se->type >= NR_PERSISTENT_LOG) { 5198 f2fs_err(sbi, "Invalid segment type: %u, segno: %u", 5199 se->type, start); 5200 f2fs_handle_error(sbi, 5201 ERROR_INCONSISTENT_SUM_TYPE); 5202 return -EFSCORRUPTED; 5203 } 5204 5205 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks; 5206 5207 if (!f2fs_block_unit_discard(sbi)) 5208 goto init_discard_map_done; 5209 5210 /* build discard map only one time */ 5211 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 5212 memset(se->discard_map, 0xff, 5213 SIT_VBLOCK_MAP_SIZE); 5214 goto init_discard_map_done; 5215 } 5216 memcpy(se->discard_map, se->cur_valid_map, 5217 SIT_VBLOCK_MAP_SIZE); 5218 sbi->discard_blks += BLKS_PER_SEG(sbi) - 5219 se->valid_blocks; 5220 init_discard_map_done: 5221 if (__is_large_section(sbi)) 5222 get_sec_entry(sbi, start)->valid_blocks += 5223 se->valid_blocks; 5224 } 5225 start_blk += readed; 5226 } while (start_blk < sit_blk_cnt); 5227 5228 down_read(&curseg->journal_rwsem); 5229 for (i = 0; i < sits_in_cursum(journal); i++) { 5230 unsigned int old_valid_blocks; 5231 5232 start = le32_to_cpu(segno_in_journal(journal, i)); 5233 if (start >= MAIN_SEGS(sbi)) { 5234 f2fs_err(sbi, "Wrong journal entry on segno %u", 5235 start); 5236 err = -EFSCORRUPTED; 5237 f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL); 5238 break; 5239 } 5240 5241 se = &sit_i->sentries[start]; 5242 sit = sit_in_journal(journal, i); 5243 5244 old_valid_blocks = se->valid_blocks; 5245 5246 sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks; 5247 5248 err = check_block_count(sbi, start, &sit); 5249 if (err) 5250 break; 5251 seg_info_from_raw_sit(se, &sit); 5252 5253 if (se->type >= NR_PERSISTENT_LOG) { 5254 f2fs_err(sbi, "Invalid segment type: %u, segno: %u", 5255 se->type, start); 5256 err = -EFSCORRUPTED; 5257 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); 5258 break; 5259 } 5260 5261 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks; 5262 5263 if (f2fs_block_unit_discard(sbi)) { 5264 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { 5265 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE); 5266 } else { 5267 memcpy(se->discard_map, se->cur_valid_map, 5268 SIT_VBLOCK_MAP_SIZE); 5269 sbi->discard_blks += old_valid_blocks; 5270 sbi->discard_blks -= se->valid_blocks; 5271 } 5272 } 5273 5274 if (__is_large_section(sbi)) { 5275 get_sec_entry(sbi, start)->valid_blocks += 5276 se->valid_blocks; 5277 get_sec_entry(sbi, start)->valid_blocks -= 5278 old_valid_blocks; 5279 } 5280 } 5281 up_read(&curseg->journal_rwsem); 5282 5283 /* update ckpt_valid_block */ 5284 if (__is_large_section(sbi)) { 5285 unsigned int segno; 5286 5287 for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) 5288 set_ckpt_valid_blocks(sbi, segno); 5289 } 5290 5291 if (err) 5292 return err; 5293 5294 if (sit_valid_blocks[NODE] != valid_node_count(sbi)) { 5295 f2fs_err(sbi, "SIT is corrupted node# %u vs %u", 5296 sit_valid_blocks[NODE], valid_node_count(sbi)); 5297 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT); 5298 return -EFSCORRUPTED; 5299 } 5300 5301 if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] > 5302 valid_user_blocks(sbi)) { 5303 f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u", 5304 sit_valid_blocks[DATA], sit_valid_blocks[NODE], 5305 valid_user_blocks(sbi)); 5306 f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT); 5307 return -EFSCORRUPTED; 5308 } 5309 5310 return 0; 5311 } 5312 5313 static void init_free_segmap(struct f2fs_sb_info *sbi) 5314 { 5315 unsigned int start; 5316 int type; 5317 struct seg_entry *sentry; 5318 5319 for (start = 0; start < MAIN_SEGS(sbi); start++) { 5320 if (f2fs_usable_blks_in_seg(sbi, start) == 0) 5321 continue; 5322 sentry = get_seg_entry(sbi, start); 5323 if (!sentry->valid_blocks) 5324 __set_free(sbi, start); 5325 else 5326 SIT_I(sbi)->written_valid_blocks += 5327 sentry->valid_blocks; 5328 } 5329 5330 /* set use the current segments */ 5331 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { 5332 struct curseg_info *curseg_t = CURSEG_I(sbi, type); 5333 5334 __set_test_and_inuse(sbi, curseg_t->segno); 5335 } 5336 } 5337 5338 static void init_dirty_segmap(struct f2fs_sb_info *sbi) 5339 { 5340 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5341 struct free_segmap_info *free_i = FREE_I(sbi); 5342 unsigned int segno = 0, offset = 0, secno; 5343 block_t valid_blocks, usable_blks_in_seg; 5344 5345 while (1) { 5346 /* find dirty segment based on free segmap */ 5347 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); 5348 if (segno >= MAIN_SEGS(sbi)) 5349 break; 5350 offset = segno + 1; 5351 valid_blocks = get_valid_blocks(sbi, segno, false); 5352 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno); 5353 if (valid_blocks == usable_blks_in_seg || !valid_blocks) 5354 continue; 5355 if (valid_blocks > usable_blks_in_seg) { 5356 f2fs_bug_on(sbi, 1); 5357 continue; 5358 } 5359 mutex_lock(&dirty_i->seglist_lock); 5360 __locate_dirty_segment(sbi, segno, DIRTY); 5361 mutex_unlock(&dirty_i->seglist_lock); 5362 } 5363 5364 if (!__is_large_section(sbi)) 5365 return; 5366 5367 mutex_lock(&dirty_i->seglist_lock); 5368 for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { 5369 valid_blocks = get_valid_blocks(sbi, segno, true); 5370 secno = GET_SEC_FROM_SEG(sbi, segno); 5371 5372 if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi)) 5373 continue; 5374 if (is_cursec(sbi, secno)) 5375 continue; 5376 set_bit(secno, dirty_i->dirty_secmap); 5377 } 5378 mutex_unlock(&dirty_i->seglist_lock); 5379 } 5380 5381 static int init_victim_secmap(struct f2fs_sb_info *sbi) 5382 { 5383 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5384 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 5385 5386 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 5387 if (!dirty_i->victim_secmap) 5388 return -ENOMEM; 5389 5390 dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); 5391 if (!dirty_i->pinned_secmap) 5392 return -ENOMEM; 5393 5394 dirty_i->pinned_secmap_cnt = 0; 5395 dirty_i->enable_pin_section = true; 5396 return 0; 5397 } 5398 5399 static int build_dirty_segmap(struct f2fs_sb_info *sbi) 5400 { 5401 struct dirty_seglist_info *dirty_i; 5402 unsigned int bitmap_size, i; 5403 5404 /* allocate memory for dirty segments list information */ 5405 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), 5406 GFP_KERNEL); 5407 if (!dirty_i) 5408 return -ENOMEM; 5409 5410 SM_I(sbi)->dirty_info = dirty_i; 5411 mutex_init(&dirty_i->seglist_lock); 5412 5413 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); 5414 5415 for (i = 0; i < NR_DIRTY_TYPE; i++) { 5416 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, 5417 GFP_KERNEL); 5418 if (!dirty_i->dirty_segmap[i]) 5419 return -ENOMEM; 5420 } 5421 5422 if (__is_large_section(sbi)) { 5423 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); 5424 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi, 5425 bitmap_size, GFP_KERNEL); 5426 if (!dirty_i->dirty_secmap) 5427 return -ENOMEM; 5428 } 5429 5430 init_dirty_segmap(sbi); 5431 return init_victim_secmap(sbi); 5432 } 5433 5434 static int sanity_check_curseg(struct f2fs_sb_info *sbi) 5435 { 5436 int i; 5437 5438 /* 5439 * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr; 5440 * In LFS curseg, all blkaddr after .next_blkoff should be unused. 5441 */ 5442 for (i = 0; i < NR_PERSISTENT_LOG; i++) { 5443 struct curseg_info *curseg = CURSEG_I(sbi, i); 5444 struct seg_entry *se = get_seg_entry(sbi, curseg->segno); 5445 unsigned int blkofs = curseg->next_blkoff; 5446 5447 if (f2fs_sb_has_readonly(sbi) && 5448 i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE) 5449 continue; 5450 5451 sanity_check_seg_type(sbi, curseg->seg_type); 5452 5453 if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) { 5454 f2fs_err(sbi, 5455 "Current segment has invalid alloc_type:%d", 5456 curseg->alloc_type); 5457 f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); 5458 return -EFSCORRUPTED; 5459 } 5460 5461 if (f2fs_test_bit(blkofs, se->cur_valid_map)) 5462 goto out; 5463 5464 if (curseg->alloc_type == SSR) 5465 continue; 5466 5467 for (blkofs += 1; blkofs < BLKS_PER_SEG(sbi); blkofs++) { 5468 if (!f2fs_test_bit(blkofs, se->cur_valid_map)) 5469 continue; 5470 out: 5471 f2fs_err(sbi, 5472 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u", 5473 i, curseg->segno, curseg->alloc_type, 5474 curseg->next_blkoff, blkofs); 5475 f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); 5476 return -EFSCORRUPTED; 5477 } 5478 } 5479 return 0; 5480 } 5481 5482 #ifdef CONFIG_BLK_DEV_ZONED 5483 static int check_zone_write_pointer(struct f2fs_sb_info *sbi, 5484 struct f2fs_dev_info *fdev, 5485 struct blk_zone *zone) 5486 { 5487 unsigned int zone_segno; 5488 block_t zone_block, valid_block_cnt; 5489 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; 5490 int ret; 5491 unsigned int nofs_flags; 5492 5493 if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ) 5494 return 0; 5495 5496 zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block); 5497 zone_segno = GET_SEGNO(sbi, zone_block); 5498 5499 /* 5500 * Skip check of zones cursegs point to, since 5501 * fix_curseg_write_pointer() checks them. 5502 */ 5503 if (zone_segno >= MAIN_SEGS(sbi)) 5504 return 0; 5505 5506 /* 5507 * Get # of valid block of the zone. 5508 */ 5509 valid_block_cnt = get_valid_blocks(sbi, zone_segno, true); 5510 if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, zone_segno))) { 5511 f2fs_notice(sbi, "Open zones: valid block[0x%x,0x%x] cond[%s]", 5512 zone_segno, valid_block_cnt, 5513 blk_zone_cond_str(zone->cond)); 5514 return 0; 5515 } 5516 5517 if ((!valid_block_cnt && zone->cond == BLK_ZONE_COND_EMPTY) || 5518 (valid_block_cnt && zone->cond == BLK_ZONE_COND_FULL)) 5519 return 0; 5520 5521 if (!valid_block_cnt) { 5522 f2fs_notice(sbi, "Zone without valid block has non-zero write " 5523 "pointer. Reset the write pointer: cond[%s]", 5524 blk_zone_cond_str(zone->cond)); 5525 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block, 5526 zone->len >> log_sectors_per_block); 5527 if (ret) 5528 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", 5529 fdev->path, ret); 5530 return ret; 5531 } 5532 5533 /* 5534 * If there are valid blocks and the write pointer doesn't match 5535 * with them, we need to report the inconsistency and fill 5536 * the zone till the end to close the zone. This inconsistency 5537 * does not cause write error because the zone will not be 5538 * selected for write operation until it get discarded. 5539 */ 5540 f2fs_notice(sbi, "Valid blocks are not aligned with write " 5541 "pointer: valid block[0x%x,0x%x] cond[%s]", 5542 zone_segno, valid_block_cnt, blk_zone_cond_str(zone->cond)); 5543 5544 nofs_flags = memalloc_nofs_save(); 5545 ret = blkdev_zone_mgmt(fdev->bdev, REQ_OP_ZONE_FINISH, 5546 zone->start, zone->len); 5547 memalloc_nofs_restore(nofs_flags); 5548 if (ret == -EOPNOTSUPP) { 5549 ret = blkdev_issue_zeroout(fdev->bdev, zone->wp, 5550 zone->len - (zone->wp - zone->start), 5551 GFP_NOFS, 0); 5552 if (ret) 5553 f2fs_err(sbi, "Fill up zone failed: %s (errno=%d)", 5554 fdev->path, ret); 5555 } else if (ret) { 5556 f2fs_err(sbi, "Finishing zone failed: %s (errno=%d)", 5557 fdev->path, ret); 5558 } 5559 5560 return ret; 5561 } 5562 5563 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi, 5564 block_t zone_blkaddr) 5565 { 5566 int i; 5567 5568 for (i = 0; i < sbi->s_ndevs; i++) { 5569 if (!bdev_is_zoned(FDEV(i).bdev)) 5570 continue; 5571 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr && 5572 zone_blkaddr <= FDEV(i).end_blk)) 5573 return &FDEV(i); 5574 } 5575 5576 return NULL; 5577 } 5578 5579 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx, 5580 void *data) 5581 { 5582 memcpy(data, zone, sizeof(struct blk_zone)); 5583 return 0; 5584 } 5585 5586 static int do_fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type) 5587 { 5588 struct curseg_info *cs = CURSEG_I(sbi, type); 5589 struct f2fs_dev_info *zbd; 5590 struct blk_zone zone; 5591 unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off; 5592 block_t cs_zone_block, wp_block; 5593 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; 5594 sector_t zone_sector; 5595 int err; 5596 5597 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); 5598 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); 5599 5600 zbd = get_target_zoned_dev(sbi, cs_zone_block); 5601 if (!zbd) 5602 return 0; 5603 5604 /* report zone for the sector the curseg points to */ 5605 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) 5606 << log_sectors_per_block; 5607 err = blkdev_report_zones(zbd->bdev, zone_sector, 1, 5608 report_one_zone_cb, &zone); 5609 if (err != 1) { 5610 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", 5611 zbd->path, err); 5612 return err; 5613 } 5614 5615 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) 5616 return 0; 5617 5618 /* 5619 * When safely unmounted in the previous mount, we could use current 5620 * segments. Otherwise, allocate new sections. 5621 */ 5622 if (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) { 5623 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block); 5624 wp_segno = GET_SEGNO(sbi, wp_block); 5625 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno); 5626 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0); 5627 5628 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff && 5629 wp_sector_off == 0) 5630 return 0; 5631 5632 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: " 5633 "curseg[0x%x,0x%x] wp[0x%x,0x%x]", type, cs->segno, 5634 cs->next_blkoff, wp_segno, wp_blkoff); 5635 } 5636 5637 /* Allocate a new section if it's not new. */ 5638 if (cs->next_blkoff || 5639 cs->segno != GET_SEG_FROM_SEC(sbi, GET_ZONE_FROM_SEC(sbi, cs_section))) { 5640 unsigned int old_segno = cs->segno, old_blkoff = cs->next_blkoff; 5641 5642 f2fs_allocate_new_section(sbi, type, true); 5643 f2fs_notice(sbi, "Assign new section to curseg[%d]: " 5644 "[0x%x,0x%x] -> [0x%x,0x%x]", 5645 type, old_segno, old_blkoff, 5646 cs->segno, cs->next_blkoff); 5647 } 5648 5649 /* check consistency of the zone curseg pointed to */ 5650 if (check_zone_write_pointer(sbi, zbd, &zone)) 5651 return -EIO; 5652 5653 /* check newly assigned zone */ 5654 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); 5655 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); 5656 5657 zbd = get_target_zoned_dev(sbi, cs_zone_block); 5658 if (!zbd) 5659 return 0; 5660 5661 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) 5662 << log_sectors_per_block; 5663 err = blkdev_report_zones(zbd->bdev, zone_sector, 1, 5664 report_one_zone_cb, &zone); 5665 if (err != 1) { 5666 f2fs_err(sbi, "Report zone failed: %s errno=(%d)", 5667 zbd->path, err); 5668 return err; 5669 } 5670 5671 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) 5672 return 0; 5673 5674 if (zone.wp != zone.start) { 5675 f2fs_notice(sbi, 5676 "New zone for curseg[%d] is not yet discarded. " 5677 "Reset the zone: curseg[0x%x,0x%x]", 5678 type, cs->segno, cs->next_blkoff); 5679 err = __f2fs_issue_discard_zone(sbi, zbd->bdev, cs_zone_block, 5680 zone.len >> log_sectors_per_block); 5681 if (err) { 5682 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", 5683 zbd->path, err); 5684 return err; 5685 } 5686 } 5687 5688 return 0; 5689 } 5690 5691 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi) 5692 { 5693 int i, ret; 5694 5695 for (i = 0; i < NR_PERSISTENT_LOG; i++) { 5696 ret = do_fix_curseg_write_pointer(sbi, i); 5697 if (ret) 5698 return ret; 5699 } 5700 5701 return 0; 5702 } 5703 5704 struct check_zone_write_pointer_args { 5705 struct f2fs_sb_info *sbi; 5706 struct f2fs_dev_info *fdev; 5707 }; 5708 5709 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx, 5710 void *data) 5711 { 5712 struct check_zone_write_pointer_args *args; 5713 5714 args = (struct check_zone_write_pointer_args *)data; 5715 5716 return check_zone_write_pointer(args->sbi, args->fdev, zone); 5717 } 5718 5719 static int check_write_pointer(struct f2fs_sb_info *sbi) 5720 { 5721 int i, ret; 5722 struct check_zone_write_pointer_args args; 5723 5724 for (i = 0; i < sbi->s_ndevs; i++) { 5725 if (!bdev_is_zoned(FDEV(i).bdev)) 5726 continue; 5727 5728 args.sbi = sbi; 5729 args.fdev = &FDEV(i); 5730 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES, 5731 check_zone_write_pointer_cb, &args); 5732 if (ret < 0) 5733 return ret; 5734 } 5735 5736 return 0; 5737 } 5738 5739 int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi) 5740 { 5741 int ret; 5742 5743 if (!f2fs_sb_has_blkzoned(sbi) || f2fs_readonly(sbi->sb) || 5744 f2fs_hw_is_readonly(sbi)) 5745 return 0; 5746 5747 f2fs_notice(sbi, "Checking entire write pointers"); 5748 ret = fix_curseg_write_pointer(sbi); 5749 if (!ret) 5750 ret = check_write_pointer(sbi); 5751 return ret; 5752 } 5753 5754 /* 5755 * Return the number of usable blocks in a segment. The number of blocks 5756 * returned is always equal to the number of blocks in a segment for 5757 * segments fully contained within a sequential zone capacity or a 5758 * conventional zone. For segments partially contained in a sequential 5759 * zone capacity, the number of usable blocks up to the zone capacity 5760 * is returned. 0 is returned in all other cases. 5761 */ 5762 static inline unsigned int f2fs_usable_zone_blks_in_seg( 5763 struct f2fs_sb_info *sbi, unsigned int segno) 5764 { 5765 block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr; 5766 unsigned int secno; 5767 5768 if (!sbi->unusable_blocks_per_sec) 5769 return BLKS_PER_SEG(sbi); 5770 5771 secno = GET_SEC_FROM_SEG(sbi, segno); 5772 seg_start = START_BLOCK(sbi, segno); 5773 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); 5774 sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi); 5775 5776 /* 5777 * If segment starts before zone capacity and spans beyond 5778 * zone capacity, then usable blocks are from seg start to 5779 * zone capacity. If the segment starts after the zone capacity, 5780 * then there are no usable blocks. 5781 */ 5782 if (seg_start >= sec_cap_blkaddr) 5783 return 0; 5784 if (seg_start + BLKS_PER_SEG(sbi) > sec_cap_blkaddr) 5785 return sec_cap_blkaddr - seg_start; 5786 5787 return BLKS_PER_SEG(sbi); 5788 } 5789 #else 5790 int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi) 5791 { 5792 return 0; 5793 } 5794 5795 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi, 5796 unsigned int segno) 5797 { 5798 return 0; 5799 } 5800 5801 #endif 5802 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, 5803 unsigned int segno) 5804 { 5805 if (f2fs_sb_has_blkzoned(sbi)) 5806 return f2fs_usable_zone_blks_in_seg(sbi, segno); 5807 5808 return BLKS_PER_SEG(sbi); 5809 } 5810 5811 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi) 5812 { 5813 if (f2fs_sb_has_blkzoned(sbi)) 5814 return CAP_SEGS_PER_SEC(sbi); 5815 5816 return SEGS_PER_SEC(sbi); 5817 } 5818 5819 unsigned long long f2fs_get_section_mtime(struct f2fs_sb_info *sbi, 5820 unsigned int segno) 5821 { 5822 unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi); 5823 unsigned int secno = 0, start = 0; 5824 unsigned int total_valid_blocks = 0; 5825 unsigned long long mtime = 0; 5826 unsigned int i = 0; 5827 5828 secno = GET_SEC_FROM_SEG(sbi, segno); 5829 start = GET_SEG_FROM_SEC(sbi, secno); 5830 5831 if (!__is_large_section(sbi)) { 5832 mtime = get_seg_entry(sbi, start + i)->mtime; 5833 goto out; 5834 } 5835 5836 for (i = 0; i < usable_segs_per_sec; i++) { 5837 /* for large section, only check the mtime of valid segments */ 5838 struct seg_entry *se = get_seg_entry(sbi, start+i); 5839 5840 mtime += se->mtime * se->valid_blocks; 5841 total_valid_blocks += se->valid_blocks; 5842 } 5843 5844 if (total_valid_blocks == 0) 5845 return INVALID_MTIME; 5846 5847 mtime = div_u64(mtime, total_valid_blocks); 5848 out: 5849 if (unlikely(mtime == INVALID_MTIME)) 5850 mtime -= 1; 5851 return mtime; 5852 } 5853 5854 /* 5855 * Update min, max modified time for cost-benefit GC algorithm 5856 */ 5857 static void init_min_max_mtime(struct f2fs_sb_info *sbi) 5858 { 5859 struct sit_info *sit_i = SIT_I(sbi); 5860 unsigned int segno; 5861 5862 down_write(&sit_i->sentry_lock); 5863 5864 sit_i->min_mtime = ULLONG_MAX; 5865 5866 for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { 5867 unsigned long long mtime = 0; 5868 5869 mtime = f2fs_get_section_mtime(sbi, segno); 5870 5871 if (sit_i->min_mtime > mtime) 5872 sit_i->min_mtime = mtime; 5873 } 5874 sit_i->max_mtime = get_mtime(sbi, false); 5875 sit_i->dirty_max_mtime = 0; 5876 up_write(&sit_i->sentry_lock); 5877 } 5878 5879 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi) 5880 { 5881 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 5882 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 5883 struct f2fs_sm_info *sm_info; 5884 int err; 5885 5886 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); 5887 if (!sm_info) 5888 return -ENOMEM; 5889 5890 /* init sm info */ 5891 sbi->sm_info = sm_info; 5892 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); 5893 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); 5894 sm_info->segment_count = le32_to_cpu(raw_super->segment_count); 5895 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); 5896 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); 5897 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); 5898 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); 5899 sm_info->rec_prefree_segments = sm_info->main_segments * 5900 DEF_RECLAIM_PREFREE_SEGMENTS / 100; 5901 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS) 5902 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS; 5903 5904 if (!f2fs_lfs_mode(sbi)) 5905 sm_info->ipu_policy = BIT(F2FS_IPU_FSYNC); 5906 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; 5907 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS; 5908 sm_info->min_seq_blocks = BLKS_PER_SEG(sbi); 5909 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS; 5910 sm_info->min_ssr_sections = reserved_sections(sbi); 5911 5912 INIT_LIST_HEAD(&sm_info->sit_entry_set); 5913 5914 init_f2fs_rwsem(&sm_info->curseg_lock); 5915 5916 err = f2fs_create_flush_cmd_control(sbi); 5917 if (err) 5918 return err; 5919 5920 err = create_discard_cmd_control(sbi); 5921 if (err) 5922 return err; 5923 5924 err = build_sit_info(sbi); 5925 if (err) 5926 return err; 5927 err = build_free_segmap(sbi); 5928 if (err) 5929 return err; 5930 err = build_curseg(sbi); 5931 if (err) 5932 return err; 5933 5934 /* reinit free segmap based on SIT */ 5935 err = build_sit_entries(sbi); 5936 if (err) 5937 return err; 5938 5939 init_free_segmap(sbi); 5940 err = build_dirty_segmap(sbi); 5941 if (err) 5942 return err; 5943 5944 err = sanity_check_curseg(sbi); 5945 if (err) 5946 return err; 5947 5948 init_min_max_mtime(sbi); 5949 return 0; 5950 } 5951 5952 static void discard_dirty_segmap(struct f2fs_sb_info *sbi, 5953 enum dirty_type dirty_type) 5954 { 5955 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5956 5957 mutex_lock(&dirty_i->seglist_lock); 5958 kvfree(dirty_i->dirty_segmap[dirty_type]); 5959 dirty_i->nr_dirty[dirty_type] = 0; 5960 mutex_unlock(&dirty_i->seglist_lock); 5961 } 5962 5963 static void destroy_victim_secmap(struct f2fs_sb_info *sbi) 5964 { 5965 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5966 5967 kvfree(dirty_i->pinned_secmap); 5968 kvfree(dirty_i->victim_secmap); 5969 } 5970 5971 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) 5972 { 5973 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 5974 int i; 5975 5976 if (!dirty_i) 5977 return; 5978 5979 /* discard pre-free/dirty segments list */ 5980 for (i = 0; i < NR_DIRTY_TYPE; i++) 5981 discard_dirty_segmap(sbi, i); 5982 5983 if (__is_large_section(sbi)) { 5984 mutex_lock(&dirty_i->seglist_lock); 5985 kvfree(dirty_i->dirty_secmap); 5986 mutex_unlock(&dirty_i->seglist_lock); 5987 } 5988 5989 destroy_victim_secmap(sbi); 5990 SM_I(sbi)->dirty_info = NULL; 5991 kfree(dirty_i); 5992 } 5993 5994 static void destroy_curseg(struct f2fs_sb_info *sbi) 5995 { 5996 struct curseg_info *array = SM_I(sbi)->curseg_array; 5997 int i; 5998 5999 if (!array) 6000 return; 6001 SM_I(sbi)->curseg_array = NULL; 6002 for (i = 0; i < NR_CURSEG_TYPE; i++) { 6003 kfree(array[i].sum_blk); 6004 kfree(array[i].journal); 6005 } 6006 kfree(array); 6007 } 6008 6009 static void destroy_free_segmap(struct f2fs_sb_info *sbi) 6010 { 6011 struct free_segmap_info *free_i = SM_I(sbi)->free_info; 6012 6013 if (!free_i) 6014 return; 6015 SM_I(sbi)->free_info = NULL; 6016 kvfree(free_i->free_segmap); 6017 kvfree(free_i->free_secmap); 6018 kfree(free_i); 6019 } 6020 6021 static void destroy_sit_info(struct f2fs_sb_info *sbi) 6022 { 6023 struct sit_info *sit_i = SIT_I(sbi); 6024 6025 if (!sit_i) 6026 return; 6027 6028 if (sit_i->sentries) 6029 kvfree(sit_i->bitmap); 6030 kfree(sit_i->tmp_map); 6031 6032 kvfree(sit_i->sentries); 6033 kvfree(sit_i->sec_entries); 6034 kvfree(sit_i->dirty_sentries_bitmap); 6035 6036 SM_I(sbi)->sit_info = NULL; 6037 kfree(sit_i->sit_bitmap); 6038 #ifdef CONFIG_F2FS_CHECK_FS 6039 kvfree(sit_i->invalid_segmap); 6040 #endif 6041 kfree(sit_i); 6042 } 6043 6044 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi) 6045 { 6046 struct f2fs_sm_info *sm_info = SM_I(sbi); 6047 6048 if (!sm_info) 6049 return; 6050 f2fs_destroy_flush_cmd_control(sbi, true); 6051 destroy_discard_cmd_control(sbi); 6052 destroy_dirty_segmap(sbi); 6053 destroy_curseg(sbi); 6054 destroy_free_segmap(sbi); 6055 destroy_sit_info(sbi); 6056 sbi->sm_info = NULL; 6057 kfree(sm_info); 6058 } 6059 6060 int __init f2fs_create_segment_manager_caches(void) 6061 { 6062 discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry", 6063 sizeof(struct discard_entry)); 6064 if (!discard_entry_slab) 6065 goto fail; 6066 6067 discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd", 6068 sizeof(struct discard_cmd)); 6069 if (!discard_cmd_slab) 6070 goto destroy_discard_entry; 6071 6072 sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set", 6073 sizeof(struct sit_entry_set)); 6074 if (!sit_entry_set_slab) 6075 goto destroy_discard_cmd; 6076 6077 revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry", 6078 sizeof(struct revoke_entry)); 6079 if (!revoke_entry_slab) 6080 goto destroy_sit_entry_set; 6081 return 0; 6082 6083 destroy_sit_entry_set: 6084 kmem_cache_destroy(sit_entry_set_slab); 6085 destroy_discard_cmd: 6086 kmem_cache_destroy(discard_cmd_slab); 6087 destroy_discard_entry: 6088 kmem_cache_destroy(discard_entry_slab); 6089 fail: 6090 return -ENOMEM; 6091 } 6092 6093 void f2fs_destroy_segment_manager_caches(void) 6094 { 6095 kmem_cache_destroy(sit_entry_set_slab); 6096 kmem_cache_destroy(discard_cmd_slab); 6097 kmem_cache_destroy(discard_entry_slab); 6098 kmem_cache_destroy(revoke_entry_slab); 6099 } 6100