Lines Matching refs:sbi
811 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_mark_free_simple() local
836 buddy + sbi->s_mb_offsets[min]); in ext4_mb_mark_free_simple()
865 struct ext4_sb_info *sbi = EXT4_SB(sb); in mb_update_avg_fragment_size() local
878 xa_erase(&sbi->s_mb_avg_fragment_size[old], grp->bb_group); in mb_update_avg_fragment_size()
887 int err = xa_insert(&sbi->s_mb_avg_fragment_size[new], in mb_update_avg_fragment_size()
915 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_scan_groups_xa_range() local
927 if (sbi->s_mb_stats) in ext4_mb_scan_groups_xa_range()
928 atomic64_inc(&sbi->s_bal_cX_groups_considered[cr]); in ext4_mb_scan_groups_xa_range()
963 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_scan_groups_p2_aligned() local
983 if (sbi->s_mb_stats) in ext4_mb_scan_groups_p2_aligned()
984 atomic64_inc(&sbi->s_bal_cX_failed[ac->ac_criteria]); in ext4_mb_scan_groups_p2_aligned()
1014 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_scan_groups_goal_fast() local
1034 if (sbi->s_mb_stats) in ext4_mb_scan_groups_goal_fast()
1035 atomic64_inc(&sbi->s_bal_cX_failed[ac->ac_criteria]); in ext4_mb_scan_groups_goal_fast()
1065 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_scan_groups_best_avail() local
1079 min_order = order - sbi->s_mb_best_avail_max_trim_order; in ext4_mb_scan_groups_best_avail()
1083 if (sbi->s_stripe > 0) { in ext4_mb_scan_groups_best_avail()
1088 num_stripe_clusters = EXT4_NUM_B2C(sbi, sbi->s_stripe); in ext4_mb_scan_groups_best_avail()
1138 if (sbi->s_mb_stats) in ext4_mb_scan_groups_best_avail()
1139 atomic64_inc(&sbi->s_bal_cX_failed[ac->ac_criteria]); in ext4_mb_scan_groups_best_avail()
1172 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_scan_groups_linear() local
1187 if (sbi->s_mb_stats && i == ngroups) in ext4_mb_scan_groups_linear()
1188 atomic64_inc(&sbi->s_bal_cX_failed[cr]); in ext4_mb_scan_groups_linear()
1197 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_scan_groups() local
1215 if (sbi->s_mb_max_linear_groups) in ext4_mb_scan_groups()
1217 sbi->s_mb_max_linear_groups); in ext4_mb_scan_groups()
1247 struct ext4_sb_info *sbi = EXT4_SB(sb); in mb_set_largest_free_order() local
1259 struct xarray *xa = &sbi->s_mb_largest_free_orders[old]; in mb_set_largest_free_order()
1272 int err = xa_insert(&sbi->s_mb_largest_free_orders[new], in mb_set_largest_free_order()
1285 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_generate_buddy() local
1332 atomic_inc(&sbi->s_mb_buddies_generated); in ext4_mb_generate_buddy()
1333 atomic64_add(period, &sbi->s_mb_generation_time); in ext4_mb_generate_buddy()
1683 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_load_buddy_gfp() local
1684 struct inode *inode = sbi->s_buddy_cache; in ext4_mb_load_buddy_gfp()
2030 struct ext4_sb_info *sbi = EXT4_SB(sb); in mb_free_blocks() local
2037 if (sbi->s_mount_state & EXT4_FC_REPLAY) { in mb_free_blocks()
2043 blocknr += EXT4_C2B(sbi, block); in mb_free_blocks()
2240 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_best_found() local
2271 int hash = (unsigned int)ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals; in ext4_mb_use_best_found()
2273 WRITE_ONCE(sbi->s_mb_last_groups[hash], ac->ac_f_ex.fe_group); in ext4_mb_use_best_found()
2290 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_check_limits() local
2299 if (ac->ac_found > sbi->s_mb_max_to_scan && in ext4_mb_check_limits()
2311 if (finish_group || ac->ac_found > sbi->s_mb_min_to_scan) in ext4_mb_check_limits()
2436 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_by_goal() local
2464 ac->ac_g_ex.fe_len == EXT4_NUM_B2C(sbi, sbi->s_stripe)) { in ext4_mb_find_by_goal()
2469 if (do_div(start, sbi->s_stripe) == 0) { in ext4_mb_find_by_goal()
2644 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_scan_aligned() local
2652 BUG_ON(sbi->s_stripe == 0); in ext4_mb_scan_aligned()
2657 a = first_group_block + sbi->s_stripe - 1; in ext4_mb_scan_aligned()
2658 do_div(a, sbi->s_stripe); in ext4_mb_scan_aligned()
2659 i = (a * sbi->s_stripe) - first_group_block; in ext4_mb_scan_aligned()
2661 stripe = EXT4_NUM_B2C(sbi, sbi->s_stripe); in ext4_mb_scan_aligned()
2662 i = EXT4_B2C(sbi, i); in ext4_mb_scan_aligned()
2682 struct ext4_sb_info *sbi; in __ext4_mb_scan_group() local
2689 sbi = EXT4_SB(ac->ac_sb); in __ext4_mb_scan_group()
2691 if ((sbi->s_stripe >= sbi->s_cluster_ratio) && in __ext4_mb_scan_group()
2692 !(ac->ac_g_ex.fe_len % EXT4_NUM_B2C(sbi, sbi->s_stripe))) in __ext4_mb_scan_group()
2782 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_good_group_nolock() local
2789 if (sbi->s_mb_stats) in ext4_mb_good_group_nolock()
2790 atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]); in ext4_mb_good_group_nolock()
2828 (!sbi->s_log_groups_per_flex || in ext4_mb_good_group_nolock()
2829 ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) && in ext4_mb_good_group_nolock()
2901 struct ext4_sb_info *sbi; in ext4_mb_might_prefetch() local
2906 sbi = EXT4_SB(ac->ac_sb); in ext4_mb_might_prefetch()
2908 ac->ac_prefetch_ios < sbi->s_mb_prefetch_limit) { in ext4_mb_might_prefetch()
2909 unsigned int nr = sbi->s_mb_prefetch; in ext4_mb_might_prefetch()
2912 nr = 1 << sbi->s_log_groups_per_flex; in ext4_mb_might_prefetch()
2914 nr = umin(nr, sbi->s_mb_prefetch); in ext4_mb_might_prefetch()
3006 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_regular_allocator() local
3033 if (i >= sbi->s_mb_order2_reqs && i <= MB_NUM_ORDERS(sb)) { in ext4_mb_regular_allocator()
3041 int hash = (unsigned int)ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals; in ext4_mb_regular_allocator()
3043 ac->ac_g_ex.fe_group = READ_ONCE(sbi->s_mb_last_groups[hash]); in ext4_mb_regular_allocator()
3085 lost = atomic_inc_return(&sbi->s_mb_lost_chunks); in ext4_mb_regular_allocator()
3100 if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) { in ext4_mb_regular_allocator()
3101 atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]); in ext4_mb_regular_allocator()
3104 atomic_inc(&sbi->s_bal_stream_goals); in ext4_mb_regular_allocator()
3210 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_seq_mb_stats_show() local
3213 if (!sbi->s_mb_stats) { in ext4_seq_mb_stats_show()
3220 seq_printf(seq, "\treqs: %u\n", atomic_read(&sbi->s_bal_reqs)); in ext4_seq_mb_stats_show()
3221 seq_printf(seq, "\tsuccess: %u\n", atomic_read(&sbi->s_bal_success)); in ext4_seq_mb_stats_show()
3224 atomic_read(&sbi->s_bal_groups_scanned)); in ext4_seq_mb_stats_show()
3229 atomic64_read(&sbi->s_bal_cX_hits[CR_POWER2_ALIGNED])); in ext4_seq_mb_stats_show()
3233 &sbi->s_bal_cX_groups_considered[CR_POWER2_ALIGNED])); in ext4_seq_mb_stats_show()
3235 atomic_read(&sbi->s_bal_cX_ex_scanned[CR_POWER2_ALIGNED])); in ext4_seq_mb_stats_show()
3237 atomic64_read(&sbi->s_bal_cX_failed[CR_POWER2_ALIGNED])); in ext4_seq_mb_stats_show()
3242 atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_FAST])); in ext4_seq_mb_stats_show()
3245 &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_FAST])); in ext4_seq_mb_stats_show()
3247 atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_FAST])); in ext4_seq_mb_stats_show()
3249 atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_FAST])); in ext4_seq_mb_stats_show()
3254 atomic64_read(&sbi->s_bal_cX_hits[CR_BEST_AVAIL_LEN])); in ext4_seq_mb_stats_show()
3258 &sbi->s_bal_cX_groups_considered[CR_BEST_AVAIL_LEN])); in ext4_seq_mb_stats_show()
3260 atomic_read(&sbi->s_bal_cX_ex_scanned[CR_BEST_AVAIL_LEN])); in ext4_seq_mb_stats_show()
3262 atomic64_read(&sbi->s_bal_cX_failed[CR_BEST_AVAIL_LEN])); in ext4_seq_mb_stats_show()
3267 atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_SLOW])); in ext4_seq_mb_stats_show()
3270 &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_SLOW])); in ext4_seq_mb_stats_show()
3272 atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_SLOW])); in ext4_seq_mb_stats_show()
3274 atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_SLOW])); in ext4_seq_mb_stats_show()
3279 atomic64_read(&sbi->s_bal_cX_hits[CR_ANY_FREE])); in ext4_seq_mb_stats_show()
3282 atomic64_read(&sbi->s_bal_cX_groups_considered[CR_ANY_FREE])); in ext4_seq_mb_stats_show()
3284 atomic_read(&sbi->s_bal_cX_ex_scanned[CR_ANY_FREE])); in ext4_seq_mb_stats_show()
3286 atomic64_read(&sbi->s_bal_cX_failed[CR_ANY_FREE])); in ext4_seq_mb_stats_show()
3290 atomic_read(&sbi->s_bal_ex_scanned)); in ext4_seq_mb_stats_show()
3291 seq_printf(seq, "\t\tgoal_hits: %u\n", atomic_read(&sbi->s_bal_goals)); in ext4_seq_mb_stats_show()
3293 atomic_read(&sbi->s_bal_stream_goals)); in ext4_seq_mb_stats_show()
3295 atomic_read(&sbi->s_bal_len_goals)); in ext4_seq_mb_stats_show()
3296 seq_printf(seq, "\t\t2^n_hits: %u\n", atomic_read(&sbi->s_bal_2orders)); in ext4_seq_mb_stats_show()
3297 seq_printf(seq, "\t\tbreaks: %u\n", atomic_read(&sbi->s_bal_breaks)); in ext4_seq_mb_stats_show()
3298 seq_printf(seq, "\t\tlost: %u\n", atomic_read(&sbi->s_mb_lost_chunks)); in ext4_seq_mb_stats_show()
3300 atomic_read(&sbi->s_mb_buddies_generated), in ext4_seq_mb_stats_show()
3303 atomic64_read(&sbi->s_mb_generation_time)); in ext4_seq_mb_stats_show()
3305 atomic_read(&sbi->s_mb_preallocated)); in ext4_seq_mb_stats_show()
3306 seq_printf(seq, "\tdiscarded: %u\n", atomic_read(&sbi->s_mb_discarded)); in ext4_seq_mb_stats_show()
3336 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_seq_structs_summary_show() local
3349 xa_for_each(&sbi->s_mb_avg_fragment_size[position], idx, grp) in ext4_mb_seq_structs_summary_show()
3362 xa_for_each(&sbi->s_mb_largest_free_orders[position], idx, grp) in ext4_mb_seq_structs_summary_show()
3396 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_alloc_groupinfo() local
3402 if (size <= sbi->s_group_info_size) in ext4_mb_alloc_groupinfo()
3405 size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size); in ext4_mb_alloc_groupinfo()
3412 old_groupinfo = rcu_dereference(sbi->s_group_info); in ext4_mb_alloc_groupinfo()
3415 sbi->s_group_info_size * sizeof(*sbi->s_group_info)); in ext4_mb_alloc_groupinfo()
3417 rcu_assign_pointer(sbi->s_group_info, new_groupinfo); in ext4_mb_alloc_groupinfo()
3418 sbi->s_group_info_size = size / sizeof(*sbi->s_group_info); in ext4_mb_alloc_groupinfo()
3422 sbi->s_group_info_size); in ext4_mb_alloc_groupinfo()
3433 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_add_groupinfo() local
3452 rcu_dereference(sbi->s_group_info)[idx] = meta_group_info; in ext4_mb_add_groupinfo() local
3456 meta_group_info = sbi_array_rcu_deref(sbi, s_group_info, idx); in ext4_mb_add_groupinfo()
3496 group_info = rcu_dereference(sbi->s_group_info); in ext4_mb_add_groupinfo()
3508 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_init_backend() local
3518 sbi->s_buddy_cache = new_inode(sb); in ext4_mb_init_backend()
3519 if (sbi->s_buddy_cache == NULL) { in ext4_mb_init_backend()
3527 sbi->s_buddy_cache->i_ino = EXT4_BAD_INO; in ext4_mb_init_backend()
3528 EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; in ext4_mb_init_backend()
3529 ext4_set_inode_mapping_order(sbi->s_buddy_cache); in ext4_mb_init_backend()
3547 if (sbi->s_es->s_log_groups_per_flex >= 32) { in ext4_mb_init_backend()
3551 sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex, in ext4_mb_init_backend()
3553 sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */ in ext4_mb_init_backend()
3555 sbi->s_mb_prefetch = 32; in ext4_mb_init_backend()
3557 if (sbi->s_mb_prefetch > ext4_get_groups_count(sb)) in ext4_mb_init_backend()
3558 sbi->s_mb_prefetch = ext4_get_groups_count(sb); in ext4_mb_init_backend()
3567 sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4; in ext4_mb_init_backend()
3568 if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb)) in ext4_mb_init_backend()
3569 sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb); in ext4_mb_init_backend()
3581 i = sbi->s_group_info_size; in ext4_mb_init_backend()
3583 group_info = rcu_dereference(sbi->s_group_info); in ext4_mb_init_backend()
3587 iput(sbi->s_buddy_cache); in ext4_mb_init_backend()
3589 kvfree(rcu_access_pointer(sbi->s_group_info)); in ext4_mb_init_backend()
3644 struct ext4_sb_info *sbi = container_of(work, in ext4_discard_work() local
3646 struct super_block *sb = sbi->s_sb; in ext4_discard_work()
3653 spin_lock(&sbi->s_md_lock); in ext4_discard_work()
3654 list_splice_init(&sbi->s_discard_list, &discard_list); in ext4_discard_work()
3655 spin_unlock(&sbi->s_md_lock); in ext4_discard_work()
3664 !atomic_read(&sbi->s_retry_alloc_pending)) { in ext4_discard_work()
3692 static inline void ext4_mb_avg_fragment_size_destroy(struct ext4_sb_info *sbi) in ext4_mb_avg_fragment_size_destroy() argument
3694 if (!sbi->s_mb_avg_fragment_size) in ext4_mb_avg_fragment_size_destroy()
3697 for (int i = 0; i < MB_NUM_ORDERS(sbi->s_sb); i++) in ext4_mb_avg_fragment_size_destroy()
3698 xa_destroy(&sbi->s_mb_avg_fragment_size[i]); in ext4_mb_avg_fragment_size_destroy()
3700 kfree(sbi->s_mb_avg_fragment_size); in ext4_mb_avg_fragment_size_destroy()
3701 sbi->s_mb_avg_fragment_size = NULL; in ext4_mb_avg_fragment_size_destroy()
3704 static inline void ext4_mb_largest_free_orders_destroy(struct ext4_sb_info *sbi) in ext4_mb_largest_free_orders_destroy() argument
3706 if (!sbi->s_mb_largest_free_orders) in ext4_mb_largest_free_orders_destroy()
3709 for (int i = 0; i < MB_NUM_ORDERS(sbi->s_sb); i++) in ext4_mb_largest_free_orders_destroy()
3710 xa_destroy(&sbi->s_mb_largest_free_orders[i]); in ext4_mb_largest_free_orders_destroy()
3712 kfree(sbi->s_mb_largest_free_orders); in ext4_mb_largest_free_orders_destroy()
3713 sbi->s_mb_largest_free_orders = NULL; in ext4_mb_largest_free_orders_destroy()
3718 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_init() local
3724 i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_offsets); in ext4_mb_init()
3726 sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); in ext4_mb_init()
3727 if (sbi->s_mb_offsets == NULL) { in ext4_mb_init()
3732 i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_maxs); in ext4_mb_init()
3733 sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); in ext4_mb_init()
3734 if (sbi->s_mb_maxs == NULL) { in ext4_mb_init()
3744 sbi->s_mb_maxs[0] = sb->s_blocksize << 3; in ext4_mb_init()
3745 sbi->s_mb_offsets[0] = 0; in ext4_mb_init()
3752 sbi->s_mb_offsets[i] = offset; in ext4_mb_init()
3753 sbi->s_mb_maxs[i] = max; in ext4_mb_init()
3760 sbi->s_mb_avg_fragment_size = in ext4_mb_init()
3762 if (!sbi->s_mb_avg_fragment_size) { in ext4_mb_init()
3767 xa_init(&sbi->s_mb_avg_fragment_size[i]); in ext4_mb_init()
3769 sbi->s_mb_largest_free_orders = in ext4_mb_init()
3771 if (!sbi->s_mb_largest_free_orders) { in ext4_mb_init()
3776 xa_init(&sbi->s_mb_largest_free_orders[i]); in ext4_mb_init()
3778 spin_lock_init(&sbi->s_md_lock); in ext4_mb_init()
3779 atomic_set(&sbi->s_mb_free_pending, 0); in ext4_mb_init()
3780 INIT_LIST_HEAD(&sbi->s_freed_data_list[0]); in ext4_mb_init()
3781 INIT_LIST_HEAD(&sbi->s_freed_data_list[1]); in ext4_mb_init()
3782 INIT_LIST_HEAD(&sbi->s_discard_list); in ext4_mb_init()
3783 INIT_WORK(&sbi->s_discard_work, ext4_discard_work); in ext4_mb_init()
3784 atomic_set(&sbi->s_retry_alloc_pending, 0); in ext4_mb_init()
3786 sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; in ext4_mb_init()
3787 sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; in ext4_mb_init()
3788 sbi->s_mb_stats = MB_DEFAULT_STATS; in ext4_mb_init()
3789 sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; in ext4_mb_init()
3790 sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; in ext4_mb_init()
3791 sbi->s_mb_best_avail_max_trim_order = MB_DEFAULT_BEST_AVAIL_TRIM_ORDER; in ext4_mb_init()
3805 sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >> in ext4_mb_init()
3806 sbi->s_cluster_bits, 32); in ext4_mb_init()
3815 if (sbi->s_stripe > 1) { in ext4_mb_init()
3816 sbi->s_mb_group_prealloc = roundup( in ext4_mb_init()
3817 sbi->s_mb_group_prealloc, EXT4_NUM_B2C(sbi, sbi->s_stripe)); in ext4_mb_init()
3820 sbi->s_mb_nr_global_goals = umin(num_possible_cpus(), in ext4_mb_init()
3821 DIV_ROUND_UP(sbi->s_groups_count, 4)); in ext4_mb_init()
3822 sbi->s_mb_last_groups = kzalloc_objs(ext4_group_t, in ext4_mb_init()
3823 sbi->s_mb_nr_global_goals); in ext4_mb_init()
3824 if (sbi->s_mb_last_groups == NULL) { in ext4_mb_init()
3829 sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); in ext4_mb_init()
3830 if (sbi->s_locality_groups == NULL) { in ext4_mb_init()
3836 lg = per_cpu_ptr(sbi->s_locality_groups, i); in ext4_mb_init()
3844 sbi->s_mb_max_linear_groups = 0; in ext4_mb_init()
3846 sbi->s_mb_max_linear_groups = MB_DEFAULT_LINEAR_LIMIT; in ext4_mb_init()
3855 free_percpu(sbi->s_locality_groups); in ext4_mb_init()
3856 sbi->s_locality_groups = NULL; in ext4_mb_init()
3858 kfree(sbi->s_mb_last_groups); in ext4_mb_init()
3859 sbi->s_mb_last_groups = NULL; in ext4_mb_init()
3861 ext4_mb_avg_fragment_size_destroy(sbi); in ext4_mb_init()
3862 ext4_mb_largest_free_orders_destroy(sbi); in ext4_mb_init()
3863 kfree(sbi->s_mb_offsets); in ext4_mb_init()
3864 sbi->s_mb_offsets = NULL; in ext4_mb_init()
3865 kfree(sbi->s_mb_maxs); in ext4_mb_init()
3866 sbi->s_mb_maxs = NULL; in ext4_mb_init()
3892 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_release() local
3899 flush_work(&sbi->s_discard_work); in ext4_mb_release()
3900 WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); in ext4_mb_release()
3902 group_info = rcu_access_pointer(sbi->s_group_info); in ext4_mb_release()
3925 ext4_mb_avg_fragment_size_destroy(sbi); in ext4_mb_release()
3926 ext4_mb_largest_free_orders_destroy(sbi); in ext4_mb_release()
3927 kfree(sbi->s_mb_offsets); in ext4_mb_release()
3928 kfree(sbi->s_mb_maxs); in ext4_mb_release()
3929 iput(sbi->s_buddy_cache); in ext4_mb_release()
3930 if (sbi->s_mb_stats) { in ext4_mb_release()
3933 atomic_read(&sbi->s_bal_allocated), in ext4_mb_release()
3934 atomic_read(&sbi->s_bal_reqs), in ext4_mb_release()
3935 atomic_read(&sbi->s_bal_success)); in ext4_mb_release()
3939 atomic_read(&sbi->s_bal_ex_scanned), in ext4_mb_release()
3940 atomic_read(&sbi->s_bal_groups_scanned), in ext4_mb_release()
3941 atomic_read(&sbi->s_bal_goals), in ext4_mb_release()
3942 atomic_read(&sbi->s_bal_2orders), in ext4_mb_release()
3943 atomic_read(&sbi->s_bal_breaks), in ext4_mb_release()
3944 atomic_read(&sbi->s_mb_lost_chunks)); in ext4_mb_release()
3947 atomic_read(&sbi->s_mb_buddies_generated), in ext4_mb_release()
3948 atomic64_read(&sbi->s_mb_generation_time)); in ext4_mb_release()
3951 atomic_read(&sbi->s_mb_preallocated), in ext4_mb_release()
3952 atomic_read(&sbi->s_mb_discarded)); in ext4_mb_release()
3955 free_percpu(sbi->s_locality_groups); in ext4_mb_release()
3956 kfree(sbi->s_mb_last_groups); in ext4_mb_release()
4021 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_process_freed_data() local
4024 struct list_head *s_freed_head = &sbi->s_freed_data_list[commit_tid & 1]; in ext4_process_freed_data()
4033 spin_lock(&sbi->s_md_lock); in ext4_process_freed_data()
4034 wake = list_empty(&sbi->s_discard_list); in ext4_process_freed_data()
4035 list_splice_tail(&freed_data_list, &sbi->s_discard_list); in ext4_process_freed_data()
4036 spin_unlock(&sbi->s_md_lock); in ext4_process_freed_data()
4038 queue_work(system_dfl_wq, &sbi->s_discard_work); in ext4_process_freed_data()
4092 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_mark_context() local
4163 if (sbi->s_log_groups_per_flex) { in ext4_mb_mark_context()
4164 ext4_group_t flex_group = ext4_flex_group(sbi, group); in ext4_mb_mark_context()
4165 struct flex_groups *fg = sbi_array_rcu_deref(sbi, in ext4_mb_mark_context()
4199 struct ext4_sb_info *sbi; in ext4_mb_mark_diskspace_used() local
4210 sbi = EXT4_SB(sb); in ext4_mb_mark_diskspace_used()
4219 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4250 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4262 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_mark_bb() local
4279 thisgrp_len = min(len, EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff)); in ext4_mb_mark_bb()
4280 clen = EXT4_NUM_B2C(sbi, thisgrp_len); in ext4_mb_mark_bb()
4344 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_pa_assert_overlap() local
4357 tmp_pa_end = pa_logical_end(sbi, tmp_pa); in ext4_mb_pa_assert_overlap()
4382 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_pa_adjust_overlap() local
4405 tmp_pa_end = pa_logical_end(sbi, tmp_pa); in ext4_mb_pa_adjust_overlap()
4485 left_pa_end = pa_logical_end(sbi, left_pa); in ext4_mb_pa_adjust_overlap()
4521 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_normalize_request() local
4522 struct ext4_super_block *es = sbi->s_es; in ext4_mb_normalize_request()
4551 size = extent_logical_end(sbi, &ac->ac_o_ex); in ext4_mb_normalize_request()
4582 } else if (NRL_CHECK_SIZE(EXT4_C2B(sbi, ac->ac_o_ex.fe_len), in ext4_mb_normalize_request()
4589 size = (loff_t) EXT4_C2B(sbi, in ext4_mb_normalize_request()
4659 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); in ext4_mb_normalize_request()
4687 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_collect_stats() local
4689 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) { in ext4_mb_collect_stats()
4690 atomic_inc(&sbi->s_bal_reqs); in ext4_mb_collect_stats()
4691 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); in ext4_mb_collect_stats()
4693 atomic_inc(&sbi->s_bal_success); in ext4_mb_collect_stats()
4695 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); in ext4_mb_collect_stats()
4697 atomic_add(ac->ac_cX_found[i], &sbi->s_bal_cX_ex_scanned[i]); in ext4_mb_collect_stats()
4700 atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned); in ext4_mb_collect_stats()
4703 atomic_inc(&sbi->s_bal_goals); in ext4_mb_collect_stats()
4706 atomic_inc(&sbi->s_bal_len_goals); in ext4_mb_collect_stats()
4708 if (ac->ac_found > sbi->s_mb_max_to_scan) in ext4_mb_collect_stats()
4709 atomic_inc(&sbi->s_bal_breaks); in ext4_mb_collect_stats()
4762 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_inode_pa() local
4769 end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len), in ext4_mb_use_inode_pa()
4770 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); in ext4_mb_use_inode_pa()
4771 len = EXT4_NUM_B2C(sbi, end - start); in ext4_mb_use_inode_pa()
4779 BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len)); in ext4_mb_use_inode_pa()
4848 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_pa_goal_check() local
4866 EXT4_B2C(sbi, ac->ac_g_ex.fe_logical - pa->pa_lstart)) in ext4_mb_pa_goal_check()
4878 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_preallocated() local
4978 if (ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, tmp_pa)) { in ext4_mb_use_preallocated()
4985 (tmp_pa->pa_pstart + EXT4_C2B(sbi, tmp_pa->pa_len) > in ext4_mb_use_preallocated()
5254 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_new_inode_pa() local
5272 loff_t orig_goal_end = extent_logical_end(sbi, &ex); in ext4_mb_new_inode_pa()
5273 loff_t o_ex_end = extent_logical_end(sbi, &ac->ac_o_ex); in ext4_mb_new_inode_pa()
5299 ex.fe_logical = orig_goal_end - EXT4_C2B(sbi, ex.fe_len); in ext4_mb_new_inode_pa()
5304 if (o_ex_end <= extent_logical_end(sbi, &ex)) in ext4_mb_new_inode_pa()
5312 BUG_ON(extent_logical_end(sbi, &ex) > orig_goal_end); in ext4_mb_new_inode_pa()
5328 atomic_add(pa->pa_free, &sbi->s_mb_preallocated); in ext4_mb_new_inode_pa()
5421 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_release_inode_pa() local
5431 grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit); in ext4_mb_release_inode_pa()
5447 EXT4_C2B(sbi, bit)), in ext4_mb_release_inode_pa()
5465 atomic_add(free, &sbi->s_mb_discarded); in ext4_mb_release_inode_pa()
5832 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_group_or_file() local
5843 group_pa_eligible = sbi->s_mb_group_prealloc > 0; in ext4_mb_group_or_file()
5845 size = extent_logical_end(sbi, &ac->ac_o_ex); in ext4_mb_group_or_file()
5850 if ((size == isize) && !ext4_fs_is_busy(sbi) && in ext4_mb_group_or_file()
5856 if (size > sbi->s_mb_stream_request) in ext4_mb_group_or_file()
5873 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); in ext4_mb_group_or_file()
5887 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_initialize_context() local
5888 struct ext4_super_block *es = sbi->s_es; in ext4_mb_initialize_context()
5909 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); in ext4_mb_initialize_context()
6068 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_release_context() local
6074 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
6075 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
6166 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_new_blocks_simple() local
6172 struct ext4_super_block *es = sbi->s_es; in ext4_mb_new_blocks_simple()
6196 EXT4_C2B(sbi, i))) { in ext4_mb_new_blocks_simple()
6216 block = ext4_group_first_block_no(sb, group) + EXT4_C2B(sbi, i); in ext4_mb_new_blocks_simple()
6233 struct ext4_sb_info *sbi; in ext4_mb_new_blocks() local
6243 sbi = EXT4_SB(sb); in ext4_mb_new_blocks()
6246 if (sbi->s_mount_state & EXT4_FC_REPLAY) in ext4_mb_new_blocks()
6259 ext4_claim_free_clusters(sbi, ar->len, ar->flags)) { in ext4_mb_new_blocks()
6273 EXT4_C2B(sbi, ar->len)); in ext4_mb_new_blocks()
6277 EXT4_C2B(sbi, ar->len))) { in ext4_mb_new_blocks()
6358 dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); in ext4_mb_new_blocks()
6361 percpu_counter_sub(&sbi->s_dirtyclusters_counter, reserv_clstrs); in ext4_mb_new_blocks()
6388 ext4_merge_freed_extents(struct ext4_sb_info *sbi, struct rb_root *root, in ext4_merge_freed_extents() argument
6393 spin_lock(&sbi->s_md_lock); in ext4_merge_freed_extents()
6395 spin_unlock(&sbi->s_md_lock); in ext4_merge_freed_extents()
6401 ext4_try_merge_freed_extent_prev(struct ext4_sb_info *sbi, struct rb_root *root, in ext4_try_merge_freed_extent_prev() argument
6413 ext4_merge_freed_extents(sbi, root, prev, entry); in ext4_try_merge_freed_extent_prev()
6417 ext4_try_merge_freed_extent_next(struct ext4_sb_info *sbi, struct rb_root *root, in ext4_try_merge_freed_extent_next() argument
6429 ext4_merge_freed_extents(sbi, root, entry, next); in ext4_try_merge_freed_extent_next()
6442 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_mb_free_metadata() local
6473 EXT4_C2B(sbi, cluster), in ext4_mb_free_metadata()
6480 atomic_add(clusters, &sbi->s_mb_free_pending); in ext4_mb_free_metadata()
6489 ext4_try_merge_freed_extent_prev(sbi, root, entry); in ext4_mb_free_metadata()
6495 ext4_try_merge_freed_extent_next(sbi, root, entry); in ext4_mb_free_metadata()
6502 spin_lock(&sbi->s_md_lock); in ext4_mb_free_metadata()
6503 list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list[new_entry->efd_tid & 1]); in ext4_mb_free_metadata()
6504 spin_unlock(&sbi->s_md_lock); in ext4_mb_free_metadata()
6539 struct ext4_sb_info *sbi; in ext4_mb_clear_bb() local
6546 sbi = EXT4_SB(sb); in ext4_mb_clear_bb()
6569 if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) { in ext4_mb_clear_bb()
6570 overflow = EXT4_C2B(sbi, bit) + count - in ext4_mb_clear_bb()
6576 count_clusters = EXT4_NUM_B2C(sbi, count); in ext4_mb_clear_bb()
6662 dquot_free_block(inode, EXT4_C2B(sbi, count_clusters)); in ext4_mb_clear_bb()
6663 percpu_counter_add(&sbi->s_freeclusters_counter, in ext4_mb_clear_bb()
6697 struct ext4_sb_info *sbi; in ext4_free_blocks() local
6699 sbi = EXT4_SB(sb); in ext4_free_blocks()
6708 if (sbi->s_mount_state & EXT4_FC_REPLAY) { in ext4_free_blocks()
6709 ext4_free_blocks_simple(inode, block, EXT4_NUM_B2C(sbi, count)); in ext4_free_blocks()
6740 overflow = EXT4_PBLK_COFF(sbi, block); in ext4_free_blocks()
6743 overflow = sbi->s_cluster_ratio - overflow; in ext4_free_blocks()
6756 overflow = EXT4_LBLK_COFF(sbi, count); in ext4_free_blocks()
6764 count += sbi->s_cluster_ratio - overflow; in ext4_free_blocks()
6799 struct ext4_sb_info *sbi = EXT4_SB(sb); in ext4_group_add_blocks() local
6802 ext4_fsblk_t first_cluster = EXT4_B2C(sbi, block); in ext4_group_add_blocks()
6803 ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); in ext4_group_add_blocks()
6848 percpu_counter_add(&sbi->s_freeclusters_counter, in ext4_group_add_blocks()