Lines Matching +full:power +full:- +full:friendly
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2023-2025 Christoph Hellwig.
4 * Copyright (c) 2024-2025, Western Digital Corporation or its affiliates.
35 xfs_rtgroup_rele(oz->oz_rtg); in xfs_open_zone_free_rcu()
43 if (atomic_dec_and_test(&oz->oz_ref)) in xfs_open_zone_put()
44 call_rcu(&oz->oz_rcu, xfs_open_zone_free_rcu); in xfs_open_zone_put()
53 mp->m_groups[XG_TYPE_RTG].blocks; in xfs_zone_bucket()
62 __set_bit(rgno, zi->zi_used_bucket_bitmap[to_bucket]); in xfs_zone_add_to_bucket()
63 zi->zi_used_bucket_entries[to_bucket]++; in xfs_zone_add_to_bucket()
72 __clear_bit(rgno, zi->zi_used_bucket_bitmap[from_bucket]); in xfs_zone_remove_from_bucket()
73 zi->zi_used_bucket_entries[from_bucket]--; in xfs_zone_remove_from_bucket()
81 struct xfs_group *xg = &rtg->rtg_group; in xfs_zone_account_reclaimable()
83 struct xfs_zone_info *zi = mp->m_zone_info; in xfs_zone_account_reclaimable()
84 uint32_t used = rtg_rmap(rtg)->i_used_blocks; in xfs_zone_account_reclaimable()
109 spin_lock(&zi->zi_used_buckets_lock); in xfs_zone_account_reclaimable()
112 spin_unlock(&zi->zi_used_buckets_lock); in xfs_zone_account_reclaimable()
114 spin_lock(&zi->zi_reset_list_lock); in xfs_zone_account_reclaimable()
115 xg->xg_next_reset = zi->zi_reset_list; in xfs_zone_account_reclaimable()
116 zi->zi_reset_list = xg; in xfs_zone_account_reclaimable()
117 spin_unlock(&zi->zi_reset_list_lock); in xfs_zone_account_reclaimable()
119 if (zi->zi_gc_thread) in xfs_zone_account_reclaimable()
120 wake_up_process(zi->zi_gc_thread); in xfs_zone_account_reclaimable()
126 spin_lock(&zi->zi_used_buckets_lock); in xfs_zone_account_reclaimable()
128 spin_unlock(&zi->zi_used_buckets_lock); in xfs_zone_account_reclaimable()
131 if (zi->zi_gc_thread && xfs_zoned_need_gc(mp)) in xfs_zone_account_reclaimable()
132 wake_up_process(zi->zi_gc_thread); in xfs_zone_account_reclaimable()
138 spin_lock(&zi->zi_used_buckets_lock); in xfs_zone_account_reclaimable()
141 spin_unlock(&zi->zi_used_buckets_lock); in xfs_zone_account_reclaimable()
149 struct xfs_rtgroup *rtg = oz->oz_rtg; in xfs_open_zone_mark_full()
151 struct xfs_zone_info *zi = mp->m_zone_info; in xfs_open_zone_mark_full()
152 uint32_t used = rtg_rmap(rtg)->i_used_blocks; in xfs_open_zone_mark_full()
156 WRITE_ONCE(rtg->rtg_open_zone, NULL); in xfs_open_zone_mark_full()
158 spin_lock(&zi->zi_open_zones_lock); in xfs_open_zone_mark_full()
159 if (oz->oz_is_gc) { in xfs_open_zone_mark_full()
160 ASSERT(current == zi->zi_gc_thread); in xfs_open_zone_mark_full()
161 zi->zi_open_gc_zone = NULL; in xfs_open_zone_mark_full()
163 zi->zi_nr_open_zones--; in xfs_open_zone_mark_full()
164 list_del_init(&oz->oz_entry); in xfs_open_zone_mark_full()
166 spin_unlock(&zi->zi_open_zones_lock); in xfs_open_zone_mark_full()
169 wake_up_all(&zi->zi_zone_wait); in xfs_open_zone_mark_full()
171 xfs_zone_account_reclaimable(rtg, rtg_blocks(rtg) - used); in xfs_open_zone_mark_full()
181 struct xfs_mount *mp = tp->t_mountp; in xfs_zone_record_blocks()
182 struct xfs_rtgroup *rtg = oz->oz_rtg; in xfs_zone_record_blocks()
189 rmapip->i_used_blocks += len; in xfs_zone_record_blocks()
190 ASSERT(rmapip->i_used_blocks <= rtg_blocks(rtg)); in xfs_zone_record_blocks()
191 oz->oz_written += len; in xfs_zone_record_blocks()
192 if (oz->oz_written == rtg_blocks(rtg)) in xfs_zone_record_blocks()
207 struct xfs_rtgroup *rtg = oz->oz_rtg; in xfs_zone_skip_blocks()
212 oz->oz_written += len; in xfs_zone_skip_blocks()
213 if (oz->oz_written == rtg_blocks(rtg)) in xfs_zone_skip_blocks()
233 error = xfs_bmapi_read(ip, new->br_startoff, new->br_blockcount, &data, in xfs_zoned_map_extent()
242 ASSERT(new->br_blockcount >= data.br_blockcount); in xfs_zoned_map_extent()
243 new->br_blockcount = data.br_blockcount; in xfs_zoned_map_extent()
287 xfs_zone_record_blocks(tp, oz, new->br_startblock, new->br_blockcount); in xfs_zoned_map_extent()
295 xfs_zone_skip_blocks(oz, new->br_blockcount); in xfs_zoned_map_extent()
308 struct xfs_mount *mp = ip->i_mount; in xfs_zoned_end_io()
321 return -EIO; in xfs_zoned_end_io()
324 new.br_blockcount = end_fsb - new.br_startoff; in xfs_zoned_end_io()
326 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, in xfs_zoned_end_io()
363 struct xfs_mount *mp = tp->t_mountp; in xfs_zone_free_blocks()
368 if (len > rmapip->i_used_blocks) { in xfs_zone_free_blocks()
371 len, rmapip->i_used_blocks); in xfs_zone_free_blocks()
372 ASSERT(len <= rmapip->i_used_blocks); in xfs_zone_free_blocks()
375 return -EFSCORRUPTED; in xfs_zone_free_blocks()
380 rmapip->i_used_blocks -= len; in xfs_zone_free_blocks()
386 if (!READ_ONCE(rtg->rtg_open_zone)) in xfs_zone_free_blocks()
399 struct xfs_zone_info *zi = mp->m_zone_info; in xfs_find_free_zone()
400 XA_STATE (xas, &mp->m_groups[XG_TYPE_RTG].xa, start); in xfs_find_free_zone()
405 if (atomic_inc_not_zero(&xg->xg_active_ref)) in xfs_find_free_zone()
412 atomic_dec(&zi->zi_nr_free_zones); in xfs_find_free_zone()
413 zi->zi_free_zone_cursor = xg->xg_gno; in xfs_find_free_zone()
428 spin_lock_init(&oz->oz_alloc_lock); in xfs_init_open_zone()
429 atomic_set(&oz->oz_ref, 1); in xfs_init_open_zone()
430 oz->oz_rtg = rtg; in xfs_init_open_zone()
431 oz->oz_allocated = write_pointer; in xfs_init_open_zone()
432 oz->oz_written = write_pointer; in xfs_init_open_zone()
433 oz->oz_write_hint = write_hint; in xfs_init_open_zone()
434 oz->oz_is_gc = is_gc; in xfs_init_open_zone()
437 * All dereferences of rtg->rtg_open_zone hold the ILOCK for the rmap in xfs_init_open_zone()
443 WRITE_ONCE(rtg->rtg_open_zone, oz); in xfs_init_open_zone()
456 struct xfs_zone_info *zi = mp->m_zone_info; in xfs_open_zone()
459 xg = xfs_find_free_zone(mp, zi->zi_free_zone_cursor, ULONG_MAX); in xfs_open_zone()
461 xg = xfs_find_free_zone(mp, 0, zi->zi_free_zone_cursor); in xfs_open_zone()
474 struct xfs_zone_info *zi = mp->m_zone_info; in xfs_try_open_zone()
477 if (zi->zi_nr_open_zones >= mp->m_max_open_zones - XFS_OPEN_GC_ZONES) in xfs_try_open_zone()
479 if (atomic_read(&zi->zi_nr_free_zones) < in xfs_try_open_zone()
480 XFS_GC_ZONES - XFS_OPEN_GC_ZONES) in xfs_try_open_zone()
487 zi->zi_nr_open_zones++; in xfs_try_open_zone()
488 spin_unlock(&zi->zi_open_zones_lock); in xfs_try_open_zone()
490 spin_lock(&zi->zi_open_zones_lock); in xfs_try_open_zone()
492 zi->zi_nr_open_zones--; in xfs_try_open_zone()
496 atomic_inc(&oz->oz_ref); in xfs_try_open_zone()
497 list_add_tail(&oz->oz_entry, &zi->zi_open_zones); in xfs_try_open_zone()
503 wake_up_all(&zi->zi_zone_wait); in xfs_try_open_zone()
506 wake_up_process(zi->zi_gc_thread); in xfs_try_open_zone()
508 trace_xfs_zone_opened(oz->oz_rtg); in xfs_try_open_zone()
524 * Life time hint co-location matrix. Fields not set default to 0
558 if (oz->oz_allocated == rtg_blocks(oz->oz_rtg)) in xfs_try_use_zone()
561 if (xfs_zoned_hint_score[oz->oz_write_hint][file_hint] < goodness) in xfs_try_use_zone()
564 if (!atomic_inc_not_zero(&oz->oz_ref)) in xfs_try_use_zone()
571 * tracking per-temperature class written block counts, which is in xfs_try_use_zone()
575 oz->oz_write_hint == WRITE_LIFE_NOT_SET) in xfs_try_use_zone()
576 oz->oz_write_hint = file_hint; in xfs_try_use_zone()
583 * even if it isn't exactly cache friendly. in xfs_try_use_zone()
585 if (!list_is_last(&oz->oz_entry, &zi->zi_open_zones)) in xfs_try_use_zone()
586 list_move_tail(&oz->oz_entry, &zi->zi_open_zones); in xfs_try_use_zone()
598 lockdep_assert_held(&zi->zi_open_zones_lock); in xfs_select_open_zone_lru()
600 list_for_each_entry(oz, &zi->zi_open_zones, oz_entry) in xfs_select_open_zone_lru()
604 cond_resched_lock(&zi->zi_open_zones_lock); in xfs_select_open_zone_lru()
615 lockdep_assert_held(&zi->zi_open_zones_lock); in xfs_select_open_zone_mru()
617 list_for_each_entry_reverse(oz, &zi->zi_open_zones, oz_entry) in xfs_select_open_zone_mru()
621 cond_resched_lock(&zi->zi_open_zones_lock); in xfs_select_open_zone_mru()
627 if (xfs_has_nolifetime(ip->i_mount)) in xfs_inode_write_hint()
629 return VFS_I(ip)->i_write_hint; in xfs_inode_write_hint()
641 struct xfs_mount *mp = ip->i_mount; in xfs_zoned_pack_tight()
643 XFS_FSB_TO_B(mp, mp->m_groups[XG_TYPE_RTG].blocks); in xfs_zoned_pack_tight()
653 !(ip->i_diflags & XFS_DIFLAG_APPEND); in xfs_zoned_pack_tight()
662 struct xfs_zone_info *zi = mp->m_zone_info; in xfs_select_zone_nowait()
670 * is better to try to co-locate data when this is favorable, so we can in xfs_select_zone_nowait()
674 spin_lock(&zi->zi_open_zones_lock); in xfs_select_zone_nowait()
704 spin_unlock(&zi->zi_open_zones_lock); in xfs_select_zone_nowait()
714 struct xfs_zone_info *zi = mp->m_zone_info; in xfs_select_zone()
723 prepare_to_wait(&zi->zi_zone_wait, &wait, TASK_UNINTERRUPTIBLE); in xfs_select_zone()
729 finish_wait(&zi->zi_zone_wait, &wait); in xfs_select_zone()
740 struct xfs_rtgroup *rtg = oz->oz_rtg; in xfs_zone_alloc_blocks()
744 spin_lock(&oz->oz_alloc_lock); in xfs_zone_alloc_blocks()
746 (xfs_filblks_t)rtg_blocks(rtg) - oz->oz_allocated); in xfs_zone_alloc_blocks()
748 spin_unlock(&oz->oz_alloc_lock); in xfs_zone_alloc_blocks()
751 allocated = oz->oz_allocated; in xfs_zone_alloc_blocks()
752 oz->oz_allocated += count_fsb; in xfs_zone_alloc_blocks()
753 spin_unlock(&oz->oz_alloc_lock); in xfs_zone_alloc_blocks()
757 *sector = xfs_gbno_to_daddr(&rtg->rtg_group, 0); in xfs_zone_alloc_blocks()
758 *is_seq = bdev_zone_is_seq(mp->m_rtdev_targp->bt_bdev, *sector); in xfs_zone_alloc_blocks()
768 struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; in xfs_mark_rtg_boundary()
769 sector_t sector = ioend->io_bio.bi_iter.bi_sector; in xfs_mark_rtg_boundary()
772 ioend->io_flags |= IOMAP_IOEND_BOUNDARY; in xfs_mark_rtg_boundary()
786 oz = VFS_I(ip)->i_private; in xfs_get_cached_zone()
792 ASSERT(!oz->oz_is_gc); in xfs_get_cached_zone()
793 if (!atomic_inc_not_zero(&oz->oz_ref)) in xfs_get_cached_zone()
820 atomic_inc(&oz->oz_ref); in xfs_set_cached_zone()
821 old_oz = xchg(&VFS_I(ip)->i_private, oz); in xfs_set_cached_zone()
832 ioend->io_bio.bi_iter.bi_sector = ioend->io_sector; in xfs_submit_zoned_bio()
833 ioend->io_private = oz; in xfs_submit_zoned_bio()
834 atomic_inc(&oz->oz_ref); /* for xfs_zoned_end_io */ in xfs_submit_zoned_bio()
837 ioend->io_bio.bi_opf &= ~REQ_OP_WRITE; in xfs_submit_zoned_bio()
838 ioend->io_bio.bi_opf |= REQ_OP_ZONE_APPEND; in xfs_submit_zoned_bio()
843 submit_bio(&ioend->io_bio); in xfs_submit_zoned_bio()
851 struct xfs_inode *ip = XFS_I(ioend->io_inode); in xfs_zone_alloc_and_submit()
852 struct xfs_mount *mp = ip->i_mount; in xfs_zone_alloc_and_submit()
877 alloc_len = xfs_zone_alloc_blocks(*oz, XFS_B_TO_FSB(mp, ioend->io_size), in xfs_zone_alloc_and_submit()
878 &ioend->io_sector, &is_seq); in xfs_zone_alloc_and_submit()
887 alloc_len -= split->io_bio.bi_iter.bi_size; in xfs_zone_alloc_and_submit()
899 ioend->io_bio.bi_status = errno_to_blk_status(PTR_ERR(split)); in xfs_zone_alloc_and_submit()
901 bio_io_error(&ioend->io_bio); in xfs_zone_alloc_and_submit()
918 if ((mp->m_super->s_flags & SB_ACTIVE) && mp->m_zone_info) in xfs_zoned_wake_all()
919 wake_up_all(&mp->m_zone_info->zi_zone_wait); in xfs_zoned_wake_all()
931 lockdep_assert_held(&rtg_rmap(rtg)->i_lock); in xfs_zone_rgbno_is_valid()
933 if (rtg->rtg_open_zone) in xfs_zone_rgbno_is_valid()
934 return rgbno < rtg->rtg_open_zone->oz_allocated; in xfs_zone_rgbno_is_valid()
935 return !xa_get_mark(&rtg_mount(rtg)->m_groups[XG_TYPE_RTG].xa, in xfs_zone_rgbno_is_valid()
945 spin_lock(&zi->zi_open_zones_lock); in xfs_free_open_zones()
946 while ((oz = list_first_entry_or_null(&zi->zi_open_zones, in xfs_free_open_zones()
948 list_del(&oz->oz_entry); in xfs_free_open_zones()
951 spin_unlock(&zi->zi_open_zones_lock); in xfs_free_open_zones()
973 struct xfs_zone_info *zi = mp->m_zone_info; in xfs_init_zone()
974 uint32_t used = rtg_rmap(rtg)->i_used_blocks; in xfs_init_zone()
979 return -EFSCORRUPTED; in xfs_init_zone()
988 * power loss this misses blocks where the data I/O has completed but in xfs_init_zone()
993 if (!zone || zone->cond == BLK_ZONE_COND_NOT_WP) { in xfs_init_zone()
1005 * we lost power before the zoned reset. In that case finish the work in xfs_init_zone()
1017 atomic_inc(&zi->zi_nr_free_zones); in xfs_init_zone()
1018 xfs_group_set_mark(&rtg->rtg_group, XFS_RTG_FREE); in xfs_init_zone()
1019 iz->available += rtg_blocks(rtg); in xfs_init_zone()
1024 atomic_inc(&rtg_group(rtg)->xg_active_ref); in xfs_init_zone()
1027 list_add_tail(&oz->oz_entry, &zi->zi_open_zones); in xfs_init_zone()
1028 zi->zi_nr_open_zones++; in xfs_init_zone()
1030 iz->available += (rtg_blocks(rtg) - write_pointer); in xfs_init_zone()
1031 iz->reclaimable += write_pointer - used; in xfs_init_zone()
1034 xfs_zone_account_reclaimable(rtg, rtg_blocks(rtg) - used); in xfs_init_zone()
1035 iz->reclaimable += (rtg_blocks(rtg) - used); in xfs_init_zone()
1048 struct xfs_mount *mp = iz->mp; in xfs_get_zone_info_cb()
1049 xfs_fsblock_t zsbno = xfs_daddr_to_rtb(mp, zone->start); in xfs_get_zone_info_cb()
1056 return -EFSCORRUPTED; in xfs_get_zone_info_cb()
1063 return -EFSCORRUPTED; in xfs_get_zone_info_cb()
1085 max_open_data_zones = (mp->m_sb.sb_rgcount - XFS_MIN_ZONES) / 2 + 1; in xfs_max_open_zones()
1093 max_open = min(max_open, mp->m_sb.sb_rgcount / 4); in xfs_max_open_zones()
1116 struct block_device *bdev = mp->m_rtdev_targp->bt_bdev; in xfs_calc_open_zones()
1119 if (!mp->m_max_open_zones) { in xfs_calc_open_zones()
1121 mp->m_max_open_zones = bdev_open_zones; in xfs_calc_open_zones()
1123 mp->m_max_open_zones = XFS_DEFAULT_MAX_OPEN_ZONES; in xfs_calc_open_zones()
1126 if (mp->m_max_open_zones < XFS_MIN_OPEN_ZONES) { in xfs_calc_open_zones()
1129 return -EIO; in xfs_calc_open_zones()
1132 if (bdev_open_zones && bdev_open_zones < mp->m_max_open_zones) { in xfs_calc_open_zones()
1133 mp->m_max_open_zones = bdev_open_zones; in xfs_calc_open_zones()
1138 if (mp->m_max_open_zones > xfs_max_open_zones(mp)) { in xfs_calc_open_zones()
1139 mp->m_max_open_zones = xfs_max_open_zones(mp); in xfs_calc_open_zones()
1142 mp->m_max_open_zones, mp->m_sb.sb_rgcount); in xfs_calc_open_zones()
1152 return kvmalloc_array(BITS_TO_LONGS(mp->m_sb.sb_rgcount), in xfs_alloc_bucket_bitmap()
1166 INIT_LIST_HEAD(&zi->zi_open_zones); in xfs_alloc_zone_info()
1167 INIT_LIST_HEAD(&zi->zi_reclaim_reservations); in xfs_alloc_zone_info()
1168 spin_lock_init(&zi->zi_reset_list_lock); in xfs_alloc_zone_info()
1169 spin_lock_init(&zi->zi_open_zones_lock); in xfs_alloc_zone_info()
1170 spin_lock_init(&zi->zi_reservation_lock); in xfs_alloc_zone_info()
1171 init_waitqueue_head(&zi->zi_zone_wait); in xfs_alloc_zone_info()
1172 spin_lock_init(&zi->zi_used_buckets_lock); in xfs_alloc_zone_info()
1174 zi->zi_used_bucket_bitmap[i] = xfs_alloc_bucket_bitmap(mp); in xfs_alloc_zone_info()
1175 if (!zi->zi_used_bucket_bitmap[i]) in xfs_alloc_zone_info()
1181 while (--i > 0) in xfs_alloc_zone_info()
1182 kvfree(zi->zi_used_bucket_bitmap[i]); in xfs_alloc_zone_info()
1195 kvfree(zi->zi_used_bucket_bitmap[i]); in xfs_free_zone_info()
1206 struct xfs_buftarg *bt = mp->m_rtdev_targp; in xfs_mount_zones()
1211 return -EINVAL; in xfs_mount_zones()
1216 return -EFSCORRUPTED; in xfs_mount_zones()
1218 if (mp->m_sb.sb_rextsize != 1) { in xfs_mount_zones()
1220 return -EFSCORRUPTED; in xfs_mount_zones()
1222 if (mp->m_sb.sb_rgcount < XFS_MIN_ZONES) { in xfs_mount_zones()
1225 return -EFSCORRUPTED; in xfs_mount_zones()
1232 mp->m_zone_info = xfs_alloc_zone_info(mp); in xfs_mount_zones()
1233 if (!mp->m_zone_info) in xfs_mount_zones()
1234 return -ENOMEM; in xfs_mount_zones()
1237 mp->m_sb.sb_rgcount, mp->m_groups[XG_TYPE_RTG].blocks, in xfs_mount_zones()
1238 mp->m_max_open_zones); in xfs_mount_zones()
1241 if (bdev_is_zoned(bt->bt_bdev)) { in xfs_mount_zones()
1242 error = blkdev_report_zones(bt->bt_bdev, in xfs_mount_zones()
1243 XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart), in xfs_mount_zones()
1244 mp->m_sb.sb_rgcount, xfs_get_zone_info_cb, &iz); in xfs_mount_zones()
1268 mp->m_zonegc_low_space = 0; in xfs_mount_zones()
1276 xfs_free_zone_info(mp->m_zone_info); in xfs_mount_zones()
1285 xfs_free_zone_info(mp->m_zone_info); in xfs_unmount_zones()