1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2023-2025 Christoph Hellwig.
4 * Copyright (c) 2024-2025, Western Digital Corporation or its affiliates.
5 */
6 #include "xfs_platform.h"
7 #include "xfs_shared.h"
8 #include "xfs_format.h"
9 #include "xfs_log_format.h"
10 #include "xfs_trans_resv.h"
11 #include "xfs_mount.h"
12 #include "xfs_inode.h"
13 #include "xfs_btree.h"
14 #include "xfs_trans.h"
15 #include "xfs_icache.h"
16 #include "xfs_rmap.h"
17 #include "xfs_rtbitmap.h"
18 #include "xfs_rtrmap_btree.h"
19 #include "xfs_errortag.h"
20 #include "xfs_error.h"
21 #include "xfs_zone_alloc.h"
22 #include "xfs_zone_priv.h"
23 #include "xfs_zones.h"
24 #include "xfs_trace.h"
25
26 /*
27 * Implement Garbage Collection (GC) of partially used zoned.
28 *
29 * To support the purely sequential writes in each zone, zoned XFS needs to be
30 * able to move data remaining in a zone out of it to reset the zone to prepare
31 * for writing to it again.
32 *
33 * This is done by the GC thread implemented in this file. To support that a
34 * number of zones (XFS_GC_ZONES) is reserved from the user visible capacity to
35 * write the garbage collected data into.
36 *
37 * Whenever the available space is below the chosen threshold, the GC thread
38 * looks for potential non-empty but not fully used zones that are worth
39 * reclaiming. Once found the rmap for the victim zone is queried, and after
40 * a bit of sorting to reduce fragmentation, the still live extents are read
41 * into memory and written to the GC target zone, and the bmap btree of the
42 * files is updated to point to the new location. To avoid taking the IOLOCK
43 * and MMAPLOCK for the entire GC process and thus affecting the latency of
44 * user reads and writes to the files, the GC writes are speculative and the
45 * I/O completion checks that no other writes happened for the affected regions
46 * before remapping.
47 *
48 * Once a zone does not contain any valid data, be that through GC or user
49 * block removal, it is queued for for a zone reset. The reset operation
50 * carefully ensures that the RT device cache is flushed and all transactions
51 * referencing the rmap have been committed to disk.
52 */
53
54 /*
55 * Size of each GC scratch allocation, and the number of buffers.
56 */
57 #define XFS_GC_BUF_SIZE SZ_1M
58 #define XFS_GC_NR_BUFS 2
59 static_assert(XFS_GC_NR_BUFS < BIO_MAX_VECS);
60
61 /*
62 * Chunk that is read and written for each GC operation.
63 *
64 * Note that for writes to actual zoned devices, the chunk can be split when
65 * reaching the hardware limit.
66 */
67 struct xfs_gc_bio {
68 struct xfs_zone_gc_data *data;
69
70 /*
71 * Entry into the reading/writing/resetting list. Only accessed from
72 * the GC thread, so no locking needed.
73 */
74 struct list_head entry;
75
76 /*
77 * State of this gc_bio. Done means the current I/O completed.
78 * Set from the bio end I/O handler, read from the GC thread.
79 */
80 enum {
81 XFS_GC_BIO_NEW,
82 XFS_GC_BIO_DONE,
83 } state;
84
85 /*
86 * Pointer to the inode and byte range in the inode that this
87 * GC chunk is operating on.
88 */
89 struct xfs_inode *ip;
90 loff_t offset;
91 unsigned int len;
92
93 /*
94 * Existing startblock (in the zone to be freed) and newly assigned
95 * daddr in the zone GCed into.
96 */
97 xfs_fsblock_t old_startblock;
98 xfs_daddr_t new_daddr;
99
100 /* Are we writing to a sequential write required zone? */
101 bool is_seq;
102
103 /* Open Zone being written to */
104 struct xfs_open_zone *oz;
105
106 /* Realtime group currently being reclaimed */
107 struct xfs_rtgroup *victim_rtg;
108
109 /* Bio used for reads and writes, including the bvec used by it */
110 struct bio bio; /* must be last */
111 };
112
113 #define XFS_ZONE_GC_RECS 1024
114
115 /* iterator, needs to be reinitialized for each victim zone */
116 struct xfs_zone_gc_iter {
117 struct xfs_rtgroup *victim_rtg;
118 unsigned int rec_count;
119 unsigned int rec_idx;
120 xfs_agblock_t next_startblock;
121 struct xfs_rmap_irec *recs;
122 };
123
124 /*
125 * Per-mount GC state.
126 */
127 struct xfs_zone_gc_data {
128 struct xfs_mount *mp;
129 struct xfs_open_zone *oz;
130
131 /* bioset used to allocate the gc_bios */
132 struct bio_set bio_set;
133
134 /* bioset used when writes need to be split to hardware limits */
135 struct bio_set split_bio_set;
136
137 /*
138 * Scratchpad to buffer GC data, organized as a ring buffer over
139 * discontiguous folios. scratch_head is where the buffer is filled,
140 * scratch_tail tracks the buffer space freed, and scratch_available
141 * counts the space available in the ring buffer between the head and
142 * the tail.
143 */
144 struct folio *scratch_folios[XFS_GC_NR_BUFS];
145 unsigned int scratch_size;
146 unsigned int scratch_available;
147 unsigned int scratch_head;
148 unsigned int scratch_tail;
149
150 /*
151 * List of bios currently being read, written and reset.
152 * These lists are only accessed by the GC thread itself, and must only
153 * be processed in order.
154 */
155 struct list_head reading;
156 struct list_head writing;
157 struct list_head resetting;
158
159 /*
160 * Iterator for the victim zone.
161 */
162 struct xfs_zone_gc_iter iter;
163 };
164
165 /*
166 * We aim to keep enough zones free in stock to fully use the open zone limit
167 * for data placement purposes. Additionally, the m_zonegc_low_space tunable
168 * can be set to make sure a fraction of the unused blocks are available for
169 * writing.
170 */
171 bool
xfs_zoned_need_gc(struct xfs_mount * mp)172 xfs_zoned_need_gc(
173 struct xfs_mount *mp)
174 {
175 s64 available, free, threshold;
176 s32 remainder;
177
178 /* If we have no reclaimable blocks, running GC is useless. */
179 if (!xfs_zoned_have_reclaimable(mp->m_zone_info))
180 return false;
181
182 /*
183 * In order to avoid file fragmentation as much as possible, we should
184 * make sure that we can open enough zones. So trigger GC if the number
185 * of blocks immediately available for writes is lower than the total
186 * number of blocks from all possible open zones.
187 */
188 available = xfs_estimate_freecounter(mp, XC_FREE_RTAVAILABLE);
189 if (available <
190 xfs_rtgs_to_rfsbs(mp, mp->m_max_open_zones - XFS_OPEN_GC_ZONES))
191 return true;
192
193 /*
194 * For cases where the user wants to be more aggressive with GC,
195 * the sysfs attribute zonegc_low_space may be set to a non zero value,
196 * to indicate that GC should try to maintain at least zonegc_low_space
197 * percent of the free space to be directly available for writing. Check
198 * this here.
199 */
200 if (!mp->m_zonegc_low_space)
201 return false;
202
203 free = xfs_estimate_freecounter(mp, XC_FREE_RTEXTENTS);
204 threshold = div_s64_rem(free, 100, &remainder);
205 threshold = threshold * mp->m_zonegc_low_space +
206 remainder * div_s64(mp->m_zonegc_low_space, 100);
207
208 return available < threshold;
209 }
210
211 static struct xfs_zone_gc_data *
xfs_zone_gc_data_alloc(struct xfs_mount * mp)212 xfs_zone_gc_data_alloc(
213 struct xfs_mount *mp)
214 {
215 struct xfs_zone_gc_data *data;
216 int i;
217
218 data = kzalloc_obj(*data);
219 if (!data)
220 return NULL;
221 data->iter.recs = kzalloc_objs(*data->iter.recs, XFS_ZONE_GC_RECS);
222 if (!data->iter.recs)
223 goto out_free_data;
224
225 if (bioset_init(&data->bio_set, 16, offsetof(struct xfs_gc_bio, bio),
226 BIOSET_NEED_BVECS))
227 goto out_free_recs;
228 if (bioset_init(&data->split_bio_set, 16,
229 offsetof(struct xfs_gc_bio, bio), 0))
230 goto out_exit_bio_set;
231 for (i = 0; i < XFS_GC_NR_BUFS; i++) {
232 data->scratch_folios[i] =
233 folio_alloc(GFP_KERNEL, get_order(XFS_GC_BUF_SIZE));
234 if (!data->scratch_folios[i])
235 goto out_free_scratch;
236 }
237 data->scratch_size = XFS_GC_BUF_SIZE * XFS_GC_NR_BUFS;
238 data->scratch_available = data->scratch_size;
239 INIT_LIST_HEAD(&data->reading);
240 INIT_LIST_HEAD(&data->writing);
241 INIT_LIST_HEAD(&data->resetting);
242 data->mp = mp;
243 return data;
244
245 out_free_scratch:
246 while (--i >= 0)
247 folio_put(data->scratch_folios[i]);
248 bioset_exit(&data->split_bio_set);
249 out_exit_bio_set:
250 bioset_exit(&data->bio_set);
251 out_free_recs:
252 kfree(data->iter.recs);
253 out_free_data:
254 kfree(data);
255 return NULL;
256 }
257
258 static void
xfs_zone_gc_data_free(struct xfs_zone_gc_data * data)259 xfs_zone_gc_data_free(
260 struct xfs_zone_gc_data *data)
261 {
262 int i;
263
264 for (i = 0; i < XFS_GC_NR_BUFS; i++)
265 folio_put(data->scratch_folios[i]);
266 bioset_exit(&data->split_bio_set);
267 bioset_exit(&data->bio_set);
268 kfree(data->iter.recs);
269 kfree(data);
270 }
271
272 static void
xfs_zone_gc_iter_init(struct xfs_zone_gc_iter * iter,struct xfs_rtgroup * victim_rtg)273 xfs_zone_gc_iter_init(
274 struct xfs_zone_gc_iter *iter,
275 struct xfs_rtgroup *victim_rtg)
276
277 {
278 iter->next_startblock = 0;
279 iter->rec_count = 0;
280 iter->rec_idx = 0;
281 iter->victim_rtg = victim_rtg;
282 atomic_inc(&victim_rtg->rtg_gccount);
283 }
284
285 /*
286 * Query the rmap of the victim zone to gather the records to evacuate.
287 */
288 static int
xfs_zone_gc_query_cb(struct xfs_btree_cur * cur,const struct xfs_rmap_irec * irec,void * private)289 xfs_zone_gc_query_cb(
290 struct xfs_btree_cur *cur,
291 const struct xfs_rmap_irec *irec,
292 void *private)
293 {
294 struct xfs_zone_gc_iter *iter = private;
295
296 ASSERT(!XFS_RMAP_NON_INODE_OWNER(irec->rm_owner));
297 ASSERT(!xfs_is_sb_inum(cur->bc_mp, irec->rm_owner));
298 ASSERT(!(irec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK)));
299
300 iter->recs[iter->rec_count] = *irec;
301 if (++iter->rec_count == XFS_ZONE_GC_RECS) {
302 iter->next_startblock =
303 irec->rm_startblock + irec->rm_blockcount;
304 return 1;
305 }
306 return 0;
307 }
308
309 static int
xfs_zone_gc_rmap_rec_cmp(const void * a,const void * b)310 xfs_zone_gc_rmap_rec_cmp(
311 const void *a,
312 const void *b)
313 {
314 const struct xfs_rmap_irec *reca = a;
315 const struct xfs_rmap_irec *recb = b;
316 int diff;
317
318 diff = cmp_int(reca->rm_owner, recb->rm_owner);
319 if (diff)
320 return diff;
321 return cmp_int(reca->rm_offset, recb->rm_offset);
322 }
323
324 static int
xfs_zone_gc_query(struct xfs_mount * mp,struct xfs_zone_gc_iter * iter)325 xfs_zone_gc_query(
326 struct xfs_mount *mp,
327 struct xfs_zone_gc_iter *iter)
328 {
329 struct xfs_rtgroup *rtg = iter->victim_rtg;
330 struct xfs_rmap_irec ri_low = { };
331 struct xfs_rmap_irec ri_high;
332 struct xfs_btree_cur *cur;
333 struct xfs_trans *tp;
334 int error;
335
336 ASSERT(iter->next_startblock <= rtg_blocks(rtg));
337 if (iter->next_startblock == rtg_blocks(rtg))
338 goto done;
339
340 ASSERT(iter->next_startblock < rtg_blocks(rtg));
341 ri_low.rm_startblock = iter->next_startblock;
342 memset(&ri_high, 0xFF, sizeof(ri_high));
343
344 iter->rec_idx = 0;
345 iter->rec_count = 0;
346
347 tp = xfs_trans_alloc_empty(mp);
348 xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP);
349 cur = xfs_rtrmapbt_init_cursor(tp, rtg);
350 error = xfs_rmap_query_range(cur, &ri_low, &ri_high,
351 xfs_zone_gc_query_cb, iter);
352 xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP);
353 xfs_btree_del_cursor(cur, error < 0 ? error : 0);
354 xfs_trans_cancel(tp);
355
356 if (error < 0)
357 return error;
358
359 /*
360 * Sort the rmap records by inode number and increasing offset to
361 * defragment the mappings.
362 *
363 * This could be further enhanced by an even bigger look ahead window,
364 * but that's better left until we have better detection of changes to
365 * inode mapping to avoid the potential of GCing already dead data.
366 */
367 sort(iter->recs, iter->rec_count, sizeof(iter->recs[0]),
368 xfs_zone_gc_rmap_rec_cmp, NULL);
369
370 if (error == 0) {
371 /*
372 * We finished iterating through the zone.
373 */
374 iter->next_startblock = rtg_blocks(rtg);
375 if (iter->rec_count == 0)
376 goto done;
377 }
378
379 return 0;
380 done:
381 atomic_dec(&iter->victim_rtg->rtg_gccount);
382 xfs_rtgroup_rele(iter->victim_rtg);
383 iter->victim_rtg = NULL;
384 return 0;
385 }
386
387 static bool
xfs_zone_gc_iter_irec(struct xfs_mount * mp,struct xfs_zone_gc_iter * iter,struct xfs_rmap_irec * chunk_rec,struct xfs_inode ** ipp)388 xfs_zone_gc_iter_irec(
389 struct xfs_mount *mp,
390 struct xfs_zone_gc_iter *iter,
391 struct xfs_rmap_irec *chunk_rec,
392 struct xfs_inode **ipp)
393 {
394 struct xfs_rmap_irec *irec;
395 int error;
396
397 retry:
398 if (iter->rec_idx == iter->rec_count) {
399 error = xfs_zone_gc_query(mp, iter);
400 if (error)
401 goto fail;
402 if (!iter->victim_rtg)
403 return false;
404 }
405
406 irec = &iter->recs[iter->rec_idx];
407 error = xfs_iget(mp, NULL, irec->rm_owner,
408 XFS_IGET_UNTRUSTED | XFS_IGET_DONTCACHE, 0, ipp);
409 if (error) {
410 /*
411 * If the inode was already deleted, skip over it.
412 */
413 if (error == -ENOENT || error == -EINVAL) {
414 iter->rec_idx++;
415 goto retry;
416 }
417 goto fail;
418 }
419
420 if (!S_ISREG(VFS_I(*ipp)->i_mode) || !XFS_IS_REALTIME_INODE(*ipp)) {
421 iter->rec_idx++;
422 xfs_irele(*ipp);
423 goto retry;
424 }
425
426 *chunk_rec = *irec;
427 return true;
428
429 fail:
430 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
431 return false;
432 }
433
434 static void
xfs_zone_gc_iter_advance(struct xfs_zone_gc_iter * iter,xfs_extlen_t count_fsb)435 xfs_zone_gc_iter_advance(
436 struct xfs_zone_gc_iter *iter,
437 xfs_extlen_t count_fsb)
438 {
439 struct xfs_rmap_irec *irec = &iter->recs[iter->rec_idx];
440
441 irec->rm_offset += count_fsb;
442 irec->rm_startblock += count_fsb;
443 irec->rm_blockcount -= count_fsb;
444 if (!irec->rm_blockcount)
445 iter->rec_idx++;
446 }
447
448 static struct xfs_rtgroup *
xfs_zone_gc_pick_victim_from(struct xfs_mount * mp,uint32_t bucket)449 xfs_zone_gc_pick_victim_from(
450 struct xfs_mount *mp,
451 uint32_t bucket)
452 {
453 struct xfs_zone_info *zi = mp->m_zone_info;
454 uint32_t victim_used = U32_MAX;
455 struct xfs_rtgroup *victim_rtg = NULL;
456 uint32_t bit;
457
458 if (!zi->zi_used_bucket_entries[bucket])
459 return NULL;
460
461 for_each_set_bit(bit, zi->zi_used_bucket_bitmap[bucket],
462 mp->m_sb.sb_rgcount) {
463 struct xfs_rtgroup *rtg = xfs_rtgroup_grab(mp, bit);
464
465 if (!rtg)
466 continue;
467
468 /*
469 * If the zone is already undergoing GC, don't pick it again.
470 *
471 * This prevents us from picking one of the zones for which we
472 * already submitted GC I/O, but for which the remapping hasn't
473 * concluded yet. This won't cause data corruption, but
474 * increases write amplification and slows down GC, so this is
475 * a bad thing.
476 */
477 if (atomic_read(&rtg->rtg_gccount)) {
478 xfs_rtgroup_rele(rtg);
479 continue;
480 }
481
482 /* skip zones that are just waiting for a reset */
483 if (rtg_rmap(rtg)->i_used_blocks == 0 ||
484 rtg_rmap(rtg)->i_used_blocks >= victim_used) {
485 xfs_rtgroup_rele(rtg);
486 continue;
487 }
488
489 if (victim_rtg)
490 xfs_rtgroup_rele(victim_rtg);
491 victim_rtg = rtg;
492 victim_used = rtg_rmap(rtg)->i_used_blocks;
493
494 /*
495 * Any zone that is less than 1 percent used is fair game for
496 * instant reclaim. All of these zones are in the last
497 * bucket, so avoid the expensive division for the zones
498 * in the other buckets.
499 */
500 if (bucket == 0 &&
501 rtg_rmap(rtg)->i_used_blocks < rtg_blocks(rtg) / 100)
502 break;
503 }
504
505 return victim_rtg;
506 }
507
508 /*
509 * Iterate through all zones marked as reclaimable and find a candidate to
510 * reclaim.
511 */
512 static bool
xfs_zone_gc_select_victim(struct xfs_zone_gc_data * data)513 xfs_zone_gc_select_victim(
514 struct xfs_zone_gc_data *data)
515 {
516 struct xfs_zone_gc_iter *iter = &data->iter;
517 struct xfs_mount *mp = data->mp;
518 struct xfs_zone_info *zi = mp->m_zone_info;
519 struct xfs_rtgroup *victim_rtg = NULL;
520 unsigned int bucket;
521
522 spin_lock(&zi->zi_used_buckets_lock);
523 for (bucket = 0; bucket < XFS_ZONE_USED_BUCKETS; bucket++) {
524 victim_rtg = xfs_zone_gc_pick_victim_from(mp, bucket);
525 if (victim_rtg)
526 break;
527 }
528 spin_unlock(&zi->zi_used_buckets_lock);
529
530 if (!victim_rtg)
531 return false;
532
533 trace_xfs_zone_gc_select_victim(victim_rtg, bucket);
534 xfs_zone_gc_iter_init(iter, victim_rtg);
535 return true;
536 }
537
538 static int
xfs_zone_gc_steal_open_zone(struct xfs_zone_gc_data * data)539 xfs_zone_gc_steal_open_zone(
540 struct xfs_zone_gc_data *data)
541 {
542 struct xfs_zone_info *zi = data->mp->m_zone_info;
543 struct xfs_open_zone *oz, *found = NULL;
544
545 spin_lock(&zi->zi_open_zones_lock);
546 list_for_each_entry(oz, &zi->zi_open_zones, oz_entry) {
547 if (!found || oz->oz_allocated < found->oz_allocated)
548 found = oz;
549 }
550 if (!found) {
551 spin_unlock(&zi->zi_open_zones_lock);
552 return -EIO;
553 }
554
555 trace_xfs_zone_gc_target_stolen(found->oz_rtg);
556 found->oz_is_gc = true;
557 zi->zi_nr_open_zones--;
558 zi->zi_nr_open_gc_zones++;
559 spin_unlock(&zi->zi_open_zones_lock);
560
561 atomic_inc(&found->oz_ref);
562 data->oz = found;
563 return 0;
564 }
565
566 /*
567 * Ensure we have a valid open zone to write to.
568 */
569 static bool
xfs_zone_gc_select_target(struct xfs_zone_gc_data * data)570 xfs_zone_gc_select_target(
571 struct xfs_zone_gc_data *data)
572 {
573 struct xfs_zone_info *zi = data->mp->m_zone_info;
574
575 if (data->oz) {
576 /*
577 * If we have space available, just keep using the existing
578 * zone.
579 */
580 if (data->oz->oz_allocated < rtg_blocks(data->oz->oz_rtg))
581 return true;
582
583 /*
584 * Wait for all writes to the current zone to finish before
585 * picking a new one.
586 */
587 if (data->oz->oz_written < rtg_blocks(data->oz->oz_rtg))
588 return false;
589
590 xfs_open_zone_put(data->oz);
591 }
592
593 /*
594 * Open a new zone when there is none currently in use.
595 */
596 ASSERT(zi->zi_nr_open_zones <=
597 data->mp->m_max_open_zones - XFS_OPEN_GC_ZONES);
598 data->oz = xfs_open_zone(data->mp, WRITE_LIFE_NOT_SET, true);
599 if (!data->oz)
600 return false;
601 trace_xfs_zone_gc_target_opened(data->oz->oz_rtg);
602 atomic_inc(&data->oz->oz_ref);
603 spin_lock(&zi->zi_open_zones_lock);
604 zi->zi_nr_open_gc_zones++;
605 list_add_tail(&data->oz->oz_entry, &zi->zi_open_zones);
606 spin_unlock(&zi->zi_open_zones_lock);
607 return true;
608 }
609
610 static void
xfs_zone_gc_end_io(struct bio * bio)611 xfs_zone_gc_end_io(
612 struct bio *bio)
613 {
614 struct xfs_gc_bio *chunk =
615 container_of(bio, struct xfs_gc_bio, bio);
616 struct xfs_zone_gc_data *data = chunk->data;
617
618 WRITE_ONCE(chunk->state, XFS_GC_BIO_DONE);
619 wake_up_process(data->mp->m_zone_info->zi_gc_thread);
620 }
621
622 static bool
xfs_zone_gc_alloc_blocks(struct xfs_zone_gc_data * data,xfs_extlen_t * count_fsb,xfs_daddr_t * daddr,bool * is_seq)623 xfs_zone_gc_alloc_blocks(
624 struct xfs_zone_gc_data *data,
625 xfs_extlen_t *count_fsb,
626 xfs_daddr_t *daddr,
627 bool *is_seq)
628 {
629 struct xfs_mount *mp = data->mp;
630 struct xfs_open_zone *oz = data->oz;
631
632 *count_fsb = min(*count_fsb, XFS_B_TO_FSB(mp, data->scratch_available));
633
634 /*
635 * Directly allocate GC blocks from the reserved pool.
636 *
637 * If we'd take them from the normal pool we could be stealing blocks
638 * from a regular writer, which would then have to wait for GC and
639 * deadlock.
640 */
641 spin_lock(&mp->m_sb_lock);
642 *count_fsb = min(*count_fsb,
643 rtg_blocks(oz->oz_rtg) - oz->oz_allocated);
644 *count_fsb = min3(*count_fsb,
645 mp->m_free[XC_FREE_RTEXTENTS].res_avail,
646 mp->m_free[XC_FREE_RTAVAILABLE].res_avail);
647 mp->m_free[XC_FREE_RTEXTENTS].res_avail -= *count_fsb;
648 mp->m_free[XC_FREE_RTAVAILABLE].res_avail -= *count_fsb;
649 spin_unlock(&mp->m_sb_lock);
650
651 if (!*count_fsb)
652 return false;
653
654 *daddr = xfs_gbno_to_daddr(rtg_group(oz->oz_rtg), 0);
655 *is_seq = bdev_zone_is_seq(mp->m_rtdev_targp->bt_bdev, *daddr);
656 if (!*is_seq)
657 *daddr += XFS_FSB_TO_BB(mp, oz->oz_allocated);
658 oz->oz_allocated += *count_fsb;
659 atomic_inc(&oz->oz_ref);
660 return true;
661 }
662
663 static void
xfs_zone_gc_add_data(struct xfs_gc_bio * chunk)664 xfs_zone_gc_add_data(
665 struct xfs_gc_bio *chunk)
666 {
667 struct xfs_zone_gc_data *data = chunk->data;
668 unsigned int len = chunk->len;
669 unsigned int off = data->scratch_head;
670
671 do {
672 unsigned int this_off = off % XFS_GC_BUF_SIZE;
673 unsigned int this_len = min(len, XFS_GC_BUF_SIZE - this_off);
674
675 bio_add_folio_nofail(&chunk->bio,
676 data->scratch_folios[off / XFS_GC_BUF_SIZE],
677 this_len, this_off);
678 len -= this_len;
679 off += this_len;
680 if (off == data->scratch_size)
681 off = 0;
682 } while (len);
683 }
684
685 static bool
xfs_zone_gc_can_start_chunk(struct xfs_zone_gc_data * data)686 xfs_zone_gc_can_start_chunk(
687 struct xfs_zone_gc_data *data)
688 {
689
690 if (xfs_is_shutdown(data->mp))
691 return false;
692 if (!data->scratch_available)
693 return false;
694
695 if (!data->iter.victim_rtg) {
696 if (kthread_should_stop() || kthread_should_park())
697 return false;
698 if (!xfs_zoned_need_gc(data->mp))
699 return false;
700 if (!xfs_zone_gc_select_victim(data))
701 return false;
702 }
703
704 return xfs_zone_gc_select_target(data);
705 }
706
707 static bool
xfs_zone_gc_start_chunk(struct xfs_zone_gc_data * data)708 xfs_zone_gc_start_chunk(
709 struct xfs_zone_gc_data *data)
710 {
711 struct xfs_zone_gc_iter *iter = &data->iter;
712 struct xfs_mount *mp = data->mp;
713 struct block_device *bdev = mp->m_rtdev_targp->bt_bdev;
714 struct xfs_rmap_irec irec;
715 struct xfs_gc_bio *chunk;
716 struct xfs_inode *ip;
717 struct bio *bio;
718 xfs_daddr_t daddr;
719 bool is_seq;
720
721 if (!xfs_zone_gc_can_start_chunk(data))
722 return false;
723
724 set_current_state(TASK_RUNNING);
725 if (!xfs_zone_gc_iter_irec(mp, iter, &irec, &ip))
726 return false;
727
728 if (!xfs_zone_gc_alloc_blocks(data, &irec.rm_blockcount, &daddr,
729 &is_seq)) {
730 xfs_irele(ip);
731 return false;
732 }
733
734 /*
735 * Scratch allocation can wrap around to the same buffer again,
736 * provision an extra bvec for that case.
737 */
738 bio = bio_alloc_bioset(bdev, XFS_GC_NR_BUFS + 1, REQ_OP_READ, GFP_NOFS,
739 &data->bio_set);
740 chunk = container_of(bio, struct xfs_gc_bio, bio);
741 chunk->ip = ip;
742 chunk->offset = XFS_FSB_TO_B(mp, irec.rm_offset);
743 chunk->len = XFS_FSB_TO_B(mp, irec.rm_blockcount);
744 chunk->old_startblock =
745 xfs_rgbno_to_rtb(iter->victim_rtg, irec.rm_startblock);
746 chunk->new_daddr = daddr;
747 chunk->is_seq = is_seq;
748 chunk->data = data;
749 chunk->oz = data->oz;
750 chunk->victim_rtg = iter->victim_rtg;
751 atomic_inc(&rtg_group(chunk->victim_rtg)->xg_active_ref);
752 atomic_inc(&chunk->victim_rtg->rtg_gccount);
753
754 bio->bi_iter.bi_sector = xfs_rtb_to_daddr(mp, chunk->old_startblock);
755 bio->bi_end_io = xfs_zone_gc_end_io;
756 xfs_zone_gc_add_data(chunk);
757 data->scratch_head =
758 (data->scratch_head + chunk->len) % data->scratch_size;
759 data->scratch_available -= chunk->len;
760
761 XFS_STATS_INC(mp, xs_gc_read_calls);
762
763 WRITE_ONCE(chunk->state, XFS_GC_BIO_NEW);
764 list_add_tail(&chunk->entry, &data->reading);
765 xfs_zone_gc_iter_advance(iter, irec.rm_blockcount);
766
767 submit_bio(bio);
768 return true;
769 }
770
771 static void
xfs_zone_gc_free_chunk(struct xfs_gc_bio * chunk)772 xfs_zone_gc_free_chunk(
773 struct xfs_gc_bio *chunk)
774 {
775 atomic_dec(&chunk->victim_rtg->rtg_gccount);
776 xfs_rtgroup_rele(chunk->victim_rtg);
777 list_del(&chunk->entry);
778 xfs_open_zone_put(chunk->oz);
779 xfs_irele(chunk->ip);
780 bio_put(&chunk->bio);
781 }
782
783 static void
xfs_zone_gc_submit_write(struct xfs_zone_gc_data * data,struct xfs_gc_bio * chunk)784 xfs_zone_gc_submit_write(
785 struct xfs_zone_gc_data *data,
786 struct xfs_gc_bio *chunk)
787 {
788 if (chunk->is_seq) {
789 chunk->bio.bi_opf &= ~REQ_OP_WRITE;
790 chunk->bio.bi_opf |= REQ_OP_ZONE_APPEND;
791 }
792 chunk->bio.bi_iter.bi_sector = chunk->new_daddr;
793 chunk->bio.bi_end_io = xfs_zone_gc_end_io;
794 submit_bio(&chunk->bio);
795 }
796
797 static struct xfs_gc_bio *
xfs_zone_gc_split_write(struct xfs_zone_gc_data * data,struct xfs_gc_bio * chunk)798 xfs_zone_gc_split_write(
799 struct xfs_zone_gc_data *data,
800 struct xfs_gc_bio *chunk)
801 {
802 struct queue_limits *lim =
803 &bdev_get_queue(chunk->bio.bi_bdev)->limits;
804 struct xfs_gc_bio *split_chunk;
805 int split_sectors;
806 unsigned int split_len;
807 struct bio *split;
808 unsigned int nsegs;
809
810 if (!chunk->is_seq)
811 return NULL;
812
813 split_sectors = bio_split_rw_at(&chunk->bio, lim, &nsegs,
814 lim->max_zone_append_sectors << SECTOR_SHIFT);
815 if (split_sectors <= 0)
816 return NULL;
817
818 /* ensure the split chunk is still block size aligned */
819 split_sectors = ALIGN_DOWN(split_sectors << SECTOR_SHIFT,
820 data->mp->m_sb.sb_blocksize) >> SECTOR_SHIFT;
821 split_len = split_sectors << SECTOR_SHIFT;
822
823 split = bio_split(&chunk->bio, split_sectors, GFP_NOFS,
824 &data->split_bio_set);
825 split_chunk = container_of(split, struct xfs_gc_bio, bio);
826 split_chunk->data = data;
827 ihold(VFS_I(chunk->ip));
828 split_chunk->ip = chunk->ip;
829 split_chunk->is_seq = chunk->is_seq;
830 split_chunk->offset = chunk->offset;
831 split_chunk->len = split_len;
832 split_chunk->old_startblock = chunk->old_startblock;
833 split_chunk->new_daddr = chunk->new_daddr;
834 split_chunk->oz = chunk->oz;
835 atomic_inc(&chunk->oz->oz_ref);
836
837 split_chunk->victim_rtg = chunk->victim_rtg;
838 atomic_inc(&rtg_group(chunk->victim_rtg)->xg_active_ref);
839 atomic_inc(&chunk->victim_rtg->rtg_gccount);
840
841 chunk->offset += split_len;
842 chunk->len -= split_len;
843 chunk->old_startblock += XFS_B_TO_FSB(data->mp, split_len);
844
845 /* add right before the original chunk */
846 WRITE_ONCE(split_chunk->state, XFS_GC_BIO_NEW);
847 list_add_tail(&split_chunk->entry, &chunk->entry);
848 return split_chunk;
849 }
850
851 static void
xfs_zone_gc_write_chunk(struct xfs_gc_bio * chunk)852 xfs_zone_gc_write_chunk(
853 struct xfs_gc_bio *chunk)
854 {
855 struct xfs_zone_gc_data *data = chunk->data;
856 struct xfs_mount *mp = chunk->ip->i_mount;
857 struct xfs_gc_bio *split_chunk;
858
859 if (chunk->bio.bi_status)
860 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
861 if (xfs_is_shutdown(mp)) {
862 xfs_zone_gc_free_chunk(chunk);
863 return;
864 }
865
866 XFS_STATS_INC(mp, xs_gc_write_calls);
867 XFS_STATS_ADD(mp, xs_gc_bytes, chunk->len);
868
869 WRITE_ONCE(chunk->state, XFS_GC_BIO_NEW);
870 list_move_tail(&chunk->entry, &data->writing);
871
872 /*
873 * If we run on top of stacked block device, the read I/O might have
874 * reset bi_bdev, restore it to the one we want.
875 */
876 bio_set_dev(&chunk->bio, mp->m_rtdev_targp->bt_bdev);
877 bio_reuse(&chunk->bio, REQ_OP_WRITE);
878 while ((split_chunk = xfs_zone_gc_split_write(data, chunk)))
879 xfs_zone_gc_submit_write(data, split_chunk);
880 xfs_zone_gc_submit_write(data, chunk);
881 }
882
883 static void
xfs_zone_gc_finish_chunk(struct xfs_gc_bio * chunk)884 xfs_zone_gc_finish_chunk(
885 struct xfs_gc_bio *chunk)
886 {
887 uint iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL;
888 struct xfs_zone_gc_data *data = chunk->data;
889 struct xfs_inode *ip = chunk->ip;
890 struct xfs_mount *mp = ip->i_mount;
891 int error;
892
893 if (chunk->bio.bi_status)
894 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
895 if (xfs_is_shutdown(mp)) {
896 xfs_zone_gc_free_chunk(chunk);
897 return;
898 }
899
900 data->scratch_tail =
901 (data->scratch_tail + chunk->len) % data->scratch_size;
902 data->scratch_available += chunk->len;
903
904 /*
905 * Cycle through the iolock and wait for direct I/O and layouts to
906 * ensure no one is reading from the old mapping before it goes away.
907 *
908 * Note that xfs_zoned_end_io() below checks that no other writer raced
909 * with us to update the mapping by checking that the old startblock
910 * didn't change.
911 */
912 xfs_ilock(ip, iolock);
913 error = xfs_break_layouts(VFS_I(ip), &iolock, BREAK_UNMAP);
914 if (!error)
915 inode_dio_wait(VFS_I(ip));
916 xfs_iunlock(ip, iolock);
917 if (error)
918 goto free;
919
920 if (chunk->is_seq)
921 chunk->new_daddr = chunk->bio.bi_iter.bi_sector;
922 error = xfs_zoned_end_io(ip, chunk->offset, chunk->len,
923 chunk->new_daddr, chunk->oz, chunk->old_startblock);
924 free:
925 if (error)
926 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
927 xfs_zone_gc_free_chunk(chunk);
928 }
929
930 static void
xfs_zone_gc_finish_reset(struct xfs_gc_bio * chunk)931 xfs_zone_gc_finish_reset(
932 struct xfs_gc_bio *chunk)
933 {
934 struct xfs_rtgroup *rtg = chunk->bio.bi_private;
935 struct xfs_mount *mp = rtg_mount(rtg);
936 struct xfs_zone_info *zi = mp->m_zone_info;
937
938 if (chunk->bio.bi_status) {
939 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
940 goto out;
941 }
942
943 xfs_zone_mark_free(rtg);
944 xfs_zoned_add_available(mp, rtg_blocks(rtg));
945
946 wake_up_all(&zi->zi_zone_wait);
947 out:
948 list_del(&chunk->entry);
949 bio_put(&chunk->bio);
950 }
951
952 static void
xfs_submit_zone_reset_bio(struct bio * bio,void * priv)953 xfs_submit_zone_reset_bio(
954 struct bio *bio,
955 void *priv)
956 {
957 struct xfs_rtgroup *rtg = priv;
958 struct xfs_mount *mp = rtg_mount(rtg);
959
960 trace_xfs_zone_reset(rtg);
961
962 ASSERT(rtg_rmap(rtg)->i_used_blocks == 0);
963
964 if (XFS_TEST_ERROR(mp, XFS_ERRTAG_ZONE_RESET)) {
965 bio_io_error(bio);
966 return;
967 }
968
969 XFS_STATS_INC(mp, xs_gc_zone_reset_calls);
970
971 bio->bi_iter.bi_sector = xfs_gbno_to_daddr(rtg_group(rtg), 0);
972 if (!bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector)) {
973 /*
974 * Also use the bio to drive the state machine when neither
975 * zone reset nor discard is supported to keep things simple.
976 */
977 if (!bdev_max_discard_sectors(bio->bi_bdev)) {
978 bio_endio(bio);
979 return;
980 }
981 bio->bi_opf &= ~REQ_OP_ZONE_RESET;
982 bio->bi_opf |= REQ_OP_DISCARD;
983 bio->bi_iter.bi_size = XFS_FSB_TO_B(mp, rtg_blocks(rtg));
984 }
985
986 submit_bio(bio);
987 }
988
989 int
xfs_zone_gc_reset_sync(struct xfs_rtgroup * rtg)990 xfs_zone_gc_reset_sync(
991 struct xfs_rtgroup *rtg)
992 {
993 struct bio bio;
994 int error;
995
996 bio_init(&bio, rtg_mount(rtg)->m_rtdev_targp->bt_bdev, NULL, 0,
997 REQ_OP_ZONE_RESET | REQ_SYNC);
998 bio_await(&bio, rtg, xfs_submit_zone_reset_bio);
999 error = blk_status_to_errno(bio.bi_status);
1000 bio_uninit(&bio);
1001 return error;
1002 }
1003
1004 static void
xfs_zone_gc_reset_zones(struct xfs_zone_gc_data * data,struct xfs_group * reset_list)1005 xfs_zone_gc_reset_zones(
1006 struct xfs_zone_gc_data *data,
1007 struct xfs_group *reset_list)
1008 {
1009 struct xfs_group *next = reset_list;
1010
1011 if (blkdev_issue_flush(data->mp->m_rtdev_targp->bt_bdev) < 0) {
1012 xfs_force_shutdown(data->mp, SHUTDOWN_META_IO_ERROR);
1013 return;
1014 }
1015
1016 do {
1017 struct xfs_rtgroup *rtg = to_rtg(next);
1018 struct xfs_gc_bio *chunk;
1019 struct bio *bio;
1020
1021 xfs_log_force_inode(rtg_rmap(rtg));
1022
1023 next = rtg_group(rtg)->xg_next_reset;
1024 rtg_group(rtg)->xg_next_reset = NULL;
1025
1026 bio = bio_alloc_bioset(rtg_mount(rtg)->m_rtdev_targp->bt_bdev,
1027 0, REQ_OP_ZONE_RESET, GFP_NOFS, &data->bio_set);
1028 bio->bi_private = rtg;
1029 bio->bi_end_io = xfs_zone_gc_end_io;
1030
1031 chunk = container_of(bio, struct xfs_gc_bio, bio);
1032 chunk->data = data;
1033 WRITE_ONCE(chunk->state, XFS_GC_BIO_NEW);
1034 list_add_tail(&chunk->entry, &data->resetting);
1035 xfs_submit_zone_reset_bio(bio, rtg);
1036 } while (next);
1037 }
1038
1039 /*
1040 * Handle the work to read and write data for GC and to reset the zones,
1041 * including handling all completions.
1042 *
1043 * Note that the order of the chunks is preserved so that we don't undo the
1044 * optimal order established by xfs_zone_gc_query().
1045 */
1046 static void
xfs_zone_gc_handle_work(struct xfs_zone_gc_data * data)1047 xfs_zone_gc_handle_work(
1048 struct xfs_zone_gc_data *data)
1049 {
1050 struct xfs_zone_info *zi = data->mp->m_zone_info;
1051 struct xfs_gc_bio *chunk, *next;
1052 struct xfs_group *reset_list;
1053 struct blk_plug plug;
1054
1055 spin_lock(&zi->zi_reset_list_lock);
1056 reset_list = zi->zi_reset_list;
1057 zi->zi_reset_list = NULL;
1058 spin_unlock(&zi->zi_reset_list_lock);
1059
1060 if (reset_list) {
1061 set_current_state(TASK_RUNNING);
1062 xfs_zone_gc_reset_zones(data, reset_list);
1063 }
1064
1065 list_for_each_entry_safe(chunk, next, &data->resetting, entry) {
1066 if (READ_ONCE(chunk->state) != XFS_GC_BIO_DONE)
1067 break;
1068 set_current_state(TASK_RUNNING);
1069 xfs_zone_gc_finish_reset(chunk);
1070 }
1071
1072 list_for_each_entry_safe(chunk, next, &data->writing, entry) {
1073 if (READ_ONCE(chunk->state) != XFS_GC_BIO_DONE)
1074 break;
1075 set_current_state(TASK_RUNNING);
1076 xfs_zone_gc_finish_chunk(chunk);
1077 }
1078
1079 blk_start_plug(&plug);
1080 list_for_each_entry_safe(chunk, next, &data->reading, entry) {
1081 if (READ_ONCE(chunk->state) != XFS_GC_BIO_DONE)
1082 break;
1083 set_current_state(TASK_RUNNING);
1084 xfs_zone_gc_write_chunk(chunk);
1085 }
1086 blk_finish_plug(&plug);
1087
1088 blk_start_plug(&plug);
1089 while (xfs_zone_gc_start_chunk(data))
1090 ;
1091 blk_finish_plug(&plug);
1092 }
1093
1094 /*
1095 * Note that the current GC algorithm would break reflinks and thus duplicate
1096 * data that was shared by multiple owners before. Because of that reflinks
1097 * are currently not supported on zoned file systems and can't be created or
1098 * mounted.
1099 */
1100 static int
xfs_zoned_gcd(void * private)1101 xfs_zoned_gcd(
1102 void *private)
1103 {
1104 struct xfs_zone_gc_data *data = private;
1105 struct xfs_mount *mp = data->mp;
1106 struct xfs_zone_info *zi = mp->m_zone_info;
1107 unsigned int nofs_flag;
1108
1109 nofs_flag = memalloc_nofs_save();
1110 set_freezable();
1111
1112 for (;;) {
1113 set_current_state(TASK_INTERRUPTIBLE | TASK_FREEZABLE);
1114 xfs_set_zonegc_running(mp);
1115
1116 xfs_zone_gc_handle_work(data);
1117
1118 /*
1119 * Only sleep if nothing set the state to running. Else check for
1120 * work again as someone might have queued up more work and woken
1121 * us in the meantime.
1122 */
1123 if (get_current_state() == TASK_RUNNING) {
1124 try_to_freeze();
1125 continue;
1126 }
1127
1128 if (list_empty(&data->reading) &&
1129 list_empty(&data->writing) &&
1130 list_empty(&data->resetting) &&
1131 !zi->zi_reset_list) {
1132 xfs_clear_zonegc_running(mp);
1133 xfs_zoned_resv_wake_all(mp);
1134
1135 if (kthread_should_stop()) {
1136 __set_current_state(TASK_RUNNING);
1137 break;
1138 }
1139
1140 if (kthread_should_park()) {
1141 __set_current_state(TASK_RUNNING);
1142 kthread_parkme();
1143 continue;
1144 }
1145 }
1146
1147 schedule();
1148 }
1149 xfs_clear_zonegc_running(mp);
1150
1151 if (data->oz)
1152 xfs_open_zone_put(data->oz);
1153 if (data->iter.victim_rtg)
1154 xfs_rtgroup_rele(data->iter.victim_rtg);
1155
1156 memalloc_nofs_restore(nofs_flag);
1157 xfs_zone_gc_data_free(data);
1158 return 0;
1159 }
1160
1161 void
xfs_zone_gc_start(struct xfs_mount * mp)1162 xfs_zone_gc_start(
1163 struct xfs_mount *mp)
1164 {
1165 if (xfs_has_zoned(mp))
1166 kthread_unpark(mp->m_zone_info->zi_gc_thread);
1167 }
1168
1169 void
xfs_zone_gc_stop(struct xfs_mount * mp)1170 xfs_zone_gc_stop(
1171 struct xfs_mount *mp)
1172 {
1173 if (xfs_has_zoned(mp))
1174 kthread_park(mp->m_zone_info->zi_gc_thread);
1175 }
1176
1177 void
xfs_zone_gc_wakeup(struct xfs_mount * mp)1178 xfs_zone_gc_wakeup(
1179 struct xfs_mount *mp)
1180 {
1181 struct super_block *sb = mp->m_super;
1182
1183 /*
1184 * If we are unmounting the file system we must not try to
1185 * wake gc as m_zone_info might have been freed already.
1186 */
1187 if (down_read_trylock(&sb->s_umount)) {
1188 if (!xfs_is_readonly(mp))
1189 wake_up_process(mp->m_zone_info->zi_gc_thread);
1190 up_read(&sb->s_umount);
1191 }
1192 }
1193
1194 int
xfs_zone_gc_mount(struct xfs_mount * mp)1195 xfs_zone_gc_mount(
1196 struct xfs_mount *mp)
1197 {
1198 struct xfs_zone_info *zi = mp->m_zone_info;
1199 struct xfs_zone_gc_data *data;
1200 int error;
1201
1202 data = xfs_zone_gc_data_alloc(mp);
1203 if (!data)
1204 return -ENOMEM;
1205
1206 /*
1207 * If there are no free zones available for GC, or the number of open
1208 * zones has reached the open zone limit, pick the open zone with
1209 * the least used space to GC into. This should only happen after an
1210 * unclean shutdown while GC was ongoing. Otherwise a GC zone will
1211 * be selected from the free zone pool on demand.
1212 */
1213 if (!xfs_group_marked(mp, XG_TYPE_RTG, XFS_RTG_FREE) ||
1214 zi->zi_nr_open_zones >= mp->m_max_open_zones) {
1215 error = xfs_zone_gc_steal_open_zone(data);
1216 if (error) {
1217 xfs_warn(mp, "unable to steal an open zone for gc");
1218 goto out_free_gc_data;
1219 }
1220 }
1221
1222 zi->zi_gc_thread = kthread_create(xfs_zoned_gcd, data,
1223 "xfs-zone-gc/%s", mp->m_super->s_id);
1224 if (IS_ERR(zi->zi_gc_thread)) {
1225 xfs_warn(mp, "unable to create zone gc thread");
1226 error = PTR_ERR(zi->zi_gc_thread);
1227 goto out_put_oz;
1228 }
1229
1230 /* xfs_zone_gc_start will unpark for rw mounts */
1231 kthread_park(zi->zi_gc_thread);
1232 return 0;
1233
1234 out_put_oz:
1235 if (data->oz)
1236 xfs_open_zone_put(data->oz);
1237 out_free_gc_data:
1238 xfs_zone_gc_data_free(data);
1239 return error;
1240 }
1241
1242 void
xfs_zone_gc_unmount(struct xfs_mount * mp)1243 xfs_zone_gc_unmount(
1244 struct xfs_mount *mp)
1245 {
1246 struct xfs_zone_info *zi = mp->m_zone_info;
1247
1248 kthread_stop(zi->zi_gc_thread);
1249 }
1250