1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2023-2025 Christoph Hellwig.
4 * Copyright (c) 2024-2025, Western Digital Corporation or its affiliates.
5 */
6 #include "xfs.h"
7 #include "xfs_shared.h"
8 #include "xfs_format.h"
9 #include "xfs_log_format.h"
10 #include "xfs_trans_resv.h"
11 #include "xfs_mount.h"
12 #include "xfs_inode.h"
13 #include "xfs_btree.h"
14 #include "xfs_trans.h"
15 #include "xfs_icache.h"
16 #include "xfs_rmap.h"
17 #include "xfs_rtbitmap.h"
18 #include "xfs_rtrmap_btree.h"
19 #include "xfs_zone_alloc.h"
20 #include "xfs_zone_priv.h"
21 #include "xfs_zones.h"
22 #include "xfs_trace.h"
23
24 /*
25 * Implement Garbage Collection (GC) of partially used zoned.
26 *
27 * To support the purely sequential writes in each zone, zoned XFS needs to be
28 * able to move data remaining in a zone out of it to reset the zone to prepare
29 * for writing to it again.
30 *
31 * This is done by the GC thread implemented in this file. To support that a
32 * number of zones (XFS_GC_ZONES) is reserved from the user visible capacity to
33 * write the garbage collected data into.
34 *
35 * Whenever the available space is below the chosen threshold, the GC thread
36 * looks for potential non-empty but not fully used zones that are worth
37 * reclaiming. Once found the rmap for the victim zone is queried, and after
38 * a bit of sorting to reduce fragmentation, the still live extents are read
39 * into memory and written to the GC target zone, and the bmap btree of the
40 * files is updated to point to the new location. To avoid taking the IOLOCK
41 * and MMAPLOCK for the entire GC process and thus affecting the latency of
42 * user reads and writes to the files, the GC writes are speculative and the
43 * I/O completion checks that no other writes happened for the affected regions
44 * before remapping.
45 *
46 * Once a zone does not contain any valid data, be that through GC or user
47 * block removal, it is queued for for a zone reset. The reset operation
48 * carefully ensures that the RT device cache is flushed and all transactions
49 * referencing the rmap have been committed to disk.
50 */
51
52 /*
53 * Size of each GC scratch pad. This is also the upper bound for each
54 * GC I/O, which helps to keep latency down.
55 */
56 #define XFS_GC_CHUNK_SIZE SZ_1M
57
58 /*
59 * Scratchpad data to read GCed data into.
60 *
61 * The offset member tracks where the next allocation starts, and freed tracks
62 * the amount of space that is not used anymore.
63 */
64 #define XFS_ZONE_GC_NR_SCRATCH 2
65 struct xfs_zone_scratch {
66 struct folio *folio;
67 unsigned int offset;
68 unsigned int freed;
69 };
70
71 /*
72 * Chunk that is read and written for each GC operation.
73 *
74 * Note that for writes to actual zoned devices, the chunk can be split when
75 * reaching the hardware limit.
76 */
77 struct xfs_gc_bio {
78 struct xfs_zone_gc_data *data;
79
80 /*
81 * Entry into the reading/writing/resetting list. Only accessed from
82 * the GC thread, so no locking needed.
83 */
84 struct list_head entry;
85
86 /*
87 * State of this gc_bio. Done means the current I/O completed.
88 * Set from the bio end I/O handler, read from the GC thread.
89 */
90 enum {
91 XFS_GC_BIO_NEW,
92 XFS_GC_BIO_DONE,
93 } state;
94
95 /*
96 * Pointer to the inode and byte range in the inode that this
97 * GC chunk is operating on.
98 */
99 struct xfs_inode *ip;
100 loff_t offset;
101 unsigned int len;
102
103 /*
104 * Existing startblock (in the zone to be freed) and newly assigned
105 * daddr in the zone GCed into.
106 */
107 xfs_fsblock_t old_startblock;
108 xfs_daddr_t new_daddr;
109 struct xfs_zone_scratch *scratch;
110
111 /* Are we writing to a sequential write required zone? */
112 bool is_seq;
113
114 /* Open Zone being written to */
115 struct xfs_open_zone *oz;
116
117 /* Bio used for reads and writes, including the bvec used by it */
118 struct bio_vec bv;
119 struct bio bio; /* must be last */
120 };
121
122 #define XFS_ZONE_GC_RECS 1024
123
124 /* iterator, needs to be reinitialized for each victim zone */
125 struct xfs_zone_gc_iter {
126 struct xfs_rtgroup *victim_rtg;
127 unsigned int rec_count;
128 unsigned int rec_idx;
129 xfs_agblock_t next_startblock;
130 struct xfs_rmap_irec *recs;
131 };
132
133 /*
134 * Per-mount GC state.
135 */
136 struct xfs_zone_gc_data {
137 struct xfs_mount *mp;
138
139 /* bioset used to allocate the gc_bios */
140 struct bio_set bio_set;
141
142 /*
143 * Scratchpad used, and index to indicated which one is used.
144 */
145 struct xfs_zone_scratch scratch[XFS_ZONE_GC_NR_SCRATCH];
146 unsigned int scratch_idx;
147
148 /*
149 * List of bios currently being read, written and reset.
150 * These lists are only accessed by the GC thread itself, and must only
151 * be processed in order.
152 */
153 struct list_head reading;
154 struct list_head writing;
155 struct list_head resetting;
156
157 /*
158 * Iterator for the victim zone.
159 */
160 struct xfs_zone_gc_iter iter;
161 };
162
163 /*
164 * We aim to keep enough zones free in stock to fully use the open zone limit
165 * for data placement purposes. Additionally, the m_zonegc_low_space tunable
166 * can be set to make sure a fraction of the unused blocks are available for
167 * writing.
168 */
169 bool
xfs_zoned_need_gc(struct xfs_mount * mp)170 xfs_zoned_need_gc(
171 struct xfs_mount *mp)
172 {
173 s64 available, free, threshold;
174 s32 remainder;
175
176 if (!xfs_group_marked(mp, XG_TYPE_RTG, XFS_RTG_RECLAIMABLE))
177 return false;
178
179 available = xfs_estimate_freecounter(mp, XC_FREE_RTAVAILABLE);
180
181 if (available <
182 mp->m_groups[XG_TYPE_RTG].blocks *
183 (mp->m_max_open_zones - XFS_OPEN_GC_ZONES))
184 return true;
185
186 free = xfs_estimate_freecounter(mp, XC_FREE_RTEXTENTS);
187
188 threshold = div_s64_rem(free, 100, &remainder);
189 threshold = threshold * mp->m_zonegc_low_space +
190 remainder * div_s64(mp->m_zonegc_low_space, 100);
191
192 if (available < threshold)
193 return true;
194
195 return false;
196 }
197
198 static struct xfs_zone_gc_data *
xfs_zone_gc_data_alloc(struct xfs_mount * mp)199 xfs_zone_gc_data_alloc(
200 struct xfs_mount *mp)
201 {
202 struct xfs_zone_gc_data *data;
203 int i;
204
205 data = kzalloc(sizeof(*data), GFP_KERNEL);
206 if (!data)
207 return NULL;
208 data->iter.recs = kcalloc(XFS_ZONE_GC_RECS, sizeof(*data->iter.recs),
209 GFP_KERNEL);
210 if (!data->iter.recs)
211 goto out_free_data;
212
213 /*
214 * We actually only need a single bio_vec. It would be nice to have
215 * a flag that only allocates the inline bvecs and not the separate
216 * bvec pool.
217 */
218 if (bioset_init(&data->bio_set, 16, offsetof(struct xfs_gc_bio, bio),
219 BIOSET_NEED_BVECS))
220 goto out_free_recs;
221 for (i = 0; i < XFS_ZONE_GC_NR_SCRATCH; i++) {
222 data->scratch[i].folio =
223 folio_alloc(GFP_KERNEL, get_order(XFS_GC_CHUNK_SIZE));
224 if (!data->scratch[i].folio)
225 goto out_free_scratch;
226 }
227 INIT_LIST_HEAD(&data->reading);
228 INIT_LIST_HEAD(&data->writing);
229 INIT_LIST_HEAD(&data->resetting);
230 data->mp = mp;
231 return data;
232
233 out_free_scratch:
234 while (--i >= 0)
235 folio_put(data->scratch[i].folio);
236 bioset_exit(&data->bio_set);
237 out_free_recs:
238 kfree(data->iter.recs);
239 out_free_data:
240 kfree(data);
241 return NULL;
242 }
243
244 static void
xfs_zone_gc_data_free(struct xfs_zone_gc_data * data)245 xfs_zone_gc_data_free(
246 struct xfs_zone_gc_data *data)
247 {
248 int i;
249
250 for (i = 0; i < XFS_ZONE_GC_NR_SCRATCH; i++)
251 folio_put(data->scratch[i].folio);
252 bioset_exit(&data->bio_set);
253 kfree(data->iter.recs);
254 kfree(data);
255 }
256
257 static void
xfs_zone_gc_iter_init(struct xfs_zone_gc_iter * iter,struct xfs_rtgroup * victim_rtg)258 xfs_zone_gc_iter_init(
259 struct xfs_zone_gc_iter *iter,
260 struct xfs_rtgroup *victim_rtg)
261
262 {
263 iter->next_startblock = 0;
264 iter->rec_count = 0;
265 iter->rec_idx = 0;
266 iter->victim_rtg = victim_rtg;
267 }
268
269 /*
270 * Query the rmap of the victim zone to gather the records to evacuate.
271 */
272 static int
xfs_zone_gc_query_cb(struct xfs_btree_cur * cur,const struct xfs_rmap_irec * irec,void * private)273 xfs_zone_gc_query_cb(
274 struct xfs_btree_cur *cur,
275 const struct xfs_rmap_irec *irec,
276 void *private)
277 {
278 struct xfs_zone_gc_iter *iter = private;
279
280 ASSERT(!XFS_RMAP_NON_INODE_OWNER(irec->rm_owner));
281 ASSERT(!xfs_is_sb_inum(cur->bc_mp, irec->rm_owner));
282 ASSERT(!(irec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK)));
283
284 iter->recs[iter->rec_count] = *irec;
285 if (++iter->rec_count == XFS_ZONE_GC_RECS) {
286 iter->next_startblock =
287 irec->rm_startblock + irec->rm_blockcount;
288 return 1;
289 }
290 return 0;
291 }
292
293 static int
xfs_zone_gc_rmap_rec_cmp(const void * a,const void * b)294 xfs_zone_gc_rmap_rec_cmp(
295 const void *a,
296 const void *b)
297 {
298 const struct xfs_rmap_irec *reca = a;
299 const struct xfs_rmap_irec *recb = b;
300 int diff;
301
302 diff = cmp_int(reca->rm_owner, recb->rm_owner);
303 if (diff)
304 return diff;
305 return cmp_int(reca->rm_offset, recb->rm_offset);
306 }
307
308 static int
xfs_zone_gc_query(struct xfs_mount * mp,struct xfs_zone_gc_iter * iter)309 xfs_zone_gc_query(
310 struct xfs_mount *mp,
311 struct xfs_zone_gc_iter *iter)
312 {
313 struct xfs_rtgroup *rtg = iter->victim_rtg;
314 struct xfs_rmap_irec ri_low = { };
315 struct xfs_rmap_irec ri_high;
316 struct xfs_btree_cur *cur;
317 struct xfs_trans *tp;
318 int error;
319
320 ASSERT(iter->next_startblock <= rtg_blocks(rtg));
321 if (iter->next_startblock == rtg_blocks(rtg))
322 goto done;
323
324 ASSERT(iter->next_startblock < rtg_blocks(rtg));
325 ri_low.rm_startblock = iter->next_startblock;
326 memset(&ri_high, 0xFF, sizeof(ri_high));
327
328 iter->rec_idx = 0;
329 iter->rec_count = 0;
330
331 error = xfs_trans_alloc_empty(mp, &tp);
332 if (error)
333 return error;
334
335 xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP);
336 cur = xfs_rtrmapbt_init_cursor(tp, rtg);
337 error = xfs_rmap_query_range(cur, &ri_low, &ri_high,
338 xfs_zone_gc_query_cb, iter);
339 xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP);
340 xfs_btree_del_cursor(cur, error < 0 ? error : 0);
341 xfs_trans_cancel(tp);
342
343 if (error < 0)
344 return error;
345
346 /*
347 * Sort the rmap records by inode number and increasing offset to
348 * defragment the mappings.
349 *
350 * This could be further enhanced by an even bigger look ahead window,
351 * but that's better left until we have better detection of changes to
352 * inode mapping to avoid the potential of GCing already dead data.
353 */
354 sort(iter->recs, iter->rec_count, sizeof(iter->recs[0]),
355 xfs_zone_gc_rmap_rec_cmp, NULL);
356
357 if (error == 0) {
358 /*
359 * We finished iterating through the zone.
360 */
361 iter->next_startblock = rtg_blocks(rtg);
362 if (iter->rec_count == 0)
363 goto done;
364 }
365
366 return 0;
367 done:
368 xfs_rtgroup_rele(iter->victim_rtg);
369 iter->victim_rtg = NULL;
370 return 0;
371 }
372
373 static bool
xfs_zone_gc_iter_next(struct xfs_mount * mp,struct xfs_zone_gc_iter * iter,struct xfs_rmap_irec * chunk_rec,struct xfs_inode ** ipp)374 xfs_zone_gc_iter_next(
375 struct xfs_mount *mp,
376 struct xfs_zone_gc_iter *iter,
377 struct xfs_rmap_irec *chunk_rec,
378 struct xfs_inode **ipp)
379 {
380 struct xfs_rmap_irec *irec;
381 int error;
382
383 if (!iter->victim_rtg)
384 return false;
385
386 retry:
387 if (iter->rec_idx == iter->rec_count) {
388 error = xfs_zone_gc_query(mp, iter);
389 if (error)
390 goto fail;
391 if (!iter->victim_rtg)
392 return false;
393 }
394
395 irec = &iter->recs[iter->rec_idx];
396 error = xfs_iget(mp, NULL, irec->rm_owner,
397 XFS_IGET_UNTRUSTED | XFS_IGET_DONTCACHE, 0, ipp);
398 if (error) {
399 /*
400 * If the inode was already deleted, skip over it.
401 */
402 if (error == -ENOENT) {
403 iter->rec_idx++;
404 goto retry;
405 }
406 goto fail;
407 }
408
409 if (!S_ISREG(VFS_I(*ipp)->i_mode) || !XFS_IS_REALTIME_INODE(*ipp)) {
410 iter->rec_idx++;
411 xfs_irele(*ipp);
412 goto retry;
413 }
414
415 *chunk_rec = *irec;
416 return true;
417
418 fail:
419 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
420 return false;
421 }
422
423 static void
xfs_zone_gc_iter_advance(struct xfs_zone_gc_iter * iter,xfs_extlen_t count_fsb)424 xfs_zone_gc_iter_advance(
425 struct xfs_zone_gc_iter *iter,
426 xfs_extlen_t count_fsb)
427 {
428 struct xfs_rmap_irec *irec = &iter->recs[iter->rec_idx];
429
430 irec->rm_offset += count_fsb;
431 irec->rm_startblock += count_fsb;
432 irec->rm_blockcount -= count_fsb;
433 if (!irec->rm_blockcount)
434 iter->rec_idx++;
435 }
436
437 static struct xfs_rtgroup *
xfs_zone_gc_pick_victim_from(struct xfs_mount * mp,uint32_t bucket)438 xfs_zone_gc_pick_victim_from(
439 struct xfs_mount *mp,
440 uint32_t bucket)
441 {
442 struct xfs_zone_info *zi = mp->m_zone_info;
443 uint32_t victim_used = U32_MAX;
444 struct xfs_rtgroup *victim_rtg = NULL;
445 uint32_t bit;
446
447 if (!zi->zi_used_bucket_entries[bucket])
448 return NULL;
449
450 for_each_set_bit(bit, zi->zi_used_bucket_bitmap[bucket],
451 mp->m_sb.sb_rgcount) {
452 struct xfs_rtgroup *rtg = xfs_rtgroup_grab(mp, bit);
453
454 if (!rtg)
455 continue;
456
457 /* skip zones that are just waiting for a reset */
458 if (rtg_rmap(rtg)->i_used_blocks == 0 ||
459 rtg_rmap(rtg)->i_used_blocks >= victim_used) {
460 xfs_rtgroup_rele(rtg);
461 continue;
462 }
463
464 if (victim_rtg)
465 xfs_rtgroup_rele(victim_rtg);
466 victim_rtg = rtg;
467 victim_used = rtg_rmap(rtg)->i_used_blocks;
468
469 /*
470 * Any zone that is less than 1 percent used is fair game for
471 * instant reclaim. All of these zones are in the last
472 * bucket, so avoid the expensive division for the zones
473 * in the other buckets.
474 */
475 if (bucket == 0 &&
476 rtg_rmap(rtg)->i_used_blocks < rtg_blocks(rtg) / 100)
477 break;
478 }
479
480 return victim_rtg;
481 }
482
483 /*
484 * Iterate through all zones marked as reclaimable and find a candidate to
485 * reclaim.
486 */
487 static bool
xfs_zone_gc_select_victim(struct xfs_zone_gc_data * data)488 xfs_zone_gc_select_victim(
489 struct xfs_zone_gc_data *data)
490 {
491 struct xfs_zone_gc_iter *iter = &data->iter;
492 struct xfs_mount *mp = data->mp;
493 struct xfs_zone_info *zi = mp->m_zone_info;
494 struct xfs_rtgroup *victim_rtg = NULL;
495 unsigned int bucket;
496
497 if (xfs_is_shutdown(mp))
498 return false;
499
500 if (iter->victim_rtg)
501 return true;
502
503 /*
504 * Don't start new work if we are asked to stop or park.
505 */
506 if (kthread_should_stop() || kthread_should_park())
507 return false;
508
509 if (!xfs_zoned_need_gc(mp))
510 return false;
511
512 spin_lock(&zi->zi_used_buckets_lock);
513 for (bucket = 0; bucket < XFS_ZONE_USED_BUCKETS; bucket++) {
514 victim_rtg = xfs_zone_gc_pick_victim_from(mp, bucket);
515 if (victim_rtg)
516 break;
517 }
518 spin_unlock(&zi->zi_used_buckets_lock);
519
520 if (!victim_rtg)
521 return false;
522
523 trace_xfs_zone_gc_select_victim(victim_rtg, bucket);
524 xfs_zone_gc_iter_init(iter, victim_rtg);
525 return true;
526 }
527
528 static struct xfs_open_zone *
xfs_zone_gc_steal_open(struct xfs_zone_info * zi)529 xfs_zone_gc_steal_open(
530 struct xfs_zone_info *zi)
531 {
532 struct xfs_open_zone *oz, *found = NULL;
533
534 spin_lock(&zi->zi_open_zones_lock);
535 list_for_each_entry(oz, &zi->zi_open_zones, oz_entry) {
536 if (!found ||
537 oz->oz_write_pointer < found->oz_write_pointer)
538 found = oz;
539 }
540
541 if (found) {
542 found->oz_is_gc = true;
543 list_del_init(&found->oz_entry);
544 zi->zi_nr_open_zones--;
545 }
546
547 spin_unlock(&zi->zi_open_zones_lock);
548 return found;
549 }
550
551 static struct xfs_open_zone *
xfs_zone_gc_select_target(struct xfs_mount * mp)552 xfs_zone_gc_select_target(
553 struct xfs_mount *mp)
554 {
555 struct xfs_zone_info *zi = mp->m_zone_info;
556 struct xfs_open_zone *oz = zi->zi_open_gc_zone;
557
558 /*
559 * We need to wait for pending writes to finish.
560 */
561 if (oz && oz->oz_written < rtg_blocks(oz->oz_rtg))
562 return NULL;
563
564 ASSERT(zi->zi_nr_open_zones <=
565 mp->m_max_open_zones - XFS_OPEN_GC_ZONES);
566 oz = xfs_open_zone(mp, WRITE_LIFE_NOT_SET, true);
567 if (oz)
568 trace_xfs_zone_gc_target_opened(oz->oz_rtg);
569 spin_lock(&zi->zi_open_zones_lock);
570 zi->zi_open_gc_zone = oz;
571 spin_unlock(&zi->zi_open_zones_lock);
572 return oz;
573 }
574
575 /*
576 * Ensure we have a valid open zone to write the GC data to.
577 *
578 * If the current target zone has space keep writing to it, else first wait for
579 * all pending writes and then pick a new one.
580 */
581 static struct xfs_open_zone *
xfs_zone_gc_ensure_target(struct xfs_mount * mp)582 xfs_zone_gc_ensure_target(
583 struct xfs_mount *mp)
584 {
585 struct xfs_open_zone *oz = mp->m_zone_info->zi_open_gc_zone;
586
587 if (!oz || oz->oz_write_pointer == rtg_blocks(oz->oz_rtg))
588 return xfs_zone_gc_select_target(mp);
589 return oz;
590 }
591
592 static unsigned int
xfs_zone_gc_scratch_available(struct xfs_zone_gc_data * data)593 xfs_zone_gc_scratch_available(
594 struct xfs_zone_gc_data *data)
595 {
596 return XFS_GC_CHUNK_SIZE - data->scratch[data->scratch_idx].offset;
597 }
598
599 static bool
xfs_zone_gc_space_available(struct xfs_zone_gc_data * data)600 xfs_zone_gc_space_available(
601 struct xfs_zone_gc_data *data)
602 {
603 struct xfs_open_zone *oz;
604
605 oz = xfs_zone_gc_ensure_target(data->mp);
606 if (!oz)
607 return false;
608 return oz->oz_write_pointer < rtg_blocks(oz->oz_rtg) &&
609 xfs_zone_gc_scratch_available(data);
610 }
611
612 static void
xfs_zone_gc_end_io(struct bio * bio)613 xfs_zone_gc_end_io(
614 struct bio *bio)
615 {
616 struct xfs_gc_bio *chunk =
617 container_of(bio, struct xfs_gc_bio, bio);
618 struct xfs_zone_gc_data *data = chunk->data;
619
620 WRITE_ONCE(chunk->state, XFS_GC_BIO_DONE);
621 wake_up_process(data->mp->m_zone_info->zi_gc_thread);
622 }
623
624 static struct xfs_open_zone *
xfs_zone_gc_alloc_blocks(struct xfs_zone_gc_data * data,xfs_extlen_t * count_fsb,xfs_daddr_t * daddr,bool * is_seq)625 xfs_zone_gc_alloc_blocks(
626 struct xfs_zone_gc_data *data,
627 xfs_extlen_t *count_fsb,
628 xfs_daddr_t *daddr,
629 bool *is_seq)
630 {
631 struct xfs_mount *mp = data->mp;
632 struct xfs_open_zone *oz;
633
634 oz = xfs_zone_gc_ensure_target(mp);
635 if (!oz)
636 return NULL;
637
638 *count_fsb = min(*count_fsb,
639 XFS_B_TO_FSB(mp, xfs_zone_gc_scratch_available(data)));
640
641 /*
642 * Directly allocate GC blocks from the reserved pool.
643 *
644 * If we'd take them from the normal pool we could be stealing blocks
645 * from a regular writer, which would then have to wait for GC and
646 * deadlock.
647 */
648 spin_lock(&mp->m_sb_lock);
649 *count_fsb = min(*count_fsb,
650 rtg_blocks(oz->oz_rtg) - oz->oz_write_pointer);
651 *count_fsb = min3(*count_fsb,
652 mp->m_free[XC_FREE_RTEXTENTS].res_avail,
653 mp->m_free[XC_FREE_RTAVAILABLE].res_avail);
654 mp->m_free[XC_FREE_RTEXTENTS].res_avail -= *count_fsb;
655 mp->m_free[XC_FREE_RTAVAILABLE].res_avail -= *count_fsb;
656 spin_unlock(&mp->m_sb_lock);
657
658 if (!*count_fsb)
659 return NULL;
660
661 *daddr = xfs_gbno_to_daddr(&oz->oz_rtg->rtg_group, 0);
662 *is_seq = bdev_zone_is_seq(mp->m_rtdev_targp->bt_bdev, *daddr);
663 if (!*is_seq)
664 *daddr += XFS_FSB_TO_BB(mp, oz->oz_write_pointer);
665 oz->oz_write_pointer += *count_fsb;
666 atomic_inc(&oz->oz_ref);
667 return oz;
668 }
669
670 static bool
xfs_zone_gc_start_chunk(struct xfs_zone_gc_data * data)671 xfs_zone_gc_start_chunk(
672 struct xfs_zone_gc_data *data)
673 {
674 struct xfs_zone_gc_iter *iter = &data->iter;
675 struct xfs_mount *mp = data->mp;
676 struct block_device *bdev = mp->m_rtdev_targp->bt_bdev;
677 struct xfs_open_zone *oz;
678 struct xfs_rmap_irec irec;
679 struct xfs_gc_bio *chunk;
680 struct xfs_inode *ip;
681 struct bio *bio;
682 xfs_daddr_t daddr;
683 bool is_seq;
684
685 if (xfs_is_shutdown(mp))
686 return false;
687
688 if (!xfs_zone_gc_iter_next(mp, iter, &irec, &ip))
689 return false;
690 oz = xfs_zone_gc_alloc_blocks(data, &irec.rm_blockcount, &daddr,
691 &is_seq);
692 if (!oz) {
693 xfs_irele(ip);
694 return false;
695 }
696
697 bio = bio_alloc_bioset(bdev, 1, REQ_OP_READ, GFP_NOFS, &data->bio_set);
698
699 chunk = container_of(bio, struct xfs_gc_bio, bio);
700 chunk->ip = ip;
701 chunk->offset = XFS_FSB_TO_B(mp, irec.rm_offset);
702 chunk->len = XFS_FSB_TO_B(mp, irec.rm_blockcount);
703 chunk->old_startblock =
704 xfs_rgbno_to_rtb(iter->victim_rtg, irec.rm_startblock);
705 chunk->new_daddr = daddr;
706 chunk->is_seq = is_seq;
707 chunk->scratch = &data->scratch[data->scratch_idx];
708 chunk->data = data;
709 chunk->oz = oz;
710
711 bio->bi_iter.bi_sector = xfs_rtb_to_daddr(mp, chunk->old_startblock);
712 bio->bi_end_io = xfs_zone_gc_end_io;
713 bio_add_folio_nofail(bio, chunk->scratch->folio, chunk->len,
714 chunk->scratch->offset);
715 chunk->scratch->offset += chunk->len;
716 if (chunk->scratch->offset == XFS_GC_CHUNK_SIZE) {
717 data->scratch_idx =
718 (data->scratch_idx + 1) % XFS_ZONE_GC_NR_SCRATCH;
719 }
720 WRITE_ONCE(chunk->state, XFS_GC_BIO_NEW);
721 list_add_tail(&chunk->entry, &data->reading);
722 xfs_zone_gc_iter_advance(iter, irec.rm_blockcount);
723
724 submit_bio(bio);
725 return true;
726 }
727
728 static void
xfs_zone_gc_free_chunk(struct xfs_gc_bio * chunk)729 xfs_zone_gc_free_chunk(
730 struct xfs_gc_bio *chunk)
731 {
732 list_del(&chunk->entry);
733 xfs_open_zone_put(chunk->oz);
734 xfs_irele(chunk->ip);
735 bio_put(&chunk->bio);
736 }
737
738 static void
xfs_zone_gc_submit_write(struct xfs_zone_gc_data * data,struct xfs_gc_bio * chunk)739 xfs_zone_gc_submit_write(
740 struct xfs_zone_gc_data *data,
741 struct xfs_gc_bio *chunk)
742 {
743 if (chunk->is_seq) {
744 chunk->bio.bi_opf &= ~REQ_OP_WRITE;
745 chunk->bio.bi_opf |= REQ_OP_ZONE_APPEND;
746 }
747 chunk->bio.bi_iter.bi_sector = chunk->new_daddr;
748 chunk->bio.bi_end_io = xfs_zone_gc_end_io;
749 submit_bio(&chunk->bio);
750 }
751
752 static struct xfs_gc_bio *
xfs_zone_gc_split_write(struct xfs_zone_gc_data * data,struct xfs_gc_bio * chunk)753 xfs_zone_gc_split_write(
754 struct xfs_zone_gc_data *data,
755 struct xfs_gc_bio *chunk)
756 {
757 struct queue_limits *lim =
758 &bdev_get_queue(chunk->bio.bi_bdev)->limits;
759 struct xfs_gc_bio *split_chunk;
760 int split_sectors;
761 unsigned int split_len;
762 struct bio *split;
763 unsigned int nsegs;
764
765 if (!chunk->is_seq)
766 return NULL;
767
768 split_sectors = bio_split_rw_at(&chunk->bio, lim, &nsegs,
769 lim->max_zone_append_sectors << SECTOR_SHIFT);
770 if (!split_sectors)
771 return NULL;
772
773 /* ensure the split chunk is still block size aligned */
774 split_sectors = ALIGN_DOWN(split_sectors << SECTOR_SHIFT,
775 data->mp->m_sb.sb_blocksize) >> SECTOR_SHIFT;
776 split_len = split_sectors << SECTOR_SHIFT;
777
778 split = bio_split(&chunk->bio, split_sectors, GFP_NOFS, &data->bio_set);
779 split_chunk = container_of(split, struct xfs_gc_bio, bio);
780 split_chunk->data = data;
781 ihold(VFS_I(chunk->ip));
782 split_chunk->ip = chunk->ip;
783 split_chunk->is_seq = chunk->is_seq;
784 split_chunk->scratch = chunk->scratch;
785 split_chunk->offset = chunk->offset;
786 split_chunk->len = split_len;
787 split_chunk->old_startblock = chunk->old_startblock;
788 split_chunk->new_daddr = chunk->new_daddr;
789 split_chunk->oz = chunk->oz;
790 atomic_inc(&chunk->oz->oz_ref);
791
792 chunk->offset += split_len;
793 chunk->len -= split_len;
794 chunk->old_startblock += XFS_B_TO_FSB(data->mp, split_len);
795
796 /* add right before the original chunk */
797 WRITE_ONCE(split_chunk->state, XFS_GC_BIO_NEW);
798 list_add_tail(&split_chunk->entry, &chunk->entry);
799 return split_chunk;
800 }
801
802 static void
xfs_zone_gc_write_chunk(struct xfs_gc_bio * chunk)803 xfs_zone_gc_write_chunk(
804 struct xfs_gc_bio *chunk)
805 {
806 struct xfs_zone_gc_data *data = chunk->data;
807 struct xfs_mount *mp = chunk->ip->i_mount;
808 phys_addr_t bvec_paddr =
809 bvec_phys(bio_first_bvec_all(&chunk->bio));
810 struct xfs_gc_bio *split_chunk;
811
812 if (chunk->bio.bi_status)
813 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
814 if (xfs_is_shutdown(mp)) {
815 xfs_zone_gc_free_chunk(chunk);
816 return;
817 }
818
819 WRITE_ONCE(chunk->state, XFS_GC_BIO_NEW);
820 list_move_tail(&chunk->entry, &data->writing);
821
822 bio_reset(&chunk->bio, mp->m_rtdev_targp->bt_bdev, REQ_OP_WRITE);
823 bio_add_folio_nofail(&chunk->bio, chunk->scratch->folio, chunk->len,
824 offset_in_folio(chunk->scratch->folio, bvec_paddr));
825
826 while ((split_chunk = xfs_zone_gc_split_write(data, chunk)))
827 xfs_zone_gc_submit_write(data, split_chunk);
828 xfs_zone_gc_submit_write(data, chunk);
829 }
830
831 static void
xfs_zone_gc_finish_chunk(struct xfs_gc_bio * chunk)832 xfs_zone_gc_finish_chunk(
833 struct xfs_gc_bio *chunk)
834 {
835 uint iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL;
836 struct xfs_inode *ip = chunk->ip;
837 struct xfs_mount *mp = ip->i_mount;
838 int error;
839
840 if (chunk->bio.bi_status)
841 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
842 if (xfs_is_shutdown(mp)) {
843 xfs_zone_gc_free_chunk(chunk);
844 return;
845 }
846
847 chunk->scratch->freed += chunk->len;
848 if (chunk->scratch->freed == chunk->scratch->offset) {
849 chunk->scratch->offset = 0;
850 chunk->scratch->freed = 0;
851 }
852
853 /*
854 * Cycle through the iolock and wait for direct I/O and layouts to
855 * ensure no one is reading from the old mapping before it goes away.
856 *
857 * Note that xfs_zoned_end_io() below checks that no other writer raced
858 * with us to update the mapping by checking that the old startblock
859 * didn't change.
860 */
861 xfs_ilock(ip, iolock);
862 error = xfs_break_layouts(VFS_I(ip), &iolock, BREAK_UNMAP);
863 if (!error)
864 inode_dio_wait(VFS_I(ip));
865 xfs_iunlock(ip, iolock);
866 if (error)
867 goto free;
868
869 if (chunk->is_seq)
870 chunk->new_daddr = chunk->bio.bi_iter.bi_sector;
871 error = xfs_zoned_end_io(ip, chunk->offset, chunk->len,
872 chunk->new_daddr, chunk->oz, chunk->old_startblock);
873 free:
874 if (error)
875 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
876 xfs_zone_gc_free_chunk(chunk);
877 }
878
879 static void
xfs_zone_gc_finish_reset(struct xfs_gc_bio * chunk)880 xfs_zone_gc_finish_reset(
881 struct xfs_gc_bio *chunk)
882 {
883 struct xfs_rtgroup *rtg = chunk->bio.bi_private;
884 struct xfs_mount *mp = rtg_mount(rtg);
885 struct xfs_zone_info *zi = mp->m_zone_info;
886
887 if (chunk->bio.bi_status) {
888 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
889 goto out;
890 }
891
892 xfs_group_set_mark(&rtg->rtg_group, XFS_RTG_FREE);
893 atomic_inc(&zi->zi_nr_free_zones);
894
895 xfs_zoned_add_available(mp, rtg_blocks(rtg));
896
897 wake_up_all(&zi->zi_zone_wait);
898 out:
899 list_del(&chunk->entry);
900 bio_put(&chunk->bio);
901 }
902
903 static bool
xfs_zone_gc_prepare_reset(struct bio * bio,struct xfs_rtgroup * rtg)904 xfs_zone_gc_prepare_reset(
905 struct bio *bio,
906 struct xfs_rtgroup *rtg)
907 {
908 trace_xfs_zone_reset(rtg);
909
910 ASSERT(rtg_rmap(rtg)->i_used_blocks == 0);
911 bio->bi_iter.bi_sector = xfs_gbno_to_daddr(&rtg->rtg_group, 0);
912 if (!bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector)) {
913 if (!bdev_max_discard_sectors(bio->bi_bdev))
914 return false;
915 bio->bi_opf = REQ_OP_DISCARD | REQ_SYNC;
916 bio->bi_iter.bi_size =
917 XFS_FSB_TO_B(rtg_mount(rtg), rtg_blocks(rtg));
918 }
919
920 return true;
921 }
922
923 int
xfs_zone_gc_reset_sync(struct xfs_rtgroup * rtg)924 xfs_zone_gc_reset_sync(
925 struct xfs_rtgroup *rtg)
926 {
927 int error = 0;
928 struct bio bio;
929
930 bio_init(&bio, rtg_mount(rtg)->m_rtdev_targp->bt_bdev, NULL, 0,
931 REQ_OP_ZONE_RESET);
932 if (xfs_zone_gc_prepare_reset(&bio, rtg))
933 error = submit_bio_wait(&bio);
934 bio_uninit(&bio);
935
936 return error;
937 }
938
939 static void
xfs_zone_gc_reset_zones(struct xfs_zone_gc_data * data,struct xfs_group * reset_list)940 xfs_zone_gc_reset_zones(
941 struct xfs_zone_gc_data *data,
942 struct xfs_group *reset_list)
943 {
944 struct xfs_group *next = reset_list;
945
946 if (blkdev_issue_flush(data->mp->m_rtdev_targp->bt_bdev) < 0) {
947 xfs_force_shutdown(data->mp, SHUTDOWN_META_IO_ERROR);
948 return;
949 }
950
951 do {
952 struct xfs_rtgroup *rtg = to_rtg(next);
953 struct xfs_gc_bio *chunk;
954 struct bio *bio;
955
956 xfs_log_force_inode(rtg_rmap(rtg));
957
958 next = rtg_group(rtg)->xg_next_reset;
959 rtg_group(rtg)->xg_next_reset = NULL;
960
961 bio = bio_alloc_bioset(rtg_mount(rtg)->m_rtdev_targp->bt_bdev,
962 0, REQ_OP_ZONE_RESET, GFP_NOFS, &data->bio_set);
963 bio->bi_private = rtg;
964 bio->bi_end_io = xfs_zone_gc_end_io;
965
966 chunk = container_of(bio, struct xfs_gc_bio, bio);
967 chunk->data = data;
968 WRITE_ONCE(chunk->state, XFS_GC_BIO_NEW);
969 list_add_tail(&chunk->entry, &data->resetting);
970
971 /*
972 * Also use the bio to drive the state machine when neither
973 * zone reset nor discard is supported to keep things simple.
974 */
975 if (xfs_zone_gc_prepare_reset(bio, rtg))
976 submit_bio(bio);
977 else
978 bio_endio(bio);
979 } while (next);
980 }
981
982 /*
983 * Handle the work to read and write data for GC and to reset the zones,
984 * including handling all completions.
985 *
986 * Note that the order of the chunks is preserved so that we don't undo the
987 * optimal order established by xfs_zone_gc_query().
988 */
989 static bool
xfs_zone_gc_handle_work(struct xfs_zone_gc_data * data)990 xfs_zone_gc_handle_work(
991 struct xfs_zone_gc_data *data)
992 {
993 struct xfs_zone_info *zi = data->mp->m_zone_info;
994 struct xfs_gc_bio *chunk, *next;
995 struct xfs_group *reset_list;
996 struct blk_plug plug;
997
998 spin_lock(&zi->zi_reset_list_lock);
999 reset_list = zi->zi_reset_list;
1000 zi->zi_reset_list = NULL;
1001 spin_unlock(&zi->zi_reset_list_lock);
1002
1003 if (!xfs_zone_gc_select_victim(data) ||
1004 !xfs_zone_gc_space_available(data)) {
1005 if (list_empty(&data->reading) &&
1006 list_empty(&data->writing) &&
1007 list_empty(&data->resetting) &&
1008 !reset_list)
1009 return false;
1010 }
1011
1012 __set_current_state(TASK_RUNNING);
1013 try_to_freeze();
1014
1015 if (reset_list)
1016 xfs_zone_gc_reset_zones(data, reset_list);
1017
1018 list_for_each_entry_safe(chunk, next, &data->resetting, entry) {
1019 if (READ_ONCE(chunk->state) != XFS_GC_BIO_DONE)
1020 break;
1021 xfs_zone_gc_finish_reset(chunk);
1022 }
1023
1024 list_for_each_entry_safe(chunk, next, &data->writing, entry) {
1025 if (READ_ONCE(chunk->state) != XFS_GC_BIO_DONE)
1026 break;
1027 xfs_zone_gc_finish_chunk(chunk);
1028 }
1029
1030 blk_start_plug(&plug);
1031 list_for_each_entry_safe(chunk, next, &data->reading, entry) {
1032 if (READ_ONCE(chunk->state) != XFS_GC_BIO_DONE)
1033 break;
1034 xfs_zone_gc_write_chunk(chunk);
1035 }
1036 blk_finish_plug(&plug);
1037
1038 blk_start_plug(&plug);
1039 while (xfs_zone_gc_start_chunk(data))
1040 ;
1041 blk_finish_plug(&plug);
1042 return true;
1043 }
1044
1045 /*
1046 * Note that the current GC algorithm would break reflinks and thus duplicate
1047 * data that was shared by multiple owners before. Because of that reflinks
1048 * are currently not supported on zoned file systems and can't be created or
1049 * mounted.
1050 */
1051 static int
xfs_zoned_gcd(void * private)1052 xfs_zoned_gcd(
1053 void *private)
1054 {
1055 struct xfs_zone_gc_data *data = private;
1056 struct xfs_mount *mp = data->mp;
1057 struct xfs_zone_info *zi = mp->m_zone_info;
1058 unsigned int nofs_flag;
1059
1060 nofs_flag = memalloc_nofs_save();
1061 set_freezable();
1062
1063 for (;;) {
1064 set_current_state(TASK_INTERRUPTIBLE | TASK_FREEZABLE);
1065 xfs_set_zonegc_running(mp);
1066 if (xfs_zone_gc_handle_work(data))
1067 continue;
1068
1069 if (list_empty(&data->reading) &&
1070 list_empty(&data->writing) &&
1071 list_empty(&data->resetting) &&
1072 !zi->zi_reset_list) {
1073 xfs_clear_zonegc_running(mp);
1074 xfs_zoned_resv_wake_all(mp);
1075
1076 if (kthread_should_stop()) {
1077 __set_current_state(TASK_RUNNING);
1078 break;
1079 }
1080
1081 if (kthread_should_park()) {
1082 __set_current_state(TASK_RUNNING);
1083 kthread_parkme();
1084 continue;
1085 }
1086 }
1087
1088 schedule();
1089 }
1090 xfs_clear_zonegc_running(mp);
1091
1092 if (data->iter.victim_rtg)
1093 xfs_rtgroup_rele(data->iter.victim_rtg);
1094
1095 memalloc_nofs_restore(nofs_flag);
1096 xfs_zone_gc_data_free(data);
1097 return 0;
1098 }
1099
1100 void
xfs_zone_gc_start(struct xfs_mount * mp)1101 xfs_zone_gc_start(
1102 struct xfs_mount *mp)
1103 {
1104 if (xfs_has_zoned(mp))
1105 kthread_unpark(mp->m_zone_info->zi_gc_thread);
1106 }
1107
1108 void
xfs_zone_gc_stop(struct xfs_mount * mp)1109 xfs_zone_gc_stop(
1110 struct xfs_mount *mp)
1111 {
1112 if (xfs_has_zoned(mp))
1113 kthread_park(mp->m_zone_info->zi_gc_thread);
1114 }
1115
1116 int
xfs_zone_gc_mount(struct xfs_mount * mp)1117 xfs_zone_gc_mount(
1118 struct xfs_mount *mp)
1119 {
1120 struct xfs_zone_info *zi = mp->m_zone_info;
1121 struct xfs_zone_gc_data *data;
1122 struct xfs_open_zone *oz;
1123 int error;
1124
1125 /*
1126 * If there are no free zones available for GC, pick the open zone with
1127 * the least used space to GC into. This should only happen after an
1128 * unclean shutdown near ENOSPC while GC was ongoing.
1129 *
1130 * We also need to do this for the first gc zone allocation if we
1131 * unmounted while at the open limit.
1132 */
1133 if (!xfs_group_marked(mp, XG_TYPE_RTG, XFS_RTG_FREE) ||
1134 zi->zi_nr_open_zones == mp->m_max_open_zones)
1135 oz = xfs_zone_gc_steal_open(zi);
1136 else
1137 oz = xfs_open_zone(mp, WRITE_LIFE_NOT_SET, true);
1138 if (!oz) {
1139 xfs_warn(mp, "unable to allocate a zone for gc");
1140 error = -EIO;
1141 goto out;
1142 }
1143
1144 trace_xfs_zone_gc_target_opened(oz->oz_rtg);
1145 zi->zi_open_gc_zone = oz;
1146
1147 data = xfs_zone_gc_data_alloc(mp);
1148 if (!data) {
1149 error = -ENOMEM;
1150 goto out_put_gc_zone;
1151 }
1152
1153 mp->m_zone_info->zi_gc_thread = kthread_create(xfs_zoned_gcd, data,
1154 "xfs-zone-gc/%s", mp->m_super->s_id);
1155 if (IS_ERR(mp->m_zone_info->zi_gc_thread)) {
1156 xfs_warn(mp, "unable to create zone gc thread");
1157 error = PTR_ERR(mp->m_zone_info->zi_gc_thread);
1158 goto out_free_gc_data;
1159 }
1160
1161 /* xfs_zone_gc_start will unpark for rw mounts */
1162 kthread_park(mp->m_zone_info->zi_gc_thread);
1163 return 0;
1164
1165 out_free_gc_data:
1166 kfree(data);
1167 out_put_gc_zone:
1168 xfs_open_zone_put(zi->zi_open_gc_zone);
1169 out:
1170 return error;
1171 }
1172
1173 void
xfs_zone_gc_unmount(struct xfs_mount * mp)1174 xfs_zone_gc_unmount(
1175 struct xfs_mount *mp)
1176 {
1177 struct xfs_zone_info *zi = mp->m_zone_info;
1178
1179 kthread_stop(zi->zi_gc_thread);
1180 if (zi->zi_open_gc_zone)
1181 xfs_open_zone_put(zi->zi_open_gc_zone);
1182 }
1183