1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2016-2025 Christoph Hellwig.
4 * All Rights Reserved.
5 */
6 #include "xfs_platform.h"
7 #include "xfs_shared.h"
8 #include "xfs_format.h"
9 #include "xfs_log_format.h"
10 #include "xfs_trans_resv.h"
11 #include "xfs_mount.h"
12 #include "xfs_inode.h"
13 #include "xfs_iomap.h"
14 #include "xfs_trace.h"
15 #include "xfs_bmap_util.h"
16 #include "xfs_reflink.h"
17 #include "xfs_zone_alloc.h"
18 #include "xfs_ioend.h"
19
20 static void
xfs_ioend_put_open_zones(struct iomap_ioend * ioend)21 xfs_ioend_put_open_zones(
22 struct iomap_ioend *ioend)
23 {
24 struct iomap_ioend *tmp;
25
26 /*
27 * Put the open zone for all ioends merged into this one (if any).
28 */
29 list_for_each_entry(tmp, &ioend->io_list, io_list)
30 xfs_open_zone_put(tmp->io_private);
31
32 /*
33 * The main ioend might not have an open zone if the submission failed
34 * before xfs_zone_alloc_and_submit got called.
35 */
36 if (ioend->io_private)
37 xfs_open_zone_put(ioend->io_private);
38 }
39
40 static void
xfs_end_ioend_write(struct iomap_ioend * ioend)41 xfs_end_ioend_write(
42 struct iomap_ioend *ioend)
43 {
44 struct xfs_inode *ip = XFS_I(ioend->io_inode);
45 struct xfs_mount *mp = ip->i_mount;
46 bool is_zoned = xfs_is_zoned_inode(ip);
47 xfs_off_t offset = ioend->io_offset;
48 size_t size = ioend->io_size;
49 unsigned int nofs_flag;
50 int error;
51
52 /*
53 * We can allocate memory here while doing writeback on behalf of
54 * memory reclaim. To avoid memory allocation deadlocks set the
55 * task-wide nofs context for the following operations.
56 */
57 nofs_flag = memalloc_nofs_save();
58
59 /*
60 * Just clean up the in-memory structures if the fs has been shut down.
61 */
62 if (xfs_is_shutdown(mp)) {
63 error = -EIO;
64 goto done;
65 }
66
67 /*
68 * Clean up all COW blocks and underlying data fork delalloc blocks on
69 * I/O error. The delalloc punch is required because this ioend was
70 * mapped to blocks in the COW fork and the associated pages are no
71 * longer dirty. If we don't remove delalloc blocks here, they become
72 * stale and can corrupt free space accounting on unmount.
73 */
74 error = blk_status_to_errno(ioend->io_bio.bi_status);
75 if (unlikely(error)) {
76 /*
77 * Zoned writes update the in-core open zone accounting before
78 * I/O submission. A failed write leaves that state
79 * inconsistent, so shut down the filesystem instead of letting
80 * later writers wait forever for open zone space to become
81 * available.
82 */
83 if (is_zoned) {
84 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
85 goto done;
86 }
87 if (ioend->io_flags & IOMAP_IOEND_SHARED) {
88 ASSERT(!is_zoned);
89 xfs_reflink_cancel_cow_range(ip, offset, size, true);
90 xfs_bmap_punch_delalloc_range(ip, XFS_DATA_FORK, offset,
91 offset + size, NULL);
92 }
93 goto done;
94 }
95
96 /*
97 * Success: commit the COW or unwritten blocks if needed.
98 */
99 if (is_zoned)
100 error = xfs_zoned_end_io(ip, offset, size, ioend->io_sector,
101 ioend->io_private, NULLFSBLOCK);
102 else if (ioend->io_flags & IOMAP_IOEND_SHARED)
103 error = xfs_reflink_end_cow(ip, offset, size);
104 else if (ioend->io_flags & IOMAP_IOEND_UNWRITTEN)
105 error = xfs_iomap_write_unwritten(ip, offset, size, false);
106
107 if (!error &&
108 !(ioend->io_flags & IOMAP_IOEND_DIRECT) &&
109 xfs_ioend_is_append(ioend))
110 error = xfs_setfilesize(ip, offset, size);
111 done:
112 if (is_zoned)
113 xfs_ioend_put_open_zones(ioend);
114 iomap_finish_ioends(ioend, error);
115 memalloc_nofs_restore(nofs_flag);
116 }
117
118 /*
119 * Finish all pending IO completions that require transactional modifications.
120 *
121 * We try to merge physical and logically contiguous ioends before completion to
122 * minimise the number of transactions we need to perform during IO completion.
123 * Both unwritten extent conversion and COW remapping need to iterate and modify
124 * one physical extent at a time, so we gain nothing by merging physically
125 * discontiguous extents here.
126 *
127 * The ioend chain length that we can be processing here is largely unbound in
128 * length and we may have to perform significant amounts of work on each ioend
129 * to complete it. Hence we have to be careful about holding the CPU for too
130 * long in this loop.
131 */
132 void
xfs_end_io(struct work_struct * work)133 xfs_end_io(
134 struct work_struct *work)
135 {
136 struct xfs_inode *ip =
137 container_of(work, struct xfs_inode, i_ioend_work);
138 struct iomap_ioend *ioend;
139 struct list_head tmp;
140 unsigned long flags;
141
142 spin_lock_irqsave(&ip->i_ioend_lock, flags);
143 list_replace_init(&ip->i_ioend_list, &tmp);
144 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
145
146 iomap_sort_ioends(&tmp);
147 while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
148 io_list))) {
149 list_del_init(&ioend->io_list);
150 iomap_ioend_try_merge(ioend, &tmp);
151 if (bio_op(&ioend->io_bio) == REQ_OP_READ)
152 iomap_finish_ioends(ioend,
153 blk_status_to_errno(ioend->io_bio.bi_status));
154 else
155 xfs_end_ioend_write(ioend);
156 cond_resched();
157 }
158 }
159
160 void
xfs_end_bio(struct bio * bio)161 xfs_end_bio(
162 struct bio *bio)
163 {
164 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
165 struct xfs_inode *ip = XFS_I(ioend->io_inode);
166 struct xfs_mount *mp = ip->i_mount;
167 unsigned long flags;
168
169 /*
170 * For Appends record the actually written block number and set the
171 * boundary flag if needed.
172 */
173 if (IS_ENABLED(CONFIG_XFS_RT) && bio_is_zone_append(bio)) {
174 ioend->io_sector = bio->bi_iter.bi_sector;
175 xfs_mark_rtg_boundary(ioend);
176 }
177
178 spin_lock_irqsave(&ip->i_ioend_lock, flags);
179 if (list_empty(&ip->i_ioend_list))
180 WARN_ON_ONCE(!queue_work(mp->m_unwritten_workqueue,
181 &ip->i_ioend_work));
182 list_add_tail(&ioend->io_list, &ip->i_ioend_list);
183 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
184 }
185