1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2026 Christoph Hellwig. 4 * All Rights Reserved. 5 */ 6 #include "xfs_platform.h" 7 #include "xfs_shared.h" 8 #include "xfs_format.h" 9 #include "xfs_log_format.h" 10 #include "xfs_trans_resv.h" 11 #include "xfs_mount.h" 12 #include "xfs_inode.h" 13 #include "xfs_iomap.h" 14 #include "xfs_trace.h" 15 #include "xfs_bmap_util.h" 16 #include "xfs_reflink.h" 17 #include "xfs_zone_alloc.h" 18 #include "xfs_ioend.h" 19 #include "xfs_error.h" 20 #include "xfs_errortag.h" 21 #include <linux/bio-integrity.h> 22 23 static void 24 xfs_dio_bounce_end_io( 25 struct bio *bio) 26 { 27 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); 28 int error = blk_status_to_errno(bio->bi_status); 29 struct bio *orig_bio = bio->bi_private; 30 31 if ((ioend->io_flags & IOMAP_IOEND_INTEGRITY) && !bio->bi_status) 32 error = iomap_ioend_integrity_verify(ioend); 33 iomap_bounce_read_end_io(ioend, orig_bio, error); 34 } 35 36 static void 37 xfs_bounce_submit_ioend( 38 struct iomap_ioend *ioend) 39 { 40 if (ioend->io_flags & IOMAP_IOEND_INTEGRITY) 41 fs_bio_integrity_alloc(&ioend->io_bio); 42 ioend->io_bio.bi_end_io = xfs_dio_bounce_end_io; 43 bio_set_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK); 44 submit_bio(&ioend->io_bio); 45 } 46 47 static void 48 xfs_end_bio_bounced( 49 struct bio *bio) 50 { 51 /* 52 * Just complete the original ioends as all verification is done by the 53 * end_io handlers for the clone bio(s). 54 */ 55 iomap_finish_ioends(iomap_ioend_from_bio(bio), 56 blk_status_to_errno(bio->bi_status)); 57 } 58 59 static void 60 xfs_read_bounce_and_resubmit( 61 struct iomap_ioend *ioend) 62 { 63 struct bio *bio = &ioend->io_bio; 64 struct xfs_inode *ip = XFS_I(ioend->io_inode); 65 unsigned int nofs_flag = memalloc_nofs_save(); 66 67 trace_xfs_bounce_reread(ip, ioend->io_offset, ioend->io_size); 68 69 /* 70 * Free the bio integrity data for the original bio, as we'll allocate 71 * a new one for each sub-I/O, which could deadlock if we keep the 72 * integrity data for the original bio around. 73 */ 74 if (bio_integrity(bio)) 75 fs_bio_integrity_free(bio); 76 77 /* 78 * Resubmit the bio through the iomap bounce machinery. The original 79 * bio itself is not resubmitted to the block layer, but just used to 80 * track I/O completion of the cloned bios. 81 */ 82 bio_prepare_reissue(bio, xfs_inode_buftarg(ip)->bt_bdev); 83 bio->bi_iter = (struct bvec_iter) { 84 .bi_sector = ioend->io_sector, 85 .bi_size = ioend->io_size, 86 .bi_offset = ioend->io_bvec_offset, 87 }; 88 bio->bi_end_io = xfs_end_bio_bounced; 89 iomap_bounce_read(ioend, bdev_logical_block_size(bio->bi_bdev), 90 xfs_bounce_submit_ioend); 91 memalloc_nofs_restore(nofs_flag); 92 } 93 94 static void 95 xfs_end_io_read( 96 struct bio *bio) 97 { 98 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); 99 struct xfs_inode *ip = XFS_I(ioend->io_inode); 100 struct xfs_mount *mp = ip->i_mount; 101 int error = blk_status_to_errno(bio->bi_status); 102 103 if (!error && (ioend->io_flags & IOMAP_IOEND_INTEGRITY)) { 104 error = iomap_ioend_integrity_verify(ioend); 105 if ((ioend->io_flags & IOMAP_IOEND_DIRECT) && 106 READ_ONCE(mp->m_read_bounce) == XFS_READ_BOUNCE_LAZY) { 107 /* 108 * We only really need to retry for guard tag errors, 109 * but right now we can't distinguish them from other 110 * (i.e, reftag) errors. 111 */ 112 if (error || 113 XFS_TEST_ERROR(mp, XFS_ERRTAG_BOUNCE_REREAD)) { 114 xfs_read_bounce_and_resubmit(ioend); 115 return; 116 } 117 } 118 } 119 120 iomap_finish_ioends(ioend, error); 121 } 122 123 void 124 xfs_ioend_submit_read( 125 struct inode *inode, 126 struct bio *bio, 127 loff_t file_offset, 128 u16 ioend_flags) 129 { 130 struct xfs_inode *ip = XFS_I(inode); 131 struct xfs_mount *mp = ip->i_mount; 132 struct iomap_ioend *ioend; 133 134 ioend = iomap_init_ioend(inode, bio, file_offset, ioend_flags); 135 if ((ioend_flags & IOMAP_IOEND_DIRECT) && 136 READ_ONCE(mp->m_read_bounce) == XFS_READ_BOUNCE_ALWAYS) { 137 iomap_bounce_read(ioend, bdev_logical_block_size(bio->bi_bdev), 138 xfs_bounce_submit_ioend); 139 return; 140 } 141 142 if (ioend_flags & IOMAP_IOEND_INTEGRITY) 143 fs_bio_integrity_alloc(bio); 144 bio->bi_end_io = xfs_end_io_read; 145 bio_set_flag(bio, BIO_COMPLETE_IN_TASK); 146 submit_bio(bio); 147 } 148 149 static void 150 xfs_end_ioend_write_zoned( 151 struct iomap_ioend *ioend) 152 { 153 struct xfs_inode *ip = XFS_I(ioend->io_inode); 154 struct xfs_open_zone *oz = ioend->io_private; 155 struct iomap_ioend *tmp; 156 int error = -EIO; 157 158 ASSERT(!(ioend->io_flags & IOMAP_IOEND_SHARED)); 159 160 if (xfs_is_shutdown(ip->i_mount)) 161 goto done; 162 163 /* 164 * Zoned writes update the in-core open zone accounting before I/O 165 * submission. A failed write leaves that state inconsistent, so shut 166 * down the filesystem instead of letting later writers wait forever for 167 * open zone space to become available. 168 */ 169 error = blk_status_to_errno(ioend->io_bio.bi_status); 170 if (unlikely(error)) { 171 xfs_force_shutdown(ip->i_mount, SHUTDOWN_META_IO_ERROR); 172 goto done; 173 } 174 175 error = xfs_zoned_end_io(ip, ioend->io_offset, ioend->io_size, 176 ioend->io_sector, oz, NULLFSBLOCK); 177 if (error) 178 goto done; 179 180 if (!(ioend->io_flags & IOMAP_IOEND_DIRECT) && 181 xfs_ioend_is_append(ioend)) 182 error = xfs_setfilesize(ip, ioend->io_offset, ioend->io_size); 183 done: 184 /* 185 * Put the open zone references for all ioends merged into this one in 186 * addition to the main one they were merged into. If the submission 187 * failed before xfs_zone_alloc_and_submit got called, there is no open 188 * zone and at least in theory that could also happen for merged ioends. 189 */ 190 list_for_each_entry(tmp, &ioend->io_list, io_list) { 191 if (tmp->io_private) 192 xfs_open_zone_put(tmp->io_private); 193 } 194 if (oz) 195 xfs_open_zone_put(oz); 196 iomap_finish_ioends(ioend, error); 197 } 198 199 static void 200 xfs_end_ioend_write( 201 struct iomap_ioend *ioend) 202 { 203 struct xfs_inode *ip = XFS_I(ioend->io_inode); 204 struct xfs_mount *mp = ip->i_mount; 205 xfs_off_t offset = ioend->io_offset; 206 size_t size = ioend->io_size; 207 int error; 208 209 /* 210 * Just clean up the in-memory structures if the fs has been shut down. 211 */ 212 if (xfs_is_shutdown(mp)) { 213 error = -EIO; 214 goto done; 215 } 216 217 /* 218 * Clean up all COW blocks and underlying data fork delalloc blocks on 219 * I/O error. The delalloc punch is required because this ioend was 220 * mapped to blocks in the COW fork and the associated pages are no 221 * longer dirty. If we don't remove delalloc blocks here, they become 222 * stale and can corrupt free space accounting on unmount. 223 */ 224 error = blk_status_to_errno(ioend->io_bio.bi_status); 225 if (unlikely(error)) { 226 if (ioend->io_flags & IOMAP_IOEND_SHARED) { 227 xfs_reflink_cancel_cow_range(ip, offset, size, true); 228 xfs_bmap_punch_delalloc_range(ip, XFS_DATA_FORK, offset, 229 offset + size, NULL); 230 } 231 goto done; 232 } 233 234 /* 235 * Success: commit the COW or unwritten blocks if needed. 236 */ 237 if (ioend->io_flags & IOMAP_IOEND_SHARED) 238 error = xfs_reflink_end_cow(ip, offset, size); 239 else if (ioend->io_flags & IOMAP_IOEND_UNWRITTEN) 240 error = xfs_iomap_write_unwritten(ip, offset, size, false); 241 242 if (!error && 243 !(ioend->io_flags & IOMAP_IOEND_DIRECT) && 244 xfs_ioend_is_append(ioend)) 245 error = xfs_setfilesize(ip, offset, size); 246 done: 247 iomap_finish_ioends(ioend, error); 248 } 249 250 /* 251 * Finish all pending IO completions that require transactional modifications. 252 * 253 * We try to merge physical and logically contiguous ioends before completion to 254 * minimise the number of transactions we need to perform during IO completion. 255 * Both unwritten extent conversion and COW remapping need to iterate and modify 256 * one physical extent at a time, so we gain nothing by merging physically 257 * discontiguous extents here. 258 * 259 * The ioend chain length that we can be processing here is largely unbound in 260 * length and we may have to perform significant amounts of work on each ioend 261 * to complete it. Hence we have to be careful about holding the CPU for too 262 * long in this loop. 263 */ 264 void 265 xfs_end_io( 266 struct work_struct *work) 267 { 268 struct xfs_inode *ip = 269 container_of(work, struct xfs_inode, i_ioend_work); 270 struct iomap_ioend *ioend; 271 struct list_head tmp; 272 unsigned int nofs_flag; 273 unsigned long flags; 274 275 spin_lock_irqsave(&ip->i_ioend_lock, flags); 276 list_replace_init(&ip->i_ioend_list, &tmp); 277 spin_unlock_irqrestore(&ip->i_ioend_lock, flags); 278 279 iomap_sort_ioends(&tmp); 280 281 /* 282 * We can allocate memory here while doing writeback on behalf of 283 * memory reclaim. To avoid memory allocation deadlocks set the 284 * task-wide nofs context for the following operations. 285 */ 286 nofs_flag = memalloc_nofs_save(); 287 while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend, 288 io_list))) { 289 list_del_init(&ioend->io_list); 290 iomap_ioend_try_merge(ioend, &tmp); 291 if (xfs_is_zoned_inode(ip)) 292 xfs_end_ioend_write_zoned(ioend); 293 else 294 xfs_end_ioend_write(ioend); 295 cond_resched(); 296 } 297 memalloc_nofs_restore(nofs_flag); 298 } 299 300 void 301 xfs_end_bio( 302 struct bio *bio) 303 { 304 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); 305 struct xfs_inode *ip = XFS_I(ioend->io_inode); 306 struct xfs_mount *mp = ip->i_mount; 307 unsigned long flags; 308 309 /* 310 * For Appends record the actually written block number and set the 311 * boundary flag if needed. 312 */ 313 if (IS_ENABLED(CONFIG_XFS_RT) && bio_is_zone_append(bio)) { 314 ioend->io_sector = bio->bi_iter.bi_sector; 315 xfs_mark_rtg_boundary(ioend); 316 } 317 318 spin_lock_irqsave(&ip->i_ioend_lock, flags); 319 if (list_empty(&ip->i_ioend_list)) 320 WARN_ON_ONCE(!queue_work(mp->m_unwritten_workqueue, 321 &ip->i_ioend_work)); 322 list_add_tail(&ioend->io_list, &ip->i_ioend_list); 323 spin_unlock_irqrestore(&ip->i_ioend_lock, flags); 324 } 325