xref: /linux/fs/xfs/xfs_file.c (revision b50ecc5aca4d18f1f0c4942f5c797bc85edef144)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2000-2005 Silicon Graphics, Inc.
4  * All Rights Reserved.
5  */
6 #include "xfs.h"
7 #include "xfs_fs.h"
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_log_format.h"
11 #include "xfs_trans_resv.h"
12 #include "xfs_mount.h"
13 #include "xfs_inode.h"
14 #include "xfs_trans.h"
15 #include "xfs_inode_item.h"
16 #include "xfs_bmap.h"
17 #include "xfs_bmap_util.h"
18 #include "xfs_dir2.h"
19 #include "xfs_dir2_priv.h"
20 #include "xfs_ioctl.h"
21 #include "xfs_trace.h"
22 #include "xfs_log.h"
23 #include "xfs_icache.h"
24 #include "xfs_pnfs.h"
25 #include "xfs_iomap.h"
26 #include "xfs_reflink.h"
27 #include "xfs_file.h"
28 
29 #include <linux/dax.h>
30 #include <linux/falloc.h>
31 #include <linux/backing-dev.h>
32 #include <linux/mman.h>
33 #include <linux/fadvise.h>
34 #include <linux/mount.h>
35 
36 static const struct vm_operations_struct xfs_file_vm_ops;
37 
38 /*
39  * Decide if the given file range is aligned to the size of the fundamental
40  * allocation unit for the file.
41  */
42 bool
43 xfs_is_falloc_aligned(
44 	struct xfs_inode	*ip,
45 	loff_t			pos,
46 	long long int		len)
47 {
48 	unsigned int		alloc_unit = xfs_inode_alloc_unitsize(ip);
49 
50 	if (!is_power_of_2(alloc_unit))
51 		return isaligned_64(pos, alloc_unit) &&
52 		       isaligned_64(len, alloc_unit);
53 
54 	return !((pos | len) & (alloc_unit - 1));
55 }
56 
57 /*
58  * Fsync operations on directories are much simpler than on regular files,
59  * as there is no file data to flush, and thus also no need for explicit
60  * cache flush operations, and there are no non-transaction metadata updates
61  * on directories either.
62  */
63 STATIC int
64 xfs_dir_fsync(
65 	struct file		*file,
66 	loff_t			start,
67 	loff_t			end,
68 	int			datasync)
69 {
70 	struct xfs_inode	*ip = XFS_I(file->f_mapping->host);
71 
72 	trace_xfs_dir_fsync(ip);
73 	return xfs_log_force_inode(ip);
74 }
75 
76 static xfs_csn_t
77 xfs_fsync_seq(
78 	struct xfs_inode	*ip,
79 	bool			datasync)
80 {
81 	if (!xfs_ipincount(ip))
82 		return 0;
83 	if (datasync && !(ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
84 		return 0;
85 	return ip->i_itemp->ili_commit_seq;
86 }
87 
88 /*
89  * All metadata updates are logged, which means that we just have to flush the
90  * log up to the latest LSN that touched the inode.
91  *
92  * If we have concurrent fsync/fdatasync() calls, we need them to all block on
93  * the log force before we clear the ili_fsync_fields field. This ensures that
94  * we don't get a racing sync operation that does not wait for the metadata to
95  * hit the journal before returning.  If we race with clearing ili_fsync_fields,
96  * then all that will happen is the log force will do nothing as the lsn will
97  * already be on disk.  We can't race with setting ili_fsync_fields because that
98  * is done under XFS_ILOCK_EXCL, and that can't happen because we hold the lock
99  * shared until after the ili_fsync_fields is cleared.
100  */
101 static  int
102 xfs_fsync_flush_log(
103 	struct xfs_inode	*ip,
104 	bool			datasync,
105 	int			*log_flushed)
106 {
107 	int			error = 0;
108 	xfs_csn_t		seq;
109 
110 	xfs_ilock(ip, XFS_ILOCK_SHARED);
111 	seq = xfs_fsync_seq(ip, datasync);
112 	if (seq) {
113 		error = xfs_log_force_seq(ip->i_mount, seq, XFS_LOG_SYNC,
114 					  log_flushed);
115 
116 		spin_lock(&ip->i_itemp->ili_lock);
117 		ip->i_itemp->ili_fsync_fields = 0;
118 		spin_unlock(&ip->i_itemp->ili_lock);
119 	}
120 	xfs_iunlock(ip, XFS_ILOCK_SHARED);
121 	return error;
122 }
123 
124 STATIC int
125 xfs_file_fsync(
126 	struct file		*file,
127 	loff_t			start,
128 	loff_t			end,
129 	int			datasync)
130 {
131 	struct xfs_inode	*ip = XFS_I(file->f_mapping->host);
132 	struct xfs_mount	*mp = ip->i_mount;
133 	int			error, err2;
134 	int			log_flushed = 0;
135 
136 	trace_xfs_file_fsync(ip);
137 
138 	error = file_write_and_wait_range(file, start, end);
139 	if (error)
140 		return error;
141 
142 	if (xfs_is_shutdown(mp))
143 		return -EIO;
144 
145 	xfs_iflags_clear(ip, XFS_ITRUNCATED);
146 
147 	/*
148 	 * If we have an RT and/or log subvolume we need to make sure to flush
149 	 * the write cache the device used for file data first.  This is to
150 	 * ensure newly written file data make it to disk before logging the new
151 	 * inode size in case of an extending write.
152 	 */
153 	if (XFS_IS_REALTIME_INODE(ip))
154 		error = blkdev_issue_flush(mp->m_rtdev_targp->bt_bdev);
155 	else if (mp->m_logdev_targp != mp->m_ddev_targp)
156 		error = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev);
157 
158 	/*
159 	 * Any inode that has dirty modifications in the log is pinned.  The
160 	 * racy check here for a pinned inode will not catch modifications
161 	 * that happen concurrently to the fsync call, but fsync semantics
162 	 * only require to sync previously completed I/O.
163 	 */
164 	if (xfs_ipincount(ip)) {
165 		err2 = xfs_fsync_flush_log(ip, datasync, &log_flushed);
166 		if (err2 && !error)
167 			error = err2;
168 	}
169 
170 	/*
171 	 * If we only have a single device, and the log force about was
172 	 * a no-op we might have to flush the data device cache here.
173 	 * This can only happen for fdatasync/O_DSYNC if we were overwriting
174 	 * an already allocated file and thus do not have any metadata to
175 	 * commit.
176 	 */
177 	if (!log_flushed && !XFS_IS_REALTIME_INODE(ip) &&
178 	    mp->m_logdev_targp == mp->m_ddev_targp) {
179 		err2 = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev);
180 		if (err2 && !error)
181 			error = err2;
182 	}
183 
184 	return error;
185 }
186 
187 static int
188 xfs_ilock_iocb(
189 	struct kiocb		*iocb,
190 	unsigned int		lock_mode)
191 {
192 	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
193 
194 	if (iocb->ki_flags & IOCB_NOWAIT) {
195 		if (!xfs_ilock_nowait(ip, lock_mode))
196 			return -EAGAIN;
197 	} else {
198 		xfs_ilock(ip, lock_mode);
199 	}
200 
201 	return 0;
202 }
203 
204 static int
205 xfs_ilock_iocb_for_write(
206 	struct kiocb		*iocb,
207 	unsigned int		*lock_mode)
208 {
209 	ssize_t			ret;
210 	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
211 
212 	ret = xfs_ilock_iocb(iocb, *lock_mode);
213 	if (ret)
214 		return ret;
215 
216 	/*
217 	 * If a reflink remap is in progress we always need to take the iolock
218 	 * exclusively to wait for it to finish.
219 	 */
220 	if (*lock_mode == XFS_IOLOCK_SHARED &&
221 	    xfs_iflags_test(ip, XFS_IREMAPPING)) {
222 		xfs_iunlock(ip, *lock_mode);
223 		*lock_mode = XFS_IOLOCK_EXCL;
224 		return xfs_ilock_iocb(iocb, *lock_mode);
225 	}
226 
227 	return 0;
228 }
229 
230 STATIC ssize_t
231 xfs_file_dio_read(
232 	struct kiocb		*iocb,
233 	struct iov_iter		*to)
234 {
235 	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
236 	ssize_t			ret;
237 
238 	trace_xfs_file_direct_read(iocb, to);
239 
240 	if (!iov_iter_count(to))
241 		return 0; /* skip atime */
242 
243 	file_accessed(iocb->ki_filp);
244 
245 	ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
246 	if (ret)
247 		return ret;
248 	ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0, NULL, 0);
249 	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
250 
251 	return ret;
252 }
253 
254 static noinline ssize_t
255 xfs_file_dax_read(
256 	struct kiocb		*iocb,
257 	struct iov_iter		*to)
258 {
259 	struct xfs_inode	*ip = XFS_I(iocb->ki_filp->f_mapping->host);
260 	ssize_t			ret = 0;
261 
262 	trace_xfs_file_dax_read(iocb, to);
263 
264 	if (!iov_iter_count(to))
265 		return 0; /* skip atime */
266 
267 	ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
268 	if (ret)
269 		return ret;
270 	ret = dax_iomap_rw(iocb, to, &xfs_read_iomap_ops);
271 	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
272 
273 	file_accessed(iocb->ki_filp);
274 	return ret;
275 }
276 
277 STATIC ssize_t
278 xfs_file_buffered_read(
279 	struct kiocb		*iocb,
280 	struct iov_iter		*to)
281 {
282 	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
283 	ssize_t			ret;
284 
285 	trace_xfs_file_buffered_read(iocb, to);
286 
287 	ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
288 	if (ret)
289 		return ret;
290 	ret = generic_file_read_iter(iocb, to);
291 	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
292 
293 	return ret;
294 }
295 
296 STATIC ssize_t
297 xfs_file_read_iter(
298 	struct kiocb		*iocb,
299 	struct iov_iter		*to)
300 {
301 	struct inode		*inode = file_inode(iocb->ki_filp);
302 	struct xfs_mount	*mp = XFS_I(inode)->i_mount;
303 	ssize_t			ret = 0;
304 
305 	XFS_STATS_INC(mp, xs_read_calls);
306 
307 	if (xfs_is_shutdown(mp))
308 		return -EIO;
309 
310 	if (IS_DAX(inode))
311 		ret = xfs_file_dax_read(iocb, to);
312 	else if (iocb->ki_flags & IOCB_DIRECT)
313 		ret = xfs_file_dio_read(iocb, to);
314 	else
315 		ret = xfs_file_buffered_read(iocb, to);
316 
317 	if (ret > 0)
318 		XFS_STATS_ADD(mp, xs_read_bytes, ret);
319 	return ret;
320 }
321 
322 STATIC ssize_t
323 xfs_file_splice_read(
324 	struct file		*in,
325 	loff_t			*ppos,
326 	struct pipe_inode_info	*pipe,
327 	size_t			len,
328 	unsigned int		flags)
329 {
330 	struct inode		*inode = file_inode(in);
331 	struct xfs_inode	*ip = XFS_I(inode);
332 	struct xfs_mount	*mp = ip->i_mount;
333 	ssize_t			ret = 0;
334 
335 	XFS_STATS_INC(mp, xs_read_calls);
336 
337 	if (xfs_is_shutdown(mp))
338 		return -EIO;
339 
340 	trace_xfs_file_splice_read(ip, *ppos, len);
341 
342 	xfs_ilock(ip, XFS_IOLOCK_SHARED);
343 	ret = filemap_splice_read(in, ppos, pipe, len, flags);
344 	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
345 	if (ret > 0)
346 		XFS_STATS_ADD(mp, xs_read_bytes, ret);
347 	return ret;
348 }
349 
350 /*
351  * Take care of zeroing post-EOF blocks when they might exist.
352  *
353  * Returns 0 if successfully, a negative error for a failure, or 1 if this
354  * function dropped the iolock and reacquired it exclusively and the caller
355  * needs to restart the write sanity checks.
356  */
357 static ssize_t
358 xfs_file_write_zero_eof(
359 	struct kiocb		*iocb,
360 	struct iov_iter		*from,
361 	unsigned int		*iolock,
362 	size_t			count,
363 	bool			*drained_dio)
364 {
365 	struct xfs_inode	*ip = XFS_I(iocb->ki_filp->f_mapping->host);
366 	loff_t			isize;
367 	int			error;
368 
369 	/*
370 	 * We need to serialise against EOF updates that occur in IO completions
371 	 * here. We want to make sure that nobody is changing the size while
372 	 * we do this check until we have placed an IO barrier (i.e. hold
373 	 * XFS_IOLOCK_EXCL) that prevents new IO from being dispatched.  The
374 	 * spinlock effectively forms a memory barrier once we have
375 	 * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value and
376 	 * hence be able to correctly determine if we need to run zeroing.
377 	 */
378 	spin_lock(&ip->i_flags_lock);
379 	isize = i_size_read(VFS_I(ip));
380 	if (iocb->ki_pos <= isize) {
381 		spin_unlock(&ip->i_flags_lock);
382 		return 0;
383 	}
384 	spin_unlock(&ip->i_flags_lock);
385 
386 	if (iocb->ki_flags & IOCB_NOWAIT)
387 		return -EAGAIN;
388 
389 	if (!*drained_dio) {
390 		/*
391 		 * If zeroing is needed and we are currently holding the iolock
392 		 * shared, we need to update it to exclusive which implies
393 		 * having to redo all checks before.
394 		 */
395 		if (*iolock == XFS_IOLOCK_SHARED) {
396 			xfs_iunlock(ip, *iolock);
397 			*iolock = XFS_IOLOCK_EXCL;
398 			xfs_ilock(ip, *iolock);
399 			iov_iter_reexpand(from, count);
400 		}
401 
402 		/*
403 		 * We now have an IO submission barrier in place, but AIO can do
404 		 * EOF updates during IO completion and hence we now need to
405 		 * wait for all of them to drain.  Non-AIO DIO will have drained
406 		 * before we are given the XFS_IOLOCK_EXCL, and so for most
407 		 * cases this wait is a no-op.
408 		 */
409 		inode_dio_wait(VFS_I(ip));
410 		*drained_dio = true;
411 		return 1;
412 	}
413 
414 	trace_xfs_zero_eof(ip, isize, iocb->ki_pos - isize);
415 
416 	xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
417 	error = xfs_zero_range(ip, isize, iocb->ki_pos - isize, NULL);
418 	xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
419 
420 	return error;
421 }
422 
423 /*
424  * Common pre-write limit and setup checks.
425  *
426  * Called with the iolock held either shared and exclusive according to
427  * @iolock, and returns with it held.  Might upgrade the iolock to exclusive
428  * if called for a direct write beyond i_size.
429  */
430 STATIC ssize_t
431 xfs_file_write_checks(
432 	struct kiocb		*iocb,
433 	struct iov_iter		*from,
434 	unsigned int		*iolock)
435 {
436 	struct inode		*inode = iocb->ki_filp->f_mapping->host;
437 	size_t			count = iov_iter_count(from);
438 	bool			drained_dio = false;
439 	ssize_t			error;
440 
441 restart:
442 	error = generic_write_checks(iocb, from);
443 	if (error <= 0)
444 		return error;
445 
446 	if (iocb->ki_flags & IOCB_NOWAIT) {
447 		error = break_layout(inode, false);
448 		if (error == -EWOULDBLOCK)
449 			error = -EAGAIN;
450 	} else {
451 		error = xfs_break_layouts(inode, iolock, BREAK_WRITE);
452 	}
453 
454 	if (error)
455 		return error;
456 
457 	/*
458 	 * For changing security info in file_remove_privs() we need i_rwsem
459 	 * exclusively.
460 	 */
461 	if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
462 		xfs_iunlock(XFS_I(inode), *iolock);
463 		*iolock = XFS_IOLOCK_EXCL;
464 		error = xfs_ilock_iocb(iocb, *iolock);
465 		if (error) {
466 			*iolock = 0;
467 			return error;
468 		}
469 		goto restart;
470 	}
471 
472 	/*
473 	 * If the offset is beyond the size of the file, we need to zero all
474 	 * blocks that fall between the existing EOF and the start of this
475 	 * write.
476 	 *
477 	 * We can do an unlocked check for i_size here safely as I/O completion
478 	 * can only extend EOF.  Truncate is locked out at this point, so the
479 	 * EOF can not move backwards, only forwards. Hence we only need to take
480 	 * the slow path when we are at or beyond the current EOF.
481 	 */
482 	if (iocb->ki_pos > i_size_read(inode)) {
483 		error = xfs_file_write_zero_eof(iocb, from, iolock, count,
484 				&drained_dio);
485 		if (error == 1)
486 			goto restart;
487 		if (error)
488 			return error;
489 	}
490 
491 	return kiocb_modified(iocb);
492 }
493 
494 static int
495 xfs_dio_write_end_io(
496 	struct kiocb		*iocb,
497 	ssize_t			size,
498 	int			error,
499 	unsigned		flags)
500 {
501 	struct inode		*inode = file_inode(iocb->ki_filp);
502 	struct xfs_inode	*ip = XFS_I(inode);
503 	loff_t			offset = iocb->ki_pos;
504 	unsigned int		nofs_flag;
505 
506 	trace_xfs_end_io_direct_write(ip, offset, size);
507 
508 	if (xfs_is_shutdown(ip->i_mount))
509 		return -EIO;
510 
511 	if (error)
512 		return error;
513 	if (!size)
514 		return 0;
515 
516 	/*
517 	 * Capture amount written on completion as we can't reliably account
518 	 * for it on submission.
519 	 */
520 	XFS_STATS_ADD(ip->i_mount, xs_write_bytes, size);
521 
522 	/*
523 	 * We can allocate memory here while doing writeback on behalf of
524 	 * memory reclaim.  To avoid memory allocation deadlocks set the
525 	 * task-wide nofs context for the following operations.
526 	 */
527 	nofs_flag = memalloc_nofs_save();
528 
529 	if (flags & IOMAP_DIO_COW) {
530 		error = xfs_reflink_end_cow(ip, offset, size);
531 		if (error)
532 			goto out;
533 	}
534 
535 	/*
536 	 * Unwritten conversion updates the in-core isize after extent
537 	 * conversion but before updating the on-disk size. Updating isize any
538 	 * earlier allows a racing dio read to find unwritten extents before
539 	 * they are converted.
540 	 */
541 	if (flags & IOMAP_DIO_UNWRITTEN) {
542 		error = xfs_iomap_write_unwritten(ip, offset, size, true);
543 		goto out;
544 	}
545 
546 	/*
547 	 * We need to update the in-core inode size here so that we don't end up
548 	 * with the on-disk inode size being outside the in-core inode size. We
549 	 * have no other method of updating EOF for AIO, so always do it here
550 	 * if necessary.
551 	 *
552 	 * We need to lock the test/set EOF update as we can be racing with
553 	 * other IO completions here to update the EOF. Failing to serialise
554 	 * here can result in EOF moving backwards and Bad Things Happen when
555 	 * that occurs.
556 	 *
557 	 * As IO completion only ever extends EOF, we can do an unlocked check
558 	 * here to avoid taking the spinlock. If we land within the current EOF,
559 	 * then we do not need to do an extending update at all, and we don't
560 	 * need to take the lock to check this. If we race with an update moving
561 	 * EOF, then we'll either still be beyond EOF and need to take the lock,
562 	 * or we'll be within EOF and we don't need to take it at all.
563 	 */
564 	if (offset + size <= i_size_read(inode))
565 		goto out;
566 
567 	spin_lock(&ip->i_flags_lock);
568 	if (offset + size > i_size_read(inode)) {
569 		i_size_write(inode, offset + size);
570 		spin_unlock(&ip->i_flags_lock);
571 		error = xfs_setfilesize(ip, offset, size);
572 	} else {
573 		spin_unlock(&ip->i_flags_lock);
574 	}
575 
576 out:
577 	memalloc_nofs_restore(nofs_flag);
578 	return error;
579 }
580 
581 static const struct iomap_dio_ops xfs_dio_write_ops = {
582 	.end_io		= xfs_dio_write_end_io,
583 };
584 
585 /*
586  * Handle block aligned direct I/O writes
587  */
588 static noinline ssize_t
589 xfs_file_dio_write_aligned(
590 	struct xfs_inode	*ip,
591 	struct kiocb		*iocb,
592 	struct iov_iter		*from)
593 {
594 	unsigned int		iolock = XFS_IOLOCK_SHARED;
595 	ssize_t			ret;
596 
597 	ret = xfs_ilock_iocb_for_write(iocb, &iolock);
598 	if (ret)
599 		return ret;
600 	ret = xfs_file_write_checks(iocb, from, &iolock);
601 	if (ret)
602 		goto out_unlock;
603 
604 	/*
605 	 * We don't need to hold the IOLOCK exclusively across the IO, so demote
606 	 * the iolock back to shared if we had to take the exclusive lock in
607 	 * xfs_file_write_checks() for other reasons.
608 	 */
609 	if (iolock == XFS_IOLOCK_EXCL) {
610 		xfs_ilock_demote(ip, XFS_IOLOCK_EXCL);
611 		iolock = XFS_IOLOCK_SHARED;
612 	}
613 	trace_xfs_file_direct_write(iocb, from);
614 	ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
615 			   &xfs_dio_write_ops, 0, NULL, 0);
616 out_unlock:
617 	if (iolock)
618 		xfs_iunlock(ip, iolock);
619 	return ret;
620 }
621 
622 /*
623  * Handle block unaligned direct I/O writes
624  *
625  * In most cases direct I/O writes will be done holding IOLOCK_SHARED, allowing
626  * them to be done in parallel with reads and other direct I/O writes.  However,
627  * if the I/O is not aligned to filesystem blocks, the direct I/O layer may need
628  * to do sub-block zeroing and that requires serialisation against other direct
629  * I/O to the same block.  In this case we need to serialise the submission of
630  * the unaligned I/O so that we don't get racing block zeroing in the dio layer.
631  * In the case where sub-block zeroing is not required, we can do concurrent
632  * sub-block dios to the same block successfully.
633  *
634  * Optimistically submit the I/O using the shared lock first, but use the
635  * IOMAP_DIO_OVERWRITE_ONLY flag to tell the lower layers to return -EAGAIN
636  * if block allocation or partial block zeroing would be required.  In that case
637  * we try again with the exclusive lock.
638  */
639 static noinline ssize_t
640 xfs_file_dio_write_unaligned(
641 	struct xfs_inode	*ip,
642 	struct kiocb		*iocb,
643 	struct iov_iter		*from)
644 {
645 	size_t			isize = i_size_read(VFS_I(ip));
646 	size_t			count = iov_iter_count(from);
647 	unsigned int		iolock = XFS_IOLOCK_SHARED;
648 	unsigned int		flags = IOMAP_DIO_OVERWRITE_ONLY;
649 	ssize_t			ret;
650 
651 	/*
652 	 * Extending writes need exclusivity because of the sub-block zeroing
653 	 * that the DIO code always does for partial tail blocks beyond EOF, so
654 	 * don't even bother trying the fast path in this case.
655 	 */
656 	if (iocb->ki_pos > isize || iocb->ki_pos + count >= isize) {
657 		if (iocb->ki_flags & IOCB_NOWAIT)
658 			return -EAGAIN;
659 retry_exclusive:
660 		iolock = XFS_IOLOCK_EXCL;
661 		flags = IOMAP_DIO_FORCE_WAIT;
662 	}
663 
664 	ret = xfs_ilock_iocb_for_write(iocb, &iolock);
665 	if (ret)
666 		return ret;
667 
668 	/*
669 	 * We can't properly handle unaligned direct I/O to reflink files yet,
670 	 * as we can't unshare a partial block.
671 	 */
672 	if (xfs_is_cow_inode(ip)) {
673 		trace_xfs_reflink_bounce_dio_write(iocb, from);
674 		ret = -ENOTBLK;
675 		goto out_unlock;
676 	}
677 
678 	ret = xfs_file_write_checks(iocb, from, &iolock);
679 	if (ret)
680 		goto out_unlock;
681 
682 	/*
683 	 * If we are doing exclusive unaligned I/O, this must be the only I/O
684 	 * in-flight.  Otherwise we risk data corruption due to unwritten extent
685 	 * conversions from the AIO end_io handler.  Wait for all other I/O to
686 	 * drain first.
687 	 */
688 	if (flags & IOMAP_DIO_FORCE_WAIT)
689 		inode_dio_wait(VFS_I(ip));
690 
691 	trace_xfs_file_direct_write(iocb, from);
692 	ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
693 			   &xfs_dio_write_ops, flags, NULL, 0);
694 
695 	/*
696 	 * Retry unaligned I/O with exclusive blocking semantics if the DIO
697 	 * layer rejected it for mapping or locking reasons. If we are doing
698 	 * nonblocking user I/O, propagate the error.
699 	 */
700 	if (ret == -EAGAIN && !(iocb->ki_flags & IOCB_NOWAIT)) {
701 		ASSERT(flags & IOMAP_DIO_OVERWRITE_ONLY);
702 		xfs_iunlock(ip, iolock);
703 		goto retry_exclusive;
704 	}
705 
706 out_unlock:
707 	if (iolock)
708 		xfs_iunlock(ip, iolock);
709 	return ret;
710 }
711 
712 static ssize_t
713 xfs_file_dio_write(
714 	struct kiocb		*iocb,
715 	struct iov_iter		*from)
716 {
717 	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
718 	struct xfs_buftarg      *target = xfs_inode_buftarg(ip);
719 	size_t			count = iov_iter_count(from);
720 
721 	/* direct I/O must be aligned to device logical sector size */
722 	if ((iocb->ki_pos | count) & target->bt_logical_sectormask)
723 		return -EINVAL;
724 	if ((iocb->ki_pos | count) & ip->i_mount->m_blockmask)
725 		return xfs_file_dio_write_unaligned(ip, iocb, from);
726 	return xfs_file_dio_write_aligned(ip, iocb, from);
727 }
728 
729 static noinline ssize_t
730 xfs_file_dax_write(
731 	struct kiocb		*iocb,
732 	struct iov_iter		*from)
733 {
734 	struct inode		*inode = iocb->ki_filp->f_mapping->host;
735 	struct xfs_inode	*ip = XFS_I(inode);
736 	unsigned int		iolock = XFS_IOLOCK_EXCL;
737 	ssize_t			ret, error = 0;
738 	loff_t			pos;
739 
740 	ret = xfs_ilock_iocb(iocb, iolock);
741 	if (ret)
742 		return ret;
743 	ret = xfs_file_write_checks(iocb, from, &iolock);
744 	if (ret)
745 		goto out;
746 
747 	pos = iocb->ki_pos;
748 
749 	trace_xfs_file_dax_write(iocb, from);
750 	ret = dax_iomap_rw(iocb, from, &xfs_dax_write_iomap_ops);
751 	if (ret > 0 && iocb->ki_pos > i_size_read(inode)) {
752 		i_size_write(inode, iocb->ki_pos);
753 		error = xfs_setfilesize(ip, pos, ret);
754 	}
755 out:
756 	if (iolock)
757 		xfs_iunlock(ip, iolock);
758 	if (error)
759 		return error;
760 
761 	if (ret > 0) {
762 		XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
763 
764 		/* Handle various SYNC-type writes */
765 		ret = generic_write_sync(iocb, ret);
766 	}
767 	return ret;
768 }
769 
770 STATIC ssize_t
771 xfs_file_buffered_write(
772 	struct kiocb		*iocb,
773 	struct iov_iter		*from)
774 {
775 	struct inode		*inode = iocb->ki_filp->f_mapping->host;
776 	struct xfs_inode	*ip = XFS_I(inode);
777 	ssize_t			ret;
778 	bool			cleared_space = false;
779 	unsigned int		iolock;
780 
781 write_retry:
782 	iolock = XFS_IOLOCK_EXCL;
783 	ret = xfs_ilock_iocb(iocb, iolock);
784 	if (ret)
785 		return ret;
786 
787 	ret = xfs_file_write_checks(iocb, from, &iolock);
788 	if (ret)
789 		goto out;
790 
791 	trace_xfs_file_buffered_write(iocb, from);
792 	ret = iomap_file_buffered_write(iocb, from,
793 			&xfs_buffered_write_iomap_ops, NULL);
794 
795 	/*
796 	 * If we hit a space limit, try to free up some lingering preallocated
797 	 * space before returning an error. In the case of ENOSPC, first try to
798 	 * write back all dirty inodes to free up some of the excess reserved
799 	 * metadata space. This reduces the chances that the eofblocks scan
800 	 * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this
801 	 * also behaves as a filter to prevent too many eofblocks scans from
802 	 * running at the same time.  Use a synchronous scan to increase the
803 	 * effectiveness of the scan.
804 	 */
805 	if (ret == -EDQUOT && !cleared_space) {
806 		xfs_iunlock(ip, iolock);
807 		xfs_blockgc_free_quota(ip, XFS_ICWALK_FLAG_SYNC);
808 		cleared_space = true;
809 		goto write_retry;
810 	} else if (ret == -ENOSPC && !cleared_space) {
811 		struct xfs_icwalk	icw = {0};
812 
813 		cleared_space = true;
814 		xfs_flush_inodes(ip->i_mount);
815 
816 		xfs_iunlock(ip, iolock);
817 		icw.icw_flags = XFS_ICWALK_FLAG_SYNC;
818 		xfs_blockgc_free_space(ip->i_mount, &icw);
819 		goto write_retry;
820 	}
821 
822 out:
823 	if (iolock)
824 		xfs_iunlock(ip, iolock);
825 
826 	if (ret > 0) {
827 		XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
828 		/* Handle various SYNC-type writes */
829 		ret = generic_write_sync(iocb, ret);
830 	}
831 	return ret;
832 }
833 
834 STATIC ssize_t
835 xfs_file_write_iter(
836 	struct kiocb		*iocb,
837 	struct iov_iter		*from)
838 {
839 	struct inode		*inode = iocb->ki_filp->f_mapping->host;
840 	struct xfs_inode	*ip = XFS_I(inode);
841 	ssize_t			ret;
842 	size_t			ocount = iov_iter_count(from);
843 
844 	XFS_STATS_INC(ip->i_mount, xs_write_calls);
845 
846 	if (ocount == 0)
847 		return 0;
848 
849 	if (xfs_is_shutdown(ip->i_mount))
850 		return -EIO;
851 
852 	if (IS_DAX(inode))
853 		return xfs_file_dax_write(iocb, from);
854 
855 	if (iocb->ki_flags & IOCB_ATOMIC) {
856 		/*
857 		 * Currently only atomic writing of a single FS block is
858 		 * supported. It would be possible to atomic write smaller than
859 		 * a FS block, but there is no requirement to support this.
860 		 * Note that iomap also does not support this yet.
861 		 */
862 		if (ocount != ip->i_mount->m_sb.sb_blocksize)
863 			return -EINVAL;
864 		ret = generic_atomic_write_valid(iocb, from);
865 		if (ret)
866 			return ret;
867 	}
868 
869 	if (iocb->ki_flags & IOCB_DIRECT) {
870 		/*
871 		 * Allow a directio write to fall back to a buffered
872 		 * write *only* in the case that we're doing a reflink
873 		 * CoW.  In all other directio scenarios we do not
874 		 * allow an operation to fall back to buffered mode.
875 		 */
876 		ret = xfs_file_dio_write(iocb, from);
877 		if (ret != -ENOTBLK)
878 			return ret;
879 	}
880 
881 	return xfs_file_buffered_write(iocb, from);
882 }
883 
884 /* Does this file, inode, or mount want synchronous writes? */
885 static inline bool xfs_file_sync_writes(struct file *filp)
886 {
887 	struct xfs_inode	*ip = XFS_I(file_inode(filp));
888 
889 	if (xfs_has_wsync(ip->i_mount))
890 		return true;
891 	if (filp->f_flags & (__O_SYNC | O_DSYNC))
892 		return true;
893 	if (IS_SYNC(file_inode(filp)))
894 		return true;
895 
896 	return false;
897 }
898 
899 static int
900 xfs_falloc_newsize(
901 	struct file		*file,
902 	int			mode,
903 	loff_t			offset,
904 	loff_t			len,
905 	loff_t			*new_size)
906 {
907 	struct inode		*inode = file_inode(file);
908 
909 	if ((mode & FALLOC_FL_KEEP_SIZE) || offset + len <= i_size_read(inode))
910 		return 0;
911 	*new_size = offset + len;
912 	return inode_newsize_ok(inode, *new_size);
913 }
914 
915 static int
916 xfs_falloc_setsize(
917 	struct file		*file,
918 	loff_t			new_size)
919 {
920 	struct iattr iattr = {
921 		.ia_valid	= ATTR_SIZE,
922 		.ia_size	= new_size,
923 	};
924 
925 	if (!new_size)
926 		return 0;
927 	return xfs_vn_setattr_size(file_mnt_idmap(file), file_dentry(file),
928 			&iattr);
929 }
930 
931 static int
932 xfs_falloc_collapse_range(
933 	struct file		*file,
934 	loff_t			offset,
935 	loff_t			len)
936 {
937 	struct inode		*inode = file_inode(file);
938 	loff_t			new_size = i_size_read(inode) - len;
939 	int			error;
940 
941 	if (!xfs_is_falloc_aligned(XFS_I(inode), offset, len))
942 		return -EINVAL;
943 
944 	/*
945 	 * There is no need to overlap collapse range with EOF, in which case it
946 	 * is effectively a truncate operation
947 	 */
948 	if (offset + len >= i_size_read(inode))
949 		return -EINVAL;
950 
951 	error = xfs_collapse_file_space(XFS_I(inode), offset, len);
952 	if (error)
953 		return error;
954 	return xfs_falloc_setsize(file, new_size);
955 }
956 
957 static int
958 xfs_falloc_insert_range(
959 	struct file		*file,
960 	loff_t			offset,
961 	loff_t			len)
962 {
963 	struct inode		*inode = file_inode(file);
964 	loff_t			isize = i_size_read(inode);
965 	int			error;
966 
967 	if (!xfs_is_falloc_aligned(XFS_I(inode), offset, len))
968 		return -EINVAL;
969 
970 	/*
971 	 * New inode size must not exceed ->s_maxbytes, accounting for
972 	 * possible signed overflow.
973 	 */
974 	if (inode->i_sb->s_maxbytes - isize < len)
975 		return -EFBIG;
976 
977 	/* Offset should be less than i_size */
978 	if (offset >= isize)
979 		return -EINVAL;
980 
981 	error = xfs_falloc_setsize(file, isize + len);
982 	if (error)
983 		return error;
984 
985 	/*
986 	 * Perform hole insertion now that the file size has been updated so
987 	 * that if we crash during the operation we don't leave shifted extents
988 	 * past EOF and hence losing access to the data that is contained within
989 	 * them.
990 	 */
991 	return xfs_insert_file_space(XFS_I(inode), offset, len);
992 }
993 
994 /*
995  * Punch a hole and prealloc the range.  We use a hole punch rather than
996  * unwritten extent conversion for two reasons:
997  *
998  *   1.) Hole punch handles partial block zeroing for us.
999  *   2.) If prealloc returns ENOSPC, the file range is still zero-valued by
1000  *	 virtue of the hole punch.
1001  */
1002 static int
1003 xfs_falloc_zero_range(
1004 	struct file		*file,
1005 	int			mode,
1006 	loff_t			offset,
1007 	loff_t			len)
1008 {
1009 	struct inode		*inode = file_inode(file);
1010 	unsigned int		blksize = i_blocksize(inode);
1011 	loff_t			new_size = 0;
1012 	int			error;
1013 
1014 	trace_xfs_zero_file_space(XFS_I(inode));
1015 
1016 	error = xfs_falloc_newsize(file, mode, offset, len, &new_size);
1017 	if (error)
1018 		return error;
1019 
1020 	error = xfs_free_file_space(XFS_I(inode), offset, len);
1021 	if (error)
1022 		return error;
1023 
1024 	len = round_up(offset + len, blksize) - round_down(offset, blksize);
1025 	offset = round_down(offset, blksize);
1026 	error = xfs_alloc_file_space(XFS_I(inode), offset, len);
1027 	if (error)
1028 		return error;
1029 	return xfs_falloc_setsize(file, new_size);
1030 }
1031 
1032 static int
1033 xfs_falloc_unshare_range(
1034 	struct file		*file,
1035 	int			mode,
1036 	loff_t			offset,
1037 	loff_t			len)
1038 {
1039 	struct inode		*inode = file_inode(file);
1040 	loff_t			new_size = 0;
1041 	int			error;
1042 
1043 	error = xfs_falloc_newsize(file, mode, offset, len, &new_size);
1044 	if (error)
1045 		return error;
1046 
1047 	error = xfs_reflink_unshare(XFS_I(inode), offset, len);
1048 	if (error)
1049 		return error;
1050 
1051 	error = xfs_alloc_file_space(XFS_I(inode), offset, len);
1052 	if (error)
1053 		return error;
1054 	return xfs_falloc_setsize(file, new_size);
1055 }
1056 
1057 static int
1058 xfs_falloc_allocate_range(
1059 	struct file		*file,
1060 	int			mode,
1061 	loff_t			offset,
1062 	loff_t			len)
1063 {
1064 	struct inode		*inode = file_inode(file);
1065 	loff_t			new_size = 0;
1066 	int			error;
1067 
1068 	/*
1069 	 * If always_cow mode we can't use preallocations and thus should not
1070 	 * create them.
1071 	 */
1072 	if (xfs_is_always_cow_inode(XFS_I(inode)))
1073 		return -EOPNOTSUPP;
1074 
1075 	error = xfs_falloc_newsize(file, mode, offset, len, &new_size);
1076 	if (error)
1077 		return error;
1078 
1079 	error = xfs_alloc_file_space(XFS_I(inode), offset, len);
1080 	if (error)
1081 		return error;
1082 	return xfs_falloc_setsize(file, new_size);
1083 }
1084 
1085 #define	XFS_FALLOC_FL_SUPPORTED						\
1086 		(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |		\
1087 		 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE |	\
1088 		 FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE)
1089 
1090 STATIC long
1091 xfs_file_fallocate(
1092 	struct file		*file,
1093 	int			mode,
1094 	loff_t			offset,
1095 	loff_t			len)
1096 {
1097 	struct inode		*inode = file_inode(file);
1098 	struct xfs_inode	*ip = XFS_I(inode);
1099 	long			error;
1100 	uint			iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL;
1101 
1102 	if (!S_ISREG(inode->i_mode))
1103 		return -EINVAL;
1104 	if (mode & ~XFS_FALLOC_FL_SUPPORTED)
1105 		return -EOPNOTSUPP;
1106 
1107 	xfs_ilock(ip, iolock);
1108 	error = xfs_break_layouts(inode, &iolock, BREAK_UNMAP);
1109 	if (error)
1110 		goto out_unlock;
1111 
1112 	/*
1113 	 * Must wait for all AIO to complete before we continue as AIO can
1114 	 * change the file size on completion without holding any locks we
1115 	 * currently hold. We must do this first because AIO can update both
1116 	 * the on disk and in memory inode sizes, and the operations that follow
1117 	 * require the in-memory size to be fully up-to-date.
1118 	 */
1119 	inode_dio_wait(inode);
1120 
1121 	error = file_modified(file);
1122 	if (error)
1123 		goto out_unlock;
1124 
1125 	switch (mode & FALLOC_FL_MODE_MASK) {
1126 	case FALLOC_FL_PUNCH_HOLE:
1127 		error = xfs_free_file_space(ip, offset, len);
1128 		break;
1129 	case FALLOC_FL_COLLAPSE_RANGE:
1130 		error = xfs_falloc_collapse_range(file, offset, len);
1131 		break;
1132 	case FALLOC_FL_INSERT_RANGE:
1133 		error = xfs_falloc_insert_range(file, offset, len);
1134 		break;
1135 	case FALLOC_FL_ZERO_RANGE:
1136 		error = xfs_falloc_zero_range(file, mode, offset, len);
1137 		break;
1138 	case FALLOC_FL_UNSHARE_RANGE:
1139 		error = xfs_falloc_unshare_range(file, mode, offset, len);
1140 		break;
1141 	case FALLOC_FL_ALLOCATE_RANGE:
1142 		error = xfs_falloc_allocate_range(file, mode, offset, len);
1143 		break;
1144 	default:
1145 		error = -EOPNOTSUPP;
1146 		break;
1147 	}
1148 
1149 	if (!error && xfs_file_sync_writes(file))
1150 		error = xfs_log_force_inode(ip);
1151 
1152 out_unlock:
1153 	xfs_iunlock(ip, iolock);
1154 	return error;
1155 }
1156 
1157 STATIC int
1158 xfs_file_fadvise(
1159 	struct file	*file,
1160 	loff_t		start,
1161 	loff_t		end,
1162 	int		advice)
1163 {
1164 	struct xfs_inode *ip = XFS_I(file_inode(file));
1165 	int ret;
1166 	int lockflags = 0;
1167 
1168 	/*
1169 	 * Operations creating pages in page cache need protection from hole
1170 	 * punching and similar ops
1171 	 */
1172 	if (advice == POSIX_FADV_WILLNEED) {
1173 		lockflags = XFS_IOLOCK_SHARED;
1174 		xfs_ilock(ip, lockflags);
1175 	}
1176 	ret = generic_fadvise(file, start, end, advice);
1177 	if (lockflags)
1178 		xfs_iunlock(ip, lockflags);
1179 	return ret;
1180 }
1181 
1182 STATIC loff_t
1183 xfs_file_remap_range(
1184 	struct file		*file_in,
1185 	loff_t			pos_in,
1186 	struct file		*file_out,
1187 	loff_t			pos_out,
1188 	loff_t			len,
1189 	unsigned int		remap_flags)
1190 {
1191 	struct inode		*inode_in = file_inode(file_in);
1192 	struct xfs_inode	*src = XFS_I(inode_in);
1193 	struct inode		*inode_out = file_inode(file_out);
1194 	struct xfs_inode	*dest = XFS_I(inode_out);
1195 	struct xfs_mount	*mp = src->i_mount;
1196 	loff_t			remapped = 0;
1197 	xfs_extlen_t		cowextsize;
1198 	int			ret;
1199 
1200 	if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY))
1201 		return -EINVAL;
1202 
1203 	if (!xfs_has_reflink(mp))
1204 		return -EOPNOTSUPP;
1205 
1206 	if (xfs_is_shutdown(mp))
1207 		return -EIO;
1208 
1209 	/* Prepare and then clone file data. */
1210 	ret = xfs_reflink_remap_prep(file_in, pos_in, file_out, pos_out,
1211 			&len, remap_flags);
1212 	if (ret || len == 0)
1213 		return ret;
1214 
1215 	trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out);
1216 
1217 	ret = xfs_reflink_remap_blocks(src, pos_in, dest, pos_out, len,
1218 			&remapped);
1219 	if (ret)
1220 		goto out_unlock;
1221 
1222 	/*
1223 	 * Carry the cowextsize hint from src to dest if we're sharing the
1224 	 * entire source file to the entire destination file, the source file
1225 	 * has a cowextsize hint, and the destination file does not.
1226 	 */
1227 	cowextsize = 0;
1228 	if (pos_in == 0 && len == i_size_read(inode_in) &&
1229 	    (src->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) &&
1230 	    pos_out == 0 && len >= i_size_read(inode_out) &&
1231 	    !(dest->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE))
1232 		cowextsize = src->i_cowextsize;
1233 
1234 	ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize,
1235 			remap_flags);
1236 	if (ret)
1237 		goto out_unlock;
1238 
1239 	if (xfs_file_sync_writes(file_in) || xfs_file_sync_writes(file_out))
1240 		xfs_log_force_inode(dest);
1241 out_unlock:
1242 	xfs_iunlock2_remapping(src, dest);
1243 	if (ret)
1244 		trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_);
1245 	return remapped > 0 ? remapped : ret;
1246 }
1247 
1248 STATIC int
1249 xfs_file_open(
1250 	struct inode	*inode,
1251 	struct file	*file)
1252 {
1253 	if (xfs_is_shutdown(XFS_M(inode->i_sb)))
1254 		return -EIO;
1255 	file->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT;
1256 	if (xfs_inode_can_atomicwrite(XFS_I(inode)))
1257 		file->f_mode |= FMODE_CAN_ATOMIC_WRITE;
1258 	return generic_file_open(inode, file);
1259 }
1260 
1261 STATIC int
1262 xfs_dir_open(
1263 	struct inode	*inode,
1264 	struct file	*file)
1265 {
1266 	struct xfs_inode *ip = XFS_I(inode);
1267 	unsigned int	mode;
1268 	int		error;
1269 
1270 	if (xfs_is_shutdown(ip->i_mount))
1271 		return -EIO;
1272 	error = generic_file_open(inode, file);
1273 	if (error)
1274 		return error;
1275 
1276 	/*
1277 	 * If there are any blocks, read-ahead block 0 as we're almost
1278 	 * certain to have the next operation be a read there.
1279 	 */
1280 	mode = xfs_ilock_data_map_shared(ip);
1281 	if (ip->i_df.if_nextents > 0)
1282 		error = xfs_dir3_data_readahead(ip, 0, 0);
1283 	xfs_iunlock(ip, mode);
1284 	return error;
1285 }
1286 
1287 /*
1288  * Don't bother propagating errors.  We're just doing cleanup, and the caller
1289  * ignores the return value anyway.
1290  */
1291 STATIC int
1292 xfs_file_release(
1293 	struct inode		*inode,
1294 	struct file		*file)
1295 {
1296 	struct xfs_inode	*ip = XFS_I(inode);
1297 	struct xfs_mount	*mp = ip->i_mount;
1298 
1299 	/*
1300 	 * If this is a read-only mount or the file system has been shut down,
1301 	 * don't generate I/O.
1302 	 */
1303 	if (xfs_is_readonly(mp) || xfs_is_shutdown(mp))
1304 		return 0;
1305 
1306 	/*
1307 	 * If we previously truncated this file and removed old data in the
1308 	 * process, we want to initiate "early" writeout on the last close.
1309 	 * This is an attempt to combat the notorious NULL files problem which
1310 	 * is particularly noticeable from a truncate down, buffered (re-)write
1311 	 * (delalloc), followed by a crash.  What we are effectively doing here
1312 	 * is significantly reducing the time window where we'd otherwise be
1313 	 * exposed to that problem.
1314 	 */
1315 	if (xfs_iflags_test_and_clear(ip, XFS_ITRUNCATED)) {
1316 		xfs_iflags_clear(ip, XFS_EOFBLOCKS_RELEASED);
1317 		if (ip->i_delayed_blks > 0)
1318 			filemap_flush(inode->i_mapping);
1319 	}
1320 
1321 	/*
1322 	 * XFS aggressively preallocates post-EOF space to generate contiguous
1323 	 * allocations for writers that append to the end of the file.
1324 	 *
1325 	 * To support workloads that close and reopen the file frequently, these
1326 	 * preallocations usually persist after a close unless it is the first
1327 	 * close for the inode.  This is a tradeoff to generate tightly packed
1328 	 * data layouts for unpacking tarballs or similar archives that write
1329 	 * one file after another without going back to it while keeping the
1330 	 * preallocation for files that have recurring open/write/close cycles.
1331 	 *
1332 	 * This heuristic is skipped for inodes with the append-only flag as
1333 	 * that flag is rather pointless for inodes written only once.
1334 	 *
1335 	 * There is no point in freeing blocks here for open but unlinked files
1336 	 * as they will be taken care of by the inactivation path soon.
1337 	 *
1338 	 * When releasing a read-only context, don't flush data or trim post-EOF
1339 	 * blocks.  This avoids open/read/close workloads from removing EOF
1340 	 * blocks that other writers depend upon to reduce fragmentation.
1341 	 *
1342 	 * If we can't get the iolock just skip truncating the blocks past EOF
1343 	 * because we could deadlock with the mmap_lock otherwise. We'll get
1344 	 * another chance to drop them once the last reference to the inode is
1345 	 * dropped, so we'll never leak blocks permanently.
1346 	 */
1347 	if (inode->i_nlink &&
1348 	    (file->f_mode & FMODE_WRITE) &&
1349 	    !(ip->i_diflags & XFS_DIFLAG_APPEND) &&
1350 	    !xfs_iflags_test(ip, XFS_EOFBLOCKS_RELEASED) &&
1351 	    xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
1352 		if (xfs_can_free_eofblocks(ip) &&
1353 		    !xfs_iflags_test_and_set(ip, XFS_EOFBLOCKS_RELEASED))
1354 			xfs_free_eofblocks(ip);
1355 		xfs_iunlock(ip, XFS_IOLOCK_EXCL);
1356 	}
1357 
1358 	return 0;
1359 }
1360 
1361 STATIC int
1362 xfs_file_readdir(
1363 	struct file	*file,
1364 	struct dir_context *ctx)
1365 {
1366 	struct inode	*inode = file_inode(file);
1367 	xfs_inode_t	*ip = XFS_I(inode);
1368 	size_t		bufsize;
1369 
1370 	/*
1371 	 * The Linux API doesn't pass down the total size of the buffer
1372 	 * we read into down to the filesystem.  With the filldir concept
1373 	 * it's not needed for correct information, but the XFS dir2 leaf
1374 	 * code wants an estimate of the buffer size to calculate it's
1375 	 * readahead window and size the buffers used for mapping to
1376 	 * physical blocks.
1377 	 *
1378 	 * Try to give it an estimate that's good enough, maybe at some
1379 	 * point we can change the ->readdir prototype to include the
1380 	 * buffer size.  For now we use the current glibc buffer size.
1381 	 */
1382 	bufsize = (size_t)min_t(loff_t, XFS_READDIR_BUFSIZE, ip->i_disk_size);
1383 
1384 	return xfs_readdir(NULL, ip, ctx, bufsize);
1385 }
1386 
1387 STATIC loff_t
1388 xfs_file_llseek(
1389 	struct file	*file,
1390 	loff_t		offset,
1391 	int		whence)
1392 {
1393 	struct inode		*inode = file->f_mapping->host;
1394 
1395 	if (xfs_is_shutdown(XFS_I(inode)->i_mount))
1396 		return -EIO;
1397 
1398 	switch (whence) {
1399 	default:
1400 		return generic_file_llseek(file, offset, whence);
1401 	case SEEK_HOLE:
1402 		offset = iomap_seek_hole(inode, offset, &xfs_seek_iomap_ops);
1403 		break;
1404 	case SEEK_DATA:
1405 		offset = iomap_seek_data(inode, offset, &xfs_seek_iomap_ops);
1406 		break;
1407 	}
1408 
1409 	if (offset < 0)
1410 		return offset;
1411 	return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
1412 }
1413 
1414 static inline vm_fault_t
1415 xfs_dax_fault_locked(
1416 	struct vm_fault		*vmf,
1417 	unsigned int		order,
1418 	bool			write_fault)
1419 {
1420 	vm_fault_t		ret;
1421 	pfn_t			pfn;
1422 
1423 	if (!IS_ENABLED(CONFIG_FS_DAX)) {
1424 		ASSERT(0);
1425 		return VM_FAULT_SIGBUS;
1426 	}
1427 	ret = dax_iomap_fault(vmf, order, &pfn, NULL,
1428 			(write_fault && !vmf->cow_page) ?
1429 				&xfs_dax_write_iomap_ops :
1430 				&xfs_read_iomap_ops);
1431 	if (ret & VM_FAULT_NEEDDSYNC)
1432 		ret = dax_finish_sync_fault(vmf, order, pfn);
1433 	return ret;
1434 }
1435 
1436 static vm_fault_t
1437 xfs_dax_read_fault(
1438 	struct vm_fault		*vmf,
1439 	unsigned int		order)
1440 {
1441 	struct xfs_inode	*ip = XFS_I(file_inode(vmf->vma->vm_file));
1442 	vm_fault_t		ret;
1443 
1444 	trace_xfs_read_fault(ip, order);
1445 
1446 	xfs_ilock(ip, XFS_MMAPLOCK_SHARED);
1447 	ret = xfs_dax_fault_locked(vmf, order, false);
1448 	xfs_iunlock(ip, XFS_MMAPLOCK_SHARED);
1449 
1450 	return ret;
1451 }
1452 
1453 /*
1454  * Locking for serialisation of IO during page faults. This results in a lock
1455  * ordering of:
1456  *
1457  * mmap_lock (MM)
1458  *   sb_start_pagefault(vfs, freeze)
1459  *     invalidate_lock (vfs/XFS_MMAPLOCK - truncate serialisation)
1460  *       page_lock (MM)
1461  *         i_lock (XFS - extent map serialisation)
1462  */
1463 static vm_fault_t
1464 xfs_write_fault(
1465 	struct vm_fault		*vmf,
1466 	unsigned int		order)
1467 {
1468 	struct inode		*inode = file_inode(vmf->vma->vm_file);
1469 	struct xfs_inode	*ip = XFS_I(inode);
1470 	unsigned int		lock_mode = XFS_MMAPLOCK_SHARED;
1471 	vm_fault_t		ret;
1472 
1473 	trace_xfs_write_fault(ip, order);
1474 
1475 	sb_start_pagefault(inode->i_sb);
1476 	file_update_time(vmf->vma->vm_file);
1477 
1478 	/*
1479 	 * Normally we only need the shared mmaplock, but if a reflink remap is
1480 	 * in progress we take the exclusive lock to wait for the remap to
1481 	 * finish before taking a write fault.
1482 	 */
1483 	xfs_ilock(ip, XFS_MMAPLOCK_SHARED);
1484 	if (xfs_iflags_test(ip, XFS_IREMAPPING)) {
1485 		xfs_iunlock(ip, XFS_MMAPLOCK_SHARED);
1486 		xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
1487 		lock_mode = XFS_MMAPLOCK_EXCL;
1488 	}
1489 
1490 	if (IS_DAX(inode))
1491 		ret = xfs_dax_fault_locked(vmf, order, true);
1492 	else
1493 		ret = iomap_page_mkwrite(vmf, &xfs_buffered_write_iomap_ops);
1494 	xfs_iunlock(ip, lock_mode);
1495 
1496 	sb_end_pagefault(inode->i_sb);
1497 	return ret;
1498 }
1499 
1500 static inline bool
1501 xfs_is_write_fault(
1502 	struct vm_fault		*vmf)
1503 {
1504 	return (vmf->flags & FAULT_FLAG_WRITE) &&
1505 	       (vmf->vma->vm_flags & VM_SHARED);
1506 }
1507 
1508 static vm_fault_t
1509 xfs_filemap_fault(
1510 	struct vm_fault		*vmf)
1511 {
1512 	struct inode		*inode = file_inode(vmf->vma->vm_file);
1513 
1514 	/* DAX can shortcut the normal fault path on write faults! */
1515 	if (IS_DAX(inode)) {
1516 		if (xfs_is_write_fault(vmf))
1517 			return xfs_write_fault(vmf, 0);
1518 		return xfs_dax_read_fault(vmf, 0);
1519 	}
1520 
1521 	trace_xfs_read_fault(XFS_I(inode), 0);
1522 	return filemap_fault(vmf);
1523 }
1524 
1525 static vm_fault_t
1526 xfs_filemap_huge_fault(
1527 	struct vm_fault		*vmf,
1528 	unsigned int		order)
1529 {
1530 	if (!IS_DAX(file_inode(vmf->vma->vm_file)))
1531 		return VM_FAULT_FALLBACK;
1532 
1533 	/* DAX can shortcut the normal fault path on write faults! */
1534 	if (xfs_is_write_fault(vmf))
1535 		return xfs_write_fault(vmf, order);
1536 	return xfs_dax_read_fault(vmf, order);
1537 }
1538 
1539 static vm_fault_t
1540 xfs_filemap_page_mkwrite(
1541 	struct vm_fault		*vmf)
1542 {
1543 	return xfs_write_fault(vmf, 0);
1544 }
1545 
1546 /*
1547  * pfn_mkwrite was originally intended to ensure we capture time stamp updates
1548  * on write faults. In reality, it needs to serialise against truncate and
1549  * prepare memory for writing so handle is as standard write fault.
1550  */
1551 static vm_fault_t
1552 xfs_filemap_pfn_mkwrite(
1553 	struct vm_fault		*vmf)
1554 {
1555 	return xfs_write_fault(vmf, 0);
1556 }
1557 
1558 static const struct vm_operations_struct xfs_file_vm_ops = {
1559 	.fault		= xfs_filemap_fault,
1560 	.huge_fault	= xfs_filemap_huge_fault,
1561 	.map_pages	= filemap_map_pages,
1562 	.page_mkwrite	= xfs_filemap_page_mkwrite,
1563 	.pfn_mkwrite	= xfs_filemap_pfn_mkwrite,
1564 };
1565 
1566 STATIC int
1567 xfs_file_mmap(
1568 	struct file		*file,
1569 	struct vm_area_struct	*vma)
1570 {
1571 	struct inode		*inode = file_inode(file);
1572 	struct xfs_buftarg	*target = xfs_inode_buftarg(XFS_I(inode));
1573 
1574 	/*
1575 	 * We don't support synchronous mappings for non-DAX files and
1576 	 * for DAX files if underneath dax_device is not synchronous.
1577 	 */
1578 	if (!daxdev_mapping_supported(vma, target->bt_daxdev))
1579 		return -EOPNOTSUPP;
1580 
1581 	file_accessed(file);
1582 	vma->vm_ops = &xfs_file_vm_ops;
1583 	if (IS_DAX(inode))
1584 		vm_flags_set(vma, VM_HUGEPAGE);
1585 	return 0;
1586 }
1587 
1588 const struct file_operations xfs_file_operations = {
1589 	.llseek		= xfs_file_llseek,
1590 	.read_iter	= xfs_file_read_iter,
1591 	.write_iter	= xfs_file_write_iter,
1592 	.splice_read	= xfs_file_splice_read,
1593 	.splice_write	= iter_file_splice_write,
1594 	.iopoll		= iocb_bio_iopoll,
1595 	.unlocked_ioctl	= xfs_file_ioctl,
1596 #ifdef CONFIG_COMPAT
1597 	.compat_ioctl	= xfs_file_compat_ioctl,
1598 #endif
1599 	.mmap		= xfs_file_mmap,
1600 	.open		= xfs_file_open,
1601 	.release	= xfs_file_release,
1602 	.fsync		= xfs_file_fsync,
1603 	.get_unmapped_area = thp_get_unmapped_area,
1604 	.fallocate	= xfs_file_fallocate,
1605 	.fadvise	= xfs_file_fadvise,
1606 	.remap_file_range = xfs_file_remap_range,
1607 	.fop_flags	= FOP_MMAP_SYNC | FOP_BUFFER_RASYNC |
1608 			  FOP_BUFFER_WASYNC | FOP_DIO_PARALLEL_WRITE,
1609 };
1610 
1611 const struct file_operations xfs_dir_file_operations = {
1612 	.open		= xfs_dir_open,
1613 	.read		= generic_read_dir,
1614 	.iterate_shared	= xfs_file_readdir,
1615 	.llseek		= generic_file_llseek,
1616 	.unlocked_ioctl	= xfs_file_ioctl,
1617 #ifdef CONFIG_COMPAT
1618 	.compat_ioctl	= xfs_file_compat_ioctl,
1619 #endif
1620 	.fsync		= xfs_dir_fsync,
1621 };
1622