xref: /linux/fs/xfs/xfs_file.c (revision 2455f0e124d317dd08d337a7550a78a224d4ba41)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2000-2005 Silicon Graphics, Inc.
4  * All Rights Reserved.
5  */
6 #include "xfs.h"
7 #include "xfs_fs.h"
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_log_format.h"
11 #include "xfs_trans_resv.h"
12 #include "xfs_mount.h"
13 #include "xfs_inode.h"
14 #include "xfs_trans.h"
15 #include "xfs_inode_item.h"
16 #include "xfs_bmap.h"
17 #include "xfs_bmap_util.h"
18 #include "xfs_dir2.h"
19 #include "xfs_dir2_priv.h"
20 #include "xfs_ioctl.h"
21 #include "xfs_trace.h"
22 #include "xfs_log.h"
23 #include "xfs_icache.h"
24 #include "xfs_pnfs.h"
25 #include "xfs_iomap.h"
26 #include "xfs_reflink.h"
27 
28 #include <linux/dax.h>
29 #include <linux/falloc.h>
30 #include <linux/backing-dev.h>
31 #include <linux/mman.h>
32 #include <linux/fadvise.h>
33 #include <linux/mount.h>
34 
35 static const struct vm_operations_struct xfs_file_vm_ops;
36 
37 /*
38  * Decide if the given file range is aligned to the size of the fundamental
39  * allocation unit for the file.
40  */
41 static bool
42 xfs_is_falloc_aligned(
43 	struct xfs_inode	*ip,
44 	loff_t			pos,
45 	long long int		len)
46 {
47 	struct xfs_mount	*mp = ip->i_mount;
48 	uint64_t		mask;
49 
50 	if (XFS_IS_REALTIME_INODE(ip)) {
51 		if (!is_power_of_2(mp->m_sb.sb_rextsize)) {
52 			u64	rextbytes;
53 			u32	mod;
54 
55 			rextbytes = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize);
56 			div_u64_rem(pos, rextbytes, &mod);
57 			if (mod)
58 				return false;
59 			div_u64_rem(len, rextbytes, &mod);
60 			return mod == 0;
61 		}
62 		mask = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize) - 1;
63 	} else {
64 		mask = mp->m_sb.sb_blocksize - 1;
65 	}
66 
67 	return !((pos | len) & mask);
68 }
69 
70 /*
71  * Fsync operations on directories are much simpler than on regular files,
72  * as there is no file data to flush, and thus also no need for explicit
73  * cache flush operations, and there are no non-transaction metadata updates
74  * on directories either.
75  */
76 STATIC int
77 xfs_dir_fsync(
78 	struct file		*file,
79 	loff_t			start,
80 	loff_t			end,
81 	int			datasync)
82 {
83 	struct xfs_inode	*ip = XFS_I(file->f_mapping->host);
84 
85 	trace_xfs_dir_fsync(ip);
86 	return xfs_log_force_inode(ip);
87 }
88 
89 static xfs_csn_t
90 xfs_fsync_seq(
91 	struct xfs_inode	*ip,
92 	bool			datasync)
93 {
94 	if (!xfs_ipincount(ip))
95 		return 0;
96 	if (datasync && !(ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
97 		return 0;
98 	return ip->i_itemp->ili_commit_seq;
99 }
100 
101 /*
102  * All metadata updates are logged, which means that we just have to flush the
103  * log up to the latest LSN that touched the inode.
104  *
105  * If we have concurrent fsync/fdatasync() calls, we need them to all block on
106  * the log force before we clear the ili_fsync_fields field. This ensures that
107  * we don't get a racing sync operation that does not wait for the metadata to
108  * hit the journal before returning.  If we race with clearing ili_fsync_fields,
109  * then all that will happen is the log force will do nothing as the lsn will
110  * already be on disk.  We can't race with setting ili_fsync_fields because that
111  * is done under XFS_ILOCK_EXCL, and that can't happen because we hold the lock
112  * shared until after the ili_fsync_fields is cleared.
113  */
114 static  int
115 xfs_fsync_flush_log(
116 	struct xfs_inode	*ip,
117 	bool			datasync,
118 	int			*log_flushed)
119 {
120 	int			error = 0;
121 	xfs_csn_t		seq;
122 
123 	xfs_ilock(ip, XFS_ILOCK_SHARED);
124 	seq = xfs_fsync_seq(ip, datasync);
125 	if (seq) {
126 		error = xfs_log_force_seq(ip->i_mount, seq, XFS_LOG_SYNC,
127 					  log_flushed);
128 
129 		spin_lock(&ip->i_itemp->ili_lock);
130 		ip->i_itemp->ili_fsync_fields = 0;
131 		spin_unlock(&ip->i_itemp->ili_lock);
132 	}
133 	xfs_iunlock(ip, XFS_ILOCK_SHARED);
134 	return error;
135 }
136 
137 STATIC int
138 xfs_file_fsync(
139 	struct file		*file,
140 	loff_t			start,
141 	loff_t			end,
142 	int			datasync)
143 {
144 	struct xfs_inode	*ip = XFS_I(file->f_mapping->host);
145 	struct xfs_mount	*mp = ip->i_mount;
146 	int			error, err2;
147 	int			log_flushed = 0;
148 
149 	trace_xfs_file_fsync(ip);
150 
151 	error = file_write_and_wait_range(file, start, end);
152 	if (error)
153 		return error;
154 
155 	if (xfs_is_shutdown(mp))
156 		return -EIO;
157 
158 	xfs_iflags_clear(ip, XFS_ITRUNCATED);
159 
160 	/*
161 	 * If we have an RT and/or log subvolume we need to make sure to flush
162 	 * the write cache the device used for file data first.  This is to
163 	 * ensure newly written file data make it to disk before logging the new
164 	 * inode size in case of an extending write.
165 	 */
166 	if (XFS_IS_REALTIME_INODE(ip))
167 		error = blkdev_issue_flush(mp->m_rtdev_targp->bt_bdev);
168 	else if (mp->m_logdev_targp != mp->m_ddev_targp)
169 		error = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev);
170 
171 	/*
172 	 * Any inode that has dirty modifications in the log is pinned.  The
173 	 * racy check here for a pinned inode will not catch modifications
174 	 * that happen concurrently to the fsync call, but fsync semantics
175 	 * only require to sync previously completed I/O.
176 	 */
177 	if (xfs_ipincount(ip)) {
178 		err2 = xfs_fsync_flush_log(ip, datasync, &log_flushed);
179 		if (err2 && !error)
180 			error = err2;
181 	}
182 
183 	/*
184 	 * If we only have a single device, and the log force about was
185 	 * a no-op we might have to flush the data device cache here.
186 	 * This can only happen for fdatasync/O_DSYNC if we were overwriting
187 	 * an already allocated file and thus do not have any metadata to
188 	 * commit.
189 	 */
190 	if (!log_flushed && !XFS_IS_REALTIME_INODE(ip) &&
191 	    mp->m_logdev_targp == mp->m_ddev_targp) {
192 		err2 = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev);
193 		if (err2 && !error)
194 			error = err2;
195 	}
196 
197 	return error;
198 }
199 
200 static int
201 xfs_ilock_iocb(
202 	struct kiocb		*iocb,
203 	unsigned int		lock_mode)
204 {
205 	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
206 
207 	if (iocb->ki_flags & IOCB_NOWAIT) {
208 		if (!xfs_ilock_nowait(ip, lock_mode))
209 			return -EAGAIN;
210 	} else {
211 		xfs_ilock(ip, lock_mode);
212 	}
213 
214 	return 0;
215 }
216 
217 STATIC ssize_t
218 xfs_file_dio_read(
219 	struct kiocb		*iocb,
220 	struct iov_iter		*to)
221 {
222 	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
223 	ssize_t			ret;
224 
225 	trace_xfs_file_direct_read(iocb, to);
226 
227 	if (!iov_iter_count(to))
228 		return 0; /* skip atime */
229 
230 	file_accessed(iocb->ki_filp);
231 
232 	ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
233 	if (ret)
234 		return ret;
235 	ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0, NULL, 0);
236 	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
237 
238 	return ret;
239 }
240 
241 static noinline ssize_t
242 xfs_file_dax_read(
243 	struct kiocb		*iocb,
244 	struct iov_iter		*to)
245 {
246 	struct xfs_inode	*ip = XFS_I(iocb->ki_filp->f_mapping->host);
247 	ssize_t			ret = 0;
248 
249 	trace_xfs_file_dax_read(iocb, to);
250 
251 	if (!iov_iter_count(to))
252 		return 0; /* skip atime */
253 
254 	ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
255 	if (ret)
256 		return ret;
257 	ret = dax_iomap_rw(iocb, to, &xfs_read_iomap_ops);
258 	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
259 
260 	file_accessed(iocb->ki_filp);
261 	return ret;
262 }
263 
264 STATIC ssize_t
265 xfs_file_buffered_read(
266 	struct kiocb		*iocb,
267 	struct iov_iter		*to)
268 {
269 	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
270 	ssize_t			ret;
271 
272 	trace_xfs_file_buffered_read(iocb, to);
273 
274 	ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED);
275 	if (ret)
276 		return ret;
277 	ret = generic_file_read_iter(iocb, to);
278 	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
279 
280 	return ret;
281 }
282 
283 STATIC ssize_t
284 xfs_file_read_iter(
285 	struct kiocb		*iocb,
286 	struct iov_iter		*to)
287 {
288 	struct inode		*inode = file_inode(iocb->ki_filp);
289 	struct xfs_mount	*mp = XFS_I(inode)->i_mount;
290 	ssize_t			ret = 0;
291 
292 	XFS_STATS_INC(mp, xs_read_calls);
293 
294 	if (xfs_is_shutdown(mp))
295 		return -EIO;
296 
297 	if (IS_DAX(inode))
298 		ret = xfs_file_dax_read(iocb, to);
299 	else if (iocb->ki_flags & IOCB_DIRECT)
300 		ret = xfs_file_dio_read(iocb, to);
301 	else
302 		ret = xfs_file_buffered_read(iocb, to);
303 
304 	if (ret > 0)
305 		XFS_STATS_ADD(mp, xs_read_bytes, ret);
306 	return ret;
307 }
308 
309 /*
310  * Common pre-write limit and setup checks.
311  *
312  * Called with the iolocked held either shared and exclusive according to
313  * @iolock, and returns with it held.  Might upgrade the iolock to exclusive
314  * if called for a direct write beyond i_size.
315  */
316 STATIC ssize_t
317 xfs_file_write_checks(
318 	struct kiocb		*iocb,
319 	struct iov_iter		*from,
320 	unsigned int		*iolock)
321 {
322 	struct file		*file = iocb->ki_filp;
323 	struct inode		*inode = file->f_mapping->host;
324 	struct xfs_inode	*ip = XFS_I(inode);
325 	ssize_t			error = 0;
326 	size_t			count = iov_iter_count(from);
327 	bool			drained_dio = false;
328 	loff_t			isize;
329 
330 restart:
331 	error = generic_write_checks(iocb, from);
332 	if (error <= 0)
333 		return error;
334 
335 	if (iocb->ki_flags & IOCB_NOWAIT) {
336 		error = break_layout(inode, false);
337 		if (error == -EWOULDBLOCK)
338 			error = -EAGAIN;
339 	} else {
340 		error = xfs_break_layouts(inode, iolock, BREAK_WRITE);
341 	}
342 
343 	if (error)
344 		return error;
345 
346 	/*
347 	 * For changing security info in file_remove_privs() we need i_rwsem
348 	 * exclusively.
349 	 */
350 	if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
351 		xfs_iunlock(ip, *iolock);
352 		*iolock = XFS_IOLOCK_EXCL;
353 		error = xfs_ilock_iocb(iocb, *iolock);
354 		if (error) {
355 			*iolock = 0;
356 			return error;
357 		}
358 		goto restart;
359 	}
360 
361 	/*
362 	 * If the offset is beyond the size of the file, we need to zero any
363 	 * blocks that fall between the existing EOF and the start of this
364 	 * write.  If zeroing is needed and we are currently holding the iolock
365 	 * shared, we need to update it to exclusive which implies having to
366 	 * redo all checks before.
367 	 *
368 	 * We need to serialise against EOF updates that occur in IO completions
369 	 * here. We want to make sure that nobody is changing the size while we
370 	 * do this check until we have placed an IO barrier (i.e.  hold the
371 	 * XFS_IOLOCK_EXCL) that prevents new IO from being dispatched.  The
372 	 * spinlock effectively forms a memory barrier once we have the
373 	 * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value and
374 	 * hence be able to correctly determine if we need to run zeroing.
375 	 *
376 	 * We can do an unlocked check here safely as IO completion can only
377 	 * extend EOF. Truncate is locked out at this point, so the EOF can
378 	 * not move backwards, only forwards. Hence we only need to take the
379 	 * slow path and spin locks when we are at or beyond the current EOF.
380 	 */
381 	if (iocb->ki_pos <= i_size_read(inode))
382 		goto out;
383 
384 	spin_lock(&ip->i_flags_lock);
385 	isize = i_size_read(inode);
386 	if (iocb->ki_pos > isize) {
387 		spin_unlock(&ip->i_flags_lock);
388 
389 		if (iocb->ki_flags & IOCB_NOWAIT)
390 			return -EAGAIN;
391 
392 		if (!drained_dio) {
393 			if (*iolock == XFS_IOLOCK_SHARED) {
394 				xfs_iunlock(ip, *iolock);
395 				*iolock = XFS_IOLOCK_EXCL;
396 				xfs_ilock(ip, *iolock);
397 				iov_iter_reexpand(from, count);
398 			}
399 			/*
400 			 * We now have an IO submission barrier in place, but
401 			 * AIO can do EOF updates during IO completion and hence
402 			 * we now need to wait for all of them to drain. Non-AIO
403 			 * DIO will have drained before we are given the
404 			 * XFS_IOLOCK_EXCL, and so for most cases this wait is a
405 			 * no-op.
406 			 */
407 			inode_dio_wait(inode);
408 			drained_dio = true;
409 			goto restart;
410 		}
411 
412 		trace_xfs_zero_eof(ip, isize, iocb->ki_pos - isize);
413 		error = xfs_zero_range(ip, isize, iocb->ki_pos - isize, NULL);
414 		if (error)
415 			return error;
416 	} else
417 		spin_unlock(&ip->i_flags_lock);
418 
419 out:
420 	return kiocb_modified(iocb);
421 }
422 
423 static int
424 xfs_dio_write_end_io(
425 	struct kiocb		*iocb,
426 	ssize_t			size,
427 	int			error,
428 	unsigned		flags)
429 {
430 	struct inode		*inode = file_inode(iocb->ki_filp);
431 	struct xfs_inode	*ip = XFS_I(inode);
432 	loff_t			offset = iocb->ki_pos;
433 	unsigned int		nofs_flag;
434 
435 	trace_xfs_end_io_direct_write(ip, offset, size);
436 
437 	if (xfs_is_shutdown(ip->i_mount))
438 		return -EIO;
439 
440 	if (error)
441 		return error;
442 	if (!size)
443 		return 0;
444 
445 	/*
446 	 * Capture amount written on completion as we can't reliably account
447 	 * for it on submission.
448 	 */
449 	XFS_STATS_ADD(ip->i_mount, xs_write_bytes, size);
450 
451 	/*
452 	 * We can allocate memory here while doing writeback on behalf of
453 	 * memory reclaim.  To avoid memory allocation deadlocks set the
454 	 * task-wide nofs context for the following operations.
455 	 */
456 	nofs_flag = memalloc_nofs_save();
457 
458 	if (flags & IOMAP_DIO_COW) {
459 		error = xfs_reflink_end_cow(ip, offset, size);
460 		if (error)
461 			goto out;
462 	}
463 
464 	/*
465 	 * Unwritten conversion updates the in-core isize after extent
466 	 * conversion but before updating the on-disk size. Updating isize any
467 	 * earlier allows a racing dio read to find unwritten extents before
468 	 * they are converted.
469 	 */
470 	if (flags & IOMAP_DIO_UNWRITTEN) {
471 		error = xfs_iomap_write_unwritten(ip, offset, size, true);
472 		goto out;
473 	}
474 
475 	/*
476 	 * We need to update the in-core inode size here so that we don't end up
477 	 * with the on-disk inode size being outside the in-core inode size. We
478 	 * have no other method of updating EOF for AIO, so always do it here
479 	 * if necessary.
480 	 *
481 	 * We need to lock the test/set EOF update as we can be racing with
482 	 * other IO completions here to update the EOF. Failing to serialise
483 	 * here can result in EOF moving backwards and Bad Things Happen when
484 	 * that occurs.
485 	 *
486 	 * As IO completion only ever extends EOF, we can do an unlocked check
487 	 * here to avoid taking the spinlock. If we land within the current EOF,
488 	 * then we do not need to do an extending update at all, and we don't
489 	 * need to take the lock to check this. If we race with an update moving
490 	 * EOF, then we'll either still be beyond EOF and need to take the lock,
491 	 * or we'll be within EOF and we don't need to take it at all.
492 	 */
493 	if (offset + size <= i_size_read(inode))
494 		goto out;
495 
496 	spin_lock(&ip->i_flags_lock);
497 	if (offset + size > i_size_read(inode)) {
498 		i_size_write(inode, offset + size);
499 		spin_unlock(&ip->i_flags_lock);
500 		error = xfs_setfilesize(ip, offset, size);
501 	} else {
502 		spin_unlock(&ip->i_flags_lock);
503 	}
504 
505 out:
506 	memalloc_nofs_restore(nofs_flag);
507 	return error;
508 }
509 
510 static const struct iomap_dio_ops xfs_dio_write_ops = {
511 	.end_io		= xfs_dio_write_end_io,
512 };
513 
514 /*
515  * Handle block aligned direct I/O writes
516  */
517 static noinline ssize_t
518 xfs_file_dio_write_aligned(
519 	struct xfs_inode	*ip,
520 	struct kiocb		*iocb,
521 	struct iov_iter		*from)
522 {
523 	unsigned int		iolock = XFS_IOLOCK_SHARED;
524 	ssize_t			ret;
525 
526 	ret = xfs_ilock_iocb(iocb, iolock);
527 	if (ret)
528 		return ret;
529 	ret = xfs_file_write_checks(iocb, from, &iolock);
530 	if (ret)
531 		goto out_unlock;
532 
533 	/*
534 	 * We don't need to hold the IOLOCK exclusively across the IO, so demote
535 	 * the iolock back to shared if we had to take the exclusive lock in
536 	 * xfs_file_write_checks() for other reasons.
537 	 */
538 	if (iolock == XFS_IOLOCK_EXCL) {
539 		xfs_ilock_demote(ip, XFS_IOLOCK_EXCL);
540 		iolock = XFS_IOLOCK_SHARED;
541 	}
542 	trace_xfs_file_direct_write(iocb, from);
543 	ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
544 			   &xfs_dio_write_ops, 0, NULL, 0);
545 out_unlock:
546 	if (iolock)
547 		xfs_iunlock(ip, iolock);
548 	return ret;
549 }
550 
551 /*
552  * Handle block unaligned direct I/O writes
553  *
554  * In most cases direct I/O writes will be done holding IOLOCK_SHARED, allowing
555  * them to be done in parallel with reads and other direct I/O writes.  However,
556  * if the I/O is not aligned to filesystem blocks, the direct I/O layer may need
557  * to do sub-block zeroing and that requires serialisation against other direct
558  * I/O to the same block.  In this case we need to serialise the submission of
559  * the unaligned I/O so that we don't get racing block zeroing in the dio layer.
560  * In the case where sub-block zeroing is not required, we can do concurrent
561  * sub-block dios to the same block successfully.
562  *
563  * Optimistically submit the I/O using the shared lock first, but use the
564  * IOMAP_DIO_OVERWRITE_ONLY flag to tell the lower layers to return -EAGAIN
565  * if block allocation or partial block zeroing would be required.  In that case
566  * we try again with the exclusive lock.
567  */
568 static noinline ssize_t
569 xfs_file_dio_write_unaligned(
570 	struct xfs_inode	*ip,
571 	struct kiocb		*iocb,
572 	struct iov_iter		*from)
573 {
574 	size_t			isize = i_size_read(VFS_I(ip));
575 	size_t			count = iov_iter_count(from);
576 	unsigned int		iolock = XFS_IOLOCK_SHARED;
577 	unsigned int		flags = IOMAP_DIO_OVERWRITE_ONLY;
578 	ssize_t			ret;
579 
580 	/*
581 	 * Extending writes need exclusivity because of the sub-block zeroing
582 	 * that the DIO code always does for partial tail blocks beyond EOF, so
583 	 * don't even bother trying the fast path in this case.
584 	 */
585 	if (iocb->ki_pos > isize || iocb->ki_pos + count >= isize) {
586 		if (iocb->ki_flags & IOCB_NOWAIT)
587 			return -EAGAIN;
588 retry_exclusive:
589 		iolock = XFS_IOLOCK_EXCL;
590 		flags = IOMAP_DIO_FORCE_WAIT;
591 	}
592 
593 	ret = xfs_ilock_iocb(iocb, iolock);
594 	if (ret)
595 		return ret;
596 
597 	/*
598 	 * We can't properly handle unaligned direct I/O to reflink files yet,
599 	 * as we can't unshare a partial block.
600 	 */
601 	if (xfs_is_cow_inode(ip)) {
602 		trace_xfs_reflink_bounce_dio_write(iocb, from);
603 		ret = -ENOTBLK;
604 		goto out_unlock;
605 	}
606 
607 	ret = xfs_file_write_checks(iocb, from, &iolock);
608 	if (ret)
609 		goto out_unlock;
610 
611 	/*
612 	 * If we are doing exclusive unaligned I/O, this must be the only I/O
613 	 * in-flight.  Otherwise we risk data corruption due to unwritten extent
614 	 * conversions from the AIO end_io handler.  Wait for all other I/O to
615 	 * drain first.
616 	 */
617 	if (flags & IOMAP_DIO_FORCE_WAIT)
618 		inode_dio_wait(VFS_I(ip));
619 
620 	trace_xfs_file_direct_write(iocb, from);
621 	ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
622 			   &xfs_dio_write_ops, flags, NULL, 0);
623 
624 	/*
625 	 * Retry unaligned I/O with exclusive blocking semantics if the DIO
626 	 * layer rejected it for mapping or locking reasons. If we are doing
627 	 * nonblocking user I/O, propagate the error.
628 	 */
629 	if (ret == -EAGAIN && !(iocb->ki_flags & IOCB_NOWAIT)) {
630 		ASSERT(flags & IOMAP_DIO_OVERWRITE_ONLY);
631 		xfs_iunlock(ip, iolock);
632 		goto retry_exclusive;
633 	}
634 
635 out_unlock:
636 	if (iolock)
637 		xfs_iunlock(ip, iolock);
638 	return ret;
639 }
640 
641 static ssize_t
642 xfs_file_dio_write(
643 	struct kiocb		*iocb,
644 	struct iov_iter		*from)
645 {
646 	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
647 	struct xfs_buftarg      *target = xfs_inode_buftarg(ip);
648 	size_t			count = iov_iter_count(from);
649 
650 	/* direct I/O must be aligned to device logical sector size */
651 	if ((iocb->ki_pos | count) & target->bt_logical_sectormask)
652 		return -EINVAL;
653 	if ((iocb->ki_pos | count) & ip->i_mount->m_blockmask)
654 		return xfs_file_dio_write_unaligned(ip, iocb, from);
655 	return xfs_file_dio_write_aligned(ip, iocb, from);
656 }
657 
658 static noinline ssize_t
659 xfs_file_dax_write(
660 	struct kiocb		*iocb,
661 	struct iov_iter		*from)
662 {
663 	struct inode		*inode = iocb->ki_filp->f_mapping->host;
664 	struct xfs_inode	*ip = XFS_I(inode);
665 	unsigned int		iolock = XFS_IOLOCK_EXCL;
666 	ssize_t			ret, error = 0;
667 	loff_t			pos;
668 
669 	ret = xfs_ilock_iocb(iocb, iolock);
670 	if (ret)
671 		return ret;
672 	ret = xfs_file_write_checks(iocb, from, &iolock);
673 	if (ret)
674 		goto out;
675 
676 	pos = iocb->ki_pos;
677 
678 	trace_xfs_file_dax_write(iocb, from);
679 	ret = dax_iomap_rw(iocb, from, &xfs_dax_write_iomap_ops);
680 	if (ret > 0 && iocb->ki_pos > i_size_read(inode)) {
681 		i_size_write(inode, iocb->ki_pos);
682 		error = xfs_setfilesize(ip, pos, ret);
683 	}
684 out:
685 	if (iolock)
686 		xfs_iunlock(ip, iolock);
687 	if (error)
688 		return error;
689 
690 	if (ret > 0) {
691 		XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
692 
693 		/* Handle various SYNC-type writes */
694 		ret = generic_write_sync(iocb, ret);
695 	}
696 	return ret;
697 }
698 
699 STATIC ssize_t
700 xfs_file_buffered_write(
701 	struct kiocb		*iocb,
702 	struct iov_iter		*from)
703 {
704 	struct inode		*inode = iocb->ki_filp->f_mapping->host;
705 	struct xfs_inode	*ip = XFS_I(inode);
706 	ssize_t			ret;
707 	bool			cleared_space = false;
708 	unsigned int		iolock;
709 
710 write_retry:
711 	iolock = XFS_IOLOCK_EXCL;
712 	ret = xfs_ilock_iocb(iocb, iolock);
713 	if (ret)
714 		return ret;
715 
716 	ret = xfs_file_write_checks(iocb, from, &iolock);
717 	if (ret)
718 		goto out;
719 
720 	/* We can write back this queue in page reclaim */
721 	current->backing_dev_info = inode_to_bdi(inode);
722 
723 	trace_xfs_file_buffered_write(iocb, from);
724 	ret = iomap_file_buffered_write(iocb, from,
725 			&xfs_buffered_write_iomap_ops);
726 	if (likely(ret >= 0))
727 		iocb->ki_pos += ret;
728 
729 	/*
730 	 * If we hit a space limit, try to free up some lingering preallocated
731 	 * space before returning an error. In the case of ENOSPC, first try to
732 	 * write back all dirty inodes to free up some of the excess reserved
733 	 * metadata space. This reduces the chances that the eofblocks scan
734 	 * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this
735 	 * also behaves as a filter to prevent too many eofblocks scans from
736 	 * running at the same time.  Use a synchronous scan to increase the
737 	 * effectiveness of the scan.
738 	 */
739 	if (ret == -EDQUOT && !cleared_space) {
740 		xfs_iunlock(ip, iolock);
741 		xfs_blockgc_free_quota(ip, XFS_ICWALK_FLAG_SYNC);
742 		cleared_space = true;
743 		goto write_retry;
744 	} else if (ret == -ENOSPC && !cleared_space) {
745 		struct xfs_icwalk	icw = {0};
746 
747 		cleared_space = true;
748 		xfs_flush_inodes(ip->i_mount);
749 
750 		xfs_iunlock(ip, iolock);
751 		icw.icw_flags = XFS_ICWALK_FLAG_SYNC;
752 		xfs_blockgc_free_space(ip->i_mount, &icw);
753 		goto write_retry;
754 	}
755 
756 	current->backing_dev_info = NULL;
757 out:
758 	if (iolock)
759 		xfs_iunlock(ip, iolock);
760 
761 	if (ret > 0) {
762 		XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
763 		/* Handle various SYNC-type writes */
764 		ret = generic_write_sync(iocb, ret);
765 	}
766 	return ret;
767 }
768 
769 STATIC ssize_t
770 xfs_file_write_iter(
771 	struct kiocb		*iocb,
772 	struct iov_iter		*from)
773 {
774 	struct inode		*inode = iocb->ki_filp->f_mapping->host;
775 	struct xfs_inode	*ip = XFS_I(inode);
776 	ssize_t			ret;
777 	size_t			ocount = iov_iter_count(from);
778 
779 	XFS_STATS_INC(ip->i_mount, xs_write_calls);
780 
781 	if (ocount == 0)
782 		return 0;
783 
784 	if (xfs_is_shutdown(ip->i_mount))
785 		return -EIO;
786 
787 	if (IS_DAX(inode))
788 		return xfs_file_dax_write(iocb, from);
789 
790 	if (iocb->ki_flags & IOCB_DIRECT) {
791 		/*
792 		 * Allow a directio write to fall back to a buffered
793 		 * write *only* in the case that we're doing a reflink
794 		 * CoW.  In all other directio scenarios we do not
795 		 * allow an operation to fall back to buffered mode.
796 		 */
797 		ret = xfs_file_dio_write(iocb, from);
798 		if (ret != -ENOTBLK)
799 			return ret;
800 	}
801 
802 	return xfs_file_buffered_write(iocb, from);
803 }
804 
805 static void
806 xfs_wait_dax_page(
807 	struct inode		*inode)
808 {
809 	struct xfs_inode        *ip = XFS_I(inode);
810 
811 	xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
812 	schedule();
813 	xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
814 }
815 
816 int
817 xfs_break_dax_layouts(
818 	struct inode		*inode,
819 	bool			*retry)
820 {
821 	struct page		*page;
822 
823 	ASSERT(xfs_isilocked(XFS_I(inode), XFS_MMAPLOCK_EXCL));
824 
825 	page = dax_layout_busy_page(inode->i_mapping);
826 	if (!page)
827 		return 0;
828 
829 	*retry = true;
830 	return ___wait_var_event(&page->_refcount,
831 			atomic_read(&page->_refcount) == 1, TASK_INTERRUPTIBLE,
832 			0, 0, xfs_wait_dax_page(inode));
833 }
834 
835 int
836 xfs_break_layouts(
837 	struct inode		*inode,
838 	uint			*iolock,
839 	enum layout_break_reason reason)
840 {
841 	bool			retry;
842 	int			error;
843 
844 	ASSERT(xfs_isilocked(XFS_I(inode), XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL));
845 
846 	do {
847 		retry = false;
848 		switch (reason) {
849 		case BREAK_UNMAP:
850 			error = xfs_break_dax_layouts(inode, &retry);
851 			if (error || retry)
852 				break;
853 			fallthrough;
854 		case BREAK_WRITE:
855 			error = xfs_break_leased_layouts(inode, iolock, &retry);
856 			break;
857 		default:
858 			WARN_ON_ONCE(1);
859 			error = -EINVAL;
860 		}
861 	} while (error == 0 && retry);
862 
863 	return error;
864 }
865 
866 /* Does this file, inode, or mount want synchronous writes? */
867 static inline bool xfs_file_sync_writes(struct file *filp)
868 {
869 	struct xfs_inode	*ip = XFS_I(file_inode(filp));
870 
871 	if (xfs_has_wsync(ip->i_mount))
872 		return true;
873 	if (filp->f_flags & (__O_SYNC | O_DSYNC))
874 		return true;
875 	if (IS_SYNC(file_inode(filp)))
876 		return true;
877 
878 	return false;
879 }
880 
881 #define	XFS_FALLOC_FL_SUPPORTED						\
882 		(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |		\
883 		 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE |	\
884 		 FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE)
885 
886 STATIC long
887 xfs_file_fallocate(
888 	struct file		*file,
889 	int			mode,
890 	loff_t			offset,
891 	loff_t			len)
892 {
893 	struct inode		*inode = file_inode(file);
894 	struct xfs_inode	*ip = XFS_I(inode);
895 	long			error;
896 	uint			iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL;
897 	loff_t			new_size = 0;
898 	bool			do_file_insert = false;
899 
900 	if (!S_ISREG(inode->i_mode))
901 		return -EINVAL;
902 	if (mode & ~XFS_FALLOC_FL_SUPPORTED)
903 		return -EOPNOTSUPP;
904 
905 	xfs_ilock(ip, iolock);
906 	error = xfs_break_layouts(inode, &iolock, BREAK_UNMAP);
907 	if (error)
908 		goto out_unlock;
909 
910 	/*
911 	 * Must wait for all AIO to complete before we continue as AIO can
912 	 * change the file size on completion without holding any locks we
913 	 * currently hold. We must do this first because AIO can update both
914 	 * the on disk and in memory inode sizes, and the operations that follow
915 	 * require the in-memory size to be fully up-to-date.
916 	 */
917 	inode_dio_wait(inode);
918 
919 	/*
920 	 * Now AIO and DIO has drained we flush and (if necessary) invalidate
921 	 * the cached range over the first operation we are about to run.
922 	 *
923 	 * We care about zero and collapse here because they both run a hole
924 	 * punch over the range first. Because that can zero data, and the range
925 	 * of invalidation for the shift operations is much larger, we still do
926 	 * the required flush for collapse in xfs_prepare_shift().
927 	 *
928 	 * Insert has the same range requirements as collapse, and we extend the
929 	 * file first which can zero data. Hence insert has the same
930 	 * flush/invalidate requirements as collapse and so they are both
931 	 * handled at the right time by xfs_prepare_shift().
932 	 */
933 	if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE |
934 		    FALLOC_FL_COLLAPSE_RANGE)) {
935 		error = xfs_flush_unmap_range(ip, offset, len);
936 		if (error)
937 			goto out_unlock;
938 	}
939 
940 	error = file_modified(file);
941 	if (error)
942 		goto out_unlock;
943 
944 	if (mode & FALLOC_FL_PUNCH_HOLE) {
945 		error = xfs_free_file_space(ip, offset, len);
946 		if (error)
947 			goto out_unlock;
948 	} else if (mode & FALLOC_FL_COLLAPSE_RANGE) {
949 		if (!xfs_is_falloc_aligned(ip, offset, len)) {
950 			error = -EINVAL;
951 			goto out_unlock;
952 		}
953 
954 		/*
955 		 * There is no need to overlap collapse range with EOF,
956 		 * in which case it is effectively a truncate operation
957 		 */
958 		if (offset + len >= i_size_read(inode)) {
959 			error = -EINVAL;
960 			goto out_unlock;
961 		}
962 
963 		new_size = i_size_read(inode) - len;
964 
965 		error = xfs_collapse_file_space(ip, offset, len);
966 		if (error)
967 			goto out_unlock;
968 	} else if (mode & FALLOC_FL_INSERT_RANGE) {
969 		loff_t		isize = i_size_read(inode);
970 
971 		if (!xfs_is_falloc_aligned(ip, offset, len)) {
972 			error = -EINVAL;
973 			goto out_unlock;
974 		}
975 
976 		/*
977 		 * New inode size must not exceed ->s_maxbytes, accounting for
978 		 * possible signed overflow.
979 		 */
980 		if (inode->i_sb->s_maxbytes - isize < len) {
981 			error = -EFBIG;
982 			goto out_unlock;
983 		}
984 		new_size = isize + len;
985 
986 		/* Offset should be less than i_size */
987 		if (offset >= isize) {
988 			error = -EINVAL;
989 			goto out_unlock;
990 		}
991 		do_file_insert = true;
992 	} else {
993 		if (!(mode & FALLOC_FL_KEEP_SIZE) &&
994 		    offset + len > i_size_read(inode)) {
995 			new_size = offset + len;
996 			error = inode_newsize_ok(inode, new_size);
997 			if (error)
998 				goto out_unlock;
999 		}
1000 
1001 		if (mode & FALLOC_FL_ZERO_RANGE) {
1002 			/*
1003 			 * Punch a hole and prealloc the range.  We use a hole
1004 			 * punch rather than unwritten extent conversion for two
1005 			 * reasons:
1006 			 *
1007 			 *   1.) Hole punch handles partial block zeroing for us.
1008 			 *   2.) If prealloc returns ENOSPC, the file range is
1009 			 *       still zero-valued by virtue of the hole punch.
1010 			 */
1011 			unsigned int blksize = i_blocksize(inode);
1012 
1013 			trace_xfs_zero_file_space(ip);
1014 
1015 			error = xfs_free_file_space(ip, offset, len);
1016 			if (error)
1017 				goto out_unlock;
1018 
1019 			len = round_up(offset + len, blksize) -
1020 			      round_down(offset, blksize);
1021 			offset = round_down(offset, blksize);
1022 		} else if (mode & FALLOC_FL_UNSHARE_RANGE) {
1023 			error = xfs_reflink_unshare(ip, offset, len);
1024 			if (error)
1025 				goto out_unlock;
1026 		} else {
1027 			/*
1028 			 * If always_cow mode we can't use preallocations and
1029 			 * thus should not create them.
1030 			 */
1031 			if (xfs_is_always_cow_inode(ip)) {
1032 				error = -EOPNOTSUPP;
1033 				goto out_unlock;
1034 			}
1035 		}
1036 
1037 		if (!xfs_is_always_cow_inode(ip)) {
1038 			error = xfs_alloc_file_space(ip, offset, len);
1039 			if (error)
1040 				goto out_unlock;
1041 		}
1042 	}
1043 
1044 	/* Change file size if needed */
1045 	if (new_size) {
1046 		struct iattr iattr;
1047 
1048 		iattr.ia_valid = ATTR_SIZE;
1049 		iattr.ia_size = new_size;
1050 		error = xfs_vn_setattr_size(file_mnt_user_ns(file),
1051 					    file_dentry(file), &iattr);
1052 		if (error)
1053 			goto out_unlock;
1054 	}
1055 
1056 	/*
1057 	 * Perform hole insertion now that the file size has been
1058 	 * updated so that if we crash during the operation we don't
1059 	 * leave shifted extents past EOF and hence losing access to
1060 	 * the data that is contained within them.
1061 	 */
1062 	if (do_file_insert) {
1063 		error = xfs_insert_file_space(ip, offset, len);
1064 		if (error)
1065 			goto out_unlock;
1066 	}
1067 
1068 	if (xfs_file_sync_writes(file))
1069 		error = xfs_log_force_inode(ip);
1070 
1071 out_unlock:
1072 	xfs_iunlock(ip, iolock);
1073 	return error;
1074 }
1075 
1076 STATIC int
1077 xfs_file_fadvise(
1078 	struct file	*file,
1079 	loff_t		start,
1080 	loff_t		end,
1081 	int		advice)
1082 {
1083 	struct xfs_inode *ip = XFS_I(file_inode(file));
1084 	int ret;
1085 	int lockflags = 0;
1086 
1087 	/*
1088 	 * Operations creating pages in page cache need protection from hole
1089 	 * punching and similar ops
1090 	 */
1091 	if (advice == POSIX_FADV_WILLNEED) {
1092 		lockflags = XFS_IOLOCK_SHARED;
1093 		xfs_ilock(ip, lockflags);
1094 	}
1095 	ret = generic_fadvise(file, start, end, advice);
1096 	if (lockflags)
1097 		xfs_iunlock(ip, lockflags);
1098 	return ret;
1099 }
1100 
1101 STATIC loff_t
1102 xfs_file_remap_range(
1103 	struct file		*file_in,
1104 	loff_t			pos_in,
1105 	struct file		*file_out,
1106 	loff_t			pos_out,
1107 	loff_t			len,
1108 	unsigned int		remap_flags)
1109 {
1110 	struct inode		*inode_in = file_inode(file_in);
1111 	struct xfs_inode	*src = XFS_I(inode_in);
1112 	struct inode		*inode_out = file_inode(file_out);
1113 	struct xfs_inode	*dest = XFS_I(inode_out);
1114 	struct xfs_mount	*mp = src->i_mount;
1115 	loff_t			remapped = 0;
1116 	xfs_extlen_t		cowextsize;
1117 	int			ret;
1118 
1119 	if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY))
1120 		return -EINVAL;
1121 
1122 	if (!xfs_has_reflink(mp))
1123 		return -EOPNOTSUPP;
1124 
1125 	if (xfs_is_shutdown(mp))
1126 		return -EIO;
1127 
1128 	/* Prepare and then clone file data. */
1129 	ret = xfs_reflink_remap_prep(file_in, pos_in, file_out, pos_out,
1130 			&len, remap_flags);
1131 	if (ret || len == 0)
1132 		return ret;
1133 
1134 	trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out);
1135 
1136 	ret = xfs_reflink_remap_blocks(src, pos_in, dest, pos_out, len,
1137 			&remapped);
1138 	if (ret)
1139 		goto out_unlock;
1140 
1141 	/*
1142 	 * Carry the cowextsize hint from src to dest if we're sharing the
1143 	 * entire source file to the entire destination file, the source file
1144 	 * has a cowextsize hint, and the destination file does not.
1145 	 */
1146 	cowextsize = 0;
1147 	if (pos_in == 0 && len == i_size_read(inode_in) &&
1148 	    (src->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) &&
1149 	    pos_out == 0 && len >= i_size_read(inode_out) &&
1150 	    !(dest->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE))
1151 		cowextsize = src->i_cowextsize;
1152 
1153 	ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize,
1154 			remap_flags);
1155 	if (ret)
1156 		goto out_unlock;
1157 
1158 	if (xfs_file_sync_writes(file_in) || xfs_file_sync_writes(file_out))
1159 		xfs_log_force_inode(dest);
1160 out_unlock:
1161 	xfs_iunlock2_io_mmap(src, dest);
1162 	if (ret)
1163 		trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_);
1164 	return remapped > 0 ? remapped : ret;
1165 }
1166 
1167 STATIC int
1168 xfs_file_open(
1169 	struct inode	*inode,
1170 	struct file	*file)
1171 {
1172 	if (xfs_is_shutdown(XFS_M(inode->i_sb)))
1173 		return -EIO;
1174 	file->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC | FMODE_BUF_WASYNC;
1175 	return generic_file_open(inode, file);
1176 }
1177 
1178 STATIC int
1179 xfs_dir_open(
1180 	struct inode	*inode,
1181 	struct file	*file)
1182 {
1183 	struct xfs_inode *ip = XFS_I(inode);
1184 	unsigned int	mode;
1185 	int		error;
1186 
1187 	error = xfs_file_open(inode, file);
1188 	if (error)
1189 		return error;
1190 
1191 	/*
1192 	 * If there are any blocks, read-ahead block 0 as we're almost
1193 	 * certain to have the next operation be a read there.
1194 	 */
1195 	mode = xfs_ilock_data_map_shared(ip);
1196 	if (ip->i_df.if_nextents > 0)
1197 		error = xfs_dir3_data_readahead(ip, 0, 0);
1198 	xfs_iunlock(ip, mode);
1199 	return error;
1200 }
1201 
1202 STATIC int
1203 xfs_file_release(
1204 	struct inode	*inode,
1205 	struct file	*filp)
1206 {
1207 	return xfs_release(XFS_I(inode));
1208 }
1209 
1210 STATIC int
1211 xfs_file_readdir(
1212 	struct file	*file,
1213 	struct dir_context *ctx)
1214 {
1215 	struct inode	*inode = file_inode(file);
1216 	xfs_inode_t	*ip = XFS_I(inode);
1217 	size_t		bufsize;
1218 
1219 	/*
1220 	 * The Linux API doesn't pass down the total size of the buffer
1221 	 * we read into down to the filesystem.  With the filldir concept
1222 	 * it's not needed for correct information, but the XFS dir2 leaf
1223 	 * code wants an estimate of the buffer size to calculate it's
1224 	 * readahead window and size the buffers used for mapping to
1225 	 * physical blocks.
1226 	 *
1227 	 * Try to give it an estimate that's good enough, maybe at some
1228 	 * point we can change the ->readdir prototype to include the
1229 	 * buffer size.  For now we use the current glibc buffer size.
1230 	 */
1231 	bufsize = (size_t)min_t(loff_t, XFS_READDIR_BUFSIZE, ip->i_disk_size);
1232 
1233 	return xfs_readdir(NULL, ip, ctx, bufsize);
1234 }
1235 
1236 STATIC loff_t
1237 xfs_file_llseek(
1238 	struct file	*file,
1239 	loff_t		offset,
1240 	int		whence)
1241 {
1242 	struct inode		*inode = file->f_mapping->host;
1243 
1244 	if (xfs_is_shutdown(XFS_I(inode)->i_mount))
1245 		return -EIO;
1246 
1247 	switch (whence) {
1248 	default:
1249 		return generic_file_llseek(file, offset, whence);
1250 	case SEEK_HOLE:
1251 		offset = iomap_seek_hole(inode, offset, &xfs_seek_iomap_ops);
1252 		break;
1253 	case SEEK_DATA:
1254 		offset = iomap_seek_data(inode, offset, &xfs_seek_iomap_ops);
1255 		break;
1256 	}
1257 
1258 	if (offset < 0)
1259 		return offset;
1260 	return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
1261 }
1262 
1263 #ifdef CONFIG_FS_DAX
1264 static inline vm_fault_t
1265 xfs_dax_fault(
1266 	struct vm_fault		*vmf,
1267 	enum page_entry_size	pe_size,
1268 	bool			write_fault,
1269 	pfn_t			*pfn)
1270 {
1271 	return dax_iomap_fault(vmf, pe_size, pfn, NULL,
1272 			(write_fault && !vmf->cow_page) ?
1273 				&xfs_dax_write_iomap_ops :
1274 				&xfs_read_iomap_ops);
1275 }
1276 #else
1277 static inline vm_fault_t
1278 xfs_dax_fault(
1279 	struct vm_fault		*vmf,
1280 	enum page_entry_size	pe_size,
1281 	bool			write_fault,
1282 	pfn_t			*pfn)
1283 {
1284 	ASSERT(0);
1285 	return VM_FAULT_SIGBUS;
1286 }
1287 #endif
1288 
1289 /*
1290  * Locking for serialisation of IO during page faults. This results in a lock
1291  * ordering of:
1292  *
1293  * mmap_lock (MM)
1294  *   sb_start_pagefault(vfs, freeze)
1295  *     invalidate_lock (vfs/XFS_MMAPLOCK - truncate serialisation)
1296  *       page_lock (MM)
1297  *         i_lock (XFS - extent map serialisation)
1298  */
1299 static vm_fault_t
1300 __xfs_filemap_fault(
1301 	struct vm_fault		*vmf,
1302 	enum page_entry_size	pe_size,
1303 	bool			write_fault)
1304 {
1305 	struct inode		*inode = file_inode(vmf->vma->vm_file);
1306 	struct xfs_inode	*ip = XFS_I(inode);
1307 	vm_fault_t		ret;
1308 
1309 	trace_xfs_filemap_fault(ip, pe_size, write_fault);
1310 
1311 	if (write_fault) {
1312 		sb_start_pagefault(inode->i_sb);
1313 		file_update_time(vmf->vma->vm_file);
1314 	}
1315 
1316 	if (IS_DAX(inode)) {
1317 		pfn_t pfn;
1318 
1319 		xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1320 		ret = xfs_dax_fault(vmf, pe_size, write_fault, &pfn);
1321 		if (ret & VM_FAULT_NEEDDSYNC)
1322 			ret = dax_finish_sync_fault(vmf, pe_size, pfn);
1323 		xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1324 	} else {
1325 		if (write_fault) {
1326 			xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1327 			ret = iomap_page_mkwrite(vmf,
1328 					&xfs_page_mkwrite_iomap_ops);
1329 			xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1330 		} else {
1331 			ret = filemap_fault(vmf);
1332 		}
1333 	}
1334 
1335 	if (write_fault)
1336 		sb_end_pagefault(inode->i_sb);
1337 	return ret;
1338 }
1339 
1340 static inline bool
1341 xfs_is_write_fault(
1342 	struct vm_fault		*vmf)
1343 {
1344 	return (vmf->flags & FAULT_FLAG_WRITE) &&
1345 	       (vmf->vma->vm_flags & VM_SHARED);
1346 }
1347 
1348 static vm_fault_t
1349 xfs_filemap_fault(
1350 	struct vm_fault		*vmf)
1351 {
1352 	/* DAX can shortcut the normal fault path on write faults! */
1353 	return __xfs_filemap_fault(vmf, PE_SIZE_PTE,
1354 			IS_DAX(file_inode(vmf->vma->vm_file)) &&
1355 			xfs_is_write_fault(vmf));
1356 }
1357 
1358 static vm_fault_t
1359 xfs_filemap_huge_fault(
1360 	struct vm_fault		*vmf,
1361 	enum page_entry_size	pe_size)
1362 {
1363 	if (!IS_DAX(file_inode(vmf->vma->vm_file)))
1364 		return VM_FAULT_FALLBACK;
1365 
1366 	/* DAX can shortcut the normal fault path on write faults! */
1367 	return __xfs_filemap_fault(vmf, pe_size,
1368 			xfs_is_write_fault(vmf));
1369 }
1370 
1371 static vm_fault_t
1372 xfs_filemap_page_mkwrite(
1373 	struct vm_fault		*vmf)
1374 {
1375 	return __xfs_filemap_fault(vmf, PE_SIZE_PTE, true);
1376 }
1377 
1378 /*
1379  * pfn_mkwrite was originally intended to ensure we capture time stamp updates
1380  * on write faults. In reality, it needs to serialise against truncate and
1381  * prepare memory for writing so handle is as standard write fault.
1382  */
1383 static vm_fault_t
1384 xfs_filemap_pfn_mkwrite(
1385 	struct vm_fault		*vmf)
1386 {
1387 
1388 	return __xfs_filemap_fault(vmf, PE_SIZE_PTE, true);
1389 }
1390 
1391 static vm_fault_t
1392 xfs_filemap_map_pages(
1393 	struct vm_fault		*vmf,
1394 	pgoff_t			start_pgoff,
1395 	pgoff_t			end_pgoff)
1396 {
1397 	struct inode		*inode = file_inode(vmf->vma->vm_file);
1398 	vm_fault_t ret;
1399 
1400 	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1401 	ret = filemap_map_pages(vmf, start_pgoff, end_pgoff);
1402 	xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1403 	return ret;
1404 }
1405 
1406 static const struct vm_operations_struct xfs_file_vm_ops = {
1407 	.fault		= xfs_filemap_fault,
1408 	.huge_fault	= xfs_filemap_huge_fault,
1409 	.map_pages	= xfs_filemap_map_pages,
1410 	.page_mkwrite	= xfs_filemap_page_mkwrite,
1411 	.pfn_mkwrite	= xfs_filemap_pfn_mkwrite,
1412 };
1413 
1414 STATIC int
1415 xfs_file_mmap(
1416 	struct file		*file,
1417 	struct vm_area_struct	*vma)
1418 {
1419 	struct inode		*inode = file_inode(file);
1420 	struct xfs_buftarg	*target = xfs_inode_buftarg(XFS_I(inode));
1421 
1422 	/*
1423 	 * We don't support synchronous mappings for non-DAX files and
1424 	 * for DAX files if underneath dax_device is not synchronous.
1425 	 */
1426 	if (!daxdev_mapping_supported(vma, target->bt_daxdev))
1427 		return -EOPNOTSUPP;
1428 
1429 	file_accessed(file);
1430 	vma->vm_ops = &xfs_file_vm_ops;
1431 	if (IS_DAX(inode))
1432 		vma->vm_flags |= VM_HUGEPAGE;
1433 	return 0;
1434 }
1435 
1436 const struct file_operations xfs_file_operations = {
1437 	.llseek		= xfs_file_llseek,
1438 	.read_iter	= xfs_file_read_iter,
1439 	.write_iter	= xfs_file_write_iter,
1440 	.splice_read	= generic_file_splice_read,
1441 	.splice_write	= iter_file_splice_write,
1442 	.iopoll		= iocb_bio_iopoll,
1443 	.unlocked_ioctl	= xfs_file_ioctl,
1444 #ifdef CONFIG_COMPAT
1445 	.compat_ioctl	= xfs_file_compat_ioctl,
1446 #endif
1447 	.mmap		= xfs_file_mmap,
1448 	.mmap_supported_flags = MAP_SYNC,
1449 	.open		= xfs_file_open,
1450 	.release	= xfs_file_release,
1451 	.fsync		= xfs_file_fsync,
1452 	.get_unmapped_area = thp_get_unmapped_area,
1453 	.fallocate	= xfs_file_fallocate,
1454 	.fadvise	= xfs_file_fadvise,
1455 	.remap_file_range = xfs_file_remap_range,
1456 };
1457 
1458 const struct file_operations xfs_dir_file_operations = {
1459 	.open		= xfs_dir_open,
1460 	.read		= generic_read_dir,
1461 	.iterate_shared	= xfs_file_readdir,
1462 	.llseek		= generic_file_llseek,
1463 	.unlocked_ioctl	= xfs_file_ioctl,
1464 #ifdef CONFIG_COMPAT
1465 	.compat_ioctl	= xfs_file_compat_ioctl,
1466 #endif
1467 	.fsync		= xfs_dir_fsync,
1468 };
1469