1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2016 Trond Myklebust
4 *
5 * I/O and data path helper functionality.
6 */
7
8 #include <linux/types.h>
9 #include <linux/kernel.h>
10 #include <linux/bitops.h>
11 #include <linux/rwsem.h>
12 #include <linux/fs.h>
13 #include <linux/nfs_fs.h>
14
15 #include "internal.h"
16
17 /**
18 * nfs_start_io_read - declare the file is being used for buffered reads
19 * @inode: file inode
20 *
21 * Declare that a buffered read operation is about to start, and ensure
22 * that we block all direct I/O.
23 * On exit, the function ensures that the NFS_INO_ODIRECT flag is unset,
24 * and holds a shared lock on inode->i_rwsem to ensure that the flag
25 * cannot be changed.
26 * In practice, this means that buffered read operations are allowed to
27 * execute in parallel, thanks to the shared lock, whereas direct I/O
28 * operations need to wait to grab an exclusive lock in order to set
29 * NFS_INO_ODIRECT.
30 * Note that buffered writes and truncates both take a write lock on
31 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
32 */
33 int
nfs_start_io_read(struct inode * inode)34 nfs_start_io_read(struct inode *inode)
35 {
36 struct nfs_inode *nfsi = NFS_I(inode);
37 int err;
38
39 /* Be an optimist! */
40 err = down_read_killable(&inode->i_rwsem);
41 if (err)
42 return err;
43 if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0)
44 return 0;
45 up_read(&inode->i_rwsem);
46
47 /* Slow path.... */
48 err = down_write_killable(&inode->i_rwsem);
49 if (err)
50 return err;
51 nfs_file_block_o_direct(nfsi);
52 downgrade_write(&inode->i_rwsem);
53
54 return 0;
55 }
56
57 /**
58 * nfs_end_io_read - declare that the buffered read operation is done
59 * @inode: file inode
60 *
61 * Declare that a buffered read operation is done, and release the shared
62 * lock on inode->i_rwsem.
63 */
64 void
nfs_end_io_read(struct inode * inode)65 nfs_end_io_read(struct inode *inode)
66 {
67 up_read(&inode->i_rwsem);
68 }
69
70 /**
71 * nfs_start_io_write - declare the file is being used for buffered writes
72 * @inode: file inode
73 *
74 * Declare that a buffered read operation is about to start, and ensure
75 * that we block all direct I/O.
76 */
77 int
nfs_start_io_write(struct inode * inode)78 nfs_start_io_write(struct inode *inode)
79 {
80 int err;
81
82 err = down_write_killable(&inode->i_rwsem);
83 if (!err)
84 nfs_file_block_o_direct(NFS_I(inode));
85 return err;
86 }
87 EXPORT_SYMBOL_GPL(nfs_start_io_write);
88
89 /**
90 * nfs_end_io_write - declare that the buffered write operation is done
91 * @inode: file inode
92 *
93 * Declare that a buffered write operation is done, and release the
94 * lock on inode->i_rwsem.
95 */
96 void
nfs_end_io_write(struct inode * inode)97 nfs_end_io_write(struct inode *inode)
98 {
99 up_write(&inode->i_rwsem);
100 }
101 EXPORT_SYMBOL_GPL(nfs_end_io_write);
102
103 /* Call with exclusively locked inode->i_rwsem */
nfs_block_buffered(struct nfs_inode * nfsi,struct inode * inode)104 static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode)
105 {
106 if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) {
107 set_bit(NFS_INO_ODIRECT, &nfsi->flags);
108 nfs_sync_mapping(inode->i_mapping);
109 }
110 }
111
112 /**
113 * nfs_start_io_direct - declare the file is being used for direct i/o
114 * @inode: file inode
115 *
116 * Declare that a direct I/O operation is about to start, and ensure
117 * that we block all buffered I/O.
118 * On exit, the function ensures that the NFS_INO_ODIRECT flag is set,
119 * and holds a shared lock on inode->i_rwsem to ensure that the flag
120 * cannot be changed.
121 * In practice, this means that direct I/O operations are allowed to
122 * execute in parallel, thanks to the shared lock, whereas buffered I/O
123 * operations need to wait to grab an exclusive lock in order to clear
124 * NFS_INO_ODIRECT.
125 * Note that buffered writes and truncates both take a write lock on
126 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
127 */
128 int
nfs_start_io_direct(struct inode * inode)129 nfs_start_io_direct(struct inode *inode)
130 {
131 struct nfs_inode *nfsi = NFS_I(inode);
132 int err;
133
134 /* Be an optimist! */
135 err = down_read_killable(&inode->i_rwsem);
136 if (err)
137 return err;
138 if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) != 0)
139 return 0;
140 up_read(&inode->i_rwsem);
141
142 /* Slow path.... */
143 err = down_write_killable(&inode->i_rwsem);
144 if (err)
145 return err;
146 nfs_block_buffered(nfsi, inode);
147 downgrade_write(&inode->i_rwsem);
148
149 return 0;
150 }
151
152 /**
153 * nfs_end_io_direct - declare that the direct i/o operation is done
154 * @inode: file inode
155 *
156 * Declare that a direct I/O operation is done, and release the shared
157 * lock on inode->i_rwsem.
158 */
159 void
nfs_end_io_direct(struct inode * inode)160 nfs_end_io_direct(struct inode *inode)
161 {
162 up_read(&inode->i_rwsem);
163 }
164