1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * I/O and data path helper functionality.
4 *
5 * Borrowed from NFS Copyright (c) 2016 Trond Myklebust
6 */
7
8 #include <linux/kernel.h>
9 #include <linux/netfs.h>
10 #include "internal.h"
11
12 /*
13 * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish
14 * @inode: inode to wait for
15 *
16 * Waits for all pending direct I/O requests to finish so that we can
17 * proceed with a truncate or equivalent operation.
18 *
19 * Must be called under a lock that serializes taking new references
20 * to i_dio_count, usually by inode->i_mutex.
21 */
netfs_inode_dio_wait_interruptible(struct inode * inode)22 static int netfs_inode_dio_wait_interruptible(struct inode *inode)
23 {
24 if (inode_dio_finished(inode))
25 return 0;
26
27 inode_dio_wait_interruptible(inode);
28 return !inode_dio_finished(inode) ? -ERESTARTSYS : 0;
29 }
30
31 /* Call with exclusively locked inode->i_rwsem */
netfs_block_o_direct(struct netfs_inode * ictx)32 static int netfs_block_o_direct(struct netfs_inode *ictx)
33 {
34 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags))
35 return 0;
36 clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags);
37 return netfs_inode_dio_wait_interruptible(&ictx->inode);
38 }
39
40 /**
41 * netfs_start_io_read - declare the file is being used for buffered reads
42 * @inode: file inode
43 *
44 * Declare that a buffered read operation is about to start, and ensure
45 * that we block all direct I/O.
46 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset,
47 * and holds a shared lock on inode->i_rwsem to ensure that the flag
48 * cannot be changed.
49 * In practice, this means that buffered read operations are allowed to
50 * execute in parallel, thanks to the shared lock, whereas direct I/O
51 * operations need to wait to grab an exclusive lock in order to set
52 * NETFS_ICTX_ODIRECT.
53 * Note that buffered writes and truncates both take a write lock on
54 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
55 */
netfs_start_io_read(struct inode * inode)56 int netfs_start_io_read(struct inode *inode)
57 __acquires(inode->i_rwsem)
58 {
59 struct netfs_inode *ictx = netfs_inode(inode);
60
61 /* Be an optimist! */
62 if (down_read_interruptible(&inode->i_rwsem) < 0)
63 return -ERESTARTSYS;
64 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0)
65 return 0;
66 up_read(&inode->i_rwsem);
67
68 /* Slow path.... */
69 if (down_write_killable(&inode->i_rwsem) < 0)
70 return -ERESTARTSYS;
71 if (netfs_block_o_direct(ictx) < 0) {
72 up_write(&inode->i_rwsem);
73 return -ERESTARTSYS;
74 }
75 downgrade_write(&inode->i_rwsem);
76 return 0;
77 }
78 EXPORT_SYMBOL(netfs_start_io_read);
79
80 /**
81 * netfs_end_io_read - declare that the buffered read operation is done
82 * @inode: file inode
83 *
84 * Declare that a buffered read operation is done, and release the shared
85 * lock on inode->i_rwsem.
86 */
netfs_end_io_read(struct inode * inode)87 void netfs_end_io_read(struct inode *inode)
88 __releases(inode->i_rwsem)
89 {
90 up_read(&inode->i_rwsem);
91 }
92 EXPORT_SYMBOL(netfs_end_io_read);
93
94 /**
95 * netfs_start_io_write - declare the file is being used for buffered writes
96 * @inode: file inode
97 *
98 * Declare that a buffered read operation is about to start, and ensure
99 * that we block all direct I/O.
100 */
netfs_start_io_write(struct inode * inode)101 int netfs_start_io_write(struct inode *inode)
102 __acquires(inode->i_rwsem)
103 {
104 struct netfs_inode *ictx = netfs_inode(inode);
105
106 if (down_write_killable(&inode->i_rwsem) < 0)
107 return -ERESTARTSYS;
108 if (netfs_block_o_direct(ictx) < 0) {
109 up_write(&inode->i_rwsem);
110 return -ERESTARTSYS;
111 }
112 downgrade_write(&inode->i_rwsem);
113 return 0;
114 }
115 EXPORT_SYMBOL(netfs_start_io_write);
116
117 /**
118 * netfs_end_io_write - declare that the buffered write operation is done
119 * @inode: file inode
120 *
121 * Declare that a buffered write operation is done, and release the
122 * lock on inode->i_rwsem.
123 */
netfs_end_io_write(struct inode * inode)124 void netfs_end_io_write(struct inode *inode)
125 __releases(inode->i_rwsem)
126 {
127 up_read(&inode->i_rwsem);
128 }
129 EXPORT_SYMBOL(netfs_end_io_write);
130
131 /* Call with exclusively locked inode->i_rwsem */
netfs_block_buffered(struct inode * inode)132 static int netfs_block_buffered(struct inode *inode)
133 {
134 struct netfs_inode *ictx = netfs_inode(inode);
135 int ret;
136
137 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) {
138 set_bit(NETFS_ICTX_ODIRECT, &ictx->flags);
139 if (inode->i_mapping->nrpages != 0) {
140 unmap_mapping_range(inode->i_mapping, 0, 0, 0);
141 ret = filemap_fdatawait(inode->i_mapping);
142 if (ret < 0) {
143 clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags);
144 return ret;
145 }
146 }
147 }
148 return 0;
149 }
150
151 /**
152 * netfs_start_io_direct - declare the file is being used for direct i/o
153 * @inode: file inode
154 *
155 * Declare that a direct I/O operation is about to start, and ensure
156 * that we block all buffered I/O.
157 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set,
158 * and holds a shared lock on inode->i_rwsem to ensure that the flag
159 * cannot be changed.
160 * In practice, this means that direct I/O operations are allowed to
161 * execute in parallel, thanks to the shared lock, whereas buffered I/O
162 * operations need to wait to grab an exclusive lock in order to clear
163 * NETFS_ICTX_ODIRECT.
164 * Note that buffered writes and truncates both take a write lock on
165 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
166 */
netfs_start_io_direct(struct inode * inode)167 int netfs_start_io_direct(struct inode *inode)
168 __acquires(inode->i_rwsem)
169 {
170 struct netfs_inode *ictx = netfs_inode(inode);
171 int ret;
172
173 /* Be an optimist! */
174 if (down_read_interruptible(&inode->i_rwsem) < 0)
175 return -ERESTARTSYS;
176 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0)
177 return 0;
178 up_read(&inode->i_rwsem);
179
180 /* Slow path.... */
181 if (down_write_killable(&inode->i_rwsem) < 0)
182 return -ERESTARTSYS;
183 ret = netfs_block_buffered(inode);
184 if (ret < 0) {
185 up_write(&inode->i_rwsem);
186 return ret;
187 }
188 downgrade_write(&inode->i_rwsem);
189 return 0;
190 }
191 EXPORT_SYMBOL(netfs_start_io_direct);
192
193 /**
194 * netfs_end_io_direct - declare that the direct i/o operation is done
195 * @inode: file inode
196 *
197 * Declare that a direct I/O operation is done, and release the shared
198 * lock on inode->i_rwsem.
199 */
netfs_end_io_direct(struct inode * inode)200 void netfs_end_io_direct(struct inode *inode)
201 __releases(inode->i_rwsem)
202 {
203 up_read(&inode->i_rwsem);
204 }
205 EXPORT_SYMBOL(netfs_end_io_direct);
206