1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * I/O and data path helper functionality. 4 * 5 * Borrowed from NFS Copyright (c) 2016 Trond Myklebust 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/netfs.h> 10 #include "internal.h" 11 12 struct netfs_wb_waiter { 13 struct list_head link; /* Link in ictx->wb_queue */ 14 struct task_struct *waiter; /* Waiter task; cleared when lock granted */ 15 }; 16 17 /* 18 * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish 19 * @inode: inode to wait for 20 * 21 * Waits for all pending direct I/O requests to finish so that we can 22 * proceed with a truncate or equivalent operation. 23 * 24 * Must be called under a lock that serializes taking new references 25 * to i_dio_count, usually by inode->i_mutex. 26 */ 27 static int netfs_inode_dio_wait_interruptible(struct inode *inode) 28 { 29 if (inode_dio_finished(inode)) 30 return 0; 31 32 inode_dio_wait_interruptible(inode); 33 return !inode_dio_finished(inode) ? -ERESTARTSYS : 0; 34 } 35 36 /* Call with exclusively locked inode->i_rwsem */ 37 static int netfs_block_o_direct(struct netfs_inode *ictx) 38 { 39 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) 40 return 0; 41 clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); 42 return netfs_inode_dio_wait_interruptible(&ictx->inode); 43 } 44 45 /** 46 * netfs_start_io_read - declare the file is being used for buffered reads 47 * @inode: file inode 48 * 49 * Declare that a buffered read operation is about to start, and ensure 50 * that we block all direct I/O. 51 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset, 52 * and holds a shared lock on inode->i_rwsem to ensure that the flag 53 * cannot be changed. 54 * In practice, this means that buffered read operations are allowed to 55 * execute in parallel, thanks to the shared lock, whereas direct I/O 56 * operations need to wait to grab an exclusive lock in order to set 57 * NETFS_ICTX_ODIRECT. 58 * Note that buffered writes and truncates both take a write lock on 59 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. 60 */ 61 int netfs_start_io_read(struct inode *inode) 62 __acquires(inode->i_rwsem) 63 { 64 struct netfs_inode *ictx = netfs_inode(inode); 65 66 /* Be an optimist! */ 67 if (down_read_interruptible(&inode->i_rwsem) < 0) 68 return -ERESTARTSYS; 69 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0) 70 return 0; 71 up_read(&inode->i_rwsem); 72 73 /* Slow path.... */ 74 if (down_write_killable(&inode->i_rwsem) < 0) 75 return -ERESTARTSYS; 76 if (netfs_block_o_direct(ictx) < 0) { 77 up_write(&inode->i_rwsem); 78 return -ERESTARTSYS; 79 } 80 downgrade_write(&inode->i_rwsem); 81 return 0; 82 } 83 EXPORT_SYMBOL(netfs_start_io_read); 84 85 /** 86 * netfs_end_io_read - declare that the buffered read operation is done 87 * @inode: file inode 88 * 89 * Declare that a buffered read operation is done, and release the shared 90 * lock on inode->i_rwsem. 91 */ 92 void netfs_end_io_read(struct inode *inode) 93 __releases(inode->i_rwsem) 94 { 95 up_read(&inode->i_rwsem); 96 } 97 EXPORT_SYMBOL(netfs_end_io_read); 98 99 /** 100 * netfs_start_io_write - declare the file is being used for buffered writes 101 * @inode: file inode 102 * 103 * Declare that a buffered read operation is about to start, and ensure 104 * that we block all direct I/O. 105 */ 106 int netfs_start_io_write(struct inode *inode) 107 __acquires(inode->i_rwsem) 108 { 109 struct netfs_inode *ictx = netfs_inode(inode); 110 111 if (down_write_killable(&inode->i_rwsem) < 0) 112 return -ERESTARTSYS; 113 if (netfs_block_o_direct(ictx) < 0) { 114 up_write(&inode->i_rwsem); 115 return -ERESTARTSYS; 116 } 117 downgrade_write(&inode->i_rwsem); 118 return 0; 119 } 120 EXPORT_SYMBOL(netfs_start_io_write); 121 122 /** 123 * netfs_end_io_write - declare that the buffered write operation is done 124 * @inode: file inode 125 * 126 * Declare that a buffered write operation is done, and release the 127 * lock on inode->i_rwsem. 128 */ 129 void netfs_end_io_write(struct inode *inode) 130 __releases(inode->i_rwsem) 131 { 132 up_read(&inode->i_rwsem); 133 } 134 EXPORT_SYMBOL(netfs_end_io_write); 135 136 /* Call with exclusively locked inode->i_rwsem */ 137 static int netfs_block_buffered(struct inode *inode) 138 { 139 struct netfs_inode *ictx = netfs_inode(inode); 140 int ret; 141 142 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) { 143 set_bit(NETFS_ICTX_ODIRECT, &ictx->flags); 144 if (inode->i_mapping->nrpages != 0) { 145 unmap_mapping_range(inode->i_mapping, 0, 0, 0); 146 ret = filemap_fdatawait(inode->i_mapping); 147 if (ret < 0) { 148 clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); 149 return ret; 150 } 151 } 152 } 153 return 0; 154 } 155 156 /** 157 * netfs_start_io_direct - declare the file is being used for direct i/o 158 * @inode: file inode 159 * 160 * Declare that a direct I/O operation is about to start, and ensure 161 * that we block all buffered I/O. 162 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set, 163 * and holds a shared lock on inode->i_rwsem to ensure that the flag 164 * cannot be changed. 165 * In practice, this means that direct I/O operations are allowed to 166 * execute in parallel, thanks to the shared lock, whereas buffered I/O 167 * operations need to wait to grab an exclusive lock in order to clear 168 * NETFS_ICTX_ODIRECT. 169 * Note that buffered writes and truncates both take a write lock on 170 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. 171 */ 172 int netfs_start_io_direct(struct inode *inode) 173 __acquires(inode->i_rwsem) 174 { 175 struct netfs_inode *ictx = netfs_inode(inode); 176 int ret; 177 178 /* Be an optimist! */ 179 if (down_read_interruptible(&inode->i_rwsem) < 0) 180 return -ERESTARTSYS; 181 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0) 182 return 0; 183 up_read(&inode->i_rwsem); 184 185 /* Slow path.... */ 186 if (down_write_killable(&inode->i_rwsem) < 0) 187 return -ERESTARTSYS; 188 ret = netfs_block_buffered(inode); 189 if (ret < 0) { 190 up_write(&inode->i_rwsem); 191 return ret; 192 } 193 downgrade_write(&inode->i_rwsem); 194 return 0; 195 } 196 EXPORT_SYMBOL(netfs_start_io_direct); 197 198 /** 199 * netfs_end_io_direct - declare that the direct i/o operation is done 200 * @inode: file inode 201 * 202 * Declare that a direct I/O operation is done, and release the shared 203 * lock on inode->i_rwsem. 204 */ 205 void netfs_end_io_direct(struct inode *inode) 206 __releases(inode->i_rwsem) 207 { 208 up_read(&inode->i_rwsem); 209 } 210 EXPORT_SYMBOL(netfs_end_io_direct); 211 212 /* 213 * Wait to have exclusive access to writeback. 214 */ 215 static bool netfs_wb_begin_wait(struct netfs_inode *ictx) 216 { 217 struct netfs_wb_waiter waiter = {}; 218 struct task_struct *tsk = current; 219 bool got = false; 220 221 spin_lock(&ictx->lock); 222 223 if (test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) { 224 get_task_struct(tsk); 225 waiter.waiter = tsk; 226 list_add_tail(&waiter.link, &ictx->wb_queue); 227 } else { 228 got = true; 229 } 230 spin_unlock(&ictx->lock); 231 232 if (!got) { 233 for (;;) { 234 set_current_state(TASK_UNINTERRUPTIBLE); 235 /* Read waiter before accessing inode state. */ 236 if (smp_load_acquire(&waiter.waiter) == NULL) 237 break; 238 schedule(); 239 } 240 } 241 __set_current_state(TASK_RUNNING); 242 return true; 243 } 244 245 /** 246 * netfs_wb_begin - Begin writeback, waiting if need be 247 * @ictx: The inode to get writeback access on 248 * @nowait: Return failure immediately rather than waiting if true 249 * 250 * Begin writeback to an inode, waiting for exclusive access if @nowait is 251 * false. This prevents collection from being done out of order with respect 252 * to the issuance of write subrequests. 253 * 254 * Note that writeback may be ended in a different process (e.g. the collection 255 * function on a workqueue) than started it. 256 * 257 * Return: True if can proceed, false if denied. 258 */ 259 bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait) 260 { 261 if (!test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) 262 return true; 263 if (nowait) { 264 netfs_stat(&netfs_n_wb_lock_skip); 265 return false; 266 } 267 netfs_stat(&netfs_n_wb_lock_wait); 268 return netfs_wb_begin_wait(ictx); 269 } 270 EXPORT_SYMBOL(netfs_wb_begin); 271 272 /* netfs_wb_end - End writeback 273 * @ictx: The inode we have writeback access to 274 * 275 * End writeback access on an inode, waking up the next writeback request. 276 */ 277 void netfs_wb_end(struct netfs_inode *ictx) 278 { 279 struct netfs_wb_waiter *waiter; 280 struct task_struct *tsk; 281 282 WARN_ON_ONCE(!test_bit(NETFS_ICTX_WB_LOCK, &ictx->flags)); 283 284 spin_lock(&ictx->lock); 285 286 waiter = list_first_entry_or_null(&ictx->wb_queue, struct netfs_wb_waiter, link); 287 if (waiter) { 288 list_del(&waiter->link); 289 tsk = waiter->waiter; 290 /* Write inode state before clearing waiter. */ 291 smp_store_release(&waiter->waiter, NULL); 292 wake_up_process(tsk); 293 put_task_struct(tsk); 294 } else { 295 clear_bit_unlock(NETFS_ICTX_WB_LOCK, &ictx->flags); 296 } 297 298 spin_unlock(&ictx->lock); 299 } 300 EXPORT_SYMBOL(netfs_wb_end); 301