1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * I/O and data path helper functionality. 4 * 5 * Borrowed from NFS Copyright (c) 2016 Trond Myklebust 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/netfs.h> 10 #include "internal.h" 11 12 /* 13 * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish 14 * @inode: inode to wait for 15 * 16 * Waits for all pending direct I/O requests to finish so that we can 17 * proceed with a truncate or equivalent operation. 18 * 19 * Must be called under a lock that serializes taking new references 20 * to i_dio_count, usually by inode->i_mutex. 21 */ 22 static int inode_dio_wait_interruptible(struct inode *inode) 23 { 24 if (!atomic_read(&inode->i_dio_count)) 25 return 0; 26 27 wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP); 28 DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP); 29 30 for (;;) { 31 prepare_to_wait(wq, &q.wq_entry, TASK_INTERRUPTIBLE); 32 if (!atomic_read(&inode->i_dio_count)) 33 break; 34 if (signal_pending(current)) 35 break; 36 schedule(); 37 } 38 finish_wait(wq, &q.wq_entry); 39 40 return atomic_read(&inode->i_dio_count) ? -ERESTARTSYS : 0; 41 } 42 43 /* Call with exclusively locked inode->i_rwsem */ 44 static int netfs_block_o_direct(struct netfs_inode *ictx) 45 { 46 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) 47 return 0; 48 clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); 49 return inode_dio_wait_interruptible(&ictx->inode); 50 } 51 52 /** 53 * netfs_start_io_read - declare the file is being used for buffered reads 54 * @inode: file inode 55 * 56 * Declare that a buffered read operation is about to start, and ensure 57 * that we block all direct I/O. 58 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset, 59 * and holds a shared lock on inode->i_rwsem to ensure that the flag 60 * cannot be changed. 61 * In practice, this means that buffered read operations are allowed to 62 * execute in parallel, thanks to the shared lock, whereas direct I/O 63 * operations need to wait to grab an exclusive lock in order to set 64 * NETFS_ICTX_ODIRECT. 65 * Note that buffered writes and truncates both take a write lock on 66 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. 67 */ 68 int netfs_start_io_read(struct inode *inode) 69 __acquires(inode->i_rwsem) 70 { 71 struct netfs_inode *ictx = netfs_inode(inode); 72 73 /* Be an optimist! */ 74 if (down_read_interruptible(&inode->i_rwsem) < 0) 75 return -ERESTARTSYS; 76 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0) 77 return 0; 78 up_read(&inode->i_rwsem); 79 80 /* Slow path.... */ 81 if (down_write_killable(&inode->i_rwsem) < 0) 82 return -ERESTARTSYS; 83 if (netfs_block_o_direct(ictx) < 0) { 84 up_write(&inode->i_rwsem); 85 return -ERESTARTSYS; 86 } 87 downgrade_write(&inode->i_rwsem); 88 return 0; 89 } 90 EXPORT_SYMBOL(netfs_start_io_read); 91 92 /** 93 * netfs_end_io_read - declare that the buffered read operation is done 94 * @inode: file inode 95 * 96 * Declare that a buffered read operation is done, and release the shared 97 * lock on inode->i_rwsem. 98 */ 99 void netfs_end_io_read(struct inode *inode) 100 __releases(inode->i_rwsem) 101 { 102 up_read(&inode->i_rwsem); 103 } 104 EXPORT_SYMBOL(netfs_end_io_read); 105 106 /** 107 * netfs_start_io_write - declare the file is being used for buffered writes 108 * @inode: file inode 109 * 110 * Declare that a buffered read operation is about to start, and ensure 111 * that we block all direct I/O. 112 */ 113 int netfs_start_io_write(struct inode *inode) 114 __acquires(inode->i_rwsem) 115 { 116 struct netfs_inode *ictx = netfs_inode(inode); 117 118 if (down_write_killable(&inode->i_rwsem) < 0) 119 return -ERESTARTSYS; 120 if (netfs_block_o_direct(ictx) < 0) { 121 up_write(&inode->i_rwsem); 122 return -ERESTARTSYS; 123 } 124 return 0; 125 } 126 EXPORT_SYMBOL(netfs_start_io_write); 127 128 /** 129 * netfs_end_io_write - declare that the buffered write operation is done 130 * @inode: file inode 131 * 132 * Declare that a buffered write operation is done, and release the 133 * lock on inode->i_rwsem. 134 */ 135 void netfs_end_io_write(struct inode *inode) 136 __releases(inode->i_rwsem) 137 { 138 up_write(&inode->i_rwsem); 139 } 140 EXPORT_SYMBOL(netfs_end_io_write); 141 142 /* Call with exclusively locked inode->i_rwsem */ 143 static int netfs_block_buffered(struct inode *inode) 144 { 145 struct netfs_inode *ictx = netfs_inode(inode); 146 int ret; 147 148 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) { 149 set_bit(NETFS_ICTX_ODIRECT, &ictx->flags); 150 if (inode->i_mapping->nrpages != 0) { 151 unmap_mapping_range(inode->i_mapping, 0, 0, 0); 152 ret = filemap_fdatawait(inode->i_mapping); 153 if (ret < 0) { 154 clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); 155 return ret; 156 } 157 } 158 } 159 return 0; 160 } 161 162 /** 163 * netfs_start_io_direct - declare the file is being used for direct i/o 164 * @inode: file inode 165 * 166 * Declare that a direct I/O operation is about to start, and ensure 167 * that we block all buffered I/O. 168 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set, 169 * and holds a shared lock on inode->i_rwsem to ensure that the flag 170 * cannot be changed. 171 * In practice, this means that direct I/O operations are allowed to 172 * execute in parallel, thanks to the shared lock, whereas buffered I/O 173 * operations need to wait to grab an exclusive lock in order to clear 174 * NETFS_ICTX_ODIRECT. 175 * Note that buffered writes and truncates both take a write lock on 176 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. 177 */ 178 int netfs_start_io_direct(struct inode *inode) 179 __acquires(inode->i_rwsem) 180 { 181 struct netfs_inode *ictx = netfs_inode(inode); 182 int ret; 183 184 /* Be an optimist! */ 185 if (down_read_interruptible(&inode->i_rwsem) < 0) 186 return -ERESTARTSYS; 187 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0) 188 return 0; 189 up_read(&inode->i_rwsem); 190 191 /* Slow path.... */ 192 if (down_write_killable(&inode->i_rwsem) < 0) 193 return -ERESTARTSYS; 194 ret = netfs_block_buffered(inode); 195 if (ret < 0) { 196 up_write(&inode->i_rwsem); 197 return ret; 198 } 199 downgrade_write(&inode->i_rwsem); 200 return 0; 201 } 202 EXPORT_SYMBOL(netfs_start_io_direct); 203 204 /** 205 * netfs_end_io_direct - declare that the direct i/o operation is done 206 * @inode: file inode 207 * 208 * Declare that a direct I/O operation is done, and release the shared 209 * lock on inode->i_rwsem. 210 */ 211 void netfs_end_io_direct(struct inode *inode) 212 __releases(inode->i_rwsem) 213 { 214 up_read(&inode->i_rwsem); 215 } 216 EXPORT_SYMBOL(netfs_end_io_direct); 217