1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016 Trond Myklebust 4 * 5 * I/O and data path helper functionality. 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/bitops.h> 11 #include <linux/rwsem.h> 12 #include <linux/fs.h> 13 #include <linux/nfs_fs.h> 14 15 #include "internal.h" 16 17 /** 18 * nfs_start_io_read - declare the file is being used for buffered reads 19 * @inode: file inode 20 * 21 * Declare that a buffered read operation is about to start, and ensure 22 * that we block all direct I/O. 23 * On exit, the function ensures that the NFS_INO_ODIRECT flag is unset, 24 * and holds a shared lock on inode->i_rwsem to ensure that the flag 25 * cannot be changed. 26 * In practice, this means that buffered read operations are allowed to 27 * execute in parallel, thanks to the shared lock, whereas direct I/O 28 * operations need to wait to grab an exclusive lock in order to set 29 * NFS_INO_ODIRECT. 30 * Note that buffered writes and truncates both take a write lock on 31 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. 32 */ 33 int 34 nfs_start_io_read(struct inode *inode) 35 { 36 struct nfs_inode *nfsi = NFS_I(inode); 37 int err; 38 39 /* Be an optimist! */ 40 err = down_read_killable(&inode->i_rwsem); 41 if (err) 42 return err; 43 if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0) 44 return 0; 45 up_read(&inode->i_rwsem); 46 47 /* Slow path.... */ 48 err = down_write_killable(&inode->i_rwsem); 49 if (err) 50 return err; 51 nfs_file_block_o_direct(nfsi); 52 downgrade_write(&inode->i_rwsem); 53 54 return 0; 55 } 56 57 /** 58 * nfs_end_io_read - declare that the buffered read operation is done 59 * @inode: file inode 60 * 61 * Declare that a buffered read operation is done, and release the shared 62 * lock on inode->i_rwsem. 63 */ 64 void 65 nfs_end_io_read(struct inode *inode) 66 { 67 up_read(&inode->i_rwsem); 68 } 69 70 /** 71 * nfs_start_io_write - declare the file is being used for buffered writes 72 * @inode: file inode 73 * 74 * Declare that a buffered read operation is about to start, and ensure 75 * that we block all direct I/O. 76 */ 77 int 78 nfs_start_io_write(struct inode *inode) 79 { 80 int err; 81 82 err = down_write_killable(&inode->i_rwsem); 83 if (!err) 84 nfs_file_block_o_direct(NFS_I(inode)); 85 return err; 86 } 87 EXPORT_SYMBOL_GPL(nfs_start_io_write); 88 89 /** 90 * nfs_end_io_write - declare that the buffered write operation is done 91 * @inode: file inode 92 * 93 * Declare that a buffered write operation is done, and release the 94 * lock on inode->i_rwsem. 95 */ 96 void 97 nfs_end_io_write(struct inode *inode) 98 { 99 up_write(&inode->i_rwsem); 100 } 101 EXPORT_SYMBOL_GPL(nfs_end_io_write); 102 103 /* Call with exclusively locked inode->i_rwsem */ 104 static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode) 105 { 106 if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { 107 set_bit(NFS_INO_ODIRECT, &nfsi->flags); 108 nfs_sync_mapping(inode->i_mapping); 109 } 110 } 111 112 static int nfs_block_buffered_nowait(struct nfs_inode *nfsi, struct inode *inode) 113 { 114 if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { 115 if (inode->i_mapping->nrpages != 0) 116 return 1; 117 set_bit(NFS_INO_ODIRECT, &nfsi->flags); 118 } 119 return 0; 120 } 121 122 /** 123 * nfs_start_io_direct - declare the file is being used for direct i/o 124 * @inode: file inode 125 * 126 * Declare that a direct I/O operation is about to start, and ensure 127 * that we block all buffered I/O. 128 * On exit, the function ensures that the NFS_INO_ODIRECT flag is set, 129 * and holds a shared lock on inode->i_rwsem to ensure that the flag 130 * cannot be changed. 131 * In practice, this means that direct I/O operations are allowed to 132 * execute in parallel, thanks to the shared lock, whereas buffered I/O 133 * operations need to wait to grab an exclusive lock in order to clear 134 * NFS_INO_ODIRECT. 135 * Note that buffered writes and truncates both take a write lock on 136 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. 137 */ 138 int 139 nfs_start_io_direct(struct inode *inode) 140 { 141 struct nfs_inode *nfsi = NFS_I(inode); 142 int err; 143 144 /* Be an optimist! */ 145 err = down_read_killable(&inode->i_rwsem); 146 if (err) 147 return err; 148 if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) != 0) 149 return 0; 150 up_read(&inode->i_rwsem); 151 152 /* Slow path.... */ 153 err = down_write_killable(&inode->i_rwsem); 154 if (err) 155 return err; 156 nfs_block_buffered(nfsi, inode); 157 downgrade_write(&inode->i_rwsem); 158 159 return 0; 160 } 161 162 /** 163 * nfs_start_io_direct_nowait - non-blocking variant of nfs_start_io_direct() 164 * @inode: file inode 165 * 166 * Try to declare that a direct I/O operation is about to start without 167 * blocking. 168 * Ensure all buffered I/O is blocked. 169 * If this could not be done without blocking then returns -EAGAIN. 170 */ 171 int 172 nfs_start_io_direct_nowait(struct inode *inode) 173 { 174 struct nfs_inode *nfsi = NFS_I(inode); 175 176 if (!down_read_trylock(&inode->i_rwsem)) 177 return -EAGAIN; 178 if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) 179 return 0; 180 up_read(&inode->i_rwsem); 181 182 /* Slow path: try to flip NFS_INO_ODIRECT without blocking. */ 183 if (!down_write_trylock(&inode->i_rwsem)) 184 return -EAGAIN; 185 if (nfs_block_buffered_nowait(nfsi, inode)) { 186 up_write(&inode->i_rwsem); 187 return -EAGAIN; 188 } 189 downgrade_write(&inode->i_rwsem); 190 return 0; 191 } 192 193 /** 194 * nfs_end_io_direct - declare that the direct i/o operation is done 195 * @inode: file inode 196 * 197 * Declare that a direct I/O operation is done, and release the shared 198 * lock on inode->i_rwsem. 199 */ 200 void 201 nfs_end_io_direct(struct inode *inode) 202 { 203 up_read(&inode->i_rwsem); 204 } 205