1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016 Trond Myklebust 4 * 5 * I/O and data path helper functionality. 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/bitops.h> 11 #include <linux/rwsem.h> 12 #include <linux/fs.h> 13 #include <linux/nfs_fs.h> 14 15 #include "internal.h" 16 17 /* Call with exclusively locked inode->i_rwsem */ 18 static void nfs_block_o_direct(struct nfs_inode *nfsi, struct inode *inode) 19 { 20 if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { 21 clear_bit(NFS_INO_ODIRECT, &nfsi->flags); 22 inode_dio_wait(inode); 23 } 24 } 25 26 /** 27 * nfs_start_io_read - declare the file is being used for buffered reads 28 * @inode: file inode 29 * 30 * Declare that a buffered read operation is about to start, and ensure 31 * that we block all direct I/O. 32 * On exit, the function ensures that the NFS_INO_ODIRECT flag is unset, 33 * and holds a shared lock on inode->i_rwsem to ensure that the flag 34 * cannot be changed. 35 * In practice, this means that buffered read operations are allowed to 36 * execute in parallel, thanks to the shared lock, whereas direct I/O 37 * operations need to wait to grab an exclusive lock in order to set 38 * NFS_INO_ODIRECT. 39 * Note that buffered writes and truncates both take a write lock on 40 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. 41 */ 42 int 43 nfs_start_io_read(struct inode *inode) 44 { 45 struct nfs_inode *nfsi = NFS_I(inode); 46 int err; 47 48 /* Be an optimist! */ 49 err = down_read_killable(&inode->i_rwsem); 50 if (err) 51 return err; 52 if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0) 53 return 0; 54 up_read(&inode->i_rwsem); 55 56 /* Slow path.... */ 57 err = down_write_killable(&inode->i_rwsem); 58 if (err) 59 return err; 60 nfs_block_o_direct(nfsi, inode); 61 downgrade_write(&inode->i_rwsem); 62 63 return 0; 64 } 65 66 /** 67 * nfs_end_io_read - declare that the buffered read operation is done 68 * @inode: file inode 69 * 70 * Declare that a buffered read operation is done, and release the shared 71 * lock on inode->i_rwsem. 72 */ 73 void 74 nfs_end_io_read(struct inode *inode) 75 { 76 up_read(&inode->i_rwsem); 77 } 78 79 /** 80 * nfs_start_io_write - declare the file is being used for buffered writes 81 * @inode: file inode 82 * 83 * Declare that a buffered read operation is about to start, and ensure 84 * that we block all direct I/O. 85 */ 86 int 87 nfs_start_io_write(struct inode *inode) 88 { 89 int err; 90 91 err = down_write_killable(&inode->i_rwsem); 92 if (!err) 93 nfs_block_o_direct(NFS_I(inode), inode); 94 return err; 95 } 96 97 /** 98 * nfs_end_io_write - declare that the buffered write operation is done 99 * @inode: file inode 100 * 101 * Declare that a buffered write operation is done, and release the 102 * lock on inode->i_rwsem. 103 */ 104 void 105 nfs_end_io_write(struct inode *inode) 106 { 107 up_write(&inode->i_rwsem); 108 } 109 110 /* Call with exclusively locked inode->i_rwsem */ 111 static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode) 112 { 113 if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { 114 set_bit(NFS_INO_ODIRECT, &nfsi->flags); 115 nfs_sync_mapping(inode->i_mapping); 116 } 117 } 118 119 /** 120 * nfs_start_io_direct - declare the file is being used for direct i/o 121 * @inode: file inode 122 * 123 * Declare that a direct I/O operation is about to start, and ensure 124 * that we block all buffered I/O. 125 * On exit, the function ensures that the NFS_INO_ODIRECT flag is set, 126 * and holds a shared lock on inode->i_rwsem to ensure that the flag 127 * cannot be changed. 128 * In practice, this means that direct I/O operations are allowed to 129 * execute in parallel, thanks to the shared lock, whereas buffered I/O 130 * operations need to wait to grab an exclusive lock in order to clear 131 * NFS_INO_ODIRECT. 132 * Note that buffered writes and truncates both take a write lock on 133 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. 134 */ 135 int 136 nfs_start_io_direct(struct inode *inode) 137 { 138 struct nfs_inode *nfsi = NFS_I(inode); 139 int err; 140 141 /* Be an optimist! */ 142 err = down_read_killable(&inode->i_rwsem); 143 if (err) 144 return err; 145 if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) != 0) 146 return 0; 147 up_read(&inode->i_rwsem); 148 149 /* Slow path.... */ 150 err = down_write_killable(&inode->i_rwsem); 151 if (err) 152 return err; 153 nfs_block_buffered(nfsi, inode); 154 downgrade_write(&inode->i_rwsem); 155 156 return 0; 157 } 158 159 /** 160 * nfs_end_io_direct - declare that the direct i/o operation is done 161 * @inode: file inode 162 * 163 * Declare that a direct I/O operation is done, and release the shared 164 * lock on inode->i_rwsem. 165 */ 166 void 167 nfs_end_io_direct(struct inode *inode) 168 { 169 up_read(&inode->i_rwsem); 170 } 171