1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PIPE_FS_I_H 3 #define _LINUX_PIPE_FS_I_H 4 5 #define PIPE_DEF_BUFFERS 16 6 7 #define PIPE_BUF_FLAG_LRU 0x01 /* page is on the LRU */ 8 #define PIPE_BUF_FLAG_ATOMIC 0x02 /* was atomically mapped */ 9 #define PIPE_BUF_FLAG_GIFT 0x04 /* page is a gift */ 10 #define PIPE_BUF_FLAG_PACKET 0x08 /* read() as a packet */ 11 #define PIPE_BUF_FLAG_CAN_MERGE 0x10 /* can merge buffers */ 12 #define PIPE_BUF_FLAG_WHOLE 0x20 /* read() must return entire buffer or error */ 13 #ifdef CONFIG_WATCH_QUEUE 14 #define PIPE_BUF_FLAG_LOSS 0x40 /* Message loss happened after this buffer */ 15 #endif 16 17 #define PIPE_PREALLOC_MAX 8 /* max pages in prealloc pool */ 18 #define PIPE_PREALLOC_KEEP 2 /* keep at least this many after trim */ 19 20 /** 21 * struct pipe_buffer - a linux kernel pipe buffer 22 * @page: the page containing the data for the pipe buffer 23 * @offset: offset of data inside the @page 24 * @len: length of data inside the @page 25 * @ops: operations associated with this buffer. See @pipe_buf_operations. 26 * @flags: pipe buffer flags. See above. 27 * @private: private data owned by the ops. 28 **/ 29 struct pipe_buffer { 30 struct page *page; 31 unsigned int offset, len; 32 const struct pipe_buf_operations *ops; 33 unsigned int flags; 34 unsigned long private; 35 }; 36 37 /* 38 * Really only alpha needs 32-bit fields, but 39 * might as well do it for 64-bit architectures 40 * since that's what we've historically done, 41 * and it makes 'head_tail' always be a simple 42 * 'unsigned long'. 43 */ 44 #ifdef CONFIG_64BIT 45 typedef unsigned int pipe_index_t; 46 #else 47 typedef unsigned short pipe_index_t; 48 #endif 49 50 /** 51 * struct pipe_index - pipe indeces 52 * @head: The point of buffer production 53 * @tail: The point of buffer consumption 54 * @head_tail: unsigned long union of @head and @tail 55 */ 56 union pipe_index { 57 unsigned long head_tail; 58 struct { 59 pipe_index_t head; 60 pipe_index_t tail; 61 }; 62 }; 63 64 /** 65 * struct anon_pipe_prealloc - per-pipe page preallocation pool 66 * @pages: array of cached pages (pool) 67 * @count: number of pages currently in the pool 68 * 69 * Each pipe keeps a small bounded pool of preallocated pages to reduce 70 * allocation overhead during writes. The pool is bounded at PIPE_PREALLOC_MAX 71 * and trimmed down to PIPE_PREALLOC_KEEP after a write completes. 72 */ 73 struct anon_pipe_prealloc { 74 struct page *pages[PIPE_PREALLOC_MAX]; 75 76 unsigned int __data_racy count; 77 }; 78 79 /** 80 * struct pipe_inode_info - a linux kernel pipe 81 * @mutex: mutex protecting the whole thing 82 * @rd_wait: reader wait point in case of empty pipe 83 * @wr_wait: writer wait point in case of full pipe 84 * @pipe_index: the pipe indeces 85 * @note_loss: The next read() should insert a data-lost message 86 * @max_usage: The maximum number of slots that may be used in the ring 87 * @ring_size: total number of buffers (should be a power of 2) 88 * @nr_accounted: The amount this pipe accounts for in user->pipe_bufs 89 * @prealloc: per-pipe page preallocation pool 90 * @readers: number of current readers of this pipe 91 * @writers: number of current writers of this pipe 92 * @files: number of struct file referring this pipe (protected by ->i_lock) 93 * @r_counter: reader counter 94 * @w_counter: writer counter 95 * @pseudo_edgetrigger: has an EPOLLET consumer, enable per-write wakeups 96 * @fasync_readers: reader side fasync 97 * @fasync_writers: writer side fasync 98 * @bufs: the circular array of pipe buffers 99 * @user: the user who created this pipe 100 * @watch_queue: If this pipe is a watch_queue, this is the stuff for that 101 **/ 102 struct pipe_inode_info { 103 struct mutex mutex; 104 wait_queue_head_t rd_wait, wr_wait; 105 106 union pipe_index; 107 108 unsigned int max_usage; 109 unsigned int ring_size; 110 unsigned int nr_accounted; 111 unsigned int readers; 112 unsigned int writers; 113 unsigned int files; 114 unsigned int r_counter; 115 unsigned int w_counter; 116 bool pseudo_edgetrigger; 117 #ifdef CONFIG_WATCH_QUEUE 118 bool note_loss; 119 #endif 120 struct anon_pipe_prealloc prealloc; 121 struct fasync_struct *fasync_readers; 122 struct fasync_struct *fasync_writers; 123 struct pipe_buffer *bufs; 124 struct user_struct *user; 125 #ifdef CONFIG_WATCH_QUEUE 126 struct watch_queue *watch_queue; 127 #endif 128 }; 129 130 /* 131 * Note on the nesting of these functions: 132 * 133 * ->confirm() 134 * ->try_steal() 135 * 136 * That is, ->try_steal() must be called on a confirmed buffer. See below for 137 * the meaning of each operation. Also see the kerneldoc in fs/pipe.c for the 138 * pipe and generic variants of these hooks. 139 */ 140 struct pipe_buf_operations { 141 /* 142 * ->confirm() verifies that the data in the pipe buffer is there 143 * and that the contents are good. If the pages in the pipe belong 144 * to a file system, we may need to wait for IO completion in this 145 * hook. Returns 0 for good, or a negative error value in case of 146 * error. If not present all pages are considered good. 147 */ 148 int (*confirm)(struct pipe_inode_info *, struct pipe_buffer *); 149 150 /* 151 * When the contents of this pipe buffer has been completely 152 * consumed by a reader, ->release() is called. 153 */ 154 void (*release)(struct pipe_inode_info *, struct pipe_buffer *); 155 156 /* 157 * Attempt to take ownership of the pipe buffer and its contents. 158 * ->try_steal() returns %true for success, in which case the contents 159 * of the pipe (the buf->page) is locked and now completely owned by the 160 * caller. The page may then be transferred to a different mapping, the 161 * most often used case is insertion into different file address space 162 * cache. 163 */ 164 bool (*try_steal)(struct pipe_inode_info *, struct pipe_buffer *); 165 166 /* 167 * Get a reference to the pipe buffer. 168 */ 169 bool (*get)(struct pipe_inode_info *, struct pipe_buffer *); 170 }; 171 172 /** 173 * pipe_has_watch_queue - Check whether the pipe is a watch_queue, 174 * i.e. it was created with O_NOTIFICATION_PIPE 175 * @pipe: The pipe to check 176 * 177 * Return: true if pipe is a watch queue, false otherwise. 178 */ 179 static inline bool pipe_has_watch_queue(const struct pipe_inode_info *pipe) 180 { 181 #ifdef CONFIG_WATCH_QUEUE 182 return pipe->watch_queue != NULL; 183 #else 184 return false; 185 #endif 186 } 187 188 /** 189 * pipe_occupancy - Return number of slots used in the pipe 190 * @head: The pipe ring head pointer 191 * @tail: The pipe ring tail pointer 192 */ 193 static inline unsigned int pipe_occupancy(unsigned int head, unsigned int tail) 194 { 195 return (pipe_index_t)(head - tail); 196 } 197 198 /** 199 * pipe_empty - Return true if the pipe is empty 200 * @head: The pipe ring head pointer 201 * @tail: The pipe ring tail pointer 202 */ 203 static inline bool pipe_empty(unsigned int head, unsigned int tail) 204 { 205 return !pipe_occupancy(head, tail); 206 } 207 208 /** 209 * pipe_full - Return true if the pipe is full 210 * @head: The pipe ring head pointer 211 * @tail: The pipe ring tail pointer 212 * @limit: The maximum amount of slots available. 213 */ 214 static inline bool pipe_full(unsigned int head, unsigned int tail, 215 unsigned int limit) 216 { 217 return pipe_occupancy(head, tail) >= limit; 218 } 219 220 /** 221 * pipe_is_full - Return true if the pipe is full 222 * @pipe: the pipe 223 */ 224 static inline bool pipe_is_full(const struct pipe_inode_info *pipe) 225 { 226 return pipe_full(pipe->head, pipe->tail, pipe->max_usage); 227 } 228 229 /** 230 * pipe_is_empty - Return true if the pipe is empty 231 * @pipe: the pipe 232 */ 233 static inline bool pipe_is_empty(const struct pipe_inode_info *pipe) 234 { 235 return pipe_empty(pipe->head, pipe->tail); 236 } 237 238 /** 239 * pipe_buf_usage - Return how many pipe buffers are in use 240 * @pipe: the pipe 241 */ 242 static inline unsigned int pipe_buf_usage(const struct pipe_inode_info *pipe) 243 { 244 return pipe_occupancy(pipe->head, pipe->tail); 245 } 246 247 /** 248 * pipe_buf - Return the pipe buffer for the specified slot in the pipe ring 249 * @pipe: The pipe to access 250 * @slot: The slot of interest 251 */ 252 static inline struct pipe_buffer *pipe_buf(const struct pipe_inode_info *pipe, 253 unsigned int slot) 254 { 255 return &pipe->bufs[slot & (pipe->ring_size - 1)]; 256 } 257 258 /** 259 * pipe_head_buf - Return the pipe buffer at the head of the pipe ring 260 * @pipe: The pipe to access 261 */ 262 static inline struct pipe_buffer *pipe_head_buf(const struct pipe_inode_info *pipe) 263 { 264 return pipe_buf(pipe, pipe->head); 265 } 266 267 /** 268 * pipe_buf_get - get a reference to a pipe_buffer 269 * @pipe: the pipe that the buffer belongs to 270 * @buf: the buffer to get a reference to 271 * 272 * Return: %true if the reference was successfully obtained. 273 */ 274 static inline __must_check bool pipe_buf_get(struct pipe_inode_info *pipe, 275 struct pipe_buffer *buf) 276 { 277 return buf->ops->get(pipe, buf); 278 } 279 280 /** 281 * pipe_buf_release - put a reference to a pipe_buffer 282 * @pipe: the pipe that the buffer belongs to 283 * @buf: the buffer to put a reference to 284 */ 285 static inline void pipe_buf_release(struct pipe_inode_info *pipe, 286 struct pipe_buffer *buf) 287 { 288 const struct pipe_buf_operations *ops = buf->ops; 289 290 buf->ops = NULL; 291 ops->release(pipe, buf); 292 } 293 294 /** 295 * pipe_buf_confirm - verify contents of the pipe buffer 296 * @pipe: the pipe that the buffer belongs to 297 * @buf: the buffer to confirm 298 */ 299 static inline int pipe_buf_confirm(struct pipe_inode_info *pipe, 300 struct pipe_buffer *buf) 301 { 302 if (!buf->ops->confirm) 303 return 0; 304 return buf->ops->confirm(pipe, buf); 305 } 306 307 /** 308 * pipe_buf_try_steal - attempt to take ownership of a pipe_buffer 309 * @pipe: the pipe that the buffer belongs to 310 * @buf: the buffer to attempt to steal 311 */ 312 static inline bool pipe_buf_try_steal(struct pipe_inode_info *pipe, 313 struct pipe_buffer *buf) 314 { 315 if (!buf->ops->try_steal) 316 return false; 317 return buf->ops->try_steal(pipe, buf); 318 } 319 320 /* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual 321 memory allocation, whereas PIPE_BUF makes atomicity guarantees. */ 322 #define PIPE_SIZE PAGE_SIZE 323 324 /* Pipe lock and unlock operations */ 325 void pipe_lock(struct pipe_inode_info *); 326 void pipe_unlock(struct pipe_inode_info *); 327 void pipe_double_lock(struct pipe_inode_info *, struct pipe_inode_info *); 328 329 /* Wait for a pipe to be readable/writable while dropping the pipe lock */ 330 void pipe_wait_readable(struct pipe_inode_info *); 331 void pipe_wait_writable(struct pipe_inode_info *); 332 333 struct pipe_inode_info *alloc_pipe_info(void); 334 void free_pipe_info(struct pipe_inode_info *); 335 336 /* Generic pipe buffer ops functions */ 337 bool generic_pipe_buf_get(struct pipe_inode_info *, struct pipe_buffer *); 338 bool generic_pipe_buf_try_steal(struct pipe_inode_info *, struct pipe_buffer *); 339 void generic_pipe_buf_release(struct pipe_inode_info *, struct pipe_buffer *); 340 341 extern const struct pipe_buf_operations nosteal_pipe_buf_ops; 342 343 unsigned long account_pipe_buffers(struct user_struct *user, 344 unsigned long old, unsigned long new); 345 bool too_many_pipe_buffers_soft(unsigned long user_bufs); 346 bool too_many_pipe_buffers_hard(unsigned long user_bufs); 347 bool pipe_is_unprivileged_user(void); 348 349 /* for F_SETPIPE_SZ and F_GETPIPE_SZ */ 350 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots); 351 long pipe_fcntl(struct file *, unsigned int, unsigned int arg); 352 struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice); 353 354 int create_pipe_files(struct file **, int); 355 unsigned int round_pipe_size(unsigned int size); 356 357 #endif 358