1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * FUSE: Filesystem in Userspace 4 * Copyright (c) 2023-2024 DataDirect Networks. 5 */ 6 7 #ifndef _FS_FUSE_DEV_URING_I_H 8 #define _FS_FUSE_DEV_URING_I_H 9 10 #include <linux/uio.h> 11 12 #include "fuse_dev_i.h" 13 14 #ifdef CONFIG_FUSE_IO_URING 15 16 #define FUSE_URING_TEARDOWN_TIMEOUT (5 * HZ) 17 #define FUSE_URING_TEARDOWN_INTERVAL (HZ/20) 18 19 enum fuse_ring_req_state { 20 FRRS_INVALID = 0, 21 22 /* The ring entry received from userspace and it is being processed */ 23 FRRS_COMMIT, 24 25 /* The ring entry is waiting for new fuse requests */ 26 FRRS_AVAILABLE, 27 28 /* The ring entry got assigned a fuse req */ 29 FRRS_FUSE_REQ, 30 31 /* The ring entry is in or on the way to user space */ 32 FRRS_USERSPACE, 33 34 /* The ring entry is in teardown */ 35 FRRS_TEARDOWN, 36 37 /* The ring entry is released, but not freed yet */ 38 FRRS_RELEASED, 39 }; 40 41 /* how a queue's payload buffers are provided */ 42 enum fuse_queue_payload_mode { 43 /* not yet committed (a bufpool may still be added) */ 44 FUSE_PAYLOAD_UNSET = 0, 45 /* each entry registers its own payload buffer */ 46 FUSE_PAYLOAD_PER_ENT, 47 /* each entry's payload buffer is assigned from a bufpool */ 48 FUSE_PAYLOAD_BUFPOOL, 49 }; 50 51 struct fuse_bufpool { 52 bool registered; 53 54 /* 55 * io_uring registered buffer table index for this pool, bound at 56 * ADD_BUFPOOL time. Only valid if the bufpool is registered 57 */ 58 u16 registered_index; 59 60 /* starting uaddr of the bufpool */ 61 uintptr_t base_uaddr; 62 63 /* size of each buffer in the pool */ 64 size_t buf_size; 65 66 /* total number of buffers in the pool */ 67 unsigned int nr_bufs; 68 69 /* bitmap tracking which buffers are free */ 70 unsigned long free_map[]; 71 }; 72 73 /** A fuse ring entry, part of the ring queue */ 74 struct fuse_ring_ent { 75 /* userspace buffer */ 76 struct fuse_uring_req_header __user *headers; 77 struct iovec payload; 78 79 /* buffer id in the pool, if bufpools are used. ignored otherwise */ 80 unsigned int buf_id; 81 82 /* true if the request's pages are being zero-copied */ 83 bool zero_copied; 84 unsigned int zero_copy_index; 85 86 /* the ring queue that owns the request */ 87 struct fuse_ring_queue *queue; 88 89 /* fields below are protected by queue->lock */ 90 91 struct io_uring_cmd *cmd; 92 93 struct list_head list; 94 95 enum fuse_ring_req_state state; 96 97 struct fuse_req *fuse_req; 98 }; 99 100 struct fuse_ring_queue { 101 /* 102 * back pointer to the main fuse uring structure that holds this 103 * queue 104 */ 105 struct fuse_ring *ring; 106 107 /* queue id, corresponds to the cpu core */ 108 unsigned int qid; 109 110 /* 111 * queue lock, taken when any value in the queue changes _and_ also 112 * a ring entry state changes. 113 */ 114 spinlock_t lock; 115 116 /* available ring entries (struct fuse_ring_ent) */ 117 struct list_head ent_avail_queue; 118 119 /* 120 * entries in the process of being committed or in the process 121 * to be sent to userspace 122 */ 123 struct list_head ent_w_req_queue; 124 struct list_head ent_commit_queue; 125 126 /* entries in userspace */ 127 struct list_head ent_in_userspace; 128 129 /* entries that are released */ 130 struct list_head ent_released; 131 132 /* fuse requests waiting for an entry slot */ 133 struct list_head fuse_req_queue; 134 135 /* background fuse requests */ 136 struct list_head fuse_req_bg_queue; 137 138 struct fuse_pqueue fpq; 139 140 unsigned int active_background; 141 142 bool stopped; 143 144 /* how this queue's payload buffers are provided */ 145 enum fuse_queue_payload_mode payload_mode; 146 147 /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */ 148 struct fuse_bufpool *bufpool; 149 150 bool zero_copy; 151 }; 152 153 /* 154 * Describes if uring is for communication and holds alls the data needed 155 * for uring communication 156 */ 157 struct fuse_ring { 158 /* back pointer */ 159 struct fuse_chan *chan; 160 161 /* number of ring queues */ 162 size_t nr_queues; 163 164 /* maximum payload/arg size */ 165 size_t max_payload_sz; 166 167 struct fuse_ring_queue **queues; 168 169 /* 170 * Log ring entry states on stop when entries cannot be released 171 */ 172 unsigned int stop_debug_log : 1; 173 174 wait_queue_head_t stop_waitq; 175 176 /* async tear down */ 177 struct delayed_work async_teardown_work; 178 179 /* log */ 180 unsigned long teardown_time; 181 182 atomic_t queue_refs; 183 184 bool ready; 185 }; 186 187 void fuse_uring_conn_init(struct fuse_chan *fch); 188 void fuse_uring_stop_queues(struct fuse_ring *ring); 189 void fuse_uring_abort_end_requests(struct fuse_ring *ring); 190 int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); 191 void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req); 192 bool fuse_uring_queue_bq_req(struct fuse_req *req); 193 bool fuse_uring_remove_pending_req(struct fuse_req *req); 194 bool fuse_uring_request_expired(struct fuse_chan *fch); 195 196 static inline void fuse_uring_abort(struct fuse_chan *fch) 197 { 198 struct fuse_ring *ring = fch->ring; 199 200 if (ring == NULL) 201 return; 202 203 fuse_uring_abort_end_requests(ring); 204 205 if (atomic_read(&ring->queue_refs) > 0) 206 fuse_uring_stop_queues(ring); 207 } 208 209 static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch) 210 { 211 struct fuse_ring *ring = fch->ring; 212 213 if (ring) 214 wait_event(ring->stop_waitq, 215 atomic_read(&ring->queue_refs) == 0); 216 } 217 218 static inline bool fuse_uring_ready(struct fuse_chan *fch) 219 { 220 struct fuse_ring *ring = READ_ONCE(fch->ring); 221 222 return ring && smp_load_acquire(&ring->ready); 223 } 224 225 #else /* CONFIG_FUSE_IO_URING */ 226 227 static inline void fuse_uring_conn_init(struct fuse_chan *fch) 228 { 229 } 230 231 static inline void fuse_uring_abort(struct fuse_chan *fch) 232 { 233 } 234 235 static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch) 236 { 237 } 238 239 static inline bool fuse_uring_ready(struct fuse_chan *fch) 240 { 241 return false; 242 } 243 244 static inline bool fuse_uring_remove_pending_req(struct fuse_req *req) 245 { 246 return false; 247 } 248 249 static inline bool fuse_uring_request_expired(struct fuse_chan *fch) 250 { 251 return false; 252 } 253 254 #endif /* CONFIG_FUSE_IO_URING */ 255 256 #endif /* _FS_FUSE_DEV_URING_I_H */ 257