1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/io_uring.h> 7 8 #include <uapi/linux/io_uring.h> 9 10 #include "../kernel/futex/futex.h" 11 #include "io_uring.h" 12 #include "alloc_cache.h" 13 #include "futex.h" 14 15 struct io_futex { 16 struct file *file; 17 void __user *uaddr; 18 unsigned long futex_val; 19 unsigned long futex_mask; 20 u32 futex_flags; 21 unsigned int futex_nr; 22 bool futexv_unqueued; 23 }; 24 25 struct io_futex_data { 26 struct futex_q q; 27 struct io_kiocb *req; 28 }; 29 30 struct io_futexv_data { 31 unsigned long owned; 32 struct futex_vector futexv[]; 33 }; 34 35 #define IO_FUTEX_ALLOC_CACHE_MAX 32 36 37 bool io_futex_cache_init(struct io_ring_ctx *ctx) 38 { 39 return io_alloc_cache_init(&ctx->futex_cache, IO_FUTEX_ALLOC_CACHE_MAX, 40 sizeof(struct io_futex_data), 0); 41 } 42 43 void io_futex_cache_free(struct io_ring_ctx *ctx) 44 { 45 io_alloc_cache_free(&ctx->futex_cache, kfree); 46 } 47 48 static void __io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw) 49 { 50 hlist_del_init(&tw_req.req->hash_node); 51 io_req_task_complete(tw_req, tw); 52 } 53 54 static void io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw) 55 { 56 struct io_kiocb *req = tw_req.req; 57 struct io_ring_ctx *ctx = req->ctx; 58 59 io_tw_lock(ctx, tw); 60 io_cache_free(&ctx->futex_cache, req->async_data); 61 io_req_async_data_clear(req, 0); 62 __io_futex_complete(tw_req, tw); 63 } 64 65 static void io_futexv_complete(struct io_tw_req tw_req, io_tw_token_t tw) 66 { 67 struct io_kiocb *req = tw_req.req; 68 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 69 struct io_futexv_data *ifd = req->async_data; 70 71 io_tw_lock(req->ctx, tw); 72 73 if (!iof->futexv_unqueued) { 74 int res; 75 76 res = futex_unqueue_multiple(ifd->futexv, iof->futex_nr); 77 if (res != -1) 78 io_req_set_res(req, res, 0); 79 } 80 81 io_req_async_data_free(req); 82 __io_futex_complete(tw_req, tw); 83 } 84 85 static bool io_futexv_claim(struct io_futexv_data *ifd) 86 { 87 if (test_bit(0, &ifd->owned) || test_and_set_bit_lock(0, &ifd->owned)) 88 return false; 89 return true; 90 } 91 92 static bool __io_futex_cancel(struct io_kiocb *req) 93 { 94 /* futex wake already done or in progress */ 95 if (req->opcode == IORING_OP_FUTEX_WAIT) { 96 struct io_futex_data *ifd = req->async_data; 97 98 if (!futex_unqueue(&ifd->q)) 99 return false; 100 req->io_task_work.func = io_futex_complete; 101 } else { 102 struct io_futexv_data *ifd = req->async_data; 103 104 if (!io_futexv_claim(ifd)) 105 return false; 106 req->io_task_work.func = io_futexv_complete; 107 } 108 109 hlist_del_init(&req->hash_node); 110 io_req_set_res(req, -ECANCELED, 0); 111 io_req_task_work_add(req); 112 return true; 113 } 114 115 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, 116 unsigned int issue_flags) 117 { 118 return io_cancel_remove(ctx, cd, issue_flags, &ctx->futex_list, __io_futex_cancel); 119 } 120 121 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, 122 bool cancel_all) 123 { 124 return io_cancel_remove_all(ctx, tctx, &ctx->futex_list, cancel_all, __io_futex_cancel); 125 } 126 127 int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 128 { 129 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 130 u32 flags; 131 132 if (unlikely(sqe->len || sqe->futex_flags || sqe->buf_index || 133 sqe->file_index)) 134 return -EINVAL; 135 136 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 137 iof->futex_val = READ_ONCE(sqe->addr2); 138 iof->futex_mask = READ_ONCE(sqe->addr3); 139 flags = READ_ONCE(sqe->fd); 140 141 if (flags & ~FUTEX2_VALID_MASK) 142 return -EINVAL; 143 144 iof->futex_flags = futex2_to_flags(flags); 145 if (!futex_flags_valid(iof->futex_flags)) 146 return -EINVAL; 147 148 if (!futex_validate_input(iof->futex_flags, iof->futex_val) || 149 !futex_validate_input(iof->futex_flags, iof->futex_mask)) 150 return -EINVAL; 151 152 /* Mark as inflight, so file exit cancelation will find it */ 153 io_req_track_inflight(req); 154 return 0; 155 } 156 157 static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) 158 { 159 struct io_kiocb *req = q->wake_data; 160 struct io_futexv_data *ifd = req->async_data; 161 162 if (!io_futexv_claim(ifd)) { 163 __futex_wake_mark(q); 164 return; 165 } 166 if (unlikely(!__futex_wake_mark(q))) 167 return; 168 169 io_req_set_res(req, 0, 0); 170 req->io_task_work.func = io_futexv_complete; 171 io_req_task_work_add(req); 172 } 173 174 int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 175 { 176 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 177 struct io_futexv_data *ifd; 178 int ret; 179 180 /* No flags or mask supported for waitv */ 181 if (unlikely(sqe->fd || sqe->buf_index || sqe->file_index || 182 sqe->addr2 || sqe->futex_flags || sqe->addr3)) 183 return -EINVAL; 184 185 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 186 iof->futex_nr = READ_ONCE(sqe->len); 187 if (!iof->futex_nr || iof->futex_nr > FUTEX_WAITV_MAX) 188 return -EINVAL; 189 190 ifd = kzalloc_flex(struct io_futexv_data, futexv, iof->futex_nr, 191 GFP_KERNEL_ACCOUNT); 192 if (!ifd) 193 return -ENOMEM; 194 195 ret = futex_parse_waitv(ifd->futexv, iof->uaddr, iof->futex_nr, 196 io_futex_wakev_fn, req); 197 if (ret) { 198 kfree(ifd); 199 return ret; 200 } 201 202 /* Mark as inflight, so file exit cancelation will find it */ 203 io_req_track_inflight(req); 204 iof->futexv_unqueued = 0; 205 req->flags |= REQ_F_ASYNC_DATA; 206 req->async_data = ifd; 207 return 0; 208 } 209 210 static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) 211 { 212 struct io_futex_data *ifd = container_of(q, struct io_futex_data, q); 213 struct io_kiocb *req = ifd->req; 214 215 if (unlikely(!__futex_wake_mark(q))) 216 return; 217 218 io_req_set_res(req, 0, 0); 219 req->io_task_work.func = io_futex_complete; 220 io_req_task_work_add(req); 221 } 222 223 int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) 224 { 225 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 226 struct io_futexv_data *ifd = req->async_data; 227 struct io_ring_ctx *ctx = req->ctx; 228 int ret, woken = -1; 229 230 io_ring_submit_lock(ctx, issue_flags); 231 232 ret = futex_wait_multiple_setup(ifd->futexv, iof->futex_nr, &woken); 233 234 /* 235 * Error case, ret is < 0. Mark the request as failed. 236 */ 237 if (unlikely(ret < 0)) { 238 io_ring_submit_unlock(ctx, issue_flags); 239 req_set_fail(req); 240 io_req_set_res(req, ret, 0); 241 io_req_async_data_free(req); 242 return IOU_COMPLETE; 243 } 244 245 /* 246 * 0 return means that we successfully setup the waiters, and that 247 * nobody triggered a wakeup while we were doing so. If the wakeup 248 * happened post setup, the task_work will be run post this issue and 249 * under the submission lock. 1 means We got woken while setting up, 250 * let that side do the completion. Note that 251 * futex_wait_multiple_setup() will have unqueued all the futexes in 252 * this case. Mark us as having done that already, since this is 253 * different from normal wakeup. 254 */ 255 if (!ret) { 256 /* 257 * If futex_wait_multiple_setup() returns 0 for a 258 * successful setup, then the task state will not be 259 * runnable. This is fine for the sync syscall, as 260 * it'll be blocking unless we already got one of the 261 * futexes woken, but it obviously won't work for an 262 * async invocation. Mark us runnable again. 263 */ 264 __set_current_state(TASK_RUNNING); 265 hlist_add_head(&req->hash_node, &ctx->futex_list); 266 } else { 267 iof->futexv_unqueued = 1; 268 if (woken != -1) 269 io_req_set_res(req, woken, 0); 270 } 271 272 io_ring_submit_unlock(ctx, issue_flags); 273 return IOU_ISSUE_SKIP_COMPLETE; 274 } 275 276 int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) 277 { 278 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 279 struct io_ring_ctx *ctx = req->ctx; 280 struct io_futex_data *ifd = NULL; 281 int ret; 282 283 if (!iof->futex_mask) { 284 ret = -EINVAL; 285 goto done; 286 } 287 288 io_ring_submit_lock(ctx, issue_flags); 289 ifd = io_cache_alloc(&ctx->futex_cache, GFP_NOWAIT); 290 if (!ifd) { 291 ret = -ENOMEM; 292 goto done_unlock; 293 } 294 295 req->flags |= REQ_F_ASYNC_DATA; 296 req->async_data = ifd; 297 ifd->q = futex_q_init; 298 ifd->q.bitset = iof->futex_mask; 299 ifd->q.wake = io_futex_wake_fn; 300 ifd->req = req; 301 302 ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags, 303 &ifd->q, NULL, NULL); 304 if (!ret) { 305 hlist_add_head(&req->hash_node, &ctx->futex_list); 306 io_ring_submit_unlock(ctx, issue_flags); 307 308 return IOU_ISSUE_SKIP_COMPLETE; 309 } 310 311 done_unlock: 312 io_ring_submit_unlock(ctx, issue_flags); 313 done: 314 if (ret < 0) 315 req_set_fail(req); 316 io_req_set_res(req, ret, 0); 317 io_req_async_data_free(req); 318 return IOU_COMPLETE; 319 } 320 321 int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags) 322 { 323 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 324 int ret; 325 326 /* 327 * Strict flags - ensure that waking 0 futexes yields a 0 result. 328 * See commit 43adf8449510 ("futex: FLAGS_STRICT") for details. 329 */ 330 ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags, 331 iof->futex_val, iof->futex_mask); 332 if (ret < 0) 333 req_set_fail(req); 334 io_req_set_res(req, ret, 0); 335 return IOU_COMPLETE; 336 } 337