1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/io_uring.h> 7 8 #include <uapi/linux/io_uring.h> 9 10 #include "../kernel/futex/futex.h" 11 #include "io_uring.h" 12 #include "alloc_cache.h" 13 #include "futex.h" 14 15 struct io_futex { 16 struct file *file; 17 void __user *uaddr; 18 unsigned long futex_val; 19 unsigned long futex_mask; 20 u32 futex_flags; 21 unsigned int futex_nr; 22 bool futexv_unqueued; 23 }; 24 25 struct io_futex_data { 26 struct futex_q q; 27 struct io_kiocb *req; 28 }; 29 30 struct io_futexv_data { 31 unsigned long owned; 32 struct futex_vector futexv[]; 33 }; 34 35 #define IO_FUTEX_ALLOC_CACHE_MAX 32 36 37 bool io_futex_cache_init(struct io_ring_ctx *ctx) 38 { 39 return io_alloc_cache_init(&ctx->futex_cache, IO_FUTEX_ALLOC_CACHE_MAX, 40 sizeof(struct io_futex_data), 0); 41 } 42 43 void io_futex_cache_free(struct io_ring_ctx *ctx) 44 { 45 io_alloc_cache_free(&ctx->futex_cache, kfree); 46 } 47 48 static void __io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw) 49 { 50 hlist_del_init(&tw_req.req->hash_node); 51 io_req_task_complete(tw_req, tw); 52 } 53 54 static void io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw) 55 { 56 struct io_kiocb *req = tw_req.req; 57 struct io_ring_ctx *ctx = req->ctx; 58 59 io_tw_lock(ctx, tw); 60 io_cache_free(&ctx->futex_cache, req->async_data); 61 io_req_async_data_clear(req, 0); 62 __io_futex_complete(tw_req, tw); 63 } 64 65 static void io_futexv_complete(struct io_tw_req tw_req, io_tw_token_t tw) 66 { 67 struct io_kiocb *req = tw_req.req; 68 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 69 struct io_futexv_data *ifd = req->async_data; 70 71 io_tw_lock(req->ctx, tw); 72 73 if (!iof->futexv_unqueued) { 74 int res; 75 76 res = futex_unqueue_multiple(ifd->futexv, iof->futex_nr); 77 if (res != -1) 78 io_req_set_res(req, res, 0); 79 } 80 81 io_req_async_data_free(req); 82 __io_futex_complete(tw_req, tw); 83 } 84 85 static bool io_futexv_claim(struct io_futexv_data *ifd) 86 { 87 if (test_bit(0, &ifd->owned) || test_and_set_bit_lock(0, &ifd->owned)) 88 return false; 89 return true; 90 } 91 92 static bool __io_futex_cancel(struct io_kiocb *req) 93 { 94 /* futex wake already done or in progress */ 95 if (req->opcode == IORING_OP_FUTEX_WAIT) { 96 struct io_futex_data *ifd = req->async_data; 97 98 if (!futex_unqueue(&ifd->q)) 99 return false; 100 req->io_task_work.func = io_futex_complete; 101 } else { 102 struct io_futexv_data *ifd = req->async_data; 103 104 if (!io_futexv_claim(ifd)) 105 return false; 106 req->io_task_work.func = io_futexv_complete; 107 } 108 109 hlist_del_init(&req->hash_node); 110 io_req_set_res(req, -ECANCELED, 0); 111 io_req_task_work_add(req); 112 return true; 113 } 114 115 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, 116 unsigned int issue_flags) 117 { 118 return io_cancel_remove(ctx, cd, issue_flags, &ctx->futex_list, __io_futex_cancel); 119 } 120 121 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, 122 bool cancel_all) 123 { 124 return io_cancel_remove_all(ctx, tctx, &ctx->futex_list, cancel_all, __io_futex_cancel); 125 } 126 127 int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 128 { 129 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 130 u32 flags; 131 132 if (unlikely(sqe->len || sqe->futex_flags || sqe->buf_index || 133 sqe->file_index)) 134 return -EINVAL; 135 136 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 137 iof->futex_val = READ_ONCE(sqe->addr2); 138 iof->futex_mask = READ_ONCE(sqe->addr3); 139 flags = READ_ONCE(sqe->fd); 140 141 if (flags & ~FUTEX2_VALID_MASK) 142 return -EINVAL; 143 144 iof->futex_flags = futex2_to_flags(flags); 145 if (!futex_flags_valid(iof->futex_flags)) 146 return -EINVAL; 147 148 if (!futex_validate_input(iof->futex_flags, iof->futex_val) || 149 !futex_validate_input(iof->futex_flags, iof->futex_mask)) 150 return -EINVAL; 151 152 /* Mark as inflight, so file exit cancelation will find it */ 153 io_req_track_inflight(req); 154 return 0; 155 } 156 157 static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) 158 { 159 struct io_kiocb *req = q->wake_data; 160 struct io_futexv_data *ifd = req->async_data; 161 162 if (!io_futexv_claim(ifd)) 163 return; 164 if (unlikely(!__futex_wake_mark(q))) 165 return; 166 167 io_req_set_res(req, 0, 0); 168 req->io_task_work.func = io_futexv_complete; 169 io_req_task_work_add(req); 170 } 171 172 int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 173 { 174 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 175 struct io_futexv_data *ifd; 176 int ret; 177 178 /* No flags or mask supported for waitv */ 179 if (unlikely(sqe->fd || sqe->buf_index || sqe->file_index || 180 sqe->addr2 || sqe->futex_flags || sqe->addr3)) 181 return -EINVAL; 182 183 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 184 iof->futex_nr = READ_ONCE(sqe->len); 185 if (!iof->futex_nr || iof->futex_nr > FUTEX_WAITV_MAX) 186 return -EINVAL; 187 188 ifd = kzalloc(struct_size_t(struct io_futexv_data, futexv, iof->futex_nr), 189 GFP_KERNEL); 190 if (!ifd) 191 return -ENOMEM; 192 193 ret = futex_parse_waitv(ifd->futexv, iof->uaddr, iof->futex_nr, 194 io_futex_wakev_fn, req); 195 if (ret) { 196 kfree(ifd); 197 return ret; 198 } 199 200 /* Mark as inflight, so file exit cancelation will find it */ 201 io_req_track_inflight(req); 202 iof->futexv_unqueued = 0; 203 req->flags |= REQ_F_ASYNC_DATA; 204 req->async_data = ifd; 205 return 0; 206 } 207 208 static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) 209 { 210 struct io_futex_data *ifd = container_of(q, struct io_futex_data, q); 211 struct io_kiocb *req = ifd->req; 212 213 if (unlikely(!__futex_wake_mark(q))) 214 return; 215 216 io_req_set_res(req, 0, 0); 217 req->io_task_work.func = io_futex_complete; 218 io_req_task_work_add(req); 219 } 220 221 int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) 222 { 223 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 224 struct io_futexv_data *ifd = req->async_data; 225 struct io_ring_ctx *ctx = req->ctx; 226 int ret, woken = -1; 227 228 io_ring_submit_lock(ctx, issue_flags); 229 230 ret = futex_wait_multiple_setup(ifd->futexv, iof->futex_nr, &woken); 231 232 /* 233 * Error case, ret is < 0. Mark the request as failed. 234 */ 235 if (unlikely(ret < 0)) { 236 io_ring_submit_unlock(ctx, issue_flags); 237 req_set_fail(req); 238 io_req_set_res(req, ret, 0); 239 io_req_async_data_free(req); 240 return IOU_COMPLETE; 241 } 242 243 /* 244 * 0 return means that we successfully setup the waiters, and that 245 * nobody triggered a wakeup while we were doing so. If the wakeup 246 * happened post setup, the task_work will be run post this issue and 247 * under the submission lock. 1 means We got woken while setting up, 248 * let that side do the completion. Note that 249 * futex_wait_multiple_setup() will have unqueued all the futexes in 250 * this case. Mark us as having done that already, since this is 251 * different from normal wakeup. 252 */ 253 if (!ret) { 254 /* 255 * If futex_wait_multiple_setup() returns 0 for a 256 * successful setup, then the task state will not be 257 * runnable. This is fine for the sync syscall, as 258 * it'll be blocking unless we already got one of the 259 * futexes woken, but it obviously won't work for an 260 * async invocation. Mark us runnable again. 261 */ 262 __set_current_state(TASK_RUNNING); 263 hlist_add_head(&req->hash_node, &ctx->futex_list); 264 } else { 265 iof->futexv_unqueued = 1; 266 if (woken != -1) 267 io_req_set_res(req, woken, 0); 268 } 269 270 io_ring_submit_unlock(ctx, issue_flags); 271 return IOU_ISSUE_SKIP_COMPLETE; 272 } 273 274 int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) 275 { 276 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 277 struct io_ring_ctx *ctx = req->ctx; 278 struct io_futex_data *ifd = NULL; 279 int ret; 280 281 if (!iof->futex_mask) { 282 ret = -EINVAL; 283 goto done; 284 } 285 286 io_ring_submit_lock(ctx, issue_flags); 287 ifd = io_cache_alloc(&ctx->futex_cache, GFP_NOWAIT); 288 if (!ifd) { 289 ret = -ENOMEM; 290 goto done_unlock; 291 } 292 293 req->flags |= REQ_F_ASYNC_DATA; 294 req->async_data = ifd; 295 ifd->q = futex_q_init; 296 ifd->q.bitset = iof->futex_mask; 297 ifd->q.wake = io_futex_wake_fn; 298 ifd->req = req; 299 300 ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags, 301 &ifd->q, NULL, NULL); 302 if (!ret) { 303 hlist_add_head(&req->hash_node, &ctx->futex_list); 304 io_ring_submit_unlock(ctx, issue_flags); 305 306 return IOU_ISSUE_SKIP_COMPLETE; 307 } 308 309 done_unlock: 310 io_ring_submit_unlock(ctx, issue_flags); 311 done: 312 if (ret < 0) 313 req_set_fail(req); 314 io_req_set_res(req, ret, 0); 315 io_req_async_data_free(req); 316 return IOU_COMPLETE; 317 } 318 319 int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags) 320 { 321 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 322 int ret; 323 324 /* 325 * Strict flags - ensure that waking 0 futexes yields a 0 result. 326 * See commit 43adf8449510 ("futex: FLAGS_STRICT") for details. 327 */ 328 ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags, 329 iof->futex_val, iof->futex_mask); 330 if (ret < 0) 331 req_set_fail(req); 332 io_req_set_res(req, ret, 0); 333 return IOU_COMPLETE; 334 } 335