1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/io_uring.h> 7 8 #include <uapi/linux/io_uring.h> 9 10 #include "../kernel/futex/futex.h" 11 #include "io_uring.h" 12 #include "alloc_cache.h" 13 #include "futex.h" 14 15 struct io_futex { 16 struct file *file; 17 void __user *uaddr; 18 unsigned long futex_val; 19 unsigned long futex_mask; 20 u32 futex_flags; 21 unsigned int futex_nr; 22 bool futexv_unqueued; 23 }; 24 25 struct io_futex_data { 26 struct futex_q q; 27 struct io_kiocb *req; 28 }; 29 30 struct io_futexv_data { 31 unsigned long owned; 32 struct futex_vector futexv[]; 33 }; 34 35 #define IO_FUTEX_ALLOC_CACHE_MAX 32 36 37 bool io_futex_cache_init(struct io_ring_ctx *ctx) 38 { 39 return io_alloc_cache_init(&ctx->futex_cache, IO_FUTEX_ALLOC_CACHE_MAX, 40 sizeof(struct io_futex_data), 0); 41 } 42 43 void io_futex_cache_free(struct io_ring_ctx *ctx) 44 { 45 io_alloc_cache_free(&ctx->futex_cache, kfree); 46 } 47 48 static void __io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw) 49 { 50 hlist_del_init(&tw_req.req->hash_node); 51 io_req_task_complete(tw_req, tw); 52 } 53 54 static void io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw) 55 { 56 struct io_kiocb *req = tw_req.req; 57 struct io_ring_ctx *ctx = req->ctx; 58 59 io_tw_lock(ctx, tw); 60 io_cache_free(&ctx->futex_cache, req->async_data); 61 io_req_async_data_clear(req, 0); 62 __io_futex_complete(tw_req, tw); 63 } 64 65 static void io_futexv_complete(struct io_tw_req tw_req, io_tw_token_t tw) 66 { 67 struct io_kiocb *req = tw_req.req; 68 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 69 struct io_futexv_data *ifd = req->async_data; 70 71 io_tw_lock(req->ctx, tw); 72 73 if (!iof->futexv_unqueued) { 74 int res; 75 76 res = futex_unqueue_multiple(ifd->futexv, iof->futex_nr); 77 if (res != -1) 78 io_req_set_res(req, res, 0); 79 } 80 81 io_req_async_data_free(req); 82 __io_futex_complete(tw_req, tw); 83 } 84 85 static bool io_futexv_claim(struct io_futexv_data *ifd) 86 { 87 if (test_bit(0, &ifd->owned) || test_and_set_bit_lock(0, &ifd->owned)) 88 return false; 89 return true; 90 } 91 92 static bool __io_futex_cancel(struct io_kiocb *req) 93 { 94 /* futex wake already done or in progress */ 95 if (req->opcode == IORING_OP_FUTEX_WAIT) { 96 struct io_futex_data *ifd = req->async_data; 97 98 if (!futex_unqueue(&ifd->q)) 99 return false; 100 req->io_task_work.func = io_futex_complete; 101 } else { 102 struct io_futexv_data *ifd = req->async_data; 103 104 if (!io_futexv_claim(ifd)) 105 return false; 106 req->io_task_work.func = io_futexv_complete; 107 } 108 109 hlist_del_init(&req->hash_node); 110 io_req_set_res(req, -ECANCELED, 0); 111 io_req_task_work_add(req); 112 return true; 113 } 114 115 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, 116 unsigned int issue_flags) 117 { 118 return io_cancel_remove(ctx, cd, issue_flags, &ctx->futex_list, __io_futex_cancel); 119 } 120 121 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, 122 bool cancel_all) 123 { 124 return io_cancel_remove_all(ctx, tctx, &ctx->futex_list, cancel_all, __io_futex_cancel); 125 } 126 127 int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 128 { 129 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 130 u32 flags; 131 132 if (unlikely(sqe->len || sqe->futex_flags || sqe->buf_index || 133 sqe->file_index)) 134 return -EINVAL; 135 136 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 137 iof->futex_val = READ_ONCE(sqe->addr2); 138 iof->futex_mask = READ_ONCE(sqe->addr3); 139 flags = READ_ONCE(sqe->fd); 140 141 if (flags & ~FUTEX2_VALID_MASK) 142 return -EINVAL; 143 144 iof->futex_flags = futex2_to_flags(flags); 145 if (!futex_flags_valid(iof->futex_flags)) 146 return -EINVAL; 147 148 if (!futex_validate_input(iof->futex_flags, iof->futex_val) || 149 !futex_validate_input(iof->futex_flags, iof->futex_mask)) 150 return -EINVAL; 151 152 return 0; 153 } 154 155 int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 156 { 157 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 158 int ret; 159 160 ret = io_futex_prep(req, sqe); 161 if (unlikely(ret)) 162 return ret; 163 164 /* inflight tracking only needed for mm private hash */ 165 if (!(iof->futex_flags & FLAGS_SHARED)) 166 io_req_track_inflight(req); 167 return 0; 168 } 169 170 static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) 171 { 172 struct io_kiocb *req = q->wake_data; 173 struct io_futexv_data *ifd = req->async_data; 174 175 if (!io_futexv_claim(ifd)) { 176 __futex_wake_mark(q); 177 return; 178 } 179 if (unlikely(!__futex_wake_mark(q))) 180 return; 181 182 io_req_set_res(req, 0, 0); 183 req->io_task_work.func = io_futexv_complete; 184 __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); 185 } 186 187 int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 188 { 189 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 190 struct io_futexv_data *ifd; 191 unsigned int i; 192 int ret; 193 194 /* No flags or mask supported for waitv */ 195 if (unlikely(sqe->fd || sqe->buf_index || sqe->file_index || 196 sqe->addr2 || sqe->futex_flags || sqe->addr3)) 197 return -EINVAL; 198 199 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 200 iof->futex_nr = READ_ONCE(sqe->len); 201 if (!iof->futex_nr || iof->futex_nr > FUTEX_WAITV_MAX) 202 return -EINVAL; 203 204 ifd = kzalloc_flex(struct io_futexv_data, futexv, iof->futex_nr, 205 GFP_KERNEL_ACCOUNT); 206 if (!ifd) 207 return -ENOMEM; 208 209 ret = futex_parse_waitv(ifd->futexv, iof->uaddr, iof->futex_nr, 210 io_futex_wakev_fn, req); 211 if (ret) { 212 kfree(ifd); 213 return ret; 214 } 215 216 /* inflight tracking only needed for mm private hash */ 217 for (i = 0; i < iof->futex_nr; i++) { 218 if (!(ifd->futexv[i].w.flags & FLAGS_SHARED)) { 219 io_req_track_inflight(req); 220 break; 221 } 222 } 223 224 iof->futexv_unqueued = 0; 225 req->flags |= REQ_F_ASYNC_DATA; 226 req->async_data = ifd; 227 return 0; 228 } 229 230 static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) 231 { 232 struct io_futex_data *ifd = container_of(q, struct io_futex_data, q); 233 struct io_kiocb *req = ifd->req; 234 235 if (unlikely(!__futex_wake_mark(q))) 236 return; 237 238 io_req_set_res(req, 0, 0); 239 req->io_task_work.func = io_futex_complete; 240 __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); 241 } 242 243 int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) 244 { 245 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 246 struct io_futexv_data *ifd = req->async_data; 247 struct io_ring_ctx *ctx = req->ctx; 248 int ret, woken = -1; 249 250 io_ring_submit_lock(ctx, issue_flags); 251 252 ret = futex_wait_multiple_setup(ifd->futexv, iof->futex_nr, &woken); 253 254 /* 255 * Error case, ret is < 0. Mark the request as failed. 256 */ 257 if (unlikely(ret < 0)) { 258 io_ring_submit_unlock(ctx, issue_flags); 259 req_set_fail(req); 260 io_req_set_res(req, ret, 0); 261 io_req_async_data_free(req); 262 return IOU_COMPLETE; 263 } 264 265 /* 266 * 0 return means that we successfully setup the waiters, and that 267 * nobody triggered a wakeup while we were doing so. If the wakeup 268 * happened post setup, the task_work will be run post this issue and 269 * under the submission lock. 1 means We got woken while setting up, 270 * let that side do the completion. Note that 271 * futex_wait_multiple_setup() will have unqueued all the futexes in 272 * this case. Mark us as having done that already, since this is 273 * different from normal wakeup. 274 */ 275 if (!ret) { 276 /* 277 * If futex_wait_multiple_setup() returns 0 for a 278 * successful setup, then the task state will not be 279 * runnable. This is fine for the sync syscall, as 280 * it'll be blocking unless we already got one of the 281 * futexes woken, but it obviously won't work for an 282 * async invocation. Mark us runnable again. 283 */ 284 __set_current_state(TASK_RUNNING); 285 hlist_add_head(&req->hash_node, &ctx->futex_list); 286 } else { 287 iof->futexv_unqueued = 1; 288 if (woken != -1) 289 io_req_set_res(req, woken, 0); 290 } 291 292 io_ring_submit_unlock(ctx, issue_flags); 293 return IOU_ISSUE_SKIP_COMPLETE; 294 } 295 296 int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) 297 { 298 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 299 struct io_ring_ctx *ctx = req->ctx; 300 struct io_futex_data *ifd = NULL; 301 int ret; 302 303 if (!iof->futex_mask) { 304 ret = -EINVAL; 305 goto done; 306 } 307 308 io_ring_submit_lock(ctx, issue_flags); 309 ifd = io_cache_alloc(&ctx->futex_cache, GFP_NOWAIT); 310 if (!ifd) { 311 ret = -ENOMEM; 312 goto done_unlock; 313 } 314 315 req->flags |= REQ_F_ASYNC_DATA; 316 req->async_data = ifd; 317 ifd->q = futex_q_init; 318 ifd->q.bitset = iof->futex_mask; 319 ifd->q.wake = io_futex_wake_fn; 320 ifd->req = req; 321 322 ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags, 323 &ifd->q, NULL, NULL); 324 if (!ret) { 325 hlist_add_head(&req->hash_node, &ctx->futex_list); 326 io_ring_submit_unlock(ctx, issue_flags); 327 328 return IOU_ISSUE_SKIP_COMPLETE; 329 } 330 331 done_unlock: 332 io_ring_submit_unlock(ctx, issue_flags); 333 done: 334 if (ret < 0) 335 req_set_fail(req); 336 io_req_set_res(req, ret, 0); 337 io_req_async_data_free(req); 338 return IOU_COMPLETE; 339 } 340 341 int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags) 342 { 343 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 344 int ret; 345 346 /* 347 * Strict flags - ensure that waking 0 futexes yields a 0 result. 348 * See commit 43adf8449510 ("futex: FLAGS_STRICT") for details. 349 */ 350 ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags, NULL, 351 iof->futex_val, iof->futex_mask); 352 if (ret < 0) 353 req_set_fail(req); 354 io_req_set_res(req, ret, 0); 355 return IOU_COMPLETE; 356 } 357