1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef IOU_CORE_H 3 #define IOU_CORE_H 4 5 #include <linux/errno.h> 6 #include <linux/lockdep.h> 7 #include <linux/resume_user_mode.h> 8 #include <linux/poll.h> 9 #include <linux/io_uring_types.h> 10 #include <uapi/linux/eventpoll.h> 11 #include "alloc_cache.h" 12 #include "io-wq.h" 13 #include "slist.h" 14 #include "tw.h" 15 #include "opdef.h" 16 17 #ifndef CREATE_TRACE_POINTS 18 #include <trace/events/io_uring.h> 19 #endif 20 21 struct io_rings_layout { 22 /* size of CQ + headers + SQ offset array */ 23 size_t rings_size; 24 size_t sq_size; 25 26 size_t sq_array_offset; 27 }; 28 29 struct io_ctx_config { 30 struct io_uring_params p; 31 struct io_rings_layout layout; 32 struct io_uring_params __user *uptr; 33 }; 34 35 #define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\ 36 IORING_FEAT_NODROP |\ 37 IORING_FEAT_SUBMIT_STABLE |\ 38 IORING_FEAT_RW_CUR_POS |\ 39 IORING_FEAT_CUR_PERSONALITY |\ 40 IORING_FEAT_FAST_POLL |\ 41 IORING_FEAT_POLL_32BITS |\ 42 IORING_FEAT_SQPOLL_NONFIXED |\ 43 IORING_FEAT_EXT_ARG |\ 44 IORING_FEAT_NATIVE_WORKERS |\ 45 IORING_FEAT_RSRC_TAGS |\ 46 IORING_FEAT_CQE_SKIP |\ 47 IORING_FEAT_LINKED_FILE |\ 48 IORING_FEAT_REG_REG_RING |\ 49 IORING_FEAT_RECVSEND_BUNDLE |\ 50 IORING_FEAT_MIN_TIMEOUT |\ 51 IORING_FEAT_RW_ATTR |\ 52 IORING_FEAT_NO_IOWAIT) 53 54 #define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\ 55 IORING_SETUP_SQPOLL |\ 56 IORING_SETUP_SQ_AFF |\ 57 IORING_SETUP_CQSIZE |\ 58 IORING_SETUP_CLAMP |\ 59 IORING_SETUP_ATTACH_WQ |\ 60 IORING_SETUP_R_DISABLED |\ 61 IORING_SETUP_SUBMIT_ALL |\ 62 IORING_SETUP_COOP_TASKRUN |\ 63 IORING_SETUP_TASKRUN_FLAG |\ 64 IORING_SETUP_SQE128 |\ 65 IORING_SETUP_CQE32 |\ 66 IORING_SETUP_SINGLE_ISSUER |\ 67 IORING_SETUP_DEFER_TASKRUN |\ 68 IORING_SETUP_NO_MMAP |\ 69 IORING_SETUP_REGISTERED_FD_ONLY |\ 70 IORING_SETUP_NO_SQARRAY |\ 71 IORING_SETUP_HYBRID_IOPOLL |\ 72 IORING_SETUP_CQE_MIXED |\ 73 IORING_SETUP_SQE_MIXED |\ 74 IORING_SETUP_SQ_REWIND) 75 76 #define IORING_ENTER_FLAGS (IORING_ENTER_GETEVENTS |\ 77 IORING_ENTER_SQ_WAKEUP |\ 78 IORING_ENTER_SQ_WAIT |\ 79 IORING_ENTER_EXT_ARG |\ 80 IORING_ENTER_REGISTERED_RING |\ 81 IORING_ENTER_ABS_TIMER |\ 82 IORING_ENTER_EXT_ARG_REG |\ 83 IORING_ENTER_NO_IOWAIT) 84 85 86 #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE |\ 87 IOSQE_IO_DRAIN |\ 88 IOSQE_IO_LINK |\ 89 IOSQE_IO_HARDLINK |\ 90 IOSQE_ASYNC |\ 91 IOSQE_BUFFER_SELECT |\ 92 IOSQE_CQE_SKIP_SUCCESS) 93 94 #define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK) 95 96 /* 97 * Complaint timeout for io_uring cancelation exits, and for io-wq exit 98 * worker waiting. 99 */ 100 #define IO_URING_EXIT_WAIT_MAX (HZ * 60 * 5) 101 102 enum { 103 IOU_COMPLETE = 0, 104 105 IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED, 106 107 /* 108 * The request has more work to do and should be retried. io_uring will 109 * attempt to wait on the file for eligible opcodes, but otherwise 110 * it'll be handed to iowq for blocking execution. It works for normal 111 * requests as well as for the multi shot mode. 112 */ 113 IOU_RETRY = -EAGAIN, 114 115 /* 116 * Requeue the task_work to restart operations on this request. The 117 * actual value isn't important, should just be not an otherwise 118 * valid error code, yet less than -MAX_ERRNO and valid internally. 119 */ 120 IOU_REQUEUE = -3072, 121 }; 122 123 struct io_defer_entry { 124 struct list_head list; 125 struct io_kiocb *req; 126 }; 127 128 struct io_wait_queue { 129 struct wait_queue_entry wq; 130 struct io_ring_ctx *ctx; 131 unsigned cq_tail; 132 unsigned cq_min_tail; 133 unsigned nr_timeouts; 134 int hit_timeout; 135 ktime_t min_timeout; 136 ktime_t timeout; 137 struct hrtimer t; 138 139 #ifdef CONFIG_NET_RX_BUSY_POLL 140 ktime_t napi_busy_poll_dt; 141 bool napi_prefer_busy_poll; 142 #endif 143 }; 144 145 static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx) 146 { 147 return rcu_dereference_check(ctx->rings_rcu, 148 lockdep_is_held(&ctx->uring_lock) || 149 lockdep_is_held(&ctx->completion_lock)); 150 } 151 152 static inline bool io_should_wake(struct io_wait_queue *iowq) 153 { 154 struct io_ring_ctx *ctx = iowq->ctx; 155 struct io_rings *rings; 156 int dist; 157 158 guard(rcu)(); 159 rings = io_get_rings(ctx); 160 161 /* 162 * Wake up if we have enough events, or if a timeout occurred since we 163 * started waiting. For timeouts, we always want to return to userspace, 164 * regardless of event count. 165 */ 166 dist = READ_ONCE(rings->cq.tail) - (int) iowq->cq_tail; 167 return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; 168 } 169 170 #define IORING_MAX_ENTRIES 32768 171 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) 172 173 int io_prepare_config(struct io_ctx_config *config); 174 175 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32); 176 void io_req_defer_failed(struct io_kiocb *req, s32 res); 177 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); 178 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); 179 bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags); 180 bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe src_cqe[2]); 181 void __io_commit_cqring_flush(struct io_ring_ctx *ctx); 182 183 unsigned io_linked_nr(struct io_kiocb *req); 184 void io_req_track_inflight(struct io_kiocb *req); 185 struct file *io_file_get_normal(struct io_kiocb *req, int fd); 186 struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 187 unsigned issue_flags); 188 struct file *io_uring_ctx_get_file(unsigned int fd, bool registered); 189 190 void io_req_task_queue(struct io_kiocb *req); 191 void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw); 192 void io_req_task_queue_fail(struct io_kiocb *req, int ret); 193 void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw); 194 __cold void io_uring_drop_tctx_refs(struct task_struct *task); 195 196 int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file, 197 int start, int end); 198 void io_req_queue_iowq(struct io_kiocb *req); 199 200 int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw); 201 int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr); 202 int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin); 203 __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx); 204 void __io_submit_flush_completions(struct io_ring_ctx *ctx); 205 206 struct io_wq_work *io_wq_free_work(struct io_wq_work *work); 207 void io_wq_submit_work(struct io_wq_work *work); 208 209 void io_free_req(struct io_kiocb *req); 210 void io_queue_next(struct io_kiocb *req); 211 void io_task_refs_refill(struct io_uring_task *tctx); 212 bool __io_alloc_req_refill(struct io_ring_ctx *ctx); 213 214 void io_activate_pollwq(struct io_ring_ctx *ctx); 215 void io_restriction_clone(struct io_restriction *dst, struct io_restriction *src); 216 217 static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx) 218 { 219 #if defined(CONFIG_PROVE_LOCKING) 220 lockdep_assert(in_task()); 221 222 if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) 223 lockdep_assert_held(&ctx->uring_lock); 224 225 if (ctx->flags & IORING_SETUP_IOPOLL) { 226 lockdep_assert_held(&ctx->uring_lock); 227 } else if (!(ctx->int_flags & IO_RING_F_TASK_COMPLETE)) { 228 lockdep_assert_held(&ctx->completion_lock); 229 } else if (ctx->submitter_task) { 230 /* 231 * ->submitter_task may be NULL and we can still post a CQE, 232 * if the ring has been setup with IORING_SETUP_R_DISABLED. 233 * Not from an SQE, as those cannot be submitted, but via 234 * updating tagged resources. 235 */ 236 if (!percpu_ref_is_dying(&ctx->refs)) 237 lockdep_assert(current == ctx->submitter_task); 238 } 239 #endif 240 } 241 242 static inline bool io_is_compat(struct io_ring_ctx *ctx) 243 { 244 return IS_ENABLED(CONFIG_COMPAT) && unlikely(ctx->int_flags & IO_RING_F_COMPAT); 245 } 246 247 static inline void io_submit_flush_completions(struct io_ring_ctx *ctx) 248 { 249 if (!wq_list_empty(&ctx->submit_state.compl_reqs) || 250 ctx->submit_state.cq_flush) 251 __io_submit_flush_completions(ctx); 252 } 253 254 #define io_for_each_link(pos, head) \ 255 for (pos = (head); pos; pos = pos->link) 256 257 static inline bool io_get_cqe_overflow(struct io_ring_ctx *ctx, 258 struct io_uring_cqe **ret, 259 bool overflow, bool cqe32) 260 { 261 io_lockdep_assert_cq_locked(ctx); 262 263 if (unlikely(ctx->cqe_sentinel - ctx->cqe_cached < (cqe32 + 1))) { 264 if (unlikely(!io_cqe_cache_refill(ctx, overflow, cqe32))) 265 return false; 266 } 267 *ret = ctx->cqe_cached; 268 ctx->cached_cq_tail++; 269 ctx->cqe_cached++; 270 if (ctx->flags & IORING_SETUP_CQE32) { 271 ctx->cqe_cached++; 272 } else if (cqe32 && ctx->flags & IORING_SETUP_CQE_MIXED) { 273 ctx->cqe_cached++; 274 ctx->cached_cq_tail++; 275 } 276 WARN_ON_ONCE(ctx->cqe_cached > ctx->cqe_sentinel); 277 return true; 278 } 279 280 static inline bool io_get_cqe(struct io_ring_ctx *ctx, struct io_uring_cqe **ret, 281 bool cqe32) 282 { 283 return io_get_cqe_overflow(ctx, ret, false, cqe32); 284 } 285 286 static inline bool io_defer_get_uncommited_cqe(struct io_ring_ctx *ctx, 287 struct io_uring_cqe **cqe_ret) 288 { 289 io_lockdep_assert_cq_locked(ctx); 290 291 ctx->submit_state.cq_flush = true; 292 return io_get_cqe(ctx, cqe_ret, ctx->flags & IORING_SETUP_CQE_MIXED); 293 } 294 295 static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx, 296 struct io_kiocb *req) 297 { 298 bool is_cqe32 = req->cqe.flags & IORING_CQE_F_32; 299 struct io_uring_cqe *cqe; 300 301 /* 302 * If we can't get a cq entry, userspace overflowed the submission 303 * (by quite a lot). 304 */ 305 if (unlikely(!io_get_cqe(ctx, &cqe, is_cqe32))) 306 return false; 307 308 memcpy(cqe, &req->cqe, sizeof(*cqe)); 309 if (ctx->flags & IORING_SETUP_CQE32 || is_cqe32) { 310 memcpy(cqe->big_cqe, &req->big_cqe, sizeof(*cqe)); 311 memset(&req->big_cqe, 0, sizeof(req->big_cqe)); 312 } 313 314 if (trace_io_uring_complete_enabled()) 315 trace_io_uring_complete(req->ctx, req, cqe); 316 return true; 317 } 318 319 static inline void req_set_fail(struct io_kiocb *req) 320 { 321 req->flags |= REQ_F_FAIL; 322 if (req->flags & REQ_F_CQE_SKIP) { 323 req->flags &= ~REQ_F_CQE_SKIP; 324 req->flags |= REQ_F_SKIP_LINK_CQES; 325 } 326 } 327 328 static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags) 329 { 330 req->cqe.res = res; 331 req->cqe.flags = cflags; 332 } 333 334 static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx) 335 { 336 if (ctx->flags & IORING_SETUP_CQE_MIXED) 337 return IORING_CQE_F_32; 338 return 0; 339 } 340 341 static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags, 342 __u64 extra1, __u64 extra2) 343 { 344 req->cqe.res = res; 345 req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx); 346 req->big_cqe.extra1 = extra1; 347 req->big_cqe.extra2 = extra2; 348 } 349 350 static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache, 351 struct io_kiocb *req) 352 { 353 if (cache) { 354 req->async_data = io_cache_alloc(cache, GFP_KERNEL); 355 } else { 356 const struct io_issue_def *def = &io_issue_defs[req->opcode]; 357 358 WARN_ON_ONCE(!def->async_size); 359 req->async_data = kmalloc(def->async_size, GFP_KERNEL); 360 } 361 if (req->async_data) 362 req->flags |= REQ_F_ASYNC_DATA; 363 return req->async_data; 364 } 365 366 static inline bool req_has_async_data(struct io_kiocb *req) 367 { 368 return req->flags & REQ_F_ASYNC_DATA; 369 } 370 371 static inline void io_req_async_data_clear(struct io_kiocb *req, 372 io_req_flags_t extra_flags) 373 { 374 req->flags &= ~(REQ_F_ASYNC_DATA|extra_flags); 375 req->async_data = NULL; 376 } 377 378 static inline void io_req_async_data_free(struct io_kiocb *req) 379 { 380 kfree(req->async_data); 381 io_req_async_data_clear(req, 0); 382 } 383 384 static inline void io_put_file(struct io_kiocb *req) 385 { 386 if (!(req->flags & REQ_F_FIXED_FILE) && req->file) 387 fput(req->file); 388 } 389 390 static inline void io_ring_submit_unlock(struct io_ring_ctx *ctx, 391 unsigned issue_flags) 392 { 393 lockdep_assert_held(&ctx->uring_lock); 394 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) 395 mutex_unlock(&ctx->uring_lock); 396 } 397 398 static inline void io_ring_submit_lock(struct io_ring_ctx *ctx, 399 unsigned issue_flags) 400 { 401 /* 402 * "Normal" inline submissions always hold the uring_lock, since we 403 * grab it from the system call. Same is true for the SQPOLL offload. 404 * The only exception is when we've detached the request and issue it 405 * from an async worker thread, grab the lock for that case. 406 */ 407 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) 408 mutex_lock(&ctx->uring_lock); 409 lockdep_assert_held(&ctx->uring_lock); 410 } 411 412 static inline void io_commit_cqring(struct io_ring_ctx *ctx) 413 { 414 /* order cqe stores with ring update */ 415 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); 416 } 417 418 static inline void __io_wq_wake(struct wait_queue_head *wq) 419 { 420 /* 421 * 422 * Pass in EPOLLIN|EPOLL_URING_WAKE as the poll wakeup key. The latter 423 * set in the mask so that if we recurse back into our own poll 424 * waitqueue handlers, we know we have a dependency between eventfd or 425 * epoll and should terminate multishot poll at that point. 426 */ 427 if (wq_has_sleeper(wq)) 428 __wake_up(wq, TASK_NORMAL, 0, poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); 429 } 430 431 static inline void io_poll_wq_wake(struct io_ring_ctx *ctx) 432 { 433 __io_wq_wake(&ctx->poll_wq); 434 } 435 436 static inline void io_cqring_wake(struct io_ring_ctx *ctx) 437 { 438 /* 439 * Trigger waitqueue handler on all waiters on our waitqueue. This 440 * won't necessarily wake up all the tasks, io_should_wake() will make 441 * that decision. 442 */ 443 444 __io_wq_wake(&ctx->cq_wait); 445 } 446 447 static inline bool __io_sqring_full(struct io_ring_ctx *ctx) 448 { 449 struct io_rings *r = io_get_rings(ctx); 450 451 /* 452 * SQPOLL must use the actual sqring head, as using the cached_sq_head 453 * is race prone if the SQPOLL thread has grabbed entries but not yet 454 * committed them to the ring. For !SQPOLL, this doesn't matter, but 455 * since this helper is just used for SQPOLL sqring waits (or POLLOUT), 456 * just read the actual sqring head unconditionally. 457 */ 458 return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries; 459 } 460 461 static inline bool io_sqring_full(struct io_ring_ctx *ctx) 462 { 463 guard(rcu)(); 464 return __io_sqring_full(ctx); 465 } 466 467 static inline unsigned int __io_sqring_entries(struct io_ring_ctx *ctx) 468 { 469 struct io_rings *rings = io_get_rings(ctx); 470 unsigned int entries; 471 472 /* make sure SQ entry isn't read before tail */ 473 entries = smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; 474 return min(entries, ctx->sq_entries); 475 } 476 477 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) 478 { 479 guard(rcu)(); 480 return __io_sqring_entries(ctx); 481 } 482 483 /* 484 * Don't complete immediately but use deferred completion infrastructure. 485 * Protected by ->uring_lock and can only be used either with 486 * IO_URING_F_COMPLETE_DEFER or inside a tw handler holding the mutex. 487 */ 488 static inline void io_req_complete_defer(struct io_kiocb *req) 489 __must_hold(&req->ctx->uring_lock) 490 { 491 struct io_submit_state *state = &req->ctx->submit_state; 492 493 lockdep_assert_held(&req->ctx->uring_lock); 494 495 wq_list_add_tail(&req->comp_list, &state->compl_reqs); 496 } 497 498 #define SHOULD_FLUSH_MASK (IO_RING_F_OFF_TIMEOUT_USED | \ 499 IO_RING_F_HAS_EVFD | IO_RING_F_POLL_ACTIVATED) 500 501 static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx) 502 { 503 if (unlikely(data_race(ctx->int_flags) & SHOULD_FLUSH_MASK)) 504 __io_commit_cqring_flush(ctx); 505 } 506 507 static inline void io_get_task_refs(int nr) 508 { 509 struct io_uring_task *tctx = current->io_uring; 510 511 tctx->cached_refs -= nr; 512 if (unlikely(tctx->cached_refs < 0)) 513 io_task_refs_refill(tctx); 514 } 515 516 static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) 517 { 518 return !ctx->submit_state.free_list.next; 519 } 520 521 extern struct kmem_cache *req_cachep; 522 523 static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx) 524 { 525 struct io_kiocb *req; 526 527 req = container_of(ctx->submit_state.free_list.next, struct io_kiocb, comp_list); 528 wq_stack_extract(&ctx->submit_state.free_list); 529 return req; 530 } 531 532 static inline bool io_alloc_req(struct io_ring_ctx *ctx, struct io_kiocb **req) 533 { 534 if (unlikely(io_req_cache_empty(ctx))) { 535 if (!__io_alloc_req_refill(ctx)) 536 return false; 537 } 538 *req = io_extract_req(ctx); 539 return true; 540 } 541 542 static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) 543 { 544 io_req_set_res(req, res, 0); 545 req->io_task_work.func = io_req_task_complete; 546 io_req_task_work_add(req); 547 } 548 549 static inline bool io_file_can_poll(struct io_kiocb *req) 550 { 551 if (req->flags & REQ_F_CAN_POLL) 552 return true; 553 if (req->file && file_can_poll(req->file)) { 554 req->flags |= REQ_F_CAN_POLL; 555 return true; 556 } 557 return false; 558 } 559 560 static inline bool io_is_uring_cmd(const struct io_kiocb *req) 561 { 562 return req->opcode == IORING_OP_URING_CMD || 563 req->opcode == IORING_OP_URING_CMD128; 564 } 565 566 static inline ktime_t io_get_time(struct io_ring_ctx *ctx) 567 { 568 if (ctx->clockid == CLOCK_MONOTONIC) 569 return ktime_get(); 570 571 return ktime_get_with_offset(ctx->clock_offset); 572 } 573 574 enum { 575 IO_CHECK_CQ_OVERFLOW_BIT, 576 IO_CHECK_CQ_DROPPED_BIT, 577 }; 578 579 static inline bool io_has_work(struct io_ring_ctx *ctx) 580 { 581 return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) || 582 io_local_work_pending(ctx); 583 } 584 #endif 585