1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef IOU_CORE_H 3 #define IOU_CORE_H 4 5 #include <linux/errno.h> 6 #include <linux/file.h> 7 #include <linux/lockdep.h> 8 #include <linux/resume_user_mode.h> 9 #include <linux/poll.h> 10 #include <linux/io_uring_types.h> 11 #include <uapi/linux/eventpoll.h> 12 #include "alloc_cache.h" 13 #include "io-wq.h" 14 #include "slist.h" 15 #include "tw.h" 16 #include "opdef.h" 17 18 #ifndef CREATE_TRACE_POINTS 19 #include <trace/events/io_uring.h> 20 #endif 21 22 struct io_rings_layout { 23 /* size of CQ + headers + SQ offset array */ 24 size_t rings_size; 25 size_t sq_size; 26 27 size_t sq_array_offset; 28 }; 29 30 struct io_ctx_config { 31 struct io_uring_params p; 32 struct io_rings_layout layout; 33 struct io_uring_params __user *uptr; 34 }; 35 36 #define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\ 37 IORING_FEAT_NODROP |\ 38 IORING_FEAT_SUBMIT_STABLE |\ 39 IORING_FEAT_RW_CUR_POS |\ 40 IORING_FEAT_CUR_PERSONALITY |\ 41 IORING_FEAT_FAST_POLL |\ 42 IORING_FEAT_POLL_32BITS |\ 43 IORING_FEAT_SQPOLL_NONFIXED |\ 44 IORING_FEAT_EXT_ARG |\ 45 IORING_FEAT_NATIVE_WORKERS |\ 46 IORING_FEAT_RSRC_TAGS |\ 47 IORING_FEAT_CQE_SKIP |\ 48 IORING_FEAT_LINKED_FILE |\ 49 IORING_FEAT_REG_REG_RING |\ 50 IORING_FEAT_RECVSEND_BUNDLE |\ 51 IORING_FEAT_MIN_TIMEOUT |\ 52 IORING_FEAT_RW_ATTR |\ 53 IORING_FEAT_NO_IOWAIT) 54 55 #define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\ 56 IORING_SETUP_SQPOLL |\ 57 IORING_SETUP_SQ_AFF |\ 58 IORING_SETUP_CQSIZE |\ 59 IORING_SETUP_CLAMP |\ 60 IORING_SETUP_ATTACH_WQ |\ 61 IORING_SETUP_R_DISABLED |\ 62 IORING_SETUP_SUBMIT_ALL |\ 63 IORING_SETUP_COOP_TASKRUN |\ 64 IORING_SETUP_TASKRUN_FLAG |\ 65 IORING_SETUP_SQE128 |\ 66 IORING_SETUP_CQE32 |\ 67 IORING_SETUP_SINGLE_ISSUER |\ 68 IORING_SETUP_DEFER_TASKRUN |\ 69 IORING_SETUP_NO_MMAP |\ 70 IORING_SETUP_REGISTERED_FD_ONLY |\ 71 IORING_SETUP_NO_SQARRAY |\ 72 IORING_SETUP_HYBRID_IOPOLL |\ 73 IORING_SETUP_CQE_MIXED |\ 74 IORING_SETUP_SQE_MIXED |\ 75 IORING_SETUP_SQ_REWIND) 76 77 #define IORING_ENTER_FLAGS (IORING_ENTER_GETEVENTS |\ 78 IORING_ENTER_SQ_WAKEUP |\ 79 IORING_ENTER_SQ_WAIT |\ 80 IORING_ENTER_EXT_ARG |\ 81 IORING_ENTER_REGISTERED_RING |\ 82 IORING_ENTER_ABS_TIMER |\ 83 IORING_ENTER_EXT_ARG_REG |\ 84 IORING_ENTER_NO_IOWAIT) 85 86 87 #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE |\ 88 IOSQE_IO_DRAIN |\ 89 IOSQE_IO_LINK |\ 90 IOSQE_IO_HARDLINK |\ 91 IOSQE_ASYNC |\ 92 IOSQE_BUFFER_SELECT |\ 93 IOSQE_CQE_SKIP_SUCCESS) 94 95 #define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK) 96 97 /* 98 * Complaint timeout for io_uring cancelation exits, and for io-wq exit 99 * worker waiting. 100 */ 101 #define IO_URING_EXIT_WAIT_MAX (HZ * 60 * 5) 102 103 enum { 104 IOU_COMPLETE = 0, 105 106 IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED, 107 108 /* 109 * The request has more work to do and should be retried. io_uring will 110 * attempt to wait on the file for eligible opcodes, but otherwise 111 * it'll be handed to iowq for blocking execution. It works for normal 112 * requests as well as for the multi shot mode. 113 */ 114 IOU_RETRY = -EAGAIN, 115 116 /* 117 * Requeue the task_work to restart operations on this request. The 118 * actual value isn't important, should just be not an otherwise 119 * valid error code, yet less than -MAX_ERRNO and valid internally. 120 */ 121 IOU_REQUEUE = -3072, 122 }; 123 124 struct io_defer_entry { 125 struct list_head list; 126 struct io_kiocb *req; 127 }; 128 129 struct io_wait_queue { 130 struct wait_queue_entry wq; 131 struct io_ring_ctx *ctx; 132 unsigned cq_tail; 133 unsigned cq_min_tail; 134 unsigned nr_timeouts; 135 int hit_timeout; 136 ktime_t min_timeout; 137 ktime_t timeout; 138 struct hrtimer t; 139 140 #ifdef CONFIG_NET_RX_BUSY_POLL 141 ktime_t napi_busy_poll_dt; 142 bool napi_prefer_busy_poll; 143 #endif 144 }; 145 146 static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx) 147 { 148 return rcu_dereference_check(ctx->rings_rcu, 149 lockdep_is_held(&ctx->uring_lock) || 150 lockdep_is_held(&ctx->completion_lock)); 151 } 152 153 static inline bool io_should_wake(struct io_wait_queue *iowq) 154 { 155 struct io_ring_ctx *ctx = iowq->ctx; 156 struct io_rings *rings; 157 int dist; 158 159 guard(rcu)(); 160 rings = io_get_rings(ctx); 161 162 /* 163 * Wake up if we have enough events, or if a timeout occurred since we 164 * started waiting. For timeouts, we always want to return to userspace, 165 * regardless of event count. 166 */ 167 dist = READ_ONCE(rings->cq.tail) - (int) iowq->cq_tail; 168 return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; 169 } 170 171 #define IORING_MAX_ENTRIES 32768 172 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) 173 174 int io_prepare_config(struct io_ctx_config *config); 175 176 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32); 177 void io_req_defer_failed(struct io_kiocb *req, s32 res); 178 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); 179 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); 180 bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags); 181 bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe src_cqe[2]); 182 void __io_commit_cqring_flush(struct io_ring_ctx *ctx); 183 184 unsigned io_linked_nr(struct io_kiocb *req); 185 void io_req_track_inflight(struct io_kiocb *req); 186 struct file *io_file_get_normal(struct io_kiocb *req, int fd); 187 struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 188 unsigned issue_flags); 189 struct file *io_uring_ctx_get_file(unsigned int fd, bool registered); 190 191 void io_req_task_queue(struct io_kiocb *req); 192 void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw); 193 void io_req_task_queue_fail(struct io_kiocb *req, int ret); 194 void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw); 195 __cold void io_uring_drop_tctx_refs(struct task_struct *task); 196 197 int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file, 198 int start, int end); 199 void io_queue_iowq(struct io_kiocb *req); 200 201 int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw); 202 int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr); 203 int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin); 204 __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx); 205 void __io_submit_flush_completions(struct io_ring_ctx *ctx); 206 207 struct io_wq_work *io_wq_free_work(struct io_wq_work *work); 208 void io_wq_submit_work(struct io_wq_work *work); 209 210 void io_free_req(struct io_kiocb *req); 211 void io_queue_next(struct io_kiocb *req); 212 void io_task_refs_refill(struct io_uring_task *tctx); 213 bool __io_alloc_req_refill(struct io_ring_ctx *ctx); 214 215 void io_activate_pollwq(struct io_ring_ctx *ctx); 216 void io_restriction_clone(struct io_restriction *dst, struct io_restriction *src); 217 void io_poison_req(struct io_kiocb *req); 218 219 static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx) 220 { 221 #if defined(CONFIG_PROVE_LOCKING) 222 lockdep_assert(in_task()); 223 224 if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) 225 lockdep_assert_held(&ctx->uring_lock); 226 227 if (ctx->flags & IORING_SETUP_IOPOLL) { 228 lockdep_assert_held(&ctx->uring_lock); 229 } else if (!(ctx->int_flags & IO_RING_F_TASK_COMPLETE)) { 230 lockdep_assert_held(&ctx->completion_lock); 231 } else if (ctx->submitter_task) { 232 /* 233 * ->submitter_task may be NULL and we can still post a CQE, 234 * if the ring has been setup with IORING_SETUP_R_DISABLED. 235 * Not from an SQE, as those cannot be submitted, but via 236 * updating tagged resources. 237 */ 238 if (!percpu_ref_is_dying(&ctx->refs)) 239 lockdep_assert(current == ctx->submitter_task); 240 } 241 #endif 242 } 243 244 static inline bool io_is_compat(struct io_ring_ctx *ctx) 245 { 246 return IS_ENABLED(CONFIG_COMPAT) && unlikely(ctx->int_flags & IO_RING_F_COMPAT); 247 } 248 249 static inline void io_submit_flush_completions(struct io_ring_ctx *ctx) 250 { 251 if (!wq_list_empty(&ctx->submit_state.compl_reqs) || 252 ctx->submit_state.cq_flush) 253 __io_submit_flush_completions(ctx); 254 } 255 256 #define io_for_each_link(pos, head) \ 257 for (pos = (head); pos; pos = pos->link) 258 259 static inline bool io_get_cqe_overflow(struct io_ring_ctx *ctx, 260 struct io_uring_cqe **ret, 261 bool overflow, bool cqe32) 262 { 263 io_lockdep_assert_cq_locked(ctx); 264 265 if (unlikely(ctx->cqe_sentinel - ctx->cqe_cached < (cqe32 + 1))) { 266 if (unlikely(!io_cqe_cache_refill(ctx, overflow, cqe32))) 267 return false; 268 } 269 *ret = ctx->cqe_cached; 270 ctx->cached_cq_tail++; 271 ctx->cqe_cached++; 272 if (ctx->flags & IORING_SETUP_CQE32) { 273 ctx->cqe_cached++; 274 } else if (cqe32 && ctx->flags & IORING_SETUP_CQE_MIXED) { 275 ctx->cqe_cached++; 276 ctx->cached_cq_tail++; 277 } 278 WARN_ON_ONCE(ctx->cqe_cached > ctx->cqe_sentinel); 279 return true; 280 } 281 282 static inline bool io_get_cqe(struct io_ring_ctx *ctx, struct io_uring_cqe **ret, 283 bool cqe32) 284 { 285 return io_get_cqe_overflow(ctx, ret, false, cqe32); 286 } 287 288 static inline bool io_defer_get_uncommited_cqe(struct io_ring_ctx *ctx, 289 struct io_uring_cqe **cqe_ret) 290 { 291 io_lockdep_assert_cq_locked(ctx); 292 293 ctx->submit_state.cq_flush = true; 294 return io_get_cqe(ctx, cqe_ret, ctx->flags & IORING_SETUP_CQE_MIXED); 295 } 296 297 static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx, 298 struct io_kiocb *req) 299 { 300 bool is_cqe32 = req->cqe.flags & IORING_CQE_F_32; 301 struct io_uring_cqe *cqe; 302 303 /* 304 * If we can't get a cq entry, userspace overflowed the submission 305 * (by quite a lot). 306 */ 307 if (unlikely(!io_get_cqe(ctx, &cqe, is_cqe32))) 308 return false; 309 310 memcpy(cqe, &req->cqe, sizeof(*cqe)); 311 if (ctx->flags & IORING_SETUP_CQE32 || is_cqe32) { 312 memcpy(cqe->big_cqe, &req->big_cqe, sizeof(*cqe)); 313 memset(&req->big_cqe, 0, sizeof(req->big_cqe)); 314 } 315 316 if (trace_io_uring_complete_enabled()) 317 trace_call__io_uring_complete(req->ctx, req, cqe); 318 return true; 319 } 320 321 static inline void req_set_fail(struct io_kiocb *req) 322 { 323 req->flags |= REQ_F_FAIL; 324 if (req->flags & REQ_F_CQE_SKIP) { 325 req->flags &= ~REQ_F_CQE_SKIP; 326 req->flags |= REQ_F_SKIP_LINK_CQES; 327 } 328 } 329 330 static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags) 331 { 332 req->cqe.res = res; 333 req->cqe.flags = cflags; 334 } 335 336 static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx) 337 { 338 if (ctx->flags & IORING_SETUP_CQE_MIXED) 339 return IORING_CQE_F_32; 340 return 0; 341 } 342 343 static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags, 344 __u64 extra1, __u64 extra2) 345 { 346 req->cqe.res = res; 347 req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx); 348 req->big_cqe.extra1 = extra1; 349 req->big_cqe.extra2 = extra2; 350 } 351 352 static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache, 353 struct io_kiocb *req) 354 { 355 if (cache) { 356 req->async_data = io_cache_alloc(cache, GFP_KERNEL); 357 } else { 358 const struct io_issue_def *def = &io_issue_defs[req->opcode]; 359 360 WARN_ON_ONCE(!def->async_size); 361 req->async_data = kmalloc(def->async_size, GFP_KERNEL); 362 } 363 if (req->async_data) 364 req->flags |= REQ_F_ASYNC_DATA; 365 return req->async_data; 366 } 367 368 static inline bool req_has_async_data(struct io_kiocb *req) 369 { 370 return req->flags & REQ_F_ASYNC_DATA; 371 } 372 373 static inline void io_req_async_data_clear(struct io_kiocb *req, 374 io_req_flags_t extra_flags) 375 { 376 req->flags &= ~(REQ_F_ASYNC_DATA|extra_flags); 377 req->async_data = NULL; 378 } 379 380 static inline void io_req_async_data_free(struct io_kiocb *req) 381 { 382 kfree(req->async_data); 383 io_req_async_data_clear(req, 0); 384 } 385 386 static inline void io_put_file(struct io_kiocb *req) 387 { 388 if (!(req->flags & REQ_F_FIXED_FILE) && req->file) 389 fput(req->file); 390 } 391 392 static inline void io_ring_submit_unlock(struct io_ring_ctx *ctx, 393 unsigned issue_flags) 394 { 395 lockdep_assert_held(&ctx->uring_lock); 396 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) 397 mutex_unlock(&ctx->uring_lock); 398 } 399 400 static inline void io_ring_submit_lock(struct io_ring_ctx *ctx, 401 unsigned issue_flags) 402 { 403 /* 404 * "Normal" inline submissions always hold the uring_lock, since we 405 * grab it from the system call. Same is true for the SQPOLL offload. 406 * The only exception is when we've detached the request and issue it 407 * from an async worker thread, grab the lock for that case. 408 */ 409 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) 410 mutex_lock(&ctx->uring_lock); 411 lockdep_assert_held(&ctx->uring_lock); 412 } 413 414 static inline void io_commit_cqring(struct io_ring_ctx *ctx) 415 { 416 /* order cqe stores with ring update */ 417 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); 418 } 419 420 static inline void __io_wq_wake(struct wait_queue_head *wq) 421 { 422 /* 423 * 424 * Pass in EPOLLIN|EPOLL_URING_WAKE as the poll wakeup key. The latter 425 * set in the mask so that if we recurse back into our own poll 426 * waitqueue handlers, we know we have a dependency between eventfd or 427 * epoll and should terminate multishot poll at that point. 428 */ 429 if (wq_has_sleeper(wq)) 430 __wake_up(wq, TASK_NORMAL, 0, poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); 431 } 432 433 static inline void io_poll_wq_wake(struct io_ring_ctx *ctx) 434 { 435 __io_wq_wake(&ctx->poll_wq); 436 } 437 438 static inline void io_cqring_wake(struct io_ring_ctx *ctx) 439 { 440 /* 441 * Trigger waitqueue handler on all waiters on our waitqueue. This 442 * won't necessarily wake up all the tasks, io_should_wake() will make 443 * that decision. 444 */ 445 446 __io_wq_wake(&ctx->cq_wait); 447 } 448 449 static inline bool __io_sqring_full(struct io_ring_ctx *ctx) 450 { 451 struct io_rings *r = io_get_rings(ctx); 452 453 /* 454 * SQPOLL must use the actual sqring head, as using the cached_sq_head 455 * is race prone if the SQPOLL thread has grabbed entries but not yet 456 * committed them to the ring. For !SQPOLL, this doesn't matter, but 457 * since this helper is just used for SQPOLL sqring waits (or POLLOUT), 458 * just read the actual sqring head unconditionally. 459 */ 460 return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries; 461 } 462 463 static inline bool io_sqring_full(struct io_ring_ctx *ctx) 464 { 465 guard(rcu)(); 466 return __io_sqring_full(ctx); 467 } 468 469 static inline unsigned int __io_sqring_entries(struct io_ring_ctx *ctx) 470 { 471 struct io_rings *rings = io_get_rings(ctx); 472 unsigned int entries; 473 474 /* make sure SQ entry isn't read before tail */ 475 entries = smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; 476 return min(entries, ctx->sq_entries); 477 } 478 479 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) 480 { 481 guard(rcu)(); 482 return __io_sqring_entries(ctx); 483 } 484 485 /* 486 * Don't complete immediately but use deferred completion infrastructure. 487 * Protected by ->uring_lock and can only be used either with 488 * IO_URING_F_COMPLETE_DEFER or inside a tw handler holding the mutex. 489 */ 490 static inline void io_req_complete_defer(struct io_kiocb *req) 491 __must_hold(&req->ctx->uring_lock) 492 { 493 struct io_submit_state *state = &req->ctx->submit_state; 494 495 lockdep_assert_held(&req->ctx->uring_lock); 496 497 wq_list_add_tail(&req->comp_list, &state->compl_reqs); 498 } 499 500 #define SHOULD_FLUSH_MASK (IO_RING_F_OFF_TIMEOUT_USED | \ 501 IO_RING_F_HAS_EVFD | IO_RING_F_POLL_ACTIVATED) 502 503 static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx) 504 { 505 if (unlikely(data_race(ctx->int_flags) & SHOULD_FLUSH_MASK)) 506 __io_commit_cqring_flush(ctx); 507 } 508 509 static inline void io_get_task_refs(int nr) 510 { 511 struct io_uring_task *tctx = current->io_uring; 512 513 tctx->cached_refs -= nr; 514 if (unlikely(tctx->cached_refs < 0)) 515 io_task_refs_refill(tctx); 516 } 517 518 static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) 519 { 520 return !ctx->submit_state.free_list.next; 521 } 522 523 extern struct kmem_cache *req_cachep; 524 525 static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx) 526 { 527 struct io_kiocb *req; 528 529 req = container_of(ctx->submit_state.free_list.next, struct io_kiocb, comp_list); 530 wq_stack_extract(&ctx->submit_state.free_list); 531 return req; 532 } 533 534 static inline bool io_alloc_req(struct io_ring_ctx *ctx, struct io_kiocb **req) 535 { 536 if (unlikely(io_req_cache_empty(ctx))) { 537 if (!__io_alloc_req_refill(ctx)) 538 return false; 539 } 540 *req = io_extract_req(ctx); 541 return true; 542 } 543 544 static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) 545 { 546 io_req_set_res(req, res, 0); 547 req->io_task_work.func = io_req_task_complete; 548 io_req_task_work_add(req); 549 } 550 551 static inline bool io_file_can_poll(struct io_kiocb *req) 552 { 553 if (req->flags & REQ_F_CAN_POLL) 554 return true; 555 if (req->file && file_can_poll(req->file)) { 556 req->flags |= REQ_F_CAN_POLL; 557 return true; 558 } 559 return false; 560 } 561 562 static inline bool io_is_uring_cmd(const struct io_kiocb *req) 563 { 564 return req->opcode == IORING_OP_URING_CMD || 565 req->opcode == IORING_OP_URING_CMD128; 566 } 567 568 static inline ktime_t io_get_time(struct io_ring_ctx *ctx) 569 { 570 if (ctx->clockid == CLOCK_MONOTONIC) 571 return ktime_get(); 572 573 return ktime_get_with_offset(ctx->clock_offset); 574 } 575 576 enum { 577 IO_CHECK_CQ_OVERFLOW_BIT, 578 IO_CHECK_CQ_DROPPED_BIT, 579 }; 580 581 static inline bool io_has_work(struct io_ring_ctx *ctx) 582 { 583 return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) || 584 io_local_work_pending(ctx); 585 } 586 #endif 587