1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef IOU_CORE_H 3 #define IOU_CORE_H 4 5 #include <linux/errno.h> 6 #include <linux/lockdep.h> 7 #include <linux/resume_user_mode.h> 8 #include <linux/poll.h> 9 #include <linux/io_uring_types.h> 10 #include <uapi/linux/eventpoll.h> 11 #include "alloc_cache.h" 12 #include "io-wq.h" 13 #include "slist.h" 14 #include "tw.h" 15 #include "opdef.h" 16 17 #ifndef CREATE_TRACE_POINTS 18 #include <trace/events/io_uring.h> 19 #endif 20 21 struct io_rings_layout { 22 /* size of CQ + headers + SQ offset array */ 23 size_t rings_size; 24 size_t sq_size; 25 26 size_t sq_array_offset; 27 }; 28 29 struct io_ctx_config { 30 struct io_uring_params p; 31 struct io_rings_layout layout; 32 struct io_uring_params __user *uptr; 33 }; 34 35 #define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\ 36 IORING_FEAT_NODROP |\ 37 IORING_FEAT_SUBMIT_STABLE |\ 38 IORING_FEAT_RW_CUR_POS |\ 39 IORING_FEAT_CUR_PERSONALITY |\ 40 IORING_FEAT_FAST_POLL |\ 41 IORING_FEAT_POLL_32BITS |\ 42 IORING_FEAT_SQPOLL_NONFIXED |\ 43 IORING_FEAT_EXT_ARG |\ 44 IORING_FEAT_NATIVE_WORKERS |\ 45 IORING_FEAT_RSRC_TAGS |\ 46 IORING_FEAT_CQE_SKIP |\ 47 IORING_FEAT_LINKED_FILE |\ 48 IORING_FEAT_REG_REG_RING |\ 49 IORING_FEAT_RECVSEND_BUNDLE |\ 50 IORING_FEAT_MIN_TIMEOUT |\ 51 IORING_FEAT_RW_ATTR |\ 52 IORING_FEAT_NO_IOWAIT) 53 54 #define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\ 55 IORING_SETUP_SQPOLL |\ 56 IORING_SETUP_SQ_AFF |\ 57 IORING_SETUP_CQSIZE |\ 58 IORING_SETUP_CLAMP |\ 59 IORING_SETUP_ATTACH_WQ |\ 60 IORING_SETUP_R_DISABLED |\ 61 IORING_SETUP_SUBMIT_ALL |\ 62 IORING_SETUP_COOP_TASKRUN |\ 63 IORING_SETUP_TASKRUN_FLAG |\ 64 IORING_SETUP_SQE128 |\ 65 IORING_SETUP_CQE32 |\ 66 IORING_SETUP_SINGLE_ISSUER |\ 67 IORING_SETUP_DEFER_TASKRUN |\ 68 IORING_SETUP_NO_MMAP |\ 69 IORING_SETUP_REGISTERED_FD_ONLY |\ 70 IORING_SETUP_NO_SQARRAY |\ 71 IORING_SETUP_HYBRID_IOPOLL |\ 72 IORING_SETUP_CQE_MIXED |\ 73 IORING_SETUP_SQE_MIXED |\ 74 IORING_SETUP_SQ_REWIND) 75 76 #define IORING_ENTER_FLAGS (IORING_ENTER_GETEVENTS |\ 77 IORING_ENTER_SQ_WAKEUP |\ 78 IORING_ENTER_SQ_WAIT |\ 79 IORING_ENTER_EXT_ARG |\ 80 IORING_ENTER_REGISTERED_RING |\ 81 IORING_ENTER_ABS_TIMER |\ 82 IORING_ENTER_EXT_ARG_REG |\ 83 IORING_ENTER_NO_IOWAIT) 84 85 86 #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE |\ 87 IOSQE_IO_DRAIN |\ 88 IOSQE_IO_LINK |\ 89 IOSQE_IO_HARDLINK |\ 90 IOSQE_ASYNC |\ 91 IOSQE_BUFFER_SELECT |\ 92 IOSQE_CQE_SKIP_SUCCESS) 93 94 #define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK) 95 96 /* 97 * Complaint timeout for io_uring cancelation exits, and for io-wq exit 98 * worker waiting. 99 */ 100 #define IO_URING_EXIT_WAIT_MAX (HZ * 60 * 5) 101 102 enum { 103 IOU_COMPLETE = 0, 104 105 IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED, 106 107 /* 108 * The request has more work to do and should be retried. io_uring will 109 * attempt to wait on the file for eligible opcodes, but otherwise 110 * it'll be handed to iowq for blocking execution. It works for normal 111 * requests as well as for the multi shot mode. 112 */ 113 IOU_RETRY = -EAGAIN, 114 115 /* 116 * Requeue the task_work to restart operations on this request. The 117 * actual value isn't important, should just be not an otherwise 118 * valid error code, yet less than -MAX_ERRNO and valid internally. 119 */ 120 IOU_REQUEUE = -3072, 121 }; 122 123 struct io_defer_entry { 124 struct list_head list; 125 struct io_kiocb *req; 126 }; 127 128 struct io_wait_queue { 129 struct wait_queue_entry wq; 130 struct io_ring_ctx *ctx; 131 unsigned cq_tail; 132 unsigned cq_min_tail; 133 unsigned nr_timeouts; 134 int hit_timeout; 135 ktime_t min_timeout; 136 ktime_t timeout; 137 struct hrtimer t; 138 139 #ifdef CONFIG_NET_RX_BUSY_POLL 140 ktime_t napi_busy_poll_dt; 141 bool napi_prefer_busy_poll; 142 #endif 143 }; 144 145 static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx) 146 { 147 return rcu_dereference_check(ctx->rings_rcu, 148 lockdep_is_held(&ctx->uring_lock) || 149 lockdep_is_held(&ctx->completion_lock)); 150 } 151 152 static inline bool io_should_wake(struct io_wait_queue *iowq) 153 { 154 struct io_ring_ctx *ctx = iowq->ctx; 155 struct io_rings *rings; 156 int dist; 157 158 guard(rcu)(); 159 rings = io_get_rings(ctx); 160 161 /* 162 * Wake up if we have enough events, or if a timeout occurred since we 163 * started waiting. For timeouts, we always want to return to userspace, 164 * regardless of event count. 165 */ 166 dist = READ_ONCE(rings->cq.tail) - (int) iowq->cq_tail; 167 return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; 168 } 169 170 #define IORING_MAX_ENTRIES 32768 171 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) 172 173 int io_prepare_config(struct io_ctx_config *config); 174 175 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32); 176 void io_req_defer_failed(struct io_kiocb *req, s32 res); 177 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); 178 void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); 179 bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags); 180 bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe src_cqe[2]); 181 void __io_commit_cqring_flush(struct io_ring_ctx *ctx); 182 183 unsigned io_linked_nr(struct io_kiocb *req); 184 void io_req_track_inflight(struct io_kiocb *req); 185 struct file *io_file_get_normal(struct io_kiocb *req, int fd); 186 struct file *io_file_get_fixed(struct io_kiocb *req, int fd, 187 unsigned issue_flags); 188 189 void io_req_task_queue(struct io_kiocb *req); 190 void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw); 191 void io_req_task_queue_fail(struct io_kiocb *req, int ret); 192 void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw); 193 __cold void io_uring_drop_tctx_refs(struct task_struct *task); 194 195 int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file, 196 int start, int end); 197 void io_req_queue_iowq(struct io_kiocb *req); 198 199 int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw); 200 int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr); 201 int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin); 202 __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx); 203 void __io_submit_flush_completions(struct io_ring_ctx *ctx); 204 205 struct io_wq_work *io_wq_free_work(struct io_wq_work *work); 206 void io_wq_submit_work(struct io_wq_work *work); 207 208 void io_free_req(struct io_kiocb *req); 209 void io_queue_next(struct io_kiocb *req); 210 void io_task_refs_refill(struct io_uring_task *tctx); 211 bool __io_alloc_req_refill(struct io_ring_ctx *ctx); 212 213 void io_activate_pollwq(struct io_ring_ctx *ctx); 214 void io_restriction_clone(struct io_restriction *dst, struct io_restriction *src); 215 216 static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx) 217 { 218 #if defined(CONFIG_PROVE_LOCKING) 219 lockdep_assert(in_task()); 220 221 if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) 222 lockdep_assert_held(&ctx->uring_lock); 223 224 if (ctx->flags & IORING_SETUP_IOPOLL) { 225 lockdep_assert_held(&ctx->uring_lock); 226 } else if (!ctx->task_complete) { 227 lockdep_assert_held(&ctx->completion_lock); 228 } else if (ctx->submitter_task) { 229 /* 230 * ->submitter_task may be NULL and we can still post a CQE, 231 * if the ring has been setup with IORING_SETUP_R_DISABLED. 232 * Not from an SQE, as those cannot be submitted, but via 233 * updating tagged resources. 234 */ 235 if (!percpu_ref_is_dying(&ctx->refs)) 236 lockdep_assert(current == ctx->submitter_task); 237 } 238 #endif 239 } 240 241 static inline bool io_is_compat(struct io_ring_ctx *ctx) 242 { 243 return IS_ENABLED(CONFIG_COMPAT) && unlikely(ctx->compat); 244 } 245 246 static inline void io_submit_flush_completions(struct io_ring_ctx *ctx) 247 { 248 if (!wq_list_empty(&ctx->submit_state.compl_reqs) || 249 ctx->submit_state.cq_flush) 250 __io_submit_flush_completions(ctx); 251 } 252 253 #define io_for_each_link(pos, head) \ 254 for (pos = (head); pos; pos = pos->link) 255 256 static inline bool io_get_cqe_overflow(struct io_ring_ctx *ctx, 257 struct io_uring_cqe **ret, 258 bool overflow, bool cqe32) 259 { 260 io_lockdep_assert_cq_locked(ctx); 261 262 if (unlikely(ctx->cqe_sentinel - ctx->cqe_cached < (cqe32 + 1))) { 263 if (unlikely(!io_cqe_cache_refill(ctx, overflow, cqe32))) 264 return false; 265 } 266 *ret = ctx->cqe_cached; 267 ctx->cached_cq_tail++; 268 ctx->cqe_cached++; 269 if (ctx->flags & IORING_SETUP_CQE32) { 270 ctx->cqe_cached++; 271 } else if (cqe32 && ctx->flags & IORING_SETUP_CQE_MIXED) { 272 ctx->cqe_cached++; 273 ctx->cached_cq_tail++; 274 } 275 WARN_ON_ONCE(ctx->cqe_cached > ctx->cqe_sentinel); 276 return true; 277 } 278 279 static inline bool io_get_cqe(struct io_ring_ctx *ctx, struct io_uring_cqe **ret, 280 bool cqe32) 281 { 282 return io_get_cqe_overflow(ctx, ret, false, cqe32); 283 } 284 285 static inline bool io_defer_get_uncommited_cqe(struct io_ring_ctx *ctx, 286 struct io_uring_cqe **cqe_ret) 287 { 288 io_lockdep_assert_cq_locked(ctx); 289 290 ctx->submit_state.cq_flush = true; 291 return io_get_cqe(ctx, cqe_ret, ctx->flags & IORING_SETUP_CQE_MIXED); 292 } 293 294 static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx, 295 struct io_kiocb *req) 296 { 297 bool is_cqe32 = req->cqe.flags & IORING_CQE_F_32; 298 struct io_uring_cqe *cqe; 299 300 /* 301 * If we can't get a cq entry, userspace overflowed the submission 302 * (by quite a lot). 303 */ 304 if (unlikely(!io_get_cqe(ctx, &cqe, is_cqe32))) 305 return false; 306 307 memcpy(cqe, &req->cqe, sizeof(*cqe)); 308 if (ctx->flags & IORING_SETUP_CQE32 || is_cqe32) { 309 memcpy(cqe->big_cqe, &req->big_cqe, sizeof(*cqe)); 310 memset(&req->big_cqe, 0, sizeof(req->big_cqe)); 311 } 312 313 if (trace_io_uring_complete_enabled()) 314 trace_io_uring_complete(req->ctx, req, cqe); 315 return true; 316 } 317 318 static inline void req_set_fail(struct io_kiocb *req) 319 { 320 req->flags |= REQ_F_FAIL; 321 if (req->flags & REQ_F_CQE_SKIP) { 322 req->flags &= ~REQ_F_CQE_SKIP; 323 req->flags |= REQ_F_SKIP_LINK_CQES; 324 } 325 } 326 327 static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags) 328 { 329 req->cqe.res = res; 330 req->cqe.flags = cflags; 331 } 332 333 static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx) 334 { 335 if (ctx->flags & IORING_SETUP_CQE_MIXED) 336 return IORING_CQE_F_32; 337 return 0; 338 } 339 340 static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags, 341 __u64 extra1, __u64 extra2) 342 { 343 req->cqe.res = res; 344 req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx); 345 req->big_cqe.extra1 = extra1; 346 req->big_cqe.extra2 = extra2; 347 } 348 349 static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache, 350 struct io_kiocb *req) 351 { 352 if (cache) { 353 req->async_data = io_cache_alloc(cache, GFP_KERNEL); 354 } else { 355 const struct io_issue_def *def = &io_issue_defs[req->opcode]; 356 357 WARN_ON_ONCE(!def->async_size); 358 req->async_data = kmalloc(def->async_size, GFP_KERNEL); 359 } 360 if (req->async_data) 361 req->flags |= REQ_F_ASYNC_DATA; 362 return req->async_data; 363 } 364 365 static inline bool req_has_async_data(struct io_kiocb *req) 366 { 367 return req->flags & REQ_F_ASYNC_DATA; 368 } 369 370 static inline void io_req_async_data_clear(struct io_kiocb *req, 371 io_req_flags_t extra_flags) 372 { 373 req->flags &= ~(REQ_F_ASYNC_DATA|extra_flags); 374 req->async_data = NULL; 375 } 376 377 static inline void io_req_async_data_free(struct io_kiocb *req) 378 { 379 kfree(req->async_data); 380 io_req_async_data_clear(req, 0); 381 } 382 383 static inline void io_put_file(struct io_kiocb *req) 384 { 385 if (!(req->flags & REQ_F_FIXED_FILE) && req->file) 386 fput(req->file); 387 } 388 389 static inline void io_ring_submit_unlock(struct io_ring_ctx *ctx, 390 unsigned issue_flags) 391 { 392 lockdep_assert_held(&ctx->uring_lock); 393 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) 394 mutex_unlock(&ctx->uring_lock); 395 } 396 397 static inline void io_ring_submit_lock(struct io_ring_ctx *ctx, 398 unsigned issue_flags) 399 { 400 /* 401 * "Normal" inline submissions always hold the uring_lock, since we 402 * grab it from the system call. Same is true for the SQPOLL offload. 403 * The only exception is when we've detached the request and issue it 404 * from an async worker thread, grab the lock for that case. 405 */ 406 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) 407 mutex_lock(&ctx->uring_lock); 408 lockdep_assert_held(&ctx->uring_lock); 409 } 410 411 static inline void io_commit_cqring(struct io_ring_ctx *ctx) 412 { 413 /* order cqe stores with ring update */ 414 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); 415 } 416 417 static inline void __io_wq_wake(struct wait_queue_head *wq) 418 { 419 /* 420 * 421 * Pass in EPOLLIN|EPOLL_URING_WAKE as the poll wakeup key. The latter 422 * set in the mask so that if we recurse back into our own poll 423 * waitqueue handlers, we know we have a dependency between eventfd or 424 * epoll and should terminate multishot poll at that point. 425 */ 426 if (wq_has_sleeper(wq)) 427 __wake_up(wq, TASK_NORMAL, 0, poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); 428 } 429 430 static inline void io_poll_wq_wake(struct io_ring_ctx *ctx) 431 { 432 __io_wq_wake(&ctx->poll_wq); 433 } 434 435 static inline void io_cqring_wake(struct io_ring_ctx *ctx) 436 { 437 /* 438 * Trigger waitqueue handler on all waiters on our waitqueue. This 439 * won't necessarily wake up all the tasks, io_should_wake() will make 440 * that decision. 441 */ 442 443 __io_wq_wake(&ctx->cq_wait); 444 } 445 446 static inline bool __io_sqring_full(struct io_ring_ctx *ctx) 447 { 448 struct io_rings *r = io_get_rings(ctx); 449 450 /* 451 * SQPOLL must use the actual sqring head, as using the cached_sq_head 452 * is race prone if the SQPOLL thread has grabbed entries but not yet 453 * committed them to the ring. For !SQPOLL, this doesn't matter, but 454 * since this helper is just used for SQPOLL sqring waits (or POLLOUT), 455 * just read the actual sqring head unconditionally. 456 */ 457 return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries; 458 } 459 460 static inline bool io_sqring_full(struct io_ring_ctx *ctx) 461 { 462 guard(rcu)(); 463 return __io_sqring_full(ctx); 464 } 465 466 static inline unsigned int __io_sqring_entries(struct io_ring_ctx *ctx) 467 { 468 struct io_rings *rings = io_get_rings(ctx); 469 unsigned int entries; 470 471 /* make sure SQ entry isn't read before tail */ 472 entries = smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; 473 return min(entries, ctx->sq_entries); 474 } 475 476 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) 477 { 478 guard(rcu)(); 479 return __io_sqring_entries(ctx); 480 } 481 482 /* 483 * Don't complete immediately but use deferred completion infrastructure. 484 * Protected by ->uring_lock and can only be used either with 485 * IO_URING_F_COMPLETE_DEFER or inside a tw handler holding the mutex. 486 */ 487 static inline void io_req_complete_defer(struct io_kiocb *req) 488 __must_hold(&req->ctx->uring_lock) 489 { 490 struct io_submit_state *state = &req->ctx->submit_state; 491 492 lockdep_assert_held(&req->ctx->uring_lock); 493 494 wq_list_add_tail(&req->comp_list, &state->compl_reqs); 495 } 496 497 static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx) 498 { 499 if (unlikely(ctx->off_timeout_used || 500 ctx->has_evfd || ctx->poll_activated)) 501 __io_commit_cqring_flush(ctx); 502 } 503 504 static inline void io_get_task_refs(int nr) 505 { 506 struct io_uring_task *tctx = current->io_uring; 507 508 tctx->cached_refs -= nr; 509 if (unlikely(tctx->cached_refs < 0)) 510 io_task_refs_refill(tctx); 511 } 512 513 static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) 514 { 515 return !ctx->submit_state.free_list.next; 516 } 517 518 extern struct kmem_cache *req_cachep; 519 520 static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx) 521 { 522 struct io_kiocb *req; 523 524 req = container_of(ctx->submit_state.free_list.next, struct io_kiocb, comp_list); 525 wq_stack_extract(&ctx->submit_state.free_list); 526 return req; 527 } 528 529 static inline bool io_alloc_req(struct io_ring_ctx *ctx, struct io_kiocb **req) 530 { 531 if (unlikely(io_req_cache_empty(ctx))) { 532 if (!__io_alloc_req_refill(ctx)) 533 return false; 534 } 535 *req = io_extract_req(ctx); 536 return true; 537 } 538 539 static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) 540 { 541 io_req_set_res(req, res, 0); 542 req->io_task_work.func = io_req_task_complete; 543 io_req_task_work_add(req); 544 } 545 546 static inline bool io_file_can_poll(struct io_kiocb *req) 547 { 548 if (req->flags & REQ_F_CAN_POLL) 549 return true; 550 if (req->file && file_can_poll(req->file)) { 551 req->flags |= REQ_F_CAN_POLL; 552 return true; 553 } 554 return false; 555 } 556 557 static inline bool io_is_uring_cmd(const struct io_kiocb *req) 558 { 559 return req->opcode == IORING_OP_URING_CMD || 560 req->opcode == IORING_OP_URING_CMD128; 561 } 562 563 static inline ktime_t io_get_time(struct io_ring_ctx *ctx) 564 { 565 if (ctx->clockid == CLOCK_MONOTONIC) 566 return ktime_get(); 567 568 return ktime_get_with_offset(ctx->clock_offset); 569 } 570 571 enum { 572 IO_CHECK_CQ_OVERFLOW_BIT, 573 IO_CHECK_CQ_DROPPED_BIT, 574 }; 575 576 static inline bool io_has_work(struct io_ring_ctx *ctx) 577 { 578 return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) || 579 io_local_work_pending(ctx); 580 } 581 #endif 582