1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code related to the io_uring_register() syscall 4 * 5 * Copyright (C) 2023 Jens Axboe 6 */ 7 #include <linux/kernel.h> 8 #include <linux/errno.h> 9 #include <linux/syscalls.h> 10 #include <linux/refcount.h> 11 #include <linux/bits.h> 12 #include <linux/fs.h> 13 #include <linux/file.h> 14 #include <linux/slab.h> 15 #include <linux/uaccess.h> 16 #include <linux/nospec.h> 17 #include <linux/compat.h> 18 #include <linux/io_uring.h> 19 #include <linux/io_uring_types.h> 20 21 #include "filetable.h" 22 #include "io_uring.h" 23 #include "opdef.h" 24 #include "tctx.h" 25 #include "rsrc.h" 26 #include "sqpoll.h" 27 #include "register.h" 28 #include "cancel.h" 29 #include "kbuf.h" 30 #include "napi.h" 31 #include "eventfd.h" 32 #include "msg_ring.h" 33 #include "memmap.h" 34 #include "zcrx.h" 35 #include "query.h" 36 #include "bpf_filter.h" 37 38 #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ 39 IORING_REGISTER_LAST + IORING_OP_LAST) 40 41 static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, 42 unsigned nr_args) 43 { 44 struct io_uring_probe *p; 45 size_t size; 46 int i, ret; 47 48 if (nr_args > IORING_OP_LAST) 49 nr_args = IORING_OP_LAST; 50 51 size = struct_size(p, ops, nr_args); 52 p = memdup_user(arg, size); 53 if (IS_ERR(p)) 54 return PTR_ERR(p); 55 ret = -EINVAL; 56 if (memchr_inv(p, 0, size)) 57 goto out; 58 59 p->last_op = IORING_OP_LAST - 1; 60 61 for (i = 0; i < nr_args; i++) { 62 p->ops[i].op = i; 63 if (io_uring_op_supported(i)) 64 p->ops[i].flags = IO_URING_OP_SUPPORTED; 65 } 66 p->ops_len = i; 67 68 ret = 0; 69 if (copy_to_user(arg, p, size)) 70 ret = -EFAULT; 71 out: 72 kfree(p); 73 return ret; 74 } 75 76 int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) 77 { 78 const struct cred *creds; 79 80 creds = xa_erase(&ctx->personalities, id); 81 if (creds) { 82 put_cred(creds); 83 return 0; 84 } 85 86 return -EINVAL; 87 } 88 89 90 static int io_register_personality(struct io_ring_ctx *ctx) 91 { 92 const struct cred *creds; 93 u32 id; 94 int ret; 95 96 creds = get_current_cred(); 97 98 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds, 99 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); 100 if (ret < 0) { 101 put_cred(creds); 102 return ret; 103 } 104 return id; 105 } 106 107 /* 108 * Returns number of restrictions parsed and added on success, or < 0 for 109 * an error. 110 */ 111 static __cold int io_parse_restrictions(void __user *arg, unsigned int nr_args, 112 struct io_restriction *restrictions) 113 { 114 struct io_uring_restriction *res; 115 size_t size; 116 int i, ret; 117 118 if (!arg || nr_args > IORING_MAX_RESTRICTIONS) 119 return -EINVAL; 120 121 size = array_size(nr_args, sizeof(*res)); 122 if (size == SIZE_MAX) 123 return -EOVERFLOW; 124 125 res = memdup_user(arg, size); 126 if (IS_ERR(res)) 127 return PTR_ERR(res); 128 129 ret = -EINVAL; 130 131 for (i = 0; i < nr_args; i++) { 132 switch (res[i].opcode) { 133 case IORING_RESTRICTION_REGISTER_OP: 134 if (res[i].register_op >= IORING_REGISTER_LAST) 135 goto err; 136 __set_bit(res[i].register_op, restrictions->register_op); 137 restrictions->reg_registered = true; 138 break; 139 case IORING_RESTRICTION_SQE_OP: 140 if (res[i].sqe_op >= IORING_OP_LAST) 141 goto err; 142 __set_bit(res[i].sqe_op, restrictions->sqe_op); 143 restrictions->op_registered = true; 144 break; 145 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: 146 restrictions->sqe_flags_allowed = res[i].sqe_flags; 147 restrictions->op_registered = true; 148 break; 149 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: 150 restrictions->sqe_flags_required = res[i].sqe_flags; 151 restrictions->op_registered = true; 152 break; 153 default: 154 goto err; 155 } 156 } 157 ret = nr_args; 158 if (!nr_args) { 159 restrictions->op_registered = true; 160 restrictions->reg_registered = true; 161 } 162 err: 163 kfree(res); 164 return ret; 165 } 166 167 static __cold int io_register_restrictions(struct io_ring_ctx *ctx, 168 void __user *arg, unsigned int nr_args) 169 { 170 int ret; 171 172 /* Restrictions allowed only if rings started disabled */ 173 if (!(ctx->flags & IORING_SETUP_R_DISABLED)) 174 return -EBADFD; 175 176 /* We allow only a single restrictions registration */ 177 if (ctx->restrictions.op_registered || ctx->restrictions.reg_registered) 178 return -EBUSY; 179 180 ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions); 181 /* 182 * Reset all restrictions if an error happened, but retain any COW'ed 183 * settings. 184 */ 185 if (ret < 0) { 186 struct io_bpf_filters *bpf = ctx->restrictions.bpf_filters; 187 bool cowed = ctx->restrictions.bpf_filters_cow; 188 189 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); 190 ctx->restrictions.bpf_filters = bpf; 191 ctx->restrictions.bpf_filters_cow = cowed; 192 return ret; 193 } 194 if (ctx->restrictions.op_registered) 195 ctx->int_flags |= IO_RING_F_OP_RESTRICTED; 196 if (ctx->restrictions.reg_registered) 197 ctx->int_flags |= IO_RING_F_REG_RESTRICTED; 198 return 0; 199 } 200 201 static int io_register_restrictions_task(void __user *arg, unsigned int nr_args) 202 { 203 struct io_uring_task_restriction __user *ures = arg; 204 struct io_uring_task_restriction tres; 205 struct io_restriction *res; 206 int ret; 207 208 /* Disallow if task already has registered restrictions */ 209 if (current->io_uring_restrict) 210 return -EPERM; 211 /* 212 * Similar to seccomp, disallow setting a filter if task_no_new_privs 213 * is false and we're not CAP_SYS_ADMIN. 214 */ 215 if (!task_no_new_privs(current) && 216 !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN)) 217 return -EACCES; 218 if (nr_args != 1) 219 return -EINVAL; 220 221 if (copy_from_user(&tres, arg, sizeof(tres))) 222 return -EFAULT; 223 224 if (tres.flags) 225 return -EINVAL; 226 if (!mem_is_zero(tres.resv, sizeof(tres.resv))) 227 return -EINVAL; 228 229 res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT); 230 if (!res) 231 return -ENOMEM; 232 233 ret = io_parse_restrictions(ures->restrictions, tres.nr_res, res); 234 if (ret < 0) { 235 kfree(res); 236 return ret; 237 } 238 current->io_uring_restrict = res; 239 return 0; 240 } 241 242 static int io_register_bpf_filter_task(void __user *arg, unsigned int nr_args) 243 { 244 struct io_restriction *res; 245 int ret; 246 247 /* 248 * Similar to seccomp, disallow setting a filter if task_no_new_privs 249 * is false and we're not CAP_SYS_ADMIN. 250 */ 251 if (!task_no_new_privs(current) && 252 !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN)) 253 return -EACCES; 254 255 if (nr_args != 1) 256 return -EINVAL; 257 258 /* If no task restrictions exist, setup a new set */ 259 res = current->io_uring_restrict; 260 if (!res) { 261 res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT); 262 if (!res) 263 return -ENOMEM; 264 } 265 266 ret = io_register_bpf_filter(res, arg); 267 if (ret) { 268 if (res != current->io_uring_restrict) 269 kfree(res); 270 return ret; 271 } 272 if (!current->io_uring_restrict) 273 current->io_uring_restrict = res; 274 return 0; 275 } 276 277 static int io_register_enable_rings(struct io_ring_ctx *ctx) 278 { 279 if (!(ctx->flags & IORING_SETUP_R_DISABLED)) 280 return -EBADFD; 281 282 if (ctx->flags & IORING_SETUP_SINGLE_ISSUER) { 283 ctx->submitter_task = get_task_struct(current); 284 /* 285 * Lazy activation attempts would fail if it was polled before 286 * submitter_task is set. 287 */ 288 if (wq_has_sleeper(&ctx->poll_wq)) 289 io_activate_pollwq(ctx); 290 } 291 292 /* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */ 293 smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED); 294 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) 295 wake_up(&ctx->sq_data->wait); 296 return 0; 297 } 298 299 static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, 300 cpumask_var_t new_mask) 301 { 302 int ret; 303 304 if (!(ctx->flags & IORING_SETUP_SQPOLL)) { 305 ret = io_wq_cpu_affinity(current->io_uring, new_mask); 306 } else { 307 mutex_unlock(&ctx->uring_lock); 308 ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask); 309 mutex_lock(&ctx->uring_lock); 310 } 311 312 return ret; 313 } 314 315 static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, 316 void __user *arg, unsigned len) 317 { 318 cpumask_var_t new_mask; 319 int ret; 320 321 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 322 return -ENOMEM; 323 324 cpumask_clear(new_mask); 325 if (len > cpumask_size()) 326 len = cpumask_size(); 327 328 #ifdef CONFIG_COMPAT 329 if (in_compat_syscall()) 330 ret = compat_get_bitmap(cpumask_bits(new_mask), 331 (const compat_ulong_t __user *)arg, 332 len * 8 /* CHAR_BIT */); 333 else 334 #endif 335 ret = copy_from_user(new_mask, arg, len); 336 337 if (ret) { 338 free_cpumask_var(new_mask); 339 return -EFAULT; 340 } 341 342 ret = __io_register_iowq_aff(ctx, new_mask); 343 free_cpumask_var(new_mask); 344 return ret; 345 } 346 347 static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx) 348 { 349 return __io_register_iowq_aff(ctx, NULL); 350 } 351 352 static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, 353 void __user *arg) 354 __must_hold(&ctx->uring_lock) 355 { 356 struct io_tctx_node *node; 357 struct io_uring_task *tctx = NULL; 358 struct io_sq_data *sqd = NULL; 359 __u32 new_count[2]; 360 int i, ret; 361 362 if (copy_from_user(new_count, arg, sizeof(new_count))) 363 return -EFAULT; 364 for (i = 0; i < ARRAY_SIZE(new_count); i++) 365 if (new_count[i] > INT_MAX) 366 return -EINVAL; 367 368 if (ctx->flags & IORING_SETUP_SQPOLL) { 369 sqd = ctx->sq_data; 370 if (sqd) { 371 struct task_struct *tsk; 372 373 /* 374 * Observe the correct sqd->lock -> ctx->uring_lock 375 * ordering. Fine to drop uring_lock here, we hold 376 * a ref to the ctx. 377 */ 378 refcount_inc(&sqd->refs); 379 mutex_unlock(&ctx->uring_lock); 380 mutex_lock(&sqd->lock); 381 mutex_lock(&ctx->uring_lock); 382 tsk = sqpoll_task_locked(sqd); 383 if (tsk) 384 tctx = tsk->io_uring; 385 } 386 } else { 387 tctx = current->io_uring; 388 } 389 390 BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits)); 391 392 for (i = 0; i < ARRAY_SIZE(new_count); i++) 393 if (new_count[i]) 394 ctx->iowq_limits[i] = new_count[i]; 395 ctx->int_flags |= IO_RING_F_IOWQ_LIMITS_SET; 396 397 if (tctx && tctx->io_wq) { 398 ret = io_wq_max_workers(tctx->io_wq, new_count); 399 if (ret) 400 goto err; 401 } else { 402 memset(new_count, 0, sizeof(new_count)); 403 } 404 405 if (sqd) { 406 mutex_unlock(&ctx->uring_lock); 407 mutex_unlock(&sqd->lock); 408 io_put_sq_data(sqd); 409 mutex_lock(&ctx->uring_lock); 410 } 411 412 if (copy_to_user(arg, new_count, sizeof(new_count))) 413 return -EFAULT; 414 415 /* that's it for SQPOLL, only the SQPOLL task creates requests */ 416 if (sqd) 417 return 0; 418 419 /* now propagate the restriction to all registered users */ 420 mutex_lock(&ctx->tctx_lock); 421 list_for_each_entry(node, &ctx->tctx_list, ctx_node) { 422 tctx = node->task->io_uring; 423 if (WARN_ON_ONCE(!tctx->io_wq)) 424 continue; 425 426 for (i = 0; i < ARRAY_SIZE(new_count); i++) 427 new_count[i] = ctx->iowq_limits[i]; 428 /* ignore errors, it always returns zero anyway */ 429 (void)io_wq_max_workers(tctx->io_wq, new_count); 430 } 431 mutex_unlock(&ctx->tctx_lock); 432 return 0; 433 err: 434 if (sqd) { 435 mutex_unlock(&ctx->uring_lock); 436 mutex_unlock(&sqd->lock); 437 io_put_sq_data(sqd); 438 mutex_lock(&ctx->uring_lock); 439 } 440 return ret; 441 } 442 443 static int io_register_clock(struct io_ring_ctx *ctx, 444 struct io_uring_clock_register __user *arg) 445 { 446 struct io_uring_clock_register reg; 447 448 if (copy_from_user(®, arg, sizeof(reg))) 449 return -EFAULT; 450 if (memchr_inv(®.__resv, 0, sizeof(reg.__resv))) 451 return -EINVAL; 452 453 switch (reg.clockid) { 454 case CLOCK_MONOTONIC: 455 ctx->clock_offset = 0; 456 break; 457 case CLOCK_BOOTTIME: 458 ctx->clock_offset = TK_OFFS_BOOT; 459 break; 460 default: 461 return -EINVAL; 462 } 463 464 ctx->clockid = reg.clockid; 465 return 0; 466 } 467 468 /* 469 * State to maintain until we can swap. Both new and old state, used for 470 * either mapping or freeing. 471 */ 472 struct io_ring_ctx_rings { 473 struct io_rings *rings; 474 struct io_uring_sqe *sq_sqes; 475 476 struct io_mapped_region sq_region; 477 struct io_mapped_region ring_region; 478 }; 479 480 static void io_register_free_rings(struct io_ring_ctx *ctx, 481 struct io_ring_ctx_rings *r) 482 { 483 /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ 484 io_free_region(NULL, &r->sq_region); 485 io_free_region(NULL, &r->ring_region); 486 } 487 488 #define swap_old(ctx, o, n, field) \ 489 do { \ 490 (o).field = (ctx)->field; \ 491 (ctx)->field = (n).field; \ 492 } while (0) 493 494 #define RESIZE_FLAGS (IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP) 495 #define COPY_FLAGS (IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \ 496 IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \ 497 IORING_SETUP_CQE_MIXED | IORING_SETUP_SQE_MIXED) 498 499 static bool io_sqe_is_128(struct io_ring_ctx *ctx, 500 const struct io_uring_sqe *sqe) 501 { 502 u8 opcode = READ_ONCE(sqe->opcode); 503 504 if (!(ctx->flags & IORING_SETUP_SQE_MIXED) || opcode >= IORING_OP_LAST) 505 return false; 506 opcode = array_index_nospec(opcode, IORING_OP_LAST); 507 return io_issue_defs[opcode].is_128; 508 } 509 510 static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) 511 { 512 struct io_ctx_config config; 513 struct io_uring_region_desc rd; 514 struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL; 515 unsigned i, tail, old_head; 516 struct io_uring_params *p = &config.p; 517 struct io_rings_layout *rl = &config.layout; 518 u32 *o_sq_array, *n_sq_array = NULL; 519 bool cqe32_tail = false; 520 unsigned int sqe128_src = 0; 521 int ret; 522 523 memset(&config, 0, sizeof(config)); 524 525 /* limited to DEFER_TASKRUN for now */ 526 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 527 return -EINVAL; 528 if (copy_from_user(p, arg, sizeof(*p))) 529 return -EFAULT; 530 if (p->flags & ~RESIZE_FLAGS) 531 return -EINVAL; 532 533 /* properties that are always inherited */ 534 p->flags |= (ctx->flags & COPY_FLAGS); 535 536 ret = io_prepare_config(&config); 537 if (unlikely(ret)) 538 return ret; 539 540 memset(&rd, 0, sizeof(rd)); 541 rd.size = PAGE_ALIGN(rl->rings_size); 542 if (p->flags & IORING_SETUP_NO_MMAP) { 543 rd.user_addr = p->cq_off.user_addr; 544 rd.flags |= IORING_MEM_REGION_TYPE_USER; 545 } 546 ret = io_create_region(NULL, &n.ring_region, &rd, IORING_OFF_CQ_RING); 547 if (ret) 548 return ret; 549 550 n.rings = io_region_get_ptr(&n.ring_region); 551 552 /* 553 * At this point n.rings is shared with userspace, just like o.rings 554 * is as well. While we don't expect userspace to modify it while 555 * a resize is in progress, and it's most likely that userspace will 556 * shoot itself in the foot if it does, we can't always assume good 557 * intent... Use read/write once helpers from here on to indicate the 558 * shared nature of it. 559 */ 560 WRITE_ONCE(n.rings->sq_ring_mask, p->sq_entries - 1); 561 WRITE_ONCE(n.rings->cq_ring_mask, p->cq_entries - 1); 562 WRITE_ONCE(n.rings->sq_ring_entries, p->sq_entries); 563 WRITE_ONCE(n.rings->cq_ring_entries, p->cq_entries); 564 565 if (copy_to_user(arg, p, sizeof(*p))) { 566 io_register_free_rings(ctx, &n); 567 return -EFAULT; 568 } 569 570 memset(&rd, 0, sizeof(rd)); 571 rd.size = PAGE_ALIGN(rl->sq_size); 572 if (p->flags & IORING_SETUP_NO_MMAP) { 573 rd.user_addr = p->sq_off.user_addr; 574 rd.flags |= IORING_MEM_REGION_TYPE_USER; 575 } 576 ret = io_create_region(NULL, &n.sq_region, &rd, IORING_OFF_SQES); 577 if (ret) { 578 io_register_free_rings(ctx, &n); 579 return ret; 580 } 581 n.sq_sqes = io_region_get_ptr(&n.sq_region); 582 583 /* 584 * If using SQPOLL, park the thread 585 */ 586 if (ctx->sq_data) { 587 mutex_unlock(&ctx->uring_lock); 588 io_sq_thread_park(ctx->sq_data); 589 mutex_lock(&ctx->uring_lock); 590 } 591 592 /* 593 * We'll do the swap. Grab the ctx->mmap_lock, which will exclude 594 * any new mmap's on the ring fd. Clear out existing mappings to prevent 595 * mmap from seeing them, as we'll unmap them. Any attempt to mmap 596 * existing rings beyond this point will fail. Not that it could proceed 597 * at this point anyway, as the io_uring mmap side needs go grab the 598 * ctx->mmap_lock as well. Likewise, hold the completion lock over the 599 * duration of the actual swap. 600 */ 601 mutex_lock(&ctx->mmap_lock); 602 spin_lock(&ctx->completion_lock); 603 o.rings = ctx->rings; 604 ctx->rings = NULL; 605 o.sq_sqes = ctx->sq_sqes; 606 ctx->sq_sqes = NULL; 607 o_sq_array = ctx->sq_array; 608 if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) 609 n_sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset); 610 611 /* 612 * Now copy SQ and CQ entries, if any. If either of the destination 613 * rings can't hold what is already there, then fail the operation. 614 */ 615 tail = READ_ONCE(o.rings->sq.tail); 616 old_head = READ_ONCE(o.rings->sq.head); 617 if (tail - old_head > p->sq_entries) 618 goto overflow; 619 for (i = old_head; i != tail; i++) { 620 bool sqe128_tail = sqe128_src; 621 unsigned int dst, src; 622 size_t sq_size; 623 624 dst = i & (p->sq_entries - 1); 625 src = i & (ctx->sq_entries - 1); 626 if (sqe128_tail) { 627 /* it follows the first half, whatever the array says */ 628 src = sqe128_src; 629 sqe128_src = 0; 630 if (n_sq_array) 631 WRITE_ONCE(n_sq_array[dst], dst); 632 } else if (n_sq_array) { 633 src = READ_ONCE(o_sq_array[src]); 634 if (unlikely(src >= ctx->sq_entries)) { 635 WRITE_ONCE(n_sq_array[dst], UINT_MAX); 636 continue; 637 } 638 WRITE_ONCE(n_sq_array[dst], dst); 639 } 640 641 /* 642 * A 128b SQE on a SQE_MIXED ring takes two contiguous entries 643 * and must not wrap. A smaller ring may place it on the last 644 * entry, fail the resize if that happens. 645 */ 646 if (!sqe128_tail && i + 1 != tail && 647 src < ctx->sq_entries - 1 && 648 io_sqe_is_128(ctx, &o.sq_sqes[src])) { 649 if (dst == p->sq_entries - 1) 650 goto overflow; 651 sqe128_src = src + 1; 652 } 653 654 sq_size = sizeof(struct io_uring_sqe); 655 if (ctx->flags & IORING_SETUP_SQE128) { 656 dst <<= 1; 657 src <<= 1; 658 sq_size <<= 1; 659 } 660 memcpy(&n.sq_sqes[dst], &o.sq_sqes[src], sq_size); 661 } 662 WRITE_ONCE(n.rings->sq.head, old_head); 663 WRITE_ONCE(n.rings->sq.tail, tail); 664 665 tail = READ_ONCE(o.rings->cq.tail); 666 old_head = READ_ONCE(o.rings->cq.head); 667 if (tail - old_head > p->cq_entries) { 668 overflow: 669 /* restore old rings, and return -EOVERFLOW via cleanup path */ 670 ctx->rings = o.rings; 671 ctx->sq_sqes = o.sq_sqes; 672 to_free = &n; 673 ret = -EOVERFLOW; 674 goto out; 675 } 676 for (i = old_head; i != tail; i++) { 677 unsigned index, dst_mask, src_mask; 678 size_t cq_size; 679 680 index = i; 681 cq_size = sizeof(struct io_uring_cqe); 682 src_mask = ctx->cq_entries - 1; 683 dst_mask = p->cq_entries - 1; 684 if (ctx->flags & IORING_SETUP_CQE32) { 685 index <<= 1; 686 cq_size <<= 1; 687 src_mask = (ctx->cq_entries << 1) - 1; 688 dst_mask = (p->cq_entries << 1) - 1; 689 } 690 /* 691 * A 32b CQE on a CQE_MIXED ring takes two contiguous entries 692 * and must not wrap. A smaller ring may place it on the last 693 * entry, fail the resize if that happens. 694 */ 695 if (ctx->flags & IORING_SETUP_CQE_MIXED) { 696 if (cqe32_tail) { 697 cqe32_tail = false; 698 } else if (READ_ONCE(o.rings->cqes[index & src_mask].flags) & 699 IORING_CQE_F_32) { 700 if ((index & dst_mask) == dst_mask) 701 goto overflow; 702 cqe32_tail = true; 703 } 704 } 705 memcpy(&n.rings->cqes[index & dst_mask], &o.rings->cqes[index & src_mask], cq_size); 706 } 707 WRITE_ONCE(n.rings->cq.head, old_head); 708 WRITE_ONCE(n.rings->cq.tail, tail); 709 /* invalidate cached cqe refill */ 710 ctx->cqe_cached = ctx->cqe_sentinel = NULL; 711 712 WRITE_ONCE(n.rings->sq_dropped, READ_ONCE(o.rings->sq_dropped)); 713 atomic_set(&n.rings->sq_flags, atomic_read(&o.rings->sq_flags)); 714 WRITE_ONCE(n.rings->cq_flags, READ_ONCE(o.rings->cq_flags)); 715 WRITE_ONCE(n.rings->cq_overflow, READ_ONCE(o.rings->cq_overflow)); 716 717 /* all done, store old pointers and assign new ones */ 718 if (n_sq_array) 719 ctx->sq_array = n_sq_array; 720 721 ctx->sq_entries = p->sq_entries; 722 ctx->cq_entries = p->cq_entries; 723 724 /* 725 * Just mark any flag we may have missed and that the application 726 * should act on unconditionally. Worst case it'll be an extra 727 * syscall. 728 */ 729 atomic_or(IORING_SQ_TASKRUN | IORING_SQ_NEED_WAKEUP, &n.rings->sq_flags); 730 ctx->rings = n.rings; 731 rcu_assign_pointer(ctx->rings_rcu, n.rings); 732 733 ctx->sq_sqes = n.sq_sqes; 734 swap_old(ctx, o, n, ring_region); 735 swap_old(ctx, o, n, sq_region); 736 to_free = &o; 737 ret = 0; 738 out: 739 spin_unlock(&ctx->completion_lock); 740 mutex_unlock(&ctx->mmap_lock); 741 /* Wait for concurrent io_ctx_mark_taskrun() */ 742 if (to_free == &o) 743 synchronize_rcu_expedited(); 744 io_register_free_rings(ctx, to_free); 745 746 if (ctx->sq_data) 747 io_sq_thread_unpark(ctx->sq_data); 748 749 return ret; 750 } 751 752 static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) 753 { 754 struct io_uring_mem_region_reg __user *reg_uptr = uarg; 755 struct io_uring_mem_region_reg reg; 756 struct io_uring_region_desc __user *rd_uptr; 757 struct io_uring_region_desc rd; 758 struct io_mapped_region region = {}; 759 int ret; 760 761 if (io_region_is_set(&ctx->param_region)) 762 return -EBUSY; 763 if (copy_from_user(®, reg_uptr, sizeof(reg))) 764 return -EFAULT; 765 rd_uptr = u64_to_user_ptr(reg.region_uptr); 766 if (copy_from_user(&rd, rd_uptr, sizeof(rd))) 767 return -EFAULT; 768 if (memchr_inv(®.__resv, 0, sizeof(reg.__resv))) 769 return -EINVAL; 770 if (reg.flags & ~IORING_MEM_REGION_REG_WAIT_ARG) 771 return -EINVAL; 772 773 /* 774 * This ensures there are no waiters. Waiters are unlocked and it's 775 * hard to synchronise with them, especially if we need to initialise 776 * the region. 777 */ 778 if ((reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) && 779 !(ctx->flags & IORING_SETUP_R_DISABLED)) 780 return -EINVAL; 781 782 ret = io_create_region(ctx->user, ®ion, &rd, 783 IORING_MAP_OFF_PARAM_REGION); 784 if (ret) 785 return ret; 786 if (copy_to_user(rd_uptr, &rd, sizeof(rd))) { 787 io_free_region(ctx->user, ®ion); 788 return -EFAULT; 789 } 790 791 if (reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) { 792 ctx->cq_wait_arg = io_region_get_ptr(®ion); 793 ctx->cq_wait_size = rd.size; 794 } 795 796 io_region_publish(ctx, ®ion, &ctx->param_region); 797 return 0; 798 } 799 800 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, 801 void __user *arg, unsigned nr_args) 802 __releases(ctx->uring_lock) 803 __acquires(ctx->uring_lock) 804 { 805 int ret; 806 807 /* 808 * We don't quiesce the refs for register anymore and so it can't be 809 * dying as we're holding a file ref here. 810 */ 811 if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs))) 812 return -ENXIO; 813 814 if (ctx->submitter_task && ctx->submitter_task != current) 815 return -EEXIST; 816 817 /* 818 * A disabled ring is exempt from its own register restrictions, so 819 * that the creator can set them up and then enable the ring. That 820 * does not apply to restrictions inherited from the task. 821 */ 822 if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) && 823 (!(ctx->flags & IORING_SETUP_R_DISABLED) || 824 (ctx->int_flags & IO_RING_F_RESTRICT_INHERITED))) { 825 opcode = array_index_nospec(opcode, IORING_REGISTER_LAST); 826 if (!test_bit(opcode, ctx->restrictions.register_op)) 827 return -EACCES; 828 } 829 830 switch (opcode) { 831 case IORING_REGISTER_BUFFERS: 832 ret = -EFAULT; 833 if (!arg) 834 break; 835 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL); 836 break; 837 case IORING_UNREGISTER_BUFFERS: 838 ret = -EINVAL; 839 if (arg || nr_args) 840 break; 841 ret = io_sqe_buffers_unregister(ctx); 842 break; 843 case IORING_REGISTER_FILES: 844 ret = -EFAULT; 845 if (!arg) 846 break; 847 ret = io_sqe_files_register(ctx, arg, nr_args, NULL); 848 break; 849 case IORING_UNREGISTER_FILES: 850 ret = -EINVAL; 851 if (arg || nr_args) 852 break; 853 ret = io_sqe_files_unregister(ctx); 854 break; 855 case IORING_REGISTER_FILES_UPDATE: 856 ret = io_register_files_update(ctx, arg, nr_args); 857 break; 858 case IORING_REGISTER_EVENTFD: 859 ret = -EINVAL; 860 if (nr_args != 1) 861 break; 862 ret = io_eventfd_register(ctx, arg, 0); 863 break; 864 case IORING_REGISTER_EVENTFD_ASYNC: 865 ret = -EINVAL; 866 if (nr_args != 1) 867 break; 868 ret = io_eventfd_register(ctx, arg, 1); 869 break; 870 case IORING_UNREGISTER_EVENTFD: 871 ret = -EINVAL; 872 if (arg || nr_args) 873 break; 874 ret = io_eventfd_unregister(ctx); 875 break; 876 case IORING_REGISTER_PROBE: 877 ret = -EINVAL; 878 if (!arg || nr_args > 256) 879 break; 880 ret = io_probe(ctx, arg, nr_args); 881 break; 882 case IORING_REGISTER_PERSONALITY: 883 ret = -EINVAL; 884 if (arg || nr_args) 885 break; 886 ret = io_register_personality(ctx); 887 break; 888 case IORING_UNREGISTER_PERSONALITY: 889 ret = -EINVAL; 890 if (arg) 891 break; 892 ret = io_unregister_personality(ctx, nr_args); 893 break; 894 case IORING_REGISTER_ENABLE_RINGS: 895 ret = -EINVAL; 896 if (arg || nr_args) 897 break; 898 ret = io_register_enable_rings(ctx); 899 break; 900 case IORING_REGISTER_RESTRICTIONS: 901 ret = io_register_restrictions(ctx, arg, nr_args); 902 break; 903 case IORING_REGISTER_FILES2: 904 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE); 905 break; 906 case IORING_REGISTER_FILES_UPDATE2: 907 ret = io_register_rsrc_update(ctx, arg, nr_args, 908 IORING_RSRC_FILE); 909 break; 910 case IORING_REGISTER_BUFFERS2: 911 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER); 912 break; 913 case IORING_REGISTER_BUFFERS_UPDATE: 914 ret = io_register_rsrc_update(ctx, arg, nr_args, 915 IORING_RSRC_BUFFER); 916 break; 917 case IORING_REGISTER_IOWQ_AFF: 918 ret = -EINVAL; 919 if (!arg || !nr_args) 920 break; 921 ret = io_register_iowq_aff(ctx, arg, nr_args); 922 break; 923 case IORING_UNREGISTER_IOWQ_AFF: 924 ret = -EINVAL; 925 if (arg || nr_args) 926 break; 927 ret = io_unregister_iowq_aff(ctx); 928 break; 929 case IORING_REGISTER_IOWQ_MAX_WORKERS: 930 ret = -EINVAL; 931 if (!arg || nr_args != 2) 932 break; 933 ret = io_register_iowq_max_workers(ctx, arg); 934 break; 935 case IORING_REGISTER_RING_FDS: 936 ret = io_ringfd_register(ctx, arg, nr_args); 937 break; 938 case IORING_UNREGISTER_RING_FDS: 939 ret = io_ringfd_unregister(ctx, arg, nr_args); 940 break; 941 case IORING_REGISTER_PBUF_RING: 942 ret = -EINVAL; 943 if (!arg || nr_args != 1) 944 break; 945 ret = io_register_pbuf_ring(ctx, arg); 946 break; 947 case IORING_UNREGISTER_PBUF_RING: 948 ret = -EINVAL; 949 if (!arg || nr_args != 1) 950 break; 951 ret = io_unregister_pbuf_ring(ctx, arg); 952 break; 953 case IORING_REGISTER_SYNC_CANCEL: 954 ret = -EINVAL; 955 if (!arg || nr_args != 1) 956 break; 957 ret = io_sync_cancel(ctx, arg); 958 break; 959 case IORING_REGISTER_FILE_ALLOC_RANGE: 960 ret = -EINVAL; 961 if (!arg || nr_args) 962 break; 963 ret = io_register_file_alloc_range(ctx, arg); 964 break; 965 case IORING_REGISTER_PBUF_STATUS: 966 ret = -EINVAL; 967 if (!arg || nr_args != 1) 968 break; 969 ret = io_register_pbuf_status(ctx, arg); 970 break; 971 case IORING_REGISTER_NAPI: 972 ret = -EINVAL; 973 if (!arg || nr_args != 1) 974 break; 975 ret = io_register_napi(ctx, arg); 976 break; 977 case IORING_UNREGISTER_NAPI: 978 ret = -EINVAL; 979 if (nr_args != 1) 980 break; 981 ret = io_unregister_napi(ctx, arg); 982 break; 983 case IORING_REGISTER_CLOCK: 984 ret = -EINVAL; 985 if (!arg || nr_args) 986 break; 987 ret = io_register_clock(ctx, arg); 988 break; 989 case IORING_REGISTER_CLONE_BUFFERS: 990 ret = -EINVAL; 991 if (!arg || nr_args != 1) 992 break; 993 ret = io_register_clone_buffers(ctx, arg); 994 break; 995 case IORING_REGISTER_ZCRX_IFQ: 996 ret = -EINVAL; 997 if (!arg || nr_args != 1) 998 break; 999 ret = io_register_zcrx(ctx, arg); 1000 break; 1001 case IORING_REGISTER_RESIZE_RINGS: 1002 ret = -EINVAL; 1003 if (!arg || nr_args != 1) 1004 break; 1005 ret = io_register_resize_rings(ctx, arg); 1006 break; 1007 case IORING_REGISTER_MEM_REGION: 1008 ret = -EINVAL; 1009 if (!arg || nr_args != 1) 1010 break; 1011 ret = io_register_mem_region(ctx, arg); 1012 break; 1013 case IORING_REGISTER_QUERY: 1014 ret = io_query(arg, nr_args); 1015 break; 1016 case IORING_REGISTER_ZCRX_CTRL: 1017 ret = io_zcrx_ctrl(ctx, arg, nr_args); 1018 break; 1019 case IORING_REGISTER_BPF_FILTER: 1020 ret = -EINVAL; 1021 1022 if (nr_args != 1) 1023 break; 1024 ret = io_register_bpf_filter(&ctx->restrictions, arg); 1025 if (!ret) 1026 WRITE_ONCE(ctx->bpf_filters, 1027 ctx->restrictions.bpf_filters->filters); 1028 break; 1029 default: 1030 ret = -EINVAL; 1031 break; 1032 } 1033 1034 return ret; 1035 } 1036 1037 static int io_uring_register_send_msg_ring(void __user *arg, unsigned int nr_args) 1038 { 1039 struct io_uring_sqe sqe; 1040 1041 if (!arg || nr_args != 1) 1042 return -EINVAL; 1043 if (copy_from_user(&sqe, arg, sizeof(sqe))) 1044 return -EFAULT; 1045 /* no flags supported */ 1046 if (sqe.flags) 1047 return -EINVAL; 1048 if (sqe.opcode != IORING_OP_MSG_RING) 1049 return -EINVAL; 1050 1051 return io_uring_sync_msg_ring(&sqe); 1052 } 1053 1054 /* 1055 * "blind" registration opcodes are ones where there's no ring given, and 1056 * hence the source fd must be -1. 1057 */ 1058 static int io_uring_register_blind(unsigned int opcode, void __user *arg, 1059 unsigned int nr_args) 1060 { 1061 switch (opcode) { 1062 case IORING_REGISTER_SEND_MSG_RING: 1063 return io_uring_register_send_msg_ring(arg, nr_args); 1064 case IORING_REGISTER_QUERY: 1065 return io_query(arg, nr_args); 1066 case IORING_REGISTER_RESTRICTIONS: 1067 return io_register_restrictions_task(arg, nr_args); 1068 case IORING_REGISTER_BPF_FILTER: 1069 return io_register_bpf_filter_task(arg, nr_args); 1070 } 1071 return -EINVAL; 1072 } 1073 1074 SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, 1075 void __user *, arg, unsigned int, nr_args) 1076 { 1077 struct io_ring_ctx *ctx; 1078 long ret = -EBADF; 1079 struct file *file; 1080 bool use_registered_ring; 1081 1082 use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING); 1083 opcode &= ~IORING_REGISTER_USE_REGISTERED_RING; 1084 1085 if (opcode >= IORING_REGISTER_LAST) 1086 return -EINVAL; 1087 1088 if (fd == -1) 1089 return io_uring_register_blind(opcode, arg, nr_args); 1090 1091 file = io_uring_ctx_get_file(fd, use_registered_ring); 1092 if (IS_ERR(file)) 1093 return PTR_ERR(file); 1094 ctx = file->private_data; 1095 1096 mutex_lock(&ctx->uring_lock); 1097 ret = __io_uring_register(ctx, opcode, arg, nr_args); 1098 1099 trace_io_uring_register(ctx, opcode, ctx->file_table.data.nr, 1100 ctx->buf_table.nr, ret); 1101 mutex_unlock(&ctx->uring_lock); 1102 1103 if (!use_registered_ring) 1104 fput(file); 1105 return ret; 1106 } 1107