1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/mm.h> 7 #include <linux/slab.h> 8 #include <linux/namei.h> 9 #include <linux/poll.h> 10 #include <linux/vmalloc.h> 11 #include <linux/io_uring.h> 12 13 #include <uapi/linux/io_uring.h> 14 15 #include "io_uring.h" 16 #include "opdef.h" 17 #include "kbuf.h" 18 #include "memmap.h" 19 20 /* BIDs are addressed by a 16-bit field in a CQE */ 21 #define MAX_BIDS_PER_BGID (1 << 16) 22 23 /* Mapped buffer ring, return io_uring_buf from head */ 24 #define io_ring_head_to_buf(br, head, mask) (&(br)->bufs[(head) & (mask)]) 25 26 struct io_provide_buf { 27 struct file *file; 28 __u64 addr; 29 __u32 len; 30 __u32 bgid; 31 __u32 nbufs; 32 __u16 bid; 33 }; 34 35 static bool io_kbuf_inc_commit(struct io_buffer_list *bl, int len) 36 { 37 /* No data consumed, return false early to avoid consuming the buffer */ 38 if (!len) 39 return false; 40 41 while (len) { 42 struct io_uring_buf *buf; 43 u32 buf_len, this_len; 44 45 buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask); 46 buf_len = READ_ONCE(buf->len); 47 this_len = min_t(u32, len, buf_len); 48 buf_len -= this_len; 49 /* Stop looping for invalid buffer length of 0 */ 50 if (buf_len > bl->min_left_sub_one || !this_len) { 51 WRITE_ONCE(buf->addr, READ_ONCE(buf->addr) + this_len); 52 WRITE_ONCE(buf->len, buf_len); 53 return false; 54 } 55 WRITE_ONCE(buf->len, 0); 56 bl->head++; 57 len -= this_len; 58 } 59 return true; 60 } 61 62 bool io_kbuf_commit(struct io_kiocb *req, 63 struct io_buffer_list *bl, int len, int nr) 64 { 65 if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT))) 66 return true; 67 68 req->flags &= ~REQ_F_BUFFERS_COMMIT; 69 70 if (unlikely(len < 0)) 71 return true; 72 if (bl->flags & IOBL_INC) 73 return io_kbuf_inc_commit(bl, len); 74 bl->head += nr; 75 return true; 76 } 77 78 static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx, 79 unsigned int bgid) 80 { 81 lockdep_assert_held(&ctx->uring_lock); 82 83 return xa_load(&ctx->io_bl_xa, bgid); 84 } 85 86 static int io_buffer_add_list(struct io_ring_ctx *ctx, 87 struct io_buffer_list *bl, unsigned int bgid) 88 { 89 /* 90 * Store buffer group ID and finally mark the list as visible. 91 * The normal lookup doesn't care about the visibility as we're 92 * always under the ->uring_lock, but lookups from mmap do. 93 */ 94 bl->bgid = bgid; 95 guard(mutex)(&ctx->mmap_lock); 96 return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL)); 97 } 98 99 void io_kbuf_drop_legacy(struct io_kiocb *req) 100 { 101 if (WARN_ON_ONCE(!(req->flags & REQ_F_BUFFER_SELECTED))) 102 return; 103 req->flags &= ~REQ_F_BUFFER_SELECTED; 104 kfree(req->kbuf); 105 req->kbuf = NULL; 106 } 107 108 bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags) 109 { 110 struct io_ring_ctx *ctx = req->ctx; 111 struct io_buffer_list *bl; 112 struct io_buffer *buf; 113 114 io_ring_submit_lock(ctx, issue_flags); 115 116 buf = req->kbuf; 117 bl = io_buffer_get_list(ctx, buf->bgid); 118 /* 119 * If the buffer list was upgraded to a ring-based one, or removed, 120 * while the request was in-flight in io-wq, drop it. 121 */ 122 if (bl && !(bl->flags & IOBL_BUF_RING)) { 123 list_add(&buf->list, &bl->buf_list); 124 bl->nbufs++; 125 } else { 126 kfree(buf); 127 } 128 req->flags &= ~REQ_F_BUFFER_SELECTED; 129 req->kbuf = NULL; 130 131 io_ring_submit_unlock(ctx, issue_flags); 132 return true; 133 } 134 135 static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len, 136 struct io_buffer_list *bl) 137 { 138 if (!list_empty(&bl->buf_list)) { 139 struct io_buffer *kbuf; 140 141 kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list); 142 list_del(&kbuf->list); 143 bl->nbufs--; 144 if (*len == 0 || *len > kbuf->len) 145 *len = kbuf->len; 146 if (list_empty(&bl->buf_list)) 147 req->flags |= REQ_F_BL_EMPTY; 148 req->flags |= REQ_F_BUFFER_SELECTED; 149 req->kbuf = kbuf; 150 req->buf_index = kbuf->bid; 151 return u64_to_user_ptr(kbuf->addr); 152 } 153 return NULL; 154 } 155 156 static int io_provided_buffers_select(struct io_kiocb *req, size_t *len, 157 struct io_buffer_list *bl, 158 struct iovec *iov) 159 { 160 void __user *buf; 161 162 buf = io_provided_buffer_select(req, len, bl); 163 if (unlikely(!buf)) 164 return -ENOBUFS; 165 166 iov[0].iov_base = buf; 167 iov[0].iov_len = *len; 168 return 1; 169 } 170 171 static bool io_should_commit(struct io_kiocb *req, unsigned int issue_flags) 172 { 173 /* 174 * If we came in unlocked, we have no choice but to consume the 175 * buffer here, otherwise nothing ensures that the buffer won't 176 * get used by others. This does mean it'll be pinned until the 177 * IO completes, coming in unlocked means we're being called from 178 * io-wq context and there may be further retries in async hybrid 179 * mode. For the locked case, the caller must call commit when 180 * the transfer completes (or if we get -EAGAIN and must poll of 181 * retry). 182 */ 183 if (issue_flags & IO_URING_F_UNLOCKED) 184 return true; 185 186 /* uring_cmd commits kbuf upfront, no need to auto-commit */ 187 if (!io_file_can_poll(req) && !io_is_uring_cmd(req)) 188 return true; 189 return false; 190 } 191 192 static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len, 193 struct io_buffer_list *bl, 194 unsigned int issue_flags) 195 { 196 struct io_uring_buf_ring *br = bl->buf_ring; 197 __u16 tail, head = bl->head; 198 struct io_br_sel sel = { }; 199 struct io_uring_buf *buf; 200 u32 buf_len; 201 202 tail = smp_load_acquire(&br->tail); 203 if (unlikely(tail == head)) 204 return sel; 205 206 if (head + 1 == tail) 207 req->flags |= REQ_F_BL_EMPTY; 208 209 buf = io_ring_head_to_buf(br, head, bl->mask); 210 buf_len = READ_ONCE(buf->len); 211 if (*len == 0 || *len > buf_len) 212 *len = buf_len; 213 sel.addr = u64_to_user_ptr(READ_ONCE(buf->addr)); 214 if (unlikely(!access_ok(sel.addr, *len))) { 215 sel.addr = NULL; 216 return sel; 217 } 218 req->flags |= REQ_F_BUFFER_RING | REQ_F_BUFFERS_COMMIT; 219 req->buf_index = READ_ONCE(buf->bid); 220 sel.buf_list = bl; 221 222 if (io_should_commit(req, issue_flags)) { 223 if (!io_kbuf_commit(req, sel.buf_list, *len, 1)) 224 req->flags |= REQ_F_BUF_MORE; 225 sel.buf_list = NULL; 226 } 227 return sel; 228 } 229 230 struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len, 231 unsigned buf_group, unsigned int issue_flags) 232 { 233 struct io_ring_ctx *ctx = req->ctx; 234 struct io_br_sel sel = { }; 235 struct io_buffer_list *bl; 236 237 io_ring_submit_lock(ctx, issue_flags); 238 239 bl = io_buffer_get_list(ctx, buf_group); 240 if (likely(bl)) { 241 if (bl->flags & IOBL_BUF_RING) 242 sel = io_ring_buffer_select(req, len, bl, issue_flags); 243 else 244 sel.addr = io_provided_buffer_select(req, len, bl); 245 } 246 io_ring_submit_unlock(ctx, issue_flags); 247 return sel; 248 } 249 250 /* cap it at a reasonable 256, will be one page even for 4K */ 251 #define PEEK_MAX_IMPORT 256 252 253 static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, 254 struct io_buffer_list *bl) 255 { 256 struct io_uring_buf_ring *br = bl->buf_ring; 257 struct iovec *org_iovs = arg->iovs; 258 struct iovec *iov = arg->iovs; 259 int nr_iovs = arg->nr_iovs; 260 __u16 nr_avail, tail, head; 261 struct io_uring_buf *buf; 262 263 tail = smp_load_acquire(&br->tail); 264 head = bl->head; 265 nr_avail = min_t(__u16, tail - head, UIO_MAXIOV); 266 if (unlikely(!nr_avail)) 267 return -ENOBUFS; 268 269 buf = io_ring_head_to_buf(br, head, bl->mask); 270 if (arg->max_len) { 271 u32 len = READ_ONCE(buf->len); 272 size_t needed; 273 274 if (unlikely(!len)) 275 return -ENOBUFS; 276 needed = (arg->max_len + len - 1) / len; 277 needed = min_not_zero(needed, (size_t) PEEK_MAX_IMPORT); 278 if (nr_avail > needed) 279 nr_avail = needed; 280 } 281 282 /* 283 * only alloc a bigger array if we know we have data to map, eg not 284 * a speculative peek operation. 285 */ 286 if (arg->mode & KBUF_MODE_EXPAND && nr_avail > nr_iovs && arg->max_len) { 287 iov = kmalloc_objs(struct iovec, nr_avail); 288 if (unlikely(!iov)) 289 return -ENOMEM; 290 if (arg->mode & KBUF_MODE_FREE) 291 kfree(arg->iovs); 292 arg->iovs = iov; 293 nr_iovs = nr_avail; 294 } else if (nr_avail < nr_iovs) { 295 nr_iovs = nr_avail; 296 } 297 298 /* set it to max, if not set, so we can use it unconditionally */ 299 if (!arg->max_len) 300 arg->max_len = INT_MAX; 301 302 req->buf_index = READ_ONCE(buf->bid); 303 do { 304 u32 len = READ_ONCE(buf->len); 305 306 /* truncate end piece, if needed, for non partial buffers */ 307 if (len > arg->max_len) { 308 len = arg->max_len; 309 if (!(bl->flags & IOBL_INC)) { 310 arg->partial_map = 1; 311 if (iov != arg->iovs) 312 break; 313 } 314 } 315 316 iov->iov_base = u64_to_user_ptr(READ_ONCE(buf->addr)); 317 iov->iov_len = len; 318 if (unlikely(!access_ok(iov->iov_base, len))) { 319 if (arg->iovs != org_iovs) 320 kfree(arg->iovs); 321 return -EFAULT; 322 } 323 iov++; 324 325 arg->out_len += len; 326 arg->max_len -= len; 327 if (!arg->max_len) 328 break; 329 330 buf = io_ring_head_to_buf(br, ++head, bl->mask); 331 } while (--nr_iovs); 332 333 if (head == tail) 334 req->flags |= REQ_F_BL_EMPTY; 335 336 req->flags |= REQ_F_BUFFER_RING; 337 return iov - arg->iovs; 338 } 339 340 int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, 341 struct io_br_sel *sel, unsigned int issue_flags) 342 { 343 struct io_ring_ctx *ctx = req->ctx; 344 int ret = -ENOENT; 345 346 io_ring_submit_lock(ctx, issue_flags); 347 sel->buf_list = io_buffer_get_list(ctx, arg->buf_group); 348 if (unlikely(!sel->buf_list)) 349 goto out_unlock; 350 351 if (sel->buf_list->flags & IOBL_BUF_RING) { 352 ret = io_ring_buffers_peek(req, arg, sel->buf_list); 353 /* 354 * Don't recycle these buffers if we need to go through poll. 355 * Nobody else can use them anyway, and holding on to provided 356 * buffers for a send/write operation would happen on the app 357 * side anyway with normal buffers. Besides, we already 358 * committed them, they cannot be put back in the queue. 359 */ 360 if (ret > 0) { 361 req->flags |= REQ_F_BUFFERS_COMMIT | REQ_F_BL_NO_RECYCLE; 362 if (!io_kbuf_commit(req, sel->buf_list, arg->out_len, ret)) 363 req->flags |= REQ_F_BUF_MORE; 364 } 365 } else { 366 ret = io_provided_buffers_select(req, &arg->out_len, sel->buf_list, arg->iovs); 367 } 368 out_unlock: 369 if (issue_flags & IO_URING_F_UNLOCKED) { 370 sel->buf_list = NULL; 371 mutex_unlock(&ctx->uring_lock); 372 } 373 return ret; 374 } 375 376 int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, 377 struct io_br_sel *sel) 378 { 379 struct io_ring_ctx *ctx = req->ctx; 380 struct io_buffer_list *bl; 381 int ret; 382 383 lockdep_assert_held(&ctx->uring_lock); 384 385 bl = io_buffer_get_list(ctx, arg->buf_group); 386 if (unlikely(!bl)) 387 return -ENOENT; 388 389 if (bl->flags & IOBL_BUF_RING) { 390 ret = io_ring_buffers_peek(req, arg, bl); 391 if (ret > 0) 392 req->flags |= REQ_F_BUFFERS_COMMIT; 393 sel->buf_list = bl; 394 return ret; 395 } 396 397 /* don't support multiple buffer selections for legacy */ 398 sel->buf_list = NULL; 399 return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs); 400 } 401 402 static inline bool __io_put_kbuf_ring(struct io_kiocb *req, 403 struct io_buffer_list *bl, int len, int nr) 404 { 405 bool ret = true; 406 407 if (bl) 408 ret = io_kbuf_commit(req, bl, len, nr); 409 if (ret && (req->flags & REQ_F_BUF_MORE)) 410 ret = false; 411 412 req->flags &= ~(REQ_F_BUFFER_RING | REQ_F_BUF_MORE); 413 return ret; 414 } 415 416 unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl, 417 int len, int nbufs) 418 { 419 unsigned int ret; 420 421 ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT); 422 423 if (unlikely(!(req->flags & REQ_F_BUFFER_RING))) { 424 io_kbuf_drop_legacy(req); 425 return ret; 426 } 427 428 if (!__io_put_kbuf_ring(req, bl, len, nbufs)) 429 ret |= IORING_CQE_F_BUF_MORE; 430 return ret; 431 } 432 433 static int io_remove_buffers_legacy(struct io_ring_ctx *ctx, 434 struct io_buffer_list *bl, 435 unsigned long nbufs) 436 { 437 unsigned long i = 0; 438 struct io_buffer *nxt; 439 440 /* protects io_buffers_cache */ 441 lockdep_assert_held(&ctx->uring_lock); 442 WARN_ON_ONCE(bl->flags & IOBL_BUF_RING); 443 444 for (i = 0; i < nbufs && !list_empty(&bl->buf_list); i++) { 445 nxt = list_first_entry(&bl->buf_list, struct io_buffer, list); 446 list_del(&nxt->list); 447 bl->nbufs--; 448 kfree(nxt); 449 cond_resched(); 450 } 451 return i; 452 } 453 454 static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) 455 { 456 if (bl->flags & IOBL_BUF_RING) 457 io_free_region(ctx->user, &bl->region); 458 else 459 io_remove_buffers_legacy(ctx, bl, -1U); 460 461 kfree(bl); 462 } 463 464 void io_destroy_buffers(struct io_ring_ctx *ctx) 465 { 466 struct io_buffer_list *bl; 467 468 while (1) { 469 unsigned long index = 0; 470 471 scoped_guard(mutex, &ctx->mmap_lock) { 472 bl = xa_find(&ctx->io_bl_xa, &index, ULONG_MAX, XA_PRESENT); 473 if (bl) 474 xa_erase(&ctx->io_bl_xa, bl->bgid); 475 } 476 if (!bl) 477 break; 478 io_put_bl(ctx, bl); 479 } 480 } 481 482 static void io_destroy_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) 483 { 484 scoped_guard(mutex, &ctx->mmap_lock) 485 WARN_ON_ONCE(xa_erase(&ctx->io_bl_xa, bl->bgid) != bl); 486 io_put_bl(ctx, bl); 487 } 488 489 int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 490 { 491 struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); 492 u64 tmp; 493 494 if (sqe->rw_flags || sqe->addr || sqe->len || sqe->off || 495 sqe->splice_fd_in) 496 return -EINVAL; 497 498 tmp = READ_ONCE(sqe->fd); 499 if (!tmp || tmp > MAX_BIDS_PER_BGID) 500 return -EINVAL; 501 502 memset(p, 0, sizeof(*p)); 503 p->nbufs = tmp; 504 p->bgid = READ_ONCE(sqe->buf_group); 505 return 0; 506 } 507 508 int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 509 { 510 unsigned long size, tmp_check; 511 struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); 512 u64 tmp; 513 514 if (sqe->rw_flags || sqe->splice_fd_in) 515 return -EINVAL; 516 517 tmp = READ_ONCE(sqe->fd); 518 if (!tmp || tmp > MAX_BIDS_PER_BGID) 519 return -E2BIG; 520 p->nbufs = tmp; 521 p->addr = READ_ONCE(sqe->addr); 522 p->len = READ_ONCE(sqe->len); 523 if (!p->len) 524 return -EINVAL; 525 526 if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs, 527 &size)) 528 return -EOVERFLOW; 529 if (check_add_overflow((unsigned long)p->addr, size, &tmp_check)) 530 return -EOVERFLOW; 531 if (!access_ok(u64_to_user_ptr(p->addr), size)) 532 return -EFAULT; 533 534 p->bgid = READ_ONCE(sqe->buf_group); 535 tmp = READ_ONCE(sqe->off); 536 if (tmp > USHRT_MAX) 537 return -E2BIG; 538 if (tmp + p->nbufs > MAX_BIDS_PER_BGID) 539 return -EINVAL; 540 p->bid = tmp; 541 return 0; 542 } 543 544 static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, 545 struct io_buffer_list *bl) 546 { 547 struct io_buffer *buf; 548 u64 addr = pbuf->addr; 549 int ret = -ENOMEM, i, bid = pbuf->bid; 550 551 for (i = 0; i < pbuf->nbufs; i++) { 552 /* 553 * Nonsensical to have more than MAX_BIDS_PER_BGID buffers in a 554 * buffer list, as the application then has no way of knowing 555 * which duplicate bid refers to what buffer. 556 */ 557 if (bl->nbufs == MAX_BIDS_PER_BGID) { 558 ret = -EOVERFLOW; 559 break; 560 } 561 buf = kmalloc_obj(*buf, GFP_KERNEL_ACCOUNT); 562 if (!buf) 563 break; 564 565 list_add_tail(&buf->list, &bl->buf_list); 566 bl->nbufs++; 567 buf->addr = addr; 568 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); 569 buf->bid = bid; 570 buf->bgid = pbuf->bgid; 571 addr += pbuf->len; 572 bid++; 573 cond_resched(); 574 } 575 576 return i ? 0 : ret; 577 } 578 579 static int __io_manage_buffers_legacy(struct io_kiocb *req, 580 struct io_buffer_list *bl) 581 { 582 struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); 583 int ret; 584 585 if (!bl) { 586 if (req->opcode != IORING_OP_PROVIDE_BUFFERS) 587 return -ENOENT; 588 bl = kzalloc_obj(*bl, GFP_KERNEL_ACCOUNT); 589 if (!bl) 590 return -ENOMEM; 591 592 INIT_LIST_HEAD(&bl->buf_list); 593 ret = io_buffer_add_list(req->ctx, bl, p->bgid); 594 if (ret) { 595 kfree(bl); 596 return ret; 597 } 598 } 599 /* can't use provide/remove buffers command on mapped buffers */ 600 if (bl->flags & IOBL_BUF_RING) 601 return -EINVAL; 602 if (req->opcode == IORING_OP_PROVIDE_BUFFERS) 603 return io_add_buffers(req->ctx, p, bl); 604 return io_remove_buffers_legacy(req->ctx, bl, p->nbufs); 605 } 606 607 int io_manage_buffers_legacy(struct io_kiocb *req, unsigned int issue_flags) 608 { 609 struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); 610 struct io_ring_ctx *ctx = req->ctx; 611 struct io_buffer_list *bl; 612 int ret; 613 614 io_ring_submit_lock(ctx, issue_flags); 615 bl = io_buffer_get_list(ctx, p->bgid); 616 ret = __io_manage_buffers_legacy(req, bl); 617 io_ring_submit_unlock(ctx, issue_flags); 618 619 if (ret < 0) 620 req_set_fail(req); 621 io_req_set_res(req, ret, 0); 622 return IOU_COMPLETE; 623 } 624 625 int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) 626 { 627 struct io_uring_buf_reg reg; 628 struct io_buffer_list *bl; 629 struct io_uring_region_desc rd; 630 struct io_uring_buf_ring *br; 631 unsigned long mmap_offset; 632 unsigned long ring_size; 633 int ret; 634 635 lockdep_assert_held(&ctx->uring_lock); 636 637 if (copy_from_user(®, arg, sizeof(reg))) 638 return -EFAULT; 639 if (!mem_is_zero(reg.resv, sizeof(reg.resv))) 640 return -EINVAL; 641 if (reg.flags & ~(IOU_PBUF_RING_MMAP | IOU_PBUF_RING_INC)) 642 return -EINVAL; 643 if (!is_power_of_2(reg.ring_entries)) 644 return -EINVAL; 645 /* cannot disambiguate full vs empty due to head/tail size */ 646 if (reg.ring_entries >= 65536) 647 return -EINVAL; 648 649 /* minimum left byte count is a property of incremental buffers */ 650 if (!(reg.flags & IOU_PBUF_RING_INC) && reg.min_left) 651 return -EINVAL; 652 653 bl = io_buffer_get_list(ctx, reg.bgid); 654 if (bl) { 655 /* if mapped buffer ring OR classic exists, don't allow */ 656 if (bl->flags & IOBL_BUF_RING || !list_empty(&bl->buf_list)) 657 return -EEXIST; 658 io_destroy_bl(ctx, bl); 659 } 660 661 bl = kzalloc_obj(*bl, GFP_KERNEL_ACCOUNT); 662 if (!bl) 663 return -ENOMEM; 664 665 mmap_offset = (unsigned long)reg.bgid << IORING_OFF_PBUF_SHIFT; 666 ring_size = flex_array_size(br, bufs, reg.ring_entries); 667 668 memset(&rd, 0, sizeof(rd)); 669 rd.size = PAGE_ALIGN(ring_size); 670 if (!(reg.flags & IOU_PBUF_RING_MMAP)) { 671 rd.user_addr = reg.ring_addr; 672 rd.flags |= IORING_MEM_REGION_TYPE_USER; 673 } 674 ret = io_create_region(ctx, &bl->region, &rd, mmap_offset); 675 if (ret) 676 goto fail; 677 br = io_region_get_ptr(&bl->region); 678 679 #ifdef SHM_COLOUR 680 /* 681 * On platforms that have specific aliasing requirements, SHM_COLOUR 682 * is set and we must guarantee that the kernel and user side align 683 * nicely. We cannot do that if IOU_PBUF_RING_MMAP isn't set and 684 * the application mmap's the provided ring buffer. Fail the request 685 * if we, by chance, don't end up with aligned addresses. The app 686 * should use IOU_PBUF_RING_MMAP instead, and liburing will handle 687 * this transparently. 688 */ 689 if (!(reg.flags & IOU_PBUF_RING_MMAP) && 690 ((reg.ring_addr | (unsigned long)br) & (SHM_COLOUR - 1))) { 691 ret = -EINVAL; 692 goto fail; 693 } 694 #endif 695 696 bl->mask = reg.ring_entries - 1; 697 bl->flags |= IOBL_BUF_RING; 698 bl->buf_ring = br; 699 if (reg.min_left) 700 bl->min_left_sub_one = reg.min_left - 1; 701 if (reg.flags & IOU_PBUF_RING_INC) 702 bl->flags |= IOBL_INC; 703 ret = io_buffer_add_list(ctx, bl, reg.bgid); 704 if (!ret) 705 return 0; 706 fail: 707 io_free_region(ctx->user, &bl->region); 708 kfree(bl); 709 return ret; 710 } 711 712 int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) 713 { 714 struct io_uring_buf_reg reg; 715 struct io_buffer_list *bl; 716 717 lockdep_assert_held(&ctx->uring_lock); 718 719 if (copy_from_user(®, arg, sizeof(reg))) 720 return -EFAULT; 721 if (!mem_is_zero(reg.resv, sizeof(reg.resv)) || reg.flags) 722 return -EINVAL; 723 724 bl = io_buffer_get_list(ctx, reg.bgid); 725 if (!bl) 726 return -ENOENT; 727 if (!(bl->flags & IOBL_BUF_RING)) 728 return -EINVAL; 729 730 scoped_guard(mutex, &ctx->mmap_lock) 731 xa_erase(&ctx->io_bl_xa, bl->bgid); 732 733 io_put_bl(ctx, bl); 734 return 0; 735 } 736 737 int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg) 738 { 739 struct io_uring_buf_status buf_status; 740 struct io_buffer_list *bl; 741 742 if (copy_from_user(&buf_status, arg, sizeof(buf_status))) 743 return -EFAULT; 744 if (!mem_is_zero(buf_status.resv, sizeof(buf_status.resv))) 745 return -EINVAL; 746 747 bl = io_buffer_get_list(ctx, buf_status.buf_group); 748 if (!bl) 749 return -ENOENT; 750 if (!(bl->flags & IOBL_BUF_RING)) 751 return -EINVAL; 752 753 buf_status.head = bl->head; 754 if (copy_to_user(arg, &buf_status, sizeof(buf_status))) 755 return -EFAULT; 756 757 return 0; 758 } 759 760 struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx, 761 unsigned int bgid) 762 { 763 struct io_buffer_list *bl; 764 765 lockdep_assert_held(&ctx->mmap_lock); 766 767 bl = xa_load(&ctx->io_bl_xa, bgid); 768 if (!bl || !(bl->flags & IOBL_BUF_RING)) 769 return NULL; 770 return &bl->region; 771 } 772