1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/mm.h> 7 #include <linux/slab.h> 8 #include <linux/namei.h> 9 #include <linux/poll.h> 10 #include <linux/vmalloc.h> 11 #include <linux/io_uring.h> 12 13 #include <uapi/linux/io_uring.h> 14 15 #include "io_uring.h" 16 #include "opdef.h" 17 #include "kbuf.h" 18 #include "memmap.h" 19 20 /* BIDs are addressed by a 16-bit field in a CQE */ 21 #define MAX_BIDS_PER_BGID (1 << 16) 22 23 /* Mapped buffer ring, return io_uring_buf from head */ 24 #define io_ring_head_to_buf(br, head, mask) (&(br)->bufs[(head) & (mask)]) 25 26 struct io_provide_buf { 27 struct file *file; 28 __u64 addr; 29 __u32 len; 30 __u32 bgid; 31 __u32 nbufs; 32 __u16 bid; 33 }; 34 35 static bool io_kbuf_inc_commit(struct io_buffer_list *bl, int len) 36 { 37 /* No data consumed, return false early to avoid consuming the buffer */ 38 if (!len) 39 return false; 40 41 while (len) { 42 struct io_uring_buf *buf; 43 u32 buf_len, this_len; 44 45 buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask); 46 buf_len = READ_ONCE(buf->len); 47 this_len = min_t(u32, len, buf_len); 48 buf_len -= this_len; 49 /* Stop looping for invalid buffer length of 0 */ 50 if (buf_len > bl->min_left_sub_one || !this_len) { 51 WRITE_ONCE(buf->addr, READ_ONCE(buf->addr) + this_len); 52 WRITE_ONCE(buf->len, buf_len); 53 return false; 54 } 55 WRITE_ONCE(buf->len, 0); 56 bl->head++; 57 len -= this_len; 58 } 59 return true; 60 } 61 62 bool io_kbuf_commit(struct io_kiocb *req, 63 struct io_buffer_list *bl, int len, int nr) 64 { 65 if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT))) 66 return true; 67 68 req->flags &= ~REQ_F_BUFFERS_COMMIT; 69 70 if (unlikely(len < 0)) 71 return true; 72 if (bl->flags & IOBL_INC) 73 return io_kbuf_inc_commit(bl, len); 74 bl->head += nr; 75 return true; 76 } 77 78 static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx, 79 unsigned int bgid) 80 { 81 lockdep_assert_held(&ctx->uring_lock); 82 83 return xa_load(&ctx->io_bl_xa, bgid); 84 } 85 86 static int io_buffer_add_list(struct io_ring_ctx *ctx, 87 struct io_buffer_list *bl, unsigned int bgid) 88 { 89 /* 90 * Store buffer group ID and finally mark the list as visible. 91 * The normal lookup doesn't care about the visibility as we're 92 * always under the ->uring_lock, but lookups from mmap do. 93 */ 94 bl->bgid = bgid; 95 guard(mutex)(&ctx->mmap_lock); 96 return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL)); 97 } 98 99 void io_kbuf_drop_legacy(struct io_kiocb *req) 100 { 101 if (WARN_ON_ONCE(!(req->flags & REQ_F_BUFFER_SELECTED))) 102 return; 103 req->flags &= ~REQ_F_BUFFER_SELECTED; 104 kfree(req->kbuf); 105 req->kbuf = NULL; 106 } 107 108 bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags) 109 { 110 struct io_ring_ctx *ctx = req->ctx; 111 struct io_buffer_list *bl; 112 struct io_buffer *buf; 113 114 io_ring_submit_lock(ctx, issue_flags); 115 116 buf = req->kbuf; 117 bl = io_buffer_get_list(ctx, buf->bgid); 118 /* 119 * If the buffer list was upgraded to a ring-based one, or removed, 120 * while the request was in-flight in io-wq, drop it. 121 */ 122 if (bl && !(bl->flags & IOBL_BUF_RING)) { 123 list_add(&buf->list, &bl->buf_list); 124 bl->nbufs++; 125 } else { 126 kfree(buf); 127 } 128 req->flags &= ~REQ_F_BUFFER_SELECTED; 129 req->kbuf = NULL; 130 131 io_ring_submit_unlock(ctx, issue_flags); 132 return true; 133 } 134 135 static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len, 136 struct io_buffer_list *bl) 137 { 138 if (!list_empty(&bl->buf_list)) { 139 struct io_buffer *kbuf; 140 141 kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list); 142 list_del(&kbuf->list); 143 bl->nbufs--; 144 if (*len == 0 || *len > kbuf->len) 145 *len = kbuf->len; 146 if (list_empty(&bl->buf_list)) 147 req->flags |= REQ_F_BL_EMPTY; 148 req->flags |= REQ_F_BUFFER_SELECTED; 149 req->kbuf = kbuf; 150 req->buf_index = kbuf->bid; 151 return u64_to_user_ptr(kbuf->addr); 152 } 153 return NULL; 154 } 155 156 static int io_provided_buffers_select(struct io_kiocb *req, size_t *len, 157 struct io_buffer_list *bl, 158 struct iovec *iov) 159 { 160 void __user *buf; 161 162 buf = io_provided_buffer_select(req, len, bl); 163 if (unlikely(!buf)) 164 return -ENOBUFS; 165 166 iov[0].iov_base = buf; 167 iov[0].iov_len = *len; 168 return 1; 169 } 170 171 static bool io_should_commit(struct io_kiocb *req, unsigned int issue_flags) 172 { 173 /* 174 * If we came in unlocked, we have no choice but to consume the 175 * buffer here, otherwise nothing ensures that the buffer won't 176 * get used by others. This does mean it'll be pinned until the 177 * IO completes, coming in unlocked means we're being called from 178 * io-wq context and there may be further retries in async hybrid 179 * mode. For the locked case, the caller must call commit when 180 * the transfer completes (or if we get -EAGAIN and must poll of 181 * retry). 182 */ 183 if (issue_flags & IO_URING_F_UNLOCKED) 184 return true; 185 186 /* uring_cmd commits kbuf upfront, no need to auto-commit */ 187 if (!io_file_can_poll(req) && !io_is_uring_cmd(req)) 188 return true; 189 return false; 190 } 191 192 static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len, 193 struct io_buffer_list *bl, 194 unsigned int issue_flags) 195 { 196 struct io_uring_buf_ring *br = bl->buf_ring; 197 __u16 tail, head = bl->head; 198 struct io_br_sel sel = { }; 199 struct io_uring_buf *buf; 200 u32 buf_len; 201 202 tail = smp_load_acquire(&br->tail); 203 if (unlikely(tail == head)) 204 return sel; 205 206 if (head + 1 == tail) 207 req->flags |= REQ_F_BL_EMPTY; 208 209 buf = io_ring_head_to_buf(br, head, bl->mask); 210 buf_len = READ_ONCE(buf->len); 211 if (*len == 0 || *len > buf_len) 212 *len = buf_len; 213 sel.addr = u64_to_user_ptr(READ_ONCE(buf->addr)); 214 if (unlikely(!access_ok(sel.addr, *len))) { 215 sel.addr = NULL; 216 return sel; 217 } 218 req->flags |= REQ_F_BUFFER_RING | REQ_F_BUFFERS_COMMIT; 219 req->buf_index = READ_ONCE(buf->bid); 220 sel.buf_list = bl; 221 222 if (io_should_commit(req, issue_flags)) { 223 if (!io_kbuf_commit(req, sel.buf_list, *len, 1)) 224 req->flags |= REQ_F_BUF_MORE; 225 sel.buf_list = NULL; 226 } 227 return sel; 228 } 229 230 struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len, 231 unsigned buf_group, unsigned int issue_flags) 232 { 233 struct io_ring_ctx *ctx = req->ctx; 234 struct io_br_sel sel = { }; 235 struct io_buffer_list *bl; 236 237 io_ring_submit_lock(ctx, issue_flags); 238 239 bl = io_buffer_get_list(ctx, buf_group); 240 if (likely(bl)) { 241 if (bl->flags & IOBL_BUF_RING) 242 sel = io_ring_buffer_select(req, len, bl, issue_flags); 243 else 244 sel.addr = io_provided_buffer_select(req, len, bl); 245 } 246 io_ring_submit_unlock(ctx, issue_flags); 247 return sel; 248 } 249 250 /* cap it at a reasonable 256, will be one page even for 4K */ 251 #define PEEK_MAX_IMPORT 256 252 253 static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, 254 struct io_buffer_list *bl) 255 { 256 struct io_uring_buf_ring *br = bl->buf_ring; 257 struct iovec *org_iovs = arg->iovs; 258 struct iovec *iov = arg->iovs; 259 int nr_iovs = arg->nr_iovs; 260 __u16 nr_avail, tail, head; 261 struct io_uring_buf *buf; 262 263 tail = smp_load_acquire(&br->tail); 264 head = bl->head; 265 nr_avail = min_t(__u16, tail - head, UIO_MAXIOV); 266 if (unlikely(!nr_avail)) 267 return -ENOBUFS; 268 269 buf = io_ring_head_to_buf(br, head, bl->mask); 270 if (arg->max_len) { 271 u32 len = READ_ONCE(buf->len); 272 size_t needed; 273 274 if (unlikely(!len)) 275 return -ENOBUFS; 276 needed = (arg->max_len + len - 1) / len; 277 needed = min_not_zero(needed, (size_t) PEEK_MAX_IMPORT); 278 if (nr_avail > needed) 279 nr_avail = needed; 280 } 281 282 /* 283 * only alloc a bigger array if we know we have data to map, eg not 284 * a speculative peek operation. 285 */ 286 if (arg->mode & KBUF_MODE_EXPAND && nr_avail > nr_iovs && arg->max_len) { 287 iov = kmalloc_objs(struct iovec, nr_avail); 288 if (unlikely(!iov)) 289 return -ENOMEM; 290 arg->iovs = iov; 291 nr_iovs = nr_avail; 292 } else if (nr_avail < nr_iovs) { 293 nr_iovs = nr_avail; 294 } 295 296 /* set it to max, if not set, so we can use it unconditionally */ 297 if (!arg->max_len) 298 arg->max_len = INT_MAX; 299 300 req->buf_index = READ_ONCE(buf->bid); 301 do { 302 u32 len = READ_ONCE(buf->len); 303 304 /* truncate end piece, if needed, for non partial buffers */ 305 if (len > arg->max_len) { 306 len = arg->max_len; 307 if (!(bl->flags & IOBL_INC)) { 308 arg->partial_map = 1; 309 if (iov != arg->iovs) 310 break; 311 } 312 } 313 314 iov->iov_base = u64_to_user_ptr(READ_ONCE(buf->addr)); 315 iov->iov_len = len; 316 if (unlikely(!access_ok(iov->iov_base, len))) { 317 if (arg->iovs != org_iovs) 318 kfree(arg->iovs); 319 return -EFAULT; 320 } 321 iov++; 322 323 arg->out_len += len; 324 arg->max_len -= len; 325 if (!arg->max_len) 326 break; 327 328 buf = io_ring_head_to_buf(br, ++head, bl->mask); 329 } while (--nr_iovs); 330 331 if (arg->iovs != org_iovs && (arg->mode & KBUF_MODE_FREE)) 332 kfree(org_iovs); 333 334 if (head == tail) 335 req->flags |= REQ_F_BL_EMPTY; 336 337 req->flags |= REQ_F_BUFFER_RING; 338 return iov - arg->iovs; 339 } 340 341 int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg, 342 struct io_br_sel *sel, unsigned int issue_flags) 343 { 344 struct io_ring_ctx *ctx = req->ctx; 345 int ret = -ENOENT; 346 347 io_ring_submit_lock(ctx, issue_flags); 348 sel->buf_list = io_buffer_get_list(ctx, arg->buf_group); 349 if (unlikely(!sel->buf_list)) 350 goto out_unlock; 351 352 if (sel->buf_list->flags & IOBL_BUF_RING) { 353 ret = io_ring_buffers_peek(req, arg, sel->buf_list); 354 /* 355 * Don't recycle these buffers if we need to go through poll. 356 * Nobody else can use them anyway, and holding on to provided 357 * buffers for a send/write operation would happen on the app 358 * side anyway with normal buffers. Besides, we already 359 * committed them, they cannot be put back in the queue. 360 */ 361 if (ret > 0) { 362 req->flags |= REQ_F_BUFFERS_COMMIT | REQ_F_BL_NO_RECYCLE; 363 if (!io_kbuf_commit(req, sel->buf_list, arg->out_len, ret)) 364 req->flags |= REQ_F_BUF_MORE; 365 } 366 } else { 367 ret = io_provided_buffers_select(req, &arg->out_len, sel->buf_list, arg->iovs); 368 } 369 out_unlock: 370 if (issue_flags & IO_URING_F_UNLOCKED) { 371 sel->buf_list = NULL; 372 mutex_unlock(&ctx->uring_lock); 373 } 374 return ret; 375 } 376 377 int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, 378 struct io_br_sel *sel) 379 { 380 struct io_ring_ctx *ctx = req->ctx; 381 struct io_buffer_list *bl; 382 int ret; 383 384 lockdep_assert_held(&ctx->uring_lock); 385 386 bl = io_buffer_get_list(ctx, arg->buf_group); 387 if (unlikely(!bl)) 388 return -ENOENT; 389 390 if (bl->flags & IOBL_BUF_RING) { 391 ret = io_ring_buffers_peek(req, arg, bl); 392 if (ret > 0) 393 req->flags |= REQ_F_BUFFERS_COMMIT; 394 sel->buf_list = bl; 395 return ret; 396 } 397 398 /* don't support multiple buffer selections for legacy */ 399 sel->buf_list = NULL; 400 return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs); 401 } 402 403 static inline bool __io_put_kbuf_ring(struct io_kiocb *req, 404 struct io_buffer_list *bl, int len, int nr) 405 { 406 bool ret = true; 407 408 if (bl) 409 ret = io_kbuf_commit(req, bl, len, nr); 410 if (ret && (req->flags & REQ_F_BUF_MORE)) 411 ret = false; 412 413 req->flags &= ~(REQ_F_BUFFER_RING | REQ_F_BUF_MORE); 414 return ret; 415 } 416 417 unsigned int __io_put_kbufs(struct io_kiocb *req, struct io_buffer_list *bl, 418 int len, int nbufs) 419 { 420 unsigned int ret; 421 422 ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT); 423 424 if (unlikely(!(req->flags & REQ_F_BUFFER_RING))) { 425 io_kbuf_drop_legacy(req); 426 return ret; 427 } 428 429 if (!__io_put_kbuf_ring(req, bl, len, nbufs)) 430 ret |= IORING_CQE_F_BUF_MORE; 431 return ret; 432 } 433 434 static int io_remove_buffers_legacy(struct io_ring_ctx *ctx, 435 struct io_buffer_list *bl, 436 unsigned long nbufs) 437 { 438 unsigned long i = 0; 439 struct io_buffer *nxt; 440 441 /* protects io_buffers_cache */ 442 lockdep_assert_held(&ctx->uring_lock); 443 WARN_ON_ONCE(bl->flags & IOBL_BUF_RING); 444 445 for (i = 0; i < nbufs && !list_empty(&bl->buf_list); i++) { 446 nxt = list_first_entry(&bl->buf_list, struct io_buffer, list); 447 list_del(&nxt->list); 448 bl->nbufs--; 449 kfree(nxt); 450 cond_resched(); 451 } 452 return i; 453 } 454 455 static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) 456 { 457 if (bl->flags & IOBL_BUF_RING) 458 io_free_region(ctx->user, &bl->region); 459 else 460 io_remove_buffers_legacy(ctx, bl, -1U); 461 462 kfree(bl); 463 } 464 465 void io_destroy_buffers(struct io_ring_ctx *ctx) 466 { 467 struct io_buffer_list *bl; 468 469 while (1) { 470 unsigned long index = 0; 471 472 scoped_guard(mutex, &ctx->mmap_lock) { 473 bl = xa_find(&ctx->io_bl_xa, &index, ULONG_MAX, XA_PRESENT); 474 if (bl) 475 xa_erase(&ctx->io_bl_xa, bl->bgid); 476 } 477 if (!bl) 478 break; 479 io_put_bl(ctx, bl); 480 } 481 } 482 483 static void io_destroy_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) 484 { 485 scoped_guard(mutex, &ctx->mmap_lock) 486 WARN_ON_ONCE(xa_erase(&ctx->io_bl_xa, bl->bgid) != bl); 487 io_put_bl(ctx, bl); 488 } 489 490 int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 491 { 492 struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); 493 u64 tmp; 494 495 if (sqe->rw_flags || sqe->addr || sqe->len || sqe->off || 496 sqe->splice_fd_in) 497 return -EINVAL; 498 499 tmp = READ_ONCE(sqe->fd); 500 if (!tmp || tmp > MAX_BIDS_PER_BGID) 501 return -EINVAL; 502 503 memset(p, 0, sizeof(*p)); 504 p->nbufs = tmp; 505 p->bgid = READ_ONCE(sqe->buf_group); 506 return 0; 507 } 508 509 int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 510 { 511 unsigned long size, tmp_check; 512 struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); 513 u64 tmp; 514 515 if (sqe->rw_flags || sqe->splice_fd_in) 516 return -EINVAL; 517 518 tmp = READ_ONCE(sqe->fd); 519 if (!tmp || tmp > MAX_BIDS_PER_BGID) 520 return -E2BIG; 521 p->nbufs = tmp; 522 p->addr = READ_ONCE(sqe->addr); 523 p->len = READ_ONCE(sqe->len); 524 if (!p->len) 525 return -EINVAL; 526 527 if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs, 528 &size)) 529 return -EOVERFLOW; 530 if (check_add_overflow((unsigned long)p->addr, size, &tmp_check)) 531 return -EOVERFLOW; 532 if (!access_ok(u64_to_user_ptr(p->addr), size)) 533 return -EFAULT; 534 535 p->bgid = READ_ONCE(sqe->buf_group); 536 tmp = READ_ONCE(sqe->off); 537 if (tmp > USHRT_MAX) 538 return -E2BIG; 539 if (tmp + p->nbufs > MAX_BIDS_PER_BGID) 540 return -EINVAL; 541 p->bid = tmp; 542 return 0; 543 } 544 545 static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, 546 struct io_buffer_list *bl) 547 { 548 struct io_buffer *buf; 549 u64 addr = pbuf->addr; 550 int ret = -ENOMEM, i, bid = pbuf->bid; 551 552 for (i = 0; i < pbuf->nbufs; i++) { 553 /* 554 * Nonsensical to have more than MAX_BIDS_PER_BGID buffers in a 555 * buffer list, as the application then has no way of knowing 556 * which duplicate bid refers to what buffer. 557 */ 558 if (bl->nbufs == MAX_BIDS_PER_BGID) { 559 ret = -EOVERFLOW; 560 break; 561 } 562 buf = kmalloc_obj(*buf, GFP_KERNEL_ACCOUNT); 563 if (!buf) 564 break; 565 566 list_add_tail(&buf->list, &bl->buf_list); 567 bl->nbufs++; 568 buf->addr = addr; 569 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); 570 buf->bid = bid; 571 buf->bgid = pbuf->bgid; 572 addr += pbuf->len; 573 bid++; 574 cond_resched(); 575 } 576 577 return i ? 0 : ret; 578 } 579 580 static int __io_manage_buffers_legacy(struct io_kiocb *req, 581 struct io_buffer_list *bl) 582 { 583 struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); 584 int ret; 585 586 if (!bl) { 587 if (req->opcode != IORING_OP_PROVIDE_BUFFERS) 588 return -ENOENT; 589 bl = kzalloc_obj(*bl, GFP_KERNEL_ACCOUNT); 590 if (!bl) 591 return -ENOMEM; 592 593 INIT_LIST_HEAD(&bl->buf_list); 594 ret = io_buffer_add_list(req->ctx, bl, p->bgid); 595 if (ret) { 596 kfree(bl); 597 return ret; 598 } 599 } 600 /* can't use provide/remove buffers command on mapped buffers */ 601 if (bl->flags & IOBL_BUF_RING) 602 return -EINVAL; 603 if (req->opcode == IORING_OP_PROVIDE_BUFFERS) 604 return io_add_buffers(req->ctx, p, bl); 605 return io_remove_buffers_legacy(req->ctx, bl, p->nbufs); 606 } 607 608 int io_manage_buffers_legacy(struct io_kiocb *req, unsigned int issue_flags) 609 { 610 struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); 611 struct io_ring_ctx *ctx = req->ctx; 612 struct io_buffer_list *bl; 613 int ret; 614 615 io_ring_submit_lock(ctx, issue_flags); 616 bl = io_buffer_get_list(ctx, p->bgid); 617 ret = __io_manage_buffers_legacy(req, bl); 618 io_ring_submit_unlock(ctx, issue_flags); 619 620 if (ret < 0) 621 req_set_fail(req); 622 io_req_set_res(req, ret, 0); 623 return IOU_COMPLETE; 624 } 625 626 int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) 627 { 628 struct io_uring_buf_reg reg; 629 struct io_buffer_list *bl; 630 struct io_uring_region_desc rd; 631 struct io_uring_buf_ring *br; 632 unsigned long mmap_offset; 633 unsigned long ring_size; 634 int ret; 635 636 lockdep_assert_held(&ctx->uring_lock); 637 638 if (copy_from_user(®, arg, sizeof(reg))) 639 return -EFAULT; 640 if (!mem_is_zero(reg.resv, sizeof(reg.resv))) 641 return -EINVAL; 642 if (reg.flags & ~(IOU_PBUF_RING_MMAP | IOU_PBUF_RING_INC)) 643 return -EINVAL; 644 if (!is_power_of_2(reg.ring_entries)) 645 return -EINVAL; 646 /* cannot disambiguate full vs empty due to head/tail size */ 647 if (reg.ring_entries >= 65536) 648 return -EINVAL; 649 650 /* minimum left byte count is a property of incremental buffers */ 651 if (!(reg.flags & IOU_PBUF_RING_INC) && reg.min_left) 652 return -EINVAL; 653 654 bl = io_buffer_get_list(ctx, reg.bgid); 655 if (bl) { 656 /* if mapped buffer ring OR classic exists, don't allow */ 657 if (bl->flags & IOBL_BUF_RING || !list_empty(&bl->buf_list)) 658 return -EEXIST; 659 io_destroy_bl(ctx, bl); 660 } 661 662 bl = kzalloc_obj(*bl, GFP_KERNEL_ACCOUNT); 663 if (!bl) 664 return -ENOMEM; 665 666 mmap_offset = (unsigned long)reg.bgid << IORING_OFF_PBUF_SHIFT; 667 ring_size = flex_array_size(br, bufs, reg.ring_entries); 668 669 memset(&rd, 0, sizeof(rd)); 670 rd.size = PAGE_ALIGN(ring_size); 671 if (!(reg.flags & IOU_PBUF_RING_MMAP)) { 672 rd.user_addr = reg.ring_addr; 673 rd.flags |= IORING_MEM_REGION_TYPE_USER; 674 } 675 ret = io_create_region(ctx, &bl->region, &rd, mmap_offset); 676 if (ret) 677 goto fail; 678 br = io_region_get_ptr(&bl->region); 679 680 #ifdef SHM_COLOUR 681 /* 682 * On platforms that have specific aliasing requirements, SHM_COLOUR 683 * is set and we must guarantee that the kernel and user side align 684 * nicely. We cannot do that if IOU_PBUF_RING_MMAP isn't set and 685 * the application mmap's the provided ring buffer. Fail the request 686 * if we, by chance, don't end up with aligned addresses. The app 687 * should use IOU_PBUF_RING_MMAP instead, and liburing will handle 688 * this transparently. 689 */ 690 if (!(reg.flags & IOU_PBUF_RING_MMAP) && 691 ((reg.ring_addr | (unsigned long)br) & (SHM_COLOUR - 1))) { 692 ret = -EINVAL; 693 goto fail; 694 } 695 #endif 696 697 bl->mask = reg.ring_entries - 1; 698 bl->flags |= IOBL_BUF_RING; 699 bl->buf_ring = br; 700 if (reg.min_left) 701 bl->min_left_sub_one = reg.min_left - 1; 702 if (reg.flags & IOU_PBUF_RING_INC) 703 bl->flags |= IOBL_INC; 704 ret = io_buffer_add_list(ctx, bl, reg.bgid); 705 if (!ret) 706 return 0; 707 fail: 708 io_free_region(ctx->user, &bl->region); 709 kfree(bl); 710 return ret; 711 } 712 713 int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) 714 { 715 struct io_uring_buf_reg reg; 716 struct io_buffer_list *bl; 717 718 lockdep_assert_held(&ctx->uring_lock); 719 720 if (copy_from_user(®, arg, sizeof(reg))) 721 return -EFAULT; 722 if (!mem_is_zero(reg.resv, sizeof(reg.resv)) || reg.flags) 723 return -EINVAL; 724 725 bl = io_buffer_get_list(ctx, reg.bgid); 726 if (!bl) 727 return -ENOENT; 728 if (!(bl->flags & IOBL_BUF_RING)) 729 return -EINVAL; 730 731 scoped_guard(mutex, &ctx->mmap_lock) 732 xa_erase(&ctx->io_bl_xa, bl->bgid); 733 734 io_put_bl(ctx, bl); 735 return 0; 736 } 737 738 int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg) 739 { 740 struct io_uring_buf_status buf_status; 741 struct io_buffer_list *bl; 742 743 if (copy_from_user(&buf_status, arg, sizeof(buf_status))) 744 return -EFAULT; 745 if (!mem_is_zero(buf_status.resv, sizeof(buf_status.resv))) 746 return -EINVAL; 747 748 bl = io_buffer_get_list(ctx, buf_status.buf_group); 749 if (!bl) 750 return -ENOENT; 751 if (!(bl->flags & IOBL_BUF_RING)) 752 return -EINVAL; 753 754 buf_status.head = bl->head; 755 if (copy_to_user(arg, &buf_status, sizeof(buf_status))) 756 return -EFAULT; 757 758 return 0; 759 } 760 761 struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx, 762 unsigned int bgid) 763 { 764 struct io_buffer_list *bl; 765 766 lockdep_assert_held(&ctx->mmap_lock); 767 768 bl = xa_load(&ctx->io_bl_xa, bgid); 769 if (!bl || !(bl->flags & IOBL_BUF_RING)) 770 return NULL; 771 return &bl->region; 772 } 773