1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/dma-map-ops.h> 5 #include <linux/mm.h> 6 #include <linux/nospec.h> 7 #include <linux/io_uring.h> 8 #include <linux/netdevice.h> 9 #include <linux/rtnetlink.h> 10 #include <linux/skbuff_ref.h> 11 #include <linux/anon_inodes.h> 12 13 #include <net/page_pool/helpers.h> 14 #include <net/page_pool/memory_provider.h> 15 #include <net/netlink.h> 16 #include <net/netdev_queues.h> 17 #include <net/netdev_rx_queue.h> 18 #include <net/tcp.h> 19 #include <net/rps.h> 20 21 #include <trace/events/page_pool.h> 22 23 #include <uapi/linux/io_uring.h> 24 25 #include "io_uring.h" 26 #include "kbuf.h" 27 #include "memmap.h" 28 #include "zcrx.h" 29 #include "rsrc.h" 30 31 #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) 32 33 #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) 34 35 static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp) 36 { 37 return pp->mp_priv; 38 } 39 40 static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *niov) 41 { 42 struct net_iov_area *owner = net_iov_owner(niov); 43 44 return container_of(owner, struct io_zcrx_area, nia); 45 } 46 47 static bool zcrx_set_ring_ctx(struct io_zcrx_ifq *zcrx, 48 struct io_ring_ctx *ctx) 49 { 50 guard(spinlock_bh)(&zcrx->ctx_lock); 51 if (zcrx->master_ctx) 52 return false; 53 percpu_ref_get(&ctx->refs); 54 zcrx->master_ctx = ctx; 55 return true; 56 } 57 58 static inline struct page *io_zcrx_iov_page(const struct net_iov *niov) 59 { 60 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); 61 unsigned niov_pages_shift; 62 63 lockdep_assert(!area->mem.is_dmabuf); 64 65 niov_pages_shift = area->ifq->niov_shift - PAGE_SHIFT; 66 return area->mem.pages[net_iov_idx(niov) << niov_pages_shift]; 67 } 68 69 static int io_area_max_shift(struct io_zcrx_mem *mem) 70 { 71 struct sg_table *sgt = mem->sgt; 72 struct scatterlist *sg; 73 unsigned shift = -1U; 74 unsigned i; 75 76 for_each_sgtable_dma_sg(sgt, sg, i) 77 shift = min(shift, __ffs(sg_dma_len(sg))); 78 return shift; 79 } 80 81 static int io_populate_area_dma(struct io_zcrx_ifq *ifq, 82 struct io_zcrx_area *area) 83 { 84 unsigned niov_size = 1U << ifq->niov_shift; 85 struct sg_table *sgt = area->mem.sgt; 86 struct scatterlist *sg; 87 unsigned i, niov_idx = 0; 88 89 for_each_sgtable_dma_sg(sgt, sg, i) { 90 dma_addr_t dma = sg_dma_address(sg); 91 unsigned long sg_len = sg_dma_len(sg); 92 93 if (WARN_ON_ONCE(sg_len % niov_size)) 94 return -EINVAL; 95 96 while (sg_len && niov_idx < area->nia.num_niovs) { 97 struct net_iov *niov = &area->nia.niovs[niov_idx]; 98 99 if (net_mp_niov_set_dma_addr(niov, dma)) 100 return -EFAULT; 101 sg_len -= niov_size; 102 dma += niov_size; 103 niov_idx++; 104 } 105 } 106 107 if (WARN_ON_ONCE(niov_idx != area->nia.num_niovs)) 108 return -EFAULT; 109 return 0; 110 } 111 112 static void io_release_dmabuf(struct io_zcrx_mem *mem) 113 { 114 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) 115 return; 116 117 if (mem->sgt) 118 dma_buf_unmap_attachment_unlocked(mem->attach, mem->sgt, 119 DMA_FROM_DEVICE); 120 if (mem->attach) 121 dma_buf_detach(mem->dmabuf, mem->attach); 122 if (mem->dmabuf) 123 dma_buf_put(mem->dmabuf); 124 125 mem->sgt = NULL; 126 mem->attach = NULL; 127 mem->dmabuf = NULL; 128 } 129 130 static int io_import_dmabuf(struct io_zcrx_ifq *ifq, 131 struct io_zcrx_mem *mem, 132 struct io_uring_zcrx_area_reg *area_reg) 133 { 134 unsigned long off = (unsigned long)area_reg->addr; 135 unsigned long len = (unsigned long)area_reg->len; 136 unsigned long total_size = 0; 137 struct scatterlist *sg; 138 int dmabuf_fd = area_reg->dmabuf_fd; 139 int i, ret; 140 141 if (!ifq->dev) 142 return -EINVAL; 143 if (off) 144 return -EINVAL; 145 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) 146 return -EINVAL; 147 148 mem->is_dmabuf = true; 149 mem->dmabuf = dma_buf_get(dmabuf_fd); 150 if (IS_ERR(mem->dmabuf)) { 151 ret = PTR_ERR(mem->dmabuf); 152 mem->dmabuf = NULL; 153 goto err; 154 } 155 156 mem->attach = dma_buf_attach(mem->dmabuf, ifq->dev); 157 if (IS_ERR(mem->attach)) { 158 ret = PTR_ERR(mem->attach); 159 mem->attach = NULL; 160 goto err; 161 } 162 163 mem->sgt = dma_buf_map_attachment_unlocked(mem->attach, DMA_FROM_DEVICE); 164 if (IS_ERR(mem->sgt)) { 165 ret = PTR_ERR(mem->sgt); 166 mem->sgt = NULL; 167 goto err; 168 } 169 170 for_each_sgtable_dma_sg(mem->sgt, sg, i) 171 total_size += sg_dma_len(sg); 172 173 if (total_size != len) { 174 ret = -EINVAL; 175 goto err; 176 } 177 178 mem->size = len; 179 return 0; 180 err: 181 io_release_dmabuf(mem); 182 return ret; 183 } 184 185 static unsigned long io_count_account_pages(struct page **pages, unsigned nr_pages) 186 { 187 struct folio *last_folio = NULL; 188 unsigned long res = 0; 189 int i; 190 191 for (i = 0; i < nr_pages; i++) { 192 struct folio *folio = page_folio(pages[i]); 193 194 if (folio == last_folio) 195 continue; 196 last_folio = folio; 197 res += folio_nr_pages(folio); 198 } 199 return res; 200 } 201 202 static int io_import_umem(struct io_zcrx_ifq *ifq, 203 struct io_zcrx_mem *mem, 204 struct io_uring_zcrx_area_reg *area_reg) 205 { 206 struct page **pages; 207 int nr_pages, ret; 208 bool mapped = false; 209 210 if (area_reg->dmabuf_fd) 211 return -EINVAL; 212 if (!area_reg->addr) 213 return -EFAULT; 214 pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len, 215 &nr_pages); 216 if (IS_ERR(pages)) 217 return PTR_ERR(pages); 218 219 ret = sg_alloc_table_from_pages(&mem->page_sg_table, pages, nr_pages, 220 0, (unsigned long)nr_pages << PAGE_SHIFT, 221 GFP_KERNEL_ACCOUNT); 222 if (ret) 223 goto out_err; 224 225 if (ifq->dev) { 226 ret = dma_map_sgtable(ifq->dev, &mem->page_sg_table, 227 DMA_FROM_DEVICE, IO_DMA_ATTR); 228 if (ret < 0) 229 goto out_err; 230 mapped = true; 231 } 232 233 mem->account_pages = io_count_account_pages(pages, nr_pages); 234 ret = io_account_mem(ifq->user, ifq->mm_account, mem->account_pages); 235 if (ret < 0) { 236 mem->account_pages = 0; 237 goto out_err; 238 } 239 240 mem->sgt = &mem->page_sg_table; 241 mem->pages = pages; 242 mem->nr_folios = nr_pages; 243 mem->size = area_reg->len; 244 return ret; 245 out_err: 246 if (mapped) 247 dma_unmap_sgtable(ifq->dev, &mem->page_sg_table, 248 DMA_FROM_DEVICE, IO_DMA_ATTR); 249 sg_free_table(&mem->page_sg_table); 250 unpin_user_pages(pages, nr_pages); 251 kvfree(pages); 252 return ret; 253 } 254 255 static void io_release_area_mem(struct io_zcrx_mem *mem) 256 { 257 if (mem->is_dmabuf) { 258 io_release_dmabuf(mem); 259 } else if (mem->pages) { 260 unpin_user_pages(mem->pages, mem->nr_folios); 261 sg_free_table(mem->sgt); 262 kvfree(mem->pages); 263 } 264 mem->pages = IO_URING_PTR_POISON; 265 mem->sgt = IO_URING_PTR_POISON; 266 } 267 268 static int io_import_area(struct io_zcrx_ifq *ifq, 269 struct io_zcrx_mem *mem, 270 struct io_uring_zcrx_area_reg *area_reg) 271 { 272 int ret; 273 274 if (area_reg->flags & ~IO_ZCRX_AREA_SUPPORTED_FLAGS) 275 return -EINVAL; 276 if (area_reg->rq_area_token) 277 return -EINVAL; 278 if (area_reg->__resv2[0] || area_reg->__resv2[1]) 279 return -EINVAL; 280 281 ret = io_validate_user_buf_range(area_reg->addr, area_reg->len); 282 if (ret) 283 return ret; 284 if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK) 285 return -EINVAL; 286 287 if (area_reg->flags & IORING_ZCRX_AREA_DMABUF) 288 return io_import_dmabuf(ifq, mem, area_reg); 289 return io_import_umem(ifq, mem, area_reg); 290 } 291 292 static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, 293 struct io_zcrx_area *area) 294 { 295 int i; 296 297 guard(mutex)(&ifq->pp_lock); 298 if (!area->is_mapped) 299 return; 300 area->is_mapped = false; 301 302 if (area->nia.niovs) { 303 for (i = 0; i < area->nia.num_niovs; i++) 304 net_mp_niov_set_dma_addr(&area->nia.niovs[i], 0); 305 } 306 307 if (area->mem.is_dmabuf) { 308 io_release_dmabuf(&area->mem); 309 } else { 310 dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table, 311 DMA_FROM_DEVICE, IO_DMA_ATTR); 312 } 313 } 314 315 static void zcrx_sync_for_device(struct page_pool *pp, struct io_zcrx_ifq *zcrx, 316 netmem_ref *netmems, unsigned nr) 317 { 318 #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) 319 struct device *dev = pp->p.dev; 320 unsigned i, niov_size; 321 dma_addr_t dma_addr; 322 323 if (!dma_dev_need_sync(dev)) 324 return; 325 niov_size = 1U << zcrx->niov_shift; 326 327 for (i = 0; i < nr; i++) { 328 dma_addr = page_pool_get_dma_addr_netmem(netmems[i]); 329 __dma_sync_single_for_device(dev, dma_addr + pp->p.offset, 330 niov_size, pp->p.dma_dir); 331 } 332 #endif 333 } 334 335 #define IO_RQ_MAX_ENTRIES 32768 336 337 #define IO_SKBS_PER_CALL_LIMIT 20 338 339 struct io_zcrx_args { 340 struct io_kiocb *req; 341 struct io_zcrx_ifq *ifq; 342 unsigned nr_skbs; 343 }; 344 345 static const struct memory_provider_ops io_uring_pp_zc_ops; 346 347 static inline atomic_t *io_get_user_counter(struct net_iov *niov) 348 { 349 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); 350 351 return &area->user_refs[net_iov_idx(niov)]; 352 } 353 354 static bool io_zcrx_put_niov_uref(struct net_iov *niov) 355 { 356 atomic_t *uref = io_get_user_counter(niov); 357 int old; 358 359 old = atomic_read(uref); 360 do { 361 if (unlikely(old == 0)) 362 return false; 363 } while (!atomic_try_cmpxchg(uref, &old, old - 1)); 364 365 return true; 366 } 367 368 static void io_zcrx_get_niov_uref(struct net_iov *niov) 369 { 370 atomic_inc(io_get_user_counter(niov)); 371 } 372 373 static void io_fill_zcrx_offsets(struct io_uring_zcrx_offsets *offsets) 374 { 375 offsets->head = offsetof(struct io_uring, head); 376 offsets->tail = offsetof(struct io_uring, tail); 377 offsets->rqes = ALIGN(sizeof(struct io_uring), L1_CACHE_BYTES); 378 } 379 380 static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, 381 struct io_zcrx_ifq *ifq, 382 struct io_uring_zcrx_ifq_reg *reg, 383 struct io_uring_region_desc *rd, 384 u32 id) 385 { 386 u64 mmap_offset; 387 size_t off, size; 388 void *ptr; 389 int ret; 390 391 io_fill_zcrx_offsets(®->offsets); 392 off = reg->offsets.rqes; 393 size = off + sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries; 394 if (size > rd->size) 395 return -EINVAL; 396 397 mmap_offset = IORING_MAP_OFF_ZCRX_REGION; 398 mmap_offset += (u64)id << IORING_OFF_ZCRX_SHIFT; 399 400 ret = io_create_region(ctx, &ifq->rq_region, rd, mmap_offset); 401 if (ret < 0) 402 return ret; 403 404 ptr = io_region_get_ptr(&ifq->rq_region); 405 ifq->rq.ring = (struct io_uring *)ptr; 406 ifq->rq.rqes = (struct io_uring_zcrx_rqe *)(ptr + off); 407 408 memset(ifq->rq.ring, 0, sizeof(*ifq->rq.ring)); 409 return 0; 410 } 411 412 static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq) 413 { 414 io_free_region(ifq->user, &ifq->rq_region); 415 ifq->rq.ring = IO_URING_PTR_POISON; 416 ifq->rq.rqes = IO_URING_PTR_POISON; 417 ifq->notif_stats = IO_URING_PTR_POISON; 418 } 419 420 static void io_zcrx_free_area(struct io_zcrx_ifq *ifq, 421 struct io_zcrx_area *area) 422 { 423 io_zcrx_unmap_area(ifq, area); 424 io_release_area_mem(&area->mem); 425 426 if (area->mem.account_pages) 427 io_unaccount_mem(ifq->user, ifq->mm_account, 428 area->mem.account_pages); 429 430 kvfree(area->freelist); 431 kvfree(area->nia.niovs); 432 kvfree(area->user_refs); 433 kfree(area); 434 } 435 436 static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, 437 struct io_zcrx_area *area) 438 { 439 bool kern_readable = !area->mem.is_dmabuf; 440 441 if (WARN_ON_ONCE(ifq->area)) 442 return -EINVAL; 443 if (WARN_ON_ONCE(ifq->kern_readable != kern_readable)) 444 return -EINVAL; 445 446 ifq->area = area; 447 return 0; 448 } 449 450 static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, 451 struct io_uring_zcrx_area_reg *area_reg, 452 struct io_uring_zcrx_ifq_reg *reg) 453 { 454 int buf_size_shift = PAGE_SHIFT; 455 struct io_zcrx_area *area; 456 unsigned nr_iovs; 457 int i, ret; 458 459 if (reg->rx_buf_len) { 460 if (!is_power_of_2(reg->rx_buf_len) || 461 reg->rx_buf_len < PAGE_SIZE) 462 return -EINVAL; 463 buf_size_shift = ilog2(reg->rx_buf_len); 464 } 465 if (!ifq->dev && buf_size_shift != PAGE_SHIFT) 466 return -EOPNOTSUPP; 467 468 ret = -ENOMEM; 469 area = kzalloc_obj(*area); 470 if (!area) 471 goto err; 472 area->ifq = ifq; 473 474 ret = io_import_area(ifq, &area->mem, area_reg); 475 if (ret) 476 goto err; 477 if (ifq->dev) 478 area->is_mapped = true; 479 480 if (ifq->dev && buf_size_shift > io_area_max_shift(&area->mem)) { 481 ret = -ERANGE; 482 goto err; 483 } 484 485 ifq->niov_shift = buf_size_shift; 486 nr_iovs = area->mem.size >> ifq->niov_shift; 487 area->nia.num_niovs = nr_iovs; 488 489 ret = -ENOMEM; 490 area->nia.niovs = kvmalloc_objs(area->nia.niovs[0], nr_iovs, 491 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 492 if (!area->nia.niovs) 493 goto err; 494 495 area->freelist = kvmalloc_array(nr_iovs, sizeof(area->freelist[0]), 496 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 497 if (!area->freelist) 498 goto err; 499 500 area->user_refs = kvmalloc_objs(area->user_refs[0], nr_iovs, 501 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 502 if (!area->user_refs) 503 goto err; 504 505 for (i = 0; i < nr_iovs; i++) { 506 struct net_iov *niov = &area->nia.niovs[i]; 507 508 net_iov_init(niov, &area->nia, NET_IOV_IOURING); 509 area->freelist[i] = i; 510 atomic_set(&area->user_refs[i], 0); 511 } 512 513 if (ifq->dev) { 514 ret = io_populate_area_dma(ifq, area); 515 if (ret) 516 goto err; 517 } 518 519 area->free_count = nr_iovs; 520 /* we're only supporting one area per ifq for now */ 521 area->area_id = 0; 522 area_reg->rq_area_token = (u64)area->area_id << IORING_ZCRX_AREA_SHIFT; 523 spin_lock_init(&area->freelist_lock); 524 525 ret = io_zcrx_append_area(ifq, area); 526 if (!ret) 527 return 0; 528 err: 529 if (area) 530 io_zcrx_free_area(ifq, area); 531 return ret; 532 } 533 534 static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) 535 { 536 struct io_zcrx_ifq *ifq; 537 538 ifq = kzalloc_obj(*ifq); 539 if (!ifq) 540 return NULL; 541 542 ifq->if_rxq = -1; 543 spin_lock_init(&ifq->ctx_lock); 544 spin_lock_init(&ifq->rq.lock); 545 mutex_init(&ifq->pp_lock); 546 refcount_set(&ifq->refs, 1); 547 refcount_set(&ifq->user_refs, 1); 548 return ifq; 549 } 550 551 static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq) 552 { 553 guard(mutex)(&ifq->pp_lock); 554 555 if (!ifq->netdev) 556 return; 557 netdev_put(ifq->netdev, &ifq->netdev_tracker); 558 ifq->netdev = NULL; 559 } 560 561 static void io_close_queue(struct io_zcrx_ifq *ifq) 562 { 563 struct net_device *netdev; 564 netdevice_tracker netdev_tracker; 565 struct pp_memory_provider_params p = { 566 .mp_ops = &io_uring_pp_zc_ops, 567 .mp_priv = ifq, 568 }; 569 570 scoped_guard(mutex, &ifq->pp_lock) { 571 netdev = ifq->netdev; 572 netdev_tracker = ifq->netdev_tracker; 573 ifq->netdev = NULL; 574 } 575 576 if (netdev) { 577 if (ifq->if_rxq != -1) { 578 netdev_lock(netdev); 579 netif_mp_close_rxq(netdev, ifq->if_rxq, &p); 580 netdev_unlock(netdev); 581 } 582 netdev_put(netdev, &netdev_tracker); 583 } 584 ifq->if_rxq = -1; 585 } 586 587 static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) 588 { 589 if (WARN_ON_ONCE(ifq->if_rxq != -1)) 590 return; 591 if (WARN_ON_ONCE(ifq->netdev != NULL)) 592 return; 593 if (WARN_ON_ONCE(ifq->master_ctx)) 594 return; 595 596 if (ifq->area) 597 io_zcrx_free_area(ifq, ifq->area); 598 if (ifq->mm_account) 599 mmdrop(ifq->mm_account); 600 if (ifq->dev) 601 put_device(ifq->dev); 602 603 io_free_rbuf_ring(ifq); 604 free_uid(ifq->user); 605 mutex_destroy(&ifq->pp_lock); 606 kfree(ifq); 607 } 608 609 static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq) 610 { 611 if (refcount_dec_and_test(&ifq->refs)) 612 io_zcrx_ifq_free(ifq); 613 } 614 615 static void io_zcrx_return_niov_freelist(struct net_iov *niov) 616 { 617 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); 618 619 guard(spinlock_bh)(&area->freelist_lock); 620 if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs)) 621 return; 622 area->freelist[area->free_count++] = net_iov_idx(niov); 623 } 624 625 static struct net_iov *zcrx_get_free_niov(struct io_zcrx_area *area) 626 { 627 unsigned niov_idx; 628 629 lockdep_assert_held(&area->freelist_lock); 630 631 if (unlikely(!area->free_count)) 632 return NULL; 633 634 niov_idx = area->freelist[--area->free_count]; 635 return &area->nia.niovs[niov_idx]; 636 } 637 638 static void io_zcrx_return_niov(struct net_iov *niov) 639 { 640 netmem_ref netmem = net_iov_to_netmem(niov); 641 642 if (!niov->desc.pp) { 643 /* copy fallback allocated niovs */ 644 io_zcrx_return_niov_freelist(niov); 645 return; 646 } 647 page_pool_put_unrefed_netmem(niov->desc.pp, netmem, -1, false); 648 } 649 650 static void io_zcrx_scrub(struct io_zcrx_ifq *ifq) 651 { 652 struct io_zcrx_area *area = ifq->area; 653 int i; 654 655 if (!area) 656 return; 657 658 /* Reclaim back all buffers given to the user space. */ 659 for (i = 0; i < area->nia.num_niovs; i++) { 660 struct net_iov *niov = &area->nia.niovs[i]; 661 int nr; 662 663 if (!atomic_read(io_get_user_counter(niov))) 664 continue; 665 nr = atomic_xchg(io_get_user_counter(niov), 0); 666 if (nr && !page_pool_unref_netmem(net_iov_to_netmem(niov), nr)) 667 io_zcrx_return_niov(niov); 668 } 669 } 670 671 static void zcrx_unregister_user(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) 672 { 673 scoped_guard(spinlock_bh, &ifq->ctx_lock) { 674 if (ctx && ifq->master_ctx == ctx) { 675 ifq->master_ctx = NULL; 676 percpu_ref_put(&ctx->refs); 677 } 678 } 679 680 if (refcount_dec_and_test(&ifq->user_refs)) { 681 io_close_queue(ifq); 682 io_zcrx_scrub(ifq); 683 } 684 } 685 686 static void zcrx_unregister(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) 687 { 688 zcrx_unregister_user(ifq, ctx); 689 io_put_zcrx_ifq(ifq); 690 } 691 692 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 693 unsigned int id) 694 { 695 struct io_zcrx_ifq *ifq = xa_load(&ctx->zcrx_ctxs, id); 696 697 lockdep_assert_held(&ctx->mmap_lock); 698 699 return ifq ? &ifq->rq_region : NULL; 700 } 701 702 static int zcrx_box_release(struct inode *inode, struct file *file) 703 { 704 struct io_zcrx_ifq *ifq = file->private_data; 705 706 if (WARN_ON_ONCE(!ifq)) 707 return -EFAULT; 708 zcrx_unregister(ifq, NULL); 709 return 0; 710 } 711 712 static const struct file_operations zcrx_box_fops = { 713 .owner = THIS_MODULE, 714 .release = zcrx_box_release, 715 }; 716 717 static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, 718 struct zcrx_ctrl *ctrl, void __user *arg) 719 { 720 struct zcrx_ctrl_export *ce = &ctrl->zc_export; 721 struct file *file; 722 int fd; 723 724 if (!mem_is_zero(ce, sizeof(*ce))) 725 return -EINVAL; 726 727 refcount_inc(&ifq->refs); 728 refcount_inc(&ifq->user_refs); 729 730 file = anon_inode_create_getfile("[zcrx]", &zcrx_box_fops, 731 ifq, O_CLOEXEC, NULL); 732 if (IS_ERR(file)) { 733 zcrx_unregister(ifq, NULL); 734 return PTR_ERR(file); 735 } 736 737 fd = get_unused_fd_flags(O_CLOEXEC); 738 if (fd < 0) { 739 fput(file); 740 return fd; 741 } 742 743 ce->zcrx_fd = fd; 744 if (copy_to_user(arg, ctrl, sizeof(*ctrl))) { 745 fput(file); 746 put_unused_fd(fd); 747 return -EFAULT; 748 } 749 750 fd_install(fd, file); 751 return 0; 752 } 753 754 static int import_zcrx(struct io_ring_ctx *ctx, 755 struct io_uring_zcrx_ifq_reg __user *arg, 756 struct io_uring_zcrx_ifq_reg *reg) 757 { 758 struct io_zcrx_ifq *ifq; 759 struct file *file; 760 int fd, ret; 761 u32 id; 762 763 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 764 return -EINVAL; 765 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))) 766 return -EINVAL; 767 if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr) 768 return -EINVAL; 769 if (reg->event_desc) 770 return -EINVAL; 771 if (reg->flags & ~ZCRX_REG_IMPORT) 772 return -EINVAL; 773 774 fd = reg->if_idx; 775 CLASS(fd, f)(fd); 776 if (fd_empty(f)) 777 return -EBADF; 778 779 file = fd_file(f); 780 if (file->f_op != &zcrx_box_fops || !file->private_data) 781 return -EBADF; 782 783 ifq = file->private_data; 784 refcount_inc(&ifq->refs); 785 refcount_inc(&ifq->user_refs); 786 787 scoped_guard(mutex, &ctx->mmap_lock) { 788 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL); 789 if (ret) 790 goto err; 791 } 792 793 reg->zcrx_id = id; 794 io_fill_zcrx_offsets(®->offsets); 795 if (copy_to_user(arg, reg, sizeof(*reg))) { 796 ret = -EFAULT; 797 goto err_xa_erase; 798 } 799 800 scoped_guard(mutex, &ctx->mmap_lock) { 801 ret = -ENOMEM; 802 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL)) 803 goto err_xa_erase; 804 } 805 806 return 0; 807 err_xa_erase: 808 scoped_guard(mutex, &ctx->mmap_lock) 809 xa_erase(&ctx->zcrx_ctxs, id); 810 err: 811 /* the import path never set the ->master_ctx ref, don't drop it */ 812 zcrx_unregister(ifq, NULL); 813 return ret; 814 } 815 816 static int zcrx_register_netdev(struct io_zcrx_ifq *ifq, 817 struct io_uring_zcrx_ifq_reg *reg, 818 struct io_uring_zcrx_area_reg *area) 819 { 820 struct pp_memory_provider_params mp_param = {}; 821 unsigned if_rxq = reg->if_rxq; 822 int ret; 823 824 ifq->netdev = netdev_get_by_index_lock(current->nsproxy->net_ns, 825 reg->if_idx); 826 if (!ifq->netdev) 827 return -ENODEV; 828 829 netdev_hold(ifq->netdev, &ifq->netdev_tracker, GFP_KERNEL); 830 831 ifq->dev = netdev_queue_get_dma_dev(ifq->netdev, if_rxq, NETDEV_QUEUE_TYPE_RX); 832 if (!ifq->dev) { 833 ret = -EOPNOTSUPP; 834 goto netdev_put_unlock; 835 } 836 get_device(ifq->dev); 837 838 ret = io_zcrx_create_area(ifq, area, reg); 839 if (ret) 840 goto netdev_put_unlock; 841 842 if (reg->rx_buf_len) 843 mp_param.rx_page_size = 1U << ifq->niov_shift; 844 mp_param.mp_ops = &io_uring_pp_zc_ops; 845 mp_param.mp_priv = ifq; 846 ret = netif_mp_open_rxq(ifq->netdev, if_rxq, &mp_param, NULL); 847 if (ret) 848 goto netdev_put_unlock; 849 850 ifq->if_rxq = if_rxq; 851 ret = 0; 852 netdev_put_unlock: 853 netdev_unlock(ifq->netdev); 854 return ret; 855 } 856 857 static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq, 858 const struct io_uring_zcrx_ifq_reg *reg, 859 const struct zcrx_event_desc *notif) 860 { 861 size_t stats_off = notif->stats_offset; 862 size_t used, end; 863 864 used = reg->offsets.rqes + 865 sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries; 866 867 if (!IS_ALIGNED(stats_off, __alignof__(struct zcrx_stats))) 868 return -EINVAL; 869 if (stats_off < used) 870 return -ERANGE; 871 if (check_add_overflow(stats_off, 872 sizeof(struct zcrx_stats), 873 &end)) 874 return -ERANGE; 875 if (end > io_region_size(&ifq->rq_region)) 876 return -ERANGE; 877 878 ifq->notif_stats = io_region_get_ptr(&ifq->rq_region) + stats_off; 879 memset(ifq->notif_stats, 0, sizeof(*ifq->notif_stats)); 880 881 return 0; 882 } 883 884 int io_register_zcrx(struct io_ring_ctx *ctx, 885 struct io_uring_zcrx_ifq_reg __user *arg) 886 { 887 struct zcrx_event_desc notif; 888 struct io_uring_zcrx_area_reg area; 889 struct io_uring_zcrx_ifq_reg reg; 890 struct io_uring_region_desc rd; 891 struct io_zcrx_ifq *ifq; 892 int ret; 893 u32 id; 894 895 /* 896 * 1. Interface queue allocation. 897 * 2. It can observe data destined for sockets of other tasks. 898 */ 899 if (!capable(CAP_NET_ADMIN)) 900 return -EPERM; 901 902 /* mandatory io_uring features for zc rx */ 903 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 904 return -EINVAL; 905 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))) 906 return -EINVAL; 907 if (copy_from_user(®, arg, sizeof(reg))) 908 return -EFAULT; 909 if (!mem_is_zero(®.__resv, sizeof(reg.__resv)) || reg.zcrx_id) 910 return -EINVAL; 911 if (reg.flags & ~ZCRX_SUPPORTED_REG_FLAGS) 912 return -EINVAL; 913 if (reg.flags & ZCRX_REG_IMPORT) 914 return import_zcrx(ctx, arg, ®); 915 if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd))) 916 return -EFAULT; 917 if (reg.if_rxq == -1 || !reg.rq_entries) 918 return -EINVAL; 919 if ((reg.if_rxq || reg.if_idx) && (reg.flags & ZCRX_REG_NODEV)) 920 return -EINVAL; 921 if (reg.rq_entries > IO_RQ_MAX_ENTRIES) { 922 if (!(ctx->flags & IORING_SETUP_CLAMP)) 923 return -EINVAL; 924 reg.rq_entries = IO_RQ_MAX_ENTRIES; 925 } 926 reg.rq_entries = roundup_pow_of_two(reg.rq_entries); 927 928 if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area))) 929 return -EFAULT; 930 931 memset(¬if, 0, sizeof(notif)); 932 if (reg.event_desc && copy_from_user(¬if, u64_to_user_ptr(reg.event_desc), 933 sizeof(notif))) 934 return -EFAULT; 935 if (notif.type_mask & ~ZCRX_EVENT_TYPE_MASK) 936 return -EINVAL; 937 if (notif.flags & ~ZCRX_EVENT_DESC_FLAG_STATS) 938 return -EINVAL; 939 if (!(notif.flags & ZCRX_EVENT_DESC_FLAG_STATS)) { 940 if (notif.stats_offset) 941 return -EINVAL; 942 } 943 if (!mem_is_zero(¬if.__resv2, sizeof(notif.__resv2))) 944 return -EINVAL; 945 946 ifq = io_zcrx_ifq_alloc(ctx); 947 if (!ifq) 948 return -ENOMEM; 949 950 ifq->notif_data = notif.user_data; 951 ifq->allowed_notif_mask = notif.type_mask; 952 953 if (ctx->user) { 954 get_uid(ctx->user); 955 ifq->user = ctx->user; 956 } 957 if (ctx->mm_account) { 958 mmgrab(ctx->mm_account); 959 ifq->mm_account = ctx->mm_account; 960 } 961 ifq->rq.nr_entries = reg.rq_entries; 962 963 scoped_guard(mutex, &ctx->mmap_lock) { 964 /* preallocate id */ 965 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL); 966 if (ret) 967 goto ifq_free; 968 } 969 970 ret = io_allocate_rbuf_ring(ctx, ifq, ®, &rd, id); 971 if (ret) 972 goto err; 973 974 if (notif.flags & ZCRX_EVENT_DESC_FLAG_STATS) { 975 ret = zcrx_validate_notif_stats(ifq, ®, ¬if); 976 if (ret) 977 goto err; 978 } 979 980 ifq->kern_readable = !(area.flags & IORING_ZCRX_AREA_DMABUF); 981 982 if (!(reg.flags & ZCRX_REG_NODEV)) { 983 ret = zcrx_register_netdev(ifq, ®, &area); 984 if (ret) 985 goto err; 986 } else { 987 ret = io_zcrx_create_area(ifq, &area, ®); 988 if (ret) 989 goto err; 990 } 991 992 reg.zcrx_id = id; 993 994 scoped_guard(mutex, &ctx->mmap_lock) { 995 /* publish ifq */ 996 ret = -ENOMEM; 997 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL)) 998 goto err; 999 } 1000 1001 reg.rx_buf_len = 1U << ifq->niov_shift; 1002 1003 if (copy_to_user(arg, ®, sizeof(reg)) || 1004 copy_to_user(u64_to_user_ptr(reg.region_ptr), &rd, sizeof(rd)) || 1005 copy_to_user(u64_to_user_ptr(reg.area_ptr), &area, sizeof(area))) { 1006 ret = -EFAULT; 1007 goto err; 1008 } 1009 1010 if (notif.type_mask) 1011 zcrx_set_ring_ctx(ifq, ctx); 1012 return 0; 1013 err: 1014 scoped_guard(mutex, &ctx->mmap_lock) 1015 xa_erase(&ctx->zcrx_ctxs, id); 1016 ifq_free: 1017 zcrx_unregister(ifq, ctx); 1018 return ret; 1019 } 1020 1021 static inline bool is_zcrx_entry_marked(struct io_ring_ctx *ctx, unsigned long id) 1022 { 1023 return xa_get_mark(&ctx->zcrx_ctxs, id, XA_MARK_1); 1024 } 1025 1026 static inline void set_zcrx_entry_mark(struct io_ring_ctx *ctx, unsigned long id) 1027 { 1028 xa_set_mark(&ctx->zcrx_ctxs, id, XA_MARK_1); 1029 } 1030 1031 void io_terminate_zcrx(struct io_ring_ctx *ctx) 1032 { 1033 struct io_zcrx_ifq *ifq; 1034 unsigned long id = 0; 1035 1036 lockdep_assert_held(&ctx->uring_lock); 1037 1038 while (1) { 1039 scoped_guard(mutex, &ctx->mmap_lock) 1040 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); 1041 if (!ifq) 1042 break; 1043 if (WARN_ON_ONCE(is_zcrx_entry_marked(ctx, id))) 1044 break; 1045 set_zcrx_entry_mark(ctx, id); 1046 id++; 1047 zcrx_unregister_user(ifq, ctx); 1048 } 1049 } 1050 1051 void io_unregister_zcrx(struct io_ring_ctx *ctx) 1052 { 1053 struct io_zcrx_ifq *ifq; 1054 1055 lockdep_assert_held(&ctx->uring_lock); 1056 1057 while (1) { 1058 scoped_guard(mutex, &ctx->mmap_lock) { 1059 unsigned long id = 0; 1060 1061 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); 1062 if (ifq) { 1063 if (WARN_ON_ONCE(!is_zcrx_entry_marked(ctx, id))) { 1064 ifq = NULL; 1065 break; 1066 } 1067 xa_erase(&ctx->zcrx_ctxs, id); 1068 } 1069 } 1070 if (!ifq) 1071 break; 1072 /* 1073 * io_uring can run requests and return buffers to the user 1074 * after termination, scrub it again. 1075 */ 1076 if (refcount_read(&ifq->user_refs) == 0) 1077 io_zcrx_scrub(ifq); 1078 io_put_zcrx_ifq(ifq); 1079 } 1080 1081 xa_destroy(&ctx->zcrx_ctxs); 1082 } 1083 1084 static inline u32 zcrx_rq_entries(struct zcrx_rq *rq) 1085 { 1086 u32 entries; 1087 1088 entries = smp_load_acquire(&rq->ring->tail) - rq->cached_head; 1089 return min(entries, rq->nr_entries); 1090 } 1091 1092 static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask) 1093 { 1094 unsigned int idx = rq->cached_head++ & mask; 1095 1096 return &rq->rqes[idx]; 1097 } 1098 1099 static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, 1100 struct io_zcrx_ifq *ifq, 1101 struct net_iov **ret_niov) 1102 { 1103 __u64 off = READ_ONCE(rqe->off); 1104 unsigned niov_idx, area_idx; 1105 struct io_zcrx_area *area; 1106 1107 area_idx = off >> IORING_ZCRX_AREA_SHIFT; 1108 niov_idx = (off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; 1109 1110 if (unlikely(rqe->__pad || area_idx)) 1111 return false; 1112 area = ifq->area; 1113 1114 if (unlikely(niov_idx >= area->nia.num_niovs)) 1115 return false; 1116 niov_idx = array_index_nospec(niov_idx, area->nia.num_niovs); 1117 1118 *ret_niov = &area->nia.niovs[niov_idx]; 1119 return true; 1120 } 1121 1122 static unsigned io_zcrx_ring_refill(struct page_pool *pp, 1123 struct io_zcrx_ifq *ifq, 1124 netmem_ref *netmems, unsigned to_alloc) 1125 { 1126 struct zcrx_rq *rq = &ifq->rq; 1127 unsigned int mask = rq->nr_entries - 1; 1128 unsigned int entries; 1129 unsigned allocated = 0; 1130 1131 guard(spinlock_bh)(&rq->lock); 1132 1133 entries = zcrx_rq_entries(rq); 1134 entries = min_t(unsigned, entries, to_alloc); 1135 if (unlikely(!entries)) 1136 return 0; 1137 1138 do { 1139 struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask); 1140 struct net_iov *niov; 1141 netmem_ref netmem; 1142 1143 if (!io_parse_rqe(rqe, ifq, &niov)) 1144 continue; 1145 if (!io_zcrx_put_niov_uref(niov)) 1146 continue; 1147 1148 netmem = net_iov_to_netmem(niov); 1149 if (!page_pool_unref_and_test(netmem)) 1150 continue; 1151 1152 if (unlikely(niov->desc.pp != pp)) { 1153 io_zcrx_return_niov(niov); 1154 continue; 1155 } 1156 1157 netmems[allocated] = netmem; 1158 allocated++; 1159 } while (--entries); 1160 1161 smp_store_release(&rq->ring->head, rq->cached_head); 1162 return allocated; 1163 } 1164 1165 static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq, 1166 netmem_ref *netmems, unsigned to_alloc) 1167 { 1168 struct io_zcrx_area *area = ifq->area; 1169 unsigned allocated = 0; 1170 1171 guard(spinlock_bh)(&area->freelist_lock); 1172 1173 for (allocated = 0; allocated < to_alloc; allocated++) { 1174 struct net_iov *niov = zcrx_get_free_niov(area); 1175 1176 if (!niov) 1177 break; 1178 net_mp_niov_set_page_pool(pp, niov); 1179 netmems[allocated] = net_iov_to_netmem(niov); 1180 } 1181 return allocated; 1182 } 1183 1184 static void zcrx_notif_tw(struct io_tw_req tw_req, io_tw_token_t tw) 1185 { 1186 struct io_kiocb *req = tw_req.req; 1187 struct io_ring_ctx *ctx = req->ctx; 1188 1189 io_post_aux_cqe(ctx, req->cqe.user_data, req->cqe.res, 0); 1190 percpu_ref_put(&ctx->refs); 1191 io_poison_req(req); 1192 kmem_cache_free(req_cachep, req); 1193 } 1194 1195 static void zcrx_stat_add(__u64 *p, s64 v) 1196 { 1197 WRITE_ONCE(*p, READ_ONCE(*p) + v); 1198 } 1199 1200 static void zcrx_send_notif(struct io_zcrx_ifq *ifq, unsigned type) 1201 { 1202 gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO; 1203 u32 type_mask = 1 << type; 1204 struct io_kiocb *req; 1205 1206 if (!(type_mask & ifq->allowed_notif_mask)) 1207 return; 1208 1209 guard(spinlock_bh)(&ifq->ctx_lock); 1210 if (!ifq->master_ctx) 1211 return; 1212 if (type_mask & ifq->fired_notifs) 1213 return; 1214 1215 req = kmem_cache_alloc(req_cachep, gfp); 1216 if (unlikely(!req)) 1217 return; 1218 1219 ifq->fired_notifs |= type_mask; 1220 1221 req->opcode = IORING_OP_NOP; 1222 req->cqe.user_data = ifq->notif_data; 1223 req->cqe.res = type; 1224 req->ctx = ifq->master_ctx; 1225 percpu_ref_get(&req->ctx->refs); 1226 req->tctx = NULL; 1227 req->io_task_work.func = zcrx_notif_tw; 1228 io_req_task_work_add(req); 1229 } 1230 1231 static netmem_ref io_pp_zc_alloc_netmems(struct page_pool *pp, gfp_t gfp) 1232 { 1233 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp); 1234 netmem_ref *netmems = pp->alloc.cache; 1235 unsigned to_alloc = PP_ALLOC_CACHE_REFILL; 1236 unsigned allocated; 1237 1238 /* pp should already be ensuring that */ 1239 if (WARN_ON_ONCE(pp->alloc.count)) 1240 return 0; 1241 1242 allocated = io_zcrx_ring_refill(pp, ifq, netmems, to_alloc); 1243 if (likely(allocated)) 1244 goto out_return; 1245 1246 allocated = io_zcrx_refill_slow(pp, ifq, netmems, to_alloc); 1247 if (!allocated) { 1248 zcrx_send_notif(ifq, ZCRX_EVENT_ALLOC_FAIL); 1249 return 0; 1250 } 1251 out_return: 1252 zcrx_sync_for_device(pp, ifq, netmems, allocated); 1253 allocated--; 1254 pp->alloc.count += allocated; 1255 return netmems[allocated]; 1256 } 1257 1258 static bool io_pp_zc_release_netmem(struct page_pool *pp, netmem_ref netmem) 1259 { 1260 struct net_iov *niov; 1261 1262 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 1263 return false; 1264 1265 niov = netmem_to_net_iov(netmem); 1266 net_mp_niov_clear_page_pool(niov); 1267 io_zcrx_return_niov_freelist(niov); 1268 return false; 1269 } 1270 1271 static int io_pp_zc_init(struct page_pool *pp) 1272 { 1273 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp); 1274 1275 if (WARN_ON_ONCE(!ifq)) 1276 return -EINVAL; 1277 if (WARN_ON_ONCE(ifq->dev != pp->p.dev)) 1278 return -EINVAL; 1279 if (WARN_ON_ONCE(!pp->dma_map)) 1280 return -EOPNOTSUPP; 1281 if (pp->p.order + PAGE_SHIFT != ifq->niov_shift) 1282 return -EINVAL; 1283 if (pp->p.dma_dir != DMA_FROM_DEVICE) 1284 return -EOPNOTSUPP; 1285 1286 refcount_inc(&ifq->refs); 1287 return 0; 1288 } 1289 1290 static void io_pp_zc_destroy(struct page_pool *pp) 1291 { 1292 io_put_zcrx_ifq(io_pp_to_ifq(pp)); 1293 } 1294 1295 static int io_pp_nl_fill(void *mp_priv, struct sk_buff *rsp, 1296 struct netdev_rx_queue *rxq) 1297 { 1298 struct io_zcrx_ifq *ifq = mp_priv; 1299 struct nlattr *nest; 1300 int type; 1301 1302 type = rxq ? NETDEV_A_QUEUE_IO_URING : NETDEV_A_PAGE_POOL_IO_URING; 1303 nest = nla_nest_start(rsp, type); 1304 if (!nest) 1305 return -EMSGSIZE; 1306 1307 if (nla_put_uint(rsp, NETDEV_A_IO_URING_PROVIDER_INFO_RX_BUF_LEN, 1308 1ULL << ifq->niov_shift)) { 1309 nla_nest_cancel(rsp, nest); 1310 return -EMSGSIZE; 1311 } 1312 1313 nla_nest_end(rsp, nest); 1314 1315 return 0; 1316 } 1317 1318 static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq) 1319 { 1320 struct pp_memory_provider_params *p = &rxq->mp_params; 1321 struct io_zcrx_ifq *ifq = mp_priv; 1322 1323 io_zcrx_drop_netdev(ifq); 1324 if (ifq->area) 1325 io_zcrx_unmap_area(ifq, ifq->area); 1326 1327 p->mp_ops = NULL; 1328 p->mp_priv = NULL; 1329 } 1330 1331 static const struct memory_provider_ops io_uring_pp_zc_ops = { 1332 .alloc_netmems = io_pp_zc_alloc_netmems, 1333 .release_netmem = io_pp_zc_release_netmem, 1334 .init = io_pp_zc_init, 1335 .destroy = io_pp_zc_destroy, 1336 .nl_fill = io_pp_nl_fill, 1337 .uninstall = io_pp_uninstall, 1338 }; 1339 1340 static unsigned zcrx_parse_rq(netmem_ref *netmem_array, unsigned nr, 1341 struct io_zcrx_ifq *zcrx, struct zcrx_rq *rq) 1342 { 1343 unsigned int mask = rq->nr_entries - 1; 1344 unsigned int i; 1345 1346 nr = min(nr, zcrx_rq_entries(rq)); 1347 for (i = 0; i < nr; i++) { 1348 struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask); 1349 struct net_iov *niov; 1350 1351 if (!io_parse_rqe(rqe, zcrx, &niov)) 1352 break; 1353 netmem_array[i] = net_iov_to_netmem(niov); 1354 } 1355 1356 smp_store_release(&rq->ring->head, rq->cached_head); 1357 return i; 1358 } 1359 1360 #define ZCRX_FLUSH_BATCH 32 1361 1362 static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr) 1363 { 1364 unsigned i; 1365 1366 for (i = 0; i < nr; i++) { 1367 netmem_ref netmem = netmems[i]; 1368 struct net_iov *niov = netmem_to_net_iov(netmem); 1369 1370 if (!io_zcrx_put_niov_uref(niov)) 1371 continue; 1372 if (!page_pool_unref_and_test(netmem)) 1373 continue; 1374 io_zcrx_return_niov(niov); 1375 } 1376 } 1377 1378 static int zcrx_flush_rq(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, 1379 struct zcrx_ctrl *ctrl) 1380 { 1381 struct zcrx_ctrl_flush_rq *frq = &ctrl->zc_flush; 1382 netmem_ref netmems[ZCRX_FLUSH_BATCH]; 1383 unsigned total = 0; 1384 unsigned nr; 1385 1386 if (!mem_is_zero(&frq->__resv, sizeof(frq->__resv))) 1387 return -EINVAL; 1388 1389 do { 1390 struct zcrx_rq *rq = &zcrx->rq; 1391 1392 scoped_guard(spinlock_bh, &rq->lock) { 1393 nr = zcrx_parse_rq(netmems, ZCRX_FLUSH_BATCH, zcrx, rq); 1394 zcrx_return_buffers(netmems, nr); 1395 } 1396 1397 total += nr; 1398 1399 if (fatal_signal_pending(current)) 1400 break; 1401 cond_resched(); 1402 } while (nr == ZCRX_FLUSH_BATCH && total < zcrx->rq.nr_entries); 1403 1404 return 0; 1405 } 1406 1407 static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, 1408 struct zcrx_ctrl *ctrl) 1409 { 1410 const struct zcrx_ctrl_arm_event *an = &ctrl->zc_arm_event; 1411 unsigned type_mask; 1412 1413 if (an->event_type >= __ZCRX_EVENT_TYPE_LAST) 1414 return -EINVAL; 1415 if (!mem_is_zero(&an->__resv, sizeof(an->__resv))) 1416 return -EINVAL; 1417 1418 guard(spinlock_bh)(&zcrx->ctx_lock); 1419 type_mask = 1U << an->event_type; 1420 if (type_mask & ~zcrx->fired_notifs) 1421 return -EINVAL; 1422 zcrx->fired_notifs &= ~type_mask; 1423 return 0; 1424 } 1425 1426 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) 1427 { 1428 struct zcrx_ctrl ctrl; 1429 struct io_zcrx_ifq *zcrx; 1430 1431 BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_flush)); 1432 BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_arm_event)); 1433 1434 if (nr_args) 1435 return -EINVAL; 1436 if (copy_from_user(&ctrl, arg, sizeof(ctrl))) 1437 return -EFAULT; 1438 if (!mem_is_zero(&ctrl.__resv, sizeof(ctrl.__resv))) 1439 return -EFAULT; 1440 1441 zcrx = xa_load(&ctx->zcrx_ctxs, ctrl.zcrx_id); 1442 if (!zcrx) 1443 return -ENXIO; 1444 1445 switch (ctrl.op) { 1446 case ZCRX_CTRL_FLUSH_RQ: 1447 return zcrx_flush_rq(ctx, zcrx, &ctrl); 1448 case ZCRX_CTRL_EXPORT: 1449 return zcrx_export(ctx, zcrx, &ctrl, arg); 1450 case ZCRX_CTRL_ARM_EVENT: 1451 return zcrx_arm_notif(ctx, zcrx, &ctrl); 1452 } 1453 1454 return -EOPNOTSUPP; 1455 } 1456 1457 static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, 1458 struct io_zcrx_ifq *ifq, int off, int len) 1459 { 1460 struct io_ring_ctx *ctx = req->ctx; 1461 struct io_uring_zcrx_cqe *rcqe; 1462 struct io_zcrx_area *area; 1463 struct io_uring_cqe *cqe; 1464 u64 offset; 1465 1466 if (!io_defer_get_uncommited_cqe(ctx, &cqe)) 1467 return false; 1468 1469 cqe->user_data = req->cqe.user_data; 1470 cqe->res = len; 1471 cqe->flags = IORING_CQE_F_MORE; 1472 if (ctx->flags & IORING_SETUP_CQE_MIXED) 1473 cqe->flags |= IORING_CQE_F_32; 1474 1475 area = io_zcrx_iov_to_area(niov); 1476 offset = off + (net_iov_idx(niov) << ifq->niov_shift); 1477 rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1); 1478 rcqe->off = offset + ((u64)area->area_id << IORING_ZCRX_AREA_SHIFT); 1479 rcqe->__pad = 0; 1480 return true; 1481 } 1482 1483 static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq) 1484 { 1485 struct io_zcrx_area *area = ifq->area; 1486 struct net_iov *niov = NULL; 1487 1488 if (!ifq->kern_readable) 1489 return NULL; 1490 1491 scoped_guard(spinlock_bh, &area->freelist_lock) 1492 niov = zcrx_get_free_niov(area); 1493 1494 if (niov) 1495 page_pool_fragment_netmem(net_iov_to_netmem(niov), 1); 1496 return niov; 1497 } 1498 1499 struct io_copy_cache { 1500 struct page *page; 1501 unsigned long offset; 1502 size_t size; 1503 }; 1504 1505 static ssize_t io_copy_page(struct io_copy_cache *cc, struct page *src_page, 1506 unsigned int src_offset, size_t len) 1507 { 1508 size_t copied = 0; 1509 1510 len = min(len, cc->size); 1511 1512 while (len) { 1513 void *src_addr, *dst_addr; 1514 struct page *dst_page = cc->page; 1515 unsigned dst_offset = cc->offset; 1516 size_t n = len; 1517 1518 if (folio_test_partial_kmap(page_folio(dst_page)) || 1519 folio_test_partial_kmap(page_folio(src_page))) { 1520 dst_page += dst_offset / PAGE_SIZE; 1521 dst_offset = offset_in_page(dst_offset); 1522 src_page += src_offset / PAGE_SIZE; 1523 src_offset = offset_in_page(src_offset); 1524 n = min(PAGE_SIZE - src_offset, PAGE_SIZE - dst_offset); 1525 n = min(n, len); 1526 } 1527 1528 dst_addr = kmap_local_page(dst_page) + dst_offset; 1529 src_addr = kmap_local_page(src_page) + src_offset; 1530 1531 memcpy(dst_addr, src_addr, n); 1532 1533 kunmap_local(src_addr); 1534 kunmap_local(dst_addr); 1535 1536 cc->size -= n; 1537 cc->offset += n; 1538 src_offset += n; 1539 len -= n; 1540 copied += n; 1541 } 1542 return copied; 1543 } 1544 1545 static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1546 struct page *src_page, unsigned int src_offset, 1547 size_t len) 1548 { 1549 size_t copied = 0; 1550 int ret = 0; 1551 1552 while (len) { 1553 struct io_copy_cache cc; 1554 struct net_iov *niov; 1555 size_t n; 1556 1557 niov = io_alloc_fallback_niov(ifq); 1558 if (!niov) { 1559 ret = -ENOMEM; 1560 break; 1561 } 1562 1563 cc.page = io_zcrx_iov_page(niov); 1564 cc.offset = 0; 1565 cc.size = PAGE_SIZE; 1566 1567 n = io_copy_page(&cc, src_page, src_offset, len); 1568 1569 if (!io_zcrx_queue_cqe(req, niov, ifq, 0, n)) { 1570 io_zcrx_return_niov(niov); 1571 ret = -ENOSPC; 1572 break; 1573 } 1574 1575 io_zcrx_get_niov_uref(niov); 1576 src_offset += n; 1577 len -= n; 1578 copied += n; 1579 } 1580 1581 return copied ? copied : ret; 1582 } 1583 1584 static int io_zcrx_copy_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1585 const skb_frag_t *frag, int off, int len) 1586 { 1587 struct page *page = skb_frag_page(frag); 1588 int ret; 1589 1590 ret = io_zcrx_copy_chunk(req, ifq, page, off + skb_frag_off(frag), len); 1591 if (ret > 0) { 1592 if (ifq->notif_stats) { 1593 zcrx_stat_add(&ifq->notif_stats->copy_count, 1); 1594 zcrx_stat_add(&ifq->notif_stats->copy_bytes, ret); 1595 } 1596 zcrx_send_notif(ifq, ZCRX_EVENT_COPY); 1597 } 1598 1599 return ret; 1600 } 1601 1602 static int io_zcrx_recv_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1603 const skb_frag_t *frag, int off, int len) 1604 { 1605 struct net_iov *niov; 1606 struct page_pool *pp; 1607 1608 if (unlikely(!skb_frag_is_net_iov(frag))) 1609 return io_zcrx_copy_frag(req, ifq, frag, off, len); 1610 1611 niov = netmem_to_net_iov(frag->netmem); 1612 pp = niov->desc.pp; 1613 1614 if (!pp || pp->mp_ops != &io_uring_pp_zc_ops || io_pp_to_ifq(pp) != ifq) 1615 return -EFAULT; 1616 1617 if (!io_zcrx_queue_cqe(req, niov, ifq, off + skb_frag_off(frag), len)) 1618 return -ENOSPC; 1619 1620 /* 1621 * Prevent it from being recycled while user is accessing it. 1622 * It has to be done before grabbing a user reference. 1623 */ 1624 page_pool_ref_netmem(net_iov_to_netmem(niov)); 1625 io_zcrx_get_niov_uref(niov); 1626 return len; 1627 } 1628 1629 static int 1630 io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 1631 unsigned int offset, size_t len) 1632 { 1633 struct io_zcrx_args *args = desc->arg.data; 1634 struct io_zcrx_ifq *ifq = args->ifq; 1635 struct io_kiocb *req = args->req; 1636 struct sk_buff *frag_iter; 1637 unsigned start, start_off = offset; 1638 int i, copy, end, off; 1639 int ret = 0; 1640 1641 len = min_t(size_t, len, desc->count); 1642 /* 1643 * __tcp_read_sock() always calls io_zcrx_recv_skb one last time, even 1644 * if desc->count is already 0. This is caused by the if (offset + 1 != 1645 * skb->len) check. Return early in this case to break out of 1646 * __tcp_read_sock(). 1647 */ 1648 if (!len) 1649 return 0; 1650 if (unlikely(args->nr_skbs++ > IO_SKBS_PER_CALL_LIMIT)) 1651 return -EAGAIN; 1652 1653 if (unlikely(offset < skb_headlen(skb))) { 1654 ssize_t copied; 1655 size_t to_copy; 1656 1657 to_copy = min_t(size_t, skb_headlen(skb) - offset, len); 1658 copied = io_zcrx_copy_chunk(req, ifq, virt_to_page(skb->data), 1659 offset_in_page(skb->data) + offset, 1660 to_copy); 1661 if (copied < 0) { 1662 ret = copied; 1663 goto out; 1664 } 1665 offset += copied; 1666 len -= copied; 1667 if (!len) 1668 goto out; 1669 if (offset != skb_headlen(skb)) 1670 goto out; 1671 } 1672 1673 start = skb_headlen(skb); 1674 1675 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1676 const skb_frag_t *frag; 1677 1678 if (WARN_ON(start > offset + len)) 1679 return -EFAULT; 1680 1681 frag = &skb_shinfo(skb)->frags[i]; 1682 end = start + skb_frag_size(frag); 1683 1684 if (offset < end) { 1685 copy = end - offset; 1686 if (copy > len) 1687 copy = len; 1688 1689 off = offset - start; 1690 ret = io_zcrx_recv_frag(req, ifq, frag, off, copy); 1691 if (ret < 0) 1692 goto out; 1693 1694 offset += ret; 1695 len -= ret; 1696 if (len == 0 || ret != copy) 1697 goto out; 1698 } 1699 start = end; 1700 } 1701 1702 skb_walk_frags(skb, frag_iter) { 1703 if (WARN_ON(start > offset + len)) 1704 return -EFAULT; 1705 1706 end = start + frag_iter->len; 1707 if (offset < end) { 1708 size_t count; 1709 1710 copy = end - offset; 1711 if (copy > len) 1712 copy = len; 1713 1714 off = offset - start; 1715 count = desc->count; 1716 ret = io_zcrx_recv_skb(desc, frag_iter, off, copy); 1717 desc->count = count; 1718 if (ret < 0) 1719 goto out; 1720 1721 offset += ret; 1722 len -= ret; 1723 if (len == 0 || ret != copy) 1724 goto out; 1725 } 1726 start = end; 1727 } 1728 1729 out: 1730 if (offset == start_off) 1731 return ret; 1732 desc->count -= (offset - start_off); 1733 return offset - start_off; 1734 } 1735 1736 static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1737 struct sock *sk, int flags, 1738 unsigned issue_flags, unsigned int *outlen) 1739 { 1740 unsigned int len = *outlen; 1741 struct io_zcrx_args args = { 1742 .req = req, 1743 .ifq = ifq, 1744 }; 1745 read_descriptor_t rd_desc = { 1746 .count = len ? len : UINT_MAX, 1747 .arg.data = &args, 1748 }; 1749 int ret; 1750 1751 lock_sock(sk); 1752 ret = tcp_read_sock(sk, &rd_desc, io_zcrx_recv_skb); 1753 if (len && ret > 0) 1754 *outlen = len - ret; 1755 if (ret <= 0) { 1756 if (ret < 0 || sock_flag(sk, SOCK_DONE)) 1757 goto out; 1758 if (sk->sk_err) 1759 ret = sock_error(sk); 1760 else if (sk->sk_shutdown & RCV_SHUTDOWN) 1761 goto out; 1762 else if (sk->sk_state == TCP_CLOSE) 1763 ret = -ENOTCONN; 1764 else 1765 ret = -EAGAIN; 1766 } else if (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) && 1767 (issue_flags & IO_URING_F_MULTISHOT)) { 1768 ret = IOU_REQUEUE; 1769 } else if (sock_flag(sk, SOCK_DONE)) { 1770 /* Make it to retry until it finally gets 0. */ 1771 if (issue_flags & IO_URING_F_MULTISHOT) 1772 ret = IOU_REQUEUE; 1773 else 1774 ret = -EAGAIN; 1775 } 1776 out: 1777 release_sock(sk); 1778 return ret; 1779 } 1780 1781 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1782 struct socket *sock, unsigned int flags, 1783 unsigned issue_flags, unsigned int *len) 1784 { 1785 struct sock *sk = sock->sk; 1786 const struct proto *prot = READ_ONCE(sk->sk_prot); 1787 1788 if (prot->recvmsg != tcp_recvmsg) 1789 return -EPROTONOSUPPORT; 1790 1791 sock_rps_record_flow(sk); 1792 return io_zcrx_tcp_recvmsg(req, ifq, sk, flags, issue_flags, len); 1793 } 1794