1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/dma-map-ops.h> 5 #include <linux/mm.h> 6 #include <linux/nospec.h> 7 #include <linux/io_uring.h> 8 #include <linux/netdevice.h> 9 #include <linux/rtnetlink.h> 10 #include <linux/skbuff_ref.h> 11 #include <linux/anon_inodes.h> 12 13 #include <net/page_pool/helpers.h> 14 #include <net/page_pool/memory_provider.h> 15 #include <net/netlink.h> 16 #include <net/netdev_queues.h> 17 #include <net/netdev_rx_queue.h> 18 #include <net/tcp.h> 19 #include <net/rps.h> 20 21 #include <trace/events/page_pool.h> 22 23 #include <uapi/linux/io_uring.h> 24 25 #include "io_uring.h" 26 #include "kbuf.h" 27 #include "memmap.h" 28 #include "zcrx.h" 29 #include "rsrc.h" 30 31 #define ZCRX_MAX_FRAGS_PER_PAGE MAX(PAGE_SIZE / 1024, 1) 32 /* 33 * We need a reasonable limit to be able to fill in 64 entries on average 34 * for 1500 byte MTU. Over-estimate it to keep it pow2. 35 */ 36 #define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024) 37 38 #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) 39 #define ZCRX_MAX_AREAS 1024 40 41 #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) 42 43 static inline u32 zcrx_next_area_id(struct io_zcrx_ifq *zcrx) 44 { 45 return zcrx->nr_areas; 46 } 47 48 static inline u64 zcrx_area_id_to_token(u32 area_id) 49 { 50 return (u64)area_id << IORING_ZCRX_AREA_SHIFT; 51 } 52 53 static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp) 54 { 55 return pp->mp_priv; 56 } 57 58 static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *niov) 59 { 60 struct net_iov_area *owner = net_iov_owner(niov); 61 62 return container_of(owner, struct io_zcrx_area, nia); 63 } 64 65 static bool zcrx_set_ring_ctx(struct io_zcrx_ifq *zcrx, 66 struct io_ring_ctx *ctx) 67 { 68 guard(spinlock_bh)(&zcrx->ctx_lock); 69 if (zcrx->master_ctx) 70 return false; 71 percpu_ref_get(&ctx->refs); 72 zcrx->master_ctx = ctx; 73 return true; 74 } 75 76 static inline struct page *io_zcrx_iov_page(const struct net_iov *niov) 77 { 78 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); 79 unsigned niov_pages_shift; 80 81 lockdep_assert(!area->mem.is_dmabuf); 82 83 niov_pages_shift = area->ifq->niov_shift - PAGE_SHIFT; 84 return area->mem.pages[net_iov_idx(niov) << niov_pages_shift]; 85 } 86 87 static int io_area_max_shift(struct io_zcrx_mem *mem) 88 { 89 struct sg_table *sgt = mem->sgt; 90 struct scatterlist *sg; 91 unsigned shift = -1U; 92 unsigned i; 93 94 for_each_sgtable_dma_sg(sgt, sg, i) 95 shift = min(shift, __ffs(sg_dma_len(sg))); 96 return shift; 97 } 98 99 static int io_populate_area_dma(struct io_zcrx_ifq *ifq, 100 struct io_zcrx_area *area) 101 { 102 unsigned niov_size = 1U << ifq->niov_shift; 103 struct sg_table *sgt = area->mem.sgt; 104 struct scatterlist *sg; 105 unsigned i, niov_idx = 0; 106 107 for_each_sgtable_dma_sg(sgt, sg, i) { 108 dma_addr_t dma = sg_dma_address(sg); 109 unsigned long sg_len = sg_dma_len(sg); 110 111 if (WARN_ON_ONCE(sg_len % niov_size)) 112 return -EINVAL; 113 114 while (sg_len && niov_idx < area->nia.num_niovs) { 115 struct net_iov *niov = &area->nia.niovs[niov_idx]; 116 117 if (net_mp_niov_set_dma_addr(niov, dma)) 118 return -EFAULT; 119 sg_len -= niov_size; 120 dma += niov_size; 121 niov_idx++; 122 } 123 } 124 125 if (WARN_ON_ONCE(niov_idx != area->nia.num_niovs)) 126 return -EFAULT; 127 return 0; 128 } 129 130 static void io_unmap_dmabuf(struct io_zcrx_mem *mem) 131 { 132 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) 133 return; 134 if (mem->sgt) 135 dma_buf_unmap_attachment_unlocked(mem->attach, mem->sgt, 136 DMA_FROM_DEVICE); 137 if (mem->attach) 138 dma_buf_detach(mem->dmabuf, mem->attach); 139 mem->sgt = NULL; 140 mem->attach = NULL; 141 } 142 143 static void io_release_dmabuf(struct io_zcrx_mem *mem) 144 { 145 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) 146 return; 147 if (mem->dmabuf) 148 dma_buf_put(mem->dmabuf); 149 mem->dmabuf = NULL; 150 } 151 152 static int io_import_dmabuf(struct io_zcrx_ifq *ifq, 153 struct io_zcrx_mem *mem, 154 const struct io_uring_zcrx_area_reg *area_reg) 155 { 156 unsigned long off = (unsigned long)area_reg->addr; 157 unsigned long len = (unsigned long)area_reg->len; 158 unsigned long total_size = 0; 159 struct scatterlist *sg; 160 int dmabuf_fd = area_reg->dmabuf_fd; 161 int i, ret; 162 163 if (!ifq->dev) 164 return -EINVAL; 165 if (off) 166 return -EINVAL; 167 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) 168 return -EINVAL; 169 170 mem->is_dmabuf = true; 171 mem->dmabuf = dma_buf_get(dmabuf_fd); 172 if (IS_ERR(mem->dmabuf)) { 173 ret = PTR_ERR(mem->dmabuf); 174 mem->dmabuf = NULL; 175 goto err; 176 } 177 178 mem->attach = dma_buf_attach(mem->dmabuf, ifq->dev); 179 if (IS_ERR(mem->attach)) { 180 ret = PTR_ERR(mem->attach); 181 mem->attach = NULL; 182 goto err; 183 } 184 185 mem->sgt = dma_buf_map_attachment_unlocked(mem->attach, DMA_FROM_DEVICE); 186 if (IS_ERR(mem->sgt)) { 187 ret = PTR_ERR(mem->sgt); 188 mem->sgt = NULL; 189 goto err; 190 } 191 192 for_each_sgtable_dma_sg(mem->sgt, sg, i) 193 total_size += sg_dma_len(sg); 194 195 if (total_size != len) { 196 ret = -EINVAL; 197 goto err; 198 } 199 200 mem->size = len; 201 return 0; 202 err: 203 io_unmap_dmabuf(mem); 204 io_release_dmabuf(mem); 205 return ret; 206 } 207 208 static unsigned long io_count_account_pages(struct page **pages, unsigned nr_pages) 209 { 210 struct folio *last_folio = NULL; 211 unsigned long res = 0; 212 int i; 213 214 for (i = 0; i < nr_pages; i++) { 215 struct folio *folio = page_folio(pages[i]); 216 217 if (folio == last_folio) 218 continue; 219 last_folio = folio; 220 res += folio_nr_pages(folio); 221 } 222 return res; 223 } 224 225 static int io_import_umem(struct io_zcrx_ifq *ifq, 226 struct io_zcrx_mem *mem, 227 const struct io_uring_zcrx_area_reg *area_reg) 228 { 229 struct page **pages; 230 int nr_pages, ret; 231 bool mapped = false; 232 233 if (area_reg->dmabuf_fd) 234 return -EINVAL; 235 if (!area_reg->addr) 236 return -EFAULT; 237 pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len, 238 &nr_pages); 239 if (IS_ERR(pages)) 240 return PTR_ERR(pages); 241 242 ret = sg_alloc_table_from_pages(&mem->page_sg_table, pages, nr_pages, 243 0, (unsigned long)nr_pages << PAGE_SHIFT, 244 GFP_KERNEL_ACCOUNT); 245 if (ret) 246 goto out_err; 247 248 if (ifq->dev) { 249 ret = dma_map_sgtable(ifq->dev, &mem->page_sg_table, 250 DMA_FROM_DEVICE, IO_DMA_ATTR); 251 if (ret < 0) 252 goto out_err; 253 mapped = true; 254 } 255 256 mem->account_pages = io_count_account_pages(pages, nr_pages); 257 ret = io_account_mem(ifq->user, ifq->mm_account, mem->account_pages); 258 if (ret < 0) { 259 mem->account_pages = 0; 260 goto out_err; 261 } 262 263 mem->sgt = &mem->page_sg_table; 264 mem->pages = pages; 265 mem->nr_folios = nr_pages; 266 mem->size = area_reg->len; 267 return ret; 268 out_err: 269 if (mapped) 270 dma_unmap_sgtable(ifq->dev, &mem->page_sg_table, 271 DMA_FROM_DEVICE, IO_DMA_ATTR); 272 sg_free_table(&mem->page_sg_table); 273 unpin_user_pages(pages, nr_pages); 274 kvfree(pages); 275 return ret; 276 } 277 278 static void io_release_area_mem(struct io_zcrx_mem *mem) 279 { 280 if (mem->is_dmabuf) { 281 io_release_dmabuf(mem); 282 } else if (mem->pages) { 283 unpin_user_pages(mem->pages, mem->nr_folios); 284 sg_free_table(mem->sgt); 285 kvfree(mem->pages); 286 } 287 mem->pages = IO_URING_PTR_POISON; 288 mem->sgt = IO_URING_PTR_POISON; 289 } 290 291 static int io_import_area(struct io_zcrx_ifq *ifq, 292 struct io_zcrx_mem *mem, 293 const struct io_uring_zcrx_area_reg *area_reg) 294 { 295 int ret; 296 297 if (area_reg->flags & ~IO_ZCRX_AREA_SUPPORTED_FLAGS) 298 return -EINVAL; 299 if (area_reg->rq_area_token) 300 return -EINVAL; 301 if (area_reg->__resv2[0] || area_reg->__resv2[1]) 302 return -EINVAL; 303 304 ret = io_validate_user_buf_range(area_reg->addr, area_reg->len); 305 if (ret) 306 return ret; 307 if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK) 308 return -EINVAL; 309 310 if (area_reg->flags & IORING_ZCRX_AREA_DMABUF) 311 return io_import_dmabuf(ifq, mem, area_reg); 312 return io_import_umem(ifq, mem, area_reg); 313 } 314 315 static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, 316 struct io_zcrx_area *area) 317 { 318 int i; 319 320 lockdep_assert_held(&ifq->pp_lock); 321 322 if (!area || !area->is_mapped) 323 return; 324 area->is_mapped = false; 325 326 if (area->nia.niovs) { 327 for (i = 0; i < area->nia.num_niovs; i++) 328 net_mp_niov_set_dma_addr(&area->nia.niovs[i], 0); 329 } 330 331 if (area->mem.is_dmabuf) { 332 io_unmap_dmabuf(&area->mem); 333 } else { 334 dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table, 335 DMA_FROM_DEVICE, IO_DMA_ATTR); 336 } 337 } 338 339 static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq) 340 { 341 unsigned area_idx; 342 343 lockdep_assert_held(&ifq->pp_lock); 344 345 for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) 346 __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]); 347 } 348 349 static void zcrx_sync_for_device(struct page_pool *pp, struct io_zcrx_ifq *zcrx, 350 netmem_ref *netmems, unsigned nr) 351 { 352 #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) 353 struct device *dev = pp->p.dev; 354 unsigned i, niov_size; 355 dma_addr_t dma_addr; 356 357 if (!dma_dev_need_sync(dev)) 358 return; 359 niov_size = 1U << zcrx->niov_shift; 360 361 for (i = 0; i < nr; i++) { 362 dma_addr = page_pool_get_dma_addr_netmem(netmems[i]); 363 __dma_sync_single_for_device(dev, dma_addr + pp->p.offset, 364 niov_size, pp->p.dma_dir); 365 } 366 #endif 367 } 368 369 #define IO_RQ_MAX_ENTRIES 32768 370 371 #define IO_SKBS_PER_CALL_LIMIT 20 372 373 struct io_zcrx_args { 374 struct io_kiocb *req; 375 struct io_zcrx_ifq *ifq; 376 unsigned nr_skbs; 377 bool stopped_early; 378 }; 379 380 static const struct memory_provider_ops io_uring_pp_zc_ops; 381 382 static inline atomic_t *io_get_user_counter(struct net_iov *niov) 383 { 384 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); 385 386 return &area->user_refs[net_iov_idx(niov)]; 387 } 388 389 static bool io_zcrx_put_niov_uref(struct net_iov *niov, unsigned refs) 390 { 391 atomic_t *uref = io_get_user_counter(niov); 392 int old; 393 394 old = atomic_read(uref); 395 do { 396 if (unlikely(old < refs)) 397 return false; 398 } while (!atomic_try_cmpxchg(uref, &old, old - refs)); 399 400 return true; 401 } 402 403 static void io_zcrx_get_niov_uref(struct net_iov *niov) 404 { 405 atomic_inc(io_get_user_counter(niov)); 406 } 407 408 static void io_fill_zcrx_offsets(struct io_uring_zcrx_offsets *offsets) 409 { 410 offsets->head = offsetof(struct zcrx_rq_hdr, head); 411 offsets->tail = offsetof(struct zcrx_rq_hdr, tail); 412 offsets->rqes = ALIGN(sizeof(struct zcrx_rq_hdr), L1_CACHE_BYTES); 413 } 414 415 static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, 416 struct io_zcrx_ifq *ifq, 417 struct io_uring_zcrx_ifq_reg *reg, 418 struct io_uring_region_desc *rd, 419 u32 id) 420 { 421 u64 mmap_offset; 422 size_t off, size; 423 void *ptr; 424 int ret; 425 426 io_fill_zcrx_offsets(®->offsets); 427 off = reg->offsets.rqes; 428 size = off + sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries; 429 if (size > rd->size) 430 return -EINVAL; 431 432 mmap_offset = IORING_MAP_OFF_ZCRX_REGION; 433 mmap_offset += (u64)id << IORING_OFF_ZCRX_SHIFT; 434 435 ret = io_create_region(ctx->user, &ifq->rq_region, rd, mmap_offset); 436 if (ret < 0) 437 return ret; 438 439 ptr = io_region_get_ptr(&ifq->rq_region); 440 ifq->rq.ring = (struct zcrx_rq_hdr *)ptr; 441 ifq->rq.rqes = (struct io_uring_zcrx_rqe *)(ptr + off); 442 443 memset(ifq->rq.ring, 0, sizeof(*ifq->rq.ring)); 444 return 0; 445 } 446 447 static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq) 448 { 449 io_free_region(ifq->user, &ifq->rq_region); 450 ifq->rq.ring = IO_URING_PTR_POISON; 451 ifq->rq.rqes = IO_URING_PTR_POISON; 452 ifq->notif_stats = IO_URING_PTR_POISON; 453 } 454 455 static void io_zcrx_free_area(struct io_zcrx_ifq *ifq, 456 struct io_zcrx_area *area) 457 { 458 if (WARN_ON_ONCE(area->is_mapped)) 459 return; 460 io_release_area_mem(&area->mem); 461 462 if (area->mem.account_pages) 463 io_unaccount_mem(ifq->user, ifq->mm_account, 464 area->mem.account_pages); 465 466 kvfree(area->freelist); 467 kvfree(area->nia.niovs); 468 kvfree(area->user_refs); 469 kfree(area); 470 } 471 472 static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, 473 struct io_zcrx_area *area) 474 { 475 bool kern_readable = !area->mem.is_dmabuf; 476 struct io_zcrx_area **areas, **old_areas; 477 unsigned old_nr; 478 479 if (ifq->kern_readable != kern_readable) 480 return -EINVAL; 481 if (ifq->nr_areas + 1 > ZCRX_MAX_AREAS) 482 return -EINVAL; 483 484 old_areas = ifq->areas; 485 old_nr = ifq->nr_areas; 486 487 areas = kmalloc_objs(areas[0], old_nr + 1, 488 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 489 if (!areas) 490 return -ENOMEM; 491 if (old_areas) 492 memcpy(areas, old_areas, old_nr * sizeof(areas[0])); 493 areas[old_nr] = area; 494 495 scoped_guard(spinlock_bh, &ifq->rq.lock) { 496 guard(spinlock_bh)(&ifq->alloc_lock); 497 ifq->areas = areas; 498 ifq->nr_areas = old_nr + 1; 499 } 500 kfree(old_areas); 501 return 0; 502 } 503 504 static int __zcrx_create_area(struct io_zcrx_ifq *ifq, 505 struct io_uring_zcrx_area_reg *area_reg, 506 u32 rx_buf_len) 507 { 508 int buf_size_shift = PAGE_SHIFT; 509 struct io_zcrx_area *area; 510 unsigned nr_iovs; 511 int i, ret; 512 513 lockdep_assert_held(&ifq->pp_lock); 514 515 if (rx_buf_len) { 516 if (!is_power_of_2(rx_buf_len) || rx_buf_len < PAGE_SIZE) 517 return -EINVAL; 518 buf_size_shift = ilog2(rx_buf_len); 519 } 520 if (ifq->niov_shift && ifq->niov_shift != buf_size_shift) 521 return -EINVAL; 522 if (!ifq->dev && buf_size_shift != PAGE_SHIFT) 523 return -EOPNOTSUPP; 524 525 ret = -ENOMEM; 526 area = kzalloc_obj(*area); 527 if (!area) 528 goto err; 529 area->ifq = ifq; 530 531 ret = io_import_area(ifq, &area->mem, area_reg); 532 if (ret) 533 goto err; 534 if (ifq->dev) 535 area->is_mapped = true; 536 537 if (ifq->dev && buf_size_shift > io_area_max_shift(&area->mem)) { 538 ret = -ERANGE; 539 goto err; 540 } 541 542 ifq->niov_shift = buf_size_shift; 543 nr_iovs = area->mem.size >> ifq->niov_shift; 544 area->nia.num_niovs = nr_iovs; 545 546 ret = -ENOMEM; 547 area->nia.niovs = kvmalloc_objs(area->nia.niovs[0], nr_iovs, 548 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 549 if (!area->nia.niovs) 550 goto err; 551 552 area->freelist = kvmalloc_array(nr_iovs, sizeof(area->freelist[0]), 553 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 554 if (!area->freelist) 555 goto err; 556 557 area->user_refs = kvmalloc_objs(area->user_refs[0], nr_iovs, 558 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 559 if (!area->user_refs) 560 goto err; 561 562 for (i = 0; i < nr_iovs; i++) { 563 struct net_iov *niov = &area->nia.niovs[i]; 564 565 net_iov_init(niov, &area->nia, NET_IOV_IOURING); 566 area->freelist[i] = i; 567 atomic_set(&area->user_refs[i], 0); 568 } 569 570 if (ifq->dev) { 571 ret = io_populate_area_dma(ifq, area); 572 if (ret) 573 goto err; 574 } 575 576 area->free_count = nr_iovs; 577 /* we're only supporting one area per ifq for now */ 578 area->area_id = zcrx_next_area_id(ifq); 579 area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id); 580 581 ret = io_zcrx_append_area(ifq, area); 582 if (!ret) 583 return 0; 584 err: 585 if (area) { 586 __io_zcrx_unmap_area(ifq, area); 587 io_zcrx_free_area(ifq, area); 588 } 589 return ret; 590 } 591 592 static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, 593 struct io_uring_zcrx_area_reg *area_reg, 594 struct io_uring_zcrx_ifq_reg *reg) 595 { 596 guard(mutex)(&ifq->pp_lock); 597 return __zcrx_create_area(ifq, area_reg, reg->rx_buf_len); 598 } 599 600 static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) 601 { 602 struct io_zcrx_ifq *ifq; 603 604 ifq = kzalloc_obj(*ifq); 605 if (!ifq) 606 return NULL; 607 608 ifq->if_rxq = -1; 609 spin_lock_init(&ifq->ctx_lock); 610 spin_lock_init(&ifq->rq.lock); 611 spin_lock_init(&ifq->alloc_lock); 612 mutex_init(&ifq->pp_lock); 613 refcount_set(&ifq->refs, 1); 614 refcount_set(&ifq->user_refs, 1); 615 return ifq; 616 } 617 618 static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq) 619 { 620 lockdep_assert_held(&ifq->pp_lock); 621 622 if (!ifq->netdev) 623 return; 624 netdev_put(ifq->netdev, &ifq->netdev_tracker); 625 ifq->netdev = NULL; 626 } 627 628 static void io_close_queue(struct io_zcrx_ifq *ifq) 629 { 630 struct net_device *netdev; 631 netdevice_tracker netdev_tracker; 632 struct pp_memory_provider_params p = { 633 .mp_ops = &io_uring_pp_zc_ops, 634 .mp_priv = ifq, 635 }; 636 637 scoped_guard(mutex, &ifq->pp_lock) { 638 netdev = ifq->netdev; 639 netdev_tracker = ifq->netdev_tracker; 640 ifq->netdev = NULL; 641 } 642 643 if (netdev) { 644 netdev_lock(netdev); 645 if (ifq->if_rxq != -1) 646 netif_mp_close_rxq(netdev, ifq->if_rxq, &p); 647 648 scoped_guard(mutex, &ifq->pp_lock) 649 io_zcrx_unmap_areas(ifq); 650 netdev_unlock(netdev); 651 netdev_put(netdev, &netdev_tracker); 652 } 653 ifq->if_rxq = -1; 654 } 655 656 static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) 657 { 658 int i; 659 660 if (WARN_ON_ONCE(ifq->if_rxq != -1)) 661 return; 662 if (WARN_ON_ONCE(ifq->netdev != NULL)) 663 return; 664 if (WARN_ON_ONCE(ifq->master_ctx)) 665 return; 666 667 for (i = 0; i < ifq->nr_areas; i++) 668 io_zcrx_free_area(ifq, ifq->areas[i]); 669 if (ifq->mm_account) 670 mmdrop(ifq->mm_account); 671 if (ifq->dev) 672 put_device(ifq->dev); 673 674 io_free_rbuf_ring(ifq); 675 free_uid(ifq->user); 676 mutex_destroy(&ifq->pp_lock); 677 kfree(ifq->areas); 678 kfree(ifq); 679 } 680 681 static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq) 682 { 683 if (refcount_dec_and_test(&ifq->refs)) 684 io_zcrx_ifq_free(ifq); 685 } 686 687 static void io_zcrx_return_niov_freelist(struct net_iov *niov) 688 { 689 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); 690 struct io_zcrx_ifq *ifq = area->ifq; 691 692 guard(spinlock_bh)(&ifq->alloc_lock); 693 if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs)) 694 return; 695 area->freelist[area->free_count++] = net_iov_idx(niov); 696 } 697 698 static struct net_iov *zcrx_get_free_niov(struct io_zcrx_area *area) 699 { 700 unsigned niov_idx; 701 702 lockdep_assert_held(&area->ifq->alloc_lock); 703 704 if (unlikely(!area->free_count)) 705 return NULL; 706 707 niov_idx = area->freelist[--area->free_count]; 708 return &area->nia.niovs[niov_idx]; 709 } 710 711 static void io_zcrx_return_niov(struct net_iov *niov) 712 { 713 netmem_ref netmem = net_iov_to_netmem(niov); 714 715 if (!niov->desc.pp) { 716 /* copy fallback allocated niovs */ 717 io_zcrx_return_niov_freelist(niov); 718 return; 719 } 720 page_pool_put_unrefed_netmem(niov->desc.pp, netmem, -1, false); 721 } 722 723 static void io_zcrx_scrub_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) 724 { 725 int i; 726 727 /* Reclaim back all buffers given to the user space. */ 728 for (i = 0; i < area->nia.num_niovs; i++) { 729 struct net_iov *niov = &area->nia.niovs[i]; 730 int nr; 731 732 if (!atomic_read(io_get_user_counter(niov))) 733 continue; 734 nr = atomic_xchg(io_get_user_counter(niov), 0); 735 if (nr && !page_pool_unref_netmem(net_iov_to_netmem(niov), nr)) 736 io_zcrx_return_niov(niov); 737 } 738 } 739 740 static void io_zcrx_scrub(struct io_zcrx_ifq *ifq) 741 { 742 int i; 743 744 guard(mutex)(&ifq->pp_lock); 745 for (i = 0; i < ifq->nr_areas; i++) 746 io_zcrx_scrub_area(ifq, ifq->areas[i]); 747 } 748 749 static void zcrx_unregister_user(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) 750 { 751 scoped_guard(spinlock_bh, &ifq->ctx_lock) { 752 if (ctx && ifq->master_ctx == ctx) { 753 ifq->master_ctx = NULL; 754 percpu_ref_put(&ctx->refs); 755 } 756 } 757 758 if (refcount_dec_and_test(&ifq->user_refs)) { 759 io_close_queue(ifq); 760 io_zcrx_scrub(ifq); 761 } 762 } 763 764 static void zcrx_unregister(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) 765 { 766 zcrx_unregister_user(ifq, ctx); 767 io_put_zcrx_ifq(ifq); 768 } 769 770 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 771 unsigned int id) 772 { 773 struct io_zcrx_ifq *ifq = xa_load(&ctx->zcrx_ctxs, id); 774 775 lockdep_assert_held(&ctx->mmap_lock); 776 777 return ifq ? &ifq->rq_region : NULL; 778 } 779 780 static int zcrx_box_release(struct inode *inode, struct file *file) 781 { 782 struct io_zcrx_ifq *ifq = file->private_data; 783 784 if (WARN_ON_ONCE(!ifq)) 785 return -EFAULT; 786 zcrx_unregister(ifq, NULL); 787 return 0; 788 } 789 790 static const struct file_operations zcrx_box_fops = { 791 .owner = THIS_MODULE, 792 .release = zcrx_box_release, 793 }; 794 795 static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, 796 struct zcrx_ctrl *ctrl, void __user *arg) 797 { 798 struct zcrx_ctrl_export *ce = &ctrl->zc_export; 799 struct file *file; 800 int fd; 801 802 if (!mem_is_zero(ce, sizeof(*ce))) 803 return -EINVAL; 804 805 refcount_inc(&ifq->refs); 806 refcount_inc(&ifq->user_refs); 807 808 file = anon_inode_create_getfile("[zcrx]", &zcrx_box_fops, 809 ifq, O_CLOEXEC, NULL); 810 if (IS_ERR(file)) { 811 zcrx_unregister(ifq, NULL); 812 return PTR_ERR(file); 813 } 814 815 fd = get_unused_fd_flags(O_CLOEXEC); 816 if (fd < 0) { 817 fput(file); 818 return fd; 819 } 820 821 ce->zcrx_fd = fd; 822 if (copy_to_user(arg, ctrl, sizeof(*ctrl))) { 823 fput(file); 824 put_unused_fd(fd); 825 return -EFAULT; 826 } 827 828 fd_install(fd, file); 829 return 0; 830 } 831 832 static int import_zcrx(struct io_ring_ctx *ctx, 833 struct io_uring_zcrx_ifq_reg __user *arg, 834 struct io_uring_zcrx_ifq_reg *reg) 835 { 836 struct io_zcrx_ifq *ifq; 837 struct file *file; 838 int fd, ret; 839 u32 id; 840 841 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 842 return -EINVAL; 843 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))) 844 return -EINVAL; 845 if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr) 846 return -EINVAL; 847 if (reg->event_desc) 848 return -EINVAL; 849 if (reg->flags & ~ZCRX_REG_IMPORT) 850 return -EINVAL; 851 852 fd = reg->if_idx; 853 CLASS(fd, f)(fd); 854 if (fd_empty(f)) 855 return -EBADF; 856 857 file = fd_file(f); 858 if (file->f_op != &zcrx_box_fops || !file->private_data) 859 return -EBADF; 860 861 ifq = file->private_data; 862 refcount_inc(&ifq->refs); 863 refcount_inc(&ifq->user_refs); 864 865 scoped_guard(mutex, &ctx->mmap_lock) { 866 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL); 867 if (ret) 868 goto err; 869 } 870 871 reg->zcrx_id = id; 872 io_fill_zcrx_offsets(®->offsets); 873 if (copy_to_user(arg, reg, sizeof(*reg))) { 874 ret = -EFAULT; 875 goto err_xa_erase; 876 } 877 878 scoped_guard(mutex, &ctx->mmap_lock) { 879 ret = -ENOMEM; 880 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL)) 881 goto err_xa_erase; 882 } 883 884 return 0; 885 err_xa_erase: 886 scoped_guard(mutex, &ctx->mmap_lock) 887 xa_erase(&ctx->zcrx_ctxs, id); 888 err: 889 /* the import path never set the ->master_ctx ref, don't drop it */ 890 zcrx_unregister(ifq, NULL); 891 return ret; 892 } 893 894 static int zcrx_register_netdev(struct io_zcrx_ifq *ifq, 895 struct io_uring_zcrx_ifq_reg *reg, 896 struct io_uring_zcrx_area_reg *area) 897 { 898 struct pp_memory_provider_params mp_param = {}; 899 unsigned if_rxq = reg->if_rxq; 900 int ret; 901 902 ifq->netdev = netdev_get_by_index_lock(current->nsproxy->net_ns, 903 reg->if_idx); 904 if (!ifq->netdev) 905 return -ENODEV; 906 907 netdev_hold(ifq->netdev, &ifq->netdev_tracker, GFP_KERNEL); 908 909 ifq->dev = netdev_queue_get_dma_dev(ifq->netdev, if_rxq, NETDEV_QUEUE_TYPE_RX); 910 if (!ifq->dev) { 911 ret = -EOPNOTSUPP; 912 goto netdev_put_unlock; 913 } 914 get_device(ifq->dev); 915 916 ret = io_zcrx_create_area(ifq, area, reg); 917 if (ret) 918 goto netdev_put_unlock; 919 920 if (reg->rx_buf_len) 921 mp_param.rx_page_size = 1U << ifq->niov_shift; 922 mp_param.mp_ops = &io_uring_pp_zc_ops; 923 mp_param.mp_priv = ifq; 924 ret = netif_mp_open_rxq(ifq->netdev, if_rxq, &mp_param, NULL); 925 if (ret) 926 goto netdev_put_unlock; 927 928 ifq->if_rxq = if_rxq; 929 ret = 0; 930 netdev_put_unlock: 931 netdev_unlock(ifq->netdev); 932 return ret; 933 } 934 935 static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq, 936 const struct io_uring_zcrx_ifq_reg *reg, 937 const struct zcrx_event_desc *notif) 938 { 939 size_t stats_off = notif->stats_offset; 940 size_t used, end; 941 942 used = reg->offsets.rqes + 943 sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries; 944 945 if (!IS_ALIGNED(stats_off, __alignof__(struct zcrx_stats))) 946 return -EINVAL; 947 if (stats_off < used) 948 return -ERANGE; 949 if (check_add_overflow(stats_off, 950 sizeof(struct zcrx_stats), 951 &end)) 952 return -ERANGE; 953 if (end > io_region_size(&ifq->rq_region)) 954 return -ERANGE; 955 956 ifq->notif_stats = io_region_get_ptr(&ifq->rq_region) + stats_off; 957 memset(ifq->notif_stats, 0, sizeof(*ifq->notif_stats)); 958 959 return 0; 960 } 961 962 int io_register_zcrx(struct io_ring_ctx *ctx, 963 struct io_uring_zcrx_ifq_reg __user *arg) 964 { 965 struct zcrx_event_desc notif; 966 struct io_uring_zcrx_area_reg area; 967 struct io_uring_zcrx_ifq_reg reg; 968 struct io_uring_region_desc rd; 969 struct io_zcrx_ifq *ifq; 970 int ret; 971 u32 id; 972 973 /* 974 * 1. Interface queue allocation. 975 * 2. It can observe data destined for sockets of other tasks. 976 */ 977 if (!capable(CAP_NET_ADMIN)) 978 return -EPERM; 979 980 /* mandatory io_uring features for zc rx */ 981 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 982 return -EINVAL; 983 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))) 984 return -EINVAL; 985 if (copy_from_user(®, arg, sizeof(reg))) 986 return -EFAULT; 987 if (!mem_is_zero(®.__resv, sizeof(reg.__resv)) || reg.zcrx_id) 988 return -EINVAL; 989 if (reg.flags & ~ZCRX_SUPPORTED_REG_FLAGS) 990 return -EINVAL; 991 if (reg.flags & ZCRX_REG_IMPORT) 992 return import_zcrx(ctx, arg, ®); 993 if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd))) 994 return -EFAULT; 995 if (reg.if_rxq == -1 || !reg.rq_entries) 996 return -EINVAL; 997 if ((reg.if_rxq || reg.if_idx) && (reg.flags & ZCRX_REG_NODEV)) 998 return -EINVAL; 999 if (reg.rq_entries > IO_RQ_MAX_ENTRIES) { 1000 if (!(ctx->flags & IORING_SETUP_CLAMP)) 1001 return -EINVAL; 1002 reg.rq_entries = IO_RQ_MAX_ENTRIES; 1003 } 1004 reg.rq_entries = roundup_pow_of_two(reg.rq_entries); 1005 1006 if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area))) 1007 return -EFAULT; 1008 if (area.rq_area_token) 1009 return -EINVAL; 1010 1011 memset(¬if, 0, sizeof(notif)); 1012 if (reg.event_desc && copy_from_user(¬if, u64_to_user_ptr(reg.event_desc), 1013 sizeof(notif))) 1014 return -EFAULT; 1015 if (notif.type_mask & ~ZCRX_EVENT_TYPE_MASK) 1016 return -EINVAL; 1017 if (notif.flags & ~ZCRX_EVENT_DESC_FLAG_STATS) 1018 return -EINVAL; 1019 if (!(notif.flags & ZCRX_EVENT_DESC_FLAG_STATS)) { 1020 if (notif.stats_offset) 1021 return -EINVAL; 1022 } 1023 if (!mem_is_zero(¬if.__resv2, sizeof(notif.__resv2))) 1024 return -EINVAL; 1025 1026 ifq = io_zcrx_ifq_alloc(ctx); 1027 if (!ifq) 1028 return -ENOMEM; 1029 1030 ifq->notif_data = notif.user_data; 1031 ifq->allowed_notif_mask = notif.type_mask; 1032 1033 if (ctx->user) { 1034 get_uid(ctx->user); 1035 ifq->user = ctx->user; 1036 } 1037 if (ctx->mm_account) { 1038 mmgrab(ctx->mm_account); 1039 ifq->mm_account = ctx->mm_account; 1040 } 1041 ifq->rq.nr_entries = reg.rq_entries; 1042 1043 scoped_guard(mutex, &ctx->mmap_lock) { 1044 /* preallocate id */ 1045 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL); 1046 if (ret) 1047 goto ifq_free; 1048 } 1049 1050 ret = io_allocate_rbuf_ring(ctx, ifq, ®, &rd, id); 1051 if (ret) 1052 goto err; 1053 1054 if (notif.flags & ZCRX_EVENT_DESC_FLAG_STATS) { 1055 ret = zcrx_validate_notif_stats(ifq, ®, ¬if); 1056 if (ret) 1057 goto err; 1058 } 1059 1060 ifq->kern_readable = !(area.flags & IORING_ZCRX_AREA_DMABUF); 1061 1062 if (!(reg.flags & ZCRX_REG_NODEV)) { 1063 ret = zcrx_register_netdev(ifq, ®, &area); 1064 if (ret) 1065 goto err; 1066 } else { 1067 ret = io_zcrx_create_area(ifq, &area, ®); 1068 if (ret) 1069 goto err; 1070 } 1071 1072 WARN_ON_ONCE(!ifq->niov_shift); 1073 1074 reg.zcrx_id = id; 1075 1076 scoped_guard(mutex, &ctx->mmap_lock) { 1077 /* publish ifq */ 1078 ret = -ENOMEM; 1079 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL)) 1080 goto err; 1081 } 1082 1083 reg.rx_buf_len = 1U << ifq->niov_shift; 1084 1085 if (copy_to_user(arg, ®, sizeof(reg)) || 1086 copy_to_user(u64_to_user_ptr(reg.region_ptr), &rd, sizeof(rd)) || 1087 copy_to_user(u64_to_user_ptr(reg.area_ptr), &area, sizeof(area))) { 1088 ret = -EFAULT; 1089 goto err; 1090 } 1091 1092 if (notif.type_mask) 1093 zcrx_set_ring_ctx(ifq, ctx); 1094 return 0; 1095 err: 1096 scoped_guard(mutex, &ctx->mmap_lock) 1097 xa_erase(&ctx->zcrx_ctxs, id); 1098 ifq_free: 1099 zcrx_unregister(ifq, ctx); 1100 return ret; 1101 } 1102 1103 static inline bool is_zcrx_entry_marked(struct io_ring_ctx *ctx, unsigned long id) 1104 { 1105 return xa_get_mark(&ctx->zcrx_ctxs, id, XA_MARK_1); 1106 } 1107 1108 static inline void set_zcrx_entry_mark(struct io_ring_ctx *ctx, unsigned long id) 1109 { 1110 xa_set_mark(&ctx->zcrx_ctxs, id, XA_MARK_1); 1111 } 1112 1113 void io_terminate_zcrx(struct io_ring_ctx *ctx) 1114 { 1115 struct io_zcrx_ifq *ifq; 1116 unsigned long id = 0; 1117 1118 lockdep_assert_held(&ctx->uring_lock); 1119 1120 while (1) { 1121 scoped_guard(mutex, &ctx->mmap_lock) 1122 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); 1123 if (!ifq) 1124 break; 1125 if (WARN_ON_ONCE(is_zcrx_entry_marked(ctx, id))) 1126 break; 1127 set_zcrx_entry_mark(ctx, id); 1128 id++; 1129 zcrx_unregister_user(ifq, ctx); 1130 } 1131 } 1132 1133 void io_unregister_zcrx(struct io_ring_ctx *ctx) 1134 { 1135 struct io_zcrx_ifq *ifq; 1136 1137 lockdep_assert_held(&ctx->uring_lock); 1138 1139 while (1) { 1140 scoped_guard(mutex, &ctx->mmap_lock) { 1141 unsigned long id = 0; 1142 1143 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); 1144 if (ifq) { 1145 if (WARN_ON_ONCE(!is_zcrx_entry_marked(ctx, id))) { 1146 ifq = NULL; 1147 break; 1148 } 1149 xa_erase(&ctx->zcrx_ctxs, id); 1150 } 1151 } 1152 if (!ifq) 1153 break; 1154 /* 1155 * io_uring can run requests and return buffers to the user 1156 * after termination, scrub it again. 1157 */ 1158 if (refcount_read(&ifq->user_refs) == 0) 1159 io_zcrx_scrub(ifq); 1160 io_put_zcrx_ifq(ifq); 1161 } 1162 1163 xa_destroy(&ctx->zcrx_ctxs); 1164 } 1165 1166 struct zcrx_rq_iter { 1167 int rqes_left; 1168 bool flushed; 1169 }; 1170 1171 static inline u32 __zcrx_rq_entries(struct zcrx_rq *rq) 1172 { 1173 u32 entries = rq->cached_tail - rq->cached_head; 1174 1175 return min(entries, rq->nr_entries); 1176 } 1177 1178 static inline u32 zcrx_rq_entries(struct zcrx_rq *rq) 1179 { 1180 rq->cached_tail = smp_load_acquire(&rq->ring->tail); 1181 return __zcrx_rq_entries(rq); 1182 } 1183 1184 static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask) 1185 { 1186 unsigned int idx = rq->cached_head++ & mask; 1187 1188 return &rq->rqes[idx]; 1189 } 1190 1191 static inline void zcrx_rq_iter_init(struct zcrx_rq_iter *it, 1192 struct zcrx_rq *rq) 1193 { 1194 it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), ZCRX_REFILL_CAP); 1195 it->flushed = false; 1196 } 1197 1198 static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it, 1199 struct zcrx_rq *rq, 1200 struct io_uring_zcrx_rqe **rqe) 1201 { 1202 it->rqes_left--; 1203 if (unlikely(it->rqes_left < 0)) { 1204 if (it->flushed) 1205 return false; 1206 rq->cached_tail = smp_load_acquire(&rq->ring->tail); 1207 it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), 1208 ZCRX_REFILL_CAP); 1209 it->flushed = true; 1210 if (--it->rqes_left < 0) 1211 return false; 1212 } 1213 1214 *rqe = zcrx_next_rqe(rq, rq->nr_entries - 1); 1215 return true; 1216 } 1217 1218 static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, 1219 struct io_zcrx_ifq *ifq, 1220 struct net_iov **ret_niov) 1221 { 1222 __u64 off = READ_ONCE(rqe->off); 1223 unsigned niov_idx, area_idx; 1224 struct io_zcrx_area *area; 1225 1226 lockdep_assert_held(&ifq->rq.lock); 1227 1228 area_idx = off >> IORING_ZCRX_AREA_SHIFT; 1229 niov_idx = (off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; 1230 1231 if (unlikely(rqe->__pad || area_idx >= ifq->nr_areas)) 1232 return false; 1233 area_idx = array_index_nospec(area_idx, ifq->nr_areas); 1234 area = ifq->areas[area_idx]; 1235 1236 if (unlikely(niov_idx >= area->nia.num_niovs)) 1237 return false; 1238 niov_idx = array_index_nospec(niov_idx, area->nia.num_niovs); 1239 1240 *ret_niov = &area->nia.niovs[niov_idx]; 1241 return true; 1242 } 1243 1244 static bool zcrx_put_refill_niov(struct net_iov *niov, struct page_pool *pp, 1245 unsigned refs) 1246 { 1247 netmem_ref netmem = net_iov_to_netmem(niov); 1248 1249 if (!io_zcrx_put_niov_uref(niov, refs)) 1250 return false; 1251 if (page_pool_unref_netmem(netmem, refs) != 0) 1252 return false; 1253 if (unlikely(niov->desc.pp != pp)) { 1254 io_zcrx_return_niov(niov); 1255 return false; 1256 } 1257 return true; 1258 } 1259 1260 static unsigned io_zcrx_ring_refill(struct page_pool *pp, 1261 struct io_zcrx_ifq *ifq, 1262 netmem_ref *netmems, unsigned to_alloc) 1263 { 1264 struct zcrx_rq *rq = &ifq->rq; 1265 struct io_uring_zcrx_rqe *rqe; 1266 struct zcrx_rq_iter it; 1267 struct net_iov *niov = NULL; 1268 unsigned niov_refs = 0; 1269 unsigned allocated = 0; 1270 1271 guard(spinlock_bh)(&rq->lock); 1272 1273 zcrx_rq_iter_init(&it, rq); 1274 1275 while (allocated < to_alloc - 1 && zcrx_rq_iter_next(&it, rq, &rqe)) { 1276 struct net_iov *next_niov; 1277 1278 if (!io_parse_rqe(rqe, ifq, &next_niov)) 1279 continue; 1280 if (niov == next_niov) { 1281 niov_refs++; 1282 continue; 1283 } 1284 if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) { 1285 netmems[allocated] = net_iov_to_netmem(niov); 1286 allocated++; 1287 } 1288 niov = next_niov; 1289 niov_refs = 1; 1290 } 1291 1292 if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) { 1293 netmems[allocated] = net_iov_to_netmem(niov); 1294 allocated++; 1295 } 1296 1297 smp_store_release(&rq->ring->head, rq->cached_head); 1298 return allocated; 1299 } 1300 1301 static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq, 1302 netmem_ref *netmems, unsigned to_alloc) 1303 { 1304 unsigned area_idx = 0; 1305 unsigned allocated = 0; 1306 1307 guard(spinlock_bh)(&ifq->alloc_lock); 1308 1309 while (allocated < to_alloc) { 1310 struct net_iov *niov = zcrx_get_free_niov(ifq->areas[area_idx]); 1311 1312 if (!niov) { 1313 area_idx++; 1314 if (area_idx >= ifq->nr_areas) 1315 break; 1316 continue; 1317 } 1318 1319 net_mp_niov_set_page_pool(pp, niov); 1320 netmems[allocated] = net_iov_to_netmem(niov); 1321 allocated++; 1322 } 1323 return allocated; 1324 } 1325 1326 static void zcrx_notif_tw(struct io_tw_req tw_req, io_tw_token_t tw) 1327 { 1328 struct io_kiocb *req = tw_req.req; 1329 struct io_ring_ctx *ctx = req->ctx; 1330 1331 io_post_aux_cqe(ctx, req->cqe.user_data, req->cqe.res, 0); 1332 percpu_ref_put(&ctx->refs); 1333 io_poison_req(req); 1334 kmem_cache_free(req_cachep, req); 1335 } 1336 1337 static void zcrx_stat_add(__u64 *p, s64 v) 1338 { 1339 WRITE_ONCE(*p, READ_ONCE(*p) + v); 1340 } 1341 1342 static void zcrx_send_notif(struct io_zcrx_ifq *ifq, unsigned type) 1343 { 1344 gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO; 1345 u32 type_mask = 1 << type; 1346 struct io_kiocb *req; 1347 1348 if (!(type_mask & ifq->allowed_notif_mask)) 1349 return; 1350 1351 guard(spinlock_bh)(&ifq->ctx_lock); 1352 if (!ifq->master_ctx) 1353 return; 1354 if (type_mask & ifq->fired_notifs) 1355 return; 1356 1357 req = kmem_cache_alloc(req_cachep, gfp); 1358 if (unlikely(!req)) 1359 return; 1360 1361 ifq->fired_notifs |= type_mask; 1362 1363 req->opcode = IORING_OP_NOP; 1364 req->cqe.user_data = ifq->notif_data; 1365 req->cqe.res = type; 1366 req->ctx = ifq->master_ctx; 1367 percpu_ref_get(&req->ctx->refs); 1368 req->tctx = NULL; 1369 req->io_task_work.func = zcrx_notif_tw; 1370 io_req_task_work_add(req); 1371 } 1372 1373 static netmem_ref io_pp_zc_alloc_netmems(struct page_pool *pp, gfp_t gfp) 1374 { 1375 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp); 1376 netmem_ref *netmems = pp->alloc.cache; 1377 unsigned to_alloc = PP_ALLOC_CACHE_REFILL; 1378 unsigned allocated; 1379 1380 /* pp should already be ensuring that */ 1381 if (WARN_ON_ONCE(pp->alloc.count)) 1382 return 0; 1383 1384 allocated = io_zcrx_ring_refill(pp, ifq, netmems, to_alloc); 1385 if (likely(allocated)) 1386 goto out_return; 1387 1388 allocated = io_zcrx_refill_slow(pp, ifq, netmems, to_alloc); 1389 if (!allocated) { 1390 zcrx_send_notif(ifq, ZCRX_EVENT_ALLOC_FAIL); 1391 return 0; 1392 } 1393 out_return: 1394 zcrx_sync_for_device(pp, ifq, netmems, allocated); 1395 allocated--; 1396 pp->alloc.count += allocated; 1397 return netmems[allocated]; 1398 } 1399 1400 static bool io_pp_zc_release_netmem(struct page_pool *pp, netmem_ref netmem) 1401 { 1402 struct net_iov *niov; 1403 1404 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 1405 return false; 1406 1407 niov = netmem_to_net_iov(netmem); 1408 net_mp_niov_clear_page_pool(niov); 1409 io_zcrx_return_niov_freelist(niov); 1410 return false; 1411 } 1412 1413 static int io_pp_zc_init(struct page_pool *pp) 1414 { 1415 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp); 1416 1417 if (WARN_ON_ONCE(!ifq)) 1418 return -EINVAL; 1419 if (WARN_ON_ONCE(ifq->dev != pp->p.dev)) 1420 return -EINVAL; 1421 if (WARN_ON_ONCE(!pp->dma_map)) 1422 return -EOPNOTSUPP; 1423 if (pp->p.order + PAGE_SHIFT != ifq->niov_shift) 1424 return -EINVAL; 1425 if (pp->p.dma_dir != DMA_FROM_DEVICE) 1426 return -EOPNOTSUPP; 1427 1428 refcount_inc(&ifq->refs); 1429 return 0; 1430 } 1431 1432 static void io_pp_zc_destroy(struct page_pool *pp) 1433 { 1434 io_put_zcrx_ifq(io_pp_to_ifq(pp)); 1435 } 1436 1437 static int io_pp_nl_fill(void *mp_priv, struct sk_buff *rsp, 1438 struct netdev_rx_queue *rxq) 1439 { 1440 struct io_zcrx_ifq *ifq = mp_priv; 1441 struct nlattr *nest; 1442 int type; 1443 1444 type = rxq ? NETDEV_A_QUEUE_IO_URING : NETDEV_A_PAGE_POOL_IO_URING; 1445 nest = nla_nest_start(rsp, type); 1446 if (!nest) 1447 return -EMSGSIZE; 1448 1449 if (nla_put_uint(rsp, NETDEV_A_IO_URING_PROVIDER_INFO_RX_BUF_LEN, 1450 1ULL << ifq->niov_shift)) { 1451 nla_nest_cancel(rsp, nest); 1452 return -EMSGSIZE; 1453 } 1454 1455 nla_nest_end(rsp, nest); 1456 1457 return 0; 1458 } 1459 1460 static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq) 1461 { 1462 struct pp_memory_provider_params *p = &rxq->mp_params; 1463 struct io_zcrx_ifq *ifq = mp_priv; 1464 1465 guard(mutex)(&ifq->pp_lock); 1466 io_zcrx_unmap_areas(ifq); 1467 io_zcrx_drop_netdev(ifq); 1468 1469 p->mp_ops = NULL; 1470 p->mp_priv = NULL; 1471 } 1472 1473 static const struct memory_provider_ops io_uring_pp_zc_ops = { 1474 .alloc_netmems = io_pp_zc_alloc_netmems, 1475 .release_netmem = io_pp_zc_release_netmem, 1476 .init = io_pp_zc_init, 1477 .destroy = io_pp_zc_destroy, 1478 .nl_fill = io_pp_nl_fill, 1479 .uninstall = io_pp_uninstall, 1480 }; 1481 1482 static unsigned zcrx_parse_rq(netmem_ref *netmem_array, unsigned nr, 1483 struct io_zcrx_ifq *zcrx, struct zcrx_rq *rq) 1484 { 1485 unsigned int mask = rq->nr_entries - 1; 1486 unsigned int i; 1487 1488 nr = min(nr, zcrx_rq_entries(rq)); 1489 for (i = 0; i < nr; i++) { 1490 struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask); 1491 struct net_iov *niov; 1492 1493 if (!io_parse_rqe(rqe, zcrx, &niov)) 1494 break; 1495 netmem_array[i] = net_iov_to_netmem(niov); 1496 } 1497 1498 smp_store_release(&rq->ring->head, rq->cached_head); 1499 return i; 1500 } 1501 1502 #define ZCRX_FLUSH_BATCH 32 1503 1504 static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr) 1505 { 1506 unsigned i; 1507 1508 for (i = 0; i < nr; i++) { 1509 netmem_ref netmem = netmems[i]; 1510 struct net_iov *niov = netmem_to_net_iov(netmem); 1511 1512 if (!io_zcrx_put_niov_uref(niov, 1)) 1513 continue; 1514 if (!page_pool_unref_and_test(netmem)) 1515 continue; 1516 io_zcrx_return_niov(niov); 1517 } 1518 } 1519 1520 static int zcrx_flush_rq(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, 1521 struct zcrx_ctrl *ctrl) 1522 { 1523 struct zcrx_ctrl_flush_rq *frq = &ctrl->zc_flush; 1524 netmem_ref netmems[ZCRX_FLUSH_BATCH]; 1525 unsigned total = 0; 1526 unsigned nr; 1527 1528 if (!mem_is_zero(&frq->__resv, sizeof(frq->__resv))) 1529 return -EINVAL; 1530 1531 do { 1532 struct zcrx_rq *rq = &zcrx->rq; 1533 1534 scoped_guard(spinlock_bh, &rq->lock) { 1535 nr = zcrx_parse_rq(netmems, ZCRX_FLUSH_BATCH, zcrx, rq); 1536 zcrx_return_buffers(netmems, nr); 1537 } 1538 1539 total += nr; 1540 1541 if (fatal_signal_pending(current)) 1542 break; 1543 cond_resched(); 1544 } while (nr == ZCRX_FLUSH_BATCH && total < zcrx->rq.nr_entries); 1545 1546 return 0; 1547 } 1548 1549 static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, 1550 struct zcrx_ctrl *ctrl) 1551 { 1552 const struct zcrx_ctrl_arm_event *an = &ctrl->zc_arm_event; 1553 unsigned type_mask; 1554 1555 if (an->event_type >= __ZCRX_EVENT_TYPE_LAST) 1556 return -EINVAL; 1557 if (!mem_is_zero(&an->__resv, sizeof(an->__resv))) 1558 return -EINVAL; 1559 1560 guard(spinlock_bh)(&zcrx->ctx_lock); 1561 type_mask = 1U << an->event_type; 1562 if (type_mask & ~zcrx->fired_notifs) 1563 return -EINVAL; 1564 zcrx->fired_notifs &= ~type_mask; 1565 return 0; 1566 } 1567 1568 static int zcrx_ctrl_add_area(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, 1569 struct zcrx_ctrl *ctrl) 1570 { 1571 struct zcrx_ctrl_add_area *ctrl_add = &ctrl->zc_area; 1572 struct io_uring_zcrx_area_reg __user *area_uptr; 1573 struct io_uring_zcrx_area_reg area_reg; 1574 1575 area_uptr = u64_to_user_ptr(ctrl_add->area_ptr); 1576 if (copy_from_user(&area_reg, area_uptr, sizeof(area_reg))) 1577 return -EFAULT; 1578 if (!mem_is_zero(&ctrl_add->__resv, sizeof(ctrl_add->__resv))) 1579 return -EINVAL; 1580 if (area_reg.rq_area_token) 1581 return -EINVAL; 1582 1583 guard(mutex)(&ifq->pp_lock); 1584 if (ifq->dev && !ifq->netdev) 1585 return -EFAULT; 1586 1587 /* we can't safely roll back area append, copy it out first */ 1588 area_reg.rq_area_token = zcrx_area_id_to_token(zcrx_next_area_id(ifq)); 1589 if (copy_to_user(area_uptr, &area_reg, sizeof(area_reg))) 1590 return -EFAULT; 1591 area_reg.rq_area_token = 0; 1592 1593 return __zcrx_create_area(ifq, &area_reg, 1U << ifq->niov_shift); 1594 } 1595 1596 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) 1597 { 1598 struct zcrx_ctrl ctrl; 1599 struct io_zcrx_ifq *zcrx; 1600 1601 BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_flush)); 1602 BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_arm_event)); 1603 1604 if (nr_args) 1605 return -EINVAL; 1606 if (copy_from_user(&ctrl, arg, sizeof(ctrl))) 1607 return -EFAULT; 1608 if (!mem_is_zero(&ctrl.__resv, sizeof(ctrl.__resv))) 1609 return -EFAULT; 1610 1611 zcrx = xa_load(&ctx->zcrx_ctxs, ctrl.zcrx_id); 1612 if (!zcrx) 1613 return -ENXIO; 1614 1615 switch (ctrl.op) { 1616 case ZCRX_CTRL_FLUSH_RQ: 1617 return zcrx_flush_rq(ctx, zcrx, &ctrl); 1618 case ZCRX_CTRL_EXPORT: 1619 return zcrx_export(ctx, zcrx, &ctrl, arg); 1620 case ZCRX_CTRL_ARM_EVENT: 1621 return zcrx_arm_notif(ctx, zcrx, &ctrl); 1622 case ZCRX_CTRL_ADD_AREA: 1623 return zcrx_ctrl_add_area(ctx, zcrx, &ctrl); 1624 } 1625 1626 return -EOPNOTSUPP; 1627 } 1628 1629 static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, 1630 struct io_zcrx_ifq *ifq, int off, int len) 1631 { 1632 struct io_ring_ctx *ctx = req->ctx; 1633 struct io_uring_zcrx_cqe *rcqe; 1634 struct io_zcrx_area *area; 1635 struct io_uring_cqe *cqe; 1636 u64 offset; 1637 1638 if (!io_defer_get_uncommited_cqe(ctx, &cqe)) 1639 return false; 1640 1641 cqe->user_data = req->cqe.user_data; 1642 cqe->res = len; 1643 cqe->flags = IORING_CQE_F_MORE; 1644 if (ctx->flags & IORING_SETUP_CQE_MIXED) 1645 cqe->flags |= IORING_CQE_F_32; 1646 1647 area = io_zcrx_iov_to_area(niov); 1648 offset = off + (net_iov_idx(niov) << ifq->niov_shift); 1649 rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1); 1650 rcqe->off = offset + zcrx_area_id_to_token(area->area_id); 1651 rcqe->__pad = 0; 1652 return true; 1653 } 1654 1655 static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq) 1656 { 1657 struct net_iov *niov = NULL; 1658 unsigned area_idx; 1659 1660 if (!ifq->kern_readable) 1661 return NULL; 1662 1663 guard(spinlock_bh)(&ifq->alloc_lock); 1664 1665 for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) { 1666 niov = zcrx_get_free_niov(ifq->areas[area_idx]); 1667 if (niov) { 1668 page_pool_fragment_netmem(net_iov_to_netmem(niov), 1); 1669 return niov; 1670 } 1671 } 1672 1673 return NULL; 1674 } 1675 1676 struct io_copy_cache { 1677 struct page *page; 1678 unsigned long offset; 1679 size_t size; 1680 }; 1681 1682 static ssize_t io_copy_page(struct io_copy_cache *cc, struct page *src_page, 1683 unsigned int src_offset, size_t len) 1684 { 1685 size_t copied = 0; 1686 1687 len = min(len, cc->size); 1688 1689 while (len) { 1690 void *src_addr, *dst_addr; 1691 struct page *dst_page = cc->page; 1692 unsigned dst_offset = cc->offset; 1693 size_t n = len; 1694 1695 if (folio_test_partial_kmap(page_folio(dst_page)) || 1696 folio_test_partial_kmap(page_folio(src_page))) { 1697 dst_page += dst_offset / PAGE_SIZE; 1698 dst_offset = offset_in_page(dst_offset); 1699 src_page += src_offset / PAGE_SIZE; 1700 src_offset = offset_in_page(src_offset); 1701 n = min(PAGE_SIZE - src_offset, PAGE_SIZE - dst_offset); 1702 n = min(n, len); 1703 } 1704 1705 dst_addr = kmap_local_page(dst_page) + dst_offset; 1706 src_addr = kmap_local_page(src_page) + src_offset; 1707 1708 memcpy(dst_addr, src_addr, n); 1709 1710 kunmap_local(src_addr); 1711 kunmap_local(dst_addr); 1712 1713 cc->size -= n; 1714 cc->offset += n; 1715 src_offset += n; 1716 len -= n; 1717 copied += n; 1718 } 1719 return copied; 1720 } 1721 1722 static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1723 struct page *src_page, unsigned int src_offset, 1724 size_t len) 1725 { 1726 size_t copied = 0; 1727 int ret = 0; 1728 1729 while (len) { 1730 struct io_copy_cache cc; 1731 struct net_iov *niov; 1732 size_t n; 1733 1734 niov = io_alloc_fallback_niov(ifq); 1735 if (!niov) { 1736 ret = -ENOMEM; 1737 break; 1738 } 1739 1740 cc.page = io_zcrx_iov_page(niov); 1741 cc.offset = 0; 1742 cc.size = PAGE_SIZE; 1743 1744 n = io_copy_page(&cc, src_page, src_offset, len); 1745 1746 if (!io_zcrx_queue_cqe(req, niov, ifq, 0, n)) { 1747 io_zcrx_return_niov(niov); 1748 ret = -ENOSPC; 1749 break; 1750 } 1751 1752 io_zcrx_get_niov_uref(niov); 1753 src_offset += n; 1754 len -= n; 1755 copied += n; 1756 } 1757 1758 return copied ? copied : ret; 1759 } 1760 1761 static int io_zcrx_copy_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1762 const skb_frag_t *frag, int off, int len) 1763 { 1764 struct page *page = skb_frag_page(frag); 1765 int ret; 1766 1767 ret = io_zcrx_copy_chunk(req, ifq, page, off + skb_frag_off(frag), len); 1768 if (ret > 0) { 1769 if (ifq->notif_stats) { 1770 zcrx_stat_add(&ifq->notif_stats->copy_count, 1); 1771 zcrx_stat_add(&ifq->notif_stats->copy_bytes, ret); 1772 } 1773 zcrx_send_notif(ifq, ZCRX_EVENT_COPY); 1774 } 1775 1776 return ret; 1777 } 1778 1779 static int io_zcrx_recv_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1780 const skb_frag_t *frag, int off, int len) 1781 { 1782 struct net_iov *niov; 1783 struct page_pool *pp; 1784 1785 if (unlikely(!skb_frag_is_net_iov(frag))) 1786 return io_zcrx_copy_frag(req, ifq, frag, off, len); 1787 1788 niov = netmem_to_net_iov(frag->netmem); 1789 pp = niov->desc.pp; 1790 1791 if (!pp || pp->mp_ops != &io_uring_pp_zc_ops || io_pp_to_ifq(pp) != ifq) 1792 return -EFAULT; 1793 1794 if (!io_zcrx_queue_cqe(req, niov, ifq, off + skb_frag_off(frag), len)) 1795 return -ENOSPC; 1796 1797 /* 1798 * Prevent it from being recycled while user is accessing it. 1799 * It has to be done before grabbing a user reference. 1800 */ 1801 page_pool_ref_netmem(net_iov_to_netmem(niov)); 1802 io_zcrx_get_niov_uref(niov); 1803 return len; 1804 } 1805 1806 static int 1807 io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 1808 unsigned int offset, size_t len) 1809 { 1810 struct io_zcrx_args *args = desc->arg.data; 1811 struct io_zcrx_ifq *ifq = args->ifq; 1812 struct io_kiocb *req = args->req; 1813 struct sk_buff *frag_iter; 1814 unsigned start, start_off = offset; 1815 struct skb_shared_info *shi; 1816 int i, ret = 0; 1817 1818 len = min_t(size_t, len, desc->count); 1819 /* 1820 * __tcp_read_sock() always calls io_zcrx_recv_skb one last time, even 1821 * if desc->count is already 0. This is caused by the if (offset + 1 != 1822 * skb->len) check. Return early in this case to break out of 1823 * __tcp_read_sock(). 1824 */ 1825 if (!len) 1826 return 0; 1827 if (unlikely(args->nr_skbs++ > IO_SKBS_PER_CALL_LIMIT)) 1828 return -EAGAIN; 1829 1830 if (unlikely(offset < skb_headlen(skb))) { 1831 ssize_t copied; 1832 size_t to_copy; 1833 1834 to_copy = min_t(size_t, skb_headlen(skb) - offset, len); 1835 copied = io_zcrx_copy_chunk(req, ifq, virt_to_page(skb->data), 1836 offset_in_page(skb->data) + offset, 1837 to_copy); 1838 if (copied < 0) { 1839 ret = copied; 1840 goto out; 1841 } 1842 offset += copied; 1843 len -= copied; 1844 if (!len) 1845 goto out; 1846 if (offset != skb_headlen(skb)) 1847 goto out; 1848 } 1849 1850 start = skb_headlen(skb); 1851 shi = skb_shinfo(skb); 1852 1853 for (i = 0; i < shi->nr_frags; i++) { 1854 const skb_frag_t *frag = &shi->frags[i]; 1855 unsigned frag_end = start + skb_frag_size(frag); 1856 1857 if (WARN_ON(start > offset + len)) 1858 return -EFAULT; 1859 1860 if (offset < frag_end) { 1861 unsigned copy = min(frag_end - offset, len); 1862 unsigned frag_off = offset - start; 1863 1864 ret = io_zcrx_recv_frag(req, ifq, frag, frag_off, copy); 1865 if (ret < 0) 1866 goto out; 1867 1868 offset += ret; 1869 len -= ret; 1870 if (len == 0 || ret != copy) 1871 goto out; 1872 } 1873 start = frag_end; 1874 } 1875 1876 skb_walk_frags(skb, frag_iter) { 1877 unsigned frag_end; 1878 1879 if (WARN_ON(start > offset + len)) 1880 return -EFAULT; 1881 1882 frag_end = start + frag_iter->len; 1883 if (offset < frag_end) { 1884 unsigned copy = min(frag_end - offset, len); 1885 unsigned frag_off = offset - start; 1886 size_t count; 1887 1888 count = desc->count; 1889 ret = io_zcrx_recv_skb(desc, frag_iter, frag_off, copy); 1890 desc->count = count; 1891 if (ret < 0) 1892 goto out; 1893 1894 offset += ret; 1895 len -= ret; 1896 if (len == 0 || ret != copy) 1897 goto out; 1898 } 1899 start = frag_end; 1900 } 1901 1902 out: 1903 /* Bytes left in len mean an error stopped the walk early. */ 1904 if (len) 1905 args->stopped_early = true; 1906 if (offset == start_off) 1907 return ret; 1908 desc->count -= (offset - start_off); 1909 return offset - start_off; 1910 } 1911 1912 static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1913 struct sock *sk, int flags, 1914 unsigned issue_flags, unsigned int *outlen) 1915 { 1916 unsigned int len = *outlen; 1917 struct io_zcrx_args args = { 1918 .req = req, 1919 .ifq = ifq, 1920 }; 1921 read_descriptor_t rd_desc = { 1922 .count = len ? len : UINT_MAX, 1923 .arg.data = &args, 1924 }; 1925 int ret; 1926 1927 lock_sock(sk); 1928 ret = tcp_read_sock(sk, &rd_desc, io_zcrx_recv_skb); 1929 if (len && ret > 0) 1930 *outlen = len - ret; 1931 if (ret <= 0) { 1932 if (ret < 0 || sock_flag(sk, SOCK_DONE)) 1933 goto out; 1934 if (sk->sk_err) 1935 ret = sock_error(sk); 1936 else if (sk->sk_shutdown & RCV_SHUTDOWN) 1937 goto out; 1938 else if (sk->sk_state == TCP_CLOSE) 1939 ret = -ENOTCONN; 1940 else 1941 ret = -EAGAIN; 1942 } else if ((issue_flags & IO_URING_F_MULTISHOT) && 1943 (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) || 1944 args.stopped_early)) { 1945 ret = IOU_REQUEUE; 1946 } else if (sock_flag(sk, SOCK_DONE)) { 1947 /* Make it to retry until it finally gets 0. */ 1948 if (issue_flags & IO_URING_F_MULTISHOT) 1949 ret = IOU_REQUEUE; 1950 else 1951 ret = -EAGAIN; 1952 } 1953 out: 1954 release_sock(sk); 1955 return ret; 1956 } 1957 1958 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1959 struct socket *sock, unsigned int flags, 1960 unsigned issue_flags, unsigned int *len) 1961 { 1962 struct sock *sk = sock->sk; 1963 const struct proto *prot = READ_ONCE(sk->sk_prot); 1964 1965 if (prot->recvmsg != tcp_recvmsg) 1966 return -EPROTONOSUPPORT; 1967 1968 sock_rps_record_flow(sk); 1969 return io_zcrx_tcp_recvmsg(req, ifq, sk, flags, issue_flags, len); 1970 } 1971