1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/dma-map-ops.h> 5 #include <linux/mm.h> 6 #include <linux/nospec.h> 7 #include <linux/io_uring.h> 8 #include <linux/netdevice.h> 9 #include <linux/rtnetlink.h> 10 #include <linux/skbuff_ref.h> 11 #include <linux/anon_inodes.h> 12 13 #include <net/page_pool/helpers.h> 14 #include <net/page_pool/memory_provider.h> 15 #include <net/netlink.h> 16 #include <net/netdev_queues.h> 17 #include <net/netdev_rx_queue.h> 18 #include <net/tcp.h> 19 #include <net/rps.h> 20 21 #include <trace/events/page_pool.h> 22 23 #include <uapi/linux/io_uring.h> 24 25 #include "io_uring.h" 26 #include "kbuf.h" 27 #include "memmap.h" 28 #include "zcrx.h" 29 #include "rsrc.h" 30 31 #define ZCRX_MAX_FRAGS_PER_PAGE MAX(PAGE_SIZE / 1024, 1) 32 /* 33 * We need a reasonable limit to be able to fill in 64 entries on average 34 * for 1500 byte MTU. Over-estimate it to keep it pow2. 35 */ 36 #define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024) 37 38 #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) 39 #define ZCRX_MAX_AREAS 1024 40 41 #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) 42 43 static inline u32 zcrx_next_area_id(struct io_zcrx_ifq *zcrx) 44 { 45 return zcrx->nr_areas; 46 } 47 48 static inline u64 zcrx_area_id_to_token(u32 area_id) 49 { 50 return (u64)area_id << IORING_ZCRX_AREA_SHIFT; 51 } 52 53 static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp) 54 { 55 return pp->mp_priv; 56 } 57 58 static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *niov) 59 { 60 struct net_iov_area *owner = net_iov_owner(niov); 61 62 return container_of(owner, struct io_zcrx_area, nia); 63 } 64 65 static bool zcrx_set_ring_ctx(struct io_zcrx_ifq *zcrx, 66 struct io_ring_ctx *ctx) 67 { 68 guard(spinlock_bh)(&zcrx->ctx_lock); 69 if (zcrx->master_ctx) 70 return false; 71 percpu_ref_get(&ctx->refs); 72 zcrx->master_ctx = ctx; 73 return true; 74 } 75 76 static inline struct page *io_zcrx_iov_page(const struct net_iov *niov) 77 { 78 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); 79 unsigned niov_pages_shift; 80 81 lockdep_assert(!area->mem.is_dmabuf); 82 83 niov_pages_shift = area->ifq->niov_shift - PAGE_SHIFT; 84 return area->mem.pages[net_iov_idx(niov) << niov_pages_shift]; 85 } 86 87 static int io_area_max_shift(struct io_zcrx_mem *mem) 88 { 89 struct sg_table *sgt = mem->sgt; 90 struct scatterlist *sg; 91 unsigned shift = -1U; 92 unsigned i; 93 94 for_each_sgtable_dma_sg(sgt, sg, i) 95 shift = min(shift, __ffs(sg_dma_len(sg))); 96 return shift; 97 } 98 99 static int io_populate_area_dma(struct io_zcrx_ifq *ifq, 100 struct io_zcrx_area *area) 101 { 102 unsigned niov_size = 1U << ifq->niov_shift; 103 struct sg_table *sgt = area->mem.sgt; 104 struct scatterlist *sg; 105 unsigned i, niov_idx = 0; 106 107 for_each_sgtable_dma_sg(sgt, sg, i) { 108 dma_addr_t dma = sg_dma_address(sg); 109 unsigned long sg_len = sg_dma_len(sg); 110 111 if (WARN_ON_ONCE(sg_len % niov_size)) 112 return -EINVAL; 113 114 while (sg_len && niov_idx < area->nia.num_niovs) { 115 struct net_iov *niov = &area->nia.niovs[niov_idx]; 116 117 if (net_mp_niov_set_dma_addr(niov, dma)) 118 return -EFAULT; 119 sg_len -= niov_size; 120 dma += niov_size; 121 niov_idx++; 122 } 123 } 124 125 if (WARN_ON_ONCE(niov_idx != area->nia.num_niovs)) 126 return -EFAULT; 127 return 0; 128 } 129 130 static void io_unmap_dmabuf(struct io_zcrx_mem *mem) 131 { 132 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) 133 return; 134 if (mem->sgt) 135 dma_buf_unmap_attachment_unlocked(mem->attach, mem->sgt, 136 DMA_FROM_DEVICE); 137 if (mem->attach) 138 dma_buf_detach(mem->dmabuf, mem->attach); 139 mem->sgt = NULL; 140 mem->attach = NULL; 141 } 142 143 static void io_release_dmabuf(struct io_zcrx_mem *mem) 144 { 145 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) 146 return; 147 if (mem->dmabuf) 148 dma_buf_put(mem->dmabuf); 149 mem->dmabuf = NULL; 150 } 151 152 static int io_import_dmabuf(struct io_zcrx_ifq *ifq, 153 struct io_zcrx_mem *mem, 154 const struct io_uring_zcrx_area_reg *area_reg) 155 { 156 unsigned long off = (unsigned long)area_reg->addr; 157 unsigned long len = (unsigned long)area_reg->len; 158 unsigned long total_size = 0; 159 struct scatterlist *sg; 160 int dmabuf_fd = area_reg->dmabuf_fd; 161 int i, ret; 162 163 if (!ifq->dev) 164 return -EINVAL; 165 if (off) 166 return -EINVAL; 167 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) 168 return -EINVAL; 169 170 mem->is_dmabuf = true; 171 mem->dmabuf = dma_buf_get(dmabuf_fd); 172 if (IS_ERR(mem->dmabuf)) { 173 ret = PTR_ERR(mem->dmabuf); 174 mem->dmabuf = NULL; 175 goto err; 176 } 177 178 mem->attach = dma_buf_attach(mem->dmabuf, ifq->dev); 179 if (IS_ERR(mem->attach)) { 180 ret = PTR_ERR(mem->attach); 181 mem->attach = NULL; 182 goto err; 183 } 184 185 mem->sgt = dma_buf_map_attachment_unlocked(mem->attach, DMA_FROM_DEVICE); 186 if (IS_ERR(mem->sgt)) { 187 ret = PTR_ERR(mem->sgt); 188 mem->sgt = NULL; 189 goto err; 190 } 191 192 for_each_sgtable_dma_sg(mem->sgt, sg, i) 193 total_size += sg_dma_len(sg); 194 195 if (total_size != len) { 196 ret = -EINVAL; 197 goto err; 198 } 199 200 mem->size = len; 201 return 0; 202 err: 203 io_unmap_dmabuf(mem); 204 io_release_dmabuf(mem); 205 return ret; 206 } 207 208 static unsigned long io_count_account_pages(struct page **pages, unsigned nr_pages) 209 { 210 struct folio *last_folio = NULL; 211 unsigned long res = 0; 212 int i; 213 214 for (i = 0; i < nr_pages; i++) { 215 struct folio *folio = page_folio(pages[i]); 216 217 if (folio == last_folio) 218 continue; 219 last_folio = folio; 220 res += folio_nr_pages(folio); 221 } 222 return res; 223 } 224 225 static int io_import_umem(struct io_zcrx_ifq *ifq, 226 struct io_zcrx_mem *mem, 227 const struct io_uring_zcrx_area_reg *area_reg) 228 { 229 struct page **pages; 230 int nr_pages, ret; 231 bool mapped = false; 232 233 if (area_reg->dmabuf_fd) 234 return -EINVAL; 235 if (!area_reg->addr) 236 return -EFAULT; 237 pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len, 238 &nr_pages); 239 if (IS_ERR(pages)) 240 return PTR_ERR(pages); 241 242 ret = sg_alloc_table_from_pages(&mem->page_sg_table, pages, nr_pages, 243 0, (unsigned long)nr_pages << PAGE_SHIFT, 244 GFP_KERNEL_ACCOUNT); 245 if (ret) 246 goto out_err; 247 248 if (ifq->dev) { 249 ret = dma_map_sgtable(ifq->dev, &mem->page_sg_table, 250 DMA_FROM_DEVICE, IO_DMA_ATTR); 251 if (ret < 0) 252 goto out_err; 253 mapped = true; 254 } 255 256 mem->account_pages = io_count_account_pages(pages, nr_pages); 257 ret = io_account_mem(ifq->user, ifq->mm_account, mem->account_pages); 258 if (ret < 0) { 259 mem->account_pages = 0; 260 goto out_err; 261 } 262 263 mem->sgt = &mem->page_sg_table; 264 mem->pages = pages; 265 mem->nr_folios = nr_pages; 266 mem->size = area_reg->len; 267 return ret; 268 out_err: 269 if (mapped) 270 dma_unmap_sgtable(ifq->dev, &mem->page_sg_table, 271 DMA_FROM_DEVICE, IO_DMA_ATTR); 272 sg_free_table(&mem->page_sg_table); 273 unpin_user_pages(pages, nr_pages); 274 kvfree(pages); 275 return ret; 276 } 277 278 static void io_release_area_mem(struct io_zcrx_mem *mem) 279 { 280 if (mem->is_dmabuf) { 281 io_release_dmabuf(mem); 282 } else if (mem->pages) { 283 unpin_user_pages(mem->pages, mem->nr_folios); 284 sg_free_table(mem->sgt); 285 kvfree(mem->pages); 286 } 287 mem->pages = IO_URING_PTR_POISON; 288 mem->sgt = IO_URING_PTR_POISON; 289 } 290 291 static int io_import_area(struct io_zcrx_ifq *ifq, 292 struct io_zcrx_mem *mem, 293 const struct io_uring_zcrx_area_reg *area_reg) 294 { 295 int ret; 296 297 if (area_reg->flags & ~IO_ZCRX_AREA_SUPPORTED_FLAGS) 298 return -EINVAL; 299 if (area_reg->rq_area_token) 300 return -EINVAL; 301 if (area_reg->__resv2[0] || area_reg->__resv2[1]) 302 return -EINVAL; 303 304 ret = io_validate_user_buf_range(area_reg->addr, area_reg->len); 305 if (ret) 306 return ret; 307 if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK) 308 return -EINVAL; 309 310 if (area_reg->flags & IORING_ZCRX_AREA_DMABUF) 311 return io_import_dmabuf(ifq, mem, area_reg); 312 return io_import_umem(ifq, mem, area_reg); 313 } 314 315 static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, 316 struct io_zcrx_area *area) 317 { 318 int i; 319 320 lockdep_assert_held(&ifq->pp_lock); 321 322 if (!area || !area->is_mapped) 323 return; 324 area->is_mapped = false; 325 326 if (area->nia.niovs) { 327 for (i = 0; i < area->nia.num_niovs; i++) 328 net_mp_niov_set_dma_addr(&area->nia.niovs[i], 0); 329 } 330 331 if (area->mem.is_dmabuf) { 332 io_unmap_dmabuf(&area->mem); 333 } else { 334 dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table, 335 DMA_FROM_DEVICE, IO_DMA_ATTR); 336 } 337 } 338 339 static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq) 340 { 341 unsigned area_idx; 342 343 lockdep_assert_held(&ifq->pp_lock); 344 345 for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) 346 __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]); 347 } 348 349 static void zcrx_sync_for_device(struct page_pool *pp, struct io_zcrx_ifq *zcrx, 350 netmem_ref *netmems, unsigned nr) 351 { 352 #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) 353 struct device *dev = pp->p.dev; 354 unsigned i, niov_size; 355 dma_addr_t dma_addr; 356 357 if (!dma_dev_need_sync(dev)) 358 return; 359 niov_size = 1U << zcrx->niov_shift; 360 361 for (i = 0; i < nr; i++) { 362 dma_addr = page_pool_get_dma_addr_netmem(netmems[i]); 363 __dma_sync_single_for_device(dev, dma_addr + pp->p.offset, 364 niov_size, pp->p.dma_dir); 365 } 366 #endif 367 } 368 369 #define IO_RQ_MAX_ENTRIES 32768 370 371 #define IO_SKBS_PER_CALL_LIMIT 20 372 373 struct io_zcrx_args { 374 struct io_kiocb *req; 375 struct io_zcrx_ifq *ifq; 376 unsigned nr_skbs; 377 }; 378 379 static const struct memory_provider_ops io_uring_pp_zc_ops; 380 381 static inline atomic_t *io_get_user_counter(struct net_iov *niov) 382 { 383 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); 384 385 return &area->user_refs[net_iov_idx(niov)]; 386 } 387 388 static bool io_zcrx_put_niov_uref(struct net_iov *niov, unsigned refs) 389 { 390 atomic_t *uref = io_get_user_counter(niov); 391 int old; 392 393 old = atomic_read(uref); 394 do { 395 if (unlikely(old < refs)) 396 return false; 397 } while (!atomic_try_cmpxchg(uref, &old, old - refs)); 398 399 return true; 400 } 401 402 static void io_zcrx_get_niov_uref(struct net_iov *niov) 403 { 404 atomic_inc(io_get_user_counter(niov)); 405 } 406 407 static void io_fill_zcrx_offsets(struct io_uring_zcrx_offsets *offsets) 408 { 409 offsets->head = offsetof(struct zcrx_rq_hdr, head); 410 offsets->tail = offsetof(struct zcrx_rq_hdr, tail); 411 offsets->rqes = ALIGN(sizeof(struct zcrx_rq_hdr), L1_CACHE_BYTES); 412 } 413 414 static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, 415 struct io_zcrx_ifq *ifq, 416 struct io_uring_zcrx_ifq_reg *reg, 417 struct io_uring_region_desc *rd, 418 u32 id) 419 { 420 u64 mmap_offset; 421 size_t off, size; 422 void *ptr; 423 int ret; 424 425 io_fill_zcrx_offsets(®->offsets); 426 off = reg->offsets.rqes; 427 size = off + sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries; 428 if (size > rd->size) 429 return -EINVAL; 430 431 mmap_offset = IORING_MAP_OFF_ZCRX_REGION; 432 mmap_offset += (u64)id << IORING_OFF_ZCRX_SHIFT; 433 434 ret = io_create_region(ctx, &ifq->rq_region, rd, mmap_offset); 435 if (ret < 0) 436 return ret; 437 438 ptr = io_region_get_ptr(&ifq->rq_region); 439 ifq->rq.ring = (struct zcrx_rq_hdr *)ptr; 440 ifq->rq.rqes = (struct io_uring_zcrx_rqe *)(ptr + off); 441 442 memset(ifq->rq.ring, 0, sizeof(*ifq->rq.ring)); 443 return 0; 444 } 445 446 static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq) 447 { 448 io_free_region(ifq->user, &ifq->rq_region); 449 ifq->rq.ring = IO_URING_PTR_POISON; 450 ifq->rq.rqes = IO_URING_PTR_POISON; 451 ifq->notif_stats = IO_URING_PTR_POISON; 452 } 453 454 static void io_zcrx_free_area(struct io_zcrx_ifq *ifq, 455 struct io_zcrx_area *area) 456 { 457 if (WARN_ON_ONCE(area->is_mapped)) 458 return; 459 io_release_area_mem(&area->mem); 460 461 if (area->mem.account_pages) 462 io_unaccount_mem(ifq->user, ifq->mm_account, 463 area->mem.account_pages); 464 465 kvfree(area->freelist); 466 kvfree(area->nia.niovs); 467 kvfree(area->user_refs); 468 kfree(area); 469 } 470 471 static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, 472 struct io_zcrx_area *area) 473 { 474 bool kern_readable = !area->mem.is_dmabuf; 475 struct io_zcrx_area **areas, **old_areas; 476 unsigned old_nr; 477 478 if (ifq->kern_readable != kern_readable) 479 return -EINVAL; 480 if (ifq->nr_areas + 1 > ZCRX_MAX_AREAS) 481 return -EINVAL; 482 483 old_areas = ifq->areas; 484 old_nr = ifq->nr_areas; 485 486 areas = kmalloc_array(old_nr + 1, sizeof(areas[0]), 487 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 488 if (!areas) 489 return -ENOMEM; 490 if (old_areas) 491 memcpy(areas, old_areas, old_nr * sizeof(areas[0])); 492 areas[old_nr] = area; 493 494 scoped_guard(spinlock_bh, &ifq->rq.lock) { 495 guard(spinlock_bh)(&ifq->alloc_lock); 496 ifq->areas = areas; 497 ifq->nr_areas = old_nr + 1; 498 } 499 kfree(old_areas); 500 return 0; 501 } 502 503 static int __zcrx_create_area(struct io_zcrx_ifq *ifq, 504 struct io_uring_zcrx_area_reg *area_reg, 505 u32 rx_buf_len) 506 { 507 int buf_size_shift = PAGE_SHIFT; 508 struct io_zcrx_area *area; 509 unsigned nr_iovs; 510 int i, ret; 511 512 lockdep_assert_held(&ifq->pp_lock); 513 514 if (rx_buf_len) { 515 if (!is_power_of_2(rx_buf_len) || rx_buf_len < PAGE_SIZE) 516 return -EINVAL; 517 buf_size_shift = ilog2(rx_buf_len); 518 } 519 if (ifq->niov_shift && ifq->niov_shift != buf_size_shift) 520 return -EINVAL; 521 if (!ifq->dev && buf_size_shift != PAGE_SHIFT) 522 return -EOPNOTSUPP; 523 524 ret = -ENOMEM; 525 area = kzalloc_obj(*area); 526 if (!area) 527 goto err; 528 area->ifq = ifq; 529 530 ret = io_import_area(ifq, &area->mem, area_reg); 531 if (ret) 532 goto err; 533 if (ifq->dev) 534 area->is_mapped = true; 535 536 if (ifq->dev && buf_size_shift > io_area_max_shift(&area->mem)) { 537 ret = -ERANGE; 538 goto err; 539 } 540 541 ifq->niov_shift = buf_size_shift; 542 nr_iovs = area->mem.size >> ifq->niov_shift; 543 area->nia.num_niovs = nr_iovs; 544 545 ret = -ENOMEM; 546 area->nia.niovs = kvmalloc_objs(area->nia.niovs[0], nr_iovs, 547 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 548 if (!area->nia.niovs) 549 goto err; 550 551 area->freelist = kvmalloc_array(nr_iovs, sizeof(area->freelist[0]), 552 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 553 if (!area->freelist) 554 goto err; 555 556 area->user_refs = kvmalloc_objs(area->user_refs[0], nr_iovs, 557 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 558 if (!area->user_refs) 559 goto err; 560 561 for (i = 0; i < nr_iovs; i++) { 562 struct net_iov *niov = &area->nia.niovs[i]; 563 564 net_iov_init(niov, &area->nia, NET_IOV_IOURING); 565 area->freelist[i] = i; 566 atomic_set(&area->user_refs[i], 0); 567 } 568 569 if (ifq->dev) { 570 ret = io_populate_area_dma(ifq, area); 571 if (ret) 572 goto err; 573 } 574 575 area->free_count = nr_iovs; 576 /* we're only supporting one area per ifq for now */ 577 area->area_id = zcrx_next_area_id(ifq); 578 area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id); 579 580 ret = io_zcrx_append_area(ifq, area); 581 if (!ret) 582 return 0; 583 err: 584 if (area) { 585 __io_zcrx_unmap_area(ifq, area); 586 io_zcrx_free_area(ifq, area); 587 } 588 return ret; 589 } 590 591 static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, 592 struct io_uring_zcrx_area_reg *area_reg, 593 struct io_uring_zcrx_ifq_reg *reg) 594 { 595 guard(mutex)(&ifq->pp_lock); 596 return __zcrx_create_area(ifq, area_reg, reg->rx_buf_len); 597 } 598 599 static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) 600 { 601 struct io_zcrx_ifq *ifq; 602 603 ifq = kzalloc_obj(*ifq); 604 if (!ifq) 605 return NULL; 606 607 ifq->if_rxq = -1; 608 spin_lock_init(&ifq->ctx_lock); 609 spin_lock_init(&ifq->rq.lock); 610 spin_lock_init(&ifq->alloc_lock); 611 mutex_init(&ifq->pp_lock); 612 refcount_set(&ifq->refs, 1); 613 refcount_set(&ifq->user_refs, 1); 614 return ifq; 615 } 616 617 static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq) 618 { 619 lockdep_assert_held(&ifq->pp_lock); 620 621 if (!ifq->netdev) 622 return; 623 netdev_put(ifq->netdev, &ifq->netdev_tracker); 624 ifq->netdev = NULL; 625 } 626 627 static void io_close_queue(struct io_zcrx_ifq *ifq) 628 { 629 struct net_device *netdev; 630 netdevice_tracker netdev_tracker; 631 struct pp_memory_provider_params p = { 632 .mp_ops = &io_uring_pp_zc_ops, 633 .mp_priv = ifq, 634 }; 635 636 scoped_guard(mutex, &ifq->pp_lock) { 637 netdev = ifq->netdev; 638 netdev_tracker = ifq->netdev_tracker; 639 ifq->netdev = NULL; 640 } 641 642 if (netdev) { 643 netdev_lock(netdev); 644 if (ifq->if_rxq != -1) 645 netif_mp_close_rxq(netdev, ifq->if_rxq, &p); 646 647 scoped_guard(mutex, &ifq->pp_lock) 648 io_zcrx_unmap_areas(ifq); 649 netdev_unlock(netdev); 650 netdev_put(netdev, &netdev_tracker); 651 } 652 ifq->if_rxq = -1; 653 } 654 655 static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) 656 { 657 int i; 658 659 if (WARN_ON_ONCE(ifq->if_rxq != -1)) 660 return; 661 if (WARN_ON_ONCE(ifq->netdev != NULL)) 662 return; 663 if (WARN_ON_ONCE(ifq->master_ctx)) 664 return; 665 666 for (i = 0; i < ifq->nr_areas; i++) 667 io_zcrx_free_area(ifq, ifq->areas[i]); 668 if (ifq->mm_account) 669 mmdrop(ifq->mm_account); 670 if (ifq->dev) 671 put_device(ifq->dev); 672 673 io_free_rbuf_ring(ifq); 674 free_uid(ifq->user); 675 mutex_destroy(&ifq->pp_lock); 676 kfree(ifq->areas); 677 kfree(ifq); 678 } 679 680 static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq) 681 { 682 if (refcount_dec_and_test(&ifq->refs)) 683 io_zcrx_ifq_free(ifq); 684 } 685 686 static void io_zcrx_return_niov_freelist(struct net_iov *niov) 687 { 688 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); 689 struct io_zcrx_ifq *ifq = area->ifq; 690 691 guard(spinlock_bh)(&ifq->alloc_lock); 692 if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs)) 693 return; 694 area->freelist[area->free_count++] = net_iov_idx(niov); 695 } 696 697 static struct net_iov *zcrx_get_free_niov(struct io_zcrx_area *area) 698 { 699 unsigned niov_idx; 700 701 lockdep_assert_held(&area->ifq->alloc_lock); 702 703 if (unlikely(!area->free_count)) 704 return NULL; 705 706 niov_idx = area->freelist[--area->free_count]; 707 return &area->nia.niovs[niov_idx]; 708 } 709 710 static void io_zcrx_return_niov(struct net_iov *niov) 711 { 712 netmem_ref netmem = net_iov_to_netmem(niov); 713 714 if (!niov->desc.pp) { 715 /* copy fallback allocated niovs */ 716 io_zcrx_return_niov_freelist(niov); 717 return; 718 } 719 page_pool_put_unrefed_netmem(niov->desc.pp, netmem, -1, false); 720 } 721 722 static void io_zcrx_scrub_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) 723 { 724 int i; 725 726 /* Reclaim back all buffers given to the user space. */ 727 for (i = 0; i < area->nia.num_niovs; i++) { 728 struct net_iov *niov = &area->nia.niovs[i]; 729 int nr; 730 731 if (!atomic_read(io_get_user_counter(niov))) 732 continue; 733 nr = atomic_xchg(io_get_user_counter(niov), 0); 734 if (nr && !page_pool_unref_netmem(net_iov_to_netmem(niov), nr)) 735 io_zcrx_return_niov(niov); 736 } 737 } 738 739 static void io_zcrx_scrub(struct io_zcrx_ifq *ifq) 740 { 741 int i; 742 743 guard(mutex)(&ifq->pp_lock); 744 for (i = 0; i < ifq->nr_areas; i++) 745 io_zcrx_scrub_area(ifq, ifq->areas[i]); 746 } 747 748 static void zcrx_unregister_user(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) 749 { 750 scoped_guard(spinlock_bh, &ifq->ctx_lock) { 751 if (ctx && ifq->master_ctx == ctx) { 752 ifq->master_ctx = NULL; 753 percpu_ref_put(&ctx->refs); 754 } 755 } 756 757 if (refcount_dec_and_test(&ifq->user_refs)) { 758 io_close_queue(ifq); 759 io_zcrx_scrub(ifq); 760 } 761 } 762 763 static void zcrx_unregister(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx) 764 { 765 zcrx_unregister_user(ifq, ctx); 766 io_put_zcrx_ifq(ifq); 767 } 768 769 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, 770 unsigned int id) 771 { 772 struct io_zcrx_ifq *ifq = xa_load(&ctx->zcrx_ctxs, id); 773 774 lockdep_assert_held(&ctx->mmap_lock); 775 776 return ifq ? &ifq->rq_region : NULL; 777 } 778 779 static int zcrx_box_release(struct inode *inode, struct file *file) 780 { 781 struct io_zcrx_ifq *ifq = file->private_data; 782 783 if (WARN_ON_ONCE(!ifq)) 784 return -EFAULT; 785 zcrx_unregister(ifq, NULL); 786 return 0; 787 } 788 789 static const struct file_operations zcrx_box_fops = { 790 .owner = THIS_MODULE, 791 .release = zcrx_box_release, 792 }; 793 794 static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, 795 struct zcrx_ctrl *ctrl, void __user *arg) 796 { 797 struct zcrx_ctrl_export *ce = &ctrl->zc_export; 798 struct file *file; 799 int fd; 800 801 if (!mem_is_zero(ce, sizeof(*ce))) 802 return -EINVAL; 803 804 refcount_inc(&ifq->refs); 805 refcount_inc(&ifq->user_refs); 806 807 file = anon_inode_create_getfile("[zcrx]", &zcrx_box_fops, 808 ifq, O_CLOEXEC, NULL); 809 if (IS_ERR(file)) { 810 zcrx_unregister(ifq, NULL); 811 return PTR_ERR(file); 812 } 813 814 fd = get_unused_fd_flags(O_CLOEXEC); 815 if (fd < 0) { 816 fput(file); 817 return fd; 818 } 819 820 ce->zcrx_fd = fd; 821 if (copy_to_user(arg, ctrl, sizeof(*ctrl))) { 822 fput(file); 823 put_unused_fd(fd); 824 return -EFAULT; 825 } 826 827 fd_install(fd, file); 828 return 0; 829 } 830 831 static int import_zcrx(struct io_ring_ctx *ctx, 832 struct io_uring_zcrx_ifq_reg __user *arg, 833 struct io_uring_zcrx_ifq_reg *reg) 834 { 835 struct io_zcrx_ifq *ifq; 836 struct file *file; 837 int fd, ret; 838 u32 id; 839 840 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 841 return -EINVAL; 842 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))) 843 return -EINVAL; 844 if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr) 845 return -EINVAL; 846 if (reg->event_desc) 847 return -EINVAL; 848 if (reg->flags & ~ZCRX_REG_IMPORT) 849 return -EINVAL; 850 851 fd = reg->if_idx; 852 CLASS(fd, f)(fd); 853 if (fd_empty(f)) 854 return -EBADF; 855 856 file = fd_file(f); 857 if (file->f_op != &zcrx_box_fops || !file->private_data) 858 return -EBADF; 859 860 ifq = file->private_data; 861 refcount_inc(&ifq->refs); 862 refcount_inc(&ifq->user_refs); 863 864 scoped_guard(mutex, &ctx->mmap_lock) { 865 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL); 866 if (ret) 867 goto err; 868 } 869 870 reg->zcrx_id = id; 871 io_fill_zcrx_offsets(®->offsets); 872 if (copy_to_user(arg, reg, sizeof(*reg))) { 873 ret = -EFAULT; 874 goto err_xa_erase; 875 } 876 877 scoped_guard(mutex, &ctx->mmap_lock) { 878 ret = -ENOMEM; 879 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL)) 880 goto err_xa_erase; 881 } 882 883 return 0; 884 err_xa_erase: 885 scoped_guard(mutex, &ctx->mmap_lock) 886 xa_erase(&ctx->zcrx_ctxs, id); 887 err: 888 /* the import path never set the ->master_ctx ref, don't drop it */ 889 zcrx_unregister(ifq, NULL); 890 return ret; 891 } 892 893 static int zcrx_register_netdev(struct io_zcrx_ifq *ifq, 894 struct io_uring_zcrx_ifq_reg *reg, 895 struct io_uring_zcrx_area_reg *area) 896 { 897 struct pp_memory_provider_params mp_param = {}; 898 unsigned if_rxq = reg->if_rxq; 899 int ret; 900 901 ifq->netdev = netdev_get_by_index_lock(current->nsproxy->net_ns, 902 reg->if_idx); 903 if (!ifq->netdev) 904 return -ENODEV; 905 906 netdev_hold(ifq->netdev, &ifq->netdev_tracker, GFP_KERNEL); 907 908 ifq->dev = netdev_queue_get_dma_dev(ifq->netdev, if_rxq, NETDEV_QUEUE_TYPE_RX); 909 if (!ifq->dev) { 910 ret = -EOPNOTSUPP; 911 goto netdev_put_unlock; 912 } 913 get_device(ifq->dev); 914 915 ret = io_zcrx_create_area(ifq, area, reg); 916 if (ret) 917 goto netdev_put_unlock; 918 919 if (reg->rx_buf_len) 920 mp_param.rx_page_size = 1U << ifq->niov_shift; 921 mp_param.mp_ops = &io_uring_pp_zc_ops; 922 mp_param.mp_priv = ifq; 923 ret = netif_mp_open_rxq(ifq->netdev, if_rxq, &mp_param, NULL); 924 if (ret) 925 goto netdev_put_unlock; 926 927 ifq->if_rxq = if_rxq; 928 ret = 0; 929 netdev_put_unlock: 930 netdev_unlock(ifq->netdev); 931 return ret; 932 } 933 934 static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq, 935 const struct io_uring_zcrx_ifq_reg *reg, 936 const struct zcrx_event_desc *notif) 937 { 938 size_t stats_off = notif->stats_offset; 939 size_t used, end; 940 941 used = reg->offsets.rqes + 942 sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries; 943 944 if (!IS_ALIGNED(stats_off, __alignof__(struct zcrx_stats))) 945 return -EINVAL; 946 if (stats_off < used) 947 return -ERANGE; 948 if (check_add_overflow(stats_off, 949 sizeof(struct zcrx_stats), 950 &end)) 951 return -ERANGE; 952 if (end > io_region_size(&ifq->rq_region)) 953 return -ERANGE; 954 955 ifq->notif_stats = io_region_get_ptr(&ifq->rq_region) + stats_off; 956 memset(ifq->notif_stats, 0, sizeof(*ifq->notif_stats)); 957 958 return 0; 959 } 960 961 int io_register_zcrx(struct io_ring_ctx *ctx, 962 struct io_uring_zcrx_ifq_reg __user *arg) 963 { 964 struct zcrx_event_desc notif; 965 struct io_uring_zcrx_area_reg area; 966 struct io_uring_zcrx_ifq_reg reg; 967 struct io_uring_region_desc rd; 968 struct io_zcrx_ifq *ifq; 969 int ret; 970 u32 id; 971 972 /* 973 * 1. Interface queue allocation. 974 * 2. It can observe data destined for sockets of other tasks. 975 */ 976 if (!capable(CAP_NET_ADMIN)) 977 return -EPERM; 978 979 /* mandatory io_uring features for zc rx */ 980 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 981 return -EINVAL; 982 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED))) 983 return -EINVAL; 984 if (copy_from_user(®, arg, sizeof(reg))) 985 return -EFAULT; 986 if (!mem_is_zero(®.__resv, sizeof(reg.__resv)) || reg.zcrx_id) 987 return -EINVAL; 988 if (reg.flags & ~ZCRX_SUPPORTED_REG_FLAGS) 989 return -EINVAL; 990 if (reg.flags & ZCRX_REG_IMPORT) 991 return import_zcrx(ctx, arg, ®); 992 if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd))) 993 return -EFAULT; 994 if (reg.if_rxq == -1 || !reg.rq_entries) 995 return -EINVAL; 996 if ((reg.if_rxq || reg.if_idx) && (reg.flags & ZCRX_REG_NODEV)) 997 return -EINVAL; 998 if (reg.rq_entries > IO_RQ_MAX_ENTRIES) { 999 if (!(ctx->flags & IORING_SETUP_CLAMP)) 1000 return -EINVAL; 1001 reg.rq_entries = IO_RQ_MAX_ENTRIES; 1002 } 1003 reg.rq_entries = roundup_pow_of_two(reg.rq_entries); 1004 1005 if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area))) 1006 return -EFAULT; 1007 if (area.rq_area_token) 1008 return -EINVAL; 1009 1010 memset(¬if, 0, sizeof(notif)); 1011 if (reg.event_desc && copy_from_user(¬if, u64_to_user_ptr(reg.event_desc), 1012 sizeof(notif))) 1013 return -EFAULT; 1014 if (notif.type_mask & ~ZCRX_EVENT_TYPE_MASK) 1015 return -EINVAL; 1016 if (notif.flags & ~ZCRX_EVENT_DESC_FLAG_STATS) 1017 return -EINVAL; 1018 if (!(notif.flags & ZCRX_EVENT_DESC_FLAG_STATS)) { 1019 if (notif.stats_offset) 1020 return -EINVAL; 1021 } 1022 if (!mem_is_zero(¬if.__resv2, sizeof(notif.__resv2))) 1023 return -EINVAL; 1024 1025 ifq = io_zcrx_ifq_alloc(ctx); 1026 if (!ifq) 1027 return -ENOMEM; 1028 1029 ifq->notif_data = notif.user_data; 1030 ifq->allowed_notif_mask = notif.type_mask; 1031 1032 if (ctx->user) { 1033 get_uid(ctx->user); 1034 ifq->user = ctx->user; 1035 } 1036 if (ctx->mm_account) { 1037 mmgrab(ctx->mm_account); 1038 ifq->mm_account = ctx->mm_account; 1039 } 1040 ifq->rq.nr_entries = reg.rq_entries; 1041 1042 scoped_guard(mutex, &ctx->mmap_lock) { 1043 /* preallocate id */ 1044 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL); 1045 if (ret) 1046 goto ifq_free; 1047 } 1048 1049 ret = io_allocate_rbuf_ring(ctx, ifq, ®, &rd, id); 1050 if (ret) 1051 goto err; 1052 1053 if (notif.flags & ZCRX_EVENT_DESC_FLAG_STATS) { 1054 ret = zcrx_validate_notif_stats(ifq, ®, ¬if); 1055 if (ret) 1056 goto err; 1057 } 1058 1059 ifq->kern_readable = !(area.flags & IORING_ZCRX_AREA_DMABUF); 1060 1061 if (!(reg.flags & ZCRX_REG_NODEV)) { 1062 ret = zcrx_register_netdev(ifq, ®, &area); 1063 if (ret) 1064 goto err; 1065 } else { 1066 ret = io_zcrx_create_area(ifq, &area, ®); 1067 if (ret) 1068 goto err; 1069 } 1070 1071 WARN_ON_ONCE(!ifq->niov_shift); 1072 1073 reg.zcrx_id = id; 1074 1075 scoped_guard(mutex, &ctx->mmap_lock) { 1076 /* publish ifq */ 1077 ret = -ENOMEM; 1078 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL)) 1079 goto err; 1080 } 1081 1082 reg.rx_buf_len = 1U << ifq->niov_shift; 1083 1084 if (copy_to_user(arg, ®, sizeof(reg)) || 1085 copy_to_user(u64_to_user_ptr(reg.region_ptr), &rd, sizeof(rd)) || 1086 copy_to_user(u64_to_user_ptr(reg.area_ptr), &area, sizeof(area))) { 1087 ret = -EFAULT; 1088 goto err; 1089 } 1090 1091 if (notif.type_mask) 1092 zcrx_set_ring_ctx(ifq, ctx); 1093 return 0; 1094 err: 1095 scoped_guard(mutex, &ctx->mmap_lock) 1096 xa_erase(&ctx->zcrx_ctxs, id); 1097 ifq_free: 1098 zcrx_unregister(ifq, ctx); 1099 return ret; 1100 } 1101 1102 static inline bool is_zcrx_entry_marked(struct io_ring_ctx *ctx, unsigned long id) 1103 { 1104 return xa_get_mark(&ctx->zcrx_ctxs, id, XA_MARK_1); 1105 } 1106 1107 static inline void set_zcrx_entry_mark(struct io_ring_ctx *ctx, unsigned long id) 1108 { 1109 xa_set_mark(&ctx->zcrx_ctxs, id, XA_MARK_1); 1110 } 1111 1112 void io_terminate_zcrx(struct io_ring_ctx *ctx) 1113 { 1114 struct io_zcrx_ifq *ifq; 1115 unsigned long id = 0; 1116 1117 lockdep_assert_held(&ctx->uring_lock); 1118 1119 while (1) { 1120 scoped_guard(mutex, &ctx->mmap_lock) 1121 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); 1122 if (!ifq) 1123 break; 1124 if (WARN_ON_ONCE(is_zcrx_entry_marked(ctx, id))) 1125 break; 1126 set_zcrx_entry_mark(ctx, id); 1127 id++; 1128 zcrx_unregister_user(ifq, ctx); 1129 } 1130 } 1131 1132 void io_unregister_zcrx(struct io_ring_ctx *ctx) 1133 { 1134 struct io_zcrx_ifq *ifq; 1135 1136 lockdep_assert_held(&ctx->uring_lock); 1137 1138 while (1) { 1139 scoped_guard(mutex, &ctx->mmap_lock) { 1140 unsigned long id = 0; 1141 1142 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); 1143 if (ifq) { 1144 if (WARN_ON_ONCE(!is_zcrx_entry_marked(ctx, id))) { 1145 ifq = NULL; 1146 break; 1147 } 1148 xa_erase(&ctx->zcrx_ctxs, id); 1149 } 1150 } 1151 if (!ifq) 1152 break; 1153 /* 1154 * io_uring can run requests and return buffers to the user 1155 * after termination, scrub it again. 1156 */ 1157 if (refcount_read(&ifq->user_refs) == 0) 1158 io_zcrx_scrub(ifq); 1159 io_put_zcrx_ifq(ifq); 1160 } 1161 1162 xa_destroy(&ctx->zcrx_ctxs); 1163 } 1164 1165 struct zcrx_rq_iter { 1166 int rqes_left; 1167 bool flushed; 1168 }; 1169 1170 static inline u32 __zcrx_rq_entries(struct zcrx_rq *rq) 1171 { 1172 u32 entries = rq->cached_tail - rq->cached_head; 1173 1174 return min(entries, rq->nr_entries); 1175 } 1176 1177 static inline u32 zcrx_rq_entries(struct zcrx_rq *rq) 1178 { 1179 rq->cached_tail = smp_load_acquire(&rq->ring->tail); 1180 return __zcrx_rq_entries(rq); 1181 } 1182 1183 static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask) 1184 { 1185 unsigned int idx = rq->cached_head++ & mask; 1186 1187 return &rq->rqes[idx]; 1188 } 1189 1190 static inline void zcrx_rq_iter_init(struct zcrx_rq_iter *it, 1191 struct zcrx_rq *rq) 1192 { 1193 it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), ZCRX_REFILL_CAP); 1194 it->flushed = false; 1195 } 1196 1197 static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it, 1198 struct zcrx_rq *rq, 1199 struct io_uring_zcrx_rqe **rqe) 1200 { 1201 it->rqes_left--; 1202 if (unlikely(it->rqes_left < 0)) { 1203 if (it->flushed) 1204 return false; 1205 rq->cached_tail = smp_load_acquire(&rq->ring->tail); 1206 it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), 1207 ZCRX_REFILL_CAP); 1208 it->flushed = true; 1209 if (--it->rqes_left < 0) 1210 return false; 1211 } 1212 1213 *rqe = zcrx_next_rqe(rq, rq->nr_entries - 1); 1214 return true; 1215 } 1216 1217 static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, 1218 struct io_zcrx_ifq *ifq, 1219 struct net_iov **ret_niov) 1220 { 1221 __u64 off = READ_ONCE(rqe->off); 1222 unsigned niov_idx, area_idx; 1223 struct io_zcrx_area *area; 1224 1225 lockdep_assert_held(&ifq->rq.lock); 1226 1227 area_idx = off >> IORING_ZCRX_AREA_SHIFT; 1228 niov_idx = (off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; 1229 1230 if (unlikely(rqe->__pad || area_idx >= ifq->nr_areas)) 1231 return false; 1232 area_idx = array_index_nospec(area_idx, ifq->nr_areas); 1233 area = ifq->areas[area_idx]; 1234 1235 if (unlikely(niov_idx >= area->nia.num_niovs)) 1236 return false; 1237 niov_idx = array_index_nospec(niov_idx, area->nia.num_niovs); 1238 1239 *ret_niov = &area->nia.niovs[niov_idx]; 1240 return true; 1241 } 1242 1243 static bool zcrx_put_refill_niov(struct net_iov *niov, struct page_pool *pp, 1244 unsigned refs) 1245 { 1246 netmem_ref netmem = net_iov_to_netmem(niov); 1247 1248 if (!io_zcrx_put_niov_uref(niov, refs)) 1249 return false; 1250 if (page_pool_unref_netmem(netmem, refs) != 0) 1251 return false; 1252 if (unlikely(niov->desc.pp != pp)) { 1253 io_zcrx_return_niov(niov); 1254 return false; 1255 } 1256 return true; 1257 } 1258 1259 static unsigned io_zcrx_ring_refill(struct page_pool *pp, 1260 struct io_zcrx_ifq *ifq, 1261 netmem_ref *netmems, unsigned to_alloc) 1262 { 1263 struct zcrx_rq *rq = &ifq->rq; 1264 struct io_uring_zcrx_rqe *rqe; 1265 struct zcrx_rq_iter it; 1266 struct net_iov *niov = NULL; 1267 unsigned niov_refs = 0; 1268 unsigned allocated = 0; 1269 1270 guard(spinlock_bh)(&rq->lock); 1271 1272 zcrx_rq_iter_init(&it, rq); 1273 1274 while (allocated < to_alloc - 1 && zcrx_rq_iter_next(&it, rq, &rqe)) { 1275 struct net_iov *next_niov; 1276 1277 if (!io_parse_rqe(rqe, ifq, &next_niov)) 1278 continue; 1279 if (niov == next_niov) { 1280 niov_refs++; 1281 continue; 1282 } 1283 if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) { 1284 netmems[allocated] = net_iov_to_netmem(niov); 1285 allocated++; 1286 } 1287 niov = next_niov; 1288 niov_refs = 1; 1289 } 1290 1291 if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) { 1292 netmems[allocated] = net_iov_to_netmem(niov); 1293 allocated++; 1294 } 1295 1296 smp_store_release(&rq->ring->head, rq->cached_head); 1297 return allocated; 1298 } 1299 1300 static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq, 1301 netmem_ref *netmems, unsigned to_alloc) 1302 { 1303 unsigned area_idx = 0; 1304 unsigned allocated = 0; 1305 1306 guard(spinlock_bh)(&ifq->alloc_lock); 1307 1308 while (allocated < to_alloc) { 1309 struct net_iov *niov = zcrx_get_free_niov(ifq->areas[area_idx]); 1310 1311 if (!niov) { 1312 area_idx++; 1313 if (area_idx >= ifq->nr_areas) 1314 break; 1315 continue; 1316 } 1317 1318 net_mp_niov_set_page_pool(pp, niov); 1319 netmems[allocated] = net_iov_to_netmem(niov); 1320 allocated++; 1321 } 1322 return allocated; 1323 } 1324 1325 static void zcrx_notif_tw(struct io_tw_req tw_req, io_tw_token_t tw) 1326 { 1327 struct io_kiocb *req = tw_req.req; 1328 struct io_ring_ctx *ctx = req->ctx; 1329 1330 io_post_aux_cqe(ctx, req->cqe.user_data, req->cqe.res, 0); 1331 percpu_ref_put(&ctx->refs); 1332 io_poison_req(req); 1333 kmem_cache_free(req_cachep, req); 1334 } 1335 1336 static void zcrx_stat_add(__u64 *p, s64 v) 1337 { 1338 WRITE_ONCE(*p, READ_ONCE(*p) + v); 1339 } 1340 1341 static void zcrx_send_notif(struct io_zcrx_ifq *ifq, unsigned type) 1342 { 1343 gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO; 1344 u32 type_mask = 1 << type; 1345 struct io_kiocb *req; 1346 1347 if (!(type_mask & ifq->allowed_notif_mask)) 1348 return; 1349 1350 guard(spinlock_bh)(&ifq->ctx_lock); 1351 if (!ifq->master_ctx) 1352 return; 1353 if (type_mask & ifq->fired_notifs) 1354 return; 1355 1356 req = kmem_cache_alloc(req_cachep, gfp); 1357 if (unlikely(!req)) 1358 return; 1359 1360 ifq->fired_notifs |= type_mask; 1361 1362 req->opcode = IORING_OP_NOP; 1363 req->cqe.user_data = ifq->notif_data; 1364 req->cqe.res = type; 1365 req->ctx = ifq->master_ctx; 1366 percpu_ref_get(&req->ctx->refs); 1367 req->tctx = NULL; 1368 req->io_task_work.func = zcrx_notif_tw; 1369 io_req_task_work_add(req); 1370 } 1371 1372 static netmem_ref io_pp_zc_alloc_netmems(struct page_pool *pp, gfp_t gfp) 1373 { 1374 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp); 1375 netmem_ref *netmems = pp->alloc.cache; 1376 unsigned to_alloc = PP_ALLOC_CACHE_REFILL; 1377 unsigned allocated; 1378 1379 /* pp should already be ensuring that */ 1380 if (WARN_ON_ONCE(pp->alloc.count)) 1381 return 0; 1382 1383 allocated = io_zcrx_ring_refill(pp, ifq, netmems, to_alloc); 1384 if (likely(allocated)) 1385 goto out_return; 1386 1387 allocated = io_zcrx_refill_slow(pp, ifq, netmems, to_alloc); 1388 if (!allocated) { 1389 zcrx_send_notif(ifq, ZCRX_EVENT_ALLOC_FAIL); 1390 return 0; 1391 } 1392 out_return: 1393 zcrx_sync_for_device(pp, ifq, netmems, allocated); 1394 allocated--; 1395 pp->alloc.count += allocated; 1396 return netmems[allocated]; 1397 } 1398 1399 static bool io_pp_zc_release_netmem(struct page_pool *pp, netmem_ref netmem) 1400 { 1401 struct net_iov *niov; 1402 1403 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 1404 return false; 1405 1406 niov = netmem_to_net_iov(netmem); 1407 net_mp_niov_clear_page_pool(niov); 1408 io_zcrx_return_niov_freelist(niov); 1409 return false; 1410 } 1411 1412 static int io_pp_zc_init(struct page_pool *pp) 1413 { 1414 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp); 1415 1416 if (WARN_ON_ONCE(!ifq)) 1417 return -EINVAL; 1418 if (WARN_ON_ONCE(ifq->dev != pp->p.dev)) 1419 return -EINVAL; 1420 if (WARN_ON_ONCE(!pp->dma_map)) 1421 return -EOPNOTSUPP; 1422 if (pp->p.order + PAGE_SHIFT != ifq->niov_shift) 1423 return -EINVAL; 1424 if (pp->p.dma_dir != DMA_FROM_DEVICE) 1425 return -EOPNOTSUPP; 1426 1427 refcount_inc(&ifq->refs); 1428 return 0; 1429 } 1430 1431 static void io_pp_zc_destroy(struct page_pool *pp) 1432 { 1433 io_put_zcrx_ifq(io_pp_to_ifq(pp)); 1434 } 1435 1436 static int io_pp_nl_fill(void *mp_priv, struct sk_buff *rsp, 1437 struct netdev_rx_queue *rxq) 1438 { 1439 struct io_zcrx_ifq *ifq = mp_priv; 1440 struct nlattr *nest; 1441 int type; 1442 1443 type = rxq ? NETDEV_A_QUEUE_IO_URING : NETDEV_A_PAGE_POOL_IO_URING; 1444 nest = nla_nest_start(rsp, type); 1445 if (!nest) 1446 return -EMSGSIZE; 1447 1448 if (nla_put_uint(rsp, NETDEV_A_IO_URING_PROVIDER_INFO_RX_BUF_LEN, 1449 1ULL << ifq->niov_shift)) { 1450 nla_nest_cancel(rsp, nest); 1451 return -EMSGSIZE; 1452 } 1453 1454 nla_nest_end(rsp, nest); 1455 1456 return 0; 1457 } 1458 1459 static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq) 1460 { 1461 struct pp_memory_provider_params *p = &rxq->mp_params; 1462 struct io_zcrx_ifq *ifq = mp_priv; 1463 1464 guard(mutex)(&ifq->pp_lock); 1465 io_zcrx_unmap_areas(ifq); 1466 io_zcrx_drop_netdev(ifq); 1467 1468 p->mp_ops = NULL; 1469 p->mp_priv = NULL; 1470 } 1471 1472 static const struct memory_provider_ops io_uring_pp_zc_ops = { 1473 .alloc_netmems = io_pp_zc_alloc_netmems, 1474 .release_netmem = io_pp_zc_release_netmem, 1475 .init = io_pp_zc_init, 1476 .destroy = io_pp_zc_destroy, 1477 .nl_fill = io_pp_nl_fill, 1478 .uninstall = io_pp_uninstall, 1479 }; 1480 1481 static unsigned zcrx_parse_rq(netmem_ref *netmem_array, unsigned nr, 1482 struct io_zcrx_ifq *zcrx, struct zcrx_rq *rq) 1483 { 1484 unsigned int mask = rq->nr_entries - 1; 1485 unsigned int i; 1486 1487 nr = min(nr, zcrx_rq_entries(rq)); 1488 for (i = 0; i < nr; i++) { 1489 struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask); 1490 struct net_iov *niov; 1491 1492 if (!io_parse_rqe(rqe, zcrx, &niov)) 1493 break; 1494 netmem_array[i] = net_iov_to_netmem(niov); 1495 } 1496 1497 smp_store_release(&rq->ring->head, rq->cached_head); 1498 return i; 1499 } 1500 1501 #define ZCRX_FLUSH_BATCH 32 1502 1503 static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr) 1504 { 1505 unsigned i; 1506 1507 for (i = 0; i < nr; i++) { 1508 netmem_ref netmem = netmems[i]; 1509 struct net_iov *niov = netmem_to_net_iov(netmem); 1510 1511 if (!io_zcrx_put_niov_uref(niov, 1)) 1512 continue; 1513 if (!page_pool_unref_and_test(netmem)) 1514 continue; 1515 io_zcrx_return_niov(niov); 1516 } 1517 } 1518 1519 static int zcrx_flush_rq(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, 1520 struct zcrx_ctrl *ctrl) 1521 { 1522 struct zcrx_ctrl_flush_rq *frq = &ctrl->zc_flush; 1523 netmem_ref netmems[ZCRX_FLUSH_BATCH]; 1524 unsigned total = 0; 1525 unsigned nr; 1526 1527 if (!mem_is_zero(&frq->__resv, sizeof(frq->__resv))) 1528 return -EINVAL; 1529 1530 do { 1531 struct zcrx_rq *rq = &zcrx->rq; 1532 1533 scoped_guard(spinlock_bh, &rq->lock) { 1534 nr = zcrx_parse_rq(netmems, ZCRX_FLUSH_BATCH, zcrx, rq); 1535 zcrx_return_buffers(netmems, nr); 1536 } 1537 1538 total += nr; 1539 1540 if (fatal_signal_pending(current)) 1541 break; 1542 cond_resched(); 1543 } while (nr == ZCRX_FLUSH_BATCH && total < zcrx->rq.nr_entries); 1544 1545 return 0; 1546 } 1547 1548 static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, 1549 struct zcrx_ctrl *ctrl) 1550 { 1551 const struct zcrx_ctrl_arm_event *an = &ctrl->zc_arm_event; 1552 unsigned type_mask; 1553 1554 if (an->event_type >= __ZCRX_EVENT_TYPE_LAST) 1555 return -EINVAL; 1556 if (!mem_is_zero(&an->__resv, sizeof(an->__resv))) 1557 return -EINVAL; 1558 1559 guard(spinlock_bh)(&zcrx->ctx_lock); 1560 type_mask = 1U << an->event_type; 1561 if (type_mask & ~zcrx->fired_notifs) 1562 return -EINVAL; 1563 zcrx->fired_notifs &= ~type_mask; 1564 return 0; 1565 } 1566 1567 static int zcrx_ctrl_add_area(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq, 1568 struct zcrx_ctrl *ctrl) 1569 { 1570 struct zcrx_ctrl_add_area *ctrl_add = &ctrl->zc_area; 1571 struct io_uring_zcrx_area_reg __user *area_uptr; 1572 struct io_uring_zcrx_area_reg area_reg; 1573 1574 area_uptr = u64_to_user_ptr(ctrl_add->area_ptr); 1575 if (copy_from_user(&area_reg, area_uptr, sizeof(area_reg))) 1576 return -EFAULT; 1577 if (!mem_is_zero(&ctrl_add->__resv, sizeof(ctrl_add->__resv))) 1578 return -EINVAL; 1579 if (area_reg.rq_area_token) 1580 return -EINVAL; 1581 1582 guard(mutex)(&ifq->pp_lock); 1583 if (ifq->dev && !ifq->netdev) 1584 return -EFAULT; 1585 1586 /* we can't safely roll back area append, copy it out first */ 1587 area_reg.rq_area_token = zcrx_area_id_to_token(zcrx_next_area_id(ifq)); 1588 if (copy_to_user(area_uptr, &area_reg, sizeof(area_reg))) 1589 return -EFAULT; 1590 area_reg.rq_area_token = 0; 1591 1592 return __zcrx_create_area(ifq, &area_reg, 1U << ifq->niov_shift); 1593 } 1594 1595 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) 1596 { 1597 struct zcrx_ctrl ctrl; 1598 struct io_zcrx_ifq *zcrx; 1599 1600 BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_flush)); 1601 BUILD_BUG_ON(sizeof(ctrl.zc_export) != sizeof(ctrl.zc_arm_event)); 1602 1603 if (nr_args) 1604 return -EINVAL; 1605 if (copy_from_user(&ctrl, arg, sizeof(ctrl))) 1606 return -EFAULT; 1607 if (!mem_is_zero(&ctrl.__resv, sizeof(ctrl.__resv))) 1608 return -EFAULT; 1609 1610 zcrx = xa_load(&ctx->zcrx_ctxs, ctrl.zcrx_id); 1611 if (!zcrx) 1612 return -ENXIO; 1613 1614 switch (ctrl.op) { 1615 case ZCRX_CTRL_FLUSH_RQ: 1616 return zcrx_flush_rq(ctx, zcrx, &ctrl); 1617 case ZCRX_CTRL_EXPORT: 1618 return zcrx_export(ctx, zcrx, &ctrl, arg); 1619 case ZCRX_CTRL_ARM_EVENT: 1620 return zcrx_arm_notif(ctx, zcrx, &ctrl); 1621 case ZCRX_CTRL_ADD_AREA: 1622 return zcrx_ctrl_add_area(ctx, zcrx, &ctrl); 1623 } 1624 1625 return -EOPNOTSUPP; 1626 } 1627 1628 static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, 1629 struct io_zcrx_ifq *ifq, int off, int len) 1630 { 1631 struct io_ring_ctx *ctx = req->ctx; 1632 struct io_uring_zcrx_cqe *rcqe; 1633 struct io_zcrx_area *area; 1634 struct io_uring_cqe *cqe; 1635 u64 offset; 1636 1637 if (!io_defer_get_uncommited_cqe(ctx, &cqe)) 1638 return false; 1639 1640 cqe->user_data = req->cqe.user_data; 1641 cqe->res = len; 1642 cqe->flags = IORING_CQE_F_MORE; 1643 if (ctx->flags & IORING_SETUP_CQE_MIXED) 1644 cqe->flags |= IORING_CQE_F_32; 1645 1646 area = io_zcrx_iov_to_area(niov); 1647 offset = off + (net_iov_idx(niov) << ifq->niov_shift); 1648 rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1); 1649 rcqe->off = offset + zcrx_area_id_to_token(area->area_id); 1650 rcqe->__pad = 0; 1651 return true; 1652 } 1653 1654 static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq) 1655 { 1656 struct net_iov *niov = NULL; 1657 unsigned area_idx; 1658 1659 if (!ifq->kern_readable) 1660 return NULL; 1661 1662 guard(spinlock_bh)(&ifq->alloc_lock); 1663 1664 for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) { 1665 niov = zcrx_get_free_niov(ifq->areas[area_idx]); 1666 if (niov) { 1667 page_pool_fragment_netmem(net_iov_to_netmem(niov), 1); 1668 return niov; 1669 } 1670 } 1671 1672 return NULL; 1673 } 1674 1675 struct io_copy_cache { 1676 struct page *page; 1677 unsigned long offset; 1678 size_t size; 1679 }; 1680 1681 static ssize_t io_copy_page(struct io_copy_cache *cc, struct page *src_page, 1682 unsigned int src_offset, size_t len) 1683 { 1684 size_t copied = 0; 1685 1686 len = min(len, cc->size); 1687 1688 while (len) { 1689 void *src_addr, *dst_addr; 1690 struct page *dst_page = cc->page; 1691 unsigned dst_offset = cc->offset; 1692 size_t n = len; 1693 1694 if (folio_test_partial_kmap(page_folio(dst_page)) || 1695 folio_test_partial_kmap(page_folio(src_page))) { 1696 dst_page += dst_offset / PAGE_SIZE; 1697 dst_offset = offset_in_page(dst_offset); 1698 src_page += src_offset / PAGE_SIZE; 1699 src_offset = offset_in_page(src_offset); 1700 n = min(PAGE_SIZE - src_offset, PAGE_SIZE - dst_offset); 1701 n = min(n, len); 1702 } 1703 1704 dst_addr = kmap_local_page(dst_page) + dst_offset; 1705 src_addr = kmap_local_page(src_page) + src_offset; 1706 1707 memcpy(dst_addr, src_addr, n); 1708 1709 kunmap_local(src_addr); 1710 kunmap_local(dst_addr); 1711 1712 cc->size -= n; 1713 cc->offset += n; 1714 src_offset += n; 1715 len -= n; 1716 copied += n; 1717 } 1718 return copied; 1719 } 1720 1721 static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1722 struct page *src_page, unsigned int src_offset, 1723 size_t len) 1724 { 1725 size_t copied = 0; 1726 int ret = 0; 1727 1728 while (len) { 1729 struct io_copy_cache cc; 1730 struct net_iov *niov; 1731 size_t n; 1732 1733 niov = io_alloc_fallback_niov(ifq); 1734 if (!niov) { 1735 ret = -ENOMEM; 1736 break; 1737 } 1738 1739 cc.page = io_zcrx_iov_page(niov); 1740 cc.offset = 0; 1741 cc.size = PAGE_SIZE; 1742 1743 n = io_copy_page(&cc, src_page, src_offset, len); 1744 1745 if (!io_zcrx_queue_cqe(req, niov, ifq, 0, n)) { 1746 io_zcrx_return_niov(niov); 1747 ret = -ENOSPC; 1748 break; 1749 } 1750 1751 io_zcrx_get_niov_uref(niov); 1752 src_offset += n; 1753 len -= n; 1754 copied += n; 1755 } 1756 1757 return copied ? copied : ret; 1758 } 1759 1760 static int io_zcrx_copy_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1761 const skb_frag_t *frag, int off, int len) 1762 { 1763 struct page *page = skb_frag_page(frag); 1764 int ret; 1765 1766 ret = io_zcrx_copy_chunk(req, ifq, page, off + skb_frag_off(frag), len); 1767 if (ret > 0) { 1768 if (ifq->notif_stats) { 1769 zcrx_stat_add(&ifq->notif_stats->copy_count, 1); 1770 zcrx_stat_add(&ifq->notif_stats->copy_bytes, ret); 1771 } 1772 zcrx_send_notif(ifq, ZCRX_EVENT_COPY); 1773 } 1774 1775 return ret; 1776 } 1777 1778 static int io_zcrx_recv_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1779 const skb_frag_t *frag, int off, int len) 1780 { 1781 struct net_iov *niov; 1782 struct page_pool *pp; 1783 1784 if (unlikely(!skb_frag_is_net_iov(frag))) 1785 return io_zcrx_copy_frag(req, ifq, frag, off, len); 1786 1787 niov = netmem_to_net_iov(frag->netmem); 1788 pp = niov->desc.pp; 1789 1790 if (!pp || pp->mp_ops != &io_uring_pp_zc_ops || io_pp_to_ifq(pp) != ifq) 1791 return -EFAULT; 1792 1793 if (!io_zcrx_queue_cqe(req, niov, ifq, off + skb_frag_off(frag), len)) 1794 return -ENOSPC; 1795 1796 /* 1797 * Prevent it from being recycled while user is accessing it. 1798 * It has to be done before grabbing a user reference. 1799 */ 1800 page_pool_ref_netmem(net_iov_to_netmem(niov)); 1801 io_zcrx_get_niov_uref(niov); 1802 return len; 1803 } 1804 1805 static int 1806 io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 1807 unsigned int offset, size_t len) 1808 { 1809 struct io_zcrx_args *args = desc->arg.data; 1810 struct io_zcrx_ifq *ifq = args->ifq; 1811 struct io_kiocb *req = args->req; 1812 struct sk_buff *frag_iter; 1813 unsigned start, start_off = offset; 1814 struct skb_shared_info *shi; 1815 int i, ret = 0; 1816 1817 len = min_t(size_t, len, desc->count); 1818 /* 1819 * __tcp_read_sock() always calls io_zcrx_recv_skb one last time, even 1820 * if desc->count is already 0. This is caused by the if (offset + 1 != 1821 * skb->len) check. Return early in this case to break out of 1822 * __tcp_read_sock(). 1823 */ 1824 if (!len) 1825 return 0; 1826 if (unlikely(args->nr_skbs++ > IO_SKBS_PER_CALL_LIMIT)) 1827 return -EAGAIN; 1828 1829 if (unlikely(offset < skb_headlen(skb))) { 1830 ssize_t copied; 1831 size_t to_copy; 1832 1833 to_copy = min_t(size_t, skb_headlen(skb) - offset, len); 1834 copied = io_zcrx_copy_chunk(req, ifq, virt_to_page(skb->data), 1835 offset_in_page(skb->data) + offset, 1836 to_copy); 1837 if (copied < 0) { 1838 ret = copied; 1839 goto out; 1840 } 1841 offset += copied; 1842 len -= copied; 1843 if (!len) 1844 goto out; 1845 if (offset != skb_headlen(skb)) 1846 goto out; 1847 } 1848 1849 start = skb_headlen(skb); 1850 shi = skb_shinfo(skb); 1851 1852 for (i = 0; i < shi->nr_frags; i++) { 1853 const skb_frag_t *frag = &shi->frags[i]; 1854 unsigned frag_end = start + skb_frag_size(frag); 1855 1856 if (WARN_ON(start > offset + len)) 1857 return -EFAULT; 1858 1859 if (offset < frag_end) { 1860 unsigned copy = min(frag_end - offset, len); 1861 unsigned frag_off = offset - start; 1862 1863 ret = io_zcrx_recv_frag(req, ifq, frag, frag_off, copy); 1864 if (ret < 0) 1865 goto out; 1866 1867 offset += ret; 1868 len -= ret; 1869 if (len == 0 || ret != copy) 1870 goto out; 1871 } 1872 start = frag_end; 1873 } 1874 1875 skb_walk_frags(skb, frag_iter) { 1876 unsigned frag_end; 1877 1878 if (WARN_ON(start > offset + len)) 1879 return -EFAULT; 1880 1881 frag_end = start + frag_iter->len; 1882 if (offset < frag_end) { 1883 unsigned copy = min(frag_end - offset, len); 1884 unsigned frag_off = offset - start; 1885 size_t count; 1886 1887 count = desc->count; 1888 ret = io_zcrx_recv_skb(desc, frag_iter, frag_off, copy); 1889 desc->count = count; 1890 if (ret < 0) 1891 goto out; 1892 1893 offset += ret; 1894 len -= ret; 1895 if (len == 0 || ret != copy) 1896 goto out; 1897 } 1898 start = frag_end; 1899 } 1900 1901 out: 1902 if (offset == start_off) 1903 return ret; 1904 desc->count -= (offset - start_off); 1905 return offset - start_off; 1906 } 1907 1908 static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1909 struct sock *sk, int flags, 1910 unsigned issue_flags, unsigned int *outlen) 1911 { 1912 unsigned int len = *outlen; 1913 struct io_zcrx_args args = { 1914 .req = req, 1915 .ifq = ifq, 1916 }; 1917 read_descriptor_t rd_desc = { 1918 .count = len ? len : UINT_MAX, 1919 .arg.data = &args, 1920 }; 1921 int ret; 1922 1923 lock_sock(sk); 1924 ret = tcp_read_sock(sk, &rd_desc, io_zcrx_recv_skb); 1925 if (len && ret > 0) 1926 *outlen = len - ret; 1927 if (ret <= 0) { 1928 if (ret < 0 || sock_flag(sk, SOCK_DONE)) 1929 goto out; 1930 if (sk->sk_err) 1931 ret = sock_error(sk); 1932 else if (sk->sk_shutdown & RCV_SHUTDOWN) 1933 goto out; 1934 else if (sk->sk_state == TCP_CLOSE) 1935 ret = -ENOTCONN; 1936 else 1937 ret = -EAGAIN; 1938 } else if (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) && 1939 (issue_flags & IO_URING_F_MULTISHOT)) { 1940 ret = IOU_REQUEUE; 1941 } else if (sock_flag(sk, SOCK_DONE)) { 1942 /* Make it to retry until it finally gets 0. */ 1943 if (issue_flags & IO_URING_F_MULTISHOT) 1944 ret = IOU_REQUEUE; 1945 else 1946 ret = -EAGAIN; 1947 } 1948 out: 1949 release_sock(sk); 1950 return ret; 1951 } 1952 1953 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, 1954 struct socket *sock, unsigned int flags, 1955 unsigned issue_flags, unsigned int *len) 1956 { 1957 struct sock *sk = sock->sk; 1958 const struct proto *prot = READ_ONCE(sk->sk_prot); 1959 1960 if (prot->recvmsg != tcp_recvmsg) 1961 return -EPROTONOSUPPORT; 1962 1963 sock_rps_record_flow(sk); 1964 return io_zcrx_tcp_recvmsg(req, ifq, sk, flags, issue_flags, len); 1965 } 1966