1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/dma-map-ops.h>
5 #include <linux/mm.h>
6 #include <linux/nospec.h>
7 #include <linux/io_uring.h>
8 #include <linux/netdevice.h>
9 #include <linux/rtnetlink.h>
10 #include <linux/skbuff_ref.h>
11 #include <linux/anon_inodes.h>
12
13 #include <net/page_pool/helpers.h>
14 #include <net/page_pool/memory_provider.h>
15 #include <net/netlink.h>
16 #include <net/netdev_queues.h>
17 #include <net/netdev_rx_queue.h>
18 #include <net/tcp.h>
19 #include <net/rps.h>
20
21 #include <trace/events/page_pool.h>
22
23 #include <uapi/linux/io_uring.h>
24
25 #include "io_uring.h"
26 #include "kbuf.h"
27 #include "memmap.h"
28 #include "zcrx.h"
29 #include "rsrc.h"
30
31 #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF)
32
33 #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
34
io_pp_to_ifq(struct page_pool * pp)35 static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)
36 {
37 return pp->mp_priv;
38 }
39
io_zcrx_iov_to_area(const struct net_iov * niov)40 static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *niov)
41 {
42 struct net_iov_area *owner = net_iov_owner(niov);
43
44 return container_of(owner, struct io_zcrx_area, nia);
45 }
46
io_zcrx_iov_page(const struct net_iov * niov)47 static inline struct page *io_zcrx_iov_page(const struct net_iov *niov)
48 {
49 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
50 unsigned niov_pages_shift;
51
52 lockdep_assert(!area->mem.is_dmabuf);
53
54 niov_pages_shift = area->ifq->niov_shift - PAGE_SHIFT;
55 return area->mem.pages[net_iov_idx(niov) << niov_pages_shift];
56 }
57
io_area_max_shift(struct io_zcrx_mem * mem)58 static int io_area_max_shift(struct io_zcrx_mem *mem)
59 {
60 struct sg_table *sgt = mem->sgt;
61 struct scatterlist *sg;
62 unsigned shift = -1U;
63 unsigned i;
64
65 for_each_sgtable_dma_sg(sgt, sg, i)
66 shift = min(shift, __ffs(sg->length));
67 return shift;
68 }
69
io_populate_area_dma(struct io_zcrx_ifq * ifq,struct io_zcrx_area * area)70 static int io_populate_area_dma(struct io_zcrx_ifq *ifq,
71 struct io_zcrx_area *area)
72 {
73 unsigned niov_size = 1U << ifq->niov_shift;
74 struct sg_table *sgt = area->mem.sgt;
75 struct scatterlist *sg;
76 unsigned i, niov_idx = 0;
77
78 for_each_sgtable_dma_sg(sgt, sg, i) {
79 dma_addr_t dma = sg_dma_address(sg);
80 unsigned long sg_len = sg_dma_len(sg);
81
82 if (WARN_ON_ONCE(sg_len % niov_size))
83 return -EINVAL;
84
85 while (sg_len && niov_idx < area->nia.num_niovs) {
86 struct net_iov *niov = &area->nia.niovs[niov_idx];
87
88 if (net_mp_niov_set_dma_addr(niov, dma))
89 return -EFAULT;
90 sg_len -= niov_size;
91 dma += niov_size;
92 niov_idx++;
93 }
94 }
95
96 if (WARN_ON_ONCE(niov_idx != area->nia.num_niovs))
97 return -EFAULT;
98 return 0;
99 }
100
io_release_dmabuf(struct io_zcrx_mem * mem)101 static void io_release_dmabuf(struct io_zcrx_mem *mem)
102 {
103 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
104 return;
105
106 if (mem->sgt)
107 dma_buf_unmap_attachment_unlocked(mem->attach, mem->sgt,
108 DMA_FROM_DEVICE);
109 if (mem->attach)
110 dma_buf_detach(mem->dmabuf, mem->attach);
111 if (mem->dmabuf)
112 dma_buf_put(mem->dmabuf);
113
114 mem->sgt = NULL;
115 mem->attach = NULL;
116 mem->dmabuf = NULL;
117 }
118
io_import_dmabuf(struct io_zcrx_ifq * ifq,struct io_zcrx_mem * mem,struct io_uring_zcrx_area_reg * area_reg)119 static int io_import_dmabuf(struct io_zcrx_ifq *ifq,
120 struct io_zcrx_mem *mem,
121 struct io_uring_zcrx_area_reg *area_reg)
122 {
123 unsigned long off = (unsigned long)area_reg->addr;
124 unsigned long len = (unsigned long)area_reg->len;
125 unsigned long total_size = 0;
126 struct scatterlist *sg;
127 int dmabuf_fd = area_reg->dmabuf_fd;
128 int i, ret;
129
130 if (off)
131 return -EINVAL;
132 if (WARN_ON_ONCE(!ifq->dev))
133 return -EFAULT;
134 if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
135 return -EINVAL;
136
137 mem->is_dmabuf = true;
138 mem->dmabuf = dma_buf_get(dmabuf_fd);
139 if (IS_ERR(mem->dmabuf)) {
140 ret = PTR_ERR(mem->dmabuf);
141 mem->dmabuf = NULL;
142 goto err;
143 }
144
145 mem->attach = dma_buf_attach(mem->dmabuf, ifq->dev);
146 if (IS_ERR(mem->attach)) {
147 ret = PTR_ERR(mem->attach);
148 mem->attach = NULL;
149 goto err;
150 }
151
152 mem->sgt = dma_buf_map_attachment_unlocked(mem->attach, DMA_FROM_DEVICE);
153 if (IS_ERR(mem->sgt)) {
154 ret = PTR_ERR(mem->sgt);
155 mem->sgt = NULL;
156 goto err;
157 }
158
159 for_each_sgtable_dma_sg(mem->sgt, sg, i)
160 total_size += sg_dma_len(sg);
161
162 if (total_size != len) {
163 ret = -EINVAL;
164 goto err;
165 }
166
167 mem->size = len;
168 return 0;
169 err:
170 io_release_dmabuf(mem);
171 return ret;
172 }
173
io_count_account_pages(struct page ** pages,unsigned nr_pages)174 static unsigned long io_count_account_pages(struct page **pages, unsigned nr_pages)
175 {
176 struct folio *last_folio = NULL;
177 unsigned long res = 0;
178 int i;
179
180 for (i = 0; i < nr_pages; i++) {
181 struct folio *folio = page_folio(pages[i]);
182
183 if (folio == last_folio)
184 continue;
185 last_folio = folio;
186 res += folio_nr_pages(folio);
187 }
188 return res;
189 }
190
io_import_umem(struct io_zcrx_ifq * ifq,struct io_zcrx_mem * mem,struct io_uring_zcrx_area_reg * area_reg)191 static int io_import_umem(struct io_zcrx_ifq *ifq,
192 struct io_zcrx_mem *mem,
193 struct io_uring_zcrx_area_reg *area_reg)
194 {
195 struct page **pages;
196 int nr_pages, ret;
197
198 if (area_reg->dmabuf_fd)
199 return -EINVAL;
200 if (!area_reg->addr)
201 return -EFAULT;
202 pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len,
203 &nr_pages);
204 if (IS_ERR(pages))
205 return PTR_ERR(pages);
206
207 ret = sg_alloc_table_from_pages(&mem->page_sg_table, pages, nr_pages,
208 0, (unsigned long)nr_pages << PAGE_SHIFT,
209 GFP_KERNEL_ACCOUNT);
210 if (ret) {
211 unpin_user_pages(pages, nr_pages);
212 kvfree(pages);
213 return ret;
214 }
215
216 mem->account_pages = io_count_account_pages(pages, nr_pages);
217 ret = io_account_mem(ifq->user, ifq->mm_account, mem->account_pages);
218 if (ret < 0)
219 mem->account_pages = 0;
220
221 mem->sgt = &mem->page_sg_table;
222 mem->pages = pages;
223 mem->nr_folios = nr_pages;
224 mem->size = area_reg->len;
225 return ret;
226 }
227
io_release_area_mem(struct io_zcrx_mem * mem)228 static void io_release_area_mem(struct io_zcrx_mem *mem)
229 {
230 if (mem->is_dmabuf) {
231 io_release_dmabuf(mem);
232 return;
233 }
234 if (mem->pages) {
235 unpin_user_pages(mem->pages, mem->nr_folios);
236 sg_free_table(mem->sgt);
237 mem->sgt = NULL;
238 kvfree(mem->pages);
239 }
240 }
241
io_import_area(struct io_zcrx_ifq * ifq,struct io_zcrx_mem * mem,struct io_uring_zcrx_area_reg * area_reg)242 static int io_import_area(struct io_zcrx_ifq *ifq,
243 struct io_zcrx_mem *mem,
244 struct io_uring_zcrx_area_reg *area_reg)
245 {
246 int ret;
247
248 if (area_reg->flags & ~IO_ZCRX_AREA_SUPPORTED_FLAGS)
249 return -EINVAL;
250 if (area_reg->rq_area_token)
251 return -EINVAL;
252 if (area_reg->__resv2[0] || area_reg->__resv2[1])
253 return -EINVAL;
254
255 ret = io_validate_user_buf_range(area_reg->addr, area_reg->len);
256 if (ret)
257 return ret;
258 if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK)
259 return -EINVAL;
260
261 if (area_reg->flags & IORING_ZCRX_AREA_DMABUF)
262 return io_import_dmabuf(ifq, mem, area_reg);
263 return io_import_umem(ifq, mem, area_reg);
264 }
265
io_zcrx_unmap_area(struct io_zcrx_ifq * ifq,struct io_zcrx_area * area)266 static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
267 struct io_zcrx_area *area)
268 {
269 int i;
270
271 guard(mutex)(&ifq->pp_lock);
272 if (!area->is_mapped)
273 return;
274 area->is_mapped = false;
275
276 for (i = 0; i < area->nia.num_niovs; i++)
277 net_mp_niov_set_dma_addr(&area->nia.niovs[i], 0);
278
279 if (area->mem.is_dmabuf) {
280 io_release_dmabuf(&area->mem);
281 } else {
282 dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table,
283 DMA_FROM_DEVICE, IO_DMA_ATTR);
284 }
285 }
286
io_zcrx_map_area(struct io_zcrx_ifq * ifq,struct io_zcrx_area * area)287 static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
288 {
289 int ret;
290
291 guard(mutex)(&ifq->pp_lock);
292 if (area->is_mapped)
293 return 0;
294
295 if (!area->mem.is_dmabuf) {
296 ret = dma_map_sgtable(ifq->dev, &area->mem.page_sg_table,
297 DMA_FROM_DEVICE, IO_DMA_ATTR);
298 if (ret < 0)
299 return ret;
300 }
301
302 ret = io_populate_area_dma(ifq, area);
303 if (ret && !area->mem.is_dmabuf)
304 dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table,
305 DMA_FROM_DEVICE, IO_DMA_ATTR);
306 if (ret == 0)
307 area->is_mapped = true;
308 return ret;
309 }
310
io_zcrx_sync_for_device(struct page_pool * pool,struct net_iov * niov)311 static void io_zcrx_sync_for_device(struct page_pool *pool,
312 struct net_iov *niov)
313 {
314 #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC)
315 dma_addr_t dma_addr;
316
317 unsigned niov_size;
318
319 if (!dma_dev_need_sync(pool->p.dev))
320 return;
321
322 niov_size = 1U << io_pp_to_ifq(pool)->niov_shift;
323 dma_addr = page_pool_get_dma_addr_netmem(net_iov_to_netmem(niov));
324 __dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset,
325 niov_size, pool->p.dma_dir);
326 #endif
327 }
328
329 #define IO_RQ_MAX_ENTRIES 32768
330
331 #define IO_SKBS_PER_CALL_LIMIT 20
332
333 struct io_zcrx_args {
334 struct io_kiocb *req;
335 struct io_zcrx_ifq *ifq;
336 struct socket *sock;
337 unsigned nr_skbs;
338 };
339
340 static const struct memory_provider_ops io_uring_pp_zc_ops;
341
io_get_user_counter(struct net_iov * niov)342 static inline atomic_t *io_get_user_counter(struct net_iov *niov)
343 {
344 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
345
346 return &area->user_refs[net_iov_idx(niov)];
347 }
348
io_zcrx_put_niov_uref(struct net_iov * niov)349 static bool io_zcrx_put_niov_uref(struct net_iov *niov)
350 {
351 atomic_t *uref = io_get_user_counter(niov);
352 int old;
353
354 old = atomic_read(uref);
355 do {
356 if (unlikely(old == 0))
357 return false;
358 } while (!atomic_try_cmpxchg(uref, &old, old - 1));
359
360 return true;
361 }
362
io_zcrx_get_niov_uref(struct net_iov * niov)363 static void io_zcrx_get_niov_uref(struct net_iov *niov)
364 {
365 atomic_inc(io_get_user_counter(niov));
366 }
367
io_fill_zcrx_offsets(struct io_uring_zcrx_offsets * offsets)368 static void io_fill_zcrx_offsets(struct io_uring_zcrx_offsets *offsets)
369 {
370 offsets->head = offsetof(struct io_uring, head);
371 offsets->tail = offsetof(struct io_uring, tail);
372 offsets->rqes = ALIGN(sizeof(struct io_uring), L1_CACHE_BYTES);
373 }
374
io_allocate_rbuf_ring(struct io_ring_ctx * ctx,struct io_zcrx_ifq * ifq,struct io_uring_zcrx_ifq_reg * reg,struct io_uring_region_desc * rd,u32 id)375 static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
376 struct io_zcrx_ifq *ifq,
377 struct io_uring_zcrx_ifq_reg *reg,
378 struct io_uring_region_desc *rd,
379 u32 id)
380 {
381 u64 mmap_offset;
382 size_t off, size;
383 void *ptr;
384 int ret;
385
386 io_fill_zcrx_offsets(®->offsets);
387 off = reg->offsets.rqes;
388 size = off + sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries;
389 if (size > rd->size)
390 return -EINVAL;
391
392 mmap_offset = IORING_MAP_OFF_ZCRX_REGION;
393 mmap_offset += id << IORING_OFF_PBUF_SHIFT;
394
395 ret = io_create_region(ctx, &ifq->region, rd, mmap_offset);
396 if (ret < 0)
397 return ret;
398
399 ptr = io_region_get_ptr(&ifq->region);
400 ifq->rq_ring = (struct io_uring *)ptr;
401 ifq->rqes = (struct io_uring_zcrx_rqe *)(ptr + off);
402
403 return 0;
404 }
405
io_free_rbuf_ring(struct io_zcrx_ifq * ifq)406 static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq)
407 {
408 io_free_region(ifq->user, &ifq->region);
409 ifq->rq_ring = NULL;
410 ifq->rqes = NULL;
411 }
412
io_zcrx_free_area(struct io_zcrx_ifq * ifq,struct io_zcrx_area * area)413 static void io_zcrx_free_area(struct io_zcrx_ifq *ifq,
414 struct io_zcrx_area *area)
415 {
416 io_zcrx_unmap_area(ifq, area);
417 io_release_area_mem(&area->mem);
418
419 if (area->mem.account_pages)
420 io_unaccount_mem(ifq->user, ifq->mm_account,
421 area->mem.account_pages);
422
423 kvfree(area->freelist);
424 kvfree(area->nia.niovs);
425 kvfree(area->user_refs);
426 kfree(area);
427 }
428
io_zcrx_append_area(struct io_zcrx_ifq * ifq,struct io_zcrx_area * area)429 static int io_zcrx_append_area(struct io_zcrx_ifq *ifq,
430 struct io_zcrx_area *area)
431 {
432 if (ifq->area)
433 return -EINVAL;
434 ifq->area = area;
435 return 0;
436 }
437
io_zcrx_create_area(struct io_zcrx_ifq * ifq,struct io_uring_zcrx_area_reg * area_reg,struct io_uring_zcrx_ifq_reg * reg)438 static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
439 struct io_uring_zcrx_area_reg *area_reg,
440 struct io_uring_zcrx_ifq_reg *reg)
441 {
442 int buf_size_shift = PAGE_SHIFT;
443 struct io_zcrx_area *area;
444 unsigned nr_iovs;
445 int i, ret;
446
447 if (reg->rx_buf_len) {
448 if (!is_power_of_2(reg->rx_buf_len) ||
449 reg->rx_buf_len < PAGE_SIZE)
450 return -EINVAL;
451 buf_size_shift = ilog2(reg->rx_buf_len);
452 }
453
454 ret = -ENOMEM;
455 area = kzalloc_obj(*area);
456 if (!area)
457 goto err;
458 area->ifq = ifq;
459
460 ret = io_import_area(ifq, &area->mem, area_reg);
461 if (ret)
462 goto err;
463
464 if (buf_size_shift > io_area_max_shift(&area->mem)) {
465 ret = -ERANGE;
466 goto err;
467 }
468
469 ifq->niov_shift = buf_size_shift;
470 nr_iovs = area->mem.size >> ifq->niov_shift;
471 area->nia.num_niovs = nr_iovs;
472
473 ret = -ENOMEM;
474 area->nia.niovs = kvmalloc_objs(area->nia.niovs[0], nr_iovs,
475 GFP_KERNEL_ACCOUNT | __GFP_ZERO);
476 if (!area->nia.niovs)
477 goto err;
478
479 area->freelist = kvmalloc_array(nr_iovs, sizeof(area->freelist[0]),
480 GFP_KERNEL_ACCOUNT | __GFP_ZERO);
481 if (!area->freelist)
482 goto err;
483
484 area->user_refs = kvmalloc_objs(area->user_refs[0], nr_iovs,
485 GFP_KERNEL_ACCOUNT | __GFP_ZERO);
486 if (!area->user_refs)
487 goto err;
488
489 for (i = 0; i < nr_iovs; i++) {
490 struct net_iov *niov = &area->nia.niovs[i];
491
492 niov->owner = &area->nia;
493 area->freelist[i] = i;
494 atomic_set(&area->user_refs[i], 0);
495 niov->type = NET_IOV_IOURING;
496 }
497
498 area->free_count = nr_iovs;
499 /* we're only supporting one area per ifq for now */
500 area->area_id = 0;
501 area_reg->rq_area_token = (u64)area->area_id << IORING_ZCRX_AREA_SHIFT;
502 spin_lock_init(&area->freelist_lock);
503
504 ret = io_zcrx_append_area(ifq, area);
505 if (!ret)
506 return 0;
507 err:
508 if (area)
509 io_zcrx_free_area(ifq, area);
510 return ret;
511 }
512
io_zcrx_ifq_alloc(struct io_ring_ctx * ctx)513 static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx)
514 {
515 struct io_zcrx_ifq *ifq;
516
517 ifq = kzalloc_obj(*ifq);
518 if (!ifq)
519 return NULL;
520
521 ifq->if_rxq = -1;
522 spin_lock_init(&ifq->rq_lock);
523 mutex_init(&ifq->pp_lock);
524 refcount_set(&ifq->refs, 1);
525 refcount_set(&ifq->user_refs, 1);
526 return ifq;
527 }
528
io_zcrx_drop_netdev(struct io_zcrx_ifq * ifq)529 static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq)
530 {
531 guard(mutex)(&ifq->pp_lock);
532
533 if (!ifq->netdev)
534 return;
535 netdev_put(ifq->netdev, &ifq->netdev_tracker);
536 ifq->netdev = NULL;
537 }
538
io_close_queue(struct io_zcrx_ifq * ifq)539 static void io_close_queue(struct io_zcrx_ifq *ifq)
540 {
541 struct net_device *netdev;
542 netdevice_tracker netdev_tracker;
543 struct pp_memory_provider_params p = {
544 .mp_ops = &io_uring_pp_zc_ops,
545 .mp_priv = ifq,
546 };
547
548 scoped_guard(mutex, &ifq->pp_lock) {
549 netdev = ifq->netdev;
550 netdev_tracker = ifq->netdev_tracker;
551 ifq->netdev = NULL;
552 }
553
554 if (netdev) {
555 if (ifq->if_rxq != -1)
556 net_mp_close_rxq(netdev, ifq->if_rxq, &p);
557 netdev_put(netdev, &netdev_tracker);
558 }
559 ifq->if_rxq = -1;
560 }
561
io_zcrx_ifq_free(struct io_zcrx_ifq * ifq)562 static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
563 {
564 io_close_queue(ifq);
565
566 if (ifq->area)
567 io_zcrx_free_area(ifq, ifq->area);
568 free_uid(ifq->user);
569 if (ifq->mm_account)
570 mmdrop(ifq->mm_account);
571 if (ifq->dev)
572 put_device(ifq->dev);
573
574 io_free_rbuf_ring(ifq);
575 mutex_destroy(&ifq->pp_lock);
576 kfree(ifq);
577 }
578
io_put_zcrx_ifq(struct io_zcrx_ifq * ifq)579 static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq)
580 {
581 if (refcount_dec_and_test(&ifq->refs))
582 io_zcrx_ifq_free(ifq);
583 }
584
io_zcrx_return_niov_freelist(struct net_iov * niov)585 static void io_zcrx_return_niov_freelist(struct net_iov *niov)
586 {
587 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
588
589 spin_lock_bh(&area->freelist_lock);
590 area->freelist[area->free_count++] = net_iov_idx(niov);
591 spin_unlock_bh(&area->freelist_lock);
592 }
593
io_zcrx_return_niov(struct net_iov * niov)594 static void io_zcrx_return_niov(struct net_iov *niov)
595 {
596 netmem_ref netmem = net_iov_to_netmem(niov);
597
598 if (!niov->desc.pp) {
599 /* copy fallback allocated niovs */
600 io_zcrx_return_niov_freelist(niov);
601 return;
602 }
603 page_pool_put_unrefed_netmem(niov->desc.pp, netmem, -1, false);
604 }
605
io_zcrx_scrub(struct io_zcrx_ifq * ifq)606 static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
607 {
608 struct io_zcrx_area *area = ifq->area;
609 int i;
610
611 if (!area)
612 return;
613
614 /* Reclaim back all buffers given to the user space. */
615 for (i = 0; i < area->nia.num_niovs; i++) {
616 struct net_iov *niov = &area->nia.niovs[i];
617 int nr;
618
619 if (!atomic_read(io_get_user_counter(niov)))
620 continue;
621 nr = atomic_xchg(io_get_user_counter(niov), 0);
622 if (nr && !page_pool_unref_netmem(net_iov_to_netmem(niov), nr))
623 io_zcrx_return_niov(niov);
624 }
625 }
626
zcrx_unregister(struct io_zcrx_ifq * ifq)627 static void zcrx_unregister(struct io_zcrx_ifq *ifq)
628 {
629 if (refcount_dec_and_test(&ifq->user_refs)) {
630 io_close_queue(ifq);
631 io_zcrx_scrub(ifq);
632 }
633 io_put_zcrx_ifq(ifq);
634 }
635
io_zcrx_get_region(struct io_ring_ctx * ctx,unsigned int id)636 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
637 unsigned int id)
638 {
639 struct io_zcrx_ifq *ifq = xa_load(&ctx->zcrx_ctxs, id);
640
641 lockdep_assert_held(&ctx->mmap_lock);
642
643 return ifq ? &ifq->region : NULL;
644 }
645
zcrx_box_release(struct inode * inode,struct file * file)646 static int zcrx_box_release(struct inode *inode, struct file *file)
647 {
648 struct io_zcrx_ifq *ifq = file->private_data;
649
650 if (WARN_ON_ONCE(!ifq))
651 return -EFAULT;
652 zcrx_unregister(ifq);
653 return 0;
654 }
655
656 static const struct file_operations zcrx_box_fops = {
657 .owner = THIS_MODULE,
658 .release = zcrx_box_release,
659 };
660
zcrx_export(struct io_ring_ctx * ctx,struct io_zcrx_ifq * ifq,struct zcrx_ctrl * ctrl,void __user * arg)661 static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq,
662 struct zcrx_ctrl *ctrl, void __user *arg)
663 {
664 struct zcrx_ctrl_export *ce = &ctrl->zc_export;
665 struct file *file;
666 int fd = -1;
667
668 if (!mem_is_zero(ce, sizeof(*ce)))
669 return -EINVAL;
670 fd = get_unused_fd_flags(O_CLOEXEC);
671 if (fd < 0)
672 return fd;
673
674 ce->zcrx_fd = fd;
675 if (copy_to_user(arg, ctrl, sizeof(*ctrl))) {
676 put_unused_fd(fd);
677 return -EFAULT;
678 }
679
680 refcount_inc(&ifq->refs);
681 refcount_inc(&ifq->user_refs);
682
683 file = anon_inode_create_getfile("[zcrx]", &zcrx_box_fops,
684 ifq, O_CLOEXEC, NULL);
685 if (IS_ERR(file)) {
686 put_unused_fd(fd);
687 zcrx_unregister(ifq);
688 return PTR_ERR(file);
689 }
690
691 fd_install(fd, file);
692 return 0;
693 }
694
import_zcrx(struct io_ring_ctx * ctx,struct io_uring_zcrx_ifq_reg __user * arg,struct io_uring_zcrx_ifq_reg * reg)695 static int import_zcrx(struct io_ring_ctx *ctx,
696 struct io_uring_zcrx_ifq_reg __user *arg,
697 struct io_uring_zcrx_ifq_reg *reg)
698 {
699 struct io_zcrx_ifq *ifq;
700 struct file *file;
701 int fd, ret;
702 u32 id;
703
704 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
705 return -EINVAL;
706 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)))
707 return -EINVAL;
708 if (reg->if_rxq || reg->rq_entries || reg->area_ptr || reg->region_ptr)
709 return -EINVAL;
710 if (reg->flags & ~ZCRX_REG_IMPORT)
711 return -EINVAL;
712
713 fd = reg->if_idx;
714 CLASS(fd, f)(fd);
715 if (fd_empty(f))
716 return -EBADF;
717
718 file = fd_file(f);
719 if (file->f_op != &zcrx_box_fops || !file->private_data)
720 return -EBADF;
721
722 ifq = file->private_data;
723 refcount_inc(&ifq->refs);
724 refcount_inc(&ifq->user_refs);
725
726 scoped_guard(mutex, &ctx->mmap_lock) {
727 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL);
728 if (ret)
729 goto err;
730 }
731
732 reg->zcrx_id = id;
733 io_fill_zcrx_offsets(®->offsets);
734 if (copy_to_user(arg, reg, sizeof(*reg))) {
735 ret = -EFAULT;
736 goto err_xa_erase;
737 }
738
739 scoped_guard(mutex, &ctx->mmap_lock) {
740 ret = -ENOMEM;
741 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL))
742 goto err_xa_erase;
743 }
744
745 return 0;
746 err_xa_erase:
747 scoped_guard(mutex, &ctx->mmap_lock)
748 xa_erase(&ctx->zcrx_ctxs, id);
749 err:
750 zcrx_unregister(ifq);
751 return ret;
752 }
753
io_register_zcrx_ifq(struct io_ring_ctx * ctx,struct io_uring_zcrx_ifq_reg __user * arg)754 int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
755 struct io_uring_zcrx_ifq_reg __user *arg)
756 {
757 struct pp_memory_provider_params mp_param = {};
758 struct io_uring_zcrx_area_reg area;
759 struct io_uring_zcrx_ifq_reg reg;
760 struct io_uring_region_desc rd;
761 struct io_zcrx_ifq *ifq;
762 int ret;
763 u32 id;
764
765 /*
766 * 1. Interface queue allocation.
767 * 2. It can observe data destined for sockets of other tasks.
768 */
769 if (!capable(CAP_NET_ADMIN))
770 return -EPERM;
771
772 /* mandatory io_uring features for zc rx */
773 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
774 return -EINVAL;
775 if (!(ctx->flags & (IORING_SETUP_CQE32|IORING_SETUP_CQE_MIXED)))
776 return -EINVAL;
777 if (copy_from_user(®, arg, sizeof(reg)))
778 return -EFAULT;
779 if (!mem_is_zero(®.__resv, sizeof(reg.__resv)) || reg.zcrx_id)
780 return -EINVAL;
781 if (reg.flags & ZCRX_REG_IMPORT)
782 return import_zcrx(ctx, arg, ®);
783 if (copy_from_user(&rd, u64_to_user_ptr(reg.region_ptr), sizeof(rd)))
784 return -EFAULT;
785 if (reg.if_rxq == -1 || !reg.rq_entries || reg.flags)
786 return -EINVAL;
787 if (reg.rq_entries > IO_RQ_MAX_ENTRIES) {
788 if (!(ctx->flags & IORING_SETUP_CLAMP))
789 return -EINVAL;
790 reg.rq_entries = IO_RQ_MAX_ENTRIES;
791 }
792 reg.rq_entries = roundup_pow_of_two(reg.rq_entries);
793
794 if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area)))
795 return -EFAULT;
796
797 ifq = io_zcrx_ifq_alloc(ctx);
798 if (!ifq)
799 return -ENOMEM;
800
801 if (ctx->user) {
802 get_uid(ctx->user);
803 ifq->user = ctx->user;
804 }
805 if (ctx->mm_account) {
806 mmgrab(ctx->mm_account);
807 ifq->mm_account = ctx->mm_account;
808 }
809 ifq->rq_entries = reg.rq_entries;
810
811 scoped_guard(mutex, &ctx->mmap_lock) {
812 /* preallocate id */
813 ret = xa_alloc(&ctx->zcrx_ctxs, &id, NULL, xa_limit_31b, GFP_KERNEL);
814 if (ret)
815 goto ifq_free;
816 }
817
818 ret = io_allocate_rbuf_ring(ctx, ifq, ®, &rd, id);
819 if (ret)
820 goto err;
821
822 ifq->netdev = netdev_get_by_index_lock(current->nsproxy->net_ns, reg.if_idx);
823 if (!ifq->netdev) {
824 ret = -ENODEV;
825 goto err;
826 }
827 netdev_hold(ifq->netdev, &ifq->netdev_tracker, GFP_KERNEL);
828
829 ifq->dev = netdev_queue_get_dma_dev(ifq->netdev, reg.if_rxq);
830 if (!ifq->dev) {
831 ret = -EOPNOTSUPP;
832 goto netdev_put_unlock;
833 }
834 get_device(ifq->dev);
835
836 ret = io_zcrx_create_area(ifq, &area, ®);
837 if (ret)
838 goto netdev_put_unlock;
839
840 mp_param.rx_page_size = 1U << ifq->niov_shift;
841 mp_param.mp_ops = &io_uring_pp_zc_ops;
842 mp_param.mp_priv = ifq;
843 ret = __net_mp_open_rxq(ifq->netdev, reg.if_rxq, &mp_param, NULL);
844 if (ret)
845 goto netdev_put_unlock;
846 netdev_unlock(ifq->netdev);
847 ifq->if_rxq = reg.if_rxq;
848
849 reg.zcrx_id = id;
850
851 scoped_guard(mutex, &ctx->mmap_lock) {
852 /* publish ifq */
853 ret = -ENOMEM;
854 if (xa_store(&ctx->zcrx_ctxs, id, ifq, GFP_KERNEL))
855 goto err;
856 }
857
858 reg.rx_buf_len = 1U << ifq->niov_shift;
859
860 if (copy_to_user(arg, ®, sizeof(reg)) ||
861 copy_to_user(u64_to_user_ptr(reg.region_ptr), &rd, sizeof(rd)) ||
862 copy_to_user(u64_to_user_ptr(reg.area_ptr), &area, sizeof(area))) {
863 ret = -EFAULT;
864 goto err;
865 }
866 return 0;
867 netdev_put_unlock:
868 netdev_unlock(ifq->netdev);
869 err:
870 scoped_guard(mutex, &ctx->mmap_lock)
871 xa_erase(&ctx->zcrx_ctxs, id);
872 ifq_free:
873 zcrx_unregister(ifq);
874 return ret;
875 }
876
__io_zcrx_get_free_niov(struct io_zcrx_area * area)877 static struct net_iov *__io_zcrx_get_free_niov(struct io_zcrx_area *area)
878 {
879 unsigned niov_idx;
880
881 lockdep_assert_held(&area->freelist_lock);
882
883 niov_idx = area->freelist[--area->free_count];
884 return &area->nia.niovs[niov_idx];
885 }
886
io_unregister_zcrx_ifqs(struct io_ring_ctx * ctx)887 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
888 {
889 struct io_zcrx_ifq *ifq;
890
891 lockdep_assert_held(&ctx->uring_lock);
892
893 while (1) {
894 scoped_guard(mutex, &ctx->mmap_lock) {
895 unsigned long id = 0;
896
897 ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT);
898 if (ifq)
899 xa_erase(&ctx->zcrx_ctxs, id);
900 }
901 if (!ifq)
902 break;
903 zcrx_unregister(ifq);
904 }
905
906 xa_destroy(&ctx->zcrx_ctxs);
907 }
908
io_zcrx_rqring_entries(struct io_zcrx_ifq * ifq)909 static inline u32 io_zcrx_rqring_entries(struct io_zcrx_ifq *ifq)
910 {
911 u32 entries;
912
913 entries = smp_load_acquire(&ifq->rq_ring->tail) - ifq->cached_rq_head;
914 return min(entries, ifq->rq_entries);
915 }
916
io_zcrx_get_rqe(struct io_zcrx_ifq * ifq,unsigned mask)917 static struct io_uring_zcrx_rqe *io_zcrx_get_rqe(struct io_zcrx_ifq *ifq,
918 unsigned mask)
919 {
920 unsigned int idx = ifq->cached_rq_head++ & mask;
921
922 return &ifq->rqes[idx];
923 }
924
io_parse_rqe(struct io_uring_zcrx_rqe * rqe,struct io_zcrx_ifq * ifq,struct net_iov ** ret_niov)925 static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe,
926 struct io_zcrx_ifq *ifq,
927 struct net_iov **ret_niov)
928 {
929 unsigned niov_idx, area_idx;
930 struct io_zcrx_area *area;
931
932 area_idx = rqe->off >> IORING_ZCRX_AREA_SHIFT;
933 niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift;
934
935 if (unlikely(rqe->__pad || area_idx))
936 return false;
937 area = ifq->area;
938
939 if (unlikely(niov_idx >= area->nia.num_niovs))
940 return false;
941 niov_idx = array_index_nospec(niov_idx, area->nia.num_niovs);
942
943 *ret_niov = &area->nia.niovs[niov_idx];
944 return true;
945 }
946
io_zcrx_ring_refill(struct page_pool * pp,struct io_zcrx_ifq * ifq)947 static void io_zcrx_ring_refill(struct page_pool *pp,
948 struct io_zcrx_ifq *ifq)
949 {
950 unsigned int mask = ifq->rq_entries - 1;
951 unsigned int entries;
952
953 guard(spinlock_bh)(&ifq->rq_lock);
954
955 entries = io_zcrx_rqring_entries(ifq);
956 entries = min_t(unsigned, entries, PP_ALLOC_CACHE_REFILL);
957 if (unlikely(!entries))
958 return;
959
960 do {
961 struct io_uring_zcrx_rqe *rqe = io_zcrx_get_rqe(ifq, mask);
962 struct net_iov *niov;
963 netmem_ref netmem;
964
965 if (!io_parse_rqe(rqe, ifq, &niov))
966 continue;
967 if (!io_zcrx_put_niov_uref(niov))
968 continue;
969
970 netmem = net_iov_to_netmem(niov);
971 if (!page_pool_unref_and_test(netmem))
972 continue;
973
974 if (unlikely(niov->desc.pp != pp)) {
975 io_zcrx_return_niov(niov);
976 continue;
977 }
978
979 io_zcrx_sync_for_device(pp, niov);
980 net_mp_netmem_place_in_cache(pp, netmem);
981 } while (--entries);
982
983 smp_store_release(&ifq->rq_ring->head, ifq->cached_rq_head);
984 }
985
io_zcrx_refill_slow(struct page_pool * pp,struct io_zcrx_ifq * ifq)986 static void io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq)
987 {
988 struct io_zcrx_area *area = ifq->area;
989
990 spin_lock_bh(&area->freelist_lock);
991 while (area->free_count && pp->alloc.count < PP_ALLOC_CACHE_REFILL) {
992 struct net_iov *niov = __io_zcrx_get_free_niov(area);
993 netmem_ref netmem = net_iov_to_netmem(niov);
994
995 net_mp_niov_set_page_pool(pp, niov);
996 io_zcrx_sync_for_device(pp, niov);
997 net_mp_netmem_place_in_cache(pp, netmem);
998 }
999 spin_unlock_bh(&area->freelist_lock);
1000 }
1001
io_pp_zc_alloc_netmems(struct page_pool * pp,gfp_t gfp)1002 static netmem_ref io_pp_zc_alloc_netmems(struct page_pool *pp, gfp_t gfp)
1003 {
1004 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp);
1005
1006 /* pp should already be ensuring that */
1007 if (unlikely(pp->alloc.count))
1008 goto out_return;
1009
1010 io_zcrx_ring_refill(pp, ifq);
1011 if (likely(pp->alloc.count))
1012 goto out_return;
1013
1014 io_zcrx_refill_slow(pp, ifq);
1015 if (!pp->alloc.count)
1016 return 0;
1017 out_return:
1018 return pp->alloc.cache[--pp->alloc.count];
1019 }
1020
io_pp_zc_release_netmem(struct page_pool * pp,netmem_ref netmem)1021 static bool io_pp_zc_release_netmem(struct page_pool *pp, netmem_ref netmem)
1022 {
1023 struct net_iov *niov;
1024
1025 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
1026 return false;
1027
1028 niov = netmem_to_net_iov(netmem);
1029 net_mp_niov_clear_page_pool(niov);
1030 io_zcrx_return_niov_freelist(niov);
1031 return false;
1032 }
1033
io_pp_zc_init(struct page_pool * pp)1034 static int io_pp_zc_init(struct page_pool *pp)
1035 {
1036 struct io_zcrx_ifq *ifq = io_pp_to_ifq(pp);
1037 int ret;
1038
1039 if (WARN_ON_ONCE(!ifq))
1040 return -EINVAL;
1041 if (WARN_ON_ONCE(ifq->dev != pp->p.dev))
1042 return -EINVAL;
1043 if (WARN_ON_ONCE(!pp->dma_map))
1044 return -EOPNOTSUPP;
1045 if (pp->p.order + PAGE_SHIFT != ifq->niov_shift)
1046 return -EINVAL;
1047 if (pp->p.dma_dir != DMA_FROM_DEVICE)
1048 return -EOPNOTSUPP;
1049
1050 ret = io_zcrx_map_area(ifq, ifq->area);
1051 if (ret)
1052 return ret;
1053
1054 refcount_inc(&ifq->refs);
1055 return 0;
1056 }
1057
io_pp_zc_destroy(struct page_pool * pp)1058 static void io_pp_zc_destroy(struct page_pool *pp)
1059 {
1060 io_put_zcrx_ifq(io_pp_to_ifq(pp));
1061 }
1062
io_pp_nl_fill(void * mp_priv,struct sk_buff * rsp,struct netdev_rx_queue * rxq)1063 static int io_pp_nl_fill(void *mp_priv, struct sk_buff *rsp,
1064 struct netdev_rx_queue *rxq)
1065 {
1066 struct nlattr *nest;
1067 int type;
1068
1069 type = rxq ? NETDEV_A_QUEUE_IO_URING : NETDEV_A_PAGE_POOL_IO_URING;
1070 nest = nla_nest_start(rsp, type);
1071 if (!nest)
1072 return -EMSGSIZE;
1073 nla_nest_end(rsp, nest);
1074
1075 return 0;
1076 }
1077
io_pp_uninstall(void * mp_priv,struct netdev_rx_queue * rxq)1078 static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq)
1079 {
1080 struct pp_memory_provider_params *p = &rxq->mp_params;
1081 struct io_zcrx_ifq *ifq = mp_priv;
1082
1083 io_zcrx_drop_netdev(ifq);
1084 if (ifq->area)
1085 io_zcrx_unmap_area(ifq, ifq->area);
1086
1087 p->mp_ops = NULL;
1088 p->mp_priv = NULL;
1089 }
1090
1091 static const struct memory_provider_ops io_uring_pp_zc_ops = {
1092 .alloc_netmems = io_pp_zc_alloc_netmems,
1093 .release_netmem = io_pp_zc_release_netmem,
1094 .init = io_pp_zc_init,
1095 .destroy = io_pp_zc_destroy,
1096 .nl_fill = io_pp_nl_fill,
1097 .uninstall = io_pp_uninstall,
1098 };
1099
zcrx_parse_rq(netmem_ref * netmem_array,unsigned nr,struct io_zcrx_ifq * zcrx)1100 static unsigned zcrx_parse_rq(netmem_ref *netmem_array, unsigned nr,
1101 struct io_zcrx_ifq *zcrx)
1102 {
1103 unsigned int mask = zcrx->rq_entries - 1;
1104 unsigned int i;
1105
1106 nr = min(nr, io_zcrx_rqring_entries(zcrx));
1107 for (i = 0; i < nr; i++) {
1108 struct io_uring_zcrx_rqe *rqe = io_zcrx_get_rqe(zcrx, mask);
1109 struct net_iov *niov;
1110
1111 if (!io_parse_rqe(rqe, zcrx, &niov))
1112 break;
1113 netmem_array[i] = net_iov_to_netmem(niov);
1114 }
1115
1116 smp_store_release(&zcrx->rq_ring->head, zcrx->cached_rq_head);
1117 return i;
1118 }
1119
1120 #define ZCRX_FLUSH_BATCH 32
1121
zcrx_return_buffers(netmem_ref * netmems,unsigned nr)1122 static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr)
1123 {
1124 unsigned i;
1125
1126 for (i = 0; i < nr; i++) {
1127 netmem_ref netmem = netmems[i];
1128 struct net_iov *niov = netmem_to_net_iov(netmem);
1129
1130 if (!io_zcrx_put_niov_uref(niov))
1131 continue;
1132 if (!page_pool_unref_and_test(netmem))
1133 continue;
1134 io_zcrx_return_niov(niov);
1135 }
1136 }
1137
zcrx_flush_rq(struct io_ring_ctx * ctx,struct io_zcrx_ifq * zcrx,struct zcrx_ctrl * ctrl)1138 static int zcrx_flush_rq(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx,
1139 struct zcrx_ctrl *ctrl)
1140 {
1141 struct zcrx_ctrl_flush_rq *frq = &ctrl->zc_flush;
1142 netmem_ref netmems[ZCRX_FLUSH_BATCH];
1143 unsigned total = 0;
1144 unsigned nr;
1145
1146 if (!mem_is_zero(&frq->__resv, sizeof(frq->__resv)))
1147 return -EINVAL;
1148
1149 do {
1150 scoped_guard(spinlock_bh, &zcrx->rq_lock) {
1151 nr = zcrx_parse_rq(netmems, ZCRX_FLUSH_BATCH, zcrx);
1152 zcrx_return_buffers(netmems, nr);
1153 }
1154
1155 total += nr;
1156
1157 if (fatal_signal_pending(current))
1158 break;
1159 cond_resched();
1160 } while (nr == ZCRX_FLUSH_BATCH && total < zcrx->rq_entries);
1161
1162 return 0;
1163 }
1164
io_zcrx_ctrl(struct io_ring_ctx * ctx,void __user * arg,unsigned nr_args)1165 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
1166 {
1167 struct zcrx_ctrl ctrl;
1168 struct io_zcrx_ifq *zcrx;
1169
1170 if (nr_args)
1171 return -EINVAL;
1172 if (copy_from_user(&ctrl, arg, sizeof(ctrl)))
1173 return -EFAULT;
1174 if (!mem_is_zero(&ctrl.__resv, sizeof(ctrl.__resv)))
1175 return -EFAULT;
1176
1177 zcrx = xa_load(&ctx->zcrx_ctxs, ctrl.zcrx_id);
1178 if (!zcrx)
1179 return -ENXIO;
1180
1181 switch (ctrl.op) {
1182 case ZCRX_CTRL_FLUSH_RQ:
1183 return zcrx_flush_rq(ctx, zcrx, &ctrl);
1184 case ZCRX_CTRL_EXPORT:
1185 return zcrx_export(ctx, zcrx, &ctrl, arg);
1186 }
1187
1188 return -EOPNOTSUPP;
1189 }
1190
io_zcrx_queue_cqe(struct io_kiocb * req,struct net_iov * niov,struct io_zcrx_ifq * ifq,int off,int len)1191 static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov,
1192 struct io_zcrx_ifq *ifq, int off, int len)
1193 {
1194 struct io_ring_ctx *ctx = req->ctx;
1195 struct io_uring_zcrx_cqe *rcqe;
1196 struct io_zcrx_area *area;
1197 struct io_uring_cqe *cqe;
1198 u64 offset;
1199
1200 if (!io_defer_get_uncommited_cqe(ctx, &cqe))
1201 return false;
1202
1203 cqe->user_data = req->cqe.user_data;
1204 cqe->res = len;
1205 cqe->flags = IORING_CQE_F_MORE;
1206 if (ctx->flags & IORING_SETUP_CQE_MIXED)
1207 cqe->flags |= IORING_CQE_F_32;
1208
1209 area = io_zcrx_iov_to_area(niov);
1210 offset = off + (net_iov_idx(niov) << ifq->niov_shift);
1211 rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1);
1212 rcqe->off = offset + ((u64)area->area_id << IORING_ZCRX_AREA_SHIFT);
1213 rcqe->__pad = 0;
1214 return true;
1215 }
1216
io_alloc_fallback_niov(struct io_zcrx_ifq * ifq)1217 static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq)
1218 {
1219 struct io_zcrx_area *area = ifq->area;
1220 struct net_iov *niov = NULL;
1221
1222 if (area->mem.is_dmabuf)
1223 return NULL;
1224
1225 spin_lock_bh(&area->freelist_lock);
1226 if (area->free_count)
1227 niov = __io_zcrx_get_free_niov(area);
1228 spin_unlock_bh(&area->freelist_lock);
1229
1230 if (niov)
1231 page_pool_fragment_netmem(net_iov_to_netmem(niov), 1);
1232 return niov;
1233 }
1234
1235 struct io_copy_cache {
1236 struct page *page;
1237 unsigned long offset;
1238 size_t size;
1239 };
1240
io_copy_page(struct io_copy_cache * cc,struct page * src_page,unsigned int src_offset,size_t len)1241 static ssize_t io_copy_page(struct io_copy_cache *cc, struct page *src_page,
1242 unsigned int src_offset, size_t len)
1243 {
1244 size_t copied = 0;
1245
1246 len = min(len, cc->size);
1247
1248 while (len) {
1249 void *src_addr, *dst_addr;
1250 struct page *dst_page = cc->page;
1251 unsigned dst_offset = cc->offset;
1252 size_t n = len;
1253
1254 if (folio_test_partial_kmap(page_folio(dst_page)) ||
1255 folio_test_partial_kmap(page_folio(src_page))) {
1256 dst_page += dst_offset / PAGE_SIZE;
1257 dst_offset = offset_in_page(dst_offset);
1258 src_page += src_offset / PAGE_SIZE;
1259 src_offset = offset_in_page(src_offset);
1260 n = min(PAGE_SIZE - src_offset, PAGE_SIZE - dst_offset);
1261 n = min(n, len);
1262 }
1263
1264 dst_addr = kmap_local_page(dst_page) + dst_offset;
1265 src_addr = kmap_local_page(src_page) + src_offset;
1266
1267 memcpy(dst_addr, src_addr, n);
1268
1269 kunmap_local(src_addr);
1270 kunmap_local(dst_addr);
1271
1272 cc->size -= n;
1273 cc->offset += n;
1274 src_offset += n;
1275 len -= n;
1276 copied += n;
1277 }
1278 return copied;
1279 }
1280
io_zcrx_copy_chunk(struct io_kiocb * req,struct io_zcrx_ifq * ifq,struct page * src_page,unsigned int src_offset,size_t len)1281 static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1282 struct page *src_page, unsigned int src_offset,
1283 size_t len)
1284 {
1285 size_t copied = 0;
1286 int ret = 0;
1287
1288 while (len) {
1289 struct io_copy_cache cc;
1290 struct net_iov *niov;
1291 size_t n;
1292
1293 niov = io_alloc_fallback_niov(ifq);
1294 if (!niov) {
1295 ret = -ENOMEM;
1296 break;
1297 }
1298
1299 cc.page = io_zcrx_iov_page(niov);
1300 cc.offset = 0;
1301 cc.size = PAGE_SIZE;
1302
1303 n = io_copy_page(&cc, src_page, src_offset, len);
1304
1305 if (!io_zcrx_queue_cqe(req, niov, ifq, 0, n)) {
1306 io_zcrx_return_niov(niov);
1307 ret = -ENOSPC;
1308 break;
1309 }
1310
1311 io_zcrx_get_niov_uref(niov);
1312 src_offset += n;
1313 len -= n;
1314 copied += n;
1315 }
1316
1317 return copied ? copied : ret;
1318 }
1319
io_zcrx_copy_frag(struct io_kiocb * req,struct io_zcrx_ifq * ifq,const skb_frag_t * frag,int off,int len)1320 static int io_zcrx_copy_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1321 const skb_frag_t *frag, int off, int len)
1322 {
1323 struct page *page = skb_frag_page(frag);
1324
1325 return io_zcrx_copy_chunk(req, ifq, page, off + skb_frag_off(frag), len);
1326 }
1327
io_zcrx_recv_frag(struct io_kiocb * req,struct io_zcrx_ifq * ifq,const skb_frag_t * frag,int off,int len)1328 static int io_zcrx_recv_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1329 const skb_frag_t *frag, int off, int len)
1330 {
1331 struct net_iov *niov;
1332 struct page_pool *pp;
1333
1334 if (unlikely(!skb_frag_is_net_iov(frag)))
1335 return io_zcrx_copy_frag(req, ifq, frag, off, len);
1336
1337 niov = netmem_to_net_iov(frag->netmem);
1338 pp = niov->desc.pp;
1339
1340 if (!pp || pp->mp_ops != &io_uring_pp_zc_ops || io_pp_to_ifq(pp) != ifq)
1341 return -EFAULT;
1342
1343 if (!io_zcrx_queue_cqe(req, niov, ifq, off + skb_frag_off(frag), len))
1344 return -ENOSPC;
1345
1346 /*
1347 * Prevent it from being recycled while user is accessing it.
1348 * It has to be done before grabbing a user reference.
1349 */
1350 page_pool_ref_netmem(net_iov_to_netmem(niov));
1351 io_zcrx_get_niov_uref(niov);
1352 return len;
1353 }
1354
1355 static int
io_zcrx_recv_skb(read_descriptor_t * desc,struct sk_buff * skb,unsigned int offset,size_t len)1356 io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
1357 unsigned int offset, size_t len)
1358 {
1359 struct io_zcrx_args *args = desc->arg.data;
1360 struct io_zcrx_ifq *ifq = args->ifq;
1361 struct io_kiocb *req = args->req;
1362 struct sk_buff *frag_iter;
1363 unsigned start, start_off = offset;
1364 int i, copy, end, off;
1365 int ret = 0;
1366
1367 len = min_t(size_t, len, desc->count);
1368 /*
1369 * __tcp_read_sock() always calls io_zcrx_recv_skb one last time, even
1370 * if desc->count is already 0. This is caused by the if (offset + 1 !=
1371 * skb->len) check. Return early in this case to break out of
1372 * __tcp_read_sock().
1373 */
1374 if (!len)
1375 return 0;
1376 if (unlikely(args->nr_skbs++ > IO_SKBS_PER_CALL_LIMIT))
1377 return -EAGAIN;
1378
1379 if (unlikely(offset < skb_headlen(skb))) {
1380 ssize_t copied;
1381 size_t to_copy;
1382
1383 to_copy = min_t(size_t, skb_headlen(skb) - offset, len);
1384 copied = io_zcrx_copy_chunk(req, ifq, virt_to_page(skb->data),
1385 offset_in_page(skb->data) + offset,
1386 to_copy);
1387 if (copied < 0) {
1388 ret = copied;
1389 goto out;
1390 }
1391 offset += copied;
1392 len -= copied;
1393 if (!len)
1394 goto out;
1395 if (offset != skb_headlen(skb))
1396 goto out;
1397 }
1398
1399 start = skb_headlen(skb);
1400
1401 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1402 const skb_frag_t *frag;
1403
1404 if (WARN_ON(start > offset + len))
1405 return -EFAULT;
1406
1407 frag = &skb_shinfo(skb)->frags[i];
1408 end = start + skb_frag_size(frag);
1409
1410 if (offset < end) {
1411 copy = end - offset;
1412 if (copy > len)
1413 copy = len;
1414
1415 off = offset - start;
1416 ret = io_zcrx_recv_frag(req, ifq, frag, off, copy);
1417 if (ret < 0)
1418 goto out;
1419
1420 offset += ret;
1421 len -= ret;
1422 if (len == 0 || ret != copy)
1423 goto out;
1424 }
1425 start = end;
1426 }
1427
1428 skb_walk_frags(skb, frag_iter) {
1429 if (WARN_ON(start > offset + len))
1430 return -EFAULT;
1431
1432 end = start + frag_iter->len;
1433 if (offset < end) {
1434 size_t count;
1435
1436 copy = end - offset;
1437 if (copy > len)
1438 copy = len;
1439
1440 off = offset - start;
1441 count = desc->count;
1442 ret = io_zcrx_recv_skb(desc, frag_iter, off, copy);
1443 desc->count = count;
1444 if (ret < 0)
1445 goto out;
1446
1447 offset += ret;
1448 len -= ret;
1449 if (len == 0 || ret != copy)
1450 goto out;
1451 }
1452 start = end;
1453 }
1454
1455 out:
1456 if (offset == start_off)
1457 return ret;
1458 desc->count -= (offset - start_off);
1459 return offset - start_off;
1460 }
1461
io_zcrx_tcp_recvmsg(struct io_kiocb * req,struct io_zcrx_ifq * ifq,struct sock * sk,int flags,unsigned issue_flags,unsigned int * outlen)1462 static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1463 struct sock *sk, int flags,
1464 unsigned issue_flags, unsigned int *outlen)
1465 {
1466 unsigned int len = *outlen;
1467 struct io_zcrx_args args = {
1468 .req = req,
1469 .ifq = ifq,
1470 .sock = sk->sk_socket,
1471 };
1472 read_descriptor_t rd_desc = {
1473 .count = len ? len : UINT_MAX,
1474 .arg.data = &args,
1475 };
1476 int ret;
1477
1478 lock_sock(sk);
1479 ret = tcp_read_sock(sk, &rd_desc, io_zcrx_recv_skb);
1480 if (len && ret > 0)
1481 *outlen = len - ret;
1482 if (ret <= 0) {
1483 if (ret < 0 || sock_flag(sk, SOCK_DONE))
1484 goto out;
1485 if (sk->sk_err)
1486 ret = sock_error(sk);
1487 else if (sk->sk_shutdown & RCV_SHUTDOWN)
1488 goto out;
1489 else if (sk->sk_state == TCP_CLOSE)
1490 ret = -ENOTCONN;
1491 else
1492 ret = -EAGAIN;
1493 } else if (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) &&
1494 (issue_flags & IO_URING_F_MULTISHOT)) {
1495 ret = IOU_REQUEUE;
1496 } else if (sock_flag(sk, SOCK_DONE)) {
1497 /* Make it to retry until it finally gets 0. */
1498 if (issue_flags & IO_URING_F_MULTISHOT)
1499 ret = IOU_REQUEUE;
1500 else
1501 ret = -EAGAIN;
1502 }
1503 out:
1504 release_sock(sk);
1505 return ret;
1506 }
1507
io_zcrx_recv(struct io_kiocb * req,struct io_zcrx_ifq * ifq,struct socket * sock,unsigned int flags,unsigned issue_flags,unsigned int * len)1508 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
1509 struct socket *sock, unsigned int flags,
1510 unsigned issue_flags, unsigned int *len)
1511 {
1512 struct sock *sk = sock->sk;
1513 const struct proto *prot = READ_ONCE(sk->sk_prot);
1514
1515 if (prot->recvmsg != tcp_recvmsg)
1516 return -EPROTONOSUPPORT;
1517
1518 sock_rps_record_flow(sk);
1519 return io_zcrx_tcp_recvmsg(req, ifq, sk, flags, issue_flags, len);
1520 }
1521