1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/netdevice.h> 4 #include <net/netdev_lock.h> 5 #include <net/xsk_buff_pool.h> 6 #include <net/xdp_sock.h> 7 #include <net/xdp_sock_drv.h> 8 9 #include "xsk_queue.h" 10 #include "xdp_umem.h" 11 #include "xsk.h" 12 13 #define ETH_PAD_LEN (ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN) 14 15 void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs) 16 { 17 if (!xs->tx) 18 return; 19 20 spin_lock(&pool->xsk_tx_list_lock); 21 list_add_rcu(&xs->tx_list, &pool->xsk_tx_list); 22 spin_unlock(&pool->xsk_tx_list_lock); 23 } 24 25 void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs) 26 { 27 if (!xs->tx) 28 return; 29 30 spin_lock(&pool->xsk_tx_list_lock); 31 list_del_rcu(&xs->tx_list); 32 spin_unlock(&pool->xsk_tx_list_lock); 33 } 34 35 void xp_destroy(struct xsk_buff_pool *pool) 36 { 37 if (!pool) 38 return; 39 40 kvfree(pool->tx_descs); 41 kvfree(pool->heads); 42 kvfree(pool); 43 } 44 45 int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs, 46 u32 max_segs) 47 { 48 u32 nentries = max(xs->tx->nentries, max_segs); 49 50 pool->tx_descs = kvzalloc_objs(*pool->tx_descs, nentries); 51 if (!pool->tx_descs) 52 return -ENOMEM; 53 54 pool->tx_descs_nentries = nentries; 55 return 0; 56 } 57 58 struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, 59 struct xdp_umem *umem, 60 u32 max_segs) 61 { 62 bool unaligned = umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG; 63 struct xsk_buff_pool *pool; 64 struct xdp_buff_xsk *xskb; 65 u32 i, entries; 66 67 entries = unaligned ? umem->chunks : 0; 68 pool = kvzalloc_flex(*pool, free_heads, entries); 69 if (!pool) 70 goto out; 71 72 pool->heads = kvzalloc_objs(*pool->heads, umem->chunks); 73 if (!pool->heads) 74 goto out; 75 76 if (xs->tx) 77 if (xp_alloc_tx_descs(pool, xs, max_segs)) 78 goto out; 79 80 pool->chunk_mask = ~((u64)umem->chunk_size - 1); 81 pool->addrs_cnt = umem->size; 82 pool->heads_cnt = umem->chunks; 83 pool->free_heads_cnt = umem->chunks; 84 pool->headroom = umem->headroom; 85 pool->chunk_size = umem->chunk_size; 86 pool->chunk_shift = ffs(umem->chunk_size) - 1; 87 pool->unaligned = unaligned; 88 pool->frame_len = umem->chunk_size - umem->headroom - 89 XDP_PACKET_HEADROOM; 90 pool->umem = umem; 91 pool->addrs = umem->addrs; 92 pool->tx_metadata_len = umem->tx_metadata_len; 93 pool->tx_sw_csum = umem->flags & XDP_UMEM_TX_SW_CSUM; 94 spin_lock_init(&pool->rx_lock); 95 INIT_LIST_HEAD(&pool->free_list); 96 INIT_LIST_HEAD(&pool->xskb_list); 97 INIT_LIST_HEAD(&pool->xsk_tx_list); 98 spin_lock_init(&pool->xsk_tx_list_lock); 99 spin_lock_init(&pool->cq_prod_lock); 100 spin_lock_init(&xs->cq_tmp->cq_cached_prod_lock); 101 refcount_set(&pool->users, 1); 102 103 pool->fq = xs->fq_tmp; 104 pool->cq = xs->cq_tmp; 105 106 for (i = 0; i < pool->free_heads_cnt; i++) { 107 xskb = &pool->heads[i]; 108 xskb->pool = pool; 109 xskb->xdp.frame_sz = umem->chunk_size - umem->headroom; 110 INIT_LIST_HEAD(&xskb->list_node); 111 if (pool->unaligned) 112 pool->free_heads[i] = xskb; 113 else 114 xp_init_xskb_addr(xskb, pool, (u64)i * pool->chunk_size); 115 } 116 117 return pool; 118 119 out: 120 xp_destroy(pool); 121 return NULL; 122 } 123 124 void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) 125 { 126 u32 i; 127 128 for (i = 0; i < pool->heads_cnt; i++) 129 pool->heads[i].xdp.rxq = rxq; 130 } 131 EXPORT_SYMBOL(xp_set_rxq_info); 132 133 void xp_fill_cb(struct xsk_buff_pool *pool, struct xsk_cb_desc *desc) 134 { 135 u32 i; 136 137 for (i = 0; i < pool->heads_cnt; i++) { 138 struct xdp_buff_xsk *xskb = &pool->heads[i]; 139 140 memcpy(xskb->cb + desc->off, desc->src, desc->bytes); 141 } 142 } 143 EXPORT_SYMBOL(xp_fill_cb); 144 145 static void xp_disable_drv_zc(struct xsk_buff_pool *pool) 146 { 147 struct netdev_bpf bpf; 148 int err; 149 150 ASSERT_RTNL(); 151 152 if (pool->umem->zc) { 153 bpf.command = XDP_SETUP_XSK_POOL; 154 bpf.xsk.pool = NULL; 155 bpf.xsk.queue_id = pool->queue_id; 156 157 err = pool->netdev->netdev_ops->ndo_bpf(pool->netdev, &bpf); 158 159 if (err) 160 WARN(1, "Failed to disable zero-copy!\n"); 161 } 162 } 163 164 int xp_assign_dev(struct xsk_buff_pool *pool, 165 struct net_device *netdev, u16 queue_id, u16 flags) 166 { 167 u32 needed = netdev->mtu + ETH_PAD_LEN; 168 u32 segs = netdev->xdp_zc_max_segs; 169 u8 umem_flags = pool->umem->flags; 170 bool mbuf = flags & XDP_USE_SG; 171 bool force_zc, force_copy; 172 struct netdev_bpf bpf; 173 u32 frame_size; 174 int err = 0; 175 176 netdev_assert_locked_ops_compat(netdev); 177 178 force_zc = flags & XDP_ZEROCOPY; 179 force_copy = flags & XDP_COPY; 180 181 if (force_zc && force_copy) 182 return -EINVAL; 183 184 if (pool->tx_sw_csum && (netdev->priv_flags & IFF_TX_SKB_NO_LINEAR)) 185 return -EOPNOTSUPP; 186 187 if (xsk_get_pool_from_qid(netdev, queue_id)) 188 return -EBUSY; 189 190 pool->netdev = netdev; 191 pool->queue_id = queue_id; 192 err = xsk_reg_pool_at_qid(netdev, pool, queue_id); 193 if (err) 194 return err; 195 196 if (mbuf) 197 pool->umem->flags |= XDP_UMEM_SG_FLAG; 198 199 if (flags & XDP_USE_NEED_WAKEUP) 200 pool->uses_need_wakeup = true; 201 /* Tx needs to be explicitly woken up the first time. Also 202 * for supporting drivers that do not implement this 203 * feature. They will always have to call sendto() or poll(). 204 */ 205 pool->cached_need_wakeup = XDP_WAKEUP_TX; 206 207 dev_hold(netdev); 208 /* 209 * Socket removal precedes the final pool put. Keep the pool visible to 210 * NETDEV_UNREGISTER even while release work is waiting for process thaw. 211 */ 212 hlist_add_head(&pool->dev_node, &netdev->xsk_pools); 213 214 if (force_copy) 215 /* For copy-mode, we are done. */ 216 return 0; 217 218 if ((netdev->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) { 219 err = -EOPNOTSUPP; 220 goto err_unreg_pool; 221 } 222 223 if (mbuf) { 224 if (segs == 1) { 225 err = -EOPNOTSUPP; 226 goto err_unreg_pool; 227 } 228 } else { 229 segs = 1; 230 } 231 232 /* open-code xsk_pool_get_rx_frame_size() as pool->dev is not 233 * set yet at this point; we are before getting down to driver 234 */ 235 frame_size = __xsk_pool_get_rx_frame_size(pool) - 236 xsk_pool_get_tailroom(mbuf); 237 frame_size = ALIGN_DOWN(frame_size, 128); 238 239 if (needed > frame_size * segs) { 240 err = -EINVAL; 241 goto err_unreg_pool; 242 } 243 244 if (dev_get_min_mp_channel_count(netdev)) { 245 err = -EBUSY; 246 goto err_unreg_pool; 247 } 248 249 bpf.command = XDP_SETUP_XSK_POOL; 250 bpf.xsk.pool = pool; 251 bpf.xsk.queue_id = queue_id; 252 253 err = netdev->netdev_ops->ndo_bpf(netdev, &bpf); 254 if (err) 255 goto err_unreg_pool; 256 257 if (!pool->dma_pages) { 258 WARN(1, "Driver did not DMA map zero-copy buffers"); 259 err = -EINVAL; 260 goto err_unreg_xsk; 261 } 262 pool->umem->zc = true; 263 pool->xdp_zc_max_segs = netdev->xdp_zc_max_segs; 264 return 0; 265 266 err_unreg_xsk: 267 xp_disable_drv_zc(pool); 268 err_unreg_pool: 269 if (!force_zc) 270 err = 0; /* fallback to copy mode */ 271 if (err) { 272 pool->umem->flags = umem_flags; 273 xsk_clear_pool_at_qid(netdev, queue_id); 274 hlist_del(&pool->dev_node); 275 pool->netdev = NULL; 276 dev_put(netdev); 277 } 278 return err; 279 } 280 281 int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs, 282 struct net_device *dev, u16 queue_id) 283 { 284 u16 flags; 285 struct xdp_umem *umem = umem_xs->umem; 286 287 flags = umem->zc ? XDP_ZEROCOPY : XDP_COPY; 288 289 if (umem->flags & XDP_UMEM_SG_FLAG) 290 flags |= XDP_USE_SG; 291 292 if (umem_xs->pool->uses_need_wakeup) 293 flags |= XDP_USE_NEED_WAKEUP; 294 295 return xp_assign_dev(pool, dev, queue_id, flags); 296 } 297 298 static void __xp_clear_dev(struct xsk_buff_pool *pool) 299 { 300 struct net_device *netdev = pool->netdev; 301 302 netdev_assert_locked_ops_compat(netdev); 303 xp_disable_drv_zc(pool); 304 xsk_clear_pool_at_qid(netdev, pool->queue_id); 305 hlist_del(&pool->dev_node); 306 pool->netdev = NULL; 307 dev_put(netdev); 308 } 309 310 void xp_clear_dev(struct xsk_buff_pool *pool) 311 { 312 struct net_device *netdev = pool->netdev; 313 314 if (!netdev) 315 return; 316 317 netdev_lock_ops(netdev); 318 __xp_clear_dev(pool); 319 netdev_unlock_ops(netdev); 320 } 321 322 void xp_clear_dev_all(struct net_device *dev) 323 { 324 struct xsk_buff_pool *pool; 325 struct hlist_node *next; 326 327 netdev_lock_ops(dev); 328 hlist_for_each_entry_safe(pool, next, &dev->xsk_pools, dev_node) 329 __xp_clear_dev(pool); 330 netdev_unlock_ops(dev); 331 } 332 333 static void xp_release_deferred(struct work_struct *work) 334 { 335 struct xsk_buff_pool *pool = container_of(work, struct xsk_buff_pool, 336 work); 337 338 rtnl_lock(); 339 xp_clear_dev(pool); 340 rtnl_unlock(); 341 342 if (pool->fq) { 343 xskq_destroy(pool->fq); 344 pool->fq = NULL; 345 } 346 347 if (pool->cq) { 348 xskq_destroy(pool->cq); 349 pool->cq = NULL; 350 } 351 352 xdp_put_umem(pool->umem, false); 353 xp_destroy(pool); 354 } 355 356 void xp_get_pool(struct xsk_buff_pool *pool) 357 { 358 refcount_inc(&pool->users); 359 } 360 361 bool xp_put_pool(struct xsk_buff_pool *pool) 362 { 363 if (!pool) 364 return false; 365 366 if (refcount_dec_and_test(&pool->users)) { 367 INIT_WORK(&pool->work, xp_release_deferred); 368 /* 369 * Teardown calls ndo_bpf(), which may need powered hardware. 370 * RTNL alone does not exclude the device's system PM callbacks. 371 */ 372 queue_work(system_freezable_wq, &pool->work); 373 return true; 374 } 375 376 return false; 377 } 378 379 static struct xsk_dma_map *xp_find_dma_map(struct xsk_buff_pool *pool) 380 { 381 struct xsk_dma_map *dma_map; 382 383 list_for_each_entry(dma_map, &pool->umem->xsk_dma_list, list) { 384 if (dma_map->netdev == pool->netdev) 385 return dma_map; 386 } 387 388 return NULL; 389 } 390 391 static struct xsk_dma_map *xp_create_dma_map(struct device *dev, struct net_device *netdev, 392 u32 nr_pages, struct xdp_umem *umem) 393 { 394 struct xsk_dma_map *dma_map; 395 396 dma_map = kzalloc_obj(*dma_map); 397 if (!dma_map) 398 return NULL; 399 400 dma_map->dma_pages = kvzalloc_objs(*dma_map->dma_pages, nr_pages); 401 if (!dma_map->dma_pages) { 402 kfree(dma_map); 403 return NULL; 404 } 405 406 dma_map->netdev = netdev; 407 dma_map->dev = dev; 408 dma_map->dma_pages_cnt = nr_pages; 409 refcount_set(&dma_map->users, 1); 410 list_add(&dma_map->list, &umem->xsk_dma_list); 411 return dma_map; 412 } 413 414 static void xp_destroy_dma_map(struct xsk_dma_map *dma_map) 415 { 416 list_del(&dma_map->list); 417 kvfree(dma_map->dma_pages); 418 kfree(dma_map); 419 } 420 421 static void __xp_dma_unmap(struct xsk_dma_map *dma_map, unsigned long attrs) 422 { 423 dma_addr_t *dma; 424 u32 i; 425 426 for (i = 0; i < dma_map->dma_pages_cnt; i++) { 427 dma = &dma_map->dma_pages[i]; 428 if (*dma) { 429 *dma &= ~XSK_NEXT_PG_CONTIG_MASK; 430 dma_unmap_page_attrs(dma_map->dev, *dma, PAGE_SIZE, 431 DMA_BIDIRECTIONAL, attrs); 432 *dma = 0; 433 } 434 } 435 436 xp_destroy_dma_map(dma_map); 437 } 438 439 void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs) 440 { 441 struct xsk_dma_map *dma_map; 442 443 if (!pool->dma_pages) 444 return; 445 446 dma_map = xp_find_dma_map(pool); 447 if (!dma_map) { 448 WARN(1, "Could not find dma_map for device"); 449 return; 450 } 451 452 if (refcount_dec_and_test(&dma_map->users)) 453 __xp_dma_unmap(dma_map, attrs); 454 455 kvfree(pool->dma_pages); 456 pool->dma_pages = NULL; 457 pool->dma_pages_cnt = 0; 458 pool->dev = NULL; 459 } 460 EXPORT_SYMBOL(xp_dma_unmap); 461 462 static void xp_check_dma_contiguity(struct xsk_dma_map *dma_map) 463 { 464 u32 i; 465 466 for (i = 0; i < dma_map->dma_pages_cnt - 1; i++) { 467 if (dma_map->dma_pages[i] + PAGE_SIZE == dma_map->dma_pages[i + 1]) 468 dma_map->dma_pages[i] |= XSK_NEXT_PG_CONTIG_MASK; 469 else 470 dma_map->dma_pages[i] &= ~XSK_NEXT_PG_CONTIG_MASK; 471 } 472 } 473 474 static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_map) 475 { 476 if (!pool->unaligned) { 477 u32 i; 478 479 for (i = 0; i < pool->heads_cnt; i++) { 480 struct xdp_buff_xsk *xskb = &pool->heads[i]; 481 u64 orig_addr; 482 483 orig_addr = xskb->xdp.data_hard_start - pool->addrs - pool->headroom; 484 xp_init_xskb_dma(xskb, pool, dma_map->dma_pages, orig_addr); 485 } 486 } 487 488 pool->dma_pages = kvzalloc_objs(*pool->dma_pages, 489 dma_map->dma_pages_cnt); 490 if (!pool->dma_pages) 491 return -ENOMEM; 492 493 pool->dev = dma_map->dev; 494 pool->dma_pages_cnt = dma_map->dma_pages_cnt; 495 memcpy(pool->dma_pages, dma_map->dma_pages, 496 pool->dma_pages_cnt * sizeof(*pool->dma_pages)); 497 498 return 0; 499 } 500 501 int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev, 502 unsigned long attrs, struct page **pages, u32 nr_pages) 503 { 504 struct xsk_dma_map *dma_map; 505 dma_addr_t dma; 506 int err; 507 u32 i; 508 509 dma_map = xp_find_dma_map(pool); 510 if (dma_map) { 511 err = xp_init_dma_info(pool, dma_map); 512 if (err) 513 return err; 514 515 refcount_inc(&dma_map->users); 516 return 0; 517 } 518 519 dma_map = xp_create_dma_map(dev, pool->netdev, nr_pages, pool->umem); 520 if (!dma_map) 521 return -ENOMEM; 522 523 for (i = 0; i < dma_map->dma_pages_cnt; i++) { 524 dma = dma_map_page_attrs(dev, pages[i], 0, PAGE_SIZE, 525 DMA_BIDIRECTIONAL, attrs); 526 if (dma_mapping_error(dev, dma)) { 527 __xp_dma_unmap(dma_map, attrs); 528 return -ENOMEM; 529 } 530 dma_map->dma_pages[i] = dma; 531 } 532 533 if (pool->unaligned) 534 xp_check_dma_contiguity(dma_map); 535 536 err = xp_init_dma_info(pool, dma_map); 537 if (err) { 538 __xp_dma_unmap(dma_map, attrs); 539 return err; 540 } 541 542 return 0; 543 } 544 EXPORT_SYMBOL(xp_dma_map); 545 546 static bool xp_addr_crosses_non_contig_pg(struct xsk_buff_pool *pool, 547 u64 addr) 548 { 549 return xp_desc_crosses_non_contig_pg(pool, addr, pool->chunk_size); 550 } 551 552 static bool xp_check_unaligned(struct xsk_buff_pool *pool, u64 *addr) 553 { 554 *addr = xp_unaligned_extract_addr(*addr); 555 if (*addr >= pool->addrs_cnt || 556 *addr + pool->chunk_size > pool->addrs_cnt || 557 xp_addr_crosses_non_contig_pg(pool, *addr)) 558 return false; 559 return true; 560 } 561 562 static bool xp_check_aligned(struct xsk_buff_pool *pool, u64 *addr) 563 { 564 *addr = xp_aligned_extract_addr(pool, *addr); 565 return *addr < pool->addrs_cnt; 566 } 567 568 static struct xdp_buff_xsk *xp_get_xskb(struct xsk_buff_pool *pool, u64 addr) 569 { 570 struct xdp_buff_xsk *xskb; 571 572 if (pool->unaligned) { 573 xskb = pool->free_heads[--pool->free_heads_cnt]; 574 xp_init_xskb_addr(xskb, pool, addr); 575 if (pool->dma_pages) 576 xp_init_xskb_dma(xskb, pool, pool->dma_pages, addr); 577 } else { 578 xskb = &pool->heads[xp_aligned_extract_idx(pool, addr)]; 579 } 580 581 return xskb; 582 } 583 584 static struct xdp_buff_xsk *__xp_alloc(struct xsk_buff_pool *pool) 585 { 586 struct xdp_buff_xsk *xskb; 587 u64 addr; 588 bool ok; 589 590 if (pool->free_heads_cnt == 0) 591 return NULL; 592 593 for (;;) { 594 if (!xskq_cons_peek_addr_unchecked(pool->fq, &addr)) { 595 pool->fq->queue_empty_descs++; 596 return NULL; 597 } 598 599 ok = pool->unaligned ? xp_check_unaligned(pool, &addr) : 600 xp_check_aligned(pool, &addr); 601 if (!ok) { 602 pool->fq->invalid_descs++; 603 xskq_cons_release(pool->fq); 604 continue; 605 } 606 break; 607 } 608 609 xskb = xp_get_xskb(pool, addr); 610 611 xskq_cons_release(pool->fq); 612 return xskb; 613 } 614 615 struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool) 616 { 617 struct xdp_buff_xsk *xskb; 618 619 if (!pool->free_list_cnt) { 620 xskb = __xp_alloc(pool); 621 if (!xskb) 622 return NULL; 623 } else { 624 pool->free_list_cnt--; 625 xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk, 626 list_node); 627 list_del_init(&xskb->list_node); 628 } 629 630 xskb->xdp.data = xskb->xdp.data_hard_start + XDP_PACKET_HEADROOM; 631 xskb->xdp.data_meta = xskb->xdp.data; 632 xskb->xdp.flags = 0; 633 634 if (pool->dev) 635 xp_dma_sync_for_device(pool, xskb->dma, pool->frame_len); 636 637 return &xskb->xdp; 638 } 639 EXPORT_SYMBOL(xp_alloc); 640 641 static u32 xp_alloc_new_from_fq(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) 642 { 643 u32 i, cached_cons, nb_entries; 644 645 if (max > pool->free_heads_cnt) 646 max = pool->free_heads_cnt; 647 max = xskq_cons_nb_entries(pool->fq, max); 648 649 cached_cons = pool->fq->cached_cons; 650 nb_entries = max; 651 i = max; 652 while (i--) { 653 struct xdp_buff_xsk *xskb; 654 u64 addr; 655 bool ok; 656 657 __xskq_cons_read_addr_unchecked(pool->fq, cached_cons++, &addr); 658 659 ok = pool->unaligned ? xp_check_unaligned(pool, &addr) : 660 xp_check_aligned(pool, &addr); 661 if (unlikely(!ok)) { 662 pool->fq->invalid_descs++; 663 nb_entries--; 664 continue; 665 } 666 667 xskb = xp_get_xskb(pool, addr); 668 669 *xdp = &xskb->xdp; 670 xdp++; 671 } 672 673 xskq_cons_release_n(pool->fq, max); 674 return nb_entries; 675 } 676 677 static u32 xp_alloc_reused(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 nb_entries) 678 { 679 struct xdp_buff_xsk *xskb; 680 u32 i; 681 682 nb_entries = min_t(u32, nb_entries, pool->free_list_cnt); 683 684 i = nb_entries; 685 while (i--) { 686 xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk, list_node); 687 list_del_init(&xskb->list_node); 688 689 *xdp = &xskb->xdp; 690 xdp++; 691 } 692 pool->free_list_cnt -= nb_entries; 693 694 return nb_entries; 695 } 696 697 static u32 xp_alloc_slow(struct xsk_buff_pool *pool, struct xdp_buff **xdp, 698 u32 max) 699 { 700 int i; 701 702 for (i = 0; i < max; i++) { 703 struct xdp_buff *buff; 704 705 buff = xp_alloc(pool); 706 if (unlikely(!buff)) 707 return i; 708 *xdp = buff; 709 xdp++; 710 } 711 712 return max; 713 } 714 715 u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) 716 { 717 u32 nb_entries1 = 0, nb_entries2; 718 719 if (unlikely(pool->dev && dma_dev_need_sync(pool->dev))) 720 return xp_alloc_slow(pool, xdp, max); 721 722 if (unlikely(pool->free_list_cnt)) { 723 nb_entries1 = xp_alloc_reused(pool, xdp, max); 724 if (nb_entries1 == max) 725 return nb_entries1; 726 727 max -= nb_entries1; 728 xdp += nb_entries1; 729 } 730 731 nb_entries2 = xp_alloc_new_from_fq(pool, xdp, max); 732 if (!nb_entries2) 733 pool->fq->queue_empty_descs++; 734 735 return nb_entries1 + nb_entries2; 736 } 737 EXPORT_SYMBOL(xp_alloc_batch); 738 739 bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count) 740 { 741 u32 req_count, avail_count; 742 743 if (pool->free_list_cnt >= count) 744 return true; 745 746 req_count = count - pool->free_list_cnt; 747 avail_count = xskq_cons_nb_entries(pool->fq, req_count); 748 if (!avail_count) 749 pool->fq->queue_empty_descs++; 750 751 return avail_count >= req_count; 752 } 753 EXPORT_SYMBOL(xp_can_alloc); 754 755 void xp_free(struct xdp_buff_xsk *xskb) 756 { 757 if (!list_empty(&xskb->list_node)) 758 return; 759 760 xskb->pool->free_list_cnt++; 761 list_add(&xskb->list_node, &xskb->pool->free_list); 762 } 763 EXPORT_SYMBOL(xp_free); 764 765 static u64 __xp_raw_get_addr(const struct xsk_buff_pool *pool, u64 addr) 766 { 767 return pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr; 768 } 769 770 static void *__xp_raw_get_data(const struct xsk_buff_pool *pool, u64 addr) 771 { 772 return pool->addrs + addr; 773 } 774 775 void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr) 776 { 777 return __xp_raw_get_data(pool, __xp_raw_get_addr(pool, addr)); 778 } 779 EXPORT_SYMBOL(xp_raw_get_data); 780 781 static dma_addr_t __xp_raw_get_dma(const struct xsk_buff_pool *pool, u64 addr) 782 { 783 return (pool->dma_pages[addr >> PAGE_SHIFT] & 784 ~XSK_NEXT_PG_CONTIG_MASK) + 785 (addr & ~PAGE_MASK); 786 } 787 788 dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr) 789 { 790 return __xp_raw_get_dma(pool, __xp_raw_get_addr(pool, addr)); 791 } 792 EXPORT_SYMBOL(xp_raw_get_dma); 793 794 /** 795 * xp_raw_get_ctx - get &xdp_desc context 796 * @pool: XSk buff pool desc address belongs to 797 * @addr: desc address (from userspace) 798 * @options: desc options (from userspace) 799 * 800 * Helper for getting desc's DMA address and metadata pointer, if present. 801 * Saves one call on hotpath and double calculation of the actual address. 802 * Metadata is validated later by xsk_tx_metadata_request(). 803 * 804 * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata 805 * pointer, if it is present (initialized to %NULL otherwise). 806 */ 807 struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, 808 u32 options) 809 { 810 struct xdp_desc_ctx ret; 811 812 addr = __xp_raw_get_addr(pool, addr); 813 814 ret.dma = __xp_raw_get_dma(pool, addr); 815 ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr), 816 options); 817 818 return ret; 819 } 820 EXPORT_SYMBOL(xp_raw_get_ctx); 821