/linux/net/core/ |
H A D | xdp.c | 463 void xdp_return_frame(struct xdp_frame *xdpf) in xdp_return_frame() argument 467 if (likely(!xdp_frame_has_frags(xdpf))) in xdp_return_frame() 470 sinfo = xdp_get_shared_info_from_frame(xdpf); in xdp_return_frame() 472 __xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type, in xdp_return_frame() 476 __xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, false, NULL); in xdp_return_frame() 480 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf) in xdp_return_frame_rx_napi() argument 484 if (likely(!xdp_frame_has_frags(xdpf))) in xdp_return_frame_rx_napi() 487 sinfo = xdp_get_shared_info_from_frame(xdpf); in xdp_return_frame_rx_napi() 489 __xdp_return(skb_frag_netmem(&sinfo->frags[i]), xdpf->mem_type, in xdp_return_frame_rx_napi() 493 __xdp_return(virt_to_netmem(xdpf->data), xdpf->mem_type, true, NULL); in xdp_return_frame_rx_napi() [all …]
|
/linux/drivers/net/ethernet/amazon/ena/ |
H A D | ena_xdp.c | 13 if (likely(tx_info->xdpf)) in validate_xdp_req_id() 21 struct xdp_frame *xdpf, in ena_xdp_tx_map_frame() argument 31 tx_info->xdpf = xdpf; in ena_xdp_tx_map_frame() 32 data = tx_info->xdpf->data; in ena_xdp_tx_map_frame() 33 size = tx_info->xdpf->len; in ena_xdp_tx_map_frame() 77 struct xdp_frame *xdpf, in ena_xdp_xmit_frame() argument 90 rc = ena_xdp_tx_map_frame(tx_ring, tx_info, xdpf, &ena_tx_ctx); in ena_xdp_xmit_frame() 101 xdpf->len); in ena_xdp_xmit_frame() 116 tx_info->xdpf = NULL; in ena_xdp_xmit_frame() 381 struct xdp_frame *xdpf; in ena_clean_xdp_irq() local [all …]
|
H A D | ena_xdp.h | 40 struct xdp_frame *xdpf, 87 struct xdp_frame *xdpf; in ena_xdp_execute() local 96 xdpf = xdp_convert_buff_to_frame(xdp); in ena_xdp_execute() 97 if (unlikely(!xdpf)) { in ena_xdp_execute() 110 if (ena_xdp_xmit_frame(xdp_ring, rx_ring->adapter, xdpf, in ena_xdp_execute() 112 xdp_return_frame(xdpf); in ena_xdp_execute()
|
/linux/drivers/net/vmxnet3/ |
H A D | vmxnet3_xdp.c | 116 struct xdp_frame *xdpf, in vmxnet3_xdp_xmit_frame() argument 129 dw2 |= xdpf->len; in vmxnet3_xdp_xmit_frame() 133 buf_size = xdpf->len; in vmxnet3_xdp_xmit_frame() 145 xdpf->data, buf_size, in vmxnet3_xdp_xmit_frame() 153 page = virt_to_page(xdpf->data); in vmxnet3_xdp_xmit_frame() 155 (xdpf->data - (void *)xdpf); in vmxnet3_xdp_xmit_frame() 160 tbi->xdpf = xdpf; in vmxnet3_xdp_xmit_frame() 207 struct xdp_frame *xdpf) in vmxnet3_xdp_xmit_back() argument 220 err = vmxnet3_xdp_xmit_frame(adapter, xdpf, tq, false); in vmxnet3_xdp_xmit_back() 264 struct xdp_frame *xdpf; in vmxnet3_run_xdp() local [all …]
|
/linux/kernel/bpf/ |
H A D | devmap.c | 345 struct xdp_frame *xdpf = frames[i]; in dev_map_bpf_prog_run() local 349 xdp_convert_frame_to_buff(xdpf, &xdp); in dev_map_bpf_prog_run() 356 err = xdp_update_frame_from_buff(&xdp, xdpf); in dev_map_bpf_prog_run() 358 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run() 360 frames[nframes++] = xdpf; in dev_map_bpf_prog_run() 369 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run() 388 struct xdp_frame *xdpf = bq->q[i]; in bq_xmit_all() local 390 prefetch(xdpf); in bq_xmit_all() 456 static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, in bq_enqueue() argument 479 bq->q[bq->count++] = xdpf; in bq_enqueue() [all …]
|
/linux/include/net/ |
H A D | xdp.h | 341 struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, 344 struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, 347 struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); 412 void xdp_return_frame(struct xdp_frame *xdpf); 413 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); 415 void xdp_return_frame_bulk(struct xdp_frame *xdpf, 428 xdp_get_frame_len(const struct xdp_frame *xdpf) in xdp_get_frame_len() argument 431 unsigned int len = xdpf->len; in xdp_get_frame_len() 433 if (likely(!xdp_frame_has_frags(xdpf))) in xdp_get_frame_len() 436 sinfo = xdp_get_shared_info_from_frame(xdpf); in xdp_get_frame_len()
|
/linux/drivers/net/ethernet/sfc/siena/ |
H A D | tx.c | 213 struct xdp_frame *xdpf; in efx_siena_xdp_tx_buffers() local 255 xdpf = xdpfs[i]; in efx_siena_xdp_tx_buffers() 263 len = xdpf->len; in efx_siena_xdp_tx_buffers() 267 xdpf->data, len, in efx_siena_xdp_tx_buffers() 274 tx_buffer->xdpf = xdpf; in efx_siena_xdp_tx_buffers()
|
H A D | rx.c | 257 struct xdp_frame *xdpf; in efx_do_xdp() local 312 xdpf = xdp_convert_buff_to_frame(&xdp); in efx_do_xdp() 313 err = efx_siena_xdp_tx_buffers(efx, 1, &xdpf, true); in efx_do_xdp()
|
/linux/drivers/net/ethernet/fungible/funeth/ |
H A D | funeth_tx.c | 500 xdp_return_frame(q->info[reclaim_idx].xdpf); in fun_xdpq_clean() 514 bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf) in fun_xdp_tx() argument 516 unsigned int idx, nfrags = 1, ndesc = 1, tot_len = xdpf->len; in fun_xdp_tx() 525 if (unlikely(xdp_frame_has_frags(xdpf))) { in fun_xdp_tx() 526 si = xdp_get_shared_info_from_frame(xdpf); in fun_xdp_tx() 527 tot_len = xdp_get_frame_len(xdpf); in fun_xdp_tx() 539 if (unlikely(fun_map_pkt(q->dma_dev, si, xdpf->data, xdpf->len, dma, in fun_xdp_tx() 558 q->info[idx].xdpf = xdpf; in fun_xdp_tx() 618 xdp_return_frame(q->info[idx].xdpf); in fun_xdpq_purge()
|
H A D | funeth_txrx.h | 101 struct xdp_frame *xdpf; /* associated XDP frame (XDP path) */ member 250 bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf);
|
H A D | funeth_rx.c | 145 struct xdp_frame *xdpf; in fun_run_xdp() local 168 xdpf = xdp_convert_buff_to_frame(&xdp); in fun_run_xdp() 169 if (!xdpf || !fun_xdp_tx(xdp_q, xdpf)) in fun_run_xdp()
|
/linux/drivers/net/ethernet/aquantia/atlantic/ |
H A D | aq_ring.c | 320 } else if (buff->xdpf) { in aq_ring_tx_clean() 323 self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf); in aq_ring_tx_clean() 325 xdp_return_frame_rx_napi(buff->xdpf); in aq_ring_tx_clean() 330 buff->xdpf = NULL; in aq_ring_tx_clean() 377 struct xdp_frame *xdpf = frames[i]; in aq_xdp_xmit() local 379 if (aq_nic_xmit_xdpf(aq_nic, ring, xdpf) == NETDEV_TX_BUSY) in aq_xdp_xmit() 390 struct xdp_frame *xdpf; in aq_xdp_build_skb() local 393 xdpf = xdp_convert_buff_to_frame(xdp); in aq_xdp_build_skb() 394 if (unlikely(!xdpf)) in aq_xdp_build_skb() 397 skb = xdp_build_skb_from_frame(xdpf, dev); in aq_xdp_build_skb() [all …]
|
H A D | aq_nic.c | 576 struct xdp_frame *xdpf, in aq_nic_map_xdp() argument 592 sinfo = xdp_get_shared_info_from_frame(xdpf); in aq_nic_map_xdp() 593 total_len = xdpf->len; in aq_nic_map_xdp() 595 if (xdp_frame_has_frags(xdpf)) { in aq_nic_map_xdp() 599 dx_buff->pa = dma_map_single(dev, xdpf->data, dx_buff->len, in aq_nic_map_xdp() 649 dx_buff->xdpf = xdpf; in aq_nic_map_xdp() 800 dx_buff->xdpf = NULL; in aq_nic_map_skb() 830 struct xdp_frame *xdpf) in aq_nic_xmit_xdpf() argument 840 if (xdp_frame_has_frags(xdpf)) { in aq_nic_xmit_xdpf() 841 sinfo = xdp_get_shared_info_from_frame(xdpf); in aq_nic_xmit_xdpf() [all …]
|
/linux/drivers/net/ethernet/ti/ |
H A D | cpsw_priv.c | 55 struct xdp_frame *xdpf; in cpsw_tx_handler() local 62 xdpf = cpsw_handle_to_xdpf(token); in cpsw_tx_handler() 63 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET; in cpsw_tx_handler() 66 xdp_return_frame(xdpf); in cpsw_tx_handler() 1291 int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf, in cpsw_xdp_tx_frame() argument 1300 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET; in cpsw_xdp_tx_frame() 1307 dma += xdpf->headroom + sizeof(struct xdp_frame); in cpsw_xdp_tx_frame() 1308 ret = cpdma_chan_submit_mapped(txch, cpsw_xdpf_to_handle(xdpf), in cpsw_xdp_tx_frame() 1309 dma, xdpf->len, port); in cpsw_xdp_tx_frame() 1311 if (sizeof(*xmeta) > xdpf->headroom) in cpsw_xdp_tx_frame() [all …]
|
H A D | am65-cpsw-nuss.c | 833 struct xdp_frame *xdpf; in am65_cpsw_nuss_tx_cleanup() local 844 xdpf = *(swdata); in am65_cpsw_nuss_tx_cleanup() 845 xdp_return_frame(xdpf); in am65_cpsw_nuss_tx_cleanup() 1095 struct xdp_frame *xdpf, in am65_cpsw_xdp_tx_frame() argument 1103 u32 pkt_len = xdpf->len; in am65_cpsw_xdp_tx_frame() 1115 dma_buf = dma_map_single(tx_chn->dma_dev, xdpf->data, in am65_cpsw_xdp_tx_frame() 1134 *(swdata) = xdpf; in am65_cpsw_xdp_tx_frame() 1178 struct xdp_frame *xdpf; in am65_cpsw_run_xdp() local 1202 xdpf = xdp_convert_buff_to_frame(xdp); in am65_cpsw_run_xdp() 1203 if (unlikely(!xdpf)) { in am65_cpsw_run_xdp() [all …]
|
/linux/drivers/net/ethernet/qlogic/qede/ |
H A D | qede_fp.c | 305 u16 len, struct page *page, struct xdp_frame *xdpf) in qede_xdp_xmit() argument 332 xdp->xdpf = xdpf; in qede_xdp_xmit() 345 struct xdp_frame *xdpf; in qede_xdp_transmit() local 362 xdpf = frames[i]; in qede_xdp_transmit() 364 mapping = dma_map_single(dmadev, xdpf->data, xdpf->len, in qede_xdp_transmit() 369 if (unlikely(qede_xdp_xmit(xdp_tx, mapping, 0, xdpf->len, in qede_xdp_transmit() 370 NULL, xdpf))) in qede_xdp_transmit() 404 struct xdp_frame *xdpf; in qede_xdp_tx_int() local 412 xdpf = xdp_info->xdpf; in qede_xdp_tx_int() 414 if (xdpf) { in qede_xdp_tx_int() [all …]
|
/linux/drivers/net/ethernet/microchip/lan966x/ |
H A D | lan966x_fdma.c | 374 xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq); in lan966x_fdma_tx_clear_buf() 605 struct xdp_frame *xdpf; in lan966x_fdma_xmit_xdpf() local 627 xdpf = ptr; in lan966x_fdma_xmit_xdpf() 629 if (xdpf->headroom < IFH_LEN_BYTES) { in lan966x_fdma_xmit_xdpf() 634 ifh = xdpf->data - IFH_LEN_BYTES; in lan966x_fdma_xmit_xdpf() 640 xdpf->data - IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf() 641 xdpf->len + IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf() 648 next_dcb_buf->data.xdpf = xdpf; in lan966x_fdma_xmit_xdpf() 649 next_dcb_buf->len = xdpf->len + IFH_LEN_BYTES; in lan966x_fdma_xmit_xdpf()
|
H A D | lan966x_xdp.c | 62 struct xdp_frame *xdpf = frames[i]; in lan966x_xdp_xmit() local 65 err = lan966x_fdma_xmit_xdpf(port, xdpf, 0); in lan966x_xdp_xmit()
|
/linux/drivers/net/ethernet/broadcom/bnxt/ |
H A D | bnxt_xdp.c | 110 struct xdp_frame *xdpf) in __bnxt_xmit_xdp_redirect() argument 116 tx_buf->xdpf = xdpf; in __bnxt_xmit_xdp_redirect() 145 xdp_return_frame(tx_buf->xdpf); in bnxt_tx_int_xdp() 147 tx_buf->xdpf = NULL; in bnxt_tx_int_xdp()
|
/linux/drivers/net/ethernet/pensando/ionic/ |
H A D | ionic_txrx.c | 284 xdp_return_frame_rx_napi(desc_info->xdpf); in ionic_xdp_tx_desc_clean() 286 xdp_return_frame(desc_info->xdpf); in ionic_xdp_tx_desc_clean() 289 xdp_return_frame_bulk(desc_info->xdpf, &bq); in ionic_xdp_tx_desc_clean() 296 desc_info->xdpf = NULL; in ionic_xdp_tx_desc_clean() 334 desc_info->xdpf = frame; in ionic_xdp_post_frame() 477 struct xdp_frame *xdpf; in ionic_run_xdp() local 544 xdpf = xdp_convert_buff_to_frame(&xdp_buf); in ionic_run_xdp() 545 if (!xdpf) { in ionic_run_xdp() 564 err = ionic_xdp_post_frame(txq, xdpf, XDP_TX, in ionic_run_xdp() 1182 if (desc_info->xdpf) { in ionic_tx_clean()
|
/linux/drivers/net/ethernet/intel/i40e/ |
H A D | i40e_txrx.c | 757 xdp_return_frame(tx_buffer->xdpf); in i40e_unmap_and_free_tx_resource() 967 xdp_return_frame(tx_buf->xdpf); in i40e_clean_tx_irq() 2246 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, 2251 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); in i40e_xmit_xdp_tx_ring() local 2253 if (unlikely(!xdpf)) in i40e_xmit_xdp_tx_ring() 2256 return i40e_xmit_xdp_ring(xdpf, xdp_ring); in i40e_xmit_xdp_tx_ring() 3778 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, in i40e_xmit_xdp_ring() argument 3781 struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); in i40e_xmit_xdp_ring() 3782 u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; in i40e_xmit_xdp_ring() 3787 void *data = xdpf->data; in i40e_xmit_xdp_ring() [all …]
|
/linux/drivers/net/ |
H A D | xen-netfront.c | 614 struct xdp_frame *xdpf) in xennet_xdp_xmit_one() argument 621 .page = virt_to_page(xdpf->data), in xennet_xdp_xmit_one() 626 offset_in_page(xdpf->data), in xennet_xdp_xmit_one() 627 xdpf->len); in xennet_xdp_xmit_one() 636 tx_stats->bytes += xdpf->len; in xennet_xdp_xmit_one() 664 struct xdp_frame *xdpf = frames[i]; in xennet_xdp_xmit() local 666 if (!xdpf) in xennet_xdp_xmit() 668 if (xennet_xdp_xmit_one(dev, queue, xdpf)) in xennet_xdp_xmit() 975 struct xdp_frame *xdpf; in xennet_run_xdp() local 989 xdpf = xdp_convert_buff_to_frame(xdp); in xennet_run_xdp() [all …]
|
/linux/drivers/net/ethernet/intel/ixgbe/ |
H A D | ixgbe_txrx_common.h | 17 struct xdp_frame *xdpf);
|
/linux/drivers/net/ethernet/intel/ice/ |
H A D | ice_txrx.c | 132 xdp_return_frame(tx_buf->xdpf); in ice_unmap_and_free_tx_buf() 584 static int ice_xmit_xdp_ring(const struct xdp_frame *xdpf, in ice_xmit_xdp_ring() argument 589 xdp.data_hard_start = (void *)xdpf; in ice_xmit_xdp_ring() 590 xdp.data = xdpf->data; in ice_xmit_xdp_ring() 591 xdp.data_end = xdp.data + xdpf->len; in ice_xmit_xdp_ring() 592 xdp.frame_sz = xdpf->frame_sz; in ice_xmit_xdp_ring() 593 xdp.flags = xdpf->flags; in ice_xmit_xdp_ring() 643 const struct xdp_frame *xdpf = frames[i]; in ice_xdp_xmit() local 646 err = ice_xmit_xdp_ring(xdpf, xdp_ring); in ice_xdp_xmit()
|
/linux/drivers/net/ethernet/stmicro/stmmac/ |
H A D | stmmac_main.c | 1580 if (tx_q->xdpf[i] && in stmmac_free_tx_buffer() 1583 xdp_return_frame(tx_q->xdpf[i]); in stmmac_free_tx_buffer() 1584 tx_q->xdpf[i] = NULL; in stmmac_free_tx_buffer() 2592 tx_q->xdpf[entry] = NULL; in stmmac_xdp_xmit_zc() 2692 struct xdp_frame *xdpf; in stmmac_tx_clean() local 2699 xdpf = tx_q->xdpf[entry]; in stmmac_tx_clean() 2702 xdpf = NULL; in stmmac_tx_clean() 2705 xdpf = NULL; in stmmac_tx_clean() 2775 if (xdpf && in stmmac_tx_clean() 2777 xdp_return_frame_rx_napi(xdpf); in stmmac_tx_clean() [all …]
|