| /linux/include/net/libeth/ |
| H A D | xdp.h | 402 #define LIBETH_XDP_ONSTACK_BULK(bq) \ argument
403 struct libeth_xdp_tx_bulk bq __uninitialized
598 static inline bool libeth_xdp_tx_queue_head(struct libeth_xdp_tx_bulk *bq, in libeth_xdp_tx_queue_head() argument
603 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xdp_tx_queue_head()
612 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_MULTI; in libeth_xdp_tx_queue_head()
622 static inline void libeth_xdp_tx_queue_frag(struct libeth_xdp_tx_bulk *bq, in libeth_xdp_tx_queue_frag() argument
625 bq->bulk[bq->count++].frag = *frag; in libeth_xdp_tx_queue_frag()
637 libeth_xdp_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq, in libeth_xdp_tx_queue_bulk() argument
639 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, in libeth_xdp_tx_queue_bulk() argument
646 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xdp_tx_queue_bulk()
[all …]
|
| H A D | xsk.h | 24 static inline bool libeth_xsk_tx_queue_head(struct libeth_xdp_tx_bulk *bq, in libeth_xsk_tx_queue_head() argument
27 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xsk_tx_queue_head()
36 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI; in libeth_xsk_tx_queue_head()
46 static inline void libeth_xsk_tx_queue_frag(struct libeth_xdp_tx_bulk *bq, in libeth_xsk_tx_queue_frag() argument
49 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xsk_tx_queue_frag()
64 libeth_xsk_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq, in libeth_xsk_tx_queue_bulk() argument
66 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, in libeth_xsk_tx_queue_bulk() argument
71 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xsk_tx_queue_bulk()
72 unlikely(!flush_bulk(bq, LIBETH_XDP_TX_XSK))) { in libeth_xsk_tx_queue_bulk()
77 if (!libeth_xsk_tx_queue_head(bq, xdp)) in libeth_xsk_tx_queue_bulk()
[all …]
|
| H A D | tx.h | 105 struct xdp_frame_bulk *bq; member
|
| /linux/drivers/net/ethernet/intel/libeth/ |
| H A D | xdp.c | 94 libeth_xdp_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, u32 count) in libeth_xdp_tx_return_bulk() argument
97 const struct libeth_xdp_tx_frame *frm = &bq[i]; in libeth_xdp_tx_return_bulk()
123 void __cold libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent, in libeth_xdp_tx_exception() argument
126 const struct libeth_xdp_tx_frame *pos = &bq->bulk[sent]; in libeth_xdp_tx_exception()
127 u32 left = bq->count - sent; in libeth_xdp_tx_exception()
130 libeth_trace_xdp_exception(bq->dev, bq->prog, XDP_TX); in libeth_xdp_tx_exception()
133 memmove(bq->bulk, pos, left * sizeof(*bq->bulk)); in libeth_xdp_tx_exception()
134 bq->count = left; in libeth_xdp_tx_exception()
144 libeth_xdp_xmit_return_bulk(pos, left, bq->dev); in libeth_xdp_tx_exception()
146 bq->count = 0; in libeth_xdp_tx_exception()
[all …]
|
| H A D | xsk.c | 14 void __cold libeth_xsk_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, in libeth_xsk_tx_return_bulk() argument
18 libeth_xsk_buff_free_slow(bq[i].xsk); in libeth_xsk_tx_return_bulk()
99 const struct libeth_xdp_tx_bulk *bq, in __libeth_xsk_run_prog_slow() argument
115 return libeth_xdp_prog_exception(bq, xdp, act, ret); in __libeth_xsk_run_prog_slow()
|
| H A D | priv.h | 19 void libeth_xsk_tx_return_bulk(const struct libeth_xdp_tx_frame *bq,
26 struct xdp_frame_bulk *bq, bool frags);
|
| /linux/kernel/bpf/ |
| H A D | cpumap.c | 441 struct xdp_bulk_queue *bq; in __cpu_map_entry_alloc() local
457 bq = per_cpu_ptr(rcpu->bulkq, i); in __cpu_map_entry_alloc()
458 bq->obj = rcpu; in __cpu_map_entry_alloc()
459 local_lock_init(&bq->bq_lock); in __cpu_map_entry_alloc()
723 static void bq_flush_to_queue(struct xdp_bulk_queue *bq) in bq_flush_to_queue() argument
725 struct bpf_cpu_map_entry *rcpu = bq->obj; in bq_flush_to_queue()
731 lockdep_assert_held(&bq->bq_lock); in bq_flush_to_queue()
733 if (unlikely(!bq->count)) in bq_flush_to_queue()
739 for (i = 0; i < bq->count; i++) { in bq_flush_to_queue()
740 struct xdp_frame *xdpf = bq->q[i]; in bq_flush_to_queue()
[all …]
|
| H A D | devmap.c | 378 static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags) in bq_xmit_all() argument
380 struct net_device *dev = bq->dev; in bq_xmit_all()
381 unsigned int cnt = bq->count; in bq_xmit_all()
386 lockdep_assert_held(&bq->bq_lock); in bq_xmit_all()
392 struct xdp_frame *xdpf = bq->q[i]; in bq_xmit_all()
397 if (bq->xdp_prog) { in bq_xmit_all()
398 to_send = dev_map_bpf_prog_run(bq->xdp_prog, bq->q, cnt, dev, bq->dev_rx); in bq_xmit_all()
403 sent = dev->netdev_ops->ndo_xdp_xmit(dev, to_send, bq->q, flags); in bq_xmit_all()
416 xdp_return_frame_rx_napi(bq->q[i]); in bq_xmit_all()
419 bq->count = 0; in bq_xmit_all()
[all …]
|
| /linux/drivers/net/ethernet/intel/idpf/ |
| H A D | xsk.c | 172 struct xdp_frame_bulk bq; in idpf_xsksq_clean() local
175 .bq = &bq, in idpf_xsksq_clean()
180 xdp_frame_bulk_init(&bq); in idpf_xsksq_clean()
194 xdp_flush_frame_bulk(&bq); in idpf_xsksq_clean()
206 struct xdp_frame_bulk bq; in idpf_xsksq_complete_slow() local
209 .bq = &bq, in idpf_xsksq_complete_slow()
215 xdp_frame_bulk_init(&bq); in idpf_xsksq_complete_slow()
229 xdp_flush_frame_bulk(&bq); in idpf_xsksq_complete_slow()
474 LIBETH_XDP_ONSTACK_BULK(bq); in idpf_xskrq_poll()
483 libeth_xsk_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, in idpf_xskrq_poll()
[all …]
|
| H A D | xdp.c | 302 struct xdp_frame_bulk bq; in idpf_xdpsq_complete() local
305 .bq = &bq, in idpf_xdpsq_complete()
315 xdp_frame_bulk_init(&bq); in idpf_xdpsq_complete()
324 xdp_flush_frame_bulk(&bq); in idpf_xdpsq_complete()
|
| H A D | idpf_txrx.c | 62 struct xdp_frame_bulk bq; in idpf_tx_buf_clean() local
65 .bq = &bq, in idpf_tx_buf_clean()
69 xdp_frame_bulk_init(&bq); in idpf_tx_buf_clean()
75 xdp_flush_frame_bulk(&bq); in idpf_tx_buf_clean()
3532 LIBETH_XDP_ONSTACK_BULK(bq); in idpf_rx_splitq_clean()
3534 libeth_xdp_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, in idpf_rx_splitq_clean()
3631 idpf_xdp_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); in idpf_rx_splitq_clean()
3634 idpf_xdp_finalize_rx(&bq); in idpf_rx_splitq_clean()
|
| /linux/drivers/net/ |
| H A D | veth.c | 575 static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) in veth_xdp_flush_bq() argument
579 sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); in veth_xdp_flush_bq()
585 for (i = sent; unlikely(i < bq->count); i++) in veth_xdp_flush_bq()
586 xdp_return_frame(bq->q[i]); in veth_xdp_flush_bq()
588 drops = bq->count - sent; in veth_xdp_flush_bq()
596 bq->count = 0; in veth_xdp_flush_bq()
599 static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) in veth_xdp_flush() argument
606 veth_xdp_flush_bq(rq, bq); in veth_xdp_flush()
623 struct veth_xdp_tx_bq *bq) in veth_xdp_tx() argument
630 if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) in veth_xdp_tx()
[all …]
|
| /linux/arch/arm/boot/dts/mediatek/ |
| H A D | mt6589-aquaris5.dts | 12 model = "bq Aquaris5";
13 compatible = "mundoreader,bq-aquaris5", "mediatek,mt6589";
|
| /linux/net/core/ |
| H A D | xdp.c | 511 struct xdp_frame_bulk *bq) in xdp_return_frame_bulk() argument
518 if (bq->count == XDP_BULK_QUEUE_SIZE) in xdp_return_frame_bulk()
519 xdp_flush_frame_bulk(bq); in xdp_return_frame_bulk()
529 bq->q[bq->count++] = skb_frag_netmem(frag); in xdp_return_frame_bulk()
530 if (bq->count == XDP_BULK_QUEUE_SIZE) in xdp_return_frame_bulk()
531 xdp_flush_frame_bulk(bq); in xdp_return_frame_bulk()
534 bq->q[bq->count++] = virt_to_netmem(xdpf->data); in xdp_return_frame_bulk()
|
| /linux/arch/arm/boot/dts/rockchip/ |
| H A D | rk3066a-bqcurie2.dts | 12 model = "bq Curie 2";
13 compatible = "mundoreader,bq-curie2", "rockchip,rk3066a";
|
| /linux/drivers/net/ethernet/microchip/lan966x/ |
| H A D | lan966x_fdma.c | 338 struct xdp_frame_bulk bq; in lan966x_fdma_tx_clear_buf() local
344 xdp_frame_bulk_init(&bq); in lan966x_fdma_tx_clear_buf()
377 xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq); in lan966x_fdma_tx_clear_buf()
386 xdp_flush_frame_bulk(&bq); in lan966x_fdma_tx_clear_buf()
|
| /linux/drivers/net/ethernet/pensando/ionic/ |
| H A D | ionic_txrx.c | 278 struct xdp_frame_bulk bq; in ionic_xdp_tx_desc_clean() local
283 xdp_frame_bulk_init(&bq); in ionic_xdp_tx_desc_clean()
293 xdp_return_frame_bulk(desc_info->xdpf, &bq); in ionic_xdp_tx_desc_clean()
296 xdp_flush_frame_bulk(&bq); in ionic_xdp_tx_desc_clean()
|
| /linux/arch/arm64/boot/dts/qcom/ |
| H A D | msm8916-longcheer-l8910.dts | 275 model = "bq-paella";
|
| /linux/drivers/net/ethernet/mediatek/ |
| H A D | mtk_eth_soc.c | 1408 struct xdp_frame_bulk *bq, bool napi) in mtk_tx_unmap() argument
1451 else if (bq) in mtk_tx_unmap()
1452 xdp_return_frame_bulk(xdpf, bq); in mtk_tx_unmap()
2475 struct xdp_frame_bulk bq; in mtk_poll_tx_qdma() local
2483 xdp_frame_bulk_init(&bq); in mtk_poll_tx_qdma()
2504 mtk_tx_unmap(eth, tx_buf, &bq, true); in mtk_poll_tx_qdma()
2511 xdp_flush_frame_bulk(&bq); in mtk_poll_tx_qdma()
2524 struct xdp_frame_bulk bq; in mtk_poll_tx_pdma() local
2530 xdp_frame_bulk_init(&bq); in mtk_poll_tx_pdma()
2542 mtk_tx_unmap(eth, tx_buf, &bq, true); in mtk_poll_tx_pdma()
[all …]
|
| /linux/drivers/net/ethernet/socionext/ |
| H A D | netsec.c | 638 struct xdp_frame_bulk bq; in netsec_clean_tx_dring() local
647 xdp_frame_bulk_init(&bq); in netsec_clean_tx_dring()
679 xdp_return_frame_bulk(desc->xdpf, &bq); in netsec_clean_tx_dring()
698 xdp_flush_frame_bulk(&bq); in netsec_clean_tx_dring()
|
| /linux/drivers/net/ethernet/marvell/mvpp2/ |
| H A D | mvpp2_main.c | 2813 struct xdp_frame_bulk bq; in mvpp2_txq_bufs_free() local
2816 xdp_frame_bulk_init(&bq); in mvpp2_txq_bufs_free()
2832 xdp_return_frame_bulk(tx_buf->xdpf, &bq); in mvpp2_txq_bufs_free()
2836 xdp_flush_frame_bulk(&bq); in mvpp2_txq_bufs_free()
|