| /linux/include/net/libeth/ |
| H A D | xdp.h | 402 #define LIBETH_XDP_ONSTACK_BULK(bq) \ argument
403 struct libeth_xdp_tx_bulk bq __uninitialized
598 static inline bool libeth_xdp_tx_queue_head(struct libeth_xdp_tx_bulk *bq, in libeth_xdp_tx_queue_head() argument
603 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xdp_tx_queue_head()
612 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_MULTI; in libeth_xdp_tx_queue_head()
622 static inline void libeth_xdp_tx_queue_frag(struct libeth_xdp_tx_bulk *bq, in libeth_xdp_tx_queue_frag() argument
625 bq->bulk[bq->count++].frag = *frag; in libeth_xdp_tx_queue_frag()
637 libeth_xdp_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq, in libeth_xdp_tx_queue_bulk() argument
639 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, in libeth_xdp_tx_queue_bulk() argument
646 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xdp_tx_queue_bulk()
[all …]
|
| H A D | xsk.h | 24 static inline bool libeth_xsk_tx_queue_head(struct libeth_xdp_tx_bulk *bq, in libeth_xsk_tx_queue_head() argument
27 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xsk_tx_queue_head()
36 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI; in libeth_xsk_tx_queue_head()
46 static inline void libeth_xsk_tx_queue_frag(struct libeth_xdp_tx_bulk *bq, in libeth_xsk_tx_queue_frag() argument
49 bq->bulk[bq->count++] = (typeof(*bq->bulk)){ in libeth_xsk_tx_queue_frag()
64 libeth_xsk_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq, in libeth_xsk_tx_queue_bulk() argument
66 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq, in libeth_xsk_tx_queue_bulk() argument
71 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) && in libeth_xsk_tx_queue_bulk()
72 unlikely(!flush_bulk(bq, LIBETH_XDP_TX_XSK))) { in libeth_xsk_tx_queue_bulk()
77 if (!libeth_xsk_tx_queue_head(bq, xdp)) in libeth_xsk_tx_queue_bulk()
[all …]
|
| H A D | tx.h | 105 struct xdp_frame_bulk *bq; member
|
| /linux/drivers/net/ethernet/intel/libeth/ |
| H A D | xdp.c | 94 libeth_xdp_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, u32 count) in libeth_xdp_tx_return_bulk() argument
97 const struct libeth_xdp_tx_frame *frm = &bq[i]; in libeth_xdp_tx_return_bulk()
123 void __cold libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent, in libeth_xdp_tx_exception() argument
126 const struct libeth_xdp_tx_frame *pos = &bq->bulk[sent]; in libeth_xdp_tx_exception()
127 u32 left = bq->count - sent; in libeth_xdp_tx_exception()
130 libeth_trace_xdp_exception(bq->dev, bq->prog, XDP_TX); in libeth_xdp_tx_exception()
133 memmove(bq->bulk, pos, left * sizeof(*bq->bulk)); in libeth_xdp_tx_exception()
134 bq->count = left; in libeth_xdp_tx_exception()
144 libeth_xdp_xmit_return_bulk(pos, left, bq->dev); in libeth_xdp_tx_exception()
146 bq->count = 0; in libeth_xdp_tx_exception()
[all …]
|
| H A D | xsk.c | 14 void __cold libeth_xsk_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, in libeth_xsk_tx_return_bulk() argument
18 libeth_xsk_buff_free_slow(bq[i].xsk); in libeth_xsk_tx_return_bulk()
99 const struct libeth_xdp_tx_bulk *bq, in __libeth_xsk_run_prog_slow() argument
115 return libeth_xdp_prog_exception(bq, xdp, act, ret); in __libeth_xsk_run_prog_slow()
|
| H A D | priv.h | 19 void libeth_xsk_tx_return_bulk(const struct libeth_xdp_tx_frame *bq,
26 struct xdp_frame_bulk *bq, bool frags);
|
| /linux/kernel/bpf/ |
| H A D | cpumap.c | 441 struct xdp_bulk_queue *bq; in __cpu_map_entry_alloc() local
457 bq = per_cpu_ptr(rcpu->bulkq, i); in __cpu_map_entry_alloc()
458 bq->obj = rcpu; in __cpu_map_entry_alloc()
459 local_lock_init(&bq->bq_lock); in __cpu_map_entry_alloc()
723 static void bq_flush_to_queue(struct xdp_bulk_queue *bq) in bq_flush_to_queue() argument
725 struct bpf_cpu_map_entry *rcpu = bq->obj; in bq_flush_to_queue()
731 lockdep_assert_held(&bq->bq_lock); in bq_flush_to_queue()
733 if (unlikely(!bq->count)) in bq_flush_to_queue()
739 for (i = 0; i < bq->count; i++) { in bq_flush_to_queue()
740 struct xdp_frame *xdpf = bq->q[i]; in bq_flush_to_queue()
[all …]
|
| /linux/drivers/net/ethernet/intel/idpf/ |
| H A D | xsk.c | 172 struct xdp_frame_bulk bq; in idpf_xsksq_clean() local
175 .bq = &bq, in idpf_xsksq_clean()
180 xdp_frame_bulk_init(&bq); in idpf_xsksq_clean()
194 xdp_flush_frame_bulk(&bq); in idpf_xsksq_clean()
206 struct xdp_frame_bulk bq; in idpf_xsksq_complete_slow() local
209 .bq = &bq, in idpf_xsksq_complete_slow()
215 xdp_frame_bulk_init(&bq); in idpf_xsksq_complete_slow()
229 xdp_flush_frame_bulk(&bq); in idpf_xsksq_complete_slow()
474 LIBETH_XDP_ONSTACK_BULK(bq); in idpf_xskrq_poll()
483 libeth_xsk_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, in idpf_xskrq_poll()
[all …]
|
| H A D | xdp.c | 302 struct xdp_frame_bulk bq; in idpf_xdpsq_complete() local
305 .bq = &bq, in idpf_xdpsq_complete()
315 xdp_frame_bulk_init(&bq); in idpf_xdpsq_complete()
324 xdp_flush_frame_bulk(&bq); in idpf_xdpsq_complete()
|
| H A D | idpf_txrx.c | 62 struct xdp_frame_bulk bq; in idpf_tx_buf_clean() local
65 .bq = &bq, in idpf_tx_buf_clean()
69 xdp_frame_bulk_init(&bq); in idpf_tx_buf_clean()
75 xdp_flush_frame_bulk(&bq); in idpf_tx_buf_clean()
3529 LIBETH_XDP_ONSTACK_BULK(bq); in idpf_rx_splitq_clean()
3531 libeth_xdp_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, in idpf_rx_splitq_clean()
3628 idpf_xdp_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); in idpf_rx_splitq_clean()
3631 idpf_xdp_finalize_rx(&bq); in idpf_rx_splitq_clean()
|
| /linux/drivers/net/ |
| H A D | veth.c | 576 static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) in veth_xdp_flush_bq() argument
580 sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); in veth_xdp_flush_bq()
586 for (i = sent; unlikely(i < bq->count); i++) in veth_xdp_flush_bq()
587 xdp_return_frame(bq->q[i]); in veth_xdp_flush_bq()
589 drops = bq->count - sent; in veth_xdp_flush_bq()
597 bq->count = 0; in veth_xdp_flush_bq()
600 static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) in veth_xdp_flush() argument
607 veth_xdp_flush_bq(rq, bq); in veth_xdp_flush()
624 struct veth_xdp_tx_bq *bq) in veth_xdp_tx() argument
631 if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) in veth_xdp_tx()
[all …]
|
| /linux/arch/arm/boot/dts/mediatek/ |
| H A D | mt6589-aquaris5.dts | 12 model = "bq Aquaris5";
13 compatible = "mundoreader,bq-aquaris5", "mediatek,mt6589";
|
| /linux/net/core/ |
| H A D | xdp.c | 511 struct xdp_frame_bulk *bq) in xdp_return_frame_bulk() argument
518 if (bq->count == XDP_BULK_QUEUE_SIZE) in xdp_return_frame_bulk()
519 xdp_flush_frame_bulk(bq); in xdp_return_frame_bulk()
529 bq->q[bq->count++] = skb_frag_netmem(frag); in xdp_return_frame_bulk()
530 if (bq->count == XDP_BULK_QUEUE_SIZE) in xdp_return_frame_bulk()
531 xdp_flush_frame_bulk(bq); in xdp_return_frame_bulk()
534 bq->q[bq->count++] = virt_to_netmem(xdpf->data); in xdp_return_frame_bulk()
|
| /linux/drivers/net/ethernet/mellanox/mlx5/core/en/ |
| H A D | xdp.c | 668 struct xdp_frame_bulk *bq, in mlx5e_free_xdpsq_desc() argument
706 xdp_return_frame_bulk(xdpf, bq); in mlx5e_free_xdpsq_desc()
757 struct xdp_frame_bulk bq; in mlx5e_poll_xdpsq_cq() local
764 xdp_frame_bulk_init(&bq); in mlx5e_poll_xdpsq_cq()
797 mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, &bq, cq, cqe); in mlx5e_poll_xdpsq_cq()
810 xdp_flush_frame_bulk(&bq); in mlx5e_poll_xdpsq_cq()
828 struct xdp_frame_bulk bq; in mlx5e_free_xdpsq_descs() local
831 xdp_frame_bulk_init(&bq); in mlx5e_free_xdpsq_descs()
844 mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, &bq, NULL, NULL); in mlx5e_free_xdpsq_descs()
847 xdp_flush_frame_bulk(&bq); in mlx5e_free_xdpsq_descs()
|
| /linux/arch/arm/boot/dts/rockchip/ |
| H A D | rk3066a-bqcurie2.dts | 12 model = "bq Curie 2";
13 compatible = "mundoreader,bq-curie2", "rockchip,rk3066a";
|
| /linux/drivers/net/ethernet/pensando/ionic/ |
| H A D | ionic_txrx.c | 278 struct xdp_frame_bulk bq; in ionic_xdp_tx_desc_clean() local
283 xdp_frame_bulk_init(&bq); in ionic_xdp_tx_desc_clean()
293 xdp_return_frame_bulk(desc_info->xdpf, &bq); in ionic_xdp_tx_desc_clean()
296 xdp_flush_frame_bulk(&bq); in ionic_xdp_tx_desc_clean()
|
| /linux/drivers/net/vmxnet3/ |
| H A D | vmxnet3_drv.c | 398 struct xdp_frame_bulk *bq) in vmxnet3_unmap_pkt() argument
427 xdp_return_frame_bulk(tbi->xdpf, bq); in vmxnet3_unmap_pkt()
443 struct xdp_frame_bulk bq; in vmxnet3_tq_tx_complete() local
446 xdp_frame_bulk_init(&bq); in vmxnet3_tq_tx_complete()
458 adapter, &bq); in vmxnet3_tq_tx_complete()
463 xdp_flush_frame_bulk(&bq); in vmxnet3_tq_tx_complete()
484 struct xdp_frame_bulk bq; in vmxnet3_tq_cleanup() local
488 xdp_frame_bulk_init(&bq); in vmxnet3_tq_cleanup()
500 xdp_return_frame_bulk(tbi->xdpf, &bq); in vmxnet3_tq_cleanup()
508 xdp_flush_frame_bulk(&bq); in vmxnet3_tq_cleanup()
|
| /linux/arch/arm64/boot/dts/qcom/ |
| H A D | msm8916-longcheer-l8910.dts | 275 model = "bq-paella";
|
| /linux/drivers/net/ethernet/mediatek/ |
| H A D | mtk_eth_soc.c | 1409 struct xdp_frame_bulk *bq, bool napi) in mtk_tx_unmap() argument
1452 else if (bq) in mtk_tx_unmap()
1453 xdp_return_frame_bulk(xdpf, bq); in mtk_tx_unmap()
2476 struct xdp_frame_bulk bq; in mtk_poll_tx_qdma() local
2484 xdp_frame_bulk_init(&bq); in mtk_poll_tx_qdma()
2505 mtk_tx_unmap(eth, tx_buf, &bq, true); in mtk_poll_tx_qdma()
2512 xdp_flush_frame_bulk(&bq); in mtk_poll_tx_qdma()
2525 struct xdp_frame_bulk bq; in mtk_poll_tx_pdma() local
2531 xdp_frame_bulk_init(&bq); in mtk_poll_tx_pdma()
2543 mtk_tx_unmap(eth, tx_buf, &bq, true); in mtk_poll_tx_pdma()
[all …]
|
| /linux/drivers/net/ethernet/socionext/ |
| H A D | netsec.c | 638 struct xdp_frame_bulk bq; in netsec_clean_tx_dring() local
647 xdp_frame_bulk_init(&bq); in netsec_clean_tx_dring()
679 xdp_return_frame_bulk(desc->xdpf, &bq); in netsec_clean_tx_dring()
698 xdp_flush_frame_bulk(&bq); in netsec_clean_tx_dring()
|
| /linux/drivers/net/ethernet/marvell/ |
| H A D | mvneta.c | 1873 struct xdp_frame_bulk bq; in mvneta_txq_bufs_free() local
1876 xdp_frame_bulk_init(&bq); in mvneta_txq_bufs_free()
1902 xdp_return_frame_bulk(buf->xdpf, &bq); in mvneta_txq_bufs_free()
1905 xdp_flush_frame_bulk(&bq); in mvneta_txq_bufs_free()
|