Lines Matching defs:bq
396 * @bq: name of the variable to declare
402 #define LIBETH_XDP_ONSTACK_BULK(bq) \
403 struct libeth_xdp_tx_bulk bq __uninitialized
593 * @bq: XDP Tx bulk to queue the head frag to
598 static inline bool libeth_xdp_tx_queue_head(struct libeth_xdp_tx_bulk *bq,
603 bq->bulk[bq->count++] = (typeof(*bq->bulk)){
612 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_MULTI;
619 * @bq: XDP Tx bulk to queue the frag to
622 static inline void libeth_xdp_tx_queue_frag(struct libeth_xdp_tx_bulk *bq,
625 bq->bulk[bq->count++].frag = *frag;
630 * @bq: XDP Tx bulk to queue the frame to
637 libeth_xdp_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq,
639 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
646 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
647 unlikely(!flush_bulk(bq, 0))) {
652 if (!libeth_xdp_tx_queue_head(bq, xdp))
659 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
660 unlikely(!flush_bulk(bq, 0))) {
665 libeth_xdp_tx_queue_frag(bq, &sinfo->frags[i]);
669 bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_LAST;
754 void libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent,
759 * @bq: bulk to flush
771 __libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags,
783 sent = libeth_xdp_tx_xmit_bulk(bq->bulk, bq->xdpsq,
784 min(bq->count, LIBETH_XDP_TX_BULK),
786 drops = bq->count - sent;
789 libeth_xdp_tx_exception(bq, sent, flags);
792 bq->count = 0;
795 trace_xdp_bulk_tx(bq->dev, sent, drops, err);
802 * @bq: bulk to flush
810 #define libeth_xdp_tx_flush_bulk(bq, flags, prep, xmit) \
811 __libeth_xdp_tx_flush_bulk(bq, flags, prep, libeth_xdp_tx_fill_buf, \
818 * @bq: bulk to initialize
823 #define libeth_xdp_xmit_init_bulk(bq, dev, xdpsqs, num) \
824 __libeth_xdp_xmit_init_bulk(bq, dev, (xdpsqs)[libeth_xdpsq_id(num)])
826 static inline void __libeth_xdp_xmit_init_bulk(struct libeth_xdp_tx_bulk *bq,
830 bq->dev = dev;
831 bq->xdpsq = xdpsq;
832 bq->count = 0;
866 * @bq: XDP Tx bulk to queue the head frag to
874 static inline u32 libeth_xdp_xmit_queue_head(struct libeth_xdp_tx_bulk *bq,
886 bq->bulk[bq->count++] = (typeof(*bq->bulk)){
894 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI;
901 * @bq: XDP Tx bulk to queue the frag to
907 static inline bool libeth_xdp_xmit_queue_frag(struct libeth_xdp_tx_bulk *bq,
917 bq->bulk[bq->count++] = (typeof(*bq->bulk)){
927 * @bq: XDP Tx bulk to queue the frame to
936 libeth_xdp_xmit_queue_bulk(struct libeth_xdp_tx_bulk *bq,
938 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
942 struct device *dev = bq->dev->dev.parent;
945 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
946 unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO)))
949 head = libeth_xdp_xmit_queue_head(bq, xdpf, dev);
959 if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
960 unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO)))
963 if (!libeth_xdp_xmit_queue_frag(bq, &sinfo->frags[i], dev))
971 bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_LAST;
1022 * @bq: bulk to flush
1030 #define libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit) \
1031 __libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_NDO, prep, \
1034 u32 libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq,
1039 * @bq: XDP Tx bulk to queue frames to
1052 __libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq,
1054 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
1066 ret = libeth_xdp_xmit_queue_bulk(bq, frames[i], flush_bulk);
1075 if (bq->count) {
1076 flush_bulk(bq, LIBETH_XDP_TX_NDO);
1077 if (unlikely(bq->count))
1078 nxmit -= libeth_xdp_xmit_return_bulk(bq->bulk,
1079 bq->count,
1080 bq->dev);
1083 finalize(bq->xdpsq, nxmit, flags & XDP_XMIT_FLUSH);
1130 * @bq: bulk to initialize
1141 #define libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num) \
1142 __libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, \
1145 #define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, ub, un) do { \
1146 typeof(bq) ub = (bq); \
1339 u32 libeth_xdp_prog_exception(const struct libeth_xdp_tx_bulk *bq,
1346 * @bq: buffer bulk for ``XDP_TX`` queueing
1356 const struct libeth_xdp_tx_bulk *bq)
1360 act = bpf_prog_run_xdp(bq->prog, &xdp->base);
1374 if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog)))
1385 return libeth_xdp_prog_exception(bq, xdp, act, 0);
1391 * @bq: buffer bulk for ``XDP_TX`` queueing
1404 struct libeth_xdp_tx_bulk *bq,
1406 const struct libeth_xdp_tx_bulk *bq),
1407 bool (*queue)(struct libeth_xdp_tx_bulk *bq,
1410 (struct libeth_xdp_tx_bulk *bq,
1412 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
1417 act = run(xdp, bq);
1418 if (act == LIBETH_XDP_TX && unlikely(!queue(bq, xdp, flush_bulk)))
1421 bq->act_mask |= act;
1429 * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers
1439 #define libeth_xdp_run_prog(xdp, bq, fl) \
1440 (__libeth_xdp_run_flush(xdp, bq, __libeth_xdp_run_prog, \
1447 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1470 struct libeth_xdp_tx_bulk *bq, struct napi_struct *napi,
1475 struct libeth_xdp_tx_bulk *bq),
1491 if (!bq || !run || !bq->prog)
1494 if (!run(xdp, bq))
1523 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1533 #define libeth_xdp_run_pass(xdp, bq, napi, ss, desc, run, populate) \
1534 __libeth_xdp_run_pass(xdp, bq, napi, ss, desc, libeth_xdp_prep_desc, \
1539 * @bq: ``XDP_TX`` frame bulk
1546 #define libeth_xdp_finalize_rx(bq, flush, finalize) \
1547 __libeth_xdp_finalize_rx(bq, 0, flush, finalize)
1550 __libeth_xdp_finalize_rx(struct libeth_xdp_tx_bulk *bq, u32 flags,
1551 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
1555 if (bq->act_mask & LIBETH_XDP_TX) {
1556 if (bq->count)
1557 flush_bulk(bq, flags | LIBETH_XDP_TX_DROP);
1558 finalize(bq->xdpsq, true, true);
1560 if (bq->act_mask & LIBETH_XDP_REDIRECT)
1586 * driver_xdp_run(xdp, &bq, napi, &rs, desc);
1588 * driver_xdp_finalize_rx(&bq);
1617 bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \
1619 return libeth_##pfx##_tx_flush_bulk(bq, flags, prep, xmit); \
1629 bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \
1631 return libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit); \
1643 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq) \
1645 return libeth_##pfx##_run_prog(xdp, bq, flush); \
1658 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq, \
1662 return libeth_##pfx##_run_pass(xdp, bq, napi, ss, desc, run, \
1690 void name(struct libeth_xdp_tx_bulk *bq) \
1692 libeth_##pfx##_finalize_rx(bq, flush, finalize); \
1745 struct xdp_frame_bulk *bq, bool frags);
1781 bulk(sqe->sinfo, cp->bq, sqe->nr_frags != 1);
1784 xdp_return_frame_bulk(sqe->xdpf, cp->bq);