1db37bc17SDimitris Michailidis // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2db37bc17SDimitris Michailidis
3db37bc17SDimitris Michailidis #include <linux/bpf_trace.h>
4db37bc17SDimitris Michailidis #include <linux/dma-mapping.h>
5db37bc17SDimitris Michailidis #include <linux/etherdevice.h>
6db37bc17SDimitris Michailidis #include <linux/filter.h>
7db37bc17SDimitris Michailidis #include <linux/irq.h>
8db37bc17SDimitris Michailidis #include <linux/pci.h>
9db37bc17SDimitris Michailidis #include <linux/skbuff.h>
10db37bc17SDimitris Michailidis #include "funeth_txrx.h"
11db37bc17SDimitris Michailidis #include "funeth.h"
12db37bc17SDimitris Michailidis #include "fun_queue.h"
13db37bc17SDimitris Michailidis
14db37bc17SDimitris Michailidis #define CREATE_TRACE_POINTS
15db37bc17SDimitris Michailidis #include "funeth_trace.h"
16db37bc17SDimitris Michailidis
17db37bc17SDimitris Michailidis /* Given the device's max supported MTU and pages of at least 4KB a packet can
18db37bc17SDimitris Michailidis * be scattered into at most 4 buffers.
19db37bc17SDimitris Michailidis */
20db37bc17SDimitris Michailidis #define RX_MAX_FRAGS 4
21db37bc17SDimitris Michailidis
22db37bc17SDimitris Michailidis /* Per packet headroom in non-XDP mode. Present only for 1-frag packets. */
23db37bc17SDimitris Michailidis #define FUN_RX_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
24db37bc17SDimitris Michailidis
25db37bc17SDimitris Michailidis /* We try to reuse pages for our buffers. To avoid frequent page ref writes we
26db37bc17SDimitris Michailidis * take EXTRA_PAGE_REFS references at once and then hand them out one per packet
27db37bc17SDimitris Michailidis * occupying the buffer.
28db37bc17SDimitris Michailidis */
29db37bc17SDimitris Michailidis #define EXTRA_PAGE_REFS 1000000
30db37bc17SDimitris Michailidis #define MIN_PAGE_REFS 1000
31db37bc17SDimitris Michailidis
32db37bc17SDimitris Michailidis enum {
33db37bc17SDimitris Michailidis FUN_XDP_FLUSH_REDIR = 1,
34db37bc17SDimitris Michailidis FUN_XDP_FLUSH_TX = 2,
35db37bc17SDimitris Michailidis };
36db37bc17SDimitris Michailidis
37db37bc17SDimitris Michailidis /* See if a page is running low on refs we are holding and if so take more. */
refresh_refs(struct funeth_rxbuf * buf)38db37bc17SDimitris Michailidis static void refresh_refs(struct funeth_rxbuf *buf)
39db37bc17SDimitris Michailidis {
40db37bc17SDimitris Michailidis if (unlikely(buf->pg_refs < MIN_PAGE_REFS)) {
41db37bc17SDimitris Michailidis buf->pg_refs += EXTRA_PAGE_REFS;
42db37bc17SDimitris Michailidis page_ref_add(buf->page, EXTRA_PAGE_REFS);
43db37bc17SDimitris Michailidis }
44db37bc17SDimitris Michailidis }
45db37bc17SDimitris Michailidis
46db37bc17SDimitris Michailidis /* Offer a buffer to the Rx buffer cache. The cache will hold the buffer if its
47db37bc17SDimitris Michailidis * page is worth retaining and there's room for it. Otherwise the page is
48db37bc17SDimitris Michailidis * unmapped and our references released.
49db37bc17SDimitris Michailidis */
cache_offer(struct funeth_rxq * q,const struct funeth_rxbuf * buf)50db37bc17SDimitris Michailidis static void cache_offer(struct funeth_rxq *q, const struct funeth_rxbuf *buf)
51db37bc17SDimitris Michailidis {
52db37bc17SDimitris Michailidis struct funeth_rx_cache *c = &q->cache;
53db37bc17SDimitris Michailidis
54db37bc17SDimitris Michailidis if (c->prod_cnt - c->cons_cnt <= c->mask && buf->node == numa_mem_id()) {
55db37bc17SDimitris Michailidis c->bufs[c->prod_cnt & c->mask] = *buf;
56db37bc17SDimitris Michailidis c->prod_cnt++;
57db37bc17SDimitris Michailidis } else {
58db37bc17SDimitris Michailidis dma_unmap_page_attrs(q->dma_dev, buf->dma_addr, PAGE_SIZE,
59db37bc17SDimitris Michailidis DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
60db37bc17SDimitris Michailidis __page_frag_cache_drain(buf->page, buf->pg_refs);
61db37bc17SDimitris Michailidis }
62db37bc17SDimitris Michailidis }
63db37bc17SDimitris Michailidis
64db37bc17SDimitris Michailidis /* Get a page from the Rx buffer cache. We only consider the next available
65db37bc17SDimitris Michailidis * page and return it if we own all its references.
66db37bc17SDimitris Michailidis */
cache_get(struct funeth_rxq * q,struct funeth_rxbuf * rb)67db37bc17SDimitris Michailidis static bool cache_get(struct funeth_rxq *q, struct funeth_rxbuf *rb)
68db37bc17SDimitris Michailidis {
69db37bc17SDimitris Michailidis struct funeth_rx_cache *c = &q->cache;
70db37bc17SDimitris Michailidis struct funeth_rxbuf *buf;
71db37bc17SDimitris Michailidis
72db37bc17SDimitris Michailidis if (c->prod_cnt == c->cons_cnt)
73db37bc17SDimitris Michailidis return false; /* empty cache */
74db37bc17SDimitris Michailidis
75db37bc17SDimitris Michailidis buf = &c->bufs[c->cons_cnt & c->mask];
76db37bc17SDimitris Michailidis if (page_ref_count(buf->page) == buf->pg_refs) {
77db37bc17SDimitris Michailidis dma_sync_single_for_device(q->dma_dev, buf->dma_addr,
78db37bc17SDimitris Michailidis PAGE_SIZE, DMA_FROM_DEVICE);
79db37bc17SDimitris Michailidis *rb = *buf;
80db37bc17SDimitris Michailidis buf->page = NULL;
81db37bc17SDimitris Michailidis refresh_refs(rb);
82db37bc17SDimitris Michailidis c->cons_cnt++;
83db37bc17SDimitris Michailidis return true;
84db37bc17SDimitris Michailidis }
85db37bc17SDimitris Michailidis
86db37bc17SDimitris Michailidis /* Page can't be reused. If the cache is full drop this page. */
87db37bc17SDimitris Michailidis if (c->prod_cnt - c->cons_cnt > c->mask) {
88db37bc17SDimitris Michailidis dma_unmap_page_attrs(q->dma_dev, buf->dma_addr, PAGE_SIZE,
89db37bc17SDimitris Michailidis DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
90db37bc17SDimitris Michailidis __page_frag_cache_drain(buf->page, buf->pg_refs);
91db37bc17SDimitris Michailidis buf->page = NULL;
92db37bc17SDimitris Michailidis c->cons_cnt++;
93db37bc17SDimitris Michailidis }
94db37bc17SDimitris Michailidis return false;
95db37bc17SDimitris Michailidis }
96db37bc17SDimitris Michailidis
97db37bc17SDimitris Michailidis /* Allocate and DMA-map a page for receive. */
funeth_alloc_page(struct funeth_rxq * q,struct funeth_rxbuf * rb,int node,gfp_t gfp)98db37bc17SDimitris Michailidis static int funeth_alloc_page(struct funeth_rxq *q, struct funeth_rxbuf *rb,
99db37bc17SDimitris Michailidis int node, gfp_t gfp)
100db37bc17SDimitris Michailidis {
101db37bc17SDimitris Michailidis struct page *p;
102db37bc17SDimitris Michailidis
103db37bc17SDimitris Michailidis if (cache_get(q, rb))
104db37bc17SDimitris Michailidis return 0;
105db37bc17SDimitris Michailidis
106db37bc17SDimitris Michailidis p = __alloc_pages_node(node, gfp | __GFP_NOWARN, 0);
107db37bc17SDimitris Michailidis if (unlikely(!p))
108db37bc17SDimitris Michailidis return -ENOMEM;
109db37bc17SDimitris Michailidis
110db37bc17SDimitris Michailidis rb->dma_addr = dma_map_page(q->dma_dev, p, 0, PAGE_SIZE,
111db37bc17SDimitris Michailidis DMA_FROM_DEVICE);
112db37bc17SDimitris Michailidis if (unlikely(dma_mapping_error(q->dma_dev, rb->dma_addr))) {
113db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, rx_map_err);
114db37bc17SDimitris Michailidis __free_page(p);
115db37bc17SDimitris Michailidis return -ENOMEM;
116db37bc17SDimitris Michailidis }
117db37bc17SDimitris Michailidis
118db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, rx_page_alloc);
119db37bc17SDimitris Michailidis
120db37bc17SDimitris Michailidis rb->page = p;
121db37bc17SDimitris Michailidis rb->pg_refs = 1;
122db37bc17SDimitris Michailidis refresh_refs(rb);
123db37bc17SDimitris Michailidis rb->node = page_is_pfmemalloc(p) ? -1 : page_to_nid(p);
124db37bc17SDimitris Michailidis return 0;
125db37bc17SDimitris Michailidis }
126db37bc17SDimitris Michailidis
funeth_free_page(struct funeth_rxq * q,struct funeth_rxbuf * rb)127db37bc17SDimitris Michailidis static void funeth_free_page(struct funeth_rxq *q, struct funeth_rxbuf *rb)
128db37bc17SDimitris Michailidis {
129db37bc17SDimitris Michailidis if (rb->page) {
130db37bc17SDimitris Michailidis dma_unmap_page(q->dma_dev, rb->dma_addr, PAGE_SIZE,
131db37bc17SDimitris Michailidis DMA_FROM_DEVICE);
132db37bc17SDimitris Michailidis __page_frag_cache_drain(rb->page, rb->pg_refs);
133db37bc17SDimitris Michailidis rb->page = NULL;
134db37bc17SDimitris Michailidis }
135db37bc17SDimitris Michailidis }
136db37bc17SDimitris Michailidis
137db37bc17SDimitris Michailidis /* Run the XDP program assigned to an Rx queue.
138db37bc17SDimitris Michailidis * Return %NULL if the buffer is consumed, or the virtual address of the packet
139db37bc17SDimitris Michailidis * to turn into an skb.
140db37bc17SDimitris Michailidis */
fun_run_xdp(struct funeth_rxq * q,skb_frag_t * frags,void * buf_va,int ref_ok,struct funeth_txq * xdp_q)141db37bc17SDimitris Michailidis static void *fun_run_xdp(struct funeth_rxq *q, skb_frag_t *frags, void *buf_va,
142db37bc17SDimitris Michailidis int ref_ok, struct funeth_txq *xdp_q)
143db37bc17SDimitris Michailidis {
144db37bc17SDimitris Michailidis struct bpf_prog *xdp_prog;
14551a83391SDimitris Michailidis struct xdp_frame *xdpf;
146db37bc17SDimitris Michailidis struct xdp_buff xdp;
147db37bc17SDimitris Michailidis u32 act;
148db37bc17SDimitris Michailidis
149db37bc17SDimitris Michailidis /* VA includes the headroom, frag size includes headroom + tailroom */
150db37bc17SDimitris Michailidis xdp_init_buff(&xdp, ALIGN(skb_frag_size(frags), FUN_EPRQ_PKT_ALIGN),
151db37bc17SDimitris Michailidis &q->xdp_rxq);
152db37bc17SDimitris Michailidis xdp_prepare_buff(&xdp, buf_va, FUN_XDP_HEADROOM, skb_frag_size(frags) -
153db37bc17SDimitris Michailidis (FUN_RX_TAILROOM + FUN_XDP_HEADROOM), false);
154db37bc17SDimitris Michailidis
155db37bc17SDimitris Michailidis xdp_prog = READ_ONCE(q->xdp_prog);
156db37bc17SDimitris Michailidis act = bpf_prog_run_xdp(xdp_prog, &xdp);
157db37bc17SDimitris Michailidis
158db37bc17SDimitris Michailidis switch (act) {
159db37bc17SDimitris Michailidis case XDP_PASS:
160db37bc17SDimitris Michailidis /* remove headroom, which may not be FUN_XDP_HEADROOM now */
161db37bc17SDimitris Michailidis skb_frag_size_set(frags, xdp.data_end - xdp.data);
162db37bc17SDimitris Michailidis skb_frag_off_add(frags, xdp.data - xdp.data_hard_start);
163db37bc17SDimitris Michailidis goto pass;
164db37bc17SDimitris Michailidis case XDP_TX:
165db37bc17SDimitris Michailidis if (unlikely(!ref_ok))
166db37bc17SDimitris Michailidis goto pass;
16751a83391SDimitris Michailidis
16851a83391SDimitris Michailidis xdpf = xdp_convert_buff_to_frame(&xdp);
16951a83391SDimitris Michailidis if (!xdpf || !fun_xdp_tx(xdp_q, xdpf))
170db37bc17SDimitris Michailidis goto xdp_error;
171db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, xdp_tx);
172db37bc17SDimitris Michailidis q->xdp_flush |= FUN_XDP_FLUSH_TX;
173db37bc17SDimitris Michailidis break;
174db37bc17SDimitris Michailidis case XDP_REDIRECT:
175db37bc17SDimitris Michailidis if (unlikely(!ref_ok))
176db37bc17SDimitris Michailidis goto pass;
177db37bc17SDimitris Michailidis if (unlikely(xdp_do_redirect(q->netdev, &xdp, xdp_prog)))
178db37bc17SDimitris Michailidis goto xdp_error;
179db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, xdp_redir);
180db37bc17SDimitris Michailidis q->xdp_flush |= FUN_XDP_FLUSH_REDIR;
181db37bc17SDimitris Michailidis break;
182db37bc17SDimitris Michailidis default:
183db37bc17SDimitris Michailidis bpf_warn_invalid_xdp_action(q->netdev, xdp_prog, act);
184db37bc17SDimitris Michailidis fallthrough;
185db37bc17SDimitris Michailidis case XDP_ABORTED:
186db37bc17SDimitris Michailidis trace_xdp_exception(q->netdev, xdp_prog, act);
187db37bc17SDimitris Michailidis xdp_error:
188db37bc17SDimitris Michailidis q->cur_buf->pg_refs++; /* return frags' page reference */
189db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, xdp_err);
190db37bc17SDimitris Michailidis break;
191db37bc17SDimitris Michailidis case XDP_DROP:
192db37bc17SDimitris Michailidis q->cur_buf->pg_refs++;
193db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, xdp_drops);
194db37bc17SDimitris Michailidis break;
195db37bc17SDimitris Michailidis }
196db37bc17SDimitris Michailidis return NULL;
197db37bc17SDimitris Michailidis
198db37bc17SDimitris Michailidis pass:
199db37bc17SDimitris Michailidis return xdp.data;
200db37bc17SDimitris Michailidis }
201db37bc17SDimitris Michailidis
202db37bc17SDimitris Michailidis /* A CQE contains a fixed completion structure along with optional metadata and
203db37bc17SDimitris Michailidis * even packet data. Given the start address of a CQE return the start of the
204db37bc17SDimitris Michailidis * contained fixed structure, which lies at the end.
205db37bc17SDimitris Michailidis */
cqe_to_info(const void * cqe)206db37bc17SDimitris Michailidis static const void *cqe_to_info(const void *cqe)
207db37bc17SDimitris Michailidis {
208db37bc17SDimitris Michailidis return cqe + FUNETH_CQE_INFO_OFFSET;
209db37bc17SDimitris Michailidis }
210db37bc17SDimitris Michailidis
211db37bc17SDimitris Michailidis /* The inverse of cqe_to_info(). */
info_to_cqe(const void * cqe_info)212db37bc17SDimitris Michailidis static const void *info_to_cqe(const void *cqe_info)
213db37bc17SDimitris Michailidis {
214db37bc17SDimitris Michailidis return cqe_info - FUNETH_CQE_INFO_OFFSET;
215db37bc17SDimitris Michailidis }
216db37bc17SDimitris Michailidis
217db37bc17SDimitris Michailidis /* Return the type of hash provided by the device based on the L3 and L4
218db37bc17SDimitris Michailidis * protocols it parsed for the packet.
219db37bc17SDimitris Michailidis */
cqe_to_pkt_hash_type(u16 pkt_parse)220db37bc17SDimitris Michailidis static enum pkt_hash_types cqe_to_pkt_hash_type(u16 pkt_parse)
221db37bc17SDimitris Michailidis {
222db37bc17SDimitris Michailidis static const enum pkt_hash_types htype_map[] = {
223db37bc17SDimitris Michailidis PKT_HASH_TYPE_NONE, PKT_HASH_TYPE_L3,
224db37bc17SDimitris Michailidis PKT_HASH_TYPE_NONE, PKT_HASH_TYPE_L4,
225db37bc17SDimitris Michailidis PKT_HASH_TYPE_NONE, PKT_HASH_TYPE_L3,
226db37bc17SDimitris Michailidis PKT_HASH_TYPE_NONE, PKT_HASH_TYPE_L3
227db37bc17SDimitris Michailidis };
228db37bc17SDimitris Michailidis u16 key;
229db37bc17SDimitris Michailidis
230db37bc17SDimitris Michailidis /* Build the key from the TCP/UDP and IP/IPv6 bits */
231db37bc17SDimitris Michailidis key = ((pkt_parse >> FUN_ETH_RX_CV_OL4_PROT_S) & 6) |
232db37bc17SDimitris Michailidis ((pkt_parse >> (FUN_ETH_RX_CV_OL3_PROT_S + 1)) & 1);
233db37bc17SDimitris Michailidis
234db37bc17SDimitris Michailidis return htype_map[key];
235db37bc17SDimitris Michailidis }
236db37bc17SDimitris Michailidis
237db37bc17SDimitris Michailidis /* Each received packet can be scattered across several Rx buffers or can
238db37bc17SDimitris Michailidis * share a buffer with previously received packets depending on the buffer
239db37bc17SDimitris Michailidis * and packet sizes and the room available in the most recently used buffer.
240db37bc17SDimitris Michailidis *
241db37bc17SDimitris Michailidis * The rules are:
242db37bc17SDimitris Michailidis * - If the buffer at the head of an RQ has not been used it gets (part of) the
243db37bc17SDimitris Michailidis * next incoming packet.
244db37bc17SDimitris Michailidis * - Otherwise, if the packet fully fits in the buffer's remaining space the
245db37bc17SDimitris Michailidis * packet is written there.
246db37bc17SDimitris Michailidis * - Otherwise, the packet goes into the next Rx buffer.
247db37bc17SDimitris Michailidis *
248db37bc17SDimitris Michailidis * This function returns the Rx buffer for a packet or fragment thereof of the
249db37bc17SDimitris Michailidis * given length. If it isn't @buf it either recycles or frees that buffer
250db37bc17SDimitris Michailidis * before advancing the queue to the next buffer.
251db37bc17SDimitris Michailidis *
252db37bc17SDimitris Michailidis * If called repeatedly with the remaining length of a packet it will walk
253db37bc17SDimitris Michailidis * through all the buffers containing the packet.
254db37bc17SDimitris Michailidis */
255db37bc17SDimitris Michailidis static struct funeth_rxbuf *
get_buf(struct funeth_rxq * q,struct funeth_rxbuf * buf,unsigned int len)256db37bc17SDimitris Michailidis get_buf(struct funeth_rxq *q, struct funeth_rxbuf *buf, unsigned int len)
257db37bc17SDimitris Michailidis {
258db37bc17SDimitris Michailidis if (q->buf_offset + len <= PAGE_SIZE || !q->buf_offset)
259db37bc17SDimitris Michailidis return buf; /* @buf holds (part of) the packet */
260db37bc17SDimitris Michailidis
261db37bc17SDimitris Michailidis /* The packet occupies part of the next buffer. Move there after
262db37bc17SDimitris Michailidis * replenishing the current buffer slot either with the spare page or
263db37bc17SDimitris Michailidis * by reusing the slot's existing page. Note that if a spare page isn't
264db37bc17SDimitris Michailidis * available and the current packet occupies @buf it is a multi-frag
265db37bc17SDimitris Michailidis * packet that will be dropped leaving @buf available for reuse.
266db37bc17SDimitris Michailidis */
267db37bc17SDimitris Michailidis if ((page_ref_count(buf->page) == buf->pg_refs &&
268db37bc17SDimitris Michailidis buf->node == numa_mem_id()) || !q->spare_buf.page) {
269db37bc17SDimitris Michailidis dma_sync_single_for_device(q->dma_dev, buf->dma_addr,
270db37bc17SDimitris Michailidis PAGE_SIZE, DMA_FROM_DEVICE);
271db37bc17SDimitris Michailidis refresh_refs(buf);
272db37bc17SDimitris Michailidis } else {
273db37bc17SDimitris Michailidis cache_offer(q, buf);
274db37bc17SDimitris Michailidis *buf = q->spare_buf;
275db37bc17SDimitris Michailidis q->spare_buf.page = NULL;
276db37bc17SDimitris Michailidis q->rqes[q->rq_cons & q->rq_mask] =
277db37bc17SDimitris Michailidis FUN_EPRQ_RQBUF_INIT(buf->dma_addr);
278db37bc17SDimitris Michailidis }
279db37bc17SDimitris Michailidis q->buf_offset = 0;
280db37bc17SDimitris Michailidis q->rq_cons++;
281db37bc17SDimitris Michailidis return &q->bufs[q->rq_cons & q->rq_mask];
282db37bc17SDimitris Michailidis }
283db37bc17SDimitris Michailidis
284db37bc17SDimitris Michailidis /* Gather the page fragments making up the first Rx packet on @q. Its total
285db37bc17SDimitris Michailidis * length @tot_len includes optional head- and tail-rooms.
286db37bc17SDimitris Michailidis *
287db37bc17SDimitris Michailidis * Return 0 if the device retains ownership of at least some of the pages.
288db37bc17SDimitris Michailidis * In this case the caller may only copy the packet.
289db37bc17SDimitris Michailidis *
290db37bc17SDimitris Michailidis * A non-zero return value gives the caller permission to use references to the
291db37bc17SDimitris Michailidis * pages, e.g., attach them to skbs. Additionally, if the value is <0 at least
292db37bc17SDimitris Michailidis * one of the pages is PF_MEMALLOC.
293db37bc17SDimitris Michailidis *
294db37bc17SDimitris Michailidis * Regardless of outcome the caller is granted a reference to each of the pages.
295db37bc17SDimitris Michailidis */
fun_gather_pkt(struct funeth_rxq * q,unsigned int tot_len,skb_frag_t * frags)296db37bc17SDimitris Michailidis static int fun_gather_pkt(struct funeth_rxq *q, unsigned int tot_len,
297db37bc17SDimitris Michailidis skb_frag_t *frags)
298db37bc17SDimitris Michailidis {
299db37bc17SDimitris Michailidis struct funeth_rxbuf *buf = q->cur_buf;
300db37bc17SDimitris Michailidis unsigned int frag_len;
301db37bc17SDimitris Michailidis int ref_ok = 1;
302db37bc17SDimitris Michailidis
303db37bc17SDimitris Michailidis for (;;) {
304db37bc17SDimitris Michailidis buf = get_buf(q, buf, tot_len);
305db37bc17SDimitris Michailidis
306db37bc17SDimitris Michailidis /* We always keep the RQ full of buffers so before we can give
307db37bc17SDimitris Michailidis * one of our pages to the stack we require that we can obtain
308db37bc17SDimitris Michailidis * a replacement page. If we can't the packet will either be
309db37bc17SDimitris Michailidis * copied or dropped so we can retain ownership of the page and
310db37bc17SDimitris Michailidis * reuse it.
311db37bc17SDimitris Michailidis */
312db37bc17SDimitris Michailidis if (!q->spare_buf.page &&
313db37bc17SDimitris Michailidis funeth_alloc_page(q, &q->spare_buf, numa_mem_id(),
314db37bc17SDimitris Michailidis GFP_ATOMIC | __GFP_MEMALLOC))
315db37bc17SDimitris Michailidis ref_ok = 0;
316db37bc17SDimitris Michailidis
317db37bc17SDimitris Michailidis frag_len = min_t(unsigned int, tot_len,
318db37bc17SDimitris Michailidis PAGE_SIZE - q->buf_offset);
319db37bc17SDimitris Michailidis dma_sync_single_for_cpu(q->dma_dev,
320db37bc17SDimitris Michailidis buf->dma_addr + q->buf_offset,
321db37bc17SDimitris Michailidis frag_len, DMA_FROM_DEVICE);
322db37bc17SDimitris Michailidis buf->pg_refs--;
323db37bc17SDimitris Michailidis if (ref_ok)
324db37bc17SDimitris Michailidis ref_ok |= buf->node;
325db37bc17SDimitris Michailidis
326*b51f4113SYunsheng Lin skb_frag_fill_page_desc(frags++, buf->page, q->buf_offset,
327*b51f4113SYunsheng Lin frag_len);
328db37bc17SDimitris Michailidis
329db37bc17SDimitris Michailidis tot_len -= frag_len;
330db37bc17SDimitris Michailidis if (!tot_len)
331db37bc17SDimitris Michailidis break;
332db37bc17SDimitris Michailidis
333db37bc17SDimitris Michailidis q->buf_offset = PAGE_SIZE;
334db37bc17SDimitris Michailidis }
335db37bc17SDimitris Michailidis q->buf_offset = ALIGN(q->buf_offset + frag_len, FUN_EPRQ_PKT_ALIGN);
336db37bc17SDimitris Michailidis q->cur_buf = buf;
337db37bc17SDimitris Michailidis return ref_ok;
338db37bc17SDimitris Michailidis }
339db37bc17SDimitris Michailidis
rx_hwtstamp_enabled(const struct net_device * dev)340db37bc17SDimitris Michailidis static bool rx_hwtstamp_enabled(const struct net_device *dev)
341db37bc17SDimitris Michailidis {
342db37bc17SDimitris Michailidis const struct funeth_priv *d = netdev_priv(dev);
343db37bc17SDimitris Michailidis
344db37bc17SDimitris Michailidis return d->hwtstamp_cfg.rx_filter == HWTSTAMP_FILTER_ALL;
345db37bc17SDimitris Michailidis }
346db37bc17SDimitris Michailidis
347db37bc17SDimitris Michailidis /* Advance the CQ pointers and phase tag to the next CQE. */
advance_cq(struct funeth_rxq * q)348db37bc17SDimitris Michailidis static void advance_cq(struct funeth_rxq *q)
349db37bc17SDimitris Michailidis {
350db37bc17SDimitris Michailidis if (unlikely(q->cq_head == q->cq_mask)) {
351db37bc17SDimitris Michailidis q->cq_head = 0;
352db37bc17SDimitris Michailidis q->phase ^= 1;
353db37bc17SDimitris Michailidis q->next_cqe_info = cqe_to_info(q->cqes);
354db37bc17SDimitris Michailidis } else {
355db37bc17SDimitris Michailidis q->cq_head++;
356db37bc17SDimitris Michailidis q->next_cqe_info += FUNETH_CQE_SIZE;
357db37bc17SDimitris Michailidis }
358db37bc17SDimitris Michailidis prefetch(q->next_cqe_info);
359db37bc17SDimitris Michailidis }
360db37bc17SDimitris Michailidis
361db37bc17SDimitris Michailidis /* Process the packet represented by the head CQE of @q. Gather the packet's
362db37bc17SDimitris Michailidis * fragments, run it through the optional XDP program, and if needed construct
363db37bc17SDimitris Michailidis * an skb and pass it to the stack.
364db37bc17SDimitris Michailidis */
fun_handle_cqe_pkt(struct funeth_rxq * q,struct funeth_txq * xdp_q)365db37bc17SDimitris Michailidis static void fun_handle_cqe_pkt(struct funeth_rxq *q, struct funeth_txq *xdp_q)
366db37bc17SDimitris Michailidis {
367db37bc17SDimitris Michailidis const struct fun_eth_cqe *rxreq = info_to_cqe(q->next_cqe_info);
368db37bc17SDimitris Michailidis unsigned int i, tot_len, pkt_len = be32_to_cpu(rxreq->pkt_len);
369db37bc17SDimitris Michailidis struct net_device *ndev = q->netdev;
370db37bc17SDimitris Michailidis skb_frag_t frags[RX_MAX_FRAGS];
371db37bc17SDimitris Michailidis struct skb_shared_info *si;
372db37bc17SDimitris Michailidis unsigned int headroom;
373db37bc17SDimitris Michailidis gro_result_t gro_res;
374db37bc17SDimitris Michailidis struct sk_buff *skb;
375db37bc17SDimitris Michailidis int ref_ok;
376db37bc17SDimitris Michailidis void *va;
377db37bc17SDimitris Michailidis u16 cv;
378db37bc17SDimitris Michailidis
379db37bc17SDimitris Michailidis u64_stats_update_begin(&q->syncp);
380db37bc17SDimitris Michailidis q->stats.rx_pkts++;
381db37bc17SDimitris Michailidis q->stats.rx_bytes += pkt_len;
382db37bc17SDimitris Michailidis u64_stats_update_end(&q->syncp);
383db37bc17SDimitris Michailidis
384db37bc17SDimitris Michailidis advance_cq(q);
385db37bc17SDimitris Michailidis
386db37bc17SDimitris Michailidis /* account for head- and tail-room, present only for 1-buffer packets */
387db37bc17SDimitris Michailidis tot_len = pkt_len;
388db37bc17SDimitris Michailidis headroom = be16_to_cpu(rxreq->headroom);
389db37bc17SDimitris Michailidis if (likely(headroom))
390db37bc17SDimitris Michailidis tot_len += FUN_RX_TAILROOM + headroom;
391db37bc17SDimitris Michailidis
392db37bc17SDimitris Michailidis ref_ok = fun_gather_pkt(q, tot_len, frags);
393db37bc17SDimitris Michailidis va = skb_frag_address(frags);
394db37bc17SDimitris Michailidis if (xdp_q && headroom == FUN_XDP_HEADROOM) {
395db37bc17SDimitris Michailidis va = fun_run_xdp(q, frags, va, ref_ok, xdp_q);
396db37bc17SDimitris Michailidis if (!va)
397db37bc17SDimitris Michailidis return;
398db37bc17SDimitris Michailidis headroom = 0; /* XDP_PASS trims it */
399db37bc17SDimitris Michailidis }
400db37bc17SDimitris Michailidis if (unlikely(!ref_ok))
401db37bc17SDimitris Michailidis goto no_mem;
402db37bc17SDimitris Michailidis
403db37bc17SDimitris Michailidis if (likely(headroom)) {
404db37bc17SDimitris Michailidis /* headroom is either FUN_RX_HEADROOM or FUN_XDP_HEADROOM */
405db37bc17SDimitris Michailidis prefetch(va + headroom);
406db37bc17SDimitris Michailidis skb = napi_build_skb(va, ALIGN(tot_len, FUN_EPRQ_PKT_ALIGN));
407db37bc17SDimitris Michailidis if (unlikely(!skb))
408db37bc17SDimitris Michailidis goto no_mem;
409db37bc17SDimitris Michailidis
410db37bc17SDimitris Michailidis skb_reserve(skb, headroom);
411db37bc17SDimitris Michailidis __skb_put(skb, pkt_len);
412db37bc17SDimitris Michailidis skb->protocol = eth_type_trans(skb, ndev);
413db37bc17SDimitris Michailidis } else {
414db37bc17SDimitris Michailidis prefetch(va);
415db37bc17SDimitris Michailidis skb = napi_get_frags(q->napi);
416db37bc17SDimitris Michailidis if (unlikely(!skb))
417db37bc17SDimitris Michailidis goto no_mem;
418db37bc17SDimitris Michailidis
419db37bc17SDimitris Michailidis if (ref_ok < 0)
420db37bc17SDimitris Michailidis skb->pfmemalloc = 1;
421db37bc17SDimitris Michailidis
422db37bc17SDimitris Michailidis si = skb_shinfo(skb);
423db37bc17SDimitris Michailidis si->nr_frags = rxreq->nsgl;
424db37bc17SDimitris Michailidis for (i = 0; i < si->nr_frags; i++)
425db37bc17SDimitris Michailidis si->frags[i] = frags[i];
426db37bc17SDimitris Michailidis
427db37bc17SDimitris Michailidis skb->len = pkt_len;
428db37bc17SDimitris Michailidis skb->data_len = pkt_len;
429db37bc17SDimitris Michailidis skb->truesize += round_up(pkt_len, FUN_EPRQ_PKT_ALIGN);
430db37bc17SDimitris Michailidis }
431db37bc17SDimitris Michailidis
432db37bc17SDimitris Michailidis skb_record_rx_queue(skb, q->qidx);
433db37bc17SDimitris Michailidis cv = be16_to_cpu(rxreq->pkt_cv);
434db37bc17SDimitris Michailidis if (likely((q->netdev->features & NETIF_F_RXHASH) && rxreq->hash))
435db37bc17SDimitris Michailidis skb_set_hash(skb, be32_to_cpu(rxreq->hash),
436db37bc17SDimitris Michailidis cqe_to_pkt_hash_type(cv));
437db37bc17SDimitris Michailidis if (likely((q->netdev->features & NETIF_F_RXCSUM) && rxreq->csum)) {
438db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, rx_cso);
439db37bc17SDimitris Michailidis skb->ip_summed = CHECKSUM_UNNECESSARY;
440db37bc17SDimitris Michailidis skb->csum_level = be16_to_cpu(rxreq->csum) - 1;
441db37bc17SDimitris Michailidis }
442db37bc17SDimitris Michailidis if (unlikely(rx_hwtstamp_enabled(q->netdev)))
443db37bc17SDimitris Michailidis skb_hwtstamps(skb)->hwtstamp = be64_to_cpu(rxreq->timestamp);
444db37bc17SDimitris Michailidis
445db37bc17SDimitris Michailidis trace_funeth_rx(q, rxreq->nsgl, pkt_len, skb->hash, cv);
446db37bc17SDimitris Michailidis
447db37bc17SDimitris Michailidis gro_res = skb->data_len ? napi_gro_frags(q->napi) :
448db37bc17SDimitris Michailidis napi_gro_receive(q->napi, skb);
449db37bc17SDimitris Michailidis if (gro_res == GRO_MERGED || gro_res == GRO_MERGED_FREE)
450db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, gro_merged);
451db37bc17SDimitris Michailidis else if (gro_res == GRO_HELD)
452db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, gro_pkts);
453db37bc17SDimitris Michailidis return;
454db37bc17SDimitris Michailidis
455db37bc17SDimitris Michailidis no_mem:
456db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, rx_mem_drops);
457db37bc17SDimitris Michailidis
458db37bc17SDimitris Michailidis /* Release the references we've been granted for the frag pages.
459db37bc17SDimitris Michailidis * We return the ref of the last frag and free the rest.
460db37bc17SDimitris Michailidis */
461db37bc17SDimitris Michailidis q->cur_buf->pg_refs++;
462db37bc17SDimitris Michailidis for (i = 0; i < rxreq->nsgl - 1; i++)
463db37bc17SDimitris Michailidis __free_page(skb_frag_page(frags + i));
464db37bc17SDimitris Michailidis }
465db37bc17SDimitris Michailidis
466db37bc17SDimitris Michailidis /* Return 0 if the phase tag of the CQE at the CQ's head matches expectations
467db37bc17SDimitris Michailidis * indicating the CQE is new.
468db37bc17SDimitris Michailidis */
cqe_phase_mismatch(const struct fun_cqe_info * ci,u16 phase)469db37bc17SDimitris Michailidis static u16 cqe_phase_mismatch(const struct fun_cqe_info *ci, u16 phase)
470db37bc17SDimitris Michailidis {
471db37bc17SDimitris Michailidis u16 sf_p = be16_to_cpu(ci->sf_p);
472db37bc17SDimitris Michailidis
473db37bc17SDimitris Michailidis return (sf_p & 1) ^ phase;
474db37bc17SDimitris Michailidis }
475db37bc17SDimitris Michailidis
476db37bc17SDimitris Michailidis /* Walk through a CQ identifying and processing fresh CQEs up to the given
477db37bc17SDimitris Michailidis * budget. Return the remaining budget.
478db37bc17SDimitris Michailidis */
fun_process_cqes(struct funeth_rxq * q,int budget)479db37bc17SDimitris Michailidis static int fun_process_cqes(struct funeth_rxq *q, int budget)
480db37bc17SDimitris Michailidis {
481db37bc17SDimitris Michailidis struct funeth_priv *fp = netdev_priv(q->netdev);
482db37bc17SDimitris Michailidis struct funeth_txq **xdpqs, *xdp_q = NULL;
483db37bc17SDimitris Michailidis
484db37bc17SDimitris Michailidis xdpqs = rcu_dereference_bh(fp->xdpqs);
485db37bc17SDimitris Michailidis if (xdpqs)
486db37bc17SDimitris Michailidis xdp_q = xdpqs[smp_processor_id()];
487db37bc17SDimitris Michailidis
488db37bc17SDimitris Michailidis while (budget && !cqe_phase_mismatch(q->next_cqe_info, q->phase)) {
489db37bc17SDimitris Michailidis /* access other descriptor fields after the phase check */
490db37bc17SDimitris Michailidis dma_rmb();
491db37bc17SDimitris Michailidis
492db37bc17SDimitris Michailidis fun_handle_cqe_pkt(q, xdp_q);
493db37bc17SDimitris Michailidis budget--;
494db37bc17SDimitris Michailidis }
495db37bc17SDimitris Michailidis
496db37bc17SDimitris Michailidis if (unlikely(q->xdp_flush)) {
497db37bc17SDimitris Michailidis if (q->xdp_flush & FUN_XDP_FLUSH_TX)
498db37bc17SDimitris Michailidis fun_txq_wr_db(xdp_q);
499db37bc17SDimitris Michailidis if (q->xdp_flush & FUN_XDP_FLUSH_REDIR)
500db37bc17SDimitris Michailidis xdp_do_flush();
501db37bc17SDimitris Michailidis q->xdp_flush = 0;
502db37bc17SDimitris Michailidis }
503db37bc17SDimitris Michailidis
504db37bc17SDimitris Michailidis return budget;
505db37bc17SDimitris Michailidis }
506db37bc17SDimitris Michailidis
507db37bc17SDimitris Michailidis /* NAPI handler for Rx queues. Calls the CQE processing loop and writes RQ/CQ
508db37bc17SDimitris Michailidis * doorbells as needed.
509db37bc17SDimitris Michailidis */
fun_rxq_napi_poll(struct napi_struct * napi,int budget)510db37bc17SDimitris Michailidis int fun_rxq_napi_poll(struct napi_struct *napi, int budget)
511db37bc17SDimitris Michailidis {
512db37bc17SDimitris Michailidis struct fun_irq *irq = container_of(napi, struct fun_irq, napi);
513db37bc17SDimitris Michailidis struct funeth_rxq *q = irq->rxq;
514db37bc17SDimitris Michailidis int work_done = budget - fun_process_cqes(q, budget);
515db37bc17SDimitris Michailidis u32 cq_db_val = q->cq_head;
516db37bc17SDimitris Michailidis
517db37bc17SDimitris Michailidis if (unlikely(work_done >= budget))
518db37bc17SDimitris Michailidis FUN_QSTAT_INC(q, rx_budget);
519db37bc17SDimitris Michailidis else if (napi_complete_done(napi, work_done))
520db37bc17SDimitris Michailidis cq_db_val |= q->irq_db_val;
521db37bc17SDimitris Michailidis
522db37bc17SDimitris Michailidis /* check whether to post new Rx buffers */
523db37bc17SDimitris Michailidis if (q->rq_cons - q->rq_cons_db >= q->rq_db_thres) {
524db37bc17SDimitris Michailidis u64_stats_update_begin(&q->syncp);
525db37bc17SDimitris Michailidis q->stats.rx_bufs += q->rq_cons - q->rq_cons_db;
526db37bc17SDimitris Michailidis u64_stats_update_end(&q->syncp);
527db37bc17SDimitris Michailidis q->rq_cons_db = q->rq_cons;
528db37bc17SDimitris Michailidis writel((q->rq_cons - 1) & q->rq_mask, q->rq_db);
529db37bc17SDimitris Michailidis }
530db37bc17SDimitris Michailidis
531db37bc17SDimitris Michailidis writel(cq_db_val, q->cq_db);
532db37bc17SDimitris Michailidis return work_done;
533db37bc17SDimitris Michailidis }
534db37bc17SDimitris Michailidis
535db37bc17SDimitris Michailidis /* Free the Rx buffers of an Rx queue. */
fun_rxq_free_bufs(struct funeth_rxq * q)536db37bc17SDimitris Michailidis static void fun_rxq_free_bufs(struct funeth_rxq *q)
537db37bc17SDimitris Michailidis {
538db37bc17SDimitris Michailidis struct funeth_rxbuf *b = q->bufs;
539db37bc17SDimitris Michailidis unsigned int i;
540db37bc17SDimitris Michailidis
541db37bc17SDimitris Michailidis for (i = 0; i <= q->rq_mask; i++, b++)
542db37bc17SDimitris Michailidis funeth_free_page(q, b);
543db37bc17SDimitris Michailidis
544db37bc17SDimitris Michailidis funeth_free_page(q, &q->spare_buf);
545db37bc17SDimitris Michailidis q->cur_buf = NULL;
546db37bc17SDimitris Michailidis }
547db37bc17SDimitris Michailidis
548db37bc17SDimitris Michailidis /* Initially provision an Rx queue with Rx buffers. */
fun_rxq_alloc_bufs(struct funeth_rxq * q,int node)549db37bc17SDimitris Michailidis static int fun_rxq_alloc_bufs(struct funeth_rxq *q, int node)
550db37bc17SDimitris Michailidis {
551db37bc17SDimitris Michailidis struct funeth_rxbuf *b = q->bufs;
552db37bc17SDimitris Michailidis unsigned int i;
553db37bc17SDimitris Michailidis
554db37bc17SDimitris Michailidis for (i = 0; i <= q->rq_mask; i++, b++) {
555db37bc17SDimitris Michailidis if (funeth_alloc_page(q, b, node, GFP_KERNEL)) {
556db37bc17SDimitris Michailidis fun_rxq_free_bufs(q);
557db37bc17SDimitris Michailidis return -ENOMEM;
558db37bc17SDimitris Michailidis }
559db37bc17SDimitris Michailidis q->rqes[i] = FUN_EPRQ_RQBUF_INIT(b->dma_addr);
560db37bc17SDimitris Michailidis }
561db37bc17SDimitris Michailidis q->cur_buf = q->bufs;
562db37bc17SDimitris Michailidis return 0;
563db37bc17SDimitris Michailidis }
564db37bc17SDimitris Michailidis
565db37bc17SDimitris Michailidis /* Initialize a used-buffer cache of the given depth. */
fun_rxq_init_cache(struct funeth_rx_cache * c,unsigned int depth,int node)566db37bc17SDimitris Michailidis static int fun_rxq_init_cache(struct funeth_rx_cache *c, unsigned int depth,
567db37bc17SDimitris Michailidis int node)
568db37bc17SDimitris Michailidis {
569db37bc17SDimitris Michailidis c->mask = depth - 1;
570db37bc17SDimitris Michailidis c->bufs = kvzalloc_node(depth * sizeof(*c->bufs), GFP_KERNEL, node);
571db37bc17SDimitris Michailidis return c->bufs ? 0 : -ENOMEM;
572db37bc17SDimitris Michailidis }
573db37bc17SDimitris Michailidis
574db37bc17SDimitris Michailidis /* Deallocate an Rx queue's used-buffer cache and its contents. */
fun_rxq_free_cache(struct funeth_rxq * q)575db37bc17SDimitris Michailidis static void fun_rxq_free_cache(struct funeth_rxq *q)
576db37bc17SDimitris Michailidis {
577db37bc17SDimitris Michailidis struct funeth_rxbuf *b = q->cache.bufs;
578db37bc17SDimitris Michailidis unsigned int i;
579db37bc17SDimitris Michailidis
580db37bc17SDimitris Michailidis for (i = 0; i <= q->cache.mask; i++, b++)
581db37bc17SDimitris Michailidis funeth_free_page(q, b);
582db37bc17SDimitris Michailidis
583db37bc17SDimitris Michailidis kvfree(q->cache.bufs);
584db37bc17SDimitris Michailidis q->cache.bufs = NULL;
585db37bc17SDimitris Michailidis }
586db37bc17SDimitris Michailidis
fun_rxq_set_bpf(struct funeth_rxq * q,struct bpf_prog * prog)587db37bc17SDimitris Michailidis int fun_rxq_set_bpf(struct funeth_rxq *q, struct bpf_prog *prog)
588db37bc17SDimitris Michailidis {
589db37bc17SDimitris Michailidis struct funeth_priv *fp = netdev_priv(q->netdev);
590db37bc17SDimitris Michailidis struct fun_admin_epcq_req cmd;
591db37bc17SDimitris Michailidis u16 headroom;
592db37bc17SDimitris Michailidis int err;
593db37bc17SDimitris Michailidis
594db37bc17SDimitris Michailidis headroom = prog ? FUN_XDP_HEADROOM : FUN_RX_HEADROOM;
595db37bc17SDimitris Michailidis if (headroom != q->headroom) {
596db37bc17SDimitris Michailidis cmd.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPCQ,
597db37bc17SDimitris Michailidis sizeof(cmd));
598db37bc17SDimitris Michailidis cmd.u.modify =
599db37bc17SDimitris Michailidis FUN_ADMIN_EPCQ_MODIFY_REQ_INIT(FUN_ADMIN_SUBOP_MODIFY,
600db37bc17SDimitris Michailidis 0, q->hw_cqid, headroom);
601db37bc17SDimitris Michailidis err = fun_submit_admin_sync_cmd(fp->fdev, &cmd.common, NULL, 0,
602db37bc17SDimitris Michailidis 0);
603db37bc17SDimitris Michailidis if (err)
604db37bc17SDimitris Michailidis return err;
605db37bc17SDimitris Michailidis q->headroom = headroom;
606db37bc17SDimitris Michailidis }
607db37bc17SDimitris Michailidis
608db37bc17SDimitris Michailidis WRITE_ONCE(q->xdp_prog, prog);
609db37bc17SDimitris Michailidis return 0;
610db37bc17SDimitris Michailidis }
611db37bc17SDimitris Michailidis
612db37bc17SDimitris Michailidis /* Create an Rx queue, allocating the host memory it needs. */
fun_rxq_create_sw(struct net_device * dev,unsigned int qidx,unsigned int ncqe,unsigned int nrqe,struct fun_irq * irq)613db37bc17SDimitris Michailidis static struct funeth_rxq *fun_rxq_create_sw(struct net_device *dev,
614db37bc17SDimitris Michailidis unsigned int qidx,
615db37bc17SDimitris Michailidis unsigned int ncqe,
616db37bc17SDimitris Michailidis unsigned int nrqe,
617db37bc17SDimitris Michailidis struct fun_irq *irq)
618db37bc17SDimitris Michailidis {
619db37bc17SDimitris Michailidis struct funeth_priv *fp = netdev_priv(dev);
620db37bc17SDimitris Michailidis struct funeth_rxq *q;
621db37bc17SDimitris Michailidis int err = -ENOMEM;
622db37bc17SDimitris Michailidis int numa_node;
623db37bc17SDimitris Michailidis
624db37bc17SDimitris Michailidis numa_node = fun_irq_node(irq);
625db37bc17SDimitris Michailidis q = kzalloc_node(sizeof(*q), GFP_KERNEL, numa_node);
626db37bc17SDimitris Michailidis if (!q)
627db37bc17SDimitris Michailidis goto err;
628db37bc17SDimitris Michailidis
629db37bc17SDimitris Michailidis q->qidx = qidx;
630db37bc17SDimitris Michailidis q->netdev = dev;
631db37bc17SDimitris Michailidis q->cq_mask = ncqe - 1;
632db37bc17SDimitris Michailidis q->rq_mask = nrqe - 1;
633db37bc17SDimitris Michailidis q->numa_node = numa_node;
634db37bc17SDimitris Michailidis q->rq_db_thres = nrqe / 4;
635db37bc17SDimitris Michailidis u64_stats_init(&q->syncp);
636db37bc17SDimitris Michailidis q->dma_dev = &fp->pdev->dev;
637db37bc17SDimitris Michailidis
638db37bc17SDimitris Michailidis q->rqes = fun_alloc_ring_mem(q->dma_dev, nrqe, sizeof(*q->rqes),
639db37bc17SDimitris Michailidis sizeof(*q->bufs), false, numa_node,
640db37bc17SDimitris Michailidis &q->rq_dma_addr, (void **)&q->bufs, NULL);
641db37bc17SDimitris Michailidis if (!q->rqes)
642db37bc17SDimitris Michailidis goto free_q;
643db37bc17SDimitris Michailidis
644db37bc17SDimitris Michailidis q->cqes = fun_alloc_ring_mem(q->dma_dev, ncqe, FUNETH_CQE_SIZE, 0,
645db37bc17SDimitris Michailidis false, numa_node, &q->cq_dma_addr, NULL,
646db37bc17SDimitris Michailidis NULL);
647db37bc17SDimitris Michailidis if (!q->cqes)
648db37bc17SDimitris Michailidis goto free_rqes;
649db37bc17SDimitris Michailidis
650db37bc17SDimitris Michailidis err = fun_rxq_init_cache(&q->cache, nrqe, numa_node);
651db37bc17SDimitris Michailidis if (err)
652db37bc17SDimitris Michailidis goto free_cqes;
653db37bc17SDimitris Michailidis
654db37bc17SDimitris Michailidis err = fun_rxq_alloc_bufs(q, numa_node);
655db37bc17SDimitris Michailidis if (err)
656db37bc17SDimitris Michailidis goto free_cache;
657db37bc17SDimitris Michailidis
658db37bc17SDimitris Michailidis q->stats.rx_bufs = q->rq_mask;
659db37bc17SDimitris Michailidis q->init_state = FUN_QSTATE_INIT_SW;
660db37bc17SDimitris Michailidis return q;
661db37bc17SDimitris Michailidis
662db37bc17SDimitris Michailidis free_cache:
663db37bc17SDimitris Michailidis fun_rxq_free_cache(q);
664db37bc17SDimitris Michailidis free_cqes:
665db37bc17SDimitris Michailidis dma_free_coherent(q->dma_dev, ncqe * FUNETH_CQE_SIZE, q->cqes,
666db37bc17SDimitris Michailidis q->cq_dma_addr);
667db37bc17SDimitris Michailidis free_rqes:
668db37bc17SDimitris Michailidis fun_free_ring_mem(q->dma_dev, nrqe, sizeof(*q->rqes), false, q->rqes,
669db37bc17SDimitris Michailidis q->rq_dma_addr, q->bufs);
670db37bc17SDimitris Michailidis free_q:
671db37bc17SDimitris Michailidis kfree(q);
672db37bc17SDimitris Michailidis err:
673db37bc17SDimitris Michailidis netdev_err(dev, "Unable to allocate memory for Rx queue %u\n", qidx);
674db37bc17SDimitris Michailidis return ERR_PTR(err);
675db37bc17SDimitris Michailidis }
676db37bc17SDimitris Michailidis
fun_rxq_free_sw(struct funeth_rxq * q)677db37bc17SDimitris Michailidis static void fun_rxq_free_sw(struct funeth_rxq *q)
678db37bc17SDimitris Michailidis {
679db37bc17SDimitris Michailidis struct funeth_priv *fp = netdev_priv(q->netdev);
680db37bc17SDimitris Michailidis
681db37bc17SDimitris Michailidis fun_rxq_free_cache(q);
682db37bc17SDimitris Michailidis fun_rxq_free_bufs(q);
683db37bc17SDimitris Michailidis fun_free_ring_mem(q->dma_dev, q->rq_mask + 1, sizeof(*q->rqes), false,
684db37bc17SDimitris Michailidis q->rqes, q->rq_dma_addr, q->bufs);
685db37bc17SDimitris Michailidis dma_free_coherent(q->dma_dev, (q->cq_mask + 1) * FUNETH_CQE_SIZE,
686db37bc17SDimitris Michailidis q->cqes, q->cq_dma_addr);
687db37bc17SDimitris Michailidis
688db37bc17SDimitris Michailidis /* Before freeing the queue transfer key counters to the device. */
689db37bc17SDimitris Michailidis fp->rx_packets += q->stats.rx_pkts;
690db37bc17SDimitris Michailidis fp->rx_bytes += q->stats.rx_bytes;
691db37bc17SDimitris Michailidis fp->rx_dropped += q->stats.rx_map_err + q->stats.rx_mem_drops;
692db37bc17SDimitris Michailidis
693db37bc17SDimitris Michailidis kfree(q);
694db37bc17SDimitris Michailidis }
695db37bc17SDimitris Michailidis
696db37bc17SDimitris Michailidis /* Create an Rx queue's resources on the device. */
fun_rxq_create_dev(struct funeth_rxq * q,struct fun_irq * irq)697db37bc17SDimitris Michailidis int fun_rxq_create_dev(struct funeth_rxq *q, struct fun_irq *irq)
698db37bc17SDimitris Michailidis {
699db37bc17SDimitris Michailidis struct funeth_priv *fp = netdev_priv(q->netdev);
700db37bc17SDimitris Michailidis unsigned int ncqe = q->cq_mask + 1;
701db37bc17SDimitris Michailidis unsigned int nrqe = q->rq_mask + 1;
702db37bc17SDimitris Michailidis int err;
703db37bc17SDimitris Michailidis
704db37bc17SDimitris Michailidis err = xdp_rxq_info_reg(&q->xdp_rxq, q->netdev, q->qidx,
705db37bc17SDimitris Michailidis irq->napi.napi_id);
706db37bc17SDimitris Michailidis if (err)
707db37bc17SDimitris Michailidis goto out;
708db37bc17SDimitris Michailidis
709db37bc17SDimitris Michailidis err = xdp_rxq_info_reg_mem_model(&q->xdp_rxq, MEM_TYPE_PAGE_SHARED,
710db37bc17SDimitris Michailidis NULL);
711db37bc17SDimitris Michailidis if (err)
712db37bc17SDimitris Michailidis goto xdp_unreg;
713db37bc17SDimitris Michailidis
714db37bc17SDimitris Michailidis q->phase = 1;
715db37bc17SDimitris Michailidis q->irq_cnt = 0;
716db37bc17SDimitris Michailidis q->cq_head = 0;
717db37bc17SDimitris Michailidis q->rq_cons = 0;
718db37bc17SDimitris Michailidis q->rq_cons_db = 0;
719db37bc17SDimitris Michailidis q->buf_offset = 0;
720db37bc17SDimitris Michailidis q->napi = &irq->napi;
721db37bc17SDimitris Michailidis q->irq_db_val = fp->cq_irq_db;
722db37bc17SDimitris Michailidis q->next_cqe_info = cqe_to_info(q->cqes);
723db37bc17SDimitris Michailidis
724db37bc17SDimitris Michailidis q->xdp_prog = fp->xdp_prog;
725db37bc17SDimitris Michailidis q->headroom = fp->xdp_prog ? FUN_XDP_HEADROOM : FUN_RX_HEADROOM;
726db37bc17SDimitris Michailidis
727db37bc17SDimitris Michailidis err = fun_sq_create(fp->fdev, FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR |
728db37bc17SDimitris Michailidis FUN_ADMIN_EPSQ_CREATE_FLAG_RQ, 0,
729db37bc17SDimitris Michailidis FUN_HCI_ID_INVALID, 0, nrqe, q->rq_dma_addr, 0, 0,
730db37bc17SDimitris Michailidis 0, 0, fp->fdev->kern_end_qid, PAGE_SHIFT,
731db37bc17SDimitris Michailidis &q->hw_sqid, &q->rq_db);
732db37bc17SDimitris Michailidis if (err)
733db37bc17SDimitris Michailidis goto xdp_unreg;
734db37bc17SDimitris Michailidis
735db37bc17SDimitris Michailidis err = fun_cq_create(fp->fdev, FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR |
736db37bc17SDimitris Michailidis FUN_ADMIN_EPCQ_CREATE_FLAG_RQ, 0,
737db37bc17SDimitris Michailidis q->hw_sqid, ilog2(FUNETH_CQE_SIZE), ncqe,
738db37bc17SDimitris Michailidis q->cq_dma_addr, q->headroom, FUN_RX_TAILROOM, 0, 0,
739db37bc17SDimitris Michailidis irq->irq_idx, 0, fp->fdev->kern_end_qid,
740db37bc17SDimitris Michailidis &q->hw_cqid, &q->cq_db);
741db37bc17SDimitris Michailidis if (err)
742db37bc17SDimitris Michailidis goto free_rq;
743db37bc17SDimitris Michailidis
744db37bc17SDimitris Michailidis irq->rxq = q;
745db37bc17SDimitris Michailidis writel(q->rq_mask, q->rq_db);
746db37bc17SDimitris Michailidis q->init_state = FUN_QSTATE_INIT_FULL;
747db37bc17SDimitris Michailidis
748db37bc17SDimitris Michailidis netif_info(fp, ifup, q->netdev,
749db37bc17SDimitris Michailidis "Rx queue %u, depth %u/%u, HW qid %u/%u, IRQ idx %u, node %d, headroom %u\n",
750db37bc17SDimitris Michailidis q->qidx, ncqe, nrqe, q->hw_cqid, q->hw_sqid, irq->irq_idx,
751db37bc17SDimitris Michailidis q->numa_node, q->headroom);
752db37bc17SDimitris Michailidis return 0;
753db37bc17SDimitris Michailidis
754db37bc17SDimitris Michailidis free_rq:
755db37bc17SDimitris Michailidis fun_destroy_sq(fp->fdev, q->hw_sqid);
756db37bc17SDimitris Michailidis xdp_unreg:
757db37bc17SDimitris Michailidis xdp_rxq_info_unreg(&q->xdp_rxq);
758db37bc17SDimitris Michailidis out:
759db37bc17SDimitris Michailidis netdev_err(q->netdev,
760db37bc17SDimitris Michailidis "Failed to create Rx queue %u on device, error %d\n",
761db37bc17SDimitris Michailidis q->qidx, err);
762db37bc17SDimitris Michailidis return err;
763db37bc17SDimitris Michailidis }
764db37bc17SDimitris Michailidis
fun_rxq_free_dev(struct funeth_rxq * q)765db37bc17SDimitris Michailidis static void fun_rxq_free_dev(struct funeth_rxq *q)
766db37bc17SDimitris Michailidis {
767db37bc17SDimitris Michailidis struct funeth_priv *fp = netdev_priv(q->netdev);
768db37bc17SDimitris Michailidis struct fun_irq *irq;
769db37bc17SDimitris Michailidis
770db37bc17SDimitris Michailidis if (q->init_state < FUN_QSTATE_INIT_FULL)
771db37bc17SDimitris Michailidis return;
772db37bc17SDimitris Michailidis
773db37bc17SDimitris Michailidis irq = container_of(q->napi, struct fun_irq, napi);
774db37bc17SDimitris Michailidis netif_info(fp, ifdown, q->netdev,
775db37bc17SDimitris Michailidis "Freeing Rx queue %u (id %u/%u), IRQ %u\n",
776db37bc17SDimitris Michailidis q->qidx, q->hw_cqid, q->hw_sqid, irq->irq_idx);
777db37bc17SDimitris Michailidis
778db37bc17SDimitris Michailidis irq->rxq = NULL;
779db37bc17SDimitris Michailidis xdp_rxq_info_unreg(&q->xdp_rxq);
780db37bc17SDimitris Michailidis fun_destroy_sq(fp->fdev, q->hw_sqid);
781db37bc17SDimitris Michailidis fun_destroy_cq(fp->fdev, q->hw_cqid);
782db37bc17SDimitris Michailidis q->init_state = FUN_QSTATE_INIT_SW;
783db37bc17SDimitris Michailidis }
784db37bc17SDimitris Michailidis
785db37bc17SDimitris Michailidis /* Create or advance an Rx queue, allocating all the host and device resources
786db37bc17SDimitris Michailidis * needed to reach the target state.
787db37bc17SDimitris Michailidis */
funeth_rxq_create(struct net_device * dev,unsigned int qidx,unsigned int ncqe,unsigned int nrqe,struct fun_irq * irq,int state,struct funeth_rxq ** qp)788db37bc17SDimitris Michailidis int funeth_rxq_create(struct net_device *dev, unsigned int qidx,
789db37bc17SDimitris Michailidis unsigned int ncqe, unsigned int nrqe, struct fun_irq *irq,
790db37bc17SDimitris Michailidis int state, struct funeth_rxq **qp)
791db37bc17SDimitris Michailidis {
792db37bc17SDimitris Michailidis struct funeth_rxq *q = *qp;
793db37bc17SDimitris Michailidis int err;
794db37bc17SDimitris Michailidis
795db37bc17SDimitris Michailidis if (!q) {
796db37bc17SDimitris Michailidis q = fun_rxq_create_sw(dev, qidx, ncqe, nrqe, irq);
797db37bc17SDimitris Michailidis if (IS_ERR(q))
798db37bc17SDimitris Michailidis return PTR_ERR(q);
799db37bc17SDimitris Michailidis }
800db37bc17SDimitris Michailidis
801db37bc17SDimitris Michailidis if (q->init_state >= state)
802db37bc17SDimitris Michailidis goto out;
803db37bc17SDimitris Michailidis
804db37bc17SDimitris Michailidis err = fun_rxq_create_dev(q, irq);
805db37bc17SDimitris Michailidis if (err) {
806db37bc17SDimitris Michailidis if (!*qp)
807db37bc17SDimitris Michailidis fun_rxq_free_sw(q);
808db37bc17SDimitris Michailidis return err;
809db37bc17SDimitris Michailidis }
810db37bc17SDimitris Michailidis
811db37bc17SDimitris Michailidis out:
812db37bc17SDimitris Michailidis *qp = q;
813db37bc17SDimitris Michailidis return 0;
814db37bc17SDimitris Michailidis }
815db37bc17SDimitris Michailidis
816db37bc17SDimitris Michailidis /* Free Rx queue resources until it reaches the target state. */
funeth_rxq_free(struct funeth_rxq * q,int state)817db37bc17SDimitris Michailidis struct funeth_rxq *funeth_rxq_free(struct funeth_rxq *q, int state)
818db37bc17SDimitris Michailidis {
819db37bc17SDimitris Michailidis if (state < FUN_QSTATE_INIT_FULL)
820db37bc17SDimitris Michailidis fun_rxq_free_dev(q);
821db37bc17SDimitris Michailidis
822db37bc17SDimitris Michailidis if (state == FUN_QSTATE_DESTROYED) {
823db37bc17SDimitris Michailidis fun_rxq_free_sw(q);
824db37bc17SDimitris Michailidis q = NULL;
825db37bc17SDimitris Michailidis }
826db37bc17SDimitris Michailidis
827db37bc17SDimitris Michailidis return q;
828db37bc17SDimitris Michailidis }
829