xref: /linux/drivers/net/ethernet/fungible/funeth/funeth_rx.c (revision cdd5b5a9761fd66d17586e4f4ba6588c70e640ea)
1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2 
3 #include <linux/bpf_trace.h>
4 #include <linux/dma-mapping.h>
5 #include <linux/etherdevice.h>
6 #include <linux/filter.h>
7 #include <linux/irq.h>
8 #include <linux/pci.h>
9 #include <linux/skbuff.h>
10 #include "funeth_txrx.h"
11 #include "funeth.h"
12 #include "fun_queue.h"
13 
14 #define CREATE_TRACE_POINTS
15 #include "funeth_trace.h"
16 
17 /* Given the device's max supported MTU and pages of at least 4KB a packet can
18  * be scattered into at most 4 buffers.
19  */
20 #define RX_MAX_FRAGS 4
21 
22 /* Per packet headroom in non-XDP mode. Present only for 1-frag packets. */
23 #define FUN_RX_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
24 
25 /* We try to reuse pages for our buffers. To avoid frequent page ref writes we
26  * take EXTRA_PAGE_REFS references at once and then hand them out one per packet
27  * occupying the buffer.
28  */
29 #define EXTRA_PAGE_REFS 1000000
30 #define MIN_PAGE_REFS 1000
31 
32 enum {
33 	FUN_XDP_FLUSH_REDIR = 1,
34 	FUN_XDP_FLUSH_TX = 2,
35 };
36 
37 /* See if a page is running low on refs we are holding and if so take more. */
refresh_refs(struct funeth_rxbuf * buf)38 static void refresh_refs(struct funeth_rxbuf *buf)
39 {
40 	if (unlikely(buf->pg_refs < MIN_PAGE_REFS)) {
41 		buf->pg_refs += EXTRA_PAGE_REFS;
42 		page_ref_add(buf->page, EXTRA_PAGE_REFS);
43 	}
44 }
45 
46 /* Offer a buffer to the Rx buffer cache. The cache will hold the buffer if its
47  * page is worth retaining and there's room for it. Otherwise the page is
48  * unmapped and our references released.
49  */
cache_offer(struct funeth_rxq * q,const struct funeth_rxbuf * buf)50 static void cache_offer(struct funeth_rxq *q, const struct funeth_rxbuf *buf)
51 {
52 	struct funeth_rx_cache *c = &q->cache;
53 
54 	if (c->prod_cnt - c->cons_cnt <= c->mask && buf->node == numa_mem_id()) {
55 		c->bufs[c->prod_cnt & c->mask] = *buf;
56 		c->prod_cnt++;
57 	} else {
58 		dma_unmap_page_attrs(q->dma_dev, buf->dma_addr, PAGE_SIZE,
59 				     DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
60 		__page_frag_cache_drain(buf->page, buf->pg_refs);
61 	}
62 }
63 
64 /* Get a page from the Rx buffer cache. We only consider the next available
65  * page and return it if we own all its references.
66  */
cache_get(struct funeth_rxq * q,struct funeth_rxbuf * rb)67 static bool cache_get(struct funeth_rxq *q, struct funeth_rxbuf *rb)
68 {
69 	struct funeth_rx_cache *c = &q->cache;
70 	struct funeth_rxbuf *buf;
71 
72 	if (c->prod_cnt == c->cons_cnt)
73 		return false;             /* empty cache */
74 
75 	buf = &c->bufs[c->cons_cnt & c->mask];
76 	if (page_ref_count(buf->page) == buf->pg_refs) {
77 		dma_sync_single_for_device(q->dma_dev, buf->dma_addr,
78 					   PAGE_SIZE, DMA_FROM_DEVICE);
79 		*rb = *buf;
80 		buf->page = NULL;
81 		refresh_refs(rb);
82 		c->cons_cnt++;
83 		return true;
84 	}
85 
86 	/* Page can't be reused. If the cache is full drop this page. */
87 	if (c->prod_cnt - c->cons_cnt > c->mask) {
88 		dma_unmap_page_attrs(q->dma_dev, buf->dma_addr, PAGE_SIZE,
89 				     DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
90 		__page_frag_cache_drain(buf->page, buf->pg_refs);
91 		buf->page = NULL;
92 		c->cons_cnt++;
93 	}
94 	return false;
95 }
96 
97 /* Allocate and DMA-map a page for receive. */
funeth_alloc_page(struct funeth_rxq * q,struct funeth_rxbuf * rb,int node,gfp_t gfp)98 static int funeth_alloc_page(struct funeth_rxq *q, struct funeth_rxbuf *rb,
99 			     int node, gfp_t gfp)
100 {
101 	struct page *p;
102 
103 	if (cache_get(q, rb))
104 		return 0;
105 
106 	p = __alloc_pages_node(node, gfp | __GFP_NOWARN, 0);
107 	if (unlikely(!p))
108 		return -ENOMEM;
109 
110 	rb->dma_addr = dma_map_page(q->dma_dev, p, 0, PAGE_SIZE,
111 				    DMA_FROM_DEVICE);
112 	if (unlikely(dma_mapping_error(q->dma_dev, rb->dma_addr))) {
113 		FUN_QSTAT_INC(q, rx_map_err);
114 		__free_page(p);
115 		return -ENOMEM;
116 	}
117 
118 	FUN_QSTAT_INC(q, rx_page_alloc);
119 
120 	rb->page = p;
121 	rb->pg_refs = 1;
122 	refresh_refs(rb);
123 	rb->node = page_is_pfmemalloc(p) ? -1 : page_to_nid(p);
124 	return 0;
125 }
126 
funeth_free_page(struct funeth_rxq * q,struct funeth_rxbuf * rb)127 static void funeth_free_page(struct funeth_rxq *q, struct funeth_rxbuf *rb)
128 {
129 	if (rb->page) {
130 		dma_unmap_page(q->dma_dev, rb->dma_addr, PAGE_SIZE,
131 			       DMA_FROM_DEVICE);
132 		__page_frag_cache_drain(rb->page, rb->pg_refs);
133 		rb->page = NULL;
134 	}
135 }
136 
137 /* Run the XDP program assigned to an Rx queue.
138  * Return %NULL if the buffer is consumed, or the virtual address of the packet
139  * to turn into an skb.
140  */
fun_run_xdp(struct funeth_rxq * q,skb_frag_t * frags,void * buf_va,int ref_ok,struct funeth_txq * xdp_q)141 static void *fun_run_xdp(struct funeth_rxq *q, skb_frag_t *frags, void *buf_va,
142 			 int ref_ok, struct funeth_txq *xdp_q)
143 {
144 	struct bpf_prog *xdp_prog;
145 	struct xdp_frame *xdpf;
146 	struct xdp_buff xdp;
147 	u32 act;
148 
149 	/* VA includes the headroom, frag size includes headroom + tailroom */
150 	xdp_init_buff(&xdp, ALIGN(skb_frag_size(frags), FUN_EPRQ_PKT_ALIGN),
151 		      &q->xdp_rxq);
152 	xdp_prepare_buff(&xdp, buf_va, FUN_XDP_HEADROOM, skb_frag_size(frags) -
153 			 (FUN_RX_TAILROOM + FUN_XDP_HEADROOM), false);
154 
155 	xdp_prog = READ_ONCE(q->xdp_prog);
156 	act = bpf_prog_run_xdp(xdp_prog, &xdp);
157 
158 	switch (act) {
159 	case XDP_PASS:
160 		/* remove headroom, which may not be FUN_XDP_HEADROOM now */
161 		skb_frag_size_set(frags, xdp.data_end - xdp.data);
162 		skb_frag_off_add(frags, xdp.data - xdp.data_hard_start);
163 		goto pass;
164 	case XDP_TX:
165 		if (unlikely(!ref_ok))
166 			goto pass;
167 
168 		xdpf = xdp_convert_buff_to_frame(&xdp);
169 		if (!xdpf || !fun_xdp_tx(xdp_q, xdpf))
170 			goto xdp_error;
171 		FUN_QSTAT_INC(q, xdp_tx);
172 		q->xdp_flush |= FUN_XDP_FLUSH_TX;
173 		break;
174 	case XDP_REDIRECT:
175 		if (unlikely(!ref_ok))
176 			goto pass;
177 		if (unlikely(xdp_do_redirect(q->netdev, &xdp, xdp_prog)))
178 			goto xdp_error;
179 		FUN_QSTAT_INC(q, xdp_redir);
180 		q->xdp_flush |= FUN_XDP_FLUSH_REDIR;
181 		break;
182 	default:
183 		bpf_warn_invalid_xdp_action(q->netdev, xdp_prog, act);
184 		fallthrough;
185 	case XDP_ABORTED:
186 		trace_xdp_exception(q->netdev, xdp_prog, act);
187 xdp_error:
188 		q->cur_buf->pg_refs++; /* return frags' page reference */
189 		FUN_QSTAT_INC(q, xdp_err);
190 		break;
191 	case XDP_DROP:
192 		q->cur_buf->pg_refs++;
193 		FUN_QSTAT_INC(q, xdp_drops);
194 		break;
195 	}
196 	return NULL;
197 
198 pass:
199 	return xdp.data;
200 }
201 
202 /* A CQE contains a fixed completion structure along with optional metadata and
203  * even packet data. Given the start address of a CQE return the start of the
204  * contained fixed structure, which lies at the end.
205  */
cqe_to_info(const void * cqe)206 static const void *cqe_to_info(const void *cqe)
207 {
208 	return cqe + FUNETH_CQE_INFO_OFFSET;
209 }
210 
211 /* The inverse of cqe_to_info(). */
info_to_cqe(const void * cqe_info)212 static const void *info_to_cqe(const void *cqe_info)
213 {
214 	return cqe_info - FUNETH_CQE_INFO_OFFSET;
215 }
216 
217 /* Return the type of hash provided by the device based on the L3 and L4
218  * protocols it parsed for the packet.
219  */
cqe_to_pkt_hash_type(u16 pkt_parse)220 static enum pkt_hash_types cqe_to_pkt_hash_type(u16 pkt_parse)
221 {
222 	static const enum pkt_hash_types htype_map[] = {
223 		PKT_HASH_TYPE_NONE, PKT_HASH_TYPE_L3,
224 		PKT_HASH_TYPE_NONE, PKT_HASH_TYPE_L4,
225 		PKT_HASH_TYPE_NONE, PKT_HASH_TYPE_L3,
226 		PKT_HASH_TYPE_NONE, PKT_HASH_TYPE_L3
227 	};
228 	u16 key;
229 
230 	/* Build the key from the TCP/UDP and IP/IPv6 bits */
231 	key = ((pkt_parse >> FUN_ETH_RX_CV_OL4_PROT_S) & 6) |
232 	      ((pkt_parse >> (FUN_ETH_RX_CV_OL3_PROT_S + 1)) & 1);
233 
234 	return htype_map[key];
235 }
236 
237 /* Each received packet can be scattered across several Rx buffers or can
238  * share a buffer with previously received packets depending on the buffer
239  * and packet sizes and the room available in the most recently used buffer.
240  *
241  * The rules are:
242  * - If the buffer at the head of an RQ has not been used it gets (part of) the
243  *   next incoming packet.
244  * - Otherwise, if the packet fully fits in the buffer's remaining space the
245  *   packet is written there.
246  * - Otherwise, the packet goes into the next Rx buffer.
247  *
248  * This function returns the Rx buffer for a packet or fragment thereof of the
249  * given length. If it isn't @buf it either recycles or frees that buffer
250  * before advancing the queue to the next buffer.
251  *
252  * If called repeatedly with the remaining length of a packet it will walk
253  * through all the buffers containing the packet.
254  */
255 static struct funeth_rxbuf *
get_buf(struct funeth_rxq * q,struct funeth_rxbuf * buf,unsigned int len)256 get_buf(struct funeth_rxq *q, struct funeth_rxbuf *buf, unsigned int len)
257 {
258 	if (q->buf_offset + len <= PAGE_SIZE || !q->buf_offset)
259 		return buf;            /* @buf holds (part of) the packet */
260 
261 	/* The packet occupies part of the next buffer. Move there after
262 	 * replenishing the current buffer slot either with the spare page or
263 	 * by reusing the slot's existing page. Note that if a spare page isn't
264 	 * available and the current packet occupies @buf it is a multi-frag
265 	 * packet that will be dropped leaving @buf available for reuse.
266 	 */
267 	if ((page_ref_count(buf->page) == buf->pg_refs &&
268 	     buf->node == numa_mem_id()) || !q->spare_buf.page) {
269 		dma_sync_single_for_device(q->dma_dev, buf->dma_addr,
270 					   PAGE_SIZE, DMA_FROM_DEVICE);
271 		refresh_refs(buf);
272 	} else {
273 		cache_offer(q, buf);
274 		*buf = q->spare_buf;
275 		q->spare_buf.page = NULL;
276 		q->rqes[q->rq_cons & q->rq_mask] =
277 			FUN_EPRQ_RQBUF_INIT(buf->dma_addr);
278 	}
279 	q->buf_offset = 0;
280 	q->rq_cons++;
281 	return &q->bufs[q->rq_cons & q->rq_mask];
282 }
283 
284 /* Gather the page fragments making up the first Rx packet on @q. Its total
285  * length @tot_len includes optional head- and tail-rooms.
286  *
287  * Return 0 if the device retains ownership of at least some of the pages.
288  * In this case the caller may only copy the packet.
289  *
290  * A non-zero return value gives the caller permission to use references to the
291  * pages, e.g., attach them to skbs. Additionally, if the value is <0 at least
292  * one of the pages is PF_MEMALLOC.
293  *
294  * Regardless of outcome the caller is granted a reference to each of the pages.
295  */
fun_gather_pkt(struct funeth_rxq * q,unsigned int tot_len,skb_frag_t * frags)296 static int fun_gather_pkt(struct funeth_rxq *q, unsigned int tot_len,
297 			  skb_frag_t *frags)
298 {
299 	struct funeth_rxbuf *buf = q->cur_buf;
300 	unsigned int frag_len;
301 	int ref_ok = 1;
302 
303 	for (;;) {
304 		buf = get_buf(q, buf, tot_len);
305 
306 		/* We always keep the RQ full of buffers so before we can give
307 		 * one of our pages to the stack we require that we can obtain
308 		 * a replacement page. If we can't the packet will either be
309 		 * copied or dropped so we can retain ownership of the page and
310 		 * reuse it.
311 		 */
312 		if (!q->spare_buf.page &&
313 		    funeth_alloc_page(q, &q->spare_buf, numa_mem_id(),
314 				      GFP_ATOMIC | __GFP_MEMALLOC))
315 			ref_ok = 0;
316 
317 		frag_len = min_t(unsigned int, tot_len,
318 				 PAGE_SIZE - q->buf_offset);
319 		dma_sync_single_for_cpu(q->dma_dev,
320 					buf->dma_addr + q->buf_offset,
321 					frag_len, DMA_FROM_DEVICE);
322 		buf->pg_refs--;
323 		if (ref_ok)
324 			ref_ok |= buf->node;
325 
326 		skb_frag_fill_page_desc(frags++, buf->page, q->buf_offset,
327 					frag_len);
328 
329 		tot_len -= frag_len;
330 		if (!tot_len)
331 			break;
332 
333 		q->buf_offset = PAGE_SIZE;
334 	}
335 	q->buf_offset = ALIGN(q->buf_offset + frag_len, FUN_EPRQ_PKT_ALIGN);
336 	q->cur_buf = buf;
337 	return ref_ok;
338 }
339 
rx_hwtstamp_enabled(const struct net_device * dev)340 static bool rx_hwtstamp_enabled(const struct net_device *dev)
341 {
342 	const struct funeth_priv *d = netdev_priv(dev);
343 
344 	return d->hwtstamp_cfg.rx_filter == HWTSTAMP_FILTER_ALL;
345 }
346 
347 /* Advance the CQ pointers and phase tag to the next CQE. */
advance_cq(struct funeth_rxq * q)348 static void advance_cq(struct funeth_rxq *q)
349 {
350 	if (unlikely(q->cq_head == q->cq_mask)) {
351 		q->cq_head = 0;
352 		q->phase ^= 1;
353 		q->next_cqe_info = cqe_to_info(q->cqes);
354 	} else {
355 		q->cq_head++;
356 		q->next_cqe_info += FUNETH_CQE_SIZE;
357 	}
358 	prefetch(q->next_cqe_info);
359 }
360 
361 /* Process the packet represented by the head CQE of @q. Gather the packet's
362  * fragments, run it through the optional XDP program, and if needed construct
363  * an skb and pass it to the stack.
364  */
fun_handle_cqe_pkt(struct funeth_rxq * q,struct funeth_txq * xdp_q)365 static void fun_handle_cqe_pkt(struct funeth_rxq *q, struct funeth_txq *xdp_q)
366 {
367 	const struct fun_eth_cqe *rxreq = info_to_cqe(q->next_cqe_info);
368 	unsigned int i, tot_len, pkt_len = be32_to_cpu(rxreq->pkt_len);
369 	struct net_device *ndev = q->netdev;
370 	skb_frag_t frags[RX_MAX_FRAGS];
371 	struct skb_shared_info *si;
372 	unsigned int headroom;
373 	gro_result_t gro_res;
374 	struct sk_buff *skb;
375 	int ref_ok;
376 	void *va;
377 	u16 cv;
378 
379 	u64_stats_update_begin(&q->syncp);
380 	q->stats.rx_pkts++;
381 	q->stats.rx_bytes += pkt_len;
382 	u64_stats_update_end(&q->syncp);
383 
384 	advance_cq(q);
385 
386 	/* account for head- and tail-room, present only for 1-buffer packets */
387 	tot_len = pkt_len;
388 	headroom = be16_to_cpu(rxreq->headroom);
389 	if (likely(headroom))
390 		tot_len += FUN_RX_TAILROOM + headroom;
391 
392 	ref_ok = fun_gather_pkt(q, tot_len, frags);
393 	va = skb_frag_address(frags);
394 	if (xdp_q && headroom == FUN_XDP_HEADROOM) {
395 		va = fun_run_xdp(q, frags, va, ref_ok, xdp_q);
396 		if (!va)
397 			return;
398 		headroom = 0;   /* XDP_PASS trims it */
399 	}
400 	if (unlikely(!ref_ok))
401 		goto no_mem;
402 
403 	if (likely(headroom)) {
404 		/* headroom is either FUN_RX_HEADROOM or FUN_XDP_HEADROOM */
405 		prefetch(va + headroom);
406 		skb = napi_build_skb(va, ALIGN(tot_len, FUN_EPRQ_PKT_ALIGN));
407 		if (unlikely(!skb))
408 			goto no_mem;
409 
410 		skb_reserve(skb, headroom);
411 		__skb_put(skb, pkt_len);
412 		skb->protocol = eth_type_trans(skb, ndev);
413 	} else {
414 		prefetch(va);
415 		skb = napi_get_frags(q->napi);
416 		if (unlikely(!skb))
417 			goto no_mem;
418 
419 		if (ref_ok < 0)
420 			skb->pfmemalloc = 1;
421 
422 		si = skb_shinfo(skb);
423 		si->nr_frags = rxreq->nsgl;
424 		for (i = 0; i < si->nr_frags; i++)
425 			si->frags[i] = frags[i];
426 
427 		skb->len = pkt_len;
428 		skb->data_len = pkt_len;
429 		skb->truesize += round_up(pkt_len, FUN_EPRQ_PKT_ALIGN);
430 	}
431 
432 	skb_record_rx_queue(skb, q->qidx);
433 	cv = be16_to_cpu(rxreq->pkt_cv);
434 	if (likely((q->netdev->features & NETIF_F_RXHASH) && rxreq->hash))
435 		skb_set_hash(skb, be32_to_cpu(rxreq->hash),
436 			     cqe_to_pkt_hash_type(cv));
437 	if (likely((q->netdev->features & NETIF_F_RXCSUM) && rxreq->csum)) {
438 		FUN_QSTAT_INC(q, rx_cso);
439 		skb->ip_summed = CHECKSUM_UNNECESSARY;
440 		skb->csum_level = be16_to_cpu(rxreq->csum) - 1;
441 	}
442 	if (unlikely(rx_hwtstamp_enabled(q->netdev)))
443 		skb_hwtstamps(skb)->hwtstamp = be64_to_cpu(rxreq->timestamp);
444 
445 	trace_funeth_rx(q, rxreq->nsgl, pkt_len, skb->hash, cv);
446 
447 	gro_res = skb->data_len ? napi_gro_frags(q->napi) :
448 				  napi_gro_receive(q->napi, skb);
449 	if (gro_res == GRO_MERGED || gro_res == GRO_MERGED_FREE)
450 		FUN_QSTAT_INC(q, gro_merged);
451 	else if (gro_res == GRO_HELD)
452 		FUN_QSTAT_INC(q, gro_pkts);
453 	return;
454 
455 no_mem:
456 	FUN_QSTAT_INC(q, rx_mem_drops);
457 
458 	/* Release the references we've been granted for the frag pages.
459 	 * We return the ref of the last frag and free the rest.
460 	 */
461 	q->cur_buf->pg_refs++;
462 	for (i = 0; i < rxreq->nsgl - 1; i++)
463 		__free_page(skb_frag_page(frags + i));
464 }
465 
466 /* Return 0 if the phase tag of the CQE at the CQ's head matches expectations
467  * indicating the CQE is new.
468  */
cqe_phase_mismatch(const struct fun_cqe_info * ci,u16 phase)469 static u16 cqe_phase_mismatch(const struct fun_cqe_info *ci, u16 phase)
470 {
471 	u16 sf_p = be16_to_cpu(ci->sf_p);
472 
473 	return (sf_p & 1) ^ phase;
474 }
475 
476 /* Walk through a CQ identifying and processing fresh CQEs up to the given
477  * budget. Return the remaining budget.
478  */
fun_process_cqes(struct funeth_rxq * q,int budget)479 static int fun_process_cqes(struct funeth_rxq *q, int budget)
480 {
481 	struct funeth_priv *fp = netdev_priv(q->netdev);
482 	struct funeth_txq **xdpqs, *xdp_q = NULL;
483 
484 	xdpqs = rcu_dereference_bh(fp->xdpqs);
485 	if (xdpqs)
486 		xdp_q = xdpqs[smp_processor_id()];
487 
488 	while (budget && !cqe_phase_mismatch(q->next_cqe_info, q->phase)) {
489 		/* access other descriptor fields after the phase check */
490 		dma_rmb();
491 
492 		fun_handle_cqe_pkt(q, xdp_q);
493 		budget--;
494 	}
495 
496 	if (unlikely(q->xdp_flush)) {
497 		if (q->xdp_flush & FUN_XDP_FLUSH_TX)
498 			fun_txq_wr_db(xdp_q);
499 		if (q->xdp_flush & FUN_XDP_FLUSH_REDIR)
500 			xdp_do_flush();
501 		q->xdp_flush = 0;
502 	}
503 
504 	return budget;
505 }
506 
507 /* NAPI handler for Rx queues. Calls the CQE processing loop and writes RQ/CQ
508  * doorbells as needed.
509  */
fun_rxq_napi_poll(struct napi_struct * napi,int budget)510 int fun_rxq_napi_poll(struct napi_struct *napi, int budget)
511 {
512 	struct fun_irq *irq = container_of(napi, struct fun_irq, napi);
513 	struct funeth_rxq *q = irq->rxq;
514 	int work_done = budget - fun_process_cqes(q, budget);
515 	u32 cq_db_val = q->cq_head;
516 
517 	if (unlikely(work_done >= budget))
518 		FUN_QSTAT_INC(q, rx_budget);
519 	else if (napi_complete_done(napi, work_done))
520 		cq_db_val |= q->irq_db_val;
521 
522 	/* check whether to post new Rx buffers */
523 	if (q->rq_cons - q->rq_cons_db >= q->rq_db_thres) {
524 		u64_stats_update_begin(&q->syncp);
525 		q->stats.rx_bufs += q->rq_cons - q->rq_cons_db;
526 		u64_stats_update_end(&q->syncp);
527 		q->rq_cons_db = q->rq_cons;
528 		writel((q->rq_cons - 1) & q->rq_mask, q->rq_db);
529 	}
530 
531 	writel(cq_db_val, q->cq_db);
532 	return work_done;
533 }
534 
535 /* Free the Rx buffers of an Rx queue. */
fun_rxq_free_bufs(struct funeth_rxq * q)536 static void fun_rxq_free_bufs(struct funeth_rxq *q)
537 {
538 	struct funeth_rxbuf *b = q->bufs;
539 	unsigned int i;
540 
541 	for (i = 0; i <= q->rq_mask; i++, b++)
542 		funeth_free_page(q, b);
543 
544 	funeth_free_page(q, &q->spare_buf);
545 	q->cur_buf = NULL;
546 }
547 
548 /* Initially provision an Rx queue with Rx buffers. */
fun_rxq_alloc_bufs(struct funeth_rxq * q,int node)549 static int fun_rxq_alloc_bufs(struct funeth_rxq *q, int node)
550 {
551 	struct funeth_rxbuf *b = q->bufs;
552 	unsigned int i;
553 
554 	for (i = 0; i <= q->rq_mask; i++, b++) {
555 		if (funeth_alloc_page(q, b, node, GFP_KERNEL)) {
556 			fun_rxq_free_bufs(q);
557 			return -ENOMEM;
558 		}
559 		q->rqes[i] = FUN_EPRQ_RQBUF_INIT(b->dma_addr);
560 	}
561 	q->cur_buf = q->bufs;
562 	return 0;
563 }
564 
565 /* Initialize a used-buffer cache of the given depth. */
fun_rxq_init_cache(struct funeth_rx_cache * c,unsigned int depth,int node)566 static int fun_rxq_init_cache(struct funeth_rx_cache *c, unsigned int depth,
567 			      int node)
568 {
569 	c->mask = depth - 1;
570 	c->bufs = kvzalloc_node(depth * sizeof(*c->bufs), GFP_KERNEL, node);
571 	return c->bufs ? 0 : -ENOMEM;
572 }
573 
574 /* Deallocate an Rx queue's used-buffer cache and its contents. */
fun_rxq_free_cache(struct funeth_rxq * q)575 static void fun_rxq_free_cache(struct funeth_rxq *q)
576 {
577 	struct funeth_rxbuf *b = q->cache.bufs;
578 	unsigned int i;
579 
580 	for (i = 0; i <= q->cache.mask; i++, b++)
581 		funeth_free_page(q, b);
582 
583 	kvfree(q->cache.bufs);
584 	q->cache.bufs = NULL;
585 }
586 
fun_rxq_set_bpf(struct funeth_rxq * q,struct bpf_prog * prog)587 int fun_rxq_set_bpf(struct funeth_rxq *q, struct bpf_prog *prog)
588 {
589 	struct funeth_priv *fp = netdev_priv(q->netdev);
590 	struct fun_admin_epcq_req cmd;
591 	u16 headroom;
592 	int err;
593 
594 	headroom = prog ? FUN_XDP_HEADROOM : FUN_RX_HEADROOM;
595 	if (headroom != q->headroom) {
596 		cmd.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPCQ,
597 							sizeof(cmd));
598 		cmd.u.modify =
599 			FUN_ADMIN_EPCQ_MODIFY_REQ_INIT(FUN_ADMIN_SUBOP_MODIFY,
600 						       0, q->hw_cqid, headroom);
601 		err = fun_submit_admin_sync_cmd(fp->fdev, &cmd.common, NULL, 0,
602 						0);
603 		if (err)
604 			return err;
605 		q->headroom = headroom;
606 	}
607 
608 	WRITE_ONCE(q->xdp_prog, prog);
609 	return 0;
610 }
611 
612 /* Create an Rx queue, allocating the host memory it needs. */
fun_rxq_create_sw(struct net_device * dev,unsigned int qidx,unsigned int ncqe,unsigned int nrqe,struct fun_irq * irq)613 static struct funeth_rxq *fun_rxq_create_sw(struct net_device *dev,
614 					    unsigned int qidx,
615 					    unsigned int ncqe,
616 					    unsigned int nrqe,
617 					    struct fun_irq *irq)
618 {
619 	struct funeth_priv *fp = netdev_priv(dev);
620 	struct funeth_rxq *q;
621 	int err = -ENOMEM;
622 	int numa_node;
623 
624 	numa_node = fun_irq_node(irq);
625 	q = kzalloc_node(sizeof(*q), GFP_KERNEL, numa_node);
626 	if (!q)
627 		goto err;
628 
629 	q->qidx = qidx;
630 	q->netdev = dev;
631 	q->cq_mask = ncqe - 1;
632 	q->rq_mask = nrqe - 1;
633 	q->numa_node = numa_node;
634 	q->rq_db_thres = nrqe / 4;
635 	u64_stats_init(&q->syncp);
636 	q->dma_dev = &fp->pdev->dev;
637 
638 	q->rqes = fun_alloc_ring_mem(q->dma_dev, nrqe, sizeof(*q->rqes),
639 				     sizeof(*q->bufs), false, numa_node,
640 				     &q->rq_dma_addr, (void **)&q->bufs, NULL);
641 	if (!q->rqes)
642 		goto free_q;
643 
644 	q->cqes = fun_alloc_ring_mem(q->dma_dev, ncqe, FUNETH_CQE_SIZE, 0,
645 				     false, numa_node, &q->cq_dma_addr, NULL,
646 				     NULL);
647 	if (!q->cqes)
648 		goto free_rqes;
649 
650 	err = fun_rxq_init_cache(&q->cache, nrqe, numa_node);
651 	if (err)
652 		goto free_cqes;
653 
654 	err = fun_rxq_alloc_bufs(q, numa_node);
655 	if (err)
656 		goto free_cache;
657 
658 	q->stats.rx_bufs = q->rq_mask;
659 	q->init_state = FUN_QSTATE_INIT_SW;
660 	return q;
661 
662 free_cache:
663 	fun_rxq_free_cache(q);
664 free_cqes:
665 	dma_free_coherent(q->dma_dev, ncqe * FUNETH_CQE_SIZE, q->cqes,
666 			  q->cq_dma_addr);
667 free_rqes:
668 	fun_free_ring_mem(q->dma_dev, nrqe, sizeof(*q->rqes), false, q->rqes,
669 			  q->rq_dma_addr, q->bufs);
670 free_q:
671 	kfree(q);
672 err:
673 	netdev_err(dev, "Unable to allocate memory for Rx queue %u\n", qidx);
674 	return ERR_PTR(err);
675 }
676 
fun_rxq_free_sw(struct funeth_rxq * q)677 static void fun_rxq_free_sw(struct funeth_rxq *q)
678 {
679 	struct funeth_priv *fp = netdev_priv(q->netdev);
680 
681 	fun_rxq_free_cache(q);
682 	fun_rxq_free_bufs(q);
683 	fun_free_ring_mem(q->dma_dev, q->rq_mask + 1, sizeof(*q->rqes), false,
684 			  q->rqes, q->rq_dma_addr, q->bufs);
685 	dma_free_coherent(q->dma_dev, (q->cq_mask + 1) * FUNETH_CQE_SIZE,
686 			  q->cqes, q->cq_dma_addr);
687 
688 	/* Before freeing the queue transfer key counters to the device. */
689 	fp->rx_packets += q->stats.rx_pkts;
690 	fp->rx_bytes   += q->stats.rx_bytes;
691 	fp->rx_dropped += q->stats.rx_map_err + q->stats.rx_mem_drops;
692 
693 	kfree(q);
694 }
695 
696 /* Create an Rx queue's resources on the device. */
fun_rxq_create_dev(struct funeth_rxq * q,struct fun_irq * irq)697 int fun_rxq_create_dev(struct funeth_rxq *q, struct fun_irq *irq)
698 {
699 	struct funeth_priv *fp = netdev_priv(q->netdev);
700 	unsigned int ncqe = q->cq_mask + 1;
701 	unsigned int nrqe = q->rq_mask + 1;
702 	int err;
703 
704 	err = xdp_rxq_info_reg(&q->xdp_rxq, q->netdev, q->qidx,
705 			       irq->napi.napi_id);
706 	if (err)
707 		goto out;
708 
709 	err = xdp_rxq_info_reg_mem_model(&q->xdp_rxq, MEM_TYPE_PAGE_SHARED,
710 					 NULL);
711 	if (err)
712 		goto xdp_unreg;
713 
714 	q->phase = 1;
715 	q->irq_cnt = 0;
716 	q->cq_head = 0;
717 	q->rq_cons = 0;
718 	q->rq_cons_db = 0;
719 	q->buf_offset = 0;
720 	q->napi = &irq->napi;
721 	q->irq_db_val = fp->cq_irq_db;
722 	q->next_cqe_info = cqe_to_info(q->cqes);
723 
724 	q->xdp_prog = fp->xdp_prog;
725 	q->headroom = fp->xdp_prog ? FUN_XDP_HEADROOM : FUN_RX_HEADROOM;
726 
727 	err = fun_sq_create(fp->fdev, FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR |
728 			    FUN_ADMIN_EPSQ_CREATE_FLAG_RQ, 0,
729 			    FUN_HCI_ID_INVALID, 0, nrqe, q->rq_dma_addr, 0, 0,
730 			    0, 0, fp->fdev->kern_end_qid, PAGE_SHIFT,
731 			    &q->hw_sqid, &q->rq_db);
732 	if (err)
733 		goto xdp_unreg;
734 
735 	err = fun_cq_create(fp->fdev, FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR |
736 			    FUN_ADMIN_EPCQ_CREATE_FLAG_RQ, 0,
737 			    q->hw_sqid, ilog2(FUNETH_CQE_SIZE), ncqe,
738 			    q->cq_dma_addr, q->headroom, FUN_RX_TAILROOM, 0, 0,
739 			    irq->irq_idx, 0, fp->fdev->kern_end_qid,
740 			    &q->hw_cqid, &q->cq_db);
741 	if (err)
742 		goto free_rq;
743 
744 	irq->rxq = q;
745 	writel(q->rq_mask, q->rq_db);
746 	q->init_state = FUN_QSTATE_INIT_FULL;
747 
748 	netif_info(fp, ifup, q->netdev,
749 		   "Rx queue %u, depth %u/%u, HW qid %u/%u, IRQ idx %u, node %d, headroom %u\n",
750 		   q->qidx, ncqe, nrqe, q->hw_cqid, q->hw_sqid, irq->irq_idx,
751 		   q->numa_node, q->headroom);
752 	return 0;
753 
754 free_rq:
755 	fun_destroy_sq(fp->fdev, q->hw_sqid);
756 xdp_unreg:
757 	xdp_rxq_info_unreg(&q->xdp_rxq);
758 out:
759 	netdev_err(q->netdev,
760 		   "Failed to create Rx queue %u on device, error %d\n",
761 		   q->qidx, err);
762 	return err;
763 }
764 
fun_rxq_free_dev(struct funeth_rxq * q)765 static void fun_rxq_free_dev(struct funeth_rxq *q)
766 {
767 	struct funeth_priv *fp = netdev_priv(q->netdev);
768 	struct fun_irq *irq;
769 
770 	if (q->init_state < FUN_QSTATE_INIT_FULL)
771 		return;
772 
773 	irq = container_of(q->napi, struct fun_irq, napi);
774 	netif_info(fp, ifdown, q->netdev,
775 		   "Freeing Rx queue %u (id %u/%u), IRQ %u\n",
776 		   q->qidx, q->hw_cqid, q->hw_sqid, irq->irq_idx);
777 
778 	irq->rxq = NULL;
779 	xdp_rxq_info_unreg(&q->xdp_rxq);
780 	fun_destroy_sq(fp->fdev, q->hw_sqid);
781 	fun_destroy_cq(fp->fdev, q->hw_cqid);
782 	q->init_state = FUN_QSTATE_INIT_SW;
783 }
784 
785 /* Create or advance an Rx queue, allocating all the host and device resources
786  * needed to reach the target state.
787  */
funeth_rxq_create(struct net_device * dev,unsigned int qidx,unsigned int ncqe,unsigned int nrqe,struct fun_irq * irq,int state,struct funeth_rxq ** qp)788 int funeth_rxq_create(struct net_device *dev, unsigned int qidx,
789 		      unsigned int ncqe, unsigned int nrqe, struct fun_irq *irq,
790 		      int state, struct funeth_rxq **qp)
791 {
792 	struct funeth_rxq *q = *qp;
793 	int err;
794 
795 	if (!q) {
796 		q = fun_rxq_create_sw(dev, qidx, ncqe, nrqe, irq);
797 		if (IS_ERR(q))
798 			return PTR_ERR(q);
799 	}
800 
801 	if (q->init_state >= state)
802 		goto out;
803 
804 	err = fun_rxq_create_dev(q, irq);
805 	if (err) {
806 		if (!*qp)
807 			fun_rxq_free_sw(q);
808 		return err;
809 	}
810 
811 out:
812 	*qp = q;
813 	return 0;
814 }
815 
816 /* Free Rx queue resources until it reaches the target state. */
funeth_rxq_free(struct funeth_rxq * q,int state)817 struct funeth_rxq *funeth_rxq_free(struct funeth_rxq *q, int state)
818 {
819 	if (state < FUN_QSTATE_INIT_FULL)
820 		fun_rxq_free_dev(q);
821 
822 	if (state == FUN_QSTATE_DESTROYED) {
823 		fun_rxq_free_sw(q);
824 		q = NULL;
825 	}
826 
827 	return q;
828 }
829