xref: /freebsd/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c (revision ec0ea6efa1ad229d75c394c1a9b9cac33af2b1d3)
1 /*-
2  * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  *
25  * $FreeBSD$
26  */
27 
28 #include "en.h"
29 #include <machine/in_cksum.h>
30 
31 static inline int
32 mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
33     struct mlx5e_rx_wqe *wqe, u16 ix)
34 {
35 	bus_dma_segment_t segs[MLX5E_MAX_BUSDMA_RX_SEGS];
36 	struct mbuf *mb;
37 	int nsegs;
38 	int err;
39 	struct mbuf *mb_head;
40 	int i;
41 
42 	if (rq->mbuf[ix].mbuf != NULL)
43 		return (0);
44 
45 	mb_head = mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR,
46 	    MLX5E_MAX_RX_BYTES);
47 	if (unlikely(mb == NULL))
48 		return (-ENOMEM);
49 
50 	mb->m_len = MLX5E_MAX_RX_BYTES;
51 	mb->m_pkthdr.len = MLX5E_MAX_RX_BYTES;
52 
53 	for (i = 1; i < rq->nsegs; i++) {
54 		if (mb_head->m_pkthdr.len >= rq->wqe_sz)
55 			break;
56 		mb = mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0,
57 		    MLX5E_MAX_RX_BYTES);
58 		if (unlikely(mb == NULL)) {
59 			m_freem(mb_head);
60 			return (-ENOMEM);
61 		}
62 		mb->m_len = MLX5E_MAX_RX_BYTES;
63 		mb_head->m_pkthdr.len += MLX5E_MAX_RX_BYTES;
64 	}
65 	/* rewind to first mbuf in chain */
66 	mb = mb_head;
67 
68 	/* get IP header aligned */
69 	m_adj(mb, MLX5E_NET_IP_ALIGN);
70 
71 	err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
72 	    mb, segs, &nsegs, BUS_DMA_NOWAIT);
73 	if (err != 0)
74 		goto err_free_mbuf;
75 	if (unlikely(nsegs == 0)) {
76 		bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map);
77 		err = -ENOMEM;
78 		goto err_free_mbuf;
79 	}
80 	wqe->data[0].addr = cpu_to_be64(segs[0].ds_addr);
81 	wqe->data[0].byte_count = cpu_to_be32(segs[0].ds_len |
82 	    MLX5_HW_START_PADDING);
83 	for (i = 1; i != nsegs; i++) {
84 		wqe->data[i].addr = cpu_to_be64(segs[i].ds_addr);
85 		wqe->data[i].byte_count = cpu_to_be32(segs[i].ds_len);
86 	}
87 	for (; i < rq->nsegs; i++) {
88 		wqe->data[i].addr = 0;
89 		wqe->data[i].byte_count = 0;
90 	}
91 
92 	rq->mbuf[ix].mbuf = mb;
93 	rq->mbuf[ix].data = mb->m_data;
94 
95 	bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map,
96 	    BUS_DMASYNC_PREREAD);
97 	return (0);
98 
99 err_free_mbuf:
100 	m_freem(mb);
101 	return (err);
102 }
103 
104 static void
105 mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
106 {
107 	if (unlikely(rq->enabled == 0))
108 		return;
109 
110 	while (!mlx5_wq_ll_is_full(&rq->wq)) {
111 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head);
112 
113 		if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) {
114 			callout_reset_curcpu(&rq->watchdog, 1, (void *)&mlx5e_post_rx_wqes, rq);
115 			break;
116 		}
117 		mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index));
118 	}
119 
120 	/* ensure wqes are visible to device before updating doorbell record */
121 	atomic_thread_fence_rel();
122 
123 	mlx5_wq_ll_update_db_record(&rq->wq);
124 }
125 
126 static void
127 mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe)
128 {
129 	/* TODO: consider vlans, ip options, ... */
130 	struct ether_header *eh;
131 	uint16_t eh_type;
132 	uint16_t tot_len;
133 	struct ip6_hdr *ip6 = NULL;
134 	struct ip *ip4 = NULL;
135 	struct tcphdr *th;
136 	uint32_t *ts_ptr;
137 	uint8_t l4_hdr_type;
138 	int tcp_ack;
139 
140 	eh = mtod(mb, struct ether_header *);
141 	eh_type = ntohs(eh->ether_type);
142 
143 	l4_hdr_type = get_cqe_l4_hdr_type(cqe);
144 	tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) ||
145 	    (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type));
146 
147 	/* TODO: consider vlan */
148 	tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN;
149 
150 	switch (eh_type) {
151 	case ETHERTYPE_IP:
152 		ip4 = (struct ip *)(eh + 1);
153 		th = (struct tcphdr *)(ip4 + 1);
154 		break;
155 	case ETHERTYPE_IPV6:
156 		ip6 = (struct ip6_hdr *)(eh + 1);
157 		th = (struct tcphdr *)(ip6 + 1);
158 		break;
159 	default:
160 		return;
161 	}
162 
163 	ts_ptr = (uint32_t *)(th + 1);
164 
165 	if (get_cqe_lro_tcppsh(cqe))
166 		th->th_flags |= TH_PUSH;
167 
168 	if (tcp_ack) {
169 		th->th_flags |= TH_ACK;
170 		th->th_ack = cqe->lro_ack_seq_num;
171 		th->th_win = cqe->lro_tcp_win;
172 
173 		/*
174 		 * FreeBSD handles only 32bit aligned timestamp right after
175 		 * the TCP hdr
176 		 * +--------+--------+--------+--------+
177 		 * |   NOP  |  NOP   |  TSopt |   10   |
178 		 * +--------+--------+--------+--------+
179 		 * |          TSval   timestamp        |
180 		 * +--------+--------+--------+--------+
181 		 * |          TSecr   timestamp        |
182 		 * +--------+--------+--------+--------+
183 		 */
184 		if (get_cqe_lro_timestamp_valid(cqe) &&
185 		    (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 |
186 		    TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 |
187 		    TCPOLEN_TIMESTAMP))) {
188 			/*
189 			 * cqe->timestamp is 64bit long.
190 			 * [0-31] - timestamp.
191 			 * [32-64] - timestamp echo replay.
192 			 */
193 			ts_ptr[1] = *(uint32_t *)&cqe->timestamp;
194 			ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1);
195 		}
196 	}
197 	if (ip4) {
198 		ip4->ip_ttl = cqe->lro_min_ttl;
199 		ip4->ip_len = cpu_to_be16(tot_len);
200 		ip4->ip_sum = 0;
201 		ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2);
202 	} else {
203 		ip6->ip6_hlim = cqe->lro_min_ttl;
204 		ip6->ip6_plen = cpu_to_be16(tot_len -
205 		    sizeof(struct ip6_hdr));
206 	}
207 	/* TODO: handle tcp checksum */
208 }
209 
210 static uint64_t
211 mlx5e_mbuf_tstmp(struct mlx5e_priv *priv, uint64_t hw_tstmp)
212 {
213 	struct mlx5e_clbr_point *cp, dcp;
214 	uint64_t a1, a2, res;
215 	u_int gen;
216 
217 	do {
218 		cp = &priv->clbr_points[priv->clbr_curr];
219 		gen = atomic_load_acq_int(&cp->clbr_gen);
220 		if (gen == 0)
221 			return (0);
222 		dcp = *cp;
223 		atomic_thread_fence_acq();
224 	} while (gen != cp->clbr_gen);
225 
226 	a1 = (hw_tstmp - dcp.clbr_hw_prev) >> MLX5E_TSTMP_PREC;
227 	a2 = (dcp.base_curr - dcp.base_prev) >> MLX5E_TSTMP_PREC;
228 	res = (a1 * a2) << MLX5E_TSTMP_PREC;
229 
230 	/*
231 	 * Divisor cannot be zero because calibration callback
232 	 * checks for the condition and disables timestamping
233 	 * if clock halted.
234 	 */
235 	res /= (dcp.clbr_hw_curr - dcp.clbr_hw_prev) >> MLX5E_TSTMP_PREC;
236 
237 	res += dcp.base_prev;
238 	return (res);
239 }
240 
241 static inline void
242 mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
243     struct mlx5e_rq *rq, struct mbuf *mb,
244     u32 cqe_bcnt)
245 {
246 	struct ifnet *ifp = rq->ifp;
247 	struct mlx5e_channel *c;
248 	struct mbuf *mb_head;
249 	int lro_num_seg;	/* HW LRO session aggregated packets counter */
250 	uint64_t tstmp;
251 
252 	lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
253 	if (lro_num_seg > 1) {
254 		mlx5e_lro_update_hdr(mb, cqe);
255 		rq->stats.lro_packets++;
256 		rq->stats.lro_bytes += cqe_bcnt;
257 	}
258 
259 	mb->m_pkthdr.len = cqe_bcnt;
260 	for (mb_head = mb; mb != NULL; mb = mb->m_next) {
261 		if (mb->m_len > cqe_bcnt)
262 			mb->m_len = cqe_bcnt;
263 		cqe_bcnt -= mb->m_len;
264 		if (likely(cqe_bcnt == 0)) {
265 			if (likely(mb->m_next != NULL)) {
266 				/* trim off empty mbufs */
267 				m_freem(mb->m_next);
268 				mb->m_next = NULL;
269 			}
270 			break;
271 		}
272 	}
273 	/* rewind to first mbuf in chain */
274 	mb = mb_head;
275 
276 	/* check if a Toeplitz hash was computed */
277 	if (cqe->rss_hash_type != 0) {
278 		mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result);
279 #ifdef RSS
280 		/* decode the RSS hash type */
281 		switch (cqe->rss_hash_type &
282 		    (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) {
283 		/* IPv4 */
284 		case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4):
285 			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4);
286 			break;
287 		case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4):
288 			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4);
289 			break;
290 		case CQE_RSS_DST_HTYPE_IPV4:
291 			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4);
292 			break;
293 		/* IPv6 */
294 		case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6):
295 			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6);
296 			break;
297 		case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6):
298 			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6);
299 			break;
300 		case CQE_RSS_DST_HTYPE_IPV6:
301 			M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6);
302 			break;
303 		default:	/* Other */
304 			M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH);
305 			break;
306 		}
307 #else
308 		M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH);
309 #endif
310 #ifdef M_HASHTYPE_SETINNER
311 		if (cqe_is_tunneled(cqe))
312 			M_HASHTYPE_SETINNER(mb);
313 #endif
314 	} else {
315 		mb->m_pkthdr.flowid = rq->ix;
316 		M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
317 	}
318 	mb->m_pkthdr.rcvif = ifp;
319 
320 	if (cqe_is_tunneled(cqe)) {
321 		/*
322 		 * CQE can be tunneled only if TIR is configured to
323 		 * enable parsing of tunneled payload, so no need to
324 		 * check for capabilities.
325 		 */
326 		if (((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK)) ==
327 		    (CQE_L2_OK | CQE_L3_OK))) {
328 			mb->m_pkthdr.csum_flags |=
329 			    CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |
330 			    CSUM_IP_CHECKED | CSUM_IP_VALID |
331 			    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
332 			mb->m_pkthdr.csum_data = htons(0xffff);
333 		}
334 		if (((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) ==
335 		    (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) {
336 			mb->m_pkthdr.csum_flags |=
337 			    CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID;
338 		}
339 	} else if (likely((ifp->if_capenable & (IFCAP_RXCSUM |
340 	    IFCAP_RXCSUM_IPV6)) != 0) &&
341 	    ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) ==
342 	    (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) {
343 		mb->m_pkthdr.csum_flags =
344 		    CSUM_IP_CHECKED | CSUM_IP_VALID |
345 		    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
346 		mb->m_pkthdr.csum_data = htons(0xffff);
347 	} else {
348 		rq->stats.csum_none++;
349 	}
350 
351 	if (cqe_has_vlan(cqe)) {
352 		mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info);
353 		mb->m_flags |= M_VLANTAG;
354 	}
355 
356 	c = container_of(rq, struct mlx5e_channel, rq);
357 	if (c->priv->clbr_done >= 2) {
358 		tstmp = mlx5e_mbuf_tstmp(c->priv, be64_to_cpu(cqe->timestamp));
359 		if ((tstmp & MLX5_CQE_TSTMP_PTP) != 0) {
360 			/*
361 			 * Timestamp was taken on the packet entrance,
362 			 * instead of the cqe generation.
363 			 */
364 			tstmp &= ~MLX5_CQE_TSTMP_PTP;
365 			mb->m_flags |= M_TSTMP_HPREC;
366 		}
367 		mb->m_pkthdr.rcv_tstmp = tstmp;
368 		mb->m_flags |= M_TSTMP;
369 	}
370 }
371 
372 static inline void
373 mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
374 {
375 	memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)),
376 	    sizeof(struct mlx5_cqe64));
377 }
378 
379 static inline void
380 mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
381 {
382 	memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1),
383 	    data, sizeof(struct mlx5_cqe64));
384 }
385 
386 static inline void
387 mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title,
388     struct mlx5_mini_cqe8 *mini,
389     u16 wqe_counter, int i)
390 {
391 	/*
392 	 * NOTE: The fields which are not set here are copied from the
393 	 * initial and common title. See memcpy() in
394 	 * mlx5e_write_cqe_slot().
395 	 */
396 	title->byte_cnt = mini->byte_cnt;
397 	title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1);
398 	title->rss_hash_result = mini->rx_hash_result;
399 	/*
400 	 * Since we use MLX5_CQE_FORMAT_HASH when creating the RX CQ,
401 	 * the value of the checksum should be ignored.
402 	 */
403 	title->check_sum = 0;
404 	title->op_own = (title->op_own & 0xf0) |
405 	    (((cq->wq.cc + i) >> cq->wq.log_sz) & 1);
406 }
407 
408 #define MLX5E_MINI_ARRAY_SZ 8
409 /* Make sure structs are not packet differently */
410 CTASSERT(sizeof(struct mlx5_cqe64) ==
411     sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ);
412 static void
413 mlx5e_decompress_cqes(struct mlx5e_cq *cq)
414 {
415 	struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ];
416 	struct mlx5_cqe64 title;
417 	u32 cqe_count;
418 	u32 i = 0;
419 	u16 title_wqe_counter;
420 
421 	mlx5e_read_cqe_slot(cq, cq->wq.cc, &title);
422 	title_wqe_counter = be16_to_cpu(title.wqe_counter);
423 	cqe_count = be32_to_cpu(title.byte_cnt);
424 
425 	/* Make sure we won't overflow */
426 	KASSERT(cqe_count <= cq->wq.sz_m1,
427 	    ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__,
428 	    cqe_count, cq->wq.sz_m1));
429 
430 	mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array);
431 	while (true) {
432 		mlx5e_decompress_cqe(cq, &title,
433 		    &mini_array[i % MLX5E_MINI_ARRAY_SZ],
434 		    title_wqe_counter, i);
435 		mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title);
436 		i++;
437 
438 		if (i == cqe_count)
439 			break;
440 		if (i % MLX5E_MINI_ARRAY_SZ == 0)
441 			mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array);
442 	}
443 }
444 
445 static int
446 mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
447 {
448 	struct pfil_head *pfil;
449 	int i, rv;
450 
451 	CURVNET_SET_QUIET(rq->ifp->if_vnet);
452 	pfil = rq->channel->priv->pfil;
453 	for (i = 0; i < budget; i++) {
454 		struct mlx5e_rx_wqe *wqe;
455 		struct mlx5_cqe64 *cqe;
456 		struct mbuf *mb;
457 		__be16 wqe_counter_be;
458 		u16 wqe_counter;
459 		u32 byte_cnt, seglen;
460 
461 		cqe = mlx5e_get_cqe(&rq->cq);
462 		if (!cqe)
463 			break;
464 
465 		if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED)
466 			mlx5e_decompress_cqes(&rq->cq);
467 
468 		mlx5_cqwq_pop(&rq->cq.wq);
469 
470 		wqe_counter_be = cqe->wqe_counter;
471 		wqe_counter = be16_to_cpu(wqe_counter_be);
472 		wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
473 		byte_cnt = be32_to_cpu(cqe->byte_cnt);
474 
475 		bus_dmamap_sync(rq->dma_tag,
476 		    rq->mbuf[wqe_counter].dma_map,
477 		    BUS_DMASYNC_POSTREAD);
478 
479 		if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
480 			rq->stats.wqe_err++;
481 			goto wq_ll_pop;
482 		}
483 		if (pfil != NULL && PFIL_HOOKED_IN(pfil)) {
484 			seglen = MIN(byte_cnt, MLX5E_MAX_RX_BYTES);
485 			rv = pfil_run_hooks(rq->channel->priv->pfil,
486 			    rq->mbuf[wqe_counter].data, rq->ifp,
487 			    seglen | PFIL_MEMPTR | PFIL_IN, NULL);
488 
489 			switch (rv) {
490 			case PFIL_DROPPED:
491 			case PFIL_CONSUMED:
492 				/*
493 				 * Filter dropped or consumed it. In
494 				 * either case, we can just recycle
495 				 * buffer; there is no more work to do.
496 				 */
497 				rq->stats.packets++;
498 				goto wq_ll_pop;
499 			case PFIL_REALLOCED:
500 				/*
501 				 * Filter copied it; recycle buffer
502 				 * and receive the new mbuf allocated
503 				 * by the Filter
504 				 */
505 				mb = pfil_mem2mbuf(rq->mbuf[wqe_counter].data);
506 				goto rx_common;
507 			default:
508 				/*
509 				 * The Filter said it was OK, so
510 				 * receive like normal.
511 				 */
512 				KASSERT(rv == PFIL_PASS,
513 					("Filter returned %d!\n", rv));
514 			}
515 		}
516 		if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt &&
517 		    (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
518 			/* set maximum mbuf length */
519 			mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN;
520 			/* get IP header aligned */
521 			mb->m_data += MLX5E_NET_IP_ALIGN;
522 
523 			bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t),
524 			    byte_cnt);
525 		} else {
526 			mb = rq->mbuf[wqe_counter].mbuf;
527 			rq->mbuf[wqe_counter].mbuf = NULL;	/* safety clear */
528 
529 			bus_dmamap_unload(rq->dma_tag,
530 			    rq->mbuf[wqe_counter].dma_map);
531 		}
532 rx_common:
533 		mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt);
534 		rq->stats.bytes += byte_cnt;
535 		rq->stats.packets++;
536 #ifdef NUMA
537 		mb->m_pkthdr.numa_domain = rq->ifp->if_numa_domain;
538 #endif
539 
540 #if !defined(HAVE_TCP_LRO_RX)
541 		tcp_lro_queue_mbuf(&rq->lro, mb);
542 #else
543 		if (mb->m_pkthdr.csum_flags == 0 ||
544 		    (rq->ifp->if_capenable & IFCAP_LRO) == 0 ||
545 		    rq->lro.lro_cnt == 0 ||
546 		    tcp_lro_rx(&rq->lro, mb, 0) != 0) {
547 			rq->ifp->if_input(rq->ifp, mb);
548 		}
549 #endif
550 wq_ll_pop:
551 		mlx5_wq_ll_pop(&rq->wq, wqe_counter_be,
552 		    &wqe->next.next_wqe_index);
553 	}
554 	CURVNET_RESTORE();
555 
556 	mlx5_cqwq_update_db_record(&rq->cq.wq);
557 
558 	/* ensure cq space is freed before enabling more cqes */
559 	atomic_thread_fence_rel();
560 	return (i);
561 }
562 
563 void
564 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
565 {
566 	struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq);
567 	int i = 0;
568 
569 #ifdef HAVE_PER_CQ_EVENT_PACKET
570 #if (MHLEN < 15)
571 #error "MHLEN is too small"
572 #endif
573 	struct mbuf *mb = m_gethdr(M_NOWAIT, MT_DATA);
574 
575 	if (mb != NULL) {
576 		/* this code is used for debugging purpose only */
577 		mb->m_pkthdr.len = mb->m_len = 15;
578 		memset(mb->m_data, 255, 14);
579 		mb->m_data[14] = rq->ix;
580 		mb->m_pkthdr.rcvif = rq->ifp;
581 		rq->ifp->if_input(rq->ifp, mb);
582 	}
583 #endif
584 
585 	mtx_lock(&rq->mtx);
586 
587 	/*
588 	 * Polling the entire CQ without posting new WQEs results in
589 	 * lack of receive WQEs during heavy traffic scenarios.
590 	 */
591 	while (1) {
592 		if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) !=
593 		    MLX5E_RX_BUDGET_MAX)
594 			break;
595 		i += MLX5E_RX_BUDGET_MAX;
596 		if (i >= MLX5E_BUDGET_MAX)
597 			break;
598 		mlx5e_post_rx_wqes(rq);
599 	}
600 	mlx5e_post_rx_wqes(rq);
601 	/* check for dynamic interrupt moderation callback */
602 	if (rq->dim.mode != NET_DIM_CQ_PERIOD_MODE_DISABLED)
603 		net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes);
604 	mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock));
605 	tcp_lro_flush_all(&rq->lro);
606 	mtx_unlock(&rq->mtx);
607 }
608