1 /*-
2 * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "opt_rss.h"
27 #include "opt_ratelimit.h"
28
29 #include <dev/mlx5/mlx5_en/en.h>
30 #include <machine/in_cksum.h>
31 #include <dev/mlx5/mlx5_accel/ipsec.h>
32
33 static inline int
mlx5e_alloc_rx_wqe(struct mlx5e_rq * rq,struct mlx5e_rx_wqe * wqe,u16 ix)34 mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
35 struct mlx5e_rx_wqe *wqe, u16 ix)
36 {
37 bus_dma_segment_t segs[MLX5E_MAX_BUSDMA_RX_SEGS];
38 struct mbuf *mb;
39 int nsegs;
40 int err;
41 struct mbuf *mb_head;
42 int i;
43
44 if (rq->mbuf[ix].mbuf != NULL)
45 return (0);
46
47 mb_head = mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR,
48 MLX5E_MAX_RX_BYTES);
49 if (unlikely(mb == NULL))
50 return (-ENOMEM);
51
52 mb->m_len = MLX5E_MAX_RX_BYTES;
53 mb->m_pkthdr.len = MLX5E_MAX_RX_BYTES;
54
55 for (i = 1; i < rq->nsegs; i++) {
56 if (mb_head->m_pkthdr.len >= rq->wqe_sz)
57 break;
58 mb = mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0,
59 MLX5E_MAX_RX_BYTES);
60 if (unlikely(mb == NULL)) {
61 m_freem(mb_head);
62 return (-ENOMEM);
63 }
64 mb->m_len = MLX5E_MAX_RX_BYTES;
65 mb_head->m_pkthdr.len += MLX5E_MAX_RX_BYTES;
66 }
67 /* rewind to first mbuf in chain */
68 mb = mb_head;
69
70 /* get IP header aligned */
71 m_adj(mb, MLX5E_NET_IP_ALIGN);
72
73 err = mlx5_accel_ipsec_rx_tag_add(rq->ifp, &rq->mbuf[ix]);
74 if (err)
75 goto err_free_mbuf;
76 err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
77 mb, segs, &nsegs, BUS_DMA_NOWAIT);
78 if (err != 0)
79 goto err_free_mbuf;
80 if (unlikely(nsegs == 0)) {
81 bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map);
82 err = -ENOMEM;
83 goto err_free_mbuf;
84 }
85 wqe->data[0].addr = cpu_to_be64(segs[0].ds_addr);
86 wqe->data[0].byte_count = cpu_to_be32(segs[0].ds_len |
87 MLX5_HW_START_PADDING);
88 for (i = 1; i != nsegs; i++) {
89 wqe->data[i].addr = cpu_to_be64(segs[i].ds_addr);
90 wqe->data[i].byte_count = cpu_to_be32(segs[i].ds_len);
91 }
92 for (; i < rq->nsegs; i++) {
93 wqe->data[i].addr = 0;
94 wqe->data[i].byte_count = 0;
95 }
96
97 rq->mbuf[ix].mbuf = mb;
98 rq->mbuf[ix].data = mb->m_data;
99
100 bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map,
101 BUS_DMASYNC_PREREAD);
102 return (0);
103
104 err_free_mbuf:
105 m_freem(mb);
106 return (err);
107 }
108
109 static void
mlx5e_post_rx_wqes(struct mlx5e_rq * rq)110 mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
111 {
112 if (unlikely(rq->enabled == 0))
113 return;
114
115 while (!mlx5_wq_ll_is_full(&rq->wq)) {
116 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head);
117
118 if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) {
119 callout_reset_curcpu(&rq->watchdog, 1, (void *)&mlx5e_post_rx_wqes, rq);
120 break;
121 }
122 mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index));
123 }
124
125 /* ensure wqes are visible to device before updating doorbell record */
126 atomic_thread_fence_rel();
127
128 mlx5_wq_ll_update_db_record(&rq->wq);
129 }
130
131 static void
mlx5e_lro_update_hdr(struct mbuf * mb,struct mlx5_cqe64 * cqe)132 mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe)
133 {
134 /* TODO: consider vlans, ip options, ... */
135 struct ether_header *eh;
136 uint16_t eh_type;
137 uint16_t tot_len;
138 struct ip6_hdr *ip6 = NULL;
139 struct ip *ip4 = NULL;
140 struct tcphdr *th;
141 uint32_t *ts_ptr;
142 uint8_t l4_hdr_type;
143 int tcp_ack;
144
145 eh = mtod(mb, struct ether_header *);
146 eh_type = ntohs(eh->ether_type);
147
148 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
149 tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) ||
150 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type));
151
152 /* TODO: consider vlan */
153 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN;
154
155 switch (eh_type) {
156 case ETHERTYPE_IP:
157 ip4 = (struct ip *)(eh + 1);
158 th = (struct tcphdr *)(ip4 + 1);
159 break;
160 case ETHERTYPE_IPV6:
161 ip6 = (struct ip6_hdr *)(eh + 1);
162 th = (struct tcphdr *)(ip6 + 1);
163 break;
164 default:
165 return;
166 }
167
168 ts_ptr = (uint32_t *)(th + 1);
169
170 if (get_cqe_lro_tcppsh(cqe))
171 th->th_flags |= TH_PUSH;
172
173 if (tcp_ack) {
174 th->th_flags |= TH_ACK;
175 th->th_ack = cqe->lro_ack_seq_num;
176 th->th_win = cqe->lro_tcp_win;
177
178 /*
179 * FreeBSD handles only 32bit aligned timestamp right after
180 * the TCP hdr
181 * +--------+--------+--------+--------+
182 * | NOP | NOP | TSopt | 10 |
183 * +--------+--------+--------+--------+
184 * | TSval timestamp |
185 * +--------+--------+--------+--------+
186 * | TSecr timestamp |
187 * +--------+--------+--------+--------+
188 */
189 if (get_cqe_lro_timestamp_valid(cqe) &&
190 (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 |
191 TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 |
192 TCPOLEN_TIMESTAMP))) {
193 /*
194 * cqe->timestamp is 64bit long.
195 * [0-31] - timestamp.
196 * [32-64] - timestamp echo replay.
197 */
198 ts_ptr[1] = *(uint32_t *)&cqe->timestamp;
199 ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1);
200 }
201 }
202 if (ip4) {
203 ip4->ip_ttl = cqe->lro_min_ttl;
204 ip4->ip_len = cpu_to_be16(tot_len);
205 ip4->ip_sum = 0;
206 ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2);
207 } else {
208 ip6->ip6_hlim = cqe->lro_min_ttl;
209 ip6->ip6_plen = cpu_to_be16(tot_len -
210 sizeof(struct ip6_hdr));
211 }
212 /* TODO: handle tcp checksum */
213 }
214
215 static uint64_t
mlx5e_mbuf_tstmp(struct mlx5e_priv * priv,uint64_t hw_tstmp)216 mlx5e_mbuf_tstmp(struct mlx5e_priv *priv, uint64_t hw_tstmp)
217 {
218 struct mlx5e_clbr_point *cp, dcp;
219 uint64_t tstmp_sec, tstmp_nsec;
220 uint64_t hw_clocks;
221 uint64_t rt_cur_to_prev, res_s, res_n, res_s_modulo, res;
222 uint64_t hw_clk_div;
223 u_int gen;
224
225 do {
226 cp = &priv->clbr_points[priv->clbr_curr];
227 gen = atomic_load_acq_int(&cp->clbr_gen);
228 if (gen == 0)
229 return (0);
230 dcp = *cp;
231 atomic_thread_fence_acq();
232 } while (gen != dcp.clbr_gen);
233 /*
234 * Our goal here is to have a result that is:
235 *
236 * ( (cur_time - prev_time) )
237 * ((hw_tstmp - hw_prev) * ----------------------------- ) + prev_time
238 * ( (hw_cur - hw_prev) )
239 *
240 * With the constraints that we cannot use float and we
241 * don't want to overflow the uint64_t numbers we are using.
242 *
243 * The plan is to take the clocking value of the hw timestamps
244 * and split them into seconds and nanosecond equivalent portions.
245 * Then we operate on the two portions seperately making sure to
246 * bring back the carry over from the seconds when we divide.
247 *
248 * First up lets get the two divided into separate entities
249 * i.e. the seconds. We use the clock frequency for this.
250 * Note that priv->cclk was setup with the clock frequency
251 * in hz so we are all set to go.
252 */
253 hw_clocks = hw_tstmp - dcp.clbr_hw_prev;
254 tstmp_sec = hw_clocks / priv->cclk;
255 tstmp_nsec = hw_clocks % priv->cclk;
256 /* Now work with them separately */
257 rt_cur_to_prev = (dcp.base_curr - dcp.base_prev);
258 res_s = tstmp_sec * rt_cur_to_prev;
259 res_n = tstmp_nsec * rt_cur_to_prev;
260 /* Now lets get our divider */
261 hw_clk_div = dcp.clbr_hw_curr - dcp.clbr_hw_prev;
262 /* Make sure to save the remainder from the seconds divide */
263 res_s_modulo = res_s % hw_clk_div;
264 res_s /= hw_clk_div;
265 /* scale the remainder to where it should be */
266 res_s_modulo *= priv->cclk;
267 /* Now add in the remainder */
268 res_n += res_s_modulo;
269 /* Now do the divide */
270 res_n /= hw_clk_div;
271 res_s *= priv->cclk;
272 /* Recombine the two */
273 res = res_s + res_n;
274 /* And now add in the base time to get to the real timestamp */
275 res += dcp.base_prev;
276 return (res);
277 }
278
279 static inline void
mlx5e_build_rx_mbuf(struct mlx5_cqe64 * cqe,struct mlx5e_rq * rq,struct mbuf * mb,struct mlx5e_rq_mbuf * mr,u32 cqe_bcnt)280 mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, struct mlx5e_rq *rq,
281 struct mbuf *mb, struct mlx5e_rq_mbuf *mr, u32 cqe_bcnt)
282 {
283 if_t ifp = rq->ifp;
284 struct mlx5e_channel *c;
285 struct mbuf *mb_head;
286 int lro_num_seg; /* HW LRO session aggregated packets counter */
287 uint64_t tstmp;
288
289 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
290 if (lro_num_seg > 1) {
291 mlx5e_lro_update_hdr(mb, cqe);
292 rq->stats.lro_packets++;
293 rq->stats.lro_bytes += cqe_bcnt;
294 }
295
296 mb->m_pkthdr.len = cqe_bcnt;
297 for (mb_head = mb; mb != NULL; mb = mb->m_next) {
298 if (mb->m_len > cqe_bcnt)
299 mb->m_len = cqe_bcnt;
300 cqe_bcnt -= mb->m_len;
301 if (likely(cqe_bcnt == 0)) {
302 if (likely(mb->m_next != NULL)) {
303 /* trim off empty mbufs */
304 m_freem(mb->m_next);
305 mb->m_next = NULL;
306 }
307 break;
308 }
309 }
310 /* rewind to first mbuf in chain */
311 mb = mb_head;
312
313 /* check if a Toeplitz hash was computed */
314 if (cqe->rss_hash_type != 0) {
315 mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result);
316 #ifdef RSS
317 /* decode the RSS hash type */
318 switch (cqe->rss_hash_type &
319 (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) {
320 /* IPv4 */
321 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4):
322 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4);
323 break;
324 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4):
325 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4);
326 break;
327 case CQE_RSS_DST_HTYPE_IPV4:
328 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4);
329 break;
330 /* IPv6 */
331 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6):
332 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6);
333 break;
334 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6):
335 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6);
336 break;
337 case CQE_RSS_DST_HTYPE_IPV6:
338 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6);
339 break;
340 default: /* Other */
341 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH);
342 break;
343 }
344 #else
345 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH);
346 #endif
347 #ifdef M_HASHTYPE_SETINNER
348 if (cqe_is_tunneled(cqe))
349 M_HASHTYPE_SETINNER(mb);
350 #endif
351 } else {
352 mb->m_pkthdr.flowid = rq->ix;
353 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
354 }
355 mb->m_pkthdr.rcvif = ifp;
356 mb->m_pkthdr.leaf_rcvif = ifp;
357
358 if (cqe_is_tunneled(cqe)) {
359 /*
360 * CQE can be tunneled only if TIR is configured to
361 * enable parsing of tunneled payload, so no need to
362 * check for capabilities.
363 */
364 if (((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK)) ==
365 (CQE_L2_OK | CQE_L3_OK))) {
366 mb->m_pkthdr.csum_flags |=
367 CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |
368 CSUM_IP_CHECKED | CSUM_IP_VALID |
369 CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
370 mb->m_pkthdr.csum_data = htons(0xffff);
371
372 if (likely((cqe->hds_ip_ext & CQE_L4_OK) == CQE_L4_OK)) {
373 mb->m_pkthdr.csum_flags |=
374 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID;
375 }
376 } else {
377 rq->stats.csum_none++;
378 }
379 } else if (likely((if_getcapenable(ifp) & (IFCAP_RXCSUM |
380 IFCAP_RXCSUM_IPV6)) != 0) &&
381 ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) ==
382 (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) {
383 mb->m_pkthdr.csum_flags =
384 CSUM_IP_CHECKED | CSUM_IP_VALID |
385 CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
386 mb->m_pkthdr.csum_data = htons(0xffff);
387 } else {
388 rq->stats.csum_none++;
389 }
390
391 if (cqe_has_vlan(cqe)) {
392 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info);
393 mb->m_flags |= M_VLANTAG;
394 }
395
396 c = container_of(rq, struct mlx5e_channel, rq);
397 if (c->priv->clbr_done >= 2) {
398 tstmp = mlx5e_mbuf_tstmp(c->priv, be64_to_cpu(cqe->timestamp));
399 if ((tstmp & MLX5_CQE_TSTMP_PTP) != 0) {
400 /*
401 * Timestamp was taken on the packet entrance,
402 * instead of the cqe generation.
403 */
404 tstmp &= ~MLX5_CQE_TSTMP_PTP;
405 mb->m_flags |= M_TSTMP_HPREC;
406 }
407 if (tstmp != 0) {
408 mb->m_pkthdr.rcv_tstmp = tstmp;
409 mb->m_flags |= M_TSTMP;
410 }
411 }
412 switch (get_cqe_tls_offload(cqe)) {
413 case CQE_TLS_OFFLOAD_DECRYPTED:
414 /* set proper checksum flag for decrypted packets */
415 mb->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED;
416 rq->stats.decrypted_ok_packets++;
417 break;
418 case CQE_TLS_OFFLOAD_ERROR:
419 rq->stats.decrypted_error_packets++;
420 break;
421 default:
422 break;
423 }
424
425 mlx5e_accel_ipsec_handle_rx(mb, cqe, mr);
426 }
427
428 static inline void
mlx5e_read_cqe_slot(struct mlx5e_cq * cq,u32 cc,void * data)429 mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
430 {
431 memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)),
432 sizeof(struct mlx5_cqe64));
433 }
434
435 static inline void
mlx5e_write_cqe_slot(struct mlx5e_cq * cq,u32 cc,void * data)436 mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
437 {
438 memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1),
439 data, sizeof(struct mlx5_cqe64));
440 }
441
442 static inline void
mlx5e_decompress_cqe(struct mlx5e_cq * cq,struct mlx5_cqe64 * title,struct mlx5_mini_cqe8 * mini,u16 wqe_counter,int i)443 mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title,
444 struct mlx5_mini_cqe8 *mini,
445 u16 wqe_counter, int i)
446 {
447 /*
448 * NOTE: The fields which are not set here are copied from the
449 * initial and common title. See memcpy() in
450 * mlx5e_write_cqe_slot().
451 */
452 title->byte_cnt = mini->byte_cnt;
453 title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1);
454 title->rss_hash_result = mini->rx_hash_result;
455 /*
456 * Since we use MLX5_CQE_FORMAT_HASH when creating the RX CQ,
457 * the value of the checksum should be ignored.
458 */
459 title->check_sum = 0;
460 title->op_own = (title->op_own & 0xf0) |
461 (((cq->wq.cc + i) >> cq->wq.log_sz) & 1);
462 }
463
464 #define MLX5E_MINI_ARRAY_SZ 8
465 /* Make sure structs are not packet differently */
466 CTASSERT(sizeof(struct mlx5_cqe64) ==
467 sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ);
468 static void
mlx5e_decompress_cqes(struct mlx5e_cq * cq)469 mlx5e_decompress_cqes(struct mlx5e_cq *cq)
470 {
471 struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ];
472 struct mlx5_cqe64 title;
473 u32 cqe_count;
474 u32 i = 0;
475 u16 title_wqe_counter;
476
477 mlx5e_read_cqe_slot(cq, cq->wq.cc, &title);
478 title_wqe_counter = be16_to_cpu(title.wqe_counter);
479 cqe_count = be32_to_cpu(title.byte_cnt);
480
481 /* Make sure we won't overflow */
482 KASSERT(cqe_count <= cq->wq.sz_m1,
483 ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__,
484 cqe_count, cq->wq.sz_m1));
485
486 mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array);
487 while (true) {
488 mlx5e_decompress_cqe(cq, &title,
489 &mini_array[i % MLX5E_MINI_ARRAY_SZ],
490 title_wqe_counter, i);
491 mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title);
492 i++;
493
494 if (i == cqe_count)
495 break;
496 if (i % MLX5E_MINI_ARRAY_SZ == 0)
497 mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array);
498 }
499 }
500
501 static int
mlx5e_poll_rx_cq(struct mlx5e_rq * rq,int budget)502 mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
503 {
504 struct pfil_head *pfil;
505 int i, rv;
506
507 CURVNET_SET_QUIET(if_getvnet(rq->ifp));
508 pfil = rq->channel->priv->pfil;
509 for (i = 0; i < budget; i++) {
510 struct mlx5e_rx_wqe *wqe;
511 struct mlx5_cqe64 *cqe;
512 struct mbuf *mb;
513 __be16 wqe_counter_be;
514 u16 wqe_counter;
515 u32 byte_cnt, seglen;
516
517 cqe = mlx5e_get_cqe(&rq->cq);
518 if (!cqe)
519 break;
520
521 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED)
522 mlx5e_decompress_cqes(&rq->cq);
523
524 mlx5_cqwq_pop(&rq->cq.wq);
525
526 wqe_counter_be = cqe->wqe_counter;
527 wqe_counter = be16_to_cpu(wqe_counter_be);
528 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
529 byte_cnt = be32_to_cpu(cqe->byte_cnt);
530
531 bus_dmamap_sync(rq->dma_tag,
532 rq->mbuf[wqe_counter].dma_map,
533 BUS_DMASYNC_POSTREAD);
534
535 if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
536 mlx5e_dump_err_cqe(&rq->cq, rq->rqn, (const void *)cqe);
537 rq->stats.wqe_err++;
538 goto wq_ll_pop;
539 }
540 if (pfil != NULL && PFIL_HOOKED_IN(pfil)) {
541 seglen = MIN(byte_cnt, MLX5E_MAX_RX_BYTES);
542 rv = pfil_mem_in(rq->channel->priv->pfil,
543 rq->mbuf[wqe_counter].data, seglen, rq->ifp, &mb);
544
545 switch (rv) {
546 case PFIL_DROPPED:
547 case PFIL_CONSUMED:
548 /*
549 * Filter dropped or consumed it. In
550 * either case, we can just recycle
551 * buffer; there is no more work to do.
552 */
553 rq->stats.packets++;
554 goto wq_ll_pop;
555 case PFIL_REALLOCED:
556 /*
557 * Filter copied it; recycle buffer
558 * and receive the new mbuf allocated
559 * by the Filter
560 */
561 goto rx_common;
562 default:
563 /*
564 * The Filter said it was OK, so
565 * receive like normal.
566 */
567 KASSERT(rv == PFIL_PASS,
568 ("Filter returned %d!\n", rv));
569 }
570 }
571 if (!mlx5e_accel_ipsec_flow(cqe) /* tag is already assigned
572 to rq->mbuf */ &&
573 MHLEN - MLX5E_NET_IP_ALIGN >= byte_cnt &&
574 (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
575 /* set maximum mbuf length */
576 mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN;
577 /* get IP header aligned */
578 mb->m_data += MLX5E_NET_IP_ALIGN;
579
580 bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t),
581 byte_cnt);
582 } else {
583 mb = rq->mbuf[wqe_counter].mbuf;
584 rq->mbuf[wqe_counter].mbuf = NULL; /* safety clear */
585
586 bus_dmamap_unload(rq->dma_tag,
587 rq->mbuf[wqe_counter].dma_map);
588 }
589 rx_common:
590 mlx5e_build_rx_mbuf(cqe, rq, mb, &rq->mbuf[wqe_counter],
591 byte_cnt);
592 rq->stats.bytes += byte_cnt;
593 rq->stats.packets++;
594 #ifdef NUMA
595 mb->m_pkthdr.numa_domain = if_getnumadomain(rq->ifp);
596 #endif
597
598 #if !defined(HAVE_TCP_LRO_RX)
599 tcp_lro_queue_mbuf(&rq->lro, mb);
600 #else
601 if (mb->m_pkthdr.csum_flags == 0 ||
602 (if_getcapenable(rq->ifp) & IFCAP_LRO) == 0 ||
603 rq->lro.lro_cnt == 0 ||
604 tcp_lro_rx(&rq->lro, mb, 0) != 0) {
605 if_input(rq->ifp, mb);
606 }
607 #endif
608 wq_ll_pop:
609 mlx5_wq_ll_pop(&rq->wq, wqe_counter_be,
610 &wqe->next.next_wqe_index);
611 }
612 CURVNET_RESTORE();
613
614 mlx5_cqwq_update_db_record(&rq->cq.wq);
615
616 /* ensure cq space is freed before enabling more cqes */
617 atomic_thread_fence_rel();
618 return (i);
619 }
620
621 void
mlx5e_rx_cq_comp(struct mlx5_core_cq * mcq,struct mlx5_eqe * eqe __unused)622 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
623 {
624 struct mlx5e_channel *c = container_of(mcq, struct mlx5e_channel, rq.cq.mcq);
625 struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq);
626 int i = 0;
627
628 #ifdef HAVE_PER_CQ_EVENT_PACKET
629 #if (MHLEN < 15)
630 #error "MHLEN is too small"
631 #endif
632 struct mbuf *mb = m_gethdr(M_NOWAIT, MT_DATA);
633
634 if (mb != NULL) {
635 /* this code is used for debugging purpose only */
636 mb->m_pkthdr.len = mb->m_len = 15;
637 memset(mb->m_data, 255, 14);
638 mb->m_data[14] = rq->ix;
639 mb->m_pkthdr.rcvif = rq->ifp;
640 mb->m_pkthdr.leaf_rcvif = rq->ifp;
641 if_input(rq->ifp, mb);
642 }
643 #endif
644 for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) {
645 mtx_lock(&c->sq[j].lock);
646 c->sq[j].db_inhibit++;
647 mtx_unlock(&c->sq[j].lock);
648 }
649
650 mtx_lock(&c->iq.lock);
651 c->iq.db_inhibit++;
652 mtx_unlock(&c->iq.lock);
653
654 mtx_lock(&rq->mtx);
655
656 /*
657 * Polling the entire CQ without posting new WQEs results in
658 * lack of receive WQEs during heavy traffic scenarios.
659 */
660 while (1) {
661 if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) !=
662 MLX5E_RX_BUDGET_MAX)
663 break;
664 i += MLX5E_RX_BUDGET_MAX;
665 if (i >= MLX5E_BUDGET_MAX)
666 break;
667 mlx5e_post_rx_wqes(rq);
668 }
669 mlx5e_post_rx_wqes(rq);
670 /* check for dynamic interrupt moderation callback */
671 if (rq->dim.mode != NET_DIM_CQ_PERIOD_MODE_DISABLED)
672 net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes);
673 mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock));
674 tcp_lro_flush_all(&rq->lro);
675 mtx_unlock(&rq->mtx);
676
677 for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) {
678 mtx_lock(&c->sq[j].lock);
679 c->sq[j].db_inhibit--;
680 /* Update the doorbell record, if any. */
681 mlx5e_tx_notify_hw(c->sq + j, true);
682 mtx_unlock(&c->sq[j].lock);
683 }
684
685 mtx_lock(&c->iq.lock);
686 c->iq.db_inhibit--;
687 mlx5e_iq_notify_hw(&c->iq);
688 mtx_unlock(&c->iq.lock);
689 }
690