1 /*-
2 * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "opt_rss.h"
27 #include "opt_ratelimit.h"
28
29 #include <dev/mlx5/mlx5_en/en.h>
30 #include <netinet/ip_var.h>
31 #include <machine/in_cksum.h>
32 #include <dev/mlx5/mlx5_accel/ipsec.h>
33
34 static inline int
mlx5e_alloc_rx_wqe(struct mlx5e_rq * rq,struct mlx5e_rx_wqe * wqe,u16 ix)35 mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
36 struct mlx5e_rx_wqe *wqe, u16 ix)
37 {
38 bus_dma_segment_t segs[MLX5E_MAX_BUSDMA_RX_SEGS];
39 struct mbuf *mb;
40 int nsegs;
41 int err;
42 struct mbuf *mb_head;
43 int i;
44
45 if (rq->mbuf[ix].mbuf != NULL)
46 return (0);
47
48 mb_head = mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz);
49 if (unlikely(mb == NULL))
50 return (-ENOMEM);
51
52 mb->m_len = rq->wqe_sz;
53 mb->m_pkthdr.len = rq->wqe_sz;
54
55 for (i = 1; i < rq->nsegs; i++) {
56 mb = mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, rq->wqe_sz);
57 if (unlikely(mb == NULL)) {
58 m_freem(mb_head);
59 return (-ENOMEM);
60 }
61 mb->m_len = rq->wqe_sz;
62 mb_head->m_pkthdr.len += rq->wqe_sz;
63 }
64 /* rewind to first mbuf in chain */
65 mb = mb_head;
66
67 /* get IP header aligned */
68 m_adj(mb, MLX5E_NET_IP_ALIGN);
69
70 err = mlx5_accel_ipsec_rx_tag_add(rq->ifp, &rq->mbuf[ix]);
71 if (err)
72 goto err_free_mbuf;
73 err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
74 mb, segs, &nsegs, BUS_DMA_NOWAIT);
75 if (err != 0)
76 goto err_free_mbuf;
77 if (unlikely(nsegs == 0)) {
78 bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map);
79 err = -ENOMEM;
80 goto err_free_mbuf;
81 }
82 wqe->data[0].addr = cpu_to_be64(segs[0].ds_addr);
83 wqe->data[0].byte_count = cpu_to_be32(segs[0].ds_len |
84 MLX5_HW_START_PADDING);
85 for (i = 1; i != nsegs; i++) {
86 wqe->data[i].addr = cpu_to_be64(segs[i].ds_addr);
87 wqe->data[i].byte_count = cpu_to_be32(segs[i].ds_len);
88 }
89 for (; i < rq->nsegs; i++) {
90 wqe->data[i].addr = 0;
91 wqe->data[i].byte_count = 0;
92 }
93
94 rq->mbuf[ix].mbuf = mb;
95 rq->mbuf[ix].data = mb->m_data;
96
97 bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map,
98 BUS_DMASYNC_PREREAD);
99 return (0);
100
101 err_free_mbuf:
102 m_freem(mb);
103 return (err);
104 }
105
106 static void
mlx5e_post_rx_wqes(struct mlx5e_rq * rq)107 mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
108 {
109 if (unlikely(rq->enabled == 0))
110 return;
111
112 while (!mlx5_wq_ll_is_full(&rq->wq)) {
113 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head);
114
115 if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) {
116 callout_reset_curcpu(&rq->watchdog, 1, (void *)&mlx5e_post_rx_wqes, rq);
117 break;
118 }
119 mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index));
120 }
121
122 /* ensure wqes are visible to device before updating doorbell record */
123 atomic_thread_fence_rel();
124
125 mlx5_wq_ll_update_db_record(&rq->wq);
126 }
127
128 static uint32_t
csum_reduce(uint32_t val)129 csum_reduce(uint32_t val)
130 {
131 while (val > 0xffff)
132 val = (val >> 16) + (val & 0xffff);
133 return (val);
134 }
135
136 static u_short
csum_buf(uint32_t val,void * buf,int len)137 csum_buf(uint32_t val, void *buf, int len)
138 {
139 u_short x;
140
141 MPASS(len % 2 == 0);
142 for (int i = 0; i < len; i += 2) {
143 bcopy((char *)buf + i, &x, 2);
144 val = csum_reduce(val + x);
145 }
146 return (val);
147 }
148
149 static void
mlx5e_lro_update_hdr(struct mbuf * mb,struct mlx5_cqe64 * cqe)150 mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe)
151 {
152 /* TODO: consider vlans, ip options, ... */
153 struct ether_header *eh;
154 uint16_t eh_type;
155 uint16_t tot_len;
156 struct ip6_hdr *ip6 = NULL;
157 struct ip *ip4 = NULL;
158 struct tcphdr *th;
159 uint32_t *ts_ptr;
160 uint32_t tcp_csum;
161 uint8_t l4_hdr_type;
162 int tcp_ack;
163
164 eh = mtod(mb, struct ether_header *);
165 eh_type = ntohs(eh->ether_type);
166
167 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
168 tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) ||
169 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type));
170
171 /* TODO: consider vlan */
172 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN;
173
174 switch (eh_type) {
175 case ETHERTYPE_IP:
176 ip4 = (struct ip *)(eh + 1);
177 th = (struct tcphdr *)(ip4 + 1);
178 break;
179 case ETHERTYPE_IPV6:
180 ip6 = (struct ip6_hdr *)(eh + 1);
181 th = (struct tcphdr *)(ip6 + 1);
182 break;
183 default:
184 return;
185 }
186
187 ts_ptr = (uint32_t *)(th + 1);
188
189 if (get_cqe_lro_tcppsh(cqe))
190 tcp_set_flags(th, tcp_get_flags(th) | TH_PUSH);
191
192 if (tcp_ack) {
193 tcp_set_flags(th, tcp_get_flags(th) | TH_ACK);
194 th->th_ack = cqe->lro_ack_seq_num;
195 th->th_win = cqe->lro_tcp_win;
196
197 /*
198 * FreeBSD handles only 32bit aligned timestamp right after
199 * the TCP hdr
200 * +--------+--------+--------+--------+
201 * | NOP | NOP | TSopt | 10 |
202 * +--------+--------+--------+--------+
203 * | TSval timestamp |
204 * +--------+--------+--------+--------+
205 * | TSecr timestamp |
206 * +--------+--------+--------+--------+
207 */
208 if (get_cqe_lro_timestamp_valid(cqe) &&
209 (__predict_true(*ts_ptr == ntohl(TCPOPT_NOP << 24 |
210 TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 |
211 TCPOLEN_TIMESTAMP)))) {
212 /*
213 * cqe->timestamp is 64bit long.
214 * [0-31] - timestamp.
215 * [32-64] - timestamp echo replay.
216 */
217 ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1);
218 }
219 }
220 if (ip4) {
221 struct ipovly io;
222
223 ip4->ip_ttl = cqe->lro_min_ttl;
224 ip4->ip_len = cpu_to_be16(tot_len);
225 ip4->ip_sum = 0;
226 ip4->ip_sum = in_cksum_skip(mb, (ip4->ip_hl << 2) +
227 ETHER_HDR_LEN, ETHER_HDR_LEN);
228
229 /* TCP checksum: data */
230 tcp_csum = cqe->check_sum;
231
232 /* TCP checksum: IP pseudoheader */
233 bzero(io.ih_x1, sizeof(io.ih_x1));
234 io.ih_pr = IPPROTO_TCP;
235 io.ih_len = htons(ntohs(ip4->ip_len) - sizeof(*ip4));
236 io.ih_src = ip4->ip_src;
237 io.ih_dst = ip4->ip_dst;
238 tcp_csum = csum_buf(tcp_csum, &io, sizeof(io));
239
240 /* TCP checksum: TCP header */
241 th->th_sum = 0;
242 tcp_csum = csum_buf(tcp_csum, th, th->th_off * 4);
243 th->th_sum = ~tcp_csum & 0xffff;
244 } else {
245 ip6->ip6_hlim = cqe->lro_min_ttl;
246 ip6->ip6_plen = cpu_to_be16(tot_len -
247 sizeof(struct ip6_hdr));
248
249 /* TCP checksum */
250 th->th_sum = 0;
251 tcp_csum = ~in6_cksum_partial_l2(mb, IPPROTO_TCP,
252 sizeof(struct ether_header),
253 sizeof(struct ether_header) + sizeof(struct ip6_hdr),
254 tot_len - sizeof(struct ip6_hdr), th->th_off * 4) & 0xffff;
255 tcp_csum = csum_reduce(tcp_csum + cqe->check_sum);
256 th->th_sum = ~tcp_csum & 0xffff;
257 }
258 }
259
260 static uint64_t
mlx5e_mbuf_tstmp(struct mlx5e_priv * priv,uint64_t hw_tstmp)261 mlx5e_mbuf_tstmp(struct mlx5e_priv *priv, uint64_t hw_tstmp)
262 {
263 struct mlx5e_clbr_point *cp, dcp;
264 uint64_t tstmp_sec, tstmp_nsec;
265 uint64_t hw_clocks;
266 uint64_t rt_cur_to_prev, res_s, res_n, res_s_modulo, res;
267 uint64_t hw_clk_div;
268 u_int gen;
269
270 do {
271 cp = &priv->clbr_points[priv->clbr_curr];
272 gen = atomic_load_acq_int(&cp->clbr_gen);
273 if (gen == 0)
274 return (0);
275 dcp = *cp;
276 atomic_thread_fence_acq();
277 } while (gen != dcp.clbr_gen);
278 /*
279 * Our goal here is to have a result that is:
280 *
281 * ( (cur_time - prev_time) )
282 * ((hw_tstmp - hw_prev) * ----------------------------- ) + prev_time
283 * ( (hw_cur - hw_prev) )
284 *
285 * With the constraints that we cannot use float and we
286 * don't want to overflow the uint64_t numbers we are using.
287 *
288 * The plan is to take the clocking value of the hw timestamps
289 * and split them into seconds and nanosecond equivalent portions.
290 * Then we operate on the two portions seperately making sure to
291 * bring back the carry over from the seconds when we divide.
292 *
293 * First up lets get the two divided into separate entities
294 * i.e. the seconds. We use the clock frequency for this.
295 * Note that priv->cclk was setup with the clock frequency
296 * in hz so we are all set to go.
297 */
298 hw_clocks = hw_tstmp - dcp.clbr_hw_prev;
299 tstmp_sec = hw_clocks / priv->cclk;
300 tstmp_nsec = hw_clocks % priv->cclk;
301 /* Now work with them separately */
302 rt_cur_to_prev = (dcp.base_curr - dcp.base_prev);
303 res_s = tstmp_sec * rt_cur_to_prev;
304 res_n = tstmp_nsec * rt_cur_to_prev;
305 /* Now lets get our divider */
306 hw_clk_div = dcp.clbr_hw_curr - dcp.clbr_hw_prev;
307 /* Make sure to save the remainder from the seconds divide */
308 res_s_modulo = res_s % hw_clk_div;
309 res_s /= hw_clk_div;
310 /* scale the remainder to where it should be */
311 res_s_modulo *= priv->cclk;
312 /* Now add in the remainder */
313 res_n += res_s_modulo;
314 /* Now do the divide */
315 res_n /= hw_clk_div;
316 res_s *= priv->cclk;
317 /* Recombine the two */
318 res = res_s + res_n;
319 /* And now add in the base time to get to the real timestamp */
320 res += dcp.base_prev;
321 return (res);
322 }
323
324 static inline void
mlx5e_build_rx_mbuf(struct mlx5_cqe64 * cqe,struct mlx5e_rq * rq,struct mbuf * mb,struct mlx5e_rq_mbuf * mr,u32 cqe_bcnt)325 mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, struct mlx5e_rq *rq,
326 struct mbuf *mb, struct mlx5e_rq_mbuf *mr, u32 cqe_bcnt)
327 {
328 if_t ifp = rq->ifp;
329 struct mlx5e_channel *c;
330 struct mbuf *mb_head;
331 int lro_num_seg; /* HW LRO session aggregated packets counter */
332 uint64_t tstmp;
333
334 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
335 if (lro_num_seg > 1) {
336 mlx5e_lro_update_hdr(mb, cqe);
337 rq->stats.lro_packets++;
338 rq->stats.lro_bytes += cqe_bcnt;
339 }
340
341 mb->m_pkthdr.len = cqe_bcnt;
342 for (mb_head = mb; mb != NULL; mb = mb->m_next) {
343 if (mb->m_len > cqe_bcnt)
344 mb->m_len = cqe_bcnt;
345 cqe_bcnt -= mb->m_len;
346 if (likely(cqe_bcnt == 0)) {
347 if (likely(mb->m_next != NULL)) {
348 /* trim off empty mbufs */
349 m_freem(mb->m_next);
350 mb->m_next = NULL;
351 }
352 break;
353 }
354 }
355 /* rewind to first mbuf in chain */
356 mb = mb_head;
357
358 /* check if a Toeplitz hash was computed */
359 if (cqe->rss_hash_type != 0) {
360 mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result);
361 /* decode the RSS hash type */
362 switch (cqe->rss_hash_type &
363 (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) {
364 /* IPv4 */
365 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4):
366 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4);
367 break;
368 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4):
369 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4);
370 break;
371 case CQE_RSS_DST_HTYPE_IPV4:
372 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4);
373 break;
374 /* IPv6 */
375 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6):
376 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6);
377 break;
378 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6):
379 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6);
380 break;
381 case CQE_RSS_DST_HTYPE_IPV6:
382 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6);
383 break;
384 default: /* Other */
385 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH);
386 break;
387 }
388 #ifdef M_HASHTYPE_SETINNER
389 if (cqe_is_tunneled(cqe))
390 M_HASHTYPE_SETINNER(mb);
391 #endif
392 } else {
393 mb->m_pkthdr.flowid = rq->ix;
394 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
395 }
396 mb->m_pkthdr.rcvif = ifp;
397 mb->m_pkthdr.leaf_rcvif = ifp;
398
399 if (cqe_is_tunneled(cqe)) {
400 /*
401 * CQE can be tunneled only if TIR is configured to
402 * enable parsing of tunneled payload, so no need to
403 * check for capabilities.
404 */
405 if (((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK)) ==
406 (CQE_L2_OK | CQE_L3_OK))) {
407 mb->m_pkthdr.csum_flags |=
408 CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |
409 CSUM_IP_CHECKED | CSUM_IP_VALID |
410 CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
411 mb->m_pkthdr.csum_data = htons(0xffff);
412
413 if (likely((cqe->hds_ip_ext & CQE_L4_OK) == CQE_L4_OK)) {
414 mb->m_pkthdr.csum_flags |=
415 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID;
416 }
417 } else {
418 rq->stats.csum_none++;
419 }
420 } else if (likely((if_getcapenable(ifp) & (IFCAP_RXCSUM |
421 IFCAP_RXCSUM_IPV6)) != 0) &&
422 ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) ==
423 (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) {
424 mb->m_pkthdr.csum_flags =
425 CSUM_IP_CHECKED | CSUM_IP_VALID |
426 CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
427 mb->m_pkthdr.csum_data = htons(0xffff);
428 } else {
429 rq->stats.csum_none++;
430 }
431
432 if (cqe_has_vlan(cqe)) {
433 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info);
434 mb->m_flags |= M_VLANTAG;
435 }
436
437 c = container_of(rq, struct mlx5e_channel, rq);
438 if (c->priv->clbr_done >= 2) {
439 tstmp = mlx5e_mbuf_tstmp(c->priv, be64_to_cpu(cqe->timestamp));
440 if ((tstmp & MLX5_CQE_TSTMP_PTP) != 0) {
441 /*
442 * Timestamp was taken on the packet entrance,
443 * instead of the cqe generation.
444 */
445 tstmp &= ~MLX5_CQE_TSTMP_PTP;
446 mb->m_flags |= M_TSTMP_HPREC;
447 }
448 if (tstmp != 0) {
449 mb->m_pkthdr.rcv_tstmp = tstmp;
450 mb->m_flags |= M_TSTMP;
451 }
452 }
453 switch (get_cqe_tls_offload(cqe)) {
454 case CQE_TLS_OFFLOAD_DECRYPTED:
455 /* set proper checksum flag for decrypted packets */
456 mb->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED;
457 rq->stats.decrypted_ok_packets++;
458 break;
459 case CQE_TLS_OFFLOAD_ERROR:
460 rq->stats.decrypted_error_packets++;
461 break;
462 default:
463 break;
464 }
465
466 mlx5e_accel_ipsec_handle_rx(ifp, mb, cqe, mr);
467 }
468
469 static inline void
mlx5e_read_cqe_slot(struct mlx5e_cq * cq,u32 cc,void * data)470 mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
471 {
472 memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)),
473 sizeof(struct mlx5_cqe64));
474 }
475
476 static inline void
mlx5e_write_cqe_slot(struct mlx5e_cq * cq,u32 cc,void * data)477 mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
478 {
479 memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1),
480 data, sizeof(struct mlx5_cqe64));
481 }
482
483 static inline void
mlx5e_decompress_cqe(struct mlx5e_cq * cq,struct mlx5_cqe64 * title,struct mlx5_mini_cqe8 * mini,u16 wqe_counter,int i)484 mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title,
485 struct mlx5_mini_cqe8 *mini,
486 u16 wqe_counter, int i)
487 {
488 /*
489 * NOTE: The fields which are not set here are copied from the
490 * initial and common title. See memcpy() in
491 * mlx5e_write_cqe_slot().
492 */
493 title->byte_cnt = mini->byte_cnt;
494 title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1);
495 title->rss_hash_result = mini->rx_hash_result;
496 /*
497 * Since we use MLX5_CQE_FORMAT_HASH when creating the RX CQ,
498 * the value of the checksum should be ignored.
499 */
500 title->check_sum = 0;
501 title->op_own = (title->op_own & 0xf0) |
502 (((cq->wq.cc + i) >> cq->wq.log_sz) & 1);
503 }
504
505 #define MLX5E_MINI_ARRAY_SZ 8
506 /* Make sure structs are not packet differently */
507 CTASSERT(sizeof(struct mlx5_cqe64) ==
508 sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ);
509 static void
mlx5e_decompress_cqes(struct mlx5e_cq * cq)510 mlx5e_decompress_cqes(struct mlx5e_cq *cq)
511 {
512 struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ];
513 struct mlx5_cqe64 title;
514 u32 cqe_count;
515 u32 i = 0;
516 u16 title_wqe_counter;
517
518 mlx5e_read_cqe_slot(cq, cq->wq.cc, &title);
519 title_wqe_counter = be16_to_cpu(title.wqe_counter);
520 cqe_count = be32_to_cpu(title.byte_cnt);
521
522 /* Make sure we won't overflow */
523 KASSERT(cqe_count <= cq->wq.sz_m1,
524 ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__,
525 cqe_count, cq->wq.sz_m1));
526
527 mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array);
528 while (true) {
529 mlx5e_decompress_cqe(cq, &title,
530 &mini_array[i % MLX5E_MINI_ARRAY_SZ],
531 title_wqe_counter, i);
532 mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title);
533 i++;
534
535 if (i == cqe_count)
536 break;
537 if (i % MLX5E_MINI_ARRAY_SZ == 0)
538 mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array);
539 }
540 }
541
542 static int
mlx5e_poll_rx_cq(struct mlx5e_rq * rq,int budget)543 mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
544 {
545 struct pfil_head *pfil;
546 int i, rv;
547
548 CURVNET_SET_QUIET(if_getvnet(rq->ifp));
549 pfil = rq->channel->priv->pfil;
550 for (i = 0; i < budget; i++) {
551 struct mlx5e_rx_wqe *wqe;
552 struct mlx5_cqe64 *cqe;
553 struct mbuf *mb;
554 __be16 wqe_counter_be;
555 u16 wqe_counter;
556 u32 byte_cnt, seglen;
557
558 cqe = mlx5e_get_cqe(&rq->cq);
559 if (!cqe)
560 break;
561
562 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED)
563 mlx5e_decompress_cqes(&rq->cq);
564
565 mlx5_cqwq_pop(&rq->cq.wq);
566
567 wqe_counter_be = cqe->wqe_counter;
568 wqe_counter = be16_to_cpu(wqe_counter_be);
569 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
570 byte_cnt = be32_to_cpu(cqe->byte_cnt);
571
572 bus_dmamap_sync(rq->dma_tag,
573 rq->mbuf[wqe_counter].dma_map,
574 BUS_DMASYNC_POSTREAD);
575
576 if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
577 mlx5e_dump_err_cqe(&rq->cq, rq->rqn, (const void *)cqe);
578 rq->stats.wqe_err++;
579 goto wq_ll_pop;
580 }
581 if (pfil != NULL && PFIL_HOOKED_IN(pfil)) {
582 seglen = MIN(byte_cnt, MLX5E_MAX_RX_BYTES);
583 rv = pfil_mem_in(rq->channel->priv->pfil,
584 rq->mbuf[wqe_counter].data, seglen, rq->ifp, &mb);
585
586 switch (rv) {
587 case PFIL_DROPPED:
588 case PFIL_CONSUMED:
589 /*
590 * Filter dropped or consumed it. In
591 * either case, we can just recycle
592 * buffer; there is no more work to do.
593 */
594 rq->stats.packets++;
595 goto wq_ll_pop;
596 case PFIL_REALLOCED:
597 /*
598 * Filter copied it; recycle buffer
599 * and receive the new mbuf allocated
600 * by the Filter
601 */
602 goto rx_common;
603 default:
604 /*
605 * The Filter said it was OK, so
606 * receive like normal.
607 */
608 KASSERT(rv == PFIL_PASS,
609 ("Filter returned %d!\n", rv));
610 }
611 }
612 if (!mlx5e_accel_ipsec_flow(cqe) /* tag is already assigned
613 to rq->mbuf */ &&
614 MHLEN - MLX5E_NET_IP_ALIGN >= byte_cnt &&
615 (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
616 /* set maximum mbuf length */
617 mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN;
618 /* get IP header aligned */
619 mb->m_data += MLX5E_NET_IP_ALIGN;
620
621 bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t),
622 byte_cnt);
623 } else {
624 mb = rq->mbuf[wqe_counter].mbuf;
625 rq->mbuf[wqe_counter].mbuf = NULL; /* safety clear */
626
627 bus_dmamap_unload(rq->dma_tag,
628 rq->mbuf[wqe_counter].dma_map);
629 }
630 rx_common:
631 mlx5e_build_rx_mbuf(cqe, rq, mb, &rq->mbuf[wqe_counter],
632 byte_cnt);
633 rq->stats.bytes += byte_cnt;
634 rq->stats.packets++;
635 #ifdef NUMA
636 mb->m_pkthdr.numa_domain = if_getnumadomain(rq->ifp);
637 #endif
638
639 #if !defined(HAVE_TCP_LRO_RX)
640 tcp_lro_queue_mbuf(&rq->lro, mb);
641 #else
642 if (mb->m_pkthdr.csum_flags == 0 ||
643 (if_getcapenable(rq->ifp) & IFCAP_LRO) == 0 ||
644 rq->lro.lro_cnt == 0 ||
645 tcp_lro_rx(&rq->lro, mb, 0) != 0) {
646 if_input(rq->ifp, mb);
647 }
648 #endif
649 wq_ll_pop:
650 mlx5_wq_ll_pop(&rq->wq, wqe_counter_be,
651 &wqe->next.next_wqe_index);
652 }
653 CURVNET_RESTORE();
654
655 mlx5_cqwq_update_db_record(&rq->cq.wq);
656
657 /* ensure cq space is freed before enabling more cqes */
658 atomic_thread_fence_rel();
659 return (i);
660 }
661
662 void
mlx5e_rx_cq_comp(struct mlx5_core_cq * mcq,struct mlx5_eqe * eqe __unused)663 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused)
664 {
665 struct mlx5e_channel *c = container_of(mcq, struct mlx5e_channel, rq.cq.mcq);
666 struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq);
667 int i = 0;
668
669 #ifdef HAVE_PER_CQ_EVENT_PACKET
670 #if (MHLEN < 15)
671 #error "MHLEN is too small"
672 #endif
673 struct mbuf *mb = m_gethdr(M_NOWAIT, MT_DATA);
674
675 if (mb != NULL) {
676 /* this code is used for debugging purpose only */
677 mb->m_pkthdr.len = mb->m_len = 15;
678 memset(mb->m_data, 255, 14);
679 mb->m_data[14] = rq->ix;
680 mb->m_pkthdr.rcvif = rq->ifp;
681 mb->m_pkthdr.leaf_rcvif = rq->ifp;
682 if_input(rq->ifp, mb);
683 }
684 #endif
685 for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) {
686 mtx_lock(&c->sq[j].lock);
687 c->sq[j].db_inhibit++;
688 mtx_unlock(&c->sq[j].lock);
689 }
690
691 mtx_lock(&c->iq.lock);
692 c->iq.db_inhibit++;
693 mtx_unlock(&c->iq.lock);
694
695 mtx_lock(&rq->mtx);
696 if (rq->enabled == 0)
697 goto out;
698 rq->processing++;
699
700 /*
701 * Polling the entire CQ without posting new WQEs results in
702 * lack of receive WQEs during heavy traffic scenarios.
703 */
704 while (1) {
705 if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) !=
706 MLX5E_RX_BUDGET_MAX)
707 break;
708 i += MLX5E_RX_BUDGET_MAX;
709 if (i >= MLX5E_BUDGET_MAX)
710 break;
711 mlx5e_post_rx_wqes(rq);
712 }
713 mlx5e_post_rx_wqes(rq);
714 /* check for dynamic interrupt moderation callback */
715 if (rq->dim.mode != NET_DIM_CQ_PERIOD_MODE_DISABLED)
716 net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes);
717 mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock));
718 tcp_lro_flush_all(&rq->lro);
719 rq->processing--;
720 out:
721 mtx_unlock(&rq->mtx);
722
723 for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) {
724 mtx_lock(&c->sq[j].lock);
725 c->sq[j].db_inhibit--;
726 /* Update the doorbell record, if any. */
727 mlx5e_tx_notify_hw(c->sq + j, true);
728 mtx_unlock(&c->sq[j].lock);
729 }
730
731 mtx_lock(&c->iq.lock);
732 c->iq.db_inhibit--;
733 mlx5e_iq_notify_hw(&c->iq);
734 mtx_unlock(&c->iq.lock);
735 }
736