1 /*- 2 * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include "opt_rss.h" 27 #include "opt_ratelimit.h" 28 29 #include <dev/mlx5/mlx5_en/en.h> 30 #include <netinet/ip_var.h> 31 #include <machine/in_cksum.h> 32 #include <dev/mlx5/mlx5_accel/ipsec.h> 33 34 static inline int 35 mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, 36 struct mlx5e_rx_wqe *wqe, u16 ix) 37 { 38 bus_dma_segment_t segs[MLX5E_MAX_BUSDMA_RX_SEGS]; 39 struct mbuf *mb; 40 int nsegs; 41 int err; 42 struct mbuf *mb_head; 43 int i; 44 45 if (rq->mbuf[ix].mbuf != NULL) 46 return (0); 47 48 mb_head = mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz); 49 if (unlikely(mb == NULL)) 50 return (-ENOMEM); 51 52 mb->m_len = rq->wqe_sz; 53 mb->m_pkthdr.len = rq->wqe_sz; 54 55 for (i = 1; i < rq->nsegs; i++) { 56 mb = mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, rq->wqe_sz); 57 if (unlikely(mb == NULL)) { 58 m_freem(mb_head); 59 return (-ENOMEM); 60 } 61 mb->m_len = rq->wqe_sz; 62 mb_head->m_pkthdr.len += rq->wqe_sz; 63 } 64 /* rewind to first mbuf in chain */ 65 mb = mb_head; 66 67 /* get IP header aligned */ 68 m_adj(mb, MLX5E_NET_IP_ALIGN); 69 70 err = mlx5_accel_ipsec_rx_tag_add(rq->ifp, &rq->mbuf[ix]); 71 if (err) 72 goto err_free_mbuf; 73 err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map, 74 mb, segs, &nsegs, BUS_DMA_NOWAIT); 75 if (err != 0) 76 goto err_free_mbuf; 77 if (unlikely(nsegs == 0)) { 78 bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map); 79 err = -ENOMEM; 80 goto err_free_mbuf; 81 } 82 wqe->data[0].addr = cpu_to_be64(segs[0].ds_addr); 83 wqe->data[0].byte_count = cpu_to_be32(segs[0].ds_len | 84 MLX5_HW_START_PADDING); 85 for (i = 1; i != nsegs; i++) { 86 wqe->data[i].addr = cpu_to_be64(segs[i].ds_addr); 87 wqe->data[i].byte_count = cpu_to_be32(segs[i].ds_len); 88 } 89 for (; i < rq->nsegs; i++) { 90 wqe->data[i].addr = 0; 91 wqe->data[i].byte_count = 0; 92 } 93 94 rq->mbuf[ix].mbuf = mb; 95 rq->mbuf[ix].data = mb->m_data; 96 97 bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map, 98 BUS_DMASYNC_PREREAD); 99 return (0); 100 101 err_free_mbuf: 102 m_freem(mb); 103 return (err); 104 } 105 106 static void 107 mlx5e_post_rx_wqes(struct mlx5e_rq *rq) 108 { 109 if (unlikely(rq->enabled == 0)) 110 return; 111 112 while (!mlx5_wq_ll_is_full(&rq->wq)) { 113 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head); 114 115 if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) { 116 callout_reset_curcpu(&rq->watchdog, 1, (void *)&mlx5e_post_rx_wqes, rq); 117 break; 118 } 119 mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index)); 120 } 121 122 /* ensure wqes are visible to device before updating doorbell record */ 123 atomic_thread_fence_rel(); 124 125 mlx5_wq_ll_update_db_record(&rq->wq); 126 } 127 128 static uint32_t 129 csum_reduce(uint32_t val) 130 { 131 while (val > 0xffff) 132 val = (val >> 16) + (val & 0xffff); 133 return (val); 134 } 135 136 static u_short 137 csum_buf(uint32_t val, void *buf, int len) 138 { 139 u_short x; 140 141 MPASS(len % 2 == 0); 142 for (int i = 0; i < len; i += 2) { 143 bcopy((char *)buf + i, &x, 2); 144 val = csum_reduce(val + x); 145 } 146 return (val); 147 } 148 149 static void 150 mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe) 151 { 152 /* TODO: consider vlans, ip options, ... */ 153 struct ether_header *eh; 154 uint16_t eh_type; 155 uint16_t tot_len; 156 struct ip6_hdr *ip6 = NULL; 157 struct ip *ip4 = NULL; 158 struct tcphdr *th; 159 uint32_t *ts_ptr; 160 uint32_t tcp_csum; 161 uint8_t l4_hdr_type; 162 int tcp_ack; 163 164 eh = mtod(mb, struct ether_header *); 165 eh_type = ntohs(eh->ether_type); 166 167 l4_hdr_type = get_cqe_l4_hdr_type(cqe); 168 tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) || 169 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type)); 170 171 /* TODO: consider vlan */ 172 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN; 173 174 switch (eh_type) { 175 case ETHERTYPE_IP: 176 ip4 = (struct ip *)(eh + 1); 177 th = (struct tcphdr *)(ip4 + 1); 178 break; 179 case ETHERTYPE_IPV6: 180 ip6 = (struct ip6_hdr *)(eh + 1); 181 th = (struct tcphdr *)(ip6 + 1); 182 break; 183 default: 184 return; 185 } 186 187 ts_ptr = (uint32_t *)(th + 1); 188 189 if (get_cqe_lro_tcppsh(cqe)) 190 tcp_set_flags(th, tcp_get_flags(th) | TH_PUSH); 191 192 if (tcp_ack) { 193 tcp_set_flags(th, tcp_get_flags(th) | TH_ACK); 194 th->th_ack = cqe->lro_ack_seq_num; 195 th->th_win = cqe->lro_tcp_win; 196 197 /* 198 * FreeBSD handles only 32bit aligned timestamp right after 199 * the TCP hdr 200 * +--------+--------+--------+--------+ 201 * | NOP | NOP | TSopt | 10 | 202 * +--------+--------+--------+--------+ 203 * | TSval timestamp | 204 * +--------+--------+--------+--------+ 205 * | TSecr timestamp | 206 * +--------+--------+--------+--------+ 207 */ 208 if (get_cqe_lro_timestamp_valid(cqe) && 209 (__predict_true(*ts_ptr == ntohl(TCPOPT_NOP << 24 | 210 TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 | 211 TCPOLEN_TIMESTAMP)))) { 212 /* 213 * cqe->timestamp is 64bit long. 214 * [0-31] - timestamp. 215 * [32-64] - timestamp echo replay. 216 */ 217 ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1); 218 } 219 } 220 if (ip4) { 221 struct ipovly io; 222 223 ip4->ip_ttl = cqe->lro_min_ttl; 224 ip4->ip_len = cpu_to_be16(tot_len); 225 ip4->ip_sum = 0; 226 ip4->ip_sum = in_cksum_skip(mb, (ip4->ip_hl << 2) + 227 ETHER_HDR_LEN, ETHER_HDR_LEN); 228 229 /* TCP checksum: data */ 230 tcp_csum = cqe->check_sum; 231 232 /* TCP checksum: IP pseudoheader */ 233 bzero(io.ih_x1, sizeof(io.ih_x1)); 234 io.ih_pr = IPPROTO_TCP; 235 io.ih_len = htons(ntohs(ip4->ip_len) - sizeof(*ip4)); 236 io.ih_src = ip4->ip_src; 237 io.ih_dst = ip4->ip_dst; 238 tcp_csum = csum_buf(tcp_csum, &io, sizeof(io)); 239 240 /* TCP checksum: TCP header */ 241 th->th_sum = 0; 242 tcp_csum = csum_buf(tcp_csum, th, th->th_off * 4); 243 th->th_sum = ~tcp_csum & 0xffff; 244 } else { 245 ip6->ip6_hlim = cqe->lro_min_ttl; 246 ip6->ip6_plen = cpu_to_be16(tot_len - 247 sizeof(struct ip6_hdr)); 248 249 /* TCP checksum */ 250 th->th_sum = 0; 251 tcp_csum = ~in6_cksum_partial_l2(mb, IPPROTO_TCP, 252 sizeof(struct ether_header), 253 sizeof(struct ether_header) + sizeof(struct ip6_hdr), 254 tot_len - sizeof(struct ip6_hdr), th->th_off * 4) & 0xffff; 255 tcp_csum = csum_reduce(tcp_csum + cqe->check_sum); 256 th->th_sum = ~tcp_csum & 0xffff; 257 } 258 } 259 260 static uint64_t 261 mlx5e_mbuf_tstmp(struct mlx5e_priv *priv, uint64_t hw_tstmp) 262 { 263 struct mlx5e_clbr_point *cp, dcp; 264 uint64_t tstmp_sec, tstmp_nsec; 265 uint64_t hw_clocks; 266 uint64_t rt_cur_to_prev, res_s, res_n, res_s_modulo, res; 267 uint64_t hw_clk_div; 268 u_int gen; 269 270 do { 271 cp = &priv->clbr_points[priv->clbr_curr]; 272 gen = atomic_load_acq_int(&cp->clbr_gen); 273 if (gen == 0) 274 return (0); 275 dcp = *cp; 276 atomic_thread_fence_acq(); 277 } while (gen != dcp.clbr_gen); 278 /* 279 * Our goal here is to have a result that is: 280 * 281 * ( (cur_time - prev_time) ) 282 * ((hw_tstmp - hw_prev) * ----------------------------- ) + prev_time 283 * ( (hw_cur - hw_prev) ) 284 * 285 * With the constraints that we cannot use float and we 286 * don't want to overflow the uint64_t numbers we are using. 287 * 288 * The plan is to take the clocking value of the hw timestamps 289 * and split them into seconds and nanosecond equivalent portions. 290 * Then we operate on the two portions seperately making sure to 291 * bring back the carry over from the seconds when we divide. 292 * 293 * First up lets get the two divided into separate entities 294 * i.e. the seconds. We use the clock frequency for this. 295 * Note that priv->cclk was setup with the clock frequency 296 * in hz so we are all set to go. 297 */ 298 hw_clocks = hw_tstmp - dcp.clbr_hw_prev; 299 tstmp_sec = hw_clocks / priv->cclk; 300 tstmp_nsec = hw_clocks % priv->cclk; 301 /* Now work with them separately */ 302 rt_cur_to_prev = (dcp.base_curr - dcp.base_prev); 303 res_s = tstmp_sec * rt_cur_to_prev; 304 res_n = tstmp_nsec * rt_cur_to_prev; 305 /* Now lets get our divider */ 306 hw_clk_div = dcp.clbr_hw_curr - dcp.clbr_hw_prev; 307 /* Make sure to save the remainder from the seconds divide */ 308 res_s_modulo = res_s % hw_clk_div; 309 res_s /= hw_clk_div; 310 /* scale the remainder to where it should be */ 311 res_s_modulo *= priv->cclk; 312 /* Now add in the remainder */ 313 res_n += res_s_modulo; 314 /* Now do the divide */ 315 res_n /= hw_clk_div; 316 res_s *= priv->cclk; 317 /* Recombine the two */ 318 res = res_s + res_n; 319 /* And now add in the base time to get to the real timestamp */ 320 res += dcp.base_prev; 321 return (res); 322 } 323 324 static inline void 325 mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, struct mlx5e_rq *rq, 326 struct mbuf *mb, struct mlx5e_rq_mbuf *mr, u32 cqe_bcnt) 327 { 328 if_t ifp = rq->ifp; 329 struct mlx5e_channel *c; 330 struct mbuf *mb_head; 331 int lro_num_seg; /* HW LRO session aggregated packets counter */ 332 uint64_t tstmp; 333 334 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24; 335 if (lro_num_seg > 1) { 336 mlx5e_lro_update_hdr(mb, cqe); 337 rq->stats.lro_packets++; 338 rq->stats.lro_bytes += cqe_bcnt; 339 } 340 341 mb->m_pkthdr.len = cqe_bcnt; 342 for (mb_head = mb; mb != NULL; mb = mb->m_next) { 343 if (mb->m_len > cqe_bcnt) 344 mb->m_len = cqe_bcnt; 345 cqe_bcnt -= mb->m_len; 346 if (likely(cqe_bcnt == 0)) { 347 if (likely(mb->m_next != NULL)) { 348 /* trim off empty mbufs */ 349 m_freem(mb->m_next); 350 mb->m_next = NULL; 351 } 352 break; 353 } 354 } 355 /* rewind to first mbuf in chain */ 356 mb = mb_head; 357 358 /* check if a Toeplitz hash was computed */ 359 if (cqe->rss_hash_type != 0) { 360 mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result); 361 /* decode the RSS hash type */ 362 switch (cqe->rss_hash_type & 363 (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) { 364 /* IPv4 */ 365 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4): 366 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4); 367 break; 368 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4): 369 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4); 370 break; 371 case CQE_RSS_DST_HTYPE_IPV4: 372 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4); 373 break; 374 /* IPv6 */ 375 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6): 376 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6); 377 break; 378 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6): 379 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6); 380 break; 381 case CQE_RSS_DST_HTYPE_IPV6: 382 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6); 383 break; 384 default: /* Other */ 385 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); 386 break; 387 } 388 #ifdef M_HASHTYPE_SETINNER 389 if (cqe_is_tunneled(cqe)) 390 M_HASHTYPE_SETINNER(mb); 391 #endif 392 } else { 393 mb->m_pkthdr.flowid = rq->ix; 394 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); 395 } 396 mb->m_pkthdr.rcvif = ifp; 397 mb->m_pkthdr.leaf_rcvif = ifp; 398 399 if (cqe_is_tunneled(cqe)) { 400 /* 401 * CQE can be tunneled only if TIR is configured to 402 * enable parsing of tunneled payload, so no need to 403 * check for capabilities. 404 */ 405 if (((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK)) == 406 (CQE_L2_OK | CQE_L3_OK))) { 407 mb->m_pkthdr.csum_flags |= 408 CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID | 409 CSUM_IP_CHECKED | CSUM_IP_VALID | 410 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 411 mb->m_pkthdr.csum_data = htons(0xffff); 412 413 if (likely((cqe->hds_ip_ext & CQE_L4_OK) == CQE_L4_OK)) { 414 mb->m_pkthdr.csum_flags |= 415 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID; 416 } 417 } else { 418 rq->stats.csum_none++; 419 } 420 } else if (likely((if_getcapenable(ifp) & (IFCAP_RXCSUM | 421 IFCAP_RXCSUM_IPV6)) != 0) && 422 ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) == 423 (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) { 424 mb->m_pkthdr.csum_flags = 425 CSUM_IP_CHECKED | CSUM_IP_VALID | 426 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 427 mb->m_pkthdr.csum_data = htons(0xffff); 428 } else { 429 rq->stats.csum_none++; 430 } 431 432 if (cqe_has_vlan(cqe)) { 433 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info); 434 mb->m_flags |= M_VLANTAG; 435 } 436 437 c = container_of(rq, struct mlx5e_channel, rq); 438 if (c->priv->clbr_done >= 2) { 439 tstmp = mlx5e_mbuf_tstmp(c->priv, be64_to_cpu(cqe->timestamp)); 440 if ((tstmp & MLX5_CQE_TSTMP_PTP) != 0) { 441 /* 442 * Timestamp was taken on the packet entrance, 443 * instead of the cqe generation. 444 */ 445 tstmp &= ~MLX5_CQE_TSTMP_PTP; 446 mb->m_flags |= M_TSTMP_HPREC; 447 } 448 if (tstmp != 0) { 449 mb->m_pkthdr.rcv_tstmp = tstmp; 450 mb->m_flags |= M_TSTMP; 451 } 452 } 453 switch (get_cqe_tls_offload(cqe)) { 454 case CQE_TLS_OFFLOAD_DECRYPTED: 455 /* set proper checksum flag for decrypted packets */ 456 mb->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; 457 rq->stats.decrypted_ok_packets++; 458 break; 459 case CQE_TLS_OFFLOAD_ERROR: 460 rq->stats.decrypted_error_packets++; 461 break; 462 default: 463 break; 464 } 465 466 mlx5e_accel_ipsec_handle_rx(ifp, mb, cqe, mr); 467 } 468 469 static inline void 470 mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 471 { 472 memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)), 473 sizeof(struct mlx5_cqe64)); 474 } 475 476 static inline void 477 mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 478 { 479 memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1), 480 data, sizeof(struct mlx5_cqe64)); 481 } 482 483 static inline void 484 mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title, 485 struct mlx5_mini_cqe8 *mini, 486 u16 wqe_counter, int i) 487 { 488 /* 489 * NOTE: The fields which are not set here are copied from the 490 * initial and common title. See memcpy() in 491 * mlx5e_write_cqe_slot(). 492 */ 493 title->byte_cnt = mini->byte_cnt; 494 title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1); 495 title->rss_hash_result = mini->rx_hash_result; 496 /* 497 * Since we use MLX5_CQE_FORMAT_HASH when creating the RX CQ, 498 * the value of the checksum should be ignored. 499 */ 500 title->check_sum = 0; 501 title->op_own = (title->op_own & 0xf0) | 502 (((cq->wq.cc + i) >> cq->wq.log_sz) & 1); 503 } 504 505 #define MLX5E_MINI_ARRAY_SZ 8 506 /* Make sure structs are not packet differently */ 507 CTASSERT(sizeof(struct mlx5_cqe64) == 508 sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ); 509 static void 510 mlx5e_decompress_cqes(struct mlx5e_cq *cq) 511 { 512 struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ]; 513 struct mlx5_cqe64 title; 514 u32 cqe_count; 515 u32 i = 0; 516 u16 title_wqe_counter; 517 518 mlx5e_read_cqe_slot(cq, cq->wq.cc, &title); 519 title_wqe_counter = be16_to_cpu(title.wqe_counter); 520 cqe_count = be32_to_cpu(title.byte_cnt); 521 522 /* Make sure we won't overflow */ 523 KASSERT(cqe_count <= cq->wq.sz_m1, 524 ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__, 525 cqe_count, cq->wq.sz_m1)); 526 527 mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array); 528 while (true) { 529 mlx5e_decompress_cqe(cq, &title, 530 &mini_array[i % MLX5E_MINI_ARRAY_SZ], 531 title_wqe_counter, i); 532 mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title); 533 i++; 534 535 if (i == cqe_count) 536 break; 537 if (i % MLX5E_MINI_ARRAY_SZ == 0) 538 mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array); 539 } 540 } 541 542 static int 543 mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) 544 { 545 struct pfil_head *pfil; 546 int i, rv; 547 548 CURVNET_SET_QUIET(if_getvnet(rq->ifp)); 549 pfil = rq->channel->priv->pfil; 550 for (i = 0; i < budget; i++) { 551 struct mlx5e_rx_wqe *wqe; 552 struct mlx5_cqe64 *cqe; 553 struct mbuf *mb; 554 __be16 wqe_counter_be; 555 u16 wqe_counter; 556 u32 byte_cnt, seglen; 557 558 cqe = mlx5e_get_cqe(&rq->cq); 559 if (!cqe) 560 break; 561 562 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) 563 mlx5e_decompress_cqes(&rq->cq); 564 565 mlx5_cqwq_pop(&rq->cq.wq); 566 567 wqe_counter_be = cqe->wqe_counter; 568 wqe_counter = be16_to_cpu(wqe_counter_be); 569 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter); 570 byte_cnt = be32_to_cpu(cqe->byte_cnt); 571 572 bus_dmamap_sync(rq->dma_tag, 573 rq->mbuf[wqe_counter].dma_map, 574 BUS_DMASYNC_POSTREAD); 575 576 if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) { 577 mlx5e_dump_err_cqe(&rq->cq, rq->rqn, (const void *)cqe); 578 rq->stats.wqe_err++; 579 goto wq_ll_pop; 580 } 581 if (pfil != NULL && PFIL_HOOKED_IN(pfil)) { 582 seglen = MIN(byte_cnt, MLX5E_MAX_RX_BYTES); 583 rv = pfil_mem_in(rq->channel->priv->pfil, 584 rq->mbuf[wqe_counter].data, seglen, rq->ifp, &mb); 585 586 switch (rv) { 587 case PFIL_DROPPED: 588 case PFIL_CONSUMED: 589 /* 590 * Filter dropped or consumed it. In 591 * either case, we can just recycle 592 * buffer; there is no more work to do. 593 */ 594 rq->stats.packets++; 595 goto wq_ll_pop; 596 case PFIL_REALLOCED: 597 /* 598 * Filter copied it; recycle buffer 599 * and receive the new mbuf allocated 600 * by the Filter 601 */ 602 goto rx_common; 603 default: 604 /* 605 * The Filter said it was OK, so 606 * receive like normal. 607 */ 608 KASSERT(rv == PFIL_PASS, 609 ("Filter returned %d!\n", rv)); 610 } 611 } 612 if (!mlx5e_accel_ipsec_flow(cqe) /* tag is already assigned 613 to rq->mbuf */ && 614 MHLEN - MLX5E_NET_IP_ALIGN >= byte_cnt && 615 (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { 616 /* set maximum mbuf length */ 617 mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN; 618 /* get IP header aligned */ 619 mb->m_data += MLX5E_NET_IP_ALIGN; 620 621 bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t), 622 byte_cnt); 623 } else { 624 mb = rq->mbuf[wqe_counter].mbuf; 625 rq->mbuf[wqe_counter].mbuf = NULL; /* safety clear */ 626 627 bus_dmamap_unload(rq->dma_tag, 628 rq->mbuf[wqe_counter].dma_map); 629 } 630 rx_common: 631 mlx5e_build_rx_mbuf(cqe, rq, mb, &rq->mbuf[wqe_counter], 632 byte_cnt); 633 rq->stats.bytes += byte_cnt; 634 rq->stats.packets++; 635 #ifdef NUMA 636 mb->m_pkthdr.numa_domain = if_getnumadomain(rq->ifp); 637 #endif 638 639 #if !defined(HAVE_TCP_LRO_RX) 640 tcp_lro_queue_mbuf(&rq->lro, mb); 641 #else 642 if (mb->m_pkthdr.csum_flags == 0 || 643 (if_getcapenable(rq->ifp) & IFCAP_LRO) == 0 || 644 rq->lro.lro_cnt == 0 || 645 tcp_lro_rx(&rq->lro, mb, 0) != 0) { 646 if_input(rq->ifp, mb); 647 } 648 #endif 649 wq_ll_pop: 650 mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, 651 &wqe->next.next_wqe_index); 652 } 653 CURVNET_RESTORE(); 654 655 mlx5_cqwq_update_db_record(&rq->cq.wq); 656 657 /* ensure cq space is freed before enabling more cqes */ 658 atomic_thread_fence_rel(); 659 return (i); 660 } 661 662 void 663 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe __unused) 664 { 665 struct mlx5e_channel *c = container_of(mcq, struct mlx5e_channel, rq.cq.mcq); 666 struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); 667 int i = 0; 668 669 #ifdef HAVE_PER_CQ_EVENT_PACKET 670 #if (MHLEN < 15) 671 #error "MHLEN is too small" 672 #endif 673 struct mbuf *mb = m_gethdr(M_NOWAIT, MT_DATA); 674 675 if (mb != NULL) { 676 /* this code is used for debugging purpose only */ 677 mb->m_pkthdr.len = mb->m_len = 15; 678 memset(mb->m_data, 255, 14); 679 mb->m_data[14] = rq->ix; 680 mb->m_pkthdr.rcvif = rq->ifp; 681 mb->m_pkthdr.leaf_rcvif = rq->ifp; 682 if_input(rq->ifp, mb); 683 } 684 #endif 685 for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) { 686 mtx_lock(&c->sq[j].lock); 687 c->sq[j].db_inhibit++; 688 mtx_unlock(&c->sq[j].lock); 689 } 690 691 mtx_lock(&c->iq.lock); 692 c->iq.db_inhibit++; 693 mtx_unlock(&c->iq.lock); 694 695 mtx_lock(&rq->mtx); 696 if (rq->enabled == 0) 697 goto out; 698 rq->processing++; 699 700 /* 701 * Polling the entire CQ without posting new WQEs results in 702 * lack of receive WQEs during heavy traffic scenarios. 703 */ 704 while (1) { 705 if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) != 706 MLX5E_RX_BUDGET_MAX) 707 break; 708 i += MLX5E_RX_BUDGET_MAX; 709 if (i >= MLX5E_BUDGET_MAX) 710 break; 711 mlx5e_post_rx_wqes(rq); 712 } 713 mlx5e_post_rx_wqes(rq); 714 /* check for dynamic interrupt moderation callback */ 715 if (rq->dim.mode != NET_DIM_CQ_PERIOD_MODE_DISABLED) 716 net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes); 717 mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock)); 718 tcp_lro_flush_all(&rq->lro); 719 rq->processing--; 720 out: 721 mtx_unlock(&rq->mtx); 722 723 for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) { 724 mtx_lock(&c->sq[j].lock); 725 c->sq[j].db_inhibit--; 726 /* Update the doorbell record, if any. */ 727 mlx5e_tx_notify_hw(c->sq + j, true); 728 mtx_unlock(&c->sq[j].lock); 729 } 730 731 mtx_lock(&c->iq.lock); 732 c->iq.db_inhibit--; 733 mlx5e_iq_notify_hw(&c->iq); 734 mtx_unlock(&c->iq.lock); 735 } 736