1 /*- 2 * Copyright (c) 2015 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include "en.h" 29 #include <machine/in_cksum.h> 30 31 static inline int 32 mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, 33 struct mlx5e_rx_wqe *wqe, u16 ix) 34 { 35 bus_dma_segment_t segs[rq->nsegs]; 36 struct mbuf *mb; 37 int nsegs; 38 int err; 39 #if (MLX5E_MAX_RX_SEGS != 1) 40 struct mbuf *mb_head; 41 int i; 42 #endif 43 if (rq->mbuf[ix].mbuf != NULL) 44 return (0); 45 46 #if (MLX5E_MAX_RX_SEGS == 1) 47 mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz); 48 if (unlikely(!mb)) 49 return (-ENOMEM); 50 51 mb->m_pkthdr.len = mb->m_len = rq->wqe_sz; 52 #else 53 mb_head = mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, 54 MLX5E_MAX_RX_BYTES); 55 if (unlikely(mb == NULL)) 56 return (-ENOMEM); 57 58 mb->m_len = MLX5E_MAX_RX_BYTES; 59 mb->m_pkthdr.len = MLX5E_MAX_RX_BYTES; 60 61 for (i = 1; i < rq->nsegs; i++) { 62 if (mb_head->m_pkthdr.len >= rq->wqe_sz) 63 break; 64 mb = mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, 65 MLX5E_MAX_RX_BYTES); 66 if (unlikely(mb == NULL)) { 67 m_freem(mb_head); 68 return (-ENOMEM); 69 } 70 mb->m_len = MLX5E_MAX_RX_BYTES; 71 mb_head->m_pkthdr.len += MLX5E_MAX_RX_BYTES; 72 } 73 /* rewind to first mbuf in chain */ 74 mb = mb_head; 75 #endif 76 /* get IP header aligned */ 77 m_adj(mb, MLX5E_NET_IP_ALIGN); 78 79 err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map, 80 mb, segs, &nsegs, BUS_DMA_NOWAIT); 81 if (err != 0) 82 goto err_free_mbuf; 83 if (unlikely(nsegs == 0)) { 84 bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map); 85 err = -ENOMEM; 86 goto err_free_mbuf; 87 } 88 #if (MLX5E_MAX_RX_SEGS == 1) 89 wqe->data[0].addr = cpu_to_be64(segs[0].ds_addr); 90 #else 91 wqe->data[0].addr = cpu_to_be64(segs[0].ds_addr); 92 wqe->data[0].byte_count = cpu_to_be32(segs[0].ds_len | 93 MLX5_HW_START_PADDING); 94 for (i = 1; i != nsegs; i++) { 95 wqe->data[i].addr = cpu_to_be64(segs[i].ds_addr); 96 wqe->data[i].byte_count = cpu_to_be32(segs[i].ds_len); 97 } 98 for (; i < rq->nsegs; i++) { 99 wqe->data[i].addr = 0; 100 wqe->data[i].byte_count = 0; 101 } 102 #endif 103 104 rq->mbuf[ix].mbuf = mb; 105 rq->mbuf[ix].data = mb->m_data; 106 107 bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map, 108 BUS_DMASYNC_PREREAD); 109 return (0); 110 111 err_free_mbuf: 112 m_freem(mb); 113 return (err); 114 } 115 116 static void 117 mlx5e_post_rx_wqes(struct mlx5e_rq *rq) 118 { 119 if (unlikely(rq->enabled == 0)) 120 return; 121 122 while (!mlx5_wq_ll_is_full(&rq->wq)) { 123 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head); 124 125 if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) { 126 callout_reset_curcpu(&rq->watchdog, 1, (void *)&mlx5e_post_rx_wqes, rq); 127 break; 128 } 129 mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index)); 130 } 131 132 /* ensure wqes are visible to device before updating doorbell record */ 133 atomic_thread_fence_rel(); 134 135 mlx5_wq_ll_update_db_record(&rq->wq); 136 } 137 138 static void 139 mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe) 140 { 141 /* TODO: consider vlans, ip options, ... */ 142 struct ether_header *eh; 143 uint16_t eh_type; 144 uint16_t tot_len; 145 struct ip6_hdr *ip6 = NULL; 146 struct ip *ip4 = NULL; 147 struct tcphdr *th; 148 uint32_t *ts_ptr; 149 uint8_t l4_hdr_type; 150 int tcp_ack; 151 152 eh = mtod(mb, struct ether_header *); 153 eh_type = ntohs(eh->ether_type); 154 155 l4_hdr_type = get_cqe_l4_hdr_type(cqe); 156 tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) || 157 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type)); 158 159 /* TODO: consider vlan */ 160 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN; 161 162 switch (eh_type) { 163 case ETHERTYPE_IP: 164 ip4 = (struct ip *)(eh + 1); 165 th = (struct tcphdr *)(ip4 + 1); 166 break; 167 case ETHERTYPE_IPV6: 168 ip6 = (struct ip6_hdr *)(eh + 1); 169 th = (struct tcphdr *)(ip6 + 1); 170 break; 171 default: 172 return; 173 } 174 175 ts_ptr = (uint32_t *)(th + 1); 176 177 if (get_cqe_lro_tcppsh(cqe)) 178 th->th_flags |= TH_PUSH; 179 180 if (tcp_ack) { 181 th->th_flags |= TH_ACK; 182 th->th_ack = cqe->lro_ack_seq_num; 183 th->th_win = cqe->lro_tcp_win; 184 185 /* 186 * FreeBSD handles only 32bit aligned timestamp right after 187 * the TCP hdr 188 * +--------+--------+--------+--------+ 189 * | NOP | NOP | TSopt | 10 | 190 * +--------+--------+--------+--------+ 191 * | TSval timestamp | 192 * +--------+--------+--------+--------+ 193 * | TSecr timestamp | 194 * +--------+--------+--------+--------+ 195 */ 196 if (get_cqe_lro_timestamp_valid(cqe) && 197 (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 | 198 TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 | 199 TCPOLEN_TIMESTAMP))) { 200 /* 201 * cqe->timestamp is 64bit long. 202 * [0-31] - timestamp. 203 * [32-64] - timestamp echo replay. 204 */ 205 ts_ptr[1] = *(uint32_t *)&cqe->timestamp; 206 ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1); 207 } 208 } 209 if (ip4) { 210 ip4->ip_ttl = cqe->lro_min_ttl; 211 ip4->ip_len = cpu_to_be16(tot_len); 212 ip4->ip_sum = 0; 213 ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2); 214 } else { 215 ip6->ip6_hlim = cqe->lro_min_ttl; 216 ip6->ip6_plen = cpu_to_be16(tot_len - 217 sizeof(struct ip6_hdr)); 218 } 219 /* TODO: handle tcp checksum */ 220 } 221 222 static uint64_t 223 mlx5e_mbuf_tstmp(struct mlx5e_priv *priv, uint64_t hw_tstmp) 224 { 225 struct mlx5e_clbr_point *cp, dcp; 226 uint64_t a1, a2, res; 227 u_int gen; 228 229 do { 230 cp = &priv->clbr_points[priv->clbr_curr]; 231 gen = atomic_load_acq_int(&cp->clbr_gen); 232 if (gen == 0) 233 return (0); 234 dcp = *cp; 235 atomic_thread_fence_acq(); 236 } while (gen != cp->clbr_gen); 237 238 a1 = (hw_tstmp - dcp.clbr_hw_prev) >> MLX5E_TSTMP_PREC; 239 a2 = (dcp.base_curr - dcp.base_prev) >> MLX5E_TSTMP_PREC; 240 res = (a1 * a2) << MLX5E_TSTMP_PREC; 241 242 /* 243 * Divisor cannot be zero because calibration callback 244 * checks for the condition and disables timestamping 245 * if clock halted. 246 */ 247 res /= (dcp.clbr_hw_curr - dcp.clbr_hw_prev) >> MLX5E_TSTMP_PREC; 248 249 res += dcp.base_prev; 250 return (res); 251 } 252 253 static inline void 254 mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, 255 struct mlx5e_rq *rq, struct mbuf *mb, 256 u32 cqe_bcnt) 257 { 258 struct ifnet *ifp = rq->ifp; 259 struct mlx5e_channel *c; 260 #if (MLX5E_MAX_RX_SEGS != 1) 261 struct mbuf *mb_head; 262 #endif 263 int lro_num_seg; /* HW LRO session aggregated packets counter */ 264 uint64_t tstmp; 265 266 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24; 267 if (lro_num_seg > 1) { 268 mlx5e_lro_update_hdr(mb, cqe); 269 rq->stats.lro_packets++; 270 rq->stats.lro_bytes += cqe_bcnt; 271 } 272 273 #if (MLX5E_MAX_RX_SEGS == 1) 274 mb->m_pkthdr.len = mb->m_len = cqe_bcnt; 275 #else 276 mb->m_pkthdr.len = cqe_bcnt; 277 for (mb_head = mb; mb != NULL; mb = mb->m_next) { 278 if (mb->m_len > cqe_bcnt) 279 mb->m_len = cqe_bcnt; 280 cqe_bcnt -= mb->m_len; 281 if (likely(cqe_bcnt == 0)) { 282 if (likely(mb->m_next != NULL)) { 283 /* trim off empty mbufs */ 284 m_freem(mb->m_next); 285 mb->m_next = NULL; 286 } 287 break; 288 } 289 } 290 /* rewind to first mbuf in chain */ 291 mb = mb_head; 292 #endif 293 /* check if a Toeplitz hash was computed */ 294 if (cqe->rss_hash_type != 0) { 295 mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result); 296 #ifdef RSS 297 /* decode the RSS hash type */ 298 switch (cqe->rss_hash_type & 299 (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) { 300 /* IPv4 */ 301 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4): 302 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4); 303 break; 304 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4): 305 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4); 306 break; 307 case CQE_RSS_DST_HTYPE_IPV4: 308 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4); 309 break; 310 /* IPv6 */ 311 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6): 312 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6); 313 break; 314 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6): 315 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6); 316 break; 317 case CQE_RSS_DST_HTYPE_IPV6: 318 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6); 319 break; 320 default: /* Other */ 321 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); 322 break; 323 } 324 #else 325 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); 326 #endif 327 } else { 328 mb->m_pkthdr.flowid = rq->ix; 329 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); 330 } 331 mb->m_pkthdr.rcvif = ifp; 332 333 if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && 334 ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) == 335 (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) { 336 mb->m_pkthdr.csum_flags = 337 CSUM_IP_CHECKED | CSUM_IP_VALID | 338 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 339 mb->m_pkthdr.csum_data = htons(0xffff); 340 } else { 341 rq->stats.csum_none++; 342 } 343 344 if (cqe_has_vlan(cqe)) { 345 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info); 346 mb->m_flags |= M_VLANTAG; 347 } 348 349 c = container_of(rq, struct mlx5e_channel, rq); 350 if (c->priv->clbr_done >= 2) { 351 tstmp = mlx5e_mbuf_tstmp(c->priv, be64_to_cpu(cqe->timestamp)); 352 if ((tstmp & MLX5_CQE_TSTMP_PTP) != 0) { 353 /* 354 * Timestamp was taken on the packet entrance, 355 * instead of the cqe generation. 356 */ 357 tstmp &= ~MLX5_CQE_TSTMP_PTP; 358 mb->m_flags |= M_TSTMP_HPREC; 359 } 360 mb->m_pkthdr.rcv_tstmp = tstmp; 361 mb->m_flags |= M_TSTMP; 362 } 363 } 364 365 static inline void 366 mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 367 { 368 memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)), 369 sizeof(struct mlx5_cqe64)); 370 } 371 372 static inline void 373 mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 374 { 375 memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1), 376 data, sizeof(struct mlx5_cqe64)); 377 } 378 379 static inline void 380 mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title, 381 struct mlx5_mini_cqe8 *mini, 382 u16 wqe_counter, int i) 383 { 384 /* 385 * NOTE: The fields which are not set here are copied from the 386 * initial and common title. See memcpy() in 387 * mlx5e_write_cqe_slot(). 388 */ 389 title->byte_cnt = mini->byte_cnt; 390 title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1); 391 title->rss_hash_result = mini->rx_hash_result; 392 /* 393 * Since we use MLX5_CQE_FORMAT_HASH when creating the RX CQ, 394 * the value of the checksum should be ignored. 395 */ 396 title->check_sum = 0; 397 title->op_own = (title->op_own & 0xf0) | 398 (((cq->wq.cc + i) >> cq->wq.log_sz) & 1); 399 } 400 401 #define MLX5E_MINI_ARRAY_SZ 8 402 /* Make sure structs are not packet differently */ 403 CTASSERT(sizeof(struct mlx5_cqe64) == 404 sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ); 405 static void 406 mlx5e_decompress_cqes(struct mlx5e_cq *cq) 407 { 408 struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ]; 409 struct mlx5_cqe64 title; 410 u32 cqe_count; 411 u32 i = 0; 412 u16 title_wqe_counter; 413 414 mlx5e_read_cqe_slot(cq, cq->wq.cc, &title); 415 title_wqe_counter = be16_to_cpu(title.wqe_counter); 416 cqe_count = be32_to_cpu(title.byte_cnt); 417 418 /* Make sure we won't overflow */ 419 KASSERT(cqe_count <= cq->wq.sz_m1, 420 ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__, 421 cqe_count, cq->wq.sz_m1)); 422 423 mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array); 424 while (true) { 425 mlx5e_decompress_cqe(cq, &title, 426 &mini_array[i % MLX5E_MINI_ARRAY_SZ], 427 title_wqe_counter, i); 428 mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title); 429 i++; 430 431 if (i == cqe_count) 432 break; 433 if (i % MLX5E_MINI_ARRAY_SZ == 0) 434 mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array); 435 } 436 } 437 438 static int 439 mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) 440 { 441 struct pfil_head *pfil; 442 int i, rv; 443 444 CURVNET_SET_QUIET(rq->ifp->if_vnet); 445 pfil = rq->channel->priv->pfil; 446 for (i = 0; i < budget; i++) { 447 struct mlx5e_rx_wqe *wqe; 448 struct mlx5_cqe64 *cqe; 449 struct mbuf *mb; 450 __be16 wqe_counter_be; 451 u16 wqe_counter; 452 u32 byte_cnt, seglen; 453 454 cqe = mlx5e_get_cqe(&rq->cq); 455 if (!cqe) 456 break; 457 458 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) 459 mlx5e_decompress_cqes(&rq->cq); 460 461 mlx5_cqwq_pop(&rq->cq.wq); 462 463 wqe_counter_be = cqe->wqe_counter; 464 wqe_counter = be16_to_cpu(wqe_counter_be); 465 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter); 466 byte_cnt = be32_to_cpu(cqe->byte_cnt); 467 468 bus_dmamap_sync(rq->dma_tag, 469 rq->mbuf[wqe_counter].dma_map, 470 BUS_DMASYNC_POSTREAD); 471 472 if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) { 473 rq->stats.wqe_err++; 474 goto wq_ll_pop; 475 } 476 if (pfil != NULL && PFIL_HOOKED_IN(pfil)) { 477 seglen = MIN(byte_cnt, MLX5E_MAX_RX_BYTES); 478 rv = pfil_run_hooks(rq->channel->priv->pfil, 479 rq->mbuf[wqe_counter].data, rq->ifp, 480 seglen | PFIL_MEMPTR | PFIL_IN, NULL); 481 482 switch (rv) { 483 case PFIL_DROPPED: 484 case PFIL_CONSUMED: 485 /* 486 * Filter dropped or consumed it. In 487 * either case, we can just recycle 488 * buffer; there is no more work to do. 489 */ 490 rq->stats.packets++; 491 goto wq_ll_pop; 492 case PFIL_REALLOCED: 493 /* 494 * Filter copied it; recycle buffer 495 * and receive the new mbuf allocated 496 * by the Filter 497 */ 498 mb = pfil_mem2mbuf(rq->mbuf[wqe_counter].data); 499 goto rx_common; 500 default: 501 /* 502 * The Filter said it was OK, so 503 * receive like normal. 504 */ 505 KASSERT(rv == PFIL_PASS, 506 ("Filter returned %d!\n", rv)); 507 } 508 } 509 if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt && 510 (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { 511 #if (MLX5E_MAX_RX_SEGS != 1) 512 /* set maximum mbuf length */ 513 mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN; 514 #endif 515 /* get IP header aligned */ 516 mb->m_data += MLX5E_NET_IP_ALIGN; 517 518 bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t), 519 byte_cnt); 520 } else { 521 mb = rq->mbuf[wqe_counter].mbuf; 522 rq->mbuf[wqe_counter].mbuf = NULL; /* safety clear */ 523 524 bus_dmamap_unload(rq->dma_tag, 525 rq->mbuf[wqe_counter].dma_map); 526 } 527 rx_common: 528 mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt); 529 rq->stats.bytes += byte_cnt; 530 rq->stats.packets++; 531 #ifdef NUMA 532 mb->m_pkthdr.numa_domain = rq->ifp->if_numa_domain; 533 #endif 534 535 #if !defined(HAVE_TCP_LRO_RX) 536 tcp_lro_queue_mbuf(&rq->lro, mb); 537 #else 538 if (mb->m_pkthdr.csum_flags == 0 || 539 (rq->ifp->if_capenable & IFCAP_LRO) == 0 || 540 rq->lro.lro_cnt == 0 || 541 tcp_lro_rx(&rq->lro, mb, 0) != 0) { 542 rq->ifp->if_input(rq->ifp, mb); 543 } 544 #endif 545 wq_ll_pop: 546 mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, 547 &wqe->next.next_wqe_index); 548 } 549 CURVNET_RESTORE(); 550 551 mlx5_cqwq_update_db_record(&rq->cq.wq); 552 553 /* ensure cq space is freed before enabling more cqes */ 554 atomic_thread_fence_rel(); 555 return (i); 556 } 557 558 void 559 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq) 560 { 561 struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); 562 int i = 0; 563 564 #ifdef HAVE_PER_CQ_EVENT_PACKET 565 #if (MHLEN < 15) 566 #error "MHLEN is too small" 567 #endif 568 struct mbuf *mb = m_gethdr(M_NOWAIT, MT_DATA); 569 570 if (mb != NULL) { 571 /* this code is used for debugging purpose only */ 572 mb->m_pkthdr.len = mb->m_len = 15; 573 memset(mb->m_data, 255, 14); 574 mb->m_data[14] = rq->ix; 575 mb->m_pkthdr.rcvif = rq->ifp; 576 rq->ifp->if_input(rq->ifp, mb); 577 } 578 #endif 579 580 mtx_lock(&rq->mtx); 581 582 /* 583 * Polling the entire CQ without posting new WQEs results in 584 * lack of receive WQEs during heavy traffic scenarios. 585 */ 586 while (1) { 587 if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) != 588 MLX5E_RX_BUDGET_MAX) 589 break; 590 i += MLX5E_RX_BUDGET_MAX; 591 if (i >= MLX5E_BUDGET_MAX) 592 break; 593 mlx5e_post_rx_wqes(rq); 594 } 595 mlx5e_post_rx_wqes(rq); 596 /* check for dynamic interrupt moderation callback */ 597 if (rq->dim.mode != NET_DIM_CQ_PERIOD_MODE_DISABLED) 598 net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes); 599 mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock)); 600 tcp_lro_flush_all(&rq->lro); 601 mtx_unlock(&rq->mtx); 602 } 603