1 /*- 2 * Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include "en.h" 29 #include <machine/atomic.h> 30 31 static inline bool 32 mlx5e_do_send_cqe(struct mlx5e_sq *sq) 33 { 34 sq->cev_counter++; 35 /* interleave the CQEs */ 36 if (sq->cev_counter >= sq->cev_factor) { 37 sq->cev_counter = 0; 38 return (1); 39 } 40 return (0); 41 } 42 43 void 44 mlx5e_send_nop(struct mlx5e_sq *sq, u32 ds_cnt) 45 { 46 u16 pi = sq->pc & sq->wq.sz_m1; 47 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 48 49 memset(&wqe->ctrl, 0, sizeof(wqe->ctrl)); 50 51 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP); 52 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 53 if (mlx5e_do_send_cqe(sq)) 54 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 55 else 56 wqe->ctrl.fm_ce_se = 0; 57 58 /* Copy data for doorbell */ 59 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 60 61 sq->mbuf[pi].mbuf = NULL; 62 sq->mbuf[pi].num_bytes = 0; 63 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 64 sq->pc += sq->mbuf[pi].num_wqebbs; 65 } 66 67 #if (__FreeBSD_version >= 1100000) 68 static uint32_t mlx5e_hash_value; 69 70 static void 71 mlx5e_hash_init(void *arg) 72 { 73 mlx5e_hash_value = m_ether_tcpip_hash_init(); 74 } 75 76 /* Make kernel call mlx5e_hash_init after the random stack finished initializing */ 77 SYSINIT(mlx5e_hash_init, SI_SUB_RANDOM, SI_ORDER_ANY, &mlx5e_hash_init, NULL); 78 #endif 79 80 static struct mlx5e_sq * 81 mlx5e_select_queue_by_send_tag(struct ifnet *ifp, struct mbuf *mb) 82 { 83 struct mlx5e_snd_tag *ptag; 84 struct mlx5e_sq *sq; 85 86 /* get pointer to sendqueue */ 87 ptag = container_of(mb->m_pkthdr.snd_tag, 88 struct mlx5e_snd_tag, m_snd_tag); 89 90 switch (ptag->type) { 91 #ifdef RATELIMIT 92 case IF_SND_TAG_TYPE_RATE_LIMIT: 93 sq = container_of(ptag, 94 struct mlx5e_rl_channel, tag)->sq; 95 break; 96 #endif 97 case IF_SND_TAG_TYPE_UNLIMITED: 98 sq = &container_of(ptag, 99 struct mlx5e_channel, tag)->sq[0]; 100 KASSERT((ptag->m_snd_tag.refcount > 0), 101 ("mlx5e_select_queue: Channel refs are zero for unlimited tag")); 102 break; 103 default: 104 sq = NULL; 105 break; 106 } 107 108 /* check if valid */ 109 if (sq != NULL && READ_ONCE(sq->running) != 0) 110 return (sq); 111 112 return (NULL); 113 } 114 115 static struct mlx5e_sq * 116 mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb) 117 { 118 struct mlx5e_priv *priv = ifp->if_softc; 119 struct mlx5e_sq *sq; 120 u32 ch; 121 u32 tc; 122 123 /* obtain VLAN information if present */ 124 if (mb->m_flags & M_VLANTAG) { 125 tc = (mb->m_pkthdr.ether_vtag >> 13); 126 if (tc >= priv->num_tc) 127 tc = priv->default_vlan_prio; 128 } else { 129 tc = priv->default_vlan_prio; 130 } 131 132 ch = priv->params.num_channels; 133 134 /* check if flowid is set */ 135 if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) { 136 #ifdef RSS 137 u32 temp; 138 139 if (rss_hash2bucket(mb->m_pkthdr.flowid, 140 M_HASHTYPE_GET(mb), &temp) == 0) 141 ch = temp % ch; 142 else 143 #endif 144 ch = (mb->m_pkthdr.flowid % 128) % ch; 145 } else { 146 #if (__FreeBSD_version >= 1100000) 147 ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 | 148 MBUF_HASHFLAG_L4, mb, mlx5e_hash_value) % ch; 149 #else 150 /* 151 * m_ether_tcpip_hash not present in stable, so just 152 * throw unhashed mbufs on queue 0 153 */ 154 ch = 0; 155 #endif 156 } 157 158 /* check if send queue is running */ 159 sq = &priv->channel[ch].sq[tc]; 160 if (likely(READ_ONCE(sq->running) != 0)) 161 return (sq); 162 return (NULL); 163 } 164 165 static inline u16 166 mlx5e_get_l2_header_size(struct mlx5e_sq *sq, struct mbuf *mb) 167 { 168 struct ether_vlan_header *eh; 169 uint16_t eth_type; 170 int min_inline; 171 172 eh = mtod(mb, struct ether_vlan_header *); 173 if (unlikely(mb->m_len < ETHER_HDR_LEN)) { 174 goto max_inline; 175 } else if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 176 if (unlikely(mb->m_len < (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN))) 177 goto max_inline; 178 eth_type = ntohs(eh->evl_proto); 179 min_inline = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 180 } else { 181 eth_type = ntohs(eh->evl_encap_proto); 182 min_inline = ETHER_HDR_LEN; 183 } 184 185 switch (eth_type) { 186 case ETHERTYPE_IP: 187 case ETHERTYPE_IPV6: 188 /* 189 * Make sure the TOS(IPv4) or traffic class(IPv6) 190 * field gets inlined. Else the SQ may stall. 191 */ 192 min_inline += 4; 193 break; 194 default: 195 goto max_inline; 196 } 197 198 /* 199 * m_copydata() will be used on the remaining header which 200 * does not need to reside within the first m_len bytes of 201 * data: 202 */ 203 if (mb->m_pkthdr.len < min_inline) 204 goto max_inline; 205 return (min_inline); 206 207 max_inline: 208 return (MIN(mb->m_pkthdr.len, sq->max_inline)); 209 } 210 211 static int 212 mlx5e_get_full_header_size(struct mbuf *mb) 213 { 214 struct ether_vlan_header *eh; 215 struct tcphdr *th; 216 struct ip *ip; 217 int ip_hlen, tcp_hlen; 218 struct ip6_hdr *ip6; 219 uint16_t eth_type; 220 int eth_hdr_len; 221 222 eh = mtod(mb, struct ether_vlan_header *); 223 if (mb->m_len < ETHER_HDR_LEN) 224 return (0); 225 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 226 if (mb->m_len < (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)) 227 return (0); 228 eth_type = ntohs(eh->evl_proto); 229 eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 230 } else { 231 eth_type = ntohs(eh->evl_encap_proto); 232 eth_hdr_len = ETHER_HDR_LEN; 233 } 234 switch (eth_type) { 235 case ETHERTYPE_IP: 236 ip = (struct ip *)(mb->m_data + eth_hdr_len); 237 if (mb->m_len < eth_hdr_len + sizeof(*ip)) 238 return (0); 239 switch (ip->ip_p) { 240 case IPPROTO_TCP: 241 ip_hlen = ip->ip_hl << 2; 242 eth_hdr_len += ip_hlen; 243 break; 244 case IPPROTO_UDP: 245 ip_hlen = ip->ip_hl << 2; 246 eth_hdr_len += ip_hlen + 8; 247 goto done; 248 default: 249 return (0); 250 } 251 break; 252 case ETHERTYPE_IPV6: 253 ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len); 254 if (mb->m_len < eth_hdr_len + sizeof(*ip6)) 255 return (0); 256 switch (ip6->ip6_nxt) { 257 case IPPROTO_TCP: 258 eth_hdr_len += sizeof(*ip6); 259 break; 260 case IPPROTO_UDP: 261 eth_hdr_len += sizeof(*ip6) + 8; 262 goto done; 263 default: 264 return (0); 265 } 266 break; 267 default: 268 return (0); 269 } 270 if (mb->m_len < eth_hdr_len + sizeof(*th)) 271 return (0); 272 th = (struct tcphdr *)(mb->m_data + eth_hdr_len); 273 tcp_hlen = th->th_off << 2; 274 eth_hdr_len += tcp_hlen; 275 done: 276 /* 277 * m_copydata() will be used on the remaining header which 278 * does not need to reside within the first m_len bytes of 279 * data: 280 */ 281 if (mb->m_pkthdr.len < eth_hdr_len) 282 return (0); 283 return (eth_hdr_len); 284 } 285 286 static int 287 mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp) 288 { 289 bus_dma_segment_t segs[MLX5E_MAX_TX_MBUF_FRAGS]; 290 struct mlx5_wqe_data_seg *dseg; 291 struct mlx5e_tx_wqe *wqe; 292 struct ifnet *ifp; 293 int nsegs; 294 int err; 295 int x; 296 struct mbuf *mb = *mbp; 297 u16 ds_cnt; 298 u16 ihs; 299 u16 pi; 300 u8 opcode; 301 302 /* Return ENOBUFS if the queue is full */ 303 if (unlikely(!mlx5e_sq_has_room_for(sq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) 304 return (ENOBUFS); 305 306 /* Align SQ edge with NOPs to avoid WQE wrap around */ 307 pi = ((~sq->pc) & sq->wq.sz_m1); 308 if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) { 309 /* Send one multi NOP message instead of many */ 310 mlx5e_send_nop(sq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS); 311 pi = ((~sq->pc) & sq->wq.sz_m1); 312 if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) 313 return (ENOMEM); 314 } 315 316 /* Setup local variables */ 317 pi = sq->pc & sq->wq.sz_m1; 318 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 319 ifp = sq->ifp; 320 321 memset(wqe, 0, sizeof(*wqe)); 322 323 /* Send a copy of the frame to the BPF listener, if any */ 324 if (ifp != NULL && ifp->if_bpf != NULL) 325 ETHER_BPF_MTAP(ifp, mb); 326 327 if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) { 328 wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_CSUM; 329 } 330 if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) { 331 wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_CSUM; 332 } 333 if (wqe->eth.cs_flags == 0) { 334 sq->stats.csum_offload_none++; 335 } 336 if (mb->m_pkthdr.csum_flags & CSUM_TSO) { 337 u32 payload_len; 338 u32 mss = mb->m_pkthdr.tso_segsz; 339 u32 num_pkts; 340 341 wqe->eth.mss = cpu_to_be16(mss); 342 opcode = MLX5_OPCODE_LSO; 343 ihs = mlx5e_get_full_header_size(mb); 344 if (unlikely(ihs == 0)) { 345 err = EINVAL; 346 goto tx_drop; 347 } 348 payload_len = mb->m_pkthdr.len - ihs; 349 if (payload_len == 0) 350 num_pkts = 1; 351 else 352 num_pkts = DIV_ROUND_UP(payload_len, mss); 353 sq->mbuf[pi].num_bytes = payload_len + (num_pkts * ihs); 354 355 sq->stats.tso_packets++; 356 sq->stats.tso_bytes += payload_len; 357 } else { 358 opcode = MLX5_OPCODE_SEND; 359 360 switch (sq->min_inline_mode) { 361 case MLX5_INLINE_MODE_IP: 362 case MLX5_INLINE_MODE_TCP_UDP: 363 ihs = mlx5e_get_full_header_size(mb); 364 if (unlikely(ihs == 0)) 365 ihs = mlx5e_get_l2_header_size(sq, mb); 366 break; 367 case MLX5_INLINE_MODE_L2: 368 ihs = mlx5e_get_l2_header_size(sq, mb); 369 break; 370 case MLX5_INLINE_MODE_NONE: 371 /* FALLTHROUGH */ 372 default: 373 if ((mb->m_flags & M_VLANTAG) != 0 && 374 (sq->min_insert_caps & MLX5E_INSERT_VLAN) != 0) { 375 /* inlining VLAN data is not required */ 376 wqe->eth.vlan_cmd = htons(0x8000); /* bit 0 CVLAN */ 377 wqe->eth.vlan_hdr = htons(mb->m_pkthdr.ether_vtag); 378 ihs = 0; 379 } else if ((mb->m_flags & M_VLANTAG) == 0 && 380 (sq->min_insert_caps & MLX5E_INSERT_NON_VLAN) != 0) { 381 /* inlining non-VLAN data is not required */ 382 ihs = 0; 383 } else { 384 /* we are forced to inlining L2 header, if any */ 385 ihs = mlx5e_get_l2_header_size(sq, mb); 386 } 387 break; 388 } 389 sq->mbuf[pi].num_bytes = max_t (unsigned int, 390 mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); 391 } 392 393 if (likely(ihs == 0)) { 394 /* nothing to inline */ 395 } else if (unlikely(ihs > sq->max_inline)) { 396 /* inline header size is too big */ 397 err = EINVAL; 398 goto tx_drop; 399 } else if ((mb->m_flags & M_VLANTAG) != 0) { 400 struct ether_vlan_header *eh = (struct ether_vlan_header *) 401 wqe->eth.inline_hdr_start; 402 403 /* Range checks */ 404 if (unlikely(ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN))) 405 ihs = (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN); 406 else if (unlikely(ihs < ETHER_HDR_LEN)) { 407 err = EINVAL; 408 goto tx_drop; 409 } 410 m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh); 411 m_adj(mb, ETHER_HDR_LEN); 412 /* Insert 4 bytes VLAN tag into data stream */ 413 eh->evl_proto = eh->evl_encap_proto; 414 eh->evl_encap_proto = htons(ETHERTYPE_VLAN); 415 eh->evl_tag = htons(mb->m_pkthdr.ether_vtag); 416 /* Copy rest of header data, if any */ 417 m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1)); 418 m_adj(mb, ihs - ETHER_HDR_LEN); 419 /* Extend header by 4 bytes */ 420 ihs += ETHER_VLAN_ENCAP_LEN; 421 wqe->eth.inline_hdr_sz = cpu_to_be16(ihs); 422 } else { 423 m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start); 424 m_adj(mb, ihs); 425 wqe->eth.inline_hdr_sz = cpu_to_be16(ihs); 426 } 427 428 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; 429 if (ihs > sizeof(wqe->eth.inline_hdr_start)) { 430 ds_cnt += DIV_ROUND_UP(ihs - sizeof(wqe->eth.inline_hdr_start), 431 MLX5_SEND_WQE_DS); 432 } 433 dseg = ((struct mlx5_wqe_data_seg *)&wqe->ctrl) + ds_cnt; 434 435 err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map, 436 mb, segs, &nsegs, BUS_DMA_NOWAIT); 437 if (err == EFBIG) { 438 /* Update statistics */ 439 sq->stats.defragged++; 440 /* Too many mbuf fragments */ 441 mb = m_defrag(*mbp, M_NOWAIT); 442 if (mb == NULL) { 443 mb = *mbp; 444 goto tx_drop; 445 } 446 /* Try again */ 447 err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map, 448 mb, segs, &nsegs, BUS_DMA_NOWAIT); 449 } 450 /* Catch errors */ 451 if (err != 0) 452 goto tx_drop; 453 454 /* Make sure all mbuf data, if any, is written to RAM */ 455 if (nsegs != 0) { 456 bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map, 457 BUS_DMASYNC_PREWRITE); 458 } else { 459 /* All data was inlined, free the mbuf. */ 460 bus_dmamap_unload(sq->dma_tag, sq->mbuf[pi].dma_map); 461 m_freem(mb); 462 mb = NULL; 463 } 464 465 for (x = 0; x != nsegs; x++) { 466 if (segs[x].ds_len == 0) 467 continue; 468 dseg->addr = cpu_to_be64((uint64_t)segs[x].ds_addr); 469 dseg->lkey = sq->mkey_be; 470 dseg->byte_count = cpu_to_be32((uint32_t)segs[x].ds_len); 471 dseg++; 472 } 473 474 ds_cnt = (dseg - ((struct mlx5_wqe_data_seg *)&wqe->ctrl)); 475 476 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode); 477 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 478 if (mlx5e_do_send_cqe(sq)) 479 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 480 else 481 wqe->ctrl.fm_ce_se = 0; 482 483 /* Copy data for doorbell */ 484 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32)); 485 486 /* Store pointer to mbuf */ 487 sq->mbuf[pi].mbuf = mb; 488 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 489 sq->pc += sq->mbuf[pi].num_wqebbs; 490 491 /* Count all traffic going out */ 492 sq->stats.packets++; 493 sq->stats.bytes += sq->mbuf[pi].num_bytes; 494 495 *mbp = NULL; /* safety clear */ 496 return (0); 497 498 tx_drop: 499 sq->stats.dropped++; 500 *mbp = NULL; 501 m_freem(mb); 502 return err; 503 } 504 505 static void 506 mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget) 507 { 508 u16 sqcc; 509 510 /* 511 * sq->cc must be updated only after mlx5_cqwq_update_db_record(), 512 * otherwise a cq overrun may occur 513 */ 514 sqcc = sq->cc; 515 516 while (budget > 0) { 517 struct mlx5_cqe64 *cqe; 518 struct mbuf *mb; 519 u16 x; 520 u16 ci; 521 522 cqe = mlx5e_get_cqe(&sq->cq); 523 if (!cqe) 524 break; 525 526 mlx5_cqwq_pop(&sq->cq.wq); 527 528 /* update budget according to the event factor */ 529 budget -= sq->cev_factor; 530 531 for (x = 0; x != sq->cev_factor; x++) { 532 ci = sqcc & sq->wq.sz_m1; 533 mb = sq->mbuf[ci].mbuf; 534 sq->mbuf[ci].mbuf = NULL; 535 536 if (mb == NULL) { 537 if (sq->mbuf[ci].num_bytes == 0) { 538 /* NOP */ 539 sq->stats.nop++; 540 } 541 } else { 542 bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map, 543 BUS_DMASYNC_POSTWRITE); 544 bus_dmamap_unload(sq->dma_tag, sq->mbuf[ci].dma_map); 545 546 /* Free transmitted mbuf */ 547 m_freem(mb); 548 } 549 sqcc += sq->mbuf[ci].num_wqebbs; 550 } 551 } 552 553 mlx5_cqwq_update_db_record(&sq->cq.wq); 554 555 /* Ensure cq space is freed before enabling more cqes */ 556 atomic_thread_fence_rel(); 557 558 sq->cc = sqcc; 559 } 560 561 static int 562 mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb) 563 { 564 int err = 0; 565 566 if (unlikely((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 567 READ_ONCE(sq->running) == 0)) { 568 m_freem(mb); 569 return (ENETDOWN); 570 } 571 572 /* Do transmit */ 573 if (mlx5e_sq_xmit(sq, &mb) != 0) { 574 /* NOTE: m_freem() is NULL safe */ 575 m_freem(mb); 576 err = ENOBUFS; 577 } 578 579 /* Check if we need to write the doorbell */ 580 if (likely(sq->doorbell.d64 != 0)) { 581 mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0); 582 sq->doorbell.d64 = 0; 583 } 584 585 /* 586 * Check if we need to start the event timer which flushes the 587 * transmit ring on timeout: 588 */ 589 if (unlikely(sq->cev_next_state == MLX5E_CEV_STATE_INITIAL && 590 sq->cev_factor != 1)) { 591 /* start the timer */ 592 mlx5e_sq_cev_timeout(sq); 593 } else { 594 /* don't send NOPs yet */ 595 sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS; 596 } 597 return (err); 598 } 599 600 int 601 mlx5e_xmit(struct ifnet *ifp, struct mbuf *mb) 602 { 603 struct mlx5e_sq *sq; 604 int ret; 605 606 if (mb->m_pkthdr.csum_flags & CSUM_SND_TAG) { 607 MPASS(mb->m_pkthdr.snd_tag->ifp == ifp); 608 sq = mlx5e_select_queue_by_send_tag(ifp, mb); 609 if (unlikely(sq == NULL)) { 610 goto select_queue; 611 } 612 } else { 613 select_queue: 614 sq = mlx5e_select_queue(ifp, mb); 615 if (unlikely(sq == NULL)) { 616 /* Free mbuf */ 617 m_freem(mb); 618 619 /* Invalid send queue */ 620 return (ENXIO); 621 } 622 } 623 624 mtx_lock(&sq->lock); 625 ret = mlx5e_xmit_locked(ifp, sq, mb); 626 mtx_unlock(&sq->lock); 627 628 return (ret); 629 } 630 631 void 632 mlx5e_tx_cq_comp(struct mlx5_core_cq *mcq) 633 { 634 struct mlx5e_sq *sq = container_of(mcq, struct mlx5e_sq, cq.mcq); 635 636 mtx_lock(&sq->comp_lock); 637 mlx5e_poll_tx_cq(sq, MLX5E_BUDGET_MAX); 638 mlx5e_cq_arm(&sq->cq, MLX5_GET_DOORBELL_LOCK(&sq->priv->doorbell_lock)); 639 mtx_unlock(&sq->comp_lock); 640 } 641