1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 #include "opt_rss.h" 32 #include "ena.h" 33 #include "ena_datapath.h" 34 #ifdef DEV_NETMAP 35 #include "ena_netmap.h" 36 #endif /* DEV_NETMAP */ 37 #include <net/rss_config.h> 38 39 #include <netinet6/ip6_var.h> 40 41 /********************************************************************* 42 * Static functions prototypes 43 *********************************************************************/ 44 45 static bool ena_tx_cleanup(struct ena_ring *); 46 static bool ena_rx_cleanup(struct ena_ring *); 47 static inline int ena_get_tx_req_id(struct ena_ring *tx_ring, 48 struct ena_com_io_cq *io_cq, uint16_t *req_id); 49 static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *, 50 struct mbuf *); 51 static struct mbuf *ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *, 52 struct ena_com_rx_ctx *, uint16_t *); 53 static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *, 54 struct mbuf *); 55 static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *, bool); 56 static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, 57 struct mbuf **mbuf); 58 static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **); 59 static void ena_start_xmit(struct ena_ring *); 60 61 /********************************************************************* 62 * Global functions 63 *********************************************************************/ 64 65 void 66 ena_cleanup(void *arg, int pending) 67 { 68 struct ena_que *que = arg; 69 struct ena_adapter *adapter = que->adapter; 70 if_t ifp = adapter->ifp; 71 struct ena_ring *tx_ring; 72 struct ena_ring *rx_ring; 73 struct ena_com_io_cq *io_cq; 74 struct ena_eth_io_intr_reg intr_reg; 75 int qid, ena_qid; 76 int i; 77 bool rx_again, tx_again; 78 79 tx_ring = que->tx_ring; 80 rx_ring = que->rx_ring; 81 qid = que->id; 82 ena_qid = ENA_IO_TXQ_IDX(qid); 83 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 84 85 atomic_store_8(&tx_ring->cleanup_running, 1); 86 /* Need to make sure that ENA_FLAG_TRIGGER_RESET is visible to ena_cleanup() and 87 * that cleanup_running is visible to check_missing_comp_in_tx_queue() to 88 * prevent the case of accessing CQ concurrently with check_cdesc_in_tx_cq() 89 */ 90 mb(); 91 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || 92 (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)))) 93 return; 94 95 ena_log_io(adapter->pdev, DBG, "MSI-X TX/RX routine\n"); 96 97 atomic_store_8(&tx_ring->first_interrupt, 1); 98 atomic_store_8(&rx_ring->first_interrupt, 1); 99 100 for (i = 0; i < ENA_CLEAN_BUDGET; ++i) { 101 tx_again = ena_tx_cleanup(tx_ring); 102 rx_again = ena_rx_cleanup(rx_ring); 103 104 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || 105 (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)))) 106 return; 107 108 if (!rx_again && !tx_again) 109 break; 110 } 111 112 /* Signal that work is done and unmask interrupt */ 113 ena_com_update_intr_reg(&intr_reg, ENA_RX_IRQ_INTERVAL, 114 ENA_TX_IRQ_INTERVAL, true, false); 115 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1); 116 ena_com_unmask_intr(io_cq, &intr_reg); 117 atomic_store_8(&tx_ring->cleanup_running, 0); 118 } 119 120 void 121 ena_deferred_mq_start(void *arg, int pending) 122 { 123 struct ena_ring *tx_ring = (struct ena_ring *)arg; 124 if_t ifp = tx_ring->adapter->ifp; 125 126 while (!drbr_empty(ifp, tx_ring->br) && tx_ring->running && 127 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 128 ENA_RING_MTX_LOCK(tx_ring); 129 ena_start_xmit(tx_ring); 130 ENA_RING_MTX_UNLOCK(tx_ring); 131 } 132 } 133 134 int 135 ena_mq_start(if_t ifp, struct mbuf *m) 136 { 137 struct ena_adapter *adapter = if_getsoftc(ifp); 138 struct ena_ring *tx_ring; 139 int ret, is_drbr_empty; 140 uint32_t i; 141 #ifdef RSS 142 uint32_t bucket_id; 143 #endif 144 145 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0)) 146 return (ENODEV); 147 148 /* Which queue to use */ 149 /* 150 * If everything is setup correctly, it should be the 151 * same bucket that the current CPU we're on is. 152 * It should improve performance. 153 */ 154 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 155 #ifdef RSS 156 if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 157 &bucket_id) == 0) 158 i = bucket_id % adapter->num_io_queues; 159 else 160 #endif 161 i = m->m_pkthdr.flowid % adapter->num_io_queues; 162 } else { 163 i = curcpu % adapter->num_io_queues; 164 } 165 tx_ring = &adapter->tx_ring[i]; 166 167 /* Check if drbr is empty before putting packet */ 168 is_drbr_empty = drbr_empty(ifp, tx_ring->br); 169 ret = drbr_enqueue(ifp, tx_ring->br, m); 170 if (unlikely(ret != 0)) { 171 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 172 return (ret); 173 } 174 175 if (is_drbr_empty && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) { 176 ena_start_xmit(tx_ring); 177 ENA_RING_MTX_UNLOCK(tx_ring); 178 } else { 179 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 180 } 181 182 return (0); 183 } 184 185 void 186 ena_qflush(if_t ifp) 187 { 188 struct ena_adapter *adapter = if_getsoftc(ifp); 189 struct ena_ring *tx_ring = adapter->tx_ring; 190 int i; 191 192 for (i = 0; i < adapter->num_io_queues; ++i, ++tx_ring) 193 if (!drbr_empty(ifp, tx_ring->br)) { 194 ENA_RING_MTX_LOCK(tx_ring); 195 drbr_flush(ifp, tx_ring->br); 196 ENA_RING_MTX_UNLOCK(tx_ring); 197 } 198 199 if_qflush(ifp); 200 } 201 202 /********************************************************************* 203 * Static functions 204 *********************************************************************/ 205 206 static inline int 207 ena_get_tx_req_id(struct ena_ring *tx_ring, struct ena_com_io_cq *io_cq, 208 uint16_t *req_id) 209 { 210 struct ena_adapter *adapter = tx_ring->adapter; 211 int rc = ena_com_tx_comp_req_id_get(io_cq, req_id); 212 213 if (unlikely(rc == ENA_COM_TRY_AGAIN)) 214 return (EAGAIN); 215 216 rc = validate_tx_req_id(tx_ring, *req_id, rc); 217 218 if (unlikely(tx_ring->tx_buffer_info[*req_id].mbuf == NULL)) { 219 ena_log(adapter->pdev, ERR, 220 "tx_info doesn't have valid mbuf. req_id %hu qid %hu\n", 221 *req_id, tx_ring->qid); 222 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_TX_REQ_ID); 223 rc = EFAULT; 224 } 225 226 return (rc); 227 } 228 229 /** 230 * ena_tx_cleanup - clear sent packets and corresponding descriptors 231 * @tx_ring: ring for which we want to clean packets 232 * 233 * Once packets are sent, we ask the device in a loop for no longer used 234 * descriptors. We find the related mbuf chain in a map (index in an array) 235 * and free it, then update ring state. 236 * This is performed in "endless" loop, updating ring pointers every 237 * TX_COMMIT. The first check of free descriptor is performed before the actual 238 * loop, then repeated at the loop end. 239 **/ 240 static bool 241 ena_tx_cleanup(struct ena_ring *tx_ring) 242 { 243 struct ena_adapter *adapter; 244 struct ena_com_io_cq *io_cq; 245 uint16_t next_to_clean; 246 uint16_t req_id; 247 uint16_t ena_qid; 248 unsigned int total_done = 0; 249 int rc; 250 int commit = ENA_TX_COMMIT; 251 int budget = ENA_TX_BUDGET; 252 bool above_thresh; 253 254 adapter = tx_ring->que->adapter; 255 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 256 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 257 next_to_clean = tx_ring->next_to_clean; 258 259 #ifdef DEV_NETMAP 260 if (netmap_tx_irq(adapter->ifp, tx_ring->qid) != NM_IRQ_PASS) 261 return (0); 262 #endif /* DEV_NETMAP */ 263 264 do { 265 struct ena_tx_buffer *tx_info; 266 struct mbuf *mbuf; 267 268 rc = ena_get_tx_req_id(tx_ring, io_cq, &req_id); 269 if (unlikely(rc != 0)) 270 break; 271 272 tx_info = &tx_ring->tx_buffer_info[req_id]; 273 274 mbuf = tx_info->mbuf; 275 276 tx_info->mbuf = NULL; 277 atomic_store_64(&tx_info->timestamp, 0); 278 279 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 280 BUS_DMASYNC_POSTWRITE); 281 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); 282 283 ena_log_io(adapter->pdev, DBG, "tx: q %d mbuf %p completed\n", 284 tx_ring->qid, mbuf); 285 286 m_freem(mbuf); 287 288 total_done += tx_info->tx_descs; 289 290 tx_ring->free_tx_ids[next_to_clean] = req_id; 291 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean, 292 tx_ring->ring_size); 293 294 if (unlikely(--commit == 0)) { 295 commit = ENA_TX_COMMIT; 296 /* update ring state every ENA_TX_COMMIT descriptor */ 297 tx_ring->next_to_clean = next_to_clean; 298 ena_com_comp_ack( 299 &adapter->ena_dev->io_sq_queues[ena_qid], 300 total_done); 301 total_done = 0; 302 } 303 } while (likely(--budget)); 304 305 ena_log_io(adapter->pdev, DBG, "tx: q %d done. total pkts: %d\n", 306 tx_ring->qid, ENA_TX_BUDGET - budget); 307 308 /* If there is still something to commit update ring state */ 309 if (likely(commit != ENA_TX_COMMIT)) { 310 tx_ring->next_to_clean = next_to_clean; 311 ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid], 312 total_done); 313 } 314 315 /* 316 * Need to make the rings circular update visible to 317 * ena_xmit_mbuf() before checking for tx_ring->running. 318 */ 319 mb(); 320 321 above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 322 ENA_TX_RESUME_THRESH); 323 if (unlikely(!tx_ring->running && above_thresh)) { 324 ENA_RING_MTX_LOCK(tx_ring); 325 above_thresh = ena_com_sq_have_enough_space( 326 tx_ring->ena_com_io_sq, ENA_TX_RESUME_THRESH); 327 if (!tx_ring->running && above_thresh) { 328 tx_ring->running = true; 329 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1); 330 taskqueue_enqueue(tx_ring->enqueue_tq, 331 &tx_ring->enqueue_task); 332 } 333 ENA_RING_MTX_UNLOCK(tx_ring); 334 } 335 336 tx_ring->tx_last_cleanup_ticks = ticks; 337 338 return (budget == 0); 339 } 340 341 static void 342 ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, 343 struct mbuf *mbuf) 344 { 345 struct ena_adapter *adapter = rx_ring->adapter; 346 347 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { 348 mbuf->m_pkthdr.flowid = ena_rx_ctx->hash; 349 350 /* 351 * Hardware and software RSS are in agreement only when both are 352 * configured to Toeplitz algorithm. This driver configures 353 * that algorithm only when software RSS is enabled and uses it. 354 */ 355 if (adapter->ena_dev->rss.hash_func != ENA_ADMIN_TOEPLITZ && 356 ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN) { 357 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 358 return; 359 } 360 361 if (ena_rx_ctx->frag && 362 (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) { 363 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 364 return; 365 } 366 367 switch (ena_rx_ctx->l3_proto) { 368 case ENA_ETH_IO_L3_PROTO_IPV4: 369 switch (ena_rx_ctx->l4_proto) { 370 case ENA_ETH_IO_L4_PROTO_TCP: 371 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4); 372 break; 373 case ENA_ETH_IO_L4_PROTO_UDP: 374 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4); 375 break; 376 default: 377 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4); 378 } 379 break; 380 case ENA_ETH_IO_L3_PROTO_IPV6: 381 switch (ena_rx_ctx->l4_proto) { 382 case ENA_ETH_IO_L4_PROTO_TCP: 383 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6); 384 break; 385 case ENA_ETH_IO_L4_PROTO_UDP: 386 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6); 387 break; 388 default: 389 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6); 390 } 391 break; 392 case ENA_ETH_IO_L3_PROTO_UNKNOWN: 393 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); 394 break; 395 default: 396 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 397 } 398 } else { 399 mbuf->m_pkthdr.flowid = rx_ring->qid; 400 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); 401 } 402 } 403 404 /** 405 * ena_rx_mbuf - assemble mbuf from descriptors 406 * @rx_ring: ring for which we want to clean packets 407 * @ena_bufs: buffer info 408 * @ena_rx_ctx: metadata for this packet(s) 409 * @next_to_clean: ring pointer, will be updated only upon success 410 * 411 **/ 412 static struct mbuf * 413 ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs, 414 struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean) 415 { 416 struct mbuf *mbuf; 417 struct ena_rx_buffer *rx_info; 418 struct ena_adapter *adapter; 419 device_t pdev; 420 unsigned int descs = ena_rx_ctx->descs; 421 uint16_t ntc, len, req_id, buf = 0; 422 423 ntc = *next_to_clean; 424 adapter = rx_ring->adapter; 425 pdev = adapter->pdev; 426 427 len = ena_bufs[buf].len; 428 req_id = ena_bufs[buf].req_id; 429 rx_info = &rx_ring->rx_buffer_info[req_id]; 430 if (unlikely(rx_info->mbuf == NULL)) { 431 ena_log(pdev, ERR, "NULL mbuf in rx_info. qid %u req_id %u\n", 432 rx_ring->qid, req_id); 433 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_RX_REQ_ID); 434 return (NULL); 435 } 436 437 ena_log_io(pdev, DBG, "rx_info %p, mbuf %p, paddr %jx\n", rx_info, 438 rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr); 439 440 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 441 BUS_DMASYNC_POSTREAD); 442 mbuf = rx_info->mbuf; 443 mbuf->m_flags |= M_PKTHDR; 444 mbuf->m_pkthdr.len = len; 445 mbuf->m_len = len; 446 /* Only for the first segment the data starts at specific offset */ 447 mbuf->m_data = mtodo(mbuf, ena_rx_ctx->pkt_offset); 448 ena_log_io(pdev, DBG, "Mbuf data offset=%u\n", ena_rx_ctx->pkt_offset); 449 mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp; 450 451 /* Fill mbuf with hash key and it's interpretation for optimization */ 452 ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf); 453 454 ena_log_io(pdev, DBG, "rx mbuf 0x%p, flags=0x%x, len: %d\n", mbuf, 455 mbuf->m_flags, mbuf->m_pkthdr.len); 456 457 /* DMA address is not needed anymore, unmap it */ 458 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map); 459 460 rx_info->mbuf = NULL; 461 rx_ring->free_rx_ids[ntc] = req_id; 462 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); 463 464 /* 465 * While we have more than 1 descriptors for one rcvd packet, append 466 * other mbufs to the main one 467 */ 468 while (--descs) { 469 ++buf; 470 len = ena_bufs[buf].len; 471 req_id = ena_bufs[buf].req_id; 472 rx_info = &rx_ring->rx_buffer_info[req_id]; 473 474 if (unlikely(rx_info->mbuf == NULL)) { 475 ena_log(pdev, ERR, "NULL mbuf in rx_info. qid %u req_id %u\n", 476 rx_ring->qid, req_id); 477 /* 478 * If one of the required mbufs was not allocated yet, 479 * we can break there. 480 * All earlier used descriptors will be reallocated 481 * later and not used mbufs can be reused. 482 * The next_to_clean pointer will not be updated in case 483 * of an error, so caller should advance it manually 484 * in error handling routine to keep it up to date 485 * with hw ring. 486 */ 487 m_freem(mbuf); 488 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_RX_REQ_ID); 489 return (NULL); 490 } 491 492 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 493 BUS_DMASYNC_POSTREAD); 494 if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) { 495 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); 496 ena_log_io(pdev, WARN, "Failed to append Rx mbuf %p\n", 497 mbuf); 498 } 499 500 ena_log_io(pdev, DBG, "rx mbuf updated. len %d\n", 501 mbuf->m_pkthdr.len); 502 503 /* Free already appended mbuf, it won't be useful anymore */ 504 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map); 505 m_freem(rx_info->mbuf); 506 rx_info->mbuf = NULL; 507 508 rx_ring->free_rx_ids[ntc] = req_id; 509 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); 510 } 511 512 *next_to_clean = ntc; 513 514 return (mbuf); 515 } 516 517 /** 518 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum 519 **/ 520 static inline void 521 ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, 522 struct mbuf *mbuf) 523 { 524 device_t pdev = rx_ring->adapter->pdev; 525 526 /* if IP and error */ 527 if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) && 528 ena_rx_ctx->l3_csum_err)) { 529 /* ipv4 checksum error */ 530 mbuf->m_pkthdr.csum_flags = 0; 531 counter_u64_add(rx_ring->rx_stats.csum_bad, 1); 532 ena_log_io(pdev, DBG, "RX IPv4 header checksum error\n"); 533 return; 534 } 535 536 /* if TCP/UDP */ 537 if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) || 538 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) { 539 if (ena_rx_ctx->l4_csum_err) { 540 /* TCP/UDP checksum error */ 541 mbuf->m_pkthdr.csum_flags = 0; 542 counter_u64_add(rx_ring->rx_stats.csum_bad, 1); 543 ena_log_io(pdev, DBG, "RX L4 checksum error\n"); 544 } else { 545 mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 546 mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID; 547 counter_u64_add(rx_ring->rx_stats.csum_good, 1); 548 } 549 } 550 } 551 552 /** 553 * ena_rx_cleanup - handle rx irq 554 * @arg: ring for which irq is being handled 555 **/ 556 static bool 557 ena_rx_cleanup(struct ena_ring *rx_ring) 558 { 559 struct ena_adapter *adapter; 560 device_t pdev; 561 struct mbuf *mbuf; 562 struct ena_com_rx_ctx ena_rx_ctx; 563 struct ena_com_io_cq *io_cq; 564 struct ena_com_io_sq *io_sq; 565 enum ena_regs_reset_reason_types reset_reason; 566 if_t ifp; 567 uint16_t ena_qid; 568 uint16_t next_to_clean; 569 uint32_t refill_required; 570 uint32_t refill_threshold; 571 uint32_t do_if_input = 0; 572 unsigned int qid; 573 int rc, i; 574 int budget = (ENA_RX_DESC_BUDGET == -1) ? INT_MAX : ENA_RX_DESC_BUDGET; 575 uint64_t total_pkts = 0; 576 uint64_t total_bytes = 0; 577 #ifdef DEV_NETMAP 578 int done; 579 #endif /* DEV_NETMAP */ 580 581 adapter = rx_ring->que->adapter; 582 pdev = adapter->pdev; 583 ifp = adapter->ifp; 584 qid = rx_ring->que->id; 585 ena_qid = ENA_IO_RXQ_IDX(qid); 586 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 587 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; 588 next_to_clean = rx_ring->next_to_clean; 589 590 #ifdef DEV_NETMAP 591 if (netmap_rx_irq(adapter->ifp, rx_ring->qid, &done) != NM_IRQ_PASS) 592 return (0); 593 #endif /* DEV_NETMAP */ 594 595 ena_log_io(pdev, DBG, "rx: qid %d\n", qid); 596 597 do { 598 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 599 ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size; 600 ena_rx_ctx.descs = 0; 601 ena_rx_ctx.pkt_offset = 0; 602 603 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag, 604 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD); 605 rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx); 606 if (unlikely(rc != 0)) { 607 if (rc == ENA_COM_NO_SPACE) { 608 counter_u64_add(rx_ring->rx_stats.bad_desc_num, 609 1); 610 reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS; 611 } else if (rc == ENA_COM_FAULT) { 612 reset_reason = ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED; 613 } else { 614 counter_u64_add(rx_ring->rx_stats.bad_req_id, 615 1); 616 reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; 617 } 618 ena_trigger_reset(adapter, reset_reason); 619 goto update_stats; 620 } 621 622 if (unlikely(ena_rx_ctx.descs == 0)) 623 break; 624 625 ena_log_io(pdev, DBG, 626 "rx: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n", 627 rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto, 628 ena_rx_ctx.l4_proto, ena_rx_ctx.hash); 629 630 /* Receive mbuf from the ring */ 631 mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs, &ena_rx_ctx, 632 &next_to_clean); 633 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag, 634 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD); 635 /* Exit if we failed to retrieve a buffer */ 636 if (unlikely(mbuf == NULL)) { 637 for (i = 0; i < ena_rx_ctx.descs; ++i) { 638 rx_ring->free_rx_ids[next_to_clean] = 639 rx_ring->ena_bufs[i].req_id; 640 next_to_clean = ENA_RX_RING_IDX_NEXT( 641 next_to_clean, rx_ring->ring_size); 642 } 643 break; 644 } 645 646 if (((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) || 647 ((if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6) != 0)) { 648 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf); 649 } 650 651 total_bytes += mbuf->m_pkthdr.len; 652 /* 653 * LRO is only for IP/TCP packets and TCP checksum of the packet 654 * should be computed by hardware. 655 */ 656 do_if_input = 1; 657 if (((if_getcapenable(ifp) & IFCAP_LRO) != 0) && 658 ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) && 659 (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) { 660 /* 661 * Send to the stack if: 662 * - LRO not enabled, or 663 * - no LRO resources, or 664 * - lro enqueue fails 665 */ 666 if ((rx_ring->lro.lro_cnt != 0) && 667 (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0)) 668 do_if_input = 0; 669 } 670 if (do_if_input != 0) { 671 ena_log_io(pdev, DBG, 672 "calling if_input() with mbuf %p\n", mbuf); 673 if_input(ifp, mbuf); 674 } 675 676 total_pkts++; 677 678 /* 679 * Adjust our budget; note that we count descriptors, not 680 * packets, since we need to ensure we don't run out of rx 681 * buffers when receiving jumbos. 682 */ 683 budget -= ena_rx_ctx.descs; 684 } while (budget > 0); 685 686 rx_ring->next_to_clean = next_to_clean; 687 688 refill_required = ena_com_free_q_entries(io_sq); 689 refill_threshold = min_t(int, 690 rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER, 691 ENA_RX_REFILL_THRESH_PACKET); 692 693 if (refill_required > refill_threshold) { 694 ena_refill_rx_bufs(rx_ring, refill_required); 695 } 696 697 tcp_lro_flush_all(&rx_ring->lro); 698 699 update_stats: 700 counter_enter(); 701 counter_u64_add_protected(rx_ring->rx_stats.cnt, total_pkts); 702 counter_u64_add_protected(rx_ring->rx_stats.bytes, total_bytes); 703 counter_u64_add_protected(adapter->hw_stats.rx_packets, total_pkts); 704 counter_u64_add_protected(adapter->hw_stats.rx_bytes, total_bytes); 705 counter_exit(); 706 return (budget <= 0); 707 } 708 709 static void 710 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf, 711 bool disable_meta_caching) 712 { 713 struct ena_com_tx_meta *ena_meta; 714 struct ether_vlan_header *eh; 715 struct mbuf *mbuf_next; 716 u32 mss; 717 bool offload; 718 uint16_t etype; 719 int ehdrlen; 720 struct ip *ip; 721 int ipproto; 722 int iphlen; 723 struct tcphdr *th; 724 int offset; 725 726 offload = false; 727 ena_meta = &ena_tx_ctx->ena_meta; 728 mss = mbuf->m_pkthdr.tso_segsz; 729 730 if (mss != 0) 731 offload = true; 732 733 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) 734 offload = true; 735 736 if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0) 737 offload = true; 738 739 if ((mbuf->m_pkthdr.csum_flags & CSUM6_OFFLOAD) != 0) 740 offload = true; 741 742 if (!offload) { 743 if (disable_meta_caching) { 744 memset(ena_meta, 0, sizeof(*ena_meta)); 745 ena_tx_ctx->meta_valid = 1; 746 } else { 747 ena_tx_ctx->meta_valid = 0; 748 } 749 return; 750 } 751 752 /* Determine where frame payload starts. */ 753 eh = mtod(mbuf, struct ether_vlan_header *); 754 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 755 etype = ntohs(eh->evl_proto); 756 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 757 } else { 758 etype = ntohs(eh->evl_encap_proto); 759 ehdrlen = ETHER_HDR_LEN; 760 } 761 762 mbuf_next = m_getptr(mbuf, ehdrlen, &offset); 763 764 switch (etype) { 765 case ETHERTYPE_IP: 766 ip = (struct ip *)(mtodo(mbuf_next, offset)); 767 iphlen = ip->ip_hl << 2; 768 ipproto = ip->ip_p; 769 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 770 if ((ip->ip_off & htons(IP_DF)) != 0) 771 ena_tx_ctx->df = 1; 772 break; 773 case ETHERTYPE_IPV6: 774 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 775 iphlen = ip6_lasthdr(mbuf, ehdrlen, IPPROTO_IPV6, &ipproto); 776 iphlen -= ehdrlen; 777 ena_tx_ctx->df = 1; 778 break; 779 default: 780 iphlen = 0; 781 ipproto = 0; 782 break; 783 } 784 785 mbuf_next = m_getptr(mbuf, iphlen + ehdrlen, &offset); 786 th = (struct tcphdr *)(mtodo(mbuf_next, offset)); 787 788 if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) { 789 ena_tx_ctx->l3_csum_enable = 1; 790 } 791 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) { 792 ena_tx_ctx->tso_enable = 1; 793 ena_meta->l4_hdr_len = (th->th_off); 794 } 795 796 if (ipproto == IPPROTO_TCP) { 797 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 798 if ((mbuf->m_pkthdr.csum_flags & 799 (CSUM_IP_TCP | CSUM_IP6_TCP)) != 0) 800 ena_tx_ctx->l4_csum_enable = 1; 801 else 802 ena_tx_ctx->l4_csum_enable = 0; 803 } else if (ipproto == IPPROTO_UDP) { 804 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 805 if ((mbuf->m_pkthdr.csum_flags & 806 (CSUM_IP_UDP | CSUM_IP6_UDP)) != 0) 807 ena_tx_ctx->l4_csum_enable = 1; 808 else 809 ena_tx_ctx->l4_csum_enable = 0; 810 } else { 811 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 812 ena_tx_ctx->l4_csum_enable = 0; 813 } 814 815 ena_meta->mss = mss; 816 ena_meta->l3_hdr_len = iphlen; 817 ena_meta->l3_hdr_offset = ehdrlen; 818 ena_tx_ctx->meta_valid = 1; 819 } 820 821 static int 822 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) 823 { 824 struct ena_adapter *adapter; 825 struct mbuf *collapsed_mbuf; 826 int num_frags; 827 828 adapter = tx_ring->adapter; 829 num_frags = ena_mbuf_count(*mbuf); 830 831 /* One segment must be reserved for configuration descriptor. */ 832 if (num_frags < adapter->max_tx_sgl_size) 833 return (0); 834 835 if ((num_frags == adapter->max_tx_sgl_size) && 836 ((*mbuf)->m_pkthdr.len < tx_ring->tx_max_header_size)) 837 return (0); 838 839 counter_u64_add(tx_ring->tx_stats.collapse, 1); 840 841 collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT, 842 adapter->max_tx_sgl_size - 1); 843 if (unlikely(collapsed_mbuf == NULL)) { 844 counter_u64_add(tx_ring->tx_stats.collapse_err, 1); 845 return (ENOMEM); 846 } 847 848 /* If mbuf was collapsed succesfully, original mbuf is released. */ 849 *mbuf = collapsed_mbuf; 850 851 return (0); 852 } 853 854 static int 855 ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info, 856 struct mbuf *mbuf, void **push_hdr, u16 *header_len) 857 { 858 struct ena_adapter *adapter = tx_ring->adapter; 859 struct ena_com_buf *ena_buf; 860 bus_dma_segment_t segs[ENA_BUS_DMA_SEGS]; 861 size_t iseg = 0; 862 uint32_t mbuf_head_len; 863 uint16_t offset; 864 int rc, nsegs; 865 866 mbuf_head_len = mbuf->m_len; 867 tx_info->mbuf = mbuf; 868 ena_buf = tx_info->bufs; 869 870 /* 871 * For easier maintaining of the DMA map, map the whole mbuf even if 872 * the LLQ is used. The descriptors will be filled using the segments. 873 */ 874 rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, 875 tx_info->dmamap, mbuf, segs, &nsegs, BUS_DMA_NOWAIT); 876 if (unlikely((rc != 0) || (nsegs == 0))) { 877 ena_log_io(adapter->pdev, WARN, 878 "dmamap load failed! err: %d nsegs: %d\n", rc, nsegs); 879 goto dma_error; 880 } 881 882 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 883 /* 884 * When the device is LLQ mode, the driver will copy 885 * the header into the device memory space. 886 * the ena_com layer assumes the header is in a linear 887 * memory space. 888 * This assumption might be wrong since part of the header 889 * can be in the fragmented buffers. 890 * First check if header fits in the mbuf. If not, copy it to 891 * separate buffer that will be holding linearized data. 892 */ 893 *header_len = min_t(uint32_t, mbuf->m_pkthdr.len, 894 tx_ring->tx_max_header_size); 895 896 /* If header is in linear space, just point into mbuf's data. */ 897 if (likely(*header_len <= mbuf_head_len)) { 898 *push_hdr = mbuf->m_data; 899 /* 900 * Otherwise, copy whole portion of header from multiple 901 * mbufs to intermediate buffer. 902 */ 903 } else { 904 m_copydata(mbuf, 0, *header_len, 905 tx_ring->push_buf_intermediate_buf); 906 *push_hdr = tx_ring->push_buf_intermediate_buf; 907 908 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1); 909 } 910 911 ena_log_io(adapter->pdev, DBG, 912 "mbuf: %p header_buf->vaddr: %p push_len: %d\n", 913 mbuf, *push_hdr, *header_len); 914 915 /* If packet is fitted in LLQ header, no need for DMA segments. */ 916 if (mbuf->m_pkthdr.len <= tx_ring->tx_max_header_size) { 917 return (0); 918 } else { 919 offset = tx_ring->tx_max_header_size; 920 /* 921 * As Header part is mapped to LLQ header, we can skip 922 * it and just map the residuum of the mbuf to DMA 923 * Segments. 924 */ 925 while (offset > 0) { 926 if (offset >= segs[iseg].ds_len) { 927 offset -= segs[iseg].ds_len; 928 } else { 929 ena_buf->paddr = segs[iseg].ds_addr + 930 offset; 931 ena_buf->len = segs[iseg].ds_len - 932 offset; 933 ena_buf++; 934 tx_info->num_of_bufs++; 935 offset = 0; 936 } 937 iseg++; 938 } 939 } 940 } else { 941 *push_hdr = NULL; 942 /* 943 * header_len is just a hint for the device. Because FreeBSD is 944 * not giving us information about packet header length and it 945 * is not guaranteed that all packet headers will be in the 1st 946 * mbuf, setting header_len to 0 is making the device ignore 947 * this value and resolve header on it's own. 948 */ 949 *header_len = 0; 950 } 951 952 /* Map rest of the mbuf */ 953 while (iseg < nsegs) { 954 ena_buf->paddr = segs[iseg].ds_addr; 955 ena_buf->len = segs[iseg].ds_len; 956 ena_buf++; 957 iseg++; 958 tx_info->num_of_bufs++; 959 } 960 961 return (0); 962 963 dma_error: 964 counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1); 965 tx_info->mbuf = NULL; 966 return (rc); 967 } 968 969 static int 970 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) 971 { 972 struct ena_adapter *adapter; 973 device_t pdev; 974 struct ena_tx_buffer *tx_info; 975 struct ena_com_tx_ctx ena_tx_ctx; 976 struct ena_com_dev *ena_dev; 977 struct ena_com_io_sq *io_sq; 978 void *push_hdr; 979 uint16_t next_to_use; 980 uint16_t req_id; 981 uint16_t ena_qid; 982 uint16_t header_len; 983 int rc; 984 int nb_hw_desc; 985 986 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 987 adapter = tx_ring->que->adapter; 988 pdev = adapter->pdev; 989 ena_dev = adapter->ena_dev; 990 io_sq = &ena_dev->io_sq_queues[ena_qid]; 991 992 rc = ena_check_and_collapse_mbuf(tx_ring, mbuf); 993 if (unlikely(rc != 0)) { 994 ena_log_io(pdev, WARN, "Failed to collapse mbuf! err: %d\n", 995 rc); 996 return (rc); 997 } 998 999 ena_log_io(pdev, DBG, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len); 1000 1001 next_to_use = tx_ring->next_to_use; 1002 req_id = tx_ring->free_tx_ids[next_to_use]; 1003 tx_info = &tx_ring->tx_buffer_info[req_id]; 1004 tx_info->num_of_bufs = 0; 1005 1006 ENA_WARN(tx_info->mbuf != NULL, adapter->ena_dev, 1007 "mbuf isn't NULL for req_id %d\n", req_id); 1008 1009 rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len); 1010 if (unlikely(rc != 0)) { 1011 ena_log_io(pdev, WARN, "Failed to map TX mbuf\n"); 1012 return (rc); 1013 } 1014 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 1015 ena_tx_ctx.ena_bufs = tx_info->bufs; 1016 ena_tx_ctx.push_header = push_hdr; 1017 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 1018 ena_tx_ctx.req_id = req_id; 1019 ena_tx_ctx.header_len = header_len; 1020 1021 /* Set flags and meta data */ 1022 ena_tx_csum(&ena_tx_ctx, *mbuf, adapter->disable_meta_caching); 1023 1024 if (tx_ring->acum_pkts == ENA_DB_THRESHOLD || 1025 ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, &ena_tx_ctx)) { 1026 ena_log_io(pdev, DBG, 1027 "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n", 1028 tx_ring->que->id); 1029 ena_ring_tx_doorbell(tx_ring); 1030 } 1031 1032 /* Prepare the packet's descriptors and send them to device */ 1033 rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc); 1034 if (unlikely(rc != 0)) { 1035 if (likely(rc == ENA_COM_NO_MEM)) { 1036 ena_log_io(pdev, DBG, "tx ring[%d] is out of space\n", 1037 tx_ring->que->id); 1038 } else { 1039 ena_log(pdev, ERR, "failed to prepare tx bufs\n"); 1040 ena_trigger_reset(adapter, 1041 ENA_REGS_RESET_DRIVER_INVALID_STATE); 1042 } 1043 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1); 1044 goto dma_error; 1045 } 1046 1047 counter_enter(); 1048 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); 1049 counter_u64_add_protected(tx_ring->tx_stats.bytes, 1050 (*mbuf)->m_pkthdr.len); 1051 1052 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1); 1053 counter_u64_add_protected(adapter->hw_stats.tx_bytes, 1054 (*mbuf)->m_pkthdr.len); 1055 counter_exit(); 1056 1057 tx_info->tx_descs = nb_hw_desc; 1058 atomic_store_64(&tx_info->timestamp, getsbinuptime()); 1059 tx_info->print_once = true; 1060 1061 tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, 1062 tx_ring->ring_size); 1063 1064 /* stop the queue when no more space available, the packet can have up 1065 * to sgl_size + 2. one for the meta descriptor and one for header 1066 * (if the header is larger than tx_max_header_size). 1067 */ 1068 if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 1069 adapter->max_tx_sgl_size + 2))) { 1070 ena_log_io(pdev, DBG, "Stop queue %d\n", tx_ring->que->id); 1071 1072 tx_ring->running = false; 1073 counter_u64_add(tx_ring->tx_stats.queue_stop, 1); 1074 1075 /* There is a rare condition where this function decides to 1076 * stop the queue but meanwhile tx_cleanup() updates 1077 * next_to_completion and terminates. 1078 * The queue will remain stopped forever. 1079 * To solve this issue this function performs mb(), checks 1080 * the wakeup condition and wakes up the queue if needed. 1081 */ 1082 mb(); 1083 1084 if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 1085 ENA_TX_RESUME_THRESH)) { 1086 tx_ring->running = true; 1087 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1); 1088 } 1089 } 1090 1091 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 1092 BUS_DMASYNC_PREWRITE); 1093 1094 return (0); 1095 1096 dma_error: 1097 tx_info->mbuf = NULL; 1098 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); 1099 1100 return (rc); 1101 } 1102 1103 static void 1104 ena_start_xmit(struct ena_ring *tx_ring) 1105 { 1106 struct mbuf *mbuf; 1107 struct ena_adapter *adapter = tx_ring->adapter; 1108 int ret = 0; 1109 1110 ENA_RING_MTX_ASSERT(tx_ring); 1111 1112 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0)) 1113 return; 1114 1115 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))) 1116 return; 1117 1118 while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) { 1119 ena_log_io(adapter->pdev, DBG, 1120 "\ndequeued mbuf %p with flags %#x and header csum flags %#jx\n", 1121 mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags); 1122 1123 if (unlikely(!tx_ring->running)) { 1124 drbr_putback(adapter->ifp, tx_ring->br, mbuf); 1125 break; 1126 } 1127 1128 if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) { 1129 if (ret == ENA_COM_NO_MEM) { 1130 drbr_putback(adapter->ifp, tx_ring->br, mbuf); 1131 } else if (ret == ENA_COM_NO_SPACE) { 1132 drbr_putback(adapter->ifp, tx_ring->br, mbuf); 1133 } else { 1134 m_freem(mbuf); 1135 drbr_advance(adapter->ifp, tx_ring->br); 1136 } 1137 1138 break; 1139 } 1140 1141 drbr_advance(adapter->ifp, tx_ring->br); 1142 1143 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0)) 1144 return; 1145 1146 tx_ring->acum_pkts++; 1147 1148 BPF_MTAP(adapter->ifp, mbuf); 1149 } 1150 1151 if (likely(tx_ring->acum_pkts != 0)) { 1152 /* Trigger the dma engine */ 1153 ena_ring_tx_doorbell(tx_ring); 1154 } 1155 1156 if (unlikely(!tx_ring->running)) 1157 taskqueue_enqueue(tx_ring->que->cleanup_tq, 1158 &tx_ring->que->cleanup_task); 1159 } 1160