1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 #include "opt_rss.h" 32 #include "ena.h" 33 #include "ena_datapath.h" 34 #ifdef DEV_NETMAP 35 #include "ena_netmap.h" 36 #endif /* DEV_NETMAP */ 37 #ifdef RSS 38 #include <net/rss_config.h> 39 #endif /* RSS */ 40 41 #include <netinet6/ip6_var.h> 42 43 /********************************************************************* 44 * Static functions prototypes 45 *********************************************************************/ 46 47 static int ena_tx_cleanup(struct ena_ring *); 48 static int ena_rx_cleanup(struct ena_ring *); 49 static inline int ena_get_tx_req_id(struct ena_ring *tx_ring, 50 struct ena_com_io_cq *io_cq, uint16_t *req_id); 51 static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *, 52 struct mbuf *); 53 static struct mbuf *ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *, 54 struct ena_com_rx_ctx *, uint16_t *); 55 static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *, 56 struct mbuf *); 57 static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *, bool); 58 static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, 59 struct mbuf **mbuf); 60 static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **); 61 static void ena_start_xmit(struct ena_ring *); 62 63 /********************************************************************* 64 * Global functions 65 *********************************************************************/ 66 67 void 68 ena_cleanup(void *arg, int pending) 69 { 70 struct ena_que *que = arg; 71 struct ena_adapter *adapter = que->adapter; 72 if_t ifp = adapter->ifp; 73 struct ena_ring *tx_ring; 74 struct ena_ring *rx_ring; 75 struct ena_com_io_cq *io_cq; 76 struct ena_eth_io_intr_reg intr_reg; 77 int qid, ena_qid; 78 int txc, rxc, i; 79 80 tx_ring = que->tx_ring; 81 rx_ring = que->rx_ring; 82 qid = que->id; 83 ena_qid = ENA_IO_TXQ_IDX(qid); 84 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 85 86 atomic_store_8(&tx_ring->cleanup_running, 1); 87 /* Need to make sure that ENA_FLAG_TRIGGER_RESET is visible to ena_cleanup() and 88 * that cleanup_running is visible to check_missing_comp_in_tx_queue() to 89 * prevent the case of accessing CQ concurrently with check_cdesc_in_tx_cq() 90 */ 91 mb(); 92 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || 93 (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)))) 94 return; 95 96 ena_log_io(adapter->pdev, DBG, "MSI-X TX/RX routine\n"); 97 98 atomic_store_8(&tx_ring->first_interrupt, 1); 99 atomic_store_8(&rx_ring->first_interrupt, 1); 100 101 for (i = 0; i < ENA_CLEAN_BUDGET; ++i) { 102 rxc = ena_rx_cleanup(rx_ring); 103 txc = ena_tx_cleanup(tx_ring); 104 105 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || 106 (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)))) 107 return; 108 109 if ((txc != ENA_TX_BUDGET) && (rxc != ENA_RX_BUDGET)) 110 break; 111 } 112 113 /* Signal that work is done and unmask interrupt */ 114 ena_com_update_intr_reg(&intr_reg, ENA_RX_IRQ_INTERVAL, 115 ENA_TX_IRQ_INTERVAL, true, false); 116 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1); 117 ena_com_unmask_intr(io_cq, &intr_reg); 118 atomic_store_8(&tx_ring->cleanup_running, 0); 119 } 120 121 void 122 ena_deferred_mq_start(void *arg, int pending) 123 { 124 struct ena_ring *tx_ring = (struct ena_ring *)arg; 125 if_t ifp = tx_ring->adapter->ifp; 126 127 while (!drbr_empty(ifp, tx_ring->br) && tx_ring->running && 128 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 129 ENA_RING_MTX_LOCK(tx_ring); 130 ena_start_xmit(tx_ring); 131 ENA_RING_MTX_UNLOCK(tx_ring); 132 } 133 } 134 135 int 136 ena_mq_start(if_t ifp, struct mbuf *m) 137 { 138 struct ena_adapter *adapter = if_getsoftc(ifp); 139 struct ena_ring *tx_ring; 140 int ret, is_drbr_empty; 141 uint32_t i; 142 #ifdef RSS 143 uint32_t bucket_id; 144 #endif 145 146 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0)) 147 return (ENODEV); 148 149 /* Which queue to use */ 150 /* 151 * If everything is setup correctly, it should be the 152 * same bucket that the current CPU we're on is. 153 * It should improve performance. 154 */ 155 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 156 #ifdef RSS 157 if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 158 &bucket_id) == 0) 159 i = bucket_id % adapter->num_io_queues; 160 else 161 #endif 162 i = m->m_pkthdr.flowid % adapter->num_io_queues; 163 } else { 164 i = curcpu % adapter->num_io_queues; 165 } 166 tx_ring = &adapter->tx_ring[i]; 167 168 /* Check if drbr is empty before putting packet */ 169 is_drbr_empty = drbr_empty(ifp, tx_ring->br); 170 ret = drbr_enqueue(ifp, tx_ring->br, m); 171 if (unlikely(ret != 0)) { 172 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 173 return (ret); 174 } 175 176 if (is_drbr_empty && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) { 177 ena_start_xmit(tx_ring); 178 ENA_RING_MTX_UNLOCK(tx_ring); 179 } else { 180 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 181 } 182 183 return (0); 184 } 185 186 void 187 ena_qflush(if_t ifp) 188 { 189 struct ena_adapter *adapter = if_getsoftc(ifp); 190 struct ena_ring *tx_ring = adapter->tx_ring; 191 int i; 192 193 for (i = 0; i < adapter->num_io_queues; ++i, ++tx_ring) 194 if (!drbr_empty(ifp, tx_ring->br)) { 195 ENA_RING_MTX_LOCK(tx_ring); 196 drbr_flush(ifp, tx_ring->br); 197 ENA_RING_MTX_UNLOCK(tx_ring); 198 } 199 200 if_qflush(ifp); 201 } 202 203 /********************************************************************* 204 * Static functions 205 *********************************************************************/ 206 207 static inline int 208 ena_get_tx_req_id(struct ena_ring *tx_ring, struct ena_com_io_cq *io_cq, 209 uint16_t *req_id) 210 { 211 struct ena_adapter *adapter = tx_ring->adapter; 212 int rc = ena_com_tx_comp_req_id_get(io_cq, req_id); 213 214 if (unlikely(rc == ENA_COM_TRY_AGAIN)) 215 return (EAGAIN); 216 217 rc = validate_tx_req_id(tx_ring, *req_id, rc); 218 219 if (unlikely(tx_ring->tx_buffer_info[*req_id].mbuf == NULL)) { 220 ena_log(adapter->pdev, ERR, 221 "tx_info doesn't have valid mbuf. req_id %hu qid %hu\n", 222 *req_id, tx_ring->qid); 223 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_TX_REQ_ID); 224 rc = EFAULT; 225 } 226 227 return (rc); 228 } 229 230 /** 231 * ena_tx_cleanup - clear sent packets and corresponding descriptors 232 * @tx_ring: ring for which we want to clean packets 233 * 234 * Once packets are sent, we ask the device in a loop for no longer used 235 * descriptors. We find the related mbuf chain in a map (index in an array) 236 * and free it, then update ring state. 237 * This is performed in "endless" loop, updating ring pointers every 238 * TX_COMMIT. The first check of free descriptor is performed before the actual 239 * loop, then repeated at the loop end. 240 **/ 241 static int 242 ena_tx_cleanup(struct ena_ring *tx_ring) 243 { 244 struct ena_adapter *adapter; 245 struct ena_com_io_cq *io_cq; 246 uint16_t next_to_clean; 247 uint16_t req_id; 248 uint16_t ena_qid; 249 unsigned int total_done = 0; 250 int rc; 251 int commit = ENA_TX_COMMIT; 252 int budget = ENA_TX_BUDGET; 253 int work_done; 254 bool above_thresh; 255 256 adapter = tx_ring->que->adapter; 257 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 258 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 259 next_to_clean = tx_ring->next_to_clean; 260 261 #ifdef DEV_NETMAP 262 if (netmap_tx_irq(adapter->ifp, tx_ring->qid) != NM_IRQ_PASS) 263 return (0); 264 #endif /* DEV_NETMAP */ 265 266 do { 267 struct ena_tx_buffer *tx_info; 268 struct mbuf *mbuf; 269 270 rc = ena_get_tx_req_id(tx_ring, io_cq, &req_id); 271 if (unlikely(rc != 0)) 272 break; 273 274 tx_info = &tx_ring->tx_buffer_info[req_id]; 275 276 mbuf = tx_info->mbuf; 277 278 tx_info->mbuf = NULL; 279 bintime_clear(&tx_info->timestamp); 280 281 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 282 BUS_DMASYNC_POSTWRITE); 283 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); 284 285 ena_log_io(adapter->pdev, DBG, "tx: q %d mbuf %p completed\n", 286 tx_ring->qid, mbuf); 287 288 m_freem(mbuf); 289 290 total_done += tx_info->tx_descs; 291 292 tx_ring->free_tx_ids[next_to_clean] = req_id; 293 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean, 294 tx_ring->ring_size); 295 296 if (unlikely(--commit == 0)) { 297 commit = ENA_TX_COMMIT; 298 /* update ring state every ENA_TX_COMMIT descriptor */ 299 tx_ring->next_to_clean = next_to_clean; 300 ena_com_comp_ack( 301 &adapter->ena_dev->io_sq_queues[ena_qid], 302 total_done); 303 total_done = 0; 304 } 305 } while (likely(--budget)); 306 307 work_done = ENA_TX_BUDGET - budget; 308 309 ena_log_io(adapter->pdev, DBG, "tx: q %d done. total pkts: %d\n", 310 tx_ring->qid, work_done); 311 312 /* If there is still something to commit update ring state */ 313 if (likely(commit != ENA_TX_COMMIT)) { 314 tx_ring->next_to_clean = next_to_clean; 315 ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid], 316 total_done); 317 } 318 319 /* 320 * Need to make the rings circular update visible to 321 * ena_xmit_mbuf() before checking for tx_ring->running. 322 */ 323 mb(); 324 325 above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 326 ENA_TX_RESUME_THRESH); 327 if (unlikely(!tx_ring->running && above_thresh)) { 328 ENA_RING_MTX_LOCK(tx_ring); 329 above_thresh = ena_com_sq_have_enough_space( 330 tx_ring->ena_com_io_sq, ENA_TX_RESUME_THRESH); 331 if (!tx_ring->running && above_thresh) { 332 tx_ring->running = true; 333 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1); 334 taskqueue_enqueue(tx_ring->enqueue_tq, 335 &tx_ring->enqueue_task); 336 } 337 ENA_RING_MTX_UNLOCK(tx_ring); 338 } 339 340 tx_ring->tx_last_cleanup_ticks = ticks; 341 342 return (work_done); 343 } 344 345 static void 346 ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, 347 struct mbuf *mbuf) 348 { 349 struct ena_adapter *adapter = rx_ring->adapter; 350 351 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { 352 mbuf->m_pkthdr.flowid = ena_rx_ctx->hash; 353 354 #ifdef RSS 355 /* 356 * Hardware and software RSS are in agreement only when both are 357 * configured to Toeplitz algorithm. This driver configures 358 * that algorithm only when software RSS is enabled and uses it. 359 */ 360 if (adapter->ena_dev->rss.hash_func != ENA_ADMIN_TOEPLITZ && 361 ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN) { 362 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 363 return; 364 } 365 #endif 366 367 if (ena_rx_ctx->frag && 368 (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) { 369 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 370 return; 371 } 372 373 switch (ena_rx_ctx->l3_proto) { 374 case ENA_ETH_IO_L3_PROTO_IPV4: 375 switch (ena_rx_ctx->l4_proto) { 376 case ENA_ETH_IO_L4_PROTO_TCP: 377 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4); 378 break; 379 case ENA_ETH_IO_L4_PROTO_UDP: 380 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4); 381 break; 382 default: 383 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4); 384 } 385 break; 386 case ENA_ETH_IO_L3_PROTO_IPV6: 387 switch (ena_rx_ctx->l4_proto) { 388 case ENA_ETH_IO_L4_PROTO_TCP: 389 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6); 390 break; 391 case ENA_ETH_IO_L4_PROTO_UDP: 392 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6); 393 break; 394 default: 395 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6); 396 } 397 break; 398 case ENA_ETH_IO_L3_PROTO_UNKNOWN: 399 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); 400 break; 401 default: 402 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 403 } 404 } else { 405 mbuf->m_pkthdr.flowid = rx_ring->qid; 406 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); 407 } 408 } 409 410 /** 411 * ena_rx_mbuf - assemble mbuf from descriptors 412 * @rx_ring: ring for which we want to clean packets 413 * @ena_bufs: buffer info 414 * @ena_rx_ctx: metadata for this packet(s) 415 * @next_to_clean: ring pointer, will be updated only upon success 416 * 417 **/ 418 static struct mbuf * 419 ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs, 420 struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean) 421 { 422 struct mbuf *mbuf; 423 struct ena_rx_buffer *rx_info; 424 struct ena_adapter *adapter; 425 device_t pdev; 426 unsigned int descs = ena_rx_ctx->descs; 427 uint16_t ntc, len, req_id, buf = 0; 428 429 ntc = *next_to_clean; 430 adapter = rx_ring->adapter; 431 pdev = adapter->pdev; 432 433 len = ena_bufs[buf].len; 434 req_id = ena_bufs[buf].req_id; 435 rx_info = &rx_ring->rx_buffer_info[req_id]; 436 if (unlikely(rx_info->mbuf == NULL)) { 437 ena_log(pdev, ERR, "NULL mbuf in rx_info. qid %u req_id %u\n", 438 rx_ring->qid, req_id); 439 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_RX_REQ_ID); 440 return (NULL); 441 } 442 443 ena_log_io(pdev, DBG, "rx_info %p, mbuf %p, paddr %jx\n", rx_info, 444 rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr); 445 446 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 447 BUS_DMASYNC_POSTREAD); 448 mbuf = rx_info->mbuf; 449 mbuf->m_flags |= M_PKTHDR; 450 mbuf->m_pkthdr.len = len; 451 mbuf->m_len = len; 452 /* Only for the first segment the data starts at specific offset */ 453 mbuf->m_data = mtodo(mbuf, ena_rx_ctx->pkt_offset); 454 ena_log_io(pdev, DBG, "Mbuf data offset=%u\n", ena_rx_ctx->pkt_offset); 455 mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp; 456 457 /* Fill mbuf with hash key and it's interpretation for optimization */ 458 ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf); 459 460 ena_log_io(pdev, DBG, "rx mbuf 0x%p, flags=0x%x, len: %d\n", mbuf, 461 mbuf->m_flags, mbuf->m_pkthdr.len); 462 463 /* DMA address is not needed anymore, unmap it */ 464 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map); 465 466 rx_info->mbuf = NULL; 467 rx_ring->free_rx_ids[ntc] = req_id; 468 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); 469 470 /* 471 * While we have more than 1 descriptors for one rcvd packet, append 472 * other mbufs to the main one 473 */ 474 while (--descs) { 475 ++buf; 476 len = ena_bufs[buf].len; 477 req_id = ena_bufs[buf].req_id; 478 rx_info = &rx_ring->rx_buffer_info[req_id]; 479 480 if (unlikely(rx_info->mbuf == NULL)) { 481 ena_log(pdev, ERR, "NULL mbuf in rx_info. qid %u req_id %u\n", 482 rx_ring->qid, req_id); 483 /* 484 * If one of the required mbufs was not allocated yet, 485 * we can break there. 486 * All earlier used descriptors will be reallocated 487 * later and not used mbufs can be reused. 488 * The next_to_clean pointer will not be updated in case 489 * of an error, so caller should advance it manually 490 * in error handling routine to keep it up to date 491 * with hw ring. 492 */ 493 m_freem(mbuf); 494 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_RX_REQ_ID); 495 return (NULL); 496 } 497 498 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 499 BUS_DMASYNC_POSTREAD); 500 if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) { 501 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); 502 ena_log_io(pdev, WARN, "Failed to append Rx mbuf %p\n", 503 mbuf); 504 } 505 506 ena_log_io(pdev, DBG, "rx mbuf updated. len %d\n", 507 mbuf->m_pkthdr.len); 508 509 /* Free already appended mbuf, it won't be useful anymore */ 510 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map); 511 m_freem(rx_info->mbuf); 512 rx_info->mbuf = NULL; 513 514 rx_ring->free_rx_ids[ntc] = req_id; 515 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); 516 } 517 518 *next_to_clean = ntc; 519 520 return (mbuf); 521 } 522 523 /** 524 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum 525 **/ 526 static inline void 527 ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, 528 struct mbuf *mbuf) 529 { 530 device_t pdev = rx_ring->adapter->pdev; 531 532 /* if IP and error */ 533 if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) && 534 ena_rx_ctx->l3_csum_err)) { 535 /* ipv4 checksum error */ 536 mbuf->m_pkthdr.csum_flags = 0; 537 counter_u64_add(rx_ring->rx_stats.csum_bad, 1); 538 ena_log_io(pdev, DBG, "RX IPv4 header checksum error\n"); 539 return; 540 } 541 542 /* if TCP/UDP */ 543 if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) || 544 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) { 545 if (ena_rx_ctx->l4_csum_err) { 546 /* TCP/UDP checksum error */ 547 mbuf->m_pkthdr.csum_flags = 0; 548 counter_u64_add(rx_ring->rx_stats.csum_bad, 1); 549 ena_log_io(pdev, DBG, "RX L4 checksum error\n"); 550 } else { 551 mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 552 mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID; 553 counter_u64_add(rx_ring->rx_stats.csum_good, 1); 554 } 555 } 556 } 557 558 /** 559 * ena_rx_cleanup - handle rx irq 560 * @arg: ring for which irq is being handled 561 **/ 562 static int 563 ena_rx_cleanup(struct ena_ring *rx_ring) 564 { 565 struct ena_adapter *adapter; 566 device_t pdev; 567 struct mbuf *mbuf; 568 struct ena_com_rx_ctx ena_rx_ctx; 569 struct ena_com_io_cq *io_cq; 570 struct ena_com_io_sq *io_sq; 571 enum ena_regs_reset_reason_types reset_reason; 572 if_t ifp; 573 uint16_t ena_qid; 574 uint16_t next_to_clean; 575 uint32_t refill_required; 576 uint32_t refill_threshold; 577 uint32_t do_if_input = 0; 578 unsigned int qid; 579 int rc, i; 580 int budget = ENA_RX_BUDGET; 581 #ifdef DEV_NETMAP 582 int done; 583 #endif /* DEV_NETMAP */ 584 585 adapter = rx_ring->que->adapter; 586 pdev = adapter->pdev; 587 ifp = adapter->ifp; 588 qid = rx_ring->que->id; 589 ena_qid = ENA_IO_RXQ_IDX(qid); 590 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 591 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; 592 next_to_clean = rx_ring->next_to_clean; 593 594 #ifdef DEV_NETMAP 595 if (netmap_rx_irq(adapter->ifp, rx_ring->qid, &done) != NM_IRQ_PASS) 596 return (0); 597 #endif /* DEV_NETMAP */ 598 599 ena_log_io(pdev, DBG, "rx: qid %d\n", qid); 600 601 do { 602 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 603 ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size; 604 ena_rx_ctx.descs = 0; 605 ena_rx_ctx.pkt_offset = 0; 606 607 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag, 608 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD); 609 rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx); 610 if (unlikely(rc != 0)) { 611 if (rc == ENA_COM_NO_SPACE) { 612 counter_u64_add(rx_ring->rx_stats.bad_desc_num, 613 1); 614 reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS; 615 } else if (rc == ENA_COM_FAULT) { 616 reset_reason = ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED; 617 } else { 618 counter_u64_add(rx_ring->rx_stats.bad_req_id, 619 1); 620 reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; 621 } 622 ena_trigger_reset(adapter, reset_reason); 623 return (0); 624 } 625 626 if (unlikely(ena_rx_ctx.descs == 0)) 627 break; 628 629 ena_log_io(pdev, DBG, 630 "rx: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n", 631 rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto, 632 ena_rx_ctx.l4_proto, ena_rx_ctx.hash); 633 634 /* Receive mbuf from the ring */ 635 mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs, &ena_rx_ctx, 636 &next_to_clean); 637 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag, 638 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD); 639 /* Exit if we failed to retrieve a buffer */ 640 if (unlikely(mbuf == NULL)) { 641 for (i = 0; i < ena_rx_ctx.descs; ++i) { 642 rx_ring->free_rx_ids[next_to_clean] = 643 rx_ring->ena_bufs[i].req_id; 644 next_to_clean = ENA_RX_RING_IDX_NEXT( 645 next_to_clean, rx_ring->ring_size); 646 } 647 break; 648 } 649 650 if (((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) || 651 ((if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6) != 0)) { 652 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf); 653 } 654 655 counter_enter(); 656 counter_u64_add_protected(rx_ring->rx_stats.bytes, 657 mbuf->m_pkthdr.len); 658 counter_u64_add_protected(adapter->hw_stats.rx_bytes, 659 mbuf->m_pkthdr.len); 660 counter_exit(); 661 /* 662 * LRO is only for IP/TCP packets and TCP checksum of the packet 663 * should be computed by hardware. 664 */ 665 do_if_input = 1; 666 if (((if_getcapenable(ifp) & IFCAP_LRO) != 0) && 667 ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) && 668 (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) { 669 /* 670 * Send to the stack if: 671 * - LRO not enabled, or 672 * - no LRO resources, or 673 * - lro enqueue fails 674 */ 675 if ((rx_ring->lro.lro_cnt != 0) && 676 (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0)) 677 do_if_input = 0; 678 } 679 if (do_if_input != 0) { 680 ena_log_io(pdev, DBG, 681 "calling if_input() with mbuf %p\n", mbuf); 682 if_input(ifp, mbuf); 683 } 684 685 counter_enter(); 686 counter_u64_add_protected(rx_ring->rx_stats.cnt, 1); 687 counter_u64_add_protected(adapter->hw_stats.rx_packets, 1); 688 counter_exit(); 689 } while (--budget); 690 691 rx_ring->next_to_clean = next_to_clean; 692 693 refill_required = ena_com_free_q_entries(io_sq); 694 refill_threshold = min_t(int, 695 rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER, 696 ENA_RX_REFILL_THRESH_PACKET); 697 698 if (refill_required > refill_threshold) { 699 ena_refill_rx_bufs(rx_ring, refill_required); 700 } 701 702 tcp_lro_flush_all(&rx_ring->lro); 703 704 return (ENA_RX_BUDGET - budget); 705 } 706 707 static void 708 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf, 709 bool disable_meta_caching) 710 { 711 struct ena_com_tx_meta *ena_meta; 712 struct ether_vlan_header *eh; 713 struct mbuf *mbuf_next; 714 u32 mss; 715 bool offload; 716 uint16_t etype; 717 int ehdrlen; 718 struct ip *ip; 719 int ipproto; 720 int iphlen; 721 struct tcphdr *th; 722 int offset; 723 724 offload = false; 725 ena_meta = &ena_tx_ctx->ena_meta; 726 mss = mbuf->m_pkthdr.tso_segsz; 727 728 if (mss != 0) 729 offload = true; 730 731 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) 732 offload = true; 733 734 if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0) 735 offload = true; 736 737 if ((mbuf->m_pkthdr.csum_flags & CSUM6_OFFLOAD) != 0) 738 offload = true; 739 740 if (!offload) { 741 if (disable_meta_caching) { 742 memset(ena_meta, 0, sizeof(*ena_meta)); 743 ena_tx_ctx->meta_valid = 1; 744 } else { 745 ena_tx_ctx->meta_valid = 0; 746 } 747 return; 748 } 749 750 /* Determine where frame payload starts. */ 751 eh = mtod(mbuf, struct ether_vlan_header *); 752 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 753 etype = ntohs(eh->evl_proto); 754 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 755 } else { 756 etype = ntohs(eh->evl_encap_proto); 757 ehdrlen = ETHER_HDR_LEN; 758 } 759 760 mbuf_next = m_getptr(mbuf, ehdrlen, &offset); 761 762 switch (etype) { 763 case ETHERTYPE_IP: 764 ip = (struct ip *)(mtodo(mbuf_next, offset)); 765 iphlen = ip->ip_hl << 2; 766 ipproto = ip->ip_p; 767 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 768 if ((ip->ip_off & htons(IP_DF)) != 0) 769 ena_tx_ctx->df = 1; 770 break; 771 case ETHERTYPE_IPV6: 772 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 773 iphlen = ip6_lasthdr(mbuf, ehdrlen, IPPROTO_IPV6, &ipproto); 774 iphlen -= ehdrlen; 775 ena_tx_ctx->df = 1; 776 break; 777 default: 778 iphlen = 0; 779 ipproto = 0; 780 break; 781 } 782 783 mbuf_next = m_getptr(mbuf, iphlen + ehdrlen, &offset); 784 th = (struct tcphdr *)(mtodo(mbuf_next, offset)); 785 786 if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) { 787 ena_tx_ctx->l3_csum_enable = 1; 788 } 789 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) { 790 ena_tx_ctx->tso_enable = 1; 791 ena_meta->l4_hdr_len = (th->th_off); 792 } 793 794 if (ipproto == IPPROTO_TCP) { 795 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 796 if ((mbuf->m_pkthdr.csum_flags & 797 (CSUM_IP_TCP | CSUM_IP6_TCP)) != 0) 798 ena_tx_ctx->l4_csum_enable = 1; 799 else 800 ena_tx_ctx->l4_csum_enable = 0; 801 } else if (ipproto == IPPROTO_UDP) { 802 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 803 if ((mbuf->m_pkthdr.csum_flags & 804 (CSUM_IP_UDP | CSUM_IP6_UDP)) != 0) 805 ena_tx_ctx->l4_csum_enable = 1; 806 else 807 ena_tx_ctx->l4_csum_enable = 0; 808 } else { 809 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 810 ena_tx_ctx->l4_csum_enable = 0; 811 } 812 813 ena_meta->mss = mss; 814 ena_meta->l3_hdr_len = iphlen; 815 ena_meta->l3_hdr_offset = ehdrlen; 816 ena_tx_ctx->meta_valid = 1; 817 } 818 819 static int 820 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) 821 { 822 struct ena_adapter *adapter; 823 struct mbuf *collapsed_mbuf; 824 int num_frags; 825 826 adapter = tx_ring->adapter; 827 num_frags = ena_mbuf_count(*mbuf); 828 829 /* One segment must be reserved for configuration descriptor. */ 830 if (num_frags < adapter->max_tx_sgl_size) 831 return (0); 832 833 if ((num_frags == adapter->max_tx_sgl_size) && 834 ((*mbuf)->m_pkthdr.len < tx_ring->tx_max_header_size)) 835 return (0); 836 837 counter_u64_add(tx_ring->tx_stats.collapse, 1); 838 839 collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT, 840 adapter->max_tx_sgl_size - 1); 841 if (unlikely(collapsed_mbuf == NULL)) { 842 counter_u64_add(tx_ring->tx_stats.collapse_err, 1); 843 return (ENOMEM); 844 } 845 846 /* If mbuf was collapsed succesfully, original mbuf is released. */ 847 *mbuf = collapsed_mbuf; 848 849 return (0); 850 } 851 852 static int 853 ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info, 854 struct mbuf *mbuf, void **push_hdr, u16 *header_len) 855 { 856 struct ena_adapter *adapter = tx_ring->adapter; 857 struct ena_com_buf *ena_buf; 858 bus_dma_segment_t segs[ENA_BUS_DMA_SEGS]; 859 size_t iseg = 0; 860 uint32_t mbuf_head_len; 861 uint16_t offset; 862 int rc, nsegs; 863 864 mbuf_head_len = mbuf->m_len; 865 tx_info->mbuf = mbuf; 866 ena_buf = tx_info->bufs; 867 868 /* 869 * For easier maintaining of the DMA map, map the whole mbuf even if 870 * the LLQ is used. The descriptors will be filled using the segments. 871 */ 872 rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, 873 tx_info->dmamap, mbuf, segs, &nsegs, BUS_DMA_NOWAIT); 874 if (unlikely((rc != 0) || (nsegs == 0))) { 875 ena_log_io(adapter->pdev, WARN, 876 "dmamap load failed! err: %d nsegs: %d\n", rc, nsegs); 877 goto dma_error; 878 } 879 880 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 881 /* 882 * When the device is LLQ mode, the driver will copy 883 * the header into the device memory space. 884 * the ena_com layer assumes the header is in a linear 885 * memory space. 886 * This assumption might be wrong since part of the header 887 * can be in the fragmented buffers. 888 * First check if header fits in the mbuf. If not, copy it to 889 * separate buffer that will be holding linearized data. 890 */ 891 *header_len = min_t(uint32_t, mbuf->m_pkthdr.len, 892 tx_ring->tx_max_header_size); 893 894 /* If header is in linear space, just point into mbuf's data. */ 895 if (likely(*header_len <= mbuf_head_len)) { 896 *push_hdr = mbuf->m_data; 897 /* 898 * Otherwise, copy whole portion of header from multiple 899 * mbufs to intermediate buffer. 900 */ 901 } else { 902 m_copydata(mbuf, 0, *header_len, 903 tx_ring->push_buf_intermediate_buf); 904 *push_hdr = tx_ring->push_buf_intermediate_buf; 905 906 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1); 907 } 908 909 ena_log_io(adapter->pdev, DBG, 910 "mbuf: %p header_buf->vaddr: %p push_len: %d\n", 911 mbuf, *push_hdr, *header_len); 912 913 /* If packet is fitted in LLQ header, no need for DMA segments. */ 914 if (mbuf->m_pkthdr.len <= tx_ring->tx_max_header_size) { 915 return (0); 916 } else { 917 offset = tx_ring->tx_max_header_size; 918 /* 919 * As Header part is mapped to LLQ header, we can skip 920 * it and just map the residuum of the mbuf to DMA 921 * Segments. 922 */ 923 while (offset > 0) { 924 if (offset >= segs[iseg].ds_len) { 925 offset -= segs[iseg].ds_len; 926 } else { 927 ena_buf->paddr = segs[iseg].ds_addr + 928 offset; 929 ena_buf->len = segs[iseg].ds_len - 930 offset; 931 ena_buf++; 932 tx_info->num_of_bufs++; 933 offset = 0; 934 } 935 iseg++; 936 } 937 } 938 } else { 939 *push_hdr = NULL; 940 /* 941 * header_len is just a hint for the device. Because FreeBSD is 942 * not giving us information about packet header length and it 943 * is not guaranteed that all packet headers will be in the 1st 944 * mbuf, setting header_len to 0 is making the device ignore 945 * this value and resolve header on it's own. 946 */ 947 *header_len = 0; 948 } 949 950 /* Map rest of the mbuf */ 951 while (iseg < nsegs) { 952 ena_buf->paddr = segs[iseg].ds_addr; 953 ena_buf->len = segs[iseg].ds_len; 954 ena_buf++; 955 iseg++; 956 tx_info->num_of_bufs++; 957 } 958 959 return (0); 960 961 dma_error: 962 counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1); 963 tx_info->mbuf = NULL; 964 return (rc); 965 } 966 967 static int 968 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) 969 { 970 struct ena_adapter *adapter; 971 device_t pdev; 972 struct ena_tx_buffer *tx_info; 973 struct ena_com_tx_ctx ena_tx_ctx; 974 struct ena_com_dev *ena_dev; 975 struct ena_com_io_sq *io_sq; 976 void *push_hdr; 977 uint16_t next_to_use; 978 uint16_t req_id; 979 uint16_t ena_qid; 980 uint16_t header_len; 981 int rc; 982 int nb_hw_desc; 983 984 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 985 adapter = tx_ring->que->adapter; 986 pdev = adapter->pdev; 987 ena_dev = adapter->ena_dev; 988 io_sq = &ena_dev->io_sq_queues[ena_qid]; 989 990 rc = ena_check_and_collapse_mbuf(tx_ring, mbuf); 991 if (unlikely(rc != 0)) { 992 ena_log_io(pdev, WARN, "Failed to collapse mbuf! err: %d\n", 993 rc); 994 return (rc); 995 } 996 997 ena_log_io(pdev, DBG, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len); 998 999 next_to_use = tx_ring->next_to_use; 1000 req_id = tx_ring->free_tx_ids[next_to_use]; 1001 tx_info = &tx_ring->tx_buffer_info[req_id]; 1002 tx_info->num_of_bufs = 0; 1003 1004 ENA_WARN(tx_info->mbuf != NULL, adapter->ena_dev, 1005 "mbuf isn't NULL for req_id %d\n", req_id); 1006 1007 rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len); 1008 if (unlikely(rc != 0)) { 1009 ena_log_io(pdev, WARN, "Failed to map TX mbuf\n"); 1010 return (rc); 1011 } 1012 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 1013 ena_tx_ctx.ena_bufs = tx_info->bufs; 1014 ena_tx_ctx.push_header = push_hdr; 1015 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 1016 ena_tx_ctx.req_id = req_id; 1017 ena_tx_ctx.header_len = header_len; 1018 1019 /* Set flags and meta data */ 1020 ena_tx_csum(&ena_tx_ctx, *mbuf, adapter->disable_meta_caching); 1021 1022 if (tx_ring->acum_pkts == ENA_DB_THRESHOLD || 1023 ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, &ena_tx_ctx)) { 1024 ena_log_io(pdev, DBG, 1025 "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n", 1026 tx_ring->que->id); 1027 ena_ring_tx_doorbell(tx_ring); 1028 } 1029 1030 /* Prepare the packet's descriptors and send them to device */ 1031 rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc); 1032 if (unlikely(rc != 0)) { 1033 if (likely(rc == ENA_COM_NO_MEM)) { 1034 ena_log_io(pdev, DBG, "tx ring[%d] is out of space\n", 1035 tx_ring->que->id); 1036 } else { 1037 ena_log(pdev, ERR, "failed to prepare tx bufs\n"); 1038 ena_trigger_reset(adapter, 1039 ENA_REGS_RESET_DRIVER_INVALID_STATE); 1040 } 1041 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1); 1042 goto dma_error; 1043 } 1044 1045 counter_enter(); 1046 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); 1047 counter_u64_add_protected(tx_ring->tx_stats.bytes, 1048 (*mbuf)->m_pkthdr.len); 1049 1050 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1); 1051 counter_u64_add_protected(adapter->hw_stats.tx_bytes, 1052 (*mbuf)->m_pkthdr.len); 1053 counter_exit(); 1054 1055 tx_info->tx_descs = nb_hw_desc; 1056 getbinuptime(&tx_info->timestamp); 1057 tx_info->print_once = true; 1058 1059 tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, 1060 tx_ring->ring_size); 1061 1062 /* stop the queue when no more space available, the packet can have up 1063 * to sgl_size + 2. one for the meta descriptor and one for header 1064 * (if the header is larger than tx_max_header_size). 1065 */ 1066 if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 1067 adapter->max_tx_sgl_size + 2))) { 1068 ena_log_io(pdev, DBG, "Stop queue %d\n", tx_ring->que->id); 1069 1070 tx_ring->running = false; 1071 counter_u64_add(tx_ring->tx_stats.queue_stop, 1); 1072 1073 /* There is a rare condition where this function decides to 1074 * stop the queue but meanwhile tx_cleanup() updates 1075 * next_to_completion and terminates. 1076 * The queue will remain stopped forever. 1077 * To solve this issue this function performs mb(), checks 1078 * the wakeup condition and wakes up the queue if needed. 1079 */ 1080 mb(); 1081 1082 if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 1083 ENA_TX_RESUME_THRESH)) { 1084 tx_ring->running = true; 1085 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1); 1086 } 1087 } 1088 1089 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 1090 BUS_DMASYNC_PREWRITE); 1091 1092 return (0); 1093 1094 dma_error: 1095 tx_info->mbuf = NULL; 1096 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); 1097 1098 return (rc); 1099 } 1100 1101 static void 1102 ena_start_xmit(struct ena_ring *tx_ring) 1103 { 1104 struct mbuf *mbuf; 1105 struct ena_adapter *adapter = tx_ring->adapter; 1106 int ret = 0; 1107 1108 ENA_RING_MTX_ASSERT(tx_ring); 1109 1110 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0)) 1111 return; 1112 1113 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))) 1114 return; 1115 1116 while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) { 1117 ena_log_io(adapter->pdev, DBG, 1118 "\ndequeued mbuf %p with flags %#x and header csum flags %#jx\n", 1119 mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags); 1120 1121 if (unlikely(!tx_ring->running)) { 1122 drbr_putback(adapter->ifp, tx_ring->br, mbuf); 1123 break; 1124 } 1125 1126 if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) { 1127 if (ret == ENA_COM_NO_MEM) { 1128 drbr_putback(adapter->ifp, tx_ring->br, mbuf); 1129 } else if (ret == ENA_COM_NO_SPACE) { 1130 drbr_putback(adapter->ifp, tx_ring->br, mbuf); 1131 } else { 1132 m_freem(mbuf); 1133 drbr_advance(adapter->ifp, tx_ring->br); 1134 } 1135 1136 break; 1137 } 1138 1139 drbr_advance(adapter->ifp, tx_ring->br); 1140 1141 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0)) 1142 return; 1143 1144 tx_ring->acum_pkts++; 1145 1146 BPF_MTAP(adapter->ifp, mbuf); 1147 } 1148 1149 if (likely(tx_ring->acum_pkts != 0)) { 1150 /* Trigger the dma engine */ 1151 ena_ring_tx_doorbell(tx_ring); 1152 } 1153 1154 if (unlikely(!tx_ring->running)) 1155 taskqueue_enqueue(tx_ring->que->cleanup_tq, 1156 &tx_ring->que->cleanup_task); 1157 } 1158