1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_rss.h" 34 #include "ena.h" 35 #include "ena_datapath.h" 36 #ifdef DEV_NETMAP 37 #include "ena_netmap.h" 38 #endif /* DEV_NETMAP */ 39 40 /********************************************************************* 41 * Static functions prototypes 42 *********************************************************************/ 43 44 static int ena_tx_cleanup(struct ena_ring *); 45 static int ena_rx_cleanup(struct ena_ring *); 46 static inline int validate_tx_req_id(struct ena_ring *, uint16_t); 47 static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *, 48 struct mbuf *); 49 static struct mbuf* ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *, 50 struct ena_com_rx_ctx *, uint16_t *); 51 static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *, 52 struct mbuf *); 53 static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *, bool); 54 static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, 55 struct mbuf **mbuf); 56 static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **); 57 static void ena_start_xmit(struct ena_ring *); 58 59 /********************************************************************* 60 * Global functions 61 *********************************************************************/ 62 63 void 64 ena_cleanup(void *arg, int pending) 65 { 66 struct ena_que *que = arg; 67 struct ena_adapter *adapter = que->adapter; 68 if_t ifp = adapter->ifp; 69 struct ena_ring *tx_ring; 70 struct ena_ring *rx_ring; 71 struct ena_com_io_cq* io_cq; 72 struct ena_eth_io_intr_reg intr_reg; 73 int qid, ena_qid; 74 int txc, rxc, i; 75 76 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) 77 return; 78 79 ena_log_io(adapter->pdev, DBG, "MSI-X TX/RX routine\n"); 80 81 tx_ring = que->tx_ring; 82 rx_ring = que->rx_ring; 83 qid = que->id; 84 ena_qid = ENA_IO_TXQ_IDX(qid); 85 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 86 87 tx_ring->first_interrupt = true; 88 rx_ring->first_interrupt = true; 89 90 for (i = 0; i < CLEAN_BUDGET; ++i) { 91 rxc = ena_rx_cleanup(rx_ring); 92 txc = ena_tx_cleanup(tx_ring); 93 94 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) 95 return; 96 97 if ((txc != TX_BUDGET) && (rxc != RX_BUDGET)) 98 break; 99 } 100 101 /* Signal that work is done and unmask interrupt */ 102 ena_com_update_intr_reg(&intr_reg, 103 RX_IRQ_INTERVAL, 104 TX_IRQ_INTERVAL, 105 true); 106 ena_com_unmask_intr(io_cq, &intr_reg); 107 } 108 109 void 110 ena_deferred_mq_start(void *arg, int pending) 111 { 112 struct ena_ring *tx_ring = (struct ena_ring *)arg; 113 struct ifnet *ifp = tx_ring->adapter->ifp; 114 115 while (!drbr_empty(ifp, tx_ring->br) && 116 tx_ring->running && 117 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 118 ENA_RING_MTX_LOCK(tx_ring); 119 ena_start_xmit(tx_ring); 120 ENA_RING_MTX_UNLOCK(tx_ring); 121 } 122 } 123 124 int 125 ena_mq_start(if_t ifp, struct mbuf *m) 126 { 127 struct ena_adapter *adapter = ifp->if_softc; 128 struct ena_ring *tx_ring; 129 int ret, is_drbr_empty; 130 uint32_t i; 131 132 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0)) 133 return (ENODEV); 134 135 /* Which queue to use */ 136 /* 137 * If everything is setup correctly, it should be the 138 * same bucket that the current CPU we're on is. 139 * It should improve performance. 140 */ 141 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 142 i = m->m_pkthdr.flowid % adapter->num_io_queues; 143 } else { 144 i = curcpu % adapter->num_io_queues; 145 } 146 tx_ring = &adapter->tx_ring[i]; 147 148 /* Check if drbr is empty before putting packet */ 149 is_drbr_empty = drbr_empty(ifp, tx_ring->br); 150 ret = drbr_enqueue(ifp, tx_ring->br, m); 151 if (unlikely(ret != 0)) { 152 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 153 return (ret); 154 } 155 156 if (is_drbr_empty && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) { 157 ena_start_xmit(tx_ring); 158 ENA_RING_MTX_UNLOCK(tx_ring); 159 } else { 160 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 161 } 162 163 return (0); 164 } 165 166 void 167 ena_qflush(if_t ifp) 168 { 169 struct ena_adapter *adapter = ifp->if_softc; 170 struct ena_ring *tx_ring = adapter->tx_ring; 171 int i; 172 173 for(i = 0; i < adapter->num_io_queues; ++i, ++tx_ring) 174 if (!drbr_empty(ifp, tx_ring->br)) { 175 ENA_RING_MTX_LOCK(tx_ring); 176 drbr_flush(ifp, tx_ring->br); 177 ENA_RING_MTX_UNLOCK(tx_ring); 178 } 179 180 if_qflush(ifp); 181 } 182 183 /********************************************************************* 184 * Static functions 185 *********************************************************************/ 186 187 static inline int 188 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id) 189 { 190 struct ena_adapter *adapter = tx_ring->adapter; 191 struct ena_tx_buffer *tx_info = NULL; 192 193 if (likely(req_id < tx_ring->ring_size)) { 194 tx_info = &tx_ring->tx_buffer_info[req_id]; 195 if (tx_info->mbuf != NULL) 196 return (0); 197 ena_log(adapter->pdev, ERR, 198 "tx_info doesn't have valid mbuf\n"); 199 } 200 201 ena_log(adapter->pdev, ERR, "Invalid req_id: %hu\n", req_id); 202 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1); 203 204 /* Trigger device reset */ 205 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_TX_REQ_ID); 206 207 return (EFAULT); 208 } 209 210 /** 211 * ena_tx_cleanup - clear sent packets and corresponding descriptors 212 * @tx_ring: ring for which we want to clean packets 213 * 214 * Once packets are sent, we ask the device in a loop for no longer used 215 * descriptors. We find the related mbuf chain in a map (index in an array) 216 * and free it, then update ring state. 217 * This is performed in "endless" loop, updating ring pointers every 218 * TX_COMMIT. The first check of free descriptor is performed before the actual 219 * loop, then repeated at the loop end. 220 **/ 221 static int 222 ena_tx_cleanup(struct ena_ring *tx_ring) 223 { 224 struct ena_adapter *adapter; 225 struct ena_com_io_cq* io_cq; 226 uint16_t next_to_clean; 227 uint16_t req_id; 228 uint16_t ena_qid; 229 unsigned int total_done = 0; 230 int rc; 231 int commit = TX_COMMIT; 232 int budget = TX_BUDGET; 233 int work_done; 234 bool above_thresh; 235 236 adapter = tx_ring->que->adapter; 237 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 238 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 239 next_to_clean = tx_ring->next_to_clean; 240 241 #ifdef DEV_NETMAP 242 if (netmap_tx_irq(adapter->ifp, tx_ring->qid) != NM_IRQ_PASS) 243 return (0); 244 #endif /* DEV_NETMAP */ 245 246 do { 247 struct ena_tx_buffer *tx_info; 248 struct mbuf *mbuf; 249 250 rc = ena_com_tx_comp_req_id_get(io_cq, &req_id); 251 if (unlikely(rc != 0)) 252 break; 253 254 rc = validate_tx_req_id(tx_ring, req_id); 255 if (unlikely(rc != 0)) 256 break; 257 258 tx_info = &tx_ring->tx_buffer_info[req_id]; 259 260 mbuf = tx_info->mbuf; 261 262 tx_info->mbuf = NULL; 263 bintime_clear(&tx_info->timestamp); 264 265 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 266 BUS_DMASYNC_POSTWRITE); 267 bus_dmamap_unload(adapter->tx_buf_tag, 268 tx_info->dmamap); 269 270 ena_log_io(adapter->pdev, DBG, "tx: q %d mbuf %p completed\n", 271 tx_ring->qid, mbuf); 272 273 m_freem(mbuf); 274 275 total_done += tx_info->tx_descs; 276 277 tx_ring->free_tx_ids[next_to_clean] = req_id; 278 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean, 279 tx_ring->ring_size); 280 281 if (unlikely(--commit == 0)) { 282 commit = TX_COMMIT; 283 /* update ring state every TX_COMMIT descriptor */ 284 tx_ring->next_to_clean = next_to_clean; 285 ena_com_comp_ack( 286 &adapter->ena_dev->io_sq_queues[ena_qid], 287 total_done); 288 ena_com_update_dev_comp_head(io_cq); 289 total_done = 0; 290 } 291 } while (likely(--budget)); 292 293 work_done = TX_BUDGET - budget; 294 295 ena_log_io(adapter->pdev, DBG, "tx: q %d done. total pkts: %d\n", 296 tx_ring->qid, work_done); 297 298 /* If there is still something to commit update ring state */ 299 if (likely(commit != TX_COMMIT)) { 300 tx_ring->next_to_clean = next_to_clean; 301 ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid], 302 total_done); 303 ena_com_update_dev_comp_head(io_cq); 304 } 305 306 /* 307 * Need to make the rings circular update visible to 308 * ena_xmit_mbuf() before checking for tx_ring->running. 309 */ 310 mb(); 311 312 above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 313 ENA_TX_RESUME_THRESH); 314 if (unlikely(!tx_ring->running && above_thresh)) { 315 ENA_RING_MTX_LOCK(tx_ring); 316 above_thresh = 317 ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 318 ENA_TX_RESUME_THRESH); 319 if (!tx_ring->running && above_thresh) { 320 tx_ring->running = true; 321 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1); 322 taskqueue_enqueue(tx_ring->enqueue_tq, 323 &tx_ring->enqueue_task); 324 } 325 ENA_RING_MTX_UNLOCK(tx_ring); 326 } 327 328 return (work_done); 329 } 330 331 static void 332 ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, 333 struct mbuf *mbuf) 334 { 335 struct ena_adapter *adapter = rx_ring->adapter; 336 337 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { 338 mbuf->m_pkthdr.flowid = ena_rx_ctx->hash; 339 340 #ifdef RSS 341 /* 342 * Hardware and software RSS are in agreement only when both are 343 * configured to Toeplitz algorithm. This driver configures 344 * that algorithm only when software RSS is enabled and uses it. 345 */ 346 if (adapter->ena_dev->rss.hash_func != ENA_ADMIN_TOEPLITZ && 347 ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN) { 348 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 349 return; 350 } 351 #endif 352 353 if (ena_rx_ctx->frag && 354 (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) { 355 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 356 return; 357 } 358 359 switch (ena_rx_ctx->l3_proto) { 360 case ENA_ETH_IO_L3_PROTO_IPV4: 361 switch (ena_rx_ctx->l4_proto) { 362 case ENA_ETH_IO_L4_PROTO_TCP: 363 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4); 364 break; 365 case ENA_ETH_IO_L4_PROTO_UDP: 366 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4); 367 break; 368 default: 369 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4); 370 } 371 break; 372 case ENA_ETH_IO_L3_PROTO_IPV6: 373 switch (ena_rx_ctx->l4_proto) { 374 case ENA_ETH_IO_L4_PROTO_TCP: 375 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6); 376 break; 377 case ENA_ETH_IO_L4_PROTO_UDP: 378 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6); 379 break; 380 default: 381 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6); 382 } 383 break; 384 case ENA_ETH_IO_L3_PROTO_UNKNOWN: 385 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); 386 break; 387 default: 388 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 389 } 390 } else { 391 mbuf->m_pkthdr.flowid = rx_ring->qid; 392 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); 393 } 394 } 395 396 /** 397 * ena_rx_mbuf - assemble mbuf from descriptors 398 * @rx_ring: ring for which we want to clean packets 399 * @ena_bufs: buffer info 400 * @ena_rx_ctx: metadata for this packet(s) 401 * @next_to_clean: ring pointer, will be updated only upon success 402 * 403 **/ 404 static struct mbuf* 405 ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs, 406 struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean) 407 { 408 struct mbuf *mbuf; 409 struct ena_rx_buffer *rx_info; 410 struct ena_adapter *adapter; 411 device_t pdev; 412 unsigned int descs = ena_rx_ctx->descs; 413 uint16_t ntc, len, req_id, buf = 0; 414 415 ntc = *next_to_clean; 416 adapter = rx_ring->adapter; 417 pdev = adapter->pdev; 418 419 len = ena_bufs[buf].len; 420 req_id = ena_bufs[buf].req_id; 421 rx_info = &rx_ring->rx_buffer_info[req_id]; 422 if (unlikely(rx_info->mbuf == NULL)) { 423 ena_log(pdev, ERR, "NULL mbuf in rx_info"); 424 return (NULL); 425 } 426 427 ena_log_io(pdev, DBG, "rx_info %p, mbuf %p, paddr %jx\n", rx_info, 428 rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr); 429 430 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 431 BUS_DMASYNC_POSTREAD); 432 mbuf = rx_info->mbuf; 433 mbuf->m_flags |= M_PKTHDR; 434 mbuf->m_pkthdr.len = len; 435 mbuf->m_len = len; 436 /* Only for the first segment the data starts at specific offset */ 437 mbuf->m_data = mtodo(mbuf, ena_rx_ctx->pkt_offset); 438 ena_log_io(pdev, DBG, "Mbuf data offset=%u\n", ena_rx_ctx->pkt_offset); 439 mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp; 440 441 /* Fill mbuf with hash key and it's interpretation for optimization */ 442 ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf); 443 444 ena_log_io(pdev, DBG, "rx mbuf 0x%p, flags=0x%x, len: %d\n", mbuf, 445 mbuf->m_flags, mbuf->m_pkthdr.len); 446 447 /* DMA address is not needed anymore, unmap it */ 448 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map); 449 450 rx_info->mbuf = NULL; 451 rx_ring->free_rx_ids[ntc] = req_id; 452 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); 453 454 /* 455 * While we have more than 1 descriptors for one rcvd packet, append 456 * other mbufs to the main one 457 */ 458 while (--descs) { 459 ++buf; 460 len = ena_bufs[buf].len; 461 req_id = ena_bufs[buf].req_id; 462 rx_info = &rx_ring->rx_buffer_info[req_id]; 463 464 if (unlikely(rx_info->mbuf == NULL)) { 465 ena_log(pdev, ERR, "NULL mbuf in rx_info"); 466 /* 467 * If one of the required mbufs was not allocated yet, 468 * we can break there. 469 * All earlier used descriptors will be reallocated 470 * later and not used mbufs can be reused. 471 * The next_to_clean pointer will not be updated in case 472 * of an error, so caller should advance it manually 473 * in error handling routine to keep it up to date 474 * with hw ring. 475 */ 476 m_freem(mbuf); 477 return (NULL); 478 } 479 480 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 481 BUS_DMASYNC_POSTREAD); 482 if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) { 483 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); 484 ena_log_io(pdev, WARN, "Failed to append Rx mbuf %p\n", 485 mbuf); 486 } 487 488 ena_log_io(pdev, DBG, "rx mbuf updated. len %d\n", 489 mbuf->m_pkthdr.len); 490 491 /* Free already appended mbuf, it won't be useful anymore */ 492 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map); 493 m_freem(rx_info->mbuf); 494 rx_info->mbuf = NULL; 495 496 rx_ring->free_rx_ids[ntc] = req_id; 497 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); 498 } 499 500 *next_to_clean = ntc; 501 502 return (mbuf); 503 } 504 505 /** 506 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum 507 **/ 508 static inline void 509 ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, 510 struct mbuf *mbuf) 511 { 512 device_t pdev = rx_ring->adapter->pdev; 513 514 /* if IP and error */ 515 if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) && 516 ena_rx_ctx->l3_csum_err)) { 517 /* ipv4 checksum error */ 518 mbuf->m_pkthdr.csum_flags = 0; 519 counter_u64_add(rx_ring->rx_stats.bad_csum, 1); 520 ena_log_io(pdev, DBG, "RX IPv4 header checksum error\n"); 521 return; 522 } 523 524 /* if TCP/UDP */ 525 if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) || 526 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) { 527 if (ena_rx_ctx->l4_csum_err) { 528 /* TCP/UDP checksum error */ 529 mbuf->m_pkthdr.csum_flags = 0; 530 counter_u64_add(rx_ring->rx_stats.bad_csum, 1); 531 ena_log_io(pdev, DBG, "RX L4 checksum error\n"); 532 } else { 533 mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 534 mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID; 535 } 536 } 537 } 538 539 /** 540 * ena_rx_cleanup - handle rx irq 541 * @arg: ring for which irq is being handled 542 **/ 543 static int 544 ena_rx_cleanup(struct ena_ring *rx_ring) 545 { 546 struct ena_adapter *adapter; 547 device_t pdev; 548 struct mbuf *mbuf; 549 struct ena_com_rx_ctx ena_rx_ctx; 550 struct ena_com_io_cq* io_cq; 551 struct ena_com_io_sq* io_sq; 552 enum ena_regs_reset_reason_types reset_reason; 553 if_t ifp; 554 uint16_t ena_qid; 555 uint16_t next_to_clean; 556 uint32_t refill_required; 557 uint32_t refill_threshold; 558 uint32_t do_if_input = 0; 559 unsigned int qid; 560 int rc, i; 561 int budget = RX_BUDGET; 562 #ifdef DEV_NETMAP 563 int done; 564 #endif /* DEV_NETMAP */ 565 566 adapter = rx_ring->que->adapter; 567 pdev = adapter->pdev; 568 ifp = adapter->ifp; 569 qid = rx_ring->que->id; 570 ena_qid = ENA_IO_RXQ_IDX(qid); 571 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 572 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; 573 next_to_clean = rx_ring->next_to_clean; 574 575 #ifdef DEV_NETMAP 576 if (netmap_rx_irq(adapter->ifp, rx_ring->qid, &done) != NM_IRQ_PASS) 577 return (0); 578 #endif /* DEV_NETMAP */ 579 580 ena_log_io(pdev, DBG, "rx: qid %d\n", qid); 581 582 do { 583 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 584 ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size; 585 ena_rx_ctx.descs = 0; 586 ena_rx_ctx.pkt_offset = 0; 587 588 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag, 589 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD); 590 rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx); 591 if (unlikely(rc != 0)) { 592 if (rc == ENA_COM_NO_SPACE) { 593 counter_u64_add(rx_ring->rx_stats.bad_desc_num, 594 1); 595 reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS; 596 } else { 597 counter_u64_add(rx_ring->rx_stats.bad_req_id, 598 1); 599 reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; 600 } 601 ena_trigger_reset(adapter, reset_reason); 602 return (0); 603 } 604 605 if (unlikely(ena_rx_ctx.descs == 0)) 606 break; 607 608 ena_log_io(pdev, DBG, "rx: q %d got packet from ena. " 609 "descs #: %d l3 proto %d l4 proto %d hash: %x\n", 610 rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto, 611 ena_rx_ctx.l4_proto, ena_rx_ctx.hash); 612 613 /* Receive mbuf from the ring */ 614 mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs, 615 &ena_rx_ctx, &next_to_clean); 616 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag, 617 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD); 618 /* Exit if we failed to retrieve a buffer */ 619 if (unlikely(mbuf == NULL)) { 620 for (i = 0; i < ena_rx_ctx.descs; ++i) { 621 rx_ring->free_rx_ids[next_to_clean] = 622 rx_ring->ena_bufs[i].req_id; 623 next_to_clean = 624 ENA_RX_RING_IDX_NEXT(next_to_clean, 625 rx_ring->ring_size); 626 627 } 628 break; 629 } 630 631 if (((ifp->if_capenable & IFCAP_RXCSUM) != 0) || 632 ((ifp->if_capenable & IFCAP_RXCSUM_IPV6) != 0)) { 633 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf); 634 } 635 636 counter_enter(); 637 counter_u64_add_protected(rx_ring->rx_stats.bytes, 638 mbuf->m_pkthdr.len); 639 counter_u64_add_protected(adapter->hw_stats.rx_bytes, 640 mbuf->m_pkthdr.len); 641 counter_exit(); 642 /* 643 * LRO is only for IP/TCP packets and TCP checksum of the packet 644 * should be computed by hardware. 645 */ 646 do_if_input = 1; 647 if (((ifp->if_capenable & IFCAP_LRO) != 0) && 648 ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) && 649 (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) { 650 /* 651 * Send to the stack if: 652 * - LRO not enabled, or 653 * - no LRO resources, or 654 * - lro enqueue fails 655 */ 656 if ((rx_ring->lro.lro_cnt != 0) && 657 (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0)) 658 do_if_input = 0; 659 } 660 if (do_if_input != 0) { 661 ena_log_io(pdev, DBG, "calling if_input() with mbuf %p\n", 662 mbuf); 663 (*ifp->if_input)(ifp, mbuf); 664 } 665 666 counter_enter(); 667 counter_u64_add_protected(rx_ring->rx_stats.cnt, 1); 668 counter_u64_add_protected(adapter->hw_stats.rx_packets, 1); 669 counter_exit(); 670 } while (--budget); 671 672 rx_ring->next_to_clean = next_to_clean; 673 674 refill_required = ena_com_free_q_entries(io_sq); 675 refill_threshold = min_t(int, 676 rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER, 677 ENA_RX_REFILL_THRESH_PACKET); 678 679 if (refill_required > refill_threshold) { 680 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); 681 ena_refill_rx_bufs(rx_ring, refill_required); 682 } 683 684 tcp_lro_flush_all(&rx_ring->lro); 685 686 return (RX_BUDGET - budget); 687 } 688 689 static void 690 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf, 691 bool disable_meta_caching) 692 { 693 struct ena_com_tx_meta *ena_meta; 694 struct ether_vlan_header *eh; 695 struct mbuf *mbuf_next; 696 u32 mss; 697 bool offload; 698 uint16_t etype; 699 int ehdrlen; 700 struct ip *ip; 701 int iphlen; 702 struct tcphdr *th; 703 int offset; 704 705 offload = false; 706 ena_meta = &ena_tx_ctx->ena_meta; 707 mss = mbuf->m_pkthdr.tso_segsz; 708 709 if (mss != 0) 710 offload = true; 711 712 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) 713 offload = true; 714 715 if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0) 716 offload = true; 717 718 if (!offload) { 719 if (disable_meta_caching) { 720 memset(ena_meta, 0, sizeof(*ena_meta)); 721 ena_tx_ctx->meta_valid = 1; 722 } else { 723 ena_tx_ctx->meta_valid = 0; 724 } 725 return; 726 } 727 728 /* Determine where frame payload starts. */ 729 eh = mtod(mbuf, struct ether_vlan_header *); 730 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 731 etype = ntohs(eh->evl_proto); 732 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 733 } else { 734 etype = ntohs(eh->evl_encap_proto); 735 ehdrlen = ETHER_HDR_LEN; 736 } 737 738 mbuf_next = m_getptr(mbuf, ehdrlen, &offset); 739 ip = (struct ip *)(mtodo(mbuf_next, offset)); 740 iphlen = ip->ip_hl << 2; 741 742 mbuf_next = m_getptr(mbuf, iphlen + ehdrlen, &offset); 743 th = (struct tcphdr *)(mtodo(mbuf_next, offset)); 744 745 if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) { 746 ena_tx_ctx->l3_csum_enable = 1; 747 } 748 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) { 749 ena_tx_ctx->tso_enable = 1; 750 ena_meta->l4_hdr_len = (th->th_off); 751 } 752 753 switch (etype) { 754 case ETHERTYPE_IP: 755 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 756 if ((ip->ip_off & htons(IP_DF)) != 0) 757 ena_tx_ctx->df = 1; 758 break; 759 case ETHERTYPE_IPV6: 760 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 761 762 default: 763 break; 764 } 765 766 if (ip->ip_p == IPPROTO_TCP) { 767 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 768 if ((mbuf->m_pkthdr.csum_flags & 769 (CSUM_IP_TCP | CSUM_IP6_TCP)) != 0) 770 ena_tx_ctx->l4_csum_enable = 1; 771 else 772 ena_tx_ctx->l4_csum_enable = 0; 773 } else if (ip->ip_p == IPPROTO_UDP) { 774 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 775 if ((mbuf->m_pkthdr.csum_flags & 776 (CSUM_IP_UDP | CSUM_IP6_UDP)) != 0) 777 ena_tx_ctx->l4_csum_enable = 1; 778 else 779 ena_tx_ctx->l4_csum_enable = 0; 780 } else { 781 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 782 ena_tx_ctx->l4_csum_enable = 0; 783 } 784 785 ena_meta->mss = mss; 786 ena_meta->l3_hdr_len = iphlen; 787 ena_meta->l3_hdr_offset = ehdrlen; 788 ena_tx_ctx->meta_valid = 1; 789 } 790 791 static int 792 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) 793 { 794 struct ena_adapter *adapter; 795 struct mbuf *collapsed_mbuf; 796 int num_frags; 797 798 adapter = tx_ring->adapter; 799 num_frags = ena_mbuf_count(*mbuf); 800 801 /* One segment must be reserved for configuration descriptor. */ 802 if (num_frags < adapter->max_tx_sgl_size) 803 return (0); 804 counter_u64_add(tx_ring->tx_stats.collapse, 1); 805 806 collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT, 807 adapter->max_tx_sgl_size - 1); 808 if (unlikely(collapsed_mbuf == NULL)) { 809 counter_u64_add(tx_ring->tx_stats.collapse_err, 1); 810 return (ENOMEM); 811 } 812 813 /* If mbuf was collapsed succesfully, original mbuf is released. */ 814 *mbuf = collapsed_mbuf; 815 816 return (0); 817 } 818 819 static int 820 ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info, 821 struct mbuf *mbuf, void **push_hdr, u16 *header_len) 822 { 823 struct ena_adapter *adapter = tx_ring->adapter; 824 struct ena_com_buf *ena_buf; 825 bus_dma_segment_t segs[ENA_BUS_DMA_SEGS]; 826 size_t iseg = 0; 827 uint32_t mbuf_head_len; 828 uint16_t offset; 829 int rc, nsegs; 830 831 mbuf_head_len = mbuf->m_len; 832 tx_info->mbuf = mbuf; 833 ena_buf = tx_info->bufs; 834 835 /* 836 * For easier maintaining of the DMA map, map the whole mbuf even if 837 * the LLQ is used. The descriptors will be filled using the segments. 838 */ 839 rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, tx_info->dmamap, mbuf, 840 segs, &nsegs, BUS_DMA_NOWAIT); 841 if (unlikely((rc != 0) || (nsegs == 0))) { 842 ena_log_io(adapter->pdev, WARN, 843 "dmamap load failed! err: %d nsegs: %d\n", rc, nsegs); 844 goto dma_error; 845 } 846 847 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 848 /* 849 * When the device is LLQ mode, the driver will copy 850 * the header into the device memory space. 851 * the ena_com layer assumes the header is in a linear 852 * memory space. 853 * This assumption might be wrong since part of the header 854 * can be in the fragmented buffers. 855 * First check if header fits in the mbuf. If not, copy it to 856 * separate buffer that will be holding linearized data. 857 */ 858 *header_len = min_t(uint32_t, mbuf->m_pkthdr.len, tx_ring->tx_max_header_size); 859 860 /* If header is in linear space, just point into mbuf's data. */ 861 if (likely(*header_len <= mbuf_head_len)) { 862 *push_hdr = mbuf->m_data; 863 /* 864 * Otherwise, copy whole portion of header from multiple mbufs 865 * to intermediate buffer. 866 */ 867 } else { 868 m_copydata(mbuf, 0, *header_len, tx_ring->push_buf_intermediate_buf); 869 *push_hdr = tx_ring->push_buf_intermediate_buf; 870 871 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1); 872 } 873 874 ena_log_io(adapter->pdev, DBG, "mbuf: %p ""header_buf->vaddr: %p " 875 "push_len: %d\n", mbuf, *push_hdr, *header_len); 876 877 /* If packet is fitted in LLQ header, no need for DMA segments. */ 878 if (mbuf->m_pkthdr.len <= tx_ring->tx_max_header_size) { 879 return (0); 880 } else { 881 offset = tx_ring->tx_max_header_size; 882 /* 883 * As Header part is mapped to LLQ header, we can skip it and just 884 * map the residuum of the mbuf to DMA Segments. 885 */ 886 while (offset > 0) { 887 if (offset >= segs[iseg].ds_len) { 888 offset -= segs[iseg].ds_len; 889 } else { 890 ena_buf->paddr = segs[iseg].ds_addr + offset; 891 ena_buf->len = segs[iseg].ds_len - offset; 892 ena_buf++; 893 tx_info->num_of_bufs++; 894 offset = 0; 895 } 896 iseg++; 897 } 898 } 899 } else { 900 *push_hdr = NULL; 901 /* 902 * header_len is just a hint for the device. Because FreeBSD is not 903 * giving us information about packet header length and it is not 904 * guaranteed that all packet headers will be in the 1st mbuf, setting 905 * header_len to 0 is making the device ignore this value and resolve 906 * header on it's own. 907 */ 908 *header_len = 0; 909 } 910 911 /* Map rest of the mbuf */ 912 while (iseg < nsegs) { 913 ena_buf->paddr = segs[iseg].ds_addr; 914 ena_buf->len = segs[iseg].ds_len; 915 ena_buf++; 916 iseg++; 917 tx_info->num_of_bufs++; 918 } 919 920 return (0); 921 922 dma_error: 923 counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1); 924 tx_info->mbuf = NULL; 925 return (rc); 926 } 927 928 static int 929 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) 930 { 931 struct ena_adapter *adapter; 932 device_t pdev; 933 struct ena_tx_buffer *tx_info; 934 struct ena_com_tx_ctx ena_tx_ctx; 935 struct ena_com_dev *ena_dev; 936 struct ena_com_io_sq* io_sq; 937 void *push_hdr; 938 uint16_t next_to_use; 939 uint16_t req_id; 940 uint16_t ena_qid; 941 uint16_t header_len; 942 int rc; 943 int nb_hw_desc; 944 945 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 946 adapter = tx_ring->que->adapter; 947 pdev = adapter->pdev; 948 ena_dev = adapter->ena_dev; 949 io_sq = &ena_dev->io_sq_queues[ena_qid]; 950 951 rc = ena_check_and_collapse_mbuf(tx_ring, mbuf); 952 if (unlikely(rc != 0)) { 953 ena_log_io(pdev, WARN, "Failed to collapse mbuf! err: %d\n", 954 rc); 955 return (rc); 956 } 957 958 ena_log_io(pdev, DBG, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len); 959 960 next_to_use = tx_ring->next_to_use; 961 req_id = tx_ring->free_tx_ids[next_to_use]; 962 tx_info = &tx_ring->tx_buffer_info[req_id]; 963 tx_info->num_of_bufs = 0; 964 965 rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len); 966 if (unlikely(rc != 0)) { 967 ena_log_io(pdev, WARN, "Failed to map TX mbuf\n"); 968 return (rc); 969 } 970 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 971 ena_tx_ctx.ena_bufs = tx_info->bufs; 972 ena_tx_ctx.push_header = push_hdr; 973 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 974 ena_tx_ctx.req_id = req_id; 975 ena_tx_ctx.header_len = header_len; 976 977 /* Set flags and meta data */ 978 ena_tx_csum(&ena_tx_ctx, *mbuf, adapter->disable_meta_caching); 979 980 if (tx_ring->acum_pkts == DB_THRESHOLD || 981 ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, &ena_tx_ctx)) { 982 ena_log_io(pdev, DBG, 983 "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n", 984 tx_ring->que->id); 985 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 986 counter_u64_add(tx_ring->tx_stats.doorbells, 1); 987 tx_ring->acum_pkts = 0; 988 } 989 990 /* Prepare the packet's descriptors and send them to device */ 991 rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc); 992 if (unlikely(rc != 0)) { 993 if (likely(rc == ENA_COM_NO_MEM)) { 994 ena_log_io(pdev, DBG, "tx ring[%d] is out of space\n", 995 tx_ring->que->id); 996 } else { 997 ena_log(pdev, ERR, "failed to prepare tx bufs\n"); 998 } 999 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1); 1000 goto dma_error; 1001 } 1002 1003 counter_enter(); 1004 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); 1005 counter_u64_add_protected(tx_ring->tx_stats.bytes, 1006 (*mbuf)->m_pkthdr.len); 1007 1008 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1); 1009 counter_u64_add_protected(adapter->hw_stats.tx_bytes, 1010 (*mbuf)->m_pkthdr.len); 1011 counter_exit(); 1012 1013 tx_info->tx_descs = nb_hw_desc; 1014 getbinuptime(&tx_info->timestamp); 1015 tx_info->print_once = true; 1016 1017 tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, 1018 tx_ring->ring_size); 1019 1020 /* stop the queue when no more space available, the packet can have up 1021 * to sgl_size + 2. one for the meta descriptor and one for header 1022 * (if the header is larger than tx_max_header_size). 1023 */ 1024 if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 1025 adapter->max_tx_sgl_size + 2))) { 1026 ena_log_io(pdev, DBG, "Stop queue %d\n", tx_ring->que->id); 1027 1028 tx_ring->running = false; 1029 counter_u64_add(tx_ring->tx_stats.queue_stop, 1); 1030 1031 /* There is a rare condition where this function decides to 1032 * stop the queue but meanwhile tx_cleanup() updates 1033 * next_to_completion and terminates. 1034 * The queue will remain stopped forever. 1035 * To solve this issue this function performs mb(), checks 1036 * the wakeup condition and wakes up the queue if needed. 1037 */ 1038 mb(); 1039 1040 if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 1041 ENA_TX_RESUME_THRESH)) { 1042 tx_ring->running = true; 1043 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1); 1044 } 1045 } 1046 1047 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 1048 BUS_DMASYNC_PREWRITE); 1049 1050 return (0); 1051 1052 dma_error: 1053 tx_info->mbuf = NULL; 1054 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); 1055 1056 return (rc); 1057 } 1058 1059 static void 1060 ena_start_xmit(struct ena_ring *tx_ring) 1061 { 1062 struct mbuf *mbuf; 1063 struct ena_adapter *adapter = tx_ring->adapter; 1064 struct ena_com_io_sq* io_sq; 1065 int ena_qid; 1066 int ret = 0; 1067 1068 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0)) 1069 return; 1070 1071 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))) 1072 return; 1073 1074 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 1075 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; 1076 1077 while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) { 1078 ena_log_io(adapter->pdev, DBG, 1079 "\ndequeued mbuf %p with flags %#x and header csum flags %#jx\n", 1080 mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags); 1081 1082 if (unlikely(!tx_ring->running)) { 1083 drbr_putback(adapter->ifp, tx_ring->br, mbuf); 1084 break; 1085 } 1086 1087 if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) { 1088 if (ret == ENA_COM_NO_MEM) { 1089 drbr_putback(adapter->ifp, tx_ring->br, mbuf); 1090 } else if (ret == ENA_COM_NO_SPACE) { 1091 drbr_putback(adapter->ifp, tx_ring->br, mbuf); 1092 } else { 1093 m_freem(mbuf); 1094 drbr_advance(adapter->ifp, tx_ring->br); 1095 } 1096 1097 break; 1098 } 1099 1100 drbr_advance(adapter->ifp, tx_ring->br); 1101 1102 if (unlikely((if_getdrvflags(adapter->ifp) & 1103 IFF_DRV_RUNNING) == 0)) 1104 return; 1105 1106 tx_ring->acum_pkts++; 1107 1108 BPF_MTAP(adapter->ifp, mbuf); 1109 } 1110 1111 if (likely(tx_ring->acum_pkts != 0)) { 1112 /* Trigger the dma engine */ 1113 ena_com_write_sq_doorbell(io_sq); 1114 counter_u64_add(tx_ring->tx_stats.doorbells, 1); 1115 tx_ring->acum_pkts = 0; 1116 } 1117 1118 if (unlikely(!tx_ring->running)) 1119 taskqueue_enqueue(tx_ring->que->cleanup_tq, 1120 &tx_ring->que->cleanup_task); 1121 } 1122