1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #ifdef DEV_NETMAP 34 35 #include "ena.h" 36 #include "ena_netmap.h" 37 38 #define ENA_NETMAP_MORE_FRAMES 1 39 #define ENA_NETMAP_NO_MORE_FRAMES 0 40 #define ENA_MAX_FRAMES 16384 41 42 struct ena_netmap_ctx { 43 struct netmap_kring *kring; 44 struct ena_adapter *adapter; 45 struct netmap_adapter *na; 46 struct netmap_slot *slots; 47 struct ena_ring *ring; 48 struct ena_com_io_cq *io_cq; 49 struct ena_com_io_sq *io_sq; 50 u_int nm_i; 51 uint16_t nt; 52 uint16_t lim; 53 }; 54 55 /* Netmap callbacks */ 56 static int ena_netmap_reg(struct netmap_adapter *, int); 57 static int ena_netmap_txsync(struct netmap_kring *, int); 58 static int ena_netmap_rxsync(struct netmap_kring *, int); 59 60 /* Helper functions */ 61 static int ena_netmap_tx_frames(struct ena_netmap_ctx *); 62 static int ena_netmap_tx_frame(struct ena_netmap_ctx *); 63 static inline uint16_t ena_netmap_count_slots(struct ena_netmap_ctx *); 64 static inline uint16_t ena_netmap_packet_len(struct netmap_slot *, u_int, 65 uint16_t); 66 static int ena_netmap_copy_data(struct netmap_adapter *, 67 struct netmap_slot *, u_int, uint16_t, uint16_t, void *); 68 static int ena_netmap_map_single_slot(struct netmap_adapter *, 69 struct netmap_slot *, bus_dma_tag_t, bus_dmamap_t, void **, uint64_t *); 70 static int ena_netmap_tx_map_slots(struct ena_netmap_ctx *, 71 struct ena_tx_buffer *, void **, uint16_t *, uint16_t *); 72 static void ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *, 73 struct ena_tx_buffer *); 74 static void ena_netmap_tx_cleanup(struct ena_netmap_ctx *); 75 static uint16_t ena_netmap_tx_clean_one(struct ena_netmap_ctx *, 76 uint16_t); 77 static inline int validate_tx_req_id(struct ena_ring *, uint16_t); 78 static int ena_netmap_rx_frames(struct ena_netmap_ctx *); 79 static int ena_netmap_rx_frame(struct ena_netmap_ctx *); 80 static int ena_netmap_rx_load_desc(struct ena_netmap_ctx *, uint16_t, 81 int *); 82 static void ena_netmap_rx_cleanup(struct ena_netmap_ctx *); 83 static void ena_netmap_fill_ctx(struct netmap_kring *, 84 struct ena_netmap_ctx *, uint16_t); 85 86 int 87 ena_netmap_attach(struct ena_adapter *adapter) 88 { 89 struct netmap_adapter na; 90 91 ena_trace(ENA_NETMAP, "netmap attach\n"); 92 93 bzero(&na, sizeof(na)); 94 na.na_flags = NAF_MOREFRAG; 95 na.ifp = adapter->ifp; 96 na.num_tx_desc = adapter->tx_ring_size; 97 na.num_rx_desc = adapter->rx_ring_size; 98 na.num_tx_rings = adapter->num_queues; 99 na.num_rx_rings = adapter->num_queues; 100 na.rx_buf_maxsize = adapter->buf_ring_size; 101 na.nm_txsync = ena_netmap_txsync; 102 na.nm_rxsync = ena_netmap_rxsync; 103 na.nm_register = ena_netmap_reg; 104 105 return (netmap_attach(&na)); 106 } 107 108 int 109 ena_netmap_alloc_rx_slot(struct ena_adapter *adapter, 110 struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) 111 { 112 struct netmap_adapter *na = NA(adapter->ifp); 113 struct netmap_kring *kring; 114 struct netmap_ring *ring; 115 struct netmap_slot *slot; 116 void *addr; 117 uint64_t paddr; 118 int nm_i, qid, head, lim, rc; 119 120 /* if previously allocated frag is not used */ 121 if (unlikely(rx_info->netmap_buf_idx != 0)) 122 return (0); 123 124 qid = rx_ring->qid; 125 kring = na->rx_rings[qid]; 126 nm_i = kring->nr_hwcur; 127 head = kring->rhead; 128 129 ena_trace(ENA_NETMAP | ENA_DBG, "nr_hwcur: %d, nr_hwtail: %d, " 130 "rhead: %d, rcur: %d, rtail: %d\n", kring->nr_hwcur, 131 kring->nr_hwtail, kring->rhead, kring->rcur, kring->rtail); 132 133 if ((nm_i == head) && rx_ring->initialized) { 134 ena_trace(ENA_NETMAP, "No free slots in netmap ring\n"); 135 return (ENOMEM); 136 } 137 138 ring = kring->ring; 139 if (ring == NULL) { 140 device_printf(adapter->pdev, "Rx ring %d is NULL\n", qid); 141 return (EFAULT); 142 } 143 slot = &ring->slot[nm_i]; 144 145 addr = PNMB(na, slot, &paddr); 146 if (addr == NETMAP_BUF_BASE(na)) { 147 device_printf(adapter->pdev, "Bad buff in slot\n"); 148 return (EFAULT); 149 } 150 151 rc = netmap_load_map(na, adapter->rx_buf_tag, rx_info->map, addr); 152 if (rc != 0) { 153 ena_trace(ENA_WARNING, "DMA mapping error\n"); 154 return (rc); 155 } 156 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD); 157 158 rx_info->ena_buf.paddr = paddr; 159 rx_info->ena_buf.len = ring->nr_buf_size; 160 rx_info->mbuf = NULL; 161 rx_info->netmap_buf_idx = slot->buf_idx; 162 163 slot->buf_idx = 0; 164 165 lim = kring->nkr_num_slots - 1; 166 kring->nr_hwcur = nm_next(nm_i, lim); 167 168 return (0); 169 } 170 171 void 172 ena_netmap_free_rx_slot(struct ena_adapter *adapter, 173 struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) 174 { 175 struct netmap_adapter *na; 176 struct netmap_kring *kring; 177 struct netmap_slot *slot; 178 int nm_i, qid, lim; 179 180 na = NA(adapter->ifp); 181 if (na == NULL) { 182 device_printf(adapter->pdev, "netmap adapter is NULL\n"); 183 return; 184 } 185 186 if (na->rx_rings == NULL) { 187 device_printf(adapter->pdev, "netmap rings are NULL\n"); 188 return; 189 } 190 191 qid = rx_ring->qid; 192 kring = na->rx_rings[qid]; 193 if (kring == NULL) { 194 device_printf(adapter->pdev, 195 "netmap kernel ring %d is NULL\n", qid); 196 return; 197 } 198 199 lim = kring->nkr_num_slots - 1; 200 nm_i = nm_prev(kring->nr_hwcur, lim); 201 202 if (kring->nr_mode != NKR_NETMAP_ON) 203 return; 204 205 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 206 BUS_DMASYNC_POSTREAD); 207 netmap_unload_map(na, adapter->rx_buf_tag, rx_info->map); 208 209 KASSERT(kring->ring == NULL, ("Netmap Rx ring is NULL\n")); 210 211 slot = &kring->ring->slot[nm_i]; 212 213 ENA_ASSERT(slot->buf_idx == 0, "Overwrite slot buf\n"); 214 slot->buf_idx = rx_info->netmap_buf_idx; 215 slot->flags = NS_BUF_CHANGED; 216 217 rx_info->netmap_buf_idx = 0; 218 kring->nr_hwcur = nm_i; 219 } 220 221 static bool 222 ena_ring_in_netmap(struct ena_adapter *adapter, int qid, enum txrx x) 223 { 224 struct netmap_adapter *na; 225 struct netmap_kring *kring; 226 227 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 228 na = NA(adapter->ifp); 229 kring = (x == NR_RX) ? na->rx_rings[qid] : na->tx_rings[qid]; 230 if (kring->nr_mode == NKR_NETMAP_ON) 231 return true; 232 } 233 return false; 234 } 235 236 bool 237 ena_tx_ring_in_netmap(struct ena_adapter *adapter, int qid) 238 { 239 return ena_ring_in_netmap(adapter, qid, NR_TX); 240 } 241 242 bool 243 ena_rx_ring_in_netmap(struct ena_adapter *adapter, int qid) 244 { 245 return ena_ring_in_netmap(adapter, qid, NR_RX); 246 } 247 248 static void 249 ena_netmap_reset_ring(struct ena_adapter *adapter, int qid, enum txrx x) 250 { 251 if (!ena_ring_in_netmap(adapter, qid, x)) 252 return; 253 254 netmap_reset(NA(adapter->ifp), x, qid, 0); 255 ena_trace(ENA_NETMAP, "%s ring %d is in netmap mode\n", 256 (x == NR_TX) ? "Tx" : "Rx", qid); 257 } 258 259 void 260 ena_netmap_reset_rx_ring(struct ena_adapter *adapter, int qid) 261 { 262 ena_netmap_reset_ring(adapter, qid, NR_RX); 263 } 264 265 void 266 ena_netmap_reset_tx_ring(struct ena_adapter *adapter, int qid) 267 { 268 ena_netmap_reset_ring(adapter, qid, NR_TX); 269 } 270 271 static int 272 ena_netmap_reg(struct netmap_adapter *na, int onoff) 273 { 274 struct ifnet *ifp = na->ifp; 275 struct ena_adapter* adapter = ifp->if_softc; 276 struct netmap_kring *kring; 277 enum txrx t; 278 int rc, i; 279 280 sx_xlock(&adapter->ioctl_sx); 281 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 282 ena_down(adapter); 283 284 if (onoff) { 285 ena_trace(ENA_NETMAP, "netmap on\n"); 286 for_rx_tx(t) { 287 for (i = 0; i <= nma_get_nrings(na, t); i++) { 288 kring = NMR(na, t)[i]; 289 if (nm_kring_pending_on(kring)) { 290 kring->nr_mode = NKR_NETMAP_ON; 291 } 292 } 293 } 294 nm_set_native_flags(na); 295 } else { 296 ena_trace(ENA_NETMAP, "netmap off\n"); 297 nm_clear_native_flags(na); 298 for_rx_tx(t) { 299 for (i = 0; i <= nma_get_nrings(na, t); i++) { 300 kring = NMR(na, t)[i]; 301 if (nm_kring_pending_off(kring)) { 302 kring->nr_mode = NKR_NETMAP_OFF; 303 } 304 } 305 } 306 } 307 308 rc = ena_up(adapter); 309 if (rc != 0) { 310 ena_trace(ENA_WARNING, "ena_up failed with rc=%d\n", rc); 311 adapter->reset_reason = ENA_REGS_RESET_DRIVER_INVALID_STATE; 312 nm_clear_native_flags(na); 313 ena_destroy_device(adapter, false); 314 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 315 rc = ena_restore_device(adapter); 316 } 317 sx_unlock(&adapter->ioctl_sx); 318 319 return (rc); 320 } 321 322 static int 323 ena_netmap_txsync(struct netmap_kring *kring, int flags) 324 { 325 struct ena_netmap_ctx ctx; 326 int rc = 0; 327 328 ena_netmap_fill_ctx(kring, &ctx, ENA_IO_TXQ_IDX(kring->ring_id)); 329 ctx.ring = &ctx.adapter->tx_ring[kring->ring_id]; 330 331 ENA_RING_MTX_LOCK(ctx.ring); 332 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, ctx.adapter))) 333 goto txsync_end; 334 335 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter))) 336 goto txsync_end; 337 338 rc = ena_netmap_tx_frames(&ctx); 339 ena_netmap_tx_cleanup(&ctx); 340 341 txsync_end: 342 ENA_RING_MTX_UNLOCK(ctx.ring); 343 return (rc); 344 } 345 346 static int 347 ena_netmap_tx_frames(struct ena_netmap_ctx *ctx) 348 { 349 struct ena_ring *tx_ring = ctx->ring; 350 int rc = 0; 351 352 ctx->nm_i = ctx->kring->nr_hwcur; 353 ctx->nt = ctx->ring->next_to_use; 354 355 __builtin_prefetch(&ctx->slots[ctx->nm_i]); 356 357 while (ctx->nm_i != ctx->kring->rhead) { 358 if ((rc = ena_netmap_tx_frame(ctx)) != 0) { 359 /* 360 * When there is no empty space in Tx ring, error is 361 * still being returned. It should not be passed to the 362 * netmap, as application knows current ring state from 363 * netmap ring pointers. Returning error there could 364 * cause application to exit, but the Tx ring is commonly 365 * being full. 366 */ 367 if (rc == ENA_COM_NO_MEM) 368 rc = 0; 369 break; 370 } 371 tx_ring->acum_pkts++; 372 } 373 374 /* If any packet was sent... */ 375 if (likely(ctx->nm_i != ctx->kring->nr_hwcur)) { 376 wmb(); 377 /* ...send the doorbell to the device. */ 378 ena_com_write_sq_doorbell(ctx->io_sq); 379 counter_u64_add(ctx->ring->tx_stats.doorbells, 1); 380 tx_ring->acum_pkts = 0; 381 382 ctx->ring->next_to_use = ctx->nt; 383 ctx->kring->nr_hwcur = ctx->nm_i; 384 } 385 386 return (rc); 387 } 388 389 static int 390 ena_netmap_tx_frame(struct ena_netmap_ctx *ctx) 391 { 392 struct ena_com_tx_ctx ena_tx_ctx; 393 struct ena_adapter *adapter; 394 struct ena_ring *tx_ring; 395 struct ena_tx_buffer *tx_info; 396 uint16_t req_id; 397 uint16_t header_len; 398 uint16_t packet_len; 399 int nb_hw_desc; 400 int rc; 401 void *push_hdr; 402 403 adapter = ctx->adapter; 404 if (ena_netmap_count_slots(ctx) > adapter->max_tx_sgl_size) { 405 ena_trace(ENA_WARNING, "Too many slots per packet\n"); 406 return (EINVAL); 407 } 408 409 tx_ring = ctx->ring; 410 411 req_id = tx_ring->free_tx_ids[ctx->nt]; 412 tx_info = &tx_ring->tx_buffer_info[req_id]; 413 tx_info->num_of_bufs = 0; 414 tx_info->nm_info.sockets_used = 0; 415 416 rc = ena_netmap_tx_map_slots(ctx, tx_info, &push_hdr, &header_len, 417 &packet_len); 418 if (unlikely(rc != 0)) { 419 device_printf(adapter->pdev, "Failed to map Tx slot\n"); 420 return (rc); 421 } 422 423 bzero(&ena_tx_ctx, sizeof(struct ena_com_tx_ctx)); 424 ena_tx_ctx.ena_bufs = tx_info->bufs; 425 ena_tx_ctx.push_header = push_hdr; 426 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 427 ena_tx_ctx.req_id = req_id; 428 ena_tx_ctx.header_len = header_len; 429 430 /* There are no any offloads, as the netmap doesn't support them */ 431 432 if (tx_ring->acum_pkts == DB_THRESHOLD || 433 ena_com_is_doorbell_needed(ctx->io_sq, &ena_tx_ctx)) { 434 wmb(); 435 ena_com_write_sq_doorbell(ctx->io_sq); 436 counter_u64_add(tx_ring->tx_stats.doorbells, 1); 437 tx_ring->acum_pkts = 0; 438 } 439 440 rc = ena_com_prepare_tx(ctx->io_sq, &ena_tx_ctx, &nb_hw_desc); 441 if (unlikely(rc != 0)) { 442 if (likely(rc == ENA_COM_NO_MEM)) { 443 ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH, 444 "Tx ring[%d] is out of space\n", tx_ring->que->id); 445 } else { 446 device_printf(adapter->pdev, 447 "Failed to prepare Tx bufs\n"); 448 } 449 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1); 450 451 ena_netmap_unmap_last_socket_chain(ctx, tx_info); 452 return (rc); 453 } 454 455 counter_enter(); 456 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); 457 counter_u64_add_protected(tx_ring->tx_stats.bytes, packet_len); 458 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1); 459 counter_u64_add_protected(adapter->hw_stats.tx_bytes, packet_len); 460 counter_exit(); 461 462 tx_info->tx_descs = nb_hw_desc; 463 464 ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size); 465 466 for (unsigned int i = 0; i < tx_info->num_of_bufs; i++) 467 bus_dmamap_sync(adapter->tx_buf_tag, 468 tx_info->nm_info.map_seg[i], BUS_DMASYNC_PREWRITE); 469 470 return (0); 471 } 472 473 static inline uint16_t 474 ena_netmap_count_slots(struct ena_netmap_ctx *ctx) 475 { 476 uint16_t slots = 1; 477 uint16_t nm = ctx->nm_i; 478 479 while ((ctx->slots[nm].flags & NS_MOREFRAG) != 0) { 480 slots++; 481 nm = nm_next(nm, ctx->lim); 482 } 483 484 return slots; 485 } 486 487 static inline uint16_t 488 ena_netmap_packet_len(struct netmap_slot *slots, u_int slot_index, 489 uint16_t limit) 490 { 491 struct netmap_slot *nm_slot; 492 uint16_t packet_size = 0; 493 494 do { 495 nm_slot = &slots[slot_index]; 496 packet_size += nm_slot->len; 497 slot_index = nm_next(slot_index, limit); 498 } while ((nm_slot->flags & NS_MOREFRAG) != 0); 499 500 return packet_size; 501 } 502 503 static int 504 ena_netmap_copy_data(struct netmap_adapter *na, struct netmap_slot *slots, 505 u_int slot_index, uint16_t limit, uint16_t bytes_to_copy, void *destination) 506 { 507 struct netmap_slot *nm_slot; 508 void *slot_vaddr; 509 uint16_t packet_size; 510 uint16_t data_amount; 511 512 packet_size = 0; 513 do { 514 nm_slot = &slots[slot_index]; 515 slot_vaddr = NMB(na, nm_slot); 516 if (unlikely(slot_vaddr == NULL)) 517 return (EINVAL); 518 519 data_amount = min_t(uint16_t, bytes_to_copy, nm_slot->len); 520 memcpy(destination, slot_vaddr, data_amount); 521 bytes_to_copy -= data_amount; 522 523 slot_index = nm_next(slot_index, limit); 524 } while ((nm_slot->flags & NS_MOREFRAG) != 0 && bytes_to_copy > 0); 525 526 return (0); 527 } 528 529 static int 530 ena_netmap_map_single_slot(struct netmap_adapter *na, struct netmap_slot *slot, 531 bus_dma_tag_t dmatag, bus_dmamap_t dmamap, void **vaddr, uint64_t *paddr) 532 { 533 int rc; 534 535 *vaddr = PNMB(na, slot, paddr); 536 if (unlikely(vaddr == NULL)) { 537 ena_trace(ENA_ALERT, "Slot address is NULL\n"); 538 return (EINVAL); 539 } 540 541 rc = netmap_load_map(na, dmatag, dmamap, *vaddr); 542 if (unlikely(rc != 0)) { 543 ena_trace(ENA_ALERT, "Failed to map slot %d for DMA\n", 544 slot->buf_idx); 545 return (EINVAL); 546 } 547 548 return (0); 549 } 550 551 static int 552 ena_netmap_tx_map_slots(struct ena_netmap_ctx *ctx, 553 struct ena_tx_buffer *tx_info, void **push_hdr, uint16_t *header_len, 554 uint16_t *packet_len) 555 { 556 struct netmap_slot *slot; 557 struct ena_com_buf *ena_buf; 558 struct ena_adapter *adapter; 559 struct ena_ring *tx_ring; 560 struct ena_netmap_tx_info *nm_info; 561 bus_dmamap_t *nm_maps; 562 void *vaddr; 563 uint64_t paddr; 564 uint32_t *nm_buf_idx; 565 uint32_t slot_head_len; 566 uint32_t frag_len; 567 uint32_t remaining_len; 568 uint16_t push_len; 569 uint16_t delta; 570 int rc; 571 572 adapter = ctx->adapter; 573 tx_ring = ctx->ring; 574 ena_buf = tx_info->bufs; 575 nm_info = &tx_info->nm_info; 576 nm_maps = nm_info->map_seg; 577 nm_buf_idx = nm_info->socket_buf_idx; 578 slot = &ctx->slots[ctx->nm_i]; 579 580 slot_head_len = slot->len; 581 *packet_len = ena_netmap_packet_len(ctx->slots, ctx->nm_i, ctx->lim); 582 remaining_len = *packet_len; 583 delta = 0; 584 585 __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]); 586 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 587 /* 588 * When the device is in LLQ mode, the driver will copy 589 * the header into the device memory space. 590 * The ena_com layer assumes that the header is in a linear 591 * memory space. 592 * This assumption might be wrong since part of the header 593 * can be in the fragmented buffers. 594 * First, check if header fits in the first slot. If not, copy 595 * it to separate buffer that will be holding linearized data. 596 */ 597 push_len = min_t(uint32_t, *packet_len, 598 tx_ring->tx_max_header_size); 599 *header_len = push_len; 600 /* If header is in linear space, just point to socket's data. */ 601 if (likely(push_len <= slot_head_len)) { 602 *push_hdr = NMB(ctx->na, slot); 603 if (unlikely(push_hdr == NULL)) { 604 device_printf(adapter->pdev, 605 "Slot vaddress is NULL\n"); 606 return (EINVAL); 607 } 608 /* 609 * Otherwise, copy whole portion of header from multiple slots 610 * to intermediate buffer. 611 */ 612 } else { 613 rc = ena_netmap_copy_data(ctx->na, 614 ctx->slots, 615 ctx->nm_i, 616 ctx->lim, 617 push_len, 618 tx_ring->push_buf_intermediate_buf); 619 if (unlikely(rc)) { 620 device_printf(adapter->pdev, 621 "Failed to copy data from slots to push_buf\n"); 622 return (EINVAL); 623 } 624 625 *push_hdr = tx_ring->push_buf_intermediate_buf; 626 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1); 627 628 delta = push_len - slot_head_len; 629 } 630 631 ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH, 632 "slot: %d header_buf->vaddr: %p push_len: %d\n", 633 slot->buf_idx, *push_hdr, push_len); 634 635 /* 636 * If header was in linear memory space, map for the dma rest of the data 637 * in the first mbuf of the mbuf chain. 638 */ 639 if (slot_head_len > push_len) { 640 rc = ena_netmap_map_single_slot(ctx->na, 641 slot, 642 adapter->tx_buf_tag, 643 *nm_maps, 644 &vaddr, 645 &paddr); 646 if (unlikely(rc != 0)) { 647 device_printf(adapter->pdev, 648 "DMA mapping error\n"); 649 return (rc); 650 } 651 nm_maps++; 652 653 ena_buf->paddr = paddr + push_len; 654 ena_buf->len = slot->len - push_len; 655 ena_buf++; 656 657 tx_info->num_of_bufs++; 658 } 659 660 remaining_len -= slot->len; 661 662 /* Save buf idx before advancing */ 663 *nm_buf_idx = slot->buf_idx; 664 nm_buf_idx++; 665 slot->buf_idx = 0; 666 667 /* Advance to the next socket */ 668 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim); 669 slot = &ctx->slots[ctx->nm_i]; 670 nm_info->sockets_used++; 671 672 /* 673 * If header is in non linear space (delta > 0), then skip mbufs 674 * containing header and map the last one containing both header 675 * and the packet data. 676 * The first segment is already counted in. 677 */ 678 while (delta > 0) { 679 __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]); 680 frag_len = slot->len; 681 682 /* 683 * If whole segment contains header just move to the 684 * next one and reduce delta. 685 */ 686 if (unlikely(delta >= frag_len)) { 687 delta -= frag_len; 688 } else { 689 /* 690 * Map the data and then assign it with the 691 * offsets 692 */ 693 rc = ena_netmap_map_single_slot(ctx->na, 694 slot, 695 adapter->tx_buf_tag, 696 *nm_maps, 697 &vaddr, 698 &paddr); 699 if (unlikely(rc != 0)) { 700 device_printf(adapter->pdev, 701 "DMA mapping error\n"); 702 goto error_map; 703 } 704 nm_maps++; 705 706 ena_buf->paddr = paddr + delta; 707 ena_buf->len = slot->len - delta; 708 ena_buf++; 709 710 tx_info->num_of_bufs++; 711 delta = 0; 712 } 713 714 remaining_len -= slot->len; 715 716 /* Save buf idx before advancing */ 717 *nm_buf_idx = slot->buf_idx; 718 nm_buf_idx++; 719 slot->buf_idx = 0; 720 721 /* Advance to the next socket */ 722 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim); 723 slot = &ctx->slots[ctx->nm_i]; 724 nm_info->sockets_used++; 725 } 726 } else { 727 *push_hdr = NULL; 728 /* 729 * header_len is just a hint for the device. Because netmap is 730 * not giving us any information about packet header length and 731 * it is not guaranteed that all packet headers will be in the 732 * 1st slot, setting header_len to 0 is making the device ignore 733 * this value and resolve header on it's own. 734 */ 735 *header_len = 0; 736 } 737 738 /* Map all remaining data (regular routine for non-LLQ mode) */ 739 while (remaining_len > 0) { 740 __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]); 741 742 rc = ena_netmap_map_single_slot(ctx->na, 743 slot, 744 adapter->tx_buf_tag, 745 *nm_maps, 746 &vaddr, 747 &paddr); 748 if (unlikely(rc != 0)) { 749 device_printf(adapter->pdev, 750 "DMA mapping error\n"); 751 goto error_map; 752 } 753 nm_maps++; 754 755 ena_buf->paddr = paddr; 756 ena_buf->len = slot->len; 757 ena_buf++; 758 759 tx_info->num_of_bufs++; 760 761 remaining_len -= slot->len; 762 763 /* Save buf idx before advancing */ 764 *nm_buf_idx = slot->buf_idx; 765 nm_buf_idx++; 766 slot->buf_idx = 0; 767 768 /* Advance to the next socket */ 769 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim); 770 slot = &ctx->slots[ctx->nm_i]; 771 nm_info->sockets_used++; 772 } 773 774 return (0); 775 776 error_map: 777 ena_netmap_unmap_last_socket_chain(ctx, tx_info); 778 779 return (rc); 780 } 781 782 static void 783 ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *ctx, 784 struct ena_tx_buffer *tx_info) 785 { 786 struct ena_netmap_tx_info *nm_info; 787 int n; 788 789 nm_info = &tx_info->nm_info; 790 791 /** 792 * As the used sockets must not be equal to the buffers used in the LLQ 793 * mode, they must be treated separately. 794 * First, unmap the DMA maps. 795 */ 796 n = tx_info->num_of_bufs; 797 while (n--) { 798 netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag, 799 nm_info->map_seg[n]); 800 } 801 tx_info->num_of_bufs = 0; 802 803 /* Next, retain the sockets back to the userspace */ 804 n = nm_info->sockets_used; 805 while (n--) { 806 ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n]; 807 ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED; 808 nm_info->socket_buf_idx[n] = 0; 809 ctx->nm_i = nm_prev(ctx->nm_i, ctx->lim); 810 } 811 nm_info->sockets_used = 0; 812 } 813 814 static void 815 ena_netmap_tx_cleanup(struct ena_netmap_ctx *ctx) 816 { 817 uint16_t req_id; 818 uint16_t total_tx_descs = 0; 819 820 ctx->nm_i = ctx->kring->nr_hwtail; 821 ctx->nt = ctx->ring->next_to_clean; 822 823 /* Reclaim buffers for completed transmissions */ 824 while (ena_com_tx_comp_req_id_get(ctx->io_cq, &req_id) >= 0) { 825 if (validate_tx_req_id(ctx->ring, req_id) != 0) 826 break; 827 total_tx_descs += ena_netmap_tx_clean_one(ctx, req_id); 828 } 829 830 ctx->kring->nr_hwtail = ctx->nm_i; 831 832 if (total_tx_descs > 0) { 833 /* acknowledge completion of sent packets */ 834 ctx->ring->next_to_clean = ctx->nt; 835 ena_com_comp_ack(ctx->ring->ena_com_io_sq, total_tx_descs); 836 ena_com_update_dev_comp_head(ctx->ring->ena_com_io_cq); 837 } 838 } 839 840 static uint16_t 841 ena_netmap_tx_clean_one(struct ena_netmap_ctx *ctx, uint16_t req_id) 842 { 843 struct ena_tx_buffer *tx_info; 844 struct ena_netmap_tx_info *nm_info; 845 int n; 846 847 tx_info = &ctx->ring->tx_buffer_info[req_id]; 848 nm_info = &tx_info->nm_info; 849 850 /** 851 * As the used sockets must not be equal to the buffers used in the LLQ 852 * mode, they must be treated separately. 853 * First, unmap the DMA maps. 854 */ 855 n = tx_info->num_of_bufs; 856 for (n = 0; n < tx_info->num_of_bufs; n++) { 857 netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag, 858 nm_info->map_seg[n]); 859 } 860 tx_info->num_of_bufs = 0; 861 862 /* Next, retain the sockets back to the userspace */ 863 for (n = 0; n < nm_info->sockets_used; n++) { 864 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim); 865 ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0, 866 "Tx idx is not 0.\n"); 867 ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n]; 868 ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED; 869 nm_info->socket_buf_idx[n] = 0; 870 } 871 nm_info->sockets_used = 0; 872 873 ctx->ring->free_tx_ids[ctx->nt] = req_id; 874 ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->lim); 875 876 return tx_info->tx_descs; 877 } 878 879 static inline int 880 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id) 881 { 882 struct ena_adapter *adapter = tx_ring->adapter; 883 884 if (likely(req_id < tx_ring->ring_size)) 885 return (0); 886 887 ena_trace(ENA_WARNING, "Invalid req_id: %hu\n", req_id); 888 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1); 889 890 adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID; 891 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 892 893 return (EFAULT); 894 } 895 896 static int 897 ena_netmap_rxsync(struct netmap_kring *kring, int flags) 898 { 899 struct ena_netmap_ctx ctx; 900 int rc; 901 902 ena_netmap_fill_ctx(kring, &ctx, ENA_IO_RXQ_IDX(kring->ring_id)); 903 ctx.ring = &ctx.adapter->rx_ring[kring->ring_id]; 904 905 if (ctx.kring->rhead > ctx.lim) { 906 /* Probably not needed to release slots from RX ring. */ 907 return (netmap_ring_reinit(ctx.kring)); 908 } 909 910 if (unlikely((if_getdrvflags(ctx.na->ifp) & IFF_DRV_RUNNING) == 0)) 911 return (0); 912 913 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter))) 914 return (0); 915 916 if ((rc = ena_netmap_rx_frames(&ctx)) != 0) 917 return (rc); 918 919 ena_netmap_rx_cleanup(&ctx); 920 921 return (0); 922 } 923 924 static inline int 925 ena_netmap_rx_frames(struct ena_netmap_ctx *ctx) 926 { 927 int rc = 0; 928 int frames_counter = 0; 929 930 ctx->nt = ctx->ring->next_to_clean; 931 ctx->nm_i = ctx->kring->nr_hwtail; 932 933 while((rc = ena_netmap_rx_frame(ctx)) == ENA_NETMAP_MORE_FRAMES) { 934 frames_counter++; 935 /* In case of multiple frames, it is not an error. */ 936 rc = 0; 937 if (frames_counter > ENA_MAX_FRAMES) { 938 device_printf(ctx->adapter->pdev, 939 "Driver is stuck in the Rx loop\n"); 940 break; 941 } 942 }; 943 944 ctx->kring->nr_hwtail = ctx->nm_i; 945 ctx->kring->nr_kflags &= ~NKR_PENDINTR; 946 ctx->ring->next_to_clean = ctx->nt; 947 948 return (rc); 949 } 950 951 static inline int 952 ena_netmap_rx_frame(struct ena_netmap_ctx *ctx) 953 { 954 struct ena_com_rx_ctx ena_rx_ctx; 955 int rc, len = 0; 956 uint16_t buf, nm; 957 958 ena_rx_ctx.ena_bufs = ctx->ring->ena_bufs; 959 ena_rx_ctx.max_bufs = ctx->adapter->max_rx_sgl_size; 960 bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag, 961 ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD); 962 963 rc = ena_com_rx_pkt(ctx->io_cq, ctx->io_sq, &ena_rx_ctx); 964 if (unlikely(rc != 0)) { 965 ena_trace(ENA_ALERT, "Too many desc from the device.\n"); 966 counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1); 967 ctx->adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS; 968 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, ctx->adapter); 969 return (rc); 970 } 971 if (unlikely(ena_rx_ctx.descs == 0)) 972 return (ENA_NETMAP_NO_MORE_FRAMES); 973 974 ena_trace(ENA_NETMAP | ENA_DBG, "Rx: q %d got packet from ena. descs #:" 975 " %d l3 proto %d l4 proto %d hash: %x\n", ctx->ring->qid, 976 ena_rx_ctx.descs, ena_rx_ctx.l3_proto, ena_rx_ctx.l4_proto, 977 ena_rx_ctx.hash); 978 979 for (buf = 0; buf < ena_rx_ctx.descs; buf++) 980 if ((rc = ena_netmap_rx_load_desc(ctx, buf, &len)) != 0) 981 break; 982 /* 983 * ena_netmap_rx_load_desc doesn't know the number of descriptors. 984 * It just set flag NS_MOREFRAG to all slots, then here flag of 985 * last slot is cleared. 986 */ 987 ctx->slots[nm_prev(ctx->nm_i, ctx->lim)].flags = NS_BUF_CHANGED; 988 989 if (rc != 0) { 990 goto rx_clear_desc; 991 } 992 993 bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag, 994 ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD); 995 996 counter_enter(); 997 counter_u64_add_protected(ctx->ring->rx_stats.bytes, len); 998 counter_u64_add_protected(ctx->adapter->hw_stats.rx_bytes, len); 999 counter_u64_add_protected(ctx->ring->rx_stats.cnt, 1); 1000 counter_u64_add_protected(ctx->adapter->hw_stats.rx_packets, 1); 1001 counter_exit(); 1002 1003 return (ENA_NETMAP_MORE_FRAMES); 1004 1005 rx_clear_desc: 1006 nm = ctx->nm_i; 1007 1008 /* Remove failed packet from ring */ 1009 while(buf--) { 1010 ctx->slots[nm].flags = 0; 1011 ctx->slots[nm].len = 0; 1012 nm = nm_prev(nm, ctx->lim); 1013 } 1014 1015 return (rc); 1016 } 1017 1018 static inline int 1019 ena_netmap_rx_load_desc(struct ena_netmap_ctx *ctx, uint16_t buf, int *len) 1020 { 1021 struct ena_rx_buffer *rx_info; 1022 uint16_t req_id; 1023 int rc; 1024 1025 req_id = ctx->ring->ena_bufs[buf].req_id; 1026 rc = validate_rx_req_id(ctx->ring, req_id); 1027 if (unlikely(rc != 0)) 1028 return (rc); 1029 1030 rx_info = &ctx->ring->rx_buffer_info[req_id]; 1031 bus_dmamap_sync(ctx->adapter->rx_buf_tag, rx_info->map, 1032 BUS_DMASYNC_POSTREAD); 1033 netmap_unload_map(ctx->na, ctx->adapter->rx_buf_tag, rx_info->map); 1034 1035 ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0, "Rx idx is not 0.\n"); 1036 1037 ctx->slots[ctx->nm_i].buf_idx = rx_info->netmap_buf_idx; 1038 rx_info->netmap_buf_idx = 0; 1039 /* 1040 * Set NS_MOREFRAG to all slots. 1041 * Then ena_netmap_rx_frame clears it from last one. 1042 */ 1043 ctx->slots[ctx->nm_i].flags |= NS_MOREFRAG | NS_BUF_CHANGED; 1044 ctx->slots[ctx->nm_i].len = ctx->ring->ena_bufs[buf].len; 1045 *len += ctx->slots[ctx->nm_i].len; 1046 ctx->ring->free_rx_ids[ctx->nt] = req_id; 1047 ena_trace(ENA_DBG, "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n", 1048 rx_info, ctx->slots[ctx->nm_i].buf_idx, 1049 (uintmax_t)rx_info->ena_buf.paddr, ctx->nm_i); 1050 1051 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim); 1052 ctx->nt = ENA_RX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size); 1053 1054 return (0); 1055 } 1056 1057 static inline void 1058 ena_netmap_rx_cleanup(struct ena_netmap_ctx *ctx) 1059 { 1060 int refill_required; 1061 1062 refill_required = ctx->kring->rhead - ctx->kring->nr_hwcur; 1063 if (ctx->kring->nr_hwcur != ctx->kring->nr_hwtail) 1064 refill_required -= 1; 1065 1066 if (refill_required == 0) 1067 return; 1068 else if (refill_required < 0) 1069 refill_required += ctx->kring->nkr_num_slots; 1070 1071 ena_refill_rx_bufs(ctx->ring, refill_required); 1072 } 1073 1074 static inline void 1075 ena_netmap_fill_ctx(struct netmap_kring *kring, struct ena_netmap_ctx *ctx, 1076 uint16_t ena_qid) 1077 { 1078 ctx->kring = kring; 1079 ctx->na = kring->na; 1080 ctx->adapter = ctx->na->ifp->if_softc; 1081 ctx->lim = kring->nkr_num_slots - 1; 1082 ctx->io_cq = &ctx->adapter->ena_dev->io_cq_queues[ena_qid]; 1083 ctx->io_sq = &ctx->adapter->ena_dev->io_sq_queues[ena_qid]; 1084 ctx->slots = kring->ring->slot; 1085 } 1086 1087 void 1088 ena_netmap_unload(struct ena_adapter *adapter, bus_dmamap_t map) 1089 { 1090 struct netmap_adapter *na = NA(adapter->ifp); 1091 1092 netmap_unload_map(na, adapter->tx_buf_tag, map); 1093 } 1094 1095 #endif /* DEV_NETMAP */ 1096