1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 #include "opt_rss.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/endian.h> 37 #include <sys/eventhandler.h> 38 #include <sys/kernel.h> 39 #include <sys/kthread.h> 40 #include <sys/malloc.h> 41 #include <sys/mbuf.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/smp.h> 45 #include <sys/socket.h> 46 #include <sys/sockio.h> 47 #include <sys/sysctl.h> 48 #include <sys/taskqueue.h> 49 #include <sys/time.h> 50 51 #include <vm/vm.h> 52 #include <vm/pmap.h> 53 54 #include <machine/atomic.h> 55 #include <machine/bus.h> 56 #include <machine/in_cksum.h> 57 #include <machine/resource.h> 58 59 #include <dev/pci/pcireg.h> 60 #include <dev/pci/pcivar.h> 61 62 #include <net/bpf.h> 63 #include <net/ethernet.h> 64 #include <net/if.h> 65 #include <net/if_arp.h> 66 #include <net/if_dl.h> 67 #include <net/if_media.h> 68 #include <net/if_types.h> 69 #include <net/if_var.h> 70 #include <net/if_vlan_var.h> 71 #include <netinet/in.h> 72 #include <netinet/in_systm.h> 73 #include <netinet/if_ether.h> 74 #include <netinet/ip.h> 75 #include <netinet/ip6.h> 76 #include <netinet/tcp.h> 77 #include <netinet/udp.h> 78 79 #include "ena.h" 80 #include "ena_datapath.h" 81 #include "ena_rss.h" 82 #include "ena_sysctl.h" 83 84 #ifdef DEV_NETMAP 85 #include "ena_netmap.h" 86 #endif /* DEV_NETMAP */ 87 88 /********************************************************* 89 * Function prototypes 90 *********************************************************/ 91 static int ena_probe(device_t); 92 static void ena_intr_msix_mgmnt(void *); 93 static void ena_free_pci_resources(struct ena_adapter *); 94 static int ena_change_mtu(if_t, int); 95 static inline void ena_alloc_counters(counter_u64_t *, int); 96 static inline void ena_free_counters(counter_u64_t *, int); 97 static inline void ena_reset_counters(counter_u64_t *, int); 98 static void ena_init_io_rings_common(struct ena_adapter *, struct ena_ring *, 99 uint16_t); 100 static void ena_init_io_rings_basic(struct ena_adapter *); 101 static void ena_init_io_rings_advanced(struct ena_adapter *); 102 static void ena_init_io_rings(struct ena_adapter *); 103 static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int); 104 static void ena_free_all_io_rings_resources(struct ena_adapter *); 105 static int ena_setup_tx_dma_tag(struct ena_adapter *); 106 static int ena_free_tx_dma_tag(struct ena_adapter *); 107 static int ena_setup_rx_dma_tag(struct ena_adapter *); 108 static int ena_free_rx_dma_tag(struct ena_adapter *); 109 static void ena_release_all_tx_dmamap(struct ena_ring *); 110 static int ena_setup_tx_resources(struct ena_adapter *, int); 111 static void ena_free_tx_resources(struct ena_adapter *, int); 112 static int ena_setup_all_tx_resources(struct ena_adapter *); 113 static void ena_free_all_tx_resources(struct ena_adapter *); 114 static int ena_setup_rx_resources(struct ena_adapter *, unsigned int); 115 static void ena_free_rx_resources(struct ena_adapter *, unsigned int); 116 static int ena_setup_all_rx_resources(struct ena_adapter *); 117 static void ena_free_all_rx_resources(struct ena_adapter *); 118 static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *, 119 struct ena_rx_buffer *); 120 static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *, 121 struct ena_rx_buffer *); 122 static void ena_free_rx_bufs(struct ena_adapter *, unsigned int); 123 static void ena_refill_all_rx_bufs(struct ena_adapter *); 124 static void ena_free_all_rx_bufs(struct ena_adapter *); 125 static void ena_free_tx_bufs(struct ena_adapter *, unsigned int); 126 static void ena_free_all_tx_bufs(struct ena_adapter *); 127 static void ena_destroy_all_tx_queues(struct ena_adapter *); 128 static void ena_destroy_all_rx_queues(struct ena_adapter *); 129 static void ena_destroy_all_io_queues(struct ena_adapter *); 130 static int ena_create_io_queues(struct ena_adapter *); 131 static int ena_handle_msix(void *); 132 static int ena_enable_msix(struct ena_adapter *); 133 static void ena_setup_mgmnt_intr(struct ena_adapter *); 134 static int ena_setup_io_intr(struct ena_adapter *); 135 static int ena_request_mgmnt_irq(struct ena_adapter *); 136 static int ena_request_io_irq(struct ena_adapter *); 137 static void ena_free_mgmnt_irq(struct ena_adapter *); 138 static void ena_free_io_irq(struct ena_adapter *); 139 static void ena_free_irqs(struct ena_adapter *); 140 static void ena_disable_msix(struct ena_adapter *); 141 static void ena_unmask_all_io_irqs(struct ena_adapter *); 142 static int ena_up_complete(struct ena_adapter *); 143 static uint64_t ena_get_counter(if_t, ift_counter); 144 static int ena_media_change(if_t); 145 static void ena_media_status(if_t, struct ifmediareq *); 146 static void ena_init(void *); 147 static int ena_ioctl(if_t, u_long, caddr_t); 148 static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *); 149 static void ena_update_host_info(struct ena_admin_host_info *, if_t); 150 static void ena_update_hwassist(struct ena_adapter *); 151 static int ena_setup_ifnet(device_t, struct ena_adapter *, 152 struct ena_com_dev_get_features_ctx *); 153 static int ena_enable_wc(device_t, struct resource *); 154 static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *, 155 struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *); 156 static int ena_map_llq_mem_bar(device_t, struct ena_com_dev *); 157 static uint32_t ena_calc_max_io_queue_num(device_t, struct ena_com_dev *, 158 struct ena_com_dev_get_features_ctx *); 159 static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *); 160 static void ena_config_host_info(struct ena_com_dev *, device_t); 161 static int ena_attach(device_t); 162 static int ena_detach(device_t); 163 static int ena_device_init(struct ena_adapter *, device_t, 164 struct ena_com_dev_get_features_ctx *, int *); 165 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *); 166 static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *); 167 static void unimplemented_aenq_handler(void *, struct ena_admin_aenq_entry *); 168 static int ena_copy_eni_metrics(struct ena_adapter *); 169 static void ena_timer_service(void *); 170 171 static char ena_version[] = ENA_DEVICE_NAME ENA_DRV_MODULE_NAME 172 " v" ENA_DRV_MODULE_VERSION; 173 174 static ena_vendor_info_t ena_vendor_info_array[] = { 175 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0 }, 176 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF_RSERV0, 0 }, 177 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0 }, 178 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF_RSERV0, 0 }, 179 /* Last entry */ 180 { 0, 0, 0 } 181 }; 182 183 struct sx ena_global_lock; 184 185 /* 186 * Contains pointers to event handlers, e.g. link state chage. 187 */ 188 static struct ena_aenq_handlers aenq_handlers; 189 190 void 191 ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 192 { 193 if (error != 0) 194 return; 195 *(bus_addr_t *)arg = segs[0].ds_addr; 196 } 197 198 int 199 ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma, 200 int mapflags, bus_size_t alignment, int domain) 201 { 202 struct ena_adapter *adapter = device_get_softc(dmadev); 203 device_t pdev = adapter->pdev; 204 uint32_t maxsize; 205 uint64_t dma_space_addr; 206 int error; 207 208 maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE; 209 210 dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width); 211 if (unlikely(dma_space_addr == 0)) 212 dma_space_addr = BUS_SPACE_MAXADDR; 213 214 error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */ 215 alignment, 0, /* alignment, bounds */ 216 dma_space_addr, /* lowaddr of exclusion window */ 217 BUS_SPACE_MAXADDR, /* highaddr of exclusion window */ 218 NULL, NULL, /* filter, filterarg */ 219 maxsize, /* maxsize */ 220 1, /* nsegments */ 221 maxsize, /* maxsegsize */ 222 BUS_DMA_ALLOCNOW, /* flags */ 223 NULL, /* lockfunc */ 224 NULL, /* lockarg */ 225 &dma->tag); 226 if (unlikely(error != 0)) { 227 ena_log(pdev, ERR, "bus_dma_tag_create failed: %d\n", error); 228 goto fail_tag; 229 } 230 231 error = bus_dma_tag_set_domain(dma->tag, domain); 232 if (unlikely(error != 0)) { 233 ena_log(pdev, ERR, "bus_dma_tag_set_domain failed: %d\n", 234 error); 235 goto fail_map_create; 236 } 237 238 error = bus_dmamem_alloc(dma->tag, (void **)&dma->vaddr, 239 BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map); 240 if (unlikely(error != 0)) { 241 ena_log(pdev, ERR, "bus_dmamem_alloc(%ju) failed: %d\n", 242 (uintmax_t)size, error); 243 goto fail_map_create; 244 } 245 246 dma->paddr = 0; 247 error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, size, 248 ena_dmamap_callback, &dma->paddr, mapflags); 249 if (unlikely((error != 0) || (dma->paddr == 0))) { 250 ena_log(pdev, ERR, "bus_dmamap_load failed: %d\n", error); 251 goto fail_map_load; 252 } 253 254 bus_dmamap_sync(dma->tag, dma->map, 255 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 256 257 return (0); 258 259 fail_map_load: 260 bus_dmamem_free(dma->tag, dma->vaddr, dma->map); 261 fail_map_create: 262 bus_dma_tag_destroy(dma->tag); 263 fail_tag: 264 dma->tag = NULL; 265 dma->vaddr = NULL; 266 dma->paddr = 0; 267 268 return (error); 269 } 270 271 static void 272 ena_free_pci_resources(struct ena_adapter *adapter) 273 { 274 device_t pdev = adapter->pdev; 275 276 if (adapter->memory != NULL) { 277 bus_release_resource(pdev, SYS_RES_MEMORY, 278 PCIR_BAR(ENA_MEM_BAR), adapter->memory); 279 } 280 281 if (adapter->registers != NULL) { 282 bus_release_resource(pdev, SYS_RES_MEMORY, 283 PCIR_BAR(ENA_REG_BAR), adapter->registers); 284 } 285 286 if (adapter->msix != NULL) { 287 bus_release_resource(pdev, SYS_RES_MEMORY, adapter->msix_rid, 288 adapter->msix); 289 } 290 } 291 292 static int 293 ena_probe(device_t dev) 294 { 295 ena_vendor_info_t *ent; 296 uint16_t pci_vendor_id = 0; 297 uint16_t pci_device_id = 0; 298 299 pci_vendor_id = pci_get_vendor(dev); 300 pci_device_id = pci_get_device(dev); 301 302 ent = ena_vendor_info_array; 303 while (ent->vendor_id != 0) { 304 if ((pci_vendor_id == ent->vendor_id) && 305 (pci_device_id == ent->device_id)) { 306 ena_log_raw(DBG, "vendor=%x device=%x\n", pci_vendor_id, 307 pci_device_id); 308 309 device_set_desc(dev, ENA_DEVICE_DESC); 310 return (BUS_PROBE_DEFAULT); 311 } 312 313 ent++; 314 } 315 316 return (ENXIO); 317 } 318 319 static int 320 ena_change_mtu(if_t ifp, int new_mtu) 321 { 322 struct ena_adapter *adapter = if_getsoftc(ifp); 323 device_t pdev = adapter->pdev; 324 int rc; 325 326 if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) { 327 ena_log(pdev, ERR, "Invalid MTU setting. new_mtu: %d max mtu: %d min mtu: %d\n", 328 new_mtu, adapter->max_mtu, ENA_MIN_MTU); 329 return (EINVAL); 330 } 331 332 rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); 333 if (likely(rc == 0)) { 334 ena_log(pdev, DBG, "set MTU to %d\n", new_mtu); 335 if_setmtu(ifp, new_mtu); 336 } else { 337 ena_log(pdev, ERR, "Failed to set MTU to %d\n", new_mtu); 338 } 339 340 return (rc); 341 } 342 343 static inline void 344 ena_alloc_counters(counter_u64_t *begin, int size) 345 { 346 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 347 348 for (; begin < end; ++begin) 349 *begin = counter_u64_alloc(M_WAITOK); 350 } 351 352 static inline void 353 ena_free_counters(counter_u64_t *begin, int size) 354 { 355 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 356 357 for (; begin < end; ++begin) 358 counter_u64_free(*begin); 359 } 360 361 static inline void 362 ena_reset_counters(counter_u64_t *begin, int size) 363 { 364 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 365 366 for (; begin < end; ++begin) 367 counter_u64_zero(*begin); 368 } 369 370 static void 371 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring, 372 uint16_t qid) 373 { 374 ring->qid = qid; 375 ring->adapter = adapter; 376 ring->ena_dev = adapter->ena_dev; 377 atomic_store_8(&ring->first_interrupt, 0); 378 ring->no_interrupt_event_cnt = 0; 379 } 380 381 static void 382 ena_init_io_rings_basic(struct ena_adapter *adapter) 383 { 384 struct ena_com_dev *ena_dev; 385 struct ena_ring *txr, *rxr; 386 struct ena_que *que; 387 int i; 388 389 ena_dev = adapter->ena_dev; 390 391 for (i = 0; i < adapter->num_io_queues; i++) { 392 txr = &adapter->tx_ring[i]; 393 rxr = &adapter->rx_ring[i]; 394 395 /* TX/RX common ring state */ 396 ena_init_io_rings_common(adapter, txr, i); 397 ena_init_io_rings_common(adapter, rxr, i); 398 399 /* TX specific ring state */ 400 txr->tx_max_header_size = ena_dev->tx_max_header_size; 401 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type; 402 403 que = &adapter->que[i]; 404 que->adapter = adapter; 405 que->id = i; 406 que->tx_ring = txr; 407 que->rx_ring = rxr; 408 409 txr->que = que; 410 rxr->que = que; 411 412 rxr->empty_rx_queue = 0; 413 rxr->rx_mbuf_sz = ena_mbuf_sz; 414 } 415 } 416 417 static void 418 ena_init_io_rings_advanced(struct ena_adapter *adapter) 419 { 420 struct ena_ring *txr, *rxr; 421 int i; 422 423 for (i = 0; i < adapter->num_io_queues; i++) { 424 txr = &adapter->tx_ring[i]; 425 rxr = &adapter->rx_ring[i]; 426 427 /* Allocate a buf ring */ 428 txr->buf_ring_size = adapter->buf_ring_size; 429 txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF, M_WAITOK, 430 &txr->ring_mtx); 431 432 /* Allocate Tx statistics. */ 433 ena_alloc_counters((counter_u64_t *)&txr->tx_stats, 434 sizeof(txr->tx_stats)); 435 txr->tx_last_cleanup_ticks = ticks; 436 437 /* Allocate Rx statistics. */ 438 ena_alloc_counters((counter_u64_t *)&rxr->rx_stats, 439 sizeof(rxr->rx_stats)); 440 441 /* Initialize locks */ 442 snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)", 443 device_get_nameunit(adapter->pdev), i); 444 snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)", 445 device_get_nameunit(adapter->pdev), i); 446 447 mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF); 448 } 449 } 450 451 static void 452 ena_init_io_rings(struct ena_adapter *adapter) 453 { 454 /* 455 * IO rings initialization can be divided into the 2 steps: 456 * 1. Initialize variables and fields with initial values and copy 457 * them from adapter/ena_dev (basic) 458 * 2. Allocate mutex, counters and buf_ring (advanced) 459 */ 460 ena_init_io_rings_basic(adapter); 461 ena_init_io_rings_advanced(adapter); 462 } 463 464 static void 465 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid) 466 { 467 struct ena_ring *txr = &adapter->tx_ring[qid]; 468 struct ena_ring *rxr = &adapter->rx_ring[qid]; 469 470 ena_free_counters((counter_u64_t *)&txr->tx_stats, 471 sizeof(txr->tx_stats)); 472 ena_free_counters((counter_u64_t *)&rxr->rx_stats, 473 sizeof(rxr->rx_stats)); 474 475 ENA_RING_MTX_LOCK(txr); 476 drbr_free(txr->br, M_DEVBUF); 477 ENA_RING_MTX_UNLOCK(txr); 478 479 mtx_destroy(&txr->ring_mtx); 480 } 481 482 static void 483 ena_free_all_io_rings_resources(struct ena_adapter *adapter) 484 { 485 int i; 486 487 for (i = 0; i < adapter->num_io_queues; i++) 488 ena_free_io_ring_resources(adapter, i); 489 } 490 491 static int 492 ena_setup_tx_dma_tag(struct ena_adapter *adapter) 493 { 494 int ret; 495 496 /* Create DMA tag for Tx buffers */ 497 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), 498 1, 0, /* alignment, bounds */ 499 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 500 BUS_SPACE_MAXADDR, /* highaddr of excl window */ 501 NULL, NULL, /* filter, filterarg */ 502 ENA_TSO_MAXSIZE, /* maxsize */ 503 adapter->max_tx_sgl_size - 1, /* nsegments */ 504 ENA_TSO_MAXSIZE, /* maxsegsize */ 505 0, /* flags */ 506 NULL, /* lockfunc */ 507 NULL, /* lockfuncarg */ 508 &adapter->tx_buf_tag); 509 510 return (ret); 511 } 512 513 static int 514 ena_free_tx_dma_tag(struct ena_adapter *adapter) 515 { 516 int ret; 517 518 ret = bus_dma_tag_destroy(adapter->tx_buf_tag); 519 520 if (likely(ret == 0)) 521 adapter->tx_buf_tag = NULL; 522 523 return (ret); 524 } 525 526 static int 527 ena_setup_rx_dma_tag(struct ena_adapter *adapter) 528 { 529 int ret; 530 531 /* Create DMA tag for Rx buffers*/ 532 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */ 533 1, 0, /* alignment, bounds */ 534 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 535 BUS_SPACE_MAXADDR, /* highaddr of excl window */ 536 NULL, NULL, /* filter, filterarg */ 537 ena_mbuf_sz, /* maxsize */ 538 adapter->max_rx_sgl_size, /* nsegments */ 539 ena_mbuf_sz, /* maxsegsize */ 540 0, /* flags */ 541 NULL, /* lockfunc */ 542 NULL, /* lockarg */ 543 &adapter->rx_buf_tag); 544 545 return (ret); 546 } 547 548 static int 549 ena_free_rx_dma_tag(struct ena_adapter *adapter) 550 { 551 int ret; 552 553 ret = bus_dma_tag_destroy(adapter->rx_buf_tag); 554 555 if (likely(ret == 0)) 556 adapter->rx_buf_tag = NULL; 557 558 return (ret); 559 } 560 561 static void 562 ena_release_all_tx_dmamap(struct ena_ring *tx_ring) 563 { 564 struct ena_adapter *adapter = tx_ring->adapter; 565 struct ena_tx_buffer *tx_info; 566 bus_dma_tag_t tx_tag = adapter->tx_buf_tag; 567 int i; 568 #ifdef DEV_NETMAP 569 struct ena_netmap_tx_info *nm_info; 570 int j; 571 #endif /* DEV_NETMAP */ 572 573 for (i = 0; i < tx_ring->ring_size; ++i) { 574 tx_info = &tx_ring->tx_buffer_info[i]; 575 #ifdef DEV_NETMAP 576 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) { 577 nm_info = &tx_info->nm_info; 578 for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) { 579 if (nm_info->map_seg[j] != NULL) { 580 bus_dmamap_destroy(tx_tag, 581 nm_info->map_seg[j]); 582 nm_info->map_seg[j] = NULL; 583 } 584 } 585 } 586 #endif /* DEV_NETMAP */ 587 if (tx_info->dmamap != NULL) { 588 bus_dmamap_destroy(tx_tag, tx_info->dmamap); 589 tx_info->dmamap = NULL; 590 } 591 } 592 } 593 594 /** 595 * ena_setup_tx_resources - allocate Tx resources (Descriptors) 596 * @adapter: network interface device structure 597 * @qid: queue index 598 * 599 * Returns 0 on success, otherwise on failure. 600 **/ 601 static int 602 ena_setup_tx_resources(struct ena_adapter *adapter, int qid) 603 { 604 device_t pdev = adapter->pdev; 605 char thread_name[MAXCOMLEN + 1]; 606 struct ena_que *que = &adapter->que[qid]; 607 struct ena_ring *tx_ring = que->tx_ring; 608 cpuset_t *cpu_mask = NULL; 609 int size, i, err; 610 #ifdef DEV_NETMAP 611 bus_dmamap_t *map; 612 int j; 613 614 ena_netmap_reset_tx_ring(adapter, qid); 615 #endif /* DEV_NETMAP */ 616 617 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; 618 619 tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 620 if (unlikely(tx_ring->tx_buffer_info == NULL)) 621 return (ENOMEM); 622 623 size = sizeof(uint16_t) * tx_ring->ring_size; 624 tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 625 if (unlikely(tx_ring->free_tx_ids == NULL)) 626 goto err_buf_info_free; 627 628 size = tx_ring->tx_max_header_size; 629 tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF, 630 M_NOWAIT | M_ZERO); 631 if (unlikely(tx_ring->push_buf_intermediate_buf == NULL)) 632 goto err_tx_ids_free; 633 634 /* Req id stack for TX OOO completions */ 635 for (i = 0; i < tx_ring->ring_size; i++) 636 tx_ring->free_tx_ids[i] = i; 637 638 /* Reset TX statistics. */ 639 ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats, 640 sizeof(tx_ring->tx_stats)); 641 642 tx_ring->next_to_use = 0; 643 tx_ring->next_to_clean = 0; 644 tx_ring->acum_pkts = 0; 645 646 /* Make sure that drbr is empty */ 647 ENA_RING_MTX_LOCK(tx_ring); 648 drbr_flush(adapter->ifp, tx_ring->br); 649 ENA_RING_MTX_UNLOCK(tx_ring); 650 651 /* ... and create the buffer DMA maps */ 652 for (i = 0; i < tx_ring->ring_size; i++) { 653 err = bus_dmamap_create(adapter->tx_buf_tag, 0, 654 &tx_ring->tx_buffer_info[i].dmamap); 655 if (unlikely(err != 0)) { 656 ena_log(pdev, ERR, 657 "Unable to create Tx DMA map for buffer %d\n", i); 658 goto err_map_release; 659 } 660 661 #ifdef DEV_NETMAP 662 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) { 663 map = tx_ring->tx_buffer_info[i].nm_info.map_seg; 664 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { 665 err = bus_dmamap_create(adapter->tx_buf_tag, 0, 666 &map[j]); 667 if (unlikely(err != 0)) { 668 ena_log(pdev, ERR, 669 "Unable to create Tx DMA for buffer %d %d\n", 670 i, j); 671 goto err_map_release; 672 } 673 } 674 } 675 #endif /* DEV_NETMAP */ 676 } 677 678 /* Allocate taskqueues */ 679 TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring); 680 tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT, 681 taskqueue_thread_enqueue, &tx_ring->enqueue_tq); 682 if (unlikely(tx_ring->enqueue_tq == NULL)) { 683 ena_log(pdev, ERR, 684 "Unable to create taskqueue for enqueue task\n"); 685 i = tx_ring->ring_size; 686 goto err_map_release; 687 } 688 689 tx_ring->running = true; 690 691 #ifdef RSS 692 cpu_mask = &que->cpu_mask; 693 snprintf(thread_name, sizeof(thread_name), "%s txeq %d", 694 device_get_nameunit(adapter->pdev), que->cpu); 695 #else 696 snprintf(thread_name, sizeof(thread_name), "%s txeq %d", 697 device_get_nameunit(adapter->pdev), que->id); 698 #endif 699 taskqueue_start_threads_cpuset(&tx_ring->enqueue_tq, 1, PI_NET, 700 cpu_mask, "%s", thread_name); 701 702 return (0); 703 704 err_map_release: 705 ena_release_all_tx_dmamap(tx_ring); 706 err_tx_ids_free: 707 free(tx_ring->free_tx_ids, M_DEVBUF); 708 tx_ring->free_tx_ids = NULL; 709 err_buf_info_free: 710 free(tx_ring->tx_buffer_info, M_DEVBUF); 711 tx_ring->tx_buffer_info = NULL; 712 713 return (ENOMEM); 714 } 715 716 /** 717 * ena_free_tx_resources - Free Tx Resources per Queue 718 * @adapter: network interface device structure 719 * @qid: queue index 720 * 721 * Free all transmit software resources 722 **/ 723 static void 724 ena_free_tx_resources(struct ena_adapter *adapter, int qid) 725 { 726 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 727 #ifdef DEV_NETMAP 728 struct ena_netmap_tx_info *nm_info; 729 int j; 730 #endif /* DEV_NETMAP */ 731 732 while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task, NULL)) 733 taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 734 735 taskqueue_free(tx_ring->enqueue_tq); 736 737 ENA_RING_MTX_LOCK(tx_ring); 738 /* Flush buffer ring, */ 739 drbr_flush(adapter->ifp, tx_ring->br); 740 741 /* Free buffer DMA maps, */ 742 for (int i = 0; i < tx_ring->ring_size; i++) { 743 bus_dmamap_sync(adapter->tx_buf_tag, 744 tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE); 745 bus_dmamap_unload(adapter->tx_buf_tag, 746 tx_ring->tx_buffer_info[i].dmamap); 747 bus_dmamap_destroy(adapter->tx_buf_tag, 748 tx_ring->tx_buffer_info[i].dmamap); 749 750 #ifdef DEV_NETMAP 751 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) { 752 nm_info = &tx_ring->tx_buffer_info[i].nm_info; 753 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { 754 if (nm_info->socket_buf_idx[j] != 0) { 755 bus_dmamap_sync(adapter->tx_buf_tag, 756 nm_info->map_seg[j], 757 BUS_DMASYNC_POSTWRITE); 758 ena_netmap_unload(adapter, 759 nm_info->map_seg[j]); 760 } 761 bus_dmamap_destroy(adapter->tx_buf_tag, 762 nm_info->map_seg[j]); 763 nm_info->socket_buf_idx[j] = 0; 764 } 765 } 766 #endif /* DEV_NETMAP */ 767 768 m_freem(tx_ring->tx_buffer_info[i].mbuf); 769 tx_ring->tx_buffer_info[i].mbuf = NULL; 770 } 771 ENA_RING_MTX_UNLOCK(tx_ring); 772 773 /* And free allocated memory. */ 774 free(tx_ring->tx_buffer_info, M_DEVBUF); 775 tx_ring->tx_buffer_info = NULL; 776 777 free(tx_ring->free_tx_ids, M_DEVBUF); 778 tx_ring->free_tx_ids = NULL; 779 780 free(tx_ring->push_buf_intermediate_buf, M_DEVBUF); 781 tx_ring->push_buf_intermediate_buf = NULL; 782 } 783 784 /** 785 * ena_setup_all_tx_resources - allocate all queues Tx resources 786 * @adapter: network interface device structure 787 * 788 * Returns 0 on success, otherwise on failure. 789 **/ 790 static int 791 ena_setup_all_tx_resources(struct ena_adapter *adapter) 792 { 793 int i, rc; 794 795 for (i = 0; i < adapter->num_io_queues; i++) { 796 rc = ena_setup_tx_resources(adapter, i); 797 if (rc != 0) { 798 ena_log(adapter->pdev, ERR, 799 "Allocation for Tx Queue %u failed\n", i); 800 goto err_setup_tx; 801 } 802 } 803 804 return (0); 805 806 err_setup_tx: 807 /* Rewind the index freeing the rings as we go */ 808 while (i--) 809 ena_free_tx_resources(adapter, i); 810 return (rc); 811 } 812 813 /** 814 * ena_free_all_tx_resources - Free Tx Resources for All Queues 815 * @adapter: network interface device structure 816 * 817 * Free all transmit software resources 818 **/ 819 static void 820 ena_free_all_tx_resources(struct ena_adapter *adapter) 821 { 822 int i; 823 824 for (i = 0; i < adapter->num_io_queues; i++) 825 ena_free_tx_resources(adapter, i); 826 } 827 828 /** 829 * ena_setup_rx_resources - allocate Rx resources (Descriptors) 830 * @adapter: network interface device structure 831 * @qid: queue index 832 * 833 * Returns 0 on success, otherwise on failure. 834 **/ 835 static int 836 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid) 837 { 838 device_t pdev = adapter->pdev; 839 struct ena_que *que = &adapter->que[qid]; 840 struct ena_ring *rx_ring = que->rx_ring; 841 int size, err, i; 842 843 size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size; 844 845 #ifdef DEV_NETMAP 846 ena_netmap_reset_rx_ring(adapter, qid); 847 rx_ring->initialized = false; 848 #endif /* DEV_NETMAP */ 849 850 /* 851 * Alloc extra element so in rx path 852 * we can always prefetch rx_info + 1 853 */ 854 size += sizeof(struct ena_rx_buffer); 855 856 rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO); 857 858 size = sizeof(uint16_t) * rx_ring->ring_size; 859 rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK); 860 861 for (i = 0; i < rx_ring->ring_size; i++) 862 rx_ring->free_rx_ids[i] = i; 863 864 /* Reset RX statistics. */ 865 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats, 866 sizeof(rx_ring->rx_stats)); 867 868 rx_ring->next_to_clean = 0; 869 rx_ring->next_to_use = 0; 870 871 /* ... and create the buffer DMA maps */ 872 for (i = 0; i < rx_ring->ring_size; i++) { 873 err = bus_dmamap_create(adapter->rx_buf_tag, 0, 874 &(rx_ring->rx_buffer_info[i].map)); 875 if (err != 0) { 876 ena_log(pdev, ERR, 877 "Unable to create Rx DMA map for buffer %d\n", i); 878 goto err_buf_info_unmap; 879 } 880 } 881 882 /* Create LRO for the ring */ 883 if ((if_getcapenable(adapter->ifp) & IFCAP_LRO) != 0) { 884 int err = tcp_lro_init(&rx_ring->lro); 885 if (err != 0) { 886 ena_log(pdev, ERR, "LRO[%d] Initialization failed!\n", 887 qid); 888 } else { 889 ena_log(pdev, DBG, "RX Soft LRO[%d] Initialized\n", 890 qid); 891 rx_ring->lro.ifp = adapter->ifp; 892 } 893 } 894 895 return (0); 896 897 err_buf_info_unmap: 898 while (i--) { 899 bus_dmamap_destroy(adapter->rx_buf_tag, 900 rx_ring->rx_buffer_info[i].map); 901 } 902 903 free(rx_ring->free_rx_ids, M_DEVBUF); 904 rx_ring->free_rx_ids = NULL; 905 free(rx_ring->rx_buffer_info, M_DEVBUF); 906 rx_ring->rx_buffer_info = NULL; 907 return (ENOMEM); 908 } 909 910 /** 911 * ena_free_rx_resources - Free Rx Resources 912 * @adapter: network interface device structure 913 * @qid: queue index 914 * 915 * Free all receive software resources 916 **/ 917 static void 918 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid) 919 { 920 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 921 922 /* Free buffer DMA maps, */ 923 for (int i = 0; i < rx_ring->ring_size; i++) { 924 bus_dmamap_sync(adapter->rx_buf_tag, 925 rx_ring->rx_buffer_info[i].map, BUS_DMASYNC_POSTREAD); 926 m_freem(rx_ring->rx_buffer_info[i].mbuf); 927 rx_ring->rx_buffer_info[i].mbuf = NULL; 928 bus_dmamap_unload(adapter->rx_buf_tag, 929 rx_ring->rx_buffer_info[i].map); 930 bus_dmamap_destroy(adapter->rx_buf_tag, 931 rx_ring->rx_buffer_info[i].map); 932 } 933 934 /* free LRO resources, */ 935 tcp_lro_free(&rx_ring->lro); 936 937 /* free allocated memory */ 938 free(rx_ring->rx_buffer_info, M_DEVBUF); 939 rx_ring->rx_buffer_info = NULL; 940 941 free(rx_ring->free_rx_ids, M_DEVBUF); 942 rx_ring->free_rx_ids = NULL; 943 } 944 945 /** 946 * ena_setup_all_rx_resources - allocate all queues Rx resources 947 * @adapter: network interface device structure 948 * 949 * Returns 0 on success, otherwise on failure. 950 **/ 951 static int 952 ena_setup_all_rx_resources(struct ena_adapter *adapter) 953 { 954 int i, rc = 0; 955 956 for (i = 0; i < adapter->num_io_queues; i++) { 957 rc = ena_setup_rx_resources(adapter, i); 958 if (rc != 0) { 959 ena_log(adapter->pdev, ERR, 960 "Allocation for Rx Queue %u failed\n", i); 961 goto err_setup_rx; 962 } 963 } 964 return (0); 965 966 err_setup_rx: 967 /* rewind the index freeing the rings as we go */ 968 while (i--) 969 ena_free_rx_resources(adapter, i); 970 return (rc); 971 } 972 973 /** 974 * ena_free_all_rx_resources - Free Rx resources for all queues 975 * @adapter: network interface device structure 976 * 977 * Free all receive software resources 978 **/ 979 static void 980 ena_free_all_rx_resources(struct ena_adapter *adapter) 981 { 982 int i; 983 984 for (i = 0; i < adapter->num_io_queues; i++) 985 ena_free_rx_resources(adapter, i); 986 } 987 988 static inline int 989 ena_alloc_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring, 990 struct ena_rx_buffer *rx_info) 991 { 992 device_t pdev = adapter->pdev; 993 struct ena_com_buf *ena_buf; 994 bus_dma_segment_t segs[1]; 995 int nsegs, error; 996 int mlen; 997 998 /* if previous allocated frag is not used */ 999 if (unlikely(rx_info->mbuf != NULL)) 1000 return (0); 1001 1002 /* Get mbuf using UMA allocator */ 1003 rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, 1004 rx_ring->rx_mbuf_sz); 1005 1006 if (unlikely(rx_info->mbuf == NULL)) { 1007 counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1); 1008 rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 1009 if (unlikely(rx_info->mbuf == NULL)) { 1010 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); 1011 return (ENOMEM); 1012 } 1013 mlen = MCLBYTES; 1014 } else { 1015 mlen = rx_ring->rx_mbuf_sz; 1016 } 1017 /* Set mbuf length*/ 1018 rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen; 1019 1020 /* Map packets for DMA */ 1021 ena_log(pdev, DBG, 1022 "Using tag %p for buffers' DMA mapping, mbuf %p len: %d\n", 1023 adapter->rx_buf_tag, rx_info->mbuf, rx_info->mbuf->m_len); 1024 error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map, 1025 rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT); 1026 if (unlikely((error != 0) || (nsegs != 1))) { 1027 ena_log(pdev, WARN, 1028 "failed to map mbuf, error: %d, nsegs: %d\n", error, nsegs); 1029 counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1); 1030 goto exit; 1031 } 1032 1033 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD); 1034 1035 ena_buf = &rx_info->ena_buf; 1036 ena_buf->paddr = segs[0].ds_addr; 1037 ena_buf->len = mlen; 1038 1039 ena_log(pdev, DBG, 1040 "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n", 1041 rx_info->mbuf, rx_info, ena_buf->len, (uintmax_t)ena_buf->paddr); 1042 1043 return (0); 1044 1045 exit: 1046 m_freem(rx_info->mbuf); 1047 rx_info->mbuf = NULL; 1048 return (EFAULT); 1049 } 1050 1051 static void 1052 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring, 1053 struct ena_rx_buffer *rx_info) 1054 { 1055 if (rx_info->mbuf == NULL) { 1056 ena_log(adapter->pdev, WARN, 1057 "Trying to free unallocated buffer\n"); 1058 return; 1059 } 1060 1061 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 1062 BUS_DMASYNC_POSTREAD); 1063 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map); 1064 m_freem(rx_info->mbuf); 1065 rx_info->mbuf = NULL; 1066 } 1067 1068 /** 1069 * ena_refill_rx_bufs - Refills ring with descriptors 1070 * @rx_ring: the ring which we want to feed with free descriptors 1071 * @num: number of descriptors to refill 1072 * Refills the ring with newly allocated DMA-mapped mbufs for receiving 1073 **/ 1074 int 1075 ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num) 1076 { 1077 struct ena_adapter *adapter = rx_ring->adapter; 1078 device_t pdev = adapter->pdev; 1079 uint16_t next_to_use, req_id; 1080 uint32_t i; 1081 int rc; 1082 1083 ena_log_io(adapter->pdev, DBG, "refill qid: %d\n", rx_ring->qid); 1084 1085 next_to_use = rx_ring->next_to_use; 1086 1087 for (i = 0; i < num; i++) { 1088 struct ena_rx_buffer *rx_info; 1089 1090 ena_log_io(pdev, DBG, "RX buffer - next to use: %d\n", 1091 next_to_use); 1092 1093 req_id = rx_ring->free_rx_ids[next_to_use]; 1094 rx_info = &rx_ring->rx_buffer_info[req_id]; 1095 #ifdef DEV_NETMAP 1096 if (ena_rx_ring_in_netmap(adapter, rx_ring->qid)) 1097 rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, 1098 rx_info); 1099 else 1100 #endif /* DEV_NETMAP */ 1101 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info); 1102 if (unlikely(rc != 0)) { 1103 ena_log_io(pdev, WARN, 1104 "failed to alloc buffer for rx queue %d\n", 1105 rx_ring->qid); 1106 break; 1107 } 1108 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq, 1109 &rx_info->ena_buf, req_id); 1110 if (unlikely(rc != 0)) { 1111 ena_log_io(pdev, WARN, 1112 "failed to add buffer for rx queue %d\n", 1113 rx_ring->qid); 1114 break; 1115 } 1116 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, 1117 rx_ring->ring_size); 1118 } 1119 1120 if (unlikely(i < num)) { 1121 counter_u64_add(rx_ring->rx_stats.refil_partial, 1); 1122 ena_log_io(pdev, WARN, 1123 "refilled rx qid %d with only %d mbufs (from %d)\n", 1124 rx_ring->qid, i, num); 1125 } 1126 1127 if (likely(i != 0)) 1128 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq); 1129 1130 rx_ring->next_to_use = next_to_use; 1131 return (i); 1132 } 1133 1134 int 1135 ena_update_buf_ring_size(struct ena_adapter *adapter, 1136 uint32_t new_buf_ring_size) 1137 { 1138 uint32_t old_buf_ring_size; 1139 int rc = 0; 1140 bool dev_was_up; 1141 1142 old_buf_ring_size = adapter->buf_ring_size; 1143 adapter->buf_ring_size = new_buf_ring_size; 1144 1145 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 1146 ena_down(adapter); 1147 1148 /* Reconfigure buf ring for all Tx rings. */ 1149 ena_free_all_io_rings_resources(adapter); 1150 ena_init_io_rings_advanced(adapter); 1151 if (dev_was_up) { 1152 /* 1153 * If ena_up() fails, it's not because of recent buf_ring size 1154 * changes. Because of that, we just want to revert old drbr 1155 * value and trigger the reset because something else had to 1156 * go wrong. 1157 */ 1158 rc = ena_up(adapter); 1159 if (unlikely(rc != 0)) { 1160 ena_log(adapter->pdev, ERR, 1161 "Failed to configure device after setting new drbr size: %u. Reverting old value: %u and triggering the reset\n", 1162 new_buf_ring_size, old_buf_ring_size); 1163 1164 /* Revert old size and trigger the reset */ 1165 adapter->buf_ring_size = old_buf_ring_size; 1166 ena_free_all_io_rings_resources(adapter); 1167 ena_init_io_rings_advanced(adapter); 1168 1169 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, 1170 adapter); 1171 ena_trigger_reset(adapter, ENA_REGS_RESET_OS_TRIGGER); 1172 } 1173 } 1174 1175 return (rc); 1176 } 1177 1178 int 1179 ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size, 1180 uint32_t new_rx_size) 1181 { 1182 uint32_t old_tx_size, old_rx_size; 1183 int rc = 0; 1184 bool dev_was_up; 1185 1186 old_tx_size = adapter->requested_tx_ring_size; 1187 old_rx_size = adapter->requested_rx_ring_size; 1188 adapter->requested_tx_ring_size = new_tx_size; 1189 adapter->requested_rx_ring_size = new_rx_size; 1190 1191 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 1192 ena_down(adapter); 1193 1194 /* Configure queues with new size. */ 1195 ena_init_io_rings_basic(adapter); 1196 if (dev_was_up) { 1197 rc = ena_up(adapter); 1198 if (unlikely(rc != 0)) { 1199 ena_log(adapter->pdev, ERR, 1200 "Failed to configure device with the new sizes - Tx: %u Rx: %u. Reverting old values - Tx: %u Rx: %u\n", 1201 new_tx_size, new_rx_size, old_tx_size, old_rx_size); 1202 1203 /* Revert old size. */ 1204 adapter->requested_tx_ring_size = old_tx_size; 1205 adapter->requested_rx_ring_size = old_rx_size; 1206 ena_init_io_rings_basic(adapter); 1207 1208 /* And try again. */ 1209 rc = ena_up(adapter); 1210 if (unlikely(rc != 0)) { 1211 ena_log(adapter->pdev, ERR, 1212 "Failed to revert old queue sizes. Triggering device reset.\n"); 1213 /* 1214 * If we've failed again, something had to go 1215 * wrong. After reset, the device should try to 1216 * go up 1217 */ 1218 ENA_FLAG_SET_ATOMIC( 1219 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 1220 ena_trigger_reset(adapter, 1221 ENA_REGS_RESET_OS_TRIGGER); 1222 } 1223 } 1224 } 1225 1226 return (rc); 1227 } 1228 1229 static void 1230 ena_update_io_rings(struct ena_adapter *adapter, uint32_t num) 1231 { 1232 ena_free_all_io_rings_resources(adapter); 1233 /* Force indirection table to be reinitialized */ 1234 ena_com_rss_destroy(adapter->ena_dev); 1235 1236 adapter->num_io_queues = num; 1237 ena_init_io_rings(adapter); 1238 } 1239 1240 /* Caller should sanitize new_num */ 1241 int 1242 ena_update_io_queue_nb(struct ena_adapter *adapter, uint32_t new_num) 1243 { 1244 uint32_t old_num; 1245 int rc = 0; 1246 bool dev_was_up; 1247 1248 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 1249 old_num = adapter->num_io_queues; 1250 ena_down(adapter); 1251 1252 ena_update_io_rings(adapter, new_num); 1253 1254 if (dev_was_up) { 1255 rc = ena_up(adapter); 1256 if (unlikely(rc != 0)) { 1257 ena_log(adapter->pdev, ERR, 1258 "Failed to configure device with %u IO queues. " 1259 "Reverting to previous value: %u\n", 1260 new_num, old_num); 1261 1262 ena_update_io_rings(adapter, old_num); 1263 1264 rc = ena_up(adapter); 1265 if (unlikely(rc != 0)) { 1266 ena_log(adapter->pdev, ERR, 1267 "Failed to revert to previous setup IO " 1268 "queues. Triggering device reset.\n"); 1269 ENA_FLAG_SET_ATOMIC( 1270 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 1271 ena_trigger_reset(adapter, 1272 ENA_REGS_RESET_OS_TRIGGER); 1273 } 1274 } 1275 } 1276 1277 return (rc); 1278 } 1279 1280 static void 1281 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid) 1282 { 1283 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 1284 unsigned int i; 1285 1286 for (i = 0; i < rx_ring->ring_size; i++) { 1287 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i]; 1288 1289 if (rx_info->mbuf != NULL) 1290 ena_free_rx_mbuf(adapter, rx_ring, rx_info); 1291 #ifdef DEV_NETMAP 1292 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) && 1293 (if_getcapenable(adapter->ifp) & IFCAP_NETMAP)) { 1294 if (rx_info->netmap_buf_idx != 0) 1295 ena_netmap_free_rx_slot(adapter, rx_ring, 1296 rx_info); 1297 } 1298 #endif /* DEV_NETMAP */ 1299 } 1300 } 1301 1302 /** 1303 * ena_refill_all_rx_bufs - allocate all queues Rx buffers 1304 * @adapter: network interface device structure 1305 * 1306 */ 1307 static void 1308 ena_refill_all_rx_bufs(struct ena_adapter *adapter) 1309 { 1310 struct ena_ring *rx_ring; 1311 int i, rc, bufs_num; 1312 1313 for (i = 0; i < adapter->num_io_queues; i++) { 1314 rx_ring = &adapter->rx_ring[i]; 1315 bufs_num = rx_ring->ring_size - 1; 1316 rc = ena_refill_rx_bufs(rx_ring, bufs_num); 1317 if (unlikely(rc != bufs_num)) 1318 ena_log_io(adapter->pdev, WARN, 1319 "refilling Queue %d failed. " 1320 "Allocated %d buffers from: %d\n", 1321 i, rc, bufs_num); 1322 #ifdef DEV_NETMAP 1323 rx_ring->initialized = true; 1324 #endif /* DEV_NETMAP */ 1325 } 1326 } 1327 1328 static void 1329 ena_free_all_rx_bufs(struct ena_adapter *adapter) 1330 { 1331 int i; 1332 1333 for (i = 0; i < adapter->num_io_queues; i++) 1334 ena_free_rx_bufs(adapter, i); 1335 } 1336 1337 /** 1338 * ena_free_tx_bufs - Free Tx Buffers per Queue 1339 * @adapter: network interface device structure 1340 * @qid: queue index 1341 **/ 1342 static void 1343 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid) 1344 { 1345 bool print_once = true; 1346 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 1347 1348 ENA_RING_MTX_LOCK(tx_ring); 1349 for (int i = 0; i < tx_ring->ring_size; i++) { 1350 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i]; 1351 1352 if (tx_info->mbuf == NULL) 1353 continue; 1354 1355 if (print_once) { 1356 ena_log(adapter->pdev, WARN, 1357 "free uncompleted tx mbuf qid %d idx 0x%x\n", qid, 1358 i); 1359 print_once = false; 1360 } else { 1361 ena_log(adapter->pdev, DBG, 1362 "free uncompleted tx mbuf qid %d idx 0x%x\n", qid, 1363 i); 1364 } 1365 1366 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 1367 BUS_DMASYNC_POSTWRITE); 1368 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); 1369 1370 m_free(tx_info->mbuf); 1371 tx_info->mbuf = NULL; 1372 } 1373 ENA_RING_MTX_UNLOCK(tx_ring); 1374 } 1375 1376 static void 1377 ena_free_all_tx_bufs(struct ena_adapter *adapter) 1378 { 1379 for (int i = 0; i < adapter->num_io_queues; i++) 1380 ena_free_tx_bufs(adapter, i); 1381 } 1382 1383 static void 1384 ena_destroy_all_tx_queues(struct ena_adapter *adapter) 1385 { 1386 uint16_t ena_qid; 1387 int i; 1388 1389 for (i = 0; i < adapter->num_io_queues; i++) { 1390 ena_qid = ENA_IO_TXQ_IDX(i); 1391 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1392 } 1393 } 1394 1395 static void 1396 ena_destroy_all_rx_queues(struct ena_adapter *adapter) 1397 { 1398 uint16_t ena_qid; 1399 int i; 1400 1401 for (i = 0; i < adapter->num_io_queues; i++) { 1402 ena_qid = ENA_IO_RXQ_IDX(i); 1403 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1404 } 1405 } 1406 1407 static void 1408 ena_destroy_all_io_queues(struct ena_adapter *adapter) 1409 { 1410 struct ena_que *queue; 1411 int i; 1412 1413 for (i = 0; i < adapter->num_io_queues; i++) { 1414 queue = &adapter->que[i]; 1415 while (taskqueue_cancel(queue->cleanup_tq, &queue->cleanup_task, NULL)) 1416 taskqueue_drain(queue->cleanup_tq, &queue->cleanup_task); 1417 taskqueue_free(queue->cleanup_tq); 1418 } 1419 1420 ena_destroy_all_tx_queues(adapter); 1421 ena_destroy_all_rx_queues(adapter); 1422 } 1423 1424 static int 1425 ena_create_io_queues(struct ena_adapter *adapter) 1426 { 1427 struct ena_com_dev *ena_dev = adapter->ena_dev; 1428 struct ena_com_create_io_ctx ctx; 1429 struct ena_ring *ring; 1430 struct ena_que *queue; 1431 uint16_t ena_qid; 1432 uint32_t msix_vector; 1433 cpuset_t *cpu_mask = NULL; 1434 int rc, i; 1435 1436 /* Create TX queues */ 1437 for (i = 0; i < adapter->num_io_queues; i++) { 1438 msix_vector = ENA_IO_IRQ_IDX(i); 1439 ena_qid = ENA_IO_TXQ_IDX(i); 1440 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1441 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1442 ctx.queue_size = adapter->requested_tx_ring_size; 1443 ctx.msix_vector = msix_vector; 1444 ctx.qid = ena_qid; 1445 ctx.numa_node = adapter->que[i].domain; 1446 1447 rc = ena_com_create_io_queue(ena_dev, &ctx); 1448 if (rc != 0) { 1449 ena_log(adapter->pdev, ERR, 1450 "Failed to create io TX queue #%d rc: %d\n", i, rc); 1451 goto err_tx; 1452 } 1453 ring = &adapter->tx_ring[i]; 1454 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1455 &ring->ena_com_io_sq, &ring->ena_com_io_cq); 1456 if (rc != 0) { 1457 ena_log(adapter->pdev, ERR, 1458 "Failed to get TX queue handlers. TX queue num" 1459 " %d rc: %d\n", 1460 i, rc); 1461 ena_com_destroy_io_queue(ena_dev, ena_qid); 1462 goto err_tx; 1463 } 1464 1465 if (ctx.numa_node >= 0) { 1466 ena_com_update_numa_node(ring->ena_com_io_cq, 1467 ctx.numa_node); 1468 } 1469 } 1470 1471 /* Create RX queues */ 1472 for (i = 0; i < adapter->num_io_queues; i++) { 1473 msix_vector = ENA_IO_IRQ_IDX(i); 1474 ena_qid = ENA_IO_RXQ_IDX(i); 1475 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1476 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1477 ctx.queue_size = adapter->requested_rx_ring_size; 1478 ctx.msix_vector = msix_vector; 1479 ctx.qid = ena_qid; 1480 ctx.numa_node = adapter->que[i].domain; 1481 1482 rc = ena_com_create_io_queue(ena_dev, &ctx); 1483 if (unlikely(rc != 0)) { 1484 ena_log(adapter->pdev, ERR, 1485 "Failed to create io RX queue[%d] rc: %d\n", i, rc); 1486 goto err_rx; 1487 } 1488 1489 ring = &adapter->rx_ring[i]; 1490 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1491 &ring->ena_com_io_sq, &ring->ena_com_io_cq); 1492 if (unlikely(rc != 0)) { 1493 ena_log(adapter->pdev, ERR, 1494 "Failed to get RX queue handlers. RX queue num" 1495 " %d rc: %d\n", 1496 i, rc); 1497 ena_com_destroy_io_queue(ena_dev, ena_qid); 1498 goto err_rx; 1499 } 1500 1501 if (ctx.numa_node >= 0) { 1502 ena_com_update_numa_node(ring->ena_com_io_cq, 1503 ctx.numa_node); 1504 } 1505 } 1506 1507 for (i = 0; i < adapter->num_io_queues; i++) { 1508 queue = &adapter->que[i]; 1509 1510 NET_TASK_INIT(&queue->cleanup_task, 0, ena_cleanup, queue); 1511 queue->cleanup_tq = taskqueue_create_fast("ena cleanup", 1512 M_WAITOK, taskqueue_thread_enqueue, &queue->cleanup_tq); 1513 1514 #ifdef RSS 1515 cpu_mask = &queue->cpu_mask; 1516 #endif 1517 taskqueue_start_threads_cpuset(&queue->cleanup_tq, 1, PI_NET, 1518 cpu_mask, "%s queue %d cleanup", 1519 device_get_nameunit(adapter->pdev), i); 1520 } 1521 1522 return (0); 1523 1524 err_rx: 1525 while (i--) 1526 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i)); 1527 i = adapter->num_io_queues; 1528 err_tx: 1529 while (i--) 1530 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i)); 1531 1532 return (ENXIO); 1533 } 1534 1535 /********************************************************************* 1536 * 1537 * MSIX & Interrupt Service routine 1538 * 1539 **********************************************************************/ 1540 1541 /** 1542 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue 1543 * @arg: interrupt number 1544 **/ 1545 static void 1546 ena_intr_msix_mgmnt(void *arg) 1547 { 1548 struct ena_adapter *adapter = (struct ena_adapter *)arg; 1549 1550 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); 1551 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))) 1552 ena_com_aenq_intr_handler(adapter->ena_dev, arg); 1553 } 1554 1555 /** 1556 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx 1557 * @arg: queue 1558 **/ 1559 static int 1560 ena_handle_msix(void *arg) 1561 { 1562 struct ena_que *queue = arg; 1563 struct ena_adapter *adapter = queue->adapter; 1564 if_t ifp = adapter->ifp; 1565 1566 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) 1567 return (FILTER_STRAY); 1568 1569 taskqueue_enqueue(queue->cleanup_tq, &queue->cleanup_task); 1570 1571 return (FILTER_HANDLED); 1572 } 1573 1574 static int 1575 ena_enable_msix(struct ena_adapter *adapter) 1576 { 1577 device_t dev = adapter->pdev; 1578 int msix_vecs, msix_req; 1579 int i, rc = 0; 1580 1581 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { 1582 ena_log(dev, ERR, "Error, MSI-X is already enabled\n"); 1583 return (EINVAL); 1584 } 1585 1586 /* Reserved the max msix vectors we might need */ 1587 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues); 1588 1589 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry), 1590 M_DEVBUF, M_WAITOK | M_ZERO); 1591 1592 ena_log(dev, DBG, "trying to enable MSI-X, vectors: %d\n", msix_vecs); 1593 1594 for (i = 0; i < msix_vecs; i++) { 1595 adapter->msix_entries[i].entry = i; 1596 /* Vectors must start from 1 */ 1597 adapter->msix_entries[i].vector = i + 1; 1598 } 1599 1600 msix_req = msix_vecs; 1601 rc = pci_alloc_msix(dev, &msix_vecs); 1602 if (unlikely(rc != 0)) { 1603 ena_log(dev, ERR, "Failed to enable MSIX, vectors %d rc %d\n", 1604 msix_vecs, rc); 1605 1606 rc = ENOSPC; 1607 goto err_msix_free; 1608 } 1609 1610 if (msix_vecs != msix_req) { 1611 if (msix_vecs == ENA_ADMIN_MSIX_VEC) { 1612 ena_log(dev, ERR, 1613 "Not enough number of MSI-x allocated: %d\n", 1614 msix_vecs); 1615 pci_release_msi(dev); 1616 rc = ENOSPC; 1617 goto err_msix_free; 1618 } 1619 ena_log(dev, ERR, 1620 "Enable only %d MSI-x (out of %d), reduce " 1621 "the number of queues\n", 1622 msix_vecs, msix_req); 1623 } 1624 1625 adapter->msix_vecs = msix_vecs; 1626 ENA_FLAG_SET_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); 1627 1628 return (0); 1629 1630 err_msix_free: 1631 free(adapter->msix_entries, M_DEVBUF); 1632 adapter->msix_entries = NULL; 1633 1634 return (rc); 1635 } 1636 1637 static void 1638 ena_setup_mgmnt_intr(struct ena_adapter *adapter) 1639 { 1640 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, ENA_IRQNAME_SIZE, 1641 "ena-mgmnt@pci:%s", device_get_nameunit(adapter->pdev)); 1642 /* 1643 * Handler is NULL on purpose, it will be set 1644 * when mgmnt interrupt is acquired 1645 */ 1646 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL; 1647 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter; 1648 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector = 1649 adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector; 1650 } 1651 1652 static int 1653 ena_setup_io_intr(struct ena_adapter *adapter) 1654 { 1655 #ifdef RSS 1656 int num_buckets = rss_getnumbuckets(); 1657 static int last_bind = 0; 1658 int cur_bind; 1659 int idx; 1660 #endif 1661 int irq_idx; 1662 1663 if (adapter->msix_entries == NULL) 1664 return (EINVAL); 1665 1666 #ifdef RSS 1667 if (adapter->first_bind < 0) { 1668 adapter->first_bind = last_bind; 1669 last_bind = (last_bind + adapter->num_io_queues) % num_buckets; 1670 } 1671 cur_bind = adapter->first_bind; 1672 #endif 1673 1674 for (int i = 0; i < adapter->num_io_queues; i++) { 1675 irq_idx = ENA_IO_IRQ_IDX(i); 1676 1677 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, 1678 "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i); 1679 adapter->irq_tbl[irq_idx].handler = ena_handle_msix; 1680 adapter->irq_tbl[irq_idx].data = &adapter->que[i]; 1681 adapter->irq_tbl[irq_idx].vector = 1682 adapter->msix_entries[irq_idx].vector; 1683 ena_log(adapter->pdev, DBG, "ena_setup_io_intr vector: %d\n", 1684 adapter->msix_entries[irq_idx].vector); 1685 1686 #ifdef RSS 1687 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = 1688 rss_getcpu(cur_bind); 1689 cur_bind = (cur_bind + 1) % num_buckets; 1690 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask); 1691 1692 for (idx = 0; idx < MAXMEMDOM; ++idx) { 1693 if (CPU_ISSET(adapter->que[i].cpu, &cpuset_domain[idx])) 1694 break; 1695 } 1696 adapter->que[i].domain = idx; 1697 #else 1698 adapter->que[i].domain = -1; 1699 #endif 1700 } 1701 1702 return (0); 1703 } 1704 1705 static int 1706 ena_request_mgmnt_irq(struct ena_adapter *adapter) 1707 { 1708 device_t pdev = adapter->pdev; 1709 struct ena_irq *irq; 1710 unsigned long flags; 1711 int rc, rcc; 1712 1713 flags = RF_ACTIVE | RF_SHAREABLE; 1714 1715 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 1716 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, 1717 &irq->vector, flags); 1718 1719 if (unlikely(irq->res == NULL)) { 1720 ena_log(pdev, ERR, "could not allocate irq vector: %d\n", 1721 irq->vector); 1722 return (ENXIO); 1723 } 1724 1725 rc = bus_setup_intr(adapter->pdev, irq->res, 1726 INTR_TYPE_NET | INTR_MPSAFE, NULL, ena_intr_msix_mgmnt, irq->data, 1727 &irq->cookie); 1728 if (unlikely(rc != 0)) { 1729 ena_log(pdev, ERR, 1730 "failed to register interrupt handler for irq %ju: %d\n", 1731 rman_get_start(irq->res), rc); 1732 goto err_res_free; 1733 } 1734 irq->requested = true; 1735 1736 return (rc); 1737 1738 err_res_free: 1739 ena_log(pdev, INFO, "releasing resource for irq %d\n", irq->vector); 1740 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, irq->vector, 1741 irq->res); 1742 if (unlikely(rcc != 0)) 1743 ena_log(pdev, ERR, 1744 "dev has no parent while releasing res for irq: %d\n", 1745 irq->vector); 1746 irq->res = NULL; 1747 1748 return (rc); 1749 } 1750 1751 static int 1752 ena_request_io_irq(struct ena_adapter *adapter) 1753 { 1754 device_t pdev = adapter->pdev; 1755 struct ena_irq *irq; 1756 unsigned long flags = 0; 1757 int rc = 0, i, rcc; 1758 1759 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter))) { 1760 ena_log(pdev, ERR, 1761 "failed to request I/O IRQ: MSI-X is not enabled\n"); 1762 return (EINVAL); 1763 } else { 1764 flags = RF_ACTIVE | RF_SHAREABLE; 1765 } 1766 1767 for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { 1768 irq = &adapter->irq_tbl[i]; 1769 1770 if (unlikely(irq->requested)) 1771 continue; 1772 1773 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, 1774 &irq->vector, flags); 1775 if (unlikely(irq->res == NULL)) { 1776 rc = ENOMEM; 1777 ena_log(pdev, ERR, 1778 "could not allocate irq vector: %d\n", irq->vector); 1779 goto err; 1780 } 1781 1782 rc = bus_setup_intr(adapter->pdev, irq->res, 1783 INTR_TYPE_NET | INTR_MPSAFE, irq->handler, NULL, irq->data, 1784 &irq->cookie); 1785 if (unlikely(rc != 0)) { 1786 ena_log(pdev, ERR, 1787 "failed to register interrupt handler for irq %ju: %d\n", 1788 rman_get_start(irq->res), rc); 1789 goto err; 1790 } 1791 irq->requested = true; 1792 1793 #ifdef RSS 1794 rc = bus_bind_intr(adapter->pdev, irq->res, irq->cpu); 1795 if (unlikely(rc != 0)) { 1796 ena_log(pdev, ERR, 1797 "failed to bind interrupt handler for irq %ju to cpu %d: %d\n", 1798 rman_get_start(irq->res), irq->cpu, rc); 1799 goto err; 1800 } 1801 1802 ena_log(pdev, INFO, "queue %d - cpu %d\n", 1803 i - ENA_IO_IRQ_FIRST_IDX, irq->cpu); 1804 #endif 1805 } 1806 1807 return (rc); 1808 1809 err: 1810 1811 for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) { 1812 irq = &adapter->irq_tbl[i]; 1813 rcc = 0; 1814 1815 /* Once we entered err: section and irq->requested is true we 1816 free both intr and resources */ 1817 if (irq->requested) 1818 rcc = bus_teardown_intr(adapter->pdev, irq->res, 1819 irq->cookie); 1820 if (unlikely(rcc != 0)) 1821 ena_log(pdev, ERR, 1822 "could not release irq: %d, error: %d\n", 1823 irq->vector, rcc); 1824 1825 /* If we entered err: section without irq->requested set we know 1826 it was bus_alloc_resource_any() that needs cleanup, provided 1827 res is not NULL. In case res is NULL no work in needed in 1828 this iteration */ 1829 rcc = 0; 1830 if (irq->res != NULL) { 1831 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1832 irq->vector, irq->res); 1833 } 1834 if (unlikely(rcc != 0)) 1835 ena_log(pdev, ERR, 1836 "dev has no parent while releasing res for irq: %d\n", 1837 irq->vector); 1838 irq->requested = false; 1839 irq->res = NULL; 1840 } 1841 1842 return (rc); 1843 } 1844 1845 static void 1846 ena_free_mgmnt_irq(struct ena_adapter *adapter) 1847 { 1848 device_t pdev = adapter->pdev; 1849 struct ena_irq *irq; 1850 int rc; 1851 1852 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 1853 if (irq->requested) { 1854 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector); 1855 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); 1856 if (unlikely(rc != 0)) 1857 ena_log(pdev, ERR, "failed to tear down irq: %d\n", 1858 irq->vector); 1859 irq->requested = 0; 1860 } 1861 1862 if (irq->res != NULL) { 1863 ena_log(pdev, DBG, "release resource irq: %d\n", irq->vector); 1864 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1865 irq->vector, irq->res); 1866 irq->res = NULL; 1867 if (unlikely(rc != 0)) 1868 ena_log(pdev, ERR, 1869 "dev has no parent while releasing res for irq: %d\n", 1870 irq->vector); 1871 } 1872 } 1873 1874 static void 1875 ena_free_io_irq(struct ena_adapter *adapter) 1876 { 1877 device_t pdev = adapter->pdev; 1878 struct ena_irq *irq; 1879 int rc; 1880 1881 for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { 1882 irq = &adapter->irq_tbl[i]; 1883 if (irq->requested) { 1884 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector); 1885 rc = bus_teardown_intr(adapter->pdev, irq->res, 1886 irq->cookie); 1887 if (unlikely(rc != 0)) { 1888 ena_log(pdev, ERR, 1889 "failed to tear down irq: %d\n", 1890 irq->vector); 1891 } 1892 irq->requested = 0; 1893 } 1894 1895 if (irq->res != NULL) { 1896 ena_log(pdev, DBG, "release resource irq: %d\n", 1897 irq->vector); 1898 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1899 irq->vector, irq->res); 1900 irq->res = NULL; 1901 if (unlikely(rc != 0)) { 1902 ena_log(pdev, ERR, 1903 "dev has no parent while releasing res for irq: %d\n", 1904 irq->vector); 1905 } 1906 } 1907 } 1908 } 1909 1910 static void 1911 ena_free_irqs(struct ena_adapter *adapter) 1912 { 1913 ena_free_io_irq(adapter); 1914 ena_free_mgmnt_irq(adapter); 1915 ena_disable_msix(adapter); 1916 } 1917 1918 static void 1919 ena_disable_msix(struct ena_adapter *adapter) 1920 { 1921 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { 1922 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); 1923 pci_release_msi(adapter->pdev); 1924 } 1925 1926 adapter->msix_vecs = 0; 1927 free(adapter->msix_entries, M_DEVBUF); 1928 adapter->msix_entries = NULL; 1929 } 1930 1931 static void 1932 ena_unmask_all_io_irqs(struct ena_adapter *adapter) 1933 { 1934 struct ena_com_io_cq *io_cq; 1935 struct ena_eth_io_intr_reg intr_reg; 1936 struct ena_ring *tx_ring; 1937 uint16_t ena_qid; 1938 int i; 1939 1940 /* Unmask interrupts for all queues */ 1941 for (i = 0; i < adapter->num_io_queues; i++) { 1942 ena_qid = ENA_IO_TXQ_IDX(i); 1943 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 1944 ena_com_update_intr_reg(&intr_reg, 0, 0, true); 1945 tx_ring = &adapter->tx_ring[i]; 1946 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1); 1947 ena_com_unmask_intr(io_cq, &intr_reg); 1948 } 1949 } 1950 1951 static int 1952 ena_up_complete(struct ena_adapter *adapter) 1953 { 1954 int rc; 1955 1956 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { 1957 rc = ena_rss_configure(adapter); 1958 if (rc != 0) { 1959 ena_log(adapter->pdev, ERR, 1960 "Failed to configure RSS\n"); 1961 return (rc); 1962 } 1963 } 1964 1965 rc = ena_change_mtu(adapter->ifp, if_getmtu(adapter->ifp)); 1966 if (unlikely(rc != 0)) 1967 return (rc); 1968 1969 ena_refill_all_rx_bufs(adapter); 1970 ena_reset_counters((counter_u64_t *)&adapter->hw_stats, 1971 sizeof(adapter->hw_stats)); 1972 1973 return (0); 1974 } 1975 1976 static void 1977 set_io_rings_size(struct ena_adapter *adapter, int new_tx_size, int new_rx_size) 1978 { 1979 int i; 1980 1981 for (i = 0; i < adapter->num_io_queues; i++) { 1982 adapter->tx_ring[i].ring_size = new_tx_size; 1983 adapter->rx_ring[i].ring_size = new_rx_size; 1984 } 1985 } 1986 1987 static int 1988 create_queues_with_size_backoff(struct ena_adapter *adapter) 1989 { 1990 device_t pdev = adapter->pdev; 1991 int rc; 1992 uint32_t cur_rx_ring_size, cur_tx_ring_size; 1993 uint32_t new_rx_ring_size, new_tx_ring_size; 1994 1995 /* 1996 * Current queue sizes might be set to smaller than the requested 1997 * ones due to past queue allocation failures. 1998 */ 1999 set_io_rings_size(adapter, adapter->requested_tx_ring_size, 2000 adapter->requested_rx_ring_size); 2001 2002 while (1) { 2003 /* Allocate transmit descriptors */ 2004 rc = ena_setup_all_tx_resources(adapter); 2005 if (unlikely(rc != 0)) { 2006 ena_log(pdev, ERR, "err_setup_tx\n"); 2007 goto err_setup_tx; 2008 } 2009 2010 /* Allocate receive descriptors */ 2011 rc = ena_setup_all_rx_resources(adapter); 2012 if (unlikely(rc != 0)) { 2013 ena_log(pdev, ERR, "err_setup_rx\n"); 2014 goto err_setup_rx; 2015 } 2016 2017 /* Create IO queues for Rx & Tx */ 2018 rc = ena_create_io_queues(adapter); 2019 if (unlikely(rc != 0)) { 2020 ena_log(pdev, ERR, "create IO queues failed\n"); 2021 goto err_io_que; 2022 } 2023 2024 return (0); 2025 2026 err_io_que: 2027 ena_free_all_rx_resources(adapter); 2028 err_setup_rx: 2029 ena_free_all_tx_resources(adapter); 2030 err_setup_tx: 2031 /* 2032 * Lower the ring size if ENOMEM. Otherwise, return the 2033 * error straightaway. 2034 */ 2035 if (unlikely(rc != ENOMEM)) { 2036 ena_log(pdev, ERR, 2037 "Queue creation failed with error code: %d\n", rc); 2038 return (rc); 2039 } 2040 2041 cur_tx_ring_size = adapter->tx_ring[0].ring_size; 2042 cur_rx_ring_size = adapter->rx_ring[0].ring_size; 2043 2044 ena_log(pdev, ERR, 2045 "Not enough memory to create queues with sizes TX=%d, RX=%d\n", 2046 cur_tx_ring_size, cur_rx_ring_size); 2047 2048 new_tx_ring_size = cur_tx_ring_size; 2049 new_rx_ring_size = cur_rx_ring_size; 2050 2051 /* 2052 * Decrease the size of a larger queue, or decrease both if they 2053 * are the same size. 2054 */ 2055 if (cur_rx_ring_size <= cur_tx_ring_size) 2056 new_tx_ring_size = cur_tx_ring_size / 2; 2057 if (cur_rx_ring_size >= cur_tx_ring_size) 2058 new_rx_ring_size = cur_rx_ring_size / 2; 2059 2060 if (new_tx_ring_size < ENA_MIN_RING_SIZE || 2061 new_rx_ring_size < ENA_MIN_RING_SIZE) { 2062 ena_log(pdev, ERR, 2063 "Queue creation failed with the smallest possible queue size" 2064 "of %d for both queues. Not retrying with smaller queues\n", 2065 ENA_MIN_RING_SIZE); 2066 return (rc); 2067 } 2068 2069 ena_log(pdev, INFO, 2070 "Retrying queue creation with sizes TX=%d, RX=%d\n", 2071 new_tx_ring_size, new_rx_ring_size); 2072 2073 set_io_rings_size(adapter, new_tx_ring_size, new_rx_ring_size); 2074 } 2075 } 2076 2077 int 2078 ena_up(struct ena_adapter *adapter) 2079 { 2080 int rc = 0; 2081 2082 ENA_LOCK_ASSERT(); 2083 2084 if (unlikely(device_is_attached(adapter->pdev) == 0)) { 2085 ena_log(adapter->pdev, ERR, "device is not attached!\n"); 2086 return (ENXIO); 2087 } 2088 2089 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 2090 return (0); 2091 2092 ena_log(adapter->pdev, INFO, "device is going UP\n"); 2093 2094 /* setup interrupts for IO queues */ 2095 rc = ena_setup_io_intr(adapter); 2096 if (unlikely(rc != 0)) { 2097 ena_log(adapter->pdev, ERR, "error setting up IO interrupt\n"); 2098 goto error; 2099 } 2100 rc = ena_request_io_irq(adapter); 2101 if (unlikely(rc != 0)) { 2102 ena_log(adapter->pdev, ERR, "err_req_irq\n"); 2103 goto error; 2104 } 2105 2106 ena_log(adapter->pdev, INFO, 2107 "Creating %u IO queues. Rx queue size: %d, Tx queue size: %d, LLQ is %s\n", 2108 adapter->num_io_queues, 2109 adapter->requested_rx_ring_size, 2110 adapter->requested_tx_ring_size, 2111 (adapter->ena_dev->tx_mem_queue_type == 2112 ENA_ADMIN_PLACEMENT_POLICY_DEV) ? "ENABLED" : "DISABLED"); 2113 2114 rc = create_queues_with_size_backoff(adapter); 2115 if (unlikely(rc != 0)) { 2116 ena_log(adapter->pdev, ERR, 2117 "error creating queues with size backoff\n"); 2118 goto err_create_queues_with_backoff; 2119 } 2120 2121 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) 2122 if_link_state_change(adapter->ifp, LINK_STATE_UP); 2123 2124 rc = ena_up_complete(adapter); 2125 if (unlikely(rc != 0)) 2126 goto err_up_complete; 2127 2128 counter_u64_add(adapter->dev_stats.interface_up, 1); 2129 2130 ena_update_hwassist(adapter); 2131 2132 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 2133 2134 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter); 2135 2136 ena_unmask_all_io_irqs(adapter); 2137 2138 return (0); 2139 2140 err_up_complete: 2141 ena_destroy_all_io_queues(adapter); 2142 ena_free_all_rx_resources(adapter); 2143 ena_free_all_tx_resources(adapter); 2144 err_create_queues_with_backoff: 2145 ena_free_io_irq(adapter); 2146 error: 2147 return (rc); 2148 } 2149 2150 static uint64_t 2151 ena_get_counter(if_t ifp, ift_counter cnt) 2152 { 2153 struct ena_adapter *adapter; 2154 struct ena_hw_stats *stats; 2155 2156 adapter = if_getsoftc(ifp); 2157 stats = &adapter->hw_stats; 2158 2159 switch (cnt) { 2160 case IFCOUNTER_IPACKETS: 2161 return (counter_u64_fetch(stats->rx_packets)); 2162 case IFCOUNTER_OPACKETS: 2163 return (counter_u64_fetch(stats->tx_packets)); 2164 case IFCOUNTER_IBYTES: 2165 return (counter_u64_fetch(stats->rx_bytes)); 2166 case IFCOUNTER_OBYTES: 2167 return (counter_u64_fetch(stats->tx_bytes)); 2168 case IFCOUNTER_IQDROPS: 2169 return (counter_u64_fetch(stats->rx_drops)); 2170 case IFCOUNTER_OQDROPS: 2171 return (counter_u64_fetch(stats->tx_drops)); 2172 default: 2173 return (if_get_counter_default(ifp, cnt)); 2174 } 2175 } 2176 2177 static int 2178 ena_media_change(if_t ifp) 2179 { 2180 /* Media Change is not supported by firmware */ 2181 return (0); 2182 } 2183 2184 static void 2185 ena_media_status(if_t ifp, struct ifmediareq *ifmr) 2186 { 2187 struct ena_adapter *adapter = if_getsoftc(ifp); 2188 ena_log(adapter->pdev, DBG, "Media status update\n"); 2189 2190 ENA_LOCK_LOCK(); 2191 2192 ifmr->ifm_status = IFM_AVALID; 2193 ifmr->ifm_active = IFM_ETHER; 2194 2195 if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) { 2196 ENA_LOCK_UNLOCK(); 2197 ena_log(adapter->pdev, INFO, "Link is down\n"); 2198 return; 2199 } 2200 2201 ifmr->ifm_status |= IFM_ACTIVE; 2202 ifmr->ifm_active |= IFM_UNKNOWN | IFM_FDX; 2203 2204 ENA_LOCK_UNLOCK(); 2205 } 2206 2207 static void 2208 ena_init(void *arg) 2209 { 2210 struct ena_adapter *adapter = (struct ena_adapter *)arg; 2211 2212 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 2213 ENA_LOCK_LOCK(); 2214 ena_up(adapter); 2215 ENA_LOCK_UNLOCK(); 2216 } 2217 } 2218 2219 static int 2220 ena_ioctl(if_t ifp, u_long command, caddr_t data) 2221 { 2222 struct ena_adapter *adapter; 2223 struct ifreq *ifr; 2224 int rc; 2225 2226 adapter = if_getsoftc(ifp); 2227 ifr = (struct ifreq *)data; 2228 2229 /* 2230 * Acquiring lock to prevent from running up and down routines parallel. 2231 */ 2232 rc = 0; 2233 switch (command) { 2234 case SIOCSIFMTU: 2235 if (if_getmtu(ifp) == ifr->ifr_mtu) 2236 break; 2237 ENA_LOCK_LOCK(); 2238 ena_down(adapter); 2239 2240 ena_change_mtu(ifp, ifr->ifr_mtu); 2241 2242 rc = ena_up(adapter); 2243 ENA_LOCK_UNLOCK(); 2244 break; 2245 2246 case SIOCSIFFLAGS: 2247 if ((if_getflags(ifp) & IFF_UP) != 0) { 2248 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2249 if ((if_getflags(ifp) & (IFF_PROMISC | 2250 IFF_ALLMULTI)) != 0) { 2251 ena_log(adapter->pdev, INFO, 2252 "ioctl promisc/allmulti\n"); 2253 } 2254 } else { 2255 ENA_LOCK_LOCK(); 2256 rc = ena_up(adapter); 2257 ENA_LOCK_UNLOCK(); 2258 } 2259 } else { 2260 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2261 ENA_LOCK_LOCK(); 2262 ena_down(adapter); 2263 ENA_LOCK_UNLOCK(); 2264 } 2265 } 2266 break; 2267 2268 case SIOCADDMULTI: 2269 case SIOCDELMULTI: 2270 break; 2271 2272 case SIOCSIFMEDIA: 2273 case SIOCGIFMEDIA: 2274 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 2275 break; 2276 2277 case SIOCSIFCAP: 2278 { 2279 int reinit = 0; 2280 2281 if (ifr->ifr_reqcap != if_getcapenable(ifp)) { 2282 if_setcapenable(ifp, ifr->ifr_reqcap); 2283 reinit = 1; 2284 } 2285 2286 if ((reinit != 0) && 2287 ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)) { 2288 ENA_LOCK_LOCK(); 2289 ena_down(adapter); 2290 rc = ena_up(adapter); 2291 ENA_LOCK_UNLOCK(); 2292 } 2293 } 2294 2295 break; 2296 default: 2297 rc = ether_ioctl(ifp, command, data); 2298 break; 2299 } 2300 2301 return (rc); 2302 } 2303 2304 static int 2305 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat) 2306 { 2307 int caps = 0; 2308 2309 if ((feat->offload.tx & 2310 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2311 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK | 2312 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)) != 0) 2313 caps |= IFCAP_TXCSUM; 2314 2315 if ((feat->offload.tx & 2316 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK | 2317 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)) != 0) 2318 caps |= IFCAP_TXCSUM_IPV6; 2319 2320 if ((feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0) 2321 caps |= IFCAP_TSO4; 2322 2323 if ((feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0) 2324 caps |= IFCAP_TSO6; 2325 2326 if ((feat->offload.rx_supported & 2327 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK | 2328 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)) != 0) 2329 caps |= IFCAP_RXCSUM; 2330 2331 if ((feat->offload.rx_supported & 2332 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) != 0) 2333 caps |= IFCAP_RXCSUM_IPV6; 2334 2335 caps |= IFCAP_LRO | IFCAP_JUMBO_MTU; 2336 2337 return (caps); 2338 } 2339 2340 static void 2341 ena_update_host_info(struct ena_admin_host_info *host_info, if_t ifp) 2342 { 2343 host_info->supported_network_features[0] = (uint32_t)if_getcapabilities(ifp); 2344 } 2345 2346 static void 2347 ena_update_hwassist(struct ena_adapter *adapter) 2348 { 2349 if_t ifp = adapter->ifp; 2350 uint32_t feat = adapter->tx_offload_cap; 2351 int cap = if_getcapenable(ifp); 2352 int flags = 0; 2353 2354 if_clearhwassist(ifp); 2355 2356 if ((cap & IFCAP_TXCSUM) != 0) { 2357 if ((feat & 2358 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK) != 0) 2359 flags |= CSUM_IP; 2360 if ((feat & 2361 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2362 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)) != 0) 2363 flags |= CSUM_IP_UDP | CSUM_IP_TCP; 2364 } 2365 2366 if ((cap & IFCAP_TXCSUM_IPV6) != 0) 2367 flags |= CSUM_IP6_UDP | CSUM_IP6_TCP; 2368 2369 if ((cap & IFCAP_TSO4) != 0) 2370 flags |= CSUM_IP_TSO; 2371 2372 if ((cap & IFCAP_TSO6) != 0) 2373 flags |= CSUM_IP6_TSO; 2374 2375 if_sethwassistbits(ifp, flags, 0); 2376 } 2377 2378 static int 2379 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter, 2380 struct ena_com_dev_get_features_ctx *feat) 2381 { 2382 if_t ifp; 2383 int caps = 0; 2384 2385 ifp = adapter->ifp = if_gethandle(IFT_ETHER); 2386 if (unlikely(ifp == NULL)) { 2387 ena_log(pdev, ERR, "can not allocate ifnet structure\n"); 2388 return (ENXIO); 2389 } 2390 if_initname(ifp, device_get_name(pdev), device_get_unit(pdev)); 2391 if_setdev(ifp, pdev); 2392 if_setsoftc(ifp, adapter); 2393 2394 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 2395 if_setinitfn(ifp, ena_init); 2396 if_settransmitfn(ifp, ena_mq_start); 2397 if_setqflushfn(ifp, ena_qflush); 2398 if_setioctlfn(ifp, ena_ioctl); 2399 if_setgetcounterfn(ifp, ena_get_counter); 2400 2401 if_setsendqlen(ifp, adapter->requested_tx_ring_size); 2402 if_setsendqready(ifp); 2403 if_setmtu(ifp, ETHERMTU); 2404 if_setbaudrate(ifp, 0); 2405 /* Zeroize capabilities... */ 2406 if_setcapabilities(ifp, 0); 2407 if_setcapenable(ifp, 0); 2408 /* check hardware support */ 2409 caps = ena_get_dev_offloads(feat); 2410 /* ... and set them */ 2411 if_setcapabilitiesbit(ifp, caps, 0); 2412 2413 /* TSO parameters */ 2414 if_sethwtsomax(ifp, ENA_TSO_MAXSIZE - 2415 (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)); 2416 if_sethwtsomaxsegcount(ifp, adapter->max_tx_sgl_size - 1); 2417 if_sethwtsomaxsegsize(ifp, ENA_TSO_MAXSIZE); 2418 2419 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 2420 if_setcapenable(ifp, if_getcapabilities(ifp)); 2421 2422 /* 2423 * Specify the media types supported by this adapter and register 2424 * callbacks to update media and link information 2425 */ 2426 ifmedia_init(&adapter->media, IFM_IMASK, ena_media_change, 2427 ena_media_status); 2428 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2429 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2430 2431 ether_ifattach(ifp, adapter->mac_addr); 2432 2433 return (0); 2434 } 2435 2436 void 2437 ena_down(struct ena_adapter *adapter) 2438 { 2439 int rc; 2440 2441 ENA_LOCK_ASSERT(); 2442 2443 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 2444 return; 2445 2446 ena_log(adapter->pdev, INFO, "device is going DOWN\n"); 2447 2448 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter); 2449 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 2450 2451 ena_free_io_irq(adapter); 2452 2453 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) { 2454 rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); 2455 if (unlikely(rc != 0)) 2456 ena_log(adapter->pdev, ERR, "Device reset failed\n"); 2457 } 2458 2459 ena_destroy_all_io_queues(adapter); 2460 2461 ena_free_all_tx_bufs(adapter); 2462 ena_free_all_rx_bufs(adapter); 2463 ena_free_all_tx_resources(adapter); 2464 ena_free_all_rx_resources(adapter); 2465 2466 counter_u64_add(adapter->dev_stats.interface_down, 1); 2467 } 2468 2469 static uint32_t 2470 ena_calc_max_io_queue_num(device_t pdev, struct ena_com_dev *ena_dev, 2471 struct ena_com_dev_get_features_ctx *get_feat_ctx) 2472 { 2473 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues; 2474 2475 /* Regular queues capabilities */ 2476 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 2477 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 2478 &get_feat_ctx->max_queue_ext.max_queue_ext; 2479 io_rx_num = min_t(int, max_queue_ext->max_rx_sq_num, 2480 max_queue_ext->max_rx_cq_num); 2481 2482 io_tx_sq_num = max_queue_ext->max_tx_sq_num; 2483 io_tx_cq_num = max_queue_ext->max_tx_cq_num; 2484 } else { 2485 struct ena_admin_queue_feature_desc *max_queues = 2486 &get_feat_ctx->max_queues; 2487 io_tx_sq_num = max_queues->max_sq_num; 2488 io_tx_cq_num = max_queues->max_cq_num; 2489 io_rx_num = min_t(int, io_tx_sq_num, io_tx_cq_num); 2490 } 2491 2492 /* In case of LLQ use the llq fields for the tx SQ/CQ */ 2493 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 2494 io_tx_sq_num = get_feat_ctx->llq.max_llq_num; 2495 2496 max_num_io_queues = min_t(uint32_t, mp_ncpus, ENA_MAX_NUM_IO_QUEUES); 2497 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_rx_num); 2498 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_sq_num); 2499 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_cq_num); 2500 /* 1 IRQ for mgmnt and 1 IRQ for each TX/RX pair */ 2501 max_num_io_queues = min_t(uint32_t, max_num_io_queues, 2502 pci_msix_count(pdev) - 1); 2503 #ifdef RSS 2504 max_num_io_queues = min_t(uint32_t, max_num_io_queues, 2505 rss_getnumbuckets()); 2506 #endif 2507 2508 return (max_num_io_queues); 2509 } 2510 2511 static int 2512 ena_enable_wc(device_t pdev, struct resource *res) 2513 { 2514 #if defined(__i386) || defined(__amd64) || defined(__aarch64__) 2515 vm_offset_t va; 2516 vm_size_t len; 2517 int rc; 2518 2519 va = (vm_offset_t)rman_get_virtual(res); 2520 len = rman_get_size(res); 2521 /* Enable write combining */ 2522 rc = pmap_change_attr(va, len, VM_MEMATTR_WRITE_COMBINING); 2523 if (unlikely(rc != 0)) { 2524 ena_log(pdev, ERR, "pmap_change_attr failed, %d\n", rc); 2525 return (rc); 2526 } 2527 2528 return (0); 2529 #endif 2530 return (EOPNOTSUPP); 2531 } 2532 2533 static int 2534 ena_set_queues_placement_policy(device_t pdev, struct ena_com_dev *ena_dev, 2535 struct ena_admin_feature_llq_desc *llq, 2536 struct ena_llq_configurations *llq_default_configurations) 2537 { 2538 int rc; 2539 uint32_t llq_feature_mask; 2540 2541 llq_feature_mask = 1 << ENA_ADMIN_LLQ; 2542 if (!(ena_dev->supported_features & llq_feature_mask)) { 2543 ena_log(pdev, WARN, 2544 "LLQ is not supported. Fallback to host mode policy.\n"); 2545 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2546 return (0); 2547 } 2548 2549 if (ena_dev->mem_bar == NULL) { 2550 ena_log(pdev, WARN, 2551 "LLQ is advertised as supported but device doesn't expose mem bar.\n"); 2552 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2553 return (0); 2554 } 2555 2556 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); 2557 if (unlikely(rc != 0)) { 2558 ena_log(pdev, WARN, 2559 "Failed to configure the device mode. " 2560 "Fallback to host mode policy.\n"); 2561 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2562 } 2563 2564 return (0); 2565 } 2566 2567 static int 2568 ena_map_llq_mem_bar(device_t pdev, struct ena_com_dev *ena_dev) 2569 { 2570 struct ena_adapter *adapter = device_get_softc(pdev); 2571 int rc, rid; 2572 2573 /* Try to allocate resources for LLQ bar */ 2574 rid = PCIR_BAR(ENA_MEM_BAR); 2575 adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid, 2576 RF_ACTIVE); 2577 if (unlikely(adapter->memory == NULL)) { 2578 ena_log(pdev, WARN, 2579 "Unable to allocate LLQ bar resource. LLQ mode won't be used.\n"); 2580 return (0); 2581 } 2582 2583 /* Enable write combining for better LLQ performance */ 2584 rc = ena_enable_wc(adapter->pdev, adapter->memory); 2585 if (unlikely(rc != 0)) { 2586 ena_log(pdev, ERR, "failed to enable write combining.\n"); 2587 return (rc); 2588 } 2589 2590 /* 2591 * Save virtual address of the device's memory region 2592 * for the ena_com layer. 2593 */ 2594 ena_dev->mem_bar = rman_get_virtual(adapter->memory); 2595 2596 return (0); 2597 } 2598 2599 static inline void 2600 set_default_llq_configurations(struct ena_llq_configurations *llq_config, 2601 struct ena_admin_feature_llq_desc *llq) 2602 { 2603 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; 2604 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; 2605 llq_config->llq_num_decs_before_header = 2606 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; 2607 if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) != 2608 0 && ena_force_large_llq_header) { 2609 llq_config->llq_ring_entry_size = 2610 ENA_ADMIN_LIST_ENTRY_SIZE_256B; 2611 llq_config->llq_ring_entry_size_value = 256; 2612 } else { 2613 llq_config->llq_ring_entry_size = 2614 ENA_ADMIN_LIST_ENTRY_SIZE_128B; 2615 llq_config->llq_ring_entry_size_value = 128; 2616 } 2617 } 2618 2619 static int 2620 ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx) 2621 { 2622 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq; 2623 struct ena_com_dev *ena_dev = ctx->ena_dev; 2624 uint32_t tx_queue_size = ENA_DEFAULT_RING_SIZE; 2625 uint32_t rx_queue_size = ENA_DEFAULT_RING_SIZE; 2626 uint32_t max_tx_queue_size; 2627 uint32_t max_rx_queue_size; 2628 2629 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 2630 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 2631 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext; 2632 max_rx_queue_size = min_t(uint32_t, 2633 max_queue_ext->max_rx_cq_depth, 2634 max_queue_ext->max_rx_sq_depth); 2635 max_tx_queue_size = max_queue_ext->max_tx_cq_depth; 2636 2637 if (ena_dev->tx_mem_queue_type == 2638 ENA_ADMIN_PLACEMENT_POLICY_DEV) 2639 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2640 llq->max_llq_depth); 2641 else 2642 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2643 max_queue_ext->max_tx_sq_depth); 2644 2645 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2646 max_queue_ext->max_per_packet_tx_descs); 2647 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2648 max_queue_ext->max_per_packet_rx_descs); 2649 } else { 2650 struct ena_admin_queue_feature_desc *max_queues = 2651 &ctx->get_feat_ctx->max_queues; 2652 max_rx_queue_size = min_t(uint32_t, max_queues->max_cq_depth, 2653 max_queues->max_sq_depth); 2654 max_tx_queue_size = max_queues->max_cq_depth; 2655 2656 if (ena_dev->tx_mem_queue_type == 2657 ENA_ADMIN_PLACEMENT_POLICY_DEV) 2658 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2659 llq->max_llq_depth); 2660 else 2661 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2662 max_queues->max_sq_depth); 2663 2664 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2665 max_queues->max_packet_tx_descs); 2666 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2667 max_queues->max_packet_rx_descs); 2668 } 2669 2670 /* round down to the nearest power of 2 */ 2671 max_tx_queue_size = 1 << (flsl(max_tx_queue_size) - 1); 2672 max_rx_queue_size = 1 << (flsl(max_rx_queue_size) - 1); 2673 2674 /* 2675 * When forcing large headers, we multiply the entry size by 2, 2676 * and therefore divide the queue size by 2, leaving the amount 2677 * of memory used by the queues unchanged. 2678 */ 2679 if (ena_force_large_llq_header) { 2680 if ((llq->entry_size_ctrl_supported & 2681 ENA_ADMIN_LIST_ENTRY_SIZE_256B) != 0 && 2682 ena_dev->tx_mem_queue_type == 2683 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 2684 max_tx_queue_size /= 2; 2685 ena_log(ctx->pdev, INFO, 2686 "Forcing large headers and decreasing maximum Tx queue size to %d\n", 2687 max_tx_queue_size); 2688 } else { 2689 ena_log(ctx->pdev, WARN, 2690 "Forcing large headers failed: LLQ is disabled or device does not support large headers\n"); 2691 } 2692 } 2693 2694 tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE, 2695 max_tx_queue_size); 2696 rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE, 2697 max_rx_queue_size); 2698 2699 tx_queue_size = 1 << (flsl(tx_queue_size) - 1); 2700 rx_queue_size = 1 << (flsl(rx_queue_size) - 1); 2701 2702 ctx->max_tx_queue_size = max_tx_queue_size; 2703 ctx->max_rx_queue_size = max_rx_queue_size; 2704 ctx->tx_queue_size = tx_queue_size; 2705 ctx->rx_queue_size = rx_queue_size; 2706 2707 return (0); 2708 } 2709 2710 static void 2711 ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev) 2712 { 2713 struct ena_admin_host_info *host_info; 2714 uintptr_t rid; 2715 int rc; 2716 2717 /* Allocate only the host info */ 2718 rc = ena_com_allocate_host_info(ena_dev); 2719 if (unlikely(rc != 0)) { 2720 ena_log(dev, ERR, "Cannot allocate host info\n"); 2721 return; 2722 } 2723 2724 host_info = ena_dev->host_attr.host_info; 2725 2726 if (pci_get_id(dev, PCI_ID_RID, &rid) == 0) 2727 host_info->bdf = rid; 2728 host_info->os_type = ENA_ADMIN_OS_FREEBSD; 2729 host_info->kernel_ver = osreldate; 2730 2731 sprintf(host_info->kernel_ver_str, "%d", osreldate); 2732 host_info->os_dist = 0; 2733 strncpy(host_info->os_dist_str, osrelease, 2734 sizeof(host_info->os_dist_str) - 1); 2735 2736 host_info->driver_version = (ENA_DRV_MODULE_VER_MAJOR) | 2737 (ENA_DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 2738 (ENA_DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 2739 host_info->num_cpus = mp_ncpus; 2740 host_info->driver_supported_features = 2741 ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK | 2742 ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK; 2743 2744 rc = ena_com_set_host_attributes(ena_dev); 2745 if (unlikely(rc != 0)) { 2746 if (rc == EOPNOTSUPP) 2747 ena_log(dev, WARN, "Cannot set host attributes\n"); 2748 else 2749 ena_log(dev, ERR, "Cannot set host attributes\n"); 2750 2751 goto err; 2752 } 2753 2754 return; 2755 2756 err: 2757 ena_com_delete_host_info(ena_dev); 2758 } 2759 2760 static int 2761 ena_device_init(struct ena_adapter *adapter, device_t pdev, 2762 struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active) 2763 { 2764 struct ena_llq_configurations llq_config; 2765 struct ena_com_dev *ena_dev = adapter->ena_dev; 2766 bool readless_supported; 2767 uint32_t aenq_groups; 2768 int dma_width; 2769 int rc; 2770 2771 rc = ena_com_mmio_reg_read_request_init(ena_dev); 2772 if (unlikely(rc != 0)) { 2773 ena_log(pdev, ERR, "failed to init mmio read less\n"); 2774 return (rc); 2775 } 2776 2777 /* 2778 * The PCIe configuration space revision id indicate if mmio reg 2779 * read is disabled 2780 */ 2781 readless_supported = !(pci_get_revid(pdev) & ENA_MMIO_DISABLE_REG_READ); 2782 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 2783 2784 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 2785 if (unlikely(rc != 0)) { 2786 ena_log(pdev, ERR, "Can not reset device\n"); 2787 goto err_mmio_read_less; 2788 } 2789 2790 rc = ena_com_validate_version(ena_dev); 2791 if (unlikely(rc != 0)) { 2792 ena_log(pdev, ERR, "device version is too low\n"); 2793 goto err_mmio_read_less; 2794 } 2795 2796 dma_width = ena_com_get_dma_width(ena_dev); 2797 if (unlikely(dma_width < 0)) { 2798 ena_log(pdev, ERR, "Invalid dma width value %d", dma_width); 2799 rc = dma_width; 2800 goto err_mmio_read_less; 2801 } 2802 adapter->dma_width = dma_width; 2803 2804 /* ENA admin level init */ 2805 rc = ena_com_admin_init(ena_dev, &aenq_handlers); 2806 if (unlikely(rc != 0)) { 2807 ena_log(pdev, ERR, 2808 "Can not initialize ena admin queue with device\n"); 2809 goto err_mmio_read_less; 2810 } 2811 2812 /* 2813 * To enable the msix interrupts the driver needs to know the number 2814 * of queues. So the driver uses polling mode to retrieve this 2815 * information 2816 */ 2817 ena_com_set_admin_polling_mode(ena_dev, true); 2818 2819 ena_config_host_info(ena_dev, pdev); 2820 2821 /* Get Device Attributes */ 2822 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 2823 if (unlikely(rc != 0)) { 2824 ena_log(pdev, ERR, 2825 "Cannot get attribute for ena device rc: %d\n", rc); 2826 goto err_admin_init; 2827 } 2828 2829 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 2830 BIT(ENA_ADMIN_FATAL_ERROR) | 2831 BIT(ENA_ADMIN_WARNING) | 2832 BIT(ENA_ADMIN_NOTIFICATION) | 2833 BIT(ENA_ADMIN_KEEP_ALIVE); 2834 2835 aenq_groups &= get_feat_ctx->aenq.supported_groups; 2836 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 2837 if (unlikely(rc != 0)) { 2838 ena_log(pdev, ERR, "Cannot configure aenq groups rc: %d\n", rc); 2839 goto err_admin_init; 2840 } 2841 2842 *wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 2843 2844 set_default_llq_configurations(&llq_config, &get_feat_ctx->llq); 2845 2846 rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq, 2847 &llq_config); 2848 if (unlikely(rc != 0)) { 2849 ena_log(pdev, ERR, "Failed to set placement policy\n"); 2850 goto err_admin_init; 2851 } 2852 2853 return (0); 2854 2855 err_admin_init: 2856 ena_com_delete_host_info(ena_dev); 2857 ena_com_admin_destroy(ena_dev); 2858 err_mmio_read_less: 2859 ena_com_mmio_reg_read_request_destroy(ena_dev); 2860 2861 return (rc); 2862 } 2863 2864 static int 2865 ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter) 2866 { 2867 struct ena_com_dev *ena_dev = adapter->ena_dev; 2868 int rc; 2869 2870 rc = ena_enable_msix(adapter); 2871 if (unlikely(rc != 0)) { 2872 ena_log(adapter->pdev, ERR, "Error with MSI-X enablement\n"); 2873 return (rc); 2874 } 2875 2876 ena_setup_mgmnt_intr(adapter); 2877 2878 rc = ena_request_mgmnt_irq(adapter); 2879 if (unlikely(rc != 0)) { 2880 ena_log(adapter->pdev, ERR, "Cannot setup mgmnt queue intr\n"); 2881 goto err_disable_msix; 2882 } 2883 2884 ena_com_set_admin_polling_mode(ena_dev, false); 2885 2886 ena_com_admin_aenq_enable(ena_dev); 2887 2888 return (0); 2889 2890 err_disable_msix: 2891 ena_disable_msix(adapter); 2892 2893 return (rc); 2894 } 2895 2896 /* Function called on ENA_ADMIN_KEEP_ALIVE event */ 2897 static void 2898 ena_keep_alive_wd(void *adapter_data, struct ena_admin_aenq_entry *aenq_e) 2899 { 2900 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 2901 struct ena_admin_aenq_keep_alive_desc *desc; 2902 sbintime_t stime; 2903 uint64_t rx_drops; 2904 uint64_t tx_drops; 2905 2906 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; 2907 2908 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; 2909 tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low; 2910 counter_u64_zero(adapter->hw_stats.rx_drops); 2911 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops); 2912 counter_u64_zero(adapter->hw_stats.tx_drops); 2913 counter_u64_add(adapter->hw_stats.tx_drops, tx_drops); 2914 2915 stime = getsbinuptime(); 2916 atomic_store_rel_64(&adapter->keep_alive_timestamp, stime); 2917 } 2918 2919 /* Check for keep alive expiration */ 2920 static void 2921 check_for_missing_keep_alive(struct ena_adapter *adapter) 2922 { 2923 sbintime_t timestamp, time; 2924 2925 if (adapter->wd_active == 0) 2926 return; 2927 2928 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2929 return; 2930 2931 timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp); 2932 time = getsbinuptime() - timestamp; 2933 if (unlikely(time > adapter->keep_alive_timeout)) { 2934 ena_log(adapter->pdev, ERR, "Keep alive watchdog timeout.\n"); 2935 counter_u64_add(adapter->dev_stats.wd_expired, 1); 2936 ena_trigger_reset(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO); 2937 } 2938 } 2939 2940 /* Check if admin queue is enabled */ 2941 static void 2942 check_for_admin_com_state(struct ena_adapter *adapter) 2943 { 2944 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == false)) { 2945 ena_log(adapter->pdev, ERR, 2946 "ENA admin queue is not in running state!\n"); 2947 counter_u64_add(adapter->dev_stats.admin_q_pause, 1); 2948 ena_trigger_reset(adapter, ENA_REGS_RESET_ADMIN_TO); 2949 } 2950 } 2951 2952 static int 2953 check_for_rx_interrupt_queue(struct ena_adapter *adapter, 2954 struct ena_ring *rx_ring) 2955 { 2956 if (likely(atomic_load_8(&rx_ring->first_interrupt))) 2957 return (0); 2958 2959 if (ena_com_cq_empty(rx_ring->ena_com_io_cq)) 2960 return (0); 2961 2962 rx_ring->no_interrupt_event_cnt++; 2963 2964 if (rx_ring->no_interrupt_event_cnt == 2965 ENA_MAX_NO_INTERRUPT_ITERATIONS) { 2966 ena_log(adapter->pdev, ERR, 2967 "Potential MSIX issue on Rx side Queue = %d. Reset the device\n", 2968 rx_ring->qid); 2969 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_INTERRUPT); 2970 return (EIO); 2971 } 2972 2973 return (0); 2974 } 2975 2976 static int 2977 check_missing_comp_in_tx_queue(struct ena_adapter *adapter, 2978 struct ena_ring *tx_ring) 2979 { 2980 device_t pdev = adapter->pdev; 2981 struct bintime curtime, time; 2982 struct ena_tx_buffer *tx_buf; 2983 int time_since_last_cleanup; 2984 int missing_tx_comp_to; 2985 sbintime_t time_offset; 2986 uint32_t missed_tx = 0; 2987 int i, rc = 0; 2988 2989 getbinuptime(&curtime); 2990 2991 for (i = 0; i < tx_ring->ring_size; i++) { 2992 tx_buf = &tx_ring->tx_buffer_info[i]; 2993 2994 if (bintime_isset(&tx_buf->timestamp) == 0) 2995 continue; 2996 2997 time = curtime; 2998 bintime_sub(&time, &tx_buf->timestamp); 2999 time_offset = bttosbt(time); 3000 3001 if (unlikely(!atomic_load_8(&tx_ring->first_interrupt) && 3002 time_offset > 2 * adapter->missing_tx_timeout)) { 3003 /* 3004 * If after graceful period interrupt is still not 3005 * received, we schedule a reset. 3006 */ 3007 ena_log(pdev, ERR, 3008 "Potential MSIX issue on Tx side Queue = %d. " 3009 "Reset the device\n", 3010 tx_ring->qid); 3011 ena_trigger_reset(adapter, 3012 ENA_REGS_RESET_MISS_INTERRUPT); 3013 return (EIO); 3014 } 3015 3016 /* Check again if packet is still waiting */ 3017 if (unlikely(time_offset > adapter->missing_tx_timeout)) { 3018 3019 if (tx_buf->print_once) { 3020 time_since_last_cleanup = TICKS_2_USEC(ticks - 3021 tx_ring->tx_last_cleanup_ticks); 3022 missing_tx_comp_to = sbttoms( 3023 adapter->missing_tx_timeout); 3024 ena_log(pdev, WARN, 3025 "Found a Tx that wasn't completed on time, qid %d, index %d. " 3026 "%d usecs have passed since last cleanup. Missing Tx timeout value %d msecs.\n", 3027 tx_ring->qid, i, time_since_last_cleanup, 3028 missing_tx_comp_to); 3029 } 3030 3031 tx_buf->print_once = false; 3032 missed_tx++; 3033 } 3034 } 3035 3036 if (unlikely(missed_tx > adapter->missing_tx_threshold)) { 3037 ena_log(pdev, ERR, 3038 "The number of lost tx completion is above the threshold " 3039 "(%d > %d). Reset the device\n", 3040 missed_tx, adapter->missing_tx_threshold); 3041 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_TX_CMPL); 3042 rc = EIO; 3043 } 3044 3045 counter_u64_add(tx_ring->tx_stats.missing_tx_comp, missed_tx); 3046 3047 return (rc); 3048 } 3049 3050 /* 3051 * Check for TX which were not completed on time. 3052 * Timeout is defined by "missing_tx_timeout". 3053 * Reset will be performed if number of incompleted 3054 * transactions exceeds "missing_tx_threshold". 3055 */ 3056 static void 3057 check_for_missing_completions(struct ena_adapter *adapter) 3058 { 3059 struct ena_ring *tx_ring; 3060 struct ena_ring *rx_ring; 3061 int i, budget, rc; 3062 3063 /* Make sure the driver doesn't turn the device in other process */ 3064 rmb(); 3065 3066 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3067 return; 3068 3069 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 3070 return; 3071 3072 if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT) 3073 return; 3074 3075 budget = adapter->missing_tx_max_queues; 3076 3077 for (i = adapter->next_monitored_tx_qid; i < adapter->num_io_queues; i++) { 3078 tx_ring = &adapter->tx_ring[i]; 3079 rx_ring = &adapter->rx_ring[i]; 3080 3081 rc = check_missing_comp_in_tx_queue(adapter, tx_ring); 3082 if (unlikely(rc != 0)) 3083 return; 3084 3085 rc = check_for_rx_interrupt_queue(adapter, rx_ring); 3086 if (unlikely(rc != 0)) 3087 return; 3088 3089 budget--; 3090 if (budget == 0) { 3091 i++; 3092 break; 3093 } 3094 } 3095 3096 adapter->next_monitored_tx_qid = i % adapter->num_io_queues; 3097 } 3098 3099 /* trigger rx cleanup after 2 consecutive detections */ 3100 #define EMPTY_RX_REFILL 2 3101 /* For the rare case where the device runs out of Rx descriptors and the 3102 * msix handler failed to refill new Rx descriptors (due to a lack of memory 3103 * for example). 3104 * This case will lead to a deadlock: 3105 * The device won't send interrupts since all the new Rx packets will be dropped 3106 * The msix handler won't allocate new Rx descriptors so the device won't be 3107 * able to send new packets. 3108 * 3109 * When such a situation is detected - execute rx cleanup task in another thread 3110 */ 3111 static void 3112 check_for_empty_rx_ring(struct ena_adapter *adapter) 3113 { 3114 struct ena_ring *rx_ring; 3115 int i, refill_required; 3116 3117 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3118 return; 3119 3120 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 3121 return; 3122 3123 for (i = 0; i < adapter->num_io_queues; i++) { 3124 rx_ring = &adapter->rx_ring[i]; 3125 3126 refill_required = ena_com_free_q_entries( 3127 rx_ring->ena_com_io_sq); 3128 if (unlikely(refill_required == (rx_ring->ring_size - 1))) { 3129 rx_ring->empty_rx_queue++; 3130 3131 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) { 3132 counter_u64_add(rx_ring->rx_stats.empty_rx_ring, 3133 1); 3134 3135 ena_log(adapter->pdev, WARN, 3136 "Rx ring %d is stalled. Triggering the refill function\n", 3137 i); 3138 3139 taskqueue_enqueue(rx_ring->que->cleanup_tq, 3140 &rx_ring->que->cleanup_task); 3141 rx_ring->empty_rx_queue = 0; 3142 } 3143 } else { 3144 rx_ring->empty_rx_queue = 0; 3145 } 3146 } 3147 } 3148 3149 static void 3150 ena_update_hints(struct ena_adapter *adapter, 3151 struct ena_admin_ena_hw_hints *hints) 3152 { 3153 struct ena_com_dev *ena_dev = adapter->ena_dev; 3154 3155 if (hints->admin_completion_tx_timeout) 3156 ena_dev->admin_queue.completion_timeout = 3157 hints->admin_completion_tx_timeout * 1000; 3158 3159 if (hints->mmio_read_timeout) 3160 /* convert to usec */ 3161 ena_dev->mmio_read.reg_read_to = hints->mmio_read_timeout * 1000; 3162 3163 if (hints->missed_tx_completion_count_threshold_to_reset) 3164 adapter->missing_tx_threshold = 3165 hints->missed_tx_completion_count_threshold_to_reset; 3166 3167 if (hints->missing_tx_completion_timeout) { 3168 if (hints->missing_tx_completion_timeout == 3169 ENA_HW_HINTS_NO_TIMEOUT) 3170 adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT; 3171 else 3172 adapter->missing_tx_timeout = SBT_1MS * 3173 hints->missing_tx_completion_timeout; 3174 } 3175 3176 if (hints->driver_watchdog_timeout) { 3177 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 3178 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 3179 else 3180 adapter->keep_alive_timeout = SBT_1MS * 3181 hints->driver_watchdog_timeout; 3182 } 3183 } 3184 3185 /** 3186 * ena_copy_eni_metrics - Get and copy ENI metrics from the HW. 3187 * @adapter: ENA device adapter 3188 * 3189 * Returns 0 on success, EOPNOTSUPP if current HW doesn't support those metrics 3190 * and other error codes on failure. 3191 * 3192 * This function can possibly cause a race with other calls to the admin queue. 3193 * Because of that, the caller should either lock this function or make sure 3194 * that there is no race in the current context. 3195 */ 3196 static int 3197 ena_copy_eni_metrics(struct ena_adapter *adapter) 3198 { 3199 static bool print_once = true; 3200 int rc; 3201 3202 rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_metrics); 3203 3204 if (rc != 0) { 3205 if (rc == ENA_COM_UNSUPPORTED) { 3206 if (print_once) { 3207 ena_log(adapter->pdev, WARN, 3208 "Retrieving ENI metrics is not supported.\n"); 3209 print_once = false; 3210 } else { 3211 ena_log(adapter->pdev, DBG, 3212 "Retrieving ENI metrics is not supported.\n"); 3213 } 3214 } else { 3215 ena_log(adapter->pdev, ERR, 3216 "Failed to get ENI metrics: %d\n", rc); 3217 } 3218 } 3219 3220 return (rc); 3221 } 3222 3223 static void 3224 ena_timer_service(void *data) 3225 { 3226 struct ena_adapter *adapter = (struct ena_adapter *)data; 3227 struct ena_admin_host_info *host_info = 3228 adapter->ena_dev->host_attr.host_info; 3229 3230 check_for_missing_keep_alive(adapter); 3231 3232 check_for_admin_com_state(adapter); 3233 3234 check_for_missing_completions(adapter); 3235 3236 check_for_empty_rx_ring(adapter); 3237 3238 /* 3239 * User controller update of the ENI metrics. 3240 * If the delay was set to 0, then the stats shouldn't be updated at 3241 * all. 3242 * Otherwise, wait 'eni_metrics_sample_interval' seconds, before 3243 * updating stats. 3244 * As timer service is executed every second, it's enough to increment 3245 * appropriate counter each time the timer service is executed. 3246 */ 3247 if ((adapter->eni_metrics_sample_interval != 0) && 3248 (++adapter->eni_metrics_sample_interval_cnt >= 3249 adapter->eni_metrics_sample_interval)) { 3250 taskqueue_enqueue(adapter->metrics_tq, &adapter->metrics_task); 3251 adapter->eni_metrics_sample_interval_cnt = 0; 3252 } 3253 3254 3255 if (host_info != NULL) 3256 ena_update_host_info(host_info, adapter->ifp); 3257 3258 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 3259 /* 3260 * Timeout when validating version indicates that the device 3261 * became unresponsive. If that happens skip the reset and 3262 * reschedule timer service, so the reset can be retried later. 3263 */ 3264 if (ena_com_validate_version(adapter->ena_dev) == 3265 ENA_COM_TIMER_EXPIRED) { 3266 ena_log(adapter->pdev, WARN, 3267 "FW unresponsive, skipping reset\n"); 3268 ENA_TIMER_RESET(adapter); 3269 return; 3270 } 3271 ena_log(adapter->pdev, WARN, "Trigger reset is on\n"); 3272 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task); 3273 return; 3274 } 3275 3276 /* 3277 * Schedule another timeout one second from now. 3278 */ 3279 ENA_TIMER_RESET(adapter); 3280 } 3281 3282 void 3283 ena_destroy_device(struct ena_adapter *adapter, bool graceful) 3284 { 3285 if_t ifp = adapter->ifp; 3286 struct ena_com_dev *ena_dev = adapter->ena_dev; 3287 bool dev_up; 3288 3289 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) 3290 return; 3291 3292 if (!graceful) 3293 if_link_state_change(ifp, LINK_STATE_DOWN); 3294 3295 ENA_TIMER_DRAIN(adapter); 3296 3297 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 3298 if (dev_up) 3299 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 3300 3301 if (!graceful) 3302 ena_com_set_admin_running_state(ena_dev, false); 3303 3304 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3305 ena_down(adapter); 3306 3307 /* 3308 * Stop the device from sending AENQ events (if the device was up, and 3309 * the trigger reset was on, ena_down already performs device reset) 3310 */ 3311 if (!(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter) && dev_up)) 3312 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); 3313 3314 ena_free_mgmnt_irq(adapter); 3315 3316 ena_disable_msix(adapter); 3317 3318 /* 3319 * IO rings resources should be freed because `ena_restore_device()` 3320 * calls (not directly) `ena_enable_msix()`, which re-allocates MSIX 3321 * vectors. The amount of MSIX vectors after destroy-restore may be 3322 * different than before. Therefore, IO rings resources should be 3323 * established from scratch each time. 3324 */ 3325 ena_free_all_io_rings_resources(adapter); 3326 3327 ena_com_abort_admin_commands(ena_dev); 3328 3329 ena_com_wait_for_abort_completion(ena_dev); 3330 3331 ena_com_admin_destroy(ena_dev); 3332 3333 ena_com_mmio_reg_read_request_destroy(ena_dev); 3334 3335 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3336 3337 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 3338 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3339 } 3340 3341 static int 3342 ena_device_validate_params(struct ena_adapter *adapter, 3343 struct ena_com_dev_get_features_ctx *get_feat_ctx) 3344 { 3345 if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr, 3346 ETHER_ADDR_LEN) != 0) { 3347 ena_log(adapter->pdev, ERR, "Error, mac addresses differ\n"); 3348 return (EINVAL); 3349 } 3350 3351 if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) { 3352 ena_log(adapter->pdev, ERR, 3353 "Error, device max mtu is smaller than ifp MTU\n"); 3354 return (EINVAL); 3355 } 3356 3357 return 0; 3358 } 3359 3360 int 3361 ena_restore_device(struct ena_adapter *adapter) 3362 { 3363 struct ena_com_dev_get_features_ctx get_feat_ctx; 3364 struct ena_com_dev *ena_dev = adapter->ena_dev; 3365 if_t ifp = adapter->ifp; 3366 device_t dev = adapter->pdev; 3367 int wd_active; 3368 int rc; 3369 3370 ENA_FLAG_SET_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3371 3372 rc = ena_device_init(adapter, dev, &get_feat_ctx, &wd_active); 3373 if (rc != 0) { 3374 ena_log(dev, ERR, "Cannot initialize device\n"); 3375 goto err; 3376 } 3377 /* 3378 * Only enable WD if it was enabled before reset, so it won't override 3379 * value set by the user by the sysctl. 3380 */ 3381 if (adapter->wd_active != 0) 3382 adapter->wd_active = wd_active; 3383 3384 rc = ena_device_validate_params(adapter, &get_feat_ctx); 3385 if (rc != 0) { 3386 ena_log(dev, ERR, "Validation of device parameters failed\n"); 3387 goto err_device_destroy; 3388 } 3389 3390 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3391 /* Make sure we don't have a race with AENQ Links state handler */ 3392 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) 3393 if_link_state_change(ifp, LINK_STATE_UP); 3394 3395 rc = ena_enable_msix_and_set_admin_interrupts(adapter); 3396 if (rc != 0) { 3397 ena_log(dev, ERR, "Enable MSI-X failed\n"); 3398 goto err_device_destroy; 3399 } 3400 3401 /* 3402 * Effective value of used MSIX vectors should be the same as before 3403 * `ena_destroy_device()`, if possible, or closest to it if less vectors 3404 * are available. 3405 */ 3406 if ((adapter->msix_vecs - ENA_ADMIN_MSIX_VEC) < adapter->num_io_queues) 3407 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; 3408 3409 /* Re-initialize rings basic information */ 3410 ena_init_io_rings(adapter); 3411 3412 /* If the interface was up before the reset bring it up */ 3413 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) { 3414 rc = ena_up(adapter); 3415 if (rc != 0) { 3416 ena_log(dev, ERR, "Failed to create I/O queues\n"); 3417 goto err_disable_msix; 3418 } 3419 } 3420 3421 /* Indicate that device is running again and ready to work */ 3422 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3423 3424 /* 3425 * As the AENQ handlers weren't executed during reset because 3426 * the flag ENA_FLAG_DEVICE_RUNNING was turned off, the 3427 * timestamp must be updated again That will prevent next reset 3428 * caused by missing keep alive. 3429 */ 3430 adapter->keep_alive_timestamp = getsbinuptime(); 3431 ENA_TIMER_RESET(adapter); 3432 3433 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 3434 3435 return (rc); 3436 3437 err_disable_msix: 3438 ena_free_mgmnt_irq(adapter); 3439 ena_disable_msix(adapter); 3440 err_device_destroy: 3441 ena_com_abort_admin_commands(ena_dev); 3442 ena_com_wait_for_abort_completion(ena_dev); 3443 ena_com_admin_destroy(ena_dev); 3444 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE); 3445 ena_com_mmio_reg_read_request_destroy(ena_dev); 3446 err: 3447 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3448 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3449 ena_log(dev, ERR, "Reset attempt failed. Can not reset the device\n"); 3450 3451 return (rc); 3452 } 3453 3454 static void 3455 ena_metrics_task(void *arg, int pending) 3456 { 3457 struct ena_adapter *adapter = (struct ena_adapter *)arg; 3458 3459 ENA_LOCK_LOCK(); 3460 (void)ena_copy_eni_metrics(adapter); 3461 ENA_LOCK_UNLOCK(); 3462 } 3463 3464 static void 3465 ena_reset_task(void *arg, int pending) 3466 { 3467 struct ena_adapter *adapter = (struct ena_adapter *)arg; 3468 3469 ENA_LOCK_LOCK(); 3470 if (likely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 3471 ena_destroy_device(adapter, false); 3472 ena_restore_device(adapter); 3473 3474 ena_log(adapter->pdev, INFO, 3475 "Device reset completed successfully, Driver info: %s\n", 3476 ena_version); 3477 } 3478 ENA_LOCK_UNLOCK(); 3479 } 3480 3481 static void 3482 ena_free_stats(struct ena_adapter *adapter) 3483 { 3484 ena_free_counters((counter_u64_t *)&adapter->hw_stats, 3485 sizeof(struct ena_hw_stats)); 3486 ena_free_counters((counter_u64_t *)&adapter->dev_stats, 3487 sizeof(struct ena_stats_dev)); 3488 3489 } 3490 /** 3491 * ena_attach - Device Initialization Routine 3492 * @pdev: device information struct 3493 * 3494 * Returns 0 on success, otherwise on failure. 3495 * 3496 * ena_attach initializes an adapter identified by a device structure. 3497 * The OS initialization, configuring of the adapter private structure, 3498 * and a hardware reset occur. 3499 **/ 3500 static int 3501 ena_attach(device_t pdev) 3502 { 3503 struct ena_com_dev_get_features_ctx get_feat_ctx; 3504 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; 3505 static int version_printed; 3506 struct ena_adapter *adapter; 3507 struct ena_com_dev *ena_dev = NULL; 3508 uint32_t max_num_io_queues; 3509 int msix_rid; 3510 int rid, rc; 3511 3512 adapter = device_get_softc(pdev); 3513 adapter->pdev = pdev; 3514 adapter->first_bind = -1; 3515 3516 /* 3517 * Set up the timer service - driver is responsible for avoiding 3518 * concurrency, as the callout won't be using any locking inside. 3519 */ 3520 ENA_TIMER_INIT(adapter); 3521 adapter->keep_alive_timeout = ENA_DEFAULT_KEEP_ALIVE_TO; 3522 adapter->missing_tx_timeout = ENA_DEFAULT_TX_CMP_TO; 3523 adapter->missing_tx_max_queues = ENA_DEFAULT_TX_MONITORED_QUEUES; 3524 adapter->missing_tx_threshold = ENA_DEFAULT_TX_CMP_THRESHOLD; 3525 3526 if (version_printed++ == 0) 3527 ena_log(pdev, INFO, "%s\n", ena_version); 3528 3529 /* Allocate memory for ena_dev structure */ 3530 ena_dev = malloc(sizeof(struct ena_com_dev), M_DEVBUF, 3531 M_WAITOK | M_ZERO); 3532 3533 adapter->ena_dev = ena_dev; 3534 ena_dev->dmadev = pdev; 3535 3536 rid = PCIR_BAR(ENA_REG_BAR); 3537 adapter->memory = NULL; 3538 adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid, 3539 RF_ACTIVE); 3540 if (unlikely(adapter->registers == NULL)) { 3541 ena_log(pdev, ERR, 3542 "unable to allocate bus resource: registers!\n"); 3543 rc = ENOMEM; 3544 goto err_dev_free; 3545 } 3546 3547 /* MSIx vector table may reside on BAR0 with registers or on BAR1. */ 3548 msix_rid = pci_msix_table_bar(pdev); 3549 if (msix_rid != rid) { 3550 adapter->msix = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, 3551 &msix_rid, RF_ACTIVE); 3552 if (unlikely(adapter->msix == NULL)) { 3553 ena_log(pdev, ERR, 3554 "unable to allocate bus resource: msix!\n"); 3555 rc = ENOMEM; 3556 goto err_pci_free; 3557 } 3558 adapter->msix_rid = msix_rid; 3559 } 3560 3561 ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF, 3562 M_WAITOK | M_ZERO); 3563 3564 /* Store register resources */ 3565 ((struct ena_bus *)(ena_dev->bus))->reg_bar_t = rman_get_bustag( 3566 adapter->registers); 3567 ((struct ena_bus *)(ena_dev->bus))->reg_bar_h = rman_get_bushandle( 3568 adapter->registers); 3569 3570 if (unlikely(((struct ena_bus *)(ena_dev->bus))->reg_bar_h == 0)) { 3571 ena_log(pdev, ERR, "failed to pmap registers bar\n"); 3572 rc = ENXIO; 3573 goto err_bus_free; 3574 } 3575 3576 rc = ena_map_llq_mem_bar(pdev, ena_dev); 3577 if (unlikely(rc != 0)) { 3578 ena_log(pdev, ERR, "Failed to map ENA mem bar"); 3579 goto err_bus_free; 3580 } 3581 3582 /* Initially clear all the flags */ 3583 ENA_FLAG_ZERO(adapter); 3584 3585 /* Device initialization */ 3586 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active); 3587 if (unlikely(rc != 0)) { 3588 ena_log(pdev, ERR, "ENA device init failed! (err: %d)\n", rc); 3589 rc = ENXIO; 3590 goto err_bus_free; 3591 } 3592 3593 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 3594 adapter->disable_meta_caching = !!( 3595 get_feat_ctx.llq.accel_mode.u.get.supported_flags & 3596 BIT(ENA_ADMIN_DISABLE_META_CACHING)); 3597 3598 adapter->keep_alive_timestamp = getsbinuptime(); 3599 3600 adapter->tx_offload_cap = get_feat_ctx.offload.tx; 3601 3602 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr, 3603 ETHER_ADDR_LEN); 3604 3605 calc_queue_ctx.pdev = pdev; 3606 calc_queue_ctx.ena_dev = ena_dev; 3607 calc_queue_ctx.get_feat_ctx = &get_feat_ctx; 3608 3609 /* Calculate initial and maximum IO queue number and size */ 3610 max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, 3611 &get_feat_ctx); 3612 rc = ena_calc_io_queue_size(&calc_queue_ctx); 3613 if (unlikely((rc != 0) || (max_num_io_queues <= 0))) { 3614 rc = EFAULT; 3615 goto err_com_free; 3616 } 3617 3618 adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size; 3619 adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size; 3620 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size; 3621 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size; 3622 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; 3623 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; 3624 3625 adapter->max_num_io_queues = max_num_io_queues; 3626 3627 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE; 3628 3629 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 3630 3631 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3632 3633 /* set up dma tags for rx and tx buffers */ 3634 rc = ena_setup_tx_dma_tag(adapter); 3635 if (unlikely(rc != 0)) { 3636 ena_log(pdev, ERR, "Failed to create TX DMA tag\n"); 3637 goto err_com_free; 3638 } 3639 3640 rc = ena_setup_rx_dma_tag(adapter); 3641 if (unlikely(rc != 0)) { 3642 ena_log(pdev, ERR, "Failed to create RX DMA tag\n"); 3643 goto err_tx_tag_free; 3644 } 3645 3646 /* 3647 * The amount of requested MSIX vectors is equal to 3648 * adapter::max_num_io_queues (see `ena_enable_msix()`), plus a constant 3649 * number of admin queue interrupts. The former is initially determined 3650 * by HW capabilities (see `ena_calc_max_io_queue_num())` but may not be 3651 * achieved if there are not enough system resources. By default, the 3652 * number of effectively used IO queues is the same but later on it can 3653 * be limited by the user using sysctl interface. 3654 */ 3655 rc = ena_enable_msix_and_set_admin_interrupts(adapter); 3656 if (unlikely(rc != 0)) { 3657 ena_log(pdev, ERR, 3658 "Failed to enable and set the admin interrupts\n"); 3659 goto err_io_free; 3660 } 3661 /* By default all of allocated MSIX vectors are actively used */ 3662 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; 3663 3664 /* initialize rings basic information */ 3665 ena_init_io_rings(adapter); 3666 3667 /* Initialize statistics */ 3668 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats, 3669 sizeof(struct ena_stats_dev)); 3670 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats, 3671 sizeof(struct ena_hw_stats)); 3672 ena_sysctl_add_nodes(adapter); 3673 3674 /* setup network interface */ 3675 rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx); 3676 if (unlikely(rc != 0)) { 3677 ena_log(pdev, ERR, "Error with network interface setup\n"); 3678 goto err_msix_free; 3679 } 3680 3681 /* Initialize reset task queue */ 3682 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter); 3683 adapter->reset_tq = taskqueue_create("ena_reset_enqueue", 3684 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq); 3685 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, "%s rstq", 3686 device_get_nameunit(adapter->pdev)); 3687 3688 /* Initialize metrics task queue */ 3689 TASK_INIT(&adapter->metrics_task, 0, ena_metrics_task, adapter); 3690 adapter->metrics_tq = taskqueue_create("ena_metrics_enqueue", 3691 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->metrics_tq); 3692 taskqueue_start_threads(&adapter->metrics_tq, 1, PI_NET, "%s metricsq", 3693 device_get_nameunit(adapter->pdev)); 3694 3695 #ifdef DEV_NETMAP 3696 rc = ena_netmap_attach(adapter); 3697 if (rc != 0) { 3698 ena_log(pdev, ERR, "netmap attach failed: %d\n", rc); 3699 goto err_detach; 3700 } 3701 #endif /* DEV_NETMAP */ 3702 3703 /* Tell the stack that the interface is not active */ 3704 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 3705 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3706 3707 /* Run the timer service */ 3708 ENA_TIMER_RESET(adapter); 3709 3710 return (0); 3711 3712 #ifdef DEV_NETMAP 3713 err_detach: 3714 ether_ifdetach(adapter->ifp); 3715 #endif /* DEV_NETMAP */ 3716 err_msix_free: 3717 ena_free_stats(adapter); 3718 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR); 3719 ena_free_mgmnt_irq(adapter); 3720 ena_disable_msix(adapter); 3721 err_io_free: 3722 ena_free_all_io_rings_resources(adapter); 3723 ena_free_rx_dma_tag(adapter); 3724 err_tx_tag_free: 3725 ena_free_tx_dma_tag(adapter); 3726 err_com_free: 3727 ena_com_admin_destroy(ena_dev); 3728 ena_com_delete_host_info(ena_dev); 3729 ena_com_mmio_reg_read_request_destroy(ena_dev); 3730 err_bus_free: 3731 free(ena_dev->bus, M_DEVBUF); 3732 err_pci_free: 3733 ena_free_pci_resources(adapter); 3734 err_dev_free: 3735 free(ena_dev, M_DEVBUF); 3736 3737 return (rc); 3738 } 3739 3740 /** 3741 * ena_detach - Device Removal Routine 3742 * @pdev: device information struct 3743 * 3744 * ena_detach is called by the device subsystem to alert the driver 3745 * that it should release a PCI device. 3746 **/ 3747 static int 3748 ena_detach(device_t pdev) 3749 { 3750 struct ena_adapter *adapter = device_get_softc(pdev); 3751 struct ena_com_dev *ena_dev = adapter->ena_dev; 3752 int rc; 3753 3754 /* Make sure VLANS are not using driver */ 3755 if (if_vlantrunkinuse(adapter->ifp)) { 3756 ena_log(adapter->pdev, ERR, "VLAN is in use, detach first\n"); 3757 return (EBUSY); 3758 } 3759 3760 ether_ifdetach(adapter->ifp); 3761 3762 /* Stop timer service */ 3763 ENA_LOCK_LOCK(); 3764 ENA_TIMER_DRAIN(adapter); 3765 ENA_LOCK_UNLOCK(); 3766 3767 /* Release metrics task */ 3768 while (taskqueue_cancel(adapter->metrics_tq, &adapter->metrics_task, NULL)) 3769 taskqueue_drain(adapter->metrics_tq, &adapter->metrics_task); 3770 taskqueue_free(adapter->metrics_tq); 3771 3772 /* Release reset task */ 3773 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL)) 3774 taskqueue_drain(adapter->reset_tq, &adapter->reset_task); 3775 taskqueue_free(adapter->reset_tq); 3776 3777 ENA_LOCK_LOCK(); 3778 ena_down(adapter); 3779 ena_destroy_device(adapter, true); 3780 ENA_LOCK_UNLOCK(); 3781 3782 /* Restore unregistered sysctl queue nodes. */ 3783 ena_sysctl_update_queue_node_nb(adapter, adapter->num_io_queues, 3784 adapter->max_num_io_queues); 3785 3786 #ifdef DEV_NETMAP 3787 netmap_detach(adapter->ifp); 3788 #endif /* DEV_NETMAP */ 3789 3790 ena_free_stats(adapter); 3791 3792 rc = ena_free_rx_dma_tag(adapter); 3793 if (unlikely(rc != 0)) 3794 ena_log(adapter->pdev, WARN, 3795 "Unmapped RX DMA tag associations\n"); 3796 3797 rc = ena_free_tx_dma_tag(adapter); 3798 if (unlikely(rc != 0)) 3799 ena_log(adapter->pdev, WARN, 3800 "Unmapped TX DMA tag associations\n"); 3801 3802 ena_free_irqs(adapter); 3803 3804 ena_free_pci_resources(adapter); 3805 3806 if (adapter->rss_indir != NULL) 3807 free(adapter->rss_indir, M_DEVBUF); 3808 3809 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) 3810 ena_com_rss_destroy(ena_dev); 3811 3812 ena_com_delete_host_info(ena_dev); 3813 3814 if_free(adapter->ifp); 3815 3816 free(ena_dev->bus, M_DEVBUF); 3817 3818 free(ena_dev, M_DEVBUF); 3819 3820 return (bus_generic_detach(pdev)); 3821 } 3822 3823 /****************************************************************************** 3824 ******************************** AENQ Handlers ******************************* 3825 *****************************************************************************/ 3826 /** 3827 * ena_update_on_link_change: 3828 * Notify the network interface about the change in link status 3829 **/ 3830 static void 3831 ena_update_on_link_change(void *adapter_data, 3832 struct ena_admin_aenq_entry *aenq_e) 3833 { 3834 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3835 struct ena_admin_aenq_link_change_desc *aenq_desc; 3836 int status; 3837 if_t ifp; 3838 3839 aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 3840 ifp = adapter->ifp; 3841 status = aenq_desc->flags & 3842 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; 3843 3844 if (status != 0) { 3845 ena_log(adapter->pdev, INFO, "link is UP\n"); 3846 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3847 if (!ENA_FLAG_ISSET(ENA_FLAG_ONGOING_RESET, adapter)) 3848 if_link_state_change(ifp, LINK_STATE_UP); 3849 } else { 3850 ena_log(adapter->pdev, INFO, "link is DOWN\n"); 3851 if_link_state_change(ifp, LINK_STATE_DOWN); 3852 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3853 } 3854 } 3855 3856 static void 3857 ena_notification(void *adapter_data, struct ena_admin_aenq_entry *aenq_e) 3858 { 3859 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3860 struct ena_admin_ena_hw_hints *hints; 3861 3862 ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION, 3863 adapter->ena_dev, "Invalid group(%x) expected %x\n", 3864 aenq_e->aenq_common_desc.group, ENA_ADMIN_NOTIFICATION); 3865 3866 switch (aenq_e->aenq_common_desc.syndrome) { 3867 case ENA_ADMIN_UPDATE_HINTS: 3868 hints = 3869 (struct ena_admin_ena_hw_hints *)(&aenq_e->inline_data_w4); 3870 ena_update_hints(adapter, hints); 3871 break; 3872 default: 3873 ena_log(adapter->pdev, ERR, 3874 "Invalid aenq notification link state %d\n", 3875 aenq_e->aenq_common_desc.syndrome); 3876 } 3877 } 3878 3879 static void 3880 ena_lock_init(void *arg) 3881 { 3882 ENA_LOCK_INIT(); 3883 } 3884 SYSINIT(ena_lock_init, SI_SUB_LOCK, SI_ORDER_FIRST, ena_lock_init, NULL); 3885 3886 static void 3887 ena_lock_uninit(void *arg) 3888 { 3889 ENA_LOCK_DESTROY(); 3890 } 3891 SYSUNINIT(ena_lock_uninit, SI_SUB_LOCK, SI_ORDER_FIRST, ena_lock_uninit, NULL); 3892 3893 /** 3894 * This handler will called for unknown event group or unimplemented handlers 3895 **/ 3896 static void 3897 unimplemented_aenq_handler(void *adapter_data, 3898 struct ena_admin_aenq_entry *aenq_e) 3899 { 3900 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3901 3902 ena_log(adapter->pdev, ERR, 3903 "Unknown event was received or event with unimplemented handler\n"); 3904 } 3905 3906 static struct ena_aenq_handlers aenq_handlers = { 3907 .handlers = { 3908 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 3909 [ENA_ADMIN_NOTIFICATION] = ena_notification, 3910 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd, 3911 }, 3912 .unimplemented_handler = unimplemented_aenq_handler 3913 }; 3914 3915 /********************************************************************* 3916 * FreeBSD Device Interface Entry Points 3917 *********************************************************************/ 3918 3919 static device_method_t ena_methods[] = { /* Device interface */ 3920 DEVMETHOD(device_probe, ena_probe), 3921 DEVMETHOD(device_attach, ena_attach), 3922 DEVMETHOD(device_detach, ena_detach), DEVMETHOD_END 3923 }; 3924 3925 static driver_t ena_driver = { 3926 "ena", 3927 ena_methods, 3928 sizeof(struct ena_adapter), 3929 }; 3930 3931 DRIVER_MODULE(ena, pci, ena_driver, 0, 0); 3932 MODULE_PNP_INFO("U16:vendor;U16:device", pci, ena, ena_vendor_info_array, 3933 nitems(ena_vendor_info_array) - 1); 3934 MODULE_DEPEND(ena, pci, 1, 1, 1); 3935 MODULE_DEPEND(ena, ether, 1, 1, 1); 3936 #ifdef DEV_NETMAP 3937 MODULE_DEPEND(ena, netmap, 1, 1, 1); 3938 #endif /* DEV_NETMAP */ 3939 3940 /*********************************************************************/ 3941