1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/endian.h> 37 #include <sys/kernel.h> 38 #include <sys/kthread.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/rman.h> 43 #include <sys/smp.h> 44 #include <sys/socket.h> 45 #include <sys/sockio.h> 46 #include <sys/sysctl.h> 47 #include <sys/taskqueue.h> 48 #include <sys/time.h> 49 #include <sys/eventhandler.h> 50 51 #include <machine/bus.h> 52 #include <machine/resource.h> 53 #include <machine/in_cksum.h> 54 55 #include <net/bpf.h> 56 #include <net/ethernet.h> 57 #include <net/if.h> 58 #include <net/if_var.h> 59 #include <net/if_arp.h> 60 #include <net/if_dl.h> 61 #include <net/if_media.h> 62 #include <net/if_types.h> 63 #include <net/if_vlan_var.h> 64 65 #include <netinet/in_systm.h> 66 #include <netinet/in.h> 67 #include <netinet/if_ether.h> 68 #include <netinet/ip.h> 69 #include <netinet/ip6.h> 70 #include <netinet/tcp.h> 71 #include <netinet/udp.h> 72 73 #include <dev/pci/pcivar.h> 74 #include <dev/pci/pcireg.h> 75 76 #include <vm/vm.h> 77 #include <vm/pmap.h> 78 79 #include "ena_datapath.h" 80 #include "ena.h" 81 #include "ena_sysctl.h" 82 83 #ifdef DEV_NETMAP 84 #include "ena_netmap.h" 85 #endif /* DEV_NETMAP */ 86 87 /********************************************************* 88 * Function prototypes 89 *********************************************************/ 90 static int ena_probe(device_t); 91 static void ena_intr_msix_mgmnt(void *); 92 static void ena_free_pci_resources(struct ena_adapter *); 93 static int ena_change_mtu(if_t, int); 94 static inline void ena_alloc_counters(counter_u64_t *, int); 95 static inline void ena_free_counters(counter_u64_t *, int); 96 static inline void ena_reset_counters(counter_u64_t *, int); 97 static void ena_init_io_rings_common(struct ena_adapter *, 98 struct ena_ring *, uint16_t); 99 static void ena_init_io_rings_basic(struct ena_adapter *); 100 static void ena_init_io_rings_advanced(struct ena_adapter *); 101 static void ena_init_io_rings(struct ena_adapter *); 102 static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int); 103 static void ena_free_all_io_rings_resources(struct ena_adapter *); 104 static int ena_setup_tx_dma_tag(struct ena_adapter *); 105 static int ena_free_tx_dma_tag(struct ena_adapter *); 106 static int ena_setup_rx_dma_tag(struct ena_adapter *); 107 static int ena_free_rx_dma_tag(struct ena_adapter *); 108 static void ena_release_all_tx_dmamap(struct ena_ring *); 109 static int ena_setup_tx_resources(struct ena_adapter *, int); 110 static void ena_free_tx_resources(struct ena_adapter *, int); 111 static int ena_setup_all_tx_resources(struct ena_adapter *); 112 static void ena_free_all_tx_resources(struct ena_adapter *); 113 static int ena_setup_rx_resources(struct ena_adapter *, unsigned int); 114 static void ena_free_rx_resources(struct ena_adapter *, unsigned int); 115 static int ena_setup_all_rx_resources(struct ena_adapter *); 116 static void ena_free_all_rx_resources(struct ena_adapter *); 117 static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *, 118 struct ena_rx_buffer *); 119 static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *, 120 struct ena_rx_buffer *); 121 static void ena_free_rx_bufs(struct ena_adapter *, unsigned int); 122 static void ena_refill_all_rx_bufs(struct ena_adapter *); 123 static void ena_free_all_rx_bufs(struct ena_adapter *); 124 static void ena_free_tx_bufs(struct ena_adapter *, unsigned int); 125 static void ena_free_all_tx_bufs(struct ena_adapter *); 126 static void ena_destroy_all_tx_queues(struct ena_adapter *); 127 static void ena_destroy_all_rx_queues(struct ena_adapter *); 128 static void ena_destroy_all_io_queues(struct ena_adapter *); 129 static int ena_create_io_queues(struct ena_adapter *); 130 static int ena_handle_msix(void *); 131 static int ena_enable_msix(struct ena_adapter *); 132 static void ena_setup_mgmnt_intr(struct ena_adapter *); 133 static int ena_setup_io_intr(struct ena_adapter *); 134 static int ena_request_mgmnt_irq(struct ena_adapter *); 135 static int ena_request_io_irq(struct ena_adapter *); 136 static void ena_free_mgmnt_irq(struct ena_adapter *); 137 static void ena_free_io_irq(struct ena_adapter *); 138 static void ena_free_irqs(struct ena_adapter*); 139 static void ena_disable_msix(struct ena_adapter *); 140 static void ena_unmask_all_io_irqs(struct ena_adapter *); 141 static int ena_rss_configure(struct ena_adapter *); 142 static int ena_up_complete(struct ena_adapter *); 143 static uint64_t ena_get_counter(if_t, ift_counter); 144 static int ena_media_change(if_t); 145 static void ena_media_status(if_t, struct ifmediareq *); 146 static void ena_init(void *); 147 static int ena_ioctl(if_t, u_long, caddr_t); 148 static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *); 149 static void ena_update_host_info(struct ena_admin_host_info *, if_t); 150 static void ena_update_hwassist(struct ena_adapter *); 151 static int ena_setup_ifnet(device_t, struct ena_adapter *, 152 struct ena_com_dev_get_features_ctx *); 153 static int ena_enable_wc(struct resource *); 154 static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *, 155 struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *); 156 static uint32_t ena_calc_max_io_queue_num(device_t, struct ena_com_dev *, 157 struct ena_com_dev_get_features_ctx *); 158 static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *); 159 static int ena_rss_init_default(struct ena_adapter *); 160 static void ena_rss_init_default_deferred(void *); 161 static void ena_config_host_info(struct ena_com_dev *, device_t); 162 static int ena_attach(device_t); 163 static int ena_detach(device_t); 164 static int ena_device_init(struct ena_adapter *, device_t, 165 struct ena_com_dev_get_features_ctx *, int *); 166 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *); 167 static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *); 168 static void unimplemented_aenq_handler(void *, 169 struct ena_admin_aenq_entry *); 170 static void ena_timer_service(void *); 171 172 static char ena_version[] = DEVICE_NAME DRV_MODULE_NAME " v" DRV_MODULE_VERSION; 173 174 static ena_vendor_info_t ena_vendor_info_array[] = { 175 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0}, 176 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_PF, 0}, 177 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0}, 178 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_VF, 0}, 179 /* Last entry */ 180 { 0, 0, 0 } 181 }; 182 183 /* 184 * Contains pointers to event handlers, e.g. link state chage. 185 */ 186 static struct ena_aenq_handlers aenq_handlers; 187 188 void 189 ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 190 { 191 if (error != 0) 192 return; 193 *(bus_addr_t *) arg = segs[0].ds_addr; 194 } 195 196 int 197 ena_dma_alloc(device_t dmadev, bus_size_t size, 198 ena_mem_handle_t *dma , int mapflags) 199 { 200 struct ena_adapter* adapter = device_get_softc(dmadev); 201 uint32_t maxsize; 202 uint64_t dma_space_addr; 203 int error; 204 205 maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE; 206 207 dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width); 208 if (unlikely(dma_space_addr == 0)) 209 dma_space_addr = BUS_SPACE_MAXADDR; 210 211 error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */ 212 8, 0, /* alignment, bounds */ 213 dma_space_addr, /* lowaddr of exclusion window */ 214 BUS_SPACE_MAXADDR,/* highaddr of exclusion window */ 215 NULL, NULL, /* filter, filterarg */ 216 maxsize, /* maxsize */ 217 1, /* nsegments */ 218 maxsize, /* maxsegsize */ 219 BUS_DMA_ALLOCNOW, /* flags */ 220 NULL, /* lockfunc */ 221 NULL, /* lockarg */ 222 &dma->tag); 223 if (unlikely(error != 0)) { 224 ena_trace(ENA_ALERT, "bus_dma_tag_create failed: %d\n", error); 225 goto fail_tag; 226 } 227 228 error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr, 229 BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map); 230 if (unlikely(error != 0)) { 231 ena_trace(ENA_ALERT, "bus_dmamem_alloc(%ju) failed: %d\n", 232 (uintmax_t)size, error); 233 goto fail_map_create; 234 } 235 236 dma->paddr = 0; 237 error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, 238 size, ena_dmamap_callback, &dma->paddr, mapflags); 239 if (unlikely((error != 0) || (dma->paddr == 0))) { 240 ena_trace(ENA_ALERT, ": bus_dmamap_load failed: %d\n", error); 241 goto fail_map_load; 242 } 243 244 bus_dmamap_sync(dma->tag, dma->map, 245 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 246 247 return (0); 248 249 fail_map_load: 250 bus_dmamem_free(dma->tag, dma->vaddr, dma->map); 251 fail_map_create: 252 bus_dma_tag_destroy(dma->tag); 253 fail_tag: 254 dma->tag = NULL; 255 dma->vaddr = NULL; 256 dma->paddr = 0; 257 258 return (error); 259 } 260 261 /* 262 * This function should generate unique key for the whole driver. 263 * If the key was already genereated in the previous call (for example 264 * for another adapter), then it should be returned instead. 265 */ 266 void 267 ena_rss_key_fill(void *key, size_t size) 268 { 269 static bool key_generated; 270 static uint8_t default_key[ENA_HASH_KEY_SIZE]; 271 272 KASSERT(size <= ENA_HASH_KEY_SIZE, ("Requested more bytes than ENA RSS key can hold")); 273 274 if (!key_generated) { 275 arc4random_buf(default_key, ENA_HASH_KEY_SIZE); 276 key_generated = true; 277 } 278 279 memcpy(key, default_key, size); 280 } 281 282 static void 283 ena_free_pci_resources(struct ena_adapter *adapter) 284 { 285 device_t pdev = adapter->pdev; 286 287 if (adapter->memory != NULL) { 288 bus_release_resource(pdev, SYS_RES_MEMORY, 289 PCIR_BAR(ENA_MEM_BAR), adapter->memory); 290 } 291 292 if (adapter->registers != NULL) { 293 bus_release_resource(pdev, SYS_RES_MEMORY, 294 PCIR_BAR(ENA_REG_BAR), adapter->registers); 295 } 296 } 297 298 static int 299 ena_probe(device_t dev) 300 { 301 ena_vendor_info_t *ent; 302 char adapter_name[60]; 303 uint16_t pci_vendor_id = 0; 304 uint16_t pci_device_id = 0; 305 306 pci_vendor_id = pci_get_vendor(dev); 307 pci_device_id = pci_get_device(dev); 308 309 ent = ena_vendor_info_array; 310 while (ent->vendor_id != 0) { 311 if ((pci_vendor_id == ent->vendor_id) && 312 (pci_device_id == ent->device_id)) { 313 ena_trace(ENA_DBG, "vendor=%x device=%x\n", 314 pci_vendor_id, pci_device_id); 315 316 sprintf(adapter_name, DEVICE_DESC); 317 device_set_desc_copy(dev, adapter_name); 318 return (BUS_PROBE_DEFAULT); 319 } 320 321 ent++; 322 323 } 324 325 return (ENXIO); 326 } 327 328 static int 329 ena_change_mtu(if_t ifp, int new_mtu) 330 { 331 struct ena_adapter *adapter = if_getsoftc(ifp); 332 int rc; 333 334 if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) { 335 device_printf(adapter->pdev, "Invalid MTU setting. " 336 "new_mtu: %d max mtu: %d min mtu: %d\n", 337 new_mtu, adapter->max_mtu, ENA_MIN_MTU); 338 return (EINVAL); 339 } 340 341 rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); 342 if (likely(rc == 0)) { 343 ena_trace(ENA_DBG, "set MTU to %d\n", new_mtu); 344 if_setmtu(ifp, new_mtu); 345 } else { 346 device_printf(adapter->pdev, "Failed to set MTU to %d\n", 347 new_mtu); 348 } 349 350 return (rc); 351 } 352 353 static inline void 354 ena_alloc_counters(counter_u64_t *begin, int size) 355 { 356 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 357 358 for (; begin < end; ++begin) 359 *begin = counter_u64_alloc(M_WAITOK); 360 } 361 362 static inline void 363 ena_free_counters(counter_u64_t *begin, int size) 364 { 365 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 366 367 for (; begin < end; ++begin) 368 counter_u64_free(*begin); 369 } 370 371 static inline void 372 ena_reset_counters(counter_u64_t *begin, int size) 373 { 374 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 375 376 for (; begin < end; ++begin) 377 counter_u64_zero(*begin); 378 } 379 380 static void 381 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring, 382 uint16_t qid) 383 { 384 385 ring->qid = qid; 386 ring->adapter = adapter; 387 ring->ena_dev = adapter->ena_dev; 388 ring->first_interrupt = false; 389 ring->no_interrupt_event_cnt = 0; 390 } 391 392 static void 393 ena_init_io_rings_basic(struct ena_adapter *adapter) 394 { 395 struct ena_com_dev *ena_dev; 396 struct ena_ring *txr, *rxr; 397 struct ena_que *que; 398 int i; 399 400 ena_dev = adapter->ena_dev; 401 402 for (i = 0; i < adapter->num_io_queues; i++) { 403 txr = &adapter->tx_ring[i]; 404 rxr = &adapter->rx_ring[i]; 405 406 /* TX/RX common ring state */ 407 ena_init_io_rings_common(adapter, txr, i); 408 ena_init_io_rings_common(adapter, rxr, i); 409 410 /* TX specific ring state */ 411 txr->ring_size = adapter->tx_ring_size; 412 txr->tx_max_header_size = ena_dev->tx_max_header_size; 413 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type; 414 415 /* RX specific ring state */ 416 rxr->ring_size = adapter->rx_ring_size; 417 418 que = &adapter->que[i]; 419 que->adapter = adapter; 420 que->id = i; 421 que->tx_ring = txr; 422 que->rx_ring = rxr; 423 424 txr->que = que; 425 rxr->que = que; 426 427 rxr->empty_rx_queue = 0; 428 rxr->rx_mbuf_sz = ena_mbuf_sz; 429 } 430 } 431 432 static void 433 ena_init_io_rings_advanced(struct ena_adapter *adapter) 434 { 435 struct ena_ring *txr, *rxr; 436 int i; 437 438 for (i = 0; i < adapter->num_io_queues; i++) { 439 txr = &adapter->tx_ring[i]; 440 rxr = &adapter->rx_ring[i]; 441 442 /* Allocate a buf ring */ 443 txr->buf_ring_size = adapter->buf_ring_size; 444 txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF, 445 M_WAITOK, &txr->ring_mtx); 446 447 /* Allocate Tx statistics. */ 448 ena_alloc_counters((counter_u64_t *)&txr->tx_stats, 449 sizeof(txr->tx_stats)); 450 451 /* Allocate Rx statistics. */ 452 ena_alloc_counters((counter_u64_t *)&rxr->rx_stats, 453 sizeof(rxr->rx_stats)); 454 455 /* Initialize locks */ 456 snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)", 457 device_get_nameunit(adapter->pdev), i); 458 snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)", 459 device_get_nameunit(adapter->pdev), i); 460 461 mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF); 462 } 463 } 464 465 static void 466 ena_init_io_rings(struct ena_adapter *adapter) 467 { 468 /* 469 * IO rings initialization can be divided into the 2 steps: 470 * 1. Initialize variables and fields with initial values and copy 471 * them from adapter/ena_dev (basic) 472 * 2. Allocate mutex, counters and buf_ring (advanced) 473 */ 474 ena_init_io_rings_basic(adapter); 475 ena_init_io_rings_advanced(adapter); 476 } 477 478 static void 479 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid) 480 { 481 struct ena_ring *txr = &adapter->tx_ring[qid]; 482 struct ena_ring *rxr = &adapter->rx_ring[qid]; 483 484 ena_free_counters((counter_u64_t *)&txr->tx_stats, 485 sizeof(txr->tx_stats)); 486 ena_free_counters((counter_u64_t *)&rxr->rx_stats, 487 sizeof(rxr->rx_stats)); 488 489 ENA_RING_MTX_LOCK(txr); 490 drbr_free(txr->br, M_DEVBUF); 491 ENA_RING_MTX_UNLOCK(txr); 492 493 mtx_destroy(&txr->ring_mtx); 494 } 495 496 static void 497 ena_free_all_io_rings_resources(struct ena_adapter *adapter) 498 { 499 int i; 500 501 for (i = 0; i < adapter->num_io_queues; i++) 502 ena_free_io_ring_resources(adapter, i); 503 504 } 505 506 static int 507 ena_setup_tx_dma_tag(struct ena_adapter *adapter) 508 { 509 int ret; 510 511 /* Create DMA tag for Tx buffers */ 512 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), 513 1, 0, /* alignment, bounds */ 514 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 515 BUS_SPACE_MAXADDR, /* highaddr of excl window */ 516 NULL, NULL, /* filter, filterarg */ 517 ENA_TSO_MAXSIZE, /* maxsize */ 518 adapter->max_tx_sgl_size - 1, /* nsegments */ 519 ENA_TSO_MAXSIZE, /* maxsegsize */ 520 0, /* flags */ 521 NULL, /* lockfunc */ 522 NULL, /* lockfuncarg */ 523 &adapter->tx_buf_tag); 524 525 return (ret); 526 } 527 528 static int 529 ena_free_tx_dma_tag(struct ena_adapter *adapter) 530 { 531 int ret; 532 533 ret = bus_dma_tag_destroy(adapter->tx_buf_tag); 534 535 if (likely(ret == 0)) 536 adapter->tx_buf_tag = NULL; 537 538 return (ret); 539 } 540 541 static int 542 ena_setup_rx_dma_tag(struct ena_adapter *adapter) 543 { 544 int ret; 545 546 /* Create DMA tag for Rx buffers*/ 547 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */ 548 1, 0, /* alignment, bounds */ 549 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 550 BUS_SPACE_MAXADDR, /* highaddr of excl window */ 551 NULL, NULL, /* filter, filterarg */ 552 ena_mbuf_sz, /* maxsize */ 553 adapter->max_rx_sgl_size, /* nsegments */ 554 ena_mbuf_sz, /* maxsegsize */ 555 0, /* flags */ 556 NULL, /* lockfunc */ 557 NULL, /* lockarg */ 558 &adapter->rx_buf_tag); 559 560 return (ret); 561 } 562 563 static int 564 ena_free_rx_dma_tag(struct ena_adapter *adapter) 565 { 566 int ret; 567 568 ret = bus_dma_tag_destroy(adapter->rx_buf_tag); 569 570 if (likely(ret == 0)) 571 adapter->rx_buf_tag = NULL; 572 573 return (ret); 574 } 575 576 static void 577 ena_release_all_tx_dmamap(struct ena_ring *tx_ring) 578 { 579 struct ena_adapter *adapter = tx_ring->adapter; 580 struct ena_tx_buffer *tx_info; 581 bus_dma_tag_t tx_tag = adapter->tx_buf_tag;; 582 int i; 583 #ifdef DEV_NETMAP 584 struct ena_netmap_tx_info *nm_info; 585 int j; 586 #endif /* DEV_NETMAP */ 587 588 for (i = 0; i < tx_ring->ring_size; ++i) { 589 tx_info = &tx_ring->tx_buffer_info[i]; 590 #ifdef DEV_NETMAP 591 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 592 nm_info = &tx_info->nm_info; 593 for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) { 594 if (nm_info->map_seg[j] != NULL) { 595 bus_dmamap_destroy(tx_tag, 596 nm_info->map_seg[j]); 597 nm_info->map_seg[j] = NULL; 598 } 599 } 600 } 601 #endif /* DEV_NETMAP */ 602 if (tx_info->dmamap != NULL) { 603 bus_dmamap_destroy(tx_tag, tx_info->dmamap); 604 tx_info->dmamap = NULL; 605 } 606 } 607 } 608 609 /** 610 * ena_setup_tx_resources - allocate Tx resources (Descriptors) 611 * @adapter: network interface device structure 612 * @qid: queue index 613 * 614 * Returns 0 on success, otherwise on failure. 615 **/ 616 static int 617 ena_setup_tx_resources(struct ena_adapter *adapter, int qid) 618 { 619 struct ena_que *que = &adapter->que[qid]; 620 struct ena_ring *tx_ring = que->tx_ring; 621 int size, i, err; 622 #ifdef DEV_NETMAP 623 bus_dmamap_t *map; 624 int j; 625 626 ena_netmap_reset_tx_ring(adapter, qid); 627 #endif /* DEV_NETMAP */ 628 629 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; 630 631 tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 632 if (unlikely(tx_ring->tx_buffer_info == NULL)) 633 return (ENOMEM); 634 635 size = sizeof(uint16_t) * tx_ring->ring_size; 636 tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 637 if (unlikely(tx_ring->free_tx_ids == NULL)) 638 goto err_buf_info_free; 639 640 size = tx_ring->tx_max_header_size; 641 tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF, 642 M_NOWAIT | M_ZERO); 643 if (unlikely(tx_ring->push_buf_intermediate_buf == NULL)) 644 goto err_tx_ids_free; 645 646 /* Req id stack for TX OOO completions */ 647 for (i = 0; i < tx_ring->ring_size; i++) 648 tx_ring->free_tx_ids[i] = i; 649 650 /* Reset TX statistics. */ 651 ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats, 652 sizeof(tx_ring->tx_stats)); 653 654 tx_ring->next_to_use = 0; 655 tx_ring->next_to_clean = 0; 656 tx_ring->acum_pkts = 0; 657 658 /* Make sure that drbr is empty */ 659 ENA_RING_MTX_LOCK(tx_ring); 660 drbr_flush(adapter->ifp, tx_ring->br); 661 ENA_RING_MTX_UNLOCK(tx_ring); 662 663 /* ... and create the buffer DMA maps */ 664 for (i = 0; i < tx_ring->ring_size; i++) { 665 err = bus_dmamap_create(adapter->tx_buf_tag, 0, 666 &tx_ring->tx_buffer_info[i].dmamap); 667 if (unlikely(err != 0)) { 668 ena_trace(ENA_ALERT, 669 "Unable to create Tx DMA map for buffer %d\n", 670 i); 671 goto err_map_release; 672 } 673 674 #ifdef DEV_NETMAP 675 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 676 map = tx_ring->tx_buffer_info[i].nm_info.map_seg; 677 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { 678 err = bus_dmamap_create(adapter->tx_buf_tag, 0, 679 &map[j]); 680 if (unlikely(err != 0)) { 681 ena_trace(ENA_ALERT, "Unable to create " 682 "Tx DMA for buffer %d %d\n", i, j); 683 goto err_map_release; 684 } 685 } 686 } 687 #endif /* DEV_NETMAP */ 688 } 689 690 /* Allocate taskqueues */ 691 TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring); 692 tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT, 693 taskqueue_thread_enqueue, &tx_ring->enqueue_tq); 694 if (unlikely(tx_ring->enqueue_tq == NULL)) { 695 ena_trace(ENA_ALERT, 696 "Unable to create taskqueue for enqueue task\n"); 697 i = tx_ring->ring_size; 698 goto err_map_release; 699 } 700 701 tx_ring->running = true; 702 703 taskqueue_start_threads(&tx_ring->enqueue_tq, 1, PI_NET, 704 "%s txeq %d", device_get_nameunit(adapter->pdev), que->cpu); 705 706 return (0); 707 708 err_map_release: 709 ena_release_all_tx_dmamap(tx_ring); 710 err_tx_ids_free: 711 free(tx_ring->free_tx_ids, M_DEVBUF); 712 tx_ring->free_tx_ids = NULL; 713 err_buf_info_free: 714 free(tx_ring->tx_buffer_info, M_DEVBUF); 715 tx_ring->tx_buffer_info = NULL; 716 717 return (ENOMEM); 718 } 719 720 /** 721 * ena_free_tx_resources - Free Tx Resources per Queue 722 * @adapter: network interface device structure 723 * @qid: queue index 724 * 725 * Free all transmit software resources 726 **/ 727 static void 728 ena_free_tx_resources(struct ena_adapter *adapter, int qid) 729 { 730 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 731 #ifdef DEV_NETMAP 732 struct ena_netmap_tx_info *nm_info; 733 int j; 734 #endif /* DEV_NETMAP */ 735 736 while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task, 737 NULL)) 738 taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 739 740 taskqueue_free(tx_ring->enqueue_tq); 741 742 ENA_RING_MTX_LOCK(tx_ring); 743 /* Flush buffer ring, */ 744 drbr_flush(adapter->ifp, tx_ring->br); 745 746 /* Free buffer DMA maps, */ 747 for (int i = 0; i < tx_ring->ring_size; i++) { 748 bus_dmamap_sync(adapter->tx_buf_tag, 749 tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE); 750 bus_dmamap_unload(adapter->tx_buf_tag, 751 tx_ring->tx_buffer_info[i].dmamap); 752 bus_dmamap_destroy(adapter->tx_buf_tag, 753 tx_ring->tx_buffer_info[i].dmamap); 754 755 #ifdef DEV_NETMAP 756 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 757 nm_info = &tx_ring->tx_buffer_info[i].nm_info; 758 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { 759 if (nm_info->socket_buf_idx[j] != 0) { 760 bus_dmamap_sync(adapter->tx_buf_tag, 761 nm_info->map_seg[j], 762 BUS_DMASYNC_POSTWRITE); 763 ena_netmap_unload(adapter, 764 nm_info->map_seg[j]); 765 } 766 bus_dmamap_destroy(adapter->tx_buf_tag, 767 nm_info->map_seg[j]); 768 nm_info->socket_buf_idx[j] = 0; 769 } 770 } 771 #endif /* DEV_NETMAP */ 772 773 m_freem(tx_ring->tx_buffer_info[i].mbuf); 774 tx_ring->tx_buffer_info[i].mbuf = NULL; 775 } 776 ENA_RING_MTX_UNLOCK(tx_ring); 777 778 /* And free allocated memory. */ 779 free(tx_ring->tx_buffer_info, M_DEVBUF); 780 tx_ring->tx_buffer_info = NULL; 781 782 free(tx_ring->free_tx_ids, M_DEVBUF); 783 tx_ring->free_tx_ids = NULL; 784 785 free(tx_ring->push_buf_intermediate_buf, M_DEVBUF); 786 tx_ring->push_buf_intermediate_buf = NULL; 787 } 788 789 /** 790 * ena_setup_all_tx_resources - allocate all queues Tx resources 791 * @adapter: network interface device structure 792 * 793 * Returns 0 on success, otherwise on failure. 794 **/ 795 static int 796 ena_setup_all_tx_resources(struct ena_adapter *adapter) 797 { 798 int i, rc; 799 800 for (i = 0; i < adapter->num_io_queues; i++) { 801 rc = ena_setup_tx_resources(adapter, i); 802 if (rc != 0) { 803 device_printf(adapter->pdev, 804 "Allocation for Tx Queue %u failed\n", i); 805 goto err_setup_tx; 806 } 807 } 808 809 return (0); 810 811 err_setup_tx: 812 /* Rewind the index freeing the rings as we go */ 813 while (i--) 814 ena_free_tx_resources(adapter, i); 815 return (rc); 816 } 817 818 /** 819 * ena_free_all_tx_resources - Free Tx Resources for All Queues 820 * @adapter: network interface device structure 821 * 822 * Free all transmit software resources 823 **/ 824 static void 825 ena_free_all_tx_resources(struct ena_adapter *adapter) 826 { 827 int i; 828 829 for (i = 0; i < adapter->num_io_queues; i++) 830 ena_free_tx_resources(adapter, i); 831 } 832 833 /** 834 * ena_setup_rx_resources - allocate Rx resources (Descriptors) 835 * @adapter: network interface device structure 836 * @qid: queue index 837 * 838 * Returns 0 on success, otherwise on failure. 839 **/ 840 static int 841 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid) 842 { 843 struct ena_que *que = &adapter->que[qid]; 844 struct ena_ring *rx_ring = que->rx_ring; 845 int size, err, i; 846 847 size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size; 848 849 #ifdef DEV_NETMAP 850 ena_netmap_reset_rx_ring(adapter, qid); 851 rx_ring->initialized = false; 852 #endif /* DEV_NETMAP */ 853 854 /* 855 * Alloc extra element so in rx path 856 * we can always prefetch rx_info + 1 857 */ 858 size += sizeof(struct ena_rx_buffer); 859 860 rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO); 861 862 size = sizeof(uint16_t) * rx_ring->ring_size; 863 rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK); 864 865 for (i = 0; i < rx_ring->ring_size; i++) 866 rx_ring->free_rx_ids[i] = i; 867 868 /* Reset RX statistics. */ 869 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats, 870 sizeof(rx_ring->rx_stats)); 871 872 rx_ring->next_to_clean = 0; 873 rx_ring->next_to_use = 0; 874 875 /* ... and create the buffer DMA maps */ 876 for (i = 0; i < rx_ring->ring_size; i++) { 877 err = bus_dmamap_create(adapter->rx_buf_tag, 0, 878 &(rx_ring->rx_buffer_info[i].map)); 879 if (err != 0) { 880 ena_trace(ENA_ALERT, 881 "Unable to create Rx DMA map for buffer %d\n", i); 882 goto err_buf_info_unmap; 883 } 884 } 885 886 /* Create LRO for the ring */ 887 if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) { 888 int err = tcp_lro_init(&rx_ring->lro); 889 if (err != 0) { 890 device_printf(adapter->pdev, 891 "LRO[%d] Initialization failed!\n", qid); 892 } else { 893 ena_trace(ENA_INFO, 894 "RX Soft LRO[%d] Initialized\n", qid); 895 rx_ring->lro.ifp = adapter->ifp; 896 } 897 } 898 899 return (0); 900 901 err_buf_info_unmap: 902 while (i--) { 903 bus_dmamap_destroy(adapter->rx_buf_tag, 904 rx_ring->rx_buffer_info[i].map); 905 } 906 907 free(rx_ring->free_rx_ids, M_DEVBUF); 908 rx_ring->free_rx_ids = NULL; 909 free(rx_ring->rx_buffer_info, M_DEVBUF); 910 rx_ring->rx_buffer_info = NULL; 911 return (ENOMEM); 912 } 913 914 /** 915 * ena_free_rx_resources - Free Rx Resources 916 * @adapter: network interface device structure 917 * @qid: queue index 918 * 919 * Free all receive software resources 920 **/ 921 static void 922 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid) 923 { 924 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 925 926 /* Free buffer DMA maps, */ 927 for (int i = 0; i < rx_ring->ring_size; i++) { 928 bus_dmamap_sync(adapter->rx_buf_tag, 929 rx_ring->rx_buffer_info[i].map, BUS_DMASYNC_POSTREAD); 930 m_freem(rx_ring->rx_buffer_info[i].mbuf); 931 rx_ring->rx_buffer_info[i].mbuf = NULL; 932 bus_dmamap_unload(adapter->rx_buf_tag, 933 rx_ring->rx_buffer_info[i].map); 934 bus_dmamap_destroy(adapter->rx_buf_tag, 935 rx_ring->rx_buffer_info[i].map); 936 } 937 938 /* free LRO resources, */ 939 tcp_lro_free(&rx_ring->lro); 940 941 /* free allocated memory */ 942 free(rx_ring->rx_buffer_info, M_DEVBUF); 943 rx_ring->rx_buffer_info = NULL; 944 945 free(rx_ring->free_rx_ids, M_DEVBUF); 946 rx_ring->free_rx_ids = NULL; 947 } 948 949 /** 950 * ena_setup_all_rx_resources - allocate all queues Rx resources 951 * @adapter: network interface device structure 952 * 953 * Returns 0 on success, otherwise on failure. 954 **/ 955 static int 956 ena_setup_all_rx_resources(struct ena_adapter *adapter) 957 { 958 int i, rc = 0; 959 960 for (i = 0; i < adapter->num_io_queues; i++) { 961 rc = ena_setup_rx_resources(adapter, i); 962 if (rc != 0) { 963 device_printf(adapter->pdev, 964 "Allocation for Rx Queue %u failed\n", i); 965 goto err_setup_rx; 966 } 967 } 968 return (0); 969 970 err_setup_rx: 971 /* rewind the index freeing the rings as we go */ 972 while (i--) 973 ena_free_rx_resources(adapter, i); 974 return (rc); 975 } 976 977 /** 978 * ena_free_all_rx_resources - Free Rx resources for all queues 979 * @adapter: network interface device structure 980 * 981 * Free all receive software resources 982 **/ 983 static void 984 ena_free_all_rx_resources(struct ena_adapter *adapter) 985 { 986 int i; 987 988 for (i = 0; i < adapter->num_io_queues; i++) 989 ena_free_rx_resources(adapter, i); 990 } 991 992 static inline int 993 ena_alloc_rx_mbuf(struct ena_adapter *adapter, 994 struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) 995 { 996 struct ena_com_buf *ena_buf; 997 bus_dma_segment_t segs[1]; 998 int nsegs, error; 999 int mlen; 1000 1001 /* if previous allocated frag is not used */ 1002 if (unlikely(rx_info->mbuf != NULL)) 1003 return (0); 1004 1005 /* Get mbuf using UMA allocator */ 1006 rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, 1007 rx_ring->rx_mbuf_sz); 1008 1009 if (unlikely(rx_info->mbuf == NULL)) { 1010 counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1); 1011 rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 1012 if (unlikely(rx_info->mbuf == NULL)) { 1013 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); 1014 return (ENOMEM); 1015 } 1016 mlen = MCLBYTES; 1017 } else { 1018 mlen = rx_ring->rx_mbuf_sz; 1019 } 1020 /* Set mbuf length*/ 1021 rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen; 1022 1023 /* Map packets for DMA */ 1024 ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH, 1025 "Using tag %p for buffers' DMA mapping, mbuf %p len: %d\n", 1026 adapter->rx_buf_tag,rx_info->mbuf, rx_info->mbuf->m_len); 1027 error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map, 1028 rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT); 1029 if (unlikely((error != 0) || (nsegs != 1))) { 1030 ena_trace(ENA_WARNING, "failed to map mbuf, error: %d, " 1031 "nsegs: %d\n", error, nsegs); 1032 counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1); 1033 goto exit; 1034 1035 } 1036 1037 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD); 1038 1039 ena_buf = &rx_info->ena_buf; 1040 ena_buf->paddr = segs[0].ds_addr; 1041 ena_buf->len = mlen; 1042 1043 ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH, 1044 "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n", 1045 rx_info->mbuf, rx_info,ena_buf->len, (uintmax_t)ena_buf->paddr); 1046 1047 return (0); 1048 1049 exit: 1050 m_freem(rx_info->mbuf); 1051 rx_info->mbuf = NULL; 1052 return (EFAULT); 1053 } 1054 1055 static void 1056 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring, 1057 struct ena_rx_buffer *rx_info) 1058 { 1059 1060 if (rx_info->mbuf == NULL) { 1061 ena_trace(ENA_WARNING, "Trying to free unallocated buffer\n"); 1062 return; 1063 } 1064 1065 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 1066 BUS_DMASYNC_POSTREAD); 1067 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map); 1068 m_freem(rx_info->mbuf); 1069 rx_info->mbuf = NULL; 1070 } 1071 1072 /** 1073 * ena_refill_rx_bufs - Refills ring with descriptors 1074 * @rx_ring: the ring which we want to feed with free descriptors 1075 * @num: number of descriptors to refill 1076 * Refills the ring with newly allocated DMA-mapped mbufs for receiving 1077 **/ 1078 int 1079 ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num) 1080 { 1081 struct ena_adapter *adapter = rx_ring->adapter; 1082 uint16_t next_to_use, req_id; 1083 uint32_t i; 1084 int rc; 1085 1086 ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d\n", 1087 rx_ring->qid); 1088 1089 next_to_use = rx_ring->next_to_use; 1090 1091 for (i = 0; i < num; i++) { 1092 struct ena_rx_buffer *rx_info; 1093 1094 ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, 1095 "RX buffer - next to use: %d\n", next_to_use); 1096 1097 req_id = rx_ring->free_rx_ids[next_to_use]; 1098 rx_info = &rx_ring->rx_buffer_info[req_id]; 1099 #ifdef DEV_NETMAP 1100 if (ena_rx_ring_in_netmap(adapter, rx_ring->qid)) 1101 rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, rx_info); 1102 else 1103 #endif /* DEV_NETMAP */ 1104 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info); 1105 if (unlikely(rc != 0)) { 1106 ena_trace(ENA_WARNING, 1107 "failed to alloc buffer for rx queue %d\n", 1108 rx_ring->qid); 1109 break; 1110 } 1111 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq, 1112 &rx_info->ena_buf, req_id); 1113 if (unlikely(rc != 0)) { 1114 ena_trace(ENA_WARNING, 1115 "failed to add buffer for rx queue %d\n", 1116 rx_ring->qid); 1117 break; 1118 } 1119 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, 1120 rx_ring->ring_size); 1121 } 1122 1123 if (unlikely(i < num)) { 1124 counter_u64_add(rx_ring->rx_stats.refil_partial, 1); 1125 ena_trace(ENA_WARNING, 1126 "refilled rx qid %d with only %d mbufs (from %d)\n", 1127 rx_ring->qid, i, num); 1128 } 1129 1130 if (likely(i != 0)) 1131 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq); 1132 1133 rx_ring->next_to_use = next_to_use; 1134 return (i); 1135 } 1136 1137 int 1138 ena_update_buf_ring_size(struct ena_adapter *adapter, 1139 uint32_t new_buf_ring_size) 1140 { 1141 uint32_t old_buf_ring_size; 1142 int rc = 0; 1143 bool dev_was_up; 1144 1145 ENA_LOCK_LOCK(adapter); 1146 1147 old_buf_ring_size = adapter->buf_ring_size; 1148 adapter->buf_ring_size = new_buf_ring_size; 1149 1150 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 1151 ena_down(adapter); 1152 1153 /* Reconfigure buf ring for all Tx rings. */ 1154 ena_free_all_io_rings_resources(adapter); 1155 ena_init_io_rings_advanced(adapter); 1156 if (dev_was_up) { 1157 /* 1158 * If ena_up() fails, it's not because of recent buf_ring size 1159 * changes. Because of that, we just want to revert old drbr 1160 * value and trigger the reset because something else had to 1161 * go wrong. 1162 */ 1163 rc = ena_up(adapter); 1164 if (unlikely(rc != 0)) { 1165 device_printf(adapter->pdev, 1166 "Failed to configure device after setting new drbr size: %u. Reverting old value: %u and triggering the reset\n", 1167 new_buf_ring_size, old_buf_ring_size); 1168 1169 /* Revert old size and trigger the reset */ 1170 adapter->buf_ring_size = old_buf_ring_size; 1171 ena_free_all_io_rings_resources(adapter); 1172 ena_init_io_rings_advanced(adapter); 1173 1174 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, 1175 adapter); 1176 ena_trigger_reset(adapter, ENA_REGS_RESET_OS_TRIGGER); 1177 1178 } 1179 } 1180 1181 ENA_LOCK_UNLOCK(adapter); 1182 1183 return (rc); 1184 } 1185 1186 int 1187 ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size, 1188 uint32_t new_rx_size) 1189 { 1190 uint32_t old_tx_size, old_rx_size; 1191 int rc = 0; 1192 bool dev_was_up; 1193 1194 ENA_LOCK_LOCK(adapter); 1195 1196 old_tx_size = adapter->tx_ring_size; 1197 old_rx_size = adapter->rx_ring_size; 1198 adapter->tx_ring_size = new_tx_size; 1199 adapter->rx_ring_size = new_rx_size; 1200 1201 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 1202 ena_down(adapter); 1203 1204 /* Configure queues with new size. */ 1205 ena_init_io_rings_basic(adapter); 1206 if (dev_was_up) { 1207 rc = ena_up(adapter); 1208 if (unlikely(rc != 0)) { 1209 device_printf(adapter->pdev, 1210 "Failed to configure device with the new sizes - Tx: %u Rx: %u. Reverting old values - Tx: %u Rx: %u\n", 1211 new_tx_size, new_rx_size, old_tx_size, old_rx_size); 1212 1213 /* Revert old size. */ 1214 adapter->tx_ring_size = old_tx_size; 1215 adapter->rx_ring_size = old_rx_size; 1216 ena_init_io_rings_basic(adapter); 1217 1218 /* And try again. */ 1219 rc = ena_up(adapter); 1220 if (unlikely(rc != 0)) { 1221 device_printf(adapter->pdev, 1222 "Failed to revert old queue sizes. Triggering device reset.\n"); 1223 /* 1224 * If we've failed again, something had to go 1225 * wrong. After reset, the device should try to 1226 * go up 1227 */ 1228 ENA_FLAG_SET_ATOMIC( 1229 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 1230 ena_trigger_reset(adapter, 1231 ENA_REGS_RESET_OS_TRIGGER); 1232 } 1233 } 1234 } 1235 1236 ENA_LOCK_UNLOCK(adapter); 1237 1238 return (rc); 1239 } 1240 1241 static void 1242 ena_update_io_rings(struct ena_adapter *adapter, uint32_t num) 1243 { 1244 ena_free_all_io_rings_resources(adapter); 1245 /* Force indirection table to be reinitialized */ 1246 ena_com_rss_destroy(adapter->ena_dev); 1247 1248 adapter->num_io_queues = num; 1249 ena_init_io_rings(adapter); 1250 } 1251 1252 /* Caller should sanitize new_num */ 1253 int 1254 ena_update_io_queue_nb(struct ena_adapter *adapter, uint32_t new_num) 1255 { 1256 uint32_t old_num; 1257 int rc = 0; 1258 bool dev_was_up; 1259 1260 ENA_LOCK_LOCK(adapter); 1261 1262 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 1263 old_num = adapter->num_io_queues; 1264 ena_down(adapter); 1265 1266 ena_update_io_rings(adapter, new_num); 1267 1268 if (dev_was_up) { 1269 rc = ena_up(adapter); 1270 if (unlikely(rc != 0)) { 1271 device_printf(adapter->pdev, 1272 "Failed to configure device with %u IO queues. " 1273 "Reverting to previous value: %u\n", 1274 new_num, old_num); 1275 1276 ena_update_io_rings(adapter, old_num); 1277 1278 rc = ena_up(adapter); 1279 if (unlikely(rc != 0)) { 1280 device_printf(adapter->pdev, 1281 "Failed to revert to previous setup IO " 1282 "queues. Triggering device reset.\n"); 1283 ENA_FLAG_SET_ATOMIC( 1284 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 1285 ena_trigger_reset(adapter, 1286 ENA_REGS_RESET_OS_TRIGGER); 1287 } 1288 } 1289 } 1290 1291 ENA_LOCK_UNLOCK(adapter); 1292 1293 return (rc); 1294 } 1295 1296 static void 1297 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid) 1298 { 1299 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 1300 unsigned int i; 1301 1302 for (i = 0; i < rx_ring->ring_size; i++) { 1303 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i]; 1304 1305 if (rx_info->mbuf != NULL) 1306 ena_free_rx_mbuf(adapter, rx_ring, rx_info); 1307 #ifdef DEV_NETMAP 1308 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) && 1309 (adapter->ifp->if_capenable & IFCAP_NETMAP)) { 1310 if (rx_info->netmap_buf_idx != 0) 1311 ena_netmap_free_rx_slot(adapter, rx_ring, 1312 rx_info); 1313 } 1314 #endif /* DEV_NETMAP */ 1315 } 1316 } 1317 1318 /** 1319 * ena_refill_all_rx_bufs - allocate all queues Rx buffers 1320 * @adapter: network interface device structure 1321 * 1322 */ 1323 static void 1324 ena_refill_all_rx_bufs(struct ena_adapter *adapter) 1325 { 1326 struct ena_ring *rx_ring; 1327 int i, rc, bufs_num; 1328 1329 for (i = 0; i < adapter->num_io_queues; i++) { 1330 rx_ring = &adapter->rx_ring[i]; 1331 bufs_num = rx_ring->ring_size - 1; 1332 rc = ena_refill_rx_bufs(rx_ring, bufs_num); 1333 if (unlikely(rc != bufs_num)) 1334 ena_trace(ENA_WARNING, "refilling Queue %d failed. " 1335 "Allocated %d buffers from: %d\n", i, rc, bufs_num); 1336 #ifdef DEV_NETMAP 1337 rx_ring->initialized = true; 1338 #endif /* DEV_NETMAP */ 1339 } 1340 } 1341 1342 static void 1343 ena_free_all_rx_bufs(struct ena_adapter *adapter) 1344 { 1345 int i; 1346 1347 for (i = 0; i < adapter->num_io_queues; i++) 1348 ena_free_rx_bufs(adapter, i); 1349 } 1350 1351 /** 1352 * ena_free_tx_bufs - Free Tx Buffers per Queue 1353 * @adapter: network interface device structure 1354 * @qid: queue index 1355 **/ 1356 static void 1357 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid) 1358 { 1359 bool print_once = true; 1360 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 1361 1362 ENA_RING_MTX_LOCK(tx_ring); 1363 for (int i = 0; i < tx_ring->ring_size; i++) { 1364 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i]; 1365 1366 if (tx_info->mbuf == NULL) 1367 continue; 1368 1369 if (print_once) { 1370 device_printf(adapter->pdev, 1371 "free uncompleted tx mbuf qid %d idx 0x%x\n", 1372 qid, i); 1373 print_once = false; 1374 } else { 1375 ena_trace(ENA_DBG, 1376 "free uncompleted tx mbuf qid %d idx 0x%x\n", 1377 qid, i); 1378 } 1379 1380 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 1381 BUS_DMASYNC_POSTWRITE); 1382 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); 1383 1384 m_free(tx_info->mbuf); 1385 tx_info->mbuf = NULL; 1386 } 1387 ENA_RING_MTX_UNLOCK(tx_ring); 1388 } 1389 1390 static void 1391 ena_free_all_tx_bufs(struct ena_adapter *adapter) 1392 { 1393 1394 for (int i = 0; i < adapter->num_io_queues; i++) 1395 ena_free_tx_bufs(adapter, i); 1396 } 1397 1398 static void 1399 ena_destroy_all_tx_queues(struct ena_adapter *adapter) 1400 { 1401 uint16_t ena_qid; 1402 int i; 1403 1404 for (i = 0; i < adapter->num_io_queues; i++) { 1405 ena_qid = ENA_IO_TXQ_IDX(i); 1406 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1407 } 1408 } 1409 1410 static void 1411 ena_destroy_all_rx_queues(struct ena_adapter *adapter) 1412 { 1413 uint16_t ena_qid; 1414 int i; 1415 1416 for (i = 0; i < adapter->num_io_queues; i++) { 1417 ena_qid = ENA_IO_RXQ_IDX(i); 1418 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1419 } 1420 } 1421 1422 static void 1423 ena_destroy_all_io_queues(struct ena_adapter *adapter) 1424 { 1425 struct ena_que *queue; 1426 int i; 1427 1428 for (i = 0; i < adapter->num_io_queues; i++) { 1429 queue = &adapter->que[i]; 1430 while (taskqueue_cancel(queue->cleanup_tq, 1431 &queue->cleanup_task, NULL)) 1432 taskqueue_drain(queue->cleanup_tq, 1433 &queue->cleanup_task); 1434 taskqueue_free(queue->cleanup_tq); 1435 } 1436 1437 ena_destroy_all_tx_queues(adapter); 1438 ena_destroy_all_rx_queues(adapter); 1439 } 1440 1441 static int 1442 ena_create_io_queues(struct ena_adapter *adapter) 1443 { 1444 struct ena_com_dev *ena_dev = adapter->ena_dev; 1445 struct ena_com_create_io_ctx ctx; 1446 struct ena_ring *ring; 1447 struct ena_que *queue; 1448 uint16_t ena_qid; 1449 uint32_t msix_vector; 1450 int rc, i; 1451 1452 /* Create TX queues */ 1453 for (i = 0; i < adapter->num_io_queues; i++) { 1454 msix_vector = ENA_IO_IRQ_IDX(i); 1455 ena_qid = ENA_IO_TXQ_IDX(i); 1456 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1457 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1458 ctx.queue_size = adapter->tx_ring_size; 1459 ctx.msix_vector = msix_vector; 1460 ctx.qid = ena_qid; 1461 rc = ena_com_create_io_queue(ena_dev, &ctx); 1462 if (rc != 0) { 1463 device_printf(adapter->pdev, 1464 "Failed to create io TX queue #%d rc: %d\n", i, rc); 1465 goto err_tx; 1466 } 1467 ring = &adapter->tx_ring[i]; 1468 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1469 &ring->ena_com_io_sq, 1470 &ring->ena_com_io_cq); 1471 if (rc != 0) { 1472 device_printf(adapter->pdev, 1473 "Failed to get TX queue handlers. TX queue num" 1474 " %d rc: %d\n", i, rc); 1475 ena_com_destroy_io_queue(ena_dev, ena_qid); 1476 goto err_tx; 1477 } 1478 } 1479 1480 /* Create RX queues */ 1481 for (i = 0; i < adapter->num_io_queues; i++) { 1482 msix_vector = ENA_IO_IRQ_IDX(i); 1483 ena_qid = ENA_IO_RXQ_IDX(i); 1484 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1485 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1486 ctx.queue_size = adapter->rx_ring_size; 1487 ctx.msix_vector = msix_vector; 1488 ctx.qid = ena_qid; 1489 rc = ena_com_create_io_queue(ena_dev, &ctx); 1490 if (unlikely(rc != 0)) { 1491 device_printf(adapter->pdev, 1492 "Failed to create io RX queue[%d] rc: %d\n", i, rc); 1493 goto err_rx; 1494 } 1495 1496 ring = &adapter->rx_ring[i]; 1497 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1498 &ring->ena_com_io_sq, 1499 &ring->ena_com_io_cq); 1500 if (unlikely(rc != 0)) { 1501 device_printf(adapter->pdev, 1502 "Failed to get RX queue handlers. RX queue num" 1503 " %d rc: %d\n", i, rc); 1504 ena_com_destroy_io_queue(ena_dev, ena_qid); 1505 goto err_rx; 1506 } 1507 } 1508 1509 for (i = 0; i < adapter->num_io_queues; i++) { 1510 queue = &adapter->que[i]; 1511 1512 NET_TASK_INIT(&queue->cleanup_task, 0, ena_cleanup, queue); 1513 queue->cleanup_tq = taskqueue_create_fast("ena cleanup", 1514 M_WAITOK, taskqueue_thread_enqueue, &queue->cleanup_tq); 1515 1516 taskqueue_start_threads(&queue->cleanup_tq, 1, PI_NET, 1517 "%s queue %d cleanup", 1518 device_get_nameunit(adapter->pdev), i); 1519 } 1520 1521 return (0); 1522 1523 err_rx: 1524 while (i--) 1525 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i)); 1526 i = adapter->num_io_queues; 1527 err_tx: 1528 while (i--) 1529 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i)); 1530 1531 return (ENXIO); 1532 } 1533 1534 /********************************************************************* 1535 * 1536 * MSIX & Interrupt Service routine 1537 * 1538 **********************************************************************/ 1539 1540 /** 1541 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue 1542 * @arg: interrupt number 1543 **/ 1544 static void 1545 ena_intr_msix_mgmnt(void *arg) 1546 { 1547 struct ena_adapter *adapter = (struct ena_adapter *)arg; 1548 1549 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); 1550 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))) 1551 ena_com_aenq_intr_handler(adapter->ena_dev, arg); 1552 } 1553 1554 /** 1555 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx 1556 * @arg: queue 1557 **/ 1558 static int 1559 ena_handle_msix(void *arg) 1560 { 1561 struct ena_que *queue = arg; 1562 struct ena_adapter *adapter = queue->adapter; 1563 if_t ifp = adapter->ifp; 1564 1565 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) 1566 return (FILTER_STRAY); 1567 1568 taskqueue_enqueue(queue->cleanup_tq, &queue->cleanup_task); 1569 1570 return (FILTER_HANDLED); 1571 } 1572 1573 static int 1574 ena_enable_msix(struct ena_adapter *adapter) 1575 { 1576 device_t dev = adapter->pdev; 1577 int msix_vecs, msix_req; 1578 int i, rc = 0; 1579 1580 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { 1581 device_printf(dev, "Error, MSI-X is already enabled\n"); 1582 return (EINVAL); 1583 } 1584 1585 /* Reserved the max msix vectors we might need */ 1586 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues); 1587 1588 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry), 1589 M_DEVBUF, M_WAITOK | M_ZERO); 1590 1591 ena_trace(ENA_DBG, "trying to enable MSI-X, vectors: %d\n", msix_vecs); 1592 1593 for (i = 0; i < msix_vecs; i++) { 1594 adapter->msix_entries[i].entry = i; 1595 /* Vectors must start from 1 */ 1596 adapter->msix_entries[i].vector = i + 1; 1597 } 1598 1599 msix_req = msix_vecs; 1600 rc = pci_alloc_msix(dev, &msix_vecs); 1601 if (unlikely(rc != 0)) { 1602 device_printf(dev, 1603 "Failed to enable MSIX, vectors %d rc %d\n", msix_vecs, rc); 1604 1605 rc = ENOSPC; 1606 goto err_msix_free; 1607 } 1608 1609 if (msix_vecs != msix_req) { 1610 if (msix_vecs == ENA_ADMIN_MSIX_VEC) { 1611 device_printf(dev, 1612 "Not enough number of MSI-x allocated: %d\n", 1613 msix_vecs); 1614 pci_release_msi(dev); 1615 rc = ENOSPC; 1616 goto err_msix_free; 1617 } 1618 device_printf(dev, "Enable only %d MSI-x (out of %d), reduce " 1619 "the number of queues\n", msix_vecs, msix_req); 1620 } 1621 1622 adapter->msix_vecs = msix_vecs; 1623 ENA_FLAG_SET_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); 1624 1625 return (0); 1626 1627 err_msix_free: 1628 free(adapter->msix_entries, M_DEVBUF); 1629 adapter->msix_entries = NULL; 1630 1631 return (rc); 1632 } 1633 1634 static void 1635 ena_setup_mgmnt_intr(struct ena_adapter *adapter) 1636 { 1637 1638 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, 1639 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s", 1640 device_get_nameunit(adapter->pdev)); 1641 /* 1642 * Handler is NULL on purpose, it will be set 1643 * when mgmnt interrupt is acquired 1644 */ 1645 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL; 1646 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter; 1647 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector = 1648 adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector; 1649 } 1650 1651 static int 1652 ena_setup_io_intr(struct ena_adapter *adapter) 1653 { 1654 static int last_bind_cpu = -1; 1655 int irq_idx; 1656 1657 if (adapter->msix_entries == NULL) 1658 return (EINVAL); 1659 1660 for (int i = 0; i < adapter->num_io_queues; i++) { 1661 irq_idx = ENA_IO_IRQ_IDX(i); 1662 1663 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, 1664 "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i); 1665 adapter->irq_tbl[irq_idx].handler = ena_handle_msix; 1666 adapter->irq_tbl[irq_idx].data = &adapter->que[i]; 1667 adapter->irq_tbl[irq_idx].vector = 1668 adapter->msix_entries[irq_idx].vector; 1669 ena_trace(ENA_INFO | ENA_IOQ, "ena_setup_io_intr vector: %d\n", 1670 adapter->msix_entries[irq_idx].vector); 1671 1672 /* 1673 * We want to bind rings to the corresponding cpu 1674 * using something similar to the RSS round-robin technique. 1675 */ 1676 if (unlikely(last_bind_cpu < 0)) 1677 last_bind_cpu = CPU_FIRST(); 1678 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = 1679 last_bind_cpu; 1680 last_bind_cpu = CPU_NEXT(last_bind_cpu); 1681 } 1682 1683 return (0); 1684 } 1685 1686 static int 1687 ena_request_mgmnt_irq(struct ena_adapter *adapter) 1688 { 1689 struct ena_irq *irq; 1690 unsigned long flags; 1691 int rc, rcc; 1692 1693 flags = RF_ACTIVE | RF_SHAREABLE; 1694 1695 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 1696 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, 1697 &irq->vector, flags); 1698 1699 if (unlikely(irq->res == NULL)) { 1700 device_printf(adapter->pdev, "could not allocate " 1701 "irq vector: %d\n", irq->vector); 1702 return (ENXIO); 1703 } 1704 1705 rc = bus_setup_intr(adapter->pdev, irq->res, 1706 INTR_TYPE_NET | INTR_MPSAFE, NULL, ena_intr_msix_mgmnt, 1707 irq->data, &irq->cookie); 1708 if (unlikely(rc != 0)) { 1709 device_printf(adapter->pdev, "failed to register " 1710 "interrupt handler for irq %ju: %d\n", 1711 rman_get_start(irq->res), rc); 1712 goto err_res_free; 1713 } 1714 irq->requested = true; 1715 1716 return (rc); 1717 1718 err_res_free: 1719 ena_trace(ENA_INFO | ENA_ADMQ, "releasing resource for irq %d\n", 1720 irq->vector); 1721 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1722 irq->vector, irq->res); 1723 if (unlikely(rcc != 0)) 1724 device_printf(adapter->pdev, "dev has no parent while " 1725 "releasing res for irq: %d\n", irq->vector); 1726 irq->res = NULL; 1727 1728 return (rc); 1729 } 1730 1731 static int 1732 ena_request_io_irq(struct ena_adapter *adapter) 1733 { 1734 struct ena_irq *irq; 1735 unsigned long flags = 0; 1736 int rc = 0, i, rcc; 1737 1738 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter))) { 1739 device_printf(adapter->pdev, 1740 "failed to request I/O IRQ: MSI-X is not enabled\n"); 1741 return (EINVAL); 1742 } else { 1743 flags = RF_ACTIVE | RF_SHAREABLE; 1744 } 1745 1746 for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { 1747 irq = &adapter->irq_tbl[i]; 1748 1749 if (unlikely(irq->requested)) 1750 continue; 1751 1752 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, 1753 &irq->vector, flags); 1754 if (unlikely(irq->res == NULL)) { 1755 rc = ENOMEM; 1756 device_printf(adapter->pdev, "could not allocate " 1757 "irq vector: %d\n", irq->vector); 1758 goto err; 1759 } 1760 1761 rc = bus_setup_intr(adapter->pdev, irq->res, 1762 INTR_TYPE_NET | INTR_MPSAFE, irq->handler, NULL, 1763 irq->data, &irq->cookie); 1764 if (unlikely(rc != 0)) { 1765 device_printf(adapter->pdev, "failed to register " 1766 "interrupt handler for irq %ju: %d\n", 1767 rman_get_start(irq->res), rc); 1768 goto err; 1769 } 1770 irq->requested = true; 1771 1772 ena_trace(ENA_INFO, "queue %d - cpu %d\n", 1773 i - ENA_IO_IRQ_FIRST_IDX, irq->cpu); 1774 } 1775 1776 return (rc); 1777 1778 err: 1779 1780 for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) { 1781 irq = &adapter->irq_tbl[i]; 1782 rcc = 0; 1783 1784 /* Once we entered err: section and irq->requested is true we 1785 free both intr and resources */ 1786 if (irq->requested) 1787 rcc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); 1788 if (unlikely(rcc != 0)) 1789 device_printf(adapter->pdev, "could not release" 1790 " irq: %d, error: %d\n", irq->vector, rcc); 1791 1792 /* If we entred err: section without irq->requested set we know 1793 it was bus_alloc_resource_any() that needs cleanup, provided 1794 res is not NULL. In case res is NULL no work in needed in 1795 this iteration */ 1796 rcc = 0; 1797 if (irq->res != NULL) { 1798 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1799 irq->vector, irq->res); 1800 } 1801 if (unlikely(rcc != 0)) 1802 device_printf(adapter->pdev, "dev has no parent while " 1803 "releasing res for irq: %d\n", irq->vector); 1804 irq->requested = false; 1805 irq->res = NULL; 1806 } 1807 1808 return (rc); 1809 } 1810 1811 static void 1812 ena_free_mgmnt_irq(struct ena_adapter *adapter) 1813 { 1814 struct ena_irq *irq; 1815 int rc; 1816 1817 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 1818 if (irq->requested) { 1819 ena_trace(ENA_INFO | ENA_ADMQ, "tear down irq: %d\n", 1820 irq->vector); 1821 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); 1822 if (unlikely(rc != 0)) 1823 device_printf(adapter->pdev, "failed to tear " 1824 "down irq: %d\n", irq->vector); 1825 irq->requested = 0; 1826 } 1827 1828 if (irq->res != NULL) { 1829 ena_trace(ENA_INFO | ENA_ADMQ, "release resource irq: %d\n", 1830 irq->vector); 1831 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1832 irq->vector, irq->res); 1833 irq->res = NULL; 1834 if (unlikely(rc != 0)) 1835 device_printf(adapter->pdev, "dev has no parent while " 1836 "releasing res for irq: %d\n", irq->vector); 1837 } 1838 } 1839 1840 static void 1841 ena_free_io_irq(struct ena_adapter *adapter) 1842 { 1843 struct ena_irq *irq; 1844 int rc; 1845 1846 for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { 1847 irq = &adapter->irq_tbl[i]; 1848 if (irq->requested) { 1849 ena_trace(ENA_INFO | ENA_IOQ, "tear down irq: %d\n", 1850 irq->vector); 1851 rc = bus_teardown_intr(adapter->pdev, irq->res, 1852 irq->cookie); 1853 if (unlikely(rc != 0)) { 1854 device_printf(adapter->pdev, "failed to tear " 1855 "down irq: %d\n", irq->vector); 1856 } 1857 irq->requested = 0; 1858 } 1859 1860 if (irq->res != NULL) { 1861 ena_trace(ENA_INFO | ENA_IOQ, "release resource irq: %d\n", 1862 irq->vector); 1863 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1864 irq->vector, irq->res); 1865 irq->res = NULL; 1866 if (unlikely(rc != 0)) { 1867 device_printf(adapter->pdev, "dev has no parent" 1868 " while releasing res for irq: %d\n", 1869 irq->vector); 1870 } 1871 } 1872 } 1873 } 1874 1875 static void 1876 ena_free_irqs(struct ena_adapter* adapter) 1877 { 1878 1879 ena_free_io_irq(adapter); 1880 ena_free_mgmnt_irq(adapter); 1881 ena_disable_msix(adapter); 1882 } 1883 1884 static void 1885 ena_disable_msix(struct ena_adapter *adapter) 1886 { 1887 1888 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { 1889 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); 1890 pci_release_msi(adapter->pdev); 1891 } 1892 1893 adapter->msix_vecs = 0; 1894 if (adapter->msix_entries != NULL) 1895 free(adapter->msix_entries, M_DEVBUF); 1896 adapter->msix_entries = NULL; 1897 } 1898 1899 static void 1900 ena_unmask_all_io_irqs(struct ena_adapter *adapter) 1901 { 1902 struct ena_com_io_cq* io_cq; 1903 struct ena_eth_io_intr_reg intr_reg; 1904 uint16_t ena_qid; 1905 int i; 1906 1907 /* Unmask interrupts for all queues */ 1908 for (i = 0; i < adapter->num_io_queues; i++) { 1909 ena_qid = ENA_IO_TXQ_IDX(i); 1910 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 1911 ena_com_update_intr_reg(&intr_reg, 0, 0, true); 1912 ena_com_unmask_intr(io_cq, &intr_reg); 1913 } 1914 } 1915 1916 /* Configure the Rx forwarding */ 1917 static int 1918 ena_rss_configure(struct ena_adapter *adapter) 1919 { 1920 struct ena_com_dev *ena_dev = adapter->ena_dev; 1921 int rc; 1922 1923 /* In case the RSS table was destroyed */ 1924 if (!ena_dev->rss.tbl_log_size) { 1925 rc = ena_rss_init_default(adapter); 1926 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 1927 device_printf(adapter->pdev, 1928 "WARNING: RSS was not properly re-initialized," 1929 " it will affect bandwidth\n"); 1930 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 1931 return (rc); 1932 } 1933 } 1934 1935 /* Set indirect table */ 1936 rc = ena_com_indirect_table_set(ena_dev); 1937 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 1938 return (rc); 1939 1940 /* Configure hash function (if supported) */ 1941 rc = ena_com_set_hash_function(ena_dev); 1942 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 1943 return (rc); 1944 1945 /* Configure hash inputs (if supported) */ 1946 rc = ena_com_set_hash_ctrl(ena_dev); 1947 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 1948 return (rc); 1949 1950 return (0); 1951 } 1952 1953 static int 1954 ena_up_complete(struct ena_adapter *adapter) 1955 { 1956 int rc; 1957 1958 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { 1959 rc = ena_rss_configure(adapter); 1960 if (rc != 0) { 1961 device_printf(adapter->pdev, 1962 "Failed to configure RSS\n"); 1963 return (rc); 1964 } 1965 } 1966 1967 rc = ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu); 1968 if (unlikely(rc != 0)) 1969 return (rc); 1970 1971 ena_refill_all_rx_bufs(adapter); 1972 ena_reset_counters((counter_u64_t *)&adapter->hw_stats, 1973 sizeof(adapter->hw_stats)); 1974 1975 return (0); 1976 } 1977 1978 int 1979 ena_up(struct ena_adapter *adapter) 1980 { 1981 int rc = 0; 1982 1983 if (unlikely(device_is_attached(adapter->pdev) == 0)) { 1984 device_printf(adapter->pdev, "device is not attached!\n"); 1985 return (ENXIO); 1986 } 1987 1988 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 1989 return (0); 1990 1991 device_printf(adapter->pdev, "device is going UP\n"); 1992 1993 /* setup interrupts for IO queues */ 1994 rc = ena_setup_io_intr(adapter); 1995 if (unlikely(rc != 0)) { 1996 ena_trace(ENA_ALERT, "error setting up IO interrupt\n"); 1997 goto error; 1998 } 1999 rc = ena_request_io_irq(adapter); 2000 if (unlikely(rc != 0)) { 2001 ena_trace(ENA_ALERT, "err_req_irq\n"); 2002 goto error; 2003 } 2004 2005 device_printf(adapter->pdev, 2006 "Creating %u IO queues. Rx queue size: %d, Tx queue size: %d, LLQ is %s\n", 2007 adapter->num_io_queues, 2008 adapter->rx_ring_size, 2009 adapter->tx_ring_size, 2010 (adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) ? 2011 "ENABLED" : "DISABLED"); 2012 2013 /* allocate transmit descriptors */ 2014 rc = ena_setup_all_tx_resources(adapter); 2015 if (unlikely(rc != 0)) { 2016 ena_trace(ENA_ALERT, "err_setup_tx\n"); 2017 goto err_setup_tx; 2018 } 2019 2020 /* allocate receive descriptors */ 2021 rc = ena_setup_all_rx_resources(adapter); 2022 if (unlikely(rc != 0)) { 2023 ena_trace(ENA_ALERT, "err_setup_rx\n"); 2024 goto err_setup_rx; 2025 } 2026 2027 /* create IO queues for Rx & Tx */ 2028 rc = ena_create_io_queues(adapter); 2029 if (unlikely(rc != 0)) { 2030 ena_trace(ENA_ALERT, 2031 "create IO queues failed\n"); 2032 goto err_io_que; 2033 } 2034 2035 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) 2036 if_link_state_change(adapter->ifp, LINK_STATE_UP); 2037 2038 rc = ena_up_complete(adapter); 2039 if (unlikely(rc != 0)) 2040 goto err_up_complete; 2041 2042 counter_u64_add(adapter->dev_stats.interface_up, 1); 2043 2044 ena_update_hwassist(adapter); 2045 2046 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, 2047 IFF_DRV_OACTIVE); 2048 2049 /* Activate timer service only if the device is running. 2050 * If this flag is not set, it means that the driver is being 2051 * reset and timer service will be activated afterwards. 2052 */ 2053 if (ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) { 2054 callout_reset_sbt(&adapter->timer_service, SBT_1S, 2055 SBT_1S, ena_timer_service, (void *)adapter, 0); 2056 } 2057 2058 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter); 2059 2060 ena_unmask_all_io_irqs(adapter); 2061 2062 return (0); 2063 2064 err_up_complete: 2065 ena_destroy_all_io_queues(adapter); 2066 err_io_que: 2067 ena_free_all_rx_resources(adapter); 2068 err_setup_rx: 2069 ena_free_all_tx_resources(adapter); 2070 err_setup_tx: 2071 ena_free_io_irq(adapter); 2072 error: 2073 return (rc); 2074 } 2075 2076 static uint64_t 2077 ena_get_counter(if_t ifp, ift_counter cnt) 2078 { 2079 struct ena_adapter *adapter; 2080 struct ena_hw_stats *stats; 2081 2082 adapter = if_getsoftc(ifp); 2083 stats = &adapter->hw_stats; 2084 2085 switch (cnt) { 2086 case IFCOUNTER_IPACKETS: 2087 return (counter_u64_fetch(stats->rx_packets)); 2088 case IFCOUNTER_OPACKETS: 2089 return (counter_u64_fetch(stats->tx_packets)); 2090 case IFCOUNTER_IBYTES: 2091 return (counter_u64_fetch(stats->rx_bytes)); 2092 case IFCOUNTER_OBYTES: 2093 return (counter_u64_fetch(stats->tx_bytes)); 2094 case IFCOUNTER_IQDROPS: 2095 return (counter_u64_fetch(stats->rx_drops)); 2096 case IFCOUNTER_OQDROPS: 2097 return (counter_u64_fetch(stats->tx_drops)); 2098 default: 2099 return (if_get_counter_default(ifp, cnt)); 2100 } 2101 } 2102 2103 static int 2104 ena_media_change(if_t ifp) 2105 { 2106 /* Media Change is not supported by firmware */ 2107 return (0); 2108 } 2109 2110 static void 2111 ena_media_status(if_t ifp, struct ifmediareq *ifmr) 2112 { 2113 struct ena_adapter *adapter = if_getsoftc(ifp); 2114 ena_trace(ENA_DBG, "enter\n"); 2115 2116 ENA_LOCK_LOCK(adapter); 2117 2118 ifmr->ifm_status = IFM_AVALID; 2119 ifmr->ifm_active = IFM_ETHER; 2120 2121 if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) { 2122 ENA_LOCK_UNLOCK(adapter); 2123 ena_trace(ENA_INFO, "Link is down\n"); 2124 return; 2125 } 2126 2127 ifmr->ifm_status |= IFM_ACTIVE; 2128 ifmr->ifm_active |= IFM_UNKNOWN | IFM_FDX; 2129 2130 ENA_LOCK_UNLOCK(adapter); 2131 } 2132 2133 static void 2134 ena_init(void *arg) 2135 { 2136 struct ena_adapter *adapter = (struct ena_adapter *)arg; 2137 2138 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 2139 ENA_LOCK_LOCK(adapter); 2140 ena_up(adapter); 2141 ENA_LOCK_UNLOCK(adapter); 2142 } 2143 } 2144 2145 static int 2146 ena_ioctl(if_t ifp, u_long command, caddr_t data) 2147 { 2148 struct ena_adapter *adapter; 2149 struct ifreq *ifr; 2150 int rc; 2151 2152 adapter = ifp->if_softc; 2153 ifr = (struct ifreq *)data; 2154 2155 /* 2156 * Acquiring lock to prevent from running up and down routines parallel. 2157 */ 2158 rc = 0; 2159 switch (command) { 2160 case SIOCSIFMTU: 2161 if (ifp->if_mtu == ifr->ifr_mtu) 2162 break; 2163 ENA_LOCK_LOCK(adapter); 2164 ena_down(adapter); 2165 2166 ena_change_mtu(ifp, ifr->ifr_mtu); 2167 2168 rc = ena_up(adapter); 2169 ENA_LOCK_UNLOCK(adapter); 2170 break; 2171 2172 case SIOCSIFFLAGS: 2173 if ((ifp->if_flags & IFF_UP) != 0) { 2174 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2175 if ((ifp->if_flags & (IFF_PROMISC | 2176 IFF_ALLMULTI)) != 0) { 2177 device_printf(adapter->pdev, 2178 "ioctl promisc/allmulti\n"); 2179 } 2180 } else { 2181 ENA_LOCK_LOCK(adapter); 2182 rc = ena_up(adapter); 2183 ENA_LOCK_UNLOCK(adapter); 2184 } 2185 } else { 2186 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2187 ENA_LOCK_LOCK(adapter); 2188 ena_down(adapter); 2189 ENA_LOCK_UNLOCK(adapter); 2190 } 2191 } 2192 break; 2193 2194 case SIOCADDMULTI: 2195 case SIOCDELMULTI: 2196 break; 2197 2198 case SIOCSIFMEDIA: 2199 case SIOCGIFMEDIA: 2200 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 2201 break; 2202 2203 case SIOCSIFCAP: 2204 { 2205 int reinit = 0; 2206 2207 if (ifr->ifr_reqcap != ifp->if_capenable) { 2208 ifp->if_capenable = ifr->ifr_reqcap; 2209 reinit = 1; 2210 } 2211 2212 if ((reinit != 0) && 2213 ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)) { 2214 ENA_LOCK_LOCK(adapter); 2215 ena_down(adapter); 2216 rc = ena_up(adapter); 2217 ENA_LOCK_UNLOCK(adapter); 2218 } 2219 } 2220 2221 break; 2222 default: 2223 rc = ether_ioctl(ifp, command, data); 2224 break; 2225 } 2226 2227 return (rc); 2228 } 2229 2230 static int 2231 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat) 2232 { 2233 int caps = 0; 2234 2235 if ((feat->offload.tx & 2236 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2237 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK | 2238 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)) != 0) 2239 caps |= IFCAP_TXCSUM; 2240 2241 if ((feat->offload.tx & 2242 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK | 2243 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)) != 0) 2244 caps |= IFCAP_TXCSUM_IPV6; 2245 2246 if ((feat->offload.tx & 2247 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0) 2248 caps |= IFCAP_TSO4; 2249 2250 if ((feat->offload.tx & 2251 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0) 2252 caps |= IFCAP_TSO6; 2253 2254 if ((feat->offload.rx_supported & 2255 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK | 2256 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)) != 0) 2257 caps |= IFCAP_RXCSUM; 2258 2259 if ((feat->offload.rx_supported & 2260 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) != 0) 2261 caps |= IFCAP_RXCSUM_IPV6; 2262 2263 caps |= IFCAP_LRO | IFCAP_JUMBO_MTU; 2264 2265 return (caps); 2266 } 2267 2268 static void 2269 ena_update_host_info(struct ena_admin_host_info *host_info, if_t ifp) 2270 { 2271 2272 host_info->supported_network_features[0] = 2273 (uint32_t)if_getcapabilities(ifp); 2274 } 2275 2276 static void 2277 ena_update_hwassist(struct ena_adapter *adapter) 2278 { 2279 if_t ifp = adapter->ifp; 2280 uint32_t feat = adapter->tx_offload_cap; 2281 int cap = if_getcapenable(ifp); 2282 int flags = 0; 2283 2284 if_clearhwassist(ifp); 2285 2286 if ((cap & IFCAP_TXCSUM) != 0) { 2287 if ((feat & 2288 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK) != 0) 2289 flags |= CSUM_IP; 2290 if ((feat & 2291 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2292 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)) != 0) 2293 flags |= CSUM_IP_UDP | CSUM_IP_TCP; 2294 } 2295 2296 if ((cap & IFCAP_TXCSUM_IPV6) != 0) 2297 flags |= CSUM_IP6_UDP | CSUM_IP6_TCP; 2298 2299 if ((cap & IFCAP_TSO4) != 0) 2300 flags |= CSUM_IP_TSO; 2301 2302 if ((cap & IFCAP_TSO6) != 0) 2303 flags |= CSUM_IP6_TSO; 2304 2305 if_sethwassistbits(ifp, flags, 0); 2306 } 2307 2308 static int 2309 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter, 2310 struct ena_com_dev_get_features_ctx *feat) 2311 { 2312 if_t ifp; 2313 int caps = 0; 2314 2315 ifp = adapter->ifp = if_gethandle(IFT_ETHER); 2316 if (unlikely(ifp == NULL)) { 2317 ena_trace(ENA_ALERT, "can not allocate ifnet structure\n"); 2318 return (ENXIO); 2319 } 2320 if_initname(ifp, device_get_name(pdev), device_get_unit(pdev)); 2321 if_setdev(ifp, pdev); 2322 if_setsoftc(ifp, adapter); 2323 2324 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | 2325 IFF_KNOWSEPOCH); 2326 if_setinitfn(ifp, ena_init); 2327 if_settransmitfn(ifp, ena_mq_start); 2328 if_setqflushfn(ifp, ena_qflush); 2329 if_setioctlfn(ifp, ena_ioctl); 2330 if_setgetcounterfn(ifp, ena_get_counter); 2331 2332 if_setsendqlen(ifp, adapter->tx_ring_size); 2333 if_setsendqready(ifp); 2334 if_setmtu(ifp, ETHERMTU); 2335 if_setbaudrate(ifp, 0); 2336 /* Zeroize capabilities... */ 2337 if_setcapabilities(ifp, 0); 2338 if_setcapenable(ifp, 0); 2339 /* check hardware support */ 2340 caps = ena_get_dev_offloads(feat); 2341 /* ... and set them */ 2342 if_setcapabilitiesbit(ifp, caps, 0); 2343 2344 /* TSO parameters */ 2345 ifp->if_hw_tsomax = ENA_TSO_MAXSIZE - 2346 (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2347 ifp->if_hw_tsomaxsegcount = adapter->max_tx_sgl_size - 1; 2348 ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE; 2349 2350 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 2351 if_setcapenable(ifp, if_getcapabilities(ifp)); 2352 2353 /* 2354 * Specify the media types supported by this adapter and register 2355 * callbacks to update media and link information 2356 */ 2357 ifmedia_init(&adapter->media, IFM_IMASK, 2358 ena_media_change, ena_media_status); 2359 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2360 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2361 2362 ether_ifattach(ifp, adapter->mac_addr); 2363 2364 return (0); 2365 } 2366 2367 void 2368 ena_down(struct ena_adapter *adapter) 2369 { 2370 int rc; 2371 2372 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 2373 return; 2374 2375 device_printf(adapter->pdev, "device is going DOWN\n"); 2376 2377 callout_drain(&adapter->timer_service); 2378 2379 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter); 2380 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, 2381 IFF_DRV_RUNNING); 2382 2383 ena_free_io_irq(adapter); 2384 2385 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) { 2386 rc = ena_com_dev_reset(adapter->ena_dev, 2387 adapter->reset_reason); 2388 if (unlikely(rc != 0)) 2389 device_printf(adapter->pdev, 2390 "Device reset failed\n"); 2391 } 2392 2393 ena_destroy_all_io_queues(adapter); 2394 2395 ena_free_all_tx_bufs(adapter); 2396 ena_free_all_rx_bufs(adapter); 2397 ena_free_all_tx_resources(adapter); 2398 ena_free_all_rx_resources(adapter); 2399 2400 counter_u64_add(adapter->dev_stats.interface_down, 1); 2401 } 2402 2403 static uint32_t 2404 ena_calc_max_io_queue_num(device_t pdev, struct ena_com_dev *ena_dev, 2405 struct ena_com_dev_get_features_ctx *get_feat_ctx) 2406 { 2407 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues; 2408 2409 /* Regular queues capabilities */ 2410 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 2411 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 2412 &get_feat_ctx->max_queue_ext.max_queue_ext; 2413 io_rx_num = min_t(int, max_queue_ext->max_rx_sq_num, 2414 max_queue_ext->max_rx_cq_num); 2415 2416 io_tx_sq_num = max_queue_ext->max_tx_sq_num; 2417 io_tx_cq_num = max_queue_ext->max_tx_cq_num; 2418 } else { 2419 struct ena_admin_queue_feature_desc *max_queues = 2420 &get_feat_ctx->max_queues; 2421 io_tx_sq_num = max_queues->max_sq_num; 2422 io_tx_cq_num = max_queues->max_cq_num; 2423 io_rx_num = min_t(int, io_tx_sq_num, io_tx_cq_num); 2424 } 2425 2426 /* In case of LLQ use the llq fields for the tx SQ/CQ */ 2427 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 2428 io_tx_sq_num = get_feat_ctx->llq.max_llq_num; 2429 2430 max_num_io_queues = min_t(uint32_t, mp_ncpus, ENA_MAX_NUM_IO_QUEUES); 2431 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_rx_num); 2432 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_sq_num); 2433 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_cq_num); 2434 /* 1 IRQ for for mgmnt and 1 IRQ for each TX/RX pair */ 2435 max_num_io_queues = min_t(uint32_t, max_num_io_queues, 2436 pci_msix_count(pdev) - 1); 2437 2438 return (max_num_io_queues); 2439 } 2440 2441 static int 2442 ena_enable_wc(struct resource *res) 2443 { 2444 #if defined(__i386) || defined(__amd64) || defined(__aarch64__) 2445 vm_offset_t va; 2446 vm_size_t len; 2447 int rc; 2448 2449 va = (vm_offset_t)rman_get_virtual(res); 2450 len = rman_get_size(res); 2451 /* Enable write combining */ 2452 rc = pmap_change_attr(va, len, VM_MEMATTR_WRITE_COMBINING); 2453 if (unlikely(rc != 0)) { 2454 ena_trace(ENA_ALERT, "pmap_change_attr failed, %d\n", rc); 2455 return (rc); 2456 } 2457 2458 return (0); 2459 #endif 2460 return (EOPNOTSUPP); 2461 } 2462 2463 static int 2464 ena_set_queues_placement_policy(device_t pdev, struct ena_com_dev *ena_dev, 2465 struct ena_admin_feature_llq_desc *llq, 2466 struct ena_llq_configurations *llq_default_configurations) 2467 { 2468 struct ena_adapter *adapter = device_get_softc(pdev); 2469 int rc, rid; 2470 uint32_t llq_feature_mask; 2471 2472 llq_feature_mask = 1 << ENA_ADMIN_LLQ; 2473 if (!(ena_dev->supported_features & llq_feature_mask)) { 2474 device_printf(pdev, 2475 "LLQ is not supported. Fallback to host mode policy.\n"); 2476 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2477 return (0); 2478 } 2479 2480 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); 2481 if (unlikely(rc != 0)) { 2482 device_printf(pdev, "Failed to configure the device mode. " 2483 "Fallback to host mode policy.\n"); 2484 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2485 return (0); 2486 } 2487 2488 /* Nothing to config, exit */ 2489 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 2490 return (0); 2491 2492 /* Try to allocate resources for LLQ bar */ 2493 rid = PCIR_BAR(ENA_MEM_BAR); 2494 adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, 2495 &rid, RF_ACTIVE); 2496 if (unlikely(adapter->memory == NULL)) { 2497 device_printf(pdev, "unable to allocate LLQ bar resource. " 2498 "Fallback to host mode policy.\n"); 2499 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2500 return (0); 2501 } 2502 2503 /* Enable write combining for better LLQ performance */ 2504 rc = ena_enable_wc(adapter->memory); 2505 if (unlikely(rc != 0)) { 2506 device_printf(pdev, "failed to enable write combining.\n"); 2507 return (rc); 2508 } 2509 2510 /* 2511 * Save virtual address of the device's memory region 2512 * for the ena_com layer. 2513 */ 2514 ena_dev->mem_bar = rman_get_virtual(adapter->memory); 2515 2516 return (0); 2517 } 2518 2519 static inline 2520 void set_default_llq_configurations(struct ena_llq_configurations *llq_config) 2521 { 2522 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; 2523 llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B; 2524 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; 2525 llq_config->llq_num_decs_before_header = 2526 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; 2527 llq_config->llq_ring_entry_size_value = 128; 2528 } 2529 2530 static int 2531 ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx) 2532 { 2533 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq; 2534 struct ena_com_dev *ena_dev = ctx->ena_dev; 2535 uint32_t tx_queue_size = ENA_DEFAULT_RING_SIZE; 2536 uint32_t rx_queue_size = ENA_DEFAULT_RING_SIZE; 2537 uint32_t max_tx_queue_size; 2538 uint32_t max_rx_queue_size; 2539 2540 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 2541 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 2542 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext; 2543 max_rx_queue_size = min_t(uint32_t, 2544 max_queue_ext->max_rx_cq_depth, 2545 max_queue_ext->max_rx_sq_depth); 2546 max_tx_queue_size = max_queue_ext->max_tx_cq_depth; 2547 2548 if (ena_dev->tx_mem_queue_type == 2549 ENA_ADMIN_PLACEMENT_POLICY_DEV) 2550 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2551 llq->max_llq_depth); 2552 else 2553 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2554 max_queue_ext->max_tx_sq_depth); 2555 2556 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2557 max_queue_ext->max_per_packet_tx_descs); 2558 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2559 max_queue_ext->max_per_packet_rx_descs); 2560 } else { 2561 struct ena_admin_queue_feature_desc *max_queues = 2562 &ctx->get_feat_ctx->max_queues; 2563 max_rx_queue_size = min_t(uint32_t, 2564 max_queues->max_cq_depth, 2565 max_queues->max_sq_depth); 2566 max_tx_queue_size = max_queues->max_cq_depth; 2567 2568 if (ena_dev->tx_mem_queue_type == 2569 ENA_ADMIN_PLACEMENT_POLICY_DEV) 2570 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2571 llq->max_llq_depth); 2572 else 2573 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2574 max_queues->max_sq_depth); 2575 2576 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2577 max_queues->max_packet_tx_descs); 2578 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2579 max_queues->max_packet_rx_descs); 2580 } 2581 2582 /* round down to the nearest power of 2 */ 2583 max_tx_queue_size = 1 << (flsl(max_tx_queue_size) - 1); 2584 max_rx_queue_size = 1 << (flsl(max_rx_queue_size) - 1); 2585 2586 tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE, 2587 max_tx_queue_size); 2588 rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE, 2589 max_rx_queue_size); 2590 2591 tx_queue_size = 1 << (flsl(tx_queue_size) - 1); 2592 rx_queue_size = 1 << (flsl(rx_queue_size) - 1); 2593 2594 ctx->max_tx_queue_size = max_tx_queue_size; 2595 ctx->max_rx_queue_size = max_rx_queue_size; 2596 ctx->tx_queue_size = tx_queue_size; 2597 ctx->rx_queue_size = rx_queue_size; 2598 2599 return (0); 2600 } 2601 2602 static int 2603 ena_rss_init_default(struct ena_adapter *adapter) 2604 { 2605 struct ena_com_dev *ena_dev = adapter->ena_dev; 2606 device_t dev = adapter->pdev; 2607 int qid, rc, i; 2608 2609 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 2610 if (unlikely(rc != 0)) { 2611 device_printf(dev, "Cannot init indirect table\n"); 2612 return (rc); 2613 } 2614 2615 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 2616 qid = i % adapter->num_io_queues; 2617 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 2618 ENA_IO_RXQ_IDX(qid)); 2619 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 2620 device_printf(dev, "Cannot fill indirect table\n"); 2621 goto err_rss_destroy; 2622 } 2623 } 2624 2625 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 2626 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 2627 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 2628 device_printf(dev, "Cannot fill hash function\n"); 2629 goto err_rss_destroy; 2630 } 2631 2632 rc = ena_com_set_default_hash_ctrl(ena_dev); 2633 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 2634 device_printf(dev, "Cannot fill hash control\n"); 2635 goto err_rss_destroy; 2636 } 2637 2638 return (0); 2639 2640 err_rss_destroy: 2641 ena_com_rss_destroy(ena_dev); 2642 return (rc); 2643 } 2644 2645 static void 2646 ena_rss_init_default_deferred(void *arg) 2647 { 2648 struct ena_adapter *adapter; 2649 devclass_t dc; 2650 int max; 2651 int rc; 2652 2653 dc = devclass_find("ena"); 2654 if (unlikely(dc == NULL)) { 2655 ena_trace(ENA_ALERT, "No devclass ena\n"); 2656 return; 2657 } 2658 2659 max = devclass_get_maxunit(dc); 2660 while (max-- >= 0) { 2661 adapter = devclass_get_softc(dc, max); 2662 if (adapter != NULL) { 2663 rc = ena_rss_init_default(adapter); 2664 ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 2665 if (unlikely(rc != 0)) { 2666 device_printf(adapter->pdev, 2667 "WARNING: RSS was not properly initialized," 2668 " it will affect bandwidth\n"); 2669 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 2670 } 2671 } 2672 } 2673 } 2674 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, ena_rss_init_default_deferred, NULL); 2675 2676 static void 2677 ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev) 2678 { 2679 struct ena_admin_host_info *host_info; 2680 uintptr_t rid; 2681 int rc; 2682 2683 /* Allocate only the host info */ 2684 rc = ena_com_allocate_host_info(ena_dev); 2685 if (unlikely(rc != 0)) { 2686 ena_trace(ENA_ALERT, "Cannot allocate host info\n"); 2687 return; 2688 } 2689 2690 host_info = ena_dev->host_attr.host_info; 2691 2692 if (pci_get_id(dev, PCI_ID_RID, &rid) == 0) 2693 host_info->bdf = rid; 2694 host_info->os_type = ENA_ADMIN_OS_FREEBSD; 2695 host_info->kernel_ver = osreldate; 2696 2697 sprintf(host_info->kernel_ver_str, "%d", osreldate); 2698 host_info->os_dist = 0; 2699 strncpy(host_info->os_dist_str, osrelease, 2700 sizeof(host_info->os_dist_str) - 1); 2701 2702 host_info->driver_version = 2703 (DRV_MODULE_VER_MAJOR) | 2704 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 2705 (DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 2706 host_info->num_cpus = mp_ncpus; 2707 2708 rc = ena_com_set_host_attributes(ena_dev); 2709 if (unlikely(rc != 0)) { 2710 if (rc == EOPNOTSUPP) 2711 ena_trace(ENA_WARNING, "Cannot set host attributes\n"); 2712 else 2713 ena_trace(ENA_ALERT, "Cannot set host attributes\n"); 2714 2715 goto err; 2716 } 2717 2718 return; 2719 2720 err: 2721 ena_com_delete_host_info(ena_dev); 2722 } 2723 2724 static int 2725 ena_device_init(struct ena_adapter *adapter, device_t pdev, 2726 struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active) 2727 { 2728 struct ena_com_dev* ena_dev = adapter->ena_dev; 2729 bool readless_supported; 2730 uint32_t aenq_groups; 2731 int dma_width; 2732 int rc; 2733 2734 rc = ena_com_mmio_reg_read_request_init(ena_dev); 2735 if (unlikely(rc != 0)) { 2736 device_printf(pdev, "failed to init mmio read less\n"); 2737 return (rc); 2738 } 2739 2740 /* 2741 * The PCIe configuration space revision id indicate if mmio reg 2742 * read is disabled 2743 */ 2744 readless_supported = !(pci_get_revid(pdev) & ENA_MMIO_DISABLE_REG_READ); 2745 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 2746 2747 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 2748 if (unlikely(rc != 0)) { 2749 device_printf(pdev, "Can not reset device\n"); 2750 goto err_mmio_read_less; 2751 } 2752 2753 rc = ena_com_validate_version(ena_dev); 2754 if (unlikely(rc != 0)) { 2755 device_printf(pdev, "device version is too low\n"); 2756 goto err_mmio_read_less; 2757 } 2758 2759 dma_width = ena_com_get_dma_width(ena_dev); 2760 if (unlikely(dma_width < 0)) { 2761 device_printf(pdev, "Invalid dma width value %d", dma_width); 2762 rc = dma_width; 2763 goto err_mmio_read_less; 2764 } 2765 adapter->dma_width = dma_width; 2766 2767 /* ENA admin level init */ 2768 rc = ena_com_admin_init(ena_dev, &aenq_handlers); 2769 if (unlikely(rc != 0)) { 2770 device_printf(pdev, 2771 "Can not initialize ena admin queue with device\n"); 2772 goto err_mmio_read_less; 2773 } 2774 2775 /* 2776 * To enable the msix interrupts the driver needs to know the number 2777 * of queues. So the driver uses polling mode to retrieve this 2778 * information 2779 */ 2780 ena_com_set_admin_polling_mode(ena_dev, true); 2781 2782 ena_config_host_info(ena_dev, pdev); 2783 2784 /* Get Device Attributes */ 2785 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 2786 if (unlikely(rc != 0)) { 2787 device_printf(pdev, 2788 "Cannot get attribute for ena device rc: %d\n", rc); 2789 goto err_admin_init; 2790 } 2791 2792 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 2793 BIT(ENA_ADMIN_FATAL_ERROR) | 2794 BIT(ENA_ADMIN_WARNING) | 2795 BIT(ENA_ADMIN_NOTIFICATION) | 2796 BIT(ENA_ADMIN_KEEP_ALIVE); 2797 2798 aenq_groups &= get_feat_ctx->aenq.supported_groups; 2799 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 2800 if (unlikely(rc != 0)) { 2801 device_printf(pdev, "Cannot configure aenq groups rc: %d\n", rc); 2802 goto err_admin_init; 2803 } 2804 2805 *wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 2806 2807 return (0); 2808 2809 err_admin_init: 2810 ena_com_delete_host_info(ena_dev); 2811 ena_com_admin_destroy(ena_dev); 2812 err_mmio_read_less: 2813 ena_com_mmio_reg_read_request_destroy(ena_dev); 2814 2815 return (rc); 2816 } 2817 2818 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter) 2819 { 2820 struct ena_com_dev *ena_dev = adapter->ena_dev; 2821 int rc; 2822 2823 rc = ena_enable_msix(adapter); 2824 if (unlikely(rc != 0)) { 2825 device_printf(adapter->pdev, "Error with MSI-X enablement\n"); 2826 return (rc); 2827 } 2828 2829 ena_setup_mgmnt_intr(adapter); 2830 2831 rc = ena_request_mgmnt_irq(adapter); 2832 if (unlikely(rc != 0)) { 2833 device_printf(adapter->pdev, "Cannot setup mgmnt queue intr\n"); 2834 goto err_disable_msix; 2835 } 2836 2837 ena_com_set_admin_polling_mode(ena_dev, false); 2838 2839 ena_com_admin_aenq_enable(ena_dev); 2840 2841 return (0); 2842 2843 err_disable_msix: 2844 ena_disable_msix(adapter); 2845 2846 return (rc); 2847 } 2848 2849 /* Function called on ENA_ADMIN_KEEP_ALIVE event */ 2850 static void ena_keep_alive_wd(void *adapter_data, 2851 struct ena_admin_aenq_entry *aenq_e) 2852 { 2853 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 2854 struct ena_admin_aenq_keep_alive_desc *desc; 2855 sbintime_t stime; 2856 uint64_t rx_drops; 2857 uint64_t tx_drops; 2858 2859 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; 2860 2861 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; 2862 tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low; 2863 counter_u64_zero(adapter->hw_stats.rx_drops); 2864 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops); 2865 counter_u64_zero(adapter->hw_stats.tx_drops); 2866 counter_u64_add(adapter->hw_stats.tx_drops, tx_drops); 2867 2868 stime = getsbinuptime(); 2869 atomic_store_rel_64(&adapter->keep_alive_timestamp, stime); 2870 } 2871 2872 /* Check for keep alive expiration */ 2873 static void check_for_missing_keep_alive(struct ena_adapter *adapter) 2874 { 2875 sbintime_t timestamp, time; 2876 2877 if (adapter->wd_active == 0) 2878 return; 2879 2880 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2881 return; 2882 2883 timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp); 2884 time = getsbinuptime() - timestamp; 2885 if (unlikely(time > adapter->keep_alive_timeout)) { 2886 device_printf(adapter->pdev, 2887 "Keep alive watchdog timeout.\n"); 2888 counter_u64_add(adapter->dev_stats.wd_expired, 1); 2889 ena_trigger_reset(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO); 2890 } 2891 } 2892 2893 /* Check if admin queue is enabled */ 2894 static void check_for_admin_com_state(struct ena_adapter *adapter) 2895 { 2896 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == 2897 false)) { 2898 device_printf(adapter->pdev, 2899 "ENA admin queue is not in running state!\n"); 2900 counter_u64_add(adapter->dev_stats.admin_q_pause, 1); 2901 ena_trigger_reset(adapter, ENA_REGS_RESET_ADMIN_TO); 2902 } 2903 } 2904 2905 static int 2906 check_for_rx_interrupt_queue(struct ena_adapter *adapter, 2907 struct ena_ring *rx_ring) 2908 { 2909 if (likely(rx_ring->first_interrupt)) 2910 return (0); 2911 2912 if (ena_com_cq_empty(rx_ring->ena_com_io_cq)) 2913 return (0); 2914 2915 rx_ring->no_interrupt_event_cnt++; 2916 2917 if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) { 2918 device_printf(adapter->pdev, "Potential MSIX issue on Rx side " 2919 "Queue = %d. Reset the device\n", rx_ring->qid); 2920 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_INTERRUPT); 2921 return (EIO); 2922 } 2923 2924 return (0); 2925 } 2926 2927 static int 2928 check_missing_comp_in_tx_queue(struct ena_adapter *adapter, 2929 struct ena_ring *tx_ring) 2930 { 2931 struct bintime curtime, time; 2932 struct ena_tx_buffer *tx_buf; 2933 sbintime_t time_offset; 2934 uint32_t missed_tx = 0; 2935 int i, rc = 0; 2936 2937 getbinuptime(&curtime); 2938 2939 for (i = 0; i < tx_ring->ring_size; i++) { 2940 tx_buf = &tx_ring->tx_buffer_info[i]; 2941 2942 if (bintime_isset(&tx_buf->timestamp) == 0) 2943 continue; 2944 2945 time = curtime; 2946 bintime_sub(&time, &tx_buf->timestamp); 2947 time_offset = bttosbt(time); 2948 2949 if (unlikely(!tx_ring->first_interrupt && 2950 time_offset > 2 * adapter->missing_tx_timeout)) { 2951 /* 2952 * If after graceful period interrupt is still not 2953 * received, we schedule a reset. 2954 */ 2955 device_printf(adapter->pdev, 2956 "Potential MSIX issue on Tx side Queue = %d. " 2957 "Reset the device\n", tx_ring->qid); 2958 ena_trigger_reset(adapter, 2959 ENA_REGS_RESET_MISS_INTERRUPT); 2960 return (EIO); 2961 } 2962 2963 /* Check again if packet is still waiting */ 2964 if (unlikely(time_offset > adapter->missing_tx_timeout)) { 2965 2966 if (!tx_buf->print_once) 2967 ena_trace(ENA_WARNING, "Found a Tx that wasn't " 2968 "completed on time, qid %d, index %d.\n", 2969 tx_ring->qid, i); 2970 2971 tx_buf->print_once = true; 2972 missed_tx++; 2973 } 2974 } 2975 2976 if (unlikely(missed_tx > adapter->missing_tx_threshold)) { 2977 device_printf(adapter->pdev, 2978 "The number of lost tx completion is above the threshold " 2979 "(%d > %d). Reset the device\n", 2980 missed_tx, adapter->missing_tx_threshold); 2981 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_TX_CMPL); 2982 rc = EIO; 2983 } 2984 2985 counter_u64_add(tx_ring->tx_stats.missing_tx_comp, missed_tx); 2986 2987 return (rc); 2988 } 2989 2990 /* 2991 * Check for TX which were not completed on time. 2992 * Timeout is defined by "missing_tx_timeout". 2993 * Reset will be performed if number of incompleted 2994 * transactions exceeds "missing_tx_threshold". 2995 */ 2996 static void 2997 check_for_missing_completions(struct ena_adapter *adapter) 2998 { 2999 struct ena_ring *tx_ring; 3000 struct ena_ring *rx_ring; 3001 int i, budget, rc; 3002 3003 /* Make sure the driver doesn't turn the device in other process */ 3004 rmb(); 3005 3006 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3007 return; 3008 3009 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 3010 return; 3011 3012 if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT) 3013 return; 3014 3015 budget = adapter->missing_tx_max_queues; 3016 3017 for (i = adapter->next_monitored_tx_qid; i < adapter->num_io_queues; i++) { 3018 tx_ring = &adapter->tx_ring[i]; 3019 rx_ring = &adapter->rx_ring[i]; 3020 3021 rc = check_missing_comp_in_tx_queue(adapter, tx_ring); 3022 if (unlikely(rc != 0)) 3023 return; 3024 3025 rc = check_for_rx_interrupt_queue(adapter, rx_ring); 3026 if (unlikely(rc != 0)) 3027 return; 3028 3029 budget--; 3030 if (budget == 0) { 3031 i++; 3032 break; 3033 } 3034 } 3035 3036 adapter->next_monitored_tx_qid = i % adapter->num_io_queues; 3037 } 3038 3039 /* trigger rx cleanup after 2 consecutive detections */ 3040 #define EMPTY_RX_REFILL 2 3041 /* For the rare case where the device runs out of Rx descriptors and the 3042 * msix handler failed to refill new Rx descriptors (due to a lack of memory 3043 * for example). 3044 * This case will lead to a deadlock: 3045 * The device won't send interrupts since all the new Rx packets will be dropped 3046 * The msix handler won't allocate new Rx descriptors so the device won't be 3047 * able to send new packets. 3048 * 3049 * When such a situation is detected - execute rx cleanup task in another thread 3050 */ 3051 static void 3052 check_for_empty_rx_ring(struct ena_adapter *adapter) 3053 { 3054 struct ena_ring *rx_ring; 3055 int i, refill_required; 3056 3057 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3058 return; 3059 3060 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 3061 return; 3062 3063 for (i = 0; i < adapter->num_io_queues; i++) { 3064 rx_ring = &adapter->rx_ring[i]; 3065 3066 refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq); 3067 if (unlikely(refill_required == (rx_ring->ring_size - 1))) { 3068 rx_ring->empty_rx_queue++; 3069 3070 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) { 3071 counter_u64_add(rx_ring->rx_stats.empty_rx_ring, 3072 1); 3073 3074 device_printf(adapter->pdev, 3075 "trigger refill for ring %d\n", i); 3076 3077 taskqueue_enqueue(rx_ring->que->cleanup_tq, 3078 &rx_ring->que->cleanup_task); 3079 rx_ring->empty_rx_queue = 0; 3080 } 3081 } else { 3082 rx_ring->empty_rx_queue = 0; 3083 } 3084 } 3085 } 3086 3087 static void ena_update_hints(struct ena_adapter *adapter, 3088 struct ena_admin_ena_hw_hints *hints) 3089 { 3090 struct ena_com_dev *ena_dev = adapter->ena_dev; 3091 3092 if (hints->admin_completion_tx_timeout) 3093 ena_dev->admin_queue.completion_timeout = 3094 hints->admin_completion_tx_timeout * 1000; 3095 3096 if (hints->mmio_read_timeout) 3097 /* convert to usec */ 3098 ena_dev->mmio_read.reg_read_to = 3099 hints->mmio_read_timeout * 1000; 3100 3101 if (hints->missed_tx_completion_count_threshold_to_reset) 3102 adapter->missing_tx_threshold = 3103 hints->missed_tx_completion_count_threshold_to_reset; 3104 3105 if (hints->missing_tx_completion_timeout) { 3106 if (hints->missing_tx_completion_timeout == 3107 ENA_HW_HINTS_NO_TIMEOUT) 3108 adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT; 3109 else 3110 adapter->missing_tx_timeout = 3111 SBT_1MS * hints->missing_tx_completion_timeout; 3112 } 3113 3114 if (hints->driver_watchdog_timeout) { 3115 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 3116 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 3117 else 3118 adapter->keep_alive_timeout = 3119 SBT_1MS * hints->driver_watchdog_timeout; 3120 } 3121 } 3122 3123 static void 3124 ena_timer_service(void *data) 3125 { 3126 struct ena_adapter *adapter = (struct ena_adapter *)data; 3127 struct ena_admin_host_info *host_info = 3128 adapter->ena_dev->host_attr.host_info; 3129 3130 check_for_missing_keep_alive(adapter); 3131 3132 check_for_admin_com_state(adapter); 3133 3134 check_for_missing_completions(adapter); 3135 3136 check_for_empty_rx_ring(adapter); 3137 3138 if (host_info != NULL) 3139 ena_update_host_info(host_info, adapter->ifp); 3140 3141 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 3142 device_printf(adapter->pdev, "Trigger reset is on\n"); 3143 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task); 3144 return; 3145 } 3146 3147 /* 3148 * Schedule another timeout one second from now. 3149 */ 3150 callout_schedule_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 0); 3151 } 3152 3153 void 3154 ena_destroy_device(struct ena_adapter *adapter, bool graceful) 3155 { 3156 if_t ifp = adapter->ifp; 3157 struct ena_com_dev *ena_dev = adapter->ena_dev; 3158 bool dev_up; 3159 3160 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) 3161 return; 3162 3163 if_link_state_change(ifp, LINK_STATE_DOWN); 3164 3165 callout_drain(&adapter->timer_service); 3166 3167 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 3168 if (dev_up) 3169 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 3170 3171 if (!graceful) 3172 ena_com_set_admin_running_state(ena_dev, false); 3173 3174 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3175 ena_down(adapter); 3176 3177 /* 3178 * Stop the device from sending AENQ events (if the device was up, and 3179 * the trigger reset was on, ena_down already performs device reset) 3180 */ 3181 if (!(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter) && dev_up)) 3182 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); 3183 3184 ena_free_mgmnt_irq(adapter); 3185 3186 ena_disable_msix(adapter); 3187 3188 /* 3189 * IO rings resources should be freed because `ena_restore_device()` 3190 * calls (not directly) `ena_enable_msix()`, which re-allocates MSIX 3191 * vectors. The amount of MSIX vectors after destroy-restore may be 3192 * different than before. Therefore, IO rings resources should be 3193 * established from scratch each time. 3194 */ 3195 ena_free_all_io_rings_resources(adapter); 3196 3197 ena_com_abort_admin_commands(ena_dev); 3198 3199 ena_com_wait_for_abort_completion(ena_dev); 3200 3201 ena_com_admin_destroy(ena_dev); 3202 3203 ena_com_mmio_reg_read_request_destroy(ena_dev); 3204 3205 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3206 3207 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 3208 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3209 } 3210 3211 static int 3212 ena_device_validate_params(struct ena_adapter *adapter, 3213 struct ena_com_dev_get_features_ctx *get_feat_ctx) 3214 { 3215 3216 if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr, 3217 ETHER_ADDR_LEN) != 0) { 3218 device_printf(adapter->pdev, 3219 "Error, mac address are different\n"); 3220 return (EINVAL); 3221 } 3222 3223 if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) { 3224 device_printf(adapter->pdev, 3225 "Error, device max mtu is smaller than ifp MTU\n"); 3226 return (EINVAL); 3227 } 3228 3229 return 0; 3230 } 3231 3232 int 3233 ena_restore_device(struct ena_adapter *adapter) 3234 { 3235 struct ena_com_dev_get_features_ctx get_feat_ctx; 3236 struct ena_com_dev *ena_dev = adapter->ena_dev; 3237 if_t ifp = adapter->ifp; 3238 device_t dev = adapter->pdev; 3239 int wd_active; 3240 int rc; 3241 3242 ENA_FLAG_SET_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3243 3244 rc = ena_device_init(adapter, dev, &get_feat_ctx, &wd_active); 3245 if (rc != 0) { 3246 device_printf(dev, "Cannot initialize device\n"); 3247 goto err; 3248 } 3249 /* 3250 * Only enable WD if it was enabled before reset, so it won't override 3251 * value set by the user by the sysctl. 3252 */ 3253 if (adapter->wd_active != 0) 3254 adapter->wd_active = wd_active; 3255 3256 rc = ena_device_validate_params(adapter, &get_feat_ctx); 3257 if (rc != 0) { 3258 device_printf(dev, "Validation of device parameters failed\n"); 3259 goto err_device_destroy; 3260 } 3261 3262 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3263 /* Make sure we don't have a race with AENQ Links state handler */ 3264 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) 3265 if_link_state_change(ifp, LINK_STATE_UP); 3266 3267 rc = ena_enable_msix_and_set_admin_interrupts(adapter); 3268 if (rc != 0) { 3269 device_printf(dev, "Enable MSI-X failed\n"); 3270 goto err_device_destroy; 3271 } 3272 3273 /* 3274 * Effective value of used MSIX vectors should be the same as before 3275 * `ena_destroy_device()`, if possible, or closest to it if less vectors 3276 * are available. 3277 */ 3278 if ((adapter->msix_vecs - ENA_ADMIN_MSIX_VEC) < adapter->num_io_queues) 3279 adapter->num_io_queues = 3280 adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; 3281 3282 /* Re-initialize rings basic information */ 3283 ena_init_io_rings(adapter); 3284 3285 /* If the interface was up before the reset bring it up */ 3286 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) { 3287 rc = ena_up(adapter); 3288 if (rc != 0) { 3289 device_printf(dev, "Failed to create I/O queues\n"); 3290 goto err_disable_msix; 3291 } 3292 } 3293 3294 /* Indicate that device is running again and ready to work */ 3295 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3296 3297 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) { 3298 /* 3299 * As the AENQ handlers weren't executed during reset because 3300 * the flag ENA_FLAG_DEVICE_RUNNING was turned off, the 3301 * timestamp must be updated again That will prevent next reset 3302 * caused by missing keep alive. 3303 */ 3304 adapter->keep_alive_timestamp = getsbinuptime(); 3305 callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 3306 ena_timer_service, (void *)adapter, 0); 3307 } 3308 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 3309 3310 device_printf(dev, 3311 "Device reset completed successfully, Driver info: %s\n", ena_version); 3312 3313 return (rc); 3314 3315 err_disable_msix: 3316 ena_free_mgmnt_irq(adapter); 3317 ena_disable_msix(adapter); 3318 err_device_destroy: 3319 ena_com_abort_admin_commands(ena_dev); 3320 ena_com_wait_for_abort_completion(ena_dev); 3321 ena_com_admin_destroy(ena_dev); 3322 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE); 3323 ena_com_mmio_reg_read_request_destroy(ena_dev); 3324 err: 3325 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3326 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3327 device_printf(dev, "Reset attempt failed. Can not reset the device\n"); 3328 3329 return (rc); 3330 } 3331 3332 static void 3333 ena_reset_task(void *arg, int pending) 3334 { 3335 struct ena_adapter *adapter = (struct ena_adapter *)arg; 3336 3337 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 3338 device_printf(adapter->pdev, 3339 "device reset scheduled but trigger_reset is off\n"); 3340 return; 3341 } 3342 3343 ENA_LOCK_LOCK(adapter); 3344 ena_destroy_device(adapter, false); 3345 ena_restore_device(adapter); 3346 ENA_LOCK_UNLOCK(adapter); 3347 } 3348 3349 /** 3350 * ena_attach - Device Initialization Routine 3351 * @pdev: device information struct 3352 * 3353 * Returns 0 on success, otherwise on failure. 3354 * 3355 * ena_attach initializes an adapter identified by a device structure. 3356 * The OS initialization, configuring of the adapter private structure, 3357 * and a hardware reset occur. 3358 **/ 3359 static int 3360 ena_attach(device_t pdev) 3361 { 3362 struct ena_com_dev_get_features_ctx get_feat_ctx; 3363 struct ena_llq_configurations llq_config; 3364 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; 3365 static int version_printed; 3366 struct ena_adapter *adapter; 3367 struct ena_com_dev *ena_dev = NULL; 3368 uint32_t max_num_io_queues; 3369 int rid, rc; 3370 3371 adapter = device_get_softc(pdev); 3372 adapter->pdev = pdev; 3373 3374 ENA_LOCK_INIT(adapter); 3375 3376 /* 3377 * Set up the timer service - driver is responsible for avoiding 3378 * concurrency, as the callout won't be using any locking inside. 3379 */ 3380 callout_init(&adapter->timer_service, true); 3381 adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO; 3382 adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO; 3383 adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES; 3384 adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD; 3385 3386 if (version_printed++ == 0) 3387 device_printf(pdev, "%s\n", ena_version); 3388 3389 /* Allocate memory for ena_dev structure */ 3390 ena_dev = malloc(sizeof(struct ena_com_dev), M_DEVBUF, 3391 M_WAITOK | M_ZERO); 3392 3393 adapter->ena_dev = ena_dev; 3394 ena_dev->dmadev = pdev; 3395 3396 rid = PCIR_BAR(ENA_REG_BAR); 3397 adapter->memory = NULL; 3398 adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, 3399 &rid, RF_ACTIVE); 3400 if (unlikely(adapter->registers == NULL)) { 3401 device_printf(pdev, 3402 "unable to allocate bus resource: registers!\n"); 3403 rc = ENOMEM; 3404 goto err_dev_free; 3405 } 3406 3407 ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF, 3408 M_WAITOK | M_ZERO); 3409 3410 /* Store register resources */ 3411 ((struct ena_bus*)(ena_dev->bus))->reg_bar_t = 3412 rman_get_bustag(adapter->registers); 3413 ((struct ena_bus*)(ena_dev->bus))->reg_bar_h = 3414 rman_get_bushandle(adapter->registers); 3415 3416 if (unlikely(((struct ena_bus*)(ena_dev->bus))->reg_bar_h == 0)) { 3417 device_printf(pdev, "failed to pmap registers bar\n"); 3418 rc = ENXIO; 3419 goto err_bus_free; 3420 } 3421 3422 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 3423 3424 /* Initially clear all the flags */ 3425 ENA_FLAG_ZERO(adapter); 3426 3427 /* Device initialization */ 3428 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active); 3429 if (unlikely(rc != 0)) { 3430 device_printf(pdev, "ENA device init failed! (err: %d)\n", rc); 3431 rc = ENXIO; 3432 goto err_bus_free; 3433 } 3434 3435 set_default_llq_configurations(&llq_config); 3436 3437 rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx.llq, 3438 &llq_config); 3439 if (unlikely(rc != 0)) { 3440 device_printf(pdev, "failed to set placement policy\n"); 3441 goto err_com_free; 3442 } 3443 3444 adapter->keep_alive_timestamp = getsbinuptime(); 3445 3446 adapter->tx_offload_cap = get_feat_ctx.offload.tx; 3447 3448 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr, 3449 ETHER_ADDR_LEN); 3450 3451 calc_queue_ctx.pdev = pdev; 3452 calc_queue_ctx.ena_dev = ena_dev; 3453 calc_queue_ctx.get_feat_ctx = &get_feat_ctx; 3454 3455 /* Calculate initial and maximum IO queue number and size */ 3456 max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, 3457 &get_feat_ctx); 3458 rc = ena_calc_io_queue_size(&calc_queue_ctx); 3459 if (unlikely((rc != 0) || (max_num_io_queues <= 0))) { 3460 rc = EFAULT; 3461 goto err_com_free; 3462 } 3463 3464 adapter->tx_ring_size = calc_queue_ctx.tx_queue_size; 3465 adapter->rx_ring_size = calc_queue_ctx.rx_queue_size; 3466 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size; 3467 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size; 3468 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; 3469 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; 3470 3471 adapter->max_num_io_queues = max_num_io_queues; 3472 3473 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE; 3474 3475 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 3476 3477 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3478 3479 /* set up dma tags for rx and tx buffers */ 3480 rc = ena_setup_tx_dma_tag(adapter); 3481 if (unlikely(rc != 0)) { 3482 device_printf(pdev, "Failed to create TX DMA tag\n"); 3483 goto err_com_free; 3484 } 3485 3486 rc = ena_setup_rx_dma_tag(adapter); 3487 if (unlikely(rc != 0)) { 3488 device_printf(pdev, "Failed to create RX DMA tag\n"); 3489 goto err_tx_tag_free; 3490 } 3491 3492 /* 3493 * The amount of requested MSIX vectors is equal to 3494 * adapter::max_num_io_queues (see `ena_enable_msix()`), plus a constant 3495 * number of admin queue interrupts. The former is initially determined 3496 * by HW capabilities (see `ena_calc_max_io_queue_num())` but may not be 3497 * achieved if there are not enough system resources. By default, the 3498 * number of effectively used IO queues is the same but later on it can 3499 * be limited by the user using sysctl interface. 3500 */ 3501 rc = ena_enable_msix_and_set_admin_interrupts(adapter); 3502 if (unlikely(rc != 0)) { 3503 device_printf(pdev, 3504 "Failed to enable and set the admin interrupts\n"); 3505 goto err_io_free; 3506 } 3507 /* By default all of allocated MSIX vectors are actively used */ 3508 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; 3509 3510 /* initialize rings basic information */ 3511 ena_init_io_rings(adapter); 3512 3513 /* setup network interface */ 3514 rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx); 3515 if (unlikely(rc != 0)) { 3516 device_printf(pdev, "Error with network interface setup\n"); 3517 goto err_msix_free; 3518 } 3519 3520 /* Initialize reset task queue */ 3521 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter); 3522 adapter->reset_tq = taskqueue_create("ena_reset_enqueue", 3523 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq); 3524 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, 3525 "%s rstq", device_get_nameunit(adapter->pdev)); 3526 3527 /* Initialize statistics */ 3528 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats, 3529 sizeof(struct ena_stats_dev)); 3530 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats, 3531 sizeof(struct ena_hw_stats)); 3532 ena_sysctl_add_nodes(adapter); 3533 3534 #ifdef DEV_NETMAP 3535 rc = ena_netmap_attach(adapter); 3536 if (rc != 0) { 3537 device_printf(pdev, "netmap attach failed: %d\n", rc); 3538 goto err_detach; 3539 } 3540 #endif /* DEV_NETMAP */ 3541 3542 /* Tell the stack that the interface is not active */ 3543 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 3544 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3545 3546 return (0); 3547 3548 #ifdef DEV_NETMAP 3549 err_detach: 3550 ether_ifdetach(adapter->ifp); 3551 #endif /* DEV_NETMAP */ 3552 err_msix_free: 3553 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR); 3554 ena_free_mgmnt_irq(adapter); 3555 ena_disable_msix(adapter); 3556 err_io_free: 3557 ena_free_all_io_rings_resources(adapter); 3558 ena_free_rx_dma_tag(adapter); 3559 err_tx_tag_free: 3560 ena_free_tx_dma_tag(adapter); 3561 err_com_free: 3562 ena_com_admin_destroy(ena_dev); 3563 ena_com_delete_host_info(ena_dev); 3564 ena_com_mmio_reg_read_request_destroy(ena_dev); 3565 err_bus_free: 3566 free(ena_dev->bus, M_DEVBUF); 3567 ena_free_pci_resources(adapter); 3568 err_dev_free: 3569 free(ena_dev, M_DEVBUF); 3570 3571 return (rc); 3572 } 3573 3574 /** 3575 * ena_detach - Device Removal Routine 3576 * @pdev: device information struct 3577 * 3578 * ena_detach is called by the device subsystem to alert the driver 3579 * that it should release a PCI device. 3580 **/ 3581 static int 3582 ena_detach(device_t pdev) 3583 { 3584 struct ena_adapter *adapter = device_get_softc(pdev); 3585 struct ena_com_dev *ena_dev = adapter->ena_dev; 3586 int rc; 3587 3588 /* Make sure VLANS are not using driver */ 3589 if (adapter->ifp->if_vlantrunk != NULL) { 3590 device_printf(adapter->pdev ,"VLAN is in use, detach first\n"); 3591 return (EBUSY); 3592 } 3593 3594 ether_ifdetach(adapter->ifp); 3595 3596 /* Stop timer service */ 3597 ENA_LOCK_LOCK(adapter); 3598 callout_drain(&adapter->timer_service); 3599 ENA_LOCK_UNLOCK(adapter); 3600 3601 /* Release reset task */ 3602 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL)) 3603 taskqueue_drain(adapter->reset_tq, &adapter->reset_task); 3604 taskqueue_free(adapter->reset_tq); 3605 3606 ENA_LOCK_LOCK(adapter); 3607 ena_down(adapter); 3608 ena_destroy_device(adapter, true); 3609 ENA_LOCK_UNLOCK(adapter); 3610 3611 #ifdef DEV_NETMAP 3612 netmap_detach(adapter->ifp); 3613 #endif /* DEV_NETMAP */ 3614 3615 ena_free_all_io_rings_resources(adapter); 3616 3617 ena_free_counters((counter_u64_t *)&adapter->hw_stats, 3618 sizeof(struct ena_hw_stats)); 3619 ena_free_counters((counter_u64_t *)&adapter->dev_stats, 3620 sizeof(struct ena_stats_dev)); 3621 3622 rc = ena_free_rx_dma_tag(adapter); 3623 if (unlikely(rc != 0)) 3624 device_printf(adapter->pdev, 3625 "Unmapped RX DMA tag associations\n"); 3626 3627 rc = ena_free_tx_dma_tag(adapter); 3628 if (unlikely(rc != 0)) 3629 device_printf(adapter->pdev, 3630 "Unmapped TX DMA tag associations\n"); 3631 3632 ena_free_irqs(adapter); 3633 3634 ena_free_pci_resources(adapter); 3635 3636 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) 3637 ena_com_rss_destroy(ena_dev); 3638 3639 ena_com_delete_host_info(ena_dev); 3640 3641 ENA_LOCK_DESTROY(adapter); 3642 3643 if_free(adapter->ifp); 3644 3645 if (ena_dev->bus != NULL) 3646 free(ena_dev->bus, M_DEVBUF); 3647 3648 if (ena_dev != NULL) 3649 free(ena_dev, M_DEVBUF); 3650 3651 return (bus_generic_detach(pdev)); 3652 } 3653 3654 /****************************************************************************** 3655 ******************************** AENQ Handlers ******************************* 3656 *****************************************************************************/ 3657 /** 3658 * ena_update_on_link_change: 3659 * Notify the network interface about the change in link status 3660 **/ 3661 static void 3662 ena_update_on_link_change(void *adapter_data, 3663 struct ena_admin_aenq_entry *aenq_e) 3664 { 3665 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3666 struct ena_admin_aenq_link_change_desc *aenq_desc; 3667 int status; 3668 if_t ifp; 3669 3670 aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 3671 ifp = adapter->ifp; 3672 status = aenq_desc->flags & 3673 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; 3674 3675 if (status != 0) { 3676 device_printf(adapter->pdev, "link is UP\n"); 3677 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3678 if (!ENA_FLAG_ISSET(ENA_FLAG_ONGOING_RESET, adapter)) 3679 if_link_state_change(ifp, LINK_STATE_UP); 3680 } else { 3681 device_printf(adapter->pdev, "link is DOWN\n"); 3682 if_link_state_change(ifp, LINK_STATE_DOWN); 3683 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3684 } 3685 } 3686 3687 static void ena_notification(void *adapter_data, 3688 struct ena_admin_aenq_entry *aenq_e) 3689 { 3690 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3691 struct ena_admin_ena_hw_hints *hints; 3692 3693 ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION, 3694 "Invalid group(%x) expected %x\n", aenq_e->aenq_common_desc.group, 3695 ENA_ADMIN_NOTIFICATION); 3696 3697 switch (aenq_e->aenq_common_desc.syndrom) { 3698 case ENA_ADMIN_UPDATE_HINTS: 3699 hints = 3700 (struct ena_admin_ena_hw_hints *)(&aenq_e->inline_data_w4); 3701 ena_update_hints(adapter, hints); 3702 break; 3703 default: 3704 device_printf(adapter->pdev, 3705 "Invalid aenq notification link state %d\n", 3706 aenq_e->aenq_common_desc.syndrom); 3707 } 3708 } 3709 3710 /** 3711 * This handler will called for unknown event group or unimplemented handlers 3712 **/ 3713 static void 3714 unimplemented_aenq_handler(void *adapter_data, 3715 struct ena_admin_aenq_entry *aenq_e) 3716 { 3717 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3718 3719 device_printf(adapter->pdev, 3720 "Unknown event was received or event with unimplemented handler\n"); 3721 } 3722 3723 static struct ena_aenq_handlers aenq_handlers = { 3724 .handlers = { 3725 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 3726 [ENA_ADMIN_NOTIFICATION] = ena_notification, 3727 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd, 3728 }, 3729 .unimplemented_handler = unimplemented_aenq_handler 3730 }; 3731 3732 /********************************************************************* 3733 * FreeBSD Device Interface Entry Points 3734 *********************************************************************/ 3735 3736 static device_method_t ena_methods[] = { 3737 /* Device interface */ 3738 DEVMETHOD(device_probe, ena_probe), 3739 DEVMETHOD(device_attach, ena_attach), 3740 DEVMETHOD(device_detach, ena_detach), 3741 DEVMETHOD_END 3742 }; 3743 3744 static driver_t ena_driver = { 3745 "ena", ena_methods, sizeof(struct ena_adapter), 3746 }; 3747 3748 devclass_t ena_devclass; 3749 DRIVER_MODULE(ena, pci, ena_driver, ena_devclass, 0, 0); 3750 MODULE_PNP_INFO("U16:vendor;U16:device", pci, ena, ena_vendor_info_array, 3751 nitems(ena_vendor_info_array) - 1); 3752 MODULE_DEPEND(ena, pci, 1, 1, 1); 3753 MODULE_DEPEND(ena, ether, 1, 1, 1); 3754 #ifdef DEV_NETMAP 3755 MODULE_DEPEND(ena, netmap, 1, 1, 1); 3756 #endif /* DEV_NETMAP */ 3757 3758 /*********************************************************************/ 3759