Lines Matching +full:irq +full:- +full:device
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates.
210 device_t pdev = adapter->pdev; in ena_dma_alloc()
215 maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE; in ena_dma_alloc()
217 dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width); in ena_dma_alloc()
232 &dma->tag); in ena_dma_alloc()
238 error = bus_dma_tag_set_domain(dma->tag, domain); in ena_dma_alloc()
245 error = bus_dmamem_alloc(dma->tag, (void **)&dma->vaddr, in ena_dma_alloc()
246 BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map); in ena_dma_alloc()
253 dma->paddr = 0; in ena_dma_alloc()
254 error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, size, in ena_dma_alloc()
255 ena_dmamap_callback, &dma->paddr, mapflags); in ena_dma_alloc()
256 if (unlikely((error != 0) || (dma->paddr == 0))) { in ena_dma_alloc()
261 bus_dmamap_sync(dma->tag, dma->map, in ena_dma_alloc()
267 bus_dmamem_free(dma->tag, dma->vaddr, dma->map); in ena_dma_alloc()
269 bus_dma_tag_destroy(dma->tag); in ena_dma_alloc()
271 dma->tag = NULL; in ena_dma_alloc()
272 dma->vaddr = NULL; in ena_dma_alloc()
273 dma->paddr = 0; in ena_dma_alloc()
281 device_t pdev = adapter->pdev; in ena_free_pci_resources()
283 if (adapter->memory != NULL) { in ena_free_pci_resources()
285 PCIR_BAR(ENA_MEM_BAR), adapter->memory); in ena_free_pci_resources()
288 if (adapter->registers != NULL) { in ena_free_pci_resources()
290 PCIR_BAR(ENA_REG_BAR), adapter->registers); in ena_free_pci_resources()
293 if (adapter->msix != NULL) { in ena_free_pci_resources()
294 bus_release_resource(pdev, SYS_RES_MEMORY, adapter->msix_rid, in ena_free_pci_resources()
295 adapter->msix); in ena_free_pci_resources()
310 while (ent->vendor_id != 0) { in ena_probe()
311 if ((pci_vendor_id == ent->vendor_id) && in ena_probe()
312 (pci_device_id == ent->device_id)) { in ena_probe()
313 ena_log_raw(DBG, "vendor=%x device=%x\n", pci_vendor_id, in ena_probe()
330 device_t pdev = adapter->pdev; in ena_change_mtu()
333 if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) { in ena_change_mtu()
335 new_mtu, adapter->max_mtu, ENA_MIN_MTU); in ena_change_mtu()
339 rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); in ena_change_mtu()
381 ring->qid = qid; in ena_init_io_rings_common()
382 ring->adapter = adapter; in ena_init_io_rings_common()
383 ring->ena_dev = adapter->ena_dev; in ena_init_io_rings_common()
384 atomic_store_8(&ring->first_interrupt, 0); in ena_init_io_rings_common()
385 ring->no_interrupt_event_cnt = 0; in ena_init_io_rings_common()
396 ena_dev = adapter->ena_dev; in ena_init_io_rings_basic()
398 for (i = 0; i < adapter->num_io_queues; i++) { in ena_init_io_rings_basic()
399 txr = &adapter->tx_ring[i]; in ena_init_io_rings_basic()
400 rxr = &adapter->rx_ring[i]; in ena_init_io_rings_basic()
407 txr->tx_max_header_size = ena_dev->tx_max_header_size; in ena_init_io_rings_basic()
408 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type; in ena_init_io_rings_basic()
410 que = &adapter->que[i]; in ena_init_io_rings_basic()
411 que->adapter = adapter; in ena_init_io_rings_basic()
412 que->id = i; in ena_init_io_rings_basic()
413 que->tx_ring = txr; in ena_init_io_rings_basic()
414 que->rx_ring = rxr; in ena_init_io_rings_basic()
416 txr->que = que; in ena_init_io_rings_basic()
417 rxr->que = que; in ena_init_io_rings_basic()
419 rxr->empty_rx_queue = 0; in ena_init_io_rings_basic()
420 rxr->rx_mbuf_sz = ena_mbuf_sz; in ena_init_io_rings_basic()
430 for (i = 0; i < adapter->num_io_queues; i++) { in ena_init_io_rings_advanced()
431 txr = &adapter->tx_ring[i]; in ena_init_io_rings_advanced()
432 rxr = &adapter->rx_ring[i]; in ena_init_io_rings_advanced()
435 txr->buf_ring_size = adapter->buf_ring_size; in ena_init_io_rings_advanced()
436 txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF, M_WAITOK, in ena_init_io_rings_advanced()
437 &txr->ring_mtx); in ena_init_io_rings_advanced()
440 ena_alloc_counters((counter_u64_t *)&txr->tx_stats, in ena_init_io_rings_advanced()
441 sizeof(txr->tx_stats)); in ena_init_io_rings_advanced()
442 txr->tx_last_cleanup_ticks = ticks; in ena_init_io_rings_advanced()
445 ena_alloc_counters((counter_u64_t *)&rxr->rx_stats, in ena_init_io_rings_advanced()
446 sizeof(rxr->rx_stats)); in ena_init_io_rings_advanced()
449 snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)", in ena_init_io_rings_advanced()
450 device_get_nameunit(adapter->pdev), i); in ena_init_io_rings_advanced()
451 snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)", in ena_init_io_rings_advanced()
452 device_get_nameunit(adapter->pdev), i); in ena_init_io_rings_advanced()
454 mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF); in ena_init_io_rings_advanced()
474 struct ena_ring *txr = &adapter->tx_ring[qid]; in ena_free_io_ring_resources()
475 struct ena_ring *rxr = &adapter->rx_ring[qid]; in ena_free_io_ring_resources()
477 ena_free_counters((counter_u64_t *)&txr->tx_stats, in ena_free_io_ring_resources()
478 sizeof(txr->tx_stats)); in ena_free_io_ring_resources()
479 ena_free_counters((counter_u64_t *)&rxr->rx_stats, in ena_free_io_ring_resources()
480 sizeof(rxr->rx_stats)); in ena_free_io_ring_resources()
483 drbr_free(txr->br, M_DEVBUF); in ena_free_io_ring_resources()
486 mtx_destroy(&txr->ring_mtx); in ena_free_io_ring_resources()
494 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_io_rings_resources()
504 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), in ena_setup_tx_dma_tag()
506 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ in ena_setup_tx_dma_tag()
510 adapter->max_tx_sgl_size - 1, /* nsegments */ in ena_setup_tx_dma_tag()
515 &adapter->tx_buf_tag); in ena_setup_tx_dma_tag()
525 ret = bus_dma_tag_destroy(adapter->tx_buf_tag); in ena_free_tx_dma_tag()
528 adapter->tx_buf_tag = NULL; in ena_free_tx_dma_tag()
539 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */ in ena_setup_rx_dma_tag()
541 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ in ena_setup_rx_dma_tag()
545 adapter->max_rx_sgl_size, /* nsegments */ in ena_setup_rx_dma_tag()
550 &adapter->rx_buf_tag); in ena_setup_rx_dma_tag()
560 ret = bus_dma_tag_destroy(adapter->rx_buf_tag); in ena_free_rx_dma_tag()
563 adapter->rx_buf_tag = NULL; in ena_free_rx_dma_tag()
571 struct ena_adapter *adapter = tx_ring->adapter; in validate_tx_req_id()
577 ena_log(adapter->pdev, ERR, in validate_tx_req_id()
579 req_id, tx_ring->qid); in validate_tx_req_id()
581 ena_log_nm(adapter->pdev, WARN, in validate_tx_req_id()
583 req_id, tx_ring->qid); in validate_tx_req_id()
584 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1); in validate_tx_req_id()
597 struct ena_adapter *adapter = tx_ring->adapter; in ena_release_all_tx_dmamap()
599 bus_dma_tag_t tx_tag = adapter->tx_buf_tag; in ena_release_all_tx_dmamap()
606 for (i = 0; i < tx_ring->ring_size; ++i) { in ena_release_all_tx_dmamap()
607 tx_info = &tx_ring->tx_buffer_info[i]; in ena_release_all_tx_dmamap()
609 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) { in ena_release_all_tx_dmamap()
610 nm_info = &tx_info->nm_info; in ena_release_all_tx_dmamap()
612 if (nm_info->map_seg[j] != NULL) { in ena_release_all_tx_dmamap()
614 nm_info->map_seg[j]); in ena_release_all_tx_dmamap()
615 nm_info->map_seg[j] = NULL; in ena_release_all_tx_dmamap()
620 if (tx_info->dmamap != NULL) { in ena_release_all_tx_dmamap()
621 bus_dmamap_destroy(tx_tag, tx_info->dmamap); in ena_release_all_tx_dmamap()
622 tx_info->dmamap = NULL; in ena_release_all_tx_dmamap()
628 * ena_setup_tx_resources - allocate Tx resources (Descriptors)
629 * @adapter: network interface device structure
637 device_t pdev = adapter->pdev; in ena_setup_tx_resources()
639 struct ena_que *que = &adapter->que[qid]; in ena_setup_tx_resources()
640 struct ena_ring *tx_ring = que->tx_ring; in ena_setup_tx_resources()
650 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; in ena_setup_tx_resources()
652 tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); in ena_setup_tx_resources()
653 if (unlikely(tx_ring->tx_buffer_info == NULL)) in ena_setup_tx_resources()
656 size = sizeof(uint16_t) * tx_ring->ring_size; in ena_setup_tx_resources()
657 tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); in ena_setup_tx_resources()
658 if (unlikely(tx_ring->free_tx_ids == NULL)) in ena_setup_tx_resources()
661 size = tx_ring->tx_max_header_size; in ena_setup_tx_resources()
662 tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF, in ena_setup_tx_resources()
664 if (unlikely(tx_ring->push_buf_intermediate_buf == NULL)) in ena_setup_tx_resources()
668 for (i = 0; i < tx_ring->ring_size; i++) in ena_setup_tx_resources()
669 tx_ring->free_tx_ids[i] = i; in ena_setup_tx_resources()
672 ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats, in ena_setup_tx_resources()
673 sizeof(tx_ring->tx_stats)); in ena_setup_tx_resources()
675 tx_ring->next_to_use = 0; in ena_setup_tx_resources()
676 tx_ring->next_to_clean = 0; in ena_setup_tx_resources()
677 tx_ring->acum_pkts = 0; in ena_setup_tx_resources()
681 drbr_flush(adapter->ifp, tx_ring->br); in ena_setup_tx_resources()
685 for (i = 0; i < tx_ring->ring_size; i++) { in ena_setup_tx_resources()
686 err = bus_dmamap_create(adapter->tx_buf_tag, 0, in ena_setup_tx_resources()
687 &tx_ring->tx_buffer_info[i].dmamap); in ena_setup_tx_resources()
695 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) { in ena_setup_tx_resources()
696 map = tx_ring->tx_buffer_info[i].nm_info.map_seg; in ena_setup_tx_resources()
698 err = bus_dmamap_create(adapter->tx_buf_tag, 0, in ena_setup_tx_resources()
712 TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring); in ena_setup_tx_resources()
713 tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT, in ena_setup_tx_resources()
714 taskqueue_thread_enqueue, &tx_ring->enqueue_tq); in ena_setup_tx_resources()
715 if (unlikely(tx_ring->enqueue_tq == NULL)) { in ena_setup_tx_resources()
718 i = tx_ring->ring_size; in ena_setup_tx_resources()
722 tx_ring->running = true; in ena_setup_tx_resources()
725 cpu_mask = &que->cpu_mask; in ena_setup_tx_resources()
727 device_get_nameunit(adapter->pdev), que->cpu); in ena_setup_tx_resources()
730 device_get_nameunit(adapter->pdev), que->id); in ena_setup_tx_resources()
732 taskqueue_start_threads_cpuset(&tx_ring->enqueue_tq, 1, PI_NET, in ena_setup_tx_resources()
740 free(tx_ring->free_tx_ids, M_DEVBUF); in ena_setup_tx_resources()
741 tx_ring->free_tx_ids = NULL; in ena_setup_tx_resources()
743 free(tx_ring->tx_buffer_info, M_DEVBUF); in ena_setup_tx_resources()
744 tx_ring->tx_buffer_info = NULL; in ena_setup_tx_resources()
750 * ena_free_tx_resources - Free Tx Resources per Queue
751 * @adapter: network interface device structure
759 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_free_tx_resources()
765 while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task, NULL)) in ena_free_tx_resources()
766 taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task); in ena_free_tx_resources()
768 taskqueue_free(tx_ring->enqueue_tq); in ena_free_tx_resources()
772 drbr_flush(adapter->ifp, tx_ring->br); in ena_free_tx_resources()
775 for (int i = 0; i < tx_ring->ring_size; i++) { in ena_free_tx_resources()
776 bus_dmamap_sync(adapter->tx_buf_tag, in ena_free_tx_resources()
777 tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE); in ena_free_tx_resources()
778 bus_dmamap_unload(adapter->tx_buf_tag, in ena_free_tx_resources()
779 tx_ring->tx_buffer_info[i].dmamap); in ena_free_tx_resources()
780 bus_dmamap_destroy(adapter->tx_buf_tag, in ena_free_tx_resources()
781 tx_ring->tx_buffer_info[i].dmamap); in ena_free_tx_resources()
784 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) { in ena_free_tx_resources()
785 nm_info = &tx_ring->tx_buffer_info[i].nm_info; in ena_free_tx_resources()
787 if (nm_info->socket_buf_idx[j] != 0) { in ena_free_tx_resources()
788 bus_dmamap_sync(adapter->tx_buf_tag, in ena_free_tx_resources()
789 nm_info->map_seg[j], in ena_free_tx_resources()
792 nm_info->map_seg[j]); in ena_free_tx_resources()
794 bus_dmamap_destroy(adapter->tx_buf_tag, in ena_free_tx_resources()
795 nm_info->map_seg[j]); in ena_free_tx_resources()
796 nm_info->socket_buf_idx[j] = 0; in ena_free_tx_resources()
801 m_freem(tx_ring->tx_buffer_info[i].mbuf); in ena_free_tx_resources()
802 tx_ring->tx_buffer_info[i].mbuf = NULL; in ena_free_tx_resources()
807 free(tx_ring->tx_buffer_info, M_DEVBUF); in ena_free_tx_resources()
808 tx_ring->tx_buffer_info = NULL; in ena_free_tx_resources()
810 free(tx_ring->free_tx_ids, M_DEVBUF); in ena_free_tx_resources()
811 tx_ring->free_tx_ids = NULL; in ena_free_tx_resources()
813 free(tx_ring->push_buf_intermediate_buf, M_DEVBUF); in ena_free_tx_resources()
814 tx_ring->push_buf_intermediate_buf = NULL; in ena_free_tx_resources()
818 * ena_setup_all_tx_resources - allocate all queues Tx resources
819 * @adapter: network interface device structure
828 for (i = 0; i < adapter->num_io_queues; i++) { in ena_setup_all_tx_resources()
831 ena_log(adapter->pdev, ERR, in ena_setup_all_tx_resources()
841 while (i--) in ena_setup_all_tx_resources()
847 * ena_free_all_tx_resources - Free Tx Resources for All Queues
848 * @adapter: network interface device structure
857 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_tx_resources()
862 * ena_setup_rx_resources - allocate Rx resources (Descriptors)
863 * @adapter: network interface device structure
871 device_t pdev = adapter->pdev; in ena_setup_rx_resources()
872 struct ena_que *que = &adapter->que[qid]; in ena_setup_rx_resources()
873 struct ena_ring *rx_ring = que->rx_ring; in ena_setup_rx_resources()
876 size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size; in ena_setup_rx_resources()
880 rx_ring->initialized = false; in ena_setup_rx_resources()
889 rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO); in ena_setup_rx_resources()
891 size = sizeof(uint16_t) * rx_ring->ring_size; in ena_setup_rx_resources()
892 rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK); in ena_setup_rx_resources()
894 for (i = 0; i < rx_ring->ring_size; i++) in ena_setup_rx_resources()
895 rx_ring->free_rx_ids[i] = i; in ena_setup_rx_resources()
898 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats, in ena_setup_rx_resources()
899 sizeof(rx_ring->rx_stats)); in ena_setup_rx_resources()
901 rx_ring->next_to_clean = 0; in ena_setup_rx_resources()
902 rx_ring->next_to_use = 0; in ena_setup_rx_resources()
905 for (i = 0; i < rx_ring->ring_size; i++) { in ena_setup_rx_resources()
906 err = bus_dmamap_create(adapter->rx_buf_tag, 0, in ena_setup_rx_resources()
907 &(rx_ring->rx_buffer_info[i].map)); in ena_setup_rx_resources()
916 if ((if_getcapenable(adapter->ifp) & IFCAP_LRO) != 0) { in ena_setup_rx_resources()
917 int err = tcp_lro_init(&rx_ring->lro); in ena_setup_rx_resources()
924 rx_ring->lro.ifp = adapter->ifp; in ena_setup_rx_resources()
931 while (i--) { in ena_setup_rx_resources()
932 bus_dmamap_destroy(adapter->rx_buf_tag, in ena_setup_rx_resources()
933 rx_ring->rx_buffer_info[i].map); in ena_setup_rx_resources()
936 free(rx_ring->free_rx_ids, M_DEVBUF); in ena_setup_rx_resources()
937 rx_ring->free_rx_ids = NULL; in ena_setup_rx_resources()
938 free(rx_ring->rx_buffer_info, M_DEVBUF); in ena_setup_rx_resources()
939 rx_ring->rx_buffer_info = NULL; in ena_setup_rx_resources()
944 * ena_free_rx_resources - Free Rx Resources
945 * @adapter: network interface device structure
953 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_resources()
956 for (int i = 0; i < rx_ring->ring_size; i++) { in ena_free_rx_resources()
957 bus_dmamap_sync(adapter->rx_buf_tag, in ena_free_rx_resources()
958 rx_ring->rx_buffer_info[i].map, BUS_DMASYNC_POSTREAD); in ena_free_rx_resources()
959 m_freem(rx_ring->rx_buffer_info[i].mbuf); in ena_free_rx_resources()
960 rx_ring->rx_buffer_info[i].mbuf = NULL; in ena_free_rx_resources()
961 bus_dmamap_unload(adapter->rx_buf_tag, in ena_free_rx_resources()
962 rx_ring->rx_buffer_info[i].map); in ena_free_rx_resources()
963 bus_dmamap_destroy(adapter->rx_buf_tag, in ena_free_rx_resources()
964 rx_ring->rx_buffer_info[i].map); in ena_free_rx_resources()
968 tcp_lro_free(&rx_ring->lro); in ena_free_rx_resources()
971 free(rx_ring->rx_buffer_info, M_DEVBUF); in ena_free_rx_resources()
972 rx_ring->rx_buffer_info = NULL; in ena_free_rx_resources()
974 free(rx_ring->free_rx_ids, M_DEVBUF); in ena_free_rx_resources()
975 rx_ring->free_rx_ids = NULL; in ena_free_rx_resources()
979 * ena_setup_all_rx_resources - allocate all queues Rx resources
980 * @adapter: network interface device structure
989 for (i = 0; i < adapter->num_io_queues; i++) { in ena_setup_all_rx_resources()
992 ena_log(adapter->pdev, ERR, in ena_setup_all_rx_resources()
1001 while (i--) in ena_setup_all_rx_resources()
1007 * ena_free_all_rx_resources - Free Rx resources for all queues
1008 * @adapter: network interface device structure
1017 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_rx_resources()
1025 device_t pdev = adapter->pdev; in ena_alloc_rx_mbuf()
1032 if (unlikely(rx_info->mbuf != NULL)) in ena_alloc_rx_mbuf()
1036 rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, in ena_alloc_rx_mbuf()
1037 rx_ring->rx_mbuf_sz); in ena_alloc_rx_mbuf()
1039 if (unlikely(rx_info->mbuf == NULL)) { in ena_alloc_rx_mbuf()
1040 counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1); in ena_alloc_rx_mbuf()
1041 rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); in ena_alloc_rx_mbuf()
1042 if (unlikely(rx_info->mbuf == NULL)) { in ena_alloc_rx_mbuf()
1043 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); in ena_alloc_rx_mbuf()
1048 mlen = rx_ring->rx_mbuf_sz; in ena_alloc_rx_mbuf()
1051 rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen; in ena_alloc_rx_mbuf()
1056 adapter->rx_buf_tag, rx_info->mbuf, rx_info->mbuf->m_len); in ena_alloc_rx_mbuf()
1057 error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map, in ena_alloc_rx_mbuf()
1058 rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT); in ena_alloc_rx_mbuf()
1062 counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1); in ena_alloc_rx_mbuf()
1066 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD); in ena_alloc_rx_mbuf()
1068 ena_buf = &rx_info->ena_buf; in ena_alloc_rx_mbuf()
1069 ena_buf->paddr = segs[0].ds_addr; in ena_alloc_rx_mbuf()
1070 ena_buf->len = mlen; in ena_alloc_rx_mbuf()
1074 rx_info->mbuf, rx_info, ena_buf->len, (uintmax_t)ena_buf->paddr); in ena_alloc_rx_mbuf()
1079 m_freem(rx_info->mbuf); in ena_alloc_rx_mbuf()
1080 rx_info->mbuf = NULL; in ena_alloc_rx_mbuf()
1088 if (rx_info->mbuf == NULL) { in ena_free_rx_mbuf()
1089 ena_log(adapter->pdev, WARN, in ena_free_rx_mbuf()
1094 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, in ena_free_rx_mbuf()
1096 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map); in ena_free_rx_mbuf()
1097 m_freem(rx_info->mbuf); in ena_free_rx_mbuf()
1098 rx_info->mbuf = NULL; in ena_free_rx_mbuf()
1102 * ena_refill_rx_bufs - Refills ring with descriptors
1105 * Refills the ring with newly allocated DMA-mapped mbufs for receiving
1110 struct ena_adapter *adapter = rx_ring->adapter; in ena_refill_rx_bufs()
1111 device_t pdev = adapter->pdev; in ena_refill_rx_bufs()
1116 ena_log_io(adapter->pdev, DBG, "refill qid: %d\n", rx_ring->qid); in ena_refill_rx_bufs()
1118 next_to_use = rx_ring->next_to_use; in ena_refill_rx_bufs()
1123 ena_log_io(pdev, DBG, "RX buffer - next to use: %d\n", in ena_refill_rx_bufs()
1126 req_id = rx_ring->free_rx_ids[next_to_use]; in ena_refill_rx_bufs()
1127 rx_info = &rx_ring->rx_buffer_info[req_id]; in ena_refill_rx_bufs()
1129 if (ena_rx_ring_in_netmap(adapter, rx_ring->qid)) in ena_refill_rx_bufs()
1138 rx_ring->qid); in ena_refill_rx_bufs()
1141 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq, in ena_refill_rx_bufs()
1142 &rx_info->ena_buf, req_id); in ena_refill_rx_bufs()
1146 rx_ring->qid); in ena_refill_rx_bufs()
1150 rx_ring->ring_size); in ena_refill_rx_bufs()
1154 counter_u64_add(rx_ring->rx_stats.refil_partial, 1); in ena_refill_rx_bufs()
1157 rx_ring->qid, i, num); in ena_refill_rx_bufs()
1161 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq); in ena_refill_rx_bufs()
1163 rx_ring->next_to_use = next_to_use; in ena_refill_rx_bufs()
1173 netmap_detach(adapter->ifp); in ena_reinit_netmap()
1176 ena_log(adapter->pdev, ERR, "netmap attach failed: %d\n", rc); in ena_reinit_netmap()
1190 old_buf_ring_size = adapter->buf_ring_size; in ena_update_buf_ring_size()
1191 adapter->buf_ring_size = new_buf_ring_size; in ena_update_buf_ring_size()
1214 ena_log(adapter->pdev, ERR, in ena_update_buf_ring_size()
1215 …"Failed to configure device after setting new drbr size: %u. Reverting old value: %u and triggerin… in ena_update_buf_ring_size()
1219 adapter->buf_ring_size = old_buf_ring_size; in ena_update_buf_ring_size()
1245 old_tx_size = adapter->requested_tx_ring_size; in ena_update_queue_size()
1246 old_rx_size = adapter->requested_rx_ring_size; in ena_update_queue_size()
1247 adapter->requested_tx_ring_size = new_tx_size; in ena_update_queue_size()
1248 adapter->requested_rx_ring_size = new_rx_size; in ena_update_queue_size()
1264 ena_log(adapter->pdev, ERR, in ena_update_queue_size()
1265 …"Failed to configure device with the new sizes - Tx: %u Rx: %u. Reverting old values - Tx: %u Rx: … in ena_update_queue_size()
1269 adapter->requested_tx_ring_size = old_tx_size; in ena_update_queue_size()
1270 adapter->requested_rx_ring_size = old_rx_size; in ena_update_queue_size()
1281 ena_log(adapter->pdev, ERR, in ena_update_queue_size()
1282 "Failed to revert old queue sizes. Triggering device reset.\n"); in ena_update_queue_size()
1285 * wrong. After reset, the device should try to in ena_update_queue_size()
1304 ena_com_rss_destroy(adapter->ena_dev); in ena_update_io_rings()
1306 adapter->num_io_queues = num; in ena_update_io_rings()
1318 old_num = adapter->irq_cpu_base; in ena_update_base_cpu()
1322 adapter->irq_cpu_base = new_num; in ena_update_base_cpu()
1327 ena_log(adapter->pdev, ERR, in ena_update_base_cpu()
1328 "Failed to configure device %d IRQ base CPU. " in ena_update_base_cpu()
1332 adapter->irq_cpu_base = old_num; in ena_update_base_cpu()
1336 ena_log(adapter->pdev, ERR, in ena_update_base_cpu()
1338 "Triggering device reset.\n"); in ena_update_base_cpu()
1357 old_num = adapter->irq_cpu_stride; in ena_update_cpu_stride()
1361 adapter->irq_cpu_stride = new_num; in ena_update_cpu_stride()
1366 ena_log(adapter->pdev, ERR, in ena_update_cpu_stride()
1367 "Failed to configure device %d IRQ CPU stride. " in ena_update_cpu_stride()
1371 adapter->irq_cpu_stride = old_num; in ena_update_cpu_stride()
1375 ena_log(adapter->pdev, ERR, in ena_update_cpu_stride()
1377 "Triggering device reset.\n"); in ena_update_cpu_stride()
1397 old_num = adapter->num_io_queues; in ena_update_io_queue_nb()
1410 ena_log(adapter->pdev, ERR, in ena_update_io_queue_nb()
1411 "Failed to configure device with %u IO queues. " in ena_update_io_queue_nb()
1424 ena_log(adapter->pdev, ERR, in ena_update_io_queue_nb()
1426 "queues. Triggering device reset.\n"); in ena_update_io_queue_nb()
1441 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_bufs()
1444 for (i = 0; i < rx_ring->ring_size; i++) { in ena_free_rx_bufs()
1445 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i]; in ena_free_rx_bufs()
1447 if (rx_info->mbuf != NULL) in ena_free_rx_bufs()
1450 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) && in ena_free_rx_bufs()
1451 (if_getcapenable(adapter->ifp) & IFCAP_NETMAP)) { in ena_free_rx_bufs()
1452 if (rx_info->netmap_buf_idx != 0) in ena_free_rx_bufs()
1461 * ena_refill_all_rx_bufs - allocate all queues Rx buffers
1462 * @adapter: network interface device structure
1471 for (i = 0; i < adapter->num_io_queues; i++) { in ena_refill_all_rx_bufs()
1472 rx_ring = &adapter->rx_ring[i]; in ena_refill_all_rx_bufs()
1473 bufs_num = rx_ring->ring_size - 1; in ena_refill_all_rx_bufs()
1476 ena_log_io(adapter->pdev, WARN, in ena_refill_all_rx_bufs()
1481 rx_ring->initialized = true; in ena_refill_all_rx_bufs()
1491 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_rx_bufs()
1496 * ena_free_tx_bufs - Free Tx Buffers per Queue
1497 * @adapter: network interface device structure
1504 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_free_tx_bufs()
1507 for (int i = 0; i < tx_ring->ring_size; i++) { in ena_free_tx_bufs()
1508 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i]; in ena_free_tx_bufs()
1510 if (tx_info->mbuf == NULL) in ena_free_tx_bufs()
1514 ena_log(adapter->pdev, WARN, in ena_free_tx_bufs()
1519 ena_log(adapter->pdev, DBG, in ena_free_tx_bufs()
1524 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, in ena_free_tx_bufs()
1526 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); in ena_free_tx_bufs()
1528 m_free(tx_info->mbuf); in ena_free_tx_bufs()
1529 tx_info->mbuf = NULL; in ena_free_tx_bufs()
1537 for (int i = 0; i < adapter->num_io_queues; i++) in ena_free_all_tx_bufs()
1547 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_tx_queues()
1549 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); in ena_destroy_all_tx_queues()
1559 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_rx_queues()
1561 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); in ena_destroy_all_rx_queues()
1571 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_io_queues()
1572 queue = &adapter->que[i]; in ena_destroy_all_io_queues()
1573 while (taskqueue_cancel(queue->cleanup_tq, &queue->cleanup_task, NULL)) in ena_destroy_all_io_queues()
1574 taskqueue_drain(queue->cleanup_tq, &queue->cleanup_task); in ena_destroy_all_io_queues()
1575 taskqueue_free(queue->cleanup_tq); in ena_destroy_all_io_queues()
1585 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_create_io_queues()
1595 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_io_queues()
1598 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; in ena_create_io_queues()
1600 ctx.queue_size = adapter->requested_tx_ring_size; in ena_create_io_queues()
1603 ctx.numa_node = adapter->que[i].domain; in ena_create_io_queues()
1607 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1611 ring = &adapter->tx_ring[i]; in ena_create_io_queues()
1613 &ring->ena_com_io_sq, &ring->ena_com_io_cq); in ena_create_io_queues()
1615 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1624 ena_com_update_numa_node(ring->ena_com_io_cq, in ena_create_io_queues()
1630 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_io_queues()
1635 ctx.queue_size = adapter->requested_rx_ring_size; in ena_create_io_queues()
1638 ctx.numa_node = adapter->que[i].domain; in ena_create_io_queues()
1642 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1647 ring = &adapter->rx_ring[i]; in ena_create_io_queues()
1649 &ring->ena_com_io_sq, &ring->ena_com_io_cq); in ena_create_io_queues()
1651 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1660 ena_com_update_numa_node(ring->ena_com_io_cq, in ena_create_io_queues()
1665 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_io_queues()
1666 queue = &adapter->que[i]; in ena_create_io_queues()
1668 NET_TASK_INIT(&queue->cleanup_task, 0, ena_cleanup, queue); in ena_create_io_queues()
1669 queue->cleanup_tq = taskqueue_create_fast("ena cleanup", in ena_create_io_queues()
1670 M_WAITOK, taskqueue_thread_enqueue, &queue->cleanup_tq); in ena_create_io_queues()
1673 cpu_mask = &queue->cpu_mask; in ena_create_io_queues()
1675 taskqueue_start_threads_cpuset(&queue->cleanup_tq, 1, PI_NET, in ena_create_io_queues()
1677 device_get_nameunit(adapter->pdev), i); in ena_create_io_queues()
1683 while (i--) in ena_create_io_queues()
1685 i = adapter->num_io_queues; in ena_create_io_queues()
1687 while (i--) in ena_create_io_queues()
1700 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue
1708 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); in ena_intr_msix_mgmnt()
1710 ena_com_aenq_intr_handler(adapter->ena_dev, arg); in ena_intr_msix_mgmnt()
1714 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
1721 struct ena_adapter *adapter = queue->adapter; in ena_handle_msix()
1722 if_t ifp = adapter->ifp; in ena_handle_msix()
1727 taskqueue_enqueue(queue->cleanup_tq, &queue->cleanup_task); in ena_handle_msix()
1735 device_t dev = adapter->pdev; in ena_enable_msix()
1740 ena_log(dev, ERR, "Error, MSI-X is already enabled\n"); in ena_enable_msix()
1745 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues); in ena_enable_msix()
1747 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry), in ena_enable_msix()
1750 ena_log(dev, DBG, "trying to enable MSI-X, vectors: %d\n", msix_vecs); in ena_enable_msix()
1753 adapter->msix_entries[i].entry = i; in ena_enable_msix()
1755 adapter->msix_entries[i].vector = i + 1; in ena_enable_msix()
1771 "Not enough number of MSI-x allocated: %d\n", in ena_enable_msix()
1778 "Enable only %d MSI-x (out of %d), reduce " in ena_enable_msix()
1783 adapter->msix_vecs = msix_vecs; in ena_enable_msix()
1789 free(adapter->msix_entries, M_DEVBUF); in ena_enable_msix()
1790 adapter->msix_entries = NULL; in ena_enable_msix()
1798 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, ENA_IRQNAME_SIZE, in ena_setup_mgmnt_intr()
1799 "ena-mgmnt@pci:%s", device_get_nameunit(adapter->pdev)); in ena_setup_mgmnt_intr()
1804 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL; in ena_setup_mgmnt_intr()
1805 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter; in ena_setup_mgmnt_intr()
1806 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector = in ena_setup_mgmnt_intr()
1807 adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector; in ena_setup_mgmnt_intr()
1821 if (adapter->msix_entries == NULL) in ena_setup_io_intr()
1825 if (adapter->first_bind < 0) { in ena_setup_io_intr()
1826 adapter->first_bind = last_bind; in ena_setup_io_intr()
1827 last_bind = (last_bind + adapter->num_io_queues) % num_buckets; in ena_setup_io_intr()
1829 cur_bind = adapter->first_bind; in ena_setup_io_intr()
1832 for (int i = 0; i < adapter->num_io_queues; i++) { in ena_setup_io_intr()
1835 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, in ena_setup_io_intr()
1836 "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i); in ena_setup_io_intr()
1837 adapter->irq_tbl[irq_idx].handler = ena_handle_msix; in ena_setup_io_intr()
1838 adapter->irq_tbl[irq_idx].data = &adapter->que[i]; in ena_setup_io_intr()
1839 adapter->irq_tbl[irq_idx].vector = in ena_setup_io_intr()
1840 adapter->msix_entries[irq_idx].vector; in ena_setup_io_intr()
1841 ena_log(adapter->pdev, DBG, "ena_setup_io_intr vector: %d\n", in ena_setup_io_intr()
1842 adapter->msix_entries[irq_idx].vector); in ena_setup_io_intr()
1844 if (adapter->irq_cpu_base > ENA_BASE_CPU_UNSPECIFIED) { in ena_setup_io_intr()
1845 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = in ena_setup_io_intr()
1846 (unsigned)(adapter->irq_cpu_base + in ena_setup_io_intr()
1847 i * adapter->irq_cpu_stride) % (unsigned)mp_ncpus; in ena_setup_io_intr()
1848 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask); in ena_setup_io_intr()
1852 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = in ena_setup_io_intr()
1855 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask); in ena_setup_io_intr()
1858 if (CPU_ISSET(adapter->que[i].cpu, &cpuset_domain[idx])) in ena_setup_io_intr()
1861 adapter->que[i].domain = idx; in ena_setup_io_intr()
1863 adapter->que[i].domain = -1; in ena_setup_io_intr()
1873 device_t pdev = adapter->pdev; in ena_request_mgmnt_irq()
1874 struct ena_irq *irq; in ena_request_mgmnt_irq() local
1880 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; in ena_request_mgmnt_irq()
1881 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, in ena_request_mgmnt_irq()
1882 &irq->vector, flags); in ena_request_mgmnt_irq()
1884 if (unlikely(irq->res == NULL)) { in ena_request_mgmnt_irq()
1885 ena_log(pdev, ERR, "could not allocate irq vector: %d\n", in ena_request_mgmnt_irq()
1886 irq->vector); in ena_request_mgmnt_irq()
1890 rc = bus_setup_intr(adapter->pdev, irq->res, in ena_request_mgmnt_irq()
1891 INTR_TYPE_NET | INTR_MPSAFE, NULL, ena_intr_msix_mgmnt, irq->data, in ena_request_mgmnt_irq()
1892 &irq->cookie); in ena_request_mgmnt_irq()
1895 "failed to register interrupt handler for irq %ju: %d\n", in ena_request_mgmnt_irq()
1896 rman_get_start(irq->res), rc); in ena_request_mgmnt_irq()
1899 irq->requested = true; in ena_request_mgmnt_irq()
1904 ena_log(pdev, INFO, "releasing resource for irq %d\n", irq->vector); in ena_request_mgmnt_irq()
1905 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, irq->vector, in ena_request_mgmnt_irq()
1906 irq->res); in ena_request_mgmnt_irq()
1909 "dev has no parent while releasing res for irq: %d\n", in ena_request_mgmnt_irq()
1910 irq->vector); in ena_request_mgmnt_irq()
1911 irq->res = NULL; in ena_request_mgmnt_irq()
1919 device_t pdev = adapter->pdev; in ena_request_io_irq()
1920 struct ena_irq *irq; in ena_request_io_irq() local
1926 "failed to request I/O IRQ: MSI-X is not enabled\n"); in ena_request_io_irq()
1932 for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { in ena_request_io_irq()
1933 irq = &adapter->irq_tbl[i]; in ena_request_io_irq()
1935 if (unlikely(irq->requested)) in ena_request_io_irq()
1938 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, in ena_request_io_irq()
1939 &irq->vector, flags); in ena_request_io_irq()
1940 if (unlikely(irq->res == NULL)) { in ena_request_io_irq()
1943 "could not allocate irq vector: %d\n", irq->vector); in ena_request_io_irq()
1947 rc = bus_setup_intr(adapter->pdev, irq->res, in ena_request_io_irq()
1948 INTR_TYPE_NET | INTR_MPSAFE, irq->handler, NULL, irq->data, in ena_request_io_irq()
1949 &irq->cookie); in ena_request_io_irq()
1952 "failed to register interrupt handler for irq %ju: %d\n", in ena_request_io_irq()
1953 rman_get_start(irq->res), rc); in ena_request_io_irq()
1956 irq->requested = true; in ena_request_io_irq()
1958 if (adapter->rss_enabled || adapter->irq_cpu_base > ENA_BASE_CPU_UNSPECIFIED) { in ena_request_io_irq()
1959 rc = bus_bind_intr(adapter->pdev, irq->res, irq->cpu); in ena_request_io_irq()
1962 "failed to bind interrupt handler for irq %ju to cpu %d: %d\n", in ena_request_io_irq()
1963 rman_get_start(irq->res), irq->cpu, rc); in ena_request_io_irq()
1967 ena_log(pdev, INFO, "queue %d - cpu %d\n", in ena_request_io_irq()
1968 i - ENA_IO_IRQ_FIRST_IDX, irq->cpu); in ena_request_io_irq()
1975 for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) { in ena_request_io_irq()
1976 irq = &adapter->irq_tbl[i]; in ena_request_io_irq()
1979 /* Once we entered err: section and irq->requested is true we in ena_request_io_irq()
1981 if (irq->requested) { in ena_request_io_irq()
1982 rcc = bus_teardown_intr(adapter->pdev, irq->res, in ena_request_io_irq()
1983 irq->cookie); in ena_request_io_irq()
1986 "could not release irq: %d, error: %d\n", in ena_request_io_irq()
1987 irq->vector, rcc); in ena_request_io_irq()
1990 /* If we entered err: section without irq->requested set we know in ena_request_io_irq()
1995 if (irq->res != NULL) { in ena_request_io_irq()
1996 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_request_io_irq()
1997 irq->vector, irq->res); in ena_request_io_irq()
2001 "dev has no parent while releasing res for irq: %d\n", in ena_request_io_irq()
2002 irq->vector); in ena_request_io_irq()
2003 irq->requested = false; in ena_request_io_irq()
2004 irq->res = NULL; in ena_request_io_irq()
2013 device_t pdev = adapter->pdev; in ena_free_mgmnt_irq()
2014 struct ena_irq *irq; in ena_free_mgmnt_irq() local
2017 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; in ena_free_mgmnt_irq()
2018 if (irq->requested) { in ena_free_mgmnt_irq()
2019 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector); in ena_free_mgmnt_irq()
2020 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); in ena_free_mgmnt_irq()
2022 ena_log(pdev, ERR, "failed to tear down irq: %d\n", in ena_free_mgmnt_irq()
2023 irq->vector); in ena_free_mgmnt_irq()
2024 irq->requested = 0; in ena_free_mgmnt_irq()
2027 if (irq->res != NULL) { in ena_free_mgmnt_irq()
2028 ena_log(pdev, DBG, "release resource irq: %d\n", irq->vector); in ena_free_mgmnt_irq()
2029 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_free_mgmnt_irq()
2030 irq->vector, irq->res); in ena_free_mgmnt_irq()
2031 irq->res = NULL; in ena_free_mgmnt_irq()
2034 "dev has no parent while releasing res for irq: %d\n", in ena_free_mgmnt_irq()
2035 irq->vector); in ena_free_mgmnt_irq()
2042 device_t pdev = adapter->pdev; in ena_free_io_irq()
2043 struct ena_irq *irq; in ena_free_io_irq() local
2046 for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { in ena_free_io_irq()
2047 irq = &adapter->irq_tbl[i]; in ena_free_io_irq()
2048 if (irq->requested) { in ena_free_io_irq()
2049 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector); in ena_free_io_irq()
2050 rc = bus_teardown_intr(adapter->pdev, irq->res, in ena_free_io_irq()
2051 irq->cookie); in ena_free_io_irq()
2054 "failed to tear down irq: %d\n", in ena_free_io_irq()
2055 irq->vector); in ena_free_io_irq()
2057 irq->requested = 0; in ena_free_io_irq()
2060 if (irq->res != NULL) { in ena_free_io_irq()
2061 ena_log(pdev, DBG, "release resource irq: %d\n", in ena_free_io_irq()
2062 irq->vector); in ena_free_io_irq()
2063 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_free_io_irq()
2064 irq->vector, irq->res); in ena_free_io_irq()
2065 irq->res = NULL; in ena_free_io_irq()
2068 "dev has no parent while releasing res for irq: %d\n", in ena_free_io_irq()
2069 irq->vector); in ena_free_io_irq()
2088 pci_release_msi(adapter->pdev); in ena_disable_msix()
2091 adapter->msix_vecs = 0; in ena_disable_msix()
2092 free(adapter->msix_entries, M_DEVBUF); in ena_disable_msix()
2093 adapter->msix_entries = NULL; in ena_disable_msix()
2106 for (i = 0; i < adapter->num_io_queues; i++) { in ena_unmask_all_io_irqs()
2108 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; in ena_unmask_all_io_irqs()
2110 tx_ring = &adapter->tx_ring[i]; in ena_unmask_all_io_irqs()
2111 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1); in ena_unmask_all_io_irqs()
2124 ena_log(adapter->pdev, ERR, in ena_up_complete()
2130 rc = ena_change_mtu(adapter->ifp, if_getmtu(adapter->ifp)); in ena_up_complete()
2135 ena_reset_counters((counter_u64_t *)&adapter->hw_stats, in ena_up_complete()
2136 sizeof(adapter->hw_stats)); in ena_up_complete()
2146 for (i = 0; i < adapter->num_io_queues; i++) { in set_io_rings_size()
2147 adapter->tx_ring[i].ring_size = new_tx_size; in set_io_rings_size()
2148 adapter->rx_ring[i].ring_size = new_rx_size; in set_io_rings_size()
2155 device_t pdev = adapter->pdev; in create_queues_with_size_backoff()
2164 set_io_rings_size(adapter, adapter->requested_tx_ring_size, in create_queues_with_size_backoff()
2165 adapter->requested_rx_ring_size); in create_queues_with_size_backoff()
2206 cur_tx_ring_size = adapter->tx_ring[0].ring_size; in create_queues_with_size_backoff()
2207 cur_rx_ring_size = adapter->rx_ring[0].ring_size; in create_queues_with_size_backoff()
2249 if (unlikely(device_is_attached(adapter->pdev) == 0)) { in ena_up()
2250 ena_log(adapter->pdev, ERR, "device is not attached!\n"); in ena_up()
2257 ena_log(adapter->pdev, INFO, "device is going UP\n"); in ena_up()
2262 ena_log(adapter->pdev, ERR, "error setting up IO interrupt\n"); in ena_up()
2267 ena_log(adapter->pdev, ERR, "err_req_irq\n"); in ena_up()
2271 ena_log(adapter->pdev, INFO, in ena_up()
2273 adapter->num_io_queues, in ena_up()
2274 adapter->requested_rx_ring_size, in ena_up()
2275 adapter->requested_tx_ring_size, in ena_up()
2276 (adapter->ena_dev->tx_mem_queue_type == in ena_up()
2281 ena_log(adapter->pdev, ERR, in ena_up()
2287 if_link_state_change(adapter->ifp, LINK_STATE_UP); in ena_up()
2293 counter_u64_add(adapter->dev_stats.interface_up, 1); in ena_up()
2297 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); in ena_up()
2322 stats = &adapter->hw_stats; in ena_get_counter()
2326 return (counter_u64_fetch(stats->rx_packets)); in ena_get_counter()
2328 return (counter_u64_fetch(stats->tx_packets)); in ena_get_counter()
2330 return (counter_u64_fetch(stats->rx_bytes)); in ena_get_counter()
2332 return (counter_u64_fetch(stats->tx_bytes)); in ena_get_counter()
2334 return (counter_u64_fetch(stats->rx_drops)); in ena_get_counter()
2336 return (counter_u64_fetch(stats->tx_drops)); in ena_get_counter()
2353 ena_log(adapter->pdev, DBG, "Media status update\n"); in ena_media_status()
2357 ifmr->ifm_status = IFM_AVALID; in ena_media_status()
2358 ifmr->ifm_active = IFM_ETHER; in ena_media_status()
2362 ena_log(adapter->pdev, INFO, "Link is down\n"); in ena_media_status()
2366 ifmr->ifm_status |= IFM_ACTIVE; in ena_media_status()
2367 ifmr->ifm_active |= IFM_UNKNOWN | IFM_FDX; in ena_media_status()
2400 if (if_getmtu(ifp) == ifr->ifr_mtu) in ena_ioctl()
2405 ena_change_mtu(ifp, ifr->ifr_mtu); in ena_ioctl()
2416 ena_log(adapter->pdev, INFO, in ena_ioctl()
2439 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command); in ena_ioctl()
2446 if (ifr->ifr_reqcap != if_getcapenable(ifp)) { in ena_ioctl()
2447 if_setcapenable(ifp, ifr->ifr_reqcap); in ena_ioctl()
2474 if ((feat->offload.tx & in ena_get_dev_offloads()
2480 if ((feat->offload.tx & in ena_get_dev_offloads()
2485 if ((feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0) in ena_get_dev_offloads()
2488 if ((feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0) in ena_get_dev_offloads()
2491 if ((feat->offload.rx_supported & in ena_get_dev_offloads()
2496 if ((feat->offload.rx_supported & in ena_get_dev_offloads()
2508 host_info->supported_network_features[0] = (uint32_t)if_getcapabilities(ifp); in ena_update_host_info()
2514 if_t ifp = adapter->ifp; in ena_update_hwassist()
2515 uint32_t feat = adapter->tx_offload_cap; in ena_update_hwassist()
2550 ifp = adapter->ifp = if_gethandle(IFT_ETHER); in ena_setup_ifnet()
2562 if_setsendqlen(ifp, adapter->requested_tx_ring_size); in ena_setup_ifnet()
2575 if_sethwtsomax(ifp, ENA_TSO_MAXSIZE - in ena_setup_ifnet()
2577 if_sethwtsomaxsegcount(ifp, adapter->max_tx_sgl_size - 1); in ena_setup_ifnet()
2587 ifmedia_init(&adapter->media, IFM_IMASK, ena_media_change, in ena_setup_ifnet()
2589 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); in ena_setup_ifnet()
2590 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); in ena_setup_ifnet()
2592 ether_ifattach(ifp, adapter->mac_addr); in ena_setup_ifnet()
2605 ena_log(adapter->pdev, INFO, "device is going DOWN\n"); in ena_down()
2608 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); in ena_down()
2613 rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); in ena_down()
2615 ena_log(adapter->pdev, ERR, "Device reset failed\n"); in ena_down()
2625 counter_u64_add(adapter->dev_stats.interface_down, 1); in ena_down()
2635 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { in ena_calc_max_io_queue_num()
2637 &get_feat_ctx->max_queue_ext.max_queue_ext; in ena_calc_max_io_queue_num()
2638 io_rx_num = min_t(int, max_queue_ext->max_rx_sq_num, in ena_calc_max_io_queue_num()
2639 max_queue_ext->max_rx_cq_num); in ena_calc_max_io_queue_num()
2641 io_tx_sq_num = max_queue_ext->max_tx_sq_num; in ena_calc_max_io_queue_num()
2642 io_tx_cq_num = max_queue_ext->max_tx_cq_num; in ena_calc_max_io_queue_num()
2645 &get_feat_ctx->max_queues; in ena_calc_max_io_queue_num()
2646 io_tx_sq_num = max_queues->max_sq_num; in ena_calc_max_io_queue_num()
2647 io_tx_cq_num = max_queues->max_cq_num; in ena_calc_max_io_queue_num()
2652 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) in ena_calc_max_io_queue_num()
2653 io_tx_sq_num = get_feat_ctx->llq.max_llq_num; in ena_calc_max_io_queue_num()
2659 /* 1 IRQ for mgmnt and 1 IRQ for each TX/RX pair */ in ena_calc_max_io_queue_num()
2661 pci_msix_count(pdev) - 1); in ena_calc_max_io_queue_num()
2701 if (!(ena_dev->supported_features & llq_feature_mask)) { in ena_set_queues_placement_policy()
2704 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; in ena_set_queues_placement_policy()
2708 if (ena_dev->mem_bar == NULL) { in ena_set_queues_placement_policy()
2710 "LLQ is advertised as supported but device doesn't expose mem bar.\n"); in ena_set_queues_placement_policy()
2711 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; in ena_set_queues_placement_policy()
2718 "Failed to configure the device mode. " in ena_set_queues_placement_policy()
2720 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; in ena_set_queues_placement_policy()
2734 adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid, in ena_map_llq_mem_bar()
2736 if (unlikely(adapter->memory == NULL)) { in ena_map_llq_mem_bar()
2743 rc = ena_enable_wc(adapter->pdev, adapter->memory); in ena_map_llq_mem_bar()
2750 * Save virtual address of the device's memory region in ena_map_llq_mem_bar()
2753 ena_dev->mem_bar = rman_get_virtual(adapter->memory); in ena_map_llq_mem_bar()
2762 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; in ena_set_llq_configurations()
2763 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; in ena_set_llq_configurations()
2764 llq_config->llq_num_decs_before_header = in ena_set_llq_configurations()
2766 if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) != 0) { in ena_set_llq_configurations()
2769 llq->entry_size_recommended == ENA_ADMIN_LIST_ENTRY_SIZE_256B)) { in ena_set_llq_configurations()
2770 llq_config->llq_ring_entry_size = in ena_set_llq_configurations()
2772 llq_config->llq_ring_entry_size_value = 256; in ena_set_llq_configurations()
2773 adapter->llq_policy = ENA_ADMIN_LIST_ENTRY_SIZE_256B; in ena_set_llq_configurations()
2776 llq_config->llq_ring_entry_size = in ena_set_llq_configurations()
2778 llq_config->llq_ring_entry_size_value = 128; in ena_set_llq_configurations()
2779 adapter->llq_policy = ENA_ADMIN_LIST_ENTRY_SIZE_128B; in ena_set_llq_configurations()
2786 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq; in ena_calc_io_queue_size()
2787 struct ena_com_dev *ena_dev = ctx->ena_dev; in ena_calc_io_queue_size()
2793 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { in ena_calc_io_queue_size()
2795 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext; in ena_calc_io_queue_size()
2797 max_queue_ext->max_rx_cq_depth, in ena_calc_io_queue_size()
2798 max_queue_ext->max_rx_sq_depth); in ena_calc_io_queue_size()
2799 max_tx_queue_size = max_queue_ext->max_tx_cq_depth; in ena_calc_io_queue_size()
2801 if (ena_dev->tx_mem_queue_type == in ena_calc_io_queue_size()
2804 llq->max_llq_depth); in ena_calc_io_queue_size()
2807 max_queue_ext->max_tx_sq_depth); in ena_calc_io_queue_size()
2809 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, in ena_calc_io_queue_size()
2810 max_queue_ext->max_per_packet_tx_descs); in ena_calc_io_queue_size()
2811 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, in ena_calc_io_queue_size()
2812 max_queue_ext->max_per_packet_rx_descs); in ena_calc_io_queue_size()
2815 &ctx->get_feat_ctx->max_queues; in ena_calc_io_queue_size()
2816 max_rx_queue_size = min_t(uint32_t, max_queues->max_cq_depth, in ena_calc_io_queue_size()
2817 max_queues->max_sq_depth); in ena_calc_io_queue_size()
2818 max_tx_queue_size = max_queues->max_cq_depth; in ena_calc_io_queue_size()
2820 if (ena_dev->tx_mem_queue_type == in ena_calc_io_queue_size()
2823 llq->max_llq_depth); in ena_calc_io_queue_size()
2826 max_queues->max_sq_depth); in ena_calc_io_queue_size()
2828 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, in ena_calc_io_queue_size()
2829 max_queues->max_packet_tx_descs); in ena_calc_io_queue_size()
2830 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, in ena_calc_io_queue_size()
2831 max_queues->max_packet_rx_descs); in ena_calc_io_queue_size()
2834 if (adapter->llq_policy == ENA_ADMIN_LIST_ENTRY_SIZE_256B) { in ena_calc_io_queue_size()
2835 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { in ena_calc_io_queue_size()
2836 if (llq->max_wide_llq_depth != max_tx_queue_size) { in ena_calc_io_queue_size()
2837 if (llq->max_wide_llq_depth == 0) { in ena_calc_io_queue_size()
2838 /* if there is no large llq max depth from device, we divide in ena_calc_io_queue_size()
2844 max_tx_queue_size = llq->max_wide_llq_depth; in ena_calc_io_queue_size()
2846 ena_log(ctx->pdev, INFO, in ena_calc_io_queue_size()
2850 ena_log(ctx->pdev, INFO, "Using large LLQ headers\n"); in ena_calc_io_queue_size()
2853 ena_log(ctx->pdev, WARN, in ena_calc_io_queue_size()
2854 "Using large headers failed: LLQ is disabled or device does not support large headers\n"); in ena_calc_io_queue_size()
2859 max_tx_queue_size = 1 << (flsl(max_tx_queue_size) - 1); in ena_calc_io_queue_size()
2860 max_rx_queue_size = 1 << (flsl(max_rx_queue_size) - 1); in ena_calc_io_queue_size()
2867 tx_queue_size = 1 << (flsl(tx_queue_size) - 1); in ena_calc_io_queue_size()
2868 rx_queue_size = 1 << (flsl(rx_queue_size) - 1); in ena_calc_io_queue_size()
2870 ctx->max_tx_queue_size = max_tx_queue_size; in ena_calc_io_queue_size()
2871 ctx->max_rx_queue_size = max_rx_queue_size; in ena_calc_io_queue_size()
2872 ctx->tx_queue_size = tx_queue_size; in ena_calc_io_queue_size()
2873 ctx->rx_queue_size = rx_queue_size; in ena_calc_io_queue_size()
2892 host_info = ena_dev->host_attr.host_info; in ena_config_host_info()
2895 host_info->bdf = rid; in ena_config_host_info()
2896 host_info->os_type = ENA_ADMIN_OS_FREEBSD; in ena_config_host_info()
2897 host_info->kernel_ver = osreldate; in ena_config_host_info()
2899 sprintf(host_info->kernel_ver_str, "%d", osreldate); in ena_config_host_info()
2900 host_info->os_dist = 0; in ena_config_host_info()
2901 strncpy(host_info->os_dist_str, osrelease, in ena_config_host_info()
2902 sizeof(host_info->os_dist_str) - 1); in ena_config_host_info()
2904 host_info->driver_version = (ENA_DRV_MODULE_VER_MAJOR) | in ena_config_host_info()
2907 host_info->num_cpus = mp_ncpus; in ena_config_host_info()
2908 host_info->driver_supported_features = in ena_config_host_info()
2933 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_device_init()
2954 ena_log(pdev, ERR, "Can not reset device\n"); in ena_device_init()
2960 ena_log(pdev, ERR, "device version is too low\n"); in ena_device_init()
2970 adapter->dma_width = dma_width; in ena_device_init()
2976 "Can not initialize ena admin queue with device\n"); in ena_device_init()
2989 /* Get Device Attributes */ in ena_device_init()
2993 "Cannot get attribute for ena device rc: %d\n", rc); in ena_device_init()
3005 aenq_groups &= get_feat_ctx->aenq.supported_groups; in ena_device_init()
3014 ena_set_llq_configurations(&llq_config, &get_feat_ctx->llq, adapter); in ena_device_init()
3016 rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq, in ena_device_init()
3037 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_enable_msix_and_set_admin_interrupts()
3042 ena_log(adapter->pdev, ERR, "Error with MSI-X enablement\n"); in ena_enable_msix_and_set_admin_interrupts()
3050 ena_log(adapter->pdev, ERR, "Cannot setup mgmnt queue intr\n"); in ena_enable_msix_and_set_admin_interrupts()
3078 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; in ena_keep_alive_wd()
3079 tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low; in ena_keep_alive_wd()
3080 counter_u64_zero(adapter->hw_stats.rx_drops); in ena_keep_alive_wd()
3081 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops); in ena_keep_alive_wd()
3082 counter_u64_zero(adapter->hw_stats.tx_drops); in ena_keep_alive_wd()
3083 counter_u64_add(adapter->hw_stats.tx_drops, tx_drops); in ena_keep_alive_wd()
3086 atomic_store_rel_64(&adapter->keep_alive_timestamp, stime); in ena_keep_alive_wd()
3096 if (adapter->wd_active == 0) in check_for_missing_keep_alive()
3099 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) in check_for_missing_keep_alive()
3102 timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp); in check_for_missing_keep_alive()
3103 time = getsbinuptime() - timestamp; in check_for_missing_keep_alive()
3104 if (unlikely(time > adapter->keep_alive_timeout)) { in check_for_missing_keep_alive()
3105 ena_log(adapter->pdev, ERR, "Keep alive watchdog timeout.\n"); in check_for_missing_keep_alive()
3106 if (ena_com_aenq_has_keep_alive(adapter->ena_dev)) in check_for_missing_keep_alive()
3118 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == false)) { in check_for_admin_com_state()
3119 ena_log(adapter->pdev, ERR, in check_for_admin_com_state()
3121 counter_u64_add(adapter->dev_stats.admin_q_pause, 1); in check_for_admin_com_state()
3122 if (ena_com_get_missing_admin_interrupt(adapter->ena_dev)) in check_for_admin_com_state()
3133 if (likely(atomic_load_8(&rx_ring->first_interrupt))) in check_for_rx_interrupt_queue()
3136 if (ena_com_cq_empty(rx_ring->ena_com_io_cq)) in check_for_rx_interrupt_queue()
3139 rx_ring->no_interrupt_event_cnt++; in check_for_rx_interrupt_queue()
3141 if (rx_ring->no_interrupt_event_cnt == in check_for_rx_interrupt_queue()
3143 ena_log(adapter->pdev, ERR, in check_for_rx_interrupt_queue()
3144 "Potential MSIX issue on Rx side Queue = %d. Reset the device\n", in check_for_rx_interrupt_queue()
3145 rx_ring->qid); in check_for_rx_interrupt_queue()
3157 device_t pdev = adapter->pdev; in check_cdesc_in_tx_cq()
3161 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id); in check_cdesc_in_tx_cq()
3166 tx_ring->qid); in check_cdesc_in_tx_cq()
3173 tx_ring->qid); in check_cdesc_in_tx_cq()
3183 device_t pdev = adapter->pdev; in check_missing_comp_in_tx_queue()
3195 for (i = 0; i < tx_ring->ring_size; i++) { in check_missing_comp_in_tx_queue()
3196 tx_buf = &tx_ring->tx_buffer_info[i]; in check_missing_comp_in_tx_queue()
3198 if (bintime_isset(&tx_buf->timestamp) == 0) in check_missing_comp_in_tx_queue()
3202 bintime_sub(&time, &tx_buf->timestamp); in check_missing_comp_in_tx_queue()
3205 if (unlikely(!atomic_load_8(&tx_ring->first_interrupt) && in check_missing_comp_in_tx_queue()
3206 time_offset > 2 * adapter->missing_tx_timeout)) { in check_missing_comp_in_tx_queue()
3213 "Reset the device\n", in check_missing_comp_in_tx_queue()
3214 tx_ring->qid); in check_missing_comp_in_tx_queue()
3221 if (unlikely(time_offset > adapter->missing_tx_timeout)) { in check_missing_comp_in_tx_queue()
3223 if (tx_buf->print_once) { in check_missing_comp_in_tx_queue()
3224 time_since_last_cleanup = TICKS_2_MSEC(ticks - in check_missing_comp_in_tx_queue()
3225 tx_ring->tx_last_cleanup_ticks); in check_missing_comp_in_tx_queue()
3227 adapter->missing_tx_timeout); in check_missing_comp_in_tx_queue()
3231 tx_ring->qid, i, time_since_last_cleanup, in check_missing_comp_in_tx_queue()
3237 tx_buf->print_once = false; in check_missing_comp_in_tx_queue()
3242 if (unlikely(missed_tx > adapter->missing_tx_threshold)) { in check_missing_comp_in_tx_queue()
3245 "(%d > %d). Reset the device\n", in check_missing_comp_in_tx_queue()
3246 missed_tx, adapter->missing_tx_threshold); in check_missing_comp_in_tx_queue()
3254 cleanup_scheduled = !!(atomic_load_16(&tx_ring->que->cleanup_task.ta_pending)); in check_missing_comp_in_tx_queue()
3255 cleanup_running = !!(atomic_load_8((&tx_ring->cleanup_running))); in check_missing_comp_in_tx_queue()
3259 adapter->reset_reason = reset_reason; in check_missing_comp_in_tx_queue()
3263 counter_u64_add(tx_ring->tx_stats.missing_tx_comp, new_missed_tx); in check_missing_comp_in_tx_queue()
3281 /* Make sure the driver doesn't turn the device in other process */ in check_for_missing_completions()
3290 if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT) in check_for_missing_completions()
3293 budget = adapter->missing_tx_max_queues; in check_for_missing_completions()
3295 for (i = adapter->next_monitored_tx_qid; i < adapter->num_io_queues; i++) { in check_for_missing_completions()
3296 tx_ring = &adapter->tx_ring[i]; in check_for_missing_completions()
3297 rx_ring = &adapter->rx_ring[i]; in check_for_missing_completions()
3307 budget--; in check_for_missing_completions()
3314 adapter->next_monitored_tx_qid = i % adapter->num_io_queues; in check_for_missing_completions()
3319 /* For the rare case where the device runs out of Rx descriptors and the
3323 * The device won't send interrupts since all the new Rx packets will be dropped
3324 * The msix handler won't allocate new Rx descriptors so the device won't be
3327 * When such a situation is detected - execute rx cleanup task in another thread
3341 for (i = 0; i < adapter->num_io_queues; i++) { in check_for_empty_rx_ring()
3342 rx_ring = &adapter->rx_ring[i]; in check_for_empty_rx_ring()
3345 rx_ring->ena_com_io_sq); in check_for_empty_rx_ring()
3346 if (unlikely(refill_required == (rx_ring->ring_size - 1))) { in check_for_empty_rx_ring()
3347 rx_ring->empty_rx_queue++; in check_for_empty_rx_ring()
3349 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) { in check_for_empty_rx_ring()
3350 counter_u64_add(rx_ring->rx_stats.empty_rx_ring, in check_for_empty_rx_ring()
3353 ena_log(adapter->pdev, WARN, in check_for_empty_rx_ring()
3357 taskqueue_enqueue(rx_ring->que->cleanup_tq, in check_for_empty_rx_ring()
3358 &rx_ring->que->cleanup_task); in check_for_empty_rx_ring()
3359 rx_ring->empty_rx_queue = 0; in check_for_empty_rx_ring()
3362 rx_ring->empty_rx_queue = 0; in check_for_empty_rx_ring()
3371 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_update_hints()
3373 if (hints->admin_completion_tx_timeout) in ena_update_hints()
3374 ena_dev->admin_queue.completion_timeout = in ena_update_hints()
3375 hints->admin_completion_tx_timeout * 1000; in ena_update_hints()
3377 if (hints->mmio_read_timeout) in ena_update_hints()
3379 ena_dev->mmio_read.reg_read_to = hints->mmio_read_timeout * 1000; in ena_update_hints()
3381 if (hints->missed_tx_completion_count_threshold_to_reset) in ena_update_hints()
3382 adapter->missing_tx_threshold = in ena_update_hints()
3383 hints->missed_tx_completion_count_threshold_to_reset; in ena_update_hints()
3385 if (hints->missing_tx_completion_timeout) { in ena_update_hints()
3386 if (hints->missing_tx_completion_timeout == in ena_update_hints()
3388 adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT; in ena_update_hints()
3390 adapter->missing_tx_timeout = SBT_1MS * in ena_update_hints()
3391 hints->missing_tx_completion_timeout; in ena_update_hints()
3394 if (hints->driver_watchdog_timeout) { in ena_update_hints()
3395 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) in ena_update_hints()
3396 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; in ena_update_hints()
3398 adapter->keep_alive_timeout = SBT_1MS * in ena_update_hints()
3399 hints->driver_watchdog_timeout; in ena_update_hints()
3404 * ena_copy_eni_metrics - Get and copy ENI metrics from the HW.
3405 * @adapter: ENA device adapter
3420 rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_metrics); in ena_copy_eni_metrics()
3425 ena_log(adapter->pdev, WARN, in ena_copy_eni_metrics()
3429 ena_log(adapter->pdev, DBG, in ena_copy_eni_metrics()
3433 ena_log(adapter->pdev, ERR, in ena_copy_eni_metrics()
3444 return ena_com_get_ena_srd_info(adapter->ena_dev, &adapter->ena_srd_info); in ena_copy_srd_metrics()
3454 dev = adapter->ena_dev; in ena_copy_customer_metrics()
3460 rc = ena_com_get_customer_metrics(adapter->ena_dev, in ena_copy_customer_metrics()
3461 (char *)(adapter->customer_metrics_array), len); in ena_copy_customer_metrics()
3471 adapter->ena_dev->host_attr.host_info; in ena_timer_service()
3490 if ((adapter->metrics_sample_interval != 0) && in ena_timer_service()
3491 (++adapter->metrics_sample_interval_cnt >= in ena_timer_service()
3492 adapter->metrics_sample_interval)) { in ena_timer_service()
3493 taskqueue_enqueue(adapter->metrics_tq, &adapter->metrics_task); in ena_timer_service()
3494 adapter->metrics_sample_interval_cnt = 0; in ena_timer_service()
3499 ena_update_host_info(host_info, adapter->ifp); in ena_timer_service()
3503 * Timeout when validating version indicates that the device in ena_timer_service()
3507 if (ena_com_validate_version(adapter->ena_dev) == in ena_timer_service()
3509 ena_log(adapter->pdev, WARN, in ena_timer_service()
3514 ena_log(adapter->pdev, WARN, "Trigger reset is on\n"); in ena_timer_service()
3515 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task); in ena_timer_service()
3528 if_t ifp = adapter->ifp; in ena_destroy_device()
3529 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_destroy_device()
3551 * Stop the device from sending AENQ events (if the device was up, and in ena_destroy_device()
3552 * the trigger reset was on, ena_down already performs device reset) in ena_destroy_device()
3555 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); in ena_destroy_device()
3563 * calls (not directly) `ena_enable_msix()`, which re-allocates MSIX in ena_destroy_device()
3564 * vectors. The amount of MSIX vectors after destroy-restore may be in ena_destroy_device()
3578 adapter->reset_reason = ENA_REGS_RESET_NORMAL; in ena_destroy_device()
3588 if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr, in ena_device_validate_params()
3590 ena_log(adapter->pdev, ERR, "Error, mac addresses differ\n"); in ena_device_validate_params()
3594 if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) { in ena_device_validate_params()
3595 ena_log(adapter->pdev, ERR, in ena_device_validate_params()
3596 "Error, device max mtu is smaller than ifp MTU\n"); in ena_device_validate_params()
3607 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_restore_device()
3608 if_t ifp = adapter->ifp; in ena_restore_device()
3609 device_t dev = adapter->pdev; in ena_restore_device()
3617 ena_log(dev, ERR, "Cannot initialize device\n"); in ena_restore_device()
3624 if (adapter->wd_active != 0) in ena_restore_device()
3625 adapter->wd_active = wd_active; in ena_restore_device()
3629 ena_log(dev, ERR, "Validation of device parameters failed\n"); in ena_restore_device()
3640 ena_log(dev, ERR, "Enable MSI-X failed\n"); in ena_restore_device()
3649 if ((adapter->msix_vecs - ENA_ADMIN_MSIX_VEC) < adapter->num_io_queues) in ena_restore_device()
3650 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; in ena_restore_device()
3652 /* Re-initialize rings basic information */ in ena_restore_device()
3664 /* Indicate that device is running again and ready to work */ in ena_restore_device()
3673 adapter->keep_alive_timestamp = getsbinuptime(); in ena_restore_device()
3692 ena_log(dev, ERR, "Reset attempt failed. Can not reset the device\n"); in ena_restore_device()
3704 if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_CUSTOMER_METRICS)) in ena_metrics_task()
3706 else if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENI_STATS)) in ena_metrics_task()
3709 if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENA_SRD_INFO)) in ena_metrics_task()
3726 ena_log(adapter->pdev, INFO, in ena_reset_task()
3727 "Device reset completed successfully, Driver info: %s\n", in ena_reset_task()
3736 ena_free_counters((counter_u64_t *)&adapter->hw_stats, in ena_free_stats()
3738 ena_free_counters((counter_u64_t *)&adapter->dev_stats, in ena_free_stats()
3743 * ena_attach - Device Initialization Routine
3744 * @pdev: device information struct
3748 * ena_attach initializes an adapter identified by a device structure.
3765 adapter->pdev = pdev; in ena_attach()
3766 adapter->first_bind = -1; in ena_attach()
3769 * Set up the timer service - driver is responsible for avoiding in ena_attach()
3773 adapter->keep_alive_timeout = ENA_DEFAULT_KEEP_ALIVE_TO; in ena_attach()
3774 adapter->missing_tx_timeout = ENA_DEFAULT_TX_CMP_TO; in ena_attach()
3775 adapter->missing_tx_max_queues = ENA_DEFAULT_TX_MONITORED_QUEUES; in ena_attach()
3776 adapter->missing_tx_threshold = ENA_DEFAULT_TX_CMP_THRESHOLD; in ena_attach()
3778 adapter->irq_cpu_base = ENA_BASE_CPU_UNSPECIFIED; in ena_attach()
3779 adapter->irq_cpu_stride = 0; in ena_attach()
3782 adapter->rss_enabled = 1; in ena_attach()
3792 adapter->ena_dev = ena_dev; in ena_attach()
3793 ena_dev->dmadev = pdev; in ena_attach()
3796 adapter->memory = NULL; in ena_attach()
3797 adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid, in ena_attach()
3799 if (unlikely(adapter->registers == NULL)) { in ena_attach()
3809 adapter->msix = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, in ena_attach()
3811 if (unlikely(adapter->msix == NULL)) { in ena_attach()
3817 adapter->msix_rid = msix_rid; in ena_attach()
3820 ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF, in ena_attach()
3824 ((struct ena_bus *)(ena_dev->bus))->reg_bar_t = rman_get_bustag( in ena_attach()
3825 adapter->registers); in ena_attach()
3826 ((struct ena_bus *)(ena_dev->bus))->reg_bar_h = rman_get_bushandle( in ena_attach()
3827 adapter->registers); in ena_attach()
3829 if (unlikely(((struct ena_bus *)(ena_dev->bus))->reg_bar_h == 0)) { in ena_attach()
3841 ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US; in ena_attach()
3846 /* Device initialization */ in ena_attach()
3847 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active); in ena_attach()
3849 ena_log(pdev, ERR, "ENA device init failed! (err: %d)\n", rc); in ena_attach()
3854 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) in ena_attach()
3855 adapter->disable_meta_caching = !!( in ena_attach()
3859 adapter->keep_alive_timestamp = getsbinuptime(); in ena_attach()
3861 adapter->tx_offload_cap = get_feat_ctx.offload.tx; in ena_attach()
3863 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr, in ena_attach()
3879 adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size; in ena_attach()
3880 adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size; in ena_attach()
3881 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size; in ena_attach()
3882 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size; in ena_attach()
3883 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; in ena_attach()
3884 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; in ena_attach()
3886 adapter->max_num_io_queues = max_num_io_queues; in ena_attach()
3888 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE; in ena_attach()
3890 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; in ena_attach()
3892 adapter->reset_reason = ENA_REGS_RESET_NORMAL; in ena_attach()
3923 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; in ena_attach()
3941 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats, in ena_attach()
3943 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats, in ena_attach()
3951 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter); in ena_attach()
3952 adapter->reset_tq = taskqueue_create("ena_reset_enqueue", in ena_attach()
3953 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq); in ena_attach()
3954 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, "%s rstq", in ena_attach()
3955 device_get_nameunit(adapter->pdev)); in ena_attach()
3958 TASK_INIT(&adapter->metrics_task, 0, ena_metrics_task, adapter); in ena_attach()
3959 adapter->metrics_tq = taskqueue_create("ena_metrics_enqueue", in ena_attach()
3960 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->metrics_tq); in ena_attach()
3961 taskqueue_start_threads(&adapter->metrics_tq, 1, PI_NET, "%s metricsq", in ena_attach()
3962 device_get_nameunit(adapter->pdev)); in ena_attach()
3973 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); in ena_attach()
3983 ether_ifdetach(adapter->ifp); in ena_attach()
3984 ifmedia_removeall(&adapter->media); in ena_attach()
3985 free(adapter->customer_metrics_array, M_DEVBUF); in ena_attach()
3991 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR); in ena_attach()
4004 free(ena_dev->bus, M_DEVBUF); in ena_attach()
4014 * ena_detach - Device Removal Routine
4015 * @pdev: device information struct
4017 * ena_detach is called by the device subsystem to alert the driver
4018 * that it should release a PCI device.
4024 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_detach()
4028 if (if_vlantrunkinuse(adapter->ifp)) { in ena_detach()
4029 ena_log(adapter->pdev, ERR, "VLAN is in use, detach first\n"); in ena_detach()
4037 ether_ifdetach(adapter->ifp); in ena_detach()
4039 ifmedia_removeall(&adapter->media); in ena_detach()
4047 while (taskqueue_cancel(adapter->metrics_tq, &adapter->metrics_task, NULL)) in ena_detach()
4048 taskqueue_drain(adapter->metrics_tq, &adapter->metrics_task); in ena_detach()
4049 taskqueue_free(adapter->metrics_tq); in ena_detach()
4052 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL)) in ena_detach()
4053 taskqueue_drain(adapter->reset_tq, &adapter->reset_task); in ena_detach()
4054 taskqueue_free(adapter->reset_tq); in ena_detach()
4062 ena_sysctl_update_queue_node_nb(adapter, adapter->num_io_queues, in ena_detach()
4063 adapter->max_num_io_queues); in ena_detach()
4066 netmap_detach(adapter->ifp); in ena_detach()
4073 ena_log(adapter->pdev, WARN, in ena_detach()
4078 ena_log(adapter->pdev, WARN, in ena_detach()
4085 if (adapter->rss_indir != NULL) in ena_detach()
4086 free(adapter->rss_indir, M_DEVBUF); in ena_detach()
4093 free(adapter->customer_metrics_array, M_DEVBUF); in ena_detach()
4097 if_free(adapter->ifp); in ena_detach()
4099 free(ena_dev->bus, M_DEVBUF); in ena_detach()
4123 ifp = adapter->ifp; in ena_update_on_link_change()
4124 status = aenq_desc->flags & in ena_update_on_link_change()
4128 ena_log(adapter->pdev, INFO, "link is UP\n"); in ena_update_on_link_change()
4133 ena_log(adapter->pdev, INFO, "link is DOWN\n"); in ena_update_on_link_change()
4145 ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION, in ena_notification()
4146 adapter->ena_dev, "Invalid group(%x) expected %x\n", in ena_notification()
4147 aenq_e->aenq_common_desc.group, ENA_ADMIN_NOTIFICATION); in ena_notification()
4149 switch (aenq_e->aenq_common_desc.syndrome) { in ena_notification()
4152 (struct ena_admin_ena_hw_hints *)(&aenq_e->inline_data_w4); in ena_notification()
4156 ena_log(adapter->pdev, ERR, in ena_notification()
4158 aenq_e->aenq_common_desc.syndrome); in ena_notification()
4185 ena_log(adapter->pdev, ERR, in unimplemented_aenq_handler()
4197 bitmap = desc->notifications_bitmap; in ena_conf_notification()
4200 ena_log(adapter->pdev, INFO, in ena_conf_notification()
4206 bit--; in ena_conf_notification()
4207 ena_log(adapter->pdev, INFO, in ena_conf_notification()
4208 …"Sub-optimal configuration notification code: %" PRIu64 " Refer to AWS ENA documentation for addit… in ena_conf_notification()
4219 ena_log(adapter->pdev, WARN, in ena_admin_device_request_reset()
4220 "The device has detected an unhealthy state, reset is requested\n"); in ena_admin_device_request_reset()
4236 * FreeBSD Device Interface Entry Points
4239 static device_method_t ena_methods[] = { /* Device interface */
4252 MODULE_PNP_INFO("U16:vendor;U16:device", pci, ena, ena_vendor_info_array,
4253 nitems(ena_vendor_info_array) - 1);