Lines Matching full:adapter
175 static int ena_reinit_netmap(struct ena_adapter *adapter);
209 struct ena_adapter *adapter = device_get_softc(dmadev); in ena_dma_alloc() local
210 device_t pdev = adapter->pdev; in ena_dma_alloc()
217 dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width); in ena_dma_alloc()
279 ena_free_pci_resources(struct ena_adapter *adapter) in ena_free_pci_resources() argument
281 device_t pdev = adapter->pdev; in ena_free_pci_resources()
283 if (adapter->memory != NULL) { in ena_free_pci_resources()
285 PCIR_BAR(ENA_MEM_BAR), adapter->memory); in ena_free_pci_resources()
288 if (adapter->registers != NULL) { in ena_free_pci_resources()
290 PCIR_BAR(ENA_REG_BAR), adapter->registers); in ena_free_pci_resources()
293 if (adapter->msix != NULL) { in ena_free_pci_resources()
294 bus_release_resource(pdev, SYS_RES_MEMORY, adapter->msix_rid, in ena_free_pci_resources()
295 adapter->msix); in ena_free_pci_resources()
329 struct ena_adapter *adapter = if_getsoftc(ifp); in ena_change_mtu() local
330 device_t pdev = adapter->pdev; in ena_change_mtu()
333 if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) { in ena_change_mtu()
335 new_mtu, adapter->max_mtu, ENA_MIN_MTU); in ena_change_mtu()
339 rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); in ena_change_mtu()
378 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring, in ena_init_io_rings_common() argument
382 ring->adapter = adapter; in ena_init_io_rings_common()
383 ring->ena_dev = adapter->ena_dev; in ena_init_io_rings_common()
389 ena_init_io_rings_basic(struct ena_adapter *adapter) in ena_init_io_rings_basic() argument
396 ena_dev = adapter->ena_dev; in ena_init_io_rings_basic()
398 for (i = 0; i < adapter->num_io_queues; i++) { in ena_init_io_rings_basic()
399 txr = &adapter->tx_ring[i]; in ena_init_io_rings_basic()
400 rxr = &adapter->rx_ring[i]; in ena_init_io_rings_basic()
403 ena_init_io_rings_common(adapter, txr, i); in ena_init_io_rings_basic()
404 ena_init_io_rings_common(adapter, rxr, i); in ena_init_io_rings_basic()
410 que = &adapter->que[i]; in ena_init_io_rings_basic()
411 que->adapter = adapter; in ena_init_io_rings_basic()
425 ena_init_io_rings_advanced(struct ena_adapter *adapter) in ena_init_io_rings_advanced() argument
430 for (i = 0; i < adapter->num_io_queues; i++) { in ena_init_io_rings_advanced()
431 txr = &adapter->tx_ring[i]; in ena_init_io_rings_advanced()
432 rxr = &adapter->rx_ring[i]; in ena_init_io_rings_advanced()
435 txr->buf_ring_size = adapter->buf_ring_size; in ena_init_io_rings_advanced()
450 device_get_nameunit(adapter->pdev), i); in ena_init_io_rings_advanced()
452 device_get_nameunit(adapter->pdev), i); in ena_init_io_rings_advanced()
459 ena_init_io_rings(struct ena_adapter *adapter) in ena_init_io_rings() argument
464 * them from adapter/ena_dev (basic) in ena_init_io_rings()
467 ena_init_io_rings_basic(adapter); in ena_init_io_rings()
468 ena_init_io_rings_advanced(adapter); in ena_init_io_rings()
472 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid) in ena_free_io_ring_resources() argument
474 struct ena_ring *txr = &adapter->tx_ring[qid]; in ena_free_io_ring_resources()
475 struct ena_ring *rxr = &adapter->rx_ring[qid]; in ena_free_io_ring_resources()
490 ena_free_all_io_rings_resources(struct ena_adapter *adapter) in ena_free_all_io_rings_resources() argument
494 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_io_rings_resources()
495 ena_free_io_ring_resources(adapter, i); in ena_free_all_io_rings_resources()
499 ena_setup_tx_dma_tag(struct ena_adapter *adapter) in ena_setup_tx_dma_tag() argument
504 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), in ena_setup_tx_dma_tag()
506 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ in ena_setup_tx_dma_tag()
510 adapter->max_tx_sgl_size - 1, /* nsegments */ in ena_setup_tx_dma_tag()
515 &adapter->tx_buf_tag); in ena_setup_tx_dma_tag()
521 ena_free_tx_dma_tag(struct ena_adapter *adapter) in ena_free_tx_dma_tag() argument
525 ret = bus_dma_tag_destroy(adapter->tx_buf_tag); in ena_free_tx_dma_tag()
528 adapter->tx_buf_tag = NULL; in ena_free_tx_dma_tag()
534 ena_setup_rx_dma_tag(struct ena_adapter *adapter) in ena_setup_rx_dma_tag() argument
539 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */ in ena_setup_rx_dma_tag()
541 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ in ena_setup_rx_dma_tag()
545 adapter->max_rx_sgl_size, /* nsegments */ in ena_setup_rx_dma_tag()
550 &adapter->rx_buf_tag); in ena_setup_rx_dma_tag()
556 ena_free_rx_dma_tag(struct ena_adapter *adapter) in ena_free_rx_dma_tag() argument
560 ret = bus_dma_tag_destroy(adapter->rx_buf_tag); in ena_free_rx_dma_tag()
563 adapter->rx_buf_tag = NULL; in ena_free_rx_dma_tag()
571 struct ena_adapter *adapter = tx_ring->adapter; in validate_tx_req_id() local
577 ena_log(adapter->pdev, ERR, in validate_tx_req_id()
581 ena_log_nm(adapter->pdev, WARN, in validate_tx_req_id()
587 ena_trigger_reset(adapter, reset_reason); in validate_tx_req_id()
597 struct ena_adapter *adapter = tx_ring->adapter; in ena_release_all_tx_dmamap() local
599 bus_dma_tag_t tx_tag = adapter->tx_buf_tag; in ena_release_all_tx_dmamap()
609 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) { in ena_release_all_tx_dmamap()
629 * @adapter: network interface device structure
635 ena_setup_tx_resources(struct ena_adapter *adapter, int qid) in ena_setup_tx_resources() argument
637 device_t pdev = adapter->pdev; in ena_setup_tx_resources()
639 struct ena_que *que = &adapter->que[qid]; in ena_setup_tx_resources()
647 ena_netmap_reset_tx_ring(adapter, qid); in ena_setup_tx_resources()
681 drbr_flush(adapter->ifp, tx_ring->br); in ena_setup_tx_resources()
686 err = bus_dmamap_create(adapter->tx_buf_tag, 0, in ena_setup_tx_resources()
695 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) { in ena_setup_tx_resources()
698 err = bus_dmamap_create(adapter->tx_buf_tag, 0, in ena_setup_tx_resources()
727 device_get_nameunit(adapter->pdev), que->cpu); in ena_setup_tx_resources()
730 device_get_nameunit(adapter->pdev), que->id); in ena_setup_tx_resources()
751 * @adapter: network interface device structure
757 ena_free_tx_resources(struct ena_adapter *adapter, int qid) in ena_free_tx_resources() argument
759 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_free_tx_resources()
772 drbr_flush(adapter->ifp, tx_ring->br); in ena_free_tx_resources()
776 bus_dmamap_sync(adapter->tx_buf_tag, in ena_free_tx_resources()
778 bus_dmamap_unload(adapter->tx_buf_tag, in ena_free_tx_resources()
780 bus_dmamap_destroy(adapter->tx_buf_tag, in ena_free_tx_resources()
784 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) { in ena_free_tx_resources()
788 bus_dmamap_sync(adapter->tx_buf_tag, in ena_free_tx_resources()
791 ena_netmap_unload(adapter, in ena_free_tx_resources()
794 bus_dmamap_destroy(adapter->tx_buf_tag, in ena_free_tx_resources()
819 * @adapter: network interface device structure
824 ena_setup_all_tx_resources(struct ena_adapter *adapter) in ena_setup_all_tx_resources() argument
828 for (i = 0; i < adapter->num_io_queues; i++) { in ena_setup_all_tx_resources()
829 rc = ena_setup_tx_resources(adapter, i); in ena_setup_all_tx_resources()
831 ena_log(adapter->pdev, ERR, in ena_setup_all_tx_resources()
842 ena_free_tx_resources(adapter, i); in ena_setup_all_tx_resources()
848 * @adapter: network interface device structure
853 ena_free_all_tx_resources(struct ena_adapter *adapter) in ena_free_all_tx_resources() argument
857 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_tx_resources()
858 ena_free_tx_resources(adapter, i); in ena_free_all_tx_resources()
863 * @adapter: network interface device structure
869 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid) in ena_setup_rx_resources() argument
871 device_t pdev = adapter->pdev; in ena_setup_rx_resources()
872 struct ena_que *que = &adapter->que[qid]; in ena_setup_rx_resources()
879 ena_netmap_reset_rx_ring(adapter, qid); in ena_setup_rx_resources()
906 err = bus_dmamap_create(adapter->rx_buf_tag, 0, in ena_setup_rx_resources()
916 if ((if_getcapenable(adapter->ifp) & IFCAP_LRO) != 0) { in ena_setup_rx_resources()
924 rx_ring->lro.ifp = adapter->ifp; in ena_setup_rx_resources()
932 bus_dmamap_destroy(adapter->rx_buf_tag, in ena_setup_rx_resources()
945 * @adapter: network interface device structure
951 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid) in ena_free_rx_resources() argument
953 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_resources()
957 bus_dmamap_sync(adapter->rx_buf_tag, in ena_free_rx_resources()
961 bus_dmamap_unload(adapter->rx_buf_tag, in ena_free_rx_resources()
963 bus_dmamap_destroy(adapter->rx_buf_tag, in ena_free_rx_resources()
980 * @adapter: network interface device structure
985 ena_setup_all_rx_resources(struct ena_adapter *adapter) in ena_setup_all_rx_resources() argument
989 for (i = 0; i < adapter->num_io_queues; i++) { in ena_setup_all_rx_resources()
990 rc = ena_setup_rx_resources(adapter, i); in ena_setup_all_rx_resources()
992 ena_log(adapter->pdev, ERR, in ena_setup_all_rx_resources()
1002 ena_free_rx_resources(adapter, i); in ena_setup_all_rx_resources()
1008 * @adapter: network interface device structure
1013 ena_free_all_rx_resources(struct ena_adapter *adapter) in ena_free_all_rx_resources() argument
1017 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_rx_resources()
1018 ena_free_rx_resources(adapter, i); in ena_free_all_rx_resources()
1022 ena_alloc_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring, in ena_alloc_rx_mbuf() argument
1025 device_t pdev = adapter->pdev; in ena_alloc_rx_mbuf()
1056 adapter->rx_buf_tag, rx_info->mbuf, rx_info->mbuf->m_len); in ena_alloc_rx_mbuf()
1057 error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map, in ena_alloc_rx_mbuf()
1066 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD); in ena_alloc_rx_mbuf()
1085 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring, in ena_free_rx_mbuf() argument
1089 ena_log(adapter->pdev, WARN, in ena_free_rx_mbuf()
1094 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, in ena_free_rx_mbuf()
1096 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map); in ena_free_rx_mbuf()
1110 struct ena_adapter *adapter = rx_ring->adapter; in ena_refill_rx_bufs() local
1111 device_t pdev = adapter->pdev; in ena_refill_rx_bufs()
1116 ena_log_io(adapter->pdev, DBG, "refill qid: %d\n", rx_ring->qid); in ena_refill_rx_bufs()
1129 if (ena_rx_ring_in_netmap(adapter, rx_ring->qid)) in ena_refill_rx_bufs()
1130 rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, in ena_refill_rx_bufs()
1134 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info); in ena_refill_rx_bufs()
1169 ena_reinit_netmap(struct ena_adapter *adapter) in ena_reinit_netmap() argument
1173 netmap_detach(adapter->ifp); in ena_reinit_netmap()
1174 rc = ena_netmap_attach(adapter); in ena_reinit_netmap()
1176 ena_log(adapter->pdev, ERR, "netmap attach failed: %d\n", rc); in ena_reinit_netmap()
1183 ena_update_buf_ring_size(struct ena_adapter *adapter, in ena_update_buf_ring_size() argument
1190 old_buf_ring_size = adapter->buf_ring_size; in ena_update_buf_ring_size()
1191 adapter->buf_ring_size = new_buf_ring_size; in ena_update_buf_ring_size()
1193 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_update_buf_ring_size()
1194 ena_down(adapter); in ena_update_buf_ring_size()
1197 ena_free_all_io_rings_resources(adapter); in ena_update_buf_ring_size()
1198 ena_init_io_rings_advanced(adapter); in ena_update_buf_ring_size()
1200 rc = ena_reinit_netmap(adapter); in ena_update_buf_ring_size()
1212 rc = ena_up(adapter); in ena_update_buf_ring_size()
1214 ena_log(adapter->pdev, ERR, in ena_update_buf_ring_size()
1219 adapter->buf_ring_size = old_buf_ring_size; in ena_update_buf_ring_size()
1220 ena_free_all_io_rings_resources(adapter); in ena_update_buf_ring_size()
1221 ena_init_io_rings_advanced(adapter); in ena_update_buf_ring_size()
1223 rc = ena_reinit_netmap(adapter); in ena_update_buf_ring_size()
1229 adapter); in ena_update_buf_ring_size()
1230 ena_trigger_reset(adapter, ENA_REGS_RESET_OS_TRIGGER); in ena_update_buf_ring_size()
1238 ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size, in ena_update_queue_size() argument
1245 old_tx_size = adapter->requested_tx_ring_size; in ena_update_queue_size()
1246 old_rx_size = adapter->requested_rx_ring_size; in ena_update_queue_size()
1247 adapter->requested_tx_ring_size = new_tx_size; in ena_update_queue_size()
1248 adapter->requested_rx_ring_size = new_rx_size; in ena_update_queue_size()
1250 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_update_queue_size()
1251 ena_down(adapter); in ena_update_queue_size()
1254 ena_init_io_rings_basic(adapter); in ena_update_queue_size()
1256 rc = ena_reinit_netmap(adapter); in ena_update_queue_size()
1262 rc = ena_up(adapter); in ena_update_queue_size()
1264 ena_log(adapter->pdev, ERR, in ena_update_queue_size()
1269 adapter->requested_tx_ring_size = old_tx_size; in ena_update_queue_size()
1270 adapter->requested_rx_ring_size = old_rx_size; in ena_update_queue_size()
1271 ena_init_io_rings_basic(adapter); in ena_update_queue_size()
1273 rc = ena_reinit_netmap(adapter); in ena_update_queue_size()
1279 rc = ena_up(adapter); in ena_update_queue_size()
1281 ena_log(adapter->pdev, ERR, in ena_update_queue_size()
1289 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_update_queue_size()
1290 ena_trigger_reset(adapter, in ena_update_queue_size()
1300 ena_update_io_rings(struct ena_adapter *adapter, uint32_t num) in ena_update_io_rings() argument
1302 ena_free_all_io_rings_resources(adapter); in ena_update_io_rings()
1304 ena_com_rss_destroy(adapter->ena_dev); in ena_update_io_rings()
1306 adapter->num_io_queues = num; in ena_update_io_rings()
1307 ena_init_io_rings(adapter); in ena_update_io_rings()
1311 ena_update_base_cpu(struct ena_adapter *adapter, int new_num) in ena_update_base_cpu() argument
1317 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_update_base_cpu()
1318 old_num = adapter->irq_cpu_base; in ena_update_base_cpu()
1320 ena_down(adapter); in ena_update_base_cpu()
1322 adapter->irq_cpu_base = new_num; in ena_update_base_cpu()
1325 rc = ena_up(adapter); in ena_update_base_cpu()
1327 ena_log(adapter->pdev, ERR, in ena_update_base_cpu()
1332 adapter->irq_cpu_base = old_num; in ena_update_base_cpu()
1334 rc = ena_up(adapter); in ena_update_base_cpu()
1336 ena_log(adapter->pdev, ERR, in ena_update_base_cpu()
1340 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_update_base_cpu()
1341 ena_trigger_reset(adapter, in ena_update_base_cpu()
1350 ena_update_cpu_stride(struct ena_adapter *adapter, uint32_t new_num) in ena_update_cpu_stride() argument
1356 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_update_cpu_stride()
1357 old_num = adapter->irq_cpu_stride; in ena_update_cpu_stride()
1359 ena_down(adapter); in ena_update_cpu_stride()
1361 adapter->irq_cpu_stride = new_num; in ena_update_cpu_stride()
1364 rc = ena_up(adapter); in ena_update_cpu_stride()
1366 ena_log(adapter->pdev, ERR, in ena_update_cpu_stride()
1371 adapter->irq_cpu_stride = old_num; in ena_update_cpu_stride()
1373 rc = ena_up(adapter); in ena_update_cpu_stride()
1375 ena_log(adapter->pdev, ERR, in ena_update_cpu_stride()
1379 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_update_cpu_stride()
1380 ena_trigger_reset(adapter, in ena_update_cpu_stride()
1390 ena_update_io_queue_nb(struct ena_adapter *adapter, uint32_t new_num) in ena_update_io_queue_nb() argument
1396 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_update_io_queue_nb()
1397 old_num = adapter->num_io_queues; in ena_update_io_queue_nb()
1398 ena_down(adapter); in ena_update_io_queue_nb()
1400 ena_update_io_rings(adapter, new_num); in ena_update_io_queue_nb()
1402 rc = ena_reinit_netmap(adapter); in ena_update_io_queue_nb()
1408 rc = ena_up(adapter); in ena_update_io_queue_nb()
1410 ena_log(adapter->pdev, ERR, in ena_update_io_queue_nb()
1415 ena_update_io_rings(adapter, old_num); in ena_update_io_queue_nb()
1417 rc = ena_reinit_netmap(adapter); in ena_update_io_queue_nb()
1422 rc = ena_up(adapter); in ena_update_io_queue_nb()
1424 ena_log(adapter->pdev, ERR, in ena_update_io_queue_nb()
1428 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_update_io_queue_nb()
1429 ena_trigger_reset(adapter, in ena_update_io_queue_nb()
1439 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid) in ena_free_rx_bufs() argument
1441 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_bufs()
1448 ena_free_rx_mbuf(adapter, rx_ring, rx_info); in ena_free_rx_bufs()
1450 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) && in ena_free_rx_bufs()
1451 (if_getcapenable(adapter->ifp) & IFCAP_NETMAP)) { in ena_free_rx_bufs()
1453 ena_netmap_free_rx_slot(adapter, rx_ring, in ena_free_rx_bufs()
1462 * @adapter: network interface device structure
1466 ena_refill_all_rx_bufs(struct ena_adapter *adapter) in ena_refill_all_rx_bufs() argument
1471 for (i = 0; i < adapter->num_io_queues; i++) { in ena_refill_all_rx_bufs()
1472 rx_ring = &adapter->rx_ring[i]; in ena_refill_all_rx_bufs()
1476 ena_log_io(adapter->pdev, WARN, in ena_refill_all_rx_bufs()
1487 ena_free_all_rx_bufs(struct ena_adapter *adapter) in ena_free_all_rx_bufs() argument
1491 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_rx_bufs()
1492 ena_free_rx_bufs(adapter, i); in ena_free_all_rx_bufs()
1497 * @adapter: network interface device structure
1501 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid) in ena_free_tx_bufs() argument
1504 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_free_tx_bufs()
1514 ena_log(adapter->pdev, WARN, in ena_free_tx_bufs()
1519 ena_log(adapter->pdev, DBG, in ena_free_tx_bufs()
1524 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, in ena_free_tx_bufs()
1526 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); in ena_free_tx_bufs()
1535 ena_free_all_tx_bufs(struct ena_adapter *adapter) in ena_free_all_tx_bufs() argument
1537 for (int i = 0; i < adapter->num_io_queues; i++) in ena_free_all_tx_bufs()
1538 ena_free_tx_bufs(adapter, i); in ena_free_all_tx_bufs()
1542 ena_destroy_all_tx_queues(struct ena_adapter *adapter) in ena_destroy_all_tx_queues() argument
1547 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_tx_queues()
1549 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); in ena_destroy_all_tx_queues()
1554 ena_destroy_all_rx_queues(struct ena_adapter *adapter) in ena_destroy_all_rx_queues() argument
1559 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_rx_queues()
1561 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); in ena_destroy_all_rx_queues()
1566 ena_destroy_all_io_queues(struct ena_adapter *adapter) in ena_destroy_all_io_queues() argument
1571 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_io_queues()
1572 queue = &adapter->que[i]; in ena_destroy_all_io_queues()
1578 ena_destroy_all_tx_queues(adapter); in ena_destroy_all_io_queues()
1579 ena_destroy_all_rx_queues(adapter); in ena_destroy_all_io_queues()
1583 ena_create_io_queues(struct ena_adapter *adapter) in ena_create_io_queues() argument
1585 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_create_io_queues()
1595 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_io_queues()
1600 ctx.queue_size = adapter->requested_tx_ring_size; in ena_create_io_queues()
1603 ctx.numa_node = adapter->que[i].domain; in ena_create_io_queues()
1607 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1611 ring = &adapter->tx_ring[i]; in ena_create_io_queues()
1615 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1630 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_io_queues()
1635 ctx.queue_size = adapter->requested_rx_ring_size; in ena_create_io_queues()
1638 ctx.numa_node = adapter->que[i].domain; in ena_create_io_queues()
1642 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1647 ring = &adapter->rx_ring[i]; in ena_create_io_queues()
1651 ena_log(adapter->pdev, ERR, in ena_create_io_queues()
1665 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_io_queues()
1666 queue = &adapter->que[i]; in ena_create_io_queues()
1677 device_get_nameunit(adapter->pdev), i); in ena_create_io_queues()
1685 i = adapter->num_io_queues; in ena_create_io_queues()
1706 struct ena_adapter *adapter = (struct ena_adapter *)arg; in ena_intr_msix_mgmnt() local
1708 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); in ena_intr_msix_mgmnt()
1709 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))) in ena_intr_msix_mgmnt()
1710 ena_com_aenq_intr_handler(adapter->ena_dev, arg); in ena_intr_msix_mgmnt()
1721 struct ena_adapter *adapter = queue->adapter; in ena_handle_msix() local
1722 if_t ifp = adapter->ifp; in ena_handle_msix()
1733 ena_enable_msix(struct ena_adapter *adapter) in ena_enable_msix() argument
1735 device_t dev = adapter->pdev; in ena_enable_msix()
1739 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { in ena_enable_msix()
1745 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues); in ena_enable_msix()
1747 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry), in ena_enable_msix()
1753 adapter->msix_entries[i].entry = i; in ena_enable_msix()
1755 adapter->msix_entries[i].vector = i + 1; in ena_enable_msix()
1783 adapter->msix_vecs = msix_vecs; in ena_enable_msix()
1784 ENA_FLAG_SET_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); in ena_enable_msix()
1789 free(adapter->msix_entries, M_DEVBUF); in ena_enable_msix()
1790 adapter->msix_entries = NULL; in ena_enable_msix()
1796 ena_setup_mgmnt_intr(struct ena_adapter *adapter) in ena_setup_mgmnt_intr() argument
1798 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, ENA_IRQNAME_SIZE, in ena_setup_mgmnt_intr()
1799 "ena-mgmnt@pci:%s", device_get_nameunit(adapter->pdev)); in ena_setup_mgmnt_intr()
1804 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL; in ena_setup_mgmnt_intr()
1805 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter; in ena_setup_mgmnt_intr()
1806 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector = in ena_setup_mgmnt_intr()
1807 adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector; in ena_setup_mgmnt_intr()
1811 ena_setup_io_intr(struct ena_adapter *adapter) in ena_setup_io_intr() argument
1821 if (adapter->msix_entries == NULL) in ena_setup_io_intr()
1825 if (adapter->first_bind < 0) { in ena_setup_io_intr()
1826 adapter->first_bind = last_bind; in ena_setup_io_intr()
1827 last_bind = (last_bind + adapter->num_io_queues) % num_buckets; in ena_setup_io_intr()
1829 cur_bind = adapter->first_bind; in ena_setup_io_intr()
1832 for (int i = 0; i < adapter->num_io_queues; i++) { in ena_setup_io_intr()
1835 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, in ena_setup_io_intr()
1836 "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i); in ena_setup_io_intr()
1837 adapter->irq_tbl[irq_idx].handler = ena_handle_msix; in ena_setup_io_intr()
1838 adapter->irq_tbl[irq_idx].data = &adapter->que[i]; in ena_setup_io_intr()
1839 adapter->irq_tbl[irq_idx].vector = in ena_setup_io_intr()
1840 adapter->msix_entries[irq_idx].vector; in ena_setup_io_intr()
1841 ena_log(adapter->pdev, DBG, "ena_setup_io_intr vector: %d\n", in ena_setup_io_intr()
1842 adapter->msix_entries[irq_idx].vector); in ena_setup_io_intr()
1844 if (adapter->irq_cpu_base > ENA_BASE_CPU_UNSPECIFIED) { in ena_setup_io_intr()
1845 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = in ena_setup_io_intr()
1846 (unsigned)(adapter->irq_cpu_base + in ena_setup_io_intr()
1847 i * adapter->irq_cpu_stride) % (unsigned)mp_ncpus; in ena_setup_io_intr()
1848 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask); in ena_setup_io_intr()
1852 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = in ena_setup_io_intr()
1855 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask); in ena_setup_io_intr()
1858 if (CPU_ISSET(adapter->que[i].cpu, &cpuset_domain[idx])) in ena_setup_io_intr()
1861 adapter->que[i].domain = idx; in ena_setup_io_intr()
1863 adapter->que[i].domain = -1; in ena_setup_io_intr()
1871 ena_request_mgmnt_irq(struct ena_adapter *adapter) in ena_request_mgmnt_irq() argument
1873 device_t pdev = adapter->pdev; in ena_request_mgmnt_irq()
1880 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; in ena_request_mgmnt_irq()
1881 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, in ena_request_mgmnt_irq()
1890 rc = bus_setup_intr(adapter->pdev, irq->res, in ena_request_mgmnt_irq()
1905 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, irq->vector, in ena_request_mgmnt_irq()
1917 ena_request_io_irq(struct ena_adapter *adapter) in ena_request_io_irq() argument
1919 device_t pdev = adapter->pdev; in ena_request_io_irq()
1924 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter))) { in ena_request_io_irq()
1932 for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { in ena_request_io_irq()
1933 irq = &adapter->irq_tbl[i]; in ena_request_io_irq()
1938 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, in ena_request_io_irq()
1947 rc = bus_setup_intr(adapter->pdev, irq->res, in ena_request_io_irq()
1958 if (adapter->rss_enabled || adapter->irq_cpu_base > ENA_BASE_CPU_UNSPECIFIED) { in ena_request_io_irq()
1959 rc = bus_bind_intr(adapter->pdev, irq->res, irq->cpu); in ena_request_io_irq()
1976 irq = &adapter->irq_tbl[i]; in ena_request_io_irq()
1982 rcc = bus_teardown_intr(adapter->pdev, irq->res, in ena_request_io_irq()
1996 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_request_io_irq()
2011 ena_free_mgmnt_irq(struct ena_adapter *adapter) in ena_free_mgmnt_irq() argument
2013 device_t pdev = adapter->pdev; in ena_free_mgmnt_irq()
2017 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; in ena_free_mgmnt_irq()
2020 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); in ena_free_mgmnt_irq()
2029 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_free_mgmnt_irq()
2040 ena_free_io_irq(struct ena_adapter *adapter) in ena_free_io_irq() argument
2042 device_t pdev = adapter->pdev; in ena_free_io_irq()
2046 for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { in ena_free_io_irq()
2047 irq = &adapter->irq_tbl[i]; in ena_free_io_irq()
2050 rc = bus_teardown_intr(adapter->pdev, irq->res, in ena_free_io_irq()
2063 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, in ena_free_io_irq()
2076 ena_free_irqs(struct ena_adapter *adapter) in ena_free_irqs() argument
2078 ena_free_io_irq(adapter); in ena_free_irqs()
2079 ena_free_mgmnt_irq(adapter); in ena_free_irqs()
2080 ena_disable_msix(adapter); in ena_free_irqs()
2084 ena_disable_msix(struct ena_adapter *adapter) in ena_disable_msix() argument
2086 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { in ena_disable_msix()
2087 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); in ena_disable_msix()
2088 pci_release_msi(adapter->pdev); in ena_disable_msix()
2091 adapter->msix_vecs = 0; in ena_disable_msix()
2092 free(adapter->msix_entries, M_DEVBUF); in ena_disable_msix()
2093 adapter->msix_entries = NULL; in ena_disable_msix()
2097 ena_unmask_all_io_irqs(struct ena_adapter *adapter) in ena_unmask_all_io_irqs() argument
2106 for (i = 0; i < adapter->num_io_queues; i++) { in ena_unmask_all_io_irqs()
2108 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; in ena_unmask_all_io_irqs()
2110 tx_ring = &adapter->tx_ring[i]; in ena_unmask_all_io_irqs()
2117 ena_up_complete(struct ena_adapter *adapter) in ena_up_complete() argument
2121 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { in ena_up_complete()
2122 rc = ena_rss_configure(adapter); in ena_up_complete()
2124 ena_log(adapter->pdev, ERR, in ena_up_complete()
2130 rc = ena_change_mtu(adapter->ifp, if_getmtu(adapter->ifp)); in ena_up_complete()
2134 ena_refill_all_rx_bufs(adapter); in ena_up_complete()
2135 ena_reset_counters((counter_u64_t *)&adapter->hw_stats, in ena_up_complete()
2136 sizeof(adapter->hw_stats)); in ena_up_complete()
2142 set_io_rings_size(struct ena_adapter *adapter, int new_tx_size, int new_rx_size) in set_io_rings_size() argument
2146 for (i = 0; i < adapter->num_io_queues; i++) { in set_io_rings_size()
2147 adapter->tx_ring[i].ring_size = new_tx_size; in set_io_rings_size()
2148 adapter->rx_ring[i].ring_size = new_rx_size; in set_io_rings_size()
2153 create_queues_with_size_backoff(struct ena_adapter *adapter) in create_queues_with_size_backoff() argument
2155 device_t pdev = adapter->pdev; in create_queues_with_size_backoff()
2164 set_io_rings_size(adapter, adapter->requested_tx_ring_size, in create_queues_with_size_backoff()
2165 adapter->requested_rx_ring_size); in create_queues_with_size_backoff()
2169 rc = ena_setup_all_tx_resources(adapter); in create_queues_with_size_backoff()
2176 rc = ena_setup_all_rx_resources(adapter); in create_queues_with_size_backoff()
2183 rc = ena_create_io_queues(adapter); in create_queues_with_size_backoff()
2192 ena_free_all_rx_resources(adapter); in create_queues_with_size_backoff()
2194 ena_free_all_tx_resources(adapter); in create_queues_with_size_backoff()
2206 cur_tx_ring_size = adapter->tx_ring[0].ring_size; in create_queues_with_size_backoff()
2207 cur_rx_ring_size = adapter->rx_ring[0].ring_size; in create_queues_with_size_backoff()
2238 set_io_rings_size(adapter, new_tx_ring_size, new_rx_ring_size); in create_queues_with_size_backoff()
2243 ena_up(struct ena_adapter *adapter) in ena_up() argument
2249 if (unlikely(device_is_attached(adapter->pdev) == 0)) { in ena_up()
2250 ena_log(adapter->pdev, ERR, "device is not attached!\n"); in ena_up()
2254 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in ena_up()
2257 ena_log(adapter->pdev, INFO, "device is going UP\n"); in ena_up()
2260 rc = ena_setup_io_intr(adapter); in ena_up()
2262 ena_log(adapter->pdev, ERR, "error setting up IO interrupt\n"); in ena_up()
2265 rc = ena_request_io_irq(adapter); in ena_up()
2267 ena_log(adapter->pdev, ERR, "err_req_irq\n"); in ena_up()
2271 ena_log(adapter->pdev, INFO, in ena_up()
2273 adapter->num_io_queues, in ena_up()
2274 adapter->requested_rx_ring_size, in ena_up()
2275 adapter->requested_tx_ring_size, in ena_up()
2276 (adapter->ena_dev->tx_mem_queue_type == in ena_up()
2279 rc = create_queues_with_size_backoff(adapter); in ena_up()
2281 ena_log(adapter->pdev, ERR, in ena_up()
2286 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) in ena_up()
2287 if_link_state_change(adapter->ifp, LINK_STATE_UP); in ena_up()
2289 rc = ena_up_complete(adapter); in ena_up()
2293 counter_u64_add(adapter->dev_stats.interface_up, 1); in ena_up()
2295 ena_update_hwassist(adapter); in ena_up()
2297 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); in ena_up()
2299 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter); in ena_up()
2301 ena_unmask_all_io_irqs(adapter); in ena_up()
2306 ena_destroy_all_io_queues(adapter); in ena_up()
2307 ena_free_all_rx_resources(adapter); in ena_up()
2308 ena_free_all_tx_resources(adapter); in ena_up()
2310 ena_free_io_irq(adapter); in ena_up()
2318 struct ena_adapter *adapter; in ena_get_counter() local
2321 adapter = if_getsoftc(ifp); in ena_get_counter()
2322 stats = &adapter->hw_stats; in ena_get_counter()
2352 struct ena_adapter *adapter = if_getsoftc(ifp); in ena_media_status() local
2353 ena_log(adapter->pdev, DBG, "Media status update\n"); in ena_media_status()
2360 if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) { in ena_media_status()
2362 ena_log(adapter->pdev, INFO, "Link is down\n"); in ena_media_status()
2375 struct ena_adapter *adapter = (struct ena_adapter *)arg; in ena_init() local
2377 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { in ena_init()
2379 ena_up(adapter); in ena_init()
2387 struct ena_adapter *adapter; in ena_ioctl() local
2391 adapter = if_getsoftc(ifp); in ena_ioctl()
2403 ena_down(adapter); in ena_ioctl()
2407 rc = ena_up(adapter); in ena_ioctl()
2416 ena_log(adapter->pdev, INFO, in ena_ioctl()
2421 rc = ena_up(adapter); in ena_ioctl()
2427 ena_down(adapter); in ena_ioctl()
2439 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command); in ena_ioctl()
2454 ena_down(adapter); in ena_ioctl()
2455 rc = ena_up(adapter); in ena_ioctl()
2512 ena_update_hwassist(struct ena_adapter *adapter) in ena_update_hwassist() argument
2514 if_t ifp = adapter->ifp; in ena_update_hwassist()
2515 uint32_t feat = adapter->tx_offload_cap; in ena_update_hwassist()
2544 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter, in ena_setup_ifnet() argument
2550 ifp = adapter->ifp = if_gethandle(IFT_ETHER); in ena_setup_ifnet()
2553 if_setsoftc(ifp, adapter); in ena_setup_ifnet()
2562 if_setsendqlen(ifp, adapter->requested_tx_ring_size); in ena_setup_ifnet()
2577 if_sethwtsomaxsegcount(ifp, adapter->max_tx_sgl_size - 1); in ena_setup_ifnet()
2584 * Specify the media types supported by this adapter and register in ena_setup_ifnet()
2587 ifmedia_init(&adapter->media, IFM_IMASK, ena_media_change, in ena_setup_ifnet()
2589 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); in ena_setup_ifnet()
2590 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); in ena_setup_ifnet()
2592 ether_ifattach(ifp, adapter->mac_addr); in ena_setup_ifnet()
2596 ena_down(struct ena_adapter *adapter) in ena_down() argument
2602 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in ena_down()
2605 ena_log(adapter->pdev, INFO, "device is going DOWN\n"); in ena_down()
2607 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter); in ena_down()
2608 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); in ena_down()
2610 ena_free_io_irq(adapter); in ena_down()
2612 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) { in ena_down()
2613 rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); in ena_down()
2615 ena_log(adapter->pdev, ERR, "Device reset failed\n"); in ena_down()
2618 ena_destroy_all_io_queues(adapter); in ena_down()
2620 ena_free_all_tx_bufs(adapter); in ena_down()
2621 ena_free_all_rx_bufs(adapter); in ena_down()
2622 ena_free_all_tx_resources(adapter); in ena_down()
2623 ena_free_all_rx_resources(adapter); in ena_down()
2625 counter_u64_add(adapter->dev_stats.interface_down, 1); in ena_down()
2729 struct ena_adapter *adapter = device_get_softc(pdev); in ena_map_llq_mem_bar() local
2734 adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid, in ena_map_llq_mem_bar()
2736 if (unlikely(adapter->memory == NULL)) { in ena_map_llq_mem_bar()
2743 rc = ena_enable_wc(adapter->pdev, adapter->memory); in ena_map_llq_mem_bar()
2753 ena_dev->mem_bar = rman_get_virtual(adapter->memory); in ena_map_llq_mem_bar()
2760 struct ena_admin_feature_llq_desc *llq, struct ena_adapter *adapter) in ena_set_llq_configurations() argument
2783 ena_log(adapter->pdev, WARN, in ena_set_llq_configurations()
2794 adapter->llq_policy = ENA_ADMIN_LIST_ENTRY_SIZE_256B; in ena_set_llq_configurations()
2798 adapter->llq_policy = ENA_ADMIN_LIST_ENTRY_SIZE_128B; in ena_set_llq_configurations()
2803 ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx, struct ena_adapter *adapter) in ena_calc_io_queue_size() argument
2853 if (adapter->llq_policy == ENA_ADMIN_LIST_ENTRY_SIZE_256B) { in ena_calc_io_queue_size()
2948 ena_device_init(struct ena_adapter *adapter, device_t pdev, in ena_device_init() argument
2952 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_device_init()
2989 adapter->dma_width = dma_width; in ena_device_init()
3033 ena_set_llq_configurations(&llq_config, &get_feat_ctx->llq, adapter); in ena_device_init()
3054 ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter) in ena_enable_msix_and_set_admin_interrupts() argument
3056 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_enable_msix_and_set_admin_interrupts()
3059 rc = ena_enable_msix(adapter); in ena_enable_msix_and_set_admin_interrupts()
3061 ena_log(adapter->pdev, ERR, "Error with MSI-X enablement\n"); in ena_enable_msix_and_set_admin_interrupts()
3065 ena_setup_mgmnt_intr(adapter); in ena_enable_msix_and_set_admin_interrupts()
3067 rc = ena_request_mgmnt_irq(adapter); in ena_enable_msix_and_set_admin_interrupts()
3069 ena_log(adapter->pdev, ERR, "Cannot setup mgmnt queue intr\n"); in ena_enable_msix_and_set_admin_interrupts()
3080 ena_disable_msix(adapter); in ena_enable_msix_and_set_admin_interrupts()
3089 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_keep_alive_wd() local
3099 counter_u64_zero(adapter->hw_stats.rx_drops); in ena_keep_alive_wd()
3100 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops); in ena_keep_alive_wd()
3101 counter_u64_zero(adapter->hw_stats.tx_drops); in ena_keep_alive_wd()
3102 counter_u64_add(adapter->hw_stats.tx_drops, tx_drops); in ena_keep_alive_wd()
3105 atomic_store_rel_64(&adapter->keep_alive_timestamp, stime); in ena_keep_alive_wd()
3110 check_for_missing_keep_alive(struct ena_adapter *adapter) in check_for_missing_keep_alive() argument
3115 if (adapter->wd_active == 0) in check_for_missing_keep_alive()
3118 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) in check_for_missing_keep_alive()
3121 timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp); in check_for_missing_keep_alive()
3123 if (unlikely(time > adapter->keep_alive_timeout)) { in check_for_missing_keep_alive()
3124 ena_log(adapter->pdev, ERR, "Keep alive watchdog timeout.\n"); in check_for_missing_keep_alive()
3125 if (ena_com_aenq_has_keep_alive(adapter->ena_dev)) in check_for_missing_keep_alive()
3128 ena_trigger_reset(adapter, reset_reason); in check_for_missing_keep_alive()
3134 check_for_admin_com_state(struct ena_adapter *adapter) in check_for_admin_com_state() argument
3137 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == false)) { in check_for_admin_com_state()
3138 ena_log(adapter->pdev, ERR, in check_for_admin_com_state()
3140 counter_u64_add(adapter->dev_stats.admin_q_pause, 1); in check_for_admin_com_state()
3141 if (ena_com_get_missing_admin_interrupt(adapter->ena_dev)) in check_for_admin_com_state()
3144 ena_trigger_reset(adapter, reset_reason); in check_for_admin_com_state()
3149 check_for_rx_interrupt_queue(struct ena_adapter *adapter, in check_for_rx_interrupt_queue() argument
3162 ena_log(adapter->pdev, ERR, in check_for_rx_interrupt_queue()
3165 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_INTERRUPT); in check_for_rx_interrupt_queue()
3173 check_cdesc_in_tx_cq(struct ena_adapter *adapter, in check_cdesc_in_tx_cq() argument
3176 device_t pdev = adapter->pdev; in check_cdesc_in_tx_cq()
3198 check_missing_comp_in_tx_queue(struct ena_adapter *adapter, in check_missing_comp_in_tx_queue() argument
3202 device_t pdev = adapter->pdev; in check_missing_comp_in_tx_queue()
3225 time_offset > 2 * adapter->missing_tx_timeout)) { in check_missing_comp_in_tx_queue()
3234 ena_trigger_reset(adapter, in check_missing_comp_in_tx_queue()
3240 if (unlikely(time_offset > adapter->missing_tx_timeout)) { in check_missing_comp_in_tx_queue()
3246 adapter->missing_tx_timeout); in check_missing_comp_in_tx_queue()
3261 if (unlikely(missed_tx > adapter->missing_tx_threshold)) { in check_missing_comp_in_tx_queue()
3265 missed_tx, adapter->missing_tx_threshold); in check_missing_comp_in_tx_queue()
3267 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); in check_missing_comp_in_tx_queue()
3276 reset_reason = check_cdesc_in_tx_cq(adapter, tx_ring); in check_missing_comp_in_tx_queue()
3278 adapter->reset_reason = reset_reason; in check_missing_comp_in_tx_queue()
3294 check_for_missing_completions(struct ena_adapter *adapter) in check_for_missing_completions() argument
3303 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in check_for_missing_completions()
3306 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) in check_for_missing_completions()
3309 if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT) in check_for_missing_completions()
3312 budget = adapter->missing_tx_max_queues; in check_for_missing_completions()
3314 for (i = adapter->next_monitored_tx_qid; i < adapter->num_io_queues; i++) { in check_for_missing_completions()
3315 tx_ring = &adapter->tx_ring[i]; in check_for_missing_completions()
3316 rx_ring = &adapter->rx_ring[i]; in check_for_missing_completions()
3318 rc = check_missing_comp_in_tx_queue(adapter, tx_ring); in check_for_missing_completions()
3322 rc = check_for_rx_interrupt_queue(adapter, rx_ring); in check_for_missing_completions()
3333 adapter->next_monitored_tx_qid = i % adapter->num_io_queues; in check_for_missing_completions()
3349 check_for_empty_rx_ring(struct ena_adapter *adapter) in check_for_empty_rx_ring() argument
3354 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in check_for_empty_rx_ring()
3357 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) in check_for_empty_rx_ring()
3360 for (i = 0; i < adapter->num_io_queues; i++) { in check_for_empty_rx_ring()
3361 rx_ring = &adapter->rx_ring[i]; in check_for_empty_rx_ring()
3372 ena_log(adapter->pdev, WARN, in check_for_empty_rx_ring()
3387 ena_update_hints(struct ena_adapter *adapter, in ena_update_hints() argument
3390 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_update_hints()
3401 adapter->missing_tx_threshold = in ena_update_hints()
3407 adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT; in ena_update_hints()
3409 adapter->missing_tx_timeout = SBT_1MS * in ena_update_hints()
3415 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; in ena_update_hints()
3417 adapter->keep_alive_timeout = SBT_1MS * in ena_update_hints()
3424 * @adapter: ENA device adapter
3434 ena_copy_eni_metrics(struct ena_adapter *adapter) in ena_copy_eni_metrics() argument
3439 rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_metrics); in ena_copy_eni_metrics()
3444 ena_log(adapter->pdev, WARN, in ena_copy_eni_metrics()
3448 ena_log(adapter->pdev, DBG, in ena_copy_eni_metrics()
3452 ena_log(adapter->pdev, ERR, in ena_copy_eni_metrics()
3461 ena_copy_srd_metrics(struct ena_adapter *adapter) in ena_copy_srd_metrics() argument
3463 return ena_com_get_ena_srd_info(adapter->ena_dev, &adapter->ena_srd_info); in ena_copy_srd_metrics()
3467 ena_copy_customer_metrics(struct ena_adapter *adapter) in ena_copy_customer_metrics() argument
3473 dev = adapter->ena_dev; in ena_copy_customer_metrics()
3479 rc = ena_com_get_customer_metrics(adapter->ena_dev, in ena_copy_customer_metrics()
3480 (char *)(adapter->customer_metrics_array), len); in ena_copy_customer_metrics()
3488 struct ena_adapter *adapter = (struct ena_adapter *)data; in ena_timer_service() local
3490 adapter->ena_dev->host_attr.host_info; in ena_timer_service()
3492 check_for_missing_keep_alive(adapter); in ena_timer_service()
3494 check_for_admin_com_state(adapter); in ena_timer_service()
3496 check_for_missing_completions(adapter); in ena_timer_service()
3498 check_for_empty_rx_ring(adapter); in ena_timer_service()
3509 if ((adapter->metrics_sample_interval != 0) && in ena_timer_service()
3510 (++adapter->metrics_sample_interval_cnt >= in ena_timer_service()
3511 adapter->metrics_sample_interval)) { in ena_timer_service()
3512 taskqueue_enqueue(adapter->metrics_tq, &adapter->metrics_task); in ena_timer_service()
3513 adapter->metrics_sample_interval_cnt = 0; in ena_timer_service()
3518 ena_update_host_info(host_info, adapter->ifp); in ena_timer_service()
3520 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { in ena_timer_service()
3526 if (ena_com_validate_version(adapter->ena_dev) == in ena_timer_service()
3528 ena_log(adapter->pdev, WARN, in ena_timer_service()
3530 ENA_TIMER_RESET(adapter); in ena_timer_service()
3533 ena_log(adapter->pdev, WARN, "Trigger reset is on\n"); in ena_timer_service()
3534 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task); in ena_timer_service()
3541 ENA_TIMER_RESET(adapter); in ena_timer_service()
3545 ena_destroy_device(struct ena_adapter *adapter, bool graceful) in ena_destroy_device() argument
3547 if_t ifp = adapter->ifp; in ena_destroy_device()
3548 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_destroy_device()
3551 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) in ena_destroy_device()
3557 ENA_TIMER_DRAIN(adapter); in ena_destroy_device()
3559 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); in ena_destroy_device()
3561 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_destroy_device()
3566 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) in ena_destroy_device()
3567 ena_down(adapter); in ena_destroy_device()
3573 if (!(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter) && dev_up)) in ena_destroy_device()
3574 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); in ena_destroy_device()
3576 ena_free_mgmnt_irq(adapter); in ena_destroy_device()
3578 ena_disable_msix(adapter); in ena_destroy_device()
3587 ena_free_all_io_rings_resources(adapter); in ena_destroy_device()
3597 adapter->reset_reason = ENA_REGS_RESET_NORMAL; in ena_destroy_device()
3599 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); in ena_destroy_device()
3600 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); in ena_destroy_device()
3604 ena_device_validate_params(struct ena_adapter *adapter, in ena_device_validate_params() argument
3607 if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr, in ena_device_validate_params()
3609 ena_log(adapter->pdev, ERR, "Error, mac addresses differ\n"); in ena_device_validate_params()
3613 if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) { in ena_device_validate_params()
3614 ena_log(adapter->pdev, ERR, in ena_device_validate_params()
3623 ena_restore_device(struct ena_adapter *adapter) in ena_restore_device() argument
3626 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_restore_device()
3627 if_t ifp = adapter->ifp; in ena_restore_device()
3628 device_t dev = adapter->pdev; in ena_restore_device()
3632 ENA_FLAG_SET_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); in ena_restore_device()
3634 rc = ena_device_init(adapter, dev, &get_feat_ctx, &wd_active); in ena_restore_device()
3643 if (adapter->wd_active != 0) in ena_restore_device()
3644 adapter->wd_active = wd_active; in ena_restore_device()
3646 rc = ena_device_validate_params(adapter, &get_feat_ctx); in ena_restore_device()
3652 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); in ena_restore_device()
3654 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) in ena_restore_device()
3657 rc = ena_enable_msix_and_set_admin_interrupts(adapter); in ena_restore_device()
3668 if ((adapter->msix_vecs - ENA_ADMIN_MSIX_VEC) < adapter->num_io_queues) in ena_restore_device()
3669 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; in ena_restore_device()
3672 ena_init_io_rings(adapter); in ena_restore_device()
3675 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) { in ena_restore_device()
3676 rc = ena_up(adapter); in ena_restore_device()
3684 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); in ena_restore_device()
3692 adapter->keep_alive_timestamp = getsbinuptime(); in ena_restore_device()
3693 ENA_TIMER_RESET(adapter); in ena_restore_device()
3695 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); in ena_restore_device()
3700 ena_free_mgmnt_irq(adapter); in ena_restore_device()
3701 ena_disable_msix(adapter); in ena_restore_device()
3709 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); in ena_restore_device()
3710 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); in ena_restore_device()
3719 struct ena_adapter *adapter = (struct ena_adapter *)arg; in ena_metrics_task() local
3723 if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_CUSTOMER_METRICS)) in ena_metrics_task()
3724 (void)ena_copy_customer_metrics(adapter); in ena_metrics_task()
3725 else if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENI_STATS)) in ena_metrics_task()
3726 (void)ena_copy_eni_metrics(adapter); in ena_metrics_task()
3728 if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENA_SRD_INFO)) in ena_metrics_task()
3729 (void)ena_copy_srd_metrics(adapter); in ena_metrics_task()
3737 struct ena_adapter *adapter = (struct ena_adapter *)arg; in ena_reset_task() local
3740 if (likely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { in ena_reset_task()
3741 ena_increment_reset_counter(adapter); in ena_reset_task()
3742 ena_destroy_device(adapter, false); in ena_reset_task()
3743 ena_restore_device(adapter); in ena_reset_task()
3745 ena_log(adapter->pdev, INFO, in ena_reset_task()
3753 ena_free_stats(struct ena_adapter *adapter) in ena_free_stats() argument
3755 ena_free_counters((counter_u64_t *)&adapter->hw_stats, in ena_free_stats()
3757 ena_free_counters((counter_u64_t *)&adapter->dev_stats, in ena_free_stats()
3767 * ena_attach initializes an adapter identified by a device structure.
3768 * The OS initialization, configuring of the adapter private structure,
3777 struct ena_adapter *adapter; in ena_attach() local
3783 adapter = device_get_softc(pdev); in ena_attach()
3784 adapter->pdev = pdev; in ena_attach()
3785 adapter->first_bind = -1; in ena_attach()
3791 ENA_TIMER_INIT(adapter); in ena_attach()
3792 adapter->keep_alive_timeout = ENA_DEFAULT_KEEP_ALIVE_TO; in ena_attach()
3793 adapter->missing_tx_timeout = ENA_DEFAULT_TX_CMP_TO; in ena_attach()
3794 adapter->missing_tx_max_queues = ENA_DEFAULT_TX_MONITORED_QUEUES; in ena_attach()
3795 adapter->missing_tx_threshold = ENA_DEFAULT_TX_CMP_THRESHOLD; in ena_attach()
3797 adapter->irq_cpu_base = ENA_BASE_CPU_UNSPECIFIED; in ena_attach()
3798 adapter->irq_cpu_stride = 0; in ena_attach()
3801 adapter->rss_enabled = 1; in ena_attach()
3811 adapter->ena_dev = ena_dev; in ena_attach()
3815 adapter->memory = NULL; in ena_attach()
3816 adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid, in ena_attach()
3818 if (unlikely(adapter->registers == NULL)) { in ena_attach()
3828 adapter->msix = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, in ena_attach()
3830 if (unlikely(adapter->msix == NULL)) { in ena_attach()
3836 adapter->msix_rid = msix_rid; in ena_attach()
3844 adapter->registers); in ena_attach()
3846 adapter->registers); in ena_attach()
3863 ENA_FLAG_ZERO(adapter); in ena_attach()
3866 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active); in ena_attach()
3874 adapter->disable_meta_caching = !!( in ena_attach()
3878 adapter->keep_alive_timestamp = getsbinuptime(); in ena_attach()
3880 adapter->tx_offload_cap = get_feat_ctx.offload.tx; in ena_attach()
3882 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr, in ena_attach()
3892 rc = ena_calc_io_queue_size(&calc_queue_ctx, adapter); in ena_attach()
3898 adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size; in ena_attach()
3899 adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size; in ena_attach()
3900 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size; in ena_attach()
3901 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size; in ena_attach()
3902 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; in ena_attach()
3903 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; in ena_attach()
3905 adapter->max_num_io_queues = max_num_io_queues; in ena_attach()
3907 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE; in ena_attach()
3909 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; in ena_attach()
3911 adapter->reset_reason = ENA_REGS_RESET_NORMAL; in ena_attach()
3914 rc = ena_setup_tx_dma_tag(adapter); in ena_attach()
3920 rc = ena_setup_rx_dma_tag(adapter); in ena_attach()
3928 * adapter::max_num_io_queues (see `ena_enable_msix()`), plus a constant in ena_attach()
3935 rc = ena_enable_msix_and_set_admin_interrupts(adapter); in ena_attach()
3942 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; in ena_attach()
3945 ena_init_io_rings(adapter); in ena_attach()
3953 rc = ena_sysctl_allocate_customer_metrics_buffer(adapter); in ena_attach()
3960 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats, in ena_attach()
3962 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats, in ena_attach()
3964 ena_sysctl_add_nodes(adapter); in ena_attach()
3967 ena_setup_ifnet(pdev, adapter, &get_feat_ctx); in ena_attach()
3970 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter); in ena_attach()
3971 adapter->reset_tq = taskqueue_create("ena_reset_enqueue", in ena_attach()
3972 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq); in ena_attach()
3973 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, "%s rstq", in ena_attach()
3974 device_get_nameunit(adapter->pdev)); in ena_attach()
3977 TASK_INIT(&adapter->metrics_task, 0, ena_metrics_task, adapter); in ena_attach()
3978 adapter->metrics_tq = taskqueue_create("ena_metrics_enqueue", in ena_attach()
3979 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->metrics_tq); in ena_attach()
3980 taskqueue_start_threads(&adapter->metrics_tq, 1, PI_NET, "%s metricsq", in ena_attach()
3981 device_get_nameunit(adapter->pdev)); in ena_attach()
3984 rc = ena_netmap_attach(adapter); in ena_attach()
3992 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); in ena_attach()
3993 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); in ena_attach()
3996 ENA_TIMER_RESET(adapter); in ena_attach()
4002 ether_ifdetach(adapter->ifp); in ena_attach()
4003 ifmedia_removeall(&adapter->media); in ena_attach()
4004 free(adapter->customer_metrics_array, M_DEVBUF); in ena_attach()
4009 ena_free_stats(adapter); in ena_attach()
4010 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR); in ena_attach()
4011 ena_free_mgmnt_irq(adapter); in ena_attach()
4012 ena_disable_msix(adapter); in ena_attach()
4014 ena_free_all_io_rings_resources(adapter); in ena_attach()
4015 ena_free_rx_dma_tag(adapter); in ena_attach()
4017 ena_free_tx_dma_tag(adapter); in ena_attach()
4025 ena_free_pci_resources(adapter); in ena_attach()
4042 struct ena_adapter *adapter = device_get_softc(pdev); in ena_detach() local
4043 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_detach()
4047 if (if_vlantrunkinuse(adapter->ifp)) { in ena_detach()
4048 ena_log(adapter->pdev, ERR, "VLAN is in use, detach first\n"); in ena_detach()
4056 ether_ifdetach(adapter->ifp); in ena_detach()
4058 ifmedia_removeall(&adapter->media); in ena_detach()
4062 ENA_TIMER_DRAIN(adapter); in ena_detach()
4066 while (taskqueue_cancel(adapter->metrics_tq, &adapter->metrics_task, NULL)) in ena_detach()
4067 taskqueue_drain(adapter->metrics_tq, &adapter->metrics_task); in ena_detach()
4068 taskqueue_free(adapter->metrics_tq); in ena_detach()
4071 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL)) in ena_detach()
4072 taskqueue_drain(adapter->reset_tq, &adapter->reset_task); in ena_detach()
4073 taskqueue_free(adapter->reset_tq); in ena_detach()
4076 ena_down(adapter); in ena_detach()
4077 ena_destroy_device(adapter, true); in ena_detach()
4081 ena_sysctl_update_queue_node_nb(adapter, adapter->num_io_queues, in ena_detach()
4082 adapter->max_num_io_queues); in ena_detach()
4085 netmap_detach(adapter->ifp); in ena_detach()
4088 ena_free_stats(adapter); in ena_detach()
4090 rc = ena_free_rx_dma_tag(adapter); in ena_detach()
4092 ena_log(adapter->pdev, WARN, in ena_detach()
4095 rc = ena_free_tx_dma_tag(adapter); in ena_detach()
4097 ena_log(adapter->pdev, WARN, in ena_detach()
4100 ena_free_irqs(adapter); in ena_detach()
4102 ena_free_pci_resources(adapter); in ena_detach()
4104 if (adapter->rss_indir != NULL) in ena_detach()
4105 free(adapter->rss_indir, M_DEVBUF); in ena_detach()
4107 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) in ena_detach()
4112 free(adapter->customer_metrics_array, M_DEVBUF); in ena_detach()
4116 if_free(adapter->ifp); in ena_detach()
4136 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_update_on_link_change() local
4142 ifp = adapter->ifp; in ena_update_on_link_change()
4147 ena_log(adapter->pdev, INFO, "link is UP\n"); in ena_update_on_link_change()
4148 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter); in ena_update_on_link_change()
4149 if (!ENA_FLAG_ISSET(ENA_FLAG_ONGOING_RESET, adapter)) in ena_update_on_link_change()
4152 ena_log(adapter->pdev, INFO, "link is DOWN\n"); in ena_update_on_link_change()
4154 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter); in ena_update_on_link_change()
4161 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_notification() local
4165 adapter->ena_dev, "Invalid group(%x) expected %x\n", in ena_notification()
4172 ena_update_hints(adapter, hints); in ena_notification()
4175 ena_log(adapter->pdev, ERR, in ena_notification()
4202 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in unimplemented_aenq_handler() local
4204 ena_log(adapter->pdev, ERR, in unimplemented_aenq_handler()
4211 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_conf_notification() local
4219 ena_log(adapter->pdev, INFO, in ena_conf_notification()
4226 ena_log(adapter->pdev, INFO, in ena_conf_notification()
4237 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_admin_device_request_reset() local
4238 ena_log(adapter->pdev, WARN, in ena_admin_device_request_reset()
4240 ena_trigger_reset(adapter, ENA_REGS_RESET_DEVICE_REQUEST); in ena_admin_device_request_reset()