Lines Matching full:adapter
40 static int ena_rss_init_default(struct ena_adapter *adapter);
41 static void check_for_admin_com_state(struct ena_adapter *adapter);
42 static int ena_destroy_device(struct ena_adapter *adapter, bool graceful);
43 static int ena_restore_device(struct ena_adapter *adapter);
48 struct ena_adapter *adapter = netdev_priv(dev); in ena_tx_timeout() local
53 if (txqueue >= adapter->num_io_queues) { in ena_tx_timeout()
59 tx_ring = &adapter->tx_ring[txqueue]; in ena_tx_timeout()
80 if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) in ena_tx_timeout()
83 ena_reset_device(adapter, reset_reason); in ena_tx_timeout()
84 ena_increase_stat(&adapter->dev_stats.tx_timeout, 1, &adapter->syncp); in ena_tx_timeout()
87 static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu) in update_rx_ring_mtu() argument
91 for (i = 0; i < adapter->num_io_queues; i++) in update_rx_ring_mtu()
92 adapter->rx_ring[i].mtu = mtu; in update_rx_ring_mtu()
97 struct ena_adapter *adapter = netdev_priv(dev); in ena_change_mtu() local
100 ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); in ena_change_mtu()
102 netif_dbg(adapter, drv, dev, "Set MTU to %d\n", new_mtu); in ena_change_mtu()
103 update_rx_ring_mtu(adapter, new_mtu); in ena_change_mtu()
106 netif_err(adapter, drv, dev, "Failed to set MTU to %d\n", in ena_change_mtu()
113 int ena_xmit_common(struct ena_adapter *adapter, in ena_xmit_common() argument
124 netif_dbg(adapter, tx_queued, adapter->netdev, in ena_xmit_common()
139 netif_err(adapter, tx_queued, adapter->netdev, in ena_xmit_common()
143 ena_reset_device(adapter, ENA_REGS_RESET_DRIVER_INVALID_STATE); in ena_xmit_common()
162 static void ena_init_io_rings_common(struct ena_adapter *adapter, in ena_init_io_rings_common() argument
166 ring->pdev = adapter->pdev; in ena_init_io_rings_common()
167 ring->dev = &adapter->pdev->dev; in ena_init_io_rings_common()
168 ring->netdev = adapter->netdev; in ena_init_io_rings_common()
169 ring->napi = &adapter->ena_napi[qid].napi; in ena_init_io_rings_common()
170 ring->adapter = adapter; in ena_init_io_rings_common()
171 ring->ena_dev = adapter->ena_dev; in ena_init_io_rings_common()
179 void ena_init_io_rings(struct ena_adapter *adapter, in ena_init_io_rings() argument
186 ena_dev = adapter->ena_dev; in ena_init_io_rings()
189 txr = &adapter->tx_ring[i]; in ena_init_io_rings()
190 rxr = &adapter->rx_ring[i]; in ena_init_io_rings()
193 ena_init_io_rings_common(adapter, txr, i); in ena_init_io_rings()
196 txr->ring_size = adapter->requested_tx_ring_size; in ena_init_io_rings()
199 txr->sgl_size = adapter->max_tx_sgl_size; in ena_init_io_rings()
202 txr->disable_meta_caching = adapter->disable_meta_caching; in ena_init_io_rings()
206 if (!ENA_IS_XDP_INDEX(adapter, i)) { in ena_init_io_rings()
208 ena_init_io_rings_common(adapter, rxr, i); in ena_init_io_rings()
211 rxr->ring_size = adapter->requested_rx_ring_size; in ena_init_io_rings()
212 rxr->rx_copybreak = adapter->rx_copybreak; in ena_init_io_rings()
213 rxr->sgl_size = adapter->max_rx_sgl_size; in ena_init_io_rings()
218 adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; in ena_init_io_rings()
219 rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues]; in ena_init_io_rings()
225 * @adapter: network interface device structure
230 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid) in ena_setup_tx_resources() argument
232 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_setup_tx_resources()
233 struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)]; in ena_setup_tx_resources()
237 netif_err(adapter, ifup, in ena_setup_tx_resources()
238 adapter->netdev, "tx_buffer_info info is not NULL"); in ena_setup_tx_resources()
292 * @adapter: network interface device structure
297 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid) in ena_free_tx_resources() argument
299 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; in ena_free_tx_resources()
311 int ena_setup_tx_resources_in_range(struct ena_adapter *adapter, in ena_setup_tx_resources_in_range() argument
317 rc = ena_setup_tx_resources(adapter, i); in ena_setup_tx_resources_in_range()
326 netif_err(adapter, ifup, adapter->netdev, in ena_setup_tx_resources_in_range()
331 ena_free_tx_resources(adapter, i); in ena_setup_tx_resources_in_range()
335 void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter, in ena_free_all_io_tx_resources_in_range() argument
341 ena_free_tx_resources(adapter, i); in ena_free_all_io_tx_resources_in_range()
345 * @adapter: board private structure
349 void ena_free_all_io_tx_resources(struct ena_adapter *adapter) in ena_free_all_io_tx_resources() argument
351 ena_free_all_io_tx_resources_in_range(adapter, in ena_free_all_io_tx_resources()
353 adapter->xdp_num_queues + in ena_free_all_io_tx_resources()
354 adapter->num_io_queues); in ena_free_all_io_tx_resources()
358 * @adapter: network interface device structure
363 static int ena_setup_rx_resources(struct ena_adapter *adapter, in ena_setup_rx_resources() argument
366 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_setup_rx_resources()
367 struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)]; in ena_setup_rx_resources()
371 netif_err(adapter, ifup, adapter->netdev, in ena_setup_rx_resources()
416 * @adapter: network interface device structure
421 static void ena_free_rx_resources(struct ena_adapter *adapter, in ena_free_rx_resources() argument
424 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_resources()
434 * @adapter: board private structure
438 static int ena_setup_all_rx_resources(struct ena_adapter *adapter) in ena_setup_all_rx_resources() argument
442 for (i = 0; i < adapter->num_io_queues; i++) { in ena_setup_all_rx_resources()
443 rc = ena_setup_rx_resources(adapter, i); in ena_setup_all_rx_resources()
452 netif_err(adapter, ifup, adapter->netdev, in ena_setup_all_rx_resources()
457 ena_free_rx_resources(adapter, i); in ena_setup_all_rx_resources()
462 * @adapter: board private structure
466 static void ena_free_all_io_rx_resources(struct ena_adapter *adapter) in ena_free_all_io_rx_resources() argument
470 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_io_rx_resources()
471 ena_free_rx_resources(adapter, i); in ena_free_all_io_rx_resources()
524 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_alloc_rx_buffer()
553 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_free_rx_page()
581 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_refill_rx_bufs()
590 netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_refill_rx_bufs()
602 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_refill_rx_bufs()
616 static void ena_free_rx_bufs(struct ena_adapter *adapter, in ena_free_rx_bufs() argument
619 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_bufs()
631 * @adapter: board private structure
633 static void ena_refill_all_rx_bufs(struct ena_adapter *adapter) in ena_refill_all_rx_bufs() argument
638 for (i = 0; i < adapter->num_io_queues; i++) { in ena_refill_all_rx_bufs()
639 rx_ring = &adapter->rx_ring[i]; in ena_refill_all_rx_bufs()
644 netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_refill_all_rx_bufs()
650 static void ena_free_all_rx_bufs(struct ena_adapter *adapter) in ena_free_all_rx_bufs() argument
654 for (i = 0; i < adapter->num_io_queues; i++) in ena_free_all_rx_bufs()
655 ena_free_rx_bufs(adapter, i); in ena_free_all_rx_bufs()
697 is_xdp_ring = ENA_IS_XDP_INDEX(tx_ring->adapter, tx_ring->qid); in ena_free_tx_bufs()
706 netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev, in ena_free_tx_bufs()
711 netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev, in ena_free_tx_bufs()
729 static void ena_free_all_tx_bufs(struct ena_adapter *adapter) in ena_free_all_tx_bufs() argument
734 for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { in ena_free_all_tx_bufs()
735 tx_ring = &adapter->tx_ring[i]; in ena_free_all_tx_bufs()
740 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter) in ena_destroy_all_tx_queues() argument
745 for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { in ena_destroy_all_tx_queues()
747 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); in ena_destroy_all_tx_queues()
751 static void ena_destroy_all_rx_queues(struct ena_adapter *adapter) in ena_destroy_all_rx_queues() argument
756 for (i = 0; i < adapter->num_io_queues; i++) { in ena_destroy_all_rx_queues()
758 cancel_work_sync(&adapter->ena_napi[i].dim.work); in ena_destroy_all_rx_queues()
759 ena_xdp_unregister_rxq_info(&adapter->rx_ring[i]); in ena_destroy_all_rx_queues()
760 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); in ena_destroy_all_rx_queues()
764 static void ena_destroy_all_io_queues(struct ena_adapter *adapter) in ena_destroy_all_io_queues() argument
766 ena_destroy_all_tx_queues(adapter); in ena_destroy_all_io_queues()
767 ena_destroy_all_rx_queues(adapter); in ena_destroy_all_io_queues()
774 netif_err(ring->adapter, in handle_invalid_req_id()
780 netif_err(ring->adapter, in handle_invalid_req_id()
787 ena_reset_device(ring->adapter, ENA_REGS_RESET_INV_TX_REQ_ID); in handle_invalid_req_id()
845 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev, in ena_clean_tx_irq()
864 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev, in ena_clean_tx_irq()
881 test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) { in ena_clean_tx_irq()
905 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_alloc_skb()
940 struct ena_adapter *adapter; in ena_rx_skb() local
956 adapter = rx_ring->adapter; in ena_rx_skb()
957 netif_err(adapter, rx_err, rx_ring->netdev, in ena_rx_skb()
960 ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID); in ena_rx_skb()
964 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_rx_skb()
985 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_rx_skb()
1013 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_rx_skb()
1061 * @adapter: structure containing adapter specific data
1088 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_rx_checksum()
1100 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_rx_checksum()
1151 netdev_err_once(rx_ring->adapter->netdev, in ena_xdp_handle_buff()
1186 struct ena_adapter *adapter; in ena_clean_rx_irq() local
1200 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_clean_rx_irq()
1226 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_clean_rx_irq()
1318 adapter = netdev_priv(rx_ring->netdev); in ena_clean_rx_irq()
1322 ena_reset_device(adapter, ENA_REGS_RESET_TOO_MANY_RX_DESCS); in ena_clean_rx_irq()
1324 ena_reset_device(adapter, ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED); in ena_clean_rx_irq()
1328 ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID); in ena_clean_rx_irq()
1448 if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) || in ena_io_poll()
1449 test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) { in ena_io_poll()
1464 if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) || in ena_io_poll()
1465 test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) { in ena_io_poll()
1506 struct ena_adapter *adapter = (struct ena_adapter *)data; in ena_intr_msix_mgmnt() local
1508 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); in ena_intr_msix_mgmnt()
1511 if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))) in ena_intr_msix_mgmnt()
1512 ena_com_aenq_intr_handler(adapter->ena_dev, data); in ena_intr_msix_mgmnt()
1541 static int ena_enable_msix(struct ena_adapter *adapter) in ena_enable_msix() argument
1545 if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) { in ena_enable_msix()
1546 netif_err(adapter, probe, adapter->netdev, in ena_enable_msix()
1552 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues); in ena_enable_msix()
1553 netif_dbg(adapter, probe, adapter->netdev, in ena_enable_msix()
1556 irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC, in ena_enable_msix()
1560 netif_err(adapter, probe, adapter->netdev, in ena_enable_msix()
1566 netif_notice(adapter, probe, adapter->netdev, in ena_enable_msix()
1569 adapter->num_io_queues = irq_cnt - ENA_ADMIN_MSIX_VEC; in ena_enable_msix()
1572 if (netif_enable_cpu_rmap(adapter->netdev, adapter->num_io_queues)) in ena_enable_msix()
1573 netif_warn(adapter, probe, adapter->netdev, in ena_enable_msix()
1576 adapter->msix_vecs = irq_cnt; in ena_enable_msix()
1577 set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags); in ena_enable_msix()
1582 static void ena_setup_mgmnt_intr(struct ena_adapter *adapter) in ena_setup_mgmnt_intr() argument
1586 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, in ena_setup_mgmnt_intr()
1588 pci_name(adapter->pdev)); in ena_setup_mgmnt_intr()
1589 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = in ena_setup_mgmnt_intr()
1591 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter; in ena_setup_mgmnt_intr()
1592 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector = in ena_setup_mgmnt_intr()
1593 pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX); in ena_setup_mgmnt_intr()
1595 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu; in ena_setup_mgmnt_intr()
1597 &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask); in ena_setup_mgmnt_intr()
1600 static void ena_setup_io_intr(struct ena_adapter *adapter) in ena_setup_io_intr() argument
1606 netdev = adapter->netdev; in ena_setup_io_intr()
1607 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; in ena_setup_io_intr()
1613 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, in ena_setup_io_intr()
1615 adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io; in ena_setup_io_intr()
1616 adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i]; in ena_setup_io_intr()
1617 adapter->irq_tbl[irq_idx].vector = in ena_setup_io_intr()
1618 pci_irq_vector(adapter->pdev, irq_idx); in ena_setup_io_intr()
1619 adapter->irq_tbl[irq_idx].cpu = cpu; in ena_setup_io_intr()
1622 &adapter->irq_tbl[irq_idx].affinity_hint_mask); in ena_setup_io_intr()
1626 static int ena_request_mgmnt_irq(struct ena_adapter *adapter) in ena_request_mgmnt_irq() argument
1632 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; in ena_request_mgmnt_irq()
1636 netif_err(adapter, probe, adapter->netdev, in ena_request_mgmnt_irq()
1641 netif_dbg(adapter, probe, adapter->netdev, in ena_request_mgmnt_irq()
1650 static int ena_request_io_irq(struct ena_adapter *adapter) in ena_request_io_irq() argument
1652 u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; in ena_request_io_irq()
1657 if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) { in ena_request_io_irq()
1658 netif_err(adapter, ifup, adapter->netdev, in ena_request_io_irq()
1664 irq = &adapter->irq_tbl[i]; in ena_request_io_irq()
1668 netif_err(adapter, ifup, adapter->netdev, in ena_request_io_irq()
1674 netif_dbg(adapter, ifup, adapter->netdev, in ena_request_io_irq()
1687 irq = &adapter->irq_tbl[irq_idx]; in ena_request_io_irq()
1688 netif_napi_set_irq(&adapter->ena_napi[i].napi, irq->vector); in ena_request_io_irq()
1695 irq = &adapter->irq_tbl[k]; in ena_request_io_irq()
1702 static void ena_free_mgmnt_irq(struct ena_adapter *adapter) in ena_free_mgmnt_irq() argument
1706 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; in ena_free_mgmnt_irq()
1712 static void ena_free_io_irq(struct ena_adapter *adapter) in ena_free_io_irq() argument
1714 u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; in ena_free_io_irq()
1721 irq = &adapter->irq_tbl[i]; in ena_free_io_irq()
1729 static void ena_disable_msix(struct ena_adapter *adapter) in ena_disable_msix() argument
1731 if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) in ena_disable_msix()
1732 pci_free_irq_vectors(adapter->pdev); in ena_disable_msix()
1735 static void ena_disable_io_intr_sync(struct ena_adapter *adapter) in ena_disable_io_intr_sync() argument
1737 u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; in ena_disable_io_intr_sync()
1740 if (!netif_running(adapter->netdev)) in ena_disable_io_intr_sync()
1744 synchronize_irq(adapter->irq_tbl[i].vector); in ena_disable_io_intr_sync()
1747 static void ena_del_napi_in_range(struct ena_adapter *adapter, in ena_del_napi_in_range() argument
1754 netif_napi_del(&adapter->ena_napi[i].napi); in ena_del_napi_in_range()
1756 WARN_ON(ENA_IS_XDP_INDEX(adapter, i) && in ena_del_napi_in_range()
1757 adapter->ena_napi[i].rx_ring); in ena_del_napi_in_range()
1761 static void ena_init_napi_in_range(struct ena_adapter *adapter, in ena_init_napi_in_range() argument
1768 struct ena_napi *napi = &adapter->ena_napi[i]; in ena_init_napi_in_range()
1773 rx_ring = &adapter->rx_ring[i]; in ena_init_napi_in_range()
1774 tx_ring = &adapter->tx_ring[i]; in ena_init_napi_in_range()
1777 if (ENA_IS_XDP_INDEX(adapter, i)) in ena_init_napi_in_range()
1780 netif_napi_add_config(adapter->netdev, &napi->napi, napi_handler, i); in ena_init_napi_in_range()
1782 if (!ENA_IS_XDP_INDEX(adapter, i)) in ena_init_napi_in_range()
1790 static void ena_napi_disable_in_range(struct ena_adapter *adapter, in ena_napi_disable_in_range() argument
1798 napi = &adapter->ena_napi[i].napi; in ena_napi_disable_in_range()
1799 if (!ENA_IS_XDP_INDEX(adapter, i)) { in ena_napi_disable_in_range()
1801 netif_queue_set_napi(adapter->netdev, i, in ena_napi_disable_in_range()
1803 netif_queue_set_napi(adapter->netdev, i, in ena_napi_disable_in_range()
1810 static void ena_napi_enable_in_range(struct ena_adapter *adapter, in ena_napi_enable_in_range() argument
1818 napi = &adapter->ena_napi[i].napi; in ena_napi_enable_in_range()
1820 if (!ENA_IS_XDP_INDEX(adapter, i)) { in ena_napi_enable_in_range()
1822 netif_queue_set_napi(adapter->netdev, i, in ena_napi_enable_in_range()
1824 netif_queue_set_napi(adapter->netdev, i, in ena_napi_enable_in_range()
1831 static int ena_rss_configure(struct ena_adapter *adapter) in ena_rss_configure() argument
1833 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_rss_configure()
1838 rc = ena_rss_init_default(adapter); in ena_rss_configure()
1840 netif_err(adapter, ifup, adapter->netdev, "Failed to init RSS rc: %d\n", rc); in ena_rss_configure()
1863 static int ena_up_complete(struct ena_adapter *adapter) in ena_up_complete() argument
1867 rc = ena_rss_configure(adapter); in ena_up_complete()
1871 ena_change_mtu(adapter->netdev, adapter->netdev->mtu); in ena_up_complete()
1873 ena_refill_all_rx_bufs(adapter); in ena_up_complete()
1876 netif_tx_start_all_queues(adapter->netdev); in ena_up_complete()
1878 ena_napi_enable_in_range(adapter, in ena_up_complete()
1880 adapter->xdp_num_queues + adapter->num_io_queues); in ena_up_complete()
1885 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid) in ena_create_io_tx_queue() argument
1894 ena_dev = adapter->ena_dev; in ena_create_io_tx_queue()
1896 tx_ring = &adapter->tx_ring[qid]; in ena_create_io_tx_queue()
1911 netif_err(adapter, ifup, adapter->netdev, in ena_create_io_tx_queue()
1921 netif_err(adapter, ifup, adapter->netdev, in ena_create_io_tx_queue()
1932 int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter, in ena_create_io_tx_queues_in_range() argument
1935 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_create_io_tx_queues_in_range()
1939 rc = ena_create_io_tx_queue(adapter, i); in ena_create_io_tx_queues_in_range()
1953 static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid) in ena_create_io_rx_queue() argument
1962 ena_dev = adapter->ena_dev; in ena_create_io_rx_queue()
1964 rx_ring = &adapter->rx_ring[qid]; in ena_create_io_rx_queue()
1979 netif_err(adapter, ifup, adapter->netdev, in ena_create_io_rx_queue()
1989 netif_err(adapter, ifup, adapter->netdev, in ena_create_io_rx_queue()
2003 static int ena_create_all_io_rx_queues(struct ena_adapter *adapter) in ena_create_all_io_rx_queues() argument
2005 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_create_all_io_rx_queues()
2008 for (i = 0; i < adapter->num_io_queues; i++) { in ena_create_all_io_rx_queues()
2009 rc = ena_create_io_rx_queue(adapter, i); in ena_create_all_io_rx_queues()
2012 INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work); in ena_create_all_io_rx_queues()
2014 ena_xdp_register_rxq_info(&adapter->rx_ring[i]); in ena_create_all_io_rx_queues()
2021 ena_xdp_unregister_rxq_info(&adapter->rx_ring[i]); in ena_create_all_io_rx_queues()
2022 cancel_work_sync(&adapter->ena_napi[i].dim.work); in ena_create_all_io_rx_queues()
2029 static void set_io_rings_size(struct ena_adapter *adapter, in set_io_rings_size() argument
2035 for (i = 0; i < adapter->num_io_queues; i++) { in set_io_rings_size()
2036 adapter->tx_ring[i].ring_size = new_tx_size; in set_io_rings_size()
2037 adapter->rx_ring[i].ring_size = new_rx_size; in set_io_rings_size()
2056 static int create_queues_with_size_backoff(struct ena_adapter *adapter) in create_queues_with_size_backoff() argument
2064 set_io_rings_size(adapter, adapter->requested_tx_ring_size, in create_queues_with_size_backoff()
2065 adapter->requested_rx_ring_size); in create_queues_with_size_backoff()
2068 if (ena_xdp_present(adapter)) { in create_queues_with_size_backoff()
2069 rc = ena_setup_and_create_all_xdp_queues(adapter); in create_queues_with_size_backoff()
2074 rc = ena_setup_tx_resources_in_range(adapter, in create_queues_with_size_backoff()
2076 adapter->num_io_queues); in create_queues_with_size_backoff()
2080 rc = ena_create_io_tx_queues_in_range(adapter, in create_queues_with_size_backoff()
2082 adapter->num_io_queues); in create_queues_with_size_backoff()
2086 rc = ena_setup_all_rx_resources(adapter); in create_queues_with_size_backoff()
2090 rc = ena_create_all_io_rx_queues(adapter); in create_queues_with_size_backoff()
2097 ena_free_all_io_rx_resources(adapter); in create_queues_with_size_backoff()
2099 ena_destroy_all_tx_queues(adapter); in create_queues_with_size_backoff()
2101 ena_free_all_io_tx_resources(adapter); in create_queues_with_size_backoff()
2104 netif_err(adapter, ifup, adapter->netdev, in create_queues_with_size_backoff()
2110 cur_tx_ring_size = adapter->tx_ring[0].ring_size; in create_queues_with_size_backoff()
2111 cur_rx_ring_size = adapter->rx_ring[0].ring_size; in create_queues_with_size_backoff()
2113 netif_err(adapter, ifup, adapter->netdev, in create_queues_with_size_backoff()
2130 netif_err(adapter, ifup, adapter->netdev, in create_queues_with_size_backoff()
2136 netif_err(adapter, ifup, adapter->netdev, in create_queues_with_size_backoff()
2141 set_io_rings_size(adapter, new_tx_ring_size, in create_queues_with_size_backoff()
2146 int ena_up(struct ena_adapter *adapter) in ena_up() argument
2150 netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__); in ena_up()
2152 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; in ena_up()
2153 ena_setup_io_intr(adapter); in ena_up()
2160 ena_init_napi_in_range(adapter, 0, io_queue_count); in ena_up()
2165 if (ena_com_interrupt_moderation_supported(adapter->ena_dev)) in ena_up()
2166 ena_com_enable_adaptive_moderation(adapter->ena_dev); in ena_up()
2168 rc = ena_request_io_irq(adapter); in ena_up()
2172 rc = create_queues_with_size_backoff(adapter); in ena_up()
2176 rc = ena_up_complete(adapter); in ena_up()
2180 if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags)) in ena_up()
2181 netif_carrier_on(adapter->netdev); in ena_up()
2183 ena_increase_stat(&adapter->dev_stats.interface_up, 1, in ena_up()
2184 &adapter->syncp); in ena_up()
2186 set_bit(ENA_FLAG_DEV_UP, &adapter->flags); in ena_up()
2189 for (i = 0; i < adapter->num_io_queues; i++) in ena_up()
2190 ena_unmask_interrupt(&adapter->tx_ring[i], in ena_up()
2191 &adapter->rx_ring[i]); in ena_up()
2197 napi_schedule(&adapter->ena_napi[i].napi); in ena_up()
2202 ena_destroy_all_tx_queues(adapter); in ena_up()
2203 ena_free_all_io_tx_resources(adapter); in ena_up()
2204 ena_destroy_all_rx_queues(adapter); in ena_up()
2205 ena_free_all_io_rx_resources(adapter); in ena_up()
2207 ena_free_io_irq(adapter); in ena_up()
2209 ena_del_napi_in_range(adapter, 0, io_queue_count); in ena_up()
2214 void ena_down(struct ena_adapter *adapter) in ena_down() argument
2216 int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; in ena_down()
2218 netif_dbg(adapter, ifdown, adapter->netdev, "%s\n", __func__); in ena_down()
2220 clear_bit(ENA_FLAG_DEV_UP, &adapter->flags); in ena_down()
2222 ena_increase_stat(&adapter->dev_stats.interface_down, 1, in ena_down()
2223 &adapter->syncp); in ena_down()
2225 netif_carrier_off(adapter->netdev); in ena_down()
2226 netif_tx_disable(adapter->netdev); in ena_down()
2229 ena_napi_disable_in_range(adapter, 0, io_queue_count); in ena_down()
2231 if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) { in ena_down()
2234 rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); in ena_down()
2236 netif_err(adapter, ifdown, adapter->netdev, in ena_down()
2239 ena_com_set_admin_running_state(adapter->ena_dev, false); in ena_down()
2242 ena_destroy_all_io_queues(adapter); in ena_down()
2244 ena_disable_io_intr_sync(adapter); in ena_down()
2245 ena_free_io_irq(adapter); in ena_down()
2246 ena_del_napi_in_range(adapter, 0, io_queue_count); in ena_down()
2248 ena_free_all_tx_bufs(adapter); in ena_down()
2249 ena_free_all_rx_bufs(adapter); in ena_down()
2250 ena_free_all_io_tx_resources(adapter); in ena_down()
2251 ena_free_all_io_rx_resources(adapter); in ena_down()
2267 struct ena_adapter *adapter = netdev_priv(netdev); in ena_open() local
2271 rc = netif_set_real_num_tx_queues(netdev, adapter->num_io_queues); in ena_open()
2273 netif_err(adapter, ifup, netdev, "Can't set num tx queues\n"); in ena_open()
2277 rc = netif_set_real_num_rx_queues(netdev, adapter->num_io_queues); in ena_open()
2279 netif_err(adapter, ifup, netdev, "Can't set num rx queues\n"); in ena_open()
2283 rc = ena_up(adapter); in ena_open()
2302 struct ena_adapter *adapter = netdev_priv(netdev); in ena_close() local
2304 netif_dbg(adapter, ifdown, netdev, "%s\n", __func__); in ena_close()
2306 if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)) in ena_close()
2309 if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) in ena_close()
2310 ena_down(adapter); in ena_close()
2313 check_for_admin_com_state(adapter); in ena_close()
2314 if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) { in ena_close()
2315 netif_err(adapter, ifdown, adapter->netdev, in ena_close()
2317 ena_dump_stats_to_dmesg(adapter); in ena_close()
2319 ena_destroy_device(adapter, false); in ena_close()
2320 ena_restore_device(adapter); in ena_close()
2326 int ena_update_queue_params(struct ena_adapter *adapter, in ena_update_queue_params() argument
2334 dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); in ena_update_queue_params()
2335 ena_close(adapter->netdev); in ena_update_queue_params()
2336 adapter->requested_tx_ring_size = new_tx_size; in ena_update_queue_params()
2337 adapter->requested_rx_ring_size = new_rx_size; in ena_update_queue_params()
2338 ena_init_io_rings(adapter, in ena_update_queue_params()
2340 adapter->xdp_num_queues + in ena_update_queue_params()
2341 adapter->num_io_queues); in ena_update_queue_params()
2343 large_llq_changed = adapter->ena_dev->tx_mem_queue_type == in ena_update_queue_params()
2346 new_llq_header_len != adapter->ena_dev->tx_max_header_size; in ena_update_queue_params()
2350 adapter->large_llq_header_enabled = !adapter->large_llq_header_enabled; in ena_update_queue_params()
2352 ena_destroy_device(adapter, false); in ena_update_queue_params()
2353 rc = ena_restore_device(adapter); in ena_update_queue_params()
2356 return dev_was_up && !rc ? ena_up(adapter) : rc; in ena_update_queue_params()
2359 int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak) in ena_set_rx_copybreak() argument
2364 if (rx_copybreak > min_t(u16, adapter->netdev->mtu, ENA_PAGE_SIZE)) in ena_set_rx_copybreak()
2367 adapter->rx_copybreak = rx_copybreak; in ena_set_rx_copybreak()
2369 for (i = 0; i < adapter->num_io_queues; i++) { in ena_set_rx_copybreak()
2370 rx_ring = &adapter->rx_ring[i]; in ena_set_rx_copybreak()
2377 int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count) in ena_update_queue_count() argument
2379 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_update_queue_count()
2383 dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); in ena_update_queue_count()
2384 ena_close(adapter->netdev); in ena_update_queue_count()
2385 prev_channel_count = adapter->num_io_queues; in ena_update_queue_count()
2386 adapter->num_io_queues = new_channel_count; in ena_update_queue_count()
2387 if (ena_xdp_present(adapter) && in ena_update_queue_count()
2388 ena_xdp_allowed(adapter) == ENA_XDP_ALLOWED) { in ena_update_queue_count()
2389 adapter->xdp_first_ring = new_channel_count; in ena_update_queue_count()
2390 adapter->xdp_num_queues = new_channel_count; in ena_update_queue_count()
2392 ena_xdp_exchange_program_rx_in_range(adapter, in ena_update_queue_count()
2397 ena_xdp_exchange_program_rx_in_range(adapter, in ena_update_queue_count()
2398 adapter->xdp_bpf_prog, in ena_update_queue_count()
2407 ena_init_io_rings(adapter, in ena_update_queue_count()
2409 adapter->xdp_num_queues + in ena_update_queue_count()
2410 adapter->num_io_queues); in ena_update_queue_count()
2411 return dev_was_up ? ena_open(adapter->netdev) : 0; in ena_update_queue_count()
2500 struct ena_adapter *adapter = tx_ring->adapter; in ena_tx_map_skb() local
2539 netif_dbg(adapter, tx_queued, adapter->netdev, in ena_tx_map_skb()
2588 netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map skb\n"); in ena_tx_map_skb()
2601 struct ena_adapter *adapter = netdev_priv(dev); in ena_start_xmit() local
2610 netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb); in ena_start_xmit()
2613 tx_ring = &adapter->tx_ring[qid]; in ena_start_xmit()
2641 rc = ena_xmit_common(adapter, in ena_start_xmit()
2658 netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n", in ena_start_xmit()
2764 static void ena_config_debug_area(struct ena_adapter *adapter) in ena_config_debug_area() argument
2769 ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS); in ena_config_debug_area()
2771 netif_err(adapter, drv, adapter->netdev, in ena_config_debug_area()
2779 rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size); in ena_config_debug_area()
2781 netif_err(adapter, drv, adapter->netdev, in ena_config_debug_area()
2786 rc = ena_com_set_host_attributes(adapter->ena_dev); in ena_config_debug_area()
2789 netif_warn(adapter, drv, adapter->netdev, "Cannot set host attributes\n"); in ena_config_debug_area()
2791 netif_err(adapter, drv, adapter->netdev, in ena_config_debug_area()
2798 ena_com_delete_debug_area(adapter->ena_dev); in ena_config_debug_area()
2804 struct ena_adapter *adapter = netdev_priv(netdev); in ena_get_stats64() local
2812 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) in ena_get_stats64()
2815 for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { in ena_get_stats64()
2818 tx_ring = &adapter->tx_ring[i]; in ena_get_stats64()
2830 if (ENA_IS_XDP_INDEX(adapter, i)) in ena_get_stats64()
2833 rx_ring = &adapter->rx_ring[i]; in ena_get_stats64()
2848 start = u64_stats_fetch_begin(&adapter->syncp); in ena_get_stats64()
2849 rx_drops = adapter->dev_stats.rx_drops; in ena_get_stats64()
2850 tx_drops = adapter->dev_stats.tx_drops; in ena_get_stats64()
2851 } while (u64_stats_fetch_retry(&adapter->syncp, start)); in ena_get_stats64()
2882 static int ena_calc_io_queue_size(struct ena_adapter *adapter, in ena_calc_io_queue_size() argument
2886 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_calc_io_queue_size()
2895 if (adapter->tx_ring->ring_size) in ena_calc_io_queue_size()
2896 tx_queue_size = adapter->tx_ring->ring_size; in ena_calc_io_queue_size()
2898 if (adapter->rx_ring->ring_size) in ena_calc_io_queue_size()
2899 rx_queue_size = adapter->rx_ring->ring_size; in ena_calc_io_queue_size()
2915 adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS, in ena_calc_io_queue_size()
2917 adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS, in ena_calc_io_queue_size()
2933 adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS, in ena_calc_io_queue_size()
2935 adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS, in ena_calc_io_queue_size()
2943 netdev_err(adapter->netdev, "Device max TX queue size: %d < minimum: %d\n", in ena_calc_io_queue_size()
2949 netdev_err(adapter->netdev, "Device max RX queue size: %d < minimum: %d\n", in ena_calc_io_queue_size()
2957 if (adapter->large_llq_header_enabled) { in ena_calc_io_queue_size()
2961 dev_info(&adapter->pdev->dev, in ena_calc_io_queue_size()
2965 dev_err(&adapter->pdev->dev, in ena_calc_io_queue_size()
2968 adapter->large_llq_header_enabled = false; in ena_calc_io_queue_size()
2980 adapter->max_tx_ring_size = max_tx_queue_size; in ena_calc_io_queue_size()
2981 adapter->max_rx_ring_size = max_rx_queue_size; in ena_calc_io_queue_size()
2982 adapter->requested_tx_ring_size = tx_queue_size; in ena_calc_io_queue_size()
2983 adapter->requested_rx_ring_size = rx_queue_size; in ena_calc_io_queue_size()
2988 static int ena_device_validate_params(struct ena_adapter *adapter, in ena_device_validate_params() argument
2991 struct net_device *netdev = adapter->netdev; in ena_device_validate_params()
2995 adapter->mac_addr); in ena_device_validate_params()
2997 netif_err(adapter, drv, netdev, in ena_device_validate_params()
3003 netif_err(adapter, drv, netdev, in ena_device_validate_params()
3011 static void set_default_llq_configurations(struct ena_adapter *adapter, in set_default_llq_configurations() argument
3015 struct ena_com_dev *ena_dev = adapter->ena_dev; in set_default_llq_configurations()
3021 adapter->large_llq_header_supported = in set_default_llq_configurations()
3023 adapter->large_llq_header_supported &= in set_default_llq_configurations()
3028 adapter->large_llq_header_enabled) { in set_default_llq_configurations()
3088 static int ena_device_init(struct ena_adapter *adapter, struct pci_dev *pdev, in ena_device_init() argument
3092 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_device_init()
3093 struct net_device *netdev = adapter->netdev; in ena_device_init()
3178 set_default_llq_configurations(adapter, &llq_config, &get_feat_ctx->llq); in ena_device_init()
3187 rc = ena_calc_io_queue_size(adapter, get_feat_ctx); in ena_device_init()
3204 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter) in ena_enable_msix_and_set_admin_interrupts() argument
3206 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_enable_msix_and_set_admin_interrupts()
3207 struct device *dev = &adapter->pdev->dev; in ena_enable_msix_and_set_admin_interrupts()
3210 rc = ena_enable_msix(adapter); in ena_enable_msix_and_set_admin_interrupts()
3216 ena_setup_mgmnt_intr(adapter); in ena_enable_msix_and_set_admin_interrupts()
3218 rc = ena_request_mgmnt_irq(adapter); in ena_enable_msix_and_set_admin_interrupts()
3231 ena_disable_msix(adapter); in ena_enable_msix_and_set_admin_interrupts()
3236 static int ena_destroy_device(struct ena_adapter *adapter, bool graceful) in ena_destroy_device() argument
3238 struct net_device *netdev = adapter->netdev; in ena_destroy_device()
3239 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_destroy_device()
3243 if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)) in ena_destroy_device()
3248 timer_delete_sync(&adapter->timer_service); in ena_destroy_device()
3250 dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); in ena_destroy_device()
3251 adapter->dev_up_before_reset = dev_up; in ena_destroy_device()
3256 ena_down(adapter); in ena_destroy_device()
3261 if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up)) in ena_destroy_device()
3262 rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); in ena_destroy_device()
3264 ena_free_mgmnt_irq(adapter); in ena_destroy_device()
3266 ena_disable_msix(adapter); in ena_destroy_device()
3277 adapter->reset_reason = ENA_REGS_RESET_NORMAL; in ena_destroy_device()
3279 clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags); in ena_destroy_device()
3280 clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags); in ena_destroy_device()
3285 static int ena_restore_device(struct ena_adapter *adapter) in ena_restore_device() argument
3288 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_restore_device()
3289 struct pci_dev *pdev = adapter->pdev; in ena_restore_device()
3294 set_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags); in ena_restore_device()
3295 rc = ena_device_init(adapter, adapter->pdev, &get_feat_ctx, &wd_state); in ena_restore_device()
3300 adapter->wd_state = wd_state; in ena_restore_device()
3302 count = adapter->xdp_num_queues + adapter->num_io_queues; in ena_restore_device()
3304 txr = &adapter->tx_ring[i]; in ena_restore_device()
3309 rc = ena_device_validate_params(adapter, &get_feat_ctx); in ena_restore_device()
3315 rc = ena_enable_msix_and_set_admin_interrupts(adapter); in ena_restore_device()
3321 if (adapter->dev_up_before_reset) { in ena_restore_device()
3322 rc = ena_up(adapter); in ena_restore_device()
3329 set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags); in ena_restore_device()
3331 clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags); in ena_restore_device()
3332 if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags)) in ena_restore_device()
3333 netif_carrier_on(adapter->netdev); in ena_restore_device()
3335 mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ)); in ena_restore_device()
3336 adapter->last_keep_alive_jiffies = jiffies; in ena_restore_device()
3340 ena_free_mgmnt_irq(adapter); in ena_restore_device()
3341 ena_disable_msix(adapter); in ena_restore_device()
3349 clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags); in ena_restore_device()
3350 clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags); in ena_restore_device()
3361 struct ena_adapter *adapter = in ena_fw_reset_device() local
3366 if (likely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) { in ena_fw_reset_device()
3367 rc |= ena_destroy_device(adapter, false); in ena_fw_reset_device()
3368 rc |= ena_restore_device(adapter); in ena_fw_reset_device()
3369 adapter->dev_stats.reset_fail += !!rc; in ena_fw_reset_device()
3371 dev_err(&adapter->pdev->dev, "Device reset completed successfully\n"); in ena_fw_reset_device()
3377 static int check_for_rx_interrupt_queue(struct ena_adapter *adapter, in check_for_rx_interrupt_queue() argument
3391 netif_err(adapter, rx_err, adapter->netdev, in check_for_rx_interrupt_queue()
3395 ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT); in check_for_rx_interrupt_queue()
3402 static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter, in check_missing_comp_in_tx_queue() argument
3416 missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to); in check_missing_comp_in_tx_queue()
3427 2 * adapter->missing_tx_completion_to); in check_missing_comp_in_tx_queue()
3433 netif_err(adapter, tx_err, adapter->netdev, in check_missing_comp_in_tx_queue()
3436 ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT); in check_missing_comp_in_tx_queue()
3441 adapter->missing_tx_completion_to); in check_missing_comp_in_tx_queue()
3454 2 * adapter->missing_tx_completion_to)) in check_missing_comp_in_tx_queue()
3465 netif_notice(adapter, tx_err, adapter->netdev, in check_missing_comp_in_tx_queue()
3473 if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) { in check_missing_comp_in_tx_queue()
3474 netif_err(adapter, tx_err, adapter->netdev, in check_missing_comp_in_tx_queue()
3477 adapter->missing_tx_completion_threshold, in check_missing_comp_in_tx_queue()
3479 netif_err(adapter, tx_err, adapter->netdev, in check_missing_comp_in_tx_queue()
3482 ena_reset_device(adapter, reset_reason); in check_missing_comp_in_tx_queue()
3492 static void check_for_missing_completions(struct ena_adapter *adapter) in check_for_missing_completions() argument
3499 io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues; in check_for_missing_completions()
3504 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) in check_for_missing_completions()
3507 if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) in check_for_missing_completions()
3510 if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT) in check_for_missing_completions()
3515 qid = adapter->last_monitored_tx_qid; in check_for_missing_completions()
3520 tx_ring = &adapter->tx_ring[qid]; in check_for_missing_completions()
3521 rx_ring = &adapter->rx_ring[qid]; in check_for_missing_completions()
3523 rc = check_missing_comp_in_tx_queue(adapter, tx_ring); in check_for_missing_completions()
3527 rc = !ENA_IS_XDP_INDEX(adapter, qid) ? in check_for_missing_completions()
3528 check_for_rx_interrupt_queue(adapter, rx_ring) : 0; in check_for_missing_completions()
3535 adapter->last_monitored_tx_qid = qid; in check_for_missing_completions()
3554 static void check_for_empty_rx_ring(struct ena_adapter *adapter) in check_for_empty_rx_ring() argument
3559 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) in check_for_empty_rx_ring()
3562 if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) in check_for_empty_rx_ring()
3565 for (i = 0; i < adapter->num_io_queues; i++) { in check_for_empty_rx_ring()
3566 rx_ring = &adapter->rx_ring[i]; in check_for_empty_rx_ring()
3576 netif_err(adapter, drv, adapter->netdev, in check_for_empty_rx_ring()
3589 static void check_for_missing_keep_alive(struct ena_adapter *adapter) in check_for_missing_keep_alive() argument
3593 if (!adapter->wd_state) in check_for_missing_keep_alive()
3596 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) in check_for_missing_keep_alive()
3599 keep_alive_expired = adapter->last_keep_alive_jiffies + in check_for_missing_keep_alive()
3600 adapter->keep_alive_timeout; in check_for_missing_keep_alive()
3602 netif_err(adapter, drv, adapter->netdev, in check_for_missing_keep_alive()
3604 ena_increase_stat(&adapter->dev_stats.wd_expired, 1, in check_for_missing_keep_alive()
3605 &adapter->syncp); in check_for_missing_keep_alive()
3606 ena_reset_device(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO); in check_for_missing_keep_alive()
3610 static void check_for_admin_com_state(struct ena_adapter *adapter) in check_for_admin_com_state() argument
3612 if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) { in check_for_admin_com_state()
3613 netif_err(adapter, drv, adapter->netdev, in check_for_admin_com_state()
3615 ena_increase_stat(&adapter->dev_stats.admin_q_pause, 1, in check_for_admin_com_state()
3616 &adapter->syncp); in check_for_admin_com_state()
3617 ena_reset_device(adapter, ENA_REGS_RESET_ADMIN_TO); in check_for_admin_com_state()
3621 static void ena_update_hints(struct ena_adapter *adapter, in ena_update_hints() argument
3624 struct net_device *netdev = adapter->netdev; in ena_update_hints()
3627 adapter->ena_dev->admin_queue.completion_timeout = in ena_update_hints()
3632 adapter->ena_dev->mmio_read.reg_read_to = in ena_update_hints()
3636 adapter->missing_tx_completion_threshold = in ena_update_hints()
3641 adapter->missing_tx_completion_to = ENA_HW_HINTS_NO_TIMEOUT; in ena_update_hints()
3643 adapter->missing_tx_completion_to = in ena_update_hints()
3652 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; in ena_update_hints()
3654 adapter->keep_alive_timeout = in ena_update_hints()
3670 struct ena_adapter *adapter = timer_container_of(adapter, t, in ena_timer_service() local
3672 u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr; in ena_timer_service()
3674 adapter->ena_dev->host_attr.host_info; in ena_timer_service()
3676 check_for_missing_keep_alive(adapter); in ena_timer_service()
3678 check_for_admin_com_state(adapter); in ena_timer_service()
3680 check_for_missing_completions(adapter); in ena_timer_service()
3682 check_for_empty_rx_ring(adapter); in ena_timer_service()
3685 ena_dump_stats_to_buf(adapter, debug_area); in ena_timer_service()
3688 ena_update_host_info(host_info, adapter->netdev); in ena_timer_service()
3690 if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) { in ena_timer_service()
3691 netif_err(adapter, drv, adapter->netdev, in ena_timer_service()
3693 ena_dump_stats_to_dmesg(adapter); in ena_timer_service()
3694 queue_work(ena_wq, &adapter->reset_task); in ena_timer_service()
3699 mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ)); in ena_timer_service()
3779 static void ena_set_conf_feat_params(struct ena_adapter *adapter, in ena_set_conf_feat_params() argument
3782 struct net_device *netdev = adapter->netdev; in ena_set_conf_feat_params()
3787 ether_addr_copy(adapter->mac_addr, netdev->dev_addr); in ena_set_conf_feat_params()
3789 ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr); in ena_set_conf_feat_params()
3790 eth_hw_addr_set(netdev, adapter->mac_addr); in ena_set_conf_feat_params()
3796 adapter->max_mtu = feat->dev_attr.max_mtu; in ena_set_conf_feat_params()
3797 netdev->max_mtu = adapter->max_mtu; in ena_set_conf_feat_params()
3801 static int ena_rss_init_default(struct ena_adapter *adapter) in ena_rss_init_default() argument
3803 struct ena_com_dev *ena_dev = adapter->ena_dev; in ena_rss_init_default()
3804 struct device *dev = &adapter->pdev->dev; in ena_rss_init_default()
3815 val = ethtool_rxfh_indir_default(i, adapter->num_io_queues); in ena_rss_init_default()
3859 * ena_probe initializes an adapter identified by a pci_dev structure.
3860 * The OS initialization, configuring of the adapter private structure,
3867 struct ena_adapter *adapter; in ena_probe() local
3925 adapter = netdev_priv(netdev); in ena_probe()
3926 adapter->ena_dev = ena_dev; in ena_probe()
3927 adapter->netdev = netdev; in ena_probe()
3928 adapter->pdev = pdev; in ena_probe()
3929 adapter->msg_enable = DEFAULT_MSG_ENABLE; in ena_probe()
3933 pci_set_drvdata(pdev, adapter); in ena_probe()
3947 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &wd_state); in ena_probe()
3967 ena_set_conf_feat_params(adapter, &get_feat_ctx); in ena_probe()
3969 adapter->reset_reason = ENA_REGS_RESET_NORMAL; in ena_probe()
3971 adapter->num_io_queues = max_num_io_queues; in ena_probe()
3972 adapter->max_num_io_queues = max_num_io_queues; in ena_probe()
3973 adapter->last_monitored_tx_qid = 0; in ena_probe()
3975 adapter->xdp_first_ring = 0; in ena_probe()
3976 adapter->xdp_num_queues = 0; in ena_probe()
3978 adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK; in ena_probe()
3980 adapter->disable_meta_caching = in ena_probe()
3984 adapter->wd_state = wd_state; in ena_probe()
3986 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found); in ena_probe()
3988 rc = ena_com_init_interrupt_moderation(adapter->ena_dev); in ena_probe()
3995 ena_init_io_rings(adapter, in ena_probe()
3997 adapter->xdp_num_queues + in ena_probe()
3998 adapter->num_io_queues); in ena_probe()
4006 u64_stats_init(&adapter->syncp); in ena_probe()
4008 rc = ena_enable_msix_and_set_admin_interrupts(adapter); in ena_probe()
4014 rc = ena_rss_init_default(adapter); in ena_probe()
4020 ena_config_debug_area(adapter); in ena_probe()
4022 if (ena_xdp_legal_queue_count(adapter, adapter->num_io_queues)) in ena_probe()
4026 memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len); in ena_probe()
4036 INIT_WORK(&adapter->reset_task, ena_fw_reset_device); in ena_probe()
4038 adapter->last_keep_alive_jiffies = jiffies; in ena_probe()
4039 adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT; in ena_probe()
4040 adapter->missing_tx_completion_to = TX_TIMEOUT; in ena_probe()
4041 adapter->missing_tx_completion_threshold = MAX_NUM_OF_TIMEOUTED_PACKETS; in ena_probe()
4043 ena_update_hints(adapter, &get_feat_ctx.hw_hints); in ena_probe()
4045 timer_setup(&adapter->timer_service, ena_timer_service, 0); in ena_probe()
4046 mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ)); in ena_probe()
4053 set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags); in ena_probe()
4066 ena_free_mgmnt_irq(adapter); in ena_probe()
4067 ena_disable_msix(adapter); in ena_probe()
4069 timer_delete(&adapter->timer_service); in ena_probe()
4098 struct ena_adapter *adapter = pci_get_drvdata(pdev); in __ena_shutoff() local
4102 ena_dev = adapter->ena_dev; in __ena_shutoff()
4103 netdev = adapter->netdev; in __ena_shutoff()
4108 timer_delete_sync(&adapter->timer_service); in __ena_shutoff()
4109 cancel_work_sync(&adapter->reset_task); in __ena_shutoff()
4112 adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN; in __ena_shutoff()
4113 ena_destroy_device(adapter, true); in __ena_shutoff()
4170 struct ena_adapter *adapter = pci_get_drvdata(pdev); in ena_suspend() local
4172 ena_increase_stat(&adapter->dev_stats.suspend, 1, &adapter->syncp); in ena_suspend()
4175 if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) { in ena_suspend()
4178 clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags); in ena_suspend()
4180 ena_destroy_device(adapter, true); in ena_suspend()
4190 struct ena_adapter *adapter = dev_get_drvdata(dev_d); in ena_resume() local
4193 ena_increase_stat(&adapter->dev_stats.resume, 1, &adapter->syncp); in ena_resume()
4196 rc = ena_restore_device(adapter); in ena_resume()
4249 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_update_on_link_change() local
4256 netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__); in ena_update_on_link_change()
4257 set_bit(ENA_FLAG_LINK_UP, &adapter->flags); in ena_update_on_link_change()
4258 if (!test_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags)) in ena_update_on_link_change()
4259 netif_carrier_on(adapter->netdev); in ena_update_on_link_change()
4261 clear_bit(ENA_FLAG_LINK_UP, &adapter->flags); in ena_update_on_link_change()
4262 netif_carrier_off(adapter->netdev); in ena_update_on_link_change()
4269 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_keep_alive_wd() local
4275 adapter->last_keep_alive_jiffies = jiffies; in ena_keep_alive_wd()
4280 u64_stats_update_begin(&adapter->syncp); in ena_keep_alive_wd()
4284 adapter->dev_stats.rx_drops = rx_drops; in ena_keep_alive_wd()
4285 adapter->dev_stats.tx_drops = tx_drops; in ena_keep_alive_wd()
4286 u64_stats_update_end(&adapter->syncp); in ena_keep_alive_wd()
4292 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; in ena_notification() local
4304 ena_update_hints(adapter, hints); in ena_notification()
4307 netif_err(adapter, drv, adapter->netdev, in ena_notification()
4317 struct ena_adapter *adapter = (struct ena_adapter *)data; in unimplemented_aenq_handler() local
4319 netif_err(adapter, drv, adapter->netdev, in unimplemented_aenq_handler()