ena.c (c2c014f24c10f90d85126ac5fbd4d8524de32b1c) ena.c (cd5d5804b16daf16a3ffe31e857be954f49d3834)
1/*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 77 unchanged lines hidden (view full) ---

86static int ena_allocate_pci_resources(struct ena_adapter*);
87static void ena_free_pci_resources(struct ena_adapter *);
88static int ena_change_mtu(if_t, int);
89static inline void ena_alloc_counters(counter_u64_t *, int);
90static inline void ena_free_counters(counter_u64_t *, int);
91static inline void ena_reset_counters(counter_u64_t *, int);
92static void ena_init_io_rings_common(struct ena_adapter *,
93 struct ena_ring *, uint16_t);
1/*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 77 unchanged lines hidden (view full) ---

86static int ena_allocate_pci_resources(struct ena_adapter*);
87static void ena_free_pci_resources(struct ena_adapter *);
88static int ena_change_mtu(if_t, int);
89static inline void ena_alloc_counters(counter_u64_t *, int);
90static inline void ena_free_counters(counter_u64_t *, int);
91static inline void ena_reset_counters(counter_u64_t *, int);
92static void ena_init_io_rings_common(struct ena_adapter *,
93 struct ena_ring *, uint16_t);
94static int ena_init_io_rings(struct ena_adapter *);
94static void ena_init_io_rings(struct ena_adapter *);
95static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int);
96static void ena_free_all_io_rings_resources(struct ena_adapter *);
97static int ena_setup_tx_dma_tag(struct ena_adapter *);
98static int ena_free_tx_dma_tag(struct ena_adapter *);
99static int ena_setup_rx_dma_tag(struct ena_adapter *);
100static int ena_free_rx_dma_tag(struct ena_adapter *);
101static int ena_setup_tx_resources(struct ena_adapter *, int);
102static void ena_free_tx_resources(struct ena_adapter *, int);

--- 312 unchanged lines hidden (view full) ---

415 uint16_t qid)
416{
417
418 ring->qid = qid;
419 ring->adapter = adapter;
420 ring->ena_dev = adapter->ena_dev;
421}
422
95static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int);
96static void ena_free_all_io_rings_resources(struct ena_adapter *);
97static int ena_setup_tx_dma_tag(struct ena_adapter *);
98static int ena_free_tx_dma_tag(struct ena_adapter *);
99static int ena_setup_rx_dma_tag(struct ena_adapter *);
100static int ena_free_rx_dma_tag(struct ena_adapter *);
101static int ena_setup_tx_resources(struct ena_adapter *, int);
102static void ena_free_tx_resources(struct ena_adapter *, int);

--- 312 unchanged lines hidden (view full) ---

415 uint16_t qid)
416{
417
418 ring->qid = qid;
419 ring->adapter = adapter;
420 ring->ena_dev = adapter->ena_dev;
421}
422
423static int
423static void
424ena_init_io_rings(struct ena_adapter *adapter)
425{
426 struct ena_com_dev *ena_dev;
427 struct ena_ring *txr, *rxr;
428 struct ena_que *que;
429 int i;
424ena_init_io_rings(struct ena_adapter *adapter)
425{
426 struct ena_com_dev *ena_dev;
427 struct ena_ring *txr, *rxr;
428 struct ena_que *que;
429 int i;
430 int rc;
431
432 ena_dev = adapter->ena_dev;
433
434 for (i = 0; i < adapter->num_queues; i++) {
435 txr = &adapter->tx_ring[i];
436 rxr = &adapter->rx_ring[i];
437
438 /* TX/RX common ring state */

--- 5 unchanged lines hidden (view full) ---

444 txr->tx_max_header_size = ena_dev->tx_max_header_size;
445 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
446 txr->smoothed_interval =
447 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
448
449 /* Allocate a buf ring */
450 txr->br = buf_ring_alloc(ena_buf_ring_size, M_DEVBUF,
451 M_WAITOK, &txr->ring_mtx);
430
431 ena_dev = adapter->ena_dev;
432
433 for (i = 0; i < adapter->num_queues; i++) {
434 txr = &adapter->tx_ring[i];
435 rxr = &adapter->rx_ring[i];
436
437 /* TX/RX common ring state */

--- 5 unchanged lines hidden (view full) ---

443 txr->tx_max_header_size = ena_dev->tx_max_header_size;
444 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
445 txr->smoothed_interval =
446 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
447
448 /* Allocate a buf ring */
449 txr->br = buf_ring_alloc(ena_buf_ring_size, M_DEVBUF,
450 M_WAITOK, &txr->ring_mtx);
452 if (txr->br == NULL) {
453 device_printf(adapter->pdev,
454 "Error while setting up bufring\n");
455 rc = ENOMEM;
456 goto err_bufr_free;
457 }
458
459 /* Alloc TX statistics. */
460 ena_alloc_counters((counter_u64_t *)&txr->tx_stats,
461 sizeof(txr->tx_stats));
462
463 /* RX specific ring state */
464 rxr->ring_size = adapter->rx_ring_size;
465 rxr->rx_small_copy_len = adapter->small_copy_len;

--- 16 unchanged lines hidden (view full) ---

482 que->adapter = adapter;
483 que->id = i;
484 que->tx_ring = txr;
485 que->rx_ring = rxr;
486
487 txr->que = que;
488 rxr->que = que;
489 }
451
452 /* Alloc TX statistics. */
453 ena_alloc_counters((counter_u64_t *)&txr->tx_stats,
454 sizeof(txr->tx_stats));
455
456 /* RX specific ring state */
457 rxr->ring_size = adapter->rx_ring_size;
458 rxr->rx_small_copy_len = adapter->small_copy_len;

--- 16 unchanged lines hidden (view full) ---

475 que->adapter = adapter;
476 que->id = i;
477 que->tx_ring = txr;
478 que->rx_ring = rxr;
479
480 txr->que = que;
481 rxr->que = que;
482 }
490
491 return 0;
492
493err_bufr_free:
494 while (i--)
495 ena_free_io_ring_resources(adapter, i);
496
497 return (rc);
498}
499
500static void
501ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid)
502{
503 struct ena_ring *txr = &adapter->tx_ring[qid];
504 struct ena_ring *rxr = &adapter->rx_ring[qid];
505

--- 110 unchanged lines hidden (view full) ---

616 int size, i, err;
617#ifdef RSS
618 cpuset_t cpu_mask;
619#endif
620
621 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
622
623 tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
483}
484
485static void
486ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid)
487{
488 struct ena_ring *txr = &adapter->tx_ring[qid];
489 struct ena_ring *rxr = &adapter->rx_ring[qid];
490

--- 110 unchanged lines hidden (view full) ---

601 int size, i, err;
602#ifdef RSS
603 cpuset_t cpu_mask;
604#endif
605
606 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
607
608 tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
624 if (!tx_ring->tx_buffer_info)
609 if (tx_ring->tx_buffer_info == NULL)
625 goto err_tx_buffer_info;
626
627 size = sizeof(uint16_t) * tx_ring->ring_size;
628 tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
610 goto err_tx_buffer_info;
611
612 size = sizeof(uint16_t) * tx_ring->ring_size;
613 tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
629 if (!tx_ring->free_tx_ids)
614 if (tx_ring->free_tx_ids == NULL)
630 goto err_tx_reqs;
631
632 /* Req id stack for TX OOO completions */
633 for (i = 0; i < tx_ring->ring_size; i++)
634 tx_ring->free_tx_ids[i] = i;
635
636 /* Reset TX statistics. */
637 ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats,

--- 42 unchanged lines hidden (view full) ---

680
681 return (0);
682
683err_tx_map:
684 while (i--) {
685 bus_dmamap_destroy(adapter->tx_buf_tag,
686 tx_ring->tx_buffer_info[i].map);
687 }
615 goto err_tx_reqs;
616
617 /* Req id stack for TX OOO completions */
618 for (i = 0; i < tx_ring->ring_size; i++)
619 tx_ring->free_tx_ids[i] = i;
620
621 /* Reset TX statistics. */
622 ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats,

--- 42 unchanged lines hidden (view full) ---

665
666 return (0);
667
668err_tx_map:
669 while (i--) {
670 bus_dmamap_destroy(adapter->tx_buf_tag,
671 tx_ring->tx_buffer_info[i].map);
672 }
688 ENA_MEM_FREE(adapter->ena_dev->dmadev, tx_ring->free_tx_ids);
673 free(tx_ring->free_tx_ids, M_DEVBUF);
689err_tx_reqs:
674err_tx_reqs:
690 ENA_MEM_FREE(adapter->ena_dev->dmadev, tx_ring->tx_buffer_info);
675 free(tx_ring->tx_buffer_info, M_DEVBUF);
691err_tx_buffer_info:
692 return (ENOMEM);
693}
694
695/**
696 * ena_free_tx_resources - Free Tx Resources per Queue
697 * @adapter: network interface device structure
698 * @qid: queue index

--- 22 unchanged lines hidden (view full) ---

721 bus_dmamap_unload(adapter->tx_buf_tag,
722 tx_ring->tx_buffer_info[i].map);
723 bus_dmamap_destroy(adapter->tx_buf_tag,
724 tx_ring->tx_buffer_info[i].map);
725 }
726 ENA_RING_MTX_UNLOCK(tx_ring);
727
728 /* And free allocated memory. */
676err_tx_buffer_info:
677 return (ENOMEM);
678}
679
680/**
681 * ena_free_tx_resources - Free Tx Resources per Queue
682 * @adapter: network interface device structure
683 * @qid: queue index

--- 22 unchanged lines hidden (view full) ---

706 bus_dmamap_unload(adapter->tx_buf_tag,
707 tx_ring->tx_buffer_info[i].map);
708 bus_dmamap_destroy(adapter->tx_buf_tag,
709 tx_ring->tx_buffer_info[i].map);
710 }
711 ENA_RING_MTX_UNLOCK(tx_ring);
712
713 /* And free allocated memory. */
729 ENA_MEM_FREE(adapter->ena_dev->dmadev, tx_ring->tx_buffer_info);
714 free(tx_ring->tx_buffer_info, M_DEVBUF);
730 tx_ring->tx_buffer_info = NULL;
731
715 tx_ring->tx_buffer_info = NULL;
716
732 ENA_MEM_FREE(adapter->ena_dev->dmadev, tx_ring->free_tx_ids);
717 free(tx_ring->free_tx_ids, M_DEVBUF);
733 tx_ring->free_tx_ids = NULL;
734}
735
736/**
737 * ena_setup_all_tx_resources - allocate all queues Tx resources
738 * @adapter: network interface device structure
739 *
740 * Returns 0 on success, otherwise on failure.
741 **/
742static int
743ena_setup_all_tx_resources(struct ena_adapter *adapter)
744{
745 int i, rc;
746
747 for (i = 0; i < adapter->num_queues; i++) {
748 rc = ena_setup_tx_resources(adapter, i);
718 tx_ring->free_tx_ids = NULL;
719}
720
721/**
722 * ena_setup_all_tx_resources - allocate all queues Tx resources
723 * @adapter: network interface device structure
724 *
725 * Returns 0 on success, otherwise on failure.
726 **/
727static int
728ena_setup_all_tx_resources(struct ena_adapter *adapter)
729{
730 int i, rc;
731
732 for (i = 0; i < adapter->num_queues; i++) {
733 rc = ena_setup_tx_resources(adapter, i);
749 if (!rc)
734 if (rc == 0)
750 continue;
751
752 device_printf(adapter->pdev,
753 "Allocation for Tx Queue %u failed\n", i);
754 goto err_setup_tx;
755 }
756
757 return (0);

--- 42 unchanged lines hidden (view full) ---

800 size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
801
802 /*
803 * Alloc extra element so in rx path
804 * we can always prefetch rx_info + 1
805 */
806 size += sizeof(struct ena_rx_buffer);
807
735 continue;
736
737 device_printf(adapter->pdev,
738 "Allocation for Tx Queue %u failed\n", i);
739 goto err_setup_tx;
740 }
741
742 return (0);

--- 42 unchanged lines hidden (view full) ---

785 size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
786
787 /*
788 * Alloc extra element so in rx path
789 * we can always prefetch rx_info + 1
790 */
791 size += sizeof(struct ena_rx_buffer);
792
808 rx_ring->rx_buffer_info = ENA_MEM_ALLOC(adapter->ena_dev->dmadev, size);
809 if (!rx_ring->rx_buffer_info)
810 return (ENOMEM);
793 rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
811
812 /* Reset RX statistics. */
813 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
814 sizeof(rx_ring->rx_stats));
815
816 rx_ring->next_to_clean = 0;
817 rx_ring->next_to_use = 0;
818

--- 24 unchanged lines hidden (view full) ---

843 return (0);
844
845err_rx_dma:
846 while (i--) {
847 bus_dmamap_destroy(adapter->rx_buf_tag,
848 rx_ring->rx_buffer_info[i].map);
849 }
850
794
795 /* Reset RX statistics. */
796 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
797 sizeof(rx_ring->rx_stats));
798
799 rx_ring->next_to_clean = 0;
800 rx_ring->next_to_use = 0;
801

--- 24 unchanged lines hidden (view full) ---

826 return (0);
827
828err_rx_dma:
829 while (i--) {
830 bus_dmamap_destroy(adapter->rx_buf_tag,
831 rx_ring->rx_buffer_info[i].map);
832 }
833
851 ENA_MEM_FREE(adapter->ena_dev->dmadev, rx_ring->rx_buffer_info);
834 free(rx_ring->rx_buffer_info, M_DEVBUF);
852 rx_ring->rx_buffer_info = NULL;
853 ena_trace(ENA_ALERT, "RX resource allocation fail");
854 return (ENOMEM);
855}
856
857/**
858 * ena_free_rx_resources - Free Rx Resources
859 * @adapter: network interface device structure

--- 17 unchanged lines hidden (view full) ---

877 bus_dmamap_destroy(adapter->rx_buf_tag,
878 rx_ring->rx_buffer_info[i].map);
879 }
880
881 /* free LRO resources, */
882 tcp_lro_free(&rx_ring->lro);
883
884 /* free allocated memory */
835 rx_ring->rx_buffer_info = NULL;
836 ena_trace(ENA_ALERT, "RX resource allocation fail");
837 return (ENOMEM);
838}
839
840/**
841 * ena_free_rx_resources - Free Rx Resources
842 * @adapter: network interface device structure

--- 17 unchanged lines hidden (view full) ---

860 bus_dmamap_destroy(adapter->rx_buf_tag,
861 rx_ring->rx_buffer_info[i].map);
862 }
863
864 /* free LRO resources, */
865 tcp_lro_free(&rx_ring->lro);
866
867 /* free allocated memory */
885 ENA_MEM_FREE(adapter->ena_dev->dmadev, rx_ring->rx_buffer_info);
868 free(rx_ring->rx_buffer_info, M_DEVBUF);
886 rx_ring->rx_buffer_info = NULL;
887
888 return;
889}
890
891/**
892 * ena_setup_all_rx_resources - allocate all queues Rx resources
893 * @adapter: network interface device structure
894 *
895 * Returns 0 on success, otherwise on failure.
896 **/
897static int
898ena_setup_all_rx_resources(struct ena_adapter *adapter)
899{
900 int i, rc = 0;
901
902 for (i = 0; i < adapter->num_queues; i++) {
903 rc = ena_setup_rx_resources(adapter, i);
869 rx_ring->rx_buffer_info = NULL;
870
871 return;
872}
873
874/**
875 * ena_setup_all_rx_resources - allocate all queues Rx resources
876 * @adapter: network interface device structure
877 *
878 * Returns 0 on success, otherwise on failure.
879 **/
880static int
881ena_setup_all_rx_resources(struct ena_adapter *adapter)
882{
883 int i, rc = 0;
884
885 for (i = 0; i < adapter->num_queues; i++) {
886 rc = ena_setup_rx_resources(adapter, i);
904 if (!rc)
887 if (rc == 0)
905 continue;
906
907 device_printf(adapter->pdev,
908 "Allocation for Rx Queue %u failed\n", i);
909 goto err_setup_rx;
910 }
911 return (0);
912

--- 31 unchanged lines hidden (view full) ---

944
945 /* if previous allocated frag is not used */
946 if (rx_info->mbuf != NULL)
947 return (0);
948
949 /* Get mbuf using UMA allocator */
950 rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUM16BYTES);
951
888 continue;
889
890 device_printf(adapter->pdev,
891 "Allocation for Rx Queue %u failed\n", i);
892 goto err_setup_rx;
893 }
894 return (0);
895

--- 31 unchanged lines hidden (view full) ---

927
928 /* if previous allocated frag is not used */
929 if (rx_info->mbuf != NULL)
930 return (0);
931
932 /* Get mbuf using UMA allocator */
933 rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUM16BYTES);
934
952 if (!rx_info->mbuf) {
935 if (rx_info->mbuf == NULL) {
953 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
954 return (ENOMEM);
955 }
956 /* Set mbuf length*/
957 rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = MJUM16BYTES;
958
959 /* Map packets for DMA */
960 ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,

--- 27 unchanged lines hidden (view full) ---

988 return (EFAULT);
989}
990
991static void
992ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
993 struct ena_rx_buffer *rx_info)
994{
995
936 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
937 return (ENOMEM);
938 }
939 /* Set mbuf length*/
940 rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = MJUM16BYTES;
941
942 /* Map packets for DMA */
943 ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,

--- 27 unchanged lines hidden (view full) ---

971 return (EFAULT);
972}
973
974static void
975ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
976 struct ena_rx_buffer *rx_info)
977{
978
996 if (!rx_info->mbuf)
979 if (rx_info->mbuf == NULL)
997 return;
998
999 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map);
1000 m_freem(rx_info->mbuf);
1001 rx_info->mbuf = NULL;
1002
1003 return;
1004}

--- 706 unchanged lines hidden (view full) ---

1711ena_enable_msix(struct ena_adapter *adapter)
1712{
1713 device_t dev = adapter->pdev;
1714 int i, msix_vecs, rc = 0;
1715
1716 /* Reserved the max msix vectors we might need */
1717 msix_vecs = ENA_MAX_MSIX_VEC(adapter->num_queues);
1718
980 return;
981
982 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map);
983 m_freem(rx_info->mbuf);
984 rx_info->mbuf = NULL;
985
986 return;
987}

--- 706 unchanged lines hidden (view full) ---

1694ena_enable_msix(struct ena_adapter *adapter)
1695{
1696 device_t dev = adapter->pdev;
1697 int i, msix_vecs, rc = 0;
1698
1699 /* Reserved the max msix vectors we might need */
1700 msix_vecs = ENA_MAX_MSIX_VEC(adapter->num_queues);
1701
1719 adapter->msix_entries = ENA_MEM_ALLOC(adapter->ena_dev->dmadev,
1720 msix_vecs * sizeof(struct msix_entry));
1721 if (!adapter->msix_entries) {
1722 device_printf(dev,
1723 "Failed to allocate msix_entries, vectors %d\n", msix_vecs);
1724 rc = ENOMEM;
1725 goto error;
1726 }
1702 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry),
1703 M_DEVBUF, M_WAITOK | M_ZERO);
1704
1727 device_printf(dev, "Allocated msix_entries, vectors (cnt: %d)\n",
1728 msix_vecs);
1729
1730 for (i = 0; i < msix_vecs; i++) {
1731 adapter->msix_entries[i].entry = i;
1732 /* Vectors must start from 1 */
1733 adapter->msix_entries[i].vector = i + 1;
1734 }
1735
1736 rc = pci_alloc_msix(dev, &msix_vecs);
1737 if (rc != 0) {
1738 device_printf(dev,
1739 "Failed to enable MSIX, vectors %d rc %d\n", msix_vecs, rc);
1705 device_printf(dev, "Allocated msix_entries, vectors (cnt: %d)\n",
1706 msix_vecs);
1707
1708 for (i = 0; i < msix_vecs; i++) {
1709 adapter->msix_entries[i].entry = i;
1710 /* Vectors must start from 1 */
1711 adapter->msix_entries[i].vector = i + 1;
1712 }
1713
1714 rc = pci_alloc_msix(dev, &msix_vecs);
1715 if (rc != 0) {
1716 device_printf(dev,
1717 "Failed to enable MSIX, vectors %d rc %d\n", msix_vecs, rc);
1740 ENA_MEM_FREE(adapter->ena_dev->dmadev, adapter->msix_entries);
1718 free(adapter->msix_entries, M_DEVBUF);
1741 adapter->msix_entries = NULL;
1742 rc = ENOSPC;
1743 goto error;
1744 }
1745
1746 adapter->msix_vecs = msix_vecs;
1747 adapter->msix_enabled = true;
1748

--- 271 unchanged lines hidden (view full) ---

2020
2021static void
2022ena_disable_msix(struct ena_adapter *adapter)
2023{
2024
2025 pci_release_msi(adapter->pdev);
2026
2027 adapter->msix_vecs = 0;
1719 adapter->msix_entries = NULL;
1720 rc = ENOSPC;
1721 goto error;
1722 }
1723
1724 adapter->msix_vecs = msix_vecs;
1725 adapter->msix_enabled = true;
1726

--- 271 unchanged lines hidden (view full) ---

1998
1999static void
2000ena_disable_msix(struct ena_adapter *adapter)
2001{
2002
2003 pci_release_msi(adapter->pdev);
2004
2005 adapter->msix_vecs = 0;
2028 ENA_MEM_FREE(adapter->ena_dev->dmadev, adapter->msix_entries);
2006 free(adapter->msix_entries, M_DEVBUF);
2029 adapter->msix_entries = NULL;
2030}
2031
2032static void
2033ena_unmask_all_io_irqs(struct ena_adapter *adapter)
2034{
2035 struct ena_com_io_cq* io_cq;
2036 struct ena_eth_io_intr_reg intr_reg;

--- 1277 unchanged lines hidden (view full) ---

3314 ENA_REGS_RESET_MISS_TX_CMPL;
3315 adapter->trigger_reset = true;
3316 return;
3317 }
3318 }
3319 }
3320
3321 budget--;
2007 adapter->msix_entries = NULL;
2008}
2009
2010static void
2011ena_unmask_all_io_irqs(struct ena_adapter *adapter)
2012{
2013 struct ena_com_io_cq* io_cq;
2014 struct ena_eth_io_intr_reg intr_reg;

--- 1277 unchanged lines hidden (view full) ---

3292 ENA_REGS_RESET_MISS_TX_CMPL;
3293 adapter->trigger_reset = true;
3294 return;
3295 }
3296 }
3297 }
3298
3299 budget--;
3322 if (!budget) {
3300 if (budget == 0) {
3323 i++;
3324 break;
3325 }
3326 }
3327
3328 adapter->next_monitored_tx_qid = i % adapter->num_queues;
3329}
3330

--- 166 unchanged lines hidden (view full) ---

3497
3498 if (version_printed++ == 0)
3499 device_printf(pdev, "%s\n", ena_version);
3500
3501 rc = ena_allocate_pci_resources(adapter);
3502 if (rc) {
3503 device_printf(pdev, "PCI resource allocation failed!\n");
3504 ena_free_pci_resources(adapter);
3301 i++;
3302 break;
3303 }
3304 }
3305
3306 adapter->next_monitored_tx_qid = i % adapter->num_queues;
3307}
3308

--- 166 unchanged lines hidden (view full) ---

3475
3476 if (version_printed++ == 0)
3477 device_printf(pdev, "%s\n", ena_version);
3478
3479 rc = ena_allocate_pci_resources(adapter);
3480 if (rc) {
3481 device_printf(pdev, "PCI resource allocation failed!\n");
3482 ena_free_pci_resources(adapter);
3505 goto err_pci_res;
3483 return (rc);
3506 }
3507
3508 /* Allocate memory for ena_dev structure */
3484 }
3485
3486 /* Allocate memory for ena_dev structure */
3509 ena_dev = ENA_MEM_ALLOC(pdev, sizeof(struct ena_com_dev));
3510 if (!ena_dev) {
3511 device_printf(pdev, "allocating ena_dev failed\n");
3512 rc = ENOMEM;
3513 goto err_select_region;
3514 }
3487 ena_dev = malloc(sizeof(struct ena_com_dev), M_DEVBUF,
3488 M_WAITOK | M_ZERO);
3515
3516 adapter->ena_dev = ena_dev;
3517 ena_dev->dmadev = pdev;
3518 ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF,
3519 M_WAITOK | M_ZERO);
3520
3521 /* Store register resources */
3522 ((struct ena_bus*)(ena_dev->bus))->reg_bar_t =
3523 rman_get_bustag(adapter->registers);
3524 ((struct ena_bus*)(ena_dev->bus))->reg_bar_h =
3525 rman_get_bushandle(adapter->registers);
3526
3527 if (((struct ena_bus*)(ena_dev->bus))->reg_bar_h == 0) {
3528 device_printf(pdev, "failed to pmap registers bar\n");
3529 rc = ENXIO;
3489
3490 adapter->ena_dev = ena_dev;
3491 ena_dev->dmadev = pdev;
3492 ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF,
3493 M_WAITOK | M_ZERO);
3494
3495 /* Store register resources */
3496 ((struct ena_bus*)(ena_dev->bus))->reg_bar_t =
3497 rman_get_bustag(adapter->registers);
3498 ((struct ena_bus*)(ena_dev->bus))->reg_bar_h =
3499 rman_get_bushandle(adapter->registers);
3500
3501 if (((struct ena_bus*)(ena_dev->bus))->reg_bar_h == 0) {
3502 device_printf(pdev, "failed to pmap registers bar\n");
3503 rc = ENXIO;
3530 goto err_dev_free;
3504 goto err_bus_free;
3531 }
3532
3533 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3534
3535 /* Device initialization */
3536 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active);
3537 if (rc) {
3538 device_printf(pdev, "ENA device init failed! (err: %d)\n", rc);

--- 35 unchanged lines hidden (view full) ---

3574 adapter->rx_ring_size = queue_size;
3575
3576 adapter->max_tx_sgl_size = tx_sgl_size;
3577 adapter->max_rx_sgl_size = rx_sgl_size;
3578
3579 /* set up dma tags for rx and tx buffers */
3580 rc = ena_setup_tx_dma_tag(adapter);
3581 if (rc)
3505 }
3506
3507 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3508
3509 /* Device initialization */
3510 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active);
3511 if (rc) {
3512 device_printf(pdev, "ENA device init failed! (err: %d)\n", rc);

--- 35 unchanged lines hidden (view full) ---

3548 adapter->rx_ring_size = queue_size;
3549
3550 adapter->max_tx_sgl_size = tx_sgl_size;
3551 adapter->max_rx_sgl_size = rx_sgl_size;
3552
3553 /* set up dma tags for rx and tx buffers */
3554 rc = ena_setup_tx_dma_tag(adapter);
3555 if (rc)
3582 goto dma_tx_err;
3556 goto err_com_free;
3583
3584 rc = ena_setup_rx_dma_tag(adapter);
3585 if (rc)
3557
3558 rc = ena_setup_rx_dma_tag(adapter);
3559 if (rc)
3586 goto dma_rx_err;
3560 goto err_tx_tag_free;
3587
3588 /* initialize rings basic information */
3589 device_printf(pdev, "initalize %d io queues\n", io_queue_num);
3561
3562 /* initialize rings basic information */
3563 device_printf(pdev, "initalize %d io queues\n", io_queue_num);
3590 rc = ena_init_io_rings(adapter);
3591 if (rc) {
3592 device_printf(pdev,"Error with initialization of IO rings\n");
3593 goto err_io_init;
3594 }
3564 ena_init_io_rings(adapter);
3595
3596 /* setup network interface */
3597 rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx);
3598 if (rc) {
3599 device_printf(pdev,"Error with network interface setup\n");
3565
3566 /* setup network interface */
3567 rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx);
3568 if (rc) {
3569 device_printf(pdev,"Error with network interface setup\n");
3600 goto err_com_free;
3570 goto err_io_free;
3601 }
3602
3603 rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num);
3604 if (rc) {
3605 device_printf(pdev,
3606 "Failed to enable and set the admin interrupts\n");
3607 goto err_ifp_free;
3608 }
3609
3610 /* Initialize reset task queue */
3611 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter);
3612 adapter->reset_tq = taskqueue_create("ena_reset_enqueue",
3613 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq);
3571 }
3572
3573 rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num);
3574 if (rc) {
3575 device_printf(pdev,
3576 "Failed to enable and set the admin interrupts\n");
3577 goto err_ifp_free;
3578 }
3579
3580 /* Initialize reset task queue */
3581 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter);
3582 adapter->reset_tq = taskqueue_create("ena_reset_enqueue",
3583 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq);
3614 if (adapter->reset_tq == NULL) {
3615 device_printf(adapter->pdev,
3616 "Unable to create reset task queue\n");
3617 goto err_reset_tq;
3618 }
3619 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET,
3620 "%s rstq", device_get_nameunit(adapter->pdev));
3621
3622 /* Initialize statistics */
3623 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats,
3624 sizeof(struct ena_stats_dev));
3625 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats,
3626 sizeof(struct ena_hw_stats));
3627 ena_sysctl_add_nodes(adapter);
3628
3629 /* Tell the stack that the interface is not active */
3630 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
3631
3632 adapter->running = true;
3633 return (0);
3634
3584 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET,
3585 "%s rstq", device_get_nameunit(adapter->pdev));
3586
3587 /* Initialize statistics */
3588 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats,
3589 sizeof(struct ena_stats_dev));
3590 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats,
3591 sizeof(struct ena_hw_stats));
3592 ena_sysctl_add_nodes(adapter);
3593
3594 /* Tell the stack that the interface is not active */
3595 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
3596
3597 adapter->running = true;
3598 return (0);
3599
3635err_reset_tq:
3636 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
3637 ena_free_mgmnt_irq(adapter);
3638 ena_disable_msix(adapter);
3639err_ifp_free:
3640 if_detach(adapter->ifp);
3641 if_free(adapter->ifp);
3600err_ifp_free:
3601 if_detach(adapter->ifp);
3602 if_free(adapter->ifp);
3642err_com_free:
3603err_io_free:
3643 ena_free_all_io_rings_resources(adapter);
3604 ena_free_all_io_rings_resources(adapter);
3644err_io_init:
3645 ena_free_rx_dma_tag(adapter);
3605 ena_free_rx_dma_tag(adapter);
3646dma_rx_err:
3606err_tx_tag_free:
3647 ena_free_tx_dma_tag(adapter);
3607 ena_free_tx_dma_tag(adapter);
3648dma_tx_err:
3608err_com_free:
3649 ena_com_admin_destroy(ena_dev);
3650 ena_com_delete_host_info(ena_dev);
3609 ena_com_admin_destroy(ena_dev);
3610 ena_com_delete_host_info(ena_dev);
3611 ena_com_mmio_reg_read_request_destroy(ena_dev);
3651err_bus_free:
3652 free(ena_dev->bus, M_DEVBUF);
3612err_bus_free:
3613 free(ena_dev->bus, M_DEVBUF);
3653err_dev_free:
3654 free(ena_dev, M_DEVBUF);
3614 free(ena_dev, M_DEVBUF);
3655err_select_region:
3656 ena_free_pci_resources(adapter);
3615 ena_free_pci_resources(adapter);
3657err_pci_res:
3616
3658 return (rc);
3659}
3660
3661/**
3662 * ena_detach - Device Removal Routine
3663 * @pdev: device information struct
3664 *
3665 * ena_detach is called by the device subsystem to alert the driver

--- 153 unchanged lines hidden --- 		3617 return (rc);
3618}
3619
3620/**
3621 * ena_detach - Device Removal Routine
3622 * @pdev: device information struct
3623 *
3624 * ena_detach is called by the device subsystem to alert the driver

--- 153 unchanged lines hidden ---