3   Copyright (c) 2001-2017, Intel Corporation
48 static const char ixgbe_driver_version[] = "5.0.1-k";
93     "Intel(R) X520-T 82599 LOM"),
105     "Intel(R) X520-1 82599EN (SFP+)"),
107     "Intel(R) X520-4 82599 (Quad SFP+)"),
109     "Intel(R) X520-Q1 82599 (QSFP+)"),
111     "Intel(R) X540-AT2"),
112 	PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1,  "Intel(R) X540-T1"),
113 	PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T,  "Intel(R) X550-T2"),
114 	PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, "Intel(R) X550-T1"),
120     "Intel(R) X552/X557-AT (10GBASE-T)"),
122     "Intel(R) X552 (1000BASE-T)"),
138     "Intel(R) X553/X557-AT (10GBASE-T)"),
144     "Intel(R) X540-T2 (Bypass)"),
388  * MSI-X should be the default for best performance,
394     "Enable MSI-X interrupts");
415 /* Receive-Side Scaling */
419     "Enable Receive-Side Scaling (RSS)");
485 	if_softc_ctx_t scctx = sc->shared;  in ixgbe_if_tx_queues_alloc()
489 	MPASS(sc->num_tx_queues > 0);  in ixgbe_if_tx_queues_alloc()
490 	MPASS(sc->num_tx_queues == ntxqsets);  in ixgbe_if_tx_queues_alloc()
494 	sc->tx_queues =  in ixgbe_if_tx_queues_alloc()
497 	if (!sc->tx_queues) {  in ixgbe_if_tx_queues_alloc()
503 	for (i = 0, que = sc->tx_queues; i < ntxqsets; i++, que++) {  in ixgbe_if_tx_queues_alloc()
504 		struct tx_ring *txr = &que->txr;  in ixgbe_if_tx_queues_alloc()
506 		/* In case SR-IOV is enabled, align the index properly */  in ixgbe_if_tx_queues_alloc()
507 		txr->me = ixgbe_vf_que_index(sc->iov_mode, sc->pool, i);  in ixgbe_if_tx_queues_alloc()
509 		txr->sc = que->sc = sc;  in ixgbe_if_tx_queues_alloc()
512 		txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) *  in ixgbe_if_tx_queues_alloc()
513 		    scctx->isc_ntxd[0], M_IXGBE, M_NOWAIT | M_ZERO);  in ixgbe_if_tx_queues_alloc()
514 		if (txr->tx_rsq == NULL) {  in ixgbe_if_tx_queues_alloc()
518 		for (j = 0; j < scctx->isc_ntxd[0]; j++)  in ixgbe_if_tx_queues_alloc()
519 			txr->tx_rsq[j] = QIDX_INVALID;  in ixgbe_if_tx_queues_alloc()
521 		txr->tail = IXGBE_TDT(txr->me);  in ixgbe_if_tx_queues_alloc()
522 		txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i];  in ixgbe_if_tx_queues_alloc()
523 		txr->tx_paddr = paddrs[i];  in ixgbe_if_tx_queues_alloc()
525 		txr->bytes = 0;  in ixgbe_if_tx_queues_alloc()
526 		txr->total_packets = 0;  in ixgbe_if_tx_queues_alloc()
529 		if (sc->feat_en & IXGBE_FEATURE_FDIR)  in ixgbe_if_tx_queues_alloc()
530 			txr->atr_sample = atr_sample_rate;  in ixgbe_if_tx_queues_alloc()
535 	    sc->num_tx_queues);  in ixgbe_if_tx_queues_alloc()
556 	MPASS(sc->num_rx_queues > 0);  in ixgbe_if_rx_queues_alloc()
557 	MPASS(sc->num_rx_queues == nrxqsets);  in ixgbe_if_rx_queues_alloc()
561 	sc->rx_queues =  in ixgbe_if_rx_queues_alloc()
564 	if (!sc->rx_queues) {  in ixgbe_if_rx_queues_alloc()
570 	for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) {  in ixgbe_if_rx_queues_alloc()
571 		struct rx_ring *rxr = &que->rxr;  in ixgbe_if_rx_queues_alloc()
573 		/* In case SR-IOV is enabled, align the index properly */  in ixgbe_if_rx_queues_alloc()
574 		rxr->me = ixgbe_vf_que_index(sc->iov_mode, sc->pool, i);  in ixgbe_if_rx_queues_alloc()
576 		rxr->sc = que->sc = sc;  in ixgbe_if_rx_queues_alloc()
579 		rxr->tail = IXGBE_RDT(rxr->me);  in ixgbe_if_rx_queues_alloc()
580 		rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i];  in ixgbe_if_rx_queues_alloc()
581 		rxr->rx_paddr = paddrs[i];  in ixgbe_if_rx_queues_alloc()
582 		rxr->bytes = 0;  in ixgbe_if_rx_queues_alloc()
583 		rxr->que = que;  in ixgbe_if_rx_queues_alloc()
587 	    sc->num_rx_queues);  in ixgbe_if_rx_queues_alloc()
599 	struct ix_tx_queue *tx_que = sc->tx_queues;  in ixgbe_if_queues_free()
600 	struct ix_rx_queue *rx_que = sc->rx_queues;  in ixgbe_if_queues_free()
604 		for (i = 0; i < sc->num_tx_queues; i++, tx_que++) {  in ixgbe_if_queues_free()
605 			struct tx_ring *txr = &tx_que->txr;  in ixgbe_if_queues_free()
606 			if (txr->tx_rsq == NULL)  in ixgbe_if_queues_free()
609 			free(txr->tx_rsq, M_IXGBE);  in ixgbe_if_queues_free()
610 			txr->tx_rsq = NULL;  in ixgbe_if_queues_free()
613 		free(sc->tx_queues, M_IXGBE);  in ixgbe_if_queues_free()
614 		sc->tx_queues = NULL;  in ixgbe_if_queues_free()
617 		free(sc->rx_queues, M_IXGBE);  in ixgbe_if_queues_free()
618 		sc->rx_queues = NULL;  in ixgbe_if_queues_free()
628 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_initialize_rss_mapping()
634 	if (sc->feat_en & IXGBE_FEATURE_RSS) {  in ixgbe_initialize_rss_mapping()
645 	switch (sc->hw.mac.type) {  in ixgbe_initialize_rss_mapping()
661 		if (j == sc->num_rx_queues)  in ixgbe_initialize_rss_mapping()
664 		if (sc->feat_en & IXGBE_FEATURE_RSS) {  in ixgbe_initialize_rss_mapping()
671 			queue_id = queue_id % sc->num_rx_queues;  in ixgbe_initialize_rss_mapping()
686 				    IXGBE_ERETA((i >> 2) - 32), reta);  in ixgbe_initialize_rss_mapping()
696 	if (sc->feat_en & IXGBE_FEATURE_RSS)  in ixgbe_initialize_rss_mapping()
700 		 * Disable UDP - IP fragments aren't currently being handled  in ixgbe_initialize_rss_mapping()
701 		 * and so we end up with a mix of 2-tuple and 4-tuple  in ixgbe_initialize_rss_mapping()
731 	mrqc |= ixgbe_get_mrqc(sc->iov_mode);  in ixgbe_initialize_rss_mapping()
736  * ixgbe_initialize_receive_units - Setup receive registers and features.
738 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
744 	if_softc_ctx_t scctx = sc->shared;  in ixgbe_initialize_receive_units()
745 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_initialize_receive_units()
761 	if (sc->hw.mac.type == ixgbe_mac_82598EB) {  in ixgbe_initialize_receive_units()
775 	bufsz = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >>  in ixgbe_initialize_receive_units()
778 	/* Setup the Base and Length of the Rx Descriptor Ring */  in ixgbe_initialize_receive_units()
779 	for (i = 0, que = sc->rx_queues; i < sc->num_rx_queues; i++, que++) {  in ixgbe_initialize_receive_units()
780 		struct rx_ring *rxr = &que->rxr;  in ixgbe_initialize_receive_units()
781 		u64 rdba = rxr->rx_paddr;  in ixgbe_initialize_receive_units()
783 		j = rxr->me;  in ixgbe_initialize_receive_units()
785 		/* Setup the Base and Length of the Rx Descriptor Ring */  in ixgbe_initialize_receive_units()
790 		     scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc));  in ixgbe_initialize_receive_units()
805 		if (sc->num_rx_queues > 1 &&  in ixgbe_initialize_receive_units()
806 		    sc->hw.fc.requested_mode == ixgbe_fc_none) {  in ixgbe_initialize_receive_units()
819 		rxr->tail =  IXGBE_RDT(rxr->me);  in ixgbe_initialize_receive_units()
822 	if (sc->hw.mac.type != ixgbe_mac_82598EB) {  in ixgbe_initialize_receive_units()
834 	if (sc->feat_en & IXGBE_FEATURE_RSS) {  in ixgbe_initialize_receive_units()
851  * ixgbe_initialize_transmit_units - Enable transmit units.
857 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_initialize_transmit_units()
858 	if_softc_ctx_t scctx = sc->shared;  in ixgbe_initialize_transmit_units()
862 	/* Setup the Base and Length of the Tx Descriptor Ring */  in ixgbe_initialize_transmit_units()
863 	for (i = 0, que = sc->tx_queues; i < sc->num_tx_queues;  in ixgbe_initialize_transmit_units()
865 		struct tx_ring	   *txr = &que->txr;  in ixgbe_initialize_transmit_units()
866 		u64 tdba = txr->tx_paddr;  in ixgbe_initialize_transmit_units()
868 		int j = txr->me;  in ixgbe_initialize_transmit_units()
874 		    scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc));  in ixgbe_initialize_transmit_units()
881 		txr->tail = IXGBE_TDT(txr->me);  in ixgbe_initialize_transmit_units()
883 		txr->tx_rs_cidx = txr->tx_rs_pidx;  in ixgbe_initialize_transmit_units()
884 		txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;  in ixgbe_initialize_transmit_units()
885 		for (int k = 0; k < scctx->isc_ntxd[0]; k++)  in ixgbe_initialize_transmit_units()
886 			txr->tx_rsq[k] = QIDX_INVALID;  in ixgbe_initialize_transmit_units()
894 		switch (hw->mac.type) {  in ixgbe_initialize_transmit_units()
904 		switch (hw->mac.type) {  in ixgbe_initialize_transmit_units()
916 	if (hw->mac.type != ixgbe_mac_82598EB) {  in ixgbe_initialize_transmit_units()
927 		    ixgbe_get_mtqc(sc->iov_mode));  in ixgbe_initialize_transmit_units()
937 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_check_fw_api_version()
938 	if (hw->api_maj_ver > IXGBE_FW_API_VER_MAJOR) {  in ixgbe_check_fw_api_version()
939 		device_printf(sc->dev,  in ixgbe_check_fw_api_version()
944 	} else if (hw->api_maj_ver == IXGBE_FW_API_VER_MAJOR &&  in ixgbe_check_fw_api_version()
945 		   hw->api_min_ver > (IXGBE_FW_API_VER_MINOR + 2)) {  in ixgbe_check_fw_api_version()
946 		device_printf(sc->dev,  in ixgbe_check_fw_api_version()
950 	} else if (hw->api_maj_ver < IXGBE_FW_API_VER_MAJOR ||  in ixgbe_check_fw_api_version()
951 		   hw->api_min_ver < IXGBE_FW_API_VER_MINOR - 2) {  in ixgbe_check_fw_api_version()
952 		device_printf(sc->dev,  in ixgbe_check_fw_api_version()
970  * ixgbe_if_attach_pre - Device initialization routine, part 1
994 	sc->hw.back = sc;  in ixgbe_if_attach_pre()
995 	sc->ctx = ctx;  in ixgbe_if_attach_pre()
996 	sc->dev = dev;  in ixgbe_if_attach_pre()
997 	scctx = sc->shared = iflib_get_softc_ctx(ctx);  in ixgbe_if_attach_pre()
998 	sc->media = iflib_get_media(ctx);  in ixgbe_if_attach_pre()
999 	hw = &sc->hw;  in ixgbe_if_attach_pre()
1002 	hw->vendor_id = pci_get_vendor(dev);  in ixgbe_if_attach_pre()
1003 	hw->device_id = pci_get_device(dev);  in ixgbe_if_attach_pre()
1004 	hw->revision_id = pci_get_revid(dev);  in ixgbe_if_attach_pre()
1005 	hw->subsystem_vendor_id = pci_get_subvendor(dev);  in ixgbe_if_attach_pre()
1006 	hw->subsystem_device_id = pci_get_subdevice(dev);  in ixgbe_if_attach_pre()
1008 	/* Do base PCI setup - map BAR0 */  in ixgbe_if_attach_pre()
1028 	if (hw->mac.type == ixgbe_mac_E610)  in ixgbe_if_attach_pre()
1031 	sc->do_debug_dump = false;  in ixgbe_if_attach_pre()
1033 	if (hw->mac.ops.fw_recovery_mode &&  in ixgbe_if_attach_pre()
1034 	    hw->mac.ops.fw_recovery_mode(hw)) {  in ixgbe_if_attach_pre()
1044 	/* 82598 Does not support SR-IOV, initialize everything else */  in ixgbe_if_attach_pre()
1045 	if (hw->mac.type >= ixgbe_mac_82599_vf) {  in ixgbe_if_attach_pre()
1046 		for (i = 0; i < sc->num_vfs; i++)  in ixgbe_if_attach_pre()
1047 			hw->mbx.ops[i].init_params(hw);  in ixgbe_if_attach_pre()
1050 	hw->allow_unsupported_sfp = allow_unsupported_sfp;  in ixgbe_if_attach_pre()
1052 	if (hw->mac.type != ixgbe_mac_82598EB)  in ixgbe_if_attach_pre()
1053 		hw->phy.smart_speed = ixgbe_smart_speed;  in ixgbe_if_attach_pre()
1061 	if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL) {  in ixgbe_if_attach_pre()
1070 	hw->fc.requested_mode = ixgbe_flow_control;  in ixgbe_if_attach_pre()
1072 	hw->phy.reset_if_overtemp = true;  in ixgbe_if_attach_pre()
1074 	hw->phy.reset_if_overtemp = false;  in ixgbe_if_attach_pre()
1081 		sc->sfp_probe = true;  in ixgbe_if_attach_pre()
1094 	if (ixgbe_validate_eeprom_checksum(&sc->hw, NULL) < 0) {  in ixgbe_if_attach_pre()
1104 		    "This device is a pre-production adapter/LOM.  Please be"  in ixgbe_if_attach_pre()
1122 	if (hw->mac.type == ixgbe_mac_E610 && ixgbe_check_fw_api_version(sc)) {  in ixgbe_if_attach_pre()
1129 	iflib_set_mac(ctx, hw->mac.addr);  in ixgbe_if_attach_pre()
1130 	switch (sc->hw.mac.type) {  in ixgbe_if_attach_pre()
1134 		scctx->isc_rss_table_size = 512;  in ixgbe_if_attach_pre()
1135 		scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 64;  in ixgbe_if_attach_pre()
1138 		scctx->isc_rss_table_size = 128;  in ixgbe_if_attach_pre()
1139 		scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 16;  in ixgbe_if_attach_pre()
1145 	scctx->isc_txqsizes[0] =  in ixgbe_if_attach_pre()
1146 	    roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) +  in ixgbe_if_attach_pre()
1148 	scctx->isc_rxqsizes[0] =  in ixgbe_if_attach_pre()
1149 	    roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc),  in ixgbe_if_attach_pre()
1153 	scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |  in ixgbe_if_attach_pre()
1155 	if (sc->hw.mac.type == ixgbe_mac_82598EB) {  in ixgbe_if_attach_pre()
1156 		scctx->isc_tx_nsegments = IXGBE_82598_SCATTER;  in ixgbe_if_attach_pre()
1158 		scctx->isc_tx_csum_flags |= CSUM_SCTP |CSUM_IP6_SCTP;  in ixgbe_if_attach_pre()
1159 		scctx->isc_tx_nsegments = IXGBE_82599_SCATTER;  in ixgbe_if_attach_pre()
1162 	scctx->isc_msix_bar = pci_msix_table_bar(dev);  in ixgbe_if_attach_pre()
1164 	scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments;  in ixgbe_if_attach_pre()
1165 	scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE;  in ixgbe_if_attach_pre()
1166 	scctx->isc_tx_tso_segsize_max = PAGE_SIZE;  in ixgbe_if_attach_pre()
1168 	scctx->isc_txrx = &ixgbe_txrx;  in ixgbe_if_attach_pre()
1170 	scctx->isc_capabilities = scctx->isc_capenable = IXGBE_CAPS;  in ixgbe_if_attach_pre()
1175 	ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);  in ixgbe_if_attach_pre()
1177 	IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);  in ixgbe_if_attach_pre()
1180 	if (hw->mac.type == ixgbe_mac_E610)  in ixgbe_if_attach_pre()
1187  * ixgbe_if_attach_post - Device initialization routine, part 2
1205 	hw = &sc->hw;  in ixgbe_if_attach_post()
1207 	if (sc->intr_type == IFLIB_INTR_LEGACY &&  in ixgbe_if_attach_post()
1208 		(sc->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) == 0) {  in ixgbe_if_attach_post()
1215 	sc->mta = malloc(sizeof(*sc->mta) * MAX_NUM_MULTICAST_ADDRESSES,  in ixgbe_if_attach_post()
1217 	if (sc->mta == NULL) {  in ixgbe_if_attach_post()
1260 	sc->dmac = 0;  in ixgbe_if_attach_post()
1262 	sc->advertise = ixgbe_get_default_advertise(sc);  in ixgbe_if_attach_post()
1264 	if (sc->feat_cap & IXGBE_FEATURE_SRIOV)  in ixgbe_if_attach_post()
1271 	if (sc->feat_en & IXGBE_FEATURE_RECOVERY_MODE) {  in ixgbe_if_attach_post()
1272 		sc->recovery_mode = 0;  in ixgbe_if_attach_post()
1275 		callout_init(&sc->fw_mode_timer, true);  in ixgbe_if_attach_post()
1278 		callout_reset(&sc->fw_mode_timer, hz, ixgbe_fw_mode_timer, sc);  in ixgbe_if_attach_post()
1292  *   Sets each port's hw->wol_enabled value depending
1298 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_check_wol_support()
1302 	sc->wol_support = hw->wol_enabled = 0;  in ixgbe_check_wol_support()
1306 	     hw->bus.func == 0))  in ixgbe_check_wol_support()
1307 		sc->wol_support = hw->wol_enabled = 1;  in ixgbe_check_wol_support()
1310 	sc->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC);  in ixgbe_check_wol_support()
1330 	sc->max_frame_size = if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN;  in ixgbe_setup_interface()
1332 	sc->phy_layer = ixgbe_get_supported_physical_layer(&sc->hw);  in ixgbe_setup_interface()
1337 	ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO);  in ixgbe_setup_interface()
1353 		return (sc->ipackets);  in ixgbe_if_get_counter()
1355 		return (sc->opackets);  in ixgbe_if_get_counter()
1357 		return (sc->ibytes);  in ixgbe_if_get_counter()
1359 		return (sc->obytes);  in ixgbe_if_get_counter()
1361 		return (sc->imcasts);  in ixgbe_if_get_counter()
1363 		return (sc->omcasts);  in ixgbe_if_get_counter()
1367 		return (sc->iqdrops);  in ixgbe_if_get_counter()
1369 		return (sc->ierrors);  in ixgbe_if_get_counter()
1382 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_if_i2c_req()
1385 	if (hw->phy.ops.read_i2c_byte == NULL)  in ixgbe_if_i2c_req()
1387 	for (i = 0; i < req->len; i++)  in ixgbe_if_i2c_req()
1388 		hw->phy.ops.read_i2c_byte(hw, req->offset + i,  in ixgbe_if_i2c_req()
1389 		    req->dev_addr, &req->data[i]);  in ixgbe_if_i2c_req()
1393 /* ixgbe_if_needs_restart - Tell iflib when the driver needs to be
1413  * ixgbe_if_priv_ioctl - Ioctl handler for driver
1424 	device_t dev = sc->dev;  in ixgbe_if_priv_ioctl()
1453 	switch (ifd->ifd_cmd) {  in ixgbe_if_priv_ioctl()
1463 		    (int)ifd->ifd_cmd);  in ixgbe_if_priv_ioctl()
1480 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_nvm_access_ioctl()
1481 	size_t ifd_len = ifd->ifd_len;  in ixgbe_nvm_access_ioctl()
1483 	device_t dev = sc->dev;  in ixgbe_nvm_access_ioctl()
1490 	 * without verifying privileges. To prevent non-privileged threads from  in ixgbe_nvm_access_ioctl()
1505 	if (ifd->ifd_data == NULL) {  in ixgbe_nvm_access_ioctl()
1518 	error = copyin(ifd->ifd_data, nvm_buffer, ifd_len);  in ixgbe_nvm_access_ioctl()
1541 	error = copyout(nvm_buffer, ifd->ifd_data, ifd_len);  in ixgbe_nvm_access_ioctl()
1563 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_debug_dump_ioctl()
1564 	size_t ifd_len = ifd->ifd_len;  in ixgbe_debug_dump_ioctl()
1565 	device_t dev = sc->dev;  in ixgbe_debug_dump_ioctl()
1568 	if (!(sc->feat_en & IXGBE_FEATURE_DBG_DUMP))  in ixgbe_debug_dump_ioctl()
1580 	 * without verifying privileges. To prevent non-privileged threads from  in ixgbe_debug_dump_ioctl()
1595 	if (ifd->ifd_data == NULL) {  in ixgbe_debug_dump_ioctl()
1604 		error = -ENOMEM;  in ixgbe_debug_dump_ioctl()
1608 	error = copyin(ifd->ifd_data, dd_cmd, ifd_len);  in ixgbe_debug_dump_ioctl()
1616 	if (dd_cmd->data_size == 0) {  in ixgbe_debug_dump_ioctl()
1624 	memset(dd_cmd->data, 0, ifd_len - sizeof(*dd_cmd));  in ixgbe_debug_dump_ioctl()
1626 	error = ixgbe_aci_get_internal_data(hw, dd_cmd->cluster_id,  in ixgbe_debug_dump_ioctl()
1627 	    dd_cmd->table_id, dd_cmd->offset, dd_cmd->data, dd_cmd->data_size,  in ixgbe_debug_dump_ioctl()
1636 	dd_cmd->cluster_id = ret_next_cluster;  in ixgbe_debug_dump_ioctl()
1637 	dd_cmd->table_id = ret_next_table;  in ixgbe_debug_dump_ioctl()
1638 	dd_cmd->offset = ret_next_index;  in ixgbe_debug_dump_ioctl()
1639 	dd_cmd->data_size = ret_buf_size;  in ixgbe_debug_dump_ioctl()
1641 	error = copyout(dd_cmd, ifd->ifd_data, ifd->ifd_len);  in ixgbe_debug_dump_ioctl()
1661 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_add_media_types()
1665 	layer = sc->phy_layer = ixgbe_get_supported_physical_layer(hw);  in ixgbe_add_media_types()
1669 		ifmedia_add(sc->media, IFM_ETHER | IFM_10G_T, 0, NULL);  in ixgbe_add_media_types()
1671 		ifmedia_add(sc->media, IFM_ETHER | IFM_5000_T, 0, NULL);  in ixgbe_add_media_types()
1673 		ifmedia_add(sc->media, IFM_ETHER | IFM_2500_T, 0, NULL);  in ixgbe_add_media_types()
1675 		ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);  in ixgbe_add_media_types()
1677 		ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX, 0, NULL);  in ixgbe_add_media_types()
1679 		ifmedia_add(sc->media, IFM_ETHER | IFM_10_T, 0, NULL);  in ixgbe_add_media_types()
1681 	if (hw->mac.type == ixgbe_mac_X550) {  in ixgbe_add_media_types()
1682 		ifmedia_add(sc->media, IFM_ETHER | IFM_2500_T, 0, NULL);  in ixgbe_add_media_types()
1683 		ifmedia_add(sc->media, IFM_ETHER | IFM_5000_T, 0, NULL);  in ixgbe_add_media_types()
1688 		ifmedia_add(sc->media, IFM_ETHER | IFM_10G_TWINAX, 0,  in ixgbe_add_media_types()
1690 		ifmedia_add(sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL);  in ixgbe_add_media_types()
1694 		ifmedia_add(sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL);  in ixgbe_add_media_types()
1695 		if (hw->phy.multispeed_fiber)  in ixgbe_add_media_types()
1696 			ifmedia_add(sc->media, IFM_ETHER | IFM_1000_LX, 0,  in ixgbe_add_media_types()
1700 		ifmedia_add(sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL);  in ixgbe_add_media_types()
1701 		if (hw->phy.multispeed_fiber)  in ixgbe_add_media_types()
1702 			ifmedia_add(sc->media, IFM_ETHER | IFM_1000_SX, 0,  in ixgbe_add_media_types()
1705 		ifmedia_add(sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL);  in ixgbe_add_media_types()
1707 		ifmedia_add(sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);  in ixgbe_add_media_types()
1711 		ifmedia_add(sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL);  in ixgbe_add_media_types()
1713 		ifmedia_add( sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);  in ixgbe_add_media_types()
1715 		ifmedia_add(sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL);  in ixgbe_add_media_types()
1717 		ifmedia_add(sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL);  in ixgbe_add_media_types()
1722 		ifmedia_add(sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL);  in ixgbe_add_media_types()
1727 		ifmedia_add(sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);  in ixgbe_add_media_types()
1730 		device_printf(dev, "Media supported: 1000baseKX\n");  in ixgbe_add_media_types()
1731 		device_printf(dev, "1000baseKX mapped to 1000baseCX\n");  in ixgbe_add_media_types()
1732 		ifmedia_add(sc->media, IFM_ETHER | IFM_1000_CX, 0, NULL);  in ixgbe_add_media_types()
1737 		ifmedia_add(sc->media, IFM_ETHER | IFM_2500_SX, 0, NULL);  in ixgbe_add_media_types()
1741 		device_printf(dev, "Media supported: 1000baseBX\n");  in ixgbe_add_media_types()
1742 		ifmedia_add(sc->media, IFM_ETHER | IFM_1000_BX, 0, NULL);  in ixgbe_add_media_types()
1745 	if (hw->device_id == IXGBE_DEV_ID_82598AT) {  in ixgbe_add_media_types()
1746 		ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX,  in ixgbe_add_media_types()
1748 		ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);  in ixgbe_add_media_types()
1751 	ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);  in ixgbe_add_media_types()
1760 	switch (hw->mac.type) {  in ixgbe_is_sfp()
1762 		if (hw->phy.type == ixgbe_phy_nl)  in ixgbe_is_sfp()
1766 		switch (hw->mac.ops.get_media_type(hw)) {  in ixgbe_is_sfp()
1776 		if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber)  in ixgbe_is_sfp()
1791 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_config_link()
1798 		sc->task_requests |= IXGBE_REQUEST_TASK_MOD;  in ixgbe_config_link()
1801 		if (hw->mac.ops.check_link)  in ixgbe_config_link()
1802 			err = ixgbe_check_link(hw, &sc->link_speed,  in ixgbe_config_link()
1803 			    &sc->link_up, false);  in ixgbe_config_link()
1806 		autoneg = hw->phy.autoneg_advertised;  in ixgbe_config_link()
1807 		if ((!autoneg) && (hw->mac.ops.get_link_capabilities))  in ixgbe_config_link()
1808 			err = hw->mac.ops.get_link_capabilities(hw, &autoneg,  in ixgbe_config_link()
1813 		if (hw->mac.type == ixgbe_mac_X550 &&  in ixgbe_config_link()
1814 		    hw->phy.autoneg_advertised == 0) {  in ixgbe_config_link()
1825 			 * If hw->phy.autoneg_advertised does not  in ixgbe_config_link()
1833 			 * Otherwise (i.e. if hw->phy.autoneg_advertised  in ixgbe_config_link()
1842 		if (hw->mac.type == ixgbe_mac_E610) {  in ixgbe_config_link()
1843 			hw->phy.ops.init(hw);  in ixgbe_config_link()
1846 				device_printf(sc->dev,  in ixgbe_config_link()
1851 		if (hw->mac.ops.setup_link)  in ixgbe_config_link()
1852 			err = hw->mac.ops.setup_link(hw, autoneg,  in ixgbe_config_link()
1853 			    sc->link_up);  in ixgbe_config_link()
1858  * ixgbe_update_stats_counters - Update board statistics counters.
1863 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_update_stats_counters()
1864 	struct ixgbe_hw_stats *stats = &sc->stats.pf;  in ixgbe_update_stats_counters()
1869 	stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);  in ixgbe_update_stats_counters()
1870 	stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);  in ixgbe_update_stats_counters()
1871 	stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);  in ixgbe_update_stats_counters()
1872 	stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);  in ixgbe_update_stats_counters()
1873 	stats->mpc[0] += IXGBE_READ_REG(hw, IXGBE_MPC(0));  in ixgbe_update_stats_counters()
1876 		stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));  in ixgbe_update_stats_counters()
1877 		stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));  in ixgbe_update_stats_counters()
1878 		stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));  in ixgbe_update_stats_counters()
1880 	stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);  in ixgbe_update_stats_counters()
1881 	stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);  in ixgbe_update_stats_counters()
1882 	stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);  in ixgbe_update_stats_counters()
1885 	stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);  in ixgbe_update_stats_counters()
1886 	stats->gprc -= missed_rx;  in ixgbe_update_stats_counters()
1888 	if (hw->mac.type != ixgbe_mac_82598EB) {  in ixgbe_update_stats_counters()
1889 		stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +  in ixgbe_update_stats_counters()
1891 		stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +  in ixgbe_update_stats_counters()
1893 		stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) +  in ixgbe_update_stats_counters()
1895 		stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);  in ixgbe_update_stats_counters()
1897 		stats->lxoffrxc += lxoffrxc;  in ixgbe_update_stats_counters()
1899 		stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);  in ixgbe_update_stats_counters()
1901 		stats->lxoffrxc += lxoffrxc;  in ixgbe_update_stats_counters()
1903 		stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);  in ixgbe_update_stats_counters()
1904 		stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);  in ixgbe_update_stats_counters()
1905 		stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);  in ixgbe_update_stats_counters()
1913 		sc->shared->isc_pause_frames = 1;  in ixgbe_update_stats_counters()
1920 	stats->bprc += bprc;  in ixgbe_update_stats_counters()
1921 	stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);  in ixgbe_update_stats_counters()
1922 	if (hw->mac.type == ixgbe_mac_82598EB)  in ixgbe_update_stats_counters()
1923 		stats->mprc -= bprc;  in ixgbe_update_stats_counters()
1925 	stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);  in ixgbe_update_stats_counters()
1926 	stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);  in ixgbe_update_stats_counters()
1927 	stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);  in ixgbe_update_stats_counters()
1928 	stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);  in ixgbe_update_stats_counters()
1929 	stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);  in ixgbe_update_stats_counters()
1930 	stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);  in ixgbe_update_stats_counters()
1933 	stats->lxontxc += lxon;  in ixgbe_update_stats_counters()
1935 	stats->lxofftxc += lxoff;  in ixgbe_update_stats_counters()
1938 	stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);  in ixgbe_update_stats_counters()
1939 	stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);  in ixgbe_update_stats_counters()
1940 	stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);  in ixgbe_update_stats_counters()
1941 	stats->gptc -= total;  in ixgbe_update_stats_counters()
1942 	stats->mptc -= total;  in ixgbe_update_stats_counters()
1943 	stats->ptc64 -= total;  in ixgbe_update_stats_counters()
1944 	stats->gotc -= total * ETHER_MIN_LEN;  in ixgbe_update_stats_counters()
1946 	stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);  in ixgbe_update_stats_counters()
1947 	stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);  in ixgbe_update_stats_counters()
1948 	stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);  in ixgbe_update_stats_counters()
1949 	stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);  in ixgbe_update_stats_counters()
1950 	stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);  in ixgbe_update_stats_counters()
1951 	stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);  in ixgbe_update_stats_counters()
1952 	stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);  in ixgbe_update_stats_counters()
1953 	stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);  in ixgbe_update_stats_counters()
1954 	stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);  in ixgbe_update_stats_counters()
1955 	stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);  in ixgbe_update_stats_counters()
1956 	stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);  in ixgbe_update_stats_counters()
1957 	stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);  in ixgbe_update_stats_counters()
1958 	stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);  in ixgbe_update_stats_counters()
1959 	stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);  in ixgbe_update_stats_counters()
1960 	stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);  in ixgbe_update_stats_counters()
1961 	stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);  in ixgbe_update_stats_counters()
1962 	stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);  in ixgbe_update_stats_counters()
1963 	stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);  in ixgbe_update_stats_counters()
1965 	if (hw->mac.type != ixgbe_mac_82598EB) {  in ixgbe_update_stats_counters()
1966 		stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);  in ixgbe_update_stats_counters()
1967 		stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);  in ixgbe_update_stats_counters()
1968 		stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);  in ixgbe_update_stats_counters()
1969 		stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);  in ixgbe_update_stats_counters()
1970 		stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);  in ixgbe_update_stats_counters()
1974 	IXGBE_SET_IPACKETS(sc, stats->gprc);  in ixgbe_update_stats_counters()
1975 	IXGBE_SET_OPACKETS(sc, stats->gptc);  in ixgbe_update_stats_counters()
1976 	IXGBE_SET_IBYTES(sc, stats->gorc);  in ixgbe_update_stats_counters()
1977 	IXGBE_SET_OBYTES(sc, stats->gotc);  in ixgbe_update_stats_counters()
1978 	IXGBE_SET_IMCASTS(sc, stats->mprc);  in ixgbe_update_stats_counters()
1979 	IXGBE_SET_OMCASTS(sc, stats->mptc);  in ixgbe_update_stats_counters()
1985 	 * - CRC error count,  in ixgbe_update_stats_counters()
1986 	 * - illegal byte error count,  in ixgbe_update_stats_counters()
1987 	 * - missed packets count,  in ixgbe_update_stats_counters()
1988 	 * - length error count,  in ixgbe_update_stats_counters()
1989 	 * - undersized packets count,  in ixgbe_update_stats_counters()
1990 	 * - fragmented packets count,  in ixgbe_update_stats_counters()
1991 	 * - oversized packets count,  in ixgbe_update_stats_counters()
1992 	 * - jabber count.  in ixgbe_update_stats_counters()
1994 	IXGBE_SET_IERRORS(sc, stats->crcerrs + stats->illerrc +  in ixgbe_update_stats_counters()
1995 	    stats->mpc[0] + stats->rlec + stats->ruc + stats->rfc +  in ixgbe_update_stats_counters()
1996 	    stats->roc + stats->rjc);  in ixgbe_update_stats_counters()
2007 	device_t dev = iflib_get_dev(sc->ctx);  in ixgbe_add_hw_stats()
2013 	struct ixgbe_hw_stats *stats = &sc->stats.pf;  in ixgbe_add_hw_stats()
2023 	    CTLFLAG_RD, &sc->dropped_pkts, "Driver dropped packets");  in ixgbe_add_hw_stats()
2025 	    CTLFLAG_RD, &sc->watchdog_events, "Watchdog timeouts");  in ixgbe_add_hw_stats()
2027 	    CTLFLAG_RD, &sc->link_irq, "Link MSI-X IRQ Handled");  in ixgbe_add_hw_stats()
2029 	for (i = 0, tx_que = sc->tx_queues; i < sc->num_tx_queues;  in ixgbe_add_hw_stats()
2031 		struct tx_ring *txr = &tx_que->txr;  in ixgbe_add_hw_stats()
2046 		    CTLFLAG_RD, &txr->tso_tx, "TSO");  in ixgbe_add_hw_stats()
2048 		    CTLFLAG_RD, &txr->total_packets,  in ixgbe_add_hw_stats()
2052 	for (i = 0, rx_que = sc->rx_queues; i < sc->num_rx_queues;  in ixgbe_add_hw_stats()
2054 		struct rx_ring *rxr = &rx_que->rxr;  in ixgbe_add_hw_stats()
2062 		    &sc->rx_queues[i], 0,  in ixgbe_add_hw_stats()
2066 		    CTLFLAG_RD, &(sc->rx_queues[i].irqs),  in ixgbe_add_hw_stats()
2077 		    CTLFLAG_RD, &rxr->rx_packets, "Queue Packets Received");  in ixgbe_add_hw_stats()
2079 		    CTLFLAG_RD, &rxr->rx_bytes, "Queue Bytes Received");  in ixgbe_add_hw_stats()
2081 		    CTLFLAG_RD, &rxr->rx_copies, "Copied RX Frames");  in ixgbe_add_hw_stats()
2083 		    CTLFLAG_RD, &rxr->rx_discarded, "Discarded RX packets");  in ixgbe_add_hw_stats()
2092 	    CTLFLAG_RD, &sc->ierrors, IXGBE_SYSCTL_DESC_RX_ERRS);  in ixgbe_add_hw_stats()
2094 	    CTLFLAG_RD, &stats->crcerrs, "CRC Errors");  in ixgbe_add_hw_stats()
2096 	    CTLFLAG_RD, &stats->illerrc, "Illegal Byte Errors");  in ixgbe_add_hw_stats()
2098 	    CTLFLAG_RD, &stats->errbc, "Byte Errors");  in ixgbe_add_hw_stats()
2100 	    CTLFLAG_RD, &stats->mspdc, "MAC Short Packets Discarded");  in ixgbe_add_hw_stats()
2102 	    CTLFLAG_RD, &stats->mlfc, "MAC Local Faults");  in ixgbe_add_hw_stats()
2104 	    CTLFLAG_RD, &stats->mrfc, "MAC Remote Faults");  in ixgbe_add_hw_stats()
2106 	    CTLFLAG_RD, &stats->rlec, "Receive Length Errors");  in ixgbe_add_hw_stats()
2108 	    CTLFLAG_RD, &stats->mpc[0], "RX Missed Packet Count");  in ixgbe_add_hw_stats()
2112 	    CTLFLAG_RD, &stats->lxontxc, "Link XON Transmitted");  in ixgbe_add_hw_stats()
2114 	    CTLFLAG_RD, &stats->lxonrxc, "Link XON Received");  in ixgbe_add_hw_stats()
2116 	    CTLFLAG_RD, &stats->lxofftxc, "Link XOFF Transmitted");  in ixgbe_add_hw_stats()
2118 	    CTLFLAG_RD, &stats->lxoffrxc, "Link XOFF Received");  in ixgbe_add_hw_stats()
2122 	    CTLFLAG_RD, &stats->tor, "Total Octets Received");  in ixgbe_add_hw_stats()
2124 	    CTLFLAG_RD, &stats->gorc, "Good Octets Received");  in ixgbe_add_hw_stats()
2126 	    CTLFLAG_RD, &stats->tpr, "Total Packets Received");  in ixgbe_add_hw_stats()
2128 	    CTLFLAG_RD, &stats->gprc, "Good Packets Received");  in ixgbe_add_hw_stats()
2130 	    CTLFLAG_RD, &stats->mprc, "Multicast Packets Received");  in ixgbe_add_hw_stats()
2132 	    CTLFLAG_RD, &stats->bprc, "Broadcast Packets Received");  in ixgbe_add_hw_stats()
2134 	    CTLFLAG_RD, &stats->prc64, "64 byte frames received ");  in ixgbe_add_hw_stats()
2136 	    CTLFLAG_RD, &stats->prc127, "65-127 byte frames received");  in ixgbe_add_hw_stats()
2138 	    CTLFLAG_RD, &stats->prc255, "128-255 byte frames received");  in ixgbe_add_hw_stats()
2140 	    CTLFLAG_RD, &stats->prc511, "256-511 byte frames received");  in ixgbe_add_hw_stats()
2142 	    CTLFLAG_RD, &stats->prc1023, "512-1023 byte frames received");  in ixgbe_add_hw_stats()
2144 	    CTLFLAG_RD, &stats->prc1522, "1023-1522 byte frames received");  in ixgbe_add_hw_stats()
2146 	    CTLFLAG_RD, &stats->ruc, "Receive Undersized");  in ixgbe_add_hw_stats()
2148 	    CTLFLAG_RD, &stats->rfc, "Fragmented Packets Received ");  in ixgbe_add_hw_stats()
2150 	    CTLFLAG_RD, &stats->roc, "Oversized Packets Received");  in ixgbe_add_hw_stats()
2152 	    CTLFLAG_RD, &stats->rjc, "Received Jabber");  in ixgbe_add_hw_stats()
2154 	    CTLFLAG_RD, &stats->mngprc, "Management Packets Received");  in ixgbe_add_hw_stats()
2156 	    CTLFLAG_RD, &stats->mngptc, "Management Packets Dropped");  in ixgbe_add_hw_stats()
2158 	    CTLFLAG_RD, &stats->xec, "Checksum Errors");  in ixgbe_add_hw_stats()
2162 	    CTLFLAG_RD, &stats->gotc, "Good Octets Transmitted");  in ixgbe_add_hw_stats()
2164 	    CTLFLAG_RD, &stats->tpt, "Total Packets Transmitted");  in ixgbe_add_hw_stats()
2166 	    CTLFLAG_RD, &stats->gptc, "Good Packets Transmitted");  in ixgbe_add_hw_stats()
2168 	    CTLFLAG_RD, &stats->bptc, "Broadcast Packets Transmitted");  in ixgbe_add_hw_stats()
2170 	    CTLFLAG_RD, &stats->mptc, "Multicast Packets Transmitted");  in ixgbe_add_hw_stats()
2172 	    CTLFLAG_RD, &stats->mngptc, "Management Packets Transmitted");  in ixgbe_add_hw_stats()
2174 	    CTLFLAG_RD, &stats->ptc64, "64 byte frames transmitted ");  in ixgbe_add_hw_stats()
2176 	    CTLFLAG_RD, &stats->ptc127, "65-127 byte frames transmitted");  in ixgbe_add_hw_stats()
2178 	    CTLFLAG_RD, &stats->ptc255, "128-255 byte frames transmitted");  in ixgbe_add_hw_stats()
2180 	    CTLFLAG_RD, &stats->ptc511, "256-511 byte frames transmitted");  in ixgbe_add_hw_stats()
2182 	    CTLFLAG_RD, &stats->ptc1023, "512-1023 byte frames transmitted");  in ixgbe_add_hw_stats()
2184 	    CTLFLAG_RD, &stats->ptc1522, "1024-1522 byte frames transmitted");  in ixgbe_add_hw_stats()
2188  * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function
2195 	struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);  in ixgbe_sysctl_tdh_handler()
2203 	if (atomic_load_acq_int(&txr->sc->recovery_mode))  in ixgbe_sysctl_tdh_handler()
2206 	val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_TDH(txr->me));  in ixgbe_sysctl_tdh_handler()
2208 	if (error || !req->newptr)  in ixgbe_sysctl_tdh_handler()
2215  * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function
2222 	struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);  in ixgbe_sysctl_tdt_handler()
2229 	if (atomic_load_acq_int(&txr->sc->recovery_mode))  in ixgbe_sysctl_tdt_handler()
2232 	val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_TDT(txr->me));  in ixgbe_sysctl_tdt_handler()
2234 	if (error || !req->newptr)  in ixgbe_sysctl_tdt_handler()
2241  * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function
2248 	struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);  in ixgbe_sysctl_rdh_handler()
2255 	if (atomic_load_acq_int(&rxr->sc->recovery_mode))  in ixgbe_sysctl_rdh_handler()
2258 	val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_RDH(rxr->me));  in ixgbe_sysctl_rdh_handler()
2260 	if (error || !req->newptr)  in ixgbe_sysctl_rdh_handler()
2267  * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function
2274 	struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);  in ixgbe_sysctl_rdt_handler()
2281 	if (atomic_load_acq_int(&rxr->sc->recovery_mode))  in ixgbe_sysctl_rdt_handler()
2284 	val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_RDT(rxr->me));  in ixgbe_sysctl_rdt_handler()
2286 	if (error || !req->newptr)  in ixgbe_sysctl_rdt_handler()
2308 	sc->shadow_vfta[index] |= (1 << bit);  in ixgbe_if_vlan_register()
2309 	++sc->num_vlans;  in ixgbe_if_vlan_register()
2326 	sc->shadow_vfta[index] &= ~(1 << bit);  in ixgbe_if_vlan_unregister()
2327 	--sc->num_vlans;  in ixgbe_if_vlan_unregister()
2328 	/* Re-init to load the changes */  in ixgbe_if_vlan_unregister()
2340 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_setup_vlan_hw_support()
2352 	if (sc->num_vlans == 0 ||  in ixgbe_setup_vlan_hw_support()
2355 		for (i = 0; i < sc->num_rx_queues; i++) {  in ixgbe_setup_vlan_hw_support()
2356 			rxr = &sc->rx_queues[i].rxr;  in ixgbe_setup_vlan_hw_support()
2358 			if (hw->mac.type != ixgbe_mac_82598EB) {  in ixgbe_setup_vlan_hw_support()
2360 				    IXGBE_RXDCTL(rxr->me));  in ixgbe_setup_vlan_hw_support()
2362 				IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me),  in ixgbe_setup_vlan_hw_support()
2365 			rxr->vtag_strip = false;  in ixgbe_setup_vlan_hw_support()
2371 		if (hw->mac.type == ixgbe_mac_82598EB)  in ixgbe_setup_vlan_hw_support()
2379 		for (i = 0; i < sc->num_rx_queues; i++) {  in ixgbe_setup_vlan_hw_support()
2380 			rxr = &sc->rx_queues[i].rxr;  in ixgbe_setup_vlan_hw_support()
2382 			if (hw->mac.type != ixgbe_mac_82598EB) {  in ixgbe_setup_vlan_hw_support()
2384 				    IXGBE_RXDCTL(rxr->me));  in ixgbe_setup_vlan_hw_support()
2386 				IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me),  in ixgbe_setup_vlan_hw_support()
2389 			rxr->vtag_strip = true;  in ixgbe_setup_vlan_hw_support()
2400 		if (sc->shadow_vfta[i] != 0)  in ixgbe_setup_vlan_hw_support()
2402 			    sc->shadow_vfta[i]);  in ixgbe_setup_vlan_hw_support()
2410 	if (hw->mac.type == ixgbe_mac_82598EB)  in ixgbe_setup_vlan_hw_support()
2424 	device_t dev = iflib_get_dev(sc->ctx);  in ixgbe_get_slot_info()
2425 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_get_slot_info()
2431 	switch (hw->device_id) {  in ixgbe_get_slot_info()
2442 	 * Some devices don't use PCI-E, but there is no need  in ixgbe_get_slot_info()
2445 	switch (hw->mac.type) {  in ixgbe_get_slot_info()
2472 		 * Hmm...can't get PCI-Express capabilities.  in ixgbe_get_slot_info()
2485 	    ((hw->bus.speed == ixgbe_bus_speed_16000)   ? "16.0GT/s" :  in ixgbe_get_slot_info()
2486 	     (hw->bus.speed == ixgbe_bus_speed_8000)    ? "8.0GT/s"  :  in ixgbe_get_slot_info()
2487 	     (hw->bus.speed == ixgbe_bus_speed_5000)    ? "5.0GT/s"  :  in ixgbe_get_slot_info()
2488 	     (hw->bus.speed == ixgbe_bus_speed_2500)    ? "2.5GT/s"  :  in ixgbe_get_slot_info()
2490 	    ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" :  in ixgbe_get_slot_info()
2491 	     (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" :  in ixgbe_get_slot_info()
2492 	     (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" :  in ixgbe_get_slot_info()
2496 		if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&  in ixgbe_get_slot_info()
2497 		    ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&  in ixgbe_get_slot_info()
2498 		    (hw->bus.speed == ixgbe_bus_speed_2500))) {  in ixgbe_get_slot_info()
2500 			    "PCI-Express bandwidth available for this card"  in ixgbe_get_slot_info()
2506 		if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&  in ixgbe_get_slot_info()
2507 		    ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&  in ixgbe_get_slot_info()
2508 		    (hw->bus.speed < ixgbe_bus_speed_8000))) {  in ixgbe_get_slot_info()
2510 			    "PCI-Express bandwidth available for this card"  in ixgbe_get_slot_info()
2527  *   Setup MSI-X Interrupt resources and handlers
2533 	struct ix_rx_queue *rx_que = sc->rx_queues;  in ixgbe_if_msix_intr_assign()
2540 	for (int i = 0; i < sc->num_rx_queues; i++, vector++, rx_que++) {  in ixgbe_if_msix_intr_assign()
2544 		error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid,  in ixgbe_if_msix_intr_assign()
2545 		    IFLIB_INTR_RXTX, ixgbe_msix_que, rx_que, rx_que->rxr.me,  in ixgbe_if_msix_intr_assign()
2552 			sc->num_rx_queues = i + 1;  in ixgbe_if_msix_intr_assign()
2556 		rx_que->msix = vector;  in ixgbe_if_msix_intr_assign()
2558 	for (int i = 0; i < sc->num_tx_queues; i++) {  in ixgbe_if_msix_intr_assign()
2560 		tx_que = &sc->tx_queues[i];  in ixgbe_if_msix_intr_assign()
2561 		tx_que->msix = i % sc->num_rx_queues;  in ixgbe_if_msix_intr_assign()
2563 		    &sc->rx_queues[tx_que->msix].que_irq,  in ixgbe_if_msix_intr_assign()
2564 		    IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf);  in ixgbe_if_msix_intr_assign()
2567 	error = iflib_irq_alloc_generic(ctx, &sc->irq, rid,  in ixgbe_if_msix_intr_assign()
2575 	sc->vector = vector;  in ixgbe_if_msix_intr_assign()
2579 	iflib_irq_free(ctx, &sc->irq);  in ixgbe_if_msix_intr_assign()
2580 	rx_que = sc->rx_queues;  in ixgbe_if_msix_intr_assign()
2581 	for (int i = 0; i < sc->num_rx_queues; i++, rx_que++)  in ixgbe_if_msix_intr_assign()
2582 		iflib_irq_free(ctx, &rx_que->que_irq);  in ixgbe_if_msix_intr_assign()
2591 	struct rx_ring *rxr = &que->rxr;  in ixgbe_perform_aim()
2592 	/* FIXME struct tx_ring *txr = ... ->txr; */  in ixgbe_perform_aim()
2596 	 *  - Write out last calculated setting  in ixgbe_perform_aim()
2597 	 *  - Calculate based on average size over  in ixgbe_perform_aim()
2600 	if (que->eitr_setting) {  in ixgbe_perform_aim()
2601 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(que->msix),  in ixgbe_perform_aim()
2602 		    que->eitr_setting);  in ixgbe_perform_aim()
2605 	que->eitr_setting = 0;  in ixgbe_perform_aim()
2607 	if (rxr->bytes == 0) {  in ixgbe_perform_aim()
2608 		/* FIXME && txr->bytes == 0 */  in ixgbe_perform_aim()
2612 	if ((rxr->bytes) && (rxr->packets))  in ixgbe_perform_aim()
2613 		newitr = rxr->bytes / rxr->packets;  in ixgbe_perform_aim()
2615 	 * if ((txr->bytes) && (txr->packets))  in ixgbe_perform_aim()
2616 	 * 	newitr = txr->bytes/txr->packets;  in ixgbe_perform_aim()
2617 	 * if ((rxr->bytes) && (rxr->packets))  in ixgbe_perform_aim()
2618 	 * 	newitr = max(newitr, (rxr->bytes / rxr->packets));  in ixgbe_perform_aim()
2632 	if (sc->hw.mac.type == ixgbe_mac_82598EB) {  in ixgbe_perform_aim()
2639 	que->eitr_setting = newitr;  in ixgbe_perform_aim()
2642 	/* FIXME txr->bytes = 0; */  in ixgbe_perform_aim()
2643 	/* FIXME txr->packets = 0; */  in ixgbe_perform_aim()
2644 	rxr->bytes = 0;  in ixgbe_perform_aim()
2645 	rxr->packets = 0;  in ixgbe_perform_aim()
2651  * ixgbe_msix_que - MSI-X Queue Interrupt Service routine
2657 	struct ixgbe_softc *sc = que->sc;  in ixgbe_msix_que()
2658 	if_t ifp = iflib_get_ifp(que->sc->ctx);  in ixgbe_msix_que()
2664 	ixgbe_disable_queue(sc, que->msix);  in ixgbe_msix_que()
2665 	++que->irqs;  in ixgbe_msix_que()
2668 	if (sc->enable_aim) {  in ixgbe_msix_que()
2676  * ixgbe_media_status - Media Ioctl callback
2685 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_if_media_status()
2690 	ifmr->ifm_status = IFM_AVALID;  in ixgbe_if_media_status()
2691 	ifmr->ifm_active = IFM_ETHER;  in ixgbe_if_media_status()
2693 	if (!sc->link_active)  in ixgbe_if_media_status()
2696 	ifmr->ifm_status |= IFM_ACTIVE;  in ixgbe_if_media_status()
2697 	layer = sc->phy_layer;  in ixgbe_if_media_status()
2700 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2702 			ifmr->ifm_active |= IFM_10G_T | IFM_FDX;  in ixgbe_if_media_status()
2705 			ifmr->ifm_active |= IFM_5000_T | IFM_FDX;  in ixgbe_if_media_status()
2708 			ifmr->ifm_active |= IFM_2500_T | IFM_FDX;  in ixgbe_if_media_status()
2711 			ifmr->ifm_active |= IFM_1000_T | IFM_FDX;  in ixgbe_if_media_status()
2714 			ifmr->ifm_active |= IFM_100_TX | IFM_FDX;  in ixgbe_if_media_status()
2717 			ifmr->ifm_active |= IFM_10_T | IFM_FDX;  in ixgbe_if_media_status()
2722 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2724 			ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX;  in ixgbe_if_media_status()
2727 			ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;  in ixgbe_if_media_status()
2731 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2733 			ifmr->ifm_active |= IFM_10G_LR | IFM_FDX;  in ixgbe_if_media_status()
2736 			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;  in ixgbe_if_media_status()
2740 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2742 			ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX;  in ixgbe_if_media_status()
2745 			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;  in ixgbe_if_media_status()
2750 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2752 			ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;  in ixgbe_if_media_status()
2755 			ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;  in ixgbe_if_media_status()
2759 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2761 			ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;  in ixgbe_if_media_status()
2770 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2772 			ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;  in ixgbe_if_media_status()
2775 			ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;  in ixgbe_if_media_status()
2778 			ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;  in ixgbe_if_media_status()
2784 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2786 			ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;  in ixgbe_if_media_status()
2789 			ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;  in ixgbe_if_media_status()
2792 			ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;  in ixgbe_if_media_status()
2797 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2799 			ifmr->ifm_active |= IFM_10G_KR | IFM_FDX;  in ixgbe_if_media_status()
2802 			ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;  in ixgbe_if_media_status()
2805 			ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;  in ixgbe_if_media_status()
2811 		switch (sc->link_speed) {  in ixgbe_if_media_status()
2813 			ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX;  in ixgbe_if_media_status()
2816 			ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;  in ixgbe_if_media_status()
2819 			ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;  in ixgbe_if_media_status()
2825 	if (IFM_SUBTYPE(ifmr->ifm_active) == 0)  in ixgbe_if_media_status()
2826 		ifmr->ifm_active |= IFM_UNKNOWN;  in ixgbe_if_media_status()
2829 	if (hw->fc.current_mode == ixgbe_fc_rx_pause ||  in ixgbe_if_media_status()
2830 	    hw->fc.current_mode == ixgbe_fc_full)  in ixgbe_if_media_status()
2831 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;  in ixgbe_if_media_status()
2832 	if (hw->fc.current_mode == ixgbe_fc_tx_pause ||  in ixgbe_if_media_status()
2833 	    hw->fc.current_mode == ixgbe_fc_full)  in ixgbe_if_media_status()
2834 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;  in ixgbe_if_media_status()
2838  * ixgbe_media_change - Media Ioctl callback
2848 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_if_media_change()
2853 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)  in ixgbe_if_media_change()
2856 	if (hw->phy.media_type == ixgbe_media_type_backplane)  in ixgbe_if_media_change()
2864 	switch (IFM_SUBTYPE(ifm->ifm_media)) {  in ixgbe_if_media_change()
2884 	case IFM_1000_CX: /* KX */  in ixgbe_if_media_change()
2916 	hw->mac.autotry_restart = true;  in ixgbe_if_media_change()
2917 	hw->mac.ops.setup_link(hw, speed, true);  in ixgbe_if_media_change()
2918 	sc->advertise =  in ixgbe_if_media_change()
2945 	rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);  in ixgbe_if_promisc_set()
2955 	IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, rctl);  in ixgbe_if_promisc_set()
2959 		IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, rctl);  in ixgbe_if_promisc_set()
2963 		IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, rctl);  in ixgbe_if_promisc_set()
2969  * ixgbe_msix_link - Link status change ISR (MSI/MSI-X)
2975 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_msix_link()
2979 	++sc->link_irq;  in ixgbe_msix_link()
2994 		sc->task_requests |= IXGBE_REQUEST_TASK_LSC;  in ixgbe_msix_link()
2999 		sc->task_requests |= IXGBE_REQUEST_TASK_FWEVENT;  in ixgbe_msix_link()
3002 	if (sc->hw.mac.type != ixgbe_mac_82598EB) {  in ixgbe_msix_link()
3003 		if ((sc->feat_en & IXGBE_FEATURE_FDIR) &&  in ixgbe_msix_link()
3006 			if (!atomic_cmpset_int(&sc->fdir_reinit, 0, 1))  in ixgbe_msix_link()
3010 			sc->task_requests |= IXGBE_REQUEST_TASK_FDIR;  in ixgbe_msix_link()
3013 				device_printf(iflib_get_dev(sc->ctx),  in ixgbe_msix_link()
3015 				hw->mac.flags |=  in ixgbe_msix_link()
3023 		if (sc->feat_en & IXGBE_FEATURE_TEMP_SENSOR) {  in ixgbe_msix_link()
3024 			switch (sc->hw.mac.type) {  in ixgbe_msix_link()
3032 				retval = hw->phy.ops.check_overtemp(hw);  in ixgbe_msix_link()
3035 				device_printf(iflib_get_dev(sc->ctx),  in ixgbe_msix_link()
3038 				device_printf(iflib_get_dev(sc->ctx),  in ixgbe_msix_link()
3044 				retval = hw->phy.ops.check_overtemp(hw);  in ixgbe_msix_link()
3047 				device_printf(iflib_get_dev(sc->ctx),  in ixgbe_msix_link()
3050 				device_printf(iflib_get_dev(sc->ctx),  in ixgbe_msix_link()
3059 		if ((sc->feat_en & IXGBE_FEATURE_SRIOV) &&  in ixgbe_msix_link()
3061 			sc->task_requests |= IXGBE_REQUEST_TASK_MBX;  in ixgbe_msix_link()
3067 	 * there is no need to perform any SFP-specific tasks.  in ixgbe_msix_link()
3069 	if (hw->mac.type != ixgbe_mac_E610 && ixgbe_is_sfp(hw)) {  in ixgbe_msix_link()
3070 		/* Pluggable optics-related interrupt */  in ixgbe_msix_link()
3071 		if (hw->mac.type >= ixgbe_mac_X540)  in ixgbe_msix_link()
3078 			sc->task_requests |= IXGBE_REQUEST_TASK_MOD;  in ixgbe_msix_link()
3081 		if ((hw->mac.type == ixgbe_mac_82599EB) &&  in ixgbe_msix_link()
3085 			sc->task_requests |= IXGBE_REQUEST_TASK_MSF;  in ixgbe_msix_link()
3090 	if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL) {  in ixgbe_msix_link()
3097 	if ((hw->phy.type == ixgbe_phy_x550em_ext_t) &&  in ixgbe_msix_link()
3100 		sc->task_requests |= IXGBE_REQUEST_TASK_PHY;  in ixgbe_msix_link()
3103 	return (sc->task_requests != 0) ?  in ixgbe_msix_link()
3113 	struct ix_rx_queue *que = ((struct ix_rx_queue *)oidp->oid_arg1);  in ixgbe_sysctl_interrupt_rate_handler()
3117 	if (atomic_load_acq_int(&que->sc->recovery_mode))  in ixgbe_sysctl_interrupt_rate_handler()
3120 	reg = IXGBE_READ_REG(&que->sc->hw, IXGBE_EITR(que->msix));  in ixgbe_sysctl_interrupt_rate_handler()
3127 	if (error || !req->newptr)  in ixgbe_sysctl_interrupt_rate_handler()
3132 		if (rate < 1000)  in ixgbe_sysctl_interrupt_rate_handler()
3133 			rate = 1000;  in ixgbe_sysctl_interrupt_rate_handler()
3137 	IXGBE_WRITE_REG(&que->sc->hw, IXGBE_EITR(que->msix), reg);  in ixgbe_sysctl_interrupt_rate_handler()
3150 	device_t dev = sc->dev;  in ixgbe_debug_dump_print_cluster()
3151 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_debug_dump_print_cluster()
3153 	int max_aci_calls = 1000;  in ixgbe_debug_dump_print_cluster()
3185 			    __func__, error, hw->aci.last_status);  in ixgbe_debug_dump_print_cluster()
3251 	u32 clusters = sc->debug_dump_cluster_mask;  in ixgbe_sysctl_debug_dump_set_clusters()
3252 	device_t dev = sc->dev;  in ixgbe_sysctl_debug_dump_set_clusters()
3256 	if ((error) || !req->newptr)  in ixgbe_sysctl_debug_dump_set_clusters()
3263 		sc->debug_dump_cluster_mask =  in ixgbe_sysctl_debug_dump_set_clusters()
3268 	sc->debug_dump_cluster_mask = clusters;  in ixgbe_sysctl_debug_dump_set_clusters()
3280 	device_t dev = sc->dev;  in ixgbe_sysctl_dump_debug_dump()
3286 	if (!sc->do_debug_dump) {  in ixgbe_sysctl_dump_debug_dump()
3287 		if (req->oldptr == NULL && req->newptr == NULL) {  in ixgbe_sysctl_dump_debug_dump()
3295 		if ((error) || (req->newptr == NULL))  in ixgbe_sysctl_dump_debug_dump()
3299 			if (sc->debug_dump_cluster_mask ==  in ixgbe_sysctl_dump_debug_dump()
3307 			sc->do_debug_dump = true;  in ixgbe_sysctl_dump_debug_dump()
3315 	if (req->oldptr == NULL && req->newptr == NULL) {  in ixgbe_sysctl_dump_debug_dump()
3317 		if (sc->debug_dump_cluster_mask & 0x2)  in ixgbe_sysctl_dump_debug_dump()
3328 	if (sc->debug_dump_cluster_mask) {  in ixgbe_sysctl_dump_debug_dump()
3330 			if (sc->debug_dump_cluster_mask & BIT(id)) {  in ixgbe_sysctl_dump_debug_dump()
3349 	sc->do_debug_dump = false;  in ixgbe_sysctl_dump_debug_dump()
3363 	device_t dev = sc->dev;  in ixgbe_add_debug_dump_sysctls()
3366 	debug_list = SYSCTL_CHILDREN(sc->debug_sysctls);  in ixgbe_add_debug_dump_sysctls()
3388 	device_t dev = sc->dev;  in ixgbe_add_debug_sysctls()
3393 	sc->debug_sysctls = SYSCTL_ADD_NODE(ctx, ctx_list, OID_AUTO, "debug",  in ixgbe_add_debug_sysctls()
3396 	if (sc->feat_en & IXGBE_FEATURE_DBG_DUMP)  in ixgbe_add_debug_sysctls()
3408 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_add_device_sysctls()
3426 	sc->enable_aim = ixgbe_enable_aim;  in ixgbe_add_device_sysctls()
3428 	    &sc->enable_aim, 0, "Interrupt Moderation");  in ixgbe_add_device_sysctls()
3464 	if (hw->mac.type >= ixgbe_mac_X550)  in ixgbe_add_device_sysctls()
3470 	/* for WoL-capable devices */  in ixgbe_add_device_sysctls()
3471 	if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {  in ixgbe_add_device_sysctls()
3483 	/* for X552/X557-AT devices */  in ixgbe_add_device_sysctls()
3484 	if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {  in ixgbe_add_device_sysctls()
3505 	if (sc->feat_cap & IXGBE_FEATURE_EEE) {  in ixgbe_add_device_sysctls()
3525 	sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,  in ixgbe_allocate_pci_resources()
3528 	if (!(sc->pci_mem)) {  in ixgbe_allocate_pci_resources()
3535 	sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->pci_mem);  in ixgbe_allocate_pci_resources()
3536 	sc->osdep.mem_bus_space_handle =  in ixgbe_allocate_pci_resources()
3537 	    rman_get_bushandle(sc->pci_mem);  in ixgbe_allocate_pci_resources()
3539 	sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle;  in ixgbe_allocate_pci_resources()
3545  * ixgbe_detach - Device removal routine
3563 		device_printf(dev, "SR-IOV in use; detach first.\n");  in ixgbe_if_detach()
3570 	ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);  in ixgbe_if_detach()
3572 	IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);  in ixgbe_if_detach()
3574 	callout_drain(&sc->fw_mode_timer);  in ixgbe_if_detach()
3576 	if (sc->hw.mac.type == ixgbe_mac_E610) {  in ixgbe_if_detach()
3578 		ixgbe_shutdown_aci(&sc->hw);  in ixgbe_if_detach()
3583 	free(sc->mta, M_IXGBE);  in ixgbe_if_detach()
3589  * ixgbe_setup_low_power_mode - LPLU/WoL preparation
3597 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_setup_low_power_mode()
3601 	if (!hw->wol_enabled)  in ixgbe_setup_low_power_mode()
3605 	if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&  in ixgbe_setup_low_power_mode()
3606 	    hw->phy.ops.enter_lplu) {  in ixgbe_setup_low_power_mode()
3622 		IXGBE_WRITE_REG(hw, IXGBE_WUFC, sc->wufc);  in ixgbe_setup_low_power_mode()
3629 		hw->phy.reset_disable = true;  in ixgbe_setup_low_power_mode()
3631 		error = hw->phy.ops.enter_lplu(hw);  in ixgbe_setup_low_power_mode()
3635 		hw->phy.reset_disable = false;  in ixgbe_setup_low_power_mode()
3645  * ixgbe_shutdown - Shutdown entry point
3687 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_if_resume()
3698 	/* And clear WUFC until next low-power transition */  in ixgbe_if_resume()
3702 	 * Required after D3->D0 transition;  in ixgbe_if_resume()
3703 	 * will re-advertise all previous advertised speeds  in ixgbe_if_resume()
3712  * ixgbe_if_mtu_set - Ioctl mtu entry point
3727 		sc->max_frame_size = mtu + IXGBE_MTU_HDR;  in ixgbe_if_mtu_set()
3740 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_if_crcstrip_set()
3780  * ixgbe_if_init - Init entry point
3795 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_if_init()
3810 	ixgbe_set_rar(hw, 0, hw->mac.addr, sc->pool, IXGBE_RAH_AV);  in ixgbe_if_init()
3813 	bcopy(if_getlladdr(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS);  in ixgbe_if_init()
3814 	ixgbe_set_rar(hw, 0, hw->mac.addr, sc->pool, 1);  in ixgbe_if_init()
3815 	hw->addr_ctrl.rar_used_count = 1;  in ixgbe_if_init()
3827 	sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx);  in ixgbe_if_init()
3834 	 * from MSI-X interrupts  in ixgbe_if_init()
3836 	sc->task_requests = 0;  in ixgbe_if_init()
3838 	/* Enable SDP & MSI-X interrupts based on adapter */  in ixgbe_if_init()
3846 		mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT;  in ixgbe_if_init()
3851 	for (i = 0, tx_que = sc->tx_queues; i < sc->num_tx_queues;  in ixgbe_if_init()
3853 		struct tx_ring *txr = &tx_que->txr;  in ixgbe_if_init()
3855 		txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me));  in ixgbe_if_init()
3867 		IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl);  in ixgbe_if_init()
3870 	for (i = 0, rx_que = sc->rx_queues; i < sc->num_rx_queues;  in ixgbe_if_init()
3872 		struct rx_ring *rxr = &rx_que->rxr;  in ixgbe_if_init()
3874 		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me));  in ixgbe_if_init()
3875 		if (hw->mac.type == ixgbe_mac_82598EB) {  in ixgbe_if_init()
3885 		IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl);  in ixgbe_if_init()
3887 			if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) &  in ixgbe_if_init()
3898 	if (hw->mac.type == ixgbe_mac_82598EB)  in ixgbe_if_init()
3903 	/* Set up MSI/MSI-X routing */  in ixgbe_if_init()
3906 		/* Set up auto-mask */  in ixgbe_if_init()
3907 		if (hw->mac.type == ixgbe_mac_82598EB)  in ixgbe_if_init()
3923 	 * need to be kick-started  in ixgbe_if_init()
3925 	if (hw->phy.type == ixgbe_phy_none) {  in ixgbe_if_init()
3926 		err = hw->phy.ops.identify(hw);  in ixgbe_if_init()
3935 	IXGBE_WRITE_REG(hw, IXGBE_EITR(sc->vector), IXGBE_LINK_ITR);  in ixgbe_if_init()
3959 	if (sc->feat_en & IXGBE_FEATURE_SRIOV) {  in ixgbe_if_init()
3970  *   Setup the correct IVAR register for a particular MSI-X interrupt
3972  *    - entry is the register array entry
3973  *    - vector is the MSI-X vector for this queue
3974  *    - type is RX/TX/MISC
3979 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_set_ivar()
3984 	switch (hw->mac.type) {  in ixgbe_set_ivar()
3986 		if (type == -1)  in ixgbe_set_ivar()
3994 		IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar);  in ixgbe_set_ivar()
4002 		if (type == -1) { /* MISC IVAR */  in ixgbe_set_ivar()
4026 	struct ix_rx_queue *rx_que = sc->rx_queues;  in ixgbe_configure_ivars()
4027 	struct ix_tx_queue *tx_que = sc->tx_queues;  in ixgbe_configure_ivars()
4037 		sc->dmac = 0;  in ixgbe_configure_ivars()
4041 	for (int i = 0; i < sc->num_rx_queues; i++, rx_que++) {  in ixgbe_configure_ivars()
4042 		struct rx_ring *rxr = &rx_que->rxr;  in ixgbe_configure_ivars()
4045 		ixgbe_set_ivar(sc, rxr->me, rx_que->msix, 0);  in ixgbe_configure_ivars()
4048 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(rx_que->msix), newitr);  in ixgbe_configure_ivars()
4050 	for (int i = 0; i < sc->num_tx_queues; i++, tx_que++) {  in ixgbe_configure_ivars()
4051 		struct tx_ring *txr = &tx_que->txr;  in ixgbe_configure_ivars()
4054 		ixgbe_set_ivar(sc, txr->me, tx_que->msix, 1);  in ixgbe_configure_ivars()
4057 	ixgbe_set_ivar(sc, 1, sc->vector, -1);  in ixgbe_configure_ivars()
4066 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_config_gpie()
4071 	if (sc->intr_type == IFLIB_INTR_MSIX) {  in ixgbe_config_gpie()
4072 		/* Enable Enhanced MSI-X mode */  in ixgbe_config_gpie()
4080 	if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL)  in ixgbe_config_gpie()
4084 	if (sc->feat_en & IXGBE_FEATURE_TEMP_SENSOR)  in ixgbe_config_gpie()
4088 	switch (hw->mac.type) {  in ixgbe_config_gpie()
4107  *   Requires sc->max_frame_size to be set.
4112 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_config_delay_values()
4115 	frame = sc->max_frame_size;  in ixgbe_config_delay_values()
4118 	switch (hw->mac.type) {  in ixgbe_config_delay_values()
4131 	hw->fc.high_water[0] = rxpb - size;  in ixgbe_config_delay_values()
4134 	switch (hw->mac.type) {  in ixgbe_config_delay_values()
4145 	hw->fc.low_water[0] = IXGBE_BT2KB(tmp);  in ixgbe_config_delay_values()
4147 	hw->fc.pause_time = IXGBE_FC_PAUSE;  in ixgbe_config_delay_values()
4148 	hw->fc.send_xon = true;  in ixgbe_config_delay_values()
4152  * ixgbe_set_multi - Multicast Update
4160 	struct ixgbe_mc_addr *mta = sc->mta;  in ixgbe_mc_filter_apply()
4165 	mta[idx].vmdq = sc->pool;  in ixgbe_mc_filter_apply()
4182 	mta = sc->mta;  in ixgbe_if_multi_set()
4190 		ixgbe_update_mc_addr_list(&sc->hw, update_ptr, mcnt,  in ixgbe_if_multi_set()
4194 	fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);  in ixgbe_if_multi_set()
4205 	IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl);  in ixgbe_if_multi_set()
4221 	*vmdq = mta->vmdq;  in ixgbe_mc_array_itr()
4225 	return (mta->addr);  in ixgbe_mc_array_itr()
4229  * ixgbe_local_timer - Timer routine
4243 	if (sc->sfp_probe)  in ixgbe_if_timer()
4247 	ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0);  in ixgbe_if_timer()
4255  * ixgbe_fw_mode_timer - FW mode timer routine
4261 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_fw_mode_timer()
4264 		if (atomic_cmpset_acq_int(&sc->recovery_mode, 0, 1)) {  in ixgbe_fw_mode_timer()
4266 			device_printf(sc->dev,  in ixgbe_fw_mode_timer()
4272 			if (hw->adapter_stopped == FALSE)  in ixgbe_fw_mode_timer()
4273 				ixgbe_if_stop(sc->ctx);  in ixgbe_fw_mode_timer()
4276 		atomic_cmpset_acq_int(&sc->recovery_mode, 1, 0);  in ixgbe_fw_mode_timer()
4279 	callout_reset(&sc->fw_mode_timer, hz,  in ixgbe_fw_mode_timer()
4292 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_sfp_probe()
4296 	if ((hw->phy.type == ixgbe_phy_nl) &&  in ixgbe_sfp_probe()
4297 	    (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {  in ixgbe_sfp_probe()
4298 		s32 ret = hw->phy.ops.identify_sfp(hw);  in ixgbe_sfp_probe()
4301 		ret = hw->phy.ops.reset(hw);  in ixgbe_sfp_probe()
4302 		sc->sfp_probe = false;  in ixgbe_sfp_probe()
4320  * ixgbe_handle_mod - Tasklet for SFP module interrupts
4327 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_handle_mod()
4331 	if (sc->hw.need_crosstalk_fix) {  in ixgbe_handle_mod()
4332 		switch (hw->mac.type) {  in ixgbe_handle_mod()
4350 	err = hw->phy.ops.identify_sfp(hw);  in ixgbe_handle_mod()
4357 	if (hw->mac.type == ixgbe_mac_82598EB)  in ixgbe_handle_mod()
4358 		err = hw->phy.ops.reset(hw);  in ixgbe_handle_mod()
4360 		err = hw->mac.ops.setup_sfp(hw);  in ixgbe_handle_mod()
4364 		    "Setup failure - unsupported SFP+ module type.\n");  in ixgbe_handle_mod()
4367 	sc->task_requests |= IXGBE_REQUEST_TASK_MSF;  in ixgbe_handle_mod()
4371 	sc->task_requests &= ~(IXGBE_REQUEST_TASK_MSF);  in ixgbe_handle_mod()
4376  * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts
4383 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_handle_msf()
4387 	/* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */  in ixgbe_handle_msf()
4388 	sc->phy_layer = ixgbe_get_supported_physical_layer(hw);  in ixgbe_handle_msf()
4390 	autoneg = hw->phy.autoneg_advertised;  in ixgbe_handle_msf()
4391 	if ((!autoneg) && (hw->mac.ops.get_link_capabilities))  in ixgbe_handle_msf()
4392 		hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);  in ixgbe_handle_msf()
4393 	if (hw->mac.ops.setup_link)  in ixgbe_handle_msf()
4394 		hw->mac.ops.setup_link(hw, autoneg, true);  in ixgbe_handle_msf()
4397 	ifmedia_removeall(sc->media);  in ixgbe_handle_msf()
4398 	ixgbe_add_media_types(sc->ctx);  in ixgbe_handle_msf()
4399 	ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO);  in ixgbe_handle_msf()
4403  * ixgbe_handle_phy - Tasklet for external PHY interrupts
4410 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_handle_phy()
4413 	error = hw->phy.ops.handle_lasi(hw);  in ixgbe_handle_phy()
4415 		device_printf(sc->dev,  in ixgbe_handle_phy()
4419 		device_printf(sc->dev,  in ixgbe_handle_phy()
4424  * ixgbe_enable_lse - enable link status events
4437 	error = ixgbe_configure_lse(&sc->hw, TRUE, mask);  in ixgbe_enable_lse()
4441 	sc->lse_mask = mask;  in ixgbe_enable_lse()
4446  * ixgbe_disable_lse - disable link status events
4452 	error = ixgbe_configure_lse(&sc->hw, false, sc->lse_mask);  in ixgbe_disable_lse()
4456 	sc->lse_mask = 0;  in ixgbe_disable_lse()
4461  * ixgbe_handle_fw_event - Tasklet for MSI-X Link Status Event interrupts
4468 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_handle_fw_event()
4476 		device_printf(sc->dev, "Can not allocate buffer for "  in ixgbe_handle_fw_event()
4484 			device_printf(sc->dev, "Error getting event from "  in ixgbe_handle_fw_event()
4491 			sc->task_requests |= IXGBE_REQUEST_TASK_LSC;  in ixgbe_handle_fw_event()
4495 			if (hw->adapter_stopped == FALSE)  in ixgbe_handle_fw_event()
4497 			device_printf(sc->dev,  in ixgbe_handle_fw_event()
4499 			device_printf(sc->dev, "System shutdown required!\n");  in ixgbe_handle_fw_event()
4503 			device_printf(sc->dev,  in ixgbe_handle_fw_event()
4514  * ixgbe_if_stop - Stop the hardware
4523 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_if_stop()
4528 	hw->adapter_stopped = false;  in ixgbe_if_stop()
4530 	if (hw->mac.type == ixgbe_mac_82599EB)  in ixgbe_if_stop()
4532 	/* Turn off the laser - noop with no optics */  in ixgbe_if_stop()
4536 	sc->link_up = false;  in ixgbe_if_stop()
4540 	ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV);  in ixgbe_if_stop()
4546  * ixgbe_link_speed_to_str - Convert link speed to string
4548  *   Helper function to convert link speed constants to human-readable
4573  * ixgbe_update_link_status - Update OS on link state
4585 	if (sc->link_up) {  in ixgbe_if_update_admin_status()
4586 		if (sc->link_active == false) {  in ixgbe_if_update_admin_status()
4590 				    ixgbe_link_speed_to_str(sc->link_speed));  in ixgbe_if_update_admin_status()
4591 			sc->link_active = true;  in ixgbe_if_update_admin_status()
4593 			ixgbe_fc_enable(&sc->hw);  in ixgbe_if_update_admin_status()
4597 			    ixgbe_link_speed_to_baudrate(sc->link_speed));  in ixgbe_if_update_admin_status()
4599 			if (sc->feat_en & IXGBE_FEATURE_SRIOV)  in ixgbe_if_update_admin_status()
4603 		if (sc->link_active == true) {  in ixgbe_if_update_admin_status()
4607 			sc->link_active = false;  in ixgbe_if_update_admin_status()
4608 			if (sc->feat_en & IXGBE_FEATURE_SRIOV)  in ixgbe_if_update_admin_status()
4614 	if (sc->task_requests & IXGBE_REQUEST_TASK_FWEVENT)  in ixgbe_if_update_admin_status()
4616 	if (sc->task_requests & IXGBE_REQUEST_TASK_MOD)  in ixgbe_if_update_admin_status()
4618 	if (sc->task_requests & IXGBE_REQUEST_TASK_MSF)  in ixgbe_if_update_admin_status()
4620 	if (sc->task_requests & IXGBE_REQUEST_TASK_MBX)  in ixgbe_if_update_admin_status()
4622 	if (sc->task_requests & IXGBE_REQUEST_TASK_FDIR)  in ixgbe_if_update_admin_status()
4624 	if (sc->task_requests & IXGBE_REQUEST_TASK_PHY)  in ixgbe_if_update_admin_status()
4626 	sc->task_requests = 0;  in ixgbe_if_update_admin_status()
4632  * ixgbe_config_dmac - Configure DMA Coalescing
4637 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_config_dmac()
4638 	struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config;  in ixgbe_config_dmac()
4640 	if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config)  in ixgbe_config_dmac()
4643 	if (dcfg->watchdog_timer ^ sc->dmac ||  in ixgbe_config_dmac()
4644 	    dcfg->link_speed ^ sc->link_speed) {  in ixgbe_config_dmac()
4645 		dcfg->watchdog_timer = sc->dmac;  in ixgbe_config_dmac()
4646 		dcfg->fcoe_en = false;  in ixgbe_config_dmac()
4647 		dcfg->link_speed = sc->link_speed;  in ixgbe_config_dmac()
4648 		dcfg->num_tcs = 1;  in ixgbe_config_dmac()
4651 		    dcfg->watchdog_timer, dcfg->link_speed);  in ixgbe_config_dmac()
4653 		hw->mac.ops.dmac_config(hw);  in ixgbe_config_dmac()
4664 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_if_enable_intr()
4665 	struct ix_rx_queue *que = sc->rx_queues;  in ixgbe_if_enable_intr()
4670 	switch (sc->hw.mac.type) {  in ixgbe_if_enable_intr()
4694 		if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||  in ixgbe_if_enable_intr()
4695 		    hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||  in ixgbe_if_enable_intr()
4696 		    hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||  in ixgbe_if_enable_intr()
4697 		    hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T)  in ixgbe_if_enable_intr()
4699 		if (hw->phy.type == ixgbe_phy_x550em_ext_t)  in ixgbe_if_enable_intr()
4711 	if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL)  in ixgbe_if_enable_intr()
4713 	/* Enable SR-IOV */  in ixgbe_if_enable_intr()
4714 	if (sc->feat_en & IXGBE_FEATURE_SRIOV)  in ixgbe_if_enable_intr()
4717 	if (sc->feat_en & IXGBE_FEATURE_FDIR)  in ixgbe_if_enable_intr()
4722 	/* With MSI-X we use auto clear */  in ixgbe_if_enable_intr()
4723 	if (sc->intr_type == IFLIB_INTR_MSIX) {  in ixgbe_if_enable_intr()
4729 		if (sc->feat_cap & IXGBE_FEATURE_SRIOV)  in ixgbe_if_enable_intr()
4736 	 * allow for handling the extended (beyond 32) MSI-X  in ixgbe_if_enable_intr()
4739 	for (int i = 0; i < sc->num_rx_queues; i++, que++)  in ixgbe_if_enable_intr()
4740 		ixgbe_enable_queue(sc, que->msix);  in ixgbe_if_enable_intr()
4754 	if (sc->intr_type == IFLIB_INTR_MSIX)  in ixgbe_if_disable_intr()
4755 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0);  in ixgbe_if_disable_intr()
4756 	if (sc->hw.mac.type == ixgbe_mac_82598EB) {  in ixgbe_if_disable_intr()
4757 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0);  in ixgbe_if_disable_intr()
4759 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000);  in ixgbe_if_disable_intr()
4760 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0);  in ixgbe_if_disable_intr()
4761 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0);  in ixgbe_if_disable_intr()
4763 	IXGBE_WRITE_FLUSH(&sc->hw);  in ixgbe_if_disable_intr()
4774 	    &((struct ixgbe_softc *)iflib_get_softc(ctx))->hw;  in ixgbe_link_intr_enable()
4776 	/* Re-enable other interrupts */  in ixgbe_link_intr_enable()
4787 	struct ix_rx_queue *que = &sc->rx_queues[rxqid];  in ixgbe_if_rx_queue_intr_enable()
4789 	ixgbe_enable_queue(sc, que->msix);  in ixgbe_if_rx_queue_intr_enable()
4800 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_enable_queue()
4804 	if (hw->mac.type == ixgbe_mac_82598EB) {  in ixgbe_enable_queue()
4823 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_disable_queue()
4827 	if (hw->mac.type == ixgbe_mac_82598EB) {  in ixgbe_disable_queue()
4841  * ixgbe_intr - Legacy Interrupt Service Routine
4847 	struct ix_rx_queue *que = sc->rx_queues;  in ixgbe_intr()
4848 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_intr()
4849 	if_ctx_t ctx = sc->ctx;  in ixgbe_intr()
4854 	++que->irqs;  in ixgbe_intr()
4861 	if ((sc->feat_en & IXGBE_FEATURE_FAN_FAIL) &&  in ixgbe_intr()
4863 		device_printf(sc->dev,  in ixgbe_intr()
4876 		/* Pluggable optics-related interrupt */  in ixgbe_intr()
4877 		if (hw->mac.type >= ixgbe_mac_X540)  in ixgbe_intr()
4884 			sc->task_requests |= IXGBE_REQUEST_TASK_MOD;  in ixgbe_intr()
4887 		if ((hw->mac.type == ixgbe_mac_82599EB) &&  in ixgbe_intr()
4891 			sc->task_requests |= IXGBE_REQUEST_TASK_MSF;  in ixgbe_intr()
4896 	if ((hw->phy.type == ixgbe_phy_x550em_ext_t) &&  in ixgbe_intr()
4898 		sc->task_requests |= IXGBE_REQUEST_TASK_PHY;  in ixgbe_intr()
4911 	struct ix_rx_queue *que = sc->rx_queues;  in ixgbe_free_pci_resources()
4914 	/* Release all MSI-X queue resources */  in ixgbe_free_pci_resources()
4915 	if (sc->intr_type == IFLIB_INTR_MSIX)  in ixgbe_free_pci_resources()
4916 		iflib_irq_free(ctx, &sc->irq);  in ixgbe_free_pci_resources()
4919 		for (int i = 0; i < sc->num_rx_queues; i++, que++) {  in ixgbe_free_pci_resources()
4920 			iflib_irq_free(ctx, &que->que_irq);  in ixgbe_free_pci_resources()
4924 	if (sc->pci_mem != NULL)  in ixgbe_free_pci_resources()
4926 		    rman_get_rid(sc->pci_mem), sc->pci_mem);  in ixgbe_free_pci_resources()
4941 	fc = sc->hw.fc.requested_mode;  in ixgbe_sysctl_flowcntl()
4944 	if ((error) || (req->newptr == NULL))  in ixgbe_sysctl_flowcntl()
4948 	if (fc == sc->hw.fc.current_mode)  in ixgbe_sysctl_flowcntl()
4955  * ixgbe_set_flowcntl - Set flow control
4958  *     0 - off
4959  *     1 - rx pause
4960  *     2 - tx pause
4961  *     3 - full
4970 		if (sc->num_rx_queues > 1)  in ixgbe_set_flowcntl()
4974 		if (sc->num_rx_queues > 1)  in ixgbe_set_flowcntl()
4981 	sc->hw.fc.requested_mode = fc;  in ixgbe_set_flowcntl()
4984 	sc->hw.fc.disable_fc_autoneg = true;  in ixgbe_set_flowcntl()
4985 	ixgbe_fc_enable(&sc->hw);  in ixgbe_set_flowcntl()
5002 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_enable_rx_drop()
5006 	for (int i = 0; i < sc->num_rx_queues; i++) {  in ixgbe_enable_rx_drop()
5007 		rxr = &sc->rx_queues[i].rxr;  in ixgbe_enable_rx_drop()
5008 		srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));  in ixgbe_enable_rx_drop()
5010 		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);  in ixgbe_enable_rx_drop()
5014 	for (int i = 0; i < sc->num_vfs; i++) {  in ixgbe_enable_rx_drop()
5028 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_disable_rx_drop()
5032 	for (int i = 0; i < sc->num_rx_queues; i++) {  in ixgbe_disable_rx_drop()
5033 		rxr = &sc->rx_queues[i].rxr;  in ixgbe_disable_rx_drop()
5034 		srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));  in ixgbe_disable_rx_drop()
5036 		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);  in ixgbe_disable_rx_drop()
5040 	for (int i = 0; i < sc->num_vfs; i++) {  in ixgbe_disable_rx_drop()
5058 	if (atomic_load_acq_int(&sc->recovery_mode))  in ixgbe_sysctl_advertise()
5061 	advertise = sc->advertise;  in ixgbe_sysctl_advertise()
5064 	if ((error) || (req->newptr == NULL))  in ixgbe_sysctl_advertise()
5071  * ixgbe_set_advertise - Control advertised link speed
5074  *     0x1  - advertise 100 Mb
5075  *     0x2  - advertise 1G
5076  *     0x4  - advertise 10G
5077  *     0x8  - advertise 10 Mb (yes, Mb)
5078  *     0x10 - advertise 2.5G (disabled by default)
5079  *     0x20 - advertise 5G (disabled by default)
5085 	device_t dev = iflib_get_dev(sc->ctx);  in ixgbe_set_advertise()
5093 	if (sc->advertise == advertise) /* no change */  in ixgbe_set_advertise()
5096 	hw = &sc->hw;  in ixgbe_set_advertise()
5099 	if (hw->phy.media_type == ixgbe_media_type_backplane)  in ixgbe_set_advertise()
5102 	if (!((hw->phy.media_type == ixgbe_media_type_copper) ||  in ixgbe_set_advertise()
5103 	      (hw->phy.multispeed_fiber))) {  in ixgbe_set_advertise()
5117 	if (hw->mac.ops.get_link_capabilities) {  in ixgbe_set_advertise()
5118 		err = hw->mac.ops.get_link_capabilities(hw, &link_caps,  in ixgbe_set_advertise()
5184 	hw->mac.autotry_restart = true;  in ixgbe_set_advertise()
5185 	hw->mac.ops.setup_link(hw, speed, true);  in ixgbe_set_advertise()
5186 	sc->advertise = advertise;  in ixgbe_set_advertise()
5192  * ixgbe_get_default_advertise - Get default advertised speed settings
5196  *     0x1 - advertise 100 Mb
5197  *     0x2 - advertise 1G
5198  *     0x4 - advertise 10G
5199  *     0x8 - advertise 10 Mb (yes, Mb)
5200  *     0x10 - advertise 2.5G (disabled by default)
5201  *     0x20 - advertise 5G (disabled by default)
5206 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_get_default_advertise()
5214 	 * multi-speed fiber  in ixgbe_get_default_advertise()
5216 	if (!(hw->phy.media_type == ixgbe_media_type_copper) &&  in ixgbe_get_default_advertise()
5217 	    !(hw->phy.multispeed_fiber))  in ixgbe_get_default_advertise()
5220 	err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate);  in ixgbe_get_default_advertise()
5224 	if (hw->mac.type == ixgbe_mac_X550) {  in ixgbe_get_default_advertise()
5246  * ixgbe_sysctl_dmac - Manage DMA Coalescing
5249  *     0/1 - off / on (use default value of 1000)
5252  *     50,100,250,500,1000,2000,5000,10000
5260 	if_t ifp = iflib_get_ifp(sc->ctx);  in ixgbe_sysctl_dmac()
5264 	newval = sc->dmac;  in ixgbe_sysctl_dmac()
5266 	if ((error) || (req->newptr == NULL))  in ixgbe_sysctl_dmac()
5272 		sc->dmac = 0;  in ixgbe_sysctl_dmac()
5276 		sc->dmac = 1000;  in ixgbe_sysctl_dmac()
5282 	case 1000:  in ixgbe_sysctl_dmac()
5286 		/* Legal values - allow */  in ixgbe_sysctl_dmac()
5287 		sc->dmac = newval;  in ixgbe_sysctl_dmac()
5294 	/* Re-initialize hardware if it's already running */  in ixgbe_sysctl_dmac()
5307  *     0      - set device to D0
5308  *     3      - set device to D3
5309  *     (none) - get current device power state
5315 	device_t dev = sc->dev;  in ixgbe_sysctl_power_state()
5321 	if ((error) || (req->newptr == NULL))  in ixgbe_sysctl_power_state()
5347  *     0 - disabled
5348  *     1 - enabled
5354 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_sysctl_wol_enable()
5358 	new_wol_enabled = hw->wol_enabled;  in ixgbe_sysctl_wol_enable()
5360 	if ((error) || (req->newptr == NULL))  in ixgbe_sysctl_wol_enable()
5363 	if (new_wol_enabled == hw->wol_enabled)  in ixgbe_sysctl_wol_enable()
5366 	if (new_wol_enabled > 0 && !sc->wol_support)  in ixgbe_sysctl_wol_enable()
5369 		hw->wol_enabled = new_wol_enabled;  in ixgbe_sysctl_wol_enable()
5375  * ixgbe_sysctl_wufc - Wake Up Filter Control
5380  *     0x1  - Link Status Change
5381  *     0x2  - Magic Packet
5382  *     0x4  - Direct Exact
5383  *     0x8  - Directed Multicast
5384  *     0x10 - Broadcast
5385  *     0x20 - ARP/IPv4 Request Packet
5386  *     0x40 - Direct IPv4 Packet
5387  *     0x80 - Direct IPv6 Packet
5398 	new_wufc = sc->wufc;  in ixgbe_sysctl_wufc()
5401 	if ((error) || (req->newptr == NULL))  in ixgbe_sysctl_wufc()
5403 	if (new_wufc == sc->wufc)  in ixgbe_sysctl_wufc()
5410 	new_wufc |= (0xffffff & sc->wufc);  in ixgbe_sysctl_wufc()
5411 	sc->wufc = new_wufc;  in ixgbe_sysctl_wufc()
5424 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_sysctl_print_rss_config()
5425 	device_t dev = sc->dev;  in ixgbe_sysctl_print_rss_config()
5430 	if (atomic_load_acq_int(&sc->recovery_mode))  in ixgbe_sysctl_print_rss_config()
5441 	switch (sc->hw.mac.type) {  in ixgbe_sysctl_print_rss_config()
5459 			reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32));  in ixgbe_sysctl_print_rss_config()
5460 			sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg);  in ixgbe_sysctl_print_rss_config()
5477  * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY
5479  *   For X552/X557-AT devices using an external PHY
5485 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_sysctl_phy_temp()
5488 	if (atomic_load_acq_int(&sc->recovery_mode))  in ixgbe_sysctl_phy_temp()
5491 	if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) {  in ixgbe_sysctl_phy_temp()
5492 		device_printf(iflib_get_dev(sc->ctx),  in ixgbe_sysctl_phy_temp()
5497 	if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP,  in ixgbe_sysctl_phy_temp()
5499 		device_printf(iflib_get_dev(sc->ctx),  in ixgbe_sysctl_phy_temp()
5521 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_sysctl_phy_overtemp_occurred()
5524 	if (atomic_load_acq_int(&sc->recovery_mode))  in ixgbe_sysctl_phy_overtemp_occurred()
5527 	if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) {  in ixgbe_sysctl_phy_overtemp_occurred()
5528 		device_printf(iflib_get_dev(sc->ctx),  in ixgbe_sysctl_phy_overtemp_occurred()
5533 	if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS,  in ixgbe_sysctl_phy_overtemp_occurred()
5535 		device_printf(iflib_get_dev(sc->ctx),  in ixgbe_sysctl_phy_overtemp_occurred()
5551  *     0      - disable EEE
5552  *     1      - enable EEE
5553  *     (none) - get current device EEE state
5559 	device_t dev = sc->dev;  in ixgbe_sysctl_eee_state()
5560 	if_t ifp = iflib_get_ifp(sc->ctx);  in ixgbe_sysctl_eee_state()
5564 	if (atomic_load_acq_int(&sc->recovery_mode))  in ixgbe_sysctl_eee_state()
5567 	curr_eee = new_eee = !!(sc->feat_en & IXGBE_FEATURE_EEE);  in ixgbe_sysctl_eee_state()
5570 	if ((error) || (req->newptr == NULL))  in ixgbe_sysctl_eee_state()
5578 	if (!(sc->feat_cap & IXGBE_FEATURE_EEE))  in ixgbe_sysctl_eee_state()
5585 	retval = ixgbe_setup_eee(&sc->hw, new_eee);  in ixgbe_sysctl_eee_state()
5591 	/* Restart auto-neg */  in ixgbe_sysctl_eee_state()
5598 		sc->feat_en |= IXGBE_FEATURE_EEE;  in ixgbe_sysctl_eee_state()
5600 		sc->feat_en &= ~IXGBE_FEATURE_EEE;  in ixgbe_sysctl_eee_state()
5612 	sc = oidp->oid_arg1;  in ixgbe_sysctl_tso_tcp_flags_mask()
5613 	switch (oidp->oid_arg2) {  in ixgbe_sysctl_tso_tcp_flags_mask()
5630 	val = IXGBE_READ_REG(&sc->hw, reg);  in ixgbe_sysctl_tso_tcp_flags_mask()
5633 	if (error != 0 || req->newptr == NULL)  in ixgbe_sysctl_tso_tcp_flags_mask()
5638 	IXGBE_WRITE_REG(&sc->hw, reg, val);  in ixgbe_sysctl_tso_tcp_flags_mask()
5648 	sc->feat_cap = IXGBE_FEATURE_NETMAP |  in ixgbe_init_device_features()
5655 	switch (sc->hw.mac.type) {  in ixgbe_init_device_features()
5657 		if (sc->hw.device_id == IXGBE_DEV_ID_82598AT)  in ixgbe_init_device_features()
5658 			sc->feat_cap |= IXGBE_FEATURE_FAN_FAIL;  in ixgbe_init_device_features()
5661 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;  in ixgbe_init_device_features()
5662 		sc->feat_cap |= IXGBE_FEATURE_FDIR;  in ixgbe_init_device_features()
5663 		if ((sc->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) &&  in ixgbe_init_device_features()
5664 		    (sc->hw.bus.func == 0))  in ixgbe_init_device_features()
5665 			sc->feat_cap |= IXGBE_FEATURE_BYPASS;  in ixgbe_init_device_features()
5668 		sc->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE;  in ixgbe_init_device_features()
5669 		sc->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;  in ixgbe_init_device_features()
5670 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;  in ixgbe_init_device_features()
5671 		sc->feat_cap |= IXGBE_FEATURE_FDIR;  in ixgbe_init_device_features()
5674 		sc->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE;  in ixgbe_init_device_features()
5675 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;  in ixgbe_init_device_features()
5676 		sc->feat_cap |= IXGBE_FEATURE_FDIR;  in ixgbe_init_device_features()
5677 		if (sc->hw.device_id == IXGBE_DEV_ID_X550EM_X_KR)  in ixgbe_init_device_features()
5678 			sc->feat_cap |= IXGBE_FEATURE_EEE;  in ixgbe_init_device_features()
5681 		sc->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE;  in ixgbe_init_device_features()
5682 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;  in ixgbe_init_device_features()
5683 		sc->feat_cap |= IXGBE_FEATURE_FDIR;  in ixgbe_init_device_features()
5684 		sc->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;  in ixgbe_init_device_features()
5685 		if ((sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) ||  in ixgbe_init_device_features()
5686 		    (sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) {  in ixgbe_init_device_features()
5687 			sc->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;  in ixgbe_init_device_features()
5688 			sc->feat_cap |= IXGBE_FEATURE_EEE;  in ixgbe_init_device_features()
5692 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;  in ixgbe_init_device_features()
5693 		sc->feat_cap |= IXGBE_FEATURE_FDIR;  in ixgbe_init_device_features()
5694 		if ((sc->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) &&  in ixgbe_init_device_features()
5695 		    (sc->hw.bus.func == 0))  in ixgbe_init_device_features()
5696 			sc->feat_cap |= IXGBE_FEATURE_BYPASS;  in ixgbe_init_device_features()
5697 		if (sc->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP)  in ixgbe_init_device_features()
5698 			sc->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;  in ixgbe_init_device_features()
5701 		sc->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE;  in ixgbe_init_device_features()
5702 		sc->feat_cap |= IXGBE_FEATURE_DBG_DUMP;  in ixgbe_init_device_features()
5710 	if (sc->feat_cap & IXGBE_FEATURE_FAN_FAIL)  in ixgbe_init_device_features()
5711 		sc->feat_en |= IXGBE_FEATURE_FAN_FAIL;  in ixgbe_init_device_features()
5713 	if (sc->feat_cap & IXGBE_FEATURE_NETMAP)  in ixgbe_init_device_features()
5714 		sc->feat_en |= IXGBE_FEATURE_NETMAP;  in ixgbe_init_device_features()
5716 	if (sc->feat_cap & IXGBE_FEATURE_EEE)  in ixgbe_init_device_features()
5717 		sc->feat_en |= IXGBE_FEATURE_EEE;  in ixgbe_init_device_features()
5719 	if (sc->feat_cap & IXGBE_FEATURE_TEMP_SENSOR)  in ixgbe_init_device_features()
5720 		sc->feat_en |= IXGBE_FEATURE_TEMP_SENSOR;  in ixgbe_init_device_features()
5722 	if (sc->feat_cap & IXGBE_FEATURE_RECOVERY_MODE)  in ixgbe_init_device_features()
5723 		sc->feat_en |= IXGBE_FEATURE_RECOVERY_MODE;  in ixgbe_init_device_features()
5725 	if (sc->feat_cap & IXGBE_FEATURE_DBG_DUMP)  in ixgbe_init_device_features()
5726 		sc->feat_en |= IXGBE_FEATURE_DBG_DUMP;  in ixgbe_init_device_features()
5731 		if (sc->feat_cap & IXGBE_FEATURE_FDIR)  in ixgbe_init_device_features()
5732 			sc->feat_en |= IXGBE_FEATURE_FDIR;  in ixgbe_init_device_features()
5734 			device_printf(sc->dev,  in ixgbe_init_device_features()
5739 	 * Message Signal Interrupts - Extended (MSI-X)  in ixgbe_init_device_features()
5740 	 * Normal MSI is only enabled if MSI-X calls fail.  in ixgbe_init_device_features()
5743 		sc->feat_cap &= ~IXGBE_FEATURE_MSIX;  in ixgbe_init_device_features()
5744 	/* Receive-Side Scaling (RSS) */  in ixgbe_init_device_features()
5745 	if ((sc->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss)  in ixgbe_init_device_features()
5746 		sc->feat_en |= IXGBE_FEATURE_RSS;  in ixgbe_init_device_features()
5749 	/* No MSI-X */  in ixgbe_init_device_features()
5750 	if (!(sc->feat_cap & IXGBE_FEATURE_MSIX)) {  in ixgbe_init_device_features()
5751 		sc->feat_cap &= ~IXGBE_FEATURE_RSS;  in ixgbe_init_device_features()
5752 		sc->feat_cap &= ~IXGBE_FEATURE_SRIOV;  in ixgbe_init_device_features()
5753 		sc->feat_en &= ~IXGBE_FEATURE_RSS;  in ixgbe_init_device_features()
5754 		sc->feat_en &= ~IXGBE_FEATURE_SRIOV;  in ixgbe_init_device_features()
5766 	mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&sc->hw) :  in ixgbe_check_fan_failure()
5770 		device_printf(sc->dev,  in ixgbe_check_fan_failure()
5826 		sbuf_printf(buf, "%sOption ROM V%d-b%d-p%d",  in ixgbe_sbuf_fw_version()
5845 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_print_fw_version()
5846 	device_t dev = sc->dev;  in ixgbe_print_fw_version()
5874 	struct ixgbe_hw *hw = &sc->hw;  in ixgbe_sysctl_print_fw_version()
5875 	device_t dev = sc->dev;  in ixgbe_sysctl_print_fw_version()