1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
31 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
57 /* i40e_pci_tbl - PCI Device ID Table
97 static int debug = -1;
116 	if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr))  in netdev_hw_addr_refcnt()
117 		ha_list = &netdev->uc;  in netdev_hw_addr_refcnt()
119 		ha_list = &netdev->mc;  in netdev_hw_addr_refcnt()
122 		if (ether_addr_equal(ha->addr, f->macaddr)) {  in netdev_hw_addr_refcnt()
123 			ha->refcount += delta;  in netdev_hw_addr_refcnt()
124 			if (ha->refcount <= 0)  in netdev_hw_addr_refcnt()
125 				ha->refcount = 1;  in netdev_hw_addr_refcnt()
132  * i40e_hw_to_dev - get device pointer from the hardware structure
139 	return &pf->pdev->dev;  in i40e_hw_to_dev()
143  * i40e_allocate_dma_mem - OS specific memory alloc for shared code
154 	mem->size = ALIGN(size, alignment);  in i40e_allocate_dma_mem()
155 	mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,  in i40e_allocate_dma_mem()
157 	if (!mem->va)  in i40e_allocate_dma_mem()
158 		return -ENOMEM;  in i40e_allocate_dma_mem()
164  * i40e_free_dma_mem - OS specific memory free for shared code
172 	dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);  in i40e_free_dma_mem()
173 	mem->va = NULL;  in i40e_free_dma_mem()
174 	mem->pa = 0;  in i40e_free_dma_mem()
175 	mem->size = 0;  in i40e_free_dma_mem()
181  * i40e_allocate_virt_mem - OS specific memory alloc for shared code
189 	mem->size = size;  in i40e_allocate_virt_mem()
190 	mem->va = kzalloc(size, GFP_KERNEL);  in i40e_allocate_virt_mem()
192 	if (!mem->va)  in i40e_allocate_virt_mem()
193 		return -ENOMEM;  in i40e_allocate_virt_mem()
199  * i40e_free_virt_mem - OS specific memory free for shared code
206 	kfree(mem->va);  in i40e_free_virt_mem()
207 	mem->va = NULL;  in i40e_free_virt_mem()
208 	mem->size = 0;  in i40e_free_virt_mem()
214  * i40e_get_lump - find a lump of free generic resource
225 	int ret = -ENOMEM;  in i40e_get_lump()
229 		dev_info(&pf->pdev->dev,  in i40e_get_lump()
232 		return -EINVAL;  in i40e_get_lump()
238 	if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {  in i40e_get_lump()
239 		if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {  in i40e_get_lump()
240 			dev_err(&pf->pdev->dev,  in i40e_get_lump()
242 				pile->num_entries - 1);  in i40e_get_lump()
243 			return -ENOMEM;  in i40e_get_lump()
245 		pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;  in i40e_get_lump()
246 		return pile->num_entries - 1;  in i40e_get_lump()
250 	while (i < pile->num_entries) {  in i40e_get_lump()
252 		if (pile->list[i] & I40E_PILE_VALID_BIT) {  in i40e_get_lump()
258 		for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {  in i40e_get_lump()
259 			if (pile->list[i+j] & I40E_PILE_VALID_BIT)  in i40e_get_lump()
266 				pile->list[i+j] = id | I40E_PILE_VALID_BIT;  in i40e_get_lump()
279  * i40e_put_lump - return a lump of generic resource
292 	if (!pile || index >= pile->num_entries)  in i40e_put_lump()
293 		return -EINVAL;  in i40e_put_lump()
296 	     i < pile->num_entries && pile->list[i] == valid_id;  in i40e_put_lump()
298 		pile->list[i] = 0;  in i40e_put_lump()
307  * i40e_find_vsi_from_id - searches for the vsi with the given id
317 		if (vsi->id == id)  in i40e_find_vsi_from_id()
324  * i40e_service_event_schedule - Schedule the service task to wake up
331 	if ((!test_bit(__I40E_DOWN, pf->state) &&  in i40e_service_event_schedule()
332 	     !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||  in i40e_service_event_schedule()
333 	      test_bit(__I40E_RECOVERY_MODE, pf->state))  in i40e_service_event_schedule()
334 		queue_work(i40e_wq, &pf->service_task);  in i40e_service_event_schedule()
338  * i40e_tx_timeout - Respond to a Tx Hang
349 	struct i40e_vsi *vsi = np->vsi;  in i40e_tx_timeout()
350 	struct i40e_pf *pf = vsi->back;  in i40e_tx_timeout()
355 	pf->tx_timeout_count++;  in i40e_tx_timeout()
358 	for (i = 0; i < vsi->num_queue_pairs; i++) {  in i40e_tx_timeout()
359 		if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {  in i40e_tx_timeout()
361 			    vsi->tx_rings[i]->queue_index) {  in i40e_tx_timeout()
362 				tx_ring = vsi->tx_rings[i];  in i40e_tx_timeout()
368 	if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))  in i40e_tx_timeout()
369 		pf->tx_timeout_recovery_level = 1;  /* reset after some time */  in i40e_tx_timeout()
371 		      (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))  in i40e_tx_timeout()
375 	if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))  in i40e_tx_timeout()
381 		if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_tx_timeout()
382 			val = rd32(&pf->hw,  in i40e_tx_timeout()
383 			     I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +  in i40e_tx_timeout()
384 						tx_ring->vsi->base_vector - 1));  in i40e_tx_timeout()
386 			val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);  in i40e_tx_timeout()
389 			    vsi->seid, txqueue, tx_ring->next_to_clean,  in i40e_tx_timeout()
390 			    head, tx_ring->next_to_use,  in i40e_tx_timeout()
391 			    readl(tx_ring->tail), val);  in i40e_tx_timeout()
394 	pf->tx_timeout_last_recovery = jiffies;  in i40e_tx_timeout()
396 		    pf->tx_timeout_recovery_level, txqueue);  in i40e_tx_timeout()
398 	switch (pf->tx_timeout_recovery_level) {  in i40e_tx_timeout()
400 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);  in i40e_tx_timeout()
403 		set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);  in i40e_tx_timeout()
406 		set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);  in i40e_tx_timeout()
409 		netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");  in i40e_tx_timeout()
410 		set_bit(__I40E_DOWN_REQUESTED, pf->state);  in i40e_tx_timeout()
411 		set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);  in i40e_tx_timeout()
416 	pf->tx_timeout_recovery_level++;  in i40e_tx_timeout()
420  * i40e_get_vsi_stats_struct - Get System Network Statistics
428 	return &vsi->net_stats;  in i40e_get_vsi_stats_struct()
432  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
443 		start = u64_stats_fetch_begin(&ring->syncp);  in i40e_get_netdev_stats_struct_tx()
444 		packets = ring->stats.packets;  in i40e_get_netdev_stats_struct_tx()
445 		bytes   = ring->stats.bytes;  in i40e_get_netdev_stats_struct_tx()
446 	} while (u64_stats_fetch_retry(&ring->syncp, start));  in i40e_get_netdev_stats_struct_tx()
448 	stats->tx_packets += packets;  in i40e_get_netdev_stats_struct_tx()
449 	stats->tx_bytes   += bytes;  in i40e_get_netdev_stats_struct_tx()
453  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
464 	struct i40e_vsi *vsi = np->vsi;  in i40e_get_netdev_stats_struct()
469 	if (test_bit(__I40E_VSI_DOWN, vsi->state))  in i40e_get_netdev_stats_struct()
472 	if (!vsi->tx_rings)  in i40e_get_netdev_stats_struct()
476 	for (i = 0; i < vsi->num_queue_pairs; i++) {  in i40e_get_netdev_stats_struct()
480 		ring = READ_ONCE(vsi->tx_rings[i]);  in i40e_get_netdev_stats_struct()
486 			ring = READ_ONCE(vsi->xdp_rings[i]);  in i40e_get_netdev_stats_struct()
492 		ring = READ_ONCE(vsi->rx_rings[i]);  in i40e_get_netdev_stats_struct()
496 			start   = u64_stats_fetch_begin(&ring->syncp);  in i40e_get_netdev_stats_struct()
497 			packets = ring->stats.packets;  in i40e_get_netdev_stats_struct()
498 			bytes   = ring->stats.bytes;  in i40e_get_netdev_stats_struct()
499 		} while (u64_stats_fetch_retry(&ring->syncp, start));  in i40e_get_netdev_stats_struct()
501 		stats->rx_packets += packets;  in i40e_get_netdev_stats_struct()
502 		stats->rx_bytes   += bytes;  in i40e_get_netdev_stats_struct()
508 	stats->multicast	= vsi_stats->multicast;  in i40e_get_netdev_stats_struct()
509 	stats->tx_errors	= vsi_stats->tx_errors;  in i40e_get_netdev_stats_struct()
510 	stats->tx_dropped	= vsi_stats->tx_dropped;  in i40e_get_netdev_stats_struct()
511 	stats->rx_errors	= vsi_stats->rx_errors;  in i40e_get_netdev_stats_struct()
512 	stats->rx_dropped	= vsi_stats->rx_dropped;  in i40e_get_netdev_stats_struct()
513 	stats->rx_missed_errors	= vsi_stats->rx_missed_errors;  in i40e_get_netdev_stats_struct()
514 	stats->rx_crc_errors	= vsi_stats->rx_crc_errors;  in i40e_get_netdev_stats_struct()
515 	stats->rx_length_errors	= vsi_stats->rx_length_errors;  in i40e_get_netdev_stats_struct()
519  * i40e_vsi_reset_stats - Resets all stats of the given vsi
532 	memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));  in i40e_vsi_reset_stats()
533 	memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));  in i40e_vsi_reset_stats()
534 	memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));  in i40e_vsi_reset_stats()
535 	if (vsi->rx_rings && vsi->rx_rings[0]) {  in i40e_vsi_reset_stats()
536 		for (i = 0; i < vsi->num_queue_pairs; i++) {  in i40e_vsi_reset_stats()
537 			memset(&vsi->rx_rings[i]->stats, 0,  in i40e_vsi_reset_stats()
538 			       sizeof(vsi->rx_rings[i]->stats));  in i40e_vsi_reset_stats()
539 			memset(&vsi->rx_rings[i]->rx_stats, 0,  in i40e_vsi_reset_stats()
540 			       sizeof(vsi->rx_rings[i]->rx_stats));  in i40e_vsi_reset_stats()
541 			memset(&vsi->tx_rings[i]->stats, 0,  in i40e_vsi_reset_stats()
542 			       sizeof(vsi->tx_rings[i]->stats));  in i40e_vsi_reset_stats()
543 			memset(&vsi->tx_rings[i]->tx_stats, 0,  in i40e_vsi_reset_stats()
544 			       sizeof(vsi->tx_rings[i]->tx_stats));  in i40e_vsi_reset_stats()
547 	vsi->stat_offsets_loaded = false;  in i40e_vsi_reset_stats()
551  * i40e_pf_reset_stats - Reset all of the stats for the given PF
559 	memset(&pf->stats, 0, sizeof(pf->stats));  in i40e_pf_reset_stats()
560 	memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));  in i40e_pf_reset_stats()
561 	pf->stat_offsets_loaded = false;  in i40e_pf_reset_stats()
564 		memset(&veb->stats, 0, sizeof(veb->stats));  in i40e_pf_reset_stats()
565 		memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets));  in i40e_pf_reset_stats()
566 		memset(&veb->tc_stats, 0, sizeof(veb->tc_stats));  in i40e_pf_reset_stats()
567 		memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets));  in i40e_pf_reset_stats()
568 		veb->stat_offsets_loaded = false;  in i40e_pf_reset_stats()
570 	pf->hw_csum_rx_error = 0;  in i40e_pf_reset_stats()
574  * i40e_compute_pci_to_hw_id - compute index form PCI function.
582 	if (vsi->type == I40E_VSI_SRIOV)  in i40e_compute_pci_to_hw_id()
583 		return (hw->port * BIT(7)) / pf_count + vsi->vf_id;  in i40e_compute_pci_to_hw_id()
585 	return hw->port + BIT(7);  in i40e_compute_pci_to_hw_id()
589  * i40e_stat_update64 - read and update a 64 bit stat from the chip.
593  * @offset_loaded: has the initial offset been loaded yet.
594  * @offset: ptr to current offset value.
603 			       bool offset_loaded, u64 *offset, u64 *stat)  in i40e_stat_update64()  argument
609 	if (!offset_loaded || new_data < *offset)  in i40e_stat_update64()
610 		*offset = new_data;  in i40e_stat_update64()
611 	*stat = new_data - *offset;  in i40e_stat_update64()
615  * i40e_stat_update48 - read and update a 48 bit stat from the chip
619  * @offset_loaded: has the initial offset been loaded yet
620  * @offset: ptr to current offset value
627  * the potential roll-over.
630 			       bool offset_loaded, u64 *offset, u64 *stat)  in i40e_stat_update48()  argument
634 	if (hw->device_id == I40E_DEV_ID_QEMU) {  in i40e_stat_update48()
641 		*offset = new_data;  in i40e_stat_update48()
642 	if (likely(new_data >= *offset))  in i40e_stat_update48()
643 		*stat = new_data - *offset;  in i40e_stat_update48()
645 		*stat = (new_data + BIT_ULL(48)) - *offset;  in i40e_stat_update48()
650  * i40e_stat_update32 - read and update a 32 bit stat from the chip
653  * @offset_loaded: has the initial offset been loaded yet
654  * @offset: ptr to current offset value
658 			       bool offset_loaded, u64 *offset, u64 *stat)  in i40e_stat_update32()  argument
664 		*offset = new_data;  in i40e_stat_update32()
665 	if (likely(new_data >= *offset))  in i40e_stat_update32()
666 		*stat = (u32)(new_data - *offset);  in i40e_stat_update32()
668 		*stat = (u32)((new_data + BIT_ULL(32)) - *offset);  in i40e_stat_update32()
672  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
686  * i40e_stats_update_rx_discards - update rx_discards.
701 			   &stat_offset->rx_discards, &stat->rx_discards);  in i40e_stats_update_rx_discards()
705 			   offset_loaded, &stat_offset->rx_discards_other,  in i40e_stats_update_rx_discards()
706 			   &stat->rx_discards_other);  in i40e_stats_update_rx_discards()
710  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
715 	int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);  in i40e_update_eth_stats()
716 	struct i40e_pf *pf = vsi->back;  in i40e_update_eth_stats()
717 	struct i40e_hw *hw = &pf->hw;  in i40e_update_eth_stats()
721 	es = &vsi->eth_stats;  in i40e_update_eth_stats()
722 	oes = &vsi->eth_stats_offsets;  in i40e_update_eth_stats()
726 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
727 			   &oes->tx_errors, &es->tx_errors);  in i40e_update_eth_stats()
729 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
730 			   &oes->rx_unknown_protocol, &es->rx_unknown_protocol);  in i40e_update_eth_stats()
734 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
735 			   &oes->rx_bytes, &es->rx_bytes);  in i40e_update_eth_stats()
738 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
739 			   &oes->rx_unicast, &es->rx_unicast);  in i40e_update_eth_stats()
742 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
743 			   &oes->rx_multicast, &es->rx_multicast);  in i40e_update_eth_stats()
746 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
747 			   &oes->rx_broadcast, &es->rx_broadcast);  in i40e_update_eth_stats()
751 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
752 			   &oes->tx_bytes, &es->tx_bytes);  in i40e_update_eth_stats()
755 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
756 			   &oes->tx_unicast, &es->tx_unicast);  in i40e_update_eth_stats()
759 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
760 			   &oes->tx_multicast, &es->tx_multicast);  in i40e_update_eth_stats()
763 			   vsi->stat_offsets_loaded,  in i40e_update_eth_stats()
764 			   &oes->tx_broadcast, &es->tx_broadcast);  in i40e_update_eth_stats()
767 				      vsi->stat_offsets_loaded, oes, es);  in i40e_update_eth_stats()
769 	vsi->stat_offsets_loaded = true;  in i40e_update_eth_stats()
773  * i40e_update_veb_stats - Update Switch component statistics
778 	struct i40e_pf *pf = veb->pf;  in i40e_update_veb_stats()
779 	struct i40e_hw *hw = &pf->hw;  in i40e_update_veb_stats()
786 	idx = veb->stats_idx;  in i40e_update_veb_stats()
787 	es = &veb->stats;  in i40e_update_veb_stats()
788 	oes = &veb->stats_offsets;  in i40e_update_veb_stats()
789 	veb_es = &veb->tc_stats;  in i40e_update_veb_stats()
790 	veb_oes = &veb->tc_stats_offsets;  in i40e_update_veb_stats()
794 			   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
795 			   &oes->tx_discards, &es->tx_discards);  in i40e_update_veb_stats()
796 	if (hw->revision_id > 0)  in i40e_update_veb_stats()
798 				   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
799 				   &oes->rx_unknown_protocol,  in i40e_update_veb_stats()
800 				   &es->rx_unknown_protocol);  in i40e_update_veb_stats()
802 			   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
803 			   &oes->rx_bytes, &es->rx_bytes);  in i40e_update_veb_stats()
805 			   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
806 			   &oes->rx_unicast, &es->rx_unicast);  in i40e_update_veb_stats()
808 			   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
809 			   &oes->rx_multicast, &es->rx_multicast);  in i40e_update_veb_stats()
811 			   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
812 			   &oes->rx_broadcast, &es->rx_broadcast);  in i40e_update_veb_stats()
815 			   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
816 			   &oes->tx_bytes, &es->tx_bytes);  in i40e_update_veb_stats()
818 			   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
819 			   &oes->tx_unicast, &es->tx_unicast);  in i40e_update_veb_stats()
821 			   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
822 			   &oes->tx_multicast, &es->tx_multicast);  in i40e_update_veb_stats()
824 			   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
825 			   &oes->tx_broadcast, &es->tx_broadcast);  in i40e_update_veb_stats()
829 				   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
830 				   &veb_oes->tc_rx_packets[i],  in i40e_update_veb_stats()
831 				   &veb_es->tc_rx_packets[i]);  in i40e_update_veb_stats()
834 				   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
835 				   &veb_oes->tc_rx_bytes[i],  in i40e_update_veb_stats()
836 				   &veb_es->tc_rx_bytes[i]);  in i40e_update_veb_stats()
839 				   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
840 				   &veb_oes->tc_tx_packets[i],  in i40e_update_veb_stats()
841 				   &veb_es->tc_tx_packets[i]);  in i40e_update_veb_stats()
844 				   veb->stat_offsets_loaded,  in i40e_update_veb_stats()
845 				   &veb_oes->tc_tx_bytes[i],  in i40e_update_veb_stats()
846 				   &veb_es->tc_tx_bytes[i]);  in i40e_update_veb_stats()
848 	veb->stat_offsets_loaded = true;  in i40e_update_veb_stats()
852  * i40e_update_vsi_stats - Update the vsi statistics counters.
864 	struct i40e_pf *pf = vsi->back;  in i40e_update_vsi_stats()
880 	if (test_bit(__I40E_VSI_DOWN, vsi->state) ||  in i40e_update_vsi_stats()
881 	    test_bit(__I40E_CONFIG_BUSY, pf->state))  in i40e_update_vsi_stats()
885 	ons = &vsi->net_stats_offsets;  in i40e_update_vsi_stats()
886 	es = &vsi->eth_stats;  in i40e_update_vsi_stats()
887 	oes = &vsi->eth_stats_offsets;  in i40e_update_vsi_stats()
903 	for (q = 0; q < vsi->num_queue_pairs; q++) {  in i40e_update_vsi_stats()
905 		p = READ_ONCE(vsi->tx_rings[q]);  in i40e_update_vsi_stats()
910 			start = u64_stats_fetch_begin(&p->syncp);  in i40e_update_vsi_stats()
911 			packets = p->stats.packets;  in i40e_update_vsi_stats()
912 			bytes = p->stats.bytes;  in i40e_update_vsi_stats()
913 		} while (u64_stats_fetch_retry(&p->syncp, start));  in i40e_update_vsi_stats()
916 		tx_restart += p->tx_stats.restart_queue;  in i40e_update_vsi_stats()
917 		tx_busy += p->tx_stats.tx_busy;  in i40e_update_vsi_stats()
918 		tx_linearize += p->tx_stats.tx_linearize;  in i40e_update_vsi_stats()
919 		tx_force_wb += p->tx_stats.tx_force_wb;  in i40e_update_vsi_stats()
920 		tx_stopped += p->tx_stats.tx_stopped;  in i40e_update_vsi_stats()
923 		p = READ_ONCE(vsi->rx_rings[q]);  in i40e_update_vsi_stats()
928 			start = u64_stats_fetch_begin(&p->syncp);  in i40e_update_vsi_stats()
929 			packets = p->stats.packets;  in i40e_update_vsi_stats()
930 			bytes = p->stats.bytes;  in i40e_update_vsi_stats()
931 		} while (u64_stats_fetch_retry(&p->syncp, start));  in i40e_update_vsi_stats()
934 		rx_buf += p->rx_stats.alloc_buff_failed;  in i40e_update_vsi_stats()
935 		rx_page += p->rx_stats.alloc_page_failed;  in i40e_update_vsi_stats()
936 		rx_reuse += p->rx_stats.page_reuse_count;  in i40e_update_vsi_stats()
937 		rx_alloc += p->rx_stats.page_alloc_count;  in i40e_update_vsi_stats()
938 		rx_waive += p->rx_stats.page_waive_count;  in i40e_update_vsi_stats()
939 		rx_busy += p->rx_stats.page_busy_count;  in i40e_update_vsi_stats()
943 			p = READ_ONCE(vsi->xdp_rings[q]);  in i40e_update_vsi_stats()
948 				start = u64_stats_fetch_begin(&p->syncp);  in i40e_update_vsi_stats()
949 				packets = p->stats.packets;  in i40e_update_vsi_stats()
950 				bytes = p->stats.bytes;  in i40e_update_vsi_stats()
951 			} while (u64_stats_fetch_retry(&p->syncp, start));  in i40e_update_vsi_stats()
954 			tx_restart += p->tx_stats.restart_queue;  in i40e_update_vsi_stats()
955 			tx_busy += p->tx_stats.tx_busy;  in i40e_update_vsi_stats()
956 			tx_linearize += p->tx_stats.tx_linearize;  in i40e_update_vsi_stats()
957 			tx_force_wb += p->tx_stats.tx_force_wb;  in i40e_update_vsi_stats()
961 	vsi->tx_restart = tx_restart;  in i40e_update_vsi_stats()
962 	vsi->tx_busy = tx_busy;  in i40e_update_vsi_stats()
963 	vsi->tx_linearize = tx_linearize;  in i40e_update_vsi_stats()
964 	vsi->tx_force_wb = tx_force_wb;  in i40e_update_vsi_stats()
965 	vsi->tx_stopped = tx_stopped;  in i40e_update_vsi_stats()
966 	vsi->rx_page_failed = rx_page;  in i40e_update_vsi_stats()
967 	vsi->rx_buf_failed = rx_buf;  in i40e_update_vsi_stats()
968 	vsi->rx_page_reuse = rx_reuse;  in i40e_update_vsi_stats()
969 	vsi->rx_page_alloc = rx_alloc;  in i40e_update_vsi_stats()
970 	vsi->rx_page_waive = rx_waive;  in i40e_update_vsi_stats()
971 	vsi->rx_page_busy = rx_busy;  in i40e_update_vsi_stats()
973 	ns->rx_packets = rx_p;  in i40e_update_vsi_stats()
974 	ns->rx_bytes = rx_b;  in i40e_update_vsi_stats()
975 	ns->tx_packets = tx_p;  in i40e_update_vsi_stats()
976 	ns->tx_bytes = tx_b;  in i40e_update_vsi_stats()
980 	ons->tx_errors = oes->tx_errors;  in i40e_update_vsi_stats()
981 	ns->tx_errors = es->tx_errors;  in i40e_update_vsi_stats()
982 	ons->multicast = oes->rx_multicast;  in i40e_update_vsi_stats()
983 	ns->multicast = es->rx_multicast;  in i40e_update_vsi_stats()
984 	ons->rx_dropped = oes->rx_discards_other;  in i40e_update_vsi_stats()
985 	ns->rx_dropped = es->rx_discards_other;  in i40e_update_vsi_stats()
986 	ons->rx_missed_errors = oes->rx_discards;  in i40e_update_vsi_stats()
987 	ns->rx_missed_errors = es->rx_discards;  in i40e_update_vsi_stats()
988 	ons->tx_dropped = oes->tx_discards;  in i40e_update_vsi_stats()
989 	ns->tx_dropped = es->tx_discards;  in i40e_update_vsi_stats()
992 	if (vsi->type == I40E_VSI_MAIN) {  in i40e_update_vsi_stats()
993 		ns->rx_crc_errors = pf->stats.crc_errors;  in i40e_update_vsi_stats()
994 		ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;  in i40e_update_vsi_stats()
995 		ns->rx_length_errors = pf->stats.rx_length_errors;  in i40e_update_vsi_stats()
1000  * i40e_update_pf_stats - Update the PF statistics counters.
1005 	struct i40e_hw_port_stats *osd = &pf->stats_offsets;  in i40e_update_pf_stats()
1006 	struct i40e_hw_port_stats *nsd = &pf->stats;  in i40e_update_pf_stats()
1007 	struct i40e_hw *hw = &pf->hw;  in i40e_update_pf_stats()
1011 	i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),  in i40e_update_pf_stats()
1012 			   I40E_GLPRT_GORCL(hw->port),  in i40e_update_pf_stats()
1013 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1014 			   &osd->eth.rx_bytes, &nsd->eth.rx_bytes);  in i40e_update_pf_stats()
1015 	i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),  in i40e_update_pf_stats()
1016 			   I40E_GLPRT_GOTCL(hw->port),  in i40e_update_pf_stats()
1017 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1018 			   &osd->eth.tx_bytes, &nsd->eth.tx_bytes);  in i40e_update_pf_stats()
1019 	i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),  in i40e_update_pf_stats()
1020 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1021 			   &osd->eth.rx_discards,  in i40e_update_pf_stats()
1022 			   &nsd->eth.rx_discards);  in i40e_update_pf_stats()
1023 	i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),  in i40e_update_pf_stats()
1024 			   I40E_GLPRT_UPRCL(hw->port),  in i40e_update_pf_stats()
1025 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1026 			   &osd->eth.rx_unicast,  in i40e_update_pf_stats()
1027 			   &nsd->eth.rx_unicast);  in i40e_update_pf_stats()
1028 	i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),  in i40e_update_pf_stats()
1029 			   I40E_GLPRT_MPRCL(hw->port),  in i40e_update_pf_stats()
1030 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1031 			   &osd->eth.rx_multicast,  in i40e_update_pf_stats()
1032 			   &nsd->eth.rx_multicast);  in i40e_update_pf_stats()
1033 	i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),  in i40e_update_pf_stats()
1034 			   I40E_GLPRT_BPRCL(hw->port),  in i40e_update_pf_stats()
1035 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1036 			   &osd->eth.rx_broadcast,  in i40e_update_pf_stats()
1037 			   &nsd->eth.rx_broadcast);  in i40e_update_pf_stats()
1038 	i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),  in i40e_update_pf_stats()
1039 			   I40E_GLPRT_UPTCL(hw->port),  in i40e_update_pf_stats()
1040 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1041 			   &osd->eth.tx_unicast,  in i40e_update_pf_stats()
1042 			   &nsd->eth.tx_unicast);  in i40e_update_pf_stats()
1043 	i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),  in i40e_update_pf_stats()
1044 			   I40E_GLPRT_MPTCL(hw->port),  in i40e_update_pf_stats()
1045 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1046 			   &osd->eth.tx_multicast,  in i40e_update_pf_stats()
1047 			   &nsd->eth.tx_multicast);  in i40e_update_pf_stats()
1048 	i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),  in i40e_update_pf_stats()
1049 			   I40E_GLPRT_BPTCL(hw->port),  in i40e_update_pf_stats()
1050 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1051 			   &osd->eth.tx_broadcast,  in i40e_update_pf_stats()
1052 			   &nsd->eth.tx_broadcast);  in i40e_update_pf_stats()
1054 	i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),  in i40e_update_pf_stats()
1055 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1056 			   &osd->tx_dropped_link_down,  in i40e_update_pf_stats()
1057 			   &nsd->tx_dropped_link_down);  in i40e_update_pf_stats()
1059 	i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),  in i40e_update_pf_stats()
1060 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1061 			   &osd->crc_errors, &nsd->crc_errors);  in i40e_update_pf_stats()
1063 	i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),  in i40e_update_pf_stats()
1064 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1065 			   &osd->illegal_bytes, &nsd->illegal_bytes);  in i40e_update_pf_stats()
1067 	i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),  in i40e_update_pf_stats()
1068 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1069 			   &osd->mac_local_faults,  in i40e_update_pf_stats()
1070 			   &nsd->mac_local_faults);  in i40e_update_pf_stats()
1071 	i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),  in i40e_update_pf_stats()
1072 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1073 			   &osd->mac_remote_faults,  in i40e_update_pf_stats()
1074 			   &nsd->mac_remote_faults);  in i40e_update_pf_stats()
1076 	i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),  in i40e_update_pf_stats()
1077 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1078 			   &osd->rx_length_errors,  in i40e_update_pf_stats()
1079 			   &nsd->rx_length_errors);  in i40e_update_pf_stats()
1081 	i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),  in i40e_update_pf_stats()
1082 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1083 			   &osd->link_xon_rx, &nsd->link_xon_rx);  in i40e_update_pf_stats()
1084 	i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),  in i40e_update_pf_stats()
1085 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1086 			   &osd->link_xon_tx, &nsd->link_xon_tx);  in i40e_update_pf_stats()
1087 	i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),  in i40e_update_pf_stats()
1088 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1089 			   &osd->link_xoff_rx, &nsd->link_xoff_rx);  in i40e_update_pf_stats()
1090 	i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),  in i40e_update_pf_stats()
1091 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1092 			   &osd->link_xoff_tx, &nsd->link_xoff_tx);  in i40e_update_pf_stats()
1095 		i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),  in i40e_update_pf_stats()
1096 				   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1097 				   &osd->priority_xoff_rx[i],  in i40e_update_pf_stats()
1098 				   &nsd->priority_xoff_rx[i]);  in i40e_update_pf_stats()
1099 		i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),  in i40e_update_pf_stats()
1100 				   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1101 				   &osd->priority_xon_rx[i],  in i40e_update_pf_stats()
1102 				   &nsd->priority_xon_rx[i]);  in i40e_update_pf_stats()
1103 		i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),  in i40e_update_pf_stats()
1104 				   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1105 				   &osd->priority_xon_tx[i],  in i40e_update_pf_stats()
1106 				   &nsd->priority_xon_tx[i]);  in i40e_update_pf_stats()
1107 		i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),  in i40e_update_pf_stats()
1108 				   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1109 				   &osd->priority_xoff_tx[i],  in i40e_update_pf_stats()
1110 				   &nsd->priority_xoff_tx[i]);  in i40e_update_pf_stats()
1112 				   I40E_GLPRT_RXON2OFFCNT(hw->port, i),  in i40e_update_pf_stats()
1113 				   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1114 				   &osd->priority_xon_2_xoff[i],  in i40e_update_pf_stats()
1115 				   &nsd->priority_xon_2_xoff[i]);  in i40e_update_pf_stats()
1118 	i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),  in i40e_update_pf_stats()
1119 			   I40E_GLPRT_PRC64L(hw->port),  in i40e_update_pf_stats()
1120 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1121 			   &osd->rx_size_64, &nsd->rx_size_64);  in i40e_update_pf_stats()
1122 	i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),  in i40e_update_pf_stats()
1123 			   I40E_GLPRT_PRC127L(hw->port),  in i40e_update_pf_stats()
1124 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1125 			   &osd->rx_size_127, &nsd->rx_size_127);  in i40e_update_pf_stats()
1126 	i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),  in i40e_update_pf_stats()
1127 			   I40E_GLPRT_PRC255L(hw->port),  in i40e_update_pf_stats()
1128 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1129 			   &osd->rx_size_255, &nsd->rx_size_255);  in i40e_update_pf_stats()
1130 	i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),  in i40e_update_pf_stats()
1131 			   I40E_GLPRT_PRC511L(hw->port),  in i40e_update_pf_stats()
1132 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1133 			   &osd->rx_size_511, &nsd->rx_size_511);  in i40e_update_pf_stats()
1134 	i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),  in i40e_update_pf_stats()
1135 			   I40E_GLPRT_PRC1023L(hw->port),  in i40e_update_pf_stats()
1136 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1137 			   &osd->rx_size_1023, &nsd->rx_size_1023);  in i40e_update_pf_stats()
1138 	i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),  in i40e_update_pf_stats()
1139 			   I40E_GLPRT_PRC1522L(hw->port),  in i40e_update_pf_stats()
1140 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1141 			   &osd->rx_size_1522, &nsd->rx_size_1522);  in i40e_update_pf_stats()
1142 	i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),  in i40e_update_pf_stats()
1143 			   I40E_GLPRT_PRC9522L(hw->port),  in i40e_update_pf_stats()
1144 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1145 			   &osd->rx_size_big, &nsd->rx_size_big);  in i40e_update_pf_stats()
1147 	i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),  in i40e_update_pf_stats()
1148 			   I40E_GLPRT_PTC64L(hw->port),  in i40e_update_pf_stats()
1149 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1150 			   &osd->tx_size_64, &nsd->tx_size_64);  in i40e_update_pf_stats()
1151 	i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),  in i40e_update_pf_stats()
1152 			   I40E_GLPRT_PTC127L(hw->port),  in i40e_update_pf_stats()
1153 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1154 			   &osd->tx_size_127, &nsd->tx_size_127);  in i40e_update_pf_stats()
1155 	i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),  in i40e_update_pf_stats()
1156 			   I40E_GLPRT_PTC255L(hw->port),  in i40e_update_pf_stats()
1157 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1158 			   &osd->tx_size_255, &nsd->tx_size_255);  in i40e_update_pf_stats()
1159 	i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),  in i40e_update_pf_stats()
1160 			   I40E_GLPRT_PTC511L(hw->port),  in i40e_update_pf_stats()
1161 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1162 			   &osd->tx_size_511, &nsd->tx_size_511);  in i40e_update_pf_stats()
1163 	i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),  in i40e_update_pf_stats()
1164 			   I40E_GLPRT_PTC1023L(hw->port),  in i40e_update_pf_stats()
1165 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1166 			   &osd->tx_size_1023, &nsd->tx_size_1023);  in i40e_update_pf_stats()
1167 	i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),  in i40e_update_pf_stats()
1168 			   I40E_GLPRT_PTC1522L(hw->port),  in i40e_update_pf_stats()
1169 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1170 			   &osd->tx_size_1522, &nsd->tx_size_1522);  in i40e_update_pf_stats()
1171 	i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),  in i40e_update_pf_stats()
1172 			   I40E_GLPRT_PTC9522L(hw->port),  in i40e_update_pf_stats()
1173 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1174 			   &osd->tx_size_big, &nsd->tx_size_big);  in i40e_update_pf_stats()
1176 	i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),  in i40e_update_pf_stats()
1177 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1178 			   &osd->rx_undersize, &nsd->rx_undersize);  in i40e_update_pf_stats()
1179 	i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),  in i40e_update_pf_stats()
1180 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1181 			   &osd->rx_fragments, &nsd->rx_fragments);  in i40e_update_pf_stats()
1182 	i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),  in i40e_update_pf_stats()
1183 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1184 			   &osd->rx_oversize, &nsd->rx_oversize);  in i40e_update_pf_stats()
1185 	i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),  in i40e_update_pf_stats()
1186 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1187 			   &osd->rx_jabber, &nsd->rx_jabber);  in i40e_update_pf_stats()
1191 			I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),  in i40e_update_pf_stats()
1192 			&nsd->fd_atr_match);  in i40e_update_pf_stats()
1194 			I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),  in i40e_update_pf_stats()
1195 			&nsd->fd_sb_match);  in i40e_update_pf_stats()
1197 			I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),  in i40e_update_pf_stats()
1198 			&nsd->fd_atr_tunnel_match);  in i40e_update_pf_stats()
1201 	nsd->tx_lpi_status =  in i40e_update_pf_stats()
1203 	nsd->rx_lpi_status =  in i40e_update_pf_stats()
1206 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1207 			   &osd->tx_lpi_count, &nsd->tx_lpi_count);  in i40e_update_pf_stats()
1209 			   pf->stat_offsets_loaded,  in i40e_update_pf_stats()
1210 			   &osd->rx_lpi_count, &nsd->rx_lpi_count);  in i40e_update_pf_stats()
1212 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&  in i40e_update_pf_stats()
1213 	    !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))  in i40e_update_pf_stats()
1214 		nsd->fd_sb_status = true;  in i40e_update_pf_stats()
1216 		nsd->fd_sb_status = false;  in i40e_update_pf_stats()
1218 	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&  in i40e_update_pf_stats()
1219 	    !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))  in i40e_update_pf_stats()
1220 		nsd->fd_atr_status = true;  in i40e_update_pf_stats()
1222 		nsd->fd_atr_status = false;  in i40e_update_pf_stats()
1224 	pf->stat_offsets_loaded = true;  in i40e_update_pf_stats()
1228  * i40e_update_stats - Update the various statistics counters.
1235 	struct i40e_pf *pf = vsi->back;  in i40e_update_stats()
1237 	if (vsi->type == I40E_VSI_MAIN)  in i40e_update_stats()
1244  * i40e_count_filters - counts VSI mac filters
1256 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {  in i40e_count_filters()
1257 		if (f->state == I40E_FILTER_NEW ||  in i40e_count_filters()
1258 		    f->state == I40E_FILTER_NEW_SYNC ||  in i40e_count_filters()
1259 		    f->state == I40E_FILTER_ACTIVE)  in i40e_count_filters()
1267  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1284 	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {  in i40e_find_filter()
1285 		if ((ether_addr_equal(macaddr, f->macaddr)) &&  in i40e_find_filter()
1286 		    (vlan == f->vlan))  in i40e_find_filter()
1293  * i40e_find_mac - Find a mac addr in the macvlan filters list
1309 	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {  in i40e_find_mac()
1310 		if ((ether_addr_equal(macaddr, f->macaddr)))  in i40e_find_mac()
1317  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1325 	if (vsi->info.pvid)  in i40e_is_vsi_in_vlan()
1348 	return vsi->has_vlan_filter;  in i40e_is_vsi_in_vlan()
1352  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1358  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1360  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1362  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1367  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1372  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1385 	s16 pvid = le16_to_cpu(vsi->info.pvid);  in i40e_correct_mac_vlan_filters()
1398 	 *    which are marked as VLAN=-1 must be replaced with  in i40e_correct_mac_vlan_filters()
1402 	 *    marked as VLAN=-1  in i40e_correct_mac_vlan_filters()
1407 		if (pvid && new->f->vlan != pvid)  in i40e_correct_mac_vlan_filters()
1408 			new->f->vlan = pvid;  in i40e_correct_mac_vlan_filters()
1409 		else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)  in i40e_correct_mac_vlan_filters()
1410 			new->f->vlan = 0;  in i40e_correct_mac_vlan_filters()
1411 		else if (!vlan_filters && new->f->vlan == 0)  in i40e_correct_mac_vlan_filters()
1412 			new->f->vlan = I40E_VLAN_ANY;  in i40e_correct_mac_vlan_filters()
1416 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {  in i40e_correct_mac_vlan_filters()
1422 		if ((pvid && f->vlan != pvid) ||  in i40e_correct_mac_vlan_filters()
1423 		    (vlan_filters && f->vlan == I40E_VLAN_ANY) ||  in i40e_correct_mac_vlan_filters()
1424 		    (!vlan_filters && f->vlan == 0)) {  in i40e_correct_mac_vlan_filters()
1434 			add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);  in i40e_correct_mac_vlan_filters()
1436 				return -ENOMEM;  in i40e_correct_mac_vlan_filters()
1441 				return -ENOMEM;  in i40e_correct_mac_vlan_filters()
1443 			new->f = add_head;  in i40e_correct_mac_vlan_filters()
1444 			new->state = add_head->state;  in i40e_correct_mac_vlan_filters()
1445 			if (add_head->state == I40E_FILTER_NEW)  in i40e_correct_mac_vlan_filters()
1446 				add_head->state = I40E_FILTER_NEW_SYNC;  in i40e_correct_mac_vlan_filters()
1449 			hlist_add_head(&new->hlist, tmp_add_list);  in i40e_correct_mac_vlan_filters()
1452 			f->state = I40E_FILTER_REMOVE;  in i40e_correct_mac_vlan_filters()
1453 			hash_del(&f->hlist);  in i40e_correct_mac_vlan_filters()
1454 			hlist_add_head(&f->hlist, tmp_del_list);  in i40e_correct_mac_vlan_filters()
1458 	vsi->has_vlan_filter = !!vlan_filters;  in i40e_correct_mac_vlan_filters()
1464  * i40e_get_vf_new_vlan - Get new vlan id on a vf
1467  * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL
1472  * and vf-vlan-prune-disable flag.
1483 	s16 pvid = le16_to_cpu(vsi->info.pvid);  in i40e_get_vf_new_vlan()
1484 	struct i40e_pf *pf = vsi->back;  in i40e_get_vf_new_vlan()
1488 		f = new_mac->f;  in i40e_get_vf_new_vlan()
1490 	if (pvid && f->vlan != pvid)  in i40e_get_vf_new_vlan()
1494 		  !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags));  in i40e_get_vf_new_vlan()
1496 	if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||  in i40e_get_vf_new_vlan()
1497 	    (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||  in i40e_get_vf_new_vlan()
1498 	    (is_any && !vlan_filters && f->vlan == 0)) {  in i40e_get_vf_new_vlan()
1505 	return f->vlan;  in i40e_get_vf_new_vlan()
1509  * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary
1517  * and vf-vlan-prune-disable flag.
1519  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1539 		new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL,  in i40e_correct_vf_mac_vlan_filters()
1543 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {  in i40e_correct_vf_mac_vlan_filters()
1546 		if (new_vlan != f->vlan) {  in i40e_correct_vf_mac_vlan_filters()
1547 			add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);  in i40e_correct_vf_mac_vlan_filters()
1549 				return -ENOMEM;  in i40e_correct_vf_mac_vlan_filters()
1553 				return -ENOMEM;  in i40e_correct_vf_mac_vlan_filters()
1554 			new_mac->f = add_head;  in i40e_correct_vf_mac_vlan_filters()
1555 			new_mac->state = add_head->state;  in i40e_correct_vf_mac_vlan_filters()
1556 			if (add_head->state == I40E_FILTER_NEW)  in i40e_correct_vf_mac_vlan_filters()
1557 				add_head->state = I40E_FILTER_NEW_SYNC;  in i40e_correct_vf_mac_vlan_filters()
1560 			hlist_add_head(&new_mac->hlist, tmp_add_list);  in i40e_correct_vf_mac_vlan_filters()
1563 			f->state = I40E_FILTER_REMOVE;  in i40e_correct_vf_mac_vlan_filters()
1564 			hash_del(&f->hlist);  in i40e_correct_vf_mac_vlan_filters()
1565 			hlist_add_head(&f->hlist, tmp_del_list);  in i40e_correct_vf_mac_vlan_filters()
1569 	vsi->has_vlan_filter = !!vlan_filters;  in i40e_correct_vf_mac_vlan_filters()
1574  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1575  * @vsi: the PF Main VSI - inappropriate for any other VSI
1584 	struct i40e_pf *pf = vsi->back;  in i40e_rm_default_mac_filter()
1587 	if (vsi->type != I40E_VSI_MAIN)  in i40e_rm_default_mac_filter()
1595 	i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);  in i40e_rm_default_mac_filter()
1603 	i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);  in i40e_rm_default_mac_filter()
1607  * i40e_add_filter - Add a mac/vlan filter to the VSI
1636 			vsi->has_vlan_filter = true;  in i40e_add_filter()
1638 		ether_addr_copy(f->macaddr, macaddr);  in i40e_add_filter()
1639 		f->vlan = vlan;  in i40e_add_filter()
1640 		f->state = I40E_FILTER_NEW;  in i40e_add_filter()
1641 		INIT_HLIST_NODE(&f->hlist);  in i40e_add_filter()
1644 		hash_add(vsi->mac_filter_hash, &f->hlist, key);  in i40e_add_filter()
1646 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;  in i40e_add_filter()
1647 		set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);  in i40e_add_filter()
1658 	if (f->state == I40E_FILTER_REMOVE)  in i40e_add_filter()
1659 		f->state = I40E_FILTER_ACTIVE;  in i40e_add_filter()
1665  * __i40e_del_filter - Remove a specific filter from the VSI
1688 	if ((f->state == I40E_FILTER_FAILED) ||  in __i40e_del_filter()
1689 	    (f->state == I40E_FILTER_NEW)) {  in __i40e_del_filter()
1690 		hash_del(&f->hlist);  in __i40e_del_filter()
1693 		f->state = I40E_FILTER_REMOVE;  in __i40e_del_filter()
1696 	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;  in __i40e_del_filter()
1697 	set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);  in __i40e_del_filter()
1701  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1724  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1742 	lockdep_assert_held(&vsi->mac_filter_hash_lock);  in i40e_add_mac_filter()
1743 	if (vsi->info.pvid)  in i40e_add_mac_filter()
1745 				       le16_to_cpu(vsi->info.pvid));  in i40e_add_mac_filter()
1750 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {  in i40e_add_mac_filter()
1751 		if (f->state == I40E_FILTER_REMOVE)  in i40e_add_mac_filter()
1753 		add = i40e_add_filter(vsi, macaddr, f->vlan);  in i40e_add_mac_filter()
1762  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1778 	lockdep_assert_held(&vsi->mac_filter_hash_lock);  in i40e_del_mac_filter()
1779 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {  in i40e_del_mac_filter()
1780 		if (ether_addr_equal(macaddr, f->macaddr)) {  in i40e_del_mac_filter()
1789 		return -ENOENT;  in i40e_del_mac_filter()
1793  * i40e_set_mac - NDO callback to set mac address
1802 	struct i40e_vsi *vsi = np->vsi;  in i40e_set_mac()
1803 	struct i40e_pf *pf = vsi->back;  in i40e_set_mac()
1804 	struct i40e_hw *hw = &pf->hw;  in i40e_set_mac()
1807 	if (!is_valid_ether_addr(addr->sa_data))  in i40e_set_mac()
1808 		return -EADDRNOTAVAIL;  in i40e_set_mac()
1810 	if (test_bit(__I40E_DOWN, pf->state) ||  in i40e_set_mac()
1811 	    test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))  in i40e_set_mac()
1812 		return -EADDRNOTAVAIL;  in i40e_set_mac()
1814 	if (ether_addr_equal(hw->mac.addr, addr->sa_data))  in i40e_set_mac()
1816 			    hw->mac.addr);  in i40e_set_mac()
1818 		netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);  in i40e_set_mac()
1822 	 * - Remove old address from MAC filter  in i40e_set_mac()
1823 	 * - Copy new address  in i40e_set_mac()
1824 	 * - Add new address to MAC filter  in i40e_set_mac()
1826 	spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_set_mac()
1827 	i40e_del_mac_filter(vsi, netdev->dev_addr);  in i40e_set_mac()
1828 	eth_hw_addr_set(netdev, addr->sa_data);  in i40e_set_mac()
1829 	i40e_add_mac_filter(vsi, netdev->dev_addr);  in i40e_set_mac()
1830 	spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_set_mac()
1832 	if (vsi->type == I40E_VSI_MAIN) {  in i40e_set_mac()
1836 						addr->sa_data, NULL);  in i40e_set_mac()
1840 				    i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_set_mac()
1851  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1860 	struct i40e_pf *pf = vsi->back;  in i40e_config_rss_aq()
1861 	struct i40e_hw *hw = &pf->hw;  in i40e_config_rss_aq()
1867 		ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);  in i40e_config_rss_aq()
1869 			dev_info(&pf->pdev->dev,  in i40e_config_rss_aq()
1872 				 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_config_rss_aq()
1877 		bool pf_lut = vsi->type == I40E_VSI_MAIN;  in i40e_config_rss_aq()
1879 		ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);  in i40e_config_rss_aq()
1881 			dev_info(&pf->pdev->dev,  in i40e_config_rss_aq()
1884 				 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_config_rss_aq()
1892  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1897 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_config_rss()
1902 	if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))  in i40e_vsi_config_rss()
1904 	if (!vsi->rss_size)  in i40e_vsi_config_rss()
1905 		vsi->rss_size = min_t(int, pf->alloc_rss_size,  in i40e_vsi_config_rss()
1906 				      vsi->num_queue_pairs);  in i40e_vsi_config_rss()
1907 	if (!vsi->rss_size)  in i40e_vsi_config_rss()
1908 		return -EINVAL;  in i40e_vsi_config_rss()
1909 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);  in i40e_vsi_config_rss()
1911 		return -ENOMEM;  in i40e_vsi_config_rss()
1916 	if (vsi->rss_lut_user)  in i40e_vsi_config_rss()
1917 		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);  in i40e_vsi_config_rss()
1919 		i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);  in i40e_vsi_config_rss()
1920 	if (vsi->rss_hkey_user)  in i40e_vsi_config_rss()
1921 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);  in i40e_vsi_config_rss()
1924 	ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);  in i40e_vsi_config_rss()
1930  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1943 	u8 netdev_tc = 0, offset = 0;  in i40e_vsi_setup_queue_map_mqprio()  local
1945 	if (vsi->type != I40E_VSI_MAIN)  in i40e_vsi_setup_queue_map_mqprio()
1946 		return -EINVAL;  in i40e_vsi_setup_queue_map_mqprio()
1949 	vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;  in i40e_vsi_setup_queue_map_mqprio()
1950 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;  in i40e_vsi_setup_queue_map_mqprio()
1951 	num_qps = vsi->mqprio_qopt.qopt.count[0];  in i40e_vsi_setup_queue_map_mqprio()
1953 	/* find the next higher power-of-2 of num queue pairs */  in i40e_vsi_setup_queue_map_mqprio()
1957 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |  in i40e_vsi_setup_queue_map_mqprio()
1960 	/* Setup queue offset/count for all TCs for given VSI */  in i40e_vsi_setup_queue_map_mqprio()
1961 	max_qcount = vsi->mqprio_qopt.qopt.count[0];  in i40e_vsi_setup_queue_map_mqprio()
1964 		if (vsi->tc_config.enabled_tc & BIT(i)) {  in i40e_vsi_setup_queue_map_mqprio()
1965 			offset = vsi->mqprio_qopt.qopt.offset[i];  in i40e_vsi_setup_queue_map_mqprio()
1966 			qcount = vsi->mqprio_qopt.qopt.count[i];  in i40e_vsi_setup_queue_map_mqprio()
1969 			vsi->tc_config.tc_info[i].qoffset = offset;  in i40e_vsi_setup_queue_map_mqprio()
1970 			vsi->tc_config.tc_info[i].qcount = qcount;  in i40e_vsi_setup_queue_map_mqprio()
1971 			vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;  in i40e_vsi_setup_queue_map_mqprio()
1973 			/* TC is not enabled so set the offset to  in i40e_vsi_setup_queue_map_mqprio()
1977 			vsi->tc_config.tc_info[i].qoffset = 0;  in i40e_vsi_setup_queue_map_mqprio()
1978 			vsi->tc_config.tc_info[i].qcount = 1;  in i40e_vsi_setup_queue_map_mqprio()
1979 			vsi->tc_config.tc_info[i].netdev_tc = 0;  in i40e_vsi_setup_queue_map_mqprio()
1984 	vsi->num_queue_pairs = offset + qcount;  in i40e_vsi_setup_queue_map_mqprio()
1987 	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);  in i40e_vsi_setup_queue_map_mqprio()
1988 	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);  in i40e_vsi_setup_queue_map_mqprio()
1989 	ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);  in i40e_vsi_setup_queue_map_mqprio()
1990 	ctxt->info.valid_sections |= cpu_to_le16(sections);  in i40e_vsi_setup_queue_map_mqprio()
1993 	vsi->rss_size = max_qcount;  in i40e_vsi_setup_queue_map_mqprio()
1996 		dev_info(&vsi->back->pdev->dev,  in i40e_vsi_setup_queue_map_mqprio()
2001 	vsi->reconfig_rss = true;  in i40e_vsi_setup_queue_map_mqprio()
2002 	dev_dbg(&vsi->back->pdev->dev,  in i40e_vsi_setup_queue_map_mqprio()
2005 	/* Find queue count available for channel VSIs and starting offset  in i40e_vsi_setup_queue_map_mqprio()
2008 	override_q = vsi->mqprio_qopt.qopt.count[0];  in i40e_vsi_setup_queue_map_mqprio()
2009 	if (override_q && override_q < vsi->num_queue_pairs) {  in i40e_vsi_setup_queue_map_mqprio()
2010 		vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;  in i40e_vsi_setup_queue_map_mqprio()
2011 		vsi->next_base_queue = override_q;  in i40e_vsi_setup_queue_map_mqprio()
2017  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
2020  * @enabled_tc: Enabled TCs bitmap
2030 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_setup_queue_map()
2036 	u8 offset;  in i40e_vsi_setup_queue_map()  local
2041 	offset = 0;  in i40e_vsi_setup_queue_map()
2043 	memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));  in i40e_vsi_setup_queue_map()
2045 	if (vsi->type == I40E_VSI_MAIN) {  in i40e_vsi_setup_queue_map()
2049 		 * non-zero req_queue_pairs says that user requested a new  in i40e_vsi_setup_queue_map()
2055 		if (vsi->req_queue_pairs > 0)  in i40e_vsi_setup_queue_map()
2056 			vsi->num_queue_pairs = vsi->req_queue_pairs;  in i40e_vsi_setup_queue_map()
2057 		else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_vsi_setup_queue_map()
2058 			vsi->num_queue_pairs = pf->num_lan_msix;  in i40e_vsi_setup_queue_map()
2060 			vsi->num_queue_pairs = 1;  in i40e_vsi_setup_queue_map()
2064 	if (vsi->type == I40E_VSI_MAIN ||  in i40e_vsi_setup_queue_map()
2065 	    (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))  in i40e_vsi_setup_queue_map()
2066 		num_tc_qps = vsi->num_queue_pairs;  in i40e_vsi_setup_queue_map()
2068 		num_tc_qps = vsi->alloc_queue_pairs;  in i40e_vsi_setup_queue_map()
2070 	if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {  in i40e_vsi_setup_queue_map()
2077 			dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");  in i40e_vsi_setup_queue_map()
2085 	vsi->tc_config.numtc = numtc;  in i40e_vsi_setup_queue_map()
2086 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;  in i40e_vsi_setup_queue_map()
2088 	/* Do not allow use more TC queue pairs than MSI-X vectors exist */  in i40e_vsi_setup_queue_map()
2089 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_vsi_setup_queue_map()
2090 		num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);  in i40e_vsi_setup_queue_map()
2092 	/* Setup queue offset/count for all TCs for given VSI */  in i40e_vsi_setup_queue_map()
2095 		if (vsi->tc_config.enabled_tc & BIT(i)) {  in i40e_vsi_setup_queue_map()
2099 			switch (vsi->type) {  in i40e_vsi_setup_queue_map()
2102 					       pf->flags) &&  in i40e_vsi_setup_queue_map()
2104 					       pf->flags)) ||  in i40e_vsi_setup_queue_map()
2105 				    vsi->tc_config.enabled_tc != 1) {  in i40e_vsi_setup_queue_map()
2106 					qcount = min_t(int, pf->alloc_rss_size,  in i40e_vsi_setup_queue_map()
2119 			vsi->tc_config.tc_info[i].qoffset = offset;  in i40e_vsi_setup_queue_map()
2120 			vsi->tc_config.tc_info[i].qcount = qcount;  in i40e_vsi_setup_queue_map()
2122 			/* find the next higher power-of-2 of num queue pairs */  in i40e_vsi_setup_queue_map()
2130 			vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;  in i40e_vsi_setup_queue_map()
2132 			    (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |  in i40e_vsi_setup_queue_map()
2135 			offset += qcount;  in i40e_vsi_setup_queue_map()
2137 			/* TC is not enabled so set the offset to  in i40e_vsi_setup_queue_map()
2141 			vsi->tc_config.tc_info[i].qoffset = 0;  in i40e_vsi_setup_queue_map()
2142 			vsi->tc_config.tc_info[i].qcount = 1;  in i40e_vsi_setup_queue_map()
2143 			vsi->tc_config.tc_info[i].netdev_tc = 0;  in i40e_vsi_setup_queue_map()
2147 		ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);  in i40e_vsi_setup_queue_map()
2150 	if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||  in i40e_vsi_setup_queue_map()
2151 	    (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||  in i40e_vsi_setup_queue_map()
2152 	    (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))  in i40e_vsi_setup_queue_map()
2153 		vsi->num_queue_pairs = offset;  in i40e_vsi_setup_queue_map()
2159 		ctxt->info.up_enable_bits = enabled_tc;  in i40e_vsi_setup_queue_map()
2161 	if (vsi->type == I40E_VSI_SRIOV) {  in i40e_vsi_setup_queue_map()
2162 		ctxt->info.mapping_flags |=  in i40e_vsi_setup_queue_map()
2164 		for (i = 0; i < vsi->num_queue_pairs; i++)  in i40e_vsi_setup_queue_map()
2165 			ctxt->info.queue_mapping[i] =  in i40e_vsi_setup_queue_map()
2166 					       cpu_to_le16(vsi->base_queue + i);  in i40e_vsi_setup_queue_map()
2168 		ctxt->info.mapping_flags |=  in i40e_vsi_setup_queue_map()
2170 		ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);  in i40e_vsi_setup_queue_map()
2172 	ctxt->info.valid_sections |= cpu_to_le16(sections);  in i40e_vsi_setup_queue_map()
2176  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
2186 	struct i40e_vsi *vsi = np->vsi;  in i40e_addr_sync()
2191 		return -ENOMEM;  in i40e_addr_sync()
2195  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2205 	struct i40e_vsi *vsi = np->vsi;  in i40e_addr_unsync()
2212 	if (ether_addr_equal(addr, netdev->dev_addr))  in i40e_addr_unsync()
2221  * i40e_set_rx_mode - NDO callback to set the netdev filters
2227 	struct i40e_vsi *vsi = np->vsi;  in i40e_set_rx_mode()
2229 	spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_set_rx_mode()
2234 	spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_set_rx_mode()
2237 	if (vsi->current_netdev_flags != vsi->netdev->flags) {  in i40e_set_rx_mode()
2238 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;  in i40e_set_rx_mode()
2239 		set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);  in i40e_set_rx_mode()
2244  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2246  * @from: Pointer to list which contains MAC filter entries - changes to
2258 		u64 key = i40e_addr_to_hkey(f->macaddr);  in i40e_undo_del_filter_entries()
2261 		hlist_del(&f->hlist);  in i40e_undo_del_filter_entries()
2262 		hash_add(vsi->mac_filter_hash, &f->hlist, key);  in i40e_undo_del_filter_entries()
2267  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2269  * @from: Pointer to list which contains MAC filter entries - changes to
2282 		hlist_del(&new->hlist);  in i40e_undo_add_filter_entries()
2283 		netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);  in i40e_undo_add_filter_entries()
2289  * i40e_next_filter - Get the next non-broadcast filter from a list
2292  * Returns the next non-broadcast filter in the list. Required so that we
2300 		if (!is_broadcast_ether_addr(next->f->macaddr))  in i40e_next_filter()
2308  * i40e_update_filter_state - Update filter state based on return data
2327 		 * the firmware return status because we pre-set the filter  in i40e_update_filter_state()
2333 			add_head->state = I40E_FILTER_FAILED;  in i40e_update_filter_state()
2335 			add_head->state = I40E_FILTER_ACTIVE;  in i40e_update_filter_state()
2348  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2353  * @retval: Set to -EIO on failure to delete
2365 	struct i40e_hw *hw = &vsi->back->hw;  in i40e_aqc_del_filters()
2369 	aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL,  in i40e_aqc_del_filters()
2374 		*retval = -EIO;  in i40e_aqc_del_filters()
2375 		dev_info(&vsi->back->pdev->dev,  in i40e_aqc_del_filters()
2383  * i40e_aqc_add_filters - Request firmware to add a set of filters
2391  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2400 	struct i40e_hw *hw = &vsi->back->hw;  in i40e_aqc_add_filters()
2404 	i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status);  in i40e_aqc_add_filters()
2408 		if (vsi->type == I40E_VSI_MAIN) {  in i40e_aqc_add_filters()
2409 			set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);  in i40e_aqc_add_filters()
2410 			dev_warn(&vsi->back->pdev->dev,  in i40e_aqc_add_filters()
2413 		} else if (vsi->type == I40E_VSI_SRIOV ||  in i40e_aqc_add_filters()
2414 			   vsi->type == I40E_VSI_VMDQ1 ||  in i40e_aqc_add_filters()
2415 			   vsi->type == I40E_VSI_VMDQ2) {  in i40e_aqc_add_filters()
2416 			dev_warn(&vsi->back->pdev->dev,  in i40e_aqc_add_filters()
2421 			dev_warn(&vsi->back->pdev->dev,  in i40e_aqc_add_filters()
2424 					     vsi->type);  in i40e_aqc_add_filters()
2430  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2445 	bool enable = f->state == I40E_FILTER_NEW ||  in i40e_aqc_broadcast_filter()
2446 		      f->state == I40E_FILTER_NEW_SYNC;  in i40e_aqc_broadcast_filter()
2447 	struct i40e_hw *hw = &vsi->back->hw;  in i40e_aqc_broadcast_filter()
2450 	if (f->vlan == I40E_VLAN_ANY) {  in i40e_aqc_broadcast_filter()
2452 						   vsi->seid,  in i40e_aqc_broadcast_filter()
2457 							    vsi->seid,  in i40e_aqc_broadcast_filter()
2459 							    f->vlan,  in i40e_aqc_broadcast_filter()
2464 		set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);  in i40e_aqc_broadcast_filter()
2465 		dev_warn(&vsi->back->pdev->dev,  in i40e_aqc_broadcast_filter()
2467 			 i40e_aq_str(hw, hw->aq.asq_last_status),  in i40e_aqc_broadcast_filter()
2475  * i40e_set_promiscuous - set promiscuous mode
2486 	struct i40e_hw *hw = &pf->hw;  in i40e_set_promiscuous()
2489 	if (vsi->type == I40E_VSI_MAIN &&  in i40e_set_promiscuous()
2491 	    !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {  in i40e_set_promiscuous()
2499 							 vsi->seid,  in i40e_set_promiscuous()
2503 							   vsi->seid,  in i40e_set_promiscuous()
2506 			dev_info(&pf->pdev->dev,  in i40e_set_promiscuous()
2509 				 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_set_promiscuous()
2514 						  vsi->seid,  in i40e_set_promiscuous()
2518 			dev_info(&pf->pdev->dev,  in i40e_set_promiscuous()
2521 				 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_set_promiscuous()
2525 						  vsi->seid,  in i40e_set_promiscuous()
2528 			dev_info(&pf->pdev->dev,  in i40e_set_promiscuous()
2531 				 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_set_promiscuous()
2536 		pf->cur_promisc = promisc;  in i40e_set_promiscuous()
2542  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2554 	struct i40e_hw *hw = &vsi->back->hw;  in i40e_sync_vsi_filters()
2575 	while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))  in i40e_sync_vsi_filters()
2577 	pf = vsi->back;  in i40e_sync_vsi_filters()
2579 	old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);  in i40e_sync_vsi_filters()
2581 	if (vsi->netdev) {  in i40e_sync_vsi_filters()
2582 		changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;  in i40e_sync_vsi_filters()
2583 		vsi->current_netdev_flags = vsi->netdev->flags;  in i40e_sync_vsi_filters()
2589 	if (vsi->type == I40E_VSI_SRIOV)  in i40e_sync_vsi_filters()
2590 		snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);  in i40e_sync_vsi_filters()
2591 	else if (vsi->type != I40E_VSI_MAIN)  in i40e_sync_vsi_filters()
2592 		snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);  in i40e_sync_vsi_filters()
2594 	if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {  in i40e_sync_vsi_filters()
2595 		vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;  in i40e_sync_vsi_filters()
2597 		spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_sync_vsi_filters()
2599 		hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {  in i40e_sync_vsi_filters()
2600 			if (f->state == I40E_FILTER_REMOVE) {  in i40e_sync_vsi_filters()
2602 				hash_del(&f->hlist);  in i40e_sync_vsi_filters()
2603 				hlist_add_head(&f->hlist, &tmp_del_list);  in i40e_sync_vsi_filters()
2608 			if (f->state == I40E_FILTER_NEW) {  in i40e_sync_vsi_filters()
2615 				new->f = f;  in i40e_sync_vsi_filters()
2616 				new->state = f->state;  in i40e_sync_vsi_filters()
2619 				hlist_add_head(&new->hlist, &tmp_add_list);  in i40e_sync_vsi_filters()
2620 				f->state = I40E_FILTER_NEW_SYNC;  in i40e_sync_vsi_filters()
2627 			if (f->vlan > 0)  in i40e_sync_vsi_filters()
2631 		if (vsi->type != I40E_VSI_SRIOV)  in i40e_sync_vsi_filters()
2635 		else if (pf->vf)  in i40e_sync_vsi_filters()
2638 				 vlan_filters, pf->vf[vsi->vf_id].trusted);  in i40e_sync_vsi_filters()
2641 			netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);  in i40e_sync_vsi_filters()
2646 		spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_sync_vsi_filters()
2651 		filter_list_len = hw->aq.asq_buf_size /  in i40e_sync_vsi_filters()
2665 			if (is_broadcast_ether_addr(f->macaddr)) {  in i40e_sync_vsi_filters()
2668 				hlist_del(&f->hlist);  in i40e_sync_vsi_filters()
2674 			ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);  in i40e_sync_vsi_filters()
2675 			if (f->vlan == I40E_VLAN_ANY) {  in i40e_sync_vsi_filters()
2680 					cpu_to_le16((u16)(f->vlan));  in i40e_sync_vsi_filters()
2697 			hlist_del(&f->hlist);  in i40e_sync_vsi_filters()
2712 		filter_list_len = hw->aq.asq_buf_size /  in i40e_sync_vsi_filters()
2725 			if (is_broadcast_ether_addr(new->f->macaddr)) {  in i40e_sync_vsi_filters()
2727 							      new->f))  in i40e_sync_vsi_filters()
2728 					new->state = I40E_FILTER_FAILED;  in i40e_sync_vsi_filters()
2730 					new->state = I40E_FILTER_ACTIVE;  in i40e_sync_vsi_filters()
2739 					new->f->macaddr);  in i40e_sync_vsi_filters()
2740 			if (new->f->vlan == I40E_VLAN_ANY) {  in i40e_sync_vsi_filters()
2745 					cpu_to_le16((u16)(new->f->vlan));  in i40e_sync_vsi_filters()
2769 		spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_sync_vsi_filters()
2772 			if (new->f->state == I40E_FILTER_NEW ||  in i40e_sync_vsi_filters()
2773 			    new->f->state == I40E_FILTER_NEW_SYNC)  in i40e_sync_vsi_filters()
2774 				new->f->state = new->state;  in i40e_sync_vsi_filters()
2775 			hlist_del(&new->hlist);  in i40e_sync_vsi_filters()
2776 			netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);  in i40e_sync_vsi_filters()
2779 		spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_sync_vsi_filters()
2785 	spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_sync_vsi_filters()
2786 	vsi->active_filters = 0;  in i40e_sync_vsi_filters()
2787 	hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {  in i40e_sync_vsi_filters()
2788 		if (f->state == I40E_FILTER_ACTIVE)  in i40e_sync_vsi_filters()
2789 			vsi->active_filters++;  in i40e_sync_vsi_filters()
2790 		else if (f->state == I40E_FILTER_FAILED)  in i40e_sync_vsi_filters()
2793 	spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_sync_vsi_filters()
2800 	    vsi->active_filters < vsi->promisc_threshold) {  in i40e_sync_vsi_filters()
2801 		dev_info(&pf->pdev->dev,  in i40e_sync_vsi_filters()
2804 		clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);  in i40e_sync_vsi_filters()
2805 		vsi->promisc_threshold = 0;  in i40e_sync_vsi_filters()
2809 	if (vsi->type == I40E_VSI_SRIOV && pf->vf &&  in i40e_sync_vsi_filters()
2810 	    !pf->vf[vsi->vf_id].trusted) {  in i40e_sync_vsi_filters()
2811 		clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);  in i40e_sync_vsi_filters()
2815 	new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);  in i40e_sync_vsi_filters()
2821 		vsi->promisc_threshold = (vsi->active_filters * 3) / 4;  in i40e_sync_vsi_filters()
2827 		cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);  in i40e_sync_vsi_filters()
2828 		aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,  in i40e_sync_vsi_filters()
2829 							       vsi->seid,  in i40e_sync_vsi_filters()
2834 						     hw->aq.asq_last_status);  in i40e_sync_vsi_filters()
2835 			dev_info(&pf->pdev->dev,  in i40e_sync_vsi_filters()
2839 				 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_sync_vsi_filters()
2841 			dev_info(&pf->pdev->dev, "%s allmulti mode.\n",  in i40e_sync_vsi_filters()
2849 		cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||  in i40e_sync_vsi_filters()
2854 						     hw->aq.asq_last_status);  in i40e_sync_vsi_filters()
2855 			dev_info(&pf->pdev->dev,  in i40e_sync_vsi_filters()
2860 				 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_sync_vsi_filters()
2866 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;  in i40e_sync_vsi_filters()
2868 	clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);  in i40e_sync_vsi_filters()
2873 	spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_sync_vsi_filters()
2877 	spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_sync_vsi_filters()
2879 	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;  in i40e_sync_vsi_filters()
2880 	clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);  in i40e_sync_vsi_filters()
2881 	return -ENOMEM;  in i40e_sync_vsi_filters()
2885  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2895 	if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))  in i40e_sync_filters_subtask()
2897 	if (test_bit(__I40E_VF_DISABLE, pf->state)) {  in i40e_sync_filters_subtask()
2898 		set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);  in i40e_sync_filters_subtask()
2903 		if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&  in i40e_sync_filters_subtask()
2904 		    !test_bit(__I40E_VSI_RELEASING, vsi->state)) {  in i40e_sync_filters_subtask()
2910 					pf->state);  in i40e_sync_filters_subtask()
2918  * i40e_calculate_vsi_rx_buf_len - Calculates buffer length
2924 	if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags))  in i40e_calculate_vsi_rx_buf_len()
2931  * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI
2941 	if (xdp_prog && !xdp_prog->aux->xdp_has_frags)  in i40e_max_vsi_frame_size()
2950  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2959 	struct i40e_vsi *vsi = np->vsi;  in i40e_change_mtu()
2960 	struct i40e_pf *pf = vsi->back;  in i40e_change_mtu()
2963 	frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);  in i40e_change_mtu()
2964 	if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) {  in i40e_change_mtu()
2966 			   new_mtu, frame_size - I40E_PACKET_HDR_PAD);  in i40e_change_mtu()
2967 		return -EINVAL;  in i40e_change_mtu()
2971 		   netdev->mtu, new_mtu);  in i40e_change_mtu()
2972 	WRITE_ONCE(netdev->mtu, new_mtu);  in i40e_change_mtu()
2975 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);  in i40e_change_mtu()
2976 	set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);  in i40e_change_mtu()
2981  * i40e_ioctl - Access the hwtstamp interface
2989 	struct i40e_pf *pf = np->vsi->back;  in i40e_ioctl()
2997 		return -EOPNOTSUPP;  in i40e_ioctl()
3002  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
3011 	if (vsi->info.pvid)  in i40e_vlan_stripping_enable()
3014 	if ((vsi->info.valid_sections &  in i40e_vlan_stripping_enable()
3016 	    ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))  in i40e_vlan_stripping_enable()
3019 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);  in i40e_vlan_stripping_enable()
3020 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |  in i40e_vlan_stripping_enable()
3023 	ctxt.seid = vsi->seid;  in i40e_vlan_stripping_enable()
3024 	ctxt.info = vsi->info;  in i40e_vlan_stripping_enable()
3025 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);  in i40e_vlan_stripping_enable()
3027 		dev_info(&vsi->back->pdev->dev,  in i40e_vlan_stripping_enable()
3030 			 i40e_aq_str(&vsi->back->hw,  in i40e_vlan_stripping_enable()
3031 				     vsi->back->hw.aq.asq_last_status));  in i40e_vlan_stripping_enable()
3036  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
3045 	if (vsi->info.pvid)  in i40e_vlan_stripping_disable()
3048 	if ((vsi->info.valid_sections &  in i40e_vlan_stripping_disable()
3050 	    ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==  in i40e_vlan_stripping_disable()
3054 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);  in i40e_vlan_stripping_disable()
3055 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |  in i40e_vlan_stripping_disable()
3058 	ctxt.seid = vsi->seid;  in i40e_vlan_stripping_disable()
3059 	ctxt.info = vsi->info;  in i40e_vlan_stripping_disable()
3060 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);  in i40e_vlan_stripping_disable()
3062 		dev_info(&vsi->back->pdev->dev,  in i40e_vlan_stripping_disable()
3065 			 i40e_aq_str(&vsi->back->hw,  in i40e_vlan_stripping_disable()
3066 				     vsi->back->hw.aq.asq_last_status));  in i40e_vlan_stripping_disable()
3071  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
3073  * @vid: vlan id to be added (0 = untagged only , -1 = any)
3089 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {  in i40e_add_vlan_all_mac()
3099 		if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) {  in i40e_add_vlan_all_mac()
3100 			f->state = I40E_FILTER_ACTIVE;  in i40e_add_vlan_all_mac()
3102 		} else if (f->state == I40E_FILTER_REMOVE) {  in i40e_add_vlan_all_mac()
3105 		add_f = i40e_add_filter(vsi, f->macaddr, vid);  in i40e_add_vlan_all_mac()
3107 			dev_info(&vsi->back->pdev->dev,  in i40e_add_vlan_all_mac()
3109 				 vid, f->macaddr);  in i40e_add_vlan_all_mac()
3110 			return -ENOMEM;  in i40e_add_vlan_all_mac()
3118  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
3126 	if (vsi->info.pvid)  in i40e_vsi_add_vlan()
3127 		return -EINVAL;  in i40e_vsi_add_vlan()
3141 	spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_vsi_add_vlan()
3143 	spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_vsi_add_vlan()
3150 	i40e_service_event_schedule(vsi->back);  in i40e_vsi_add_vlan()
3155  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
3157  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
3173 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {  in i40e_rm_vlan_all_mac()
3174 		if (f->vlan == vid)  in i40e_rm_vlan_all_mac()
3180  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
3186 	if (!vid || vsi->info.pvid)  in i40e_vsi_kill_vlan()
3189 	spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_vsi_kill_vlan()
3191 	spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_vsi_kill_vlan()
3196 	i40e_service_event_schedule(vsi->back);  in i40e_vsi_kill_vlan()
3200  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
3211 	struct i40e_vsi *vsi = np->vsi;  in i40e_vlan_rx_add_vid()
3215 		return -EINVAL;  in i40e_vlan_rx_add_vid()
3219 		set_bit(vid, vsi->active_vlans);  in i40e_vlan_rx_add_vid()
3225  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
3234 	struct i40e_vsi *vsi = np->vsi;  in i40e_vlan_rx_add_vid_up()
3238 	set_bit(vid, vsi->active_vlans);  in i40e_vlan_rx_add_vid_up()
3242  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
3253 	struct i40e_vsi *vsi = np->vsi;  in i40e_vlan_rx_kill_vid()
3261 	clear_bit(vid, vsi->active_vlans);  in i40e_vlan_rx_kill_vid()
3267  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
3274 	if (!vsi->netdev)  in i40e_restore_vlan()
3277 	if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)  in i40e_restore_vlan()
3282 	for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)  in i40e_restore_vlan()
3283 		i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),  in i40e_restore_vlan()
3288  * i40e_vsi_add_pvid - Add pvid for the VSI
3297 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);  in i40e_vsi_add_pvid()
3298 	vsi->info.pvid = cpu_to_le16(vid);  in i40e_vsi_add_pvid()
3299 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |  in i40e_vsi_add_pvid()
3303 	ctxt.seid = vsi->seid;  in i40e_vsi_add_pvid()
3304 	ctxt.info = vsi->info;  in i40e_vsi_add_pvid()
3305 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);  in i40e_vsi_add_pvid()
3307 		dev_info(&vsi->back->pdev->dev,  in i40e_vsi_add_pvid()
3310 			 i40e_aq_str(&vsi->back->hw,  in i40e_vsi_add_pvid()
3311 				     vsi->back->hw.aq.asq_last_status));  in i40e_vsi_add_pvid()
3312 		return -ENOENT;  in i40e_vsi_add_pvid()
3319  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3326 	vsi->info.pvid = 0;  in i40e_vsi_remove_pvid()
3332  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3345 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)  in i40e_vsi_setup_tx_resources()
3346 		err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);  in i40e_vsi_setup_tx_resources()
3351 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)  in i40e_vsi_setup_tx_resources()
3352 		err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);  in i40e_vsi_setup_tx_resources()
3358  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3367 	if (vsi->tx_rings) {  in i40e_vsi_free_tx_resources()
3368 		for (i = 0; i < vsi->num_queue_pairs; i++)  in i40e_vsi_free_tx_resources()
3369 			if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)  in i40e_vsi_free_tx_resources()
3370 				i40e_free_tx_resources(vsi->tx_rings[i]);  in i40e_vsi_free_tx_resources()
3373 	if (vsi->xdp_rings) {  in i40e_vsi_free_tx_resources()
3374 		for (i = 0; i < vsi->num_queue_pairs; i++)  in i40e_vsi_free_tx_resources()
3375 			if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)  in i40e_vsi_free_tx_resources()
3376 				i40e_free_tx_resources(vsi->xdp_rings[i]);  in i40e_vsi_free_tx_resources()
3381  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3394 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)  in i40e_vsi_setup_rx_resources()
3395 		err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);  in i40e_vsi_setup_rx_resources()
3400  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3409 	if (!vsi->rx_rings)  in i40e_vsi_free_rx_resources()
3412 	for (i = 0; i < vsi->num_queue_pairs; i++)  in i40e_vsi_free_rx_resources()
3413 		if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)  in i40e_vsi_free_rx_resources()
3414 			i40e_free_rx_resources(vsi->rx_rings[i]);  in i40e_vsi_free_rx_resources()
3418  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3422  * based on the TCs enabled for the VSI that ring belongs to.
3428 	if (!ring->q_vector || !ring->netdev || ring->ch)  in i40e_config_xps_tx_ring()
3432 	if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))  in i40e_config_xps_tx_ring()
3435 	cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);  in i40e_config_xps_tx_ring()
3436 	netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),  in i40e_config_xps_tx_ring()
3437 			    ring->queue_index);  in i40e_config_xps_tx_ring()
3441  * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3448 	bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);  in i40e_xsk_pool()
3449 	int qid = ring->queue_index;  in i40e_xsk_pool()
3452 		qid -= ring->vsi->alloc_queue_pairs;  in i40e_xsk_pool()
3454 	if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))  in i40e_xsk_pool()
3457 	return xsk_get_pool_from_qid(ring->vsi->netdev, qid);  in i40e_xsk_pool()
3461  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3468 	struct i40e_vsi *vsi = ring->vsi;  in i40e_configure_tx_ring()
3469 	u16 pf_q = vsi->base_queue + ring->queue_index;  in i40e_configure_tx_ring()
3470 	struct i40e_hw *hw = &vsi->back->hw;  in i40e_configure_tx_ring()
3476 		ring->xsk_pool = i40e_xsk_pool(ring);  in i40e_configure_tx_ring()
3479 	if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) {  in i40e_configure_tx_ring()
3480 		ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;  in i40e_configure_tx_ring()
3481 		ring->atr_count = 0;  in i40e_configure_tx_ring()
3483 		ring->atr_sample_rate = 0;  in i40e_configure_tx_ring()
3493 	tx_ctx.base = (ring->dma / 128);  in i40e_configure_tx_ring()
3494 	tx_ctx.qlen = ring->count;  in i40e_configure_tx_ring()
3495 	if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) ||  in i40e_configure_tx_ring()
3496 	    test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags))  in i40e_configure_tx_ring()
3498 	if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags))  in i40e_configure_tx_ring()
3501 	if (vsi->type != I40E_VSI_FDIR)  in i40e_configure_tx_ring()
3503 	tx_ctx.head_wb_addr = ring->dma +  in i40e_configure_tx_ring()
3504 			      (ring->count * sizeof(struct i40e_tx_desc));  in i40e_configure_tx_ring()
3517 	if (ring->ch)  in i40e_configure_tx_ring()
3519 			le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);  in i40e_configure_tx_ring()
3522 		tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);  in i40e_configure_tx_ring()
3529 		dev_info(&vsi->back->pdev->dev,  in i40e_configure_tx_ring()
3531 			 ring->queue_index, pf_q, err);  in i40e_configure_tx_ring()
3532 		return -ENOMEM;  in i40e_configure_tx_ring()
3538 		dev_info(&vsi->back->pdev->dev,  in i40e_configure_tx_ring()
3540 			 ring->queue_index, pf_q, err);  in i40e_configure_tx_ring()
3541 		return -ENOMEM;  in i40e_configure_tx_ring()
3545 	if (ring->ch) {  in i40e_configure_tx_ring()
3546 		if (ring->ch->type == I40E_VSI_VMDQ2)  in i40e_configure_tx_ring()
3549 			return -EINVAL;  in i40e_configure_tx_ring()
3552 				      ring->ch->vsi_number);  in i40e_configure_tx_ring()
3554 		if (vsi->type == I40E_VSI_VMDQ2) {  in i40e_configure_tx_ring()
3557 					      vsi->id);  in i40e_configure_tx_ring()
3563 	qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id);  in i40e_configure_tx_ring()
3568 	ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);  in i40e_configure_tx_ring()
3574  * i40e_rx_offset - Return expected offset into page to access data
3575  * @rx_ring: Ring we are requesting offset of
3577  * Returns the offset value for ring into the data buffer.
3585  * i40e_configure_rx_ring - Configure a receive ring context
3592 	struct i40e_vsi *vsi = ring->vsi;  in i40e_configure_rx_ring()
3593 	u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;  in i40e_configure_rx_ring()
3594 	u16 pf_q = vsi->base_queue + ring->queue_index;  in i40e_configure_rx_ring()
3595 	struct i40e_hw *hw = &vsi->back->hw;  in i40e_configure_rx_ring()
3600 	bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);  in i40e_configure_rx_ring()
3605 	ring->rx_buf_len = vsi->rx_buf_len;  in i40e_configure_rx_ring()
3607 	/* XDP RX-queue info only needed for RX rings exposed to XDP */  in i40e_configure_rx_ring()
3608 	if (ring->vsi->type != I40E_VSI_MAIN)  in i40e_configure_rx_ring()
3611 	if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) {  in i40e_configure_rx_ring()
3612 		err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,  in i40e_configure_rx_ring()
3613 					 ring->queue_index,  in i40e_configure_rx_ring()
3614 					 ring->q_vector->napi.napi_id,  in i40e_configure_rx_ring()
3615 					 ring->rx_buf_len);  in i40e_configure_rx_ring()
3620 	ring->xsk_pool = i40e_xsk_pool(ring);  in i40e_configure_rx_ring()
3621 	if (ring->xsk_pool) {  in i40e_configure_rx_ring()
3622 		xdp_rxq_info_unreg(&ring->xdp_rxq);  in i40e_configure_rx_ring()
3623 		ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool);  in i40e_configure_rx_ring()
3624 		err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,  in i40e_configure_rx_ring()
3625 					 ring->queue_index,  in i40e_configure_rx_ring()
3626 					 ring->q_vector->napi.napi_id,  in i40e_configure_rx_ring()
3627 					 ring->rx_buf_len);  in i40e_configure_rx_ring()
3630 		err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,  in i40e_configure_rx_ring()
3635 		dev_info(&vsi->back->pdev->dev,  in i40e_configure_rx_ring()
3637 			 ring->queue_index);  in i40e_configure_rx_ring()
3640 		err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,  in i40e_configure_rx_ring()
3648 	xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq);  in i40e_configure_rx_ring()
3650 	rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,  in i40e_configure_rx_ring()
3653 	rx_ctx.base = (ring->dma / 128);  in i40e_configure_rx_ring()
3654 	rx_ctx.qlen = ring->count;  in i40e_configure_rx_ring()
3664 	rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);  in i40e_configure_rx_ring()
3665 	if (hw->revision_id == 0)  in i40e_configure_rx_ring()
3679 		dev_info(&vsi->back->pdev->dev,  in i40e_configure_rx_ring()
3681 			 ring->queue_index, pf_q, err);  in i40e_configure_rx_ring()
3682 		return -ENOMEM;  in i40e_configure_rx_ring()
3688 		dev_info(&vsi->back->pdev->dev,  in i40e_configure_rx_ring()
3690 			 ring->queue_index, pf_q, err);  in i40e_configure_rx_ring()
3691 		return -ENOMEM;  in i40e_configure_rx_ring()
3695 	if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) {  in i40e_configure_rx_ring()
3697 			dev_info(&vsi->back->pdev->dev,  in i40e_configure_rx_ring()
3699 			return -EOPNOTSUPP;  in i40e_configure_rx_ring()
3706 	ring->rx_offset = i40e_rx_offset(ring);  in i40e_configure_rx_ring()
3709 	ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);  in i40e_configure_rx_ring()
3710 	writel(0, ring->tail);  in i40e_configure_rx_ring()
3712 	if (ring->xsk_pool) {  in i40e_configure_rx_ring()
3713 		xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);  in i40e_configure_rx_ring()
3722 		dev_info(&vsi->back->pdev->dev,  in i40e_configure_rx_ring()
3724 			 ring->xsk_pool ? "AF_XDP ZC enabled " : "",  in i40e_configure_rx_ring()
3725 			 ring->queue_index, pf_q);  in i40e_configure_rx_ring()
3732  * i40e_vsi_configure_tx - Configure the VSI for Tx
3742 	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)  in i40e_vsi_configure_tx()
3743 		err = i40e_configure_tx_ring(vsi->tx_rings[i]);  in i40e_vsi_configure_tx()
3748 	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)  in i40e_vsi_configure_tx()
3749 		err = i40e_configure_tx_ring(vsi->xdp_rings[i]);  in i40e_vsi_configure_tx()
3755  * i40e_vsi_configure_rx - Configure the VSI for Rx
3765 	vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);  in i40e_vsi_configure_rx()
3766 	vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);  in i40e_vsi_configure_rx()
3769 	if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&  in i40e_vsi_configure_rx()
3770 	    vsi->netdev->mtu <= ETH_DATA_LEN) {  in i40e_vsi_configure_rx()
3771 		vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;  in i40e_vsi_configure_rx()
3772 		vsi->max_frame = vsi->rx_buf_len;  in i40e_vsi_configure_rx()
3777 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)  in i40e_vsi_configure_rx()
3778 		err = i40e_configure_rx_ring(vsi->rx_rings[i]);  in i40e_vsi_configure_rx()
3784  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3793 	if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {  in i40e_vsi_config_dcb_rings()
3795 		for (i = 0; i < vsi->num_queue_pairs; i++) {  in i40e_vsi_config_dcb_rings()
3796 			rx_ring = vsi->rx_rings[i];  in i40e_vsi_config_dcb_rings()
3797 			tx_ring = vsi->tx_rings[i];  in i40e_vsi_config_dcb_rings()
3798 			rx_ring->dcb_tc = 0;  in i40e_vsi_config_dcb_rings()
3799 			tx_ring->dcb_tc = 0;  in i40e_vsi_config_dcb_rings()
3805 		if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))  in i40e_vsi_config_dcb_rings()
3808 		qoffset = vsi->tc_config.tc_info[n].qoffset;  in i40e_vsi_config_dcb_rings()
3809 		qcount = vsi->tc_config.tc_info[n].qcount;  in i40e_vsi_config_dcb_rings()
3811 			rx_ring = vsi->rx_rings[i];  in i40e_vsi_config_dcb_rings()
3812 			tx_ring = vsi->tx_rings[i];  in i40e_vsi_config_dcb_rings()
3813 			rx_ring->dcb_tc = n;  in i40e_vsi_config_dcb_rings()
3814 			tx_ring->dcb_tc = n;  in i40e_vsi_config_dcb_rings()
3820  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3825 	if (vsi->netdev)  in i40e_set_vsi_rx_mode()
3826 		i40e_set_rx_mode(vsi->netdev);  in i40e_set_vsi_rx_mode()
3830  * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3837 	pf->fd_tcp4_filter_cnt = 0;  in i40e_reset_fdir_filter_cnt()
3838 	pf->fd_udp4_filter_cnt = 0;  in i40e_reset_fdir_filter_cnt()
3839 	pf->fd_sctp4_filter_cnt = 0;  in i40e_reset_fdir_filter_cnt()
3840 	pf->fd_ip4_filter_cnt = 0;  in i40e_reset_fdir_filter_cnt()
3841 	pf->fd_tcp6_filter_cnt = 0;  in i40e_reset_fdir_filter_cnt()
3842 	pf->fd_udp6_filter_cnt = 0;  in i40e_reset_fdir_filter_cnt()
3843 	pf->fd_sctp6_filter_cnt = 0;  in i40e_reset_fdir_filter_cnt()
3844 	pf->fd_ip6_filter_cnt = 0;  in i40e_reset_fdir_filter_cnt()
3848  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3857 	struct i40e_pf *pf = vsi->back;  in i40e_fdir_filter_restore()
3860 	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))  in i40e_fdir_filter_restore()
3867 				  &pf->fdir_filter_list, fdir_node) {  in i40e_fdir_filter_restore()
3873  * i40e_vsi_configure - Set up the VSI for action
3891  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3897 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_configure_msix()
3898 	struct i40e_hw *hw = &pf->hw;  in i40e_vsi_configure_msix()
3903 	/* The interrupt indexing is offset by 1 in the PFINT_ITRn  in i40e_vsi_configure_msix()
3905 	 *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)  in i40e_vsi_configure_msix()
3907 	qp = vsi->base_queue;  in i40e_vsi_configure_msix()
3908 	vector = vsi->base_vector;  in i40e_vsi_configure_msix()
3909 	for (i = 0; i < vsi->num_q_vectors; i++, vector++) {  in i40e_vsi_configure_msix()
3910 		struct i40e_q_vector *q_vector = vsi->q_vectors[i];  in i40e_vsi_configure_msix()
3912 		q_vector->rx.next_update = jiffies + 1;  in i40e_vsi_configure_msix()
3913 		q_vector->rx.target_itr =  in i40e_vsi_configure_msix()
3914 			ITR_TO_REG(vsi->rx_rings[i]->itr_setting);  in i40e_vsi_configure_msix()
3915 		wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),  in i40e_vsi_configure_msix()
3916 		     q_vector->rx.target_itr >> 1);  in i40e_vsi_configure_msix()
3917 		q_vector->rx.current_itr = q_vector->rx.target_itr;  in i40e_vsi_configure_msix()
3919 		q_vector->tx.next_update = jiffies + 1;  in i40e_vsi_configure_msix()
3920 		q_vector->tx.target_itr =  in i40e_vsi_configure_msix()
3921 			ITR_TO_REG(vsi->tx_rings[i]->itr_setting);  in i40e_vsi_configure_msix()
3922 		wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),  in i40e_vsi_configure_msix()
3923 		     q_vector->tx.target_itr >> 1);  in i40e_vsi_configure_msix()
3924 		q_vector->tx.current_itr = q_vector->tx.target_itr;  in i40e_vsi_configure_msix()
3927 		 * busy-loop polling.  in i40e_vsi_configure_msix()
3929 		wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1),  in i40e_vsi_configure_msix()
3932 		wr32(hw, I40E_PFINT_RATEN(vector - 1),  in i40e_vsi_configure_msix()
3933 		     i40e_intrl_usec_to_reg(vsi->int_rate_limit));  in i40e_vsi_configure_msix()
3936 		wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);  in i40e_vsi_configure_msix()
3937 		for (q = 0; q < q_vector->num_ringpairs; q++) {  in i40e_vsi_configure_msix()
3938 			u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;  in i40e_vsi_configure_msix()
3970 			if (q == (q_vector->num_ringpairs - 1))  in i40e_vsi_configure_msix()
3983  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3988 	struct i40e_hw *hw = &pf->hw;  in i40e_enable_misc_int_causes()
4004 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))  in i40e_enable_misc_int_causes()
4007 	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))  in i40e_enable_misc_int_causes()
4021  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
4026 	u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;  in i40e_configure_msi_and_legacy()
4027 	struct i40e_q_vector *q_vector = vsi->q_vectors[0];  in i40e_configure_msi_and_legacy()
4028 	struct i40e_pf *pf = vsi->back;  in i40e_configure_msi_and_legacy()
4029 	struct i40e_hw *hw = &pf->hw;  in i40e_configure_msi_and_legacy()
4032 	q_vector->rx.next_update = jiffies + 1;  in i40e_configure_msi_and_legacy()
4033 	q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);  in i40e_configure_msi_and_legacy()
4034 	wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);  in i40e_configure_msi_and_legacy()
4035 	q_vector->rx.current_itr = q_vector->rx.target_itr;  in i40e_configure_msi_and_legacy()
4036 	q_vector->tx.next_update = jiffies + 1;  in i40e_configure_msi_and_legacy()
4037 	q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);  in i40e_configure_msi_and_legacy()
4038 	wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);  in i40e_configure_msi_and_legacy()
4039 	q_vector->tx.current_itr = q_vector->tx.target_itr;  in i40e_configure_msi_and_legacy()
4064  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
4069 	struct i40e_hw *hw = &pf->hw;  in i40e_irq_dynamic_disable_icr0()
4077  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
4082 	struct i40e_hw *hw = &pf->hw;  in i40e_irq_dynamic_enable_icr0()
4094  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
4102 	if (!q_vector->tx.ring && !q_vector->rx.ring)  in i40e_msix_clean_rings()
4105 	napi_schedule_irqoff(&q_vector->napi);  in i40e_msix_clean_rings()
4111  * i40e_irq_affinity_notify - Callback for affinity changes
4124 	cpumask_copy(&q_vector->affinity_mask, mask);  in i40e_irq_affinity_notify()
4128  * i40e_irq_affinity_release - Callback for affinity notifier release
4138  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
4142  * Allocates MSI-X vectors and requests interrupts from the kernel.
4146 	int q_vectors = vsi->num_q_vectors;  in i40e_vsi_request_irq_msix()
4147 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_request_irq_msix()
4148 	int base = vsi->base_vector;  in i40e_vsi_request_irq_msix()
4156 		struct i40e_q_vector *q_vector = vsi->q_vectors[vector];  in i40e_vsi_request_irq_msix()
4158 		irq_num = pf->msix_entries[base + vector].vector;  in i40e_vsi_request_irq_msix()
4160 		if (q_vector->tx.ring && q_vector->rx.ring) {  in i40e_vsi_request_irq_msix()
4161 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,  in i40e_vsi_request_irq_msix()
4162 				 "%s-%s-%d", basename, "TxRx", rx_int_idx++);  in i40e_vsi_request_irq_msix()
4164 		} else if (q_vector->rx.ring) {  in i40e_vsi_request_irq_msix()
4165 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,  in i40e_vsi_request_irq_msix()
4166 				 "%s-%s-%d", basename, "rx", rx_int_idx++);  in i40e_vsi_request_irq_msix()
4167 		} else if (q_vector->tx.ring) {  in i40e_vsi_request_irq_msix()
4168 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,  in i40e_vsi_request_irq_msix()
4169 				 "%s-%s-%d", basename, "tx", tx_int_idx++);  in i40e_vsi_request_irq_msix()
4175 				  vsi->irq_handler,  in i40e_vsi_request_irq_msix()
4177 				  q_vector->name,  in i40e_vsi_request_irq_msix()
4180 			dev_info(&pf->pdev->dev,  in i40e_vsi_request_irq_msix()
4186 		q_vector->irq_num = irq_num;  in i40e_vsi_request_irq_msix()
4187 		q_vector->affinity_notify.notify = i40e_irq_affinity_notify;  in i40e_vsi_request_irq_msix()
4188 		q_vector->affinity_notify.release = i40e_irq_affinity_release;  in i40e_vsi_request_irq_msix()
4189 		irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);  in i40e_vsi_request_irq_msix()
4196 		cpu = cpumask_local_spread(q_vector->v_idx, -1);  in i40e_vsi_request_irq_msix()
4200 	vsi->irqs_ready = true;  in i40e_vsi_request_irq_msix()
4205 		vector--;  in i40e_vsi_request_irq_msix()
4206 		irq_num = pf->msix_entries[base + vector].vector;  in i40e_vsi_request_irq_msix()
4209 		free_irq(irq_num, &vsi->q_vectors[vector]);  in i40e_vsi_request_irq_msix()
4215  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
4216  * @vsi: the VSI being un-configured
4220 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_disable_irq()
4221 	struct i40e_hw *hw = &pf->hw;  in i40e_vsi_disable_irq()
4222 	int base = vsi->base_vector;  in i40e_vsi_disable_irq()
4226 	for (i = 0; i < vsi->num_queue_pairs; i++) {  in i40e_vsi_disable_irq()
4229 		val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));  in i40e_vsi_disable_irq()
4231 		wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);  in i40e_vsi_disable_irq()
4233 		val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));  in i40e_vsi_disable_irq()
4235 		wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);  in i40e_vsi_disable_irq()
4239 		wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);  in i40e_vsi_disable_irq()
4243 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_vsi_disable_irq()
4244 		for (i = vsi->base_vector;  in i40e_vsi_disable_irq()
4245 		     i < (vsi->num_q_vectors + vsi->base_vector); i++)  in i40e_vsi_disable_irq()
4246 			wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);  in i40e_vsi_disable_irq()
4249 		for (i = 0; i < vsi->num_q_vectors; i++)  in i40e_vsi_disable_irq()
4250 			synchronize_irq(pf->msix_entries[i + base].vector);  in i40e_vsi_disable_irq()
4252 		/* Legacy and MSI mode - this stops all interrupt handling */  in i40e_vsi_disable_irq()
4256 		synchronize_irq(pf->pdev->irq);  in i40e_vsi_disable_irq()
4261  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
4266 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_enable_irq()
4269 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_vsi_enable_irq()
4270 		for (i = 0; i < vsi->num_q_vectors; i++)  in i40e_vsi_enable_irq()
4276 	i40e_flush(&pf->hw);  in i40e_vsi_enable_irq()
4281  * i40e_free_misc_vector - Free the vector that handles non-queue events
4287 	wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);  in i40e_free_misc_vector()
4288 	i40e_flush(&pf->hw);  in i40e_free_misc_vector()
4290 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {  in i40e_free_misc_vector()
4291 		free_irq(pf->msix_entries[0].vector, pf);  in i40e_free_misc_vector()
4292 		clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);  in i40e_free_misc_vector()
4297  * i40e_intr - MSI/Legacy and non-queue interrupt handler
4302  * with both queue and non-queue interrupts.  This is also used in
4303  * MSIX mode to handle the non-queue interrupts.
4308 	struct i40e_hw *hw = &pf->hw;  in i40e_intr()
4323 		pf->sw_int_count++;  in i40e_intr()
4325 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&  in i40e_intr()
4328 		dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");  in i40e_intr()
4329 		set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);  in i40e_intr()
4335 		struct i40e_q_vector *q_vector = vsi->q_vectors[0];  in i40e_intr()
4343 		if (!test_bit(__I40E_DOWN, pf->state))  in i40e_intr()
4344 			napi_schedule_irqoff(&q_vector->napi);  in i40e_intr()
4349 		set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);  in i40e_intr()
4350 		i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");  in i40e_intr()
4355 		set_bit(__I40E_MDD_EVENT_PENDING, pf->state);  in i40e_intr()
4360 		if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {  in i40e_intr()
4367 			set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);  in i40e_intr()
4372 		if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))  in i40e_intr()
4373 			set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);  in i40e_intr()
4378 			pf->corer_count++;  in i40e_intr()
4380 			pf->globr_count++;  in i40e_intr()
4382 			pf->empr_count++;  in i40e_intr()
4383 			set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);  in i40e_intr()
4389 		dev_info(&pf->pdev->dev, "HMC error interrupt\n");  in i40e_intr()
4390 		dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",  in i40e_intr()
4399 			schedule_work(&pf->ptp_extts0_work);  in i40e_intr()
4413 		dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",  in i40e_intr()
4418 			dev_info(&pf->pdev->dev, "device will be reset\n");  in i40e_intr()
4419 			set_bit(__I40E_PF_RESET_REQUESTED, pf->state);  in i40e_intr()
4427 	/* re-enable interrupt causes */  in i40e_intr()
4429 	if (!test_bit(__I40E_DOWN, pf->state) ||  in i40e_intr()
4430 	    test_bit(__I40E_RECOVERY_MODE, pf->state)) {  in i40e_intr()
4439  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4447 	struct i40e_vsi *vsi = tx_ring->vsi;  in i40e_clean_fdir_tx_irq()
4448 	u16 i = tx_ring->next_to_clean;  in i40e_clean_fdir_tx_irq()
4452 	tx_buf = &tx_ring->tx_bi[i];  in i40e_clean_fdir_tx_irq()
4454 	i -= tx_ring->count;  in i40e_clean_fdir_tx_irq()
4457 		struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;  in i40e_clean_fdir_tx_irq()
4467 		if (!(eop_desc->cmd_type_offset_bsz &  in i40e_clean_fdir_tx_irq()
4472 		tx_buf->next_to_watch = NULL;  in i40e_clean_fdir_tx_irq()
4474 		tx_desc->buffer_addr = 0;  in i40e_clean_fdir_tx_irq()
4475 		tx_desc->cmd_type_offset_bsz = 0;  in i40e_clean_fdir_tx_irq()
4481 			i -= tx_ring->count;  in i40e_clean_fdir_tx_irq()
4482 			tx_buf = tx_ring->tx_bi;  in i40e_clean_fdir_tx_irq()
4486 		dma_unmap_single(tx_ring->dev,  in i40e_clean_fdir_tx_irq()
4490 		if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)  in i40e_clean_fdir_tx_irq()
4491 			kfree(tx_buf->raw_buf);  in i40e_clean_fdir_tx_irq()
4493 		tx_buf->raw_buf = NULL;  in i40e_clean_fdir_tx_irq()
4494 		tx_buf->tx_flags = 0;  in i40e_clean_fdir_tx_irq()
4495 		tx_buf->next_to_watch = NULL;  in i40e_clean_fdir_tx_irq()
4497 		tx_desc->buffer_addr = 0;  in i40e_clean_fdir_tx_irq()
4498 		tx_desc->cmd_type_offset_bsz = 0;  in i40e_clean_fdir_tx_irq()
4505 			i -= tx_ring->count;  in i40e_clean_fdir_tx_irq()
4506 			tx_buf = tx_ring->tx_bi;  in i40e_clean_fdir_tx_irq()
4511 		budget--;  in i40e_clean_fdir_tx_irq()
4514 	i += tx_ring->count;  in i40e_clean_fdir_tx_irq()
4515 	tx_ring->next_to_clean = i;  in i40e_clean_fdir_tx_irq()
4517 	if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags))  in i40e_clean_fdir_tx_irq()
4518 		i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);  in i40e_clean_fdir_tx_irq()
4524  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4533 	if (!q_vector->tx.ring)  in i40e_fdir_clean_ring()
4536 	vsi = q_vector->tx.ring->vsi;  in i40e_fdir_clean_ring()
4537 	i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);  in i40e_fdir_clean_ring()
4543  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4550 	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];  in i40e_map_vector_to_qp()
4551 	struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];  in i40e_map_vector_to_qp()
4552 	struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];  in i40e_map_vector_to_qp()
4554 	tx_ring->q_vector = q_vector;  in i40e_map_vector_to_qp()
4555 	tx_ring->next = q_vector->tx.ring;  in i40e_map_vector_to_qp()
4556 	q_vector->tx.ring = tx_ring;  in i40e_map_vector_to_qp()
4557 	q_vector->tx.count++;  in i40e_map_vector_to_qp()
4561 		struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];  in i40e_map_vector_to_qp()
4563 		xdp_ring->q_vector = q_vector;  in i40e_map_vector_to_qp()
4564 		xdp_ring->next = q_vector->tx.ring;  in i40e_map_vector_to_qp()
4565 		q_vector->tx.ring = xdp_ring;  in i40e_map_vector_to_qp()
4566 		q_vector->tx.count++;  in i40e_map_vector_to_qp()
4569 	rx_ring->q_vector = q_vector;  in i40e_map_vector_to_qp()
4570 	rx_ring->next = q_vector->rx.ring;  in i40e_map_vector_to_qp()
4571 	q_vector->rx.ring = rx_ring;  in i40e_map_vector_to_qp()
4572 	q_vector->rx.count++;  in i40e_map_vector_to_qp()
4576  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4579  * This function maps descriptor rings to the queue-specific vectors
4580  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4586 	int qp_remaining = vsi->num_queue_pairs;  in i40e_vsi_map_rings_to_vectors()
4587 	int q_vectors = vsi->num_q_vectors;  in i40e_vsi_map_rings_to_vectors()
4592 	/* If we don't have enough vectors for a 1-to-1 mapping, we'll have to  in i40e_vsi_map_rings_to_vectors()
4600 		struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];  in i40e_vsi_map_rings_to_vectors()
4602 		num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);  in i40e_vsi_map_rings_to_vectors()
4604 		q_vector->num_ringpairs = num_ringpairs;  in i40e_vsi_map_rings_to_vectors()
4605 		q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;  in i40e_vsi_map_rings_to_vectors()
4607 		q_vector->rx.count = 0;  in i40e_vsi_map_rings_to_vectors()
4608 		q_vector->tx.count = 0;  in i40e_vsi_map_rings_to_vectors()
4609 		q_vector->rx.ring = NULL;  in i40e_vsi_map_rings_to_vectors()
4610 		q_vector->tx.ring = NULL;  in i40e_vsi_map_rings_to_vectors()
4612 		while (num_ringpairs--) {  in i40e_vsi_map_rings_to_vectors()
4615 			qp_remaining--;  in i40e_vsi_map_rings_to_vectors()
4621  * i40e_vsi_request_irq - Request IRQ from the OS
4627 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_request_irq()
4630 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_vsi_request_irq()
4632 	else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags))  in i40e_vsi_request_irq()
4633 		err = request_irq(pf->pdev->irq, i40e_intr, 0,  in i40e_vsi_request_irq()
4634 				  pf->int_name, pf);  in i40e_vsi_request_irq()
4636 		err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,  in i40e_vsi_request_irq()
4637 				  pf->int_name, pf);  in i40e_vsi_request_irq()
4640 		dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);  in i40e_vsi_request_irq()
4647  * i40e_netpoll - A Polling 'interrupt' handler
4650  * This is used by netconsole to send skbs without having to re-enable
4656 	struct i40e_vsi *vsi = np->vsi;  in i40e_netpoll()
4657 	struct i40e_pf *pf = vsi->back;  in i40e_netpoll()
4661 	if (test_bit(__I40E_VSI_DOWN, vsi->state))  in i40e_netpoll()
4664 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_netpoll()
4665 		for (i = 0; i < vsi->num_q_vectors; i++)  in i40e_netpoll()
4666 			i40e_msix_clean_rings(0, vsi->q_vectors[i]);  in i40e_netpoll()
4668 		i40e_intr(pf->pdev->irq, netdev);  in i40e_netpoll()
4676  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4683  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4692 		tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));  in i40e_pf_txq_wait()
4699 		return -ETIMEDOUT;  in i40e_pf_txq_wait()
4705  * i40e_control_tx_q - Start or stop a particular Tx queue
4716 	struct i40e_hw *hw = &pf->hw;  in i40e_control_tx_q()
4721 	i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);  in i40e_control_tx_q()
4749  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4766 		dev_info(&pf->pdev->dev,  in i40e_control_wait_tx_q()
4776  * i40e_vsi_enable_tx - Start a VSI's rings
4781 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_enable_tx()
4784 	pf_q = vsi->base_queue;  in i40e_vsi_enable_tx()
4785 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {  in i40e_vsi_enable_tx()
4786 		ret = i40e_control_wait_tx_q(vsi->seid, pf,  in i40e_vsi_enable_tx()
4795 		ret = i40e_control_wait_tx_q(vsi->seid, pf,  in i40e_vsi_enable_tx()
4796 					     pf_q + vsi->alloc_queue_pairs,  in i40e_vsi_enable_tx()
4805  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4812  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4821 		rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));  in i40e_pf_rxq_wait()
4828 		return -ETIMEDOUT;  in i40e_pf_rxq_wait()
4834  * i40e_control_rx_q - Start or stop a particular Rx queue
4845 	struct i40e_hw *hw = &pf->hw;  in i40e_control_rx_q()
4895  * i40e_vsi_enable_rx - Start a VSI's rings
4900 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_enable_rx()
4903 	pf_q = vsi->base_queue;  in i40e_vsi_enable_rx()
4904 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {  in i40e_vsi_enable_rx()
4907 			dev_info(&pf->pdev->dev,  in i40e_vsi_enable_rx()
4909 				 vsi->seid, pf_q);  in i40e_vsi_enable_rx()
4918  * i40e_vsi_start_rings - Start a VSI's rings
4937  * i40e_vsi_stop_rings - Stop a VSI's rings
4942 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_stop_rings()
4946 	if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))  in i40e_vsi_stop_rings()
4949 	tx_q_end = vsi->base_queue +  in i40e_vsi_stop_rings()
4950 		vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);  in i40e_vsi_stop_rings()
4951 	for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)  in i40e_vsi_stop_rings()
4952 		i40e_pre_tx_queue_cfg(&pf->hw, pf_q, false);  in i40e_vsi_stop_rings()
4954 	rx_q_end = vsi->base_queue + vsi->num_queue_pairs;  in i40e_vsi_stop_rings()
4955 	for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++)  in i40e_vsi_stop_rings()
4959 	for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)  in i40e_vsi_stop_rings()
4960 		wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);  in i40e_vsi_stop_rings()
4966  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4978 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_stop_rings_no_wait()
4981 	pf_q = vsi->base_queue;  in i40e_vsi_stop_rings_no_wait()
4982 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {  in i40e_vsi_stop_rings_no_wait()
4989  * i40e_vsi_free_irq - Free the irq association with the OS
4994 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_free_irq()
4995 	struct i40e_hw *hw = &pf->hw;  in i40e_vsi_free_irq()
4996 	int base = vsi->base_vector;  in i40e_vsi_free_irq()
5000 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_vsi_free_irq()
5001 		if (!vsi->q_vectors)  in i40e_vsi_free_irq()
5004 		if (!vsi->irqs_ready)  in i40e_vsi_free_irq()
5007 		vsi->irqs_ready = false;  in i40e_vsi_free_irq()
5008 		for (i = 0; i < vsi->num_q_vectors; i++) {  in i40e_vsi_free_irq()
5013 			irq_num = pf->msix_entries[vector].vector;  in i40e_vsi_free_irq()
5016 			if (!vsi->q_vectors[i] ||  in i40e_vsi_free_irq()
5017 			    !vsi->q_vectors[i]->num_ringpairs)  in i40e_vsi_free_irq()
5024 			free_irq(irq_num, vsi->q_vectors[i]);  in i40e_vsi_free_irq()
5033 			val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));  in i40e_vsi_free_irq()
5038 			wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);  in i40e_vsi_free_irq()
5073 		free_irq(pf->pdev->irq, pf);  in i40e_vsi_free_irq()
5107  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
5117 	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];  in i40e_free_q_vector()
5124 	i40e_for_each_ring(ring, q_vector->tx)  in i40e_free_q_vector()
5125 		ring->q_vector = NULL;  in i40e_free_q_vector()
5127 	i40e_for_each_ring(ring, q_vector->rx)  in i40e_free_q_vector()
5128 		ring->q_vector = NULL;  in i40e_free_q_vector()
5131 	if (vsi->netdev)  in i40e_free_q_vector()
5132 		netif_napi_del(&q_vector->napi);  in i40e_free_q_vector()
5134 	vsi->q_vectors[v_idx] = NULL;  in i40e_free_q_vector()
5140  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
5141  * @vsi: the VSI being un-configured
5150 	for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)  in i40e_vsi_free_q_vectors()
5155  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
5161 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_reset_interrupt_capability()
5162 		pci_disable_msix(pf->pdev);  in i40e_reset_interrupt_capability()
5163 		kfree(pf->msix_entries);  in i40e_reset_interrupt_capability()
5164 		pf->msix_entries = NULL;  in i40e_reset_interrupt_capability()
5165 		kfree(pf->irq_pile);  in i40e_reset_interrupt_capability()
5166 		pf->irq_pile = NULL;  in i40e_reset_interrupt_capability()
5167 	} else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {  in i40e_reset_interrupt_capability()
5168 		pci_disable_msi(pf->pdev);  in i40e_reset_interrupt_capability()
5170 	clear_bit(I40E_FLAG_MSI_ENA, pf->flags);  in i40e_reset_interrupt_capability()
5171 	clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);  in i40e_reset_interrupt_capability()
5175  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
5179  * to pre-load conditions
5186 	if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))  in i40e_clear_interrupt_scheme()
5189 	i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,  in i40e_clear_interrupt_scheme()
5192 	i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);  in i40e_clear_interrupt_scheme()
5201  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
5208 	if (!vsi->netdev)  in i40e_napi_enable_all()
5211 	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {  in i40e_napi_enable_all()
5212 		struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];  in i40e_napi_enable_all()
5214 		if (q_vector->rx.ring || q_vector->tx.ring)  in i40e_napi_enable_all()
5215 			napi_enable(&q_vector->napi);  in i40e_napi_enable_all()
5220  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
5227 	if (!vsi->netdev)  in i40e_napi_disable_all()
5230 	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {  in i40e_napi_disable_all()
5231 		struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];  in i40e_napi_disable_all()
5233 		if (q_vector->rx.ring || q_vector->tx.ring)  in i40e_napi_disable_all()
5234 			napi_disable(&q_vector->napi);  in i40e_napi_disable_all()
5239  * i40e_vsi_close - Shut down a VSI
5244 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_close()
5245 	if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))  in i40e_vsi_close()
5250 	vsi->current_netdev_flags = 0;  in i40e_vsi_close()
5251 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);  in i40e_vsi_close()
5252 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))  in i40e_vsi_close()
5253 		set_bit(__I40E_CLIENT_RESET, pf->state);  in i40e_vsi_close()
5257  * i40e_quiesce_vsi - Pause a given VSI
5262 	if (test_bit(__I40E_VSI_DOWN, vsi->state))  in i40e_quiesce_vsi()
5265 	set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);  in i40e_quiesce_vsi()
5266 	if (vsi->netdev && netif_running(vsi->netdev))  in i40e_quiesce_vsi()
5267 		vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);  in i40e_quiesce_vsi()
5273  * i40e_unquiesce_vsi - Resume a given VSI
5278 	if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))  in i40e_unquiesce_vsi()
5281 	if (vsi->netdev && netif_running(vsi->netdev))  in i40e_unquiesce_vsi()
5282 		vsi->netdev->netdev_ops->ndo_open(vsi->netdev);  in i40e_unquiesce_vsi()
5288  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5301  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5314  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5321 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_wait_queues_disabled()
5324 	pf_q = vsi->base_queue;  in i40e_vsi_wait_queues_disabled()
5325 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {  in i40e_vsi_wait_queues_disabled()
5329 			dev_info(&pf->pdev->dev,  in i40e_vsi_wait_queues_disabled()
5331 				 vsi->seid, pf_q);  in i40e_vsi_wait_queues_disabled()
5339 		ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,  in i40e_vsi_wait_queues_disabled()
5342 			dev_info(&pf->pdev->dev,  in i40e_vsi_wait_queues_disabled()
5344 				 vsi->seid, pf_q);  in i40e_vsi_wait_queues_disabled()
5351 			dev_info(&pf->pdev->dev,  in i40e_vsi_wait_queues_disabled()
5353 				 vsi->seid, pf_q);  in i40e_vsi_wait_queues_disabled()
5363  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5386  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5395 	struct i40e_hw *hw = &pf->hw;  in i40e_get_iscsi_tc_map()
5399 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;  in i40e_get_iscsi_tc_map()
5401 	for (i = 0; i < dcbcfg->numapps; i++) {  in i40e_get_iscsi_tc_map()
5402 		app = dcbcfg->app[i];  in i40e_get_iscsi_tc_map()
5405 			tc = dcbcfg->etscfg.prioritytable[app.priority];  in i40e_get_iscsi_tc_map()
5415  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5418  * Return the number of TCs from given DCBx configuration
5428 	 * and create a bitmask of enabled TCs  in i40e_dcb_get_num_tc()
5431 		num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);  in i40e_dcb_get_num_tc()
5434 	 * contiguous TCs starting with TC0  in i40e_dcb_get_num_tc()
5441 				pr_err("Non-contiguous TC - Disabling DCB\n");  in i40e_dcb_get_num_tc()
5457  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5476  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5485 	u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;  in i40e_mqprio_get_enabled_tc()
5494  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5507 		return vsi->mqprio_qopt.qopt.num_tc;  in i40e_pf_get_num_tc()
5511 	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))  in i40e_pf_get_num_tc()
5514 	/* SFP mode will be enabled for all TCs on port */  in i40e_pf_get_num_tc()
5515 	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))  in i40e_pf_get_num_tc()
5516 		return i40e_dcb_get_num_tc(&pf->hw.local_dcbx_config);  in i40e_pf_get_num_tc()
5518 	/* MFP mode return count of enabled TCs for this PF */  in i40e_pf_get_num_tc()
5519 	if (pf->hw.func_caps.iscsi)  in i40e_pf_get_num_tc()
5532  * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5545 	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))  in i40e_pf_get_tc_map()
5548 	/* SFP mode we want PF to be enabled for all TCs */  in i40e_pf_get_tc_map()
5549 	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))  in i40e_pf_get_tc_map()
5550 		return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);  in i40e_pf_get_tc_map()
5553 	if (pf->hw.func_caps.iscsi)  in i40e_pf_get_tc_map()
5560  * i40e_vsi_get_bw_info - Query VSI BW Information
5569 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_get_bw_info()
5570 	struct i40e_hw *hw = &pf->hw;  in i40e_vsi_get_bw_info()
5576 	ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);  in i40e_vsi_get_bw_info()
5578 		dev_info(&pf->pdev->dev,  in i40e_vsi_get_bw_info()
5581 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_vsi_get_bw_info()
5582 		return -EINVAL;  in i40e_vsi_get_bw_info()
5586 	ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,  in i40e_vsi_get_bw_info()
5589 		dev_info(&pf->pdev->dev,  in i40e_vsi_get_bw_info()
5592 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_vsi_get_bw_info()
5593 		return -EINVAL;  in i40e_vsi_get_bw_info()
5597 		dev_info(&pf->pdev->dev,  in i40e_vsi_get_bw_info()
5598 			 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",  in i40e_vsi_get_bw_info()
5604 	vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);  in i40e_vsi_get_bw_info()
5605 	vsi->bw_max_quanta = bw_config.max_bw;  in i40e_vsi_get_bw_info()
5609 		vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];  in i40e_vsi_get_bw_info()
5610 		vsi->bw_ets_limit_credits[i] =  in i40e_vsi_get_bw_info()
5613 		vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);  in i40e_vsi_get_bw_info()
5620  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5631 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_configure_bw_alloc()
5638 	if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {  in i40e_vsi_configure_bw_alloc()
5639 		ret = i40e_set_bw_limit(vsi, vsi->seid, 0);  in i40e_vsi_configure_bw_alloc()
5641 			dev_info(&pf->pdev->dev,  in i40e_vsi_configure_bw_alloc()
5642 				 "Failed to reset tx rate for vsi->seid %u\n",  in i40e_vsi_configure_bw_alloc()
5643 				 vsi->seid);  in i40e_vsi_configure_bw_alloc()
5651 	ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);  in i40e_vsi_configure_bw_alloc()
5653 		dev_info(&pf->pdev->dev,  in i40e_vsi_configure_bw_alloc()
5655 			 pf->hw.aq.asq_last_status);  in i40e_vsi_configure_bw_alloc()
5656 		return -EINVAL;  in i40e_vsi_configure_bw_alloc()
5660 		vsi->info.qs_handle[i] = bw_data.qs_handles[i];  in i40e_vsi_configure_bw_alloc()
5666  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5673 	struct net_device *netdev = vsi->netdev;  in i40e_vsi_config_netdev_tc()
5674 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_config_netdev_tc()
5675 	struct i40e_hw *hw = &pf->hw;  in i40e_vsi_config_netdev_tc()
5678 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;  in i40e_vsi_config_netdev_tc()
5688 	/* Set up actual enabled TCs on the VSI */  in i40e_vsi_config_netdev_tc()
5689 	if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))  in i40e_vsi_config_netdev_tc()
5694 		/* Only set TC queues for enabled tcs  in i40e_vsi_config_netdev_tc()
5701 		if (vsi->tc_config.enabled_tc & BIT(i))  in i40e_vsi_config_netdev_tc()
5703 					vsi->tc_config.tc_info[i].netdev_tc,  in i40e_vsi_config_netdev_tc()
5704 					vsi->tc_config.tc_info[i].qcount,  in i40e_vsi_config_netdev_tc()
5705 					vsi->tc_config.tc_info[i].qoffset);  in i40e_vsi_config_netdev_tc()
5714 		u8 ets_tc = dcbcfg->etscfg.prioritytable[i];  in i40e_vsi_config_netdev_tc()
5716 		netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;  in i40e_vsi_config_netdev_tc()
5722  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5733 	vsi->info.mapping_flags = ctxt->info.mapping_flags;  in i40e_vsi_update_queue_map()
5734 	memcpy(&vsi->info.queue_mapping,  in i40e_vsi_update_queue_map()
5735 	       &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));  in i40e_vsi_update_queue_map()
5736 	memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,  in i40e_vsi_update_queue_map()
5737 	       sizeof(vsi->info.tc_mapping));  in i40e_vsi_update_queue_map()
5741  * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5743  * @vsi_offset: offset from main VF VSI
5753 		return -EINVAL;  in i40e_update_adq_vsi_queues()
5754 	pf = vsi->back;  in i40e_update_adq_vsi_queues()
5755 	hw = &pf->hw;  in i40e_update_adq_vsi_queues()
5757 	ctxt.seid = vsi->seid;  in i40e_update_adq_vsi_queues()
5758 	ctxt.pf_num = hw->pf_id;  in i40e_update_adq_vsi_queues()
5759 	ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;  in i40e_update_adq_vsi_queues()
5760 	ctxt.uplink_seid = vsi->uplink_seid;  in i40e_update_adq_vsi_queues()
5763 	ctxt.info = vsi->info;  in i40e_update_adq_vsi_queues()
5765 	i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,  in i40e_update_adq_vsi_queues()
5767 	if (vsi->reconfig_rss) {  in i40e_update_adq_vsi_queues()
5768 		vsi->rss_size = min_t(int, pf->alloc_rss_size,  in i40e_update_adq_vsi_queues()
5769 				      vsi->num_queue_pairs);  in i40e_update_adq_vsi_queues()
5772 			dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");  in i40e_update_adq_vsi_queues()
5775 		vsi->reconfig_rss = false;  in i40e_update_adq_vsi_queues()
5780 		dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n",  in i40e_update_adq_vsi_queues()
5782 			 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_update_adq_vsi_queues()
5787 	vsi->info.valid_sections = 0;  in i40e_update_adq_vsi_queues()
5793  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5797  * This configures a particular VSI for TCs that are mapped to the
5798  * given TC bitmap. It uses default bandwidth share for TCs across
5808 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_config_tc()
5809 	struct i40e_hw *hw = &pf->hw;  in i40e_vsi_config_tc()
5814 	/* Check if enabled_tc is same as existing or new TCs */  in i40e_vsi_config_tc()
5815 	if (vsi->tc_config.enabled_tc == enabled_tc &&  in i40e_vsi_config_tc()
5816 	    vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)  in i40e_vsi_config_tc()
5819 	/* Enable ETS TCs with equal BW Share for now across all VSIs */  in i40e_vsi_config_tc()
5829 		dev_info(&pf->pdev->dev,  in i40e_vsi_config_tc()
5831 			 enabled_tc, vsi->seid);  in i40e_vsi_config_tc()
5832 		ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,  in i40e_vsi_config_tc()
5835 			dev_info(&pf->pdev->dev,  in i40e_vsi_config_tc()
5838 				 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_vsi_config_tc()
5848 			dev_info(&pf->pdev->dev,  in i40e_vsi_config_tc()
5856 			dev_err(&pf->pdev->dev,  in i40e_vsi_config_tc()
5858 				enabled_tc, vsi->seid);  in i40e_vsi_config_tc()
5864 	ctxt.seid = vsi->seid;  in i40e_vsi_config_tc()
5865 	ctxt.pf_num = vsi->back->hw.pf_id;  in i40e_vsi_config_tc()
5867 	ctxt.uplink_seid = vsi->uplink_seid;  in i40e_vsi_config_tc()
5868 	ctxt.info = vsi->info;  in i40e_vsi_config_tc()
5877 	/* On destroying the qdisc, reset vsi->rss_size, as number of enabled  in i40e_vsi_config_tc()
5880 	if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {  in i40e_vsi_config_tc()
5881 		vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,  in i40e_vsi_config_tc()
5882 				      vsi->num_queue_pairs);  in i40e_vsi_config_tc()
5885 			dev_info(&vsi->back->pdev->dev,  in i40e_vsi_config_tc()
5889 		vsi->reconfig_rss = false;  in i40e_vsi_config_tc()
5891 	if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {  in i40e_vsi_config_tc()
5897 	/* Update the VSI after updating the VSI queue-mapping  in i40e_vsi_config_tc()
5902 		dev_info(&pf->pdev->dev,  in i40e_vsi_config_tc()
5905 			 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_vsi_config_tc()
5910 	vsi->info.valid_sections = 0;  in i40e_vsi_config_tc()
5915 		dev_info(&pf->pdev->dev,  in i40e_vsi_config_tc()
5918 			 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_vsi_config_tc()
5929  * i40e_vsi_reconfig_tc - Reconfigure VSI Tx Scheduler for stored TC map
5932  * This reconfigures a particular VSI for TCs that are mapped to the
5944 	enabled_tc = vsi->tc_config.enabled_tc;  in i40e_vsi_reconfig_tc()
5945 	vsi->tc_config.enabled_tc = 0;  in i40e_vsi_reconfig_tc()
5951  * i40e_get_link_speed - Returns link speed for the interface
5957 	struct i40e_pf *pf = vsi->back;  in i40e_get_link_speed()
5959 	switch (pf->hw.phy.link_info.link_speed) {  in i40e_get_link_speed()
5971 		return -EINVAL;  in i40e_get_link_speed()
5976  * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5985 		dev_warn(&vsi->back->pdev->dev,  in i40e_bw_bytes_to_mbits()
5996  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
6005 	struct i40e_pf *pf = vsi->back;  in i40e_set_bw_limit()
6012 		dev_err(&pf->pdev->dev,  in i40e_set_bw_limit()
6015 		return -EINVAL;  in i40e_set_bw_limit()
6018 		dev_warn(&pf->pdev->dev,  in i40e_set_bw_limit()
6026 	ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,  in i40e_set_bw_limit()
6029 		dev_err(&pf->pdev->dev,  in i40e_set_bw_limit()
6030 			"Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n",  in i40e_set_bw_limit()
6032 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_set_bw_limit()
6037  * i40e_remove_queue_channels - Remove queue channels for the TCs
6040  * Remove queue channels for the TCs
6047 	struct i40e_pf *pf = vsi->back;  in i40e_remove_queue_channels()
6052 	 * channel VSIs with non-power-of-2 queue count.  in i40e_remove_queue_channels()
6054 	vsi->current_rss_size = 0;  in i40e_remove_queue_channels()
6057 	if (list_empty(&vsi->ch_list))  in i40e_remove_queue_channels()
6060 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {  in i40e_remove_queue_channels()
6063 		list_del(&ch->list);  in i40e_remove_queue_channels()
6064 		p_vsi = ch->parent_vsi;  in i40e_remove_queue_channels()
6065 		if (!p_vsi || !ch->initialized) {  in i40e_remove_queue_channels()
6070 		for (i = 0; i < ch->num_queue_pairs; i++) {  in i40e_remove_queue_channels()
6074 			pf_q = ch->base_queue + i;  in i40e_remove_queue_channels()
6075 			tx_ring = vsi->tx_rings[pf_q];  in i40e_remove_queue_channels()
6076 			tx_ring->ch = NULL;  in i40e_remove_queue_channels()
6078 			rx_ring = vsi->rx_rings[pf_q];  in i40e_remove_queue_channels()
6079 			rx_ring->ch = NULL;  in i40e_remove_queue_channels()
6083 		ret = i40e_set_bw_limit(vsi, ch->seid, 0);  in i40e_remove_queue_channels()
6085 			dev_info(&vsi->back->pdev->dev,  in i40e_remove_queue_channels()
6086 				 "Failed to reset tx rate for ch->seid %u\n",  in i40e_remove_queue_channels()
6087 				 ch->seid);  in i40e_remove_queue_channels()
6091 					  &pf->cloud_filter_list, cloud_node) {  in i40e_remove_queue_channels()
6092 			if (cfilter->seid != ch->seid)  in i40e_remove_queue_channels()
6095 			hash_del(&cfilter->cloud_node);  in i40e_remove_queue_channels()
6096 			if (cfilter->dst_port)  in i40e_remove_queue_channels()
6103 			last_aq_status = pf->hw.aq.asq_last_status;  in i40e_remove_queue_channels()
6105 				dev_info(&pf->pdev->dev,  in i40e_remove_queue_channels()
6108 					 i40e_aq_str(&pf->hw, last_aq_status));  in i40e_remove_queue_channels()
6113 		ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,  in i40e_remove_queue_channels()
6116 			dev_err(&vsi->back->pdev->dev,  in i40e_remove_queue_channels()
6118 				ch->seid, p_vsi->seid);  in i40e_remove_queue_channels()
6121 	INIT_LIST_HEAD(&vsi->ch_list);  in i40e_remove_queue_channels()
6129  * channels/TCs created.
6136 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {  in i40e_get_max_queues_for_channel()
6137 		if (!ch->initialized)  in i40e_get_max_queues_for_channel()
6139 		if (ch->num_queue_pairs > max)  in i40e_get_max_queues_for_channel()
6140 			max = ch->num_queue_pairs;  in i40e_get_max_queues_for_channel()
6147  * i40e_validate_num_queues - validate num_queues w.r.t channel
6163 		return -EINVAL;  in i40e_validate_num_queues()
6166 	if (vsi->current_rss_size) {  in i40e_validate_num_queues()
6167 		if (num_queues > vsi->current_rss_size) {  in i40e_validate_num_queues()
6168 			dev_dbg(&pf->pdev->dev,  in i40e_validate_num_queues()
6170 				num_queues, vsi->current_rss_size);  in i40e_validate_num_queues()
6171 			return -EINVAL;  in i40e_validate_num_queues()
6172 		} else if ((num_queues < vsi->current_rss_size) &&  in i40e_validate_num_queues()
6174 			dev_dbg(&pf->pdev->dev,  in i40e_validate_num_queues()
6176 				num_queues, vsi->current_rss_size);  in i40e_validate_num_queues()
6177 			return -EINVAL;  in i40e_validate_num_queues()
6189 			dev_dbg(&pf->pdev->dev,  in i40e_validate_num_queues()
6192 			return -EINVAL;  in i40e_validate_num_queues()
6201  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
6209 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_reconfig_rss()
6211 	struct i40e_hw *hw = &pf->hw;  in i40e_vsi_reconfig_rss()
6216 	if (!vsi->rss_size)  in i40e_vsi_reconfig_rss()
6217 		return -EINVAL;  in i40e_vsi_reconfig_rss()
6219 	if (rss_size > vsi->rss_size)  in i40e_vsi_reconfig_rss()
6220 		return -EINVAL;  in i40e_vsi_reconfig_rss()
6222 	local_rss_size = min_t(int, vsi->rss_size, rss_size);  in i40e_vsi_reconfig_rss()
6223 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);  in i40e_vsi_reconfig_rss()
6225 		return -ENOMEM;  in i40e_vsi_reconfig_rss()
6228 	i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);  in i40e_vsi_reconfig_rss()
6233 	if (vsi->rss_hkey_user)  in i40e_vsi_reconfig_rss()
6234 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);  in i40e_vsi_reconfig_rss()
6238 	ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);  in i40e_vsi_reconfig_rss()
6240 		dev_info(&pf->pdev->dev,  in i40e_vsi_reconfig_rss()
6243 			 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_vsi_reconfig_rss()
6250 	if (!vsi->orig_rss_size)  in i40e_vsi_reconfig_rss()
6251 		vsi->orig_rss_size = vsi->rss_size;  in i40e_vsi_reconfig_rss()
6252 	vsi->current_rss_size = local_rss_size;  in i40e_vsi_reconfig_rss()
6258  * i40e_channel_setup_queue_map - Setup a channel queue map
6270 	u8 offset = 0;  in i40e_channel_setup_queue_map()  local
6276 	qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);  in i40e_channel_setup_queue_map()
6277 	ch->num_queue_pairs = qcount;  in i40e_channel_setup_queue_map()
6279 	/* find the next higher power-of-2 of num queue pairs */  in i40e_channel_setup_queue_map()
6284 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |  in i40e_channel_setup_queue_map()
6288 	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);  in i40e_channel_setup_queue_map()
6290 	ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */  in i40e_channel_setup_queue_map()
6291 	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);  in i40e_channel_setup_queue_map()
6292 	ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);  in i40e_channel_setup_queue_map()
6293 	ctxt->info.valid_sections |= cpu_to_le16(sections);  in i40e_channel_setup_queue_map()
6297  * i40e_add_channel - add a channel by adding VSI
6307 	struct i40e_hw *hw = &pf->hw;  in i40e_add_channel()
6312 	if (ch->type != I40E_VSI_VMDQ2) {  in i40e_add_channel()
6313 		dev_info(&pf->pdev->dev,  in i40e_add_channel()
6314 			 "add new vsi failed, ch->type %d\n", ch->type);  in i40e_add_channel()
6315 		return -EINVAL;  in i40e_add_channel()
6319 	ctxt.pf_num = hw->pf_id;  in i40e_add_channel()
6323 	if (ch->type == I40E_VSI_VMDQ2)  in i40e_add_channel()
6326 	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {  in i40e_add_channel()
6339 		dev_info(&pf->pdev->dev,  in i40e_add_channel()
6342 			 i40e_aq_str(&pf->hw,  in i40e_add_channel()
6343 				     pf->hw.aq.asq_last_status));  in i40e_add_channel()
6344 		return -ENOENT;  in i40e_add_channel()
6350 	ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;  in i40e_add_channel()
6351 	ch->seid = ctxt.seid;  in i40e_add_channel()
6352 	ch->vsi_number = ctxt.vsi_number;  in i40e_add_channel()
6353 	ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);  in i40e_add_channel()
6359 	ch->info.mapping_flags = ctxt.info.mapping_flags;  in i40e_add_channel()
6360 	memcpy(&ch->info.queue_mapping,  in i40e_add_channel()
6362 	memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,  in i40e_add_channel()
6376 	bw_data.tc_valid_bits = ch->enabled_tc;  in i40e_channel_config_bw()
6380 	ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,  in i40e_channel_config_bw()
6383 		dev_info(&vsi->back->pdev->dev,  in i40e_channel_config_bw()
6384 			 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",  in i40e_channel_config_bw()
6385 			 vsi->back->hw.aq.asq_last_status, ch->seid);  in i40e_channel_config_bw()
6386 		return -EINVAL;  in i40e_channel_config_bw()
6390 		ch->info.qs_handle[i] = bw_data.qs_handles[i];  in i40e_channel_config_bw()
6396  * i40e_channel_config_tx_ring - config TX ring associated with new channel
6412 	/* Enable ETS TCs with equal BW Share for now across all VSIs */  in i40e_channel_config_tx_ring()
6414 		if (ch->enabled_tc & BIT(i))  in i40e_channel_config_tx_ring()
6421 		dev_info(&vsi->back->pdev->dev,  in i40e_channel_config_tx_ring()
6423 			 ch->enabled_tc, ch->seid);  in i40e_channel_config_tx_ring()
6427 	for (i = 0; i < ch->num_queue_pairs; i++) {  in i40e_channel_config_tx_ring()
6431 		pf_q = ch->base_queue + i;  in i40e_channel_config_tx_ring()
6433 		/* Get to TX ring ptr of main VSI, for re-setup TX queue  in i40e_channel_config_tx_ring()
6436 		tx_ring = vsi->tx_rings[pf_q];  in i40e_channel_config_tx_ring()
6437 		tx_ring->ch = ch;  in i40e_channel_config_tx_ring()
6440 		rx_ring = vsi->rx_rings[pf_q];  in i40e_channel_config_tx_ring()
6441 		rx_ring->ch = ch;  in i40e_channel_config_tx_ring()
6448  * i40e_setup_hw_channel - setup new channel
6465 	ch->initialized = false;  in i40e_setup_hw_channel()
6466 	ch->base_queue = vsi->next_base_queue;  in i40e_setup_hw_channel()
6467 	ch->type = type;  in i40e_setup_hw_channel()
6472 		dev_info(&pf->pdev->dev,  in i40e_setup_hw_channel()
6479 	ch->initialized = true;  in i40e_setup_hw_channel()
6484 		dev_info(&pf->pdev->dev,  in i40e_setup_hw_channel()
6486 			 ch->seid);  in i40e_setup_hw_channel()
6491 	vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;  in i40e_setup_hw_channel()
6492 	dev_dbg(&pf->pdev->dev,  in i40e_setup_hw_channel()
6493 …"Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base…  in i40e_setup_hw_channel()
6494 		ch->seid, ch->vsi_number, ch->stat_counter_idx,  in i40e_setup_hw_channel()
6495 		ch->num_queue_pairs,  in i40e_setup_hw_channel()
6496 		vsi->next_base_queue);  in i40e_setup_hw_channel()
6501  * i40e_setup_channel - setup new channel using uplink element
6517 	if (vsi->type == I40E_VSI_MAIN) {  in i40e_setup_channel()
6520 		dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",  in i40e_setup_channel()
6521 			vsi->type);  in i40e_setup_channel()
6527 	seid = main_vsi->uplink_seid;  in i40e_setup_channel()
6532 		dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");  in i40e_setup_channel()
6536 	return ch->initialized ? true : false;  in i40e_setup_channel()
6540  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6549 	struct i40e_pf *pf = vsi->back;  in i40e_validate_and_set_switch_mode()
6550 	struct i40e_hw *hw = &pf->hw;  in i40e_validate_and_set_switch_mode()
6555 		return -EINVAL;  in i40e_validate_and_set_switch_mode()
6557 	if (hw->dev_caps.switch_mode) {  in i40e_validate_and_set_switch_mode()
6558 		/* if switch mode is set, support mode2 (non-tunneled for  in i40e_validate_and_set_switch_mode()
6561 		u32 switch_mode = hw->dev_caps.switch_mode &  in i40e_validate_and_set_switch_mode()
6566 			dev_err(&pf->pdev->dev,  in i40e_validate_and_set_switch_mode()
6567 				"Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",  in i40e_validate_and_set_switch_mode()
6568 				hw->dev_caps.switch_mode);  in i40e_validate_and_set_switch_mode()
6569 			return -EINVAL;  in i40e_validate_and_set_switch_mode()
6583 	ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,  in i40e_validate_and_set_switch_mode()
6584 					pf->last_sw_conf_valid_flags,  in i40e_validate_and_set_switch_mode()
6586 	if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)  in i40e_validate_and_set_switch_mode()
6587 		dev_err(&pf->pdev->dev,  in i40e_validate_and_set_switch_mode()
6591 				    hw->aq.asq_last_status));  in i40e_validate_and_set_switch_mode()
6597  * i40e_create_queue_channel - function to create channel
6607 	struct i40e_pf *pf = vsi->back;  in i40e_create_queue_channel()
6612 		return -EINVAL;  in i40e_create_queue_channel()
6614 	if (!ch->num_queue_pairs) {  in i40e_create_queue_channel()
6615 		dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",  in i40e_create_queue_channel()
6616 			ch->num_queue_pairs);  in i40e_create_queue_channel()
6617 		return -EINVAL;  in i40e_create_queue_channel()
6621 	err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,  in i40e_create_queue_channel()
6624 		dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",  in i40e_create_queue_channel()
6625 			 ch->num_queue_pairs);  in i40e_create_queue_channel()
6626 		return -EINVAL;  in i40e_create_queue_channel()
6633 	if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {  in i40e_create_queue_channel()
6634 		set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);  in i40e_create_queue_channel()
6636 		if (vsi->type == I40E_VSI_MAIN) {  in i40e_create_queue_channel()
6647 	/* By this time, vsi->cnt_q_avail shall be set to non-zero and  in i40e_create_queue_channel()
6650 	if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {  in i40e_create_queue_channel()
6651 		dev_dbg(&pf->pdev->dev,  in i40e_create_queue_channel()
6653 			vsi->cnt_q_avail, ch->num_queue_pairs);  in i40e_create_queue_channel()
6654 		return -EINVAL;  in i40e_create_queue_channel()
6658 	if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {  in i40e_create_queue_channel()
6659 		err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);  in i40e_create_queue_channel()
6661 			dev_info(&pf->pdev->dev,  in i40e_create_queue_channel()
6663 				 ch->num_queue_pairs);  in i40e_create_queue_channel()
6664 			return -EINVAL;  in i40e_create_queue_channel()
6669 		dev_info(&pf->pdev->dev, "Failed to setup channel\n");  in i40e_create_queue_channel()
6670 		return -EINVAL;  in i40e_create_queue_channel()
6673 	dev_info(&pf->pdev->dev,  in i40e_create_queue_channel()
6675 		 ch->seid, ch->num_queue_pairs);  in i40e_create_queue_channel()
6678 	if (ch->max_tx_rate) {  in i40e_create_queue_channel()
6679 		u64 credits = ch->max_tx_rate;  in i40e_create_queue_channel()
6681 		if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))  in i40e_create_queue_channel()
6682 			return -EINVAL;  in i40e_create_queue_channel()
6685 		dev_dbg(&pf->pdev->dev,  in i40e_create_queue_channel()
6686 			"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",  in i40e_create_queue_channel()
6687 			ch->max_tx_rate,  in i40e_create_queue_channel()
6689 			ch->seid);  in i40e_create_queue_channel()
6693 	ch->parent_vsi = vsi;  in i40e_create_queue_channel()
6696 	vsi->cnt_q_avail -= ch->num_queue_pairs;  in i40e_create_queue_channel()
6702  * i40e_configure_queue_channels - Add queue channel for the given TCs
6705  * Configures queue channel mapping to the given TCs
6713 	/* Create app vsi with the TCs. Main VSI with TC0 is already set up */  in i40e_configure_queue_channels()
6714 	vsi->tc_seid_map[0] = vsi->seid;  in i40e_configure_queue_channels()
6716 		if (vsi->tc_config.enabled_tc & BIT(i)) {  in i40e_configure_queue_channels()
6719 				ret = -ENOMEM;  in i40e_configure_queue_channels()
6723 			INIT_LIST_HEAD(&ch->list);  in i40e_configure_queue_channels()
6724 			ch->num_queue_pairs =  in i40e_configure_queue_channels()
6725 				vsi->tc_config.tc_info[i].qcount;  in i40e_configure_queue_channels()
6726 			ch->base_queue =  in i40e_configure_queue_channels()
6727 				vsi->tc_config.tc_info[i].qoffset;  in i40e_configure_queue_channels()
6732 			max_rate = vsi->mqprio_qopt.max_rate[i];  in i40e_configure_queue_channels()
6734 			ch->max_tx_rate = max_rate;  in i40e_configure_queue_channels()
6736 			list_add_tail(&ch->list, &vsi->ch_list);  in i40e_configure_queue_channels()
6740 				dev_err(&vsi->back->pdev->dev,  in i40e_configure_queue_channels()
6742 					i, ch->num_queue_pairs);  in i40e_configure_queue_channels()
6745 			vsi->tc_seid_map[i] = ch->seid;  in i40e_configure_queue_channels()
6750 	i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true);  in i40e_configure_queue_channels()
6759  * i40e_veb_config_tc - Configure TCs for given VEB
6768 	struct i40e_pf *pf = veb->pf;  in i40e_veb_config_tc()
6772 	/* No TCs or already enabled TCs just return */  in i40e_veb_config_tc()
6773 	if (!enabled_tc || veb->enabled_tc == enabled_tc)  in i40e_veb_config_tc()
6779 	/* Enable ETS TCs with equal BW Share for now */  in i40e_veb_config_tc()
6785 	ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,  in i40e_veb_config_tc()
6788 		dev_info(&pf->pdev->dev,  in i40e_veb_config_tc()
6791 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_veb_config_tc()
6798 		dev_info(&pf->pdev->dev,  in i40e_veb_config_tc()
6801 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_veb_config_tc()
6810  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6825 	/* Enable the TCs available on PF to all VEBs */  in i40e_dcb_reconfigure()
6833 			dev_info(&pf->pdev->dev,  in i40e_dcb_reconfigure()
6835 				 veb->seid);  in i40e_dcb_reconfigure()
6842 		/* - Enable all TCs for the LAN VSI  in i40e_dcb_reconfigure()
6843 		 * - For all others keep them at TC0 for now  in i40e_dcb_reconfigure()
6845 		if (vsi->type == I40E_VSI_MAIN)  in i40e_dcb_reconfigure()
6852 			dev_info(&pf->pdev->dev,  in i40e_dcb_reconfigure()
6854 				 vsi->seid);  in i40e_dcb_reconfigure()
6857 			/* Re-configure VSI vectors based on updated TC map */  in i40e_dcb_reconfigure()
6859 			if (vsi->netdev)  in i40e_dcb_reconfigure()
6866  * i40e_resume_port_tx - Resume port Tx
6874 	struct i40e_hw *hw = &pf->hw;  in i40e_resume_port_tx()
6879 		dev_info(&pf->pdev->dev,  in i40e_resume_port_tx()
6882 			  i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_resume_port_tx()
6884 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);  in i40e_resume_port_tx()
6892  * i40e_suspend_port_tx - Suspend port Tx
6899 	struct i40e_hw *hw = &pf->hw;  in i40e_suspend_port_tx()
6902 	ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);  in i40e_suspend_port_tx()
6904 		dev_info(&pf->pdev->dev,  in i40e_suspend_port_tx()
6907 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_suspend_port_tx()
6909 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);  in i40e_suspend_port_tx()
6917  * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6927 	struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;  in i40e_hw_set_dcb_config()
6932 		dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");  in i40e_hw_set_dcb_config()
6941 	old_cfg->etsrec = old_cfg->etscfg;  in i40e_hw_set_dcb_config()
6942 	ret = i40e_set_dcb_config(&pf->hw);  in i40e_hw_set_dcb_config()
6944 		dev_info(&pf->pdev->dev,  in i40e_hw_set_dcb_config()
6947 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_hw_set_dcb_config()
6955 	if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {  in i40e_hw_set_dcb_config()
6956 		/* Re-start the VSIs if disabled */  in i40e_hw_set_dcb_config()
6968  * i40e_hw_dcb_config - Program new DCBX settings into HW
6983 	struct i40e_hw *hw = &pf->hw;  in i40e_hw_dcb_config()
6984 	u8 num_ports = hw->num_ports;  in i40e_hw_dcb_config()
6986 	int ret = -EINVAL;  in i40e_hw_dcb_config()
6992 	dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");  in i40e_hw_dcb_config()
6993 	/* Un-pack information to Program ETS HW via shared API  in i40e_hw_dcb_config()
6996 	 * ETS/NON-ETS arbiter mode  in i40e_hw_dcb_config()
6999 	 * PFC priority bit-map  in i40e_hw_dcb_config()
7003 	 * TSA table (ETS or non-ETS)  in i40e_hw_dcb_config()
7013 		switch (new_cfg->etscfg.tsatable[i]) {  in i40e_hw_dcb_config()
7017 					new_cfg->etscfg.tcbwtable[i];  in i40e_hw_dcb_config()
7032 	old_cfg = &hw->local_dcbx_config;  in i40e_hw_dcb_config()
7042 			set_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_hw_dcb_config()
7044 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_hw_dcb_config()
7046 		set_bit(__I40E_PORT_SUSPENDED, pf->state);  in i40e_hw_dcb_config()
7058 		(hw, pf->mac_seid, &ets_data,  in i40e_hw_dcb_config()
7061 		dev_info(&pf->pdev->dev,  in i40e_hw_dcb_config()
7064 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_hw_dcb_config()
7076 	i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,  in i40e_hw_dcb_config()
7078 	i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,  in i40e_hw_dcb_config()
7079 			       new_cfg->etscfg.prioritytable);  in i40e_hw_dcb_config()
7080 	i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);  in i40e_hw_dcb_config()
7086 		mfs_tc[i] = main_vsi->netdev->mtu;  in i40e_hw_dcb_config()
7091 					 false, new_cfg->pfc.pfcenable,  in i40e_hw_dcb_config()
7093 	i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);  in i40e_hw_dcb_config()
7096 	pf->pb_cfg = pb_cfg;  in i40e_hw_dcb_config()
7099 	ret = i40e_aq_dcb_updated(&pf->hw, NULL);  in i40e_hw_dcb_config()
7101 		dev_info(&pf->pdev->dev,  in i40e_hw_dcb_config()
7104 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_hw_dcb_config()
7114 	/* Re-start the VSIs if disabled */  in i40e_hw_dcb_config()
7118 		clear_bit(__I40E_PORT_SUSPENDED, pf->state);  in i40e_hw_dcb_config()
7127 			set_bit(__I40E_PF_RESET_REQUESTED, pf->state);  in i40e_hw_dcb_config()
7132 			set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);  in i40e_hw_dcb_config()
7133 			set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);  in i40e_hw_dcb_config()
7136 		if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps))  in i40e_hw_dcb_config()
7145  * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
7152 	struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;  in i40e_dcb_sw_default_config()
7154 	struct i40e_hw *hw = &pf->hw;  in i40e_dcb_sw_default_config()
7157 	if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) {  in i40e_dcb_sw_default_config()
7159 		memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));  in i40e_dcb_sw_default_config()
7160 		pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;  in i40e_dcb_sw_default_config()
7161 		pf->tmp_cfg.etscfg.maxtcs = 0;  in i40e_dcb_sw_default_config()
7162 		pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;  in i40e_dcb_sw_default_config()
7163 		pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;  in i40e_dcb_sw_default_config()
7164 		pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;  in i40e_dcb_sw_default_config()
7165 		pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;  in i40e_dcb_sw_default_config()
7167 		pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;  in i40e_dcb_sw_default_config()
7168 		pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;  in i40e_dcb_sw_default_config()
7169 		pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;  in i40e_dcb_sw_default_config()
7170 		pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;  in i40e_dcb_sw_default_config()
7172 		return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);  in i40e_dcb_sw_default_config()
7182 		(hw, pf->mac_seid, &ets_data,  in i40e_dcb_sw_default_config()
7185 		dev_info(&pf->pdev->dev,  in i40e_dcb_sw_default_config()
7188 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_dcb_sw_default_config()
7189 		err = -ENOENT;  in i40e_dcb_sw_default_config()
7194 	dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;  in i40e_dcb_sw_default_config()
7195 	dcb_cfg->etscfg.cbs = 0;  in i40e_dcb_sw_default_config()
7196 	dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;  in i40e_dcb_sw_default_config()
7197 	dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;  in i40e_dcb_sw_default_config()
7204  * i40e_init_pf_dcb - Initialize DCB configuration
7212 	struct i40e_hw *hw = &pf->hw;  in i40e_init_pf_dcb()
7218 	if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) {  in i40e_init_pf_dcb()
7219 		dev_info(&pf->pdev->dev, "DCB is not supported.\n");  in i40e_init_pf_dcb()
7220 		err = -EOPNOTSUPP;  in i40e_init_pf_dcb()
7223 	if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) {  in i40e_init_pf_dcb()
7224 		dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");  in i40e_init_pf_dcb()
7227 			dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");  in i40e_init_pf_dcb()
7230 		dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");  in i40e_init_pf_dcb()
7231 		pf->dcbx_cap = DCB_CAP_DCBX_HOST |  in i40e_init_pf_dcb()
7234 		set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_init_pf_dcb()
7235 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_init_pf_dcb()
7241 		if ((!hw->func_caps.dcb) ||  in i40e_init_pf_dcb()
7242 		    (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {  in i40e_init_pf_dcb()
7243 			dev_info(&pf->pdev->dev,  in i40e_init_pf_dcb()
7247 			pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |  in i40e_init_pf_dcb()
7250 			set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_init_pf_dcb()
7254 			if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)  in i40e_init_pf_dcb()
7255 				set_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_init_pf_dcb()
7257 				clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_init_pf_dcb()
7258 			dev_dbg(&pf->pdev->dev,  in i40e_init_pf_dcb()
7261 	} else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {  in i40e_init_pf_dcb()
7262 		dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");  in i40e_init_pf_dcb()
7263 		set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags);  in i40e_init_pf_dcb()
7265 		dev_info(&pf->pdev->dev,  in i40e_init_pf_dcb()
7268 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_init_pf_dcb()
7282 	if (vsi->netdev->ethtool_ops->get_eee)  in i40e_print_link_message_eee()
7283 		vsi->netdev->ethtool_ops->get_eee(vsi->netdev, &kedata);  in i40e_print_link_message_eee()
7286 		netdev_info(vsi->netdev,  in i40e_print_link_message_eee()
7291 		netdev_info(vsi->netdev,  in i40e_print_link_message_eee()
7297  * i40e_print_link_message - print link up or down
7304 	struct i40e_pf *pf = vsi->back;  in i40e_print_link_message()
7312 		new_speed = pf->hw.phy.link_info.link_speed;  in i40e_print_link_message()
7316 	if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))  in i40e_print_link_message()
7318 	vsi->current_isup = isup;  in i40e_print_link_message()
7319 	vsi->current_speed = new_speed;  in i40e_print_link_message()
7321 		netdev_info(vsi->netdev, "NIC Link is Down\n");  in i40e_print_link_message()
7328 	if (pf->hw.func_caps.npar_enable &&  in i40e_print_link_message()
7329 	    (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||  in i40e_print_link_message()
7330 	     pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))  in i40e_print_link_message()
7331 		netdev_warn(vsi->netdev,  in i40e_print_link_message()
7334 	switch (pf->hw.phy.link_info.link_speed) {  in i40e_print_link_message()
7363 	switch (pf->hw.fc.current_mode) {  in i40e_print_link_message()
7378 	if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {  in i40e_print_link_message()
7383 		if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)  in i40e_print_link_message()
7386 		if (pf->hw.phy.link_info.fec_info &  in i40e_print_link_message()
7388 			fec = "CL74 FC-FEC/BASE-R";  in i40e_print_link_message()
7389 		else if (pf->hw.phy.link_info.fec_info &  in i40e_print_link_message()
7391 			fec = "CL108 RS-FEC";  in i40e_print_link_message()
7393 		/* 'CL108 RS-FEC' should be displayed when RS is requested, or  in i40e_print_link_message()
7396 		if (vsi->back->hw.phy.link_info.req_fec_info &  in i40e_print_link_message()
7398 			if (vsi->back->hw.phy.link_info.req_fec_info &  in i40e_print_link_message()
7400 				req_fec = "CL108 RS-FEC";  in i40e_print_link_message()
7402 				req_fec = "CL74 FC-FEC/BASE-R";  in i40e_print_link_message()
7404 		netdev_info(vsi->netdev,  in i40e_print_link_message()
7407 	} else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {  in i40e_print_link_message()
7412 		if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)  in i40e_print_link_message()
7415 		if (pf->hw.phy.link_info.fec_info &  in i40e_print_link_message()
7417 			fec = "CL74 FC-FEC/BASE-R";  in i40e_print_link_message()
7419 		if (pf->hw.phy.link_info.req_fec_info &  in i40e_print_link_message()
7421 			req_fec = "CL74 FC-FEC/BASE-R";  in i40e_print_link_message()
7423 		netdev_info(vsi->netdev,  in i40e_print_link_message()
7433  * i40e_up_complete - Finish the last steps of bringing up a connection
7438 	struct i40e_pf *pf = vsi->back;  in i40e_up_complete()
7441 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_up_complete()
7451 	clear_bit(__I40E_VSI_DOWN, vsi->state);  in i40e_up_complete()
7455 	if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&  in i40e_up_complete()
7456 	    (vsi->netdev)) {  in i40e_up_complete()
7458 		netif_tx_start_all_queues(vsi->netdev);  in i40e_up_complete()
7459 		netif_carrier_on(vsi->netdev);  in i40e_up_complete()
7463 	if (vsi->type == I40E_VSI_FDIR) {  in i40e_up_complete()
7465 		pf->fd_add_err = 0;  in i40e_up_complete()
7466 		pf->fd_atr_cnt = 0;  in i40e_up_complete()
7473 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);  in i40e_up_complete()
7480  * i40e_vsi_reinit_locked - Reset the VSI
7488 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_reinit_locked()
7490 	while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))  in i40e_vsi_reinit_locked()
7495 	clear_bit(__I40E_CONFIG_BUSY, pf->state);  in i40e_vsi_reinit_locked()
7499  * i40e_force_link_state - Force the link status
7508 	struct i40e_hw *hw = &pf->hw;  in i40e_force_link_state()
7522 		dev_err(&pf->pdev->dev,  in i40e_force_link_state()
7525 			i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_force_link_state()
7534 		dev_err(&pf->pdev->dev,  in i40e_force_link_state()
7537 			i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_force_link_state()
7545 	if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags))  in i40e_force_link_state()
7561 	if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) {  in i40e_force_link_state()
7579 		dev_err(&pf->pdev->dev,  in i40e_force_link_state()
7582 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_force_link_state()
7603  * i40e_up - Bring the connection back up after being down
7610 	if (vsi->type == I40E_VSI_MAIN &&  in i40e_up()
7611 	    (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||  in i40e_up()
7612 	     test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))  in i40e_up()
7613 		i40e_force_link_state(vsi->back, true);  in i40e_up()
7623  * i40e_down - Shutdown the connection processing
7631 	 * sets the vsi->state __I40E_VSI_DOWN bit.  in i40e_down()
7633 	if (vsi->netdev) {  in i40e_down()
7634 		netif_carrier_off(vsi->netdev);  in i40e_down()
7635 		netif_tx_disable(vsi->netdev);  in i40e_down()
7639 	if (vsi->type == I40E_VSI_MAIN &&  in i40e_down()
7640 	   (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||  in i40e_down()
7641 	    test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))  in i40e_down()
7642 		i40e_force_link_state(vsi->back, false);  in i40e_down()
7645 	for (i = 0; i < vsi->num_queue_pairs; i++) {  in i40e_down()
7646 		i40e_clean_tx_ring(vsi->tx_rings[i]);  in i40e_down()
7648 			/* Make sure that in-progress ndo_xdp_xmit and  in i40e_down()
7652 			i40e_clean_tx_ring(vsi->xdp_rings[i]);  in i40e_down()
7654 		i40e_clean_rx_ring(vsi->rx_rings[i]);  in i40e_down()
7660  * i40e_validate_mqprio_qopt- validate queue mapping info
7671 	if (mqprio_qopt->qopt.offset[0] != 0 ||  in i40e_validate_mqprio_qopt()
7672 	    mqprio_qopt->qopt.num_tc < 1 ||  in i40e_validate_mqprio_qopt()
7673 	    mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)  in i40e_validate_mqprio_qopt()
7674 		return -EINVAL;  in i40e_validate_mqprio_qopt()
7676 		if (!mqprio_qopt->qopt.count[i])  in i40e_validate_mqprio_qopt()
7677 			return -EINVAL;  in i40e_validate_mqprio_qopt()
7678 		if (mqprio_qopt->min_rate[i]) {  in i40e_validate_mqprio_qopt()
7679 			dev_err(&vsi->back->pdev->dev,  in i40e_validate_mqprio_qopt()
7681 			return -EINVAL;  in i40e_validate_mqprio_qopt()
7683 		max_rate = mqprio_qopt->max_rate[i];  in i40e_validate_mqprio_qopt()
7687 		if (i >= mqprio_qopt->qopt.num_tc - 1)  in i40e_validate_mqprio_qopt()
7689 		if (mqprio_qopt->qopt.offset[i + 1] !=  in i40e_validate_mqprio_qopt()
7690 		    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))  in i40e_validate_mqprio_qopt()
7691 			return -EINVAL;  in i40e_validate_mqprio_qopt()
7693 	if (vsi->num_queue_pairs <  in i40e_validate_mqprio_qopt()
7694 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {  in i40e_validate_mqprio_qopt()
7695 		dev_err(&vsi->back->pdev->dev,  in i40e_validate_mqprio_qopt()
7697 		return -EINVAL;  in i40e_validate_mqprio_qopt()
7700 		dev_err(&vsi->back->pdev->dev,  in i40e_validate_mqprio_qopt()
7702 		return -EINVAL;  in i40e_validate_mqprio_qopt()
7708  * i40e_vsi_set_default_tc_config - set default values for tc configuration
7717 	vsi->tc_config.numtc = 1;  in i40e_vsi_set_default_tc_config()
7718 	vsi->tc_config.enabled_tc = 1;  in i40e_vsi_set_default_tc_config()
7719 	qcount = min_t(int, vsi->alloc_queue_pairs,  in i40e_vsi_set_default_tc_config()
7720 		       i40e_pf_get_max_q_per_tc(vsi->back));  in i40e_vsi_set_default_tc_config()
7722 		/* For the TC that is not enabled set the offset to default  in i40e_vsi_set_default_tc_config()
7725 		vsi->tc_config.tc_info[i].qoffset = 0;  in i40e_vsi_set_default_tc_config()
7727 			vsi->tc_config.tc_info[i].qcount = qcount;  in i40e_vsi_set_default_tc_config()
7729 			vsi->tc_config.tc_info[i].qcount = 1;  in i40e_vsi_set_default_tc_config()
7730 		vsi->tc_config.tc_info[i].netdev_tc = 0;  in i40e_vsi_set_default_tc_config()
7755 	*aq_err = hw->aq.asq_last_status;  in i40e_del_macvlan_filter()
7784 	*aq_err = hw->aq.asq_last_status;  in i40e_add_macvlan_filter()
7790  * i40e_reset_ch_rings - Reset the queue contexts in a channel
7800 	for (i = 0; i < ch->num_queue_pairs; i++) {  in i40e_reset_ch_rings()
7801 		pf_q = ch->base_queue + i;  in i40e_reset_ch_rings()
7802 		tx_ring = vsi->tx_rings[pf_q];  in i40e_reset_ch_rings()
7803 		tx_ring->ch = NULL;  in i40e_reset_ch_rings()
7804 		rx_ring = vsi->rx_rings[pf_q];  in i40e_reset_ch_rings()
7805 		rx_ring->ch = NULL;  in i40e_reset_ch_rings()
7822 	if (list_empty(&vsi->macvlan_list))  in i40e_free_macvlan_channels()
7825 	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {  in i40e_free_macvlan_channels()
7830 			clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);  in i40e_free_macvlan_channels()
7831 			netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);  in i40e_free_macvlan_channels()
7832 			netdev_set_sb_channel(ch->fwd->netdev, 0);  in i40e_free_macvlan_channels()
7833 			kfree(ch->fwd);  in i40e_free_macvlan_channels()
7834 			ch->fwd = NULL;  in i40e_free_macvlan_channels()
7837 		list_del(&ch->list);  in i40e_free_macvlan_channels()
7838 		parent_vsi = ch->parent_vsi;  in i40e_free_macvlan_channels()
7839 		if (!parent_vsi || !ch->initialized) {  in i40e_free_macvlan_channels()
7845 		ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,  in i40e_free_macvlan_channels()
7848 			dev_err(&vsi->back->pdev->dev,  in i40e_free_macvlan_channels()
7850 				ch->seid, parent_vsi->seid);  in i40e_free_macvlan_channels()
7853 	vsi->macvlan_cnt = 0;  in i40e_free_macvlan_channels()
7857  * i40e_fwd_ring_up - bring the macvlan device up
7867 	struct i40e_pf *pf = vsi->back;  in i40e_fwd_ring_up()
7868 	struct i40e_hw *hw = &pf->hw;  in i40e_fwd_ring_up()
7871 	list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {  in i40e_fwd_ring_up()
7873 			iter->fwd = fwd;  in i40e_fwd_ring_up()
7876 				netdev_bind_sb_channel_queue(vsi->netdev, vdev,  in i40e_fwd_ring_up()
7878 							     iter->num_queue_pairs,  in i40e_fwd_ring_up()
7879 							     iter->base_queue);  in i40e_fwd_ring_up()
7880 			for (i = 0; i < iter->num_queue_pairs; i++) {  in i40e_fwd_ring_up()
7884 				pf_q = iter->base_queue + i;  in i40e_fwd_ring_up()
7887 				tx_ring = vsi->tx_rings[pf_q];  in i40e_fwd_ring_up()
7888 				tx_ring->ch = iter;  in i40e_fwd_ring_up()
7891 				rx_ring = vsi->rx_rings[pf_q];  in i40e_fwd_ring_up()
7892 				rx_ring->ch = iter;  in i40e_fwd_ring_up()
7900 		return -EINVAL;  in i40e_fwd_ring_up()
7908 	ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);  in i40e_fwd_ring_up()
7912 		for (i = 0; i < ch->num_queue_pairs; i++) {  in i40e_fwd_ring_up()
7916 			pf_q = ch->base_queue + i;  in i40e_fwd_ring_up()
7917 			rx_ring = vsi->rx_rings[pf_q];  in i40e_fwd_ring_up()
7918 			rx_ring->netdev = NULL;  in i40e_fwd_ring_up()
7920 		dev_info(&pf->pdev->dev,  in i40e_fwd_ring_up()
7931  * i40e_setup_macvlans - create the channels which will be macvlans
7940 	struct i40e_pf *pf = vsi->back;  in i40e_setup_macvlans()
7941 	struct i40e_hw *hw = &pf->hw;  in i40e_setup_macvlans()
7946 	u8 offset = 0;  in i40e_setup_macvlans()  local
7948 	if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)  in i40e_setup_macvlans()
7949 		return -EINVAL;  in i40e_setup_macvlans()
7951 	num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);  in i40e_setup_macvlans()
7953 	/* find the next higher power-of-2 of num queue pairs */  in i40e_setup_macvlans()
7954 	pow = fls(roundup_pow_of_two(num_qps) - 1);  in i40e_setup_macvlans()
7956 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |  in i40e_setup_macvlans()
7963 	ctxt.seid = vsi->seid;  in i40e_setup_macvlans()
7964 	ctxt.pf_num = vsi->back->hw.pf_id;  in i40e_setup_macvlans()
7966 	ctxt.uplink_seid = vsi->uplink_seid;  in i40e_setup_macvlans()
7967 	ctxt.info = vsi->info;  in i40e_setup_macvlans()
7970 	ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);  in i40e_setup_macvlans()
7974 	vsi->rss_size = max_t(u16, num_qps, qcnt);  in i40e_setup_macvlans()
7977 		dev_info(&pf->pdev->dev,  in i40e_setup_macvlans()
7979 			 vsi->rss_size);  in i40e_setup_macvlans()
7982 	vsi->reconfig_rss = true;  in i40e_setup_macvlans()
7983 	dev_dbg(&vsi->back->pdev->dev,  in i40e_setup_macvlans()
7984 		"Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);  in i40e_setup_macvlans()
7985 	vsi->next_base_queue = num_qps;  in i40e_setup_macvlans()
7986 	vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;  in i40e_setup_macvlans()
7988 	/* Update the VSI after updating the VSI queue-mapping  in i40e_setup_macvlans()
7993 		dev_info(&pf->pdev->dev,  in i40e_setup_macvlans()
7996 			 i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_setup_macvlans()
8001 	vsi->info.valid_sections = 0;  in i40e_setup_macvlans()
8004 	INIT_LIST_HEAD(&vsi->macvlan_list);  in i40e_setup_macvlans()
8008 			ret = -ENOMEM;  in i40e_setup_macvlans()
8011 		INIT_LIST_HEAD(&ch->list);  in i40e_setup_macvlans()
8012 		ch->num_queue_pairs = qcnt;  in i40e_setup_macvlans()
8014 			ret = -EINVAL;  in i40e_setup_macvlans()
8018 		ch->parent_vsi = vsi;  in i40e_setup_macvlans()
8019 		vsi->cnt_q_avail -= ch->num_queue_pairs;  in i40e_setup_macvlans()
8020 		vsi->macvlan_cnt++;  in i40e_setup_macvlans()
8021 		list_add_tail(&ch->list, &vsi->macvlan_list);  in i40e_setup_macvlans()
8027 	dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");  in i40e_setup_macvlans()
8034  * i40e_fwd_add - configure macvlans
8042 	struct i40e_vsi *vsi = np->vsi;  in i40e_fwd_add()
8043 	struct i40e_pf *pf = vsi->back;  in i40e_fwd_add()
8047 	if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {  in i40e_fwd_add()
8049 		return ERR_PTR(-EINVAL);  in i40e_fwd_add()
8053 		return ERR_PTR(-EINVAL);  in i40e_fwd_add()
8055 	if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {  in i40e_fwd_add()
8057 		return ERR_PTR(-EINVAL);  in i40e_fwd_add()
8064 		return ERR_PTR(-ERANGE);  in i40e_fwd_add()
8066 	if (!vsi->macvlan_cnt) {  in i40e_fwd_add()
8068 		set_bit(0, vsi->fwd_bitmask);  in i40e_fwd_add()
8074 		vectors = pf->num_lan_msix;  in i40e_fwd_add()
8078 			macvlan_cnt = (vectors - 32) / 4;  in i40e_fwd_add()
8082 			macvlan_cnt = (vectors - 16) / 2;  in i40e_fwd_add()
8086 			macvlan_cnt = vectors - 16;  in i40e_fwd_add()
8090 			macvlan_cnt = vectors - 8;  in i40e_fwd_add()
8094 			macvlan_cnt = vectors - 1;  in i40e_fwd_add()
8098 			return ERR_PTR(-EBUSY);  in i40e_fwd_add()
8112 	avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,  in i40e_fwd_add()
8113 					    vsi->macvlan_cnt);  in i40e_fwd_add()
8115 		return ERR_PTR(-EBUSY);  in i40e_fwd_add()
8120 		return ERR_PTR(-ENOMEM);  in i40e_fwd_add()
8122 	set_bit(avail_macvlan, vsi->fwd_bitmask);  in i40e_fwd_add()
8123 	fwd->bit_no = avail_macvlan;  in i40e_fwd_add()
8125 	fwd->netdev = vdev;  in i40e_fwd_add()
8138 		return ERR_PTR(-EINVAL);  in i40e_fwd_add()
8145  * i40e_del_all_macvlans - Delete all the mac filters on the channels
8151 	struct i40e_pf *pf = vsi->back;  in i40e_del_all_macvlans()
8152 	struct i40e_hw *hw = &pf->hw;  in i40e_del_all_macvlans()
8155 	if (list_empty(&vsi->macvlan_list))  in i40e_del_all_macvlans()
8158 	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {  in i40e_del_all_macvlans()
8160 			ret = i40e_del_macvlan_filter(hw, ch->seid,  in i40e_del_all_macvlans()
8166 				clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);  in i40e_del_all_macvlans()
8167 				netdev_unbind_sb_channel(vsi->netdev,  in i40e_del_all_macvlans()
8168 							 ch->fwd->netdev);  in i40e_del_all_macvlans()
8169 				netdev_set_sb_channel(ch->fwd->netdev, 0);  in i40e_del_all_macvlans()
8170 				kfree(ch->fwd);  in i40e_del_all_macvlans()
8171 				ch->fwd = NULL;  in i40e_del_all_macvlans()
8178  * i40e_fwd_del - delete macvlan interfaces
8187 	struct i40e_vsi *vsi = np->vsi;  in i40e_fwd_del()
8188 	struct i40e_pf *pf = vsi->back;  in i40e_fwd_del()
8189 	struct i40e_hw *hw = &pf->hw;  in i40e_fwd_del()
8193 	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {  in i40e_fwd_del()
8196 				     fwd->netdev->dev_addr)) {  in i40e_fwd_del()
8197 			ret = i40e_del_macvlan_filter(hw, ch->seid,  in i40e_fwd_del()
8203 				clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);  in i40e_fwd_del()
8204 				netdev_unbind_sb_channel(netdev, fwd->netdev);  in i40e_fwd_del()
8205 				netdev_set_sb_channel(fwd->netdev, 0);  in i40e_fwd_del()
8206 				kfree(ch->fwd);  in i40e_fwd_del()
8207 				ch->fwd = NULL;  in i40e_fwd_del()
8209 				dev_info(&pf->pdev->dev,  in i40e_fwd_del()
8220  * i40e_setup_tc - configure multiple traffic classes
8228 	struct i40e_vsi *vsi = np->vsi;  in i40e_setup_tc()
8229 	struct i40e_pf *pf = vsi->back;  in i40e_setup_tc()
8233 	int ret = -EINVAL;  in i40e_setup_tc()
8237 	old_queue_pairs = vsi->num_queue_pairs;  in i40e_setup_tc()
8238 	num_tc = mqprio_qopt->qopt.num_tc;  in i40e_setup_tc()
8239 	hw = mqprio_qopt->qopt.hw;  in i40e_setup_tc()
8240 	mode = mqprio_qopt->mode;  in i40e_setup_tc()
8242 		clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);  in i40e_setup_tc()
8243 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));  in i40e_setup_tc()
8248 	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {  in i40e_setup_tc()
8255 		clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);  in i40e_setup_tc()
8258 		if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {  in i40e_setup_tc()
8272 		if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {  in i40e_setup_tc()
8277 		if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_setup_tc()
8282 		memcpy(&vsi->mqprio_qopt, mqprio_qopt,  in i40e_setup_tc()
8284 		set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);  in i40e_setup_tc()
8285 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_setup_tc()
8288 		return -EINVAL;  in i40e_setup_tc()
8297 	if (enabled_tc == vsi->tc_config.enabled_tc &&  in i40e_setup_tc()
8307 	/* Configure VSI for enabled TCs */  in i40e_setup_tc()
8311 			    vsi->seid);  in i40e_setup_tc()
8315 		   (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {  in i40e_setup_tc()
8318 			    vsi->tc_config.tc_info[0].qcount);  in i40e_setup_tc()
8319 		ret = -EINVAL;  in i40e_setup_tc()
8324 	dev_info(&vsi->back->pdev->dev,  in i40e_setup_tc()
8326 		 vsi->seid, vsi->tc_config.tc_info[0].qcount);  in i40e_setup_tc()
8329 		if (vsi->mqprio_qopt.max_rate[0]) {  in i40e_setup_tc()
8331 						  vsi->mqprio_qopt.max_rate[0]);  in i40e_setup_tc()
8333 			ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);  in i40e_setup_tc()
8338 				dev_dbg(&vsi->back->pdev->dev,  in i40e_setup_tc()
8339 					"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",  in i40e_setup_tc()
8342 					vsi->seid);  in i40e_setup_tc()
8350 			vsi->num_queue_pairs = old_queue_pairs;  in i40e_setup_tc()
8371  * i40e_set_cld_element - sets cloud filter element data
8385 	ether_addr_copy(cld->outer_mac, filter->dst_mac);  in i40e_set_cld_element()
8386 	ether_addr_copy(cld->inner_mac, filter->src_mac);  in i40e_set_cld_element()
8388 	if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)  in i40e_set_cld_element()
8391 	if (filter->n_proto == ETH_P_IPV6) {  in i40e_set_cld_element()
8392 #define IPV6_MAX_INDEX	(ARRAY_SIZE(filter->dst_ipv6) - 1)  in i40e_set_cld_element()
8393 		for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {  in i40e_set_cld_element()
8394 			ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);  in i40e_set_cld_element()
8396 			*(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);  in i40e_set_cld_element()
8399 		ipa = be32_to_cpu(filter->dst_ipv4);  in i40e_set_cld_element()
8401 		memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));  in i40e_set_cld_element()
8404 	cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));  in i40e_set_cld_element()
8407 	 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)  in i40e_set_cld_element()
8409 	if (filter->tenant_id)  in i40e_set_cld_element()
8414  * i40e_add_del_cloud_filter - Add/del cloud filter
8426 	struct i40e_pf *pf = vsi->back;  in i40e_add_del_cloud_filter()
8445 	if (filter->flags >= ARRAY_SIZE(flag_table))  in i40e_add_del_cloud_filter()
8446 		return -EIO;  in i40e_add_del_cloud_filter()
8453 	if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)  in i40e_add_del_cloud_filter()
8454 		cld_filter.flags = cpu_to_le16(filter->tunnel_type <<  in i40e_add_del_cloud_filter()
8457 	if (filter->n_proto == ETH_P_IPV6)  in i40e_add_del_cloud_filter()
8458 		cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |  in i40e_add_del_cloud_filter()
8461 		cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |  in i40e_add_del_cloud_filter()
8465 		ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,  in i40e_add_del_cloud_filter()
8468 		ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,  in i40e_add_del_cloud_filter()
8471 		dev_dbg(&pf->pdev->dev,  in i40e_add_del_cloud_filter()
8473 			add ? "add" : "delete", filter->dst_port, ret,  in i40e_add_del_cloud_filter()
8474 			pf->hw.aq.asq_last_status);  in i40e_add_del_cloud_filter()
8476 		dev_info(&pf->pdev->dev,  in i40e_add_del_cloud_filter()
8478 			 add ? "Added" : "Deleted", filter->seid);  in i40e_add_del_cloud_filter()
8483  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8496 	struct i40e_pf *pf = vsi->back;  in i40e_add_del_cloud_filter_big_buf()
8500 	if ((is_valid_ether_addr(filter->dst_mac) &&  in i40e_add_del_cloud_filter_big_buf()
8501 	     is_valid_ether_addr(filter->src_mac)) ||  in i40e_add_del_cloud_filter_big_buf()
8502 	    (is_multicast_ether_addr(filter->dst_mac) &&  in i40e_add_del_cloud_filter_big_buf()
8503 	     is_multicast_ether_addr(filter->src_mac)))  in i40e_add_del_cloud_filter_big_buf()
8504 		return -EOPNOTSUPP;  in i40e_add_del_cloud_filter_big_buf()
8506 	/* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP  in i40e_add_del_cloud_filter_big_buf()
8509 	if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)  in i40e_add_del_cloud_filter_big_buf()
8510 		return -EOPNOTSUPP;  in i40e_add_del_cloud_filter_big_buf()
8513 	if (filter->src_port ||  in i40e_add_del_cloud_filter_big_buf()
8514 	    (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||  in i40e_add_del_cloud_filter_big_buf()
8515 	    !ipv6_addr_any(&filter->ip.v6.src_ip6))  in i40e_add_del_cloud_filter_big_buf()
8516 		return -EOPNOTSUPP;  in i40e_add_del_cloud_filter_big_buf()
8523 	if (is_valid_ether_addr(filter->dst_mac) ||  in i40e_add_del_cloud_filter_big_buf()
8524 	    is_valid_ether_addr(filter->src_mac) ||  in i40e_add_del_cloud_filter_big_buf()
8525 	    is_multicast_ether_addr(filter->dst_mac) ||  in i40e_add_del_cloud_filter_big_buf()
8526 	    is_multicast_ether_addr(filter->src_mac)) {  in i40e_add_del_cloud_filter_big_buf()
8528 		if (filter->dst_ipv4)  in i40e_add_del_cloud_filter_big_buf()
8529 			return -EOPNOTSUPP;  in i40e_add_del_cloud_filter_big_buf()
8538 		if (filter->vlan_id) {  in i40e_add_del_cloud_filter_big_buf()
8543 	} else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||  in i40e_add_del_cloud_filter_big_buf()
8544 		   !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {  in i40e_add_del_cloud_filter_big_buf()
8547 		if (filter->n_proto == ETH_P_IPV6)  in i40e_add_del_cloud_filter_big_buf()
8554 		dev_err(&pf->pdev->dev,  in i40e_add_del_cloud_filter_big_buf()
8556 		return -EINVAL;  in i40e_add_del_cloud_filter_big_buf()
8561 						be16_to_cpu(filter->dst_port);  in i40e_add_del_cloud_filter_big_buf()
8567 			dev_err(&pf->pdev->dev,  in i40e_add_del_cloud_filter_big_buf()
8573 		ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,  in i40e_add_del_cloud_filter_big_buf()
8576 		ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,  in i40e_add_del_cloud_filter_big_buf()
8581 		dev_dbg(&pf->pdev->dev,  in i40e_add_del_cloud_filter_big_buf()
8583 			add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);  in i40e_add_del_cloud_filter_big_buf()
8585 		dev_info(&pf->pdev->dev,  in i40e_add_del_cloud_filter_big_buf()
8587 			 add ? "add" : "delete", filter->seid,  in i40e_add_del_cloud_filter_big_buf()
8588 			 ntohs(filter->dst_port));  in i40e_add_del_cloud_filter_big_buf()
8593  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8604 	struct flow_dissector *dissector = rule->match.dissector;  in i40e_parse_cls_flower()
8606 	struct i40e_pf *pf = vsi->back;  in i40e_parse_cls_flower()
8609 	if (dissector->used_keys &  in i40e_parse_cls_flower()
8618 		dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n",  in i40e_parse_cls_flower()
8619 			dissector->used_keys);  in i40e_parse_cls_flower()
8620 		return -EOPNOTSUPP;  in i40e_parse_cls_flower()
8627 		if (match.mask->keyid != 0)  in i40e_parse_cls_flower()
8630 		filter->tenant_id = be32_to_cpu(match.key->keyid);  in i40e_parse_cls_flower()
8637 		n_proto_key = ntohs(match.key->n_proto);  in i40e_parse_cls_flower()
8638 		n_proto_mask = ntohs(match.mask->n_proto);  in i40e_parse_cls_flower()
8644 		filter->n_proto = n_proto_key & n_proto_mask;  in i40e_parse_cls_flower()
8645 		filter->ip_proto = match.key->ip_proto;  in i40e_parse_cls_flower()
8654 		if (!is_zero_ether_addr(match.mask->dst)) {  in i40e_parse_cls_flower()
8655 			if (is_broadcast_ether_addr(match.mask->dst)) {  in i40e_parse_cls_flower()
8658 				dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",  in i40e_parse_cls_flower()
8659 					match.mask->dst);  in i40e_parse_cls_flower()
8660 				return -EIO;  in i40e_parse_cls_flower()
8664 		if (!is_zero_ether_addr(match.mask->src)) {  in i40e_parse_cls_flower()
8665 			if (is_broadcast_ether_addr(match.mask->src)) {  in i40e_parse_cls_flower()
8668 				dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",  in i40e_parse_cls_flower()
8669 					match.mask->src);  in i40e_parse_cls_flower()
8670 				return -EIO;  in i40e_parse_cls_flower()
8673 		ether_addr_copy(filter->dst_mac, match.key->dst);  in i40e_parse_cls_flower()
8674 		ether_addr_copy(filter->src_mac, match.key->src);  in i40e_parse_cls_flower()
8681 		if (match.mask->vlan_id) {  in i40e_parse_cls_flower()
8682 			if (match.mask->vlan_id == VLAN_VID_MASK) {  in i40e_parse_cls_flower()
8686 				dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",  in i40e_parse_cls_flower()
8687 					match.mask->vlan_id);  in i40e_parse_cls_flower()
8688 				return -EIO;  in i40e_parse_cls_flower()
8692 		filter->vlan_id = cpu_to_be16(match.key->vlan_id);  in i40e_parse_cls_flower()
8699 		addr_type = match.key->addr_type;  in i40e_parse_cls_flower()
8701 		if (flow_rule_has_control_flags(match.mask->flags,  in i40e_parse_cls_flower()
8702 						f->common.extack))  in i40e_parse_cls_flower()
8703 			return -EOPNOTSUPP;  in i40e_parse_cls_flower()
8710 		if (match.mask->dst) {  in i40e_parse_cls_flower()
8711 			if (match.mask->dst == cpu_to_be32(0xffffffff)) {  in i40e_parse_cls_flower()
8714 				dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",  in i40e_parse_cls_flower()
8715 					&match.mask->dst);  in i40e_parse_cls_flower()
8716 				return -EIO;  in i40e_parse_cls_flower()
8720 		if (match.mask->src) {  in i40e_parse_cls_flower()
8721 			if (match.mask->src == cpu_to_be32(0xffffffff)) {  in i40e_parse_cls_flower()
8724 				dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",  in i40e_parse_cls_flower()
8725 					&match.mask->src);  in i40e_parse_cls_flower()
8726 				return -EIO;  in i40e_parse_cls_flower()
8731 			dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");  in i40e_parse_cls_flower()
8732 			return -EIO;  in i40e_parse_cls_flower()
8734 		filter->dst_ipv4 = match.key->dst;  in i40e_parse_cls_flower()
8735 		filter->src_ipv4 = match.key->src;  in i40e_parse_cls_flower()
8746 		if (ipv6_addr_loopback(&match.key->dst) ||  in i40e_parse_cls_flower()
8747 		    ipv6_addr_loopback(&match.key->src)) {  in i40e_parse_cls_flower()
8748 			dev_err(&pf->pdev->dev,  in i40e_parse_cls_flower()
8750 			return -EIO;  in i40e_parse_cls_flower()
8752 		if (!ipv6_addr_any(&match.mask->dst) ||  in i40e_parse_cls_flower()
8753 		    !ipv6_addr_any(&match.mask->src))  in i40e_parse_cls_flower()
8756 		memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,  in i40e_parse_cls_flower()
8757 		       sizeof(filter->src_ipv6));  in i40e_parse_cls_flower()
8758 		memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,  in i40e_parse_cls_flower()
8759 		       sizeof(filter->dst_ipv6));  in i40e_parse_cls_flower()
8766 		if (match.mask->src) {  in i40e_parse_cls_flower()
8767 			if (match.mask->src == cpu_to_be16(0xffff)) {  in i40e_parse_cls_flower()
8770 				dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",  in i40e_parse_cls_flower()
8771 					be16_to_cpu(match.mask->src));  in i40e_parse_cls_flower()
8772 				return -EIO;  in i40e_parse_cls_flower()
8776 		if (match.mask->dst) {  in i40e_parse_cls_flower()
8777 			if (match.mask->dst == cpu_to_be16(0xffff)) {  in i40e_parse_cls_flower()
8780 				dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",  in i40e_parse_cls_flower()
8781 					be16_to_cpu(match.mask->dst));  in i40e_parse_cls_flower()
8782 				return -EIO;  in i40e_parse_cls_flower()
8786 		filter->dst_port = match.key->dst;  in i40e_parse_cls_flower()
8787 		filter->src_port = match.key->src;  in i40e_parse_cls_flower()
8789 		switch (filter->ip_proto) {  in i40e_parse_cls_flower()
8794 			dev_err(&pf->pdev->dev,  in i40e_parse_cls_flower()
8796 			return -EINVAL;  in i40e_parse_cls_flower()
8799 	filter->flags = field_flags;  in i40e_parse_cls_flower()
8817 		filter->seid = vsi->seid;  in i40e_handle_tclass()
8819 	} else if (vsi->tc_config.enabled_tc & BIT(tc)) {  in i40e_handle_tclass()
8820 		if (!filter->dst_port) {  in i40e_handle_tclass()
8821 			dev_err(&vsi->back->pdev->dev,  in i40e_handle_tclass()
8823 			return -EINVAL;  in i40e_handle_tclass()
8825 		if (list_empty(&vsi->ch_list))  in i40e_handle_tclass()
8826 			return -EINVAL;  in i40e_handle_tclass()
8827 		list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,  in i40e_handle_tclass()
8829 			if (ch->seid == vsi->tc_seid_map[tc])  in i40e_handle_tclass()
8830 				filter->seid = ch->seid;  in i40e_handle_tclass()
8834 	dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");  in i40e_handle_tclass()
8835 	return -EINVAL;  in i40e_handle_tclass()
8839  * i40e_configure_clsflower - Configure tc flower filters
8847 	int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);  in i40e_configure_clsflower()
8849 	struct i40e_pf *pf = vsi->back;  in i40e_configure_clsflower()
8853 		dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");  in i40e_configure_clsflower()
8854 		return -EOPNOTSUPP;  in i40e_configure_clsflower()
8858 		dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");  in i40e_configure_clsflower()
8859 		return -EINVAL;  in i40e_configure_clsflower()
8862 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||  in i40e_configure_clsflower()
8863 	    test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))  in i40e_configure_clsflower()
8864 		return -EBUSY;  in i40e_configure_clsflower()
8866 	if (pf->fdir_pf_active_filters ||  in i40e_configure_clsflower()
8867 	    (!hlist_empty(&pf->fdir_filter_list))) {  in i40e_configure_clsflower()
8868 		dev_err(&vsi->back->pdev->dev,  in i40e_configure_clsflower()
8870 		return -EINVAL;  in i40e_configure_clsflower()
8873 	if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) {  in i40e_configure_clsflower()
8874 		dev_err(&vsi->back->pdev->dev,  in i40e_configure_clsflower()
8875 			"Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");  in i40e_configure_clsflower()
8876 		clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags);  in i40e_configure_clsflower()
8877 		clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags);  in i40e_configure_clsflower()
8882 		return -ENOMEM;  in i40e_configure_clsflower()
8884 	filter->cookie = cls_flower->cookie;  in i40e_configure_clsflower()
8895 	if (filter->dst_port)  in i40e_configure_clsflower()
8901 		dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",  in i40e_configure_clsflower()
8907 	INIT_HLIST_NODE(&filter->cloud_node);  in i40e_configure_clsflower()
8909 	hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);  in i40e_configure_clsflower()
8911 	pf->num_cloud_filters++;  in i40e_configure_clsflower()
8920  * i40e_find_cloud_filter - Find the could filter in the list
8932 				  &vsi->back->cloud_filter_list, cloud_node)  in i40e_find_cloud_filter()
8933 		if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))  in i40e_find_cloud_filter()
8939  * i40e_delete_clsflower - Remove tc flower filters
8948 	struct i40e_pf *pf = vsi->back;  in i40e_delete_clsflower()
8951 	filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);  in i40e_delete_clsflower()
8954 		return -EINVAL;  in i40e_delete_clsflower()
8956 	hash_del(&filter->cloud_node);  in i40e_delete_clsflower()
8958 	if (filter->dst_port)  in i40e_delete_clsflower()
8965 		dev_err(&pf->pdev->dev,  in i40e_delete_clsflower()
8968 		return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);  in i40e_delete_clsflower()
8971 	pf->num_cloud_filters--;  in i40e_delete_clsflower()
8972 	if (!pf->num_cloud_filters)  in i40e_delete_clsflower()
8973 		if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&  in i40e_delete_clsflower()
8974 		    !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {  in i40e_delete_clsflower()
8975 			set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_delete_clsflower()
8976 			clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);  in i40e_delete_clsflower()
8977 			clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_delete_clsflower()
8983  * i40e_setup_tc_cls_flower - flower classifier offloads
8990 	struct i40e_vsi *vsi = np->vsi;  in i40e_setup_tc_cls_flower()
8992 	switch (cls_flower->command) {  in i40e_setup_tc_cls_flower()
8998 		return -EOPNOTSUPP;  in i40e_setup_tc_cls_flower()
9000 		return -EOPNOTSUPP;  in i40e_setup_tc_cls_flower()
9009 	if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))  in i40e_setup_tc_block_cb()
9010 		return -EOPNOTSUPP;  in i40e_setup_tc_block_cb()
9017 		return -EOPNOTSUPP;  in i40e_setup_tc_block_cb()
9037 		return -EOPNOTSUPP;  in __i40e_setup_tc()
9042  * i40e_open - Called when a network interface is made active
9056 	struct i40e_vsi *vsi = np->vsi;  in i40e_open()
9057 	struct i40e_pf *pf = vsi->back;  in i40e_open()
9061 	if (test_bit(__I40E_TESTING, pf->state) ||  in i40e_open()
9062 	    test_bit(__I40E_BAD_EEPROM, pf->state))  in i40e_open()
9063 		return -EBUSY;  in i40e_open()
9068 		return -EAGAIN;  in i40e_open()
9075 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |  in i40e_open()
9077 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |  in i40e_open()
9080 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);  in i40e_open()
9087  * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
9098 	ret = netif_set_real_num_rx_queues(vsi->netdev,  in i40e_netif_set_realnum_tx_rx_queues()
9099 					   vsi->num_queue_pairs);  in i40e_netif_set_realnum_tx_rx_queues()
9103 	return netif_set_real_num_tx_queues(vsi->netdev,  in i40e_netif_set_realnum_tx_rx_queues()
9104 					    vsi->num_queue_pairs);  in i40e_netif_set_realnum_tx_rx_queues()
9108  * i40e_vsi_open -
9119 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_open()
9135 	if (vsi->netdev) {  in i40e_vsi_open()
9136 		snprintf(int_name, sizeof(int_name) - 1, "%s-%s",  in i40e_vsi_open()
9137 			 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);  in i40e_vsi_open()
9147 	} else if (vsi->type == I40E_VSI_FDIR) {  in i40e_vsi_open()
9148 		snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",  in i40e_vsi_open()
9149 			 dev_driver_string(&pf->pdev->dev),  in i40e_vsi_open()
9150 			 dev_name(&pf->pdev->dev));  in i40e_vsi_open()
9156 		err = -EINVAL;  in i40e_vsi_open()
9174 	if (vsi->type == I40E_VSI_MAIN)  in i40e_vsi_open()
9181  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
9194 				  &pf->fdir_filter_list, fdir_node) {  in i40e_fdir_filter_exit()
9195 		hlist_del(&filter->fdir_node);  in i40e_fdir_filter_exit()
9199 	list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {  in i40e_fdir_filter_exit()
9200 		list_del(&pit_entry->list);  in i40e_fdir_filter_exit()
9203 	INIT_LIST_HEAD(&pf->l3_flex_pit_list);  in i40e_fdir_filter_exit()
9205 	list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {  in i40e_fdir_filter_exit()
9206 		list_del(&pit_entry->list);  in i40e_fdir_filter_exit()
9209 	INIT_LIST_HEAD(&pf->l4_flex_pit_list);  in i40e_fdir_filter_exit()
9211 	pf->fdir_pf_active_filters = 0;  in i40e_fdir_filter_exit()
9260  * i40e_cloud_filter_exit - Cleans up the cloud filters
9272 				  &pf->cloud_filter_list, cloud_node) {  in i40e_cloud_filter_exit()
9273 		hlist_del(&cfilter->cloud_node);  in i40e_cloud_filter_exit()
9276 	pf->num_cloud_filters = 0;  in i40e_cloud_filter_exit()
9278 	if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&  in i40e_cloud_filter_exit()
9279 	    !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {  in i40e_cloud_filter_exit()
9280 		set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_cloud_filter_exit()
9281 		clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);  in i40e_cloud_filter_exit()
9282 		clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_cloud_filter_exit()
9287  * i40e_close - Disables a network interface
9290  * The close entry point is called when an interface is de-activated
9299 	struct i40e_vsi *vsi = np->vsi;  in i40e_close()
9307  * i40e_do_reset - Start a PF or Core Reset sequence
9334 		dev_dbg(&pf->pdev->dev, "GlobalR requested\n");  in i40e_do_reset()
9335 		val = rd32(&pf->hw, I40E_GLGEN_RTRIG);  in i40e_do_reset()
9337 		wr32(&pf->hw, I40E_GLGEN_RTRIG, val);  in i40e_do_reset()
9345 		dev_dbg(&pf->pdev->dev, "CoreR requested\n");  in i40e_do_reset()
9346 		val = rd32(&pf->hw, I40E_GLGEN_RTRIG);  in i40e_do_reset()
9348 		wr32(&pf->hw, I40E_GLGEN_RTRIG, val);  in i40e_do_reset()
9349 		i40e_flush(&pf->hw);  in i40e_do_reset()
9355 		 * Resets only the PF-specific registers  in i40e_do_reset()
9357 		 * This goes directly to the tear-down and rebuild of  in i40e_do_reset()
9361 		dev_dbg(&pf->pdev->dev, "PFR requested\n");  in i40e_do_reset()
9371 		dev_info(&pf->pdev->dev,  in i40e_do_reset()
9372 			 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?  in i40e_do_reset()
9377 		/* Find the VSI(s) that requested a re-init */  in i40e_do_reset()
9378 		dev_info(&pf->pdev->dev, "VSI reinit requested\n");  in i40e_do_reset()
9382 					       vsi->state))  in i40e_do_reset()
9387 		dev_info(&pf->pdev->dev, "VSI down requested\n");  in i40e_do_reset()
9391 					       vsi->state)) {  in i40e_do_reset()
9392 				set_bit(__I40E_VSI_DOWN, vsi->state);  in i40e_do_reset()
9397 		dev_info(&pf->pdev->dev,  in i40e_do_reset()
9404  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
9416 	if (memcmp(&new_cfg->etscfg,  in i40e_dcb_need_reconfig()
9417 		   &old_cfg->etscfg,  in i40e_dcb_need_reconfig()
9418 		   sizeof(new_cfg->etscfg))) {  in i40e_dcb_need_reconfig()
9420 		if (memcmp(&new_cfg->etscfg.prioritytable,  in i40e_dcb_need_reconfig()
9421 			   &old_cfg->etscfg.prioritytable,  in i40e_dcb_need_reconfig()
9422 			   sizeof(new_cfg->etscfg.prioritytable))) {  in i40e_dcb_need_reconfig()
9424 			dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");  in i40e_dcb_need_reconfig()
9427 		if (memcmp(&new_cfg->etscfg.tcbwtable,  in i40e_dcb_need_reconfig()
9428 			   &old_cfg->etscfg.tcbwtable,  in i40e_dcb_need_reconfig()
9429 			   sizeof(new_cfg->etscfg.tcbwtable)))  in i40e_dcb_need_reconfig()
9430 			dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");  in i40e_dcb_need_reconfig()
9432 		if (memcmp(&new_cfg->etscfg.tsatable,  in i40e_dcb_need_reconfig()
9433 			   &old_cfg->etscfg.tsatable,  in i40e_dcb_need_reconfig()
9434 			   sizeof(new_cfg->etscfg.tsatable)))  in i40e_dcb_need_reconfig()
9435 			dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");  in i40e_dcb_need_reconfig()
9439 	if (memcmp(&new_cfg->pfc,  in i40e_dcb_need_reconfig()
9440 		   &old_cfg->pfc,  in i40e_dcb_need_reconfig()
9441 		   sizeof(new_cfg->pfc))) {  in i40e_dcb_need_reconfig()
9443 		dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");  in i40e_dcb_need_reconfig()
9447 	if (memcmp(&new_cfg->app,  in i40e_dcb_need_reconfig()
9448 		   &old_cfg->app,  in i40e_dcb_need_reconfig()
9449 		   sizeof(new_cfg->app))) {  in i40e_dcb_need_reconfig()
9451 		dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");  in i40e_dcb_need_reconfig()
9454 	dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);  in i40e_dcb_need_reconfig()
9459  * i40e_handle_lldp_event - Handle LLDP Change MIB event
9467 		(struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;  in i40e_handle_lldp_event()
9468 	struct i40e_hw *hw = &pf->hw;  in i40e_handle_lldp_event()
9474 	/* X710-T*L 2.5G and 5G speeds don't support DCB */  in i40e_handle_lldp_event()
9475 	if (I40E_IS_X710TL_DEVICE(hw->device_id) &&  in i40e_handle_lldp_event()
9476 	    (hw->phy.link_info.link_speed &  in i40e_handle_lldp_event()
9478 	     !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))  in i40e_handle_lldp_event()
9480 		set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_handle_lldp_event()
9483 	if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))  in i40e_handle_lldp_event()
9487 	type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)  in i40e_handle_lldp_event()
9489 	dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);  in i40e_handle_lldp_event()
9494 	type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;  in i40e_handle_lldp_event()
9495 	dev_dbg(&pf->pdev->dev,  in i40e_handle_lldp_event()
9501 				&hw->remote_dcbx_config);  in i40e_handle_lldp_event()
9506 	tmp_dcbx_cfg = hw->local_dcbx_config;  in i40e_handle_lldp_event()
9509 	memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));  in i40e_handle_lldp_event()
9511 	ret = i40e_get_dcb_config(&pf->hw);  in i40e_handle_lldp_event()
9513 		/* X710-T*L 2.5G and 5G speeds don't support DCB */  in i40e_handle_lldp_event()
9514 		if (I40E_IS_X710TL_DEVICE(hw->device_id) &&  in i40e_handle_lldp_event()
9515 		    (hw->phy.link_info.link_speed &  in i40e_handle_lldp_event()
9517 			dev_warn(&pf->pdev->dev,  in i40e_handle_lldp_event()
9518 				 "DCB is not supported for X710-T*L 2.5/5G speeds\n");  in i40e_handle_lldp_event()
9519 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_handle_lldp_event()
9521 			dev_info(&pf->pdev->dev,  in i40e_handle_lldp_event()
9524 				 i40e_aq_str(&pf->hw,  in i40e_handle_lldp_event()
9525 					     pf->hw.aq.asq_last_status));  in i40e_handle_lldp_event()
9531 	if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,  in i40e_handle_lldp_event()
9533 		dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");  in i40e_handle_lldp_event()
9538 					       &hw->local_dcbx_config);  in i40e_handle_lldp_event()
9540 	i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);  in i40e_handle_lldp_event()
9546 	if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)  in i40e_handle_lldp_event()
9547 		set_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_handle_lldp_event()
9549 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_handle_lldp_event()
9551 	set_bit(__I40E_PORT_SUSPENDED, pf->state);  in i40e_handle_lldp_event()
9560 	clear_bit(__I40E_PORT_SUSPENDED, pf->state);  in i40e_handle_lldp_event()
9569 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);  in i40e_handle_lldp_event()
9573 		set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);  in i40e_handle_lldp_event()
9574 		set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);  in i40e_handle_lldp_event()
9583  * i40e_do_reset_safe - Protected reset path for userland calls.
9596  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9607 		(struct i40e_aqc_lan_overflow *)&e->desc.params.raw;  in i40e_handle_lan_overflow_event()
9608 	u32 queue = le32_to_cpu(data->prtdcb_rupto);  in i40e_handle_lan_overflow_event()
9609 	u32 qtx_ctl = le32_to_cpu(data->otx_ctl);  in i40e_handle_lan_overflow_event()
9610 	struct i40e_hw *hw = &pf->hw;  in i40e_handle_lan_overflow_event()
9614 	dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",  in i40e_handle_lan_overflow_event()
9623 	vf_id -= hw->func_caps.vf_base_id;  in i40e_handle_lan_overflow_event()
9624 	vf = &pf->vf[vf_id];  in i40e_handle_lan_overflow_event()
9632  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9639 	val = rd32(&pf->hw, I40E_PFQF_FDSTAT);  in i40e_get_cur_guaranteed_fd_count()
9645  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9652 	val = rd32(&pf->hw, I40E_PFQF_FDSTAT);  in i40e_get_current_fd_count()
9659  * i40e_get_global_fd_count - Get total FD filters programmed on device
9666 	val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);  in i40e_get_global_fd_count()
9673  * i40e_reenable_fdir_sb - Restore FDir SB capability
9678 	if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))  in i40e_reenable_fdir_sb()
9679 		if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&  in i40e_reenable_fdir_sb()
9680 		    (I40E_DEBUG_FD & pf->hw.debug_mask))  in i40e_reenable_fdir_sb()
9681 …dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now…  in i40e_reenable_fdir_sb()
9685  * i40e_reenable_fdir_atr - Restore FDir ATR capability
9690 	if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {  in i40e_reenable_fdir_atr()
9700 		if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&  in i40e_reenable_fdir_atr()
9701 		    (I40E_DEBUG_FD & pf->hw.debug_mask))  in i40e_reenable_fdir_atr()
9702 …dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no c…  in i40e_reenable_fdir_atr()
9707  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9715 	pf->fdir_pf_active_filters--;  in i40e_delete_invalid_filter()
9716 	pf->fd_inv = 0;  in i40e_delete_invalid_filter()
9718 	switch (filter->flow_type) {  in i40e_delete_invalid_filter()
9720 		pf->fd_tcp4_filter_cnt--;  in i40e_delete_invalid_filter()
9723 		pf->fd_udp4_filter_cnt--;  in i40e_delete_invalid_filter()
9726 		pf->fd_sctp4_filter_cnt--;  in i40e_delete_invalid_filter()
9729 		pf->fd_tcp6_filter_cnt--;  in i40e_delete_invalid_filter()
9732 		pf->fd_udp6_filter_cnt--;  in i40e_delete_invalid_filter()
9735 		pf->fd_udp6_filter_cnt--;  in i40e_delete_invalid_filter()
9738 		switch (filter->ipl4_proto) {  in i40e_delete_invalid_filter()
9740 			pf->fd_tcp4_filter_cnt--;  in i40e_delete_invalid_filter()
9743 			pf->fd_udp4_filter_cnt--;  in i40e_delete_invalid_filter()
9746 			pf->fd_sctp4_filter_cnt--;  in i40e_delete_invalid_filter()
9749 			pf->fd_ip4_filter_cnt--;  in i40e_delete_invalid_filter()
9754 		switch (filter->ipl4_proto) {  in i40e_delete_invalid_filter()
9756 			pf->fd_tcp6_filter_cnt--;  in i40e_delete_invalid_filter()
9759 			pf->fd_udp6_filter_cnt--;  in i40e_delete_invalid_filter()
9762 			pf->fd_sctp6_filter_cnt--;  in i40e_delete_invalid_filter()
9765 			pf->fd_ip6_filter_cnt--;  in i40e_delete_invalid_filter()
9772 	hlist_del(&filter->fdir_node);  in i40e_delete_invalid_filter()
9777  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9786 	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))  in i40e_fdir_check_and_reenable()
9789 	/* Check if we have enough room to re-enable FDir SB capability. */  in i40e_fdir_check_and_reenable()
9791 	fcnt_avail = pf->fdir_pf_filter_count;  in i40e_fdir_check_and_reenable()
9792 	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||  in i40e_fdir_check_and_reenable()
9793 	    (pf->fd_add_err == 0) ||  in i40e_fdir_check_and_reenable()
9794 	    (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))  in i40e_fdir_check_and_reenable()
9797 	/* We should wait for even more space before re-enabling ATR.  in i40e_fdir_check_and_reenable()
9801 	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&  in i40e_fdir_check_and_reenable()
9802 	    pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)  in i40e_fdir_check_and_reenable()
9806 	if (pf->fd_inv > 0) {  in i40e_fdir_check_and_reenable()
9808 					  &pf->fdir_filter_list, fdir_node)  in i40e_fdir_check_and_reenable()
9809 			if (filter->fd_id == pf->fd_inv)  in i40e_fdir_check_and_reenable()
9817  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9828 	if (!time_after(jiffies, pf->fd_flush_timestamp +  in i40e_fdir_flush_and_replay()
9833 	 * should not re-enable ATR for some time.  in i40e_fdir_flush_and_replay()
9835 	min_flush_time = pf->fd_flush_timestamp +  in i40e_fdir_flush_and_replay()
9837 	fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;  in i40e_fdir_flush_and_replay()
9841 		if (I40E_DEBUG_FD & pf->hw.debug_mask)  in i40e_fdir_flush_and_replay()
9842 			dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");  in i40e_fdir_flush_and_replay()
9846 	pf->fd_flush_timestamp = jiffies;  in i40e_fdir_flush_and_replay()
9847 	set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);  in i40e_fdir_flush_and_replay()
9849 	wr32(&pf->hw, I40E_PFQF_CTL_1,  in i40e_fdir_flush_and_replay()
9851 	i40e_flush(&pf->hw);  in i40e_fdir_flush_and_replay()
9852 	pf->fd_flush_cnt++;  in i40e_fdir_flush_and_replay()
9853 	pf->fd_add_err = 0;  in i40e_fdir_flush_and_replay()
9855 		/* Check FD flush status every 5-6msec */  in i40e_fdir_flush_and_replay()
9857 		reg = rd32(&pf->hw, I40E_PFQF_CTL_1);  in i40e_fdir_flush_and_replay()
9860 	} while (flush_wait_retry--);  in i40e_fdir_flush_and_replay()
9862 		dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");  in i40e_fdir_flush_and_replay()
9866 		if (!disable_atr && !pf->fd_tcp4_filter_cnt)  in i40e_fdir_flush_and_replay()
9867 			clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);  in i40e_fdir_flush_and_replay()
9868 		clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);  in i40e_fdir_flush_and_replay()
9869 		if (I40E_DEBUG_FD & pf->hw.debug_mask)  in i40e_fdir_flush_and_replay()
9870 			dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");  in i40e_fdir_flush_and_replay()
9875  * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9880 	return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;  in i40e_get_current_atr_cnt()
9884  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9891 	if (test_bit(__I40E_DOWN, pf->state))  in i40e_fdir_reinit_subtask()
9894 	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))  in i40e_fdir_reinit_subtask()
9902  * i40e_vsi_link_event - notify VSI of a link event
9908 	if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))  in i40e_vsi_link_event()
9911 	switch (vsi->type) {  in i40e_vsi_link_event()
9913 		if (!vsi->netdev || !vsi->netdev_registered)  in i40e_vsi_link_event()
9917 			netif_carrier_on(vsi->netdev);  in i40e_vsi_link_event()
9918 			netif_tx_wake_all_queues(vsi->netdev);  in i40e_vsi_link_event()
9920 			netif_carrier_off(vsi->netdev);  in i40e_vsi_link_event()
9921 			netif_tx_stop_all_queues(vsi->netdev);  in i40e_vsi_link_event()
9937  * i40e_veb_link_event - notify elements on the veb of a link event
9947 	if (!veb || !veb->pf)  in i40e_veb_link_event()
9949 	pf = veb->pf;  in i40e_veb_link_event()
9953 		if (vsi->uplink_seid == veb->seid)  in i40e_veb_link_event()
9958  * i40e_link_event - Update netif_carrier status
9973 	pf->hw.phy.get_link_info = true;  in i40e_link_event()
9974 	old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);  in i40e_link_event()
9975 	status = i40e_get_link_status(&pf->hw, &new_link);  in i40e_link_event()
9979 		clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);  in i40e_link_event()
9984 		set_bit(__I40E_TEMP_LINK_POLLING, pf->state);  in i40e_link_event()
9985 		dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",  in i40e_link_event()
9990 	old_link_speed = pf->hw.phy.link_info_old.link_speed;  in i40e_link_event()
9991 	new_link_speed = pf->hw.phy.link_info.link_speed;  in i40e_link_event()
9995 	    (test_bit(__I40E_VSI_DOWN, vsi->state) ||  in i40e_link_event()
9996 	     new_link == netif_carrier_ok(vsi->netdev)))  in i40e_link_event()
10009 	if (pf->vf)  in i40e_link_event()
10012 	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))  in i40e_link_event()
10018 	if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)  in i40e_link_event()
10025 		dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");  in i40e_link_event()
10026 		memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));  in i40e_link_event()
10029 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_link_event()
10030 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_link_event()
10032 			pf->dcbx_cap = DCB_CAP_DCBX_HOST |  in i40e_link_event()
10034 			set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_link_event()
10035 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_link_event()
10042  * i40e_watchdog_subtask - periodic checks not using event driven response
10052 	if (test_bit(__I40E_DOWN, pf->state) ||  in i40e_watchdog_subtask()
10053 	    test_bit(__I40E_CONFIG_BUSY, pf->state))  in i40e_watchdog_subtask()
10057 	if (time_before(jiffies, (pf->service_timer_previous +  in i40e_watchdog_subtask()
10058 				  pf->service_timer_period)))  in i40e_watchdog_subtask()
10060 	pf->service_timer_previous = jiffies;  in i40e_watchdog_subtask()
10062 	if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) ||  in i40e_watchdog_subtask()
10063 	    test_bit(__I40E_TEMP_LINK_POLLING, pf->state))  in i40e_watchdog_subtask()
10070 		if (vsi->netdev)  in i40e_watchdog_subtask()
10073 	if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) {  in i40e_watchdog_subtask()
10084  * i40e_reset_subtask - Set up for resetting the device and driver
10091 	if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {  in i40e_reset_subtask()
10093 		clear_bit(__I40E_REINIT_REQUESTED, pf->state);  in i40e_reset_subtask()
10095 	if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {  in i40e_reset_subtask()
10097 		clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);  in i40e_reset_subtask()
10099 	if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {  in i40e_reset_subtask()
10101 		clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);  in i40e_reset_subtask()
10103 	if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {  in i40e_reset_subtask()
10105 		clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);  in i40e_reset_subtask()
10107 	if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {  in i40e_reset_subtask()
10109 		clear_bit(__I40E_DOWN_REQUESTED, pf->state);  in i40e_reset_subtask()
10115 	if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {  in i40e_reset_subtask()
10123 	    !test_bit(__I40E_DOWN, pf->state) &&  in i40e_reset_subtask()
10124 	    !test_bit(__I40E_CONFIG_BUSY, pf->state)) {  in i40e_reset_subtask()
10130  * i40e_handle_link_event - Handle link event
10138 		(struct i40e_aqc_get_link_status *)&e->desc.params.raw;  in i40e_handle_link_event()
10140 	/* Do a new status request to re-enable LSE reporting  in i40e_handle_link_event()
10149 	if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {  in i40e_handle_link_event()
10150 		dev_err(&pf->pdev->dev,  in i40e_handle_link_event()
10152 		dev_err(&pf->pdev->dev,  in i40e_handle_link_event()
10158 		if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&  in i40e_handle_link_event()
10159 		    (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&  in i40e_handle_link_event()
10160 		    (!(status->link_info & I40E_AQ_LINK_UP)) &&  in i40e_handle_link_event()
10161 		    (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) {  in i40e_handle_link_event()
10162 			dev_err(&pf->pdev->dev,  in i40e_handle_link_event()
10164 			dev_err(&pf->pdev->dev,  in i40e_handle_link_event()
10171  * i40e_clean_adminq_subtask - Clean the AdminQ rings
10177 	struct i40e_hw *hw = &pf->hw;  in i40e_clean_adminq_subtask()
10185 	if (test_bit(__I40E_RESET_FAILED, pf->state))  in i40e_clean_adminq_subtask()
10189 	val = rd32(&pf->hw, I40E_PF_ARQLEN);  in i40e_clean_adminq_subtask()
10192 		if (hw->debug_mask & I40E_DEBUG_AQ)  in i40e_clean_adminq_subtask()
10193 			dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");  in i40e_clean_adminq_subtask()
10197 		if (hw->debug_mask & I40E_DEBUG_AQ)  in i40e_clean_adminq_subtask()
10198 			dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");  in i40e_clean_adminq_subtask()
10200 		pf->arq_overflows++;  in i40e_clean_adminq_subtask()
10203 		if (hw->debug_mask & I40E_DEBUG_AQ)  in i40e_clean_adminq_subtask()
10204 			dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");  in i40e_clean_adminq_subtask()
10208 		wr32(&pf->hw, I40E_PF_ARQLEN, val);  in i40e_clean_adminq_subtask()
10210 	val = rd32(&pf->hw, I40E_PF_ATQLEN);  in i40e_clean_adminq_subtask()
10213 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)  in i40e_clean_adminq_subtask()
10214 			dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");  in i40e_clean_adminq_subtask()
10218 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)  in i40e_clean_adminq_subtask()
10219 			dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");  in i40e_clean_adminq_subtask()
10223 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)  in i40e_clean_adminq_subtask()
10224 			dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");  in i40e_clean_adminq_subtask()
10228 		wr32(&pf->hw, I40E_PF_ATQLEN, val);  in i40e_clean_adminq_subtask()
10237 		if (ret == -EALREADY)  in i40e_clean_adminq_subtask()
10240 			dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);  in i40e_clean_adminq_subtask()
10261 			dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");  in i40e_clean_adminq_subtask()
10269 			dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");  in i40e_clean_adminq_subtask()
10273 			dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");  in i40e_clean_adminq_subtask()
10278 			i40e_debug(&pf->hw, I40E_DEBUG_NVM,  in i40e_clean_adminq_subtask()
10283 			dev_info(&pf->pdev->dev,  in i40e_clean_adminq_subtask()
10291 		clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);  in i40e_clean_adminq_subtask()
10293 	/* re-enable Admin queue interrupt cause */  in i40e_clean_adminq_subtask()
10303  * i40e_verify_eeprom - make sure eeprom is good to use
10310 	err = i40e_diag_eeprom_test(&pf->hw);  in i40e_verify_eeprom()
10313 		err = i40e_diag_eeprom_test(&pf->hw);  in i40e_verify_eeprom()
10315 			dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",  in i40e_verify_eeprom()
10317 			set_bit(__I40E_BAD_EEPROM, pf->state);  in i40e_verify_eeprom()
10321 	if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {  in i40e_verify_eeprom()
10322 		dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");  in i40e_verify_eeprom()
10323 		clear_bit(__I40E_BAD_EEPROM, pf->state);  in i40e_verify_eeprom()
10331  * enable switch loop back or die - no point in a return value
10339 	ctxt.seid = pf->main_vsi_seid;  in i40e_enable_pf_switch_lb()
10340 	ctxt.pf_num = pf->hw.pf_id;  in i40e_enable_pf_switch_lb()
10342 	ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);  in i40e_enable_pf_switch_lb()
10344 		dev_info(&pf->pdev->dev,  in i40e_enable_pf_switch_lb()
10347 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_enable_pf_switch_lb()
10354 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);  in i40e_enable_pf_switch_lb()
10356 		dev_info(&pf->pdev->dev,  in i40e_enable_pf_switch_lb()
10359 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_enable_pf_switch_lb()
10367  * disable switch loop back or die - no point in a return value
10375 	ctxt.seid = pf->main_vsi_seid;  in i40e_disable_pf_switch_lb()
10376 	ctxt.pf_num = pf->hw.pf_id;  in i40e_disable_pf_switch_lb()
10378 	ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);  in i40e_disable_pf_switch_lb()
10380 		dev_info(&pf->pdev->dev,  in i40e_disable_pf_switch_lb()
10383 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_disable_pf_switch_lb()
10390 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);  in i40e_disable_pf_switch_lb()
10392 		dev_info(&pf->pdev->dev,  in i40e_disable_pf_switch_lb()
10395 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_disable_pf_switch_lb()
10400  * i40e_config_bridge_mode - Configure the HW bridge mode
10409 	struct i40e_pf *pf = veb->pf;  in i40e_config_bridge_mode()
10411 	if (pf->hw.debug_mask & I40E_DEBUG_LAN)  in i40e_config_bridge_mode()
10412 		dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",  in i40e_config_bridge_mode()
10413 			 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");  in i40e_config_bridge_mode()
10414 	if (veb->bridge_mode & BRIDGE_MODE_VEPA)  in i40e_config_bridge_mode()
10421  * i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it
10431 	struct i40e_pf *pf = veb->pf;  in i40e_reconstitute_veb()
10436 	 * there can be only one non-floating VEB that have uplink to MAC SEID  in i40e_reconstitute_veb()
10439 	if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) {  in i40e_reconstitute_veb()
10440 		dev_err(&pf->pdev->dev,  in i40e_reconstitute_veb()
10441 			"Invalid uplink SEID for VEB %d\n", veb->idx);  in i40e_reconstitute_veb()
10442 		return -ENOENT;  in i40e_reconstitute_veb()
10445 	if (veb->uplink_seid == pf->mac_seid) {  in i40e_reconstitute_veb()
10449 		if (WARN_ON(ctl_vsi->veb_idx != veb->idx ||  in i40e_reconstitute_veb()
10450 			    !(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) {  in i40e_reconstitute_veb()
10451 			dev_err(&pf->pdev->dev,  in i40e_reconstitute_veb()
10452 				"Invalid control VSI for VEB %d\n", veb->idx);  in i40e_reconstitute_veb()
10453 			return -ENOENT;  in i40e_reconstitute_veb()
10459 			dev_err(&pf->pdev->dev,  in i40e_reconstitute_veb()
10461 				veb->idx, ret);  in i40e_reconstitute_veb()
10473 	if (veb->uplink_seid) {  in i40e_reconstitute_veb()
10474 		if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))  in i40e_reconstitute_veb()
10475 			veb->bridge_mode = BRIDGE_MODE_VEB;  in i40e_reconstitute_veb()
10477 			veb->bridge_mode = BRIDGE_MODE_VEPA;  in i40e_reconstitute_veb()
10486 		if (vsi->veb_idx == veb->idx) {  in i40e_reconstitute_veb()
10487 			vsi->uplink_seid = veb->seid;  in i40e_reconstitute_veb()
10490 				dev_info(&pf->pdev->dev,  in i40e_reconstitute_veb()
10503  * i40e_get_capabilities - get info about the HW
10519 			return -ENOMEM;  in i40e_get_capabilities()
10522 		err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,  in i40e_get_capabilities()
10528 		if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {  in i40e_get_capabilities()
10531 		} else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {  in i40e_get_capabilities()
10532 			dev_info(&pf->pdev->dev,  in i40e_get_capabilities()
10535 				 i40e_aq_str(&pf->hw,  in i40e_get_capabilities()
10536 					     pf->hw.aq.asq_last_status));  in i40e_get_capabilities()
10537 			return -ENODEV;  in i40e_get_capabilities()
10541 	if (pf->hw.debug_mask & I40E_DEBUG_USER) {  in i40e_get_capabilities()
10543 			dev_info(&pf->pdev->dev,  in i40e_get_capabilities()
10545 				 pf->hw.pf_id, pf->hw.func_caps.num_vfs,  in i40e_get_capabilities()
10546 				 pf->hw.func_caps.num_msix_vectors,  in i40e_get_capabilities()
10547 				 pf->hw.func_caps.num_msix_vectors_vf,  in i40e_get_capabilities()
10548 				 pf->hw.func_caps.fd_filters_guaranteed,  in i40e_get_capabilities()
10549 				 pf->hw.func_caps.fd_filters_best_effort,  in i40e_get_capabilities()
10550 				 pf->hw.func_caps.num_tx_qp,  in i40e_get_capabilities()
10551 				 pf->hw.func_caps.num_vsis);  in i40e_get_capabilities()
10553 			dev_info(&pf->pdev->dev,  in i40e_get_capabilities()
10555 				 pf->hw.dev_caps.switch_mode,  in i40e_get_capabilities()
10556 				 pf->hw.dev_caps.valid_functions);  in i40e_get_capabilities()
10557 			dev_info(&pf->pdev->dev,  in i40e_get_capabilities()
10558 				 "SR-IOV=%d, num_vfs for all function=%u\n",  in i40e_get_capabilities()
10559 				 pf->hw.dev_caps.sr_iov_1_1,  in i40e_get_capabilities()
10560 				 pf->hw.dev_caps.num_vfs);  in i40e_get_capabilities()
10561 			dev_info(&pf->pdev->dev,  in i40e_get_capabilities()
10563 				 pf->hw.dev_caps.num_vsis,  in i40e_get_capabilities()
10564 				 pf->hw.dev_caps.num_rx_qp,  in i40e_get_capabilities()
10565 				 pf->hw.dev_caps.num_tx_qp);  in i40e_get_capabilities()
10569 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \  in i40e_get_capabilities()
10570 		       + pf->hw.func_caps.num_vfs)  in i40e_get_capabilities()
10571 		if (pf->hw.revision_id == 0 &&  in i40e_get_capabilities()
10572 		    pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {  in i40e_get_capabilities()
10573 			dev_info(&pf->pdev->dev,  in i40e_get_capabilities()
10575 				 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);  in i40e_get_capabilities()
10576 			pf->hw.func_caps.num_vsis = DEF_NUM_VSI;  in i40e_get_capabilities()
10585  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10595 	if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {  in i40e_fdir_sb_setup()
10604 			wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);  in i40e_fdir_sb_setup()
10607 	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))  in i40e_fdir_sb_setup()
10616 		vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, main_vsi->seid, 0);  in i40e_fdir_sb_setup()
10618 			dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");  in i40e_fdir_sb_setup()
10619 			clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_fdir_sb_setup()
10620 			set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_fdir_sb_setup()
10629  * i40e_fdir_teardown - release the Flow Director resources
10643  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10653 	struct i40e_pf *pf = vsi->back;  in i40e_rebuild_cloud_filters()
10658 	hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,  in i40e_rebuild_cloud_filters()
10660 		if (cfilter->seid != seid)  in i40e_rebuild_cloud_filters()
10663 		if (cfilter->dst_port)  in i40e_rebuild_cloud_filters()
10670 			dev_dbg(&pf->pdev->dev,  in i40e_rebuild_cloud_filters()
10673 				i40e_aq_str(&pf->hw,  in i40e_rebuild_cloud_filters()
10674 					    pf->hw.aq.asq_last_status));  in i40e_rebuild_cloud_filters()
10682  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10692 	if (list_empty(&vsi->ch_list))  in i40e_rebuild_channels()
10695 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {  in i40e_rebuild_channels()
10696 		if (!ch->initialized)  in i40e_rebuild_channels()
10699 		ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);  in i40e_rebuild_channels()
10701 			dev_info(&vsi->back->pdev->dev,  in i40e_rebuild_channels()
10703 				 vsi->uplink_seid);  in i40e_rebuild_channels()
10707 		ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);  in i40e_rebuild_channels()
10709 			dev_info(&vsi->back->pdev->dev,  in i40e_rebuild_channels()
10711 				 ch->seid);  in i40e_rebuild_channels()
10715 		vsi->next_base_queue = vsi->next_base_queue +  in i40e_rebuild_channels()
10716 							ch->num_queue_pairs;  in i40e_rebuild_channels()
10717 		if (ch->max_tx_rate) {  in i40e_rebuild_channels()
10718 			u64 credits = ch->max_tx_rate;  in i40e_rebuild_channels()
10720 			if (i40e_set_bw_limit(vsi, ch->seid,  in i40e_rebuild_channels()
10721 					      ch->max_tx_rate))  in i40e_rebuild_channels()
10722 				return -EINVAL;  in i40e_rebuild_channels()
10725 			dev_dbg(&vsi->back->pdev->dev,  in i40e_rebuild_channels()
10726 				"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",  in i40e_rebuild_channels()
10727 				ch->max_tx_rate,  in i40e_rebuild_channels()
10729 				ch->seid);  in i40e_rebuild_channels()
10731 		ret = i40e_rebuild_cloud_filters(vsi, ch->seid);  in i40e_rebuild_channels()
10733 			dev_dbg(&vsi->back->pdev->dev,  in i40e_rebuild_channels()
10735 				ch->seid);  in i40e_rebuild_channels()
10743  * i40e_clean_xps_state - clean xps state for every tx_ring
10750 	if (vsi->tx_rings)  in i40e_clean_xps_state()
10751 		for (i = 0; i < vsi->num_queue_pairs; i++)  in i40e_clean_xps_state()
10752 			if (vsi->tx_rings[i])  in i40e_clean_xps_state()
10754 					  vsi->tx_rings[i]->state);  in i40e_clean_xps_state()
10758  * i40e_prep_for_reset - prep for the core to reset
10765 	struct i40e_hw *hw = &pf->hw;  in i40e_prep_for_reset()
10770 	clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);  in i40e_prep_for_reset()
10771 	if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))  in i40e_prep_for_reset()
10773 	if (i40e_check_asq_alive(&pf->hw))  in i40e_prep_for_reset()
10776 	dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");  in i40e_prep_for_reset()
10783 		vsi->seid = 0;  in i40e_prep_for_reset()
10786 	i40e_shutdown_adminq(&pf->hw);  in i40e_prep_for_reset()
10789 	if (hw->hmc.hmc_obj) {  in i40e_prep_for_reset()
10792 			dev_warn(&pf->pdev->dev,  in i40e_prep_for_reset()
10803  * i40e_send_version - update firmware with driver version
10815 	i40e_aq_send_driver_version(&pf->hw, &dv, NULL);  in i40e_send_version()
10819  * i40e_get_oem_version - get OEM specific version information
10859 	hw->nvm.oem_ver =  in i40e_get_oem_version()
10862 	hw->nvm.eetrack = I40E_OEM_EETRACK_ID;  in i40e_get_oem_version()
10866  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10871 	struct i40e_hw *hw = &pf->hw;  in i40e_reset()
10876 		dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);  in i40e_reset()
10877 		set_bit(__I40E_RESET_FAILED, pf->state);  in i40e_reset()
10878 		clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);  in i40e_reset()
10880 		pf->pfr_count++;  in i40e_reset()
10886  * i40e_rebuild - rebuild using a saved config
10888  * @reinit: if the Main VSI needs to re-initialized.
10896 	struct i40e_hw *hw = &pf->hw;  in i40e_rebuild()
10902 	if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&  in i40e_rebuild()
10904 		i40e_set_ethtool_ops(vsi->netdev);  in i40e_rebuild()
10906 	if (test_bit(__I40E_DOWN, pf->state) &&  in i40e_rebuild()
10907 	    !test_bit(__I40E_RECOVERY_MODE, pf->state))  in i40e_rebuild()
10909 	dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");  in i40e_rebuild()
10912 	ret = i40e_init_adminq(&pf->hw);  in i40e_rebuild()
10914 		dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n",  in i40e_rebuild()
10916 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_rebuild()
10919 	i40e_get_oem_version(&pf->hw);  in i40e_rebuild()
10921 	if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {  in i40e_rebuild()
10926 	/* re-verify the eeprom if we just had an EMP reset */  in i40e_rebuild()
10927 	if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))  in i40e_rebuild()
10934 	if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {  in i40e_rebuild()
10952 			free_irq(pf->pdev->irq, pf);  in i40e_rebuild()
10972 	ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,  in i40e_rebuild()
10973 				hw->func_caps.num_rx_qp, 0, 0);  in i40e_rebuild()
10975 		dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);  in i40e_rebuild()
10980 		dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);  in i40e_rebuild()
10985 	/* Enable FW to write a default DCB config on link-up  in i40e_rebuild()
10992 		if (I40E_IS_X710TL_DEVICE(hw->device_id) &&  in i40e_rebuild()
10993 		    (hw->phy.link_info.link_speed &  in i40e_rebuild()
10996 			dev_warn(&pf->pdev->dev,  in i40e_rebuild()
10997 				 "DCB is not supported for X710-T*L 2.5/5G speeds\n");  in i40e_rebuild()
10998 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_rebuild()
11003 				dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",  in i40e_rebuild()
11005 				clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_rebuild()
11021 	ret = i40e_aq_set_phy_int_mask(&pf->hw,  in i40e_rebuild()
11026 		dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",  in i40e_rebuild()
11028 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_rebuild()
11037 	if (vsi->uplink_seid != pf->mac_seid) {  in i40e_rebuild()
11038 		dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");  in i40e_rebuild()
11052 			if (veb->uplink_seid == pf->mac_seid) {  in i40e_rebuild()
11053 				dev_info(&pf->pdev->dev,  in i40e_rebuild()
11056 				vsi->uplink_seid = pf->mac_seid;  in i40e_rebuild()
11058 			} else if (veb->uplink_seid == 0) {  in i40e_rebuild()
11059 				dev_info(&pf->pdev->dev,  in i40e_rebuild()
11066 	if (vsi->uplink_seid == pf->mac_seid) {  in i40e_rebuild()
11067 		dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");  in i40e_rebuild()
11071 			dev_info(&pf->pdev->dev,  in i40e_rebuild()
11077 	if (vsi->mqprio_qopt.max_rate[0]) {  in i40e_rebuild()
11079 						  vsi->mqprio_qopt.max_rate[0]);  in i40e_rebuild()
11082 		ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);  in i40e_rebuild()
11088 		dev_dbg(&vsi->back->pdev->dev,  in i40e_rebuild()
11089 			"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",  in i40e_rebuild()
11092 			vsi->seid);  in i40e_rebuild()
11095 	ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);  in i40e_rebuild()
11120 	if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {  in i40e_rebuild()
11122 		ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);  in i40e_rebuild()
11124 			dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",  in i40e_rebuild()
11126 				 i40e_aq_str(&pf->hw,  in i40e_rebuild()
11127 					     pf->hw.aq.asq_last_status));  in i40e_rebuild()
11130 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_rebuild()
11142 	i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,  in i40e_rebuild()
11143 						       pf->main_vsi_seid);  in i40e_rebuild()
11153 	ret = i40e_set_promiscuous(pf, pf->cur_promisc);  in i40e_rebuild()
11155 		dev_warn(&pf->pdev->dev,  in i40e_rebuild()
11157 			 pf->cur_promisc ? "on" : "off",  in i40e_rebuild()
11159 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_rebuild()
11173 	clear_bit(__I40E_RESET_FAILED, pf->state);  in i40e_rebuild()
11175 	clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);  in i40e_rebuild()
11176 	clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);  in i40e_rebuild()
11180  * i40e_reset_and_rebuild - reset and rebuild using a saved config
11182  * @reinit: if the Main VSI needs to re-initialized.
11191 	if (test_bit(__I40E_IN_REMOVE, pf->state))  in i40e_reset_and_rebuild()
11201 		dev_err(&pf->pdev->dev, "%s: i40e_reset() FAILED", __func__);  in i40e_reset_and_rebuild()
11205  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
11227 	struct i40e_hw *hw = &pf->hw;  in i40e_handle_mdd_event()
11233 	if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))  in i40e_handle_mdd_event()
11242 		u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) -  in i40e_handle_mdd_event()
11243 				pf->hw.func_caps.base_queue;  in i40e_handle_mdd_event()
11245 …dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x …  in i40e_handle_mdd_event()
11254 		u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) -  in i40e_handle_mdd_event()
11255 				pf->hw.func_caps.base_queue;  in i40e_handle_mdd_event()
11257 …dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02…  in i40e_handle_mdd_event()
11267 			dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");  in i40e_handle_mdd_event()
11272 			dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");  in i40e_handle_mdd_event()
11277 	for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {  in i40e_handle_mdd_event()
11278 		vf = &(pf->vf[i]);  in i40e_handle_mdd_event()
11282 			vf->num_mdd_events++;  in i40e_handle_mdd_event()
11283 			dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",  in i40e_handle_mdd_event()
11285 			dev_info(&pf->pdev->dev,  in i40e_handle_mdd_event()
11286 				 "Use PF Control I/F to re-enable the VF\n");  in i40e_handle_mdd_event()
11287 			set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);  in i40e_handle_mdd_event()
11293 			vf->num_mdd_events++;  in i40e_handle_mdd_event()
11294 			dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",  in i40e_handle_mdd_event()
11296 			dev_info(&pf->pdev->dev,  in i40e_handle_mdd_event()
11297 				 "Use PF Control I/F to re-enable the VF\n");  in i40e_handle_mdd_event()
11298 			set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);  in i40e_handle_mdd_event()
11302 	/* re-enable mdd interrupt cause */  in i40e_handle_mdd_event()
11303 	clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);  in i40e_handle_mdd_event()
11311  * i40e_service_task - Run the driver's async subtasks
11322 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||  in i40e_service_task()
11323 	    test_bit(__I40E_SUSPENDED, pf->state))  in i40e_service_task()
11326 	if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))  in i40e_service_task()
11329 	if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {  in i40e_service_task()
11337 		if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {  in i40e_service_task()
11343 					       pf->state))  in i40e_service_task()
11355 	clear_bit(__I40E_SERVICE_SCHED, pf->state);  in i40e_service_task()
11361 	if (time_after(jiffies, (start_time + pf->service_timer_period)) ||  in i40e_service_task()
11362 	    test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)		 ||  in i40e_service_task()
11363 	    test_bit(__I40E_MDD_EVENT_PENDING, pf->state)		 ||  in i40e_service_task()
11364 	    test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))  in i40e_service_task()
11369  * i40e_service_timer - timer callback
11376 	mod_timer(&pf->service_timer,  in i40e_service_timer()
11377 		  round_jiffies(jiffies + pf->service_timer_period));  in i40e_service_timer()
11382  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11387 	struct i40e_pf *pf = vsi->back;  in i40e_set_num_rings_in_vsi()
11389 	switch (vsi->type) {  in i40e_set_num_rings_in_vsi()
11391 		vsi->alloc_queue_pairs = pf->num_lan_qps;  in i40e_set_num_rings_in_vsi()
11392 		if (!vsi->num_tx_desc)  in i40e_set_num_rings_in_vsi()
11393 			vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,  in i40e_set_num_rings_in_vsi()
11395 		if (!vsi->num_rx_desc)  in i40e_set_num_rings_in_vsi()
11396 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,  in i40e_set_num_rings_in_vsi()
11398 		if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_set_num_rings_in_vsi()
11399 			vsi->num_q_vectors = pf->num_lan_msix;  in i40e_set_num_rings_in_vsi()
11401 			vsi->num_q_vectors = 1;  in i40e_set_num_rings_in_vsi()
11406 		vsi->alloc_queue_pairs = 1;  in i40e_set_num_rings_in_vsi()
11407 		vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,  in i40e_set_num_rings_in_vsi()
11409 		vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,  in i40e_set_num_rings_in_vsi()
11411 		vsi->num_q_vectors = pf->num_fdsb_msix;  in i40e_set_num_rings_in_vsi()
11415 		vsi->alloc_queue_pairs = pf->num_vmdq_qps;  in i40e_set_num_rings_in_vsi()
11416 		if (!vsi->num_tx_desc)  in i40e_set_num_rings_in_vsi()
11417 			vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,  in i40e_set_num_rings_in_vsi()
11419 		if (!vsi->num_rx_desc)  in i40e_set_num_rings_in_vsi()
11420 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,  in i40e_set_num_rings_in_vsi()
11422 		vsi->num_q_vectors = pf->num_vmdq_msix;  in i40e_set_num_rings_in_vsi()
11426 		vsi->alloc_queue_pairs = pf->num_vf_qps;  in i40e_set_num_rings_in_vsi()
11427 		if (!vsi->num_tx_desc)  in i40e_set_num_rings_in_vsi()
11428 			vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,  in i40e_set_num_rings_in_vsi()
11430 		if (!vsi->num_rx_desc)  in i40e_set_num_rings_in_vsi()
11431 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,  in i40e_set_num_rings_in_vsi()
11437 		return -ENODATA;  in i40e_set_num_rings_in_vsi()
11441 		vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;  in i40e_set_num_rings_in_vsi()
11442 		vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;  in i40e_set_num_rings_in_vsi()
11449  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11463 	size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *  in i40e_vsi_alloc_arrays()
11465 	vsi->tx_rings = kzalloc(size, GFP_KERNEL);  in i40e_vsi_alloc_arrays()
11466 	if (!vsi->tx_rings)  in i40e_vsi_alloc_arrays()
11467 		return -ENOMEM;  in i40e_vsi_alloc_arrays()
11468 	next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;  in i40e_vsi_alloc_arrays()
11470 		vsi->xdp_rings = next_rings;  in i40e_vsi_alloc_arrays()
11471 		next_rings += vsi->alloc_queue_pairs;  in i40e_vsi_alloc_arrays()
11473 	vsi->rx_rings = next_rings;  in i40e_vsi_alloc_arrays()
11477 		size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;  in i40e_vsi_alloc_arrays()
11478 		vsi->q_vectors = kzalloc(size, GFP_KERNEL);  in i40e_vsi_alloc_arrays()
11479 		if (!vsi->q_vectors) {  in i40e_vsi_alloc_arrays()
11480 			ret = -ENOMEM;  in i40e_vsi_alloc_arrays()
11487 	kfree(vsi->tx_rings);  in i40e_vsi_alloc_arrays()
11492  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11501 	int ret = -ENODEV;  in i40e_vsi_mem_alloc()
11507 	mutex_lock(&pf->switch_mutex);  in i40e_vsi_mem_alloc()
11515 	i = pf->next_vsi;  in i40e_vsi_mem_alloc()
11516 	while (i < pf->num_alloc_vsi && pf->vsi[i])  in i40e_vsi_mem_alloc()
11518 	if (i >= pf->num_alloc_vsi) {  in i40e_vsi_mem_alloc()
11520 		while (i < pf->next_vsi && pf->vsi[i])  in i40e_vsi_mem_alloc()
11524 	if (i < pf->num_alloc_vsi && !pf->vsi[i]) {  in i40e_vsi_mem_alloc()
11527 		ret = -ENODEV;  in i40e_vsi_mem_alloc()
11530 	pf->next_vsi = ++i;  in i40e_vsi_mem_alloc()
11534 		ret = -ENOMEM;  in i40e_vsi_mem_alloc()
11537 	vsi->type = type;  in i40e_vsi_mem_alloc()
11538 	vsi->back = pf;  in i40e_vsi_mem_alloc()
11539 	set_bit(__I40E_VSI_DOWN, vsi->state);  in i40e_vsi_mem_alloc()
11540 	vsi->flags = 0;  in i40e_vsi_mem_alloc()
11541 	vsi->idx = vsi_idx;  in i40e_vsi_mem_alloc()
11542 	vsi->int_rate_limit = 0;  in i40e_vsi_mem_alloc()
11543 	vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?  in i40e_vsi_mem_alloc()
11544 				pf->rss_table_size : 64;  in i40e_vsi_mem_alloc()
11545 	vsi->netdev_registered = false;  in i40e_vsi_mem_alloc()
11546 	vsi->work_limit = I40E_DEFAULT_IRQ_WORK;  in i40e_vsi_mem_alloc()
11547 	hash_init(vsi->mac_filter_hash);  in i40e_vsi_mem_alloc()
11548 	vsi->irqs_ready = false;  in i40e_vsi_mem_alloc()
11551 		vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);  in i40e_vsi_mem_alloc()
11552 		if (!vsi->af_xdp_zc_qps)  in i40e_vsi_mem_alloc()
11568 	spin_lock_init(&vsi->mac_filter_hash_lock);  in i40e_vsi_mem_alloc()
11569 	pf->vsi[vsi_idx] = vsi;  in i40e_vsi_mem_alloc()
11574 	bitmap_free(vsi->af_xdp_zc_qps);  in i40e_vsi_mem_alloc()
11575 	pf->next_vsi = i - 1;  in i40e_vsi_mem_alloc()
11578 	mutex_unlock(&pf->switch_mutex);  in i40e_vsi_mem_alloc()
11583  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11594 		kfree(vsi->q_vectors);  in i40e_vsi_free_arrays()
11595 		vsi->q_vectors = NULL;  in i40e_vsi_free_arrays()
11597 	kfree(vsi->tx_rings);  in i40e_vsi_free_arrays()
11598 	vsi->tx_rings = NULL;  in i40e_vsi_free_arrays()
11599 	vsi->rx_rings = NULL;  in i40e_vsi_free_arrays()
11600 	vsi->xdp_rings = NULL;  in i40e_vsi_free_arrays()
11604  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11613 	kfree(vsi->rss_hkey_user);  in i40e_clear_rss_config_user()
11614 	vsi->rss_hkey_user = NULL;  in i40e_clear_rss_config_user()
11616 	kfree(vsi->rss_lut_user);  in i40e_clear_rss_config_user()
11617 	vsi->rss_lut_user = NULL;  in i40e_clear_rss_config_user()
11621  * i40e_vsi_clear - Deallocate the VSI provided
11622  * @vsi: the VSI being un-configured
11631 	if (!vsi->back)  in i40e_vsi_clear()
11633 	pf = vsi->back;  in i40e_vsi_clear()
11635 	mutex_lock(&pf->switch_mutex);  in i40e_vsi_clear()
11636 	if (!pf->vsi[vsi->idx]) {  in i40e_vsi_clear()
11637 		dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",  in i40e_vsi_clear()
11638 			vsi->idx, vsi->idx, vsi->type);  in i40e_vsi_clear()
11642 	if (pf->vsi[vsi->idx] != vsi) {  in i40e_vsi_clear()
11643 		dev_err(&pf->pdev->dev,  in i40e_vsi_clear()
11644 			"pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",  in i40e_vsi_clear()
11645 			pf->vsi[vsi->idx]->idx,  in i40e_vsi_clear()
11646 			pf->vsi[vsi->idx]->type,  in i40e_vsi_clear()
11647 			vsi->idx, vsi->type);  in i40e_vsi_clear()
11652 	i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);  in i40e_vsi_clear()
11653 	i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);  in i40e_vsi_clear()
11655 	bitmap_free(vsi->af_xdp_zc_qps);  in i40e_vsi_clear()
11659 	pf->vsi[vsi->idx] = NULL;  in i40e_vsi_clear()
11660 	if (vsi->idx < pf->next_vsi)  in i40e_vsi_clear()
11661 		pf->next_vsi = vsi->idx;  in i40e_vsi_clear()
11664 	mutex_unlock(&pf->switch_mutex);  in i40e_vsi_clear()
11672  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11679 	if (vsi->tx_rings && vsi->tx_rings[0]) {  in i40e_vsi_clear_rings()
11680 		for (i = 0; i < vsi->alloc_queue_pairs; i++) {  in i40e_vsi_clear_rings()
11681 			kfree_rcu(vsi->tx_rings[i], rcu);  in i40e_vsi_clear_rings()
11682 			WRITE_ONCE(vsi->tx_rings[i], NULL);  in i40e_vsi_clear_rings()
11683 			WRITE_ONCE(vsi->rx_rings[i], NULL);  in i40e_vsi_clear_rings()
11684 			if (vsi->xdp_rings)  in i40e_vsi_clear_rings()
11685 				WRITE_ONCE(vsi->xdp_rings[i], NULL);  in i40e_vsi_clear_rings()
11691  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11697 	struct i40e_pf *pf = vsi->back;  in i40e_alloc_rings()
11701 	for (i = 0; i < vsi->alloc_queue_pairs; i++) {  in i40e_alloc_rings()
11707 		ring->queue_index = i;  in i40e_alloc_rings()
11708 		ring->reg_idx = vsi->base_queue + i;  in i40e_alloc_rings()
11709 		ring->ring_active = false;  in i40e_alloc_rings()
11710 		ring->vsi = vsi;  in i40e_alloc_rings()
11711 		ring->netdev = vsi->netdev;  in i40e_alloc_rings()
11712 		ring->dev = &pf->pdev->dev;  in i40e_alloc_rings()
11713 		ring->count = vsi->num_tx_desc;  in i40e_alloc_rings()
11714 		ring->size = 0;  in i40e_alloc_rings()
11715 		ring->dcb_tc = 0;  in i40e_alloc_rings()
11716 		if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))  in i40e_alloc_rings()
11717 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;  in i40e_alloc_rings()
11718 		ring->itr_setting = pf->tx_itr_default;  in i40e_alloc_rings()
11719 		WRITE_ONCE(vsi->tx_rings[i], ring++);  in i40e_alloc_rings()
11724 		ring->queue_index = vsi->alloc_queue_pairs + i;  in i40e_alloc_rings()
11725 		ring->reg_idx = vsi->base_queue + ring->queue_index;  in i40e_alloc_rings()
11726 		ring->ring_active = false;  in i40e_alloc_rings()
11727 		ring->vsi = vsi;  in i40e_alloc_rings()
11728 		ring->netdev = NULL;  in i40e_alloc_rings()
11729 		ring->dev = &pf->pdev->dev;  in i40e_alloc_rings()
11730 		ring->count = vsi->num_tx_desc;  in i40e_alloc_rings()
11731 		ring->size = 0;  in i40e_alloc_rings()
11732 		ring->dcb_tc = 0;  in i40e_alloc_rings()
11733 		if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))  in i40e_alloc_rings()
11734 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;  in i40e_alloc_rings()
11736 		ring->itr_setting = pf->tx_itr_default;  in i40e_alloc_rings()
11737 		WRITE_ONCE(vsi->xdp_rings[i], ring++);  in i40e_alloc_rings()
11740 		ring->queue_index = i;  in i40e_alloc_rings()
11741 		ring->reg_idx = vsi->base_queue + i;  in i40e_alloc_rings()
11742 		ring->ring_active = false;  in i40e_alloc_rings()
11743 		ring->vsi = vsi;  in i40e_alloc_rings()
11744 		ring->netdev = vsi->netdev;  in i40e_alloc_rings()
11745 		ring->dev = &pf->pdev->dev;  in i40e_alloc_rings()
11746 		ring->count = vsi->num_rx_desc;  in i40e_alloc_rings()
11747 		ring->size = 0;  in i40e_alloc_rings()
11748 		ring->dcb_tc = 0;  in i40e_alloc_rings()
11749 		ring->itr_setting = pf->rx_itr_default;  in i40e_alloc_rings()
11750 		WRITE_ONCE(vsi->rx_rings[i], ring);  in i40e_alloc_rings()
11757 	return -ENOMEM;  in i40e_alloc_rings()
11761  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11763  * @vectors: the number of MSI-X vectors to request
11769 	vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,  in i40e_reserve_msix_vectors()
11772 		dev_info(&pf->pdev->dev,  in i40e_reserve_msix_vectors()
11773 			 "MSI-X vector reservation failed: %d\n", vectors);  in i40e_reserve_msix_vectors()
11781  * i40e_init_msix - Setup the MSIX capability
11790 	struct i40e_hw *hw = &pf->hw;  in i40e_init_msix()
11797 	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_init_msix()
11798 		return -ENODEV;  in i40e_init_msix()
11801 	 *   - Add 1 for "other" cause for Admin Queue events, etc.  in i40e_init_msix()
11802 	 *   - The number of LAN queue pairs  in i40e_init_msix()
11803 	 *	- Queues being used for RSS.  in i40e_init_msix()
11807 	 *	- assumes symmetric Tx/Rx pairing  in i40e_init_msix()
11808 	 *   - The number of VMDq pairs  in i40e_init_msix()
11809 	 *   - The CPU count within the NUMA node if iWARP is enabled  in i40e_init_msix()
11815 	vectors_left = hw->func_caps.num_msix_vectors;  in i40e_init_msix()
11821 		vectors_left--;  in i40e_init_msix()
11832 	pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);  in i40e_init_msix()
11833 	vectors_left -= pf->num_lan_msix;  in i40e_init_msix()
11836 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {  in i40e_init_msix()
11838 			pf->num_fdsb_msix = 1;  in i40e_init_msix()
11840 			vectors_left--;  in i40e_init_msix()
11842 			pf->num_fdsb_msix = 0;  in i40e_init_msix()
11847 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {  in i40e_init_msix()
11848 		iwarp_requested = pf->num_iwarp_msix;  in i40e_init_msix()
11851 			pf->num_iwarp_msix = 0;  in i40e_init_msix()
11852 		else if (vectors_left < pf->num_iwarp_msix)  in i40e_init_msix()
11853 			pf->num_iwarp_msix = 1;  in i40e_init_msix()
11854 		v_budget += pf->num_iwarp_msix;  in i40e_init_msix()
11855 		vectors_left -= pf->num_iwarp_msix;  in i40e_init_msix()
11859 	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) {  in i40e_init_msix()
11861 			pf->num_vmdq_msix = 0;  in i40e_init_msix()
11862 			pf->num_vmdq_qps = 0;  in i40e_init_msix()
11865 				pf->num_vmdq_vsis * pf->num_vmdq_qps;  in i40e_init_msix()
11876 				pf->num_vmdq_qps = 1;  in i40e_init_msix()
11877 				vmdq_vecs_wanted = pf->num_vmdq_vsis;  in i40e_init_msix()
11882 			pf->num_vmdq_msix = pf->num_vmdq_qps;  in i40e_init_msix()
11885 			vectors_left -= vmdq_vecs;  in i40e_init_msix()
11898 	extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);  in i40e_init_msix()
11899 	pf->num_lan_msix += extra_vectors;  in i40e_init_msix()
11900 	vectors_left -= extra_vectors;  in i40e_init_msix()
11903 …remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");  in i40e_init_msix()
11905 	v_budget += pf->num_lan_msix;  in i40e_init_msix()
11906 	pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),  in i40e_init_msix()
11908 	if (!pf->msix_entries)  in i40e_init_msix()
11909 		return -ENOMEM;  in i40e_init_msix()
11912 		pf->msix_entries[i].entry = i;  in i40e_init_msix()
11916 		clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);  in i40e_init_msix()
11917 		kfree(pf->msix_entries);  in i40e_init_msix()
11918 		pf->msix_entries = NULL;  in i40e_init_msix()
11919 		pci_disable_msix(pf->pdev);  in i40e_init_msix()
11920 		return -ENODEV;  in i40e_init_msix()
11924 		pf->num_vmdq_vsis = 0;  in i40e_init_msix()
11925 		pf->num_vmdq_qps = 0;  in i40e_init_msix()
11926 		pf->num_lan_qps = 1;  in i40e_init_msix()
11927 		pf->num_lan_msix = 1;  in i40e_init_msix()
11937 		dev_info(&pf->pdev->dev,  in i40e_init_msix()
11938 			 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",  in i40e_init_msix()
11941 		vec = v_actual - 1;  in i40e_init_msix()
11944 		pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */  in i40e_init_msix()
11945 		pf->num_vmdq_vsis = 1;  in i40e_init_msix()
11946 		pf->num_vmdq_qps = 1;  in i40e_init_msix()
11951 			pf->num_lan_msix = 1;  in i40e_init_msix()
11954 			if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {  in i40e_init_msix()
11955 				pf->num_lan_msix = 1;  in i40e_init_msix()
11956 				pf->num_iwarp_msix = 1;  in i40e_init_msix()
11958 				pf->num_lan_msix = 2;  in i40e_init_msix()
11962 			if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {  in i40e_init_msix()
11963 				pf->num_iwarp_msix = min_t(int, (vec / 3),  in i40e_init_msix()
11965 				pf->num_vmdq_vsis = min_t(int, (vec / 3),  in i40e_init_msix()
11968 				pf->num_vmdq_vsis = min_t(int, (vec / 2),  in i40e_init_msix()
11971 			if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {  in i40e_init_msix()
11972 				pf->num_fdsb_msix = 1;  in i40e_init_msix()
11973 				vec--;  in i40e_init_msix()
11975 			pf->num_lan_msix = min_t(int,  in i40e_init_msix()
11976 			       (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),  in i40e_init_msix()
11977 							      pf->num_lan_msix);  in i40e_init_msix()
11978 			pf->num_lan_qps = pf->num_lan_msix;  in i40e_init_msix()
11983 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) {  in i40e_init_msix()
11984 		dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");  in i40e_init_msix()
11985 		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_init_msix()
11986 		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_init_msix()
11988 	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) {  in i40e_init_msix()
11989 		dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");  in i40e_init_msix()
11990 		clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);  in i40e_init_msix()
11993 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&  in i40e_init_msix()
11994 	    pf->num_iwarp_msix == 0) {  in i40e_init_msix()
11995 		dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");  in i40e_init_msix()
11996 		clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);  in i40e_init_msix()
11998 	i40e_debug(&pf->hw, I40E_DEBUG_INIT,  in i40e_init_msix()
11999 		   "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",  in i40e_init_msix()
12000 		   pf->num_lan_msix,  in i40e_init_msix()
12001 		   pf->num_vmdq_msix * pf->num_vmdq_vsis,  in i40e_init_msix()
12002 		   pf->num_fdsb_msix,  in i40e_init_msix()
12003 		   pf->num_iwarp_msix);  in i40e_init_msix()
12009  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
12013  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
12022 		return -ENOMEM;  in i40e_vsi_alloc_q_vector()
12024 	q_vector->vsi = vsi;  in i40e_vsi_alloc_q_vector()
12025 	q_vector->v_idx = v_idx;  in i40e_vsi_alloc_q_vector()
12026 	cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);  in i40e_vsi_alloc_q_vector()
12028 	if (vsi->netdev)  in i40e_vsi_alloc_q_vector()
12029 		netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll);  in i40e_vsi_alloc_q_vector()
12032 	vsi->q_vectors[v_idx] = q_vector;  in i40e_vsi_alloc_q_vector()
12038  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
12042  * return -ENOMEM.
12046 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_alloc_q_vectors()
12050 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_vsi_alloc_q_vectors()
12051 		num_q_vectors = vsi->num_q_vectors;  in i40e_vsi_alloc_q_vectors()
12052 	else if (vsi->type == I40E_VSI_MAIN)  in i40e_vsi_alloc_q_vectors()
12055 		return -EINVAL;  in i40e_vsi_alloc_q_vectors()
12066 	while (v_idx--)  in i40e_vsi_alloc_q_vectors()
12073  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
12081 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_init_interrupt_scheme()
12084 			clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);  in i40e_init_interrupt_scheme()
12085 			clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);  in i40e_init_interrupt_scheme()
12086 			clear_bit(I40E_FLAG_RSS_ENA, pf->flags);  in i40e_init_interrupt_scheme()
12087 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_init_interrupt_scheme()
12088 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_init_interrupt_scheme()
12089 			clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);  in i40e_init_interrupt_scheme()
12090 			clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_init_interrupt_scheme()
12091 			clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);  in i40e_init_interrupt_scheme()
12092 			clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);  in i40e_init_interrupt_scheme()
12093 			set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_init_interrupt_scheme()
12100 	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&  in i40e_init_interrupt_scheme()
12101 	    test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {  in i40e_init_interrupt_scheme()
12102 		dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");  in i40e_init_interrupt_scheme()
12103 		vectors = pci_enable_msi(pf->pdev);  in i40e_init_interrupt_scheme()
12105 			dev_info(&pf->pdev->dev, "MSI init failed - %d\n",  in i40e_init_interrupt_scheme()
12107 			clear_bit(I40E_FLAG_MSI_ENA, pf->flags);  in i40e_init_interrupt_scheme()
12112 	if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) &&  in i40e_init_interrupt_scheme()
12113 	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_init_interrupt_scheme()
12114 		dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");  in i40e_init_interrupt_scheme()
12118 	pf->irq_pile = kzalloc(size, GFP_KERNEL);  in i40e_init_interrupt_scheme()
12119 	if (!pf->irq_pile)  in i40e_init_interrupt_scheme()
12120 		return -ENOMEM;  in i40e_init_interrupt_scheme()
12122 	pf->irq_pile->num_entries = vectors;  in i40e_init_interrupt_scheme()
12125 	(void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);  in i40e_init_interrupt_scheme()
12131  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
12135  * device. This should be called during resume to re-allocate the q_vectors
12143 	/* We cleared the MSI and MSI-X flags when disabling the old interrupt  in i40e_restore_interrupt_scheme()
12144 	 * scheme. We need to re-enabled them here in order to attempt to  in i40e_restore_interrupt_scheme()
12145 	 * re-acquire the MSI or MSI-X vectors  in i40e_restore_interrupt_scheme()
12147 	set_bit(I40E_FLAG_MSI_ENA, pf->flags);  in i40e_restore_interrupt_scheme()
12148 	set_bit(I40E_FLAG_MSIX_ENA, pf->flags);  in i40e_restore_interrupt_scheme()
12154 	/* Now that we've re-acquired IRQs, we need to remap the vectors and  in i40e_restore_interrupt_scheme()
12169 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))  in i40e_restore_interrupt_scheme()
12175 	while (i--) {  in i40e_restore_interrupt_scheme()
12176 		if (pf->vsi[i])  in i40e_restore_interrupt_scheme()
12177 			i40e_vsi_free_q_vectors(pf->vsi[i]);  in i40e_restore_interrupt_scheme()
12184  * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
12189  * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
12197 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_setup_misc_vector_for_recovery_mode()
12201 			dev_info(&pf->pdev->dev,  in i40e_setup_misc_vector_for_recovery_mode()
12202 				 "MSI-X misc vector request failed, error %d\n",  in i40e_setup_misc_vector_for_recovery_mode()
12207 		u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED;  in i40e_setup_misc_vector_for_recovery_mode()
12209 		err = request_irq(pf->pdev->irq, i40e_intr, flags,  in i40e_setup_misc_vector_for_recovery_mode()
12210 				  pf->int_name, pf);  in i40e_setup_misc_vector_for_recovery_mode()
12213 			dev_info(&pf->pdev->dev,  in i40e_setup_misc_vector_for_recovery_mode()
12226  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
12230  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
12235 	struct i40e_hw *hw = &pf->hw;  in i40e_setup_misc_vector()
12239 	if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {  in i40e_setup_misc_vector()
12240 		err = request_irq(pf->msix_entries[0].vector,  in i40e_setup_misc_vector()
12241 				  i40e_intr, 0, pf->int_name, pf);  in i40e_setup_misc_vector()
12243 			clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);  in i40e_setup_misc_vector()
12244 			dev_info(&pf->pdev->dev,  in i40e_setup_misc_vector()
12246 				 pf->int_name, err);  in i40e_setup_misc_vector()
12247 			return -EFAULT;  in i40e_setup_misc_vector()
12265  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
12276 	struct i40e_pf *pf = vsi->back;  in i40e_get_rss_aq()
12277 	struct i40e_hw *hw = &pf->hw;  in i40e_get_rss_aq()
12281 		ret = i40e_aq_get_rss_key(hw, vsi->id,  in i40e_get_rss_aq()
12284 			dev_info(&pf->pdev->dev,  in i40e_get_rss_aq()
12287 				 i40e_aq_str(&pf->hw,  in i40e_get_rss_aq()
12288 					     pf->hw.aq.asq_last_status));  in i40e_get_rss_aq()
12294 		bool pf_lut = vsi->type == I40E_VSI_MAIN;  in i40e_get_rss_aq()
12296 		ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);  in i40e_get_rss_aq()
12298 			dev_info(&pf->pdev->dev,  in i40e_get_rss_aq()
12301 				 i40e_aq_str(&pf->hw,  in i40e_get_rss_aq()
12302 					     pf->hw.aq.asq_last_status));  in i40e_get_rss_aq()
12311  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
12322 	struct i40e_pf *pf = vsi->back;  in i40e_config_rss_reg()
12323 	struct i40e_hw *hw = &pf->hw;  in i40e_config_rss_reg()
12324 	u16 vf_id = vsi->vf_id;  in i40e_config_rss_reg()
12331 		if (vsi->type == I40E_VSI_MAIN) {  in i40e_config_rss_reg()
12334 		} else if (vsi->type == I40E_VSI_SRIOV) {  in i40e_config_rss_reg()
12338 			dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");  in i40e_config_rss_reg()
12345 		if (vsi->type == I40E_VSI_MAIN) {  in i40e_config_rss_reg()
12347 				return -EINVAL;  in i40e_config_rss_reg()
12350 		} else if (vsi->type == I40E_VSI_SRIOV) {  in i40e_config_rss_reg()
12352 				return -EINVAL;  in i40e_config_rss_reg()
12356 			dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");  in i40e_config_rss_reg()
12365  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
12376 	struct i40e_pf *pf = vsi->back;  in i40e_get_rss_reg()
12377 	struct i40e_hw *hw = &pf->hw;  in i40e_get_rss_reg()
12390 			return -EINVAL;  in i40e_get_rss_reg()
12399  * i40e_config_rss - Configure RSS keys and lut
12409 	struct i40e_pf *pf = vsi->back;  in i40e_config_rss()
12411 	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))  in i40e_config_rss()
12418  * i40e_get_rss - Get RSS keys and lut
12428 	struct i40e_pf *pf = vsi->back;  in i40e_get_rss()
12430 	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))  in i40e_get_rss()
12437  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12453  * i40e_pf_config_rss - Prepare for RSS if used
12461 	struct i40e_hw *hw = &pf->hw;  in i40e_pf_config_rss()
12476 	reg_val = (pf->rss_table_size == 512) ?  in i40e_pf_config_rss()
12482 	if (!vsi->rss_size) {  in i40e_pf_config_rss()
12485 		 * could end up with zero TCs. Check for that to avoid  in i40e_pf_config_rss()
12486 		 * divide-by-zero. It probably won't pass traffic, but it also  in i40e_pf_config_rss()
12489 		qcount = vsi->num_queue_pairs /  in i40e_pf_config_rss()
12490 			 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);  in i40e_pf_config_rss()
12491 		vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);  in i40e_pf_config_rss()
12493 	if (!vsi->rss_size)  in i40e_pf_config_rss()
12494 		return -EINVAL;  in i40e_pf_config_rss()
12496 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);  in i40e_pf_config_rss()
12498 		return -ENOMEM;  in i40e_pf_config_rss()
12501 	if (vsi->rss_lut_user)  in i40e_pf_config_rss()
12502 		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);  in i40e_pf_config_rss()
12504 		i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);  in i40e_pf_config_rss()
12509 	if (vsi->rss_hkey_user)  in i40e_pf_config_rss()
12510 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);  in i40e_pf_config_rss()
12513 	ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);  in i40e_pf_config_rss()
12520  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12533 	if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags))  in i40e_reconfig_rss_queues()
12537 	new_rss_size = min_t(int, queue_count, pf->rss_size_max);  in i40e_reconfig_rss_queues()
12539 	if (queue_count != vsi->num_queue_pairs) {  in i40e_reconfig_rss_queues()
12542 		vsi->req_queue_pairs = queue_count;  in i40e_reconfig_rss_queues()
12544 		if (test_bit(__I40E_IN_REMOVE, pf->state))  in i40e_reconfig_rss_queues()
12545 			return pf->alloc_rss_size;  in i40e_reconfig_rss_queues()
12547 		pf->alloc_rss_size = new_rss_size;  in i40e_reconfig_rss_queues()
12554 		if (queue_count < vsi->rss_size) {  in i40e_reconfig_rss_queues()
12556 			dev_dbg(&pf->pdev->dev,  in i40e_reconfig_rss_queues()
12560 		/* Reset vsi->rss_size, as number of enabled queues changed */  in i40e_reconfig_rss_queues()
12561 		qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;  in i40e_reconfig_rss_queues()
12562 		vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);  in i40e_reconfig_rss_queues()
12566 	dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",  in i40e_reconfig_rss_queues()
12567 		 vsi->req_queue_pairs, pf->rss_size_max);  in i40e_reconfig_rss_queues()
12568 	return pf->alloc_rss_size;  in i40e_reconfig_rss_queues()
12572  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12581 	status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,  in i40e_get_partition_bw_setting()
12586 			pf->min_bw = min_bw;  in i40e_get_partition_bw_setting()
12588 			pf->max_bw = max_bw;  in i40e_get_partition_bw_setting()
12595  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12606 	bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));  in i40e_set_partition_bw_setting()
12607 	bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;  in i40e_set_partition_bw_setting()
12608 	bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;  in i40e_set_partition_bw_setting()
12611 	status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);  in i40e_set_partition_bw_setting()
12617  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12627 	if (pf->hw.partition_id != 1) {  in i40e_commit_partition_bw_setting()
12628 		dev_info(&pf->pdev->dev,  in i40e_commit_partition_bw_setting()
12630 			 pf->hw.partition_id);  in i40e_commit_partition_bw_setting()
12631 		ret = -EOPNOTSUPP;  in i40e_commit_partition_bw_setting()
12636 	ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);  in i40e_commit_partition_bw_setting()
12637 	last_aq_status = pf->hw.aq.asq_last_status;  in i40e_commit_partition_bw_setting()
12639 		dev_info(&pf->pdev->dev,  in i40e_commit_partition_bw_setting()
12642 			 i40e_aq_str(&pf->hw, last_aq_status));  in i40e_commit_partition_bw_setting()
12646 	/* Read word 0x10 of NVM - SW compatibility word 1 */  in i40e_commit_partition_bw_setting()
12647 	ret = i40e_aq_read_nvm(&pf->hw,  in i40e_commit_partition_bw_setting()
12654 	last_aq_status = pf->hw.aq.asq_last_status;  in i40e_commit_partition_bw_setting()
12655 	i40e_release_nvm(&pf->hw);  in i40e_commit_partition_bw_setting()
12657 		dev_info(&pf->pdev->dev, "NVM read error, err %pe aq_err %s\n",  in i40e_commit_partition_bw_setting()
12659 			 i40e_aq_str(&pf->hw, last_aq_status));  in i40e_commit_partition_bw_setting()
12667 	ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);  in i40e_commit_partition_bw_setting()
12668 	last_aq_status = pf->hw.aq.asq_last_status;  in i40e_commit_partition_bw_setting()
12670 		dev_info(&pf->pdev->dev,  in i40e_commit_partition_bw_setting()
12673 			 i40e_aq_str(&pf->hw, last_aq_status));  in i40e_commit_partition_bw_setting()
12678 	 * the NVM - thus storing the bandwidth values permanently.  in i40e_commit_partition_bw_setting()
12680 	ret = i40e_aq_update_nvm(&pf->hw,  in i40e_commit_partition_bw_setting()
12687 	last_aq_status = pf->hw.aq.asq_last_status;  in i40e_commit_partition_bw_setting()
12688 	i40e_release_nvm(&pf->hw);  in i40e_commit_partition_bw_setting()
12690 		dev_info(&pf->pdev->dev,  in i40e_commit_partition_bw_setting()
12693 			 i40e_aq_str(&pf->hw, last_aq_status));  in i40e_commit_partition_bw_setting()
12700  * i40e_is_total_port_shutdown_enabled - read NVM and return value
12719 	read_status = i40e_read_nvm_word(&pf->hw,  in i40e_is_total_port_shutdown_enabled()
12724 	read_status = i40e_read_nvm_word(&pf->hw,  in i40e_is_total_port_shutdown_enabled()
12731 		read_status = i40e_read_nvm_module_data(&pf->hw,  in i40e_is_total_port_shutdown_enabled()
12739 		link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);  in i40e_is_total_port_shutdown_enabled()
12745 	dev_warn(&pf->pdev->dev,  in i40e_is_total_port_shutdown_enabled()
12746 		 "total-port-shutdown feature is off due to read nvm error: %pe\n",  in i40e_is_total_port_shutdown_enabled()
12752  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12766 	bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS);  in i40e_sw_init()
12767 	set_bit(I40E_FLAG_MSI_ENA, pf->flags);  in i40e_sw_init()
12768 	set_bit(I40E_FLAG_MSIX_ENA, pf->flags);  in i40e_sw_init()
12771 	pf->rx_itr_default = I40E_ITR_RX_DEF;  in i40e_sw_init()
12772 	pf->tx_itr_default = I40E_ITR_TX_DEF;  in i40e_sw_init()
12777 	pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);  in i40e_sw_init()
12778 	pf->alloc_rss_size = 1;  in i40e_sw_init()
12779 	pf->rss_table_size = pf->hw.func_caps.rss_table_size;  in i40e_sw_init()
12780 	pf->rss_size_max = min_t(int, pf->rss_size_max,  in i40e_sw_init()
12781 				 pf->hw.func_caps.num_tx_qp);  in i40e_sw_init()
12783 	/* find the next higher power-of-2 of num cpus */  in i40e_sw_init()
12785 	pf->rss_size_max = min_t(int, pf->rss_size_max, pow);  in i40e_sw_init()
12787 	if (pf->hw.func_caps.rss) {  in i40e_sw_init()
12788 		set_bit(I40E_FLAG_RSS_ENA, pf->flags);  in i40e_sw_init()
12789 		pf->alloc_rss_size = min_t(int, pf->rss_size_max,  in i40e_sw_init()
12794 	if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {  in i40e_sw_init()
12795 		set_bit(I40E_FLAG_MFP_ENA, pf->flags);  in i40e_sw_init()
12796 		dev_info(&pf->pdev->dev, "MFP mode Enabled\n");  in i40e_sw_init()
12798 			dev_warn(&pf->pdev->dev,  in i40e_sw_init()
12801 			dev_info(&pf->pdev->dev,  in i40e_sw_init()
12803 				 pf->min_bw, pf->max_bw);  in i40e_sw_init()
12810 	if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||  in i40e_sw_init()
12811 	    (pf->hw.func_caps.fd_filters_best_effort > 0)) {  in i40e_sw_init()
12812 		set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);  in i40e_sw_init()
12813 		if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&  in i40e_sw_init()
12814 		    pf->hw.num_partitions > 1)  in i40e_sw_init()
12815 			dev_info(&pf->pdev->dev,  in i40e_sw_init()
12818 			set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_sw_init()
12819 		pf->fdir_pf_filter_count =  in i40e_sw_init()
12820 				 pf->hw.func_caps.fd_filters_guaranteed;  in i40e_sw_init()
12821 		pf->hw.fdir_shared_filter_count =  in i40e_sw_init()
12822 				 pf->hw.func_caps.fd_filters_best_effort;  in i40e_sw_init()
12826 	if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps))  in i40e_sw_init()
12827 		set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags);  in i40e_sw_init()
12829 	if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {  in i40e_sw_init()
12830 		pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;  in i40e_sw_init()
12831 		set_bit(I40E_FLAG_VMDQ_ENA, pf->flags);  in i40e_sw_init()
12832 		pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);  in i40e_sw_init()
12835 	if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {  in i40e_sw_init()
12836 		set_bit(I40E_FLAG_IWARP_ENA, pf->flags);  in i40e_sw_init()
12838 		pf->num_iwarp_msix = (int)num_online_cpus() + 1;  in i40e_sw_init()
12845 	if (pf->hw.mac.type == I40E_MAC_XL710 &&  in i40e_sw_init()
12846 	    pf->hw.func_caps.npar_enable)  in i40e_sw_init()
12847 		clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps);  in i40e_sw_init()
12850 	if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {  in i40e_sw_init()
12851 		pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;  in i40e_sw_init()
12852 		set_bit(I40E_FLAG_SRIOV_ENA, pf->flags);  in i40e_sw_init()
12853 		pf->num_req_vfs = min_t(int,  in i40e_sw_init()
12854 					pf->hw.func_caps.num_vfs,  in i40e_sw_init()
12858 	pf->lan_veb = I40E_NO_VEB;  in i40e_sw_init()
12859 	pf->lan_vsi = I40E_NO_VSI;  in i40e_sw_init()
12862 	clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);  in i40e_sw_init()
12866 		+ (sizeof(u16) * pf->hw.func_caps.num_tx_qp);  in i40e_sw_init()
12867 	pf->qp_pile = kzalloc(size, GFP_KERNEL);  in i40e_sw_init()
12868 	if (!pf->qp_pile) {  in i40e_sw_init()
12869 		err = -ENOMEM;  in i40e_sw_init()
12872 	pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;  in i40e_sw_init()
12874 	pf->tx_timeout_recovery_level = 1;  in i40e_sw_init()
12876 	if (pf->hw.mac.type != I40E_MAC_X722 &&  in i40e_sw_init()
12881 		set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);  in i40e_sw_init()
12882 		set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);  in i40e_sw_init()
12883 		dev_info(&pf->pdev->dev,  in i40e_sw_init()
12884 			 "total-port-shutdown was enabled, link-down-on-close is forced on\n");  in i40e_sw_init()
12886 	mutex_init(&pf->switch_mutex);  in i40e_sw_init()
12893  * i40e_set_ntuple - set the ntuple feature flag and take action
12903 	/* Check if Flow Director n-tuple support was enabled or disabled.  If  in i40e_set_ntuple()
12908 		if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))  in i40e_set_ntuple()
12910 		/* enable FD_SB only if there is MSI-X vector and no cloud  in i40e_set_ntuple()
12913 		if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {  in i40e_set_ntuple()
12914 			set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_set_ntuple()
12915 			clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_set_ntuple()
12919 		if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {  in i40e_set_ntuple()
12923 		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_set_ntuple()
12924 		clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);  in i40e_set_ntuple()
12925 		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_set_ntuple()
12928 		pf->fd_add_err = 0;  in i40e_set_ntuple()
12929 		pf->fd_atr_cnt = 0;  in i40e_set_ntuple()
12930 		/* if ATR was auto disabled it can be re-enabled. */  in i40e_set_ntuple()
12931 		if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))  in i40e_set_ntuple()
12932 			if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&  in i40e_set_ntuple()
12933 			    (I40E_DEBUG_FD & pf->hw.debug_mask))  in i40e_set_ntuple()
12934 				dev_info(&pf->pdev->dev, "ATR re-enabled.\n");  in i40e_set_ntuple()
12940  * i40e_clear_rss_lut - clear the rx hash lookup table
12945 	struct i40e_pf *pf = vsi->back;  in i40e_clear_rss_lut()
12946 	struct i40e_hw *hw = &pf->hw;  in i40e_clear_rss_lut()
12947 	u16 vf_id = vsi->vf_id;  in i40e_clear_rss_lut()
12950 	if (vsi->type == I40E_VSI_MAIN) {  in i40e_clear_rss_lut()
12953 	} else if (vsi->type == I40E_VSI_SRIOV) {  in i40e_clear_rss_lut()
12957 		dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");  in i40e_clear_rss_lut()
12962  * i40e_set_loopback - turn on/off loopback mode on underlying PF
12968 	bool if_running = netif_running(vsi->netdev) &&  in i40e_set_loopback()
12969 			  !test_and_set_bit(__I40E_VSI_DOWN, vsi->state);  in i40e_set_loopback()
12975 	ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);  in i40e_set_loopback()
12977 		netdev_err(vsi->netdev, "Failed to toggle loopback state\n");  in i40e_set_loopback()
12985  * i40e_set_features - set the netdev feature flags
12994 	struct i40e_vsi *vsi = np->vsi;  in i40e_set_features()
12995 	struct i40e_pf *pf = vsi->back;  in i40e_set_features()
12998 	if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))  in i40e_set_features()
13001 		 netdev->features & NETIF_F_RXHASH)  in i40e_set_features()
13010 	    (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) {  in i40e_set_features()
13011 		dev_err(&pf->pdev->dev,  in i40e_set_features()
13013 		return -EINVAL;  in i40e_set_features()
13016 	if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)  in i40e_set_features()
13024 	if ((features ^ netdev->features) & NETIF_F_LOOPBACK)  in i40e_set_features()
13035 	struct i40e_hw *hw = &np->vsi->back->hw;  in i40e_udp_tunnel_set_port()
13039 	type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :  in i40e_udp_tunnel_set_port()
13042 	ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,  in i40e_udp_tunnel_set_port()
13047 			    i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_udp_tunnel_set_port()
13048 		return -EIO;  in i40e_udp_tunnel_set_port()
13060 	struct i40e_hw *hw = &np->vsi->back->hw;  in i40e_udp_tunnel_unset_port()
13063 	ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);  in i40e_udp_tunnel_unset_port()
13067 			    i40e_aq_str(hw, hw->aq.asq_last_status));  in i40e_udp_tunnel_unset_port()
13068 		return -EIO;  in i40e_udp_tunnel_unset_port()
13078 	struct i40e_pf *pf = np->vsi->back;  in i40e_get_phys_port_id()
13079 	struct i40e_hw *hw = &pf->hw;  in i40e_get_phys_port_id()
13081 	if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps))  in i40e_get_phys_port_id()
13082 		return -EOPNOTSUPP;  in i40e_get_phys_port_id()
13084 	ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));  in i40e_get_phys_port_id()
13085 	memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);  in i40e_get_phys_port_id()
13091  * i40e_ndo_fdb_add - add an entry to the hardware database
13107 	struct i40e_pf *pf = np->vsi->back;  in i40e_ndo_fdb_add()
13110 	if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags))  in i40e_ndo_fdb_add()
13111 		return -EOPNOTSUPP;  in i40e_ndo_fdb_add()
13114 		pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);  in i40e_ndo_fdb_add()
13115 		return -EINVAL;  in i40e_ndo_fdb_add()
13121 	if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {  in i40e_ndo_fdb_add()
13123 		return -EINVAL;  in i40e_ndo_fdb_add()
13131 		err = -EINVAL;  in i40e_ndo_fdb_add()
13134 	if (err == -EEXIST && !(flags & NLM_F_EXCL))  in i40e_ndo_fdb_add()
13141  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
13162 	struct i40e_vsi *vsi = np->vsi;  in i40e_ndo_bridge_setlink()
13163 	struct i40e_pf *pf = vsi->back;  in i40e_ndo_bridge_setlink()
13169 	if (vsi->type != I40E_VSI_MAIN)  in i40e_ndo_bridge_setlink()
13170 		return -EOPNOTSUPP;  in i40e_ndo_bridge_setlink()
13173 	veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);  in i40e_ndo_bridge_setlink()
13177 		return -EINVAL;  in i40e_ndo_bridge_setlink()
13184 			return -EINVAL;  in i40e_ndo_bridge_setlink()
13188 			veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,  in i40e_ndo_bridge_setlink()
13189 					     vsi->tc_config.enabled_tc);  in i40e_ndo_bridge_setlink()
13191 				veb->bridge_mode = mode;  in i40e_ndo_bridge_setlink()
13195 				return -ENOENT;  in i40e_ndo_bridge_setlink()
13198 		} else if (mode != veb->bridge_mode) {  in i40e_ndo_bridge_setlink()
13200 			veb->bridge_mode = mode;  in i40e_ndo_bridge_setlink()
13203 				set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);  in i40e_ndo_bridge_setlink()
13205 				clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);  in i40e_ndo_bridge_setlink()
13215  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
13232 	struct i40e_vsi *vsi = np->vsi;  in i40e_ndo_bridge_getlink()
13233 	struct i40e_pf *pf = vsi->back;  in i40e_ndo_bridge_getlink()
13237 	if (vsi->type != I40E_VSI_MAIN)  in i40e_ndo_bridge_getlink()
13238 		return -EOPNOTSUPP;  in i40e_ndo_bridge_getlink()
13241 	veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);  in i40e_ndo_bridge_getlink()
13245 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,  in i40e_ndo_bridge_getlink()
13250  * i40e_features_check - Validate encapsulated packet conforms to limits
13265 	if (skb->ip_summed != CHECKSUM_PARTIAL)  in i40e_features_check()
13271 	if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))  in i40e_features_check()
13284 	if (skb->encapsulation) {  in i40e_features_check()
13286 		len = skb_inner_network_header(skb) - skb_transport_header(skb);  in i40e_features_check()
13291 		len = skb_inner_transport_header(skb) -  in i40e_features_check()
13308  * i40e_xdp_setup - add/remove an XDP program
13317 	struct i40e_pf *pf = vsi->back;  in i40e_xdp_setup()
13323 	if (prog && test_bit(__I40E_IN_REMOVE, pf->state))  in i40e_xdp_setup()
13324 		return -EINVAL;  in i40e_xdp_setup()
13327 	if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) {  in i40e_xdp_setup()
13329 		return -EINVAL;  in i40e_xdp_setup()
13332 	/* When turning XDP on->off/off->on we reset and rebuild the rings. */  in i40e_xdp_setup()
13337 	old_prog = xchg(&vsi->xdp_prog, prog);  in i40e_xdp_setup()
13341 			xdp_features_clear_redirect_target(vsi->netdev);  in i40e_xdp_setup()
13350 			return -ENOMEM;  in i40e_xdp_setup()
13353 			return -ENOMEM;  in i40e_xdp_setup()
13356 	for (i = 0; i < vsi->num_queue_pairs; i++)  in i40e_xdp_setup()
13357 		WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);  in i40e_xdp_setup()
13366 		for (i = 0; i < vsi->num_queue_pairs; i++)  in i40e_xdp_setup()
13367 			if (vsi->xdp_rings[i]->xsk_pool)  in i40e_xdp_setup()
13368 				(void)i40e_xsk_wakeup(vsi->netdev, i,  in i40e_xdp_setup()
13370 		xdp_features_set_redirect_target(vsi->netdev, true);  in i40e_xdp_setup()
13377  * i40e_enter_busy_conf - Enters busy config state
13384 	struct i40e_pf *pf = vsi->back;  in i40e_enter_busy_conf()
13387 	while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {  in i40e_enter_busy_conf()
13388 		timeout--;  in i40e_enter_busy_conf()
13390 			return -EBUSY;  in i40e_enter_busy_conf()
13398  * i40e_exit_busy_conf - Exits busy config state
13403 	struct i40e_pf *pf = vsi->back;  in i40e_exit_busy_conf()
13405 	clear_bit(__I40E_CONFIG_BUSY, pf->state);  in i40e_exit_busy_conf()
13409  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13415 	memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,  in i40e_queue_pair_reset_stats()
13416 	       sizeof(vsi->rx_rings[queue_pair]->rx_stats));  in i40e_queue_pair_reset_stats()
13417 	memset(&vsi->tx_rings[queue_pair]->stats, 0,  in i40e_queue_pair_reset_stats()
13418 	       sizeof(vsi->tx_rings[queue_pair]->stats));  in i40e_queue_pair_reset_stats()
13420 		memset(&vsi->xdp_rings[queue_pair]->stats, 0,  in i40e_queue_pair_reset_stats()
13421 		       sizeof(vsi->xdp_rings[queue_pair]->stats));  in i40e_queue_pair_reset_stats()
13426  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13432 	i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);  in i40e_queue_pair_clean_rings()
13434 		/* Make sure that in-progress ndo_xdp_xmit calls are  in i40e_queue_pair_clean_rings()
13438 		i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);  in i40e_queue_pair_clean_rings()
13440 	i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);  in i40e_queue_pair_clean_rings()
13444  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13452 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];  in i40e_queue_pair_toggle_napi()
13453 	struct i40e_q_vector *q_vector = rxr->q_vector;  in i40e_queue_pair_toggle_napi()
13455 	if (!vsi->netdev)  in i40e_queue_pair_toggle_napi()
13459 	if (q_vector->rx.ring || q_vector->tx.ring) {  in i40e_queue_pair_toggle_napi()
13461 			napi_enable(&q_vector->napi);  in i40e_queue_pair_toggle_napi()
13463 			napi_disable(&q_vector->napi);  in i40e_queue_pair_toggle_napi()
13468  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13478 	struct i40e_pf *pf = vsi->back;  in i40e_queue_pair_toggle_rings()
13481 	pf_q = vsi->base_queue + queue_pair;  in i40e_queue_pair_toggle_rings()
13482 	ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,  in i40e_queue_pair_toggle_rings()
13485 		dev_info(&pf->pdev->dev,  in i40e_queue_pair_toggle_rings()
13487 			 vsi->seid, pf_q, (enable ? "en" : "dis"));  in i40e_queue_pair_toggle_rings()
13494 		dev_info(&pf->pdev->dev,  in i40e_queue_pair_toggle_rings()
13496 			 vsi->seid, pf_q, (enable ? "en" : "dis"));  in i40e_queue_pair_toggle_rings()
13509 	ret = i40e_control_wait_tx_q(vsi->seid, pf,  in i40e_queue_pair_toggle_rings()
13510 				     pf_q + vsi->alloc_queue_pairs,  in i40e_queue_pair_toggle_rings()
13513 		dev_info(&pf->pdev->dev,  in i40e_queue_pair_toggle_rings()
13515 			 vsi->seid, pf_q, (enable ? "en" : "dis"));  in i40e_queue_pair_toggle_rings()
13522  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13528 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];  in i40e_queue_pair_enable_irq()
13529 	struct i40e_pf *pf = vsi->back;  in i40e_queue_pair_enable_irq()
13530 	struct i40e_hw *hw = &pf->hw;  in i40e_queue_pair_enable_irq()
13533 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_queue_pair_enable_irq()
13534 		i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);  in i40e_queue_pair_enable_irq()
13542  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13548 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];  in i40e_queue_pair_disable_irq()
13549 	struct i40e_pf *pf = vsi->back;  in i40e_queue_pair_disable_irq()
13550 	struct i40e_hw *hw = &pf->hw;  in i40e_queue_pair_disable_irq()
13558 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_queue_pair_disable_irq()
13559 		u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;  in i40e_queue_pair_disable_irq()
13561 		wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);  in i40e_queue_pair_disable_irq()
13563 		synchronize_irq(pf->msix_entries[intpf].vector);  in i40e_queue_pair_disable_irq()
13565 		/* Legacy and MSI mode - this stops all interrupt handling */  in i40e_queue_pair_disable_irq()
13569 		synchronize_irq(pf->pdev->irq);  in i40e_queue_pair_disable_irq()
13574  * i40e_queue_pair_disable - Disables a queue pair
13591 	i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);  in i40e_queue_pair_disable()
13599  * i40e_queue_pair_enable - Enables a queue pair
13609 	err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);  in i40e_queue_pair_enable()
13614 		err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);  in i40e_queue_pair_enable()
13619 	err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);  in i40e_queue_pair_enable()
13633  * i40e_xdp - implements ndo_bpf for i40e
13641 	struct i40e_vsi *vsi = np->vsi;  in i40e_xdp()
13643 	if (vsi->type != I40E_VSI_MAIN)  in i40e_xdp()
13644 		return -EINVAL;  in i40e_xdp()
13646 	switch (xdp->command) {  in i40e_xdp()
13648 		return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);  in i40e_xdp()
13650 		return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,  in i40e_xdp()
13651 					   xdp->xsk.queue_id);  in i40e_xdp()
13653 		return -EINVAL;  in i40e_xdp()
13697  * i40e_config_netdev - Setup the netdev flags
13704 	struct i40e_pf *pf = vsi->back;  in i40e_config_netdev()
13705 	struct i40e_hw *hw = &pf->hw;  in i40e_config_netdev()
13715 	netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);  in i40e_config_netdev()
13717 		return -ENOMEM;  in i40e_config_netdev()
13719 	vsi->netdev = netdev;  in i40e_config_netdev()
13721 	np->vsi = vsi;  in i40e_config_netdev()
13743 	if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps))  in i40e_config_netdev()
13744 		netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;  in i40e_config_netdev()
13746 	netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;  in i40e_config_netdev()
13748 	netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;  in i40e_config_netdev()
13750 	netdev->hw_enc_features |= hw_enc_features;  in i40e_config_netdev()
13753 	netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;  in i40e_config_netdev()
13762 	netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES;  in i40e_config_netdev()
13763 	netdev->features |= NETIF_F_GSO_PARTIAL |  in i40e_config_netdev()
13766 	netdev->mpls_features |= NETIF_F_SG;  in i40e_config_netdev()
13767 	netdev->mpls_features |= NETIF_F_HW_CSUM;  in i40e_config_netdev()
13768 	netdev->mpls_features |= NETIF_F_TSO;  in i40e_config_netdev()
13769 	netdev->mpls_features |= NETIF_F_TSO6;  in i40e_config_netdev()
13770 	netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES;  in i40e_config_netdev()
13773 	netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;  in i40e_config_netdev()
13779 	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))  in i40e_config_netdev()
13782 	netdev->hw_features |= hw_features | NETIF_F_LOOPBACK;  in i40e_config_netdev()
13784 	netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;  in i40e_config_netdev()
13785 	netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;  in i40e_config_netdev()
13787 	netdev->features &= ~NETIF_F_HW_TC;  in i40e_config_netdev()
13789 	if (vsi->type == I40E_VSI_MAIN) {  in i40e_config_netdev()
13790 		SET_NETDEV_DEV(netdev, &pf->pdev->dev);  in i40e_config_netdev()
13791 		ether_addr_copy(mac_addr, hw->mac.perm_addr);  in i40e_config_netdev()
13793 		 * some older NVM configurations load a default MAC-VLAN  in i40e_config_netdev()
13803 		spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_config_netdev()
13805 		spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_config_netdev()
13807 		netdev->xdp_features = NETDEV_XDP_ACT_BASIC |  in i40e_config_netdev()
13811 		netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD;  in i40e_config_netdev()
13816 		 * original name by IFNAMSIZ - 4  in i40e_config_netdev()
13820 		snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", IFNAMSIZ - 4,  in i40e_config_netdev()
13821 			 main_vsi->netdev->name);  in i40e_config_netdev()
13824 		spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_config_netdev()
13826 		spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_config_netdev()
13843 	spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_config_netdev()
13845 	spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_config_netdev()
13848 	ether_addr_copy(netdev->perm_addr, mac_addr);  in i40e_config_netdev()
13850 	/* i40iw_net_event() reads 16 bytes from neigh->primary_key */  in i40e_config_netdev()
13851 	netdev->neigh_priv_len = sizeof(u32) * 4;  in i40e_config_netdev()
13853 	netdev->priv_flags |= IFF_UNICAST_FLT;  in i40e_config_netdev()
13854 	netdev->priv_flags |= IFF_SUPP_NOFCS;  in i40e_config_netdev()
13856 	i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);  in i40e_config_netdev()
13858 	netdev->netdev_ops = &i40e_netdev_ops;  in i40e_config_netdev()
13859 	netdev->watchdog_timeo = 5 * HZ;  in i40e_config_netdev()
13862 	/* MTU range: 68 - 9706 */  in i40e_config_netdev()
13863 	netdev->min_mtu = ETH_MIN_MTU;  in i40e_config_netdev()
13864 	netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;  in i40e_config_netdev()
13870  * i40e_vsi_delete - Delete a VSI from the switch
13878 	if (vsi == vsi->back->vsi[vsi->back->lan_vsi])  in i40e_vsi_delete()
13881 	i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);  in i40e_vsi_delete()
13885  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13893 	struct i40e_pf *pf = vsi->back;  in i40e_is_vsi_uplink_mode_veb()
13896 	if (vsi->veb_idx >= I40E_MAX_VEB)  in i40e_is_vsi_uplink_mode_veb()
13899 	veb = pf->veb[vsi->veb_idx];  in i40e_is_vsi_uplink_mode_veb()
13901 		dev_info(&pf->pdev->dev,  in i40e_is_vsi_uplink_mode_veb()
13903 		return -ENOENT;  in i40e_is_vsi_uplink_mode_veb()
13907 	if (veb->bridge_mode & BRIDGE_MODE_VEPA) {  in i40e_is_vsi_uplink_mode_veb()
13919  * i40e_add_vsi - Add a VSI to the switch
13927 	int ret = -ENODEV;  in i40e_add_vsi()
13928 	struct i40e_pf *pf = vsi->back;  in i40e_add_vsi()
13929 	struct i40e_hw *hw = &pf->hw;  in i40e_add_vsi()
13939 	switch (vsi->type) {  in i40e_add_vsi()
13946 		ctxt.seid = pf->main_vsi_seid;  in i40e_add_vsi()
13947 		ctxt.pf_num = pf->hw.pf_id;  in i40e_add_vsi()
13949 		ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);  in i40e_add_vsi()
13952 			dev_info(&pf->pdev->dev,  in i40e_add_vsi()
13955 				 i40e_aq_str(&pf->hw,  in i40e_add_vsi()
13956 					     pf->hw.aq.asq_last_status));  in i40e_add_vsi()
13957 			return -ENOENT;  in i40e_add_vsi()
13959 		vsi->info = ctxt.info;  in i40e_add_vsi()
13960 		vsi->info.valid_sections = 0;  in i40e_add_vsi()
13962 		vsi->seid = ctxt.seid;  in i40e_add_vsi()
13963 		vsi->id = ctxt.vsi_number;  in i40e_add_vsi()
13968 		 * negative logic - if it's set, we need to fiddle with  in i40e_add_vsi()
13971 		if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) {  in i40e_add_vsi()
13973 			ctxt.seid = pf->main_vsi_seid;  in i40e_add_vsi()
13974 			ctxt.pf_num = pf->hw.pf_id;  in i40e_add_vsi()
13982 				dev_info(&pf->pdev->dev,  in i40e_add_vsi()
13985 					 i40e_aq_str(&pf->hw,  in i40e_add_vsi()
13986 						     pf->hw.aq.asq_last_status));  in i40e_add_vsi()
13987 				ret = -ENOENT;  in i40e_add_vsi()
13993 		if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&  in i40e_add_vsi()
13994 		    !(pf->hw.func_caps.iscsi)) { /* NIC type PF */  in i40e_add_vsi()
13996 			ctxt.seid = pf->main_vsi_seid;  in i40e_add_vsi()
13997 			ctxt.pf_num = pf->hw.pf_id;  in i40e_add_vsi()
14002 				dev_info(&pf->pdev->dev,  in i40e_add_vsi()
14005 					 i40e_aq_str(&pf->hw,  in i40e_add_vsi()
14006 						    pf->hw.aq.asq_last_status));  in i40e_add_vsi()
14007 				ret = -ENOENT;  in i40e_add_vsi()
14012 			vsi->info.valid_sections = 0;  in i40e_add_vsi()
14015 			 * reconfigure it to enable all TCs that are  in i40e_add_vsi()
14025 				dev_info(&pf->pdev->dev,  in i40e_add_vsi()
14026 					 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n",  in i40e_add_vsi()
14029 					 i40e_aq_str(&pf->hw,  in i40e_add_vsi()
14030 						    pf->hw.aq.asq_last_status));  in i40e_add_vsi()
14036 		ctxt.pf_num = hw->pf_id;  in i40e_add_vsi()
14038 		ctxt.uplink_seid = vsi->uplink_seid;  in i40e_add_vsi()
14041 		if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) &&  in i40e_add_vsi()
14052 		ctxt.pf_num = hw->pf_id;  in i40e_add_vsi()
14054 		ctxt.uplink_seid = vsi->uplink_seid;  in i40e_add_vsi()
14073 		ctxt.pf_num = hw->pf_id;  in i40e_add_vsi()
14074 		ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;  in i40e_add_vsi()
14075 		ctxt.uplink_seid = vsi->uplink_seid;  in i40e_add_vsi()
14089 		if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {  in i40e_add_vsi()
14099 		if (pf->vf[vsi->vf_id].spoofchk) {  in i40e_add_vsi()
14115 		return -ENODEV;  in i40e_add_vsi()
14118 	if (vsi->type != I40E_VSI_MAIN) {  in i40e_add_vsi()
14121 			dev_info(&vsi->back->pdev->dev,  in i40e_add_vsi()
14124 				 i40e_aq_str(&pf->hw,  in i40e_add_vsi()
14125 					     pf->hw.aq.asq_last_status));  in i40e_add_vsi()
14126 			ret = -ENOENT;  in i40e_add_vsi()
14129 		vsi->info = ctxt.info;  in i40e_add_vsi()
14130 		vsi->info.valid_sections = 0;  in i40e_add_vsi()
14131 		vsi->seid = ctxt.seid;  in i40e_add_vsi()
14132 		vsi->id = ctxt.vsi_number;  in i40e_add_vsi()
14135 	spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_add_vsi()
14136 	vsi->active_filters = 0;  in i40e_add_vsi()
14138 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {  in i40e_add_vsi()
14139 		f->state = I40E_FILTER_NEW;  in i40e_add_vsi()
14142 	spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_add_vsi()
14143 	clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);  in i40e_add_vsi()
14146 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;  in i40e_add_vsi()
14147 		set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);  in i40e_add_vsi()
14153 		dev_info(&pf->pdev->dev,  in i40e_add_vsi()
14156 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_add_vsi()
14166  * i40e_vsi_release - Delete a VSI and free its resources
14180 	pf = vsi->back;  in i40e_vsi_release()
14182 	/* release of a VEB-owner or last VSI is not allowed */  in i40e_vsi_release()
14183 	if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {  in i40e_vsi_release()
14184 		dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",  in i40e_vsi_release()
14185 			 vsi->seid, vsi->uplink_seid);  in i40e_vsi_release()
14186 		return -ENODEV;  in i40e_vsi_release()
14188 	if (vsi->type == I40E_VSI_MAIN && !test_bit(__I40E_DOWN, pf->state)) {  in i40e_vsi_release()
14189 		dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");  in i40e_vsi_release()
14190 		return -ENODEV;  in i40e_vsi_release()
14192 	set_bit(__I40E_VSI_RELEASING, vsi->state);  in i40e_vsi_release()
14193 	uplink_seid = vsi->uplink_seid;  in i40e_vsi_release()
14195 	if (vsi->type != I40E_VSI_SRIOV) {  in i40e_vsi_release()
14196 		if (vsi->netdev_registered) {  in i40e_vsi_release()
14197 			vsi->netdev_registered = false;  in i40e_vsi_release()
14198 			if (vsi->netdev) {  in i40e_vsi_release()
14200 				unregister_netdev(vsi->netdev);  in i40e_vsi_release()
14208 	if (vsi->type == I40E_VSI_MAIN)  in i40e_vsi_release()
14211 	spin_lock_bh(&vsi->mac_filter_hash_lock);  in i40e_vsi_release()
14214 	if (vsi->netdev) {  in i40e_vsi_release()
14215 		__dev_uc_unsync(vsi->netdev, NULL);  in i40e_vsi_release()
14216 		__dev_mc_unsync(vsi->netdev, NULL);  in i40e_vsi_release()
14220 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)  in i40e_vsi_release()
14223 	spin_unlock_bh(&vsi->mac_filter_hash_lock);  in i40e_vsi_release()
14229 	if (vsi->netdev) {  in i40e_vsi_release()
14230 		free_netdev(vsi->netdev);  in i40e_vsi_release()
14231 		vsi->netdev = NULL;  in i40e_vsi_release()
14245 	if (veb && veb->uplink_seid) {  in i40e_vsi_release()
14248 		/* Count non-controlling VSIs present on  the VEB */  in i40e_vsi_release()
14250 			if (vsi->uplink_seid == uplink_seid &&  in i40e_vsi_release()
14251 			    (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)  in i40e_vsi_release()
14265  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
14276 	int ret = -ENOENT;  in i40e_vsi_setup_vectors()
14277 	struct i40e_pf *pf = vsi->back;  in i40e_vsi_setup_vectors()
14279 	if (vsi->q_vectors[0]) {  in i40e_vsi_setup_vectors()
14280 		dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",  in i40e_vsi_setup_vectors()
14281 			 vsi->seid);  in i40e_vsi_setup_vectors()
14282 		return -EEXIST;  in i40e_vsi_setup_vectors()
14285 	if (vsi->base_vector) {  in i40e_vsi_setup_vectors()
14286 		dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",  in i40e_vsi_setup_vectors()
14287 			 vsi->seid, vsi->base_vector);  in i40e_vsi_setup_vectors()
14288 		return -EEXIST;  in i40e_vsi_setup_vectors()
14293 		dev_info(&pf->pdev->dev,  in i40e_vsi_setup_vectors()
14295 			 vsi->num_q_vectors, vsi->seid, ret);  in i40e_vsi_setup_vectors()
14296 		vsi->num_q_vectors = 0;  in i40e_vsi_setup_vectors()
14303 	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_vsi_setup_vectors()
14305 	if (vsi->num_q_vectors)  in i40e_vsi_setup_vectors()
14306 		vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,  in i40e_vsi_setup_vectors()
14307 						 vsi->num_q_vectors, vsi->idx);  in i40e_vsi_setup_vectors()
14308 	if (vsi->base_vector < 0) {  in i40e_vsi_setup_vectors()
14309 		dev_info(&pf->pdev->dev,  in i40e_vsi_setup_vectors()
14311 			 vsi->num_q_vectors, vsi->seid, vsi->base_vector);  in i40e_vsi_setup_vectors()
14313 		ret = -ENOENT;  in i40e_vsi_setup_vectors()
14322  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
14325  * This re-allocates a vsi's queue resources.
14340 	pf = vsi->back;  in i40e_vsi_reinit_setup()
14342 	i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);  in i40e_vsi_reinit_setup()
14351 	alloc_queue_pairs = vsi->alloc_queue_pairs *  in i40e_vsi_reinit_setup()
14354 	ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);  in i40e_vsi_reinit_setup()
14356 		dev_info(&pf->pdev->dev,  in i40e_vsi_reinit_setup()
14358 			 alloc_queue_pairs, vsi->seid, ret);  in i40e_vsi_reinit_setup()
14361 	vsi->base_queue = ret;  in i40e_vsi_reinit_setup()
14367 	main_vsi->seid = pf->main_vsi_seid;  in i40e_vsi_reinit_setup()
14370 	if (vsi->type == I40E_VSI_MAIN)  in i40e_vsi_reinit_setup()
14371 		i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);  in i40e_vsi_reinit_setup()
14384 	if (vsi->netdev_registered) {  in i40e_vsi_reinit_setup()
14385 		vsi->netdev_registered = false;  in i40e_vsi_reinit_setup()
14386 		unregister_netdev(vsi->netdev);  in i40e_vsi_reinit_setup()
14387 		free_netdev(vsi->netdev);  in i40e_vsi_reinit_setup()
14388 		vsi->netdev = NULL;  in i40e_vsi_reinit_setup()
14390 	if (vsi->type == I40E_VSI_MAIN)  in i40e_vsi_reinit_setup()
14392 	i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);  in i40e_vsi_reinit_setup()
14399  * i40e_vsi_setup - Set up a VSI by a given type
14421 	 *     - the PF's port seid  in i40e_vsi_setup()
14424 	 *     - seid of an existing VEB  in i40e_vsi_setup()
14425 	 *     - seid of a VSI that owns an existing VEB  in i40e_vsi_setup()
14426 	 *     - seid of a VSI that doesn't own a VEB  in i40e_vsi_setup()
14428 	 *     - seid of the PF VSI, which is what creates the first VEB  in i40e_vsi_setup()
14434 	if (!veb && uplink_seid != pf->mac_seid) {  in i40e_vsi_setup()
14437 			dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",  in i40e_vsi_setup()
14442 		if (vsi->uplink_seid == pf->mac_seid)  in i40e_vsi_setup()
14443 			veb = i40e_veb_setup(pf, pf->mac_seid, vsi->seid,  in i40e_vsi_setup()
14444 					     vsi->tc_config.enabled_tc);  in i40e_vsi_setup()
14445 		else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)  in i40e_vsi_setup()
14446 			veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,  in i40e_vsi_setup()
14447 					     vsi->tc_config.enabled_tc);  in i40e_vsi_setup()
14449 			if (vsi->type != I40E_VSI_MAIN) {  in i40e_vsi_setup()
14450 				dev_info(&vsi->back->pdev->dev,  in i40e_vsi_setup()
14458 			if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {  in i40e_vsi_setup()
14459 				veb->bridge_mode = BRIDGE_MODE_VEPA;  in i40e_vsi_setup()
14460 				clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);  in i40e_vsi_setup()
14464 		veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);  in i40e_vsi_setup()
14466 			dev_info(&pf->pdev->dev, "couldn't add VEB\n");  in i40e_vsi_setup()
14470 		vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;  in i40e_vsi_setup()
14471 		uplink_seid = veb->seid;  in i40e_vsi_setup()
14478 	vsi = pf->vsi[v_idx];  in i40e_vsi_setup()
14481 	vsi->type = type;  in i40e_vsi_setup()
14482 	vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);  in i40e_vsi_setup()
14485 		pf->lan_vsi = v_idx;  in i40e_vsi_setup()
14487 		vsi->vf_id = param1;  in i40e_vsi_setup()
14489 	alloc_queue_pairs = vsi->alloc_queue_pairs *  in i40e_vsi_setup()
14492 	ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);  in i40e_vsi_setup()
14494 		dev_info(&pf->pdev->dev,  in i40e_vsi_setup()
14496 			 alloc_queue_pairs, vsi->seid, ret);  in i40e_vsi_setup()
14499 	vsi->base_queue = ret;  in i40e_vsi_setup()
14502 	vsi->uplink_seid = uplink_seid;  in i40e_vsi_setup()
14507 	switch (vsi->type) {  in i40e_vsi_setup()
14517 		if (vsi->type == I40E_VSI_MAIN) {  in i40e_vsi_setup()
14521 			SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);  in i40e_vsi_setup()
14523 		ret = register_netdev(vsi->netdev);  in i40e_vsi_setup()
14526 		vsi->netdev_registered = true;  in i40e_vsi_setup()
14527 		netif_carrier_off(vsi->netdev);  in i40e_vsi_setup()
14553 	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) &&  in i40e_vsi_setup()
14554 	    vsi->type == I40E_VSI_VMDQ2) {  in i40e_vsi_setup()
14566 	if (vsi->netdev_registered) {  in i40e_vsi_setup()
14567 		vsi->netdev_registered = false;  in i40e_vsi_setup()
14568 		unregister_netdev(vsi->netdev);  in i40e_vsi_setup()
14569 		free_netdev(vsi->netdev);  in i40e_vsi_setup()
14570 		vsi->netdev = NULL;  in i40e_vsi_setup()
14573 	if (vsi->type == I40E_VSI_MAIN)  in i40e_vsi_setup()
14576 	i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);  in i40e_vsi_setup()
14584  * i40e_veb_get_bw_info - Query VEB BW information
14593 	struct i40e_pf *pf = veb->pf;  in i40e_veb_get_bw_info()
14594 	struct i40e_hw *hw = &pf->hw;  in i40e_veb_get_bw_info()
14599 	ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,  in i40e_veb_get_bw_info()
14602 		dev_info(&pf->pdev->dev,  in i40e_veb_get_bw_info()
14605 			 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));  in i40e_veb_get_bw_info()
14609 	ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,  in i40e_veb_get_bw_info()
14612 		dev_info(&pf->pdev->dev,  in i40e_veb_get_bw_info()
14615 			 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));  in i40e_veb_get_bw_info()
14619 	veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);  in i40e_veb_get_bw_info()
14620 	veb->bw_max_quanta = ets_data.tc_bw_max;  in i40e_veb_get_bw_info()
14621 	veb->is_abs_credits = bw_data.absolute_credits_enable;  in i40e_veb_get_bw_info()
14622 	veb->enabled_tc = ets_data.tc_valid_bits;  in i40e_veb_get_bw_info()
14626 		veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];  in i40e_veb_get_bw_info()
14627 		veb->bw_tc_limit_credits[i] =  in i40e_veb_get_bw_info()
14629 		veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);  in i40e_veb_get_bw_info()
14637  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14645 	int ret = -ENOENT;  in i40e_veb_mem_alloc()
14650 	mutex_lock(&pf->switch_mutex);  in i40e_veb_mem_alloc()
14659 	while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))  in i40e_veb_mem_alloc()
14662 		ret = -ENOMEM;  in i40e_veb_mem_alloc()
14668 		ret = -ENOMEM;  in i40e_veb_mem_alloc()
14671 	veb->pf = pf;  in i40e_veb_mem_alloc()
14672 	veb->idx = i;  in i40e_veb_mem_alloc()
14673 	veb->enabled_tc = 1;  in i40e_veb_mem_alloc()
14675 	pf->veb[i] = veb;  in i40e_veb_mem_alloc()
14678 	mutex_unlock(&pf->switch_mutex);  in i40e_veb_mem_alloc()
14683  * i40e_switch_branch_release - Delete a branch of the switch tree
14691 	struct i40e_pf *pf = branch->pf;  in i40e_switch_branch_release()
14692 	u16 branch_seid = branch->seid;  in i40e_switch_branch_release()
14693 	u16 veb_idx = branch->idx;  in i40e_switch_branch_release()
14698 	/* release any VEBs on this VEB - RECURSION */  in i40e_switch_branch_release()
14700 		if (veb->uplink_seid == branch->seid)  in i40e_switch_branch_release()
14709 		if (vsi->uplink_seid == branch_seid &&  in i40e_switch_branch_release()
14710 		    (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)  in i40e_switch_branch_release()
14718 	if (pf->veb[veb_idx])  in i40e_switch_branch_release()
14719 		i40e_veb_release(pf->veb[veb_idx]);  in i40e_switch_branch_release()
14723  * i40e_veb_clear - remove veb struct
14731 	if (veb->pf) {  in i40e_veb_clear()
14732 		struct i40e_pf *pf = veb->pf;  in i40e_veb_clear()
14734 		mutex_lock(&pf->switch_mutex);  in i40e_veb_clear()
14735 		if (pf->veb[veb->idx] == veb)  in i40e_veb_clear()
14736 			pf->veb[veb->idx] = NULL;  in i40e_veb_clear()
14737 		mutex_unlock(&pf->switch_mutex);  in i40e_veb_clear()
14744  * i40e_veb_release - Delete a VEB and free its resources
14753 	pf = veb->pf;  in i40e_veb_release()
14757 		if (vsi_it->uplink_seid == veb->seid) {  in i40e_veb_release()
14758 			if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER)  in i40e_veb_release()
14766 	if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) {  in i40e_veb_release()
14767 		dev_info(&pf->pdev->dev,  in i40e_veb_release()
14769 			 veb->seid, n);  in i40e_veb_release()
14774 	if (veb->uplink_seid) {  in i40e_veb_release()
14775 		vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;  in i40e_veb_release()
14776 		vsi->uplink_seid = veb->uplink_seid;  in i40e_veb_release()
14777 		vsi->veb_idx = I40E_NO_VEB;  in i40e_veb_release()
14780 	i40e_aq_delete_element(&pf->hw, veb->seid, NULL);  in i40e_veb_release()
14785  * i40e_add_veb - create the VEB in the switch
14791 	struct i40e_pf *pf = veb->pf;  in i40e_add_veb()
14792 	bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);  in i40e_add_veb()
14795 	ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi ? vsi->seid : 0,  in i40e_add_veb()
14796 			      veb->enabled_tc, vsi ? false : true,  in i40e_add_veb()
14797 			      &veb->seid, enable_stats, NULL);  in i40e_add_veb()
14801 		dev_info(&pf->pdev->dev,  in i40e_add_veb()
14804 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_add_veb()
14805 		return -EPERM;  in i40e_add_veb()
14809 	ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,  in i40e_add_veb()
14810 					 &veb->stats_idx, NULL, NULL, NULL);  in i40e_add_veb()
14812 		dev_info(&pf->pdev->dev,  in i40e_add_veb()
14815 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_add_veb()
14816 		return -EPERM;  in i40e_add_veb()
14820 		dev_info(&pf->pdev->dev,  in i40e_add_veb()
14823 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_add_veb()
14824 		i40e_aq_delete_element(&pf->hw, veb->seid, NULL);  in i40e_add_veb()
14825 		return -ENOENT;  in i40e_add_veb()
14829 		vsi->uplink_seid = veb->seid;  in i40e_add_veb()
14830 		vsi->veb_idx = veb->idx;  in i40e_add_veb()
14831 		vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;  in i40e_add_veb()
14838  * i40e_veb_setup - Set up a VEB
14842  * @enabled_tc: Enabled TC bit-map
14863 		dev_info(&pf->pdev->dev,  in i40e_veb_setup()
14873 			dev_err(&pf->pdev->dev, "vsi seid %d not found\n",  in i40e_veb_setup()
14883 	veb = pf->veb[veb_idx];  in i40e_veb_setup()
14884 	veb->uplink_seid = uplink_seid;  in i40e_veb_setup()
14885 	veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);  in i40e_veb_setup()
14892 	if (vsi && vsi->idx == pf->lan_vsi)  in i40e_veb_setup()
14893 		pf->lan_veb = veb->idx;  in i40e_veb_setup()
14904  * i40e_setup_pf_switch_element - set PF vars based on switch type
14916 	u16 downlink_seid = le16_to_cpu(ele->downlink_seid);  in i40e_setup_pf_switch_element()
14917 	u16 uplink_seid = le16_to_cpu(ele->uplink_seid);  in i40e_setup_pf_switch_element()
14918 	u8 element_type = ele->element_type;  in i40e_setup_pf_switch_element()
14919 	u16 seid = le16_to_cpu(ele->seid);  in i40e_setup_pf_switch_element()
14923 		dev_info(&pf->pdev->dev,  in i40e_setup_pf_switch_element()
14929 		pf->mac_seid = seid;  in i40e_setup_pf_switch_element()
14933 		if (uplink_seid != pf->mac_seid)  in i40e_setup_pf_switch_element()
14942 				pf->lan_veb = veb->idx;  in i40e_setup_pf_switch_element()
14947 				pf->lan_veb = v;  in i40e_setup_pf_switch_element()
14951 		/* Try to get again main VEB as pf->lan_veb may have changed */  in i40e_setup_pf_switch_element()
14956 		veb->seid = seid;  in i40e_setup_pf_switch_element()
14957 		veb->uplink_seid = pf->mac_seid;  in i40e_setup_pf_switch_element()
14958 		veb->pf = pf;  in i40e_setup_pf_switch_element()
14966 		pf->mac_seid = uplink_seid;  in i40e_setup_pf_switch_element()
14967 		pf->main_vsi_seid = seid;  in i40e_setup_pf_switch_element()
14969 			dev_info(&pf->pdev->dev,  in i40e_setup_pf_switch_element()
14971 				 downlink_seid, pf->main_vsi_seid);  in i40e_setup_pf_switch_element()
14982 		dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",  in i40e_setup_pf_switch_element()
14989  * i40e_fetch_switch_configuration - Get switch config from firmware
15006 		return -ENOMEM;  in i40e_fetch_switch_configuration()
15012 		ret = i40e_aq_get_switch_config(&pf->hw, sw_config,  in i40e_fetch_switch_configuration()
15016 			dev_info(&pf->pdev->dev,  in i40e_fetch_switch_configuration()
15019 				 i40e_aq_str(&pf->hw,  in i40e_fetch_switch_configuration()
15020 					     pf->hw.aq.asq_last_status));  in i40e_fetch_switch_configuration()
15022 			return -ENOENT;  in i40e_fetch_switch_configuration()
15025 		num_reported = le16_to_cpu(sw_config->header.num_reported);  in i40e_fetch_switch_configuration()
15026 		num_total = le16_to_cpu(sw_config->header.num_total);  in i40e_fetch_switch_configuration()
15029 			dev_info(&pf->pdev->dev,  in i40e_fetch_switch_configuration()
15035 				&sw_config->element[i];  in i40e_fetch_switch_configuration()
15047  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
15049  * @reinit: if the Main VSI needs to re-initialized.
15063 		dev_info(&pf->pdev->dev,  in i40e_setup_pf_switch()
15066 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_setup_pf_switch()
15077 	if ((pf->hw.pf_id == 0) &&  in i40e_setup_pf_switch()
15078 	    !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) {  in i40e_setup_pf_switch()
15080 		pf->last_sw_conf_flags = flags;  in i40e_setup_pf_switch()
15083 	if (pf->hw.pf_id == 0) {  in i40e_setup_pf_switch()
15087 		ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,  in i40e_setup_pf_switch()
15089 		if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {  in i40e_setup_pf_switch()
15090 			dev_info(&pf->pdev->dev,  in i40e_setup_pf_switch()
15093 				 i40e_aq_str(&pf->hw,  in i40e_setup_pf_switch()
15094 					     pf->hw.aq.asq_last_status));  in i40e_setup_pf_switch()
15097 		pf->last_sw_conf_valid_flags = valid_flags;  in i40e_setup_pf_switch()
15111 			uplink_seid = veb->seid;  in i40e_setup_pf_switch()
15113 			uplink_seid = pf->mac_seid;  in i40e_setup_pf_switch()
15120 			dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");  in i40e_setup_pf_switch()
15123 			return -EAGAIN;  in i40e_setup_pf_switch()
15127 		main_vsi->seid = pf->main_vsi_seid;  in i40e_setup_pf_switch()
15137 		dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",  in i40e_setup_pf_switch()
15145 	if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))  in i40e_setup_pf_switch()
15157 	udp_tunnel_nic_reset_ntf(main_vsi->netdev);  in i40e_setup_pf_switch()
15166  * i40e_determine_queue_usage - Work out queue distribution
15174 	pf->num_lan_qps = 0;  in i40e_determine_queue_usage()
15180 	queues_left = pf->hw.func_caps.num_tx_qp;  in i40e_determine_queue_usage()
15183 	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_determine_queue_usage()
15186 		pf->alloc_rss_size = pf->num_lan_qps = 1;  in i40e_determine_queue_usage()
15189 		clear_bit(I40E_FLAG_RSS_ENA, pf->flags);  in i40e_determine_queue_usage()
15190 		clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);  in i40e_determine_queue_usage()
15191 		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_determine_queue_usage()
15192 		clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);  in i40e_determine_queue_usage()
15193 		clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_determine_queue_usage()
15194 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_determine_queue_usage()
15195 		clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);  in i40e_determine_queue_usage()
15196 		clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);  in i40e_determine_queue_usage()
15197 		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_determine_queue_usage()
15198 	} else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) &&  in i40e_determine_queue_usage()
15199 		   !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&  in i40e_determine_queue_usage()
15200 		   !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&  in i40e_determine_queue_usage()
15201 		   !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) {  in i40e_determine_queue_usage()
15203 		pf->alloc_rss_size = pf->num_lan_qps = 1;  in i40e_determine_queue_usage()
15204 		queues_left -= pf->num_lan_qps;  in i40e_determine_queue_usage()
15206 		clear_bit(I40E_FLAG_RSS_ENA, pf->flags);  in i40e_determine_queue_usage()
15207 		clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);  in i40e_determine_queue_usage()
15208 		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_determine_queue_usage()
15209 		clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);  in i40e_determine_queue_usage()
15210 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_determine_queue_usage()
15211 		clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);  in i40e_determine_queue_usage()
15212 		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_determine_queue_usage()
15214 		/* Not enough queues for all TCs */  in i40e_determine_queue_usage()
15215 		if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) &&  in i40e_determine_queue_usage()
15217 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_determine_queue_usage()
15218 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_determine_queue_usage()
15219 			dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");  in i40e_determine_queue_usage()
15223 		q_max = max_t(int, pf->rss_size_max, num_online_cpus());  in i40e_determine_queue_usage()
15224 		q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);  in i40e_determine_queue_usage()
15225 		q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);  in i40e_determine_queue_usage()
15226 		pf->num_lan_qps = q_max;  in i40e_determine_queue_usage()
15228 		queues_left -= pf->num_lan_qps;  in i40e_determine_queue_usage()
15231 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {  in i40e_determine_queue_usage()
15233 			queues_left -= 1; /* save 1 queue for FD */  in i40e_determine_queue_usage()
15235 			clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);  in i40e_determine_queue_usage()
15236 			set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);  in i40e_determine_queue_usage()
15237 …dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n…  in i40e_determine_queue_usage()
15241 	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&  in i40e_determine_queue_usage()
15242 	    pf->num_vf_qps && pf->num_req_vfs && queues_left) {  in i40e_determine_queue_usage()
15243 		pf->num_req_vfs = min_t(int, pf->num_req_vfs,  in i40e_determine_queue_usage()
15244 					(queues_left / pf->num_vf_qps));  in i40e_determine_queue_usage()
15245 		queues_left -= (pf->num_req_vfs * pf->num_vf_qps);  in i40e_determine_queue_usage()
15248 	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) &&  in i40e_determine_queue_usage()
15249 	    pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {  in i40e_determine_queue_usage()
15250 		pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,  in i40e_determine_queue_usage()
15251 					  (queues_left / pf->num_vmdq_qps));  in i40e_determine_queue_usage()
15252 		queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);  in i40e_determine_queue_usage()
15255 	pf->queues_left = queues_left;  in i40e_determine_queue_usage()
15256 	dev_dbg(&pf->pdev->dev,  in i40e_determine_queue_usage()
15258 		pf->hw.func_caps.num_tx_qp,  in i40e_determine_queue_usage()
15259 		!!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags),  in i40e_determine_queue_usage()
15260 		pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,  in i40e_determine_queue_usage()
15261 		pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,  in i40e_determine_queue_usage()
15266  * i40e_setup_pf_filter_control - Setup PF static filter control
15278 	struct i40e_filter_control_settings *settings = &pf->filter_settings;  in i40e_setup_pf_filter_control()
15280 	settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;  in i40e_setup_pf_filter_control()
15283 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ||  in i40e_setup_pf_filter_control()
15284 	    test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))  in i40e_setup_pf_filter_control()
15285 		settings->enable_fdir = true;  in i40e_setup_pf_filter_control()
15288 	settings->enable_ethtype = true;  in i40e_setup_pf_filter_control()
15289 	settings->enable_macvlan = true;  in i40e_setup_pf_filter_control()
15291 	if (i40e_set_filter_control(&pf->hw, settings))  in i40e_setup_pf_filter_control()
15292 		return -ENOENT;  in i40e_setup_pf_filter_control()
15298 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
15302 	struct i40e_hw *hw = &pf->hw;  in i40e_print_features()
15310 	i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);  in i40e_print_features()
15312 	i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);  in i40e_print_features()
15315 		       pf->hw.func_caps.num_vsis, main_vsi->num_queue_pairs);  in i40e_print_features()
15316 	if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))  in i40e_print_features()
15318 	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))  in i40e_print_features()
15320 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {  in i40e_print_features()
15324 	if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))  in i40e_print_features()
15328 	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))  in i40e_print_features()
15330 	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))  in i40e_print_features()
15335 	dev_info(&pf->pdev->dev, "%s\n", buf);  in i40e_print_features()
15341  * i40e_get_platform_mac_addr - get platform-specific MAC address
15352 	if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))  in i40e_get_platform_mac_addr()
15353 		i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);  in i40e_get_platform_mac_addr()
15357  * i40e_set_fec_in_flags - helper function for setting FEC options in flags
15384  * i40e_check_recovery_mode - check if we are running transition firmware
15394 	u32 val = rd32(&pf->hw, I40E_GL_FWSTS);  in i40e_check_recovery_mode()
15397 		dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");  in i40e_check_recovery_mode()
15398 …dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for detai…  in i40e_check_recovery_mode()
15399 		set_bit(__I40E_RECOVERY_MODE, pf->state);  in i40e_check_recovery_mode()
15403 	if (test_bit(__I40E_RECOVERY_MODE, pf->state))  in i40e_check_recovery_mode()
15404 …dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full …  in i40e_check_recovery_mode()
15410  * i40e_pf_loop_reset - perform reset in a loop.
15420  * state is to issue a series of pf-resets and check a return value.
15434 	struct i40e_hw *hw = &pf->hw;  in i40e_pf_loop_reset()
15444 		pf->pfr_count++;  in i40e_pf_loop_reset()
15446 		dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);  in i40e_pf_loop_reset()
15452  * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15464 	const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &  in i40e_check_fw_empr()
15471  * i40e_handle_resets - handle EMP resets and PF resets
15487 …dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several …  in i40e_handle_resets()
15489 	return is_empr ? -EIO : pfr;  in i40e_handle_resets()
15493  * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15508 	pci_set_drvdata(pf->pdev, pf);  in i40e_init_recovery_mode()
15509 	pci_save_state(pf->pdev);  in i40e_init_recovery_mode()
15512 	timer_setup(&pf->service_timer, i40e_service_timer, 0);  in i40e_init_recovery_mode()
15513 	pf->service_timer_period = HZ;  in i40e_init_recovery_mode()
15515 	INIT_WORK(&pf->service_task, i40e_service_task);  in i40e_init_recovery_mode()
15516 	clear_bit(__I40E_SERVICE_SCHED, pf->state);  in i40e_init_recovery_mode()
15527 	if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)  in i40e_init_recovery_mode()
15528 		pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;  in i40e_init_recovery_mode()
15530 		pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;  in i40e_init_recovery_mode()
15533 	pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),  in i40e_init_recovery_mode()
15535 	if (!pf->vsi) {  in i40e_init_recovery_mode()
15536 		err = -ENOMEM;  in i40e_init_recovery_mode()
15548 	pf->lan_vsi = v_idx;  in i40e_init_recovery_mode()
15549 	vsi = pf->vsi[v_idx];  in i40e_init_recovery_mode()
15551 		err = -EFAULT;  in i40e_init_recovery_mode()
15554 	vsi->alloc_queue_pairs = 1;  in i40e_init_recovery_mode()
15558 	err = register_netdev(vsi->netdev);  in i40e_init_recovery_mode()
15561 	vsi->netdev_registered = true;  in i40e_init_recovery_mode()
15572 	mod_timer(&pf->service_timer,  in i40e_init_recovery_mode()
15573 		  round_jiffies(jiffies + pf->service_timer_period));  in i40e_init_recovery_mode()
15579 	timer_shutdown_sync(&pf->service_timer);  in i40e_init_recovery_mode()
15581 	iounmap(hw->hw_addr);  in i40e_init_recovery_mode()
15582 	pci_release_mem_regions(pf->pdev);  in i40e_init_recovery_mode()
15583 	pci_disable_device(pf->pdev);  in i40e_init_recovery_mode()
15590  * i40e_set_subsystem_device_id - set subsystem device id
15600 	hw->subsystem_device_id = pf->pdev->subsystem_device ?  in i40e_set_subsystem_device_id()
15601 		pf->pdev->subsystem_device :  in i40e_set_subsystem_device_id()
15606  * i40e_probe - Device initialization routine
15639 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));  in i40e_probe()
15641 		dev_err(&pdev->dev,  in i40e_probe()
15649 		dev_info(&pdev->dev,  in i40e_probe()
15661 	pf = i40e_alloc_pf(&pdev->dev);  in i40e_probe()
15663 		err = -ENOMEM;  in i40e_probe()
15666 	pf->next_vsi = 0;  in i40e_probe()
15667 	pf->pdev = pdev;  in i40e_probe()
15668 	set_bit(__I40E_DOWN, pf->state);  in i40e_probe()
15670 	hw = &pf->hw;  in i40e_probe()
15672 	pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),  in i40e_probe()
15679 	if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {  in i40e_probe()
15680 		dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",  in i40e_probe()
15681 			pf->ioremap_len);  in i40e_probe()
15682 		err = -ENOMEM;  in i40e_probe()
15685 	hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);  in i40e_probe()
15686 	if (!hw->hw_addr) {  in i40e_probe()
15687 		err = -EIO;  in i40e_probe()
15688 		dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",  in i40e_probe()
15690 			 pf->ioremap_len, err);  in i40e_probe()
15693 	hw->vendor_id = pdev->vendor;  in i40e_probe()
15694 	hw->device_id = pdev->device;  in i40e_probe()
15695 	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);  in i40e_probe()
15696 	hw->subsystem_vendor_id = pdev->subsystem_vendor;  in i40e_probe()
15698 	hw->bus.device = PCI_SLOT(pdev->devfn);  in i40e_probe()
15699 	hw->bus.func = PCI_FUNC(pdev->devfn);  in i40e_probe()
15700 	hw->bus.bus_id = pdev->bus->number;  in i40e_probe()
15705 	hw->switch_tag = 0xffff;  in i40e_probe()
15706 	hw->first_tag = ETH_P_8021AD;  in i40e_probe()
15707 	hw->second_tag = ETH_P_8021Q;  in i40e_probe()
15709 	INIT_LIST_HEAD(&pf->l3_flex_pit_list);  in i40e_probe()
15710 	INIT_LIST_HEAD(&pf->l4_flex_pit_list);  in i40e_probe()
15711 	INIT_LIST_HEAD(&pf->ddp_old_prof);  in i40e_probe()
15716 	mutex_init(&hw->aq.asq_mutex);  in i40e_probe()
15717 	mutex_init(&hw->aq.arq_mutex);  in i40e_probe()
15719 	pf->msg_enable = netif_msg_init(debug,  in i40e_probe()
15723 	if (debug < -1)  in i40e_probe()
15724 		pf->hw.debug_mask = debug;  in i40e_probe()
15727 	if (hw->revision_id == 0 &&  in i40e_probe()
15732 		pf->corer_count++;  in i40e_probe()
15742 		dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",  in i40e_probe()
15754 		hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;  in i40e_probe()
15755 		hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;  in i40e_probe()
15757 		hw->aq.num_arq_entries = I40E_AQ_LEN;  in i40e_probe()
15758 		hw->aq.num_asq_entries = I40E_AQ_LEN;  in i40e_probe()
15760 	hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;  in i40e_probe()
15761 	hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;  in i40e_probe()
15763 	snprintf(pf->int_name, sizeof(pf->int_name) - 1,  in i40e_probe()
15764 		 "%s-%s:misc",  in i40e_probe()
15765 		 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));  in i40e_probe()
15769 		dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",  in i40e_probe()
15775 	pf->hw.fc.requested_mode = I40E_FC_NONE;  in i40e_probe()
15779 		if (err == -EIO)  in i40e_probe()
15780 			dev_info(&pdev->dev,  in i40e_probe()
15782 				 hw->aq.api_maj_ver,  in i40e_probe()
15783 				 hw->aq.api_min_ver,  in i40e_probe()
15787 			dev_info(&pdev->dev,  in i40e_probe()
15797 	dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",  in i40e_probe()
15798 		 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,  in i40e_probe()
15799 		 hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver,  in i40e_probe()
15800 		 hw->vendor_id, hw->device_id, hw->subsystem_vendor_id,  in i40e_probe()
15801 		 hw->subsystem_device_id);  in i40e_probe()
15805 		dev_dbg(&pdev->dev,  in i40e_probe()
15807 			 hw->aq.api_maj_ver,  in i40e_probe()
15808 			 hw->aq.api_min_ver,  in i40e_probe()
15812 		dev_info(&pdev->dev,  in i40e_probe()
15814 			 hw->aq.api_maj_ver,  in i40e_probe()
15815 			 hw->aq.api_min_ver,  in i40e_probe()
15822 	if (hw->revision_id < 1)  in i40e_probe()
15823 …dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be is…  in i40e_probe()
15833 		dev_info(&pdev->dev, "sw_init failed: %d\n", err);  in i40e_probe()
15837 	if (test_bit(__I40E_RECOVERY_MODE, pf->state))  in i40e_probe()
15840 	err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,  in i40e_probe()
15841 				hw->func_caps.num_rx_qp, 0, 0);  in i40e_probe()
15843 		dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);  in i40e_probe()
15849 		dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);  in i40e_probe()
15850 		err = -ENOENT;  in i40e_probe()
15858 	if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) {  in i40e_probe()
15859 		dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");  in i40e_probe()
15866 	if (!is_valid_ether_addr(hw->mac.addr)) {  in i40e_probe()
15867 		dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);  in i40e_probe()
15868 		err = -EIO;  in i40e_probe()
15871 	dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);  in i40e_probe()
15872 	ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);  in i40e_probe()
15873 	i40e_get_port_mac_addr(hw, hw->mac.port_addr);  in i40e_probe()
15874 	if (is_valid_ether_addr(hw->mac.port_addr))  in i40e_probe()
15875 		set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps);  in i40e_probe()
15882 	status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);  in i40e_probe()
15885 		(clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) :  in i40e_probe()
15886 		(set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags));  in i40e_probe()
15887 	dev_info(&pdev->dev,  in i40e_probe()
15888 		 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?  in i40e_probe()
15892 	/* Enable FW to write default DCB config on link-up */  in i40e_probe()
15897 		dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);  in i40e_probe()
15898 		clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);  in i40e_probe()
15899 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);  in i40e_probe()
15905 	timer_setup(&pf->service_timer, i40e_service_timer, 0);  in i40e_probe()
15906 	pf->service_timer_period = HZ;  in i40e_probe()
15908 	INIT_WORK(&pf->service_task, i40e_service_task);  in i40e_probe()
15909 	clear_bit(__I40E_SERVICE_SCHED, pf->state);  in i40e_probe()
15913 	if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)  in i40e_probe()
15914 		pf->wol_en = false;  in i40e_probe()
15916 		pf->wol_en = true;  in i40e_probe()
15917 	device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);  in i40e_probe()
15926 	 * When MSI-X is enabled, it's not allowed to use more TC queue  in i40e_probe()
15927 	 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus  in i40e_probe()
15928 	 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.  in i40e_probe()
15931 		pf->num_lan_msix = 1;  in i40e_probe()
15933 	pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;  in i40e_probe()
15934 	pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;  in i40e_probe()
15935 	pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;  in i40e_probe()
15936 	pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;  in i40e_probe()
15937 	pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;  in i40e_probe()
15938 	pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |  in i40e_probe()
15946 	if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)  in i40e_probe()
15947 		pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;  in i40e_probe()
15949 		pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;  in i40e_probe()
15950 	if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {  in i40e_probe()
15951 		dev_warn(&pf->pdev->dev,  in i40e_probe()
15953 			 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);  in i40e_probe()
15954 		pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;  in i40e_probe()
15958 	pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),  in i40e_probe()
15960 	if (!pf->vsi) {  in i40e_probe()
15961 		err = -ENOMEM;  in i40e_probe()
15967 	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&  in i40e_probe()
15968 	    test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&  in i40e_probe()
15969 	    !test_bit(__I40E_BAD_EEPROM, pf->state)) {  in i40e_probe()
15971 			set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);  in i40e_probe()
15976 		dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);  in i40e_probe()
15981 	INIT_LIST_HEAD(&vsi->ch_list);  in i40e_probe()
15991 	err = i40e_aq_set_phy_int_mask(&pf->hw,  in i40e_probe()
15996 		dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",  in i40e_probe()
15998 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_probe()
16011 	if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {  in i40e_probe()
16013 		err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);  in i40e_probe()
16015 			dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",  in i40e_probe()
16017 				 i40e_aq_str(&pf->hw,  in i40e_probe()
16018 					     pf->hw.aq.asq_last_status));  in i40e_probe()
16024 	clear_bit(__I40E_DOWN, pf->state);  in i40e_probe()
16031 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {  in i40e_probe()
16034 			dev_info(&pdev->dev,  in i40e_probe()
16044 	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&  in i40e_probe()
16045 	    test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&  in i40e_probe()
16046 	    !test_bit(__I40E_BAD_EEPROM, pf->state)) {  in i40e_probe()
16054 			dev_info(&pdev->dev,  in i40e_probe()
16058 				dev_info(&pdev->dev,  in i40e_probe()
16065 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {  in i40e_probe()
16066 		pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,  in i40e_probe()
16067 						      pf->num_iwarp_msix,  in i40e_probe()
16069 		if (pf->iwarp_base_vector < 0) {  in i40e_probe()
16070 			dev_info(&pdev->dev,  in i40e_probe()
16072 				 pf->num_iwarp_msix, pf->iwarp_base_vector);  in i40e_probe()
16073 			clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);  in i40e_probe()
16083 	mod_timer(&pf->service_timer,  in i40e_probe()
16084 		  round_jiffies(jiffies + pf->service_timer_period));  in i40e_probe()
16087 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {  in i40e_probe()
16090 			dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",  in i40e_probe()
16100 	if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) {  in i40e_probe()
16107 		pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,  in i40e_probe()
16112 		switch (hw->bus.speed) {  in i40e_probe()
16122 		switch (hw->bus.width) {  in i40e_probe()
16135 		dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",  in i40e_probe()
16138 		if (hw->bus.width < i40e_bus_width_pcie_x8 ||  in i40e_probe()
16139 		    hw->bus.speed < i40e_bus_speed_8000) {  in i40e_probe()
16140 …dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for opti…  in i40e_probe()
16141 …dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or high…  in i40e_probe()
16148 		dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %pe last_status =  %s\n",  in i40e_probe()
16150 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_probe()
16151 	pf->hw.phy.link_info.requested_speeds = abilities.link_speed;  in i40e_probe()
16154 	i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags);  in i40e_probe()
16159 		dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %pe last_status =  %s\n",  in i40e_probe()
16161 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));  in i40e_probe()
16166 			rd32(&pf->hw, I40E_PRTGL_SAH));  in i40e_probe()
16168 		dev_warn(&pdev->dev, "MFS for port %x (%d) has been set below the default (%d)\n",  in i40e_probe()
16169 			 pf->hw.port, val, MAX_FRAME_SIZE_DEFAULT);  in i40e_probe()
16177 	i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,  in i40e_probe()
16178 						       pf->main_vsi_seid);  in i40e_probe()
16180 	if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||  in i40e_probe()
16181 	    (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))  in i40e_probe()
16182 		set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps);  in i40e_probe()
16183 	if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)  in i40e_probe()
16184 		set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps);  in i40e_probe()
16194 	set_bit(__I40E_DOWN, pf->state);  in i40e_probe()
16196 	kfree(pf->vsi);  in i40e_probe()
16199 	timer_shutdown_sync(&pf->service_timer);  in i40e_probe()
16204 	kfree(pf->qp_pile);  in i40e_probe()
16208 	iounmap(hw->hw_addr);  in i40e_probe()
16220  * i40e_remove - Device removal routine
16225  * Hot-Plug event, or because the driver is going to be removed from
16231 	struct i40e_hw *hw = &pf->hw;  in i40e_remove()
16251 	while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))  in i40e_remove()
16253 	set_bit(__I40E_IN_REMOVE, pf->state);  in i40e_remove()
16255 	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) {  in i40e_remove()
16256 		set_bit(__I40E_VF_RESETS_DISABLED, pf->state);  in i40e_remove()
16258 		clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);  in i40e_remove()
16261 	set_bit(__I40E_SUSPENDED, pf->state);  in i40e_remove()
16262 	set_bit(__I40E_DOWN, pf->state);  in i40e_remove()
16263 	if (pf->service_timer.function)  in i40e_remove()
16264 		timer_shutdown_sync(&pf->service_timer);  in i40e_remove()
16265 	if (pf->service_task.func)  in i40e_remove()
16266 		cancel_work_sync(&pf->service_task);  in i40e_remove()
16268 	if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {  in i40e_remove()
16269 		struct i40e_vsi *vsi = pf->vsi[0];  in i40e_remove()
16275 		unregister_netdev(vsi->netdev);  in i40e_remove()
16276 		free_netdev(vsi->netdev);  in i40e_remove()
16292 		if (veb->uplink_seid == pf->mac_seid ||  in i40e_remove()
16293 		    veb->uplink_seid == 0)  in i40e_remove()
16302 		pf->vsi[i] = NULL;  in i40e_remove()
16308 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {  in i40e_remove()
16311 			dev_warn(&pdev->dev, "Failed to delete client device: %d\n",  in i40e_remove()
16316 	if (hw->hmc.hmc_obj) {  in i40e_remove()
16319 			dev_warn(&pdev->dev,  in i40e_remove()
16326 	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&  in i40e_remove()
16327 	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_remove()
16328 		free_irq(pf->pdev->irq, pf);  in i40e_remove()
16334 	mutex_destroy(&hw->aq.arq_mutex);  in i40e_remove()
16335 	mutex_destroy(&hw->aq.asq_mutex);  in i40e_remove()
16341 		if (!test_bit(__I40E_RECOVERY_MODE, pf->state))  in i40e_remove()
16345 		pf->vsi[i] = NULL;  in i40e_remove()
16351 		pf->veb[i] = NULL;  in i40e_remove()
16354 	kfree(pf->qp_pile);  in i40e_remove()
16355 	kfree(pf->vsi);  in i40e_remove()
16357 	iounmap(hw->hw_addr);  in i40e_remove()
16365  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16372 	struct i40e_hw *hw = &pf->hw;  in i40e_enable_mc_magic_wake()
16378 	if (main_vsi && main_vsi->netdev) {  in i40e_enable_mc_magic_wake()
16379 		ether_addr_copy(mac_addr, main_vsi->netdev->dev_addr);  in i40e_enable_mc_magic_wake()
16381 		dev_err(&pf->pdev->dev,  in i40e_enable_mc_magic_wake()
16383 		ether_addr_copy(mac_addr, hw->mac.addr);  in i40e_enable_mc_magic_wake()
16392 	if (hw->func_caps.flex10_enable && hw->partition_id != 1)  in i40e_enable_mc_magic_wake()
16397 		dev_err(&pf->pdev->dev,  in i40e_enable_mc_magic_wake()
16407 		dev_err(&pf->pdev->dev,  in i40e_enable_mc_magic_wake()
16412  * i40e_io_suspend - suspend all IO operations
16418 	struct i40e_hw *hw = &pf->hw;  in i40e_io_suspend()
16420 	set_bit(__I40E_DOWN, pf->state);  in i40e_io_suspend()
16423 	del_timer_sync(&pf->service_timer);  in i40e_io_suspend()
16424 	cancel_work_sync(&pf->service_task);  in i40e_io_suspend()
16431 	if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&  in i40e_io_suspend()
16432 	    pf->wol_en)  in i40e_io_suspend()
16443 	wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));  in i40e_io_suspend()
16444 	wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));  in i40e_io_suspend()
16459  * i40e_io_resume - resume IO operations
16465 	struct device *dev = &pf->pdev->dev;  in i40e_io_resume()
16482 	clear_bit(__I40E_DOWN, pf->state);  in i40e_io_resume()
16488 	clear_bit(__I40E_SUSPENDED, pf->state);  in i40e_io_resume()
16491 	mod_timer(&pf->service_timer,  in i40e_io_resume()
16492 		  round_jiffies(jiffies + pf->service_timer_period));  in i40e_io_resume()
16498  * i40e_pci_error_detected - warning that something funky happened in PCI land
16511 	dev_info(&pdev->dev, "%s: error %d\n", __func__, error);  in i40e_pci_error_detected()
16514 		dev_info(&pdev->dev,  in i40e_pci_error_detected()
16515 			 "Cannot recover - error happened during device probe\n");  in i40e_pci_error_detected()
16520 	if (!test_bit(__I40E_SUSPENDED, pf->state))  in i40e_pci_error_detected()
16528  * i40e_pci_error_slot_reset - a PCI slot reset just happened
16542 	dev_dbg(&pdev->dev, "%s\n", __func__);  in i40e_pci_error_slot_reset()
16545 		dev_info(&pdev->dev,  in i40e_pci_error_slot_reset()
16546 			 "Cannot re-enable PCI device after reset.\n");  in i40e_pci_error_slot_reset()
16554 		reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);  in i40e_pci_error_slot_reset()
16565  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
16576  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16583 	if (test_bit(__I40E_IN_REMOVE, pf->state))  in i40e_pci_error_reset_done()
16593  * i40e_pci_error_resume - restart operations after PCI error recovery
16603 	dev_dbg(&pdev->dev, "%s\n", __func__);  in i40e_pci_error_resume()
16604 	if (test_bit(__I40E_SUSPENDED, pf->state))  in i40e_pci_error_resume()
16611  * i40e_shutdown - PCI callback for shutting down
16617 	struct i40e_hw *hw = &pf->hw;  in i40e_shutdown()
16619 	set_bit(__I40E_SUSPENDED, pf->state);  in i40e_shutdown()
16620 	set_bit(__I40E_DOWN, pf->state);  in i40e_shutdown()
16622 	del_timer_sync(&pf->service_timer);  in i40e_shutdown()
16623 	cancel_work_sync(&pf->service_task);  in i40e_shutdown()
16632 	if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&  in i40e_shutdown()
16633 	    pf->wol_en)  in i40e_shutdown()
16639 	     (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));  in i40e_shutdown()
16641 	     (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));  in i40e_shutdown()
16644 	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&  in i40e_shutdown()
16645 	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))  in i40e_shutdown()
16646 		free_irq(pf->pdev->irq, pf);  in i40e_shutdown()
16657 		pci_wake_from_d3(pdev, pf->wol_en);  in i40e_shutdown()
16663  * i40e_suspend - PM callback for moving to D3
16671 	if (test_and_set_bit(__I40E_SUSPENDED, pf->state))  in i40e_suspend()
16677  * i40e_resume - PM callback for waking up from D3
16685 	if (!test_bit(__I40E_SUSPENDED, pf->state))  in i40e_resume()
16712  * i40e_init_module - Driver registration routine
16734 		return -ENOMEM;  in i40e_init_module()
16750  * i40e_exit_module - Driver exit cleanup routine