xref: /linux/drivers/net/ethernet/intel/i40e/i40e_main.c (revision 442bc81bd344dc52c37d8f80b854cc6da062b2d0)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
3 
4 #include <generated/utsrelease.h>
5 #include <linux/crash_dump.h>
6 #include <linux/if_bridge.h>
7 #include <linux/if_macvlan.h>
8 #include <linux/module.h>
9 #include <net/pkt_cls.h>
10 #include <net/xdp_sock_drv.h>
11 
12 /* Local includes */
13 #include "i40e.h"
14 #include "i40e_devids.h"
15 #include "i40e_diag.h"
16 #include "i40e_lan_hmc.h"
17 #include "i40e_virtchnl_pf.h"
18 #include "i40e_xsk.h"
19 
20 /* All i40e tracepoints are defined by the include below, which
21  * must be included exactly once across the whole kernel with
22  * CREATE_TRACE_POINTS defined
23  */
24 #define CREATE_TRACE_POINTS
25 #include "i40e_trace.h"
26 
27 const char i40e_driver_name[] = "i40e";
28 static const char i40e_driver_string[] =
29 			"Intel(R) Ethernet Connection XL710 Network Driver";
30 
31 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
32 
33 /* a bit of forward declarations */
34 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
35 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
36 static int i40e_add_vsi(struct i40e_vsi *vsi);
37 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
38 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
39 static int i40e_setup_misc_vector(struct i40e_pf *pf);
40 static void i40e_determine_queue_usage(struct i40e_pf *pf);
41 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
42 static void i40e_prep_for_reset(struct i40e_pf *pf);
43 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
44 				   bool lock_acquired);
45 static int i40e_reset(struct i40e_pf *pf);
46 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
47 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
48 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
49 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
50 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
51 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
52 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
53 static int i40e_get_capabilities(struct i40e_pf *pf,
54 				 enum i40e_admin_queue_opc list_type);
55 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
56 
57 /* i40e_pci_tbl - PCI Device ID Table
58  *
59  * Last entry must be all 0s
60  *
61  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
62  *   Class, Class Mask, private data (not used) }
63  */
64 static const struct pci_device_id i40e_pci_tbl[] = {
65 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
66 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
67 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
68 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
69 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
70 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
71 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
72 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0},
73 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
74 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
75 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
76 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
77 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
78 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
79 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
80 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
81 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
82 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
83 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
84 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0},
85 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
86 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
87 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
88 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
89 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
90 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
91 	/* required last entry */
92 	{0, }
93 };
94 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
95 
96 #define I40E_MAX_VF_COUNT 128
97 static int debug = -1;
98 module_param(debug, uint, 0);
99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
100 
101 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
102 MODULE_IMPORT_NS("LIBIE");
103 MODULE_LICENSE("GPL v2");
104 
105 static struct workqueue_struct *i40e_wq;
106 
netdev_hw_addr_refcnt(struct i40e_mac_filter * f,struct net_device * netdev,int delta)107 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
108 				  struct net_device *netdev, int delta)
109 {
110 	struct netdev_hw_addr_list *ha_list;
111 	struct netdev_hw_addr *ha;
112 
113 	if (!f || !netdev)
114 		return;
115 
116 	if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr))
117 		ha_list = &netdev->uc;
118 	else
119 		ha_list = &netdev->mc;
120 
121 	netdev_hw_addr_list_for_each(ha, ha_list) {
122 		if (ether_addr_equal(ha->addr, f->macaddr)) {
123 			ha->refcount += delta;
124 			if (ha->refcount <= 0)
125 				ha->refcount = 1;
126 			break;
127 		}
128 	}
129 }
130 
131 /**
132  * i40e_hw_to_dev - get device pointer from the hardware structure
133  * @hw: pointer to the device HW structure
134  **/
i40e_hw_to_dev(struct i40e_hw * hw)135 struct device *i40e_hw_to_dev(struct i40e_hw *hw)
136 {
137 	struct i40e_pf *pf = i40e_hw_to_pf(hw);
138 
139 	return &pf->pdev->dev;
140 }
141 
142 /**
143  * i40e_allocate_dma_mem - OS specific memory alloc for shared code
144  * @hw:   pointer to the HW structure
145  * @mem:  ptr to mem struct to fill out
146  * @size: size of memory requested
147  * @alignment: what to align the allocation to
148  **/
i40e_allocate_dma_mem(struct i40e_hw * hw,struct i40e_dma_mem * mem,u64 size,u32 alignment)149 int i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem,
150 			  u64 size, u32 alignment)
151 {
152 	struct i40e_pf *pf = i40e_hw_to_pf(hw);
153 
154 	mem->size = ALIGN(size, alignment);
155 	mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
156 				     GFP_KERNEL);
157 	if (!mem->va)
158 		return -ENOMEM;
159 
160 	return 0;
161 }
162 
163 /**
164  * i40e_free_dma_mem - OS specific memory free for shared code
165  * @hw:   pointer to the HW structure
166  * @mem:  ptr to mem struct to free
167  **/
i40e_free_dma_mem(struct i40e_hw * hw,struct i40e_dma_mem * mem)168 int i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem)
169 {
170 	struct i40e_pf *pf = i40e_hw_to_pf(hw);
171 
172 	dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
173 	mem->va = NULL;
174 	mem->pa = 0;
175 	mem->size = 0;
176 
177 	return 0;
178 }
179 
180 /**
181  * i40e_allocate_virt_mem - OS specific memory alloc for shared code
182  * @hw:   pointer to the HW structure
183  * @mem:  ptr to mem struct to fill out
184  * @size: size of memory requested
185  **/
i40e_allocate_virt_mem(struct i40e_hw * hw,struct i40e_virt_mem * mem,u32 size)186 int i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem,
187 			   u32 size)
188 {
189 	mem->size = size;
190 	mem->va = kzalloc(size, GFP_KERNEL);
191 
192 	if (!mem->va)
193 		return -ENOMEM;
194 
195 	return 0;
196 }
197 
198 /**
199  * i40e_free_virt_mem - OS specific memory free for shared code
200  * @hw:   pointer to the HW structure
201  * @mem:  ptr to mem struct to free
202  **/
i40e_free_virt_mem(struct i40e_hw * hw,struct i40e_virt_mem * mem)203 int i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem)
204 {
205 	/* it's ok to kfree a NULL pointer */
206 	kfree(mem->va);
207 	mem->va = NULL;
208 	mem->size = 0;
209 
210 	return 0;
211 }
212 
213 /**
214  * i40e_get_lump - find a lump of free generic resource
215  * @pf: board private structure
216  * @pile: the pile of resource to search
217  * @needed: the number of items needed
218  * @id: an owner id to stick on the items assigned
219  *
220  * Returns the base item index of the lump, or negative for error
221  **/
i40e_get_lump(struct i40e_pf * pf,struct i40e_lump_tracking * pile,u16 needed,u16 id)222 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
223 			 u16 needed, u16 id)
224 {
225 	int ret = -ENOMEM;
226 	int i, j;
227 
228 	if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
229 		dev_info(&pf->pdev->dev,
230 			 "param err: pile=%s needed=%d id=0x%04x\n",
231 			 pile ? "<valid>" : "<null>", needed, id);
232 		return -EINVAL;
233 	}
234 
235 	/* Allocate last queue in the pile for FDIR VSI queue
236 	 * so it doesn't fragment the qp_pile
237 	 */
238 	if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
239 		if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
240 			dev_err(&pf->pdev->dev,
241 				"Cannot allocate queue %d for I40E_VSI_FDIR\n",
242 				pile->num_entries - 1);
243 			return -ENOMEM;
244 		}
245 		pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
246 		return pile->num_entries - 1;
247 	}
248 
249 	i = 0;
250 	while (i < pile->num_entries) {
251 		/* skip already allocated entries */
252 		if (pile->list[i] & I40E_PILE_VALID_BIT) {
253 			i++;
254 			continue;
255 		}
256 
257 		/* do we have enough in this lump? */
258 		for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
259 			if (pile->list[i+j] & I40E_PILE_VALID_BIT)
260 				break;
261 		}
262 
263 		if (j == needed) {
264 			/* there was enough, so assign it to the requestor */
265 			for (j = 0; j < needed; j++)
266 				pile->list[i+j] = id | I40E_PILE_VALID_BIT;
267 			ret = i;
268 			break;
269 		}
270 
271 		/* not enough, so skip over it and continue looking */
272 		i += j;
273 	}
274 
275 	return ret;
276 }
277 
278 /**
279  * i40e_put_lump - return a lump of generic resource
280  * @pile: the pile of resource to search
281  * @index: the base item index
282  * @id: the owner id of the items assigned
283  *
284  * Returns the count of items in the lump
285  **/
i40e_put_lump(struct i40e_lump_tracking * pile,u16 index,u16 id)286 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
287 {
288 	int valid_id = (id | I40E_PILE_VALID_BIT);
289 	int count = 0;
290 	u16 i;
291 
292 	if (!pile || index >= pile->num_entries)
293 		return -EINVAL;
294 
295 	for (i = index;
296 	     i < pile->num_entries && pile->list[i] == valid_id;
297 	     i++) {
298 		pile->list[i] = 0;
299 		count++;
300 	}
301 
302 
303 	return count;
304 }
305 
306 /**
307  * i40e_find_vsi_from_id - searches for the vsi with the given id
308  * @pf: the pf structure to search for the vsi
309  * @id: id of the vsi it is searching for
310  **/
i40e_find_vsi_from_id(struct i40e_pf * pf,u16 id)311 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
312 {
313 	struct i40e_vsi *vsi;
314 	int i;
315 
316 	i40e_pf_for_each_vsi(pf, i, vsi)
317 		if (vsi->id == id)
318 			return vsi;
319 
320 	return NULL;
321 }
322 
323 /**
324  * i40e_service_event_schedule - Schedule the service task to wake up
325  * @pf: board private structure
326  *
327  * If not already scheduled, this puts the task into the work queue
328  **/
i40e_service_event_schedule(struct i40e_pf * pf)329 void i40e_service_event_schedule(struct i40e_pf *pf)
330 {
331 	if ((!test_bit(__I40E_DOWN, pf->state) &&
332 	     !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
333 	      test_bit(__I40E_RECOVERY_MODE, pf->state))
334 		queue_work(i40e_wq, &pf->service_task);
335 }
336 
337 /**
338  * i40e_tx_timeout - Respond to a Tx Hang
339  * @netdev: network interface device structure
340  * @txqueue: queue number timing out
341  *
342  * If any port has noticed a Tx timeout, it is likely that the whole
343  * device is munged, not just the one netdev port, so go for the full
344  * reset.
345  **/
i40e_tx_timeout(struct net_device * netdev,unsigned int txqueue)346 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
347 {
348 	struct i40e_netdev_priv *np = netdev_priv(netdev);
349 	struct i40e_vsi *vsi = np->vsi;
350 	struct i40e_pf *pf = vsi->back;
351 	struct i40e_ring *tx_ring = NULL;
352 	unsigned int i;
353 	u32 head, val;
354 
355 	pf->tx_timeout_count++;
356 
357 	/* with txqueue index, find the tx_ring struct */
358 	for (i = 0; i < vsi->num_queue_pairs; i++) {
359 		if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
360 			if (txqueue ==
361 			    vsi->tx_rings[i]->queue_index) {
362 				tx_ring = vsi->tx_rings[i];
363 				break;
364 			}
365 		}
366 	}
367 
368 	if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
369 		pf->tx_timeout_recovery_level = 1;  /* reset after some time */
370 	else if (time_before(jiffies,
371 		      (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
372 		return;   /* don't do any new action before the next timeout */
373 
374 	/* don't kick off another recovery if one is already pending */
375 	if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
376 		return;
377 
378 	if (tx_ring) {
379 		head = i40e_get_head(tx_ring);
380 		/* Read interrupt register */
381 		if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
382 			val = rd32(&pf->hw,
383 			     I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
384 						tx_ring->vsi->base_vector - 1));
385 		else
386 			val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
387 
388 		netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
389 			    vsi->seid, txqueue, tx_ring->next_to_clean,
390 			    head, tx_ring->next_to_use,
391 			    readl(tx_ring->tail), val);
392 	}
393 
394 	pf->tx_timeout_last_recovery = jiffies;
395 	netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
396 		    pf->tx_timeout_recovery_level, txqueue);
397 
398 	switch (pf->tx_timeout_recovery_level) {
399 	case 1:
400 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
401 		break;
402 	case 2:
403 		set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
404 		break;
405 	case 3:
406 		set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
407 		break;
408 	default:
409 		netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");
410 		set_bit(__I40E_DOWN_REQUESTED, pf->state);
411 		set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);
412 		break;
413 	}
414 
415 	i40e_service_event_schedule(pf);
416 	pf->tx_timeout_recovery_level++;
417 }
418 
419 /**
420  * i40e_get_vsi_stats_struct - Get System Network Statistics
421  * @vsi: the VSI we care about
422  *
423  * Returns the address of the device statistics structure.
424  * The statistics are actually updated from the service task.
425  **/
i40e_get_vsi_stats_struct(struct i40e_vsi * vsi)426 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
427 {
428 	return &vsi->net_stats;
429 }
430 
431 /**
432  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
433  * @ring: Tx ring to get statistics from
434  * @stats: statistics entry to be updated
435  **/
i40e_get_netdev_stats_struct_tx(struct i40e_ring * ring,struct rtnl_link_stats64 * stats)436 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
437 					    struct rtnl_link_stats64 *stats)
438 {
439 	u64 bytes, packets;
440 	unsigned int start;
441 
442 	do {
443 		start = u64_stats_fetch_begin(&ring->syncp);
444 		packets = ring->stats.packets;
445 		bytes   = ring->stats.bytes;
446 	} while (u64_stats_fetch_retry(&ring->syncp, start));
447 
448 	stats->tx_packets += packets;
449 	stats->tx_bytes   += bytes;
450 }
451 
452 /**
453  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
454  * @netdev: network interface device structure
455  * @stats: data structure to store statistics
456  *
457  * Returns the address of the device statistics structure.
458  * The statistics are actually updated from the service task.
459  **/
i40e_get_netdev_stats_struct(struct net_device * netdev,struct rtnl_link_stats64 * stats)460 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
461 				  struct rtnl_link_stats64 *stats)
462 {
463 	struct i40e_netdev_priv *np = netdev_priv(netdev);
464 	struct i40e_vsi *vsi = np->vsi;
465 	struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
466 	struct i40e_ring *ring;
467 	int i;
468 
469 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
470 		return;
471 
472 	if (!vsi->tx_rings)
473 		return;
474 
475 	rcu_read_lock();
476 	for (i = 0; i < vsi->num_queue_pairs; i++) {
477 		u64 bytes, packets;
478 		unsigned int start;
479 
480 		ring = READ_ONCE(vsi->tx_rings[i]);
481 		if (!ring)
482 			continue;
483 		i40e_get_netdev_stats_struct_tx(ring, stats);
484 
485 		if (i40e_enabled_xdp_vsi(vsi)) {
486 			ring = READ_ONCE(vsi->xdp_rings[i]);
487 			if (!ring)
488 				continue;
489 			i40e_get_netdev_stats_struct_tx(ring, stats);
490 		}
491 
492 		ring = READ_ONCE(vsi->rx_rings[i]);
493 		if (!ring)
494 			continue;
495 		do {
496 			start   = u64_stats_fetch_begin(&ring->syncp);
497 			packets = ring->stats.packets;
498 			bytes   = ring->stats.bytes;
499 		} while (u64_stats_fetch_retry(&ring->syncp, start));
500 
501 		stats->rx_packets += packets;
502 		stats->rx_bytes   += bytes;
503 
504 	}
505 	rcu_read_unlock();
506 
507 	/* following stats updated by i40e_watchdog_subtask() */
508 	stats->multicast	= vsi_stats->multicast;
509 	stats->tx_errors	= vsi_stats->tx_errors;
510 	stats->tx_dropped	= vsi_stats->tx_dropped;
511 	stats->rx_errors	= vsi_stats->rx_errors;
512 	stats->rx_dropped	= vsi_stats->rx_dropped;
513 	stats->rx_missed_errors	= vsi_stats->rx_missed_errors;
514 	stats->rx_crc_errors	= vsi_stats->rx_crc_errors;
515 	stats->rx_length_errors	= vsi_stats->rx_length_errors;
516 }
517 
518 /**
519  * i40e_vsi_reset_stats - Resets all stats of the given vsi
520  * @vsi: the VSI to have its stats reset
521  **/
i40e_vsi_reset_stats(struct i40e_vsi * vsi)522 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
523 {
524 	struct rtnl_link_stats64 *ns;
525 	int i;
526 
527 	if (!vsi)
528 		return;
529 
530 	ns = i40e_get_vsi_stats_struct(vsi);
531 	memset(ns, 0, sizeof(*ns));
532 	memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
533 	memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
534 	memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
535 	if (vsi->rx_rings && vsi->rx_rings[0]) {
536 		for (i = 0; i < vsi->num_queue_pairs; i++) {
537 			memset(&vsi->rx_rings[i]->stats, 0,
538 			       sizeof(vsi->rx_rings[i]->stats));
539 			memset(&vsi->rx_rings[i]->rx_stats, 0,
540 			       sizeof(vsi->rx_rings[i]->rx_stats));
541 			memset(&vsi->tx_rings[i]->stats, 0,
542 			       sizeof(vsi->tx_rings[i]->stats));
543 			memset(&vsi->tx_rings[i]->tx_stats, 0,
544 			       sizeof(vsi->tx_rings[i]->tx_stats));
545 		}
546 	}
547 	vsi->stat_offsets_loaded = false;
548 }
549 
550 /**
551  * i40e_pf_reset_stats - Reset all of the stats for the given PF
552  * @pf: the PF to be reset
553  **/
i40e_pf_reset_stats(struct i40e_pf * pf)554 void i40e_pf_reset_stats(struct i40e_pf *pf)
555 {
556 	struct i40e_veb *veb;
557 	int i;
558 
559 	memset(&pf->stats, 0, sizeof(pf->stats));
560 	memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
561 	pf->stat_offsets_loaded = false;
562 
563 	i40e_pf_for_each_veb(pf, i, veb) {
564 		memset(&veb->stats, 0, sizeof(veb->stats));
565 		memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets));
566 		memset(&veb->tc_stats, 0, sizeof(veb->tc_stats));
567 		memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets));
568 		veb->stat_offsets_loaded = false;
569 	}
570 	pf->hw_csum_rx_error = 0;
571 }
572 
573 /**
574  * i40e_compute_pci_to_hw_id - compute index form PCI function.
575  * @vsi: ptr to the VSI to read from.
576  * @hw: ptr to the hardware info.
577  **/
i40e_compute_pci_to_hw_id(struct i40e_vsi * vsi,struct i40e_hw * hw)578 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw)
579 {
580 	int pf_count = i40e_get_pf_count(hw);
581 
582 	if (vsi->type == I40E_VSI_SRIOV)
583 		return (hw->port * BIT(7)) / pf_count + vsi->vf_id;
584 
585 	return hw->port + BIT(7);
586 }
587 
588 /**
589  * i40e_stat_update64 - read and update a 64 bit stat from the chip.
590  * @hw: ptr to the hardware info.
591  * @hireg: the high 32 bit reg to read.
592  * @loreg: the low 32 bit reg to read.
593  * @offset_loaded: has the initial offset been loaded yet.
594  * @offset: ptr to current offset value.
595  * @stat: ptr to the stat.
596  *
597  * Since the device stats are not reset at PFReset, they will not
598  * be zeroed when the driver starts.  We'll save the first values read
599  * and use them as offsets to be subtracted from the raw values in order
600  * to report stats that count from zero.
601  **/
i40e_stat_update64(struct i40e_hw * hw,u32 hireg,u32 loreg,bool offset_loaded,u64 * offset,u64 * stat)602 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg,
603 			       bool offset_loaded, u64 *offset, u64 *stat)
604 {
605 	u64 new_data;
606 
607 	new_data = rd64(hw, loreg);
608 
609 	if (!offset_loaded || new_data < *offset)
610 		*offset = new_data;
611 	*stat = new_data - *offset;
612 }
613 
614 /**
615  * i40e_stat_update48 - read and update a 48 bit stat from the chip
616  * @hw: ptr to the hardware info
617  * @hireg: the high 32 bit reg to read
618  * @loreg: the low 32 bit reg to read
619  * @offset_loaded: has the initial offset been loaded yet
620  * @offset: ptr to current offset value
621  * @stat: ptr to the stat
622  *
623  * Since the device stats are not reset at PFReset, they likely will not
624  * be zeroed when the driver starts.  We'll save the first values read
625  * and use them as offsets to be subtracted from the raw values in order
626  * to report stats that count from zero.  In the process, we also manage
627  * the potential roll-over.
628  **/
i40e_stat_update48(struct i40e_hw * hw,u32 hireg,u32 loreg,bool offset_loaded,u64 * offset,u64 * stat)629 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
630 			       bool offset_loaded, u64 *offset, u64 *stat)
631 {
632 	u64 new_data;
633 
634 	if (hw->device_id == I40E_DEV_ID_QEMU) {
635 		new_data = rd32(hw, loreg);
636 		new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
637 	} else {
638 		new_data = rd64(hw, loreg);
639 	}
640 	if (!offset_loaded)
641 		*offset = new_data;
642 	if (likely(new_data >= *offset))
643 		*stat = new_data - *offset;
644 	else
645 		*stat = (new_data + BIT_ULL(48)) - *offset;
646 	*stat &= 0xFFFFFFFFFFFFULL;
647 }
648 
649 /**
650  * i40e_stat_update32 - read and update a 32 bit stat from the chip
651  * @hw: ptr to the hardware info
652  * @reg: the hw reg to read
653  * @offset_loaded: has the initial offset been loaded yet
654  * @offset: ptr to current offset value
655  * @stat: ptr to the stat
656  **/
i40e_stat_update32(struct i40e_hw * hw,u32 reg,bool offset_loaded,u64 * offset,u64 * stat)657 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
658 			       bool offset_loaded, u64 *offset, u64 *stat)
659 {
660 	u32 new_data;
661 
662 	new_data = rd32(hw, reg);
663 	if (!offset_loaded)
664 		*offset = new_data;
665 	if (likely(new_data >= *offset))
666 		*stat = (u32)(new_data - *offset);
667 	else
668 		*stat = (u32)((new_data + BIT_ULL(32)) - *offset);
669 }
670 
671 /**
672  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
673  * @hw: ptr to the hardware info
674  * @reg: the hw reg to read and clear
675  * @stat: ptr to the stat
676  **/
i40e_stat_update_and_clear32(struct i40e_hw * hw,u32 reg,u64 * stat)677 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
678 {
679 	u32 new_data = rd32(hw, reg);
680 
681 	wr32(hw, reg, 1); /* must write a nonzero value to clear register */
682 	*stat += new_data;
683 }
684 
685 /**
686  * i40e_stats_update_rx_discards - update rx_discards.
687  * @vsi: ptr to the VSI to be updated.
688  * @hw: ptr to the hardware info.
689  * @stat_idx: VSI's stat_counter_idx.
690  * @offset_loaded: ptr to the VSI's stat_offsets_loaded.
691  * @stat_offset: ptr to stat_offset to store first read of specific register.
692  * @stat: ptr to VSI's stat to be updated.
693  **/
694 static void
i40e_stats_update_rx_discards(struct i40e_vsi * vsi,struct i40e_hw * hw,int stat_idx,bool offset_loaded,struct i40e_eth_stats * stat_offset,struct i40e_eth_stats * stat)695 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw,
696 			      int stat_idx, bool offset_loaded,
697 			      struct i40e_eth_stats *stat_offset,
698 			      struct i40e_eth_stats *stat)
699 {
700 	i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded,
701 			   &stat_offset->rx_discards, &stat->rx_discards);
702 	i40e_stat_update64(hw,
703 			   I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)),
704 			   I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)),
705 			   offset_loaded, &stat_offset->rx_discards_other,
706 			   &stat->rx_discards_other);
707 }
708 
709 /**
710  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
711  * @vsi: the VSI to be updated
712  **/
i40e_update_eth_stats(struct i40e_vsi * vsi)713 void i40e_update_eth_stats(struct i40e_vsi *vsi)
714 {
715 	int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
716 	struct i40e_pf *pf = vsi->back;
717 	struct i40e_hw *hw = &pf->hw;
718 	struct i40e_eth_stats *oes;
719 	struct i40e_eth_stats *es;     /* device's eth stats */
720 
721 	es = &vsi->eth_stats;
722 	oes = &vsi->eth_stats_offsets;
723 
724 	/* Gather up the stats that the hw collects */
725 	i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
726 			   vsi->stat_offsets_loaded,
727 			   &oes->tx_errors, &es->tx_errors);
728 	i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
729 			   vsi->stat_offsets_loaded,
730 			   &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
731 
732 	i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
733 			   I40E_GLV_GORCL(stat_idx),
734 			   vsi->stat_offsets_loaded,
735 			   &oes->rx_bytes, &es->rx_bytes);
736 	i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
737 			   I40E_GLV_UPRCL(stat_idx),
738 			   vsi->stat_offsets_loaded,
739 			   &oes->rx_unicast, &es->rx_unicast);
740 	i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
741 			   I40E_GLV_MPRCL(stat_idx),
742 			   vsi->stat_offsets_loaded,
743 			   &oes->rx_multicast, &es->rx_multicast);
744 	i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
745 			   I40E_GLV_BPRCL(stat_idx),
746 			   vsi->stat_offsets_loaded,
747 			   &oes->rx_broadcast, &es->rx_broadcast);
748 
749 	i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
750 			   I40E_GLV_GOTCL(stat_idx),
751 			   vsi->stat_offsets_loaded,
752 			   &oes->tx_bytes, &es->tx_bytes);
753 	i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
754 			   I40E_GLV_UPTCL(stat_idx),
755 			   vsi->stat_offsets_loaded,
756 			   &oes->tx_unicast, &es->tx_unicast);
757 	i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
758 			   I40E_GLV_MPTCL(stat_idx),
759 			   vsi->stat_offsets_loaded,
760 			   &oes->tx_multicast, &es->tx_multicast);
761 	i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
762 			   I40E_GLV_BPTCL(stat_idx),
763 			   vsi->stat_offsets_loaded,
764 			   &oes->tx_broadcast, &es->tx_broadcast);
765 
766 	i40e_stats_update_rx_discards(vsi, hw, stat_idx,
767 				      vsi->stat_offsets_loaded, oes, es);
768 
769 	vsi->stat_offsets_loaded = true;
770 }
771 
772 /**
773  * i40e_update_veb_stats - Update Switch component statistics
774  * @veb: the VEB being updated
775  **/
i40e_update_veb_stats(struct i40e_veb * veb)776 void i40e_update_veb_stats(struct i40e_veb *veb)
777 {
778 	struct i40e_pf *pf = veb->pf;
779 	struct i40e_hw *hw = &pf->hw;
780 	struct i40e_eth_stats *oes;
781 	struct i40e_eth_stats *es;     /* device's eth stats */
782 	struct i40e_veb_tc_stats *veb_oes;
783 	struct i40e_veb_tc_stats *veb_es;
784 	int i, idx = 0;
785 
786 	idx = veb->stats_idx;
787 	es = &veb->stats;
788 	oes = &veb->stats_offsets;
789 	veb_es = &veb->tc_stats;
790 	veb_oes = &veb->tc_stats_offsets;
791 
792 	/* Gather up the stats that the hw collects */
793 	i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
794 			   veb->stat_offsets_loaded,
795 			   &oes->tx_discards, &es->tx_discards);
796 	if (hw->revision_id > 0)
797 		i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
798 				   veb->stat_offsets_loaded,
799 				   &oes->rx_unknown_protocol,
800 				   &es->rx_unknown_protocol);
801 	i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
802 			   veb->stat_offsets_loaded,
803 			   &oes->rx_bytes, &es->rx_bytes);
804 	i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
805 			   veb->stat_offsets_loaded,
806 			   &oes->rx_unicast, &es->rx_unicast);
807 	i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
808 			   veb->stat_offsets_loaded,
809 			   &oes->rx_multicast, &es->rx_multicast);
810 	i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
811 			   veb->stat_offsets_loaded,
812 			   &oes->rx_broadcast, &es->rx_broadcast);
813 
814 	i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
815 			   veb->stat_offsets_loaded,
816 			   &oes->tx_bytes, &es->tx_bytes);
817 	i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
818 			   veb->stat_offsets_loaded,
819 			   &oes->tx_unicast, &es->tx_unicast);
820 	i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
821 			   veb->stat_offsets_loaded,
822 			   &oes->tx_multicast, &es->tx_multicast);
823 	i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
824 			   veb->stat_offsets_loaded,
825 			   &oes->tx_broadcast, &es->tx_broadcast);
826 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
827 		i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
828 				   I40E_GLVEBTC_RPCL(i, idx),
829 				   veb->stat_offsets_loaded,
830 				   &veb_oes->tc_rx_packets[i],
831 				   &veb_es->tc_rx_packets[i]);
832 		i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
833 				   I40E_GLVEBTC_RBCL(i, idx),
834 				   veb->stat_offsets_loaded,
835 				   &veb_oes->tc_rx_bytes[i],
836 				   &veb_es->tc_rx_bytes[i]);
837 		i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
838 				   I40E_GLVEBTC_TPCL(i, idx),
839 				   veb->stat_offsets_loaded,
840 				   &veb_oes->tc_tx_packets[i],
841 				   &veb_es->tc_tx_packets[i]);
842 		i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
843 				   I40E_GLVEBTC_TBCL(i, idx),
844 				   veb->stat_offsets_loaded,
845 				   &veb_oes->tc_tx_bytes[i],
846 				   &veb_es->tc_tx_bytes[i]);
847 	}
848 	veb->stat_offsets_loaded = true;
849 }
850 
851 /**
852  * i40e_update_vsi_stats - Update the vsi statistics counters.
853  * @vsi: the VSI to be updated
854  *
855  * There are a few instances where we store the same stat in a
856  * couple of different structs.  This is partly because we have
857  * the netdev stats that need to be filled out, which is slightly
858  * different from the "eth_stats" defined by the chip and used in
859  * VF communications.  We sort it out here.
860  **/
i40e_update_vsi_stats(struct i40e_vsi * vsi)861 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
862 {
863 	u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy;
864 	struct i40e_pf *pf = vsi->back;
865 	struct rtnl_link_stats64 *ons;
866 	struct rtnl_link_stats64 *ns;   /* netdev stats */
867 	struct i40e_eth_stats *oes;
868 	struct i40e_eth_stats *es;     /* device's eth stats */
869 	u64 tx_restart, tx_busy;
870 	struct i40e_ring *p;
871 	u64 bytes, packets;
872 	unsigned int start;
873 	u64 tx_linearize;
874 	u64 tx_force_wb;
875 	u64 tx_stopped;
876 	u64 rx_p, rx_b;
877 	u64 tx_p, tx_b;
878 	u16 q;
879 
880 	if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
881 	    test_bit(__I40E_CONFIG_BUSY, pf->state))
882 		return;
883 
884 	ns = i40e_get_vsi_stats_struct(vsi);
885 	ons = &vsi->net_stats_offsets;
886 	es = &vsi->eth_stats;
887 	oes = &vsi->eth_stats_offsets;
888 
889 	/* Gather up the netdev and vsi stats that the driver collects
890 	 * on the fly during packet processing
891 	 */
892 	rx_b = rx_p = 0;
893 	tx_b = tx_p = 0;
894 	tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
895 	tx_stopped = 0;
896 	rx_page = 0;
897 	rx_buf = 0;
898 	rx_reuse = 0;
899 	rx_alloc = 0;
900 	rx_waive = 0;
901 	rx_busy = 0;
902 	rcu_read_lock();
903 	for (q = 0; q < vsi->num_queue_pairs; q++) {
904 		/* locate Tx ring */
905 		p = READ_ONCE(vsi->tx_rings[q]);
906 		if (!p)
907 			continue;
908 
909 		do {
910 			start = u64_stats_fetch_begin(&p->syncp);
911 			packets = p->stats.packets;
912 			bytes = p->stats.bytes;
913 		} while (u64_stats_fetch_retry(&p->syncp, start));
914 		tx_b += bytes;
915 		tx_p += packets;
916 		tx_restart += p->tx_stats.restart_queue;
917 		tx_busy += p->tx_stats.tx_busy;
918 		tx_linearize += p->tx_stats.tx_linearize;
919 		tx_force_wb += p->tx_stats.tx_force_wb;
920 		tx_stopped += p->tx_stats.tx_stopped;
921 
922 		/* locate Rx ring */
923 		p = READ_ONCE(vsi->rx_rings[q]);
924 		if (!p)
925 			continue;
926 
927 		do {
928 			start = u64_stats_fetch_begin(&p->syncp);
929 			packets = p->stats.packets;
930 			bytes = p->stats.bytes;
931 		} while (u64_stats_fetch_retry(&p->syncp, start));
932 		rx_b += bytes;
933 		rx_p += packets;
934 		rx_buf += p->rx_stats.alloc_buff_failed;
935 		rx_page += p->rx_stats.alloc_page_failed;
936 		rx_reuse += p->rx_stats.page_reuse_count;
937 		rx_alloc += p->rx_stats.page_alloc_count;
938 		rx_waive += p->rx_stats.page_waive_count;
939 		rx_busy += p->rx_stats.page_busy_count;
940 
941 		if (i40e_enabled_xdp_vsi(vsi)) {
942 			/* locate XDP ring */
943 			p = READ_ONCE(vsi->xdp_rings[q]);
944 			if (!p)
945 				continue;
946 
947 			do {
948 				start = u64_stats_fetch_begin(&p->syncp);
949 				packets = p->stats.packets;
950 				bytes = p->stats.bytes;
951 			} while (u64_stats_fetch_retry(&p->syncp, start));
952 			tx_b += bytes;
953 			tx_p += packets;
954 			tx_restart += p->tx_stats.restart_queue;
955 			tx_busy += p->tx_stats.tx_busy;
956 			tx_linearize += p->tx_stats.tx_linearize;
957 			tx_force_wb += p->tx_stats.tx_force_wb;
958 		}
959 	}
960 	rcu_read_unlock();
961 	vsi->tx_restart = tx_restart;
962 	vsi->tx_busy = tx_busy;
963 	vsi->tx_linearize = tx_linearize;
964 	vsi->tx_force_wb = tx_force_wb;
965 	vsi->tx_stopped = tx_stopped;
966 	vsi->rx_page_failed = rx_page;
967 	vsi->rx_buf_failed = rx_buf;
968 	vsi->rx_page_reuse = rx_reuse;
969 	vsi->rx_page_alloc = rx_alloc;
970 	vsi->rx_page_waive = rx_waive;
971 	vsi->rx_page_busy = rx_busy;
972 
973 	ns->rx_packets = rx_p;
974 	ns->rx_bytes = rx_b;
975 	ns->tx_packets = tx_p;
976 	ns->tx_bytes = tx_b;
977 
978 	/* update netdev stats from eth stats */
979 	i40e_update_eth_stats(vsi);
980 	ons->tx_errors = oes->tx_errors;
981 	ns->tx_errors = es->tx_errors;
982 	ons->multicast = oes->rx_multicast;
983 	ns->multicast = es->rx_multicast;
984 	ons->rx_dropped = oes->rx_discards_other;
985 	ns->rx_dropped = es->rx_discards_other;
986 	ons->rx_missed_errors = oes->rx_discards;
987 	ns->rx_missed_errors = es->rx_discards;
988 	ons->tx_dropped = oes->tx_discards;
989 	ns->tx_dropped = es->tx_discards;
990 
991 	/* pull in a couple PF stats if this is the main vsi */
992 	if (vsi->type == I40E_VSI_MAIN) {
993 		ns->rx_crc_errors = pf->stats.crc_errors;
994 		ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
995 		ns->rx_length_errors = pf->stats.rx_length_errors;
996 	}
997 }
998 
999 /**
1000  * i40e_update_pf_stats - Update the PF statistics counters.
1001  * @pf: the PF to be updated
1002  **/
i40e_update_pf_stats(struct i40e_pf * pf)1003 static void i40e_update_pf_stats(struct i40e_pf *pf)
1004 {
1005 	struct i40e_hw_port_stats *osd = &pf->stats_offsets;
1006 	struct i40e_hw_port_stats *nsd = &pf->stats;
1007 	struct i40e_hw *hw = &pf->hw;
1008 	u32 val;
1009 	int i;
1010 
1011 	i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
1012 			   I40E_GLPRT_GORCL(hw->port),
1013 			   pf->stat_offsets_loaded,
1014 			   &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
1015 	i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
1016 			   I40E_GLPRT_GOTCL(hw->port),
1017 			   pf->stat_offsets_loaded,
1018 			   &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
1019 	i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
1020 			   pf->stat_offsets_loaded,
1021 			   &osd->eth.rx_discards,
1022 			   &nsd->eth.rx_discards);
1023 	i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
1024 			   I40E_GLPRT_UPRCL(hw->port),
1025 			   pf->stat_offsets_loaded,
1026 			   &osd->eth.rx_unicast,
1027 			   &nsd->eth.rx_unicast);
1028 	i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
1029 			   I40E_GLPRT_MPRCL(hw->port),
1030 			   pf->stat_offsets_loaded,
1031 			   &osd->eth.rx_multicast,
1032 			   &nsd->eth.rx_multicast);
1033 	i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
1034 			   I40E_GLPRT_BPRCL(hw->port),
1035 			   pf->stat_offsets_loaded,
1036 			   &osd->eth.rx_broadcast,
1037 			   &nsd->eth.rx_broadcast);
1038 	i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
1039 			   I40E_GLPRT_UPTCL(hw->port),
1040 			   pf->stat_offsets_loaded,
1041 			   &osd->eth.tx_unicast,
1042 			   &nsd->eth.tx_unicast);
1043 	i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
1044 			   I40E_GLPRT_MPTCL(hw->port),
1045 			   pf->stat_offsets_loaded,
1046 			   &osd->eth.tx_multicast,
1047 			   &nsd->eth.tx_multicast);
1048 	i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
1049 			   I40E_GLPRT_BPTCL(hw->port),
1050 			   pf->stat_offsets_loaded,
1051 			   &osd->eth.tx_broadcast,
1052 			   &nsd->eth.tx_broadcast);
1053 
1054 	i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
1055 			   pf->stat_offsets_loaded,
1056 			   &osd->tx_dropped_link_down,
1057 			   &nsd->tx_dropped_link_down);
1058 
1059 	i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
1060 			   pf->stat_offsets_loaded,
1061 			   &osd->crc_errors, &nsd->crc_errors);
1062 
1063 	i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
1064 			   pf->stat_offsets_loaded,
1065 			   &osd->illegal_bytes, &nsd->illegal_bytes);
1066 
1067 	i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
1068 			   pf->stat_offsets_loaded,
1069 			   &osd->mac_local_faults,
1070 			   &nsd->mac_local_faults);
1071 	i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
1072 			   pf->stat_offsets_loaded,
1073 			   &osd->mac_remote_faults,
1074 			   &nsd->mac_remote_faults);
1075 
1076 	i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
1077 			   pf->stat_offsets_loaded,
1078 			   &osd->rx_length_errors,
1079 			   &nsd->rx_length_errors);
1080 
1081 	i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
1082 			   pf->stat_offsets_loaded,
1083 			   &osd->link_xon_rx, &nsd->link_xon_rx);
1084 	i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
1085 			   pf->stat_offsets_loaded,
1086 			   &osd->link_xon_tx, &nsd->link_xon_tx);
1087 	i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1088 			   pf->stat_offsets_loaded,
1089 			   &osd->link_xoff_rx, &nsd->link_xoff_rx);
1090 	i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1091 			   pf->stat_offsets_loaded,
1092 			   &osd->link_xoff_tx, &nsd->link_xoff_tx);
1093 
1094 	for (i = 0; i < 8; i++) {
1095 		i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1096 				   pf->stat_offsets_loaded,
1097 				   &osd->priority_xoff_rx[i],
1098 				   &nsd->priority_xoff_rx[i]);
1099 		i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1100 				   pf->stat_offsets_loaded,
1101 				   &osd->priority_xon_rx[i],
1102 				   &nsd->priority_xon_rx[i]);
1103 		i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1104 				   pf->stat_offsets_loaded,
1105 				   &osd->priority_xon_tx[i],
1106 				   &nsd->priority_xon_tx[i]);
1107 		i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1108 				   pf->stat_offsets_loaded,
1109 				   &osd->priority_xoff_tx[i],
1110 				   &nsd->priority_xoff_tx[i]);
1111 		i40e_stat_update32(hw,
1112 				   I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1113 				   pf->stat_offsets_loaded,
1114 				   &osd->priority_xon_2_xoff[i],
1115 				   &nsd->priority_xon_2_xoff[i]);
1116 	}
1117 
1118 	i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1119 			   I40E_GLPRT_PRC64L(hw->port),
1120 			   pf->stat_offsets_loaded,
1121 			   &osd->rx_size_64, &nsd->rx_size_64);
1122 	i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1123 			   I40E_GLPRT_PRC127L(hw->port),
1124 			   pf->stat_offsets_loaded,
1125 			   &osd->rx_size_127, &nsd->rx_size_127);
1126 	i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1127 			   I40E_GLPRT_PRC255L(hw->port),
1128 			   pf->stat_offsets_loaded,
1129 			   &osd->rx_size_255, &nsd->rx_size_255);
1130 	i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1131 			   I40E_GLPRT_PRC511L(hw->port),
1132 			   pf->stat_offsets_loaded,
1133 			   &osd->rx_size_511, &nsd->rx_size_511);
1134 	i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1135 			   I40E_GLPRT_PRC1023L(hw->port),
1136 			   pf->stat_offsets_loaded,
1137 			   &osd->rx_size_1023, &nsd->rx_size_1023);
1138 	i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1139 			   I40E_GLPRT_PRC1522L(hw->port),
1140 			   pf->stat_offsets_loaded,
1141 			   &osd->rx_size_1522, &nsd->rx_size_1522);
1142 	i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1143 			   I40E_GLPRT_PRC9522L(hw->port),
1144 			   pf->stat_offsets_loaded,
1145 			   &osd->rx_size_big, &nsd->rx_size_big);
1146 
1147 	i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1148 			   I40E_GLPRT_PTC64L(hw->port),
1149 			   pf->stat_offsets_loaded,
1150 			   &osd->tx_size_64, &nsd->tx_size_64);
1151 	i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1152 			   I40E_GLPRT_PTC127L(hw->port),
1153 			   pf->stat_offsets_loaded,
1154 			   &osd->tx_size_127, &nsd->tx_size_127);
1155 	i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1156 			   I40E_GLPRT_PTC255L(hw->port),
1157 			   pf->stat_offsets_loaded,
1158 			   &osd->tx_size_255, &nsd->tx_size_255);
1159 	i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1160 			   I40E_GLPRT_PTC511L(hw->port),
1161 			   pf->stat_offsets_loaded,
1162 			   &osd->tx_size_511, &nsd->tx_size_511);
1163 	i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1164 			   I40E_GLPRT_PTC1023L(hw->port),
1165 			   pf->stat_offsets_loaded,
1166 			   &osd->tx_size_1023, &nsd->tx_size_1023);
1167 	i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1168 			   I40E_GLPRT_PTC1522L(hw->port),
1169 			   pf->stat_offsets_loaded,
1170 			   &osd->tx_size_1522, &nsd->tx_size_1522);
1171 	i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1172 			   I40E_GLPRT_PTC9522L(hw->port),
1173 			   pf->stat_offsets_loaded,
1174 			   &osd->tx_size_big, &nsd->tx_size_big);
1175 
1176 	i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1177 			   pf->stat_offsets_loaded,
1178 			   &osd->rx_undersize, &nsd->rx_undersize);
1179 	i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1180 			   pf->stat_offsets_loaded,
1181 			   &osd->rx_fragments, &nsd->rx_fragments);
1182 	i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1183 			   pf->stat_offsets_loaded,
1184 			   &osd->rx_oversize, &nsd->rx_oversize);
1185 	i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1186 			   pf->stat_offsets_loaded,
1187 			   &osd->rx_jabber, &nsd->rx_jabber);
1188 
1189 	/* FDIR stats */
1190 	i40e_stat_update_and_clear32(hw,
1191 			I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1192 			&nsd->fd_atr_match);
1193 	i40e_stat_update_and_clear32(hw,
1194 			I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1195 			&nsd->fd_sb_match);
1196 	i40e_stat_update_and_clear32(hw,
1197 			I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1198 			&nsd->fd_atr_tunnel_match);
1199 
1200 	val = rd32(hw, I40E_PRTPM_EEE_STAT);
1201 	nsd->tx_lpi_status =
1202 		       FIELD_GET(I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK, val);
1203 	nsd->rx_lpi_status =
1204 		       FIELD_GET(I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK, val);
1205 	i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1206 			   pf->stat_offsets_loaded,
1207 			   &osd->tx_lpi_count, &nsd->tx_lpi_count);
1208 	i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1209 			   pf->stat_offsets_loaded,
1210 			   &osd->rx_lpi_count, &nsd->rx_lpi_count);
1211 
1212 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
1213 	    !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1214 		nsd->fd_sb_status = true;
1215 	else
1216 		nsd->fd_sb_status = false;
1217 
1218 	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
1219 	    !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1220 		nsd->fd_atr_status = true;
1221 	else
1222 		nsd->fd_atr_status = false;
1223 
1224 	pf->stat_offsets_loaded = true;
1225 }
1226 
1227 /**
1228  * i40e_update_stats - Update the various statistics counters.
1229  * @vsi: the VSI to be updated
1230  *
1231  * Update the various stats for this VSI and its related entities.
1232  **/
i40e_update_stats(struct i40e_vsi * vsi)1233 void i40e_update_stats(struct i40e_vsi *vsi)
1234 {
1235 	struct i40e_pf *pf = vsi->back;
1236 
1237 	if (vsi->type == I40E_VSI_MAIN)
1238 		i40e_update_pf_stats(pf);
1239 
1240 	i40e_update_vsi_stats(vsi);
1241 }
1242 
1243 /**
1244  * i40e_count_filters - counts VSI mac filters
1245  * @vsi: the VSI to be searched
1246  *
1247  * Returns count of mac filters
1248  **/
i40e_count_filters(struct i40e_vsi * vsi)1249 int i40e_count_filters(struct i40e_vsi *vsi)
1250 {
1251 	struct i40e_mac_filter *f;
1252 	struct hlist_node *h;
1253 	int bkt;
1254 	int cnt = 0;
1255 
1256 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1257 		if (f->state == I40E_FILTER_NEW ||
1258 		    f->state == I40E_FILTER_NEW_SYNC ||
1259 		    f->state == I40E_FILTER_ACTIVE)
1260 			++cnt;
1261 	}
1262 
1263 	return cnt;
1264 }
1265 
1266 /**
1267  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1268  * @vsi: the VSI to be searched
1269  * @macaddr: the MAC address
1270  * @vlan: the vlan
1271  *
1272  * Returns ptr to the filter object or NULL
1273  **/
i40e_find_filter(struct i40e_vsi * vsi,const u8 * macaddr,s16 vlan)1274 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1275 						const u8 *macaddr, s16 vlan)
1276 {
1277 	struct i40e_mac_filter *f;
1278 	u64 key;
1279 
1280 	if (!vsi || !macaddr)
1281 		return NULL;
1282 
1283 	key = i40e_addr_to_hkey(macaddr);
1284 	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1285 		if ((ether_addr_equal(macaddr, f->macaddr)) &&
1286 		    (vlan == f->vlan))
1287 			return f;
1288 	}
1289 	return NULL;
1290 }
1291 
1292 /**
1293  * i40e_find_mac - Find a mac addr in the macvlan filters list
1294  * @vsi: the VSI to be searched
1295  * @macaddr: the MAC address we are searching for
1296  *
1297  * Returns the first filter with the provided MAC address or NULL if
1298  * MAC address was not found
1299  **/
i40e_find_mac(struct i40e_vsi * vsi,const u8 * macaddr)1300 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1301 {
1302 	struct i40e_mac_filter *f;
1303 	u64 key;
1304 
1305 	if (!vsi || !macaddr)
1306 		return NULL;
1307 
1308 	key = i40e_addr_to_hkey(macaddr);
1309 	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1310 		if ((ether_addr_equal(macaddr, f->macaddr)))
1311 			return f;
1312 	}
1313 	return NULL;
1314 }
1315 
1316 /**
1317  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1318  * @vsi: the VSI to be searched
1319  *
1320  * Returns true if VSI is in vlan mode or false otherwise
1321  **/
i40e_is_vsi_in_vlan(struct i40e_vsi * vsi)1322 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1323 {
1324 	/* If we have a PVID, always operate in VLAN mode */
1325 	if (vsi->info.pvid)
1326 		return true;
1327 
1328 	/* We need to operate in VLAN mode whenever we have any filters with
1329 	 * a VLAN other than I40E_VLAN_ALL. We could check the table each
1330 	 * time, incurring search cost repeatedly. However, we can notice two
1331 	 * things:
1332 	 *
1333 	 * 1) the only place where we can gain a VLAN filter is in
1334 	 *    i40e_add_filter.
1335 	 *
1336 	 * 2) the only place where filters are actually removed is in
1337 	 *    i40e_sync_filters_subtask.
1338 	 *
1339 	 * Thus, we can simply use a boolean value, has_vlan_filters which we
1340 	 * will set to true when we add a VLAN filter in i40e_add_filter. Then
1341 	 * we have to perform the full search after deleting filters in
1342 	 * i40e_sync_filters_subtask, but we already have to search
1343 	 * filters here and can perform the check at the same time. This
1344 	 * results in avoiding embedding a loop for VLAN mode inside another
1345 	 * loop over all the filters, and should maintain correctness as noted
1346 	 * above.
1347 	 */
1348 	return vsi->has_vlan_filter;
1349 }
1350 
1351 /**
1352  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1353  * @vsi: the VSI to configure
1354  * @tmp_add_list: list of filters ready to be added
1355  * @tmp_del_list: list of filters ready to be deleted
1356  * @vlan_filters: the number of active VLAN filters
1357  *
1358  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1359  * behave as expected. If we have any active VLAN filters remaining or about
1360  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1361  * so that they only match against untagged traffic. If we no longer have any
1362  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1363  * so that they match against both tagged and untagged traffic. In this way,
1364  * we ensure that we correctly receive the desired traffic. This ensures that
1365  * when we have an active VLAN we will receive only untagged traffic and
1366  * traffic matching active VLANs. If we have no active VLANs then we will
1367  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1368  *
1369  * Finally, in a similar fashion, this function also corrects filters when
1370  * there is an active PVID assigned to this VSI.
1371  *
1372  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1373  *
1374  * This function is only expected to be called from within
1375  * i40e_sync_vsi_filters.
1376  *
1377  * NOTE: This function expects to be called while under the
1378  * mac_filter_hash_lock
1379  */
i40e_correct_mac_vlan_filters(struct i40e_vsi * vsi,struct hlist_head * tmp_add_list,struct hlist_head * tmp_del_list,int vlan_filters)1380 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1381 					 struct hlist_head *tmp_add_list,
1382 					 struct hlist_head *tmp_del_list,
1383 					 int vlan_filters)
1384 {
1385 	s16 pvid = le16_to_cpu(vsi->info.pvid);
1386 	struct i40e_mac_filter *f, *add_head;
1387 	struct i40e_new_mac_filter *new;
1388 	struct hlist_node *h;
1389 	int bkt, new_vlan;
1390 
1391 	/* To determine if a particular filter needs to be replaced we
1392 	 * have the three following conditions:
1393 	 *
1394 	 * a) if we have a PVID assigned, then all filters which are
1395 	 *    not marked as VLAN=PVID must be replaced with filters that
1396 	 *    are.
1397 	 * b) otherwise, if we have any active VLANS, all filters
1398 	 *    which are marked as VLAN=-1 must be replaced with
1399 	 *    filters marked as VLAN=0
1400 	 * c) finally, if we do not have any active VLANS, all filters
1401 	 *    which are marked as VLAN=0 must be replaced with filters
1402 	 *    marked as VLAN=-1
1403 	 */
1404 
1405 	/* Update the filters about to be added in place */
1406 	hlist_for_each_entry(new, tmp_add_list, hlist) {
1407 		if (pvid && new->f->vlan != pvid)
1408 			new->f->vlan = pvid;
1409 		else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1410 			new->f->vlan = 0;
1411 		else if (!vlan_filters && new->f->vlan == 0)
1412 			new->f->vlan = I40E_VLAN_ANY;
1413 	}
1414 
1415 	/* Update the remaining active filters */
1416 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1417 		/* Combine the checks for whether a filter needs to be changed
1418 		 * and then determine the new VLAN inside the if block, in
1419 		 * order to avoid duplicating code for adding the new filter
1420 		 * then deleting the old filter.
1421 		 */
1422 		if ((pvid && f->vlan != pvid) ||
1423 		    (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1424 		    (!vlan_filters && f->vlan == 0)) {
1425 			/* Determine the new vlan we will be adding */
1426 			if (pvid)
1427 				new_vlan = pvid;
1428 			else if (vlan_filters)
1429 				new_vlan = 0;
1430 			else
1431 				new_vlan = I40E_VLAN_ANY;
1432 
1433 			/* Create the new filter */
1434 			add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1435 			if (!add_head)
1436 				return -ENOMEM;
1437 
1438 			/* Create a temporary i40e_new_mac_filter */
1439 			new = kzalloc(sizeof(*new), GFP_ATOMIC);
1440 			if (!new)
1441 				return -ENOMEM;
1442 
1443 			new->f = add_head;
1444 			new->state = add_head->state;
1445 			if (add_head->state == I40E_FILTER_NEW)
1446 				add_head->state = I40E_FILTER_NEW_SYNC;
1447 
1448 			/* Add the new filter to the tmp list */
1449 			hlist_add_head(&new->hlist, tmp_add_list);
1450 
1451 			/* Put the original filter into the delete list */
1452 			f->state = I40E_FILTER_REMOVE;
1453 			hash_del(&f->hlist);
1454 			hlist_add_head(&f->hlist, tmp_del_list);
1455 		}
1456 	}
1457 
1458 	vsi->has_vlan_filter = !!vlan_filters;
1459 
1460 	return 0;
1461 }
1462 
1463 /**
1464  * i40e_get_vf_new_vlan - Get new vlan id on a vf
1465  * @vsi: the vsi to configure
1466  * @new_mac: new mac filter to be added
1467  * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL
1468  * @vlan_filters: the number of active VLAN filters
1469  * @trusted: flag if the VF is trusted
1470  *
1471  * Get new VLAN id based on current VLAN filters, trust, PVID
1472  * and vf-vlan-prune-disable flag.
1473  *
1474  * Returns the value of the new vlan filter or
1475  * the old value if no new filter is needed.
1476  */
i40e_get_vf_new_vlan(struct i40e_vsi * vsi,struct i40e_new_mac_filter * new_mac,struct i40e_mac_filter * f,int vlan_filters,bool trusted)1477 static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi,
1478 				struct i40e_new_mac_filter *new_mac,
1479 				struct i40e_mac_filter *f,
1480 				int vlan_filters,
1481 				bool trusted)
1482 {
1483 	s16 pvid = le16_to_cpu(vsi->info.pvid);
1484 	struct i40e_pf *pf = vsi->back;
1485 	bool is_any;
1486 
1487 	if (new_mac)
1488 		f = new_mac->f;
1489 
1490 	if (pvid && f->vlan != pvid)
1491 		return pvid;
1492 
1493 	is_any = (trusted ||
1494 		  !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags));
1495 
1496 	if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1497 	    (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1498 	    (is_any && !vlan_filters && f->vlan == 0)) {
1499 		if (is_any)
1500 			return I40E_VLAN_ANY;
1501 		else
1502 			return 0;
1503 	}
1504 
1505 	return f->vlan;
1506 }
1507 
1508 /**
1509  * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary
1510  * @vsi: the vsi to configure
1511  * @tmp_add_list: list of filters ready to be added
1512  * @tmp_del_list: list of filters ready to be deleted
1513  * @vlan_filters: the number of active VLAN filters
1514  * @trusted: flag if the VF is trusted
1515  *
1516  * Correct VF VLAN filters based on current VLAN filters, trust, PVID
1517  * and vf-vlan-prune-disable flag.
1518  *
1519  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1520  *
1521  * This function is only expected to be called from within
1522  * i40e_sync_vsi_filters.
1523  *
1524  * NOTE: This function expects to be called while under the
1525  * mac_filter_hash_lock
1526  */
i40e_correct_vf_mac_vlan_filters(struct i40e_vsi * vsi,struct hlist_head * tmp_add_list,struct hlist_head * tmp_del_list,int vlan_filters,bool trusted)1527 static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi,
1528 					    struct hlist_head *tmp_add_list,
1529 					    struct hlist_head *tmp_del_list,
1530 					    int vlan_filters,
1531 					    bool trusted)
1532 {
1533 	struct i40e_mac_filter *f, *add_head;
1534 	struct i40e_new_mac_filter *new_mac;
1535 	struct hlist_node *h;
1536 	int bkt, new_vlan;
1537 
1538 	hlist_for_each_entry(new_mac, tmp_add_list, hlist) {
1539 		new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL,
1540 							vlan_filters, trusted);
1541 	}
1542 
1543 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1544 		new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters,
1545 						trusted);
1546 		if (new_vlan != f->vlan) {
1547 			add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1548 			if (!add_head)
1549 				return -ENOMEM;
1550 			/* Create a temporary i40e_new_mac_filter */
1551 			new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC);
1552 			if (!new_mac)
1553 				return -ENOMEM;
1554 			new_mac->f = add_head;
1555 			new_mac->state = add_head->state;
1556 			if (add_head->state == I40E_FILTER_NEW)
1557 				add_head->state = I40E_FILTER_NEW_SYNC;
1558 
1559 			/* Add the new filter to the tmp list */
1560 			hlist_add_head(&new_mac->hlist, tmp_add_list);
1561 
1562 			/* Put the original filter into the delete list */
1563 			f->state = I40E_FILTER_REMOVE;
1564 			hash_del(&f->hlist);
1565 			hlist_add_head(&f->hlist, tmp_del_list);
1566 		}
1567 	}
1568 
1569 	vsi->has_vlan_filter = !!vlan_filters;
1570 	return 0;
1571 }
1572 
1573 /**
1574  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1575  * @vsi: the PF Main VSI - inappropriate for any other VSI
1576  * @macaddr: the MAC address
1577  *
1578  * Remove whatever filter the firmware set up so the driver can manage
1579  * its own filtering intelligently.
1580  **/
i40e_rm_default_mac_filter(struct i40e_vsi * vsi,u8 * macaddr)1581 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1582 {
1583 	struct i40e_aqc_remove_macvlan_element_data element;
1584 	struct i40e_pf *pf = vsi->back;
1585 
1586 	/* Only appropriate for the PF main VSI */
1587 	if (vsi->type != I40E_VSI_MAIN)
1588 		return;
1589 
1590 	memset(&element, 0, sizeof(element));
1591 	ether_addr_copy(element.mac_addr, macaddr);
1592 	element.vlan_tag = 0;
1593 	/* Ignore error returns, some firmware does it this way... */
1594 	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1595 	i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1596 
1597 	memset(&element, 0, sizeof(element));
1598 	ether_addr_copy(element.mac_addr, macaddr);
1599 	element.vlan_tag = 0;
1600 	/* ...and some firmware does it this way. */
1601 	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1602 			I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1603 	i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1604 }
1605 
1606 /**
1607  * i40e_add_filter - Add a mac/vlan filter to the VSI
1608  * @vsi: the VSI to be searched
1609  * @macaddr: the MAC address
1610  * @vlan: the vlan
1611  *
1612  * Returns ptr to the filter object or NULL when no memory available.
1613  *
1614  * NOTE: This function is expected to be called with mac_filter_hash_lock
1615  * being held.
1616  **/
i40e_add_filter(struct i40e_vsi * vsi,const u8 * macaddr,s16 vlan)1617 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1618 					const u8 *macaddr, s16 vlan)
1619 {
1620 	struct i40e_mac_filter *f;
1621 	u64 key;
1622 
1623 	if (!vsi || !macaddr)
1624 		return NULL;
1625 
1626 	f = i40e_find_filter(vsi, macaddr, vlan);
1627 	if (!f) {
1628 		f = kzalloc(sizeof(*f), GFP_ATOMIC);
1629 		if (!f)
1630 			return NULL;
1631 
1632 		/* Update the boolean indicating if we need to function in
1633 		 * VLAN mode.
1634 		 */
1635 		if (vlan >= 0)
1636 			vsi->has_vlan_filter = true;
1637 
1638 		ether_addr_copy(f->macaddr, macaddr);
1639 		f->vlan = vlan;
1640 		f->state = I40E_FILTER_NEW;
1641 		INIT_HLIST_NODE(&f->hlist);
1642 
1643 		key = i40e_addr_to_hkey(macaddr);
1644 		hash_add(vsi->mac_filter_hash, &f->hlist, key);
1645 
1646 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1647 		set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1648 	}
1649 
1650 	/* If we're asked to add a filter that has been marked for removal, it
1651 	 * is safe to simply restore it to active state. __i40e_del_filter
1652 	 * will have simply deleted any filters which were previously marked
1653 	 * NEW or FAILED, so if it is currently marked REMOVE it must have
1654 	 * previously been ACTIVE. Since we haven't yet run the sync filters
1655 	 * task, just restore this filter to the ACTIVE state so that the
1656 	 * sync task leaves it in place
1657 	 */
1658 	if (f->state == I40E_FILTER_REMOVE)
1659 		f->state = I40E_FILTER_ACTIVE;
1660 
1661 	return f;
1662 }
1663 
1664 /**
1665  * __i40e_del_filter - Remove a specific filter from the VSI
1666  * @vsi: VSI to remove from
1667  * @f: the filter to remove from the list
1668  *
1669  * This function requires you've found * the exact filter you will remove
1670  * already, such as via i40e_find_filter or i40e_find_mac.
1671  *
1672  * NOTE: This function is expected to be called with mac_filter_hash_lock
1673  * being held.
1674  * ANOTHER NOTE: This function MUST be called from within the context of
1675  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1676  * instead of list_for_each_entry().
1677  **/
__i40e_del_filter(struct i40e_vsi * vsi,struct i40e_mac_filter * f)1678 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1679 {
1680 	if (!f)
1681 		return;
1682 
1683 	/* If the filter was never added to firmware then we can just delete it
1684 	 * directly and we don't want to set the status to remove or else an
1685 	 * admin queue command will unnecessarily fire.
1686 	 */
1687 	if ((f->state == I40E_FILTER_FAILED) ||
1688 	    (f->state == I40E_FILTER_NEW)) {
1689 		hash_del(&f->hlist);
1690 		kfree(f);
1691 	} else {
1692 		f->state = I40E_FILTER_REMOVE;
1693 	}
1694 
1695 	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1696 	set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1697 }
1698 
1699 /**
1700  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1701  * @vsi: the VSI to be searched
1702  * @macaddr: the mac address to be filtered
1703  *
1704  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1705  * go through all the macvlan filters and add a macvlan filter for each
1706  * unique vlan that already exists. If a PVID has been assigned, instead only
1707  * add the macaddr to that VLAN.
1708  *
1709  * Returns last filter added on success, else NULL
1710  **/
i40e_add_mac_filter(struct i40e_vsi * vsi,const u8 * macaddr)1711 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1712 					    const u8 *macaddr)
1713 {
1714 	struct i40e_mac_filter *f, *add = NULL;
1715 	struct hlist_node *h;
1716 	int bkt;
1717 
1718 	lockdep_assert_held(&vsi->mac_filter_hash_lock);
1719 	if (vsi->info.pvid)
1720 		return i40e_add_filter(vsi, macaddr,
1721 				       le16_to_cpu(vsi->info.pvid));
1722 
1723 	if (!i40e_is_vsi_in_vlan(vsi))
1724 		return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1725 
1726 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1727 		if (f->state == I40E_FILTER_REMOVE)
1728 			continue;
1729 		add = i40e_add_filter(vsi, macaddr, f->vlan);
1730 		if (!add)
1731 			return NULL;
1732 	}
1733 
1734 	return add;
1735 }
1736 
1737 /**
1738  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1739  * @vsi: the VSI to be searched
1740  * @macaddr: the mac address to be removed
1741  *
1742  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1743  * associated with.
1744  *
1745  * Returns 0 for success, or error
1746  **/
i40e_del_mac_filter(struct i40e_vsi * vsi,const u8 * macaddr)1747 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1748 {
1749 	struct i40e_mac_filter *f;
1750 	struct hlist_node *h;
1751 	bool found = false;
1752 	int bkt;
1753 
1754 	lockdep_assert_held(&vsi->mac_filter_hash_lock);
1755 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1756 		if (ether_addr_equal(macaddr, f->macaddr)) {
1757 			__i40e_del_filter(vsi, f);
1758 			found = true;
1759 		}
1760 	}
1761 
1762 	if (found)
1763 		return 0;
1764 	else
1765 		return -ENOENT;
1766 }
1767 
1768 /**
1769  * i40e_set_mac - NDO callback to set mac address
1770  * @netdev: network interface device structure
1771  * @p: pointer to an address structure
1772  *
1773  * Returns 0 on success, negative on failure
1774  **/
i40e_set_mac(struct net_device * netdev,void * p)1775 static int i40e_set_mac(struct net_device *netdev, void *p)
1776 {
1777 	struct i40e_netdev_priv *np = netdev_priv(netdev);
1778 	struct i40e_vsi *vsi = np->vsi;
1779 	struct i40e_pf *pf = vsi->back;
1780 	struct i40e_hw *hw = &pf->hw;
1781 	struct sockaddr *addr = p;
1782 
1783 	if (!is_valid_ether_addr(addr->sa_data))
1784 		return -EADDRNOTAVAIL;
1785 
1786 	if (test_bit(__I40E_DOWN, pf->state) ||
1787 	    test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1788 		return -EADDRNOTAVAIL;
1789 
1790 	if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1791 		netdev_info(netdev, "returning to hw mac address %pM\n",
1792 			    hw->mac.addr);
1793 	else
1794 		netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1795 
1796 	/* Copy the address first, so that we avoid a possible race with
1797 	 * .set_rx_mode().
1798 	 * - Remove old address from MAC filter
1799 	 * - Copy new address
1800 	 * - Add new address to MAC filter
1801 	 */
1802 	spin_lock_bh(&vsi->mac_filter_hash_lock);
1803 	i40e_del_mac_filter(vsi, netdev->dev_addr);
1804 	eth_hw_addr_set(netdev, addr->sa_data);
1805 	i40e_add_mac_filter(vsi, netdev->dev_addr);
1806 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
1807 
1808 	if (vsi->type == I40E_VSI_MAIN) {
1809 		int ret;
1810 
1811 		ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1812 						addr->sa_data, NULL);
1813 		if (ret)
1814 			netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n",
1815 				    ERR_PTR(ret),
1816 				    i40e_aq_str(hw, hw->aq.asq_last_status));
1817 	}
1818 
1819 	/* schedule our worker thread which will take care of
1820 	 * applying the new filter changes
1821 	 */
1822 	i40e_service_event_schedule(pf);
1823 	return 0;
1824 }
1825 
1826 /**
1827  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1828  * @vsi: vsi structure
1829  * @seed: RSS hash seed
1830  * @lut: pointer to lookup table of lut_size
1831  * @lut_size: size of the lookup table
1832  **/
i40e_config_rss_aq(struct i40e_vsi * vsi,const u8 * seed,u8 * lut,u16 lut_size)1833 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1834 			      u8 *lut, u16 lut_size)
1835 {
1836 	struct i40e_pf *pf = vsi->back;
1837 	struct i40e_hw *hw = &pf->hw;
1838 	int ret = 0;
1839 
1840 	if (seed) {
1841 		struct i40e_aqc_get_set_rss_key_data *seed_dw =
1842 			(struct i40e_aqc_get_set_rss_key_data *)seed;
1843 		ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1844 		if (ret) {
1845 			dev_info(&pf->pdev->dev,
1846 				 "Cannot set RSS key, err %pe aq_err %s\n",
1847 				 ERR_PTR(ret),
1848 				 i40e_aq_str(hw, hw->aq.asq_last_status));
1849 			return ret;
1850 		}
1851 	}
1852 	if (lut) {
1853 		bool pf_lut = vsi->type == I40E_VSI_MAIN;
1854 
1855 		ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1856 		if (ret) {
1857 			dev_info(&pf->pdev->dev,
1858 				 "Cannot set RSS lut, err %pe aq_err %s\n",
1859 				 ERR_PTR(ret),
1860 				 i40e_aq_str(hw, hw->aq.asq_last_status));
1861 			return ret;
1862 		}
1863 	}
1864 	return ret;
1865 }
1866 
1867 /**
1868  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1869  * @vsi: VSI structure
1870  **/
i40e_vsi_config_rss(struct i40e_vsi * vsi)1871 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1872 {
1873 	struct i40e_pf *pf = vsi->back;
1874 	u8 seed[I40E_HKEY_ARRAY_SIZE];
1875 	u8 *lut;
1876 	int ret;
1877 
1878 	if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
1879 		return 0;
1880 	if (!vsi->rss_size)
1881 		vsi->rss_size = min_t(int, pf->alloc_rss_size,
1882 				      vsi->num_queue_pairs);
1883 	if (!vsi->rss_size)
1884 		return -EINVAL;
1885 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1886 	if (!lut)
1887 		return -ENOMEM;
1888 
1889 	/* Use the user configured hash keys and lookup table if there is one,
1890 	 * otherwise use default
1891 	 */
1892 	if (vsi->rss_lut_user)
1893 		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1894 	else
1895 		i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1896 	if (vsi->rss_hkey_user)
1897 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1898 	else
1899 		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1900 	ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1901 	kfree(lut);
1902 	return ret;
1903 }
1904 
1905 /**
1906  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1907  * @vsi: the VSI being configured,
1908  * @ctxt: VSI context structure
1909  * @enabled_tc: number of traffic classes to enable
1910  *
1911  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1912  **/
i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi * vsi,struct i40e_vsi_context * ctxt,u8 enabled_tc)1913 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1914 					   struct i40e_vsi_context *ctxt,
1915 					   u8 enabled_tc)
1916 {
1917 	u16 qcount = 0, max_qcount, qmap, sections = 0;
1918 	int i, override_q, pow, num_qps, ret;
1919 	u8 netdev_tc = 0, offset = 0;
1920 
1921 	if (vsi->type != I40E_VSI_MAIN)
1922 		return -EINVAL;
1923 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1924 	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1925 	vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1926 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1927 	num_qps = vsi->mqprio_qopt.qopt.count[0];
1928 
1929 	/* find the next higher power-of-2 of num queue pairs */
1930 	pow = ilog2(num_qps);
1931 	if (!is_power_of_2(num_qps))
1932 		pow++;
1933 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1934 		(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1935 
1936 	/* Setup queue offset/count for all TCs for given VSI */
1937 	max_qcount = vsi->mqprio_qopt.qopt.count[0];
1938 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1939 		/* See if the given TC is enabled for the given VSI */
1940 		if (vsi->tc_config.enabled_tc & BIT(i)) {
1941 			offset = vsi->mqprio_qopt.qopt.offset[i];
1942 			qcount = vsi->mqprio_qopt.qopt.count[i];
1943 			if (qcount > max_qcount)
1944 				max_qcount = qcount;
1945 			vsi->tc_config.tc_info[i].qoffset = offset;
1946 			vsi->tc_config.tc_info[i].qcount = qcount;
1947 			vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1948 		} else {
1949 			/* TC is not enabled so set the offset to
1950 			 * default queue and allocate one queue
1951 			 * for the given TC.
1952 			 */
1953 			vsi->tc_config.tc_info[i].qoffset = 0;
1954 			vsi->tc_config.tc_info[i].qcount = 1;
1955 			vsi->tc_config.tc_info[i].netdev_tc = 0;
1956 		}
1957 	}
1958 
1959 	/* Set actual Tx/Rx queue pairs */
1960 	vsi->num_queue_pairs = offset + qcount;
1961 
1962 	/* Setup queue TC[0].qmap for given VSI context */
1963 	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1964 	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1965 	ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1966 	ctxt->info.valid_sections |= cpu_to_le16(sections);
1967 
1968 	/* Reconfigure RSS for main VSI with max queue count */
1969 	vsi->rss_size = max_qcount;
1970 	ret = i40e_vsi_config_rss(vsi);
1971 	if (ret) {
1972 		dev_info(&vsi->back->pdev->dev,
1973 			 "Failed to reconfig rss for num_queues (%u)\n",
1974 			 max_qcount);
1975 		return ret;
1976 	}
1977 	vsi->reconfig_rss = true;
1978 	dev_dbg(&vsi->back->pdev->dev,
1979 		"Reconfigured rss with num_queues (%u)\n", max_qcount);
1980 
1981 	/* Find queue count available for channel VSIs and starting offset
1982 	 * for channel VSIs
1983 	 */
1984 	override_q = vsi->mqprio_qopt.qopt.count[0];
1985 	if (override_q && override_q < vsi->num_queue_pairs) {
1986 		vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1987 		vsi->next_base_queue = override_q;
1988 	}
1989 	return 0;
1990 }
1991 
1992 /**
1993  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1994  * @vsi: the VSI being setup
1995  * @ctxt: VSI context structure
1996  * @enabled_tc: Enabled TCs bitmap
1997  * @is_add: True if called before Add VSI
1998  *
1999  * Setup VSI queue mapping for enabled traffic classes.
2000  **/
i40e_vsi_setup_queue_map(struct i40e_vsi * vsi,struct i40e_vsi_context * ctxt,u8 enabled_tc,bool is_add)2001 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
2002 				     struct i40e_vsi_context *ctxt,
2003 				     u8 enabled_tc,
2004 				     bool is_add)
2005 {
2006 	struct i40e_pf *pf = vsi->back;
2007 	u16 num_tc_qps = 0;
2008 	u16 sections = 0;
2009 	u8 netdev_tc = 0;
2010 	u16 numtc = 1;
2011 	u16 qcount;
2012 	u8 offset;
2013 	u16 qmap;
2014 	int i;
2015 
2016 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2017 	offset = 0;
2018 	/* zero out queue mapping, it will get updated on the end of the function */
2019 	memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
2020 
2021 	if (vsi->type == I40E_VSI_MAIN) {
2022 		/* This code helps add more queue to the VSI if we have
2023 		 * more cores than RSS can support, the higher cores will
2024 		 * be served by ATR or other filters. Furthermore, the
2025 		 * non-zero req_queue_pairs says that user requested a new
2026 		 * queue count via ethtool's set_channels, so use this
2027 		 * value for queues distribution across traffic classes
2028 		 * We need at least one queue pair for the interface
2029 		 * to be usable as we see in else statement.
2030 		 */
2031 		if (vsi->req_queue_pairs > 0)
2032 			vsi->num_queue_pairs = vsi->req_queue_pairs;
2033 		else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2034 			vsi->num_queue_pairs = pf->num_lan_msix;
2035 		else
2036 			vsi->num_queue_pairs = 1;
2037 	}
2038 
2039 	/* Number of queues per enabled TC */
2040 	if (vsi->type == I40E_VSI_MAIN ||
2041 	    (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
2042 		num_tc_qps = vsi->num_queue_pairs;
2043 	else
2044 		num_tc_qps = vsi->alloc_queue_pairs;
2045 
2046 	if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
2047 		/* Find numtc from enabled TC bitmap */
2048 		for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2049 			if (enabled_tc & BIT(i)) /* TC is enabled */
2050 				numtc++;
2051 		}
2052 		if (!numtc) {
2053 			dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
2054 			numtc = 1;
2055 		}
2056 		num_tc_qps = num_tc_qps / numtc;
2057 		num_tc_qps = min_t(int, num_tc_qps,
2058 				   i40e_pf_get_max_q_per_tc(pf));
2059 	}
2060 
2061 	vsi->tc_config.numtc = numtc;
2062 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
2063 
2064 	/* Do not allow use more TC queue pairs than MSI-X vectors exist */
2065 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2066 		num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
2067 
2068 	/* Setup queue offset/count for all TCs for given VSI */
2069 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2070 		/* See if the given TC is enabled for the given VSI */
2071 		if (vsi->tc_config.enabled_tc & BIT(i)) {
2072 			/* TC is enabled */
2073 			int pow, num_qps;
2074 
2075 			switch (vsi->type) {
2076 			case I40E_VSI_MAIN:
2077 				if ((!test_bit(I40E_FLAG_FD_SB_ENA,
2078 					       pf->flags) &&
2079 				     !test_bit(I40E_FLAG_FD_ATR_ENA,
2080 					       pf->flags)) ||
2081 				    vsi->tc_config.enabled_tc != 1) {
2082 					qcount = min_t(int, pf->alloc_rss_size,
2083 						       num_tc_qps);
2084 					break;
2085 				}
2086 				fallthrough;
2087 			case I40E_VSI_FDIR:
2088 			case I40E_VSI_SRIOV:
2089 			case I40E_VSI_VMDQ2:
2090 			default:
2091 				qcount = num_tc_qps;
2092 				WARN_ON(i != 0);
2093 				break;
2094 			}
2095 			vsi->tc_config.tc_info[i].qoffset = offset;
2096 			vsi->tc_config.tc_info[i].qcount = qcount;
2097 
2098 			/* find the next higher power-of-2 of num queue pairs */
2099 			num_qps = qcount;
2100 			pow = 0;
2101 			while (num_qps && (BIT_ULL(pow) < qcount)) {
2102 				pow++;
2103 				num_qps >>= 1;
2104 			}
2105 
2106 			vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
2107 			qmap =
2108 			    (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2109 			    (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
2110 
2111 			offset += qcount;
2112 		} else {
2113 			/* TC is not enabled so set the offset to
2114 			 * default queue and allocate one queue
2115 			 * for the given TC.
2116 			 */
2117 			vsi->tc_config.tc_info[i].qoffset = 0;
2118 			vsi->tc_config.tc_info[i].qcount = 1;
2119 			vsi->tc_config.tc_info[i].netdev_tc = 0;
2120 
2121 			qmap = 0;
2122 		}
2123 		ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
2124 	}
2125 	/* Do not change previously set num_queue_pairs for PFs and VFs*/
2126 	if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
2127 	    (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
2128 	    (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
2129 		vsi->num_queue_pairs = offset;
2130 
2131 	/* Scheduler section valid can only be set for ADD VSI */
2132 	if (is_add) {
2133 		sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
2134 
2135 		ctxt->info.up_enable_bits = enabled_tc;
2136 	}
2137 	if (vsi->type == I40E_VSI_SRIOV) {
2138 		ctxt->info.mapping_flags |=
2139 				     cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2140 		for (i = 0; i < vsi->num_queue_pairs; i++)
2141 			ctxt->info.queue_mapping[i] =
2142 					       cpu_to_le16(vsi->base_queue + i);
2143 	} else {
2144 		ctxt->info.mapping_flags |=
2145 					cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2146 		ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
2147 	}
2148 	ctxt->info.valid_sections |= cpu_to_le16(sections);
2149 }
2150 
2151 /**
2152  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
2153  * @netdev: the netdevice
2154  * @addr: address to add
2155  *
2156  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2157  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2158  */
i40e_addr_sync(struct net_device * netdev,const u8 * addr)2159 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
2160 {
2161 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2162 	struct i40e_vsi *vsi = np->vsi;
2163 
2164 	if (i40e_add_mac_filter(vsi, addr))
2165 		return 0;
2166 	else
2167 		return -ENOMEM;
2168 }
2169 
2170 /**
2171  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2172  * @netdev: the netdevice
2173  * @addr: address to add
2174  *
2175  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
2176  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2177  */
i40e_addr_unsync(struct net_device * netdev,const u8 * addr)2178 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
2179 {
2180 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2181 	struct i40e_vsi *vsi = np->vsi;
2182 
2183 	/* Under some circumstances, we might receive a request to delete
2184 	 * our own device address from our uc list. Because we store the
2185 	 * device address in the VSI's MAC/VLAN filter list, we need to ignore
2186 	 * such requests and not delete our device address from this list.
2187 	 */
2188 	if (ether_addr_equal(addr, netdev->dev_addr))
2189 		return 0;
2190 
2191 	i40e_del_mac_filter(vsi, addr);
2192 
2193 	return 0;
2194 }
2195 
2196 /**
2197  * i40e_set_rx_mode - NDO callback to set the netdev filters
2198  * @netdev: network interface device structure
2199  **/
i40e_set_rx_mode(struct net_device * netdev)2200 static void i40e_set_rx_mode(struct net_device *netdev)
2201 {
2202 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2203 	struct i40e_vsi *vsi = np->vsi;
2204 
2205 	spin_lock_bh(&vsi->mac_filter_hash_lock);
2206 
2207 	__dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2208 	__dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2209 
2210 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
2211 
2212 	/* check for other flag changes */
2213 	if (vsi->current_netdev_flags != vsi->netdev->flags) {
2214 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2215 		set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
2216 	}
2217 }
2218 
2219 /**
2220  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2221  * @vsi: Pointer to VSI struct
2222  * @from: Pointer to list which contains MAC filter entries - changes to
2223  *        those entries needs to be undone.
2224  *
2225  * MAC filter entries from this list were slated for deletion.
2226  **/
i40e_undo_del_filter_entries(struct i40e_vsi * vsi,struct hlist_head * from)2227 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2228 					 struct hlist_head *from)
2229 {
2230 	struct i40e_mac_filter *f;
2231 	struct hlist_node *h;
2232 
2233 	hlist_for_each_entry_safe(f, h, from, hlist) {
2234 		u64 key = i40e_addr_to_hkey(f->macaddr);
2235 
2236 		/* Move the element back into MAC filter list*/
2237 		hlist_del(&f->hlist);
2238 		hash_add(vsi->mac_filter_hash, &f->hlist, key);
2239 	}
2240 }
2241 
2242 /**
2243  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2244  * @vsi: Pointer to vsi struct
2245  * @from: Pointer to list which contains MAC filter entries - changes to
2246  *        those entries needs to be undone.
2247  *
2248  * MAC filter entries from this list were slated for addition.
2249  **/
i40e_undo_add_filter_entries(struct i40e_vsi * vsi,struct hlist_head * from)2250 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2251 					 struct hlist_head *from)
2252 {
2253 	struct i40e_new_mac_filter *new;
2254 	struct hlist_node *h;
2255 
2256 	hlist_for_each_entry_safe(new, h, from, hlist) {
2257 		/* We can simply free the wrapper structure */
2258 		hlist_del(&new->hlist);
2259 		netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2260 		kfree(new);
2261 	}
2262 }
2263 
2264 /**
2265  * i40e_next_filter - Get the next non-broadcast filter from a list
2266  * @next: pointer to filter in list
2267  *
2268  * Returns the next non-broadcast filter in the list. Required so that we
2269  * ignore broadcast filters within the list, since these are not handled via
2270  * the normal firmware update path.
2271  */
2272 static
i40e_next_filter(struct i40e_new_mac_filter * next)2273 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2274 {
2275 	hlist_for_each_entry_continue(next, hlist) {
2276 		if (!is_broadcast_ether_addr(next->f->macaddr))
2277 			return next;
2278 	}
2279 
2280 	return NULL;
2281 }
2282 
2283 /**
2284  * i40e_update_filter_state - Update filter state based on return data
2285  * from firmware
2286  * @count: Number of filters added
2287  * @add_list: return data from fw
2288  * @add_head: pointer to first filter in current batch
2289  *
2290  * MAC filter entries from list were slated to be added to device. Returns
2291  * number of successful filters. Note that 0 does NOT mean success!
2292  **/
2293 static int
i40e_update_filter_state(int count,struct i40e_aqc_add_macvlan_element_data * add_list,struct i40e_new_mac_filter * add_head)2294 i40e_update_filter_state(int count,
2295 			 struct i40e_aqc_add_macvlan_element_data *add_list,
2296 			 struct i40e_new_mac_filter *add_head)
2297 {
2298 	int retval = 0;
2299 	int i;
2300 
2301 	for (i = 0; i < count; i++) {
2302 		/* Always check status of each filter. We don't need to check
2303 		 * the firmware return status because we pre-set the filter
2304 		 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2305 		 * request to the adminq. Thus, if it no longer matches then
2306 		 * we know the filter is active.
2307 		 */
2308 		if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2309 			add_head->state = I40E_FILTER_FAILED;
2310 		} else {
2311 			add_head->state = I40E_FILTER_ACTIVE;
2312 			retval++;
2313 		}
2314 
2315 		add_head = i40e_next_filter(add_head);
2316 		if (!add_head)
2317 			break;
2318 	}
2319 
2320 	return retval;
2321 }
2322 
2323 /**
2324  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2325  * @vsi: ptr to the VSI
2326  * @vsi_name: name to display in messages
2327  * @list: the list of filters to send to firmware
2328  * @num_del: the number of filters to delete
2329  * @retval: Set to -EIO on failure to delete
2330  *
2331  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2332  * *retval instead of a return value so that success does not force ret_val to
2333  * be set to 0. This ensures that a sequence of calls to this function
2334  * preserve the previous value of *retval on successful delete.
2335  */
2336 static
i40e_aqc_del_filters(struct i40e_vsi * vsi,const char * vsi_name,struct i40e_aqc_remove_macvlan_element_data * list,int num_del,int * retval)2337 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2338 			  struct i40e_aqc_remove_macvlan_element_data *list,
2339 			  int num_del, int *retval)
2340 {
2341 	struct i40e_hw *hw = &vsi->back->hw;
2342 	enum i40e_admin_queue_err aq_status;
2343 	int aq_ret;
2344 
2345 	aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL,
2346 					   &aq_status);
2347 
2348 	/* Explicitly ignore and do not report when firmware returns ENOENT */
2349 	if (aq_ret && !(aq_status == I40E_AQ_RC_ENOENT)) {
2350 		*retval = -EIO;
2351 		dev_info(&vsi->back->pdev->dev,
2352 			 "ignoring delete macvlan error on %s, err %pe, aq_err %s\n",
2353 			 vsi_name, ERR_PTR(aq_ret),
2354 			 i40e_aq_str(hw, aq_status));
2355 	}
2356 }
2357 
2358 /**
2359  * i40e_aqc_add_filters - Request firmware to add a set of filters
2360  * @vsi: ptr to the VSI
2361  * @vsi_name: name to display in messages
2362  * @list: the list of filters to send to firmware
2363  * @add_head: Position in the add hlist
2364  * @num_add: the number of filters to add
2365  *
2366  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2367  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2368  * space for more filters.
2369  */
2370 static
i40e_aqc_add_filters(struct i40e_vsi * vsi,const char * vsi_name,struct i40e_aqc_add_macvlan_element_data * list,struct i40e_new_mac_filter * add_head,int num_add)2371 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2372 			  struct i40e_aqc_add_macvlan_element_data *list,
2373 			  struct i40e_new_mac_filter *add_head,
2374 			  int num_add)
2375 {
2376 	struct i40e_hw *hw = &vsi->back->hw;
2377 	enum i40e_admin_queue_err aq_status;
2378 	int fcnt;
2379 
2380 	i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status);
2381 	fcnt = i40e_update_filter_state(num_add, list, add_head);
2382 
2383 	if (fcnt != num_add) {
2384 		if (vsi->type == I40E_VSI_MAIN) {
2385 			set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2386 			dev_warn(&vsi->back->pdev->dev,
2387 				 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2388 				 i40e_aq_str(hw, aq_status), vsi_name);
2389 		} else if (vsi->type == I40E_VSI_SRIOV ||
2390 			   vsi->type == I40E_VSI_VMDQ1 ||
2391 			   vsi->type == I40E_VSI_VMDQ2) {
2392 			dev_warn(&vsi->back->pdev->dev,
2393 				 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2394 				 i40e_aq_str(hw, aq_status), vsi_name,
2395 					     vsi_name);
2396 		} else {
2397 			dev_warn(&vsi->back->pdev->dev,
2398 				 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2399 				 i40e_aq_str(hw, aq_status), vsi_name,
2400 					     vsi->type);
2401 		}
2402 	}
2403 }
2404 
2405 /**
2406  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2407  * @vsi: pointer to the VSI
2408  * @vsi_name: the VSI name
2409  * @f: filter data
2410  *
2411  * This function sets or clears the promiscuous broadcast flags for VLAN
2412  * filters in order to properly receive broadcast frames. Assumes that only
2413  * broadcast filters are passed.
2414  *
2415  * Returns status indicating success or failure;
2416  **/
2417 static int
i40e_aqc_broadcast_filter(struct i40e_vsi * vsi,const char * vsi_name,struct i40e_mac_filter * f)2418 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2419 			  struct i40e_mac_filter *f)
2420 {
2421 	bool enable = f->state == I40E_FILTER_NEW ||
2422 		      f->state == I40E_FILTER_NEW_SYNC;
2423 	struct i40e_hw *hw = &vsi->back->hw;
2424 	int aq_ret;
2425 
2426 	if (f->vlan == I40E_VLAN_ANY) {
2427 		aq_ret = i40e_aq_set_vsi_broadcast(hw,
2428 						   vsi->seid,
2429 						   enable,
2430 						   NULL);
2431 	} else {
2432 		aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2433 							    vsi->seid,
2434 							    enable,
2435 							    f->vlan,
2436 							    NULL);
2437 	}
2438 
2439 	if (aq_ret) {
2440 		set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2441 		dev_warn(&vsi->back->pdev->dev,
2442 			 "Error %s, forcing overflow promiscuous on %s\n",
2443 			 i40e_aq_str(hw, hw->aq.asq_last_status),
2444 			 vsi_name);
2445 	}
2446 
2447 	return aq_ret;
2448 }
2449 
2450 /**
2451  * i40e_set_promiscuous - set promiscuous mode
2452  * @pf: board private structure
2453  * @promisc: promisc on or off
2454  *
2455  * There are different ways of setting promiscuous mode on a PF depending on
2456  * what state/environment we're in.  This identifies and sets it appropriately.
2457  * Returns 0 on success.
2458  **/
i40e_set_promiscuous(struct i40e_pf * pf,bool promisc)2459 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2460 {
2461 	struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
2462 	struct i40e_hw *hw = &pf->hw;
2463 	int aq_ret;
2464 
2465 	if (vsi->type == I40E_VSI_MAIN &&
2466 	    i40e_pf_get_main_veb(pf) &&
2467 	    !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
2468 		/* set defport ON for Main VSI instead of true promisc
2469 		 * this way we will get all unicast/multicast and VLAN
2470 		 * promisc behavior but will not get VF or VMDq traffic
2471 		 * replicated on the Main VSI.
2472 		 */
2473 		if (promisc)
2474 			aq_ret = i40e_aq_set_default_vsi(hw,
2475 							 vsi->seid,
2476 							 NULL);
2477 		else
2478 			aq_ret = i40e_aq_clear_default_vsi(hw,
2479 							   vsi->seid,
2480 							   NULL);
2481 		if (aq_ret) {
2482 			dev_info(&pf->pdev->dev,
2483 				 "Set default VSI failed, err %pe, aq_err %s\n",
2484 				 ERR_PTR(aq_ret),
2485 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2486 		}
2487 	} else {
2488 		aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2489 						  hw,
2490 						  vsi->seid,
2491 						  promisc, NULL,
2492 						  true);
2493 		if (aq_ret) {
2494 			dev_info(&pf->pdev->dev,
2495 				 "set unicast promisc failed, err %pe, aq_err %s\n",
2496 				 ERR_PTR(aq_ret),
2497 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2498 		}
2499 		aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2500 						  hw,
2501 						  vsi->seid,
2502 						  promisc, NULL);
2503 		if (aq_ret) {
2504 			dev_info(&pf->pdev->dev,
2505 				 "set multicast promisc failed, err %pe, aq_err %s\n",
2506 				 ERR_PTR(aq_ret),
2507 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2508 		}
2509 	}
2510 
2511 	if (!aq_ret)
2512 		pf->cur_promisc = promisc;
2513 
2514 	return aq_ret;
2515 }
2516 
2517 /**
2518  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2519  * @vsi: ptr to the VSI
2520  *
2521  * Push any outstanding VSI filter changes through the AdminQ.
2522  *
2523  * Returns 0 or error value
2524  **/
i40e_sync_vsi_filters(struct i40e_vsi * vsi)2525 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2526 {
2527 	struct hlist_head tmp_add_list, tmp_del_list;
2528 	struct i40e_mac_filter *f;
2529 	struct i40e_new_mac_filter *new, *add_head = NULL;
2530 	struct i40e_hw *hw = &vsi->back->hw;
2531 	bool old_overflow, new_overflow;
2532 	unsigned int failed_filters = 0;
2533 	unsigned int vlan_filters = 0;
2534 	char vsi_name[16] = "PF";
2535 	int filter_list_len = 0;
2536 	u32 changed_flags = 0;
2537 	struct hlist_node *h;
2538 	struct i40e_pf *pf;
2539 	int num_add = 0;
2540 	int num_del = 0;
2541 	int aq_ret = 0;
2542 	int retval = 0;
2543 	u16 cmd_flags;
2544 	int list_size;
2545 	int bkt;
2546 
2547 	/* empty array typed pointers, kcalloc later */
2548 	struct i40e_aqc_add_macvlan_element_data *add_list;
2549 	struct i40e_aqc_remove_macvlan_element_data *del_list;
2550 
2551 	while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2552 		usleep_range(1000, 2000);
2553 	pf = vsi->back;
2554 
2555 	old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2556 
2557 	if (vsi->netdev) {
2558 		changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2559 		vsi->current_netdev_flags = vsi->netdev->flags;
2560 	}
2561 
2562 	INIT_HLIST_HEAD(&tmp_add_list);
2563 	INIT_HLIST_HEAD(&tmp_del_list);
2564 
2565 	if (vsi->type == I40E_VSI_SRIOV)
2566 		snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2567 	else if (vsi->type != I40E_VSI_MAIN)
2568 		snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2569 
2570 	if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2571 		vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2572 
2573 		spin_lock_bh(&vsi->mac_filter_hash_lock);
2574 		/* Create a list of filters to delete. */
2575 		hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2576 			if (f->state == I40E_FILTER_REMOVE) {
2577 				/* Move the element into temporary del_list */
2578 				hash_del(&f->hlist);
2579 				hlist_add_head(&f->hlist, &tmp_del_list);
2580 
2581 				/* Avoid counting removed filters */
2582 				continue;
2583 			}
2584 			if (f->state == I40E_FILTER_NEW) {
2585 				/* Create a temporary i40e_new_mac_filter */
2586 				new = kzalloc(sizeof(*new), GFP_ATOMIC);
2587 				if (!new)
2588 					goto err_no_memory_locked;
2589 
2590 				/* Store pointer to the real filter */
2591 				new->f = f;
2592 				new->state = f->state;
2593 
2594 				/* Add it to the hash list */
2595 				hlist_add_head(&new->hlist, &tmp_add_list);
2596 				f->state = I40E_FILTER_NEW_SYNC;
2597 			}
2598 
2599 			/* Count the number of active (current and new) VLAN
2600 			 * filters we have now. Does not count filters which
2601 			 * are marked for deletion.
2602 			 */
2603 			if (f->vlan > 0)
2604 				vlan_filters++;
2605 		}
2606 
2607 		if (vsi->type != I40E_VSI_SRIOV)
2608 			retval = i40e_correct_mac_vlan_filters
2609 				(vsi, &tmp_add_list, &tmp_del_list,
2610 				 vlan_filters);
2611 		else if (pf->vf)
2612 			retval = i40e_correct_vf_mac_vlan_filters
2613 				(vsi, &tmp_add_list, &tmp_del_list,
2614 				 vlan_filters, pf->vf[vsi->vf_id].trusted);
2615 
2616 		hlist_for_each_entry(new, &tmp_add_list, hlist)
2617 			netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
2618 
2619 		if (retval)
2620 			goto err_no_memory_locked;
2621 
2622 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
2623 	}
2624 
2625 	/* Now process 'del_list' outside the lock */
2626 	if (!hlist_empty(&tmp_del_list)) {
2627 		filter_list_len = hw->aq.asq_buf_size /
2628 			    sizeof(struct i40e_aqc_remove_macvlan_element_data);
2629 		list_size = filter_list_len *
2630 			    sizeof(struct i40e_aqc_remove_macvlan_element_data);
2631 		del_list = kzalloc(list_size, GFP_ATOMIC);
2632 		if (!del_list)
2633 			goto err_no_memory;
2634 
2635 		hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2636 			cmd_flags = 0;
2637 
2638 			/* handle broadcast filters by updating the broadcast
2639 			 * promiscuous flag and release filter list.
2640 			 */
2641 			if (is_broadcast_ether_addr(f->macaddr)) {
2642 				i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2643 
2644 				hlist_del(&f->hlist);
2645 				kfree(f);
2646 				continue;
2647 			}
2648 
2649 			/* add to delete list */
2650 			ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2651 			if (f->vlan == I40E_VLAN_ANY) {
2652 				del_list[num_del].vlan_tag = 0;
2653 				cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2654 			} else {
2655 				del_list[num_del].vlan_tag =
2656 					cpu_to_le16((u16)(f->vlan));
2657 			}
2658 
2659 			cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2660 			del_list[num_del].flags = cmd_flags;
2661 			num_del++;
2662 
2663 			/* flush a full buffer */
2664 			if (num_del == filter_list_len) {
2665 				i40e_aqc_del_filters(vsi, vsi_name, del_list,
2666 						     num_del, &retval);
2667 				memset(del_list, 0, list_size);
2668 				num_del = 0;
2669 			}
2670 			/* Release memory for MAC filter entries which were
2671 			 * synced up with HW.
2672 			 */
2673 			hlist_del(&f->hlist);
2674 			kfree(f);
2675 		}
2676 
2677 		if (num_del) {
2678 			i40e_aqc_del_filters(vsi, vsi_name, del_list,
2679 					     num_del, &retval);
2680 		}
2681 
2682 		kfree(del_list);
2683 		del_list = NULL;
2684 	}
2685 
2686 	if (!hlist_empty(&tmp_add_list)) {
2687 		/* Do all the adds now. */
2688 		filter_list_len = hw->aq.asq_buf_size /
2689 			       sizeof(struct i40e_aqc_add_macvlan_element_data);
2690 		list_size = filter_list_len *
2691 			       sizeof(struct i40e_aqc_add_macvlan_element_data);
2692 		add_list = kzalloc(list_size, GFP_ATOMIC);
2693 		if (!add_list)
2694 			goto err_no_memory;
2695 
2696 		num_add = 0;
2697 		hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2698 			/* handle broadcast filters by updating the broadcast
2699 			 * promiscuous flag instead of adding a MAC filter.
2700 			 */
2701 			if (is_broadcast_ether_addr(new->f->macaddr)) {
2702 				if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2703 							      new->f))
2704 					new->state = I40E_FILTER_FAILED;
2705 				else
2706 					new->state = I40E_FILTER_ACTIVE;
2707 				continue;
2708 			}
2709 
2710 			/* add to add array */
2711 			if (num_add == 0)
2712 				add_head = new;
2713 			cmd_flags = 0;
2714 			ether_addr_copy(add_list[num_add].mac_addr,
2715 					new->f->macaddr);
2716 			if (new->f->vlan == I40E_VLAN_ANY) {
2717 				add_list[num_add].vlan_tag = 0;
2718 				cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2719 			} else {
2720 				add_list[num_add].vlan_tag =
2721 					cpu_to_le16((u16)(new->f->vlan));
2722 			}
2723 			add_list[num_add].queue_number = 0;
2724 			/* set invalid match method for later detection */
2725 			add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2726 			cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2727 			add_list[num_add].flags = cpu_to_le16(cmd_flags);
2728 			num_add++;
2729 
2730 			/* flush a full buffer */
2731 			if (num_add == filter_list_len) {
2732 				i40e_aqc_add_filters(vsi, vsi_name, add_list,
2733 						     add_head, num_add);
2734 				memset(add_list, 0, list_size);
2735 				num_add = 0;
2736 			}
2737 		}
2738 		if (num_add) {
2739 			i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2740 					     num_add);
2741 		}
2742 		/* Now move all of the filters from the temp add list back to
2743 		 * the VSI's list.
2744 		 */
2745 		spin_lock_bh(&vsi->mac_filter_hash_lock);
2746 		hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2747 			/* Only update the state if we're still NEW */
2748 			if (new->f->state == I40E_FILTER_NEW ||
2749 			    new->f->state == I40E_FILTER_NEW_SYNC)
2750 				new->f->state = new->state;
2751 			hlist_del(&new->hlist);
2752 			netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2753 			kfree(new);
2754 		}
2755 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
2756 		kfree(add_list);
2757 		add_list = NULL;
2758 	}
2759 
2760 	/* Determine the number of active and failed filters. */
2761 	spin_lock_bh(&vsi->mac_filter_hash_lock);
2762 	vsi->active_filters = 0;
2763 	hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2764 		if (f->state == I40E_FILTER_ACTIVE)
2765 			vsi->active_filters++;
2766 		else if (f->state == I40E_FILTER_FAILED)
2767 			failed_filters++;
2768 	}
2769 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
2770 
2771 	/* Check if we are able to exit overflow promiscuous mode. We can
2772 	 * safely exit if we didn't just enter, we no longer have any failed
2773 	 * filters, and we have reduced filters below the threshold value.
2774 	 */
2775 	if (old_overflow && !failed_filters &&
2776 	    vsi->active_filters < vsi->promisc_threshold) {
2777 		dev_info(&pf->pdev->dev,
2778 			 "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2779 			 vsi_name);
2780 		clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2781 		vsi->promisc_threshold = 0;
2782 	}
2783 
2784 	/* if the VF is not trusted do not do promisc */
2785 	if (vsi->type == I40E_VSI_SRIOV && pf->vf &&
2786 	    !pf->vf[vsi->vf_id].trusted) {
2787 		clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2788 		goto out;
2789 	}
2790 
2791 	new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2792 
2793 	/* If we are entering overflow promiscuous, we need to calculate a new
2794 	 * threshold for when we are safe to exit
2795 	 */
2796 	if (!old_overflow && new_overflow)
2797 		vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2798 
2799 	/* check for changes in promiscuous modes */
2800 	if (changed_flags & IFF_ALLMULTI) {
2801 		bool cur_multipromisc;
2802 
2803 		cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2804 		aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2805 							       vsi->seid,
2806 							       cur_multipromisc,
2807 							       NULL);
2808 		if (aq_ret) {
2809 			retval = i40e_aq_rc_to_posix(aq_ret,
2810 						     hw->aq.asq_last_status);
2811 			dev_info(&pf->pdev->dev,
2812 				 "set multi promisc failed on %s, err %pe aq_err %s\n",
2813 				 vsi_name,
2814 				 ERR_PTR(aq_ret),
2815 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2816 		} else {
2817 			dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2818 				 cur_multipromisc ? "entering" : "leaving");
2819 		}
2820 	}
2821 
2822 	if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2823 		bool cur_promisc;
2824 
2825 		cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2826 			       new_overflow);
2827 		aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2828 		if (aq_ret) {
2829 			retval = i40e_aq_rc_to_posix(aq_ret,
2830 						     hw->aq.asq_last_status);
2831 			dev_info(&pf->pdev->dev,
2832 				 "Setting promiscuous %s failed on %s, err %pe aq_err %s\n",
2833 				 cur_promisc ? "on" : "off",
2834 				 vsi_name,
2835 				 ERR_PTR(aq_ret),
2836 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2837 		}
2838 	}
2839 out:
2840 	/* if something went wrong then set the changed flag so we try again */
2841 	if (retval)
2842 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2843 
2844 	clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2845 	return retval;
2846 
2847 err_no_memory:
2848 	/* Restore elements on the temporary add and delete lists */
2849 	spin_lock_bh(&vsi->mac_filter_hash_lock);
2850 err_no_memory_locked:
2851 	i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2852 	i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2853 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
2854 
2855 	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2856 	clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2857 	return -ENOMEM;
2858 }
2859 
2860 /**
2861  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2862  * @pf: board private structure
2863  **/
i40e_sync_filters_subtask(struct i40e_pf * pf)2864 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2865 {
2866 	struct i40e_vsi *vsi;
2867 	int v;
2868 
2869 	if (!pf)
2870 		return;
2871 	if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2872 		return;
2873 	if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2874 		set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2875 		return;
2876 	}
2877 
2878 	i40e_pf_for_each_vsi(pf, v, vsi) {
2879 		if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2880 		    !test_bit(__I40E_VSI_RELEASING, vsi->state)) {
2881 			int ret = i40e_sync_vsi_filters(vsi);
2882 
2883 			if (ret) {
2884 				/* come back and try again later */
2885 				set_bit(__I40E_MACVLAN_SYNC_PENDING,
2886 					pf->state);
2887 				break;
2888 			}
2889 		}
2890 	}
2891 }
2892 
2893 /**
2894  * i40e_calculate_vsi_rx_buf_len - Calculates buffer length
2895  *
2896  * @vsi: VSI to calculate rx_buf_len from
2897  */
i40e_calculate_vsi_rx_buf_len(struct i40e_vsi * vsi)2898 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
2899 {
2900 	if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags))
2901 		return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048);
2902 
2903 	return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
2904 }
2905 
2906 /**
2907  * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI
2908  * @vsi: the vsi
2909  * @xdp_prog: XDP program
2910  **/
i40e_max_vsi_frame_size(struct i40e_vsi * vsi,struct bpf_prog * xdp_prog)2911 static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi,
2912 				   struct bpf_prog *xdp_prog)
2913 {
2914 	u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
2915 	u16 chain_len;
2916 
2917 	if (xdp_prog && !xdp_prog->aux->xdp_has_frags)
2918 		chain_len = 1;
2919 	else
2920 		chain_len = I40E_MAX_CHAINED_RX_BUFFERS;
2921 
2922 	return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER);
2923 }
2924 
2925 /**
2926  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2927  * @netdev: network interface device structure
2928  * @new_mtu: new value for maximum frame size
2929  *
2930  * Returns 0 on success, negative on failure
2931  **/
i40e_change_mtu(struct net_device * netdev,int new_mtu)2932 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2933 {
2934 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2935 	struct i40e_vsi *vsi = np->vsi;
2936 	struct i40e_pf *pf = vsi->back;
2937 	int frame_size;
2938 
2939 	frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
2940 	if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) {
2941 		netdev_err(netdev, "Error changing mtu to %d, Max is %d\n",
2942 			   new_mtu, frame_size - I40E_PACKET_HDR_PAD);
2943 		return -EINVAL;
2944 	}
2945 
2946 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
2947 		   netdev->mtu, new_mtu);
2948 	WRITE_ONCE(netdev->mtu, new_mtu);
2949 	if (netif_running(netdev))
2950 		i40e_vsi_reinit_locked(vsi);
2951 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2952 	set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2953 	return 0;
2954 }
2955 
2956 /**
2957  * i40e_ioctl - Access the hwtstamp interface
2958  * @netdev: network interface device structure
2959  * @ifr: interface request data
2960  * @cmd: ioctl command
2961  **/
i40e_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)2962 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2963 {
2964 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2965 	struct i40e_pf *pf = np->vsi->back;
2966 
2967 	switch (cmd) {
2968 	case SIOCGHWTSTAMP:
2969 		return i40e_ptp_get_ts_config(pf, ifr);
2970 	case SIOCSHWTSTAMP:
2971 		return i40e_ptp_set_ts_config(pf, ifr);
2972 	default:
2973 		return -EOPNOTSUPP;
2974 	}
2975 }
2976 
2977 /**
2978  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2979  * @vsi: the vsi being adjusted
2980  **/
i40e_vlan_stripping_enable(struct i40e_vsi * vsi)2981 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2982 {
2983 	struct i40e_vsi_context ctxt;
2984 	int ret;
2985 
2986 	/* Don't modify stripping options if a port VLAN is active */
2987 	if (vsi->info.pvid)
2988 		return;
2989 
2990 	if ((vsi->info.valid_sections &
2991 	     cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2992 	    ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2993 		return;  /* already enabled */
2994 
2995 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2996 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2997 				    I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2998 
2999 	ctxt.seid = vsi->seid;
3000 	ctxt.info = vsi->info;
3001 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3002 	if (ret) {
3003 		dev_info(&vsi->back->pdev->dev,
3004 			 "update vlan stripping failed, err %pe aq_err %s\n",
3005 			 ERR_PTR(ret),
3006 			 i40e_aq_str(&vsi->back->hw,
3007 				     vsi->back->hw.aq.asq_last_status));
3008 	}
3009 }
3010 
3011 /**
3012  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
3013  * @vsi: the vsi being adjusted
3014  **/
i40e_vlan_stripping_disable(struct i40e_vsi * vsi)3015 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
3016 {
3017 	struct i40e_vsi_context ctxt;
3018 	int ret;
3019 
3020 	/* Don't modify stripping options if a port VLAN is active */
3021 	if (vsi->info.pvid)
3022 		return;
3023 
3024 	if ((vsi->info.valid_sections &
3025 	     cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3026 	    ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
3027 	     I40E_AQ_VSI_PVLAN_EMOD_MASK))
3028 		return;  /* already disabled */
3029 
3030 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3031 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3032 				    I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3033 
3034 	ctxt.seid = vsi->seid;
3035 	ctxt.info = vsi->info;
3036 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3037 	if (ret) {
3038 		dev_info(&vsi->back->pdev->dev,
3039 			 "update vlan stripping failed, err %pe aq_err %s\n",
3040 			 ERR_PTR(ret),
3041 			 i40e_aq_str(&vsi->back->hw,
3042 				     vsi->back->hw.aq.asq_last_status));
3043 	}
3044 }
3045 
3046 /**
3047  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
3048  * @vsi: the vsi being configured
3049  * @vid: vlan id to be added (0 = untagged only , -1 = any)
3050  *
3051  * This is a helper function for adding a new MAC/VLAN filter with the
3052  * specified VLAN for each existing MAC address already in the hash table.
3053  * This function does *not* perform any accounting to update filters based on
3054  * VLAN mode.
3055  *
3056  * NOTE: this function expects to be called while under the
3057  * mac_filter_hash_lock
3058  **/
i40e_add_vlan_all_mac(struct i40e_vsi * vsi,s16 vid)3059 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3060 {
3061 	struct i40e_mac_filter *f, *add_f;
3062 	struct hlist_node *h;
3063 	int bkt;
3064 
3065 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3066 		/* If we're asked to add a filter that has been marked for
3067 		 * removal, it is safe to simply restore it to active state.
3068 		 * __i40e_del_filter will have simply deleted any filters which
3069 		 * were previously marked NEW or FAILED, so if it is currently
3070 		 * marked REMOVE it must have previously been ACTIVE. Since we
3071 		 * haven't yet run the sync filters task, just restore this
3072 		 * filter to the ACTIVE state so that the sync task leaves it
3073 		 * in place.
3074 		 */
3075 		if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) {
3076 			f->state = I40E_FILTER_ACTIVE;
3077 			continue;
3078 		} else if (f->state == I40E_FILTER_REMOVE) {
3079 			continue;
3080 		}
3081 		add_f = i40e_add_filter(vsi, f->macaddr, vid);
3082 		if (!add_f) {
3083 			dev_info(&vsi->back->pdev->dev,
3084 				 "Could not add vlan filter %d for %pM\n",
3085 				 vid, f->macaddr);
3086 			return -ENOMEM;
3087 		}
3088 	}
3089 
3090 	return 0;
3091 }
3092 
3093 /**
3094  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
3095  * @vsi: the VSI being configured
3096  * @vid: VLAN id to be added
3097  **/
i40e_vsi_add_vlan(struct i40e_vsi * vsi,u16 vid)3098 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
3099 {
3100 	int err;
3101 
3102 	if (vsi->info.pvid)
3103 		return -EINVAL;
3104 
3105 	/* The network stack will attempt to add VID=0, with the intention to
3106 	 * receive priority tagged packets with a VLAN of 0. Our HW receives
3107 	 * these packets by default when configured to receive untagged
3108 	 * packets, so we don't need to add a filter for this case.
3109 	 * Additionally, HW interprets adding a VID=0 filter as meaning to
3110 	 * receive *only* tagged traffic and stops receiving untagged traffic.
3111 	 * Thus, we do not want to actually add a filter for VID=0
3112 	 */
3113 	if (!vid)
3114 		return 0;
3115 
3116 	/* Locked once because all functions invoked below iterates list*/
3117 	spin_lock_bh(&vsi->mac_filter_hash_lock);
3118 	err = i40e_add_vlan_all_mac(vsi, vid);
3119 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
3120 	if (err)
3121 		return err;
3122 
3123 	/* schedule our worker thread which will take care of
3124 	 * applying the new filter changes
3125 	 */
3126 	i40e_service_event_schedule(vsi->back);
3127 	return 0;
3128 }
3129 
3130 /**
3131  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
3132  * @vsi: the vsi being configured
3133  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
3134  *
3135  * This function should be used to remove all VLAN filters which match the
3136  * given VID. It does not schedule the service event and does not take the
3137  * mac_filter_hash_lock so it may be combined with other operations under
3138  * a single invocation of the mac_filter_hash_lock.
3139  *
3140  * NOTE: this function expects to be called while under the
3141  * mac_filter_hash_lock
3142  */
i40e_rm_vlan_all_mac(struct i40e_vsi * vsi,s16 vid)3143 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3144 {
3145 	struct i40e_mac_filter *f;
3146 	struct hlist_node *h;
3147 	int bkt;
3148 
3149 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3150 		if (f->vlan == vid)
3151 			__i40e_del_filter(vsi, f);
3152 	}
3153 }
3154 
3155 /**
3156  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
3157  * @vsi: the VSI being configured
3158  * @vid: VLAN id to be removed
3159  **/
i40e_vsi_kill_vlan(struct i40e_vsi * vsi,u16 vid)3160 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
3161 {
3162 	if (!vid || vsi->info.pvid)
3163 		return;
3164 
3165 	spin_lock_bh(&vsi->mac_filter_hash_lock);
3166 	i40e_rm_vlan_all_mac(vsi, vid);
3167 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
3168 
3169 	/* schedule our worker thread which will take care of
3170 	 * applying the new filter changes
3171 	 */
3172 	i40e_service_event_schedule(vsi->back);
3173 }
3174 
3175 /**
3176  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
3177  * @netdev: network interface to be adjusted
3178  * @proto: unused protocol value
3179  * @vid: vlan id to be added
3180  *
3181  * net_device_ops implementation for adding vlan ids
3182  **/
i40e_vlan_rx_add_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)3183 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
3184 				__always_unused __be16 proto, u16 vid)
3185 {
3186 	struct i40e_netdev_priv *np = netdev_priv(netdev);
3187 	struct i40e_vsi *vsi = np->vsi;
3188 	int ret = 0;
3189 
3190 	if (vid >= VLAN_N_VID)
3191 		return -EINVAL;
3192 
3193 	ret = i40e_vsi_add_vlan(vsi, vid);
3194 	if (!ret)
3195 		set_bit(vid, vsi->active_vlans);
3196 
3197 	return ret;
3198 }
3199 
3200 /**
3201  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
3202  * @netdev: network interface to be adjusted
3203  * @proto: unused protocol value
3204  * @vid: vlan id to be added
3205  **/
i40e_vlan_rx_add_vid_up(struct net_device * netdev,__always_unused __be16 proto,u16 vid)3206 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
3207 				    __always_unused __be16 proto, u16 vid)
3208 {
3209 	struct i40e_netdev_priv *np = netdev_priv(netdev);
3210 	struct i40e_vsi *vsi = np->vsi;
3211 
3212 	if (vid >= VLAN_N_VID)
3213 		return;
3214 	set_bit(vid, vsi->active_vlans);
3215 }
3216 
3217 /**
3218  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
3219  * @netdev: network interface to be adjusted
3220  * @proto: unused protocol value
3221  * @vid: vlan id to be removed
3222  *
3223  * net_device_ops implementation for removing vlan ids
3224  **/
i40e_vlan_rx_kill_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)3225 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
3226 				 __always_unused __be16 proto, u16 vid)
3227 {
3228 	struct i40e_netdev_priv *np = netdev_priv(netdev);
3229 	struct i40e_vsi *vsi = np->vsi;
3230 
3231 	/* return code is ignored as there is nothing a user
3232 	 * can do about failure to remove and a log message was
3233 	 * already printed from the other function
3234 	 */
3235 	i40e_vsi_kill_vlan(vsi, vid);
3236 
3237 	clear_bit(vid, vsi->active_vlans);
3238 
3239 	return 0;
3240 }
3241 
3242 /**
3243  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
3244  * @vsi: the vsi being brought back up
3245  **/
i40e_restore_vlan(struct i40e_vsi * vsi)3246 static void i40e_restore_vlan(struct i40e_vsi *vsi)
3247 {
3248 	u16 vid;
3249 
3250 	if (!vsi->netdev)
3251 		return;
3252 
3253 	if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3254 		i40e_vlan_stripping_enable(vsi);
3255 	else
3256 		i40e_vlan_stripping_disable(vsi);
3257 
3258 	for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
3259 		i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
3260 					vid);
3261 }
3262 
3263 /**
3264  * i40e_vsi_add_pvid - Add pvid for the VSI
3265  * @vsi: the vsi being adjusted
3266  * @vid: the vlan id to set as a PVID
3267  **/
i40e_vsi_add_pvid(struct i40e_vsi * vsi,u16 vid)3268 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
3269 {
3270 	struct i40e_vsi_context ctxt;
3271 	int ret;
3272 
3273 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3274 	vsi->info.pvid = cpu_to_le16(vid);
3275 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3276 				    I40E_AQ_VSI_PVLAN_INSERT_PVID |
3277 				    I40E_AQ_VSI_PVLAN_EMOD_STR;
3278 
3279 	ctxt.seid = vsi->seid;
3280 	ctxt.info = vsi->info;
3281 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3282 	if (ret) {
3283 		dev_info(&vsi->back->pdev->dev,
3284 			 "add pvid failed, err %pe aq_err %s\n",
3285 			 ERR_PTR(ret),
3286 			 i40e_aq_str(&vsi->back->hw,
3287 				     vsi->back->hw.aq.asq_last_status));
3288 		return -ENOENT;
3289 	}
3290 
3291 	return 0;
3292 }
3293 
3294 /**
3295  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3296  * @vsi: the vsi being adjusted
3297  *
3298  * Just use the vlan_rx_register() service to put it back to normal
3299  **/
i40e_vsi_remove_pvid(struct i40e_vsi * vsi)3300 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3301 {
3302 	vsi->info.pvid = 0;
3303 
3304 	i40e_vlan_stripping_disable(vsi);
3305 }
3306 
3307 /**
3308  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3309  * @vsi: ptr to the VSI
3310  *
3311  * If this function returns with an error, then it's possible one or
3312  * more of the rings is populated (while the rest are not).  It is the
3313  * callers duty to clean those orphaned rings.
3314  *
3315  * Return 0 on success, negative on failure
3316  **/
i40e_vsi_setup_tx_resources(struct i40e_vsi * vsi)3317 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3318 {
3319 	int i, err = 0;
3320 
3321 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3322 		err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3323 
3324 	if (!i40e_enabled_xdp_vsi(vsi))
3325 		return err;
3326 
3327 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3328 		err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3329 
3330 	return err;
3331 }
3332 
3333 /**
3334  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3335  * @vsi: ptr to the VSI
3336  *
3337  * Free VSI's transmit software resources
3338  **/
i40e_vsi_free_tx_resources(struct i40e_vsi * vsi)3339 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3340 {
3341 	int i;
3342 
3343 	if (vsi->tx_rings) {
3344 		for (i = 0; i < vsi->num_queue_pairs; i++)
3345 			if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3346 				i40e_free_tx_resources(vsi->tx_rings[i]);
3347 	}
3348 
3349 	if (vsi->xdp_rings) {
3350 		for (i = 0; i < vsi->num_queue_pairs; i++)
3351 			if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3352 				i40e_free_tx_resources(vsi->xdp_rings[i]);
3353 	}
3354 }
3355 
3356 /**
3357  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3358  * @vsi: ptr to the VSI
3359  *
3360  * If this function returns with an error, then it's possible one or
3361  * more of the rings is populated (while the rest are not).  It is the
3362  * callers duty to clean those orphaned rings.
3363  *
3364  * Return 0 on success, negative on failure
3365  **/
i40e_vsi_setup_rx_resources(struct i40e_vsi * vsi)3366 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3367 {
3368 	int i, err = 0;
3369 
3370 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3371 		err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3372 	return err;
3373 }
3374 
3375 /**
3376  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3377  * @vsi: ptr to the VSI
3378  *
3379  * Free all receive software resources
3380  **/
i40e_vsi_free_rx_resources(struct i40e_vsi * vsi)3381 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3382 {
3383 	int i;
3384 
3385 	if (!vsi->rx_rings)
3386 		return;
3387 
3388 	for (i = 0; i < vsi->num_queue_pairs; i++)
3389 		if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3390 			i40e_free_rx_resources(vsi->rx_rings[i]);
3391 }
3392 
3393 /**
3394  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3395  * @ring: The Tx ring to configure
3396  *
3397  * This enables/disables XPS for a given Tx descriptor ring
3398  * based on the TCs enabled for the VSI that ring belongs to.
3399  **/
i40e_config_xps_tx_ring(struct i40e_ring * ring)3400 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3401 {
3402 	int cpu;
3403 
3404 	if (!ring->q_vector || !ring->netdev || ring->ch)
3405 		return;
3406 
3407 	/* We only initialize XPS once, so as not to overwrite user settings */
3408 	if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3409 		return;
3410 
3411 	cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3412 	netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3413 			    ring->queue_index);
3414 }
3415 
3416 /**
3417  * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3418  * @ring: The Tx or Rx ring
3419  *
3420  * Returns the AF_XDP buffer pool or NULL.
3421  **/
i40e_xsk_pool(struct i40e_ring * ring)3422 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3423 {
3424 	bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3425 	int qid = ring->queue_index;
3426 
3427 	if (ring_is_xdp(ring))
3428 		qid -= ring->vsi->alloc_queue_pairs;
3429 
3430 	if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3431 		return NULL;
3432 
3433 	return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3434 }
3435 
3436 /**
3437  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3438  * @ring: The Tx ring to configure
3439  *
3440  * Configure the Tx descriptor ring in the HMC context.
3441  **/
i40e_configure_tx_ring(struct i40e_ring * ring)3442 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3443 {
3444 	struct i40e_vsi *vsi = ring->vsi;
3445 	u16 pf_q = vsi->base_queue + ring->queue_index;
3446 	struct i40e_hw *hw = &vsi->back->hw;
3447 	struct i40e_hmc_obj_txq tx_ctx;
3448 	u32 qtx_ctl = 0;
3449 	int err = 0;
3450 
3451 	if (ring_is_xdp(ring))
3452 		ring->xsk_pool = i40e_xsk_pool(ring);
3453 
3454 	/* some ATR related tx ring init */
3455 	if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) {
3456 		ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
3457 		ring->atr_count = 0;
3458 	} else {
3459 		ring->atr_sample_rate = 0;
3460 	}
3461 
3462 	/* configure XPS */
3463 	i40e_config_xps_tx_ring(ring);
3464 
3465 	/* clear the context structure first */
3466 	memset(&tx_ctx, 0, sizeof(tx_ctx));
3467 
3468 	tx_ctx.new_context = 1;
3469 	tx_ctx.base = (ring->dma / 128);
3470 	tx_ctx.qlen = ring->count;
3471 	if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) ||
3472 	    test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags))
3473 		tx_ctx.fd_ena = 1;
3474 	if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags))
3475 		tx_ctx.timesync_ena = 1;
3476 	/* FDIR VSI tx ring can still use RS bit and writebacks */
3477 	if (vsi->type != I40E_VSI_FDIR)
3478 		tx_ctx.head_wb_ena = 1;
3479 	tx_ctx.head_wb_addr = ring->dma +
3480 			      (ring->count * sizeof(struct i40e_tx_desc));
3481 
3482 	/* As part of VSI creation/update, FW allocates certain
3483 	 * Tx arbitration queue sets for each TC enabled for
3484 	 * the VSI. The FW returns the handles to these queue
3485 	 * sets as part of the response buffer to Add VSI,
3486 	 * Update VSI, etc. AQ commands. It is expected that
3487 	 * these queue set handles be associated with the Tx
3488 	 * queues by the driver as part of the TX queue context
3489 	 * initialization. This has to be done regardless of
3490 	 * DCB as by default everything is mapped to TC0.
3491 	 */
3492 
3493 	if (ring->ch)
3494 		tx_ctx.rdylist =
3495 			le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3496 
3497 	else
3498 		tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3499 
3500 	tx_ctx.rdylist_act = 0;
3501 
3502 	/* clear the context in the HMC */
3503 	err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3504 	if (err) {
3505 		dev_info(&vsi->back->pdev->dev,
3506 			 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3507 			 ring->queue_index, pf_q, err);
3508 		return -ENOMEM;
3509 	}
3510 
3511 	/* set the context in the HMC */
3512 	err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3513 	if (err) {
3514 		dev_info(&vsi->back->pdev->dev,
3515 			 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3516 			 ring->queue_index, pf_q, err);
3517 		return -ENOMEM;
3518 	}
3519 
3520 	/* Now associate this queue with this PCI function */
3521 	if (ring->ch) {
3522 		if (ring->ch->type == I40E_VSI_VMDQ2)
3523 			qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3524 		else
3525 			return -EINVAL;
3526 
3527 		qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3528 				      ring->ch->vsi_number);
3529 	} else {
3530 		if (vsi->type == I40E_VSI_VMDQ2) {
3531 			qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3532 			qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3533 					      vsi->id);
3534 		} else {
3535 			qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3536 		}
3537 	}
3538 
3539 	qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id);
3540 	wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3541 	i40e_flush(hw);
3542 
3543 	/* cache tail off for easier writes later */
3544 	ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3545 
3546 	return 0;
3547 }
3548 
3549 /**
3550  * i40e_rx_offset - Return expected offset into page to access data
3551  * @rx_ring: Ring we are requesting offset of
3552  *
3553  * Returns the offset value for ring into the data buffer.
3554  */
i40e_rx_offset(struct i40e_ring * rx_ring)3555 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
3556 {
3557 	return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
3558 }
3559 
3560 /**
3561  * i40e_configure_rx_ring - Configure a receive ring context
3562  * @ring: The Rx ring to configure
3563  *
3564  * Configure the Rx descriptor ring in the HMC context.
3565  **/
i40e_configure_rx_ring(struct i40e_ring * ring)3566 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3567 {
3568 	struct i40e_vsi *vsi = ring->vsi;
3569 	u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3570 	u16 pf_q = vsi->base_queue + ring->queue_index;
3571 	struct i40e_hw *hw = &vsi->back->hw;
3572 	struct i40e_hmc_obj_rxq rx_ctx;
3573 	int err = 0;
3574 	bool ok;
3575 
3576 	bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3577 
3578 	/* clear the context structure first */
3579 	memset(&rx_ctx, 0, sizeof(rx_ctx));
3580 
3581 	ring->rx_buf_len = vsi->rx_buf_len;
3582 
3583 	/* XDP RX-queue info only needed for RX rings exposed to XDP */
3584 	if (ring->vsi->type != I40E_VSI_MAIN)
3585 		goto skip;
3586 
3587 	if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) {
3588 		err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3589 					 ring->queue_index,
3590 					 ring->q_vector->napi.napi_id,
3591 					 ring->rx_buf_len);
3592 		if (err)
3593 			return err;
3594 	}
3595 
3596 	ring->xsk_pool = i40e_xsk_pool(ring);
3597 	if (ring->xsk_pool) {
3598 		xdp_rxq_info_unreg(&ring->xdp_rxq);
3599 		ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool);
3600 		err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3601 					 ring->queue_index,
3602 					 ring->q_vector->napi.napi_id,
3603 					 ring->rx_buf_len);
3604 		if (err)
3605 			return err;
3606 		err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3607 						 MEM_TYPE_XSK_BUFF_POOL,
3608 						 NULL);
3609 		if (err)
3610 			return err;
3611 		dev_info(&vsi->back->pdev->dev,
3612 			 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3613 			 ring->queue_index);
3614 
3615 	} else {
3616 		err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3617 						 MEM_TYPE_PAGE_SHARED,
3618 						 NULL);
3619 		if (err)
3620 			return err;
3621 	}
3622 
3623 skip:
3624 	xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq);
3625 
3626 	rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3627 				    BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3628 
3629 	rx_ctx.base = (ring->dma / 128);
3630 	rx_ctx.qlen = ring->count;
3631 
3632 	/* use 16 byte descriptors */
3633 	rx_ctx.dsize = 0;
3634 
3635 	/* descriptor type is always zero
3636 	 * rx_ctx.dtype = 0;
3637 	 */
3638 	rx_ctx.hsplit_0 = 0;
3639 
3640 	rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3641 	if (hw->revision_id == 0)
3642 		rx_ctx.lrxqthresh = 0;
3643 	else
3644 		rx_ctx.lrxqthresh = 1;
3645 	rx_ctx.crcstrip = 1;
3646 	rx_ctx.l2tsel = 1;
3647 	/* this controls whether VLAN is stripped from inner headers */
3648 	rx_ctx.showiv = 0;
3649 	/* set the prefena field to 1 because the manual says to */
3650 	rx_ctx.prefena = 1;
3651 
3652 	/* clear the context in the HMC */
3653 	err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3654 	if (err) {
3655 		dev_info(&vsi->back->pdev->dev,
3656 			 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3657 			 ring->queue_index, pf_q, err);
3658 		return -ENOMEM;
3659 	}
3660 
3661 	/* set the context in the HMC */
3662 	err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3663 	if (err) {
3664 		dev_info(&vsi->back->pdev->dev,
3665 			 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3666 			 ring->queue_index, pf_q, err);
3667 		return -ENOMEM;
3668 	}
3669 
3670 	/* configure Rx buffer alignment */
3671 	if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) {
3672 		if (I40E_2K_TOO_SMALL_WITH_PADDING) {
3673 			dev_info(&vsi->back->pdev->dev,
3674 				 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
3675 			return -EOPNOTSUPP;
3676 		}
3677 		clear_ring_build_skb_enabled(ring);
3678 	} else {
3679 		set_ring_build_skb_enabled(ring);
3680 	}
3681 
3682 	ring->rx_offset = i40e_rx_offset(ring);
3683 
3684 	/* cache tail for quicker writes, and clear the reg before use */
3685 	ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3686 	writel(0, ring->tail);
3687 
3688 	if (ring->xsk_pool) {
3689 		xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3690 		ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3691 	} else {
3692 		ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3693 	}
3694 	if (!ok) {
3695 		/* Log this in case the user has forgotten to give the kernel
3696 		 * any buffers, even later in the application.
3697 		 */
3698 		dev_info(&vsi->back->pdev->dev,
3699 			 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3700 			 ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3701 			 ring->queue_index, pf_q);
3702 	}
3703 
3704 	return 0;
3705 }
3706 
3707 /**
3708  * i40e_vsi_configure_tx - Configure the VSI for Tx
3709  * @vsi: VSI structure describing this set of rings and resources
3710  *
3711  * Configure the Tx VSI for operation.
3712  **/
i40e_vsi_configure_tx(struct i40e_vsi * vsi)3713 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3714 {
3715 	int err = 0;
3716 	u16 i;
3717 
3718 	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3719 		err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3720 
3721 	if (err || !i40e_enabled_xdp_vsi(vsi))
3722 		return err;
3723 
3724 	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3725 		err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3726 
3727 	return err;
3728 }
3729 
3730 /**
3731  * i40e_vsi_configure_rx - Configure the VSI for Rx
3732  * @vsi: the VSI being configured
3733  *
3734  * Configure the Rx VSI for operation.
3735  **/
i40e_vsi_configure_rx(struct i40e_vsi * vsi)3736 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3737 {
3738 	int err = 0;
3739 	u16 i;
3740 
3741 	vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
3742 	vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
3743 
3744 #if (PAGE_SIZE < 8192)
3745 	if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&
3746 	    vsi->netdev->mtu <= ETH_DATA_LEN) {
3747 		vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3748 		vsi->max_frame = vsi->rx_buf_len;
3749 	}
3750 #endif
3751 
3752 	/* set up individual rings */
3753 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3754 		err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3755 
3756 	return err;
3757 }
3758 
3759 /**
3760  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3761  * @vsi: ptr to the VSI
3762  **/
i40e_vsi_config_dcb_rings(struct i40e_vsi * vsi)3763 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3764 {
3765 	struct i40e_ring *tx_ring, *rx_ring;
3766 	u16 qoffset, qcount;
3767 	int i, n;
3768 
3769 	if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
3770 		/* Reset the TC information */
3771 		for (i = 0; i < vsi->num_queue_pairs; i++) {
3772 			rx_ring = vsi->rx_rings[i];
3773 			tx_ring = vsi->tx_rings[i];
3774 			rx_ring->dcb_tc = 0;
3775 			tx_ring->dcb_tc = 0;
3776 		}
3777 		return;
3778 	}
3779 
3780 	for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3781 		if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3782 			continue;
3783 
3784 		qoffset = vsi->tc_config.tc_info[n].qoffset;
3785 		qcount = vsi->tc_config.tc_info[n].qcount;
3786 		for (i = qoffset; i < (qoffset + qcount); i++) {
3787 			rx_ring = vsi->rx_rings[i];
3788 			tx_ring = vsi->tx_rings[i];
3789 			rx_ring->dcb_tc = n;
3790 			tx_ring->dcb_tc = n;
3791 		}
3792 	}
3793 }
3794 
3795 /**
3796  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3797  * @vsi: ptr to the VSI
3798  **/
i40e_set_vsi_rx_mode(struct i40e_vsi * vsi)3799 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3800 {
3801 	if (vsi->netdev)
3802 		i40e_set_rx_mode(vsi->netdev);
3803 }
3804 
3805 /**
3806  * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3807  * @pf: Pointer to the targeted PF
3808  *
3809  * Set all flow director counters to 0.
3810  */
i40e_reset_fdir_filter_cnt(struct i40e_pf * pf)3811 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3812 {
3813 	pf->fd_tcp4_filter_cnt = 0;
3814 	pf->fd_udp4_filter_cnt = 0;
3815 	pf->fd_sctp4_filter_cnt = 0;
3816 	pf->fd_ip4_filter_cnt = 0;
3817 	pf->fd_tcp6_filter_cnt = 0;
3818 	pf->fd_udp6_filter_cnt = 0;
3819 	pf->fd_sctp6_filter_cnt = 0;
3820 	pf->fd_ip6_filter_cnt = 0;
3821 }
3822 
3823 /**
3824  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3825  * @vsi: Pointer to the targeted VSI
3826  *
3827  * This function replays the hlist on the hw where all the SB Flow Director
3828  * filters were saved.
3829  **/
i40e_fdir_filter_restore(struct i40e_vsi * vsi)3830 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3831 {
3832 	struct i40e_fdir_filter *filter;
3833 	struct i40e_pf *pf = vsi->back;
3834 	struct hlist_node *node;
3835 
3836 	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
3837 		return;
3838 
3839 	/* Reset FDir counters as we're replaying all existing filters */
3840 	i40e_reset_fdir_filter_cnt(pf);
3841 
3842 	hlist_for_each_entry_safe(filter, node,
3843 				  &pf->fdir_filter_list, fdir_node) {
3844 		i40e_add_del_fdir(vsi, filter, true);
3845 	}
3846 }
3847 
3848 /**
3849  * i40e_vsi_configure - Set up the VSI for action
3850  * @vsi: the VSI being configured
3851  **/
i40e_vsi_configure(struct i40e_vsi * vsi)3852 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3853 {
3854 	int err;
3855 
3856 	i40e_set_vsi_rx_mode(vsi);
3857 	i40e_restore_vlan(vsi);
3858 	i40e_vsi_config_dcb_rings(vsi);
3859 	err = i40e_vsi_configure_tx(vsi);
3860 	if (!err)
3861 		err = i40e_vsi_configure_rx(vsi);
3862 
3863 	return err;
3864 }
3865 
3866 /**
3867  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3868  * @vsi: the VSI being configured
3869  **/
i40e_vsi_configure_msix(struct i40e_vsi * vsi)3870 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3871 {
3872 	bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3873 	struct i40e_pf *pf = vsi->back;
3874 	struct i40e_hw *hw = &pf->hw;
3875 	u16 vector;
3876 	int i, q;
3877 	u32 qp;
3878 
3879 	/* The interrupt indexing is offset by 1 in the PFINT_ITRn
3880 	 * and PFINT_LNKLSTn registers, e.g.:
3881 	 *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3882 	 */
3883 	qp = vsi->base_queue;
3884 	vector = vsi->base_vector;
3885 	for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3886 		struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3887 
3888 		q_vector->rx.next_update = jiffies + 1;
3889 		q_vector->rx.target_itr =
3890 			ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3891 		wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3892 		     q_vector->rx.target_itr >> 1);
3893 		q_vector->rx.current_itr = q_vector->rx.target_itr;
3894 
3895 		q_vector->tx.next_update = jiffies + 1;
3896 		q_vector->tx.target_itr =
3897 			ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3898 		wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3899 		     q_vector->tx.target_itr >> 1);
3900 		q_vector->tx.current_itr = q_vector->tx.target_itr;
3901 
3902 		/* Set ITR for software interrupts triggered after exiting
3903 		 * busy-loop polling.
3904 		 */
3905 		wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1),
3906 		     I40E_ITR_20K);
3907 
3908 		wr32(hw, I40E_PFINT_RATEN(vector - 1),
3909 		     i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3910 
3911 		/* begin of linked list for RX queue assigned to this vector */
3912 		wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3913 		for (q = 0; q < q_vector->num_ringpairs; q++) {
3914 			u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3915 			u32 val;
3916 
3917 			val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3918 			      (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3919 			      (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3920 			      (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3921 			      (I40E_QUEUE_TYPE_TX <<
3922 			       I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3923 
3924 			wr32(hw, I40E_QINT_RQCTL(qp), val);
3925 
3926 			if (has_xdp) {
3927 				/* TX queue with next queue set to TX */
3928 				val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3929 				      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3930 				      (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3931 				      (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3932 				      (I40E_QUEUE_TYPE_TX <<
3933 				       I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3934 
3935 				wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3936 			}
3937 			/* TX queue with next RX or end of linked list */
3938 			val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3939 			      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3940 			      (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3941 			      ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3942 			      (I40E_QUEUE_TYPE_RX <<
3943 			       I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3944 
3945 			/* Terminate the linked list */
3946 			if (q == (q_vector->num_ringpairs - 1))
3947 				val |= (I40E_QUEUE_END_OF_LIST <<
3948 					I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3949 
3950 			wr32(hw, I40E_QINT_TQCTL(qp), val);
3951 			qp++;
3952 		}
3953 	}
3954 
3955 	i40e_flush(hw);
3956 }
3957 
3958 /**
3959  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3960  * @pf: pointer to private device data structure
3961  **/
i40e_enable_misc_int_causes(struct i40e_pf * pf)3962 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3963 {
3964 	struct i40e_hw *hw = &pf->hw;
3965 	u32 val;
3966 
3967 	/* clear things first */
3968 	wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3969 	rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3970 
3971 	val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3972 	      I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3973 	      I40E_PFINT_ICR0_ENA_GRST_MASK          |
3974 	      I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3975 	      I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3976 	      I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3977 	      I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3978 	      I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3979 
3980 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
3981 		val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3982 
3983 	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
3984 		val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3985 
3986 	wr32(hw, I40E_PFINT_ICR0_ENA, val);
3987 
3988 	/* SW_ITR_IDX = 0, but don't change INTENA */
3989 	wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3990 					I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3991 
3992 	/* OTHER_ITR_IDX = 0 */
3993 	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3994 }
3995 
3996 /**
3997  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3998  * @vsi: the VSI being configured
3999  **/
i40e_configure_msi_and_legacy(struct i40e_vsi * vsi)4000 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
4001 {
4002 	u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
4003 	struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4004 	struct i40e_pf *pf = vsi->back;
4005 	struct i40e_hw *hw = &pf->hw;
4006 
4007 	/* set the ITR configuration */
4008 	q_vector->rx.next_update = jiffies + 1;
4009 	q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
4010 	wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
4011 	q_vector->rx.current_itr = q_vector->rx.target_itr;
4012 	q_vector->tx.next_update = jiffies + 1;
4013 	q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
4014 	wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
4015 	q_vector->tx.current_itr = q_vector->tx.target_itr;
4016 
4017 	i40e_enable_misc_int_causes(pf);
4018 
4019 	/* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
4020 	wr32(hw, I40E_PFINT_LNKLST0, 0);
4021 
4022 	/* Associate the queue pair to the vector and enable the queue
4023 	 * interrupt RX queue in linked list with next queue set to TX
4024 	 */
4025 	wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX));
4026 
4027 	if (i40e_enabled_xdp_vsi(vsi)) {
4028 		/* TX queue in linked list with next queue set to TX */
4029 		wr32(hw, I40E_QINT_TQCTL(nextqp),
4030 		     I40E_QINT_TQCTL_VAL(nextqp, 0, TX));
4031 	}
4032 
4033 	/* last TX queue so the next RX queue doesn't matter */
4034 	wr32(hw, I40E_QINT_TQCTL(0),
4035 	     I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX));
4036 	i40e_flush(hw);
4037 }
4038 
4039 /**
4040  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
4041  * @pf: board private structure
4042  **/
i40e_irq_dynamic_disable_icr0(struct i40e_pf * pf)4043 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
4044 {
4045 	struct i40e_hw *hw = &pf->hw;
4046 
4047 	wr32(hw, I40E_PFINT_DYN_CTL0,
4048 	     I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4049 	i40e_flush(hw);
4050 }
4051 
4052 /**
4053  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
4054  * @pf: board private structure
4055  **/
i40e_irq_dynamic_enable_icr0(struct i40e_pf * pf)4056 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
4057 {
4058 	struct i40e_hw *hw = &pf->hw;
4059 	u32 val;
4060 
4061 	val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
4062 	      I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4063 	      (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4064 
4065 	wr32(hw, I40E_PFINT_DYN_CTL0, val);
4066 	i40e_flush(hw);
4067 }
4068 
4069 /**
4070  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
4071  * @irq: interrupt number
4072  * @data: pointer to a q_vector
4073  **/
i40e_msix_clean_rings(int irq,void * data)4074 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
4075 {
4076 	struct i40e_q_vector *q_vector = data;
4077 
4078 	if (!q_vector->tx.ring && !q_vector->rx.ring)
4079 		return IRQ_HANDLED;
4080 
4081 	napi_schedule_irqoff(&q_vector->napi);
4082 
4083 	return IRQ_HANDLED;
4084 }
4085 
4086 /**
4087  * i40e_irq_affinity_notify - Callback for affinity changes
4088  * @notify: context as to what irq was changed
4089  * @mask: the new affinity mask
4090  *
4091  * This is a callback function used by the irq_set_affinity_notifier function
4092  * so that we may register to receive changes to the irq affinity masks.
4093  **/
i40e_irq_affinity_notify(struct irq_affinity_notify * notify,const cpumask_t * mask)4094 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
4095 				     const cpumask_t *mask)
4096 {
4097 	struct i40e_q_vector *q_vector =
4098 		container_of(notify, struct i40e_q_vector, affinity_notify);
4099 
4100 	cpumask_copy(&q_vector->affinity_mask, mask);
4101 }
4102 
4103 /**
4104  * i40e_irq_affinity_release - Callback for affinity notifier release
4105  * @ref: internal core kernel usage
4106  *
4107  * This is a callback function used by the irq_set_affinity_notifier function
4108  * to inform the current notification subscriber that they will no longer
4109  * receive notifications.
4110  **/
i40e_irq_affinity_release(struct kref * ref)4111 static void i40e_irq_affinity_release(struct kref *ref) {}
4112 
4113 /**
4114  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
4115  * @vsi: the VSI being configured
4116  * @basename: name for the vector
4117  *
4118  * Allocates MSI-X vectors and requests interrupts from the kernel.
4119  **/
i40e_vsi_request_irq_msix(struct i40e_vsi * vsi,char * basename)4120 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
4121 {
4122 	int q_vectors = vsi->num_q_vectors;
4123 	struct i40e_pf *pf = vsi->back;
4124 	int base = vsi->base_vector;
4125 	int rx_int_idx = 0;
4126 	int tx_int_idx = 0;
4127 	int vector, err;
4128 	int irq_num;
4129 	int cpu;
4130 
4131 	for (vector = 0; vector < q_vectors; vector++) {
4132 		struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
4133 
4134 		irq_num = pf->msix_entries[base + vector].vector;
4135 
4136 		if (q_vector->tx.ring && q_vector->rx.ring) {
4137 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4138 				 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
4139 			tx_int_idx++;
4140 		} else if (q_vector->rx.ring) {
4141 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4142 				 "%s-%s-%d", basename, "rx", rx_int_idx++);
4143 		} else if (q_vector->tx.ring) {
4144 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4145 				 "%s-%s-%d", basename, "tx", tx_int_idx++);
4146 		} else {
4147 			/* skip this unused q_vector */
4148 			continue;
4149 		}
4150 		err = request_irq(irq_num,
4151 				  vsi->irq_handler,
4152 				  0,
4153 				  q_vector->name,
4154 				  q_vector);
4155 		if (err) {
4156 			dev_info(&pf->pdev->dev,
4157 				 "MSIX request_irq failed, error: %d\n", err);
4158 			goto free_queue_irqs;
4159 		}
4160 
4161 		/* register for affinity change notifications */
4162 		q_vector->irq_num = irq_num;
4163 		q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
4164 		q_vector->affinity_notify.release = i40e_irq_affinity_release;
4165 		irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
4166 		/* Spread affinity hints out across online CPUs.
4167 		 *
4168 		 * get_cpu_mask returns a static constant mask with
4169 		 * a permanent lifetime so it's ok to pass to
4170 		 * irq_update_affinity_hint without making a copy.
4171 		 */
4172 		cpu = cpumask_local_spread(q_vector->v_idx, -1);
4173 		irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
4174 	}
4175 
4176 	vsi->irqs_ready = true;
4177 	return 0;
4178 
4179 free_queue_irqs:
4180 	while (vector) {
4181 		vector--;
4182 		irq_num = pf->msix_entries[base + vector].vector;
4183 		irq_set_affinity_notifier(irq_num, NULL);
4184 		irq_update_affinity_hint(irq_num, NULL);
4185 		free_irq(irq_num, &vsi->q_vectors[vector]);
4186 	}
4187 	return err;
4188 }
4189 
4190 /**
4191  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
4192  * @vsi: the VSI being un-configured
4193  **/
i40e_vsi_disable_irq(struct i40e_vsi * vsi)4194 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
4195 {
4196 	struct i40e_pf *pf = vsi->back;
4197 	struct i40e_hw *hw = &pf->hw;
4198 	int base = vsi->base_vector;
4199 	int i;
4200 
4201 	/* disable interrupt causation from each queue */
4202 	for (i = 0; i < vsi->num_queue_pairs; i++) {
4203 		u32 val;
4204 
4205 		val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4206 		val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
4207 		wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4208 
4209 		val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
4210 		val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
4211 		wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
4212 
4213 		if (!i40e_enabled_xdp_vsi(vsi))
4214 			continue;
4215 		wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
4216 	}
4217 
4218 	/* disable each interrupt */
4219 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4220 		for (i = vsi->base_vector;
4221 		     i < (vsi->num_q_vectors + vsi->base_vector); i++)
4222 			wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
4223 
4224 		i40e_flush(hw);
4225 		for (i = 0; i < vsi->num_q_vectors; i++)
4226 			synchronize_irq(pf->msix_entries[i + base].vector);
4227 	} else {
4228 		/* Legacy and MSI mode - this stops all interrupt handling */
4229 		wr32(hw, I40E_PFINT_ICR0_ENA, 0);
4230 		wr32(hw, I40E_PFINT_DYN_CTL0, 0);
4231 		i40e_flush(hw);
4232 		synchronize_irq(pf->pdev->irq);
4233 	}
4234 }
4235 
4236 /**
4237  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
4238  * @vsi: the VSI being configured
4239  **/
i40e_vsi_enable_irq(struct i40e_vsi * vsi)4240 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
4241 {
4242 	struct i40e_pf *pf = vsi->back;
4243 	int i;
4244 
4245 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4246 		for (i = 0; i < vsi->num_q_vectors; i++)
4247 			i40e_irq_dynamic_enable(vsi, i);
4248 	} else {
4249 		i40e_irq_dynamic_enable_icr0(pf);
4250 	}
4251 
4252 	i40e_flush(&pf->hw);
4253 	return 0;
4254 }
4255 
4256 /**
4257  * i40e_free_misc_vector - Free the vector that handles non-queue events
4258  * @pf: board private structure
4259  **/
i40e_free_misc_vector(struct i40e_pf * pf)4260 static void i40e_free_misc_vector(struct i40e_pf *pf)
4261 {
4262 	/* Disable ICR 0 */
4263 	wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
4264 	i40e_flush(&pf->hw);
4265 
4266 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
4267 		free_irq(pf->msix_entries[0].vector, pf);
4268 		clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
4269 	}
4270 }
4271 
4272 /**
4273  * i40e_intr - MSI/Legacy and non-queue interrupt handler
4274  * @irq: interrupt number
4275  * @data: pointer to a q_vector
4276  *
4277  * This is the handler used for all MSI/Legacy interrupts, and deals
4278  * with both queue and non-queue interrupts.  This is also used in
4279  * MSIX mode to handle the non-queue interrupts.
4280  **/
i40e_intr(int irq,void * data)4281 static irqreturn_t i40e_intr(int irq, void *data)
4282 {
4283 	struct i40e_pf *pf = (struct i40e_pf *)data;
4284 	struct i40e_hw *hw = &pf->hw;
4285 	irqreturn_t ret = IRQ_NONE;
4286 	u32 icr0, icr0_remaining;
4287 	u32 val, ena_mask;
4288 
4289 	icr0 = rd32(hw, I40E_PFINT_ICR0);
4290 	ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4291 
4292 	/* if sharing a legacy IRQ, we might get called w/o an intr pending */
4293 	if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4294 		goto enable_intr;
4295 
4296 	/* if interrupt but no bits showing, must be SWINT */
4297 	if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4298 	    (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4299 		pf->sw_int_count++;
4300 
4301 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
4302 	    (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4303 		ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4304 		dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4305 		set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4306 	}
4307 
4308 	/* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4309 	if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4310 		struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
4311 		struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4312 
4313 		/* We do not have a way to disarm Queue causes while leaving
4314 		 * interrupt enabled for all other causes, ideally
4315 		 * interrupt should be disabled while we are in NAPI but
4316 		 * this is not a performance path and napi_schedule()
4317 		 * can deal with rescheduling.
4318 		 */
4319 		if (!test_bit(__I40E_DOWN, pf->state))
4320 			napi_schedule_irqoff(&q_vector->napi);
4321 	}
4322 
4323 	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4324 		ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4325 		set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4326 		i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4327 	}
4328 
4329 	if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4330 		ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4331 		set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4332 	}
4333 
4334 	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4335 		/* disable any further VFLR event notifications */
4336 		if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4337 			u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4338 
4339 			reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4340 			wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4341 		} else {
4342 			ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4343 			set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4344 		}
4345 	}
4346 
4347 	if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4348 		if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4349 			set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4350 		ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4351 		val = rd32(hw, I40E_GLGEN_RSTAT);
4352 		val = FIELD_GET(I40E_GLGEN_RSTAT_RESET_TYPE_MASK, val);
4353 		if (val == I40E_RESET_CORER) {
4354 			pf->corer_count++;
4355 		} else if (val == I40E_RESET_GLOBR) {
4356 			pf->globr_count++;
4357 		} else if (val == I40E_RESET_EMPR) {
4358 			pf->empr_count++;
4359 			set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4360 		}
4361 	}
4362 
4363 	if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4364 		icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4365 		dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4366 		dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4367 			 rd32(hw, I40E_PFHMC_ERRORINFO),
4368 			 rd32(hw, I40E_PFHMC_ERRORDATA));
4369 	}
4370 
4371 	if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4372 		u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4373 
4374 		if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)
4375 			schedule_work(&pf->ptp_extts0_work);
4376 
4377 		if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)
4378 			i40e_ptp_tx_hwtstamp(pf);
4379 
4380 		icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4381 	}
4382 
4383 	/* If a critical error is pending we have no choice but to reset the
4384 	 * device.
4385 	 * Report and mask out any remaining unexpected interrupts.
4386 	 */
4387 	icr0_remaining = icr0 & ena_mask;
4388 	if (icr0_remaining) {
4389 		dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4390 			 icr0_remaining);
4391 		if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4392 		    (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4393 		    (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4394 			dev_info(&pf->pdev->dev, "device will be reset\n");
4395 			set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4396 			i40e_service_event_schedule(pf);
4397 		}
4398 		ena_mask &= ~icr0_remaining;
4399 	}
4400 	ret = IRQ_HANDLED;
4401 
4402 enable_intr:
4403 	/* re-enable interrupt causes */
4404 	wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4405 	if (!test_bit(__I40E_DOWN, pf->state) ||
4406 	    test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4407 		i40e_service_event_schedule(pf);
4408 		i40e_irq_dynamic_enable_icr0(pf);
4409 	}
4410 
4411 	return ret;
4412 }
4413 
4414 /**
4415  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4416  * @tx_ring:  tx ring to clean
4417  * @budget:   how many cleans we're allowed
4418  *
4419  * Returns true if there's any budget left (e.g. the clean is finished)
4420  **/
i40e_clean_fdir_tx_irq(struct i40e_ring * tx_ring,int budget)4421 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4422 {
4423 	struct i40e_vsi *vsi = tx_ring->vsi;
4424 	u16 i = tx_ring->next_to_clean;
4425 	struct i40e_tx_buffer *tx_buf;
4426 	struct i40e_tx_desc *tx_desc;
4427 
4428 	tx_buf = &tx_ring->tx_bi[i];
4429 	tx_desc = I40E_TX_DESC(tx_ring, i);
4430 	i -= tx_ring->count;
4431 
4432 	do {
4433 		struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4434 
4435 		/* if next_to_watch is not set then there is no work pending */
4436 		if (!eop_desc)
4437 			break;
4438 
4439 		/* prevent any other reads prior to eop_desc */
4440 		smp_rmb();
4441 
4442 		/* if the descriptor isn't done, no work yet to do */
4443 		if (!(eop_desc->cmd_type_offset_bsz &
4444 		      cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4445 			break;
4446 
4447 		/* clear next_to_watch to prevent false hangs */
4448 		tx_buf->next_to_watch = NULL;
4449 
4450 		tx_desc->buffer_addr = 0;
4451 		tx_desc->cmd_type_offset_bsz = 0;
4452 		/* move past filter desc */
4453 		tx_buf++;
4454 		tx_desc++;
4455 		i++;
4456 		if (unlikely(!i)) {
4457 			i -= tx_ring->count;
4458 			tx_buf = tx_ring->tx_bi;
4459 			tx_desc = I40E_TX_DESC(tx_ring, 0);
4460 		}
4461 		/* unmap skb header data */
4462 		dma_unmap_single(tx_ring->dev,
4463 				 dma_unmap_addr(tx_buf, dma),
4464 				 dma_unmap_len(tx_buf, len),
4465 				 DMA_TO_DEVICE);
4466 		if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4467 			kfree(tx_buf->raw_buf);
4468 
4469 		tx_buf->raw_buf = NULL;
4470 		tx_buf->tx_flags = 0;
4471 		tx_buf->next_to_watch = NULL;
4472 		dma_unmap_len_set(tx_buf, len, 0);
4473 		tx_desc->buffer_addr = 0;
4474 		tx_desc->cmd_type_offset_bsz = 0;
4475 
4476 		/* move us past the eop_desc for start of next FD desc */
4477 		tx_buf++;
4478 		tx_desc++;
4479 		i++;
4480 		if (unlikely(!i)) {
4481 			i -= tx_ring->count;
4482 			tx_buf = tx_ring->tx_bi;
4483 			tx_desc = I40E_TX_DESC(tx_ring, 0);
4484 		}
4485 
4486 		/* update budget accounting */
4487 		budget--;
4488 	} while (likely(budget));
4489 
4490 	i += tx_ring->count;
4491 	tx_ring->next_to_clean = i;
4492 
4493 	if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags))
4494 		i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4495 
4496 	return budget > 0;
4497 }
4498 
4499 /**
4500  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4501  * @irq: interrupt number
4502  * @data: pointer to a q_vector
4503  **/
i40e_fdir_clean_ring(int irq,void * data)4504 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4505 {
4506 	struct i40e_q_vector *q_vector = data;
4507 	struct i40e_vsi *vsi;
4508 
4509 	if (!q_vector->tx.ring)
4510 		return IRQ_HANDLED;
4511 
4512 	vsi = q_vector->tx.ring->vsi;
4513 	i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4514 
4515 	return IRQ_HANDLED;
4516 }
4517 
4518 /**
4519  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4520  * @vsi: the VSI being configured
4521  * @v_idx: vector index
4522  * @qp_idx: queue pair index
4523  **/
i40e_map_vector_to_qp(struct i40e_vsi * vsi,int v_idx,int qp_idx)4524 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4525 {
4526 	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4527 	struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4528 	struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4529 
4530 	tx_ring->q_vector = q_vector;
4531 	tx_ring->next = q_vector->tx.ring;
4532 	q_vector->tx.ring = tx_ring;
4533 	q_vector->tx.count++;
4534 
4535 	/* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4536 	if (i40e_enabled_xdp_vsi(vsi)) {
4537 		struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4538 
4539 		xdp_ring->q_vector = q_vector;
4540 		xdp_ring->next = q_vector->tx.ring;
4541 		q_vector->tx.ring = xdp_ring;
4542 		q_vector->tx.count++;
4543 	}
4544 
4545 	rx_ring->q_vector = q_vector;
4546 	rx_ring->next = q_vector->rx.ring;
4547 	q_vector->rx.ring = rx_ring;
4548 	q_vector->rx.count++;
4549 }
4550 
4551 /**
4552  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4553  * @vsi: the VSI being configured
4554  *
4555  * This function maps descriptor rings to the queue-specific vectors
4556  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4557  * one vector per queue pair, but on a constrained vector budget, we
4558  * group the queue pairs as "efficiently" as possible.
4559  **/
i40e_vsi_map_rings_to_vectors(struct i40e_vsi * vsi)4560 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4561 {
4562 	int qp_remaining = vsi->num_queue_pairs;
4563 	int q_vectors = vsi->num_q_vectors;
4564 	int num_ringpairs;
4565 	int v_start = 0;
4566 	int qp_idx = 0;
4567 
4568 	/* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4569 	 * group them so there are multiple queues per vector.
4570 	 * It is also important to go through all the vectors available to be
4571 	 * sure that if we don't use all the vectors, that the remaining vectors
4572 	 * are cleared. This is especially important when decreasing the
4573 	 * number of queues in use.
4574 	 */
4575 	for (; v_start < q_vectors; v_start++) {
4576 		struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4577 
4578 		num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4579 
4580 		q_vector->num_ringpairs = num_ringpairs;
4581 		q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4582 
4583 		q_vector->rx.count = 0;
4584 		q_vector->tx.count = 0;
4585 		q_vector->rx.ring = NULL;
4586 		q_vector->tx.ring = NULL;
4587 
4588 		while (num_ringpairs--) {
4589 			i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4590 			qp_idx++;
4591 			qp_remaining--;
4592 		}
4593 	}
4594 }
4595 
4596 /**
4597  * i40e_vsi_request_irq - Request IRQ from the OS
4598  * @vsi: the VSI being configured
4599  * @basename: name for the vector
4600  **/
i40e_vsi_request_irq(struct i40e_vsi * vsi,char * basename)4601 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4602 {
4603 	struct i40e_pf *pf = vsi->back;
4604 	int err;
4605 
4606 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
4607 		err = i40e_vsi_request_irq_msix(vsi, basename);
4608 	else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags))
4609 		err = request_irq(pf->pdev->irq, i40e_intr, 0,
4610 				  pf->int_name, pf);
4611 	else
4612 		err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4613 				  pf->int_name, pf);
4614 
4615 	if (err)
4616 		dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4617 
4618 	return err;
4619 }
4620 
4621 #ifdef CONFIG_NET_POLL_CONTROLLER
4622 /**
4623  * i40e_netpoll - A Polling 'interrupt' handler
4624  * @netdev: network interface device structure
4625  *
4626  * This is used by netconsole to send skbs without having to re-enable
4627  * interrupts.  It's not called while the normal interrupt routine is executing.
4628  **/
i40e_netpoll(struct net_device * netdev)4629 static void i40e_netpoll(struct net_device *netdev)
4630 {
4631 	struct i40e_netdev_priv *np = netdev_priv(netdev);
4632 	struct i40e_vsi *vsi = np->vsi;
4633 	struct i40e_pf *pf = vsi->back;
4634 	int i;
4635 
4636 	/* if interface is down do nothing */
4637 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
4638 		return;
4639 
4640 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4641 		for (i = 0; i < vsi->num_q_vectors; i++)
4642 			i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4643 	} else {
4644 		i40e_intr(pf->pdev->irq, netdev);
4645 	}
4646 }
4647 #endif
4648 
4649 #define I40E_QTX_ENA_WAIT_COUNT 50
4650 
4651 /**
4652  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4653  * @pf: the PF being configured
4654  * @pf_q: the PF queue
4655  * @enable: enable or disable state of the queue
4656  *
4657  * This routine will wait for the given Tx queue of the PF to reach the
4658  * enabled or disabled state.
4659  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4660  * multiple retries; else will return 0 in case of success.
4661  **/
i40e_pf_txq_wait(struct i40e_pf * pf,int pf_q,bool enable)4662 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4663 {
4664 	int i;
4665 	u32 tx_reg;
4666 
4667 	for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4668 		tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4669 		if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4670 			break;
4671 
4672 		usleep_range(10, 20);
4673 	}
4674 	if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4675 		return -ETIMEDOUT;
4676 
4677 	return 0;
4678 }
4679 
4680 /**
4681  * i40e_control_tx_q - Start or stop a particular Tx queue
4682  * @pf: the PF structure
4683  * @pf_q: the PF queue to configure
4684  * @enable: start or stop the queue
4685  *
4686  * This function enables or disables a single queue. Note that any delay
4687  * required after the operation is expected to be handled by the caller of
4688  * this function.
4689  **/
i40e_control_tx_q(struct i40e_pf * pf,int pf_q,bool enable)4690 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4691 {
4692 	struct i40e_hw *hw = &pf->hw;
4693 	u32 tx_reg;
4694 	int i;
4695 
4696 	/* warn the TX unit of coming changes */
4697 	i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4698 	if (!enable)
4699 		usleep_range(10, 20);
4700 
4701 	for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4702 		tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4703 		if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4704 		    ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4705 			break;
4706 		usleep_range(1000, 2000);
4707 	}
4708 
4709 	/* Skip if the queue is already in the requested state */
4710 	if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4711 		return;
4712 
4713 	/* turn on/off the queue */
4714 	if (enable) {
4715 		wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4716 		tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4717 	} else {
4718 		tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4719 	}
4720 
4721 	wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4722 }
4723 
4724 /**
4725  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4726  * @seid: VSI SEID
4727  * @pf: the PF structure
4728  * @pf_q: the PF queue to configure
4729  * @is_xdp: true if the queue is used for XDP
4730  * @enable: start or stop the queue
4731  **/
i40e_control_wait_tx_q(int seid,struct i40e_pf * pf,int pf_q,bool is_xdp,bool enable)4732 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4733 			   bool is_xdp, bool enable)
4734 {
4735 	int ret;
4736 
4737 	i40e_control_tx_q(pf, pf_q, enable);
4738 
4739 	/* wait for the change to finish */
4740 	ret = i40e_pf_txq_wait(pf, pf_q, enable);
4741 	if (ret) {
4742 		dev_info(&pf->pdev->dev,
4743 			 "VSI seid %d %sTx ring %d %sable timeout\n",
4744 			 seid, (is_xdp ? "XDP " : ""), pf_q,
4745 			 (enable ? "en" : "dis"));
4746 	}
4747 
4748 	return ret;
4749 }
4750 
4751 /**
4752  * i40e_vsi_enable_tx - Start a VSI's rings
4753  * @vsi: the VSI being configured
4754  **/
i40e_vsi_enable_tx(struct i40e_vsi * vsi)4755 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4756 {
4757 	struct i40e_pf *pf = vsi->back;
4758 	int i, pf_q, ret = 0;
4759 
4760 	pf_q = vsi->base_queue;
4761 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4762 		ret = i40e_control_wait_tx_q(vsi->seid, pf,
4763 					     pf_q,
4764 					     false /*is xdp*/, true);
4765 		if (ret)
4766 			break;
4767 
4768 		if (!i40e_enabled_xdp_vsi(vsi))
4769 			continue;
4770 
4771 		ret = i40e_control_wait_tx_q(vsi->seid, pf,
4772 					     pf_q + vsi->alloc_queue_pairs,
4773 					     true /*is xdp*/, true);
4774 		if (ret)
4775 			break;
4776 	}
4777 	return ret;
4778 }
4779 
4780 /**
4781  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4782  * @pf: the PF being configured
4783  * @pf_q: the PF queue
4784  * @enable: enable or disable state of the queue
4785  *
4786  * This routine will wait for the given Rx queue of the PF to reach the
4787  * enabled or disabled state.
4788  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4789  * multiple retries; else will return 0 in case of success.
4790  **/
i40e_pf_rxq_wait(struct i40e_pf * pf,int pf_q,bool enable)4791 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4792 {
4793 	int i;
4794 	u32 rx_reg;
4795 
4796 	for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4797 		rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4798 		if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4799 			break;
4800 
4801 		usleep_range(10, 20);
4802 	}
4803 	if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4804 		return -ETIMEDOUT;
4805 
4806 	return 0;
4807 }
4808 
4809 /**
4810  * i40e_control_rx_q - Start or stop a particular Rx queue
4811  * @pf: the PF structure
4812  * @pf_q: the PF queue to configure
4813  * @enable: start or stop the queue
4814  *
4815  * This function enables or disables a single queue. Note that
4816  * any delay required after the operation is expected to be
4817  * handled by the caller of this function.
4818  **/
i40e_control_rx_q(struct i40e_pf * pf,int pf_q,bool enable)4819 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4820 {
4821 	struct i40e_hw *hw = &pf->hw;
4822 	u32 rx_reg;
4823 	int i;
4824 
4825 	for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4826 		rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4827 		if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4828 		    ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4829 			break;
4830 		usleep_range(1000, 2000);
4831 	}
4832 
4833 	/* Skip if the queue is already in the requested state */
4834 	if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4835 		return;
4836 
4837 	/* turn on/off the queue */
4838 	if (enable)
4839 		rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4840 	else
4841 		rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4842 
4843 	wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4844 }
4845 
4846 /**
4847  * i40e_control_wait_rx_q
4848  * @pf: the PF structure
4849  * @pf_q: queue being configured
4850  * @enable: start or stop the rings
4851  *
4852  * This function enables or disables a single queue along with waiting
4853  * for the change to finish. The caller of this function should handle
4854  * the delays needed in the case of disabling queues.
4855  **/
i40e_control_wait_rx_q(struct i40e_pf * pf,int pf_q,bool enable)4856 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4857 {
4858 	int ret = 0;
4859 
4860 	i40e_control_rx_q(pf, pf_q, enable);
4861 
4862 	/* wait for the change to finish */
4863 	ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4864 	if (ret)
4865 		return ret;
4866 
4867 	return ret;
4868 }
4869 
4870 /**
4871  * i40e_vsi_enable_rx - Start a VSI's rings
4872  * @vsi: the VSI being configured
4873  **/
i40e_vsi_enable_rx(struct i40e_vsi * vsi)4874 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4875 {
4876 	struct i40e_pf *pf = vsi->back;
4877 	int i, pf_q, ret = 0;
4878 
4879 	pf_q = vsi->base_queue;
4880 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4881 		ret = i40e_control_wait_rx_q(pf, pf_q, true);
4882 		if (ret) {
4883 			dev_info(&pf->pdev->dev,
4884 				 "VSI seid %d Rx ring %d enable timeout\n",
4885 				 vsi->seid, pf_q);
4886 			break;
4887 		}
4888 	}
4889 
4890 	return ret;
4891 }
4892 
4893 /**
4894  * i40e_vsi_start_rings - Start a VSI's rings
4895  * @vsi: the VSI being configured
4896  **/
i40e_vsi_start_rings(struct i40e_vsi * vsi)4897 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4898 {
4899 	int ret = 0;
4900 
4901 	/* do rx first for enable and last for disable */
4902 	ret = i40e_vsi_enable_rx(vsi);
4903 	if (ret)
4904 		return ret;
4905 	ret = i40e_vsi_enable_tx(vsi);
4906 
4907 	return ret;
4908 }
4909 
4910 #define I40E_DISABLE_TX_GAP_MSEC	50
4911 
4912 /**
4913  * i40e_vsi_stop_rings - Stop a VSI's rings
4914  * @vsi: the VSI being configured
4915  **/
i40e_vsi_stop_rings(struct i40e_vsi * vsi)4916 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4917 {
4918 	struct i40e_pf *pf = vsi->back;
4919 	u32 pf_q, tx_q_end, rx_q_end;
4920 
4921 	/* When port TX is suspended, don't wait */
4922 	if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4923 		return i40e_vsi_stop_rings_no_wait(vsi);
4924 
4925 	tx_q_end = vsi->base_queue +
4926 		vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
4927 	for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4928 		i40e_pre_tx_queue_cfg(&pf->hw, pf_q, false);
4929 
4930 	rx_q_end = vsi->base_queue + vsi->num_queue_pairs;
4931 	for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++)
4932 		i40e_control_rx_q(pf, pf_q, false);
4933 
4934 	msleep(I40E_DISABLE_TX_GAP_MSEC);
4935 	for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4936 		wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4937 
4938 	i40e_vsi_wait_queues_disabled(vsi);
4939 }
4940 
4941 /**
4942  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4943  * @vsi: the VSI being shutdown
4944  *
4945  * This function stops all the rings for a VSI but does not delay to verify
4946  * that rings have been disabled. It is expected that the caller is shutting
4947  * down multiple VSIs at once and will delay together for all the VSIs after
4948  * initiating the shutdown. This is particularly useful for shutting down lots
4949  * of VFs together. Otherwise, a large delay can be incurred while configuring
4950  * each VSI in serial.
4951  **/
i40e_vsi_stop_rings_no_wait(struct i40e_vsi * vsi)4952 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4953 {
4954 	struct i40e_pf *pf = vsi->back;
4955 	int i, pf_q;
4956 
4957 	pf_q = vsi->base_queue;
4958 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4959 		i40e_control_tx_q(pf, pf_q, false);
4960 		i40e_control_rx_q(pf, pf_q, false);
4961 	}
4962 }
4963 
4964 /**
4965  * i40e_vsi_free_irq - Free the irq association with the OS
4966  * @vsi: the VSI being configured
4967  **/
i40e_vsi_free_irq(struct i40e_vsi * vsi)4968 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4969 {
4970 	struct i40e_pf *pf = vsi->back;
4971 	struct i40e_hw *hw = &pf->hw;
4972 	int base = vsi->base_vector;
4973 	u32 val, qp;
4974 	int i;
4975 
4976 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4977 		if (!vsi->q_vectors)
4978 			return;
4979 
4980 		if (!vsi->irqs_ready)
4981 			return;
4982 
4983 		vsi->irqs_ready = false;
4984 		for (i = 0; i < vsi->num_q_vectors; i++) {
4985 			int irq_num;
4986 			u16 vector;
4987 
4988 			vector = i + base;
4989 			irq_num = pf->msix_entries[vector].vector;
4990 
4991 			/* free only the irqs that were actually requested */
4992 			if (!vsi->q_vectors[i] ||
4993 			    !vsi->q_vectors[i]->num_ringpairs)
4994 				continue;
4995 
4996 			/* clear the affinity notifier in the IRQ descriptor */
4997 			irq_set_affinity_notifier(irq_num, NULL);
4998 			/* remove our suggested affinity mask for this IRQ */
4999 			irq_update_affinity_hint(irq_num, NULL);
5000 			free_irq(irq_num, vsi->q_vectors[i]);
5001 
5002 			/* Tear down the interrupt queue link list
5003 			 *
5004 			 * We know that they come in pairs and always
5005 			 * the Rx first, then the Tx.  To clear the
5006 			 * link list, stick the EOL value into the
5007 			 * next_q field of the registers.
5008 			 */
5009 			val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
5010 			qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK,
5011 				       val);
5012 			val |= I40E_QUEUE_END_OF_LIST
5013 				<< I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
5014 			wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
5015 
5016 			while (qp != I40E_QUEUE_END_OF_LIST) {
5017 				u32 next;
5018 
5019 				val = rd32(hw, I40E_QINT_RQCTL(qp));
5020 
5021 				val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
5022 					 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5023 					 I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
5024 					 I40E_QINT_RQCTL_INTEVENT_MASK);
5025 
5026 				val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5027 					 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5028 
5029 				wr32(hw, I40E_QINT_RQCTL(qp), val);
5030 
5031 				val = rd32(hw, I40E_QINT_TQCTL(qp));
5032 
5033 				next = FIELD_GET(I40E_QINT_TQCTL_NEXTQ_INDX_MASK,
5034 						 val);
5035 
5036 				val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
5037 					 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5038 					 I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
5039 					 I40E_QINT_TQCTL_INTEVENT_MASK);
5040 
5041 				val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5042 					 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5043 
5044 				wr32(hw, I40E_QINT_TQCTL(qp), val);
5045 				qp = next;
5046 			}
5047 		}
5048 	} else {
5049 		free_irq(pf->pdev->irq, pf);
5050 
5051 		val = rd32(hw, I40E_PFINT_LNKLST0);
5052 		qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, val);
5053 		val |= I40E_QUEUE_END_OF_LIST
5054 			<< I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
5055 		wr32(hw, I40E_PFINT_LNKLST0, val);
5056 
5057 		val = rd32(hw, I40E_QINT_RQCTL(qp));
5058 		val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
5059 			 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5060 			 I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
5061 			 I40E_QINT_RQCTL_INTEVENT_MASK);
5062 
5063 		val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5064 			I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5065 
5066 		wr32(hw, I40E_QINT_RQCTL(qp), val);
5067 
5068 		val = rd32(hw, I40E_QINT_TQCTL(qp));
5069 
5070 		val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
5071 			 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5072 			 I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
5073 			 I40E_QINT_TQCTL_INTEVENT_MASK);
5074 
5075 		val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5076 			I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5077 
5078 		wr32(hw, I40E_QINT_TQCTL(qp), val);
5079 	}
5080 }
5081 
5082 /**
5083  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
5084  * @vsi: the VSI being configured
5085  * @v_idx: Index of vector to be freed
5086  *
5087  * This function frees the memory allocated to the q_vector.  In addition if
5088  * NAPI is enabled it will delete any references to the NAPI struct prior
5089  * to freeing the q_vector.
5090  **/
i40e_free_q_vector(struct i40e_vsi * vsi,int v_idx)5091 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
5092 {
5093 	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
5094 	struct i40e_ring *ring;
5095 
5096 	if (!q_vector)
5097 		return;
5098 
5099 	/* disassociate q_vector from rings */
5100 	i40e_for_each_ring(ring, q_vector->tx)
5101 		ring->q_vector = NULL;
5102 
5103 	i40e_for_each_ring(ring, q_vector->rx)
5104 		ring->q_vector = NULL;
5105 
5106 	/* only VSI w/ an associated netdev is set up w/ NAPI */
5107 	if (vsi->netdev)
5108 		netif_napi_del(&q_vector->napi);
5109 
5110 	vsi->q_vectors[v_idx] = NULL;
5111 
5112 	kfree_rcu(q_vector, rcu);
5113 }
5114 
5115 /**
5116  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
5117  * @vsi: the VSI being un-configured
5118  *
5119  * This frees the memory allocated to the q_vectors and
5120  * deletes references to the NAPI struct.
5121  **/
i40e_vsi_free_q_vectors(struct i40e_vsi * vsi)5122 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
5123 {
5124 	int v_idx;
5125 
5126 	for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
5127 		i40e_free_q_vector(vsi, v_idx);
5128 }
5129 
5130 /**
5131  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
5132  * @pf: board private structure
5133  **/
i40e_reset_interrupt_capability(struct i40e_pf * pf)5134 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
5135 {
5136 	/* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
5137 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
5138 		pci_disable_msix(pf->pdev);
5139 		kfree(pf->msix_entries);
5140 		pf->msix_entries = NULL;
5141 		kfree(pf->irq_pile);
5142 		pf->irq_pile = NULL;
5143 	} else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
5144 		pci_disable_msi(pf->pdev);
5145 	}
5146 	clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
5147 	clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
5148 }
5149 
5150 /**
5151  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
5152  * @pf: board private structure
5153  *
5154  * We go through and clear interrupt specific resources and reset the structure
5155  * to pre-load conditions
5156  **/
i40e_clear_interrupt_scheme(struct i40e_pf * pf)5157 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
5158 {
5159 	struct i40e_vsi *vsi;
5160 	int i;
5161 
5162 	if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
5163 		i40e_free_misc_vector(pf);
5164 
5165 	i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
5166 		      I40E_IWARP_IRQ_PILE_ID);
5167 
5168 	i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
5169 
5170 	i40e_pf_for_each_vsi(pf, i, vsi)
5171 		i40e_vsi_free_q_vectors(vsi);
5172 
5173 	i40e_reset_interrupt_capability(pf);
5174 }
5175 
5176 /**
5177  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
5178  * @vsi: the VSI being configured
5179  **/
i40e_napi_enable_all(struct i40e_vsi * vsi)5180 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
5181 {
5182 	int q_idx;
5183 
5184 	if (!vsi->netdev)
5185 		return;
5186 
5187 	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5188 		struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5189 
5190 		if (q_vector->rx.ring || q_vector->tx.ring)
5191 			napi_enable(&q_vector->napi);
5192 	}
5193 }
5194 
5195 /**
5196  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
5197  * @vsi: the VSI being configured
5198  **/
i40e_napi_disable_all(struct i40e_vsi * vsi)5199 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
5200 {
5201 	int q_idx;
5202 
5203 	if (!vsi->netdev)
5204 		return;
5205 
5206 	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5207 		struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5208 
5209 		if (q_vector->rx.ring || q_vector->tx.ring)
5210 			napi_disable(&q_vector->napi);
5211 	}
5212 }
5213 
5214 /**
5215  * i40e_vsi_close - Shut down a VSI
5216  * @vsi: the vsi to be quelled
5217  **/
i40e_vsi_close(struct i40e_vsi * vsi)5218 static void i40e_vsi_close(struct i40e_vsi *vsi)
5219 {
5220 	struct i40e_pf *pf = vsi->back;
5221 	if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
5222 		i40e_down(vsi);
5223 	i40e_vsi_free_irq(vsi);
5224 	i40e_vsi_free_tx_resources(vsi);
5225 	i40e_vsi_free_rx_resources(vsi);
5226 	vsi->current_netdev_flags = 0;
5227 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
5228 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
5229 		set_bit(__I40E_CLIENT_RESET, pf->state);
5230 }
5231 
5232 /**
5233  * i40e_quiesce_vsi - Pause a given VSI
5234  * @vsi: the VSI being paused
5235  **/
i40e_quiesce_vsi(struct i40e_vsi * vsi)5236 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
5237 {
5238 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
5239 		return;
5240 
5241 	set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
5242 	if (vsi->netdev && netif_running(vsi->netdev))
5243 		vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5244 	else
5245 		i40e_vsi_close(vsi);
5246 }
5247 
5248 /**
5249  * i40e_unquiesce_vsi - Resume a given VSI
5250  * @vsi: the VSI being resumed
5251  **/
i40e_unquiesce_vsi(struct i40e_vsi * vsi)5252 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
5253 {
5254 	if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
5255 		return;
5256 
5257 	if (vsi->netdev && netif_running(vsi->netdev))
5258 		vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5259 	else
5260 		i40e_vsi_open(vsi);   /* this clears the DOWN bit */
5261 }
5262 
5263 /**
5264  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5265  * @pf: the PF
5266  **/
i40e_pf_quiesce_all_vsi(struct i40e_pf * pf)5267 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
5268 {
5269 	struct i40e_vsi *vsi;
5270 	int v;
5271 
5272 	i40e_pf_for_each_vsi(pf, v, vsi)
5273 		i40e_quiesce_vsi(vsi);
5274 }
5275 
5276 /**
5277  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5278  * @pf: the PF
5279  **/
i40e_pf_unquiesce_all_vsi(struct i40e_pf * pf)5280 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5281 {
5282 	struct i40e_vsi *vsi;
5283 	int v;
5284 
5285 	i40e_pf_for_each_vsi(pf, v, vsi)
5286 		i40e_unquiesce_vsi(vsi);
5287 }
5288 
5289 /**
5290  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5291  * @vsi: the VSI being configured
5292  *
5293  * Wait until all queues on a given VSI have been disabled.
5294  **/
i40e_vsi_wait_queues_disabled(struct i40e_vsi * vsi)5295 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5296 {
5297 	struct i40e_pf *pf = vsi->back;
5298 	int i, pf_q, ret;
5299 
5300 	pf_q = vsi->base_queue;
5301 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5302 		/* Check and wait for the Tx queue */
5303 		ret = i40e_pf_txq_wait(pf, pf_q, false);
5304 		if (ret) {
5305 			dev_info(&pf->pdev->dev,
5306 				 "VSI seid %d Tx ring %d disable timeout\n",
5307 				 vsi->seid, pf_q);
5308 			return ret;
5309 		}
5310 
5311 		if (!i40e_enabled_xdp_vsi(vsi))
5312 			goto wait_rx;
5313 
5314 		/* Check and wait for the XDP Tx queue */
5315 		ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5316 				       false);
5317 		if (ret) {
5318 			dev_info(&pf->pdev->dev,
5319 				 "VSI seid %d XDP Tx ring %d disable timeout\n",
5320 				 vsi->seid, pf_q);
5321 			return ret;
5322 		}
5323 wait_rx:
5324 		/* Check and wait for the Rx queue */
5325 		ret = i40e_pf_rxq_wait(pf, pf_q, false);
5326 		if (ret) {
5327 			dev_info(&pf->pdev->dev,
5328 				 "VSI seid %d Rx ring %d disable timeout\n",
5329 				 vsi->seid, pf_q);
5330 			return ret;
5331 		}
5332 	}
5333 
5334 	return 0;
5335 }
5336 
5337 #ifdef CONFIG_I40E_DCB
5338 /**
5339  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5340  * @pf: the PF
5341  *
5342  * This function waits for the queues to be in disabled state for all the
5343  * VSIs that are managed by this PF.
5344  **/
i40e_pf_wait_queues_disabled(struct i40e_pf * pf)5345 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5346 {
5347 	struct i40e_vsi *vsi;
5348 	int v, ret = 0;
5349 
5350 	i40e_pf_for_each_vsi(pf, v, vsi) {
5351 		ret = i40e_vsi_wait_queues_disabled(vsi);
5352 		if (ret)
5353 			break;
5354 	}
5355 
5356 	return ret;
5357 }
5358 
5359 #endif
5360 
5361 /**
5362  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5363  * @pf: pointer to PF
5364  *
5365  * Get TC map for ISCSI PF type that will include iSCSI TC
5366  * and LAN TC.
5367  **/
i40e_get_iscsi_tc_map(struct i40e_pf * pf)5368 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5369 {
5370 	struct i40e_dcb_app_priority_table app;
5371 	struct i40e_hw *hw = &pf->hw;
5372 	u8 enabled_tc = 1; /* TC0 is always enabled */
5373 	u8 tc, i;
5374 	/* Get the iSCSI APP TLV */
5375 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5376 
5377 	for (i = 0; i < dcbcfg->numapps; i++) {
5378 		app = dcbcfg->app[i];
5379 		if (app.selector == I40E_APP_SEL_TCPIP &&
5380 		    app.protocolid == I40E_APP_PROTOID_ISCSI) {
5381 			tc = dcbcfg->etscfg.prioritytable[app.priority];
5382 			enabled_tc |= BIT(tc);
5383 			break;
5384 		}
5385 	}
5386 
5387 	return enabled_tc;
5388 }
5389 
5390 /**
5391  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5392  * @dcbcfg: the corresponding DCBx configuration structure
5393  *
5394  * Return the number of TCs from given DCBx configuration
5395  **/
i40e_dcb_get_num_tc(struct i40e_dcbx_config * dcbcfg)5396 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5397 {
5398 	int i, tc_unused = 0;
5399 	u8 num_tc = 0;
5400 	u8 ret = 0;
5401 
5402 	/* Scan the ETS Config Priority Table to find
5403 	 * traffic class enabled for a given priority
5404 	 * and create a bitmask of enabled TCs
5405 	 */
5406 	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5407 		num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5408 
5409 	/* Now scan the bitmask to check for
5410 	 * contiguous TCs starting with TC0
5411 	 */
5412 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5413 		if (num_tc & BIT(i)) {
5414 			if (!tc_unused) {
5415 				ret++;
5416 			} else {
5417 				pr_err("Non-contiguous TC - Disabling DCB\n");
5418 				return 1;
5419 			}
5420 		} else {
5421 			tc_unused = 1;
5422 		}
5423 	}
5424 
5425 	/* There is always at least TC0 */
5426 	if (!ret)
5427 		ret = 1;
5428 
5429 	return ret;
5430 }
5431 
5432 /**
5433  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5434  * @dcbcfg: the corresponding DCBx configuration structure
5435  *
5436  * Query the current DCB configuration and return the number of
5437  * traffic classes enabled from the given DCBX config
5438  **/
i40e_dcb_get_enabled_tc(struct i40e_dcbx_config * dcbcfg)5439 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5440 {
5441 	u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5442 	u8 enabled_tc = 1;
5443 	u8 i;
5444 
5445 	for (i = 0; i < num_tc; i++)
5446 		enabled_tc |= BIT(i);
5447 
5448 	return enabled_tc;
5449 }
5450 
5451 /**
5452  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5453  * @pf: PF being queried
5454  *
5455  * Query the current MQPRIO configuration and return the number of
5456  * traffic classes enabled.
5457  **/
i40e_mqprio_get_enabled_tc(struct i40e_pf * pf)5458 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5459 {
5460 	struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
5461 	u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5462 	u8 enabled_tc = 1, i;
5463 
5464 	for (i = 1; i < num_tc; i++)
5465 		enabled_tc |= BIT(i);
5466 	return enabled_tc;
5467 }
5468 
5469 /**
5470  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5471  * @pf: PF being queried
5472  *
5473  * Return number of traffic classes enabled for the given PF
5474  **/
i40e_pf_get_num_tc(struct i40e_pf * pf)5475 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5476 {
5477 	u8 i, enabled_tc = 1;
5478 	u8 num_tc = 0;
5479 
5480 	if (i40e_is_tc_mqprio_enabled(pf)) {
5481 		struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
5482 
5483 		return vsi->mqprio_qopt.qopt.num_tc;
5484 	}
5485 
5486 	/* If neither MQPRIO nor DCB is enabled, then always use single TC */
5487 	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5488 		return 1;
5489 
5490 	/* SFP mode will be enabled for all TCs on port */
5491 	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5492 		return i40e_dcb_get_num_tc(&pf->hw.local_dcbx_config);
5493 
5494 	/* MFP mode return count of enabled TCs for this PF */
5495 	if (pf->hw.func_caps.iscsi)
5496 		enabled_tc =  i40e_get_iscsi_tc_map(pf);
5497 	else
5498 		return 1; /* Only TC0 */
5499 
5500 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5501 		if (enabled_tc & BIT(i))
5502 			num_tc++;
5503 	}
5504 	return num_tc;
5505 }
5506 
5507 /**
5508  * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5509  * @pf: PF being queried
5510  *
5511  * Return a bitmap for enabled traffic classes for this PF.
5512  **/
i40e_pf_get_tc_map(struct i40e_pf * pf)5513 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5514 {
5515 	if (i40e_is_tc_mqprio_enabled(pf))
5516 		return i40e_mqprio_get_enabled_tc(pf);
5517 
5518 	/* If neither MQPRIO nor DCB is enabled for this PF then just return
5519 	 * default TC
5520 	 */
5521 	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5522 		return I40E_DEFAULT_TRAFFIC_CLASS;
5523 
5524 	/* SFP mode we want PF to be enabled for all TCs */
5525 	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5526 		return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5527 
5528 	/* MFP enabled and iSCSI PF type */
5529 	if (pf->hw.func_caps.iscsi)
5530 		return i40e_get_iscsi_tc_map(pf);
5531 	else
5532 		return I40E_DEFAULT_TRAFFIC_CLASS;
5533 }
5534 
5535 /**
5536  * i40e_vsi_get_bw_info - Query VSI BW Information
5537  * @vsi: the VSI being queried
5538  *
5539  * Returns 0 on success, negative value on failure
5540  **/
i40e_vsi_get_bw_info(struct i40e_vsi * vsi)5541 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5542 {
5543 	struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5544 	struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5545 	struct i40e_pf *pf = vsi->back;
5546 	struct i40e_hw *hw = &pf->hw;
5547 	u32 tc_bw_max;
5548 	int ret;
5549 	int i;
5550 
5551 	/* Get the VSI level BW configuration */
5552 	ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5553 	if (ret) {
5554 		dev_info(&pf->pdev->dev,
5555 			 "couldn't get PF vsi bw config, err %pe aq_err %s\n",
5556 			 ERR_PTR(ret),
5557 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5558 		return -EINVAL;
5559 	}
5560 
5561 	/* Get the VSI level BW configuration per TC */
5562 	ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5563 					       NULL);
5564 	if (ret) {
5565 		dev_info(&pf->pdev->dev,
5566 			 "couldn't get PF vsi ets bw config, err %pe aq_err %s\n",
5567 			 ERR_PTR(ret),
5568 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5569 		return -EINVAL;
5570 	}
5571 
5572 	if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5573 		dev_info(&pf->pdev->dev,
5574 			 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5575 			 bw_config.tc_valid_bits,
5576 			 bw_ets_config.tc_valid_bits);
5577 		/* Still continuing */
5578 	}
5579 
5580 	vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5581 	vsi->bw_max_quanta = bw_config.max_bw;
5582 	tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5583 		    (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5584 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5585 		vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5586 		vsi->bw_ets_limit_credits[i] =
5587 					le16_to_cpu(bw_ets_config.credits[i]);
5588 		/* 3 bits out of 4 for each TC */
5589 		vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5590 	}
5591 
5592 	return 0;
5593 }
5594 
5595 /**
5596  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5597  * @vsi: the VSI being configured
5598  * @enabled_tc: TC bitmap
5599  * @bw_share: BW shared credits per TC
5600  *
5601  * Returns 0 on success, negative value on failure
5602  **/
i40e_vsi_configure_bw_alloc(struct i40e_vsi * vsi,u8 enabled_tc,u8 * bw_share)5603 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5604 				       u8 *bw_share)
5605 {
5606 	struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5607 	struct i40e_pf *pf = vsi->back;
5608 	int ret;
5609 	int i;
5610 
5611 	/* There is no need to reset BW when mqprio mode is on.  */
5612 	if (i40e_is_tc_mqprio_enabled(pf))
5613 		return 0;
5614 	if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
5615 		ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5616 		if (ret)
5617 			dev_info(&pf->pdev->dev,
5618 				 "Failed to reset tx rate for vsi->seid %u\n",
5619 				 vsi->seid);
5620 		return ret;
5621 	}
5622 	memset(&bw_data, 0, sizeof(bw_data));
5623 	bw_data.tc_valid_bits = enabled_tc;
5624 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5625 		bw_data.tc_bw_credits[i] = bw_share[i];
5626 
5627 	ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5628 	if (ret) {
5629 		dev_info(&pf->pdev->dev,
5630 			 "AQ command Config VSI BW allocation per TC failed = %d\n",
5631 			 pf->hw.aq.asq_last_status);
5632 		return -EINVAL;
5633 	}
5634 
5635 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5636 		vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5637 
5638 	return 0;
5639 }
5640 
5641 /**
5642  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5643  * @vsi: the VSI being configured
5644  * @enabled_tc: TC map to be enabled
5645  *
5646  **/
i40e_vsi_config_netdev_tc(struct i40e_vsi * vsi,u8 enabled_tc)5647 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5648 {
5649 	struct net_device *netdev = vsi->netdev;
5650 	struct i40e_pf *pf = vsi->back;
5651 	struct i40e_hw *hw = &pf->hw;
5652 	u8 netdev_tc = 0;
5653 	int i;
5654 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5655 
5656 	if (!netdev)
5657 		return;
5658 
5659 	if (!enabled_tc) {
5660 		netdev_reset_tc(netdev);
5661 		return;
5662 	}
5663 
5664 	/* Set up actual enabled TCs on the VSI */
5665 	if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5666 		return;
5667 
5668 	/* set per TC queues for the VSI */
5669 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5670 		/* Only set TC queues for enabled tcs
5671 		 *
5672 		 * e.g. For a VSI that has TC0 and TC3 enabled the
5673 		 * enabled_tc bitmap would be 0x00001001; the driver
5674 		 * will set the numtc for netdev as 2 that will be
5675 		 * referenced by the netdev layer as TC 0 and 1.
5676 		 */
5677 		if (vsi->tc_config.enabled_tc & BIT(i))
5678 			netdev_set_tc_queue(netdev,
5679 					vsi->tc_config.tc_info[i].netdev_tc,
5680 					vsi->tc_config.tc_info[i].qcount,
5681 					vsi->tc_config.tc_info[i].qoffset);
5682 	}
5683 
5684 	if (i40e_is_tc_mqprio_enabled(pf))
5685 		return;
5686 
5687 	/* Assign UP2TC map for the VSI */
5688 	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5689 		/* Get the actual TC# for the UP */
5690 		u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5691 		/* Get the mapped netdev TC# for the UP */
5692 		netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5693 		netdev_set_prio_tc_map(netdev, i, netdev_tc);
5694 	}
5695 }
5696 
5697 /**
5698  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5699  * @vsi: the VSI being configured
5700  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5701  **/
i40e_vsi_update_queue_map(struct i40e_vsi * vsi,struct i40e_vsi_context * ctxt)5702 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5703 				      struct i40e_vsi_context *ctxt)
5704 {
5705 	/* copy just the sections touched not the entire info
5706 	 * since not all sections are valid as returned by
5707 	 * update vsi params
5708 	 */
5709 	vsi->info.mapping_flags = ctxt->info.mapping_flags;
5710 	memcpy(&vsi->info.queue_mapping,
5711 	       &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5712 	memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5713 	       sizeof(vsi->info.tc_mapping));
5714 }
5715 
5716 /**
5717  * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5718  * @vsi: the VSI being reconfigured
5719  * @vsi_offset: offset from main VF VSI
5720  */
i40e_update_adq_vsi_queues(struct i40e_vsi * vsi,int vsi_offset)5721 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5722 {
5723 	struct i40e_vsi_context ctxt = {};
5724 	struct i40e_pf *pf;
5725 	struct i40e_hw *hw;
5726 	int ret;
5727 
5728 	if (!vsi)
5729 		return -EINVAL;
5730 	pf = vsi->back;
5731 	hw = &pf->hw;
5732 
5733 	ctxt.seid = vsi->seid;
5734 	ctxt.pf_num = hw->pf_id;
5735 	ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5736 	ctxt.uplink_seid = vsi->uplink_seid;
5737 	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5738 	ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5739 	ctxt.info = vsi->info;
5740 
5741 	i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
5742 				 false);
5743 	if (vsi->reconfig_rss) {
5744 		vsi->rss_size = min_t(int, pf->alloc_rss_size,
5745 				      vsi->num_queue_pairs);
5746 		ret = i40e_vsi_config_rss(vsi);
5747 		if (ret) {
5748 			dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5749 			return ret;
5750 		}
5751 		vsi->reconfig_rss = false;
5752 	}
5753 
5754 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5755 	if (ret) {
5756 		dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n",
5757 			 ERR_PTR(ret),
5758 			 i40e_aq_str(hw, hw->aq.asq_last_status));
5759 		return ret;
5760 	}
5761 	/* update the local VSI info with updated queue map */
5762 	i40e_vsi_update_queue_map(vsi, &ctxt);
5763 	vsi->info.valid_sections = 0;
5764 
5765 	return ret;
5766 }
5767 
5768 /**
5769  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5770  * @vsi: VSI to be configured
5771  * @enabled_tc: TC bitmap
5772  *
5773  * This configures a particular VSI for TCs that are mapped to the
5774  * given TC bitmap. It uses default bandwidth share for TCs across
5775  * VSIs to configure TC for a particular VSI.
5776  *
5777  * NOTE:
5778  * It is expected that the VSI queues have been quisced before calling
5779  * this function.
5780  **/
i40e_vsi_config_tc(struct i40e_vsi * vsi,u8 enabled_tc)5781 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5782 {
5783 	u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5784 	struct i40e_pf *pf = vsi->back;
5785 	struct i40e_hw *hw = &pf->hw;
5786 	struct i40e_vsi_context ctxt;
5787 	int ret = 0;
5788 	int i;
5789 
5790 	/* Check if enabled_tc is same as existing or new TCs */
5791 	if (vsi->tc_config.enabled_tc == enabled_tc &&
5792 	    vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5793 		return ret;
5794 
5795 	/* Enable ETS TCs with equal BW Share for now across all VSIs */
5796 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5797 		if (enabled_tc & BIT(i))
5798 			bw_share[i] = 1;
5799 	}
5800 
5801 	ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5802 	if (ret) {
5803 		struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5804 
5805 		dev_info(&pf->pdev->dev,
5806 			 "Failed configuring TC map %d for VSI %d\n",
5807 			 enabled_tc, vsi->seid);
5808 		ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5809 						  &bw_config, NULL);
5810 		if (ret) {
5811 			dev_info(&pf->pdev->dev,
5812 				 "Failed querying vsi bw info, err %pe aq_err %s\n",
5813 				 ERR_PTR(ret),
5814 				 i40e_aq_str(hw, hw->aq.asq_last_status));
5815 			goto out;
5816 		}
5817 		if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5818 			u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5819 
5820 			if (!valid_tc)
5821 				valid_tc = bw_config.tc_valid_bits;
5822 			/* Always enable TC0, no matter what */
5823 			valid_tc |= 1;
5824 			dev_info(&pf->pdev->dev,
5825 				 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5826 				 enabled_tc, bw_config.tc_valid_bits, valid_tc);
5827 			enabled_tc = valid_tc;
5828 		}
5829 
5830 		ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5831 		if (ret) {
5832 			dev_err(&pf->pdev->dev,
5833 				"Unable to  configure TC map %d for VSI %d\n",
5834 				enabled_tc, vsi->seid);
5835 			goto out;
5836 		}
5837 	}
5838 
5839 	/* Update Queue Pairs Mapping for currently enabled UPs */
5840 	ctxt.seid = vsi->seid;
5841 	ctxt.pf_num = vsi->back->hw.pf_id;
5842 	ctxt.vf_num = 0;
5843 	ctxt.uplink_seid = vsi->uplink_seid;
5844 	ctxt.info = vsi->info;
5845 	if (i40e_is_tc_mqprio_enabled(pf)) {
5846 		ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5847 		if (ret)
5848 			goto out;
5849 	} else {
5850 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5851 	}
5852 
5853 	/* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5854 	 * queues changed.
5855 	 */
5856 	if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5857 		vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5858 				      vsi->num_queue_pairs);
5859 		ret = i40e_vsi_config_rss(vsi);
5860 		if (ret) {
5861 			dev_info(&vsi->back->pdev->dev,
5862 				 "Failed to reconfig rss for num_queues\n");
5863 			return ret;
5864 		}
5865 		vsi->reconfig_rss = false;
5866 	}
5867 	if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
5868 		ctxt.info.valid_sections |=
5869 				cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5870 		ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5871 	}
5872 
5873 	/* Update the VSI after updating the VSI queue-mapping
5874 	 * information
5875 	 */
5876 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5877 	if (ret) {
5878 		dev_info(&pf->pdev->dev,
5879 			 "Update vsi tc config failed, err %pe aq_err %s\n",
5880 			 ERR_PTR(ret),
5881 			 i40e_aq_str(hw, hw->aq.asq_last_status));
5882 		goto out;
5883 	}
5884 	/* update the local VSI info with updated queue map */
5885 	i40e_vsi_update_queue_map(vsi, &ctxt);
5886 	vsi->info.valid_sections = 0;
5887 
5888 	/* Update current VSI BW information */
5889 	ret = i40e_vsi_get_bw_info(vsi);
5890 	if (ret) {
5891 		dev_info(&pf->pdev->dev,
5892 			 "Failed updating vsi bw info, err %pe aq_err %s\n",
5893 			 ERR_PTR(ret),
5894 			 i40e_aq_str(hw, hw->aq.asq_last_status));
5895 		goto out;
5896 	}
5897 
5898 	/* Update the netdev TC setup */
5899 	i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5900 out:
5901 	return ret;
5902 }
5903 
5904 /**
5905  * i40e_vsi_reconfig_tc - Reconfigure VSI Tx Scheduler for stored TC map
5906  * @vsi: VSI to be reconfigured
5907  *
5908  * This reconfigures a particular VSI for TCs that are mapped to the
5909  * TC bitmap stored previously for the VSI.
5910  *
5911  * Context: It is expected that the VSI queues have been quisced before
5912  *          calling this function.
5913  *
5914  * Return: 0 on success, negative value on failure
5915  **/
i40e_vsi_reconfig_tc(struct i40e_vsi * vsi)5916 static int i40e_vsi_reconfig_tc(struct i40e_vsi *vsi)
5917 {
5918 	u8 enabled_tc;
5919 
5920 	enabled_tc = vsi->tc_config.enabled_tc;
5921 	vsi->tc_config.enabled_tc = 0;
5922 
5923 	return i40e_vsi_config_tc(vsi, enabled_tc);
5924 }
5925 
5926 /**
5927  * i40e_get_link_speed - Returns link speed for the interface
5928  * @vsi: VSI to be configured
5929  *
5930  **/
i40e_get_link_speed(struct i40e_vsi * vsi)5931 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5932 {
5933 	struct i40e_pf *pf = vsi->back;
5934 
5935 	switch (pf->hw.phy.link_info.link_speed) {
5936 	case I40E_LINK_SPEED_40GB:
5937 		return 40000;
5938 	case I40E_LINK_SPEED_25GB:
5939 		return 25000;
5940 	case I40E_LINK_SPEED_20GB:
5941 		return 20000;
5942 	case I40E_LINK_SPEED_10GB:
5943 		return 10000;
5944 	case I40E_LINK_SPEED_1GB:
5945 		return 1000;
5946 	default:
5947 		return -EINVAL;
5948 	}
5949 }
5950 
5951 /**
5952  * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5953  * @vsi: Pointer to vsi structure
5954  * @max_tx_rate: max TX rate in bytes to be converted into Mbits
5955  *
5956  * Helper function to convert units before send to set BW limit
5957  **/
i40e_bw_bytes_to_mbits(struct i40e_vsi * vsi,u64 max_tx_rate)5958 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)
5959 {
5960 	if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {
5961 		dev_warn(&vsi->back->pdev->dev,
5962 			 "Setting max tx rate to minimum usable value of 50Mbps.\n");
5963 		max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5964 	} else {
5965 		do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
5966 	}
5967 
5968 	return max_tx_rate;
5969 }
5970 
5971 /**
5972  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5973  * @vsi: VSI to be configured
5974  * @seid: seid of the channel/VSI
5975  * @max_tx_rate: max TX rate to be configured as BW limit
5976  *
5977  * Helper function to set BW limit for a given VSI
5978  **/
i40e_set_bw_limit(struct i40e_vsi * vsi,u16 seid,u64 max_tx_rate)5979 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5980 {
5981 	struct i40e_pf *pf = vsi->back;
5982 	u64 credits = 0;
5983 	int speed = 0;
5984 	int ret = 0;
5985 
5986 	speed = i40e_get_link_speed(vsi);
5987 	if (max_tx_rate > speed) {
5988 		dev_err(&pf->pdev->dev,
5989 			"Invalid max tx rate %llu specified for VSI seid %d.",
5990 			max_tx_rate, seid);
5991 		return -EINVAL;
5992 	}
5993 	if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {
5994 		dev_warn(&pf->pdev->dev,
5995 			 "Setting max tx rate to minimum usable value of 50Mbps.\n");
5996 		max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5997 	}
5998 
5999 	/* Tx rate credits are in values of 50Mbps, 0 is disabled */
6000 	credits = max_tx_rate;
6001 	do_div(credits, I40E_BW_CREDIT_DIVISOR);
6002 	ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
6003 					  I40E_MAX_BW_INACTIVE_ACCUM, NULL);
6004 	if (ret)
6005 		dev_err(&pf->pdev->dev,
6006 			"Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n",
6007 			max_tx_rate, seid, ERR_PTR(ret),
6008 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6009 	return ret;
6010 }
6011 
6012 /**
6013  * i40e_remove_queue_channels - Remove queue channels for the TCs
6014  * @vsi: VSI to be configured
6015  *
6016  * Remove queue channels for the TCs
6017  **/
i40e_remove_queue_channels(struct i40e_vsi * vsi)6018 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
6019 {
6020 	enum i40e_admin_queue_err last_aq_status;
6021 	struct i40e_cloud_filter *cfilter;
6022 	struct i40e_channel *ch, *ch_tmp;
6023 	struct i40e_pf *pf = vsi->back;
6024 	struct hlist_node *node;
6025 	int ret, i;
6026 
6027 	/* Reset rss size that was stored when reconfiguring rss for
6028 	 * channel VSIs with non-power-of-2 queue count.
6029 	 */
6030 	vsi->current_rss_size = 0;
6031 
6032 	/* perform cleanup for channels if they exist */
6033 	if (list_empty(&vsi->ch_list))
6034 		return;
6035 
6036 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6037 		struct i40e_vsi *p_vsi;
6038 
6039 		list_del(&ch->list);
6040 		p_vsi = ch->parent_vsi;
6041 		if (!p_vsi || !ch->initialized) {
6042 			kfree(ch);
6043 			continue;
6044 		}
6045 		/* Reset queue contexts */
6046 		for (i = 0; i < ch->num_queue_pairs; i++) {
6047 			struct i40e_ring *tx_ring, *rx_ring;
6048 			u16 pf_q;
6049 
6050 			pf_q = ch->base_queue + i;
6051 			tx_ring = vsi->tx_rings[pf_q];
6052 			tx_ring->ch = NULL;
6053 
6054 			rx_ring = vsi->rx_rings[pf_q];
6055 			rx_ring->ch = NULL;
6056 		}
6057 
6058 		/* Reset BW configured for this VSI via mqprio */
6059 		ret = i40e_set_bw_limit(vsi, ch->seid, 0);
6060 		if (ret)
6061 			dev_info(&vsi->back->pdev->dev,
6062 				 "Failed to reset tx rate for ch->seid %u\n",
6063 				 ch->seid);
6064 
6065 		/* delete cloud filters associated with this channel */
6066 		hlist_for_each_entry_safe(cfilter, node,
6067 					  &pf->cloud_filter_list, cloud_node) {
6068 			if (cfilter->seid != ch->seid)
6069 				continue;
6070 
6071 			hash_del(&cfilter->cloud_node);
6072 			if (cfilter->dst_port)
6073 				ret = i40e_add_del_cloud_filter_big_buf(vsi,
6074 									cfilter,
6075 									false);
6076 			else
6077 				ret = i40e_add_del_cloud_filter(vsi, cfilter,
6078 								false);
6079 			last_aq_status = pf->hw.aq.asq_last_status;
6080 			if (ret)
6081 				dev_info(&pf->pdev->dev,
6082 					 "Failed to delete cloud filter, err %pe aq_err %s\n",
6083 					 ERR_PTR(ret),
6084 					 i40e_aq_str(&pf->hw, last_aq_status));
6085 			kfree(cfilter);
6086 		}
6087 
6088 		/* delete VSI from FW */
6089 		ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
6090 					     NULL);
6091 		if (ret)
6092 			dev_err(&vsi->back->pdev->dev,
6093 				"unable to remove channel (%d) for parent VSI(%d)\n",
6094 				ch->seid, p_vsi->seid);
6095 		kfree(ch);
6096 	}
6097 	INIT_LIST_HEAD(&vsi->ch_list);
6098 }
6099 
6100 /**
6101  * i40e_get_max_queues_for_channel
6102  * @vsi: ptr to VSI to which channels are associated with
6103  *
6104  * Helper function which returns max value among the queue counts set on the
6105  * channels/TCs created.
6106  **/
i40e_get_max_queues_for_channel(struct i40e_vsi * vsi)6107 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
6108 {
6109 	struct i40e_channel *ch, *ch_tmp;
6110 	int max = 0;
6111 
6112 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6113 		if (!ch->initialized)
6114 			continue;
6115 		if (ch->num_queue_pairs > max)
6116 			max = ch->num_queue_pairs;
6117 	}
6118 
6119 	return max;
6120 }
6121 
6122 /**
6123  * i40e_validate_num_queues - validate num_queues w.r.t channel
6124  * @pf: ptr to PF device
6125  * @num_queues: number of queues
6126  * @vsi: the parent VSI
6127  * @reconfig_rss: indicates should the RSS be reconfigured or not
6128  *
6129  * This function validates number of queues in the context of new channel
6130  * which is being established and determines if RSS should be reconfigured
6131  * or not for parent VSI.
6132  **/
i40e_validate_num_queues(struct i40e_pf * pf,int num_queues,struct i40e_vsi * vsi,bool * reconfig_rss)6133 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
6134 				    struct i40e_vsi *vsi, bool *reconfig_rss)
6135 {
6136 	int max_ch_queues;
6137 
6138 	if (!reconfig_rss)
6139 		return -EINVAL;
6140 
6141 	*reconfig_rss = false;
6142 	if (vsi->current_rss_size) {
6143 		if (num_queues > vsi->current_rss_size) {
6144 			dev_dbg(&pf->pdev->dev,
6145 				"Error: num_queues (%d) > vsi's current_size(%d)\n",
6146 				num_queues, vsi->current_rss_size);
6147 			return -EINVAL;
6148 		} else if ((num_queues < vsi->current_rss_size) &&
6149 			   (!is_power_of_2(num_queues))) {
6150 			dev_dbg(&pf->pdev->dev,
6151 				"Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
6152 				num_queues, vsi->current_rss_size);
6153 			return -EINVAL;
6154 		}
6155 	}
6156 
6157 	if (!is_power_of_2(num_queues)) {
6158 		/* Find the max num_queues configured for channel if channel
6159 		 * exist.
6160 		 * if channel exist, then enforce 'num_queues' to be more than
6161 		 * max ever queues configured for channel.
6162 		 */
6163 		max_ch_queues = i40e_get_max_queues_for_channel(vsi);
6164 		if (num_queues < max_ch_queues) {
6165 			dev_dbg(&pf->pdev->dev,
6166 				"Error: num_queues (%d) < max queues configured for channel(%d)\n",
6167 				num_queues, max_ch_queues);
6168 			return -EINVAL;
6169 		}
6170 		*reconfig_rss = true;
6171 	}
6172 
6173 	return 0;
6174 }
6175 
6176 /**
6177  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
6178  * @vsi: the VSI being setup
6179  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
6180  *
6181  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
6182  **/
i40e_vsi_reconfig_rss(struct i40e_vsi * vsi,u16 rss_size)6183 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
6184 {
6185 	struct i40e_pf *pf = vsi->back;
6186 	u8 seed[I40E_HKEY_ARRAY_SIZE];
6187 	struct i40e_hw *hw = &pf->hw;
6188 	int local_rss_size;
6189 	u8 *lut;
6190 	int ret;
6191 
6192 	if (!vsi->rss_size)
6193 		return -EINVAL;
6194 
6195 	if (rss_size > vsi->rss_size)
6196 		return -EINVAL;
6197 
6198 	local_rss_size = min_t(int, vsi->rss_size, rss_size);
6199 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
6200 	if (!lut)
6201 		return -ENOMEM;
6202 
6203 	/* Ignoring user configured lut if there is one */
6204 	i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
6205 
6206 	/* Use user configured hash key if there is one, otherwise
6207 	 * use default.
6208 	 */
6209 	if (vsi->rss_hkey_user)
6210 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
6211 	else
6212 		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
6213 
6214 	ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
6215 	if (ret) {
6216 		dev_info(&pf->pdev->dev,
6217 			 "Cannot set RSS lut, err %pe aq_err %s\n",
6218 			 ERR_PTR(ret),
6219 			 i40e_aq_str(hw, hw->aq.asq_last_status));
6220 		kfree(lut);
6221 		return ret;
6222 	}
6223 	kfree(lut);
6224 
6225 	/* Do the update w.r.t. storing rss_size */
6226 	if (!vsi->orig_rss_size)
6227 		vsi->orig_rss_size = vsi->rss_size;
6228 	vsi->current_rss_size = local_rss_size;
6229 
6230 	return ret;
6231 }
6232 
6233 /**
6234  * i40e_channel_setup_queue_map - Setup a channel queue map
6235  * @pf: ptr to PF device
6236  * @ctxt: VSI context structure
6237  * @ch: ptr to channel structure
6238  *
6239  * Setup queue map for a specific channel
6240  **/
i40e_channel_setup_queue_map(struct i40e_pf * pf,struct i40e_vsi_context * ctxt,struct i40e_channel * ch)6241 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
6242 					 struct i40e_vsi_context *ctxt,
6243 					 struct i40e_channel *ch)
6244 {
6245 	u16 qcount, qmap, sections = 0;
6246 	u8 offset = 0;
6247 	int pow;
6248 
6249 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
6250 	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
6251 
6252 	qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
6253 	ch->num_queue_pairs = qcount;
6254 
6255 	/* find the next higher power-of-2 of num queue pairs */
6256 	pow = ilog2(qcount);
6257 	if (!is_power_of_2(qcount))
6258 		pow++;
6259 
6260 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6261 		(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
6262 
6263 	/* Setup queue TC[0].qmap for given VSI context */
6264 	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
6265 
6266 	ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
6267 	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
6268 	ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
6269 	ctxt->info.valid_sections |= cpu_to_le16(sections);
6270 }
6271 
6272 /**
6273  * i40e_add_channel - add a channel by adding VSI
6274  * @pf: ptr to PF device
6275  * @uplink_seid: underlying HW switching element (VEB) ID
6276  * @ch: ptr to channel structure
6277  *
6278  * Add a channel (VSI) using add_vsi and queue_map
6279  **/
i40e_add_channel(struct i40e_pf * pf,u16 uplink_seid,struct i40e_channel * ch)6280 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
6281 			    struct i40e_channel *ch)
6282 {
6283 	struct i40e_hw *hw = &pf->hw;
6284 	struct i40e_vsi_context ctxt;
6285 	u8 enabled_tc = 0x1; /* TC0 enabled */
6286 	int ret;
6287 
6288 	if (ch->type != I40E_VSI_VMDQ2) {
6289 		dev_info(&pf->pdev->dev,
6290 			 "add new vsi failed, ch->type %d\n", ch->type);
6291 		return -EINVAL;
6292 	}
6293 
6294 	memset(&ctxt, 0, sizeof(ctxt));
6295 	ctxt.pf_num = hw->pf_id;
6296 	ctxt.vf_num = 0;
6297 	ctxt.uplink_seid = uplink_seid;
6298 	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
6299 	if (ch->type == I40E_VSI_VMDQ2)
6300 		ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6301 
6302 	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6303 		ctxt.info.valid_sections |=
6304 		     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6305 		ctxt.info.switch_id =
6306 		   cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6307 	}
6308 
6309 	/* Set queue map for a given VSI context */
6310 	i40e_channel_setup_queue_map(pf, &ctxt, ch);
6311 
6312 	/* Now time to create VSI */
6313 	ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6314 	if (ret) {
6315 		dev_info(&pf->pdev->dev,
6316 			 "add new vsi failed, err %pe aq_err %s\n",
6317 			 ERR_PTR(ret),
6318 			 i40e_aq_str(&pf->hw,
6319 				     pf->hw.aq.asq_last_status));
6320 		return -ENOENT;
6321 	}
6322 
6323 	/* Success, update channel, set enabled_tc only if the channel
6324 	 * is not a macvlan
6325 	 */
6326 	ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6327 	ch->seid = ctxt.seid;
6328 	ch->vsi_number = ctxt.vsi_number;
6329 	ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6330 
6331 	/* copy just the sections touched not the entire info
6332 	 * since not all sections are valid as returned by
6333 	 * update vsi params
6334 	 */
6335 	ch->info.mapping_flags = ctxt.info.mapping_flags;
6336 	memcpy(&ch->info.queue_mapping,
6337 	       &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6338 	memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6339 	       sizeof(ctxt.info.tc_mapping));
6340 
6341 	return 0;
6342 }
6343 
i40e_channel_config_bw(struct i40e_vsi * vsi,struct i40e_channel * ch,u8 * bw_share)6344 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6345 				  u8 *bw_share)
6346 {
6347 	struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6348 	int ret;
6349 	int i;
6350 
6351 	memset(&bw_data, 0, sizeof(bw_data));
6352 	bw_data.tc_valid_bits = ch->enabled_tc;
6353 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6354 		bw_data.tc_bw_credits[i] = bw_share[i];
6355 
6356 	ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
6357 				       &bw_data, NULL);
6358 	if (ret) {
6359 		dev_info(&vsi->back->pdev->dev,
6360 			 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6361 			 vsi->back->hw.aq.asq_last_status, ch->seid);
6362 		return -EINVAL;
6363 	}
6364 
6365 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6366 		ch->info.qs_handle[i] = bw_data.qs_handles[i];
6367 
6368 	return 0;
6369 }
6370 
6371 /**
6372  * i40e_channel_config_tx_ring - config TX ring associated with new channel
6373  * @pf: ptr to PF device
6374  * @vsi: the VSI being setup
6375  * @ch: ptr to channel structure
6376  *
6377  * Configure TX rings associated with channel (VSI) since queues are being
6378  * from parent VSI.
6379  **/
i40e_channel_config_tx_ring(struct i40e_pf * pf,struct i40e_vsi * vsi,struct i40e_channel * ch)6380 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6381 				       struct i40e_vsi *vsi,
6382 				       struct i40e_channel *ch)
6383 {
6384 	u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6385 	int ret;
6386 	int i;
6387 
6388 	/* Enable ETS TCs with equal BW Share for now across all VSIs */
6389 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6390 		if (ch->enabled_tc & BIT(i))
6391 			bw_share[i] = 1;
6392 	}
6393 
6394 	/* configure BW for new VSI */
6395 	ret = i40e_channel_config_bw(vsi, ch, bw_share);
6396 	if (ret) {
6397 		dev_info(&vsi->back->pdev->dev,
6398 			 "Failed configuring TC map %d for channel (seid %u)\n",
6399 			 ch->enabled_tc, ch->seid);
6400 		return ret;
6401 	}
6402 
6403 	for (i = 0; i < ch->num_queue_pairs; i++) {
6404 		struct i40e_ring *tx_ring, *rx_ring;
6405 		u16 pf_q;
6406 
6407 		pf_q = ch->base_queue + i;
6408 
6409 		/* Get to TX ring ptr of main VSI, for re-setup TX queue
6410 		 * context
6411 		 */
6412 		tx_ring = vsi->tx_rings[pf_q];
6413 		tx_ring->ch = ch;
6414 
6415 		/* Get the RX ring ptr */
6416 		rx_ring = vsi->rx_rings[pf_q];
6417 		rx_ring->ch = ch;
6418 	}
6419 
6420 	return 0;
6421 }
6422 
6423 /**
6424  * i40e_setup_hw_channel - setup new channel
6425  * @pf: ptr to PF device
6426  * @vsi: the VSI being setup
6427  * @ch: ptr to channel structure
6428  * @uplink_seid: underlying HW switching element (VEB) ID
6429  * @type: type of channel to be created (VMDq2/VF)
6430  *
6431  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6432  * and configures TX rings accordingly
6433  **/
i40e_setup_hw_channel(struct i40e_pf * pf,struct i40e_vsi * vsi,struct i40e_channel * ch,u16 uplink_seid,u8 type)6434 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6435 					struct i40e_vsi *vsi,
6436 					struct i40e_channel *ch,
6437 					u16 uplink_seid, u8 type)
6438 {
6439 	int ret;
6440 
6441 	ch->initialized = false;
6442 	ch->base_queue = vsi->next_base_queue;
6443 	ch->type = type;
6444 
6445 	/* Proceed with creation of channel (VMDq2) VSI */
6446 	ret = i40e_add_channel(pf, uplink_seid, ch);
6447 	if (ret) {
6448 		dev_info(&pf->pdev->dev,
6449 			 "failed to add_channel using uplink_seid %u\n",
6450 			 uplink_seid);
6451 		return ret;
6452 	}
6453 
6454 	/* Mark the successful creation of channel */
6455 	ch->initialized = true;
6456 
6457 	/* Reconfigure TX queues using QTX_CTL register */
6458 	ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6459 	if (ret) {
6460 		dev_info(&pf->pdev->dev,
6461 			 "failed to configure TX rings for channel %u\n",
6462 			 ch->seid);
6463 		return ret;
6464 	}
6465 
6466 	/* update 'next_base_queue' */
6467 	vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6468 	dev_dbg(&pf->pdev->dev,
6469 		"Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6470 		ch->seid, ch->vsi_number, ch->stat_counter_idx,
6471 		ch->num_queue_pairs,
6472 		vsi->next_base_queue);
6473 	return ret;
6474 }
6475 
6476 /**
6477  * i40e_setup_channel - setup new channel using uplink element
6478  * @pf: ptr to PF device
6479  * @vsi: pointer to the VSI to set up the channel within
6480  * @ch: ptr to channel structure
6481  *
6482  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6483  * and uplink switching element (uplink_seid)
6484  **/
i40e_setup_channel(struct i40e_pf * pf,struct i40e_vsi * vsi,struct i40e_channel * ch)6485 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6486 			       struct i40e_channel *ch)
6487 {
6488 	struct i40e_vsi *main_vsi;
6489 	u8 vsi_type;
6490 	u16 seid;
6491 	int ret;
6492 
6493 	if (vsi->type == I40E_VSI_MAIN) {
6494 		vsi_type = I40E_VSI_VMDQ2;
6495 	} else {
6496 		dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6497 			vsi->type);
6498 		return false;
6499 	}
6500 
6501 	/* underlying switching element */
6502 	main_vsi = i40e_pf_get_main_vsi(pf);
6503 	seid = main_vsi->uplink_seid;
6504 
6505 	/* create channel (VSI), configure TX rings */
6506 	ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6507 	if (ret) {
6508 		dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6509 		return false;
6510 	}
6511 
6512 	return ch->initialized ? true : false;
6513 }
6514 
6515 /**
6516  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6517  * @vsi: ptr to VSI which has PF backing
6518  *
6519  * Sets up switch mode correctly if it needs to be changed and perform
6520  * what are allowed modes.
6521  **/
i40e_validate_and_set_switch_mode(struct i40e_vsi * vsi)6522 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6523 {
6524 	u8 mode;
6525 	struct i40e_pf *pf = vsi->back;
6526 	struct i40e_hw *hw = &pf->hw;
6527 	int ret;
6528 
6529 	ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6530 	if (ret)
6531 		return -EINVAL;
6532 
6533 	if (hw->dev_caps.switch_mode) {
6534 		/* if switch mode is set, support mode2 (non-tunneled for
6535 		 * cloud filter) for now
6536 		 */
6537 		u32 switch_mode = hw->dev_caps.switch_mode &
6538 				  I40E_SWITCH_MODE_MASK;
6539 		if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6540 			if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6541 				return 0;
6542 			dev_err(&pf->pdev->dev,
6543 				"Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6544 				hw->dev_caps.switch_mode);
6545 			return -EINVAL;
6546 		}
6547 	}
6548 
6549 	/* Set Bit 7 to be valid */
6550 	mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6551 
6552 	/* Set L4type for TCP support */
6553 	mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6554 
6555 	/* Set cloud filter mode */
6556 	mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6557 
6558 	/* Prep mode field for set_switch_config */
6559 	ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6560 					pf->last_sw_conf_valid_flags,
6561 					mode, NULL);
6562 	if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6563 		dev_err(&pf->pdev->dev,
6564 			"couldn't set switch config bits, err %pe aq_err %s\n",
6565 			ERR_PTR(ret),
6566 			i40e_aq_str(hw,
6567 				    hw->aq.asq_last_status));
6568 
6569 	return ret;
6570 }
6571 
6572 /**
6573  * i40e_create_queue_channel - function to create channel
6574  * @vsi: VSI to be configured
6575  * @ch: ptr to channel (it contains channel specific params)
6576  *
6577  * This function creates channel (VSI) using num_queues specified by user,
6578  * reconfigs RSS if needed.
6579  **/
i40e_create_queue_channel(struct i40e_vsi * vsi,struct i40e_channel * ch)6580 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6581 			      struct i40e_channel *ch)
6582 {
6583 	struct i40e_pf *pf = vsi->back;
6584 	bool reconfig_rss;
6585 	int err;
6586 
6587 	if (!ch)
6588 		return -EINVAL;
6589 
6590 	if (!ch->num_queue_pairs) {
6591 		dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6592 			ch->num_queue_pairs);
6593 		return -EINVAL;
6594 	}
6595 
6596 	/* validate user requested num_queues for channel */
6597 	err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6598 				       &reconfig_rss);
6599 	if (err) {
6600 		dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6601 			 ch->num_queue_pairs);
6602 		return -EINVAL;
6603 	}
6604 
6605 	/* By default we are in VEPA mode, if this is the first VF/VMDq
6606 	 * VSI to be added switch to VEB mode.
6607 	 */
6608 
6609 	if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6610 		set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
6611 
6612 		if (vsi->type == I40E_VSI_MAIN) {
6613 			if (i40e_is_tc_mqprio_enabled(pf))
6614 				i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
6615 			else
6616 				i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6617 		}
6618 		/* now onwards for main VSI, number of queues will be value
6619 		 * of TC0's queue count
6620 		 */
6621 	}
6622 
6623 	/* By this time, vsi->cnt_q_avail shall be set to non-zero and
6624 	 * it should be more than num_queues
6625 	 */
6626 	if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6627 		dev_dbg(&pf->pdev->dev,
6628 			"Error: cnt_q_avail (%u) less than num_queues %d\n",
6629 			vsi->cnt_q_avail, ch->num_queue_pairs);
6630 		return -EINVAL;
6631 	}
6632 
6633 	/* reconfig_rss only if vsi type is MAIN_VSI */
6634 	if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6635 		err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6636 		if (err) {
6637 			dev_info(&pf->pdev->dev,
6638 				 "Error: unable to reconfig rss for num_queues (%u)\n",
6639 				 ch->num_queue_pairs);
6640 			return -EINVAL;
6641 		}
6642 	}
6643 
6644 	if (!i40e_setup_channel(pf, vsi, ch)) {
6645 		dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6646 		return -EINVAL;
6647 	}
6648 
6649 	dev_info(&pf->pdev->dev,
6650 		 "Setup channel (id:%u) utilizing num_queues %d\n",
6651 		 ch->seid, ch->num_queue_pairs);
6652 
6653 	/* configure VSI for BW limit */
6654 	if (ch->max_tx_rate) {
6655 		u64 credits = ch->max_tx_rate;
6656 
6657 		if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6658 			return -EINVAL;
6659 
6660 		do_div(credits, I40E_BW_CREDIT_DIVISOR);
6661 		dev_dbg(&pf->pdev->dev,
6662 			"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6663 			ch->max_tx_rate,
6664 			credits,
6665 			ch->seid);
6666 	}
6667 
6668 	/* in case of VF, this will be main SRIOV VSI */
6669 	ch->parent_vsi = vsi;
6670 
6671 	/* and update main_vsi's count for queue_available to use */
6672 	vsi->cnt_q_avail -= ch->num_queue_pairs;
6673 
6674 	return 0;
6675 }
6676 
6677 /**
6678  * i40e_configure_queue_channels - Add queue channel for the given TCs
6679  * @vsi: VSI to be configured
6680  *
6681  * Configures queue channel mapping to the given TCs
6682  **/
i40e_configure_queue_channels(struct i40e_vsi * vsi)6683 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6684 {
6685 	struct i40e_channel *ch;
6686 	u64 max_rate = 0;
6687 	int ret = 0, i;
6688 
6689 	/* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6690 	vsi->tc_seid_map[0] = vsi->seid;
6691 	for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6692 		if (vsi->tc_config.enabled_tc & BIT(i)) {
6693 			ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6694 			if (!ch) {
6695 				ret = -ENOMEM;
6696 				goto err_free;
6697 			}
6698 
6699 			INIT_LIST_HEAD(&ch->list);
6700 			ch->num_queue_pairs =
6701 				vsi->tc_config.tc_info[i].qcount;
6702 			ch->base_queue =
6703 				vsi->tc_config.tc_info[i].qoffset;
6704 
6705 			/* Bandwidth limit through tc interface is in bytes/s,
6706 			 * change to Mbit/s
6707 			 */
6708 			max_rate = vsi->mqprio_qopt.max_rate[i];
6709 			do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6710 			ch->max_tx_rate = max_rate;
6711 
6712 			list_add_tail(&ch->list, &vsi->ch_list);
6713 
6714 			ret = i40e_create_queue_channel(vsi, ch);
6715 			if (ret) {
6716 				dev_err(&vsi->back->pdev->dev,
6717 					"Failed creating queue channel with TC%d: queues %d\n",
6718 					i, ch->num_queue_pairs);
6719 				goto err_free;
6720 			}
6721 			vsi->tc_seid_map[i] = ch->seid;
6722 		}
6723 	}
6724 
6725 	/* reset to reconfigure TX queue contexts */
6726 	i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true);
6727 	return ret;
6728 
6729 err_free:
6730 	i40e_remove_queue_channels(vsi);
6731 	return ret;
6732 }
6733 
6734 /**
6735  * i40e_veb_config_tc - Configure TCs for given VEB
6736  * @veb: given VEB
6737  * @enabled_tc: TC bitmap
6738  *
6739  * Configures given TC bitmap for VEB (switching) element
6740  **/
i40e_veb_config_tc(struct i40e_veb * veb,u8 enabled_tc)6741 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6742 {
6743 	struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6744 	struct i40e_pf *pf = veb->pf;
6745 	int ret = 0;
6746 	int i;
6747 
6748 	/* No TCs or already enabled TCs just return */
6749 	if (!enabled_tc || veb->enabled_tc == enabled_tc)
6750 		return ret;
6751 
6752 	bw_data.tc_valid_bits = enabled_tc;
6753 	/* bw_data.absolute_credits is not set (relative) */
6754 
6755 	/* Enable ETS TCs with equal BW Share for now */
6756 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6757 		if (enabled_tc & BIT(i))
6758 			bw_data.tc_bw_share_credits[i] = 1;
6759 	}
6760 
6761 	ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6762 						   &bw_data, NULL);
6763 	if (ret) {
6764 		dev_info(&pf->pdev->dev,
6765 			 "VEB bw config failed, err %pe aq_err %s\n",
6766 			 ERR_PTR(ret),
6767 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6768 		goto out;
6769 	}
6770 
6771 	/* Update the BW information */
6772 	ret = i40e_veb_get_bw_info(veb);
6773 	if (ret) {
6774 		dev_info(&pf->pdev->dev,
6775 			 "Failed getting veb bw config, err %pe aq_err %s\n",
6776 			 ERR_PTR(ret),
6777 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6778 	}
6779 
6780 out:
6781 	return ret;
6782 }
6783 
6784 #ifdef CONFIG_I40E_DCB
6785 /**
6786  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6787  * @pf: PF struct
6788  *
6789  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6790  * the caller would've quiesce all the VSIs before calling
6791  * this function
6792  **/
i40e_dcb_reconfigure(struct i40e_pf * pf)6793 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6794 {
6795 	struct i40e_vsi *vsi;
6796 	struct i40e_veb *veb;
6797 	u8 tc_map = 0;
6798 	int ret;
6799 	int v;
6800 
6801 	/* Enable the TCs available on PF to all VEBs */
6802 	tc_map = i40e_pf_get_tc_map(pf);
6803 	if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6804 		return;
6805 
6806 	i40e_pf_for_each_veb(pf, v, veb) {
6807 		ret = i40e_veb_config_tc(veb, tc_map);
6808 		if (ret) {
6809 			dev_info(&pf->pdev->dev,
6810 				 "Failed configuring TC for VEB seid=%d\n",
6811 				 veb->seid);
6812 			/* Will try to configure as many components */
6813 		}
6814 	}
6815 
6816 	/* Update each VSI */
6817 	i40e_pf_for_each_vsi(pf, v, vsi) {
6818 		/* - Enable all TCs for the LAN VSI
6819 		 * - For all others keep them at TC0 for now
6820 		 */
6821 		if (vsi->type == I40E_VSI_MAIN)
6822 			tc_map = i40e_pf_get_tc_map(pf);
6823 		else
6824 			tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6825 
6826 		ret = i40e_vsi_config_tc(vsi, tc_map);
6827 		if (ret) {
6828 			dev_info(&pf->pdev->dev,
6829 				 "Failed configuring TC for VSI seid=%d\n",
6830 				 vsi->seid);
6831 			/* Will try to configure as many components */
6832 		} else {
6833 			/* Re-configure VSI vectors based on updated TC map */
6834 			i40e_vsi_map_rings_to_vectors(vsi);
6835 			if (vsi->netdev)
6836 				i40e_dcbnl_set_all(vsi);
6837 		}
6838 	}
6839 }
6840 
6841 /**
6842  * i40e_resume_port_tx - Resume port Tx
6843  * @pf: PF struct
6844  *
6845  * Resume a port's Tx and issue a PF reset in case of failure to
6846  * resume.
6847  **/
i40e_resume_port_tx(struct i40e_pf * pf)6848 static int i40e_resume_port_tx(struct i40e_pf *pf)
6849 {
6850 	struct i40e_hw *hw = &pf->hw;
6851 	int ret;
6852 
6853 	ret = i40e_aq_resume_port_tx(hw, NULL);
6854 	if (ret) {
6855 		dev_info(&pf->pdev->dev,
6856 			 "Resume Port Tx failed, err %pe aq_err %s\n",
6857 			  ERR_PTR(ret),
6858 			  i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6859 		/* Schedule PF reset to recover */
6860 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6861 		i40e_service_event_schedule(pf);
6862 	}
6863 
6864 	return ret;
6865 }
6866 
6867 /**
6868  * i40e_suspend_port_tx - Suspend port Tx
6869  * @pf: PF struct
6870  *
6871  * Suspend a port's Tx and issue a PF reset in case of failure.
6872  **/
i40e_suspend_port_tx(struct i40e_pf * pf)6873 static int i40e_suspend_port_tx(struct i40e_pf *pf)
6874 {
6875 	struct i40e_hw *hw = &pf->hw;
6876 	int ret;
6877 
6878 	ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);
6879 	if (ret) {
6880 		dev_info(&pf->pdev->dev,
6881 			 "Suspend Port Tx failed, err %pe aq_err %s\n",
6882 			 ERR_PTR(ret),
6883 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6884 		/* Schedule PF reset to recover */
6885 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6886 		i40e_service_event_schedule(pf);
6887 	}
6888 
6889 	return ret;
6890 }
6891 
6892 /**
6893  * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6894  * @pf: PF being configured
6895  * @new_cfg: New DCBX configuration
6896  *
6897  * Program DCB settings into HW and reconfigure VEB/VSIs on
6898  * given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6899  **/
i40e_hw_set_dcb_config(struct i40e_pf * pf,struct i40e_dcbx_config * new_cfg)6900 static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6901 				  struct i40e_dcbx_config *new_cfg)
6902 {
6903 	struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6904 	int ret;
6905 
6906 	/* Check if need reconfiguration */
6907 	if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {
6908 		dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6909 		return 0;
6910 	}
6911 
6912 	/* Config change disable all VSIs */
6913 	i40e_pf_quiesce_all_vsi(pf);
6914 
6915 	/* Copy the new config to the current config */
6916 	*old_cfg = *new_cfg;
6917 	old_cfg->etsrec = old_cfg->etscfg;
6918 	ret = i40e_set_dcb_config(&pf->hw);
6919 	if (ret) {
6920 		dev_info(&pf->pdev->dev,
6921 			 "Set DCB Config failed, err %pe aq_err %s\n",
6922 			 ERR_PTR(ret),
6923 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6924 		goto out;
6925 	}
6926 
6927 	/* Changes in configuration update VEB/VSI */
6928 	i40e_dcb_reconfigure(pf);
6929 out:
6930 	/* In case of reset do not try to resume anything */
6931 	if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6932 		/* Re-start the VSIs if disabled */
6933 		ret = i40e_resume_port_tx(pf);
6934 		/* In case of error no point in resuming VSIs */
6935 		if (ret)
6936 			goto err;
6937 		i40e_pf_unquiesce_all_vsi(pf);
6938 	}
6939 err:
6940 	return ret;
6941 }
6942 
6943 /**
6944  * i40e_hw_dcb_config - Program new DCBX settings into HW
6945  * @pf: PF being configured
6946  * @new_cfg: New DCBX configuration
6947  *
6948  * Program DCB settings into HW and reconfigure VEB/VSIs on
6949  * given PF
6950  **/
i40e_hw_dcb_config(struct i40e_pf * pf,struct i40e_dcbx_config * new_cfg)6951 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6952 {
6953 	struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6954 	u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6955 	u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6956 	struct i40e_dcbx_config *old_cfg;
6957 	u8 mode[I40E_MAX_TRAFFIC_CLASS];
6958 	struct i40e_rx_pb_config pb_cfg;
6959 	struct i40e_hw *hw = &pf->hw;
6960 	u8 num_ports = hw->num_ports;
6961 	bool need_reconfig;
6962 	int ret = -EINVAL;
6963 	u8 lltc_map = 0;
6964 	u8 tc_map = 0;
6965 	u8 new_numtc;
6966 	u8 i;
6967 
6968 	dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6969 	/* Un-pack information to Program ETS HW via shared API
6970 	 * numtc, tcmap
6971 	 * LLTC map
6972 	 * ETS/NON-ETS arbiter mode
6973 	 * max exponent (credit refills)
6974 	 * Total number of ports
6975 	 * PFC priority bit-map
6976 	 * Priority Table
6977 	 * BW % per TC
6978 	 * Arbiter mode between UPs sharing same TC
6979 	 * TSA table (ETS or non-ETS)
6980 	 * EEE enabled or not
6981 	 * MFS TC table
6982 	 */
6983 
6984 	new_numtc = i40e_dcb_get_num_tc(new_cfg);
6985 
6986 	memset(&ets_data, 0, sizeof(ets_data));
6987 	for (i = 0; i < new_numtc; i++) {
6988 		tc_map |= BIT(i);
6989 		switch (new_cfg->etscfg.tsatable[i]) {
6990 		case I40E_IEEE_TSA_ETS:
6991 			prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
6992 			ets_data.tc_bw_share_credits[i] =
6993 					new_cfg->etscfg.tcbwtable[i];
6994 			break;
6995 		case I40E_IEEE_TSA_STRICT:
6996 			prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
6997 			lltc_map |= BIT(i);
6998 			ets_data.tc_bw_share_credits[i] =
6999 					I40E_DCB_STRICT_PRIO_CREDITS;
7000 			break;
7001 		default:
7002 			/* Invalid TSA type */
7003 			need_reconfig = false;
7004 			goto out;
7005 		}
7006 	}
7007 
7008 	old_cfg = &hw->local_dcbx_config;
7009 	/* Check if need reconfiguration */
7010 	need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
7011 
7012 	/* If needed, enable/disable frame tagging, disable all VSIs
7013 	 * and suspend port tx
7014 	 */
7015 	if (need_reconfig) {
7016 		/* Enable DCB tagging only when more than one TC */
7017 		if (new_numtc > 1)
7018 			set_bit(I40E_FLAG_DCB_ENA, pf->flags);
7019 		else
7020 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7021 
7022 		set_bit(__I40E_PORT_SUSPENDED, pf->state);
7023 		/* Reconfiguration needed quiesce all VSIs */
7024 		i40e_pf_quiesce_all_vsi(pf);
7025 		ret = i40e_suspend_port_tx(pf);
7026 		if (ret)
7027 			goto err;
7028 	}
7029 
7030 	/* Configure Port ETS Tx Scheduler */
7031 	ets_data.tc_valid_bits = tc_map;
7032 	ets_data.tc_strict_priority_flags = lltc_map;
7033 	ret = i40e_aq_config_switch_comp_ets
7034 		(hw, pf->mac_seid, &ets_data,
7035 		 i40e_aqc_opc_modify_switching_comp_ets, NULL);
7036 	if (ret) {
7037 		dev_info(&pf->pdev->dev,
7038 			 "Modify Port ETS failed, err %pe aq_err %s\n",
7039 			 ERR_PTR(ret),
7040 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7041 		goto out;
7042 	}
7043 
7044 	/* Configure Rx ETS HW */
7045 	memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
7046 	i40e_dcb_hw_set_num_tc(hw, new_numtc);
7047 	i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,
7048 				   I40E_DCB_ARB_MODE_STRICT_PRIORITY,
7049 				   I40E_DCB_DEFAULT_MAX_EXPONENT,
7050 				   lltc_map);
7051 	i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);
7052 	i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,
7053 				     prio_type);
7054 	i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,
7055 			       new_cfg->etscfg.prioritytable);
7056 	i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);
7057 
7058 	/* Configure Rx Packet Buffers in HW */
7059 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7060 		struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
7061 
7062 		mfs_tc[i] = main_vsi->netdev->mtu;
7063 		mfs_tc[i] += I40E_PACKET_HDR_PAD;
7064 	}
7065 
7066 	i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
7067 					 false, new_cfg->pfc.pfcenable,
7068 					 mfs_tc, &pb_cfg);
7069 	i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);
7070 
7071 	/* Update the local Rx Packet buffer config */
7072 	pf->pb_cfg = pb_cfg;
7073 
7074 	/* Inform the FW about changes to DCB configuration */
7075 	ret = i40e_aq_dcb_updated(&pf->hw, NULL);
7076 	if (ret) {
7077 		dev_info(&pf->pdev->dev,
7078 			 "DCB Updated failed, err %pe aq_err %s\n",
7079 			 ERR_PTR(ret),
7080 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7081 		goto out;
7082 	}
7083 
7084 	/* Update the port DCBx configuration */
7085 	*old_cfg = *new_cfg;
7086 
7087 	/* Changes in configuration update VEB/VSI */
7088 	i40e_dcb_reconfigure(pf);
7089 out:
7090 	/* Re-start the VSIs if disabled */
7091 	if (need_reconfig) {
7092 		ret = i40e_resume_port_tx(pf);
7093 
7094 		clear_bit(__I40E_PORT_SUSPENDED, pf->state);
7095 		/* In case of error no point in resuming VSIs */
7096 		if (ret)
7097 			goto err;
7098 
7099 		/* Wait for the PF's queues to be disabled */
7100 		ret = i40e_pf_wait_queues_disabled(pf);
7101 		if (ret) {
7102 			/* Schedule PF reset to recover */
7103 			set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7104 			i40e_service_event_schedule(pf);
7105 			goto err;
7106 		} else {
7107 			i40e_pf_unquiesce_all_vsi(pf);
7108 			set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7109 			set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
7110 		}
7111 		/* registers are set, lets apply */
7112 		if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps))
7113 			ret = i40e_hw_set_dcb_config(pf, new_cfg);
7114 	}
7115 
7116 err:
7117 	return ret;
7118 }
7119 
7120 /**
7121  * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
7122  * @pf: PF being queried
7123  *
7124  * Set default DCB configuration in case DCB is to be done in SW.
7125  **/
i40e_dcb_sw_default_config(struct i40e_pf * pf)7126 int i40e_dcb_sw_default_config(struct i40e_pf *pf)
7127 {
7128 	struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
7129 	struct i40e_aqc_configure_switching_comp_ets_data ets_data;
7130 	struct i40e_hw *hw = &pf->hw;
7131 	int err;
7132 
7133 	if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) {
7134 		/* Update the local cached instance with TC0 ETS */
7135 		memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
7136 		pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7137 		pf->tmp_cfg.etscfg.maxtcs = 0;
7138 		pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7139 		pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
7140 		pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
7141 		pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
7142 		/* FW needs one App to configure HW */
7143 		pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
7144 		pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
7145 		pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
7146 		pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
7147 
7148 		return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);
7149 	}
7150 
7151 	memset(&ets_data, 0, sizeof(ets_data));
7152 	ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
7153 	ets_data.tc_strict_priority_flags = 0; /* ETS */
7154 	ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
7155 
7156 	/* Enable ETS on the Physical port */
7157 	err = i40e_aq_config_switch_comp_ets
7158 		(hw, pf->mac_seid, &ets_data,
7159 		 i40e_aqc_opc_enable_switching_comp_ets, NULL);
7160 	if (err) {
7161 		dev_info(&pf->pdev->dev,
7162 			 "Enable Port ETS failed, err %pe aq_err %s\n",
7163 			 ERR_PTR(err),
7164 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7165 		err = -ENOENT;
7166 		goto out;
7167 	}
7168 
7169 	/* Update the local cached instance with TC0 ETS */
7170 	dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7171 	dcb_cfg->etscfg.cbs = 0;
7172 	dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
7173 	dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7174 
7175 out:
7176 	return err;
7177 }
7178 
7179 /**
7180  * i40e_init_pf_dcb - Initialize DCB configuration
7181  * @pf: PF being configured
7182  *
7183  * Query the current DCB configuration and cache it
7184  * in the hardware structure
7185  **/
i40e_init_pf_dcb(struct i40e_pf * pf)7186 static int i40e_init_pf_dcb(struct i40e_pf *pf)
7187 {
7188 	struct i40e_hw *hw = &pf->hw;
7189 	int err;
7190 
7191 	/* Do not enable DCB for SW1 and SW2 images even if the FW is capable
7192 	 * Also do not enable DCBx if FW LLDP agent is disabled
7193 	 */
7194 	if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) {
7195 		dev_info(&pf->pdev->dev, "DCB is not supported.\n");
7196 		err = -EOPNOTSUPP;
7197 		goto out;
7198 	}
7199 	if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) {
7200 		dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
7201 		err = i40e_dcb_sw_default_config(pf);
7202 		if (err) {
7203 			dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
7204 			goto out;
7205 		}
7206 		dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
7207 		pf->dcbx_cap = DCB_CAP_DCBX_HOST |
7208 			       DCB_CAP_DCBX_VER_IEEE;
7209 		/* at init capable but disabled */
7210 		set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
7211 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7212 		goto out;
7213 	}
7214 	err = i40e_init_dcb(hw, true);
7215 	if (!err) {
7216 		/* Device/Function is not DCBX capable */
7217 		if ((!hw->func_caps.dcb) ||
7218 		    (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
7219 			dev_info(&pf->pdev->dev,
7220 				 "DCBX offload is not supported or is disabled for this PF.\n");
7221 		} else {
7222 			/* When status is not DISABLED then DCBX in FW */
7223 			pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
7224 				       DCB_CAP_DCBX_VER_IEEE;
7225 
7226 			set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
7227 			/* Enable DCB tagging only when more than one TC
7228 			 * or explicitly disable if only one TC
7229 			 */
7230 			if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
7231 				set_bit(I40E_FLAG_DCB_ENA, pf->flags);
7232 			else
7233 				clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7234 			dev_dbg(&pf->pdev->dev,
7235 				"DCBX offload is supported for this PF.\n");
7236 		}
7237 	} else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
7238 		dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
7239 		set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags);
7240 	} else {
7241 		dev_info(&pf->pdev->dev,
7242 			 "Query for DCB configuration failed, err %pe aq_err %s\n",
7243 			 ERR_PTR(err),
7244 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7245 	}
7246 
7247 out:
7248 	return err;
7249 }
7250 #endif /* CONFIG_I40E_DCB */
7251 
i40e_print_link_message_eee(struct i40e_vsi * vsi,const char * speed,const char * fc)7252 static void i40e_print_link_message_eee(struct i40e_vsi *vsi,
7253 					const char *speed, const char *fc)
7254 {
7255 	struct ethtool_keee kedata;
7256 
7257 	memzero_explicit(&kedata, sizeof(kedata));
7258 	if (vsi->netdev->ethtool_ops->get_eee)
7259 		vsi->netdev->ethtool_ops->get_eee(vsi->netdev, &kedata);
7260 
7261 	if (!linkmode_empty(kedata.supported))
7262 		netdev_info(vsi->netdev,
7263 			    "NIC Link is Up, %sbps Full Duplex, Flow Control: %s, EEE: %s\n",
7264 			    speed, fc,
7265 			    kedata.eee_enabled ? "Enabled" : "Disabled");
7266 	else
7267 		netdev_info(vsi->netdev,
7268 			    "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7269 			    speed, fc);
7270 }
7271 
7272 /**
7273  * i40e_print_link_message - print link up or down
7274  * @vsi: the VSI for which link needs a message
7275  * @isup: true of link is up, false otherwise
7276  */
i40e_print_link_message(struct i40e_vsi * vsi,bool isup)7277 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
7278 {
7279 	enum i40e_aq_link_speed new_speed;
7280 	struct i40e_pf *pf = vsi->back;
7281 	char *speed = "Unknown";
7282 	char *fc = "Unknown";
7283 	char *fec = "";
7284 	char *req_fec = "";
7285 	char *an = "";
7286 
7287 	if (isup)
7288 		new_speed = pf->hw.phy.link_info.link_speed;
7289 	else
7290 		new_speed = I40E_LINK_SPEED_UNKNOWN;
7291 
7292 	if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
7293 		return;
7294 	vsi->current_isup = isup;
7295 	vsi->current_speed = new_speed;
7296 	if (!isup) {
7297 		netdev_info(vsi->netdev, "NIC Link is Down\n");
7298 		return;
7299 	}
7300 
7301 	/* Warn user if link speed on NPAR enabled partition is not at
7302 	 * least 10GB
7303 	 */
7304 	if (pf->hw.func_caps.npar_enable &&
7305 	    (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
7306 	     pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
7307 		netdev_warn(vsi->netdev,
7308 			    "The partition detected link speed that is less than 10Gbps\n");
7309 
7310 	switch (pf->hw.phy.link_info.link_speed) {
7311 	case I40E_LINK_SPEED_40GB:
7312 		speed = "40 G";
7313 		break;
7314 	case I40E_LINK_SPEED_20GB:
7315 		speed = "20 G";
7316 		break;
7317 	case I40E_LINK_SPEED_25GB:
7318 		speed = "25 G";
7319 		break;
7320 	case I40E_LINK_SPEED_10GB:
7321 		speed = "10 G";
7322 		break;
7323 	case I40E_LINK_SPEED_5GB:
7324 		speed = "5 G";
7325 		break;
7326 	case I40E_LINK_SPEED_2_5GB:
7327 		speed = "2.5 G";
7328 		break;
7329 	case I40E_LINK_SPEED_1GB:
7330 		speed = "1000 M";
7331 		break;
7332 	case I40E_LINK_SPEED_100MB:
7333 		speed = "100 M";
7334 		break;
7335 	default:
7336 		break;
7337 	}
7338 
7339 	switch (pf->hw.fc.current_mode) {
7340 	case I40E_FC_FULL:
7341 		fc = "RX/TX";
7342 		break;
7343 	case I40E_FC_TX_PAUSE:
7344 		fc = "TX";
7345 		break;
7346 	case I40E_FC_RX_PAUSE:
7347 		fc = "RX";
7348 		break;
7349 	default:
7350 		fc = "None";
7351 		break;
7352 	}
7353 
7354 	if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
7355 		req_fec = "None";
7356 		fec = "None";
7357 		an = "False";
7358 
7359 		if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7360 			an = "True";
7361 
7362 		if (pf->hw.phy.link_info.fec_info &
7363 		    I40E_AQ_CONFIG_FEC_KR_ENA)
7364 			fec = "CL74 FC-FEC/BASE-R";
7365 		else if (pf->hw.phy.link_info.fec_info &
7366 			 I40E_AQ_CONFIG_FEC_RS_ENA)
7367 			fec = "CL108 RS-FEC";
7368 
7369 		/* 'CL108 RS-FEC' should be displayed when RS is requested, or
7370 		 * both RS and FC are requested
7371 		 */
7372 		if (vsi->back->hw.phy.link_info.req_fec_info &
7373 		    (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7374 			if (vsi->back->hw.phy.link_info.req_fec_info &
7375 			    I40E_AQ_REQUEST_FEC_RS)
7376 				req_fec = "CL108 RS-FEC";
7377 			else
7378 				req_fec = "CL74 FC-FEC/BASE-R";
7379 		}
7380 		netdev_info(vsi->netdev,
7381 			    "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7382 			    speed, req_fec, fec, an, fc);
7383 	} else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7384 		req_fec = "None";
7385 		fec = "None";
7386 		an = "False";
7387 
7388 		if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7389 			an = "True";
7390 
7391 		if (pf->hw.phy.link_info.fec_info &
7392 		    I40E_AQ_CONFIG_FEC_KR_ENA)
7393 			fec = "CL74 FC-FEC/BASE-R";
7394 
7395 		if (pf->hw.phy.link_info.req_fec_info &
7396 		    I40E_AQ_REQUEST_FEC_KR)
7397 			req_fec = "CL74 FC-FEC/BASE-R";
7398 
7399 		netdev_info(vsi->netdev,
7400 			    "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7401 			    speed, req_fec, fec, an, fc);
7402 	} else {
7403 		i40e_print_link_message_eee(vsi, speed, fc);
7404 	}
7405 
7406 }
7407 
7408 /**
7409  * i40e_up_complete - Finish the last steps of bringing up a connection
7410  * @vsi: the VSI being configured
7411  **/
i40e_up_complete(struct i40e_vsi * vsi)7412 static int i40e_up_complete(struct i40e_vsi *vsi)
7413 {
7414 	struct i40e_pf *pf = vsi->back;
7415 	int err;
7416 
7417 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
7418 		i40e_vsi_configure_msix(vsi);
7419 	else
7420 		i40e_configure_msi_and_legacy(vsi);
7421 
7422 	/* start rings */
7423 	err = i40e_vsi_start_rings(vsi);
7424 	if (err)
7425 		return err;
7426 
7427 	clear_bit(__I40E_VSI_DOWN, vsi->state);
7428 	i40e_napi_enable_all(vsi);
7429 	i40e_vsi_enable_irq(vsi);
7430 
7431 	if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7432 	    (vsi->netdev)) {
7433 		i40e_print_link_message(vsi, true);
7434 		netif_tx_start_all_queues(vsi->netdev);
7435 		netif_carrier_on(vsi->netdev);
7436 	}
7437 
7438 	/* replay FDIR SB filters */
7439 	if (vsi->type == I40E_VSI_FDIR) {
7440 		/* reset fd counters */
7441 		pf->fd_add_err = 0;
7442 		pf->fd_atr_cnt = 0;
7443 		i40e_fdir_filter_restore(vsi);
7444 	}
7445 
7446 	/* On the next run of the service_task, notify any clients of the new
7447 	 * opened netdev
7448 	 */
7449 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7450 	i40e_service_event_schedule(pf);
7451 
7452 	return 0;
7453 }
7454 
7455 /**
7456  * i40e_vsi_reinit_locked - Reset the VSI
7457  * @vsi: the VSI being configured
7458  *
7459  * Rebuild the ring structs after some configuration
7460  * has changed, e.g. MTU size.
7461  **/
i40e_vsi_reinit_locked(struct i40e_vsi * vsi)7462 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7463 {
7464 	struct i40e_pf *pf = vsi->back;
7465 
7466 	while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
7467 		usleep_range(1000, 2000);
7468 	i40e_down(vsi);
7469 
7470 	i40e_up(vsi);
7471 	clear_bit(__I40E_CONFIG_BUSY, pf->state);
7472 }
7473 
7474 /**
7475  * i40e_force_link_state - Force the link status
7476  * @pf: board private structure
7477  * @is_up: whether the link state should be forced up or down
7478  **/
i40e_force_link_state(struct i40e_pf * pf,bool is_up)7479 static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7480 {
7481 	struct i40e_aq_get_phy_abilities_resp abilities;
7482 	struct i40e_aq_set_phy_config config = {0};
7483 	bool non_zero_phy_type = is_up;
7484 	struct i40e_hw *hw = &pf->hw;
7485 	u64 mask;
7486 	u8 speed;
7487 	int err;
7488 
7489 	/* Card might've been put in an unstable state by other drivers
7490 	 * and applications, which causes incorrect speed values being
7491 	 * set on startup. In order to clear speed registers, we call
7492 	 * get_phy_capabilities twice, once to get initial state of
7493 	 * available speeds, and once to get current PHY config.
7494 	 */
7495 	err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
7496 					   NULL);
7497 	if (err) {
7498 		dev_err(&pf->pdev->dev,
7499 			"failed to get phy cap., ret =  %pe last_status =  %s\n",
7500 			ERR_PTR(err),
7501 			i40e_aq_str(hw, hw->aq.asq_last_status));
7502 		return err;
7503 	}
7504 	speed = abilities.link_speed;
7505 
7506 	/* Get the current phy config */
7507 	err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
7508 					   NULL);
7509 	if (err) {
7510 		dev_err(&pf->pdev->dev,
7511 			"failed to get phy cap., ret =  %pe last_status =  %s\n",
7512 			ERR_PTR(err),
7513 			i40e_aq_str(hw, hw->aq.asq_last_status));
7514 		return err;
7515 	}
7516 
7517 	/* If link needs to go up, but was not forced to go down,
7518 	 * and its speed values are OK, no need for a flap
7519 	 * if non_zero_phy_type was set, still need to force up
7520 	 */
7521 	if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags))
7522 		non_zero_phy_type = true;
7523 	else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7524 		return 0;
7525 
7526 	/* To force link we need to set bits for all supported PHY types,
7527 	 * but there are now more than 32, so we need to split the bitmap
7528 	 * across two fields.
7529 	 */
7530 	mask = I40E_PHY_TYPES_BITMASK;
7531 	config.phy_type =
7532 		non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7533 	config.phy_type_ext =
7534 		non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7535 	/* Copy the old settings, except of phy_type */
7536 	config.abilities = abilities.abilities;
7537 	if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) {
7538 		if (is_up)
7539 			config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7540 		else
7541 			config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7542 	}
7543 	if (abilities.link_speed != 0)
7544 		config.link_speed = abilities.link_speed;
7545 	else
7546 		config.link_speed = speed;
7547 	config.eee_capability = abilities.eee_capability;
7548 	config.eeer = abilities.eeer_val;
7549 	config.low_power_ctrl = abilities.d3_lpan;
7550 	config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7551 			    I40E_AQ_PHY_FEC_CONFIG_MASK;
7552 	err = i40e_aq_set_phy_config(hw, &config, NULL);
7553 
7554 	if (err) {
7555 		dev_err(&pf->pdev->dev,
7556 			"set phy config ret =  %pe last_status =  %s\n",
7557 			ERR_PTR(err),
7558 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7559 		return err;
7560 	}
7561 
7562 	/* Update the link info */
7563 	err = i40e_update_link_info(hw);
7564 	if (err) {
7565 		/* Wait a little bit (on 40G cards it sometimes takes a really
7566 		 * long time for link to come back from the atomic reset)
7567 		 * and try once more
7568 		 */
7569 		msleep(1000);
7570 		i40e_update_link_info(hw);
7571 	}
7572 
7573 	i40e_aq_set_link_restart_an(hw, is_up, NULL);
7574 
7575 	return 0;
7576 }
7577 
7578 /**
7579  * i40e_up - Bring the connection back up after being down
7580  * @vsi: the VSI being configured
7581  **/
i40e_up(struct i40e_vsi * vsi)7582 int i40e_up(struct i40e_vsi *vsi)
7583 {
7584 	int err;
7585 
7586 	if (vsi->type == I40E_VSI_MAIN &&
7587 	    (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7588 	     test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7589 		i40e_force_link_state(vsi->back, true);
7590 
7591 	err = i40e_vsi_configure(vsi);
7592 	if (!err)
7593 		err = i40e_up_complete(vsi);
7594 
7595 	return err;
7596 }
7597 
7598 /**
7599  * i40e_down - Shutdown the connection processing
7600  * @vsi: the VSI being stopped
7601  **/
i40e_down(struct i40e_vsi * vsi)7602 void i40e_down(struct i40e_vsi *vsi)
7603 {
7604 	int i;
7605 
7606 	/* It is assumed that the caller of this function
7607 	 * sets the vsi->state __I40E_VSI_DOWN bit.
7608 	 */
7609 	if (vsi->netdev) {
7610 		netif_carrier_off(vsi->netdev);
7611 		netif_tx_disable(vsi->netdev);
7612 	}
7613 	i40e_vsi_disable_irq(vsi);
7614 	i40e_vsi_stop_rings(vsi);
7615 	if (vsi->type == I40E_VSI_MAIN &&
7616 	   (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7617 	    test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7618 		i40e_force_link_state(vsi->back, false);
7619 	i40e_napi_disable_all(vsi);
7620 
7621 	for (i = 0; i < vsi->num_queue_pairs; i++) {
7622 		i40e_clean_tx_ring(vsi->tx_rings[i]);
7623 		if (i40e_enabled_xdp_vsi(vsi)) {
7624 			/* Make sure that in-progress ndo_xdp_xmit and
7625 			 * ndo_xsk_wakeup calls are completed.
7626 			 */
7627 			synchronize_rcu();
7628 			i40e_clean_tx_ring(vsi->xdp_rings[i]);
7629 		}
7630 		i40e_clean_rx_ring(vsi->rx_rings[i]);
7631 	}
7632 
7633 }
7634 
7635 /**
7636  * i40e_validate_mqprio_qopt- validate queue mapping info
7637  * @vsi: the VSI being configured
7638  * @mqprio_qopt: queue parametrs
7639  **/
i40e_validate_mqprio_qopt(struct i40e_vsi * vsi,struct tc_mqprio_qopt_offload * mqprio_qopt)7640 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7641 				     struct tc_mqprio_qopt_offload *mqprio_qopt)
7642 {
7643 	u64 sum_max_rate = 0;
7644 	u64 max_rate = 0;
7645 	int i;
7646 
7647 	if (mqprio_qopt->qopt.offset[0] != 0 ||
7648 	    mqprio_qopt->qopt.num_tc < 1 ||
7649 	    mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7650 		return -EINVAL;
7651 	for (i = 0; ; i++) {
7652 		if (!mqprio_qopt->qopt.count[i])
7653 			return -EINVAL;
7654 		if (mqprio_qopt->min_rate[i]) {
7655 			dev_err(&vsi->back->pdev->dev,
7656 				"Invalid min tx rate (greater than 0) specified\n");
7657 			return -EINVAL;
7658 		}
7659 		max_rate = mqprio_qopt->max_rate[i];
7660 		do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7661 		sum_max_rate += max_rate;
7662 
7663 		if (i >= mqprio_qopt->qopt.num_tc - 1)
7664 			break;
7665 		if (mqprio_qopt->qopt.offset[i + 1] !=
7666 		    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7667 			return -EINVAL;
7668 	}
7669 	if (vsi->num_queue_pairs <
7670 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7671 		dev_err(&vsi->back->pdev->dev,
7672 			"Failed to create traffic channel, insufficient number of queues.\n");
7673 		return -EINVAL;
7674 	}
7675 	if (sum_max_rate > i40e_get_link_speed(vsi)) {
7676 		dev_err(&vsi->back->pdev->dev,
7677 			"Invalid max tx rate specified\n");
7678 		return -EINVAL;
7679 	}
7680 	return 0;
7681 }
7682 
7683 /**
7684  * i40e_vsi_set_default_tc_config - set default values for tc configuration
7685  * @vsi: the VSI being configured
7686  **/
i40e_vsi_set_default_tc_config(struct i40e_vsi * vsi)7687 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7688 {
7689 	u16 qcount;
7690 	int i;
7691 
7692 	/* Only TC0 is enabled */
7693 	vsi->tc_config.numtc = 1;
7694 	vsi->tc_config.enabled_tc = 1;
7695 	qcount = min_t(int, vsi->alloc_queue_pairs,
7696 		       i40e_pf_get_max_q_per_tc(vsi->back));
7697 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7698 		/* For the TC that is not enabled set the offset to default
7699 		 * queue and allocate one queue for the given TC.
7700 		 */
7701 		vsi->tc_config.tc_info[i].qoffset = 0;
7702 		if (i == 0)
7703 			vsi->tc_config.tc_info[i].qcount = qcount;
7704 		else
7705 			vsi->tc_config.tc_info[i].qcount = 1;
7706 		vsi->tc_config.tc_info[i].netdev_tc = 0;
7707 	}
7708 }
7709 
7710 /**
7711  * i40e_del_macvlan_filter
7712  * @hw: pointer to the HW structure
7713  * @seid: seid of the channel VSI
7714  * @macaddr: the mac address to apply as a filter
7715  * @aq_err: store the admin Q error
7716  *
7717  * This function deletes a mac filter on the channel VSI which serves as the
7718  * macvlan. Returns 0 on success.
7719  **/
i40e_del_macvlan_filter(struct i40e_hw * hw,u16 seid,const u8 * macaddr,int * aq_err)7720 static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7721 				   const u8 *macaddr, int *aq_err)
7722 {
7723 	struct i40e_aqc_remove_macvlan_element_data element;
7724 	int status;
7725 
7726 	memset(&element, 0, sizeof(element));
7727 	ether_addr_copy(element.mac_addr, macaddr);
7728 	element.vlan_tag = 0;
7729 	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7730 	status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7731 	*aq_err = hw->aq.asq_last_status;
7732 
7733 	return status;
7734 }
7735 
7736 /**
7737  * i40e_add_macvlan_filter
7738  * @hw: pointer to the HW structure
7739  * @seid: seid of the channel VSI
7740  * @macaddr: the mac address to apply as a filter
7741  * @aq_err: store the admin Q error
7742  *
7743  * This function adds a mac filter on the channel VSI which serves as the
7744  * macvlan. Returns 0 on success.
7745  **/
i40e_add_macvlan_filter(struct i40e_hw * hw,u16 seid,const u8 * macaddr,int * aq_err)7746 static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7747 				   const u8 *macaddr, int *aq_err)
7748 {
7749 	struct i40e_aqc_add_macvlan_element_data element;
7750 	u16 cmd_flags = 0;
7751 	int status;
7752 
7753 	ether_addr_copy(element.mac_addr, macaddr);
7754 	element.vlan_tag = 0;
7755 	element.queue_number = 0;
7756 	element.match_method = I40E_AQC_MM_ERR_NO_RES;
7757 	cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7758 	element.flags = cpu_to_le16(cmd_flags);
7759 	status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7760 	*aq_err = hw->aq.asq_last_status;
7761 
7762 	return status;
7763 }
7764 
7765 /**
7766  * i40e_reset_ch_rings - Reset the queue contexts in a channel
7767  * @vsi: the VSI we want to access
7768  * @ch: the channel we want to access
7769  */
i40e_reset_ch_rings(struct i40e_vsi * vsi,struct i40e_channel * ch)7770 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7771 {
7772 	struct i40e_ring *tx_ring, *rx_ring;
7773 	u16 pf_q;
7774 	int i;
7775 
7776 	for (i = 0; i < ch->num_queue_pairs; i++) {
7777 		pf_q = ch->base_queue + i;
7778 		tx_ring = vsi->tx_rings[pf_q];
7779 		tx_ring->ch = NULL;
7780 		rx_ring = vsi->rx_rings[pf_q];
7781 		rx_ring->ch = NULL;
7782 	}
7783 }
7784 
7785 /**
7786  * i40e_free_macvlan_channels
7787  * @vsi: the VSI we want to access
7788  *
7789  * This function frees the Qs of the channel VSI from
7790  * the stack and also deletes the channel VSIs which
7791  * serve as macvlans.
7792  */
i40e_free_macvlan_channels(struct i40e_vsi * vsi)7793 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7794 {
7795 	struct i40e_channel *ch, *ch_tmp;
7796 	int ret;
7797 
7798 	if (list_empty(&vsi->macvlan_list))
7799 		return;
7800 
7801 	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7802 		struct i40e_vsi *parent_vsi;
7803 
7804 		if (i40e_is_channel_macvlan(ch)) {
7805 			i40e_reset_ch_rings(vsi, ch);
7806 			clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7807 			netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7808 			netdev_set_sb_channel(ch->fwd->netdev, 0);
7809 			kfree(ch->fwd);
7810 			ch->fwd = NULL;
7811 		}
7812 
7813 		list_del(&ch->list);
7814 		parent_vsi = ch->parent_vsi;
7815 		if (!parent_vsi || !ch->initialized) {
7816 			kfree(ch);
7817 			continue;
7818 		}
7819 
7820 		/* remove the VSI */
7821 		ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7822 					     NULL);
7823 		if (ret)
7824 			dev_err(&vsi->back->pdev->dev,
7825 				"unable to remove channel (%d) for parent VSI(%d)\n",
7826 				ch->seid, parent_vsi->seid);
7827 		kfree(ch);
7828 	}
7829 	vsi->macvlan_cnt = 0;
7830 }
7831 
7832 /**
7833  * i40e_fwd_ring_up - bring the macvlan device up
7834  * @vsi: the VSI we want to access
7835  * @vdev: macvlan netdevice
7836  * @fwd: the private fwd structure
7837  */
i40e_fwd_ring_up(struct i40e_vsi * vsi,struct net_device * vdev,struct i40e_fwd_adapter * fwd)7838 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7839 			    struct i40e_fwd_adapter *fwd)
7840 {
7841 	struct i40e_channel *ch = NULL, *ch_tmp, *iter;
7842 	int ret = 0, num_tc = 1,  i, aq_err;
7843 	struct i40e_pf *pf = vsi->back;
7844 	struct i40e_hw *hw = &pf->hw;
7845 
7846 	/* Go through the list and find an available channel */
7847 	list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {
7848 		if (!i40e_is_channel_macvlan(iter)) {
7849 			iter->fwd = fwd;
7850 			/* record configuration for macvlan interface in vdev */
7851 			for (i = 0; i < num_tc; i++)
7852 				netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7853 							     i,
7854 							     iter->num_queue_pairs,
7855 							     iter->base_queue);
7856 			for (i = 0; i < iter->num_queue_pairs; i++) {
7857 				struct i40e_ring *tx_ring, *rx_ring;
7858 				u16 pf_q;
7859 
7860 				pf_q = iter->base_queue + i;
7861 
7862 				/* Get to TX ring ptr */
7863 				tx_ring = vsi->tx_rings[pf_q];
7864 				tx_ring->ch = iter;
7865 
7866 				/* Get the RX ring ptr */
7867 				rx_ring = vsi->rx_rings[pf_q];
7868 				rx_ring->ch = iter;
7869 			}
7870 			ch = iter;
7871 			break;
7872 		}
7873 	}
7874 
7875 	if (!ch)
7876 		return -EINVAL;
7877 
7878 	/* Guarantee all rings are updated before we update the
7879 	 * MAC address filter.
7880 	 */
7881 	wmb();
7882 
7883 	/* Add a mac filter */
7884 	ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7885 	if (ret) {
7886 		/* if we cannot add the MAC rule then disable the offload */
7887 		macvlan_release_l2fw_offload(vdev);
7888 		for (i = 0; i < ch->num_queue_pairs; i++) {
7889 			struct i40e_ring *rx_ring;
7890 			u16 pf_q;
7891 
7892 			pf_q = ch->base_queue + i;
7893 			rx_ring = vsi->rx_rings[pf_q];
7894 			rx_ring->netdev = NULL;
7895 		}
7896 		dev_info(&pf->pdev->dev,
7897 			 "Error adding mac filter on macvlan err %pe, aq_err %s\n",
7898 			  ERR_PTR(ret),
7899 			  i40e_aq_str(hw, aq_err));
7900 		netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7901 	}
7902 
7903 	return ret;
7904 }
7905 
7906 /**
7907  * i40e_setup_macvlans - create the channels which will be macvlans
7908  * @vsi: the VSI we want to access
7909  * @macvlan_cnt: no. of macvlans to be setup
7910  * @qcnt: no. of Qs per macvlan
7911  * @vdev: macvlan netdevice
7912  */
i40e_setup_macvlans(struct i40e_vsi * vsi,u16 macvlan_cnt,u16 qcnt,struct net_device * vdev)7913 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7914 			       struct net_device *vdev)
7915 {
7916 	struct i40e_pf *pf = vsi->back;
7917 	struct i40e_hw *hw = &pf->hw;
7918 	struct i40e_vsi_context ctxt;
7919 	u16 sections, qmap, num_qps;
7920 	struct i40e_channel *ch;
7921 	int i, pow, ret = 0;
7922 	u8 offset = 0;
7923 
7924 	if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7925 		return -EINVAL;
7926 
7927 	num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7928 
7929 	/* find the next higher power-of-2 of num queue pairs */
7930 	pow = fls(roundup_pow_of_two(num_qps) - 1);
7931 
7932 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7933 		(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7934 
7935 	/* Setup context bits for the main VSI */
7936 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7937 	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7938 	memset(&ctxt, 0, sizeof(ctxt));
7939 	ctxt.seid = vsi->seid;
7940 	ctxt.pf_num = vsi->back->hw.pf_id;
7941 	ctxt.vf_num = 0;
7942 	ctxt.uplink_seid = vsi->uplink_seid;
7943 	ctxt.info = vsi->info;
7944 	ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7945 	ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7946 	ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7947 	ctxt.info.valid_sections |= cpu_to_le16(sections);
7948 
7949 	/* Reconfigure RSS for main VSI with new max queue count */
7950 	vsi->rss_size = max_t(u16, num_qps, qcnt);
7951 	ret = i40e_vsi_config_rss(vsi);
7952 	if (ret) {
7953 		dev_info(&pf->pdev->dev,
7954 			 "Failed to reconfig RSS for num_queues (%u)\n",
7955 			 vsi->rss_size);
7956 		return ret;
7957 	}
7958 	vsi->reconfig_rss = true;
7959 	dev_dbg(&vsi->back->pdev->dev,
7960 		"Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7961 	vsi->next_base_queue = num_qps;
7962 	vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7963 
7964 	/* Update the VSI after updating the VSI queue-mapping
7965 	 * information
7966 	 */
7967 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7968 	if (ret) {
7969 		dev_info(&pf->pdev->dev,
7970 			 "Update vsi tc config failed, err %pe aq_err %s\n",
7971 			 ERR_PTR(ret),
7972 			 i40e_aq_str(hw, hw->aq.asq_last_status));
7973 		return ret;
7974 	}
7975 	/* update the local VSI info with updated queue map */
7976 	i40e_vsi_update_queue_map(vsi, &ctxt);
7977 	vsi->info.valid_sections = 0;
7978 
7979 	/* Create channels for macvlans */
7980 	INIT_LIST_HEAD(&vsi->macvlan_list);
7981 	for (i = 0; i < macvlan_cnt; i++) {
7982 		ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7983 		if (!ch) {
7984 			ret = -ENOMEM;
7985 			goto err_free;
7986 		}
7987 		INIT_LIST_HEAD(&ch->list);
7988 		ch->num_queue_pairs = qcnt;
7989 		if (!i40e_setup_channel(pf, vsi, ch)) {
7990 			ret = -EINVAL;
7991 			kfree(ch);
7992 			goto err_free;
7993 		}
7994 		ch->parent_vsi = vsi;
7995 		vsi->cnt_q_avail -= ch->num_queue_pairs;
7996 		vsi->macvlan_cnt++;
7997 		list_add_tail(&ch->list, &vsi->macvlan_list);
7998 	}
7999 
8000 	return ret;
8001 
8002 err_free:
8003 	dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
8004 	i40e_free_macvlan_channels(vsi);
8005 
8006 	return ret;
8007 }
8008 
8009 /**
8010  * i40e_fwd_add - configure macvlans
8011  * @netdev: net device to configure
8012  * @vdev: macvlan netdevice
8013  **/
i40e_fwd_add(struct net_device * netdev,struct net_device * vdev)8014 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
8015 {
8016 	struct i40e_netdev_priv *np = netdev_priv(netdev);
8017 	u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
8018 	struct i40e_vsi *vsi = np->vsi;
8019 	struct i40e_pf *pf = vsi->back;
8020 	struct i40e_fwd_adapter *fwd;
8021 	int avail_macvlan, ret;
8022 
8023 	if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8024 		netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
8025 		return ERR_PTR(-EINVAL);
8026 	}
8027 	if (i40e_is_tc_mqprio_enabled(pf)) {
8028 		netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
8029 		return ERR_PTR(-EINVAL);
8030 	}
8031 	if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
8032 		netdev_info(netdev, "Not enough vectors available to support macvlans\n");
8033 		return ERR_PTR(-EINVAL);
8034 	}
8035 
8036 	/* The macvlan device has to be a single Q device so that the
8037 	 * tc_to_txq field can be reused to pick the tx queue.
8038 	 */
8039 	if (netif_is_multiqueue(vdev))
8040 		return ERR_PTR(-ERANGE);
8041 
8042 	if (!vsi->macvlan_cnt) {
8043 		/* reserve bit 0 for the pf device */
8044 		set_bit(0, vsi->fwd_bitmask);
8045 
8046 		/* Try to reserve as many queues as possible for macvlans. First
8047 		 * reserve 3/4th of max vectors, then half, then quarter and
8048 		 * calculate Qs per macvlan as you go
8049 		 */
8050 		vectors = pf->num_lan_msix;
8051 		if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
8052 			/* allocate 4 Qs per macvlan and 32 Qs to the PF*/
8053 			q_per_macvlan = 4;
8054 			macvlan_cnt = (vectors - 32) / 4;
8055 		} else if (vectors <= 64 && vectors > 32) {
8056 			/* allocate 2 Qs per macvlan and 16 Qs to the PF*/
8057 			q_per_macvlan = 2;
8058 			macvlan_cnt = (vectors - 16) / 2;
8059 		} else if (vectors <= 32 && vectors > 16) {
8060 			/* allocate 1 Q per macvlan and 16 Qs to the PF*/
8061 			q_per_macvlan = 1;
8062 			macvlan_cnt = vectors - 16;
8063 		} else if (vectors <= 16 && vectors > 8) {
8064 			/* allocate 1 Q per macvlan and 8 Qs to the PF */
8065 			q_per_macvlan = 1;
8066 			macvlan_cnt = vectors - 8;
8067 		} else {
8068 			/* allocate 1 Q per macvlan and 1 Q to the PF */
8069 			q_per_macvlan = 1;
8070 			macvlan_cnt = vectors - 1;
8071 		}
8072 
8073 		if (macvlan_cnt == 0)
8074 			return ERR_PTR(-EBUSY);
8075 
8076 		/* Quiesce VSI queues */
8077 		i40e_quiesce_vsi(vsi);
8078 
8079 		/* sets up the macvlans but does not "enable" them */
8080 		ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
8081 					  vdev);
8082 		if (ret)
8083 			return ERR_PTR(ret);
8084 
8085 		/* Unquiesce VSI */
8086 		i40e_unquiesce_vsi(vsi);
8087 	}
8088 	avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
8089 					    vsi->macvlan_cnt);
8090 	if (avail_macvlan >= I40E_MAX_MACVLANS)
8091 		return ERR_PTR(-EBUSY);
8092 
8093 	/* create the fwd struct */
8094 	fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
8095 	if (!fwd)
8096 		return ERR_PTR(-ENOMEM);
8097 
8098 	set_bit(avail_macvlan, vsi->fwd_bitmask);
8099 	fwd->bit_no = avail_macvlan;
8100 	netdev_set_sb_channel(vdev, avail_macvlan);
8101 	fwd->netdev = vdev;
8102 
8103 	if (!netif_running(netdev))
8104 		return fwd;
8105 
8106 	/* Set fwd ring up */
8107 	ret = i40e_fwd_ring_up(vsi, vdev, fwd);
8108 	if (ret) {
8109 		/* unbind the queues and drop the subordinate channel config */
8110 		netdev_unbind_sb_channel(netdev, vdev);
8111 		netdev_set_sb_channel(vdev, 0);
8112 
8113 		kfree(fwd);
8114 		return ERR_PTR(-EINVAL);
8115 	}
8116 
8117 	return fwd;
8118 }
8119 
8120 /**
8121  * i40e_del_all_macvlans - Delete all the mac filters on the channels
8122  * @vsi: the VSI we want to access
8123  */
i40e_del_all_macvlans(struct i40e_vsi * vsi)8124 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
8125 {
8126 	struct i40e_channel *ch, *ch_tmp;
8127 	struct i40e_pf *pf = vsi->back;
8128 	struct i40e_hw *hw = &pf->hw;
8129 	int aq_err, ret = 0;
8130 
8131 	if (list_empty(&vsi->macvlan_list))
8132 		return;
8133 
8134 	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8135 		if (i40e_is_channel_macvlan(ch)) {
8136 			ret = i40e_del_macvlan_filter(hw, ch->seid,
8137 						      i40e_channel_mac(ch),
8138 						      &aq_err);
8139 			if (!ret) {
8140 				/* Reset queue contexts */
8141 				i40e_reset_ch_rings(vsi, ch);
8142 				clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8143 				netdev_unbind_sb_channel(vsi->netdev,
8144 							 ch->fwd->netdev);
8145 				netdev_set_sb_channel(ch->fwd->netdev, 0);
8146 				kfree(ch->fwd);
8147 				ch->fwd = NULL;
8148 			}
8149 		}
8150 	}
8151 }
8152 
8153 /**
8154  * i40e_fwd_del - delete macvlan interfaces
8155  * @netdev: net device to configure
8156  * @vdev: macvlan netdevice
8157  */
i40e_fwd_del(struct net_device * netdev,void * vdev)8158 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
8159 {
8160 	struct i40e_netdev_priv *np = netdev_priv(netdev);
8161 	struct i40e_fwd_adapter *fwd = vdev;
8162 	struct i40e_channel *ch, *ch_tmp;
8163 	struct i40e_vsi *vsi = np->vsi;
8164 	struct i40e_pf *pf = vsi->back;
8165 	struct i40e_hw *hw = &pf->hw;
8166 	int aq_err, ret = 0;
8167 
8168 	/* Find the channel associated with the macvlan and del mac filter */
8169 	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8170 		if (i40e_is_channel_macvlan(ch) &&
8171 		    ether_addr_equal(i40e_channel_mac(ch),
8172 				     fwd->netdev->dev_addr)) {
8173 			ret = i40e_del_macvlan_filter(hw, ch->seid,
8174 						      i40e_channel_mac(ch),
8175 						      &aq_err);
8176 			if (!ret) {
8177 				/* Reset queue contexts */
8178 				i40e_reset_ch_rings(vsi, ch);
8179 				clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8180 				netdev_unbind_sb_channel(netdev, fwd->netdev);
8181 				netdev_set_sb_channel(fwd->netdev, 0);
8182 				kfree(ch->fwd);
8183 				ch->fwd = NULL;
8184 			} else {
8185 				dev_info(&pf->pdev->dev,
8186 					 "Error deleting mac filter on macvlan err %pe, aq_err %s\n",
8187 					  ERR_PTR(ret),
8188 					  i40e_aq_str(hw, aq_err));
8189 			}
8190 			break;
8191 		}
8192 	}
8193 }
8194 
8195 /**
8196  * i40e_setup_tc - configure multiple traffic classes
8197  * @netdev: net device to configure
8198  * @type_data: tc offload data
8199  **/
i40e_setup_tc(struct net_device * netdev,void * type_data)8200 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
8201 {
8202 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8203 	struct i40e_netdev_priv *np = netdev_priv(netdev);
8204 	struct i40e_vsi *vsi = np->vsi;
8205 	struct i40e_pf *pf = vsi->back;
8206 	u8 enabled_tc = 0, num_tc, hw;
8207 	bool need_reset = false;
8208 	int old_queue_pairs;
8209 	int ret = -EINVAL;
8210 	u16 mode;
8211 	int i;
8212 
8213 	old_queue_pairs = vsi->num_queue_pairs;
8214 	num_tc = mqprio_qopt->qopt.num_tc;
8215 	hw = mqprio_qopt->qopt.hw;
8216 	mode = mqprio_qopt->mode;
8217 	if (!hw) {
8218 		clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8219 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8220 		goto config_tc;
8221 	}
8222 
8223 	/* Check if MFP enabled */
8224 	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
8225 		netdev_info(netdev,
8226 			    "Configuring TC not supported in MFP mode\n");
8227 		return ret;
8228 	}
8229 	switch (mode) {
8230 	case TC_MQPRIO_MODE_DCB:
8231 		clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8232 
8233 		/* Check if DCB enabled to continue */
8234 		if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8235 			netdev_info(netdev,
8236 				    "DCB is not enabled for adapter\n");
8237 			return ret;
8238 		}
8239 
8240 		/* Check whether tc count is within enabled limit */
8241 		if (num_tc > i40e_pf_get_num_tc(pf)) {
8242 			netdev_info(netdev,
8243 				    "TC count greater than enabled on link for adapter\n");
8244 			return ret;
8245 		}
8246 		break;
8247 	case TC_MQPRIO_MODE_CHANNEL:
8248 		if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8249 			netdev_info(netdev,
8250 				    "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
8251 			return ret;
8252 		}
8253 		if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
8254 			return ret;
8255 		ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
8256 		if (ret)
8257 			return ret;
8258 		memcpy(&vsi->mqprio_qopt, mqprio_qopt,
8259 		       sizeof(*mqprio_qopt));
8260 		set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8261 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
8262 		break;
8263 	default:
8264 		return -EINVAL;
8265 	}
8266 
8267 config_tc:
8268 	/* Generate TC map for number of tc requested */
8269 	for (i = 0; i < num_tc; i++)
8270 		enabled_tc |= BIT(i);
8271 
8272 	/* Requesting same TC configuration as already enabled */
8273 	if (enabled_tc == vsi->tc_config.enabled_tc &&
8274 	    mode != TC_MQPRIO_MODE_CHANNEL)
8275 		return 0;
8276 
8277 	/* Quiesce VSI queues */
8278 	i40e_quiesce_vsi(vsi);
8279 
8280 	if (!hw && !i40e_is_tc_mqprio_enabled(pf))
8281 		i40e_remove_queue_channels(vsi);
8282 
8283 	/* Configure VSI for enabled TCs */
8284 	ret = i40e_vsi_config_tc(vsi, enabled_tc);
8285 	if (ret) {
8286 		netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
8287 			    vsi->seid);
8288 		need_reset = true;
8289 		goto exit;
8290 	} else if (enabled_tc &&
8291 		   (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
8292 		netdev_info(netdev,
8293 			    "Failed to create channel. Override queues (%u) not power of 2\n",
8294 			    vsi->tc_config.tc_info[0].qcount);
8295 		ret = -EINVAL;
8296 		need_reset = true;
8297 		goto exit;
8298 	}
8299 
8300 	dev_info(&vsi->back->pdev->dev,
8301 		 "Setup channel (id:%u) utilizing num_queues %d\n",
8302 		 vsi->seid, vsi->tc_config.tc_info[0].qcount);
8303 
8304 	if (i40e_is_tc_mqprio_enabled(pf)) {
8305 		if (vsi->mqprio_qopt.max_rate[0]) {
8306 			u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
8307 						  vsi->mqprio_qopt.max_rate[0]);
8308 
8309 			ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
8310 			if (!ret) {
8311 				u64 credits = max_tx_rate;
8312 
8313 				do_div(credits, I40E_BW_CREDIT_DIVISOR);
8314 				dev_dbg(&vsi->back->pdev->dev,
8315 					"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
8316 					max_tx_rate,
8317 					credits,
8318 					vsi->seid);
8319 			} else {
8320 				need_reset = true;
8321 				goto exit;
8322 			}
8323 		}
8324 		ret = i40e_configure_queue_channels(vsi);
8325 		if (ret) {
8326 			vsi->num_queue_pairs = old_queue_pairs;
8327 			netdev_info(netdev,
8328 				    "Failed configuring queue channels\n");
8329 			need_reset = true;
8330 			goto exit;
8331 		}
8332 	}
8333 
8334 exit:
8335 	/* Reset the configuration data to defaults, only TC0 is enabled */
8336 	if (need_reset) {
8337 		i40e_vsi_set_default_tc_config(vsi);
8338 		need_reset = false;
8339 	}
8340 
8341 	/* Unquiesce VSI */
8342 	i40e_unquiesce_vsi(vsi);
8343 	return ret;
8344 }
8345 
8346 /**
8347  * i40e_set_cld_element - sets cloud filter element data
8348  * @filter: cloud filter rule
8349  * @cld: ptr to cloud filter element data
8350  *
8351  * This is helper function to copy data into cloud filter element
8352  **/
8353 static inline void
i40e_set_cld_element(struct i40e_cloud_filter * filter,struct i40e_aqc_cloud_filters_element_data * cld)8354 i40e_set_cld_element(struct i40e_cloud_filter *filter,
8355 		     struct i40e_aqc_cloud_filters_element_data *cld)
8356 {
8357 	u32 ipa;
8358 	int i;
8359 
8360 	memset(cld, 0, sizeof(*cld));
8361 	ether_addr_copy(cld->outer_mac, filter->dst_mac);
8362 	ether_addr_copy(cld->inner_mac, filter->src_mac);
8363 
8364 	if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
8365 		return;
8366 
8367 	if (filter->n_proto == ETH_P_IPV6) {
8368 #define IPV6_MAX_INDEX	(ARRAY_SIZE(filter->dst_ipv6) - 1)
8369 		for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
8370 			ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
8371 
8372 			*(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
8373 		}
8374 	} else {
8375 		ipa = be32_to_cpu(filter->dst_ipv4);
8376 
8377 		memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8378 	}
8379 
8380 	cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8381 
8382 	/* tenant_id is not supported by FW now, once the support is enabled
8383 	 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8384 	 */
8385 	if (filter->tenant_id)
8386 		return;
8387 }
8388 
8389 /**
8390  * i40e_add_del_cloud_filter - Add/del cloud filter
8391  * @vsi: pointer to VSI
8392  * @filter: cloud filter rule
8393  * @add: if true, add, if false, delete
8394  *
8395  * Add or delete a cloud filter for a specific flow spec.
8396  * Returns 0 if the filter were successfully added.
8397  **/
i40e_add_del_cloud_filter(struct i40e_vsi * vsi,struct i40e_cloud_filter * filter,bool add)8398 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8399 			      struct i40e_cloud_filter *filter, bool add)
8400 {
8401 	struct i40e_aqc_cloud_filters_element_data cld_filter;
8402 	struct i40e_pf *pf = vsi->back;
8403 	int ret;
8404 	static const u16 flag_table[128] = {
8405 		[I40E_CLOUD_FILTER_FLAGS_OMAC]  =
8406 			I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8407 		[I40E_CLOUD_FILTER_FLAGS_IMAC]  =
8408 			I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8409 		[I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
8410 			I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8411 		[I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8412 			I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8413 		[I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8414 			I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8415 		[I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8416 			I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8417 		[I40E_CLOUD_FILTER_FLAGS_IIP] =
8418 			I40E_AQC_ADD_CLOUD_FILTER_IIP,
8419 	};
8420 
8421 	if (filter->flags >= ARRAY_SIZE(flag_table))
8422 		return -EIO;
8423 
8424 	memset(&cld_filter, 0, sizeof(cld_filter));
8425 
8426 	/* copy element needed to add cloud filter from filter */
8427 	i40e_set_cld_element(filter, &cld_filter);
8428 
8429 	if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8430 		cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8431 					     I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8432 
8433 	if (filter->n_proto == ETH_P_IPV6)
8434 		cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8435 						I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8436 	else
8437 		cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8438 						I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8439 
8440 	if (add)
8441 		ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
8442 						&cld_filter, 1);
8443 	else
8444 		ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
8445 						&cld_filter, 1);
8446 	if (ret)
8447 		dev_dbg(&pf->pdev->dev,
8448 			"Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8449 			add ? "add" : "delete", filter->dst_port, ret,
8450 			pf->hw.aq.asq_last_status);
8451 	else
8452 		dev_info(&pf->pdev->dev,
8453 			 "%s cloud filter for VSI: %d\n",
8454 			 add ? "Added" : "Deleted", filter->seid);
8455 	return ret;
8456 }
8457 
8458 /**
8459  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8460  * @vsi: pointer to VSI
8461  * @filter: cloud filter rule
8462  * @add: if true, add, if false, delete
8463  *
8464  * Add or delete a cloud filter for a specific flow spec using big buffer.
8465  * Returns 0 if the filter were successfully added.
8466  **/
i40e_add_del_cloud_filter_big_buf(struct i40e_vsi * vsi,struct i40e_cloud_filter * filter,bool add)8467 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8468 				      struct i40e_cloud_filter *filter,
8469 				      bool add)
8470 {
8471 	struct i40e_aqc_cloud_filters_element_bb cld_filter;
8472 	struct i40e_pf *pf = vsi->back;
8473 	int ret;
8474 
8475 	/* Both (src/dst) valid mac_addr are not supported */
8476 	if ((is_valid_ether_addr(filter->dst_mac) &&
8477 	     is_valid_ether_addr(filter->src_mac)) ||
8478 	    (is_multicast_ether_addr(filter->dst_mac) &&
8479 	     is_multicast_ether_addr(filter->src_mac)))
8480 		return -EOPNOTSUPP;
8481 
8482 	/* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8483 	 * ports are not supported via big buffer now.
8484 	 */
8485 	if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8486 		return -EOPNOTSUPP;
8487 
8488 	/* adding filter using src_port/src_ip is not supported at this stage */
8489 	if (filter->src_port ||
8490 	    (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8491 	    !ipv6_addr_any(&filter->ip.v6.src_ip6))
8492 		return -EOPNOTSUPP;
8493 
8494 	memset(&cld_filter, 0, sizeof(cld_filter));
8495 
8496 	/* copy element needed to add cloud filter from filter */
8497 	i40e_set_cld_element(filter, &cld_filter.element);
8498 
8499 	if (is_valid_ether_addr(filter->dst_mac) ||
8500 	    is_valid_ether_addr(filter->src_mac) ||
8501 	    is_multicast_ether_addr(filter->dst_mac) ||
8502 	    is_multicast_ether_addr(filter->src_mac)) {
8503 		/* MAC + IP : unsupported mode */
8504 		if (filter->dst_ipv4)
8505 			return -EOPNOTSUPP;
8506 
8507 		/* since we validated that L4 port must be valid before
8508 		 * we get here, start with respective "flags" value
8509 		 * and update if vlan is present or not
8510 		 */
8511 		cld_filter.element.flags =
8512 			cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8513 
8514 		if (filter->vlan_id) {
8515 			cld_filter.element.flags =
8516 			cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8517 		}
8518 
8519 	} else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8520 		   !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
8521 		cld_filter.element.flags =
8522 				cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8523 		if (filter->n_proto == ETH_P_IPV6)
8524 			cld_filter.element.flags |=
8525 				cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8526 		else
8527 			cld_filter.element.flags |=
8528 				cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8529 	} else {
8530 		dev_err(&pf->pdev->dev,
8531 			"either mac or ip has to be valid for cloud filter\n");
8532 		return -EINVAL;
8533 	}
8534 
8535 	/* Now copy L4 port in Byte 6..7 in general fields */
8536 	cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8537 						be16_to_cpu(filter->dst_port);
8538 
8539 	if (add) {
8540 		/* Validate current device switch mode, change if necessary */
8541 		ret = i40e_validate_and_set_switch_mode(vsi);
8542 		if (ret) {
8543 			dev_err(&pf->pdev->dev,
8544 				"failed to set switch mode, ret %d\n",
8545 				ret);
8546 			return ret;
8547 		}
8548 
8549 		ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
8550 						   &cld_filter, 1);
8551 	} else {
8552 		ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
8553 						   &cld_filter, 1);
8554 	}
8555 
8556 	if (ret)
8557 		dev_dbg(&pf->pdev->dev,
8558 			"Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8559 			add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8560 	else
8561 		dev_info(&pf->pdev->dev,
8562 			 "%s cloud filter for VSI: %d, L4 port: %d\n",
8563 			 add ? "add" : "delete", filter->seid,
8564 			 ntohs(filter->dst_port));
8565 	return ret;
8566 }
8567 
8568 /**
8569  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8570  * @vsi: Pointer to VSI
8571  * @f: Pointer to struct flow_cls_offload
8572  * @filter: Pointer to cloud filter structure
8573  *
8574  **/
i40e_parse_cls_flower(struct i40e_vsi * vsi,struct flow_cls_offload * f,struct i40e_cloud_filter * filter)8575 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8576 				 struct flow_cls_offload *f,
8577 				 struct i40e_cloud_filter *filter)
8578 {
8579 	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8580 	struct flow_dissector *dissector = rule->match.dissector;
8581 	u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8582 	struct i40e_pf *pf = vsi->back;
8583 	u8 field_flags = 0;
8584 
8585 	if (dissector->used_keys &
8586 	    ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
8587 	      BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
8588 	      BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8589 	      BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
8590 	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8591 	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8592 	      BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
8593 	      BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8594 		dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n",
8595 			dissector->used_keys);
8596 		return -EOPNOTSUPP;
8597 	}
8598 
8599 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8600 		struct flow_match_enc_keyid match;
8601 
8602 		flow_rule_match_enc_keyid(rule, &match);
8603 		if (match.mask->keyid != 0)
8604 			field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8605 
8606 		filter->tenant_id = be32_to_cpu(match.key->keyid);
8607 	}
8608 
8609 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8610 		struct flow_match_basic match;
8611 
8612 		flow_rule_match_basic(rule, &match);
8613 		n_proto_key = ntohs(match.key->n_proto);
8614 		n_proto_mask = ntohs(match.mask->n_proto);
8615 
8616 		if (n_proto_key == ETH_P_ALL) {
8617 			n_proto_key = 0;
8618 			n_proto_mask = 0;
8619 		}
8620 		filter->n_proto = n_proto_key & n_proto_mask;
8621 		filter->ip_proto = match.key->ip_proto;
8622 	}
8623 
8624 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8625 		struct flow_match_eth_addrs match;
8626 
8627 		flow_rule_match_eth_addrs(rule, &match);
8628 
8629 		/* use is_broadcast and is_zero to check for all 0xf or 0 */
8630 		if (!is_zero_ether_addr(match.mask->dst)) {
8631 			if (is_broadcast_ether_addr(match.mask->dst)) {
8632 				field_flags |= I40E_CLOUD_FIELD_OMAC;
8633 			} else {
8634 				dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8635 					match.mask->dst);
8636 				return -EIO;
8637 			}
8638 		}
8639 
8640 		if (!is_zero_ether_addr(match.mask->src)) {
8641 			if (is_broadcast_ether_addr(match.mask->src)) {
8642 				field_flags |= I40E_CLOUD_FIELD_IMAC;
8643 			} else {
8644 				dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8645 					match.mask->src);
8646 				return -EIO;
8647 			}
8648 		}
8649 		ether_addr_copy(filter->dst_mac, match.key->dst);
8650 		ether_addr_copy(filter->src_mac, match.key->src);
8651 	}
8652 
8653 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8654 		struct flow_match_vlan match;
8655 
8656 		flow_rule_match_vlan(rule, &match);
8657 		if (match.mask->vlan_id) {
8658 			if (match.mask->vlan_id == VLAN_VID_MASK) {
8659 				field_flags |= I40E_CLOUD_FIELD_IVLAN;
8660 
8661 			} else {
8662 				dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8663 					match.mask->vlan_id);
8664 				return -EIO;
8665 			}
8666 		}
8667 
8668 		filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8669 	}
8670 
8671 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8672 		struct flow_match_control match;
8673 
8674 		flow_rule_match_control(rule, &match);
8675 		addr_type = match.key->addr_type;
8676 
8677 		if (flow_rule_has_control_flags(match.mask->flags,
8678 						f->common.extack))
8679 			return -EOPNOTSUPP;
8680 	}
8681 
8682 	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8683 		struct flow_match_ipv4_addrs match;
8684 
8685 		flow_rule_match_ipv4_addrs(rule, &match);
8686 		if (match.mask->dst) {
8687 			if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8688 				field_flags |= I40E_CLOUD_FIELD_IIP;
8689 			} else {
8690 				dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8691 					&match.mask->dst);
8692 				return -EIO;
8693 			}
8694 		}
8695 
8696 		if (match.mask->src) {
8697 			if (match.mask->src == cpu_to_be32(0xffffffff)) {
8698 				field_flags |= I40E_CLOUD_FIELD_IIP;
8699 			} else {
8700 				dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8701 					&match.mask->src);
8702 				return -EIO;
8703 			}
8704 		}
8705 
8706 		if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8707 			dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8708 			return -EIO;
8709 		}
8710 		filter->dst_ipv4 = match.key->dst;
8711 		filter->src_ipv4 = match.key->src;
8712 	}
8713 
8714 	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8715 		struct flow_match_ipv6_addrs match;
8716 
8717 		flow_rule_match_ipv6_addrs(rule, &match);
8718 
8719 		/* src and dest IPV6 address should not be LOOPBACK
8720 		 * (0:0:0:0:0:0:0:1), which can be represented as ::1
8721 		 */
8722 		if (ipv6_addr_loopback(&match.key->dst) ||
8723 		    ipv6_addr_loopback(&match.key->src)) {
8724 			dev_err(&pf->pdev->dev,
8725 				"Bad ipv6, addr is LOOPBACK\n");
8726 			return -EIO;
8727 		}
8728 		if (!ipv6_addr_any(&match.mask->dst) ||
8729 		    !ipv6_addr_any(&match.mask->src))
8730 			field_flags |= I40E_CLOUD_FIELD_IIP;
8731 
8732 		memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8733 		       sizeof(filter->src_ipv6));
8734 		memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8735 		       sizeof(filter->dst_ipv6));
8736 	}
8737 
8738 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8739 		struct flow_match_ports match;
8740 
8741 		flow_rule_match_ports(rule, &match);
8742 		if (match.mask->src) {
8743 			if (match.mask->src == cpu_to_be16(0xffff)) {
8744 				field_flags |= I40E_CLOUD_FIELD_IIP;
8745 			} else {
8746 				dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8747 					be16_to_cpu(match.mask->src));
8748 				return -EIO;
8749 			}
8750 		}
8751 
8752 		if (match.mask->dst) {
8753 			if (match.mask->dst == cpu_to_be16(0xffff)) {
8754 				field_flags |= I40E_CLOUD_FIELD_IIP;
8755 			} else {
8756 				dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8757 					be16_to_cpu(match.mask->dst));
8758 				return -EIO;
8759 			}
8760 		}
8761 
8762 		filter->dst_port = match.key->dst;
8763 		filter->src_port = match.key->src;
8764 
8765 		switch (filter->ip_proto) {
8766 		case IPPROTO_TCP:
8767 		case IPPROTO_UDP:
8768 			break;
8769 		default:
8770 			dev_err(&pf->pdev->dev,
8771 				"Only UDP and TCP transport are supported\n");
8772 			return -EINVAL;
8773 		}
8774 	}
8775 	filter->flags = field_flags;
8776 	return 0;
8777 }
8778 
8779 /**
8780  * i40e_handle_tclass: Forward to a traffic class on the device
8781  * @vsi: Pointer to VSI
8782  * @tc: traffic class index on the device
8783  * @filter: Pointer to cloud filter structure
8784  *
8785  **/
i40e_handle_tclass(struct i40e_vsi * vsi,u32 tc,struct i40e_cloud_filter * filter)8786 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8787 			      struct i40e_cloud_filter *filter)
8788 {
8789 	struct i40e_channel *ch, *ch_tmp;
8790 
8791 	/* direct to a traffic class on the same device */
8792 	if (tc == 0) {
8793 		filter->seid = vsi->seid;
8794 		return 0;
8795 	} else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8796 		if (!filter->dst_port) {
8797 			dev_err(&vsi->back->pdev->dev,
8798 				"Specify destination port to direct to traffic class that is not default\n");
8799 			return -EINVAL;
8800 		}
8801 		if (list_empty(&vsi->ch_list))
8802 			return -EINVAL;
8803 		list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8804 					 list) {
8805 			if (ch->seid == vsi->tc_seid_map[tc])
8806 				filter->seid = ch->seid;
8807 		}
8808 		return 0;
8809 	}
8810 	dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8811 	return -EINVAL;
8812 }
8813 
8814 /**
8815  * i40e_configure_clsflower - Configure tc flower filters
8816  * @vsi: Pointer to VSI
8817  * @cls_flower: Pointer to struct flow_cls_offload
8818  *
8819  **/
i40e_configure_clsflower(struct i40e_vsi * vsi,struct flow_cls_offload * cls_flower)8820 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8821 				    struct flow_cls_offload *cls_flower)
8822 {
8823 	int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8824 	struct i40e_cloud_filter *filter = NULL;
8825 	struct i40e_pf *pf = vsi->back;
8826 	int err = 0;
8827 
8828 	if (tc < 0) {
8829 		dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8830 		return -EOPNOTSUPP;
8831 	}
8832 
8833 	if (!tc) {
8834 		dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");
8835 		return -EINVAL;
8836 	}
8837 
8838 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8839 	    test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8840 		return -EBUSY;
8841 
8842 	if (pf->fdir_pf_active_filters ||
8843 	    (!hlist_empty(&pf->fdir_filter_list))) {
8844 		dev_err(&vsi->back->pdev->dev,
8845 			"Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8846 		return -EINVAL;
8847 	}
8848 
8849 	if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) {
8850 		dev_err(&vsi->back->pdev->dev,
8851 			"Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8852 		clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags);
8853 		clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags);
8854 	}
8855 
8856 	filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8857 	if (!filter)
8858 		return -ENOMEM;
8859 
8860 	filter->cookie = cls_flower->cookie;
8861 
8862 	err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8863 	if (err < 0)
8864 		goto err;
8865 
8866 	err = i40e_handle_tclass(vsi, tc, filter);
8867 	if (err < 0)
8868 		goto err;
8869 
8870 	/* Add cloud filter */
8871 	if (filter->dst_port)
8872 		err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8873 	else
8874 		err = i40e_add_del_cloud_filter(vsi, filter, true);
8875 
8876 	if (err) {
8877 		dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8878 			err);
8879 		goto err;
8880 	}
8881 
8882 	/* add filter to the ordered list */
8883 	INIT_HLIST_NODE(&filter->cloud_node);
8884 
8885 	hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8886 
8887 	pf->num_cloud_filters++;
8888 
8889 	return err;
8890 err:
8891 	kfree(filter);
8892 	return err;
8893 }
8894 
8895 /**
8896  * i40e_find_cloud_filter - Find the could filter in the list
8897  * @vsi: Pointer to VSI
8898  * @cookie: filter specific cookie
8899  *
8900  **/
i40e_find_cloud_filter(struct i40e_vsi * vsi,unsigned long * cookie)8901 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8902 							unsigned long *cookie)
8903 {
8904 	struct i40e_cloud_filter *filter = NULL;
8905 	struct hlist_node *node2;
8906 
8907 	hlist_for_each_entry_safe(filter, node2,
8908 				  &vsi->back->cloud_filter_list, cloud_node)
8909 		if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8910 			return filter;
8911 	return NULL;
8912 }
8913 
8914 /**
8915  * i40e_delete_clsflower - Remove tc flower filters
8916  * @vsi: Pointer to VSI
8917  * @cls_flower: Pointer to struct flow_cls_offload
8918  *
8919  **/
i40e_delete_clsflower(struct i40e_vsi * vsi,struct flow_cls_offload * cls_flower)8920 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8921 				 struct flow_cls_offload *cls_flower)
8922 {
8923 	struct i40e_cloud_filter *filter = NULL;
8924 	struct i40e_pf *pf = vsi->back;
8925 	int err = 0;
8926 
8927 	filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8928 
8929 	if (!filter)
8930 		return -EINVAL;
8931 
8932 	hash_del(&filter->cloud_node);
8933 
8934 	if (filter->dst_port)
8935 		err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8936 	else
8937 		err = i40e_add_del_cloud_filter(vsi, filter, false);
8938 
8939 	kfree(filter);
8940 	if (err) {
8941 		dev_err(&pf->pdev->dev,
8942 			"Failed to delete cloud filter, err %pe\n",
8943 			ERR_PTR(err));
8944 		return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8945 	}
8946 
8947 	pf->num_cloud_filters--;
8948 	if (!pf->num_cloud_filters)
8949 		if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
8950 		    !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
8951 			set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
8952 			clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
8953 			clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
8954 		}
8955 	return 0;
8956 }
8957 
8958 /**
8959  * i40e_setup_tc_cls_flower - flower classifier offloads
8960  * @np: net device to configure
8961  * @cls_flower: offload data
8962  **/
i40e_setup_tc_cls_flower(struct i40e_netdev_priv * np,struct flow_cls_offload * cls_flower)8963 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8964 				    struct flow_cls_offload *cls_flower)
8965 {
8966 	struct i40e_vsi *vsi = np->vsi;
8967 
8968 	switch (cls_flower->command) {
8969 	case FLOW_CLS_REPLACE:
8970 		return i40e_configure_clsflower(vsi, cls_flower);
8971 	case FLOW_CLS_DESTROY:
8972 		return i40e_delete_clsflower(vsi, cls_flower);
8973 	case FLOW_CLS_STATS:
8974 		return -EOPNOTSUPP;
8975 	default:
8976 		return -EOPNOTSUPP;
8977 	}
8978 }
8979 
i40e_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)8980 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8981 				  void *cb_priv)
8982 {
8983 	struct i40e_netdev_priv *np = cb_priv;
8984 
8985 	if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8986 		return -EOPNOTSUPP;
8987 
8988 	switch (type) {
8989 	case TC_SETUP_CLSFLOWER:
8990 		return i40e_setup_tc_cls_flower(np, type_data);
8991 
8992 	default:
8993 		return -EOPNOTSUPP;
8994 	}
8995 }
8996 
8997 static LIST_HEAD(i40e_block_cb_list);
8998 
__i40e_setup_tc(struct net_device * netdev,enum tc_setup_type type,void * type_data)8999 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
9000 			   void *type_data)
9001 {
9002 	struct i40e_netdev_priv *np = netdev_priv(netdev);
9003 
9004 	switch (type) {
9005 	case TC_SETUP_QDISC_MQPRIO:
9006 		return i40e_setup_tc(netdev, type_data);
9007 	case TC_SETUP_BLOCK:
9008 		return flow_block_cb_setup_simple(type_data,
9009 						  &i40e_block_cb_list,
9010 						  i40e_setup_tc_block_cb,
9011 						  np, np, true);
9012 	default:
9013 		return -EOPNOTSUPP;
9014 	}
9015 }
9016 
9017 /**
9018  * i40e_open - Called when a network interface is made active
9019  * @netdev: network interface device structure
9020  *
9021  * The open entry point is called when a network interface is made
9022  * active by the system (IFF_UP).  At this point all resources needed
9023  * for transmit and receive operations are allocated, the interrupt
9024  * handler is registered with the OS, the netdev watchdog subtask is
9025  * enabled, and the stack is notified that the interface is ready.
9026  *
9027  * Returns 0 on success, negative value on failure
9028  **/
i40e_open(struct net_device * netdev)9029 int i40e_open(struct net_device *netdev)
9030 {
9031 	struct i40e_netdev_priv *np = netdev_priv(netdev);
9032 	struct i40e_vsi *vsi = np->vsi;
9033 	struct i40e_pf *pf = vsi->back;
9034 	int err;
9035 
9036 	/* disallow open during test or if eeprom is broken */
9037 	if (test_bit(__I40E_TESTING, pf->state) ||
9038 	    test_bit(__I40E_BAD_EEPROM, pf->state))
9039 		return -EBUSY;
9040 
9041 	netif_carrier_off(netdev);
9042 
9043 	if (i40e_force_link_state(pf, true))
9044 		return -EAGAIN;
9045 
9046 	err = i40e_vsi_open(vsi);
9047 	if (err)
9048 		return err;
9049 
9050 	/* configure global TSO hardware offload settings */
9051 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
9052 						       TCP_FLAG_FIN) >> 16);
9053 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
9054 						       TCP_FLAG_FIN |
9055 						       TCP_FLAG_CWR) >> 16);
9056 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
9057 	udp_tunnel_get_rx_info(netdev);
9058 
9059 	return 0;
9060 }
9061 
9062 /**
9063  * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
9064  * @vsi: vsi structure
9065  *
9066  * This updates netdev's number of tx/rx queues
9067  *
9068  * Returns status of setting tx/rx queues
9069  **/
i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi * vsi)9070 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
9071 {
9072 	int ret;
9073 
9074 	ret = netif_set_real_num_rx_queues(vsi->netdev,
9075 					   vsi->num_queue_pairs);
9076 	if (ret)
9077 		return ret;
9078 
9079 	return netif_set_real_num_tx_queues(vsi->netdev,
9080 					    vsi->num_queue_pairs);
9081 }
9082 
9083 /**
9084  * i40e_vsi_open -
9085  * @vsi: the VSI to open
9086  *
9087  * Finish initialization of the VSI.
9088  *
9089  * Returns 0 on success, negative value on failure
9090  *
9091  * Note: expects to be called while under rtnl_lock()
9092  **/
i40e_vsi_open(struct i40e_vsi * vsi)9093 int i40e_vsi_open(struct i40e_vsi *vsi)
9094 {
9095 	struct i40e_pf *pf = vsi->back;
9096 	char int_name[I40E_INT_NAME_STR_LEN];
9097 	int err;
9098 
9099 	/* allocate descriptors */
9100 	err = i40e_vsi_setup_tx_resources(vsi);
9101 	if (err)
9102 		goto err_setup_tx;
9103 	err = i40e_vsi_setup_rx_resources(vsi);
9104 	if (err)
9105 		goto err_setup_rx;
9106 
9107 	err = i40e_vsi_configure(vsi);
9108 	if (err)
9109 		goto err_setup_rx;
9110 
9111 	if (vsi->netdev) {
9112 		snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
9113 			 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
9114 		err = i40e_vsi_request_irq(vsi, int_name);
9115 		if (err)
9116 			goto err_setup_rx;
9117 
9118 		/* Notify the stack of the actual queue counts. */
9119 		err = i40e_netif_set_realnum_tx_rx_queues(vsi);
9120 		if (err)
9121 			goto err_set_queues;
9122 
9123 	} else if (vsi->type == I40E_VSI_FDIR) {
9124 		snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
9125 			 dev_driver_string(&pf->pdev->dev),
9126 			 dev_name(&pf->pdev->dev));
9127 		err = i40e_vsi_request_irq(vsi, int_name);
9128 		if (err)
9129 			goto err_setup_rx;
9130 
9131 	} else {
9132 		err = -EINVAL;
9133 		goto err_setup_rx;
9134 	}
9135 
9136 	err = i40e_up_complete(vsi);
9137 	if (err)
9138 		goto err_up_complete;
9139 
9140 	return 0;
9141 
9142 err_up_complete:
9143 	i40e_down(vsi);
9144 err_set_queues:
9145 	i40e_vsi_free_irq(vsi);
9146 err_setup_rx:
9147 	i40e_vsi_free_rx_resources(vsi);
9148 err_setup_tx:
9149 	i40e_vsi_free_tx_resources(vsi);
9150 	if (vsi->type == I40E_VSI_MAIN)
9151 		i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
9152 
9153 	return err;
9154 }
9155 
9156 /**
9157  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
9158  * @pf: Pointer to PF
9159  *
9160  * This function destroys the hlist where all the Flow Director
9161  * filters were saved.
9162  **/
i40e_fdir_filter_exit(struct i40e_pf * pf)9163 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
9164 {
9165 	struct i40e_fdir_filter *filter;
9166 	struct i40e_flex_pit *pit_entry, *tmp;
9167 	struct hlist_node *node2;
9168 
9169 	hlist_for_each_entry_safe(filter, node2,
9170 				  &pf->fdir_filter_list, fdir_node) {
9171 		hlist_del(&filter->fdir_node);
9172 		kfree(filter);
9173 	}
9174 
9175 	list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
9176 		list_del(&pit_entry->list);
9177 		kfree(pit_entry);
9178 	}
9179 	INIT_LIST_HEAD(&pf->l3_flex_pit_list);
9180 
9181 	list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
9182 		list_del(&pit_entry->list);
9183 		kfree(pit_entry);
9184 	}
9185 	INIT_LIST_HEAD(&pf->l4_flex_pit_list);
9186 
9187 	pf->fdir_pf_active_filters = 0;
9188 	i40e_reset_fdir_filter_cnt(pf);
9189 
9190 	/* Reprogram the default input set for TCP/IPv4 */
9191 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9192 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9193 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9194 
9195 	/* Reprogram the default input set for TCP/IPv6 */
9196 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
9197 				I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9198 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9199 
9200 	/* Reprogram the default input set for UDP/IPv4 */
9201 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
9202 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9203 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9204 
9205 	/* Reprogram the default input set for UDP/IPv6 */
9206 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
9207 				I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9208 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9209 
9210 	/* Reprogram the default input set for SCTP/IPv4 */
9211 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
9212 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9213 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9214 
9215 	/* Reprogram the default input set for SCTP/IPv6 */
9216 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
9217 				I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9218 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9219 
9220 	/* Reprogram the default input set for Other/IPv4 */
9221 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
9222 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9223 
9224 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
9225 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9226 
9227 	/* Reprogram the default input set for Other/IPv6 */
9228 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
9229 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9230 
9231 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,
9232 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9233 }
9234 
9235 /**
9236  * i40e_cloud_filter_exit - Cleans up the cloud filters
9237  * @pf: Pointer to PF
9238  *
9239  * This function destroys the hlist where all the cloud filters
9240  * were saved.
9241  **/
i40e_cloud_filter_exit(struct i40e_pf * pf)9242 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
9243 {
9244 	struct i40e_cloud_filter *cfilter;
9245 	struct hlist_node *node;
9246 
9247 	hlist_for_each_entry_safe(cfilter, node,
9248 				  &pf->cloud_filter_list, cloud_node) {
9249 		hlist_del(&cfilter->cloud_node);
9250 		kfree(cfilter);
9251 	}
9252 	pf->num_cloud_filters = 0;
9253 
9254 	if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
9255 	    !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
9256 		set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
9257 		clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
9258 		clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
9259 	}
9260 }
9261 
9262 /**
9263  * i40e_close - Disables a network interface
9264  * @netdev: network interface device structure
9265  *
9266  * The close entry point is called when an interface is de-activated
9267  * by the OS.  The hardware is still under the driver's control, but
9268  * this netdev interface is disabled.
9269  *
9270  * Returns 0, this is not allowed to fail
9271  **/
i40e_close(struct net_device * netdev)9272 int i40e_close(struct net_device *netdev)
9273 {
9274 	struct i40e_netdev_priv *np = netdev_priv(netdev);
9275 	struct i40e_vsi *vsi = np->vsi;
9276 
9277 	i40e_vsi_close(vsi);
9278 
9279 	return 0;
9280 }
9281 
9282 /**
9283  * i40e_do_reset - Start a PF or Core Reset sequence
9284  * @pf: board private structure
9285  * @reset_flags: which reset is requested
9286  * @lock_acquired: indicates whether or not the lock has been acquired
9287  * before this function was called.
9288  *
9289  * The essential difference in resets is that the PF Reset
9290  * doesn't clear the packet buffers, doesn't reset the PE
9291  * firmware, and doesn't bother the other PFs on the chip.
9292  **/
i40e_do_reset(struct i40e_pf * pf,u32 reset_flags,bool lock_acquired)9293 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
9294 {
9295 	struct i40e_vsi *vsi;
9296 	u32 val;
9297 	int i;
9298 
9299 	/* do the biggest reset indicated */
9300 	if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
9301 
9302 		/* Request a Global Reset
9303 		 *
9304 		 * This will start the chip's countdown to the actual full
9305 		 * chip reset event, and a warning interrupt to be sent
9306 		 * to all PFs, including the requestor.  Our handler
9307 		 * for the warning interrupt will deal with the shutdown
9308 		 * and recovery of the switch setup.
9309 		 */
9310 		dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
9311 		val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9312 		val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
9313 		wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9314 
9315 	} else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
9316 
9317 		/* Request a Core Reset
9318 		 *
9319 		 * Same as Global Reset, except does *not* include the MAC/PHY
9320 		 */
9321 		dev_dbg(&pf->pdev->dev, "CoreR requested\n");
9322 		val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9323 		val |= I40E_GLGEN_RTRIG_CORER_MASK;
9324 		wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9325 		i40e_flush(&pf->hw);
9326 
9327 	} else if (reset_flags & I40E_PF_RESET_FLAG) {
9328 
9329 		/* Request a PF Reset
9330 		 *
9331 		 * Resets only the PF-specific registers
9332 		 *
9333 		 * This goes directly to the tear-down and rebuild of
9334 		 * the switch, since we need to do all the recovery as
9335 		 * for the Core Reset.
9336 		 */
9337 		dev_dbg(&pf->pdev->dev, "PFR requested\n");
9338 		i40e_handle_reset_warning(pf, lock_acquired);
9339 
9340 	} else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
9341 		/* Request a PF Reset
9342 		 *
9343 		 * Resets PF and reinitializes PFs VSI.
9344 		 */
9345 		i40e_prep_for_reset(pf);
9346 		i40e_reset_and_rebuild(pf, true, lock_acquired);
9347 		dev_info(&pf->pdev->dev,
9348 			 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
9349 			 "FW LLDP is disabled\n" :
9350 			 "FW LLDP is enabled\n");
9351 
9352 	} else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
9353 		/* Find the VSI(s) that requested a re-init */
9354 		dev_info(&pf->pdev->dev, "VSI reinit requested\n");
9355 
9356 		i40e_pf_for_each_vsi(pf, i, vsi) {
9357 			if (test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
9358 					       vsi->state))
9359 				i40e_vsi_reinit_locked(vsi);
9360 		}
9361 	} else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
9362 		/* Find the VSI(s) that needs to be brought down */
9363 		dev_info(&pf->pdev->dev, "VSI down requested\n");
9364 
9365 		i40e_pf_for_each_vsi(pf, i, vsi) {
9366 			if (test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
9367 					       vsi->state)) {
9368 				set_bit(__I40E_VSI_DOWN, vsi->state);
9369 				i40e_down(vsi);
9370 			}
9371 		}
9372 	} else {
9373 		dev_info(&pf->pdev->dev,
9374 			 "bad reset request 0x%08x\n", reset_flags);
9375 	}
9376 }
9377 
9378 #ifdef CONFIG_I40E_DCB
9379 /**
9380  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
9381  * @pf: board private structure
9382  * @old_cfg: current DCB config
9383  * @new_cfg: new DCB config
9384  **/
i40e_dcb_need_reconfig(struct i40e_pf * pf,struct i40e_dcbx_config * old_cfg,struct i40e_dcbx_config * new_cfg)9385 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
9386 			    struct i40e_dcbx_config *old_cfg,
9387 			    struct i40e_dcbx_config *new_cfg)
9388 {
9389 	bool need_reconfig = false;
9390 
9391 	/* Check if ETS configuration has changed */
9392 	if (memcmp(&new_cfg->etscfg,
9393 		   &old_cfg->etscfg,
9394 		   sizeof(new_cfg->etscfg))) {
9395 		/* If Priority Table has changed reconfig is needed */
9396 		if (memcmp(&new_cfg->etscfg.prioritytable,
9397 			   &old_cfg->etscfg.prioritytable,
9398 			   sizeof(new_cfg->etscfg.prioritytable))) {
9399 			need_reconfig = true;
9400 			dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9401 		}
9402 
9403 		if (memcmp(&new_cfg->etscfg.tcbwtable,
9404 			   &old_cfg->etscfg.tcbwtable,
9405 			   sizeof(new_cfg->etscfg.tcbwtable)))
9406 			dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9407 
9408 		if (memcmp(&new_cfg->etscfg.tsatable,
9409 			   &old_cfg->etscfg.tsatable,
9410 			   sizeof(new_cfg->etscfg.tsatable)))
9411 			dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9412 	}
9413 
9414 	/* Check if PFC configuration has changed */
9415 	if (memcmp(&new_cfg->pfc,
9416 		   &old_cfg->pfc,
9417 		   sizeof(new_cfg->pfc))) {
9418 		need_reconfig = true;
9419 		dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9420 	}
9421 
9422 	/* Check if APP Table has changed */
9423 	if (memcmp(&new_cfg->app,
9424 		   &old_cfg->app,
9425 		   sizeof(new_cfg->app))) {
9426 		need_reconfig = true;
9427 		dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9428 	}
9429 
9430 	dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9431 	return need_reconfig;
9432 }
9433 
9434 /**
9435  * i40e_handle_lldp_event - Handle LLDP Change MIB event
9436  * @pf: board private structure
9437  * @e: event info posted on ARQ
9438  **/
i40e_handle_lldp_event(struct i40e_pf * pf,struct i40e_arq_event_info * e)9439 static int i40e_handle_lldp_event(struct i40e_pf *pf,
9440 				  struct i40e_arq_event_info *e)
9441 {
9442 	struct i40e_aqc_lldp_get_mib *mib =
9443 		(struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9444 	struct i40e_hw *hw = &pf->hw;
9445 	struct i40e_dcbx_config tmp_dcbx_cfg;
9446 	bool need_reconfig = false;
9447 	int ret = 0;
9448 	u8 type;
9449 
9450 	/* X710-T*L 2.5G and 5G speeds don't support DCB */
9451 	if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9452 	    (hw->phy.link_info.link_speed &
9453 	     ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9454 	     !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9455 		/* let firmware decide if the DCB should be disabled */
9456 		set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9457 
9458 	/* Not DCB capable or capability disabled */
9459 	if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9460 		return ret;
9461 
9462 	/* Ignore if event is not for Nearest Bridge */
9463 	type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9464 		& I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9465 	dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9466 	if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9467 		return ret;
9468 
9469 	/* Check MIB Type and return if event for Remote MIB update */
9470 	type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9471 	dev_dbg(&pf->pdev->dev,
9472 		"LLDP event mib type %s\n", type ? "remote" : "local");
9473 	if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9474 		/* Update the remote cached instance and return */
9475 		ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9476 				I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9477 				&hw->remote_dcbx_config);
9478 		goto exit;
9479 	}
9480 
9481 	/* Store the old configuration */
9482 	tmp_dcbx_cfg = hw->local_dcbx_config;
9483 
9484 	/* Reset the old DCBx configuration data */
9485 	memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9486 	/* Get updated DCBX data from firmware */
9487 	ret = i40e_get_dcb_config(&pf->hw);
9488 	if (ret) {
9489 		/* X710-T*L 2.5G and 5G speeds don't support DCB */
9490 		if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9491 		    (hw->phy.link_info.link_speed &
9492 		     (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9493 			dev_warn(&pf->pdev->dev,
9494 				 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
9495 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9496 		} else {
9497 			dev_info(&pf->pdev->dev,
9498 				 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n",
9499 				 ERR_PTR(ret),
9500 				 i40e_aq_str(&pf->hw,
9501 					     pf->hw.aq.asq_last_status));
9502 		}
9503 		goto exit;
9504 	}
9505 
9506 	/* No change detected in DCBX configs */
9507 	if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
9508 		    sizeof(tmp_dcbx_cfg))) {
9509 		dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9510 		goto exit;
9511 	}
9512 
9513 	need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
9514 					       &hw->local_dcbx_config);
9515 
9516 	i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
9517 
9518 	if (!need_reconfig)
9519 		goto exit;
9520 
9521 	/* Enable DCB tagging only when more than one TC */
9522 	if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
9523 		set_bit(I40E_FLAG_DCB_ENA, pf->flags);
9524 	else
9525 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
9526 
9527 	set_bit(__I40E_PORT_SUSPENDED, pf->state);
9528 	/* Reconfiguration needed quiesce all VSIs */
9529 	i40e_pf_quiesce_all_vsi(pf);
9530 
9531 	/* Changes in configuration update VEB/VSI */
9532 	i40e_dcb_reconfigure(pf);
9533 
9534 	ret = i40e_resume_port_tx(pf);
9535 
9536 	clear_bit(__I40E_PORT_SUSPENDED, pf->state);
9537 	/* In case of error no point in resuming VSIs */
9538 	if (ret)
9539 		goto exit;
9540 
9541 	/* Wait for the PF's queues to be disabled */
9542 	ret = i40e_pf_wait_queues_disabled(pf);
9543 	if (ret) {
9544 		/* Schedule PF reset to recover */
9545 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9546 		i40e_service_event_schedule(pf);
9547 	} else {
9548 		i40e_pf_unquiesce_all_vsi(pf);
9549 		set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
9550 		set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
9551 	}
9552 
9553 exit:
9554 	return ret;
9555 }
9556 #endif /* CONFIG_I40E_DCB */
9557 
9558 /**
9559  * i40e_do_reset_safe - Protected reset path for userland calls.
9560  * @pf: board private structure
9561  * @reset_flags: which reset is requested
9562  *
9563  **/
i40e_do_reset_safe(struct i40e_pf * pf,u32 reset_flags)9564 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9565 {
9566 	rtnl_lock();
9567 	i40e_do_reset(pf, reset_flags, true);
9568 	rtnl_unlock();
9569 }
9570 
9571 /**
9572  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9573  * @pf: board private structure
9574  * @e: event info posted on ARQ
9575  *
9576  * Handler for LAN Queue Overflow Event generated by the firmware for PF
9577  * and VF queues
9578  **/
i40e_handle_lan_overflow_event(struct i40e_pf * pf,struct i40e_arq_event_info * e)9579 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9580 					   struct i40e_arq_event_info *e)
9581 {
9582 	struct i40e_aqc_lan_overflow *data =
9583 		(struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9584 	u32 queue = le32_to_cpu(data->prtdcb_rupto);
9585 	u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9586 	struct i40e_hw *hw = &pf->hw;
9587 	struct i40e_vf *vf;
9588 	u16 vf_id;
9589 
9590 	dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9591 		queue, qtx_ctl);
9592 
9593 	if (FIELD_GET(I40E_QTX_CTL_PFVF_Q_MASK, qtx_ctl) !=
9594 	    I40E_QTX_CTL_VF_QUEUE)
9595 		return;
9596 
9597 	/* Queue belongs to VF, find the VF and issue VF reset */
9598 	vf_id = FIELD_GET(I40E_QTX_CTL_VFVM_INDX_MASK, qtx_ctl);
9599 	vf_id -= hw->func_caps.vf_base_id;
9600 	vf = &pf->vf[vf_id];
9601 	i40e_vc_notify_vf_reset(vf);
9602 	/* Allow VF to process pending reset notification */
9603 	msleep(20);
9604 	i40e_reset_vf(vf, false);
9605 }
9606 
9607 /**
9608  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9609  * @pf: board private structure
9610  **/
i40e_get_current_fd_count(struct i40e_pf * pf)9611 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9612 {
9613 	u32 val, fcnt_prog;
9614 
9615 	val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9616 	fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9617 		    FIELD_GET(I40E_PFQF_FDSTAT_BEST_CNT_MASK, val);
9618 	return fcnt_prog;
9619 }
9620 
9621 /**
9622  * i40e_get_global_fd_count - Get total FD filters programmed on device
9623  * @pf: board private structure
9624  **/
i40e_get_global_fd_count(struct i40e_pf * pf)9625 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9626 {
9627 	u32 val, fcnt_prog;
9628 
9629 	val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9630 	fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9631 		    FIELD_GET(I40E_GLQF_FDCNT_0_BESTCNT_MASK, val);
9632 	return fcnt_prog;
9633 }
9634 
9635 /**
9636  * i40e_reenable_fdir_sb - Restore FDir SB capability
9637  * @pf: board private structure
9638  **/
i40e_reenable_fdir_sb(struct i40e_pf * pf)9639 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9640 {
9641 	if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9642 		if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
9643 		    (I40E_DEBUG_FD & pf->hw.debug_mask))
9644 			dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9645 }
9646 
9647 /**
9648  * i40e_reenable_fdir_atr - Restore FDir ATR capability
9649  * @pf: board private structure
9650  **/
i40e_reenable_fdir_atr(struct i40e_pf * pf)9651 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9652 {
9653 	if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9654 		/* ATR uses the same filtering logic as SB rules. It only
9655 		 * functions properly if the input set mask is at the default
9656 		 * settings. It is safe to restore the default input set
9657 		 * because there are no active TCPv4 filter rules.
9658 		 */
9659 		i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9660 					I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9661 					I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9662 
9663 		if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
9664 		    (I40E_DEBUG_FD & pf->hw.debug_mask))
9665 			dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9666 	}
9667 }
9668 
9669 /**
9670  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9671  * @pf: board private structure
9672  * @filter: FDir filter to remove
9673  */
i40e_delete_invalid_filter(struct i40e_pf * pf,struct i40e_fdir_filter * filter)9674 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9675 				       struct i40e_fdir_filter *filter)
9676 {
9677 	/* Update counters */
9678 	pf->fdir_pf_active_filters--;
9679 	pf->fd_inv = 0;
9680 
9681 	switch (filter->flow_type) {
9682 	case TCP_V4_FLOW:
9683 		pf->fd_tcp4_filter_cnt--;
9684 		break;
9685 	case UDP_V4_FLOW:
9686 		pf->fd_udp4_filter_cnt--;
9687 		break;
9688 	case SCTP_V4_FLOW:
9689 		pf->fd_sctp4_filter_cnt--;
9690 		break;
9691 	case TCP_V6_FLOW:
9692 		pf->fd_tcp6_filter_cnt--;
9693 		break;
9694 	case UDP_V6_FLOW:
9695 		pf->fd_udp6_filter_cnt--;
9696 		break;
9697 	case SCTP_V6_FLOW:
9698 		pf->fd_udp6_filter_cnt--;
9699 		break;
9700 	case IP_USER_FLOW:
9701 		switch (filter->ipl4_proto) {
9702 		case IPPROTO_TCP:
9703 			pf->fd_tcp4_filter_cnt--;
9704 			break;
9705 		case IPPROTO_UDP:
9706 			pf->fd_udp4_filter_cnt--;
9707 			break;
9708 		case IPPROTO_SCTP:
9709 			pf->fd_sctp4_filter_cnt--;
9710 			break;
9711 		case IPPROTO_IP:
9712 			pf->fd_ip4_filter_cnt--;
9713 			break;
9714 		}
9715 		break;
9716 	case IPV6_USER_FLOW:
9717 		switch (filter->ipl4_proto) {
9718 		case IPPROTO_TCP:
9719 			pf->fd_tcp6_filter_cnt--;
9720 			break;
9721 		case IPPROTO_UDP:
9722 			pf->fd_udp6_filter_cnt--;
9723 			break;
9724 		case IPPROTO_SCTP:
9725 			pf->fd_sctp6_filter_cnt--;
9726 			break;
9727 		case IPPROTO_IP:
9728 			pf->fd_ip6_filter_cnt--;
9729 			break;
9730 		}
9731 		break;
9732 	}
9733 
9734 	/* Remove the filter from the list and free memory */
9735 	hlist_del(&filter->fdir_node);
9736 	kfree(filter);
9737 }
9738 
9739 /**
9740  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9741  * @pf: board private structure
9742  **/
i40e_fdir_check_and_reenable(struct i40e_pf * pf)9743 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9744 {
9745 	struct i40e_fdir_filter *filter;
9746 	u32 fcnt_prog, fcnt_avail;
9747 	struct hlist_node *node;
9748 
9749 	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9750 		return;
9751 
9752 	/* Check if we have enough room to re-enable FDir SB capability. */
9753 	fcnt_prog = i40e_get_global_fd_count(pf);
9754 	fcnt_avail = pf->fdir_pf_filter_count;
9755 	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9756 	    (pf->fd_add_err == 0) ||
9757 	    (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9758 		i40e_reenable_fdir_sb(pf);
9759 
9760 	/* We should wait for even more space before re-enabling ATR.
9761 	 * Additionally, we cannot enable ATR as long as we still have TCP SB
9762 	 * rules active.
9763 	 */
9764 	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9765 	    pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9766 		i40e_reenable_fdir_atr(pf);
9767 
9768 	/* if hw had a problem adding a filter, delete it */
9769 	if (pf->fd_inv > 0) {
9770 		hlist_for_each_entry_safe(filter, node,
9771 					  &pf->fdir_filter_list, fdir_node)
9772 			if (filter->fd_id == pf->fd_inv)
9773 				i40e_delete_invalid_filter(pf, filter);
9774 	}
9775 }
9776 
9777 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9778 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9779 /**
9780  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9781  * @pf: board private structure
9782  **/
i40e_fdir_flush_and_replay(struct i40e_pf * pf)9783 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9784 {
9785 	unsigned long min_flush_time;
9786 	int flush_wait_retry = 50;
9787 	bool disable_atr = false;
9788 	int fd_room;
9789 	int reg;
9790 
9791 	if (!time_after(jiffies, pf->fd_flush_timestamp +
9792 				 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9793 		return;
9794 
9795 	/* If the flush is happening too quick and we have mostly SB rules we
9796 	 * should not re-enable ATR for some time.
9797 	 */
9798 	min_flush_time = pf->fd_flush_timestamp +
9799 			 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9800 	fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9801 
9802 	if (!(time_after(jiffies, min_flush_time)) &&
9803 	    (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9804 		if (I40E_DEBUG_FD & pf->hw.debug_mask)
9805 			dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9806 		disable_atr = true;
9807 	}
9808 
9809 	pf->fd_flush_timestamp = jiffies;
9810 	set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9811 	/* flush all filters */
9812 	wr32(&pf->hw, I40E_PFQF_CTL_1,
9813 	     I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9814 	i40e_flush(&pf->hw);
9815 	pf->fd_flush_cnt++;
9816 	pf->fd_add_err = 0;
9817 	do {
9818 		/* Check FD flush status every 5-6msec */
9819 		usleep_range(5000, 6000);
9820 		reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9821 		if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9822 			break;
9823 	} while (flush_wait_retry--);
9824 	if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9825 		dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9826 	} else {
9827 		/* replay sideband filters */
9828 		i40e_fdir_filter_restore(i40e_pf_get_main_vsi(pf));
9829 		if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9830 			clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9831 		clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9832 		if (I40E_DEBUG_FD & pf->hw.debug_mask)
9833 			dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9834 	}
9835 }
9836 
9837 /**
9838  * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9839  * @pf: board private structure
9840  **/
i40e_get_current_atr_cnt(struct i40e_pf * pf)9841 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9842 {
9843 	return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9844 }
9845 
9846 /**
9847  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9848  * @pf: board private structure
9849  **/
i40e_fdir_reinit_subtask(struct i40e_pf * pf)9850 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9851 {
9852 
9853 	/* if interface is down do nothing */
9854 	if (test_bit(__I40E_DOWN, pf->state))
9855 		return;
9856 
9857 	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9858 		i40e_fdir_flush_and_replay(pf);
9859 
9860 	i40e_fdir_check_and_reenable(pf);
9861 
9862 }
9863 
9864 /**
9865  * i40e_vsi_link_event - notify VSI of a link event
9866  * @vsi: vsi to be notified
9867  * @link_up: link up or down
9868  **/
i40e_vsi_link_event(struct i40e_vsi * vsi,bool link_up)9869 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9870 {
9871 	if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9872 		return;
9873 
9874 	switch (vsi->type) {
9875 	case I40E_VSI_MAIN:
9876 		if (!vsi->netdev || !vsi->netdev_registered)
9877 			break;
9878 
9879 		if (link_up) {
9880 			netif_carrier_on(vsi->netdev);
9881 			netif_tx_wake_all_queues(vsi->netdev);
9882 		} else {
9883 			netif_carrier_off(vsi->netdev);
9884 			netif_tx_stop_all_queues(vsi->netdev);
9885 		}
9886 		break;
9887 
9888 	case I40E_VSI_SRIOV:
9889 	case I40E_VSI_VMDQ2:
9890 	case I40E_VSI_CTRL:
9891 	case I40E_VSI_IWARP:
9892 	case I40E_VSI_MIRROR:
9893 	default:
9894 		/* there is no notification for other VSIs */
9895 		break;
9896 	}
9897 }
9898 
9899 /**
9900  * i40e_veb_link_event - notify elements on the veb of a link event
9901  * @veb: veb to be notified
9902  * @link_up: link up or down
9903  **/
i40e_veb_link_event(struct i40e_veb * veb,bool link_up)9904 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9905 {
9906 	struct i40e_vsi *vsi;
9907 	struct i40e_pf *pf;
9908 	int i;
9909 
9910 	if (!veb || !veb->pf)
9911 		return;
9912 	pf = veb->pf;
9913 
9914 	/* Send link event to contained VSIs */
9915 	i40e_pf_for_each_vsi(pf, i, vsi)
9916 		if (vsi->uplink_seid == veb->seid)
9917 			i40e_vsi_link_event(vsi, link_up);
9918 }
9919 
9920 /**
9921  * i40e_link_event - Update netif_carrier status
9922  * @pf: board private structure
9923  **/
i40e_link_event(struct i40e_pf * pf)9924 static void i40e_link_event(struct i40e_pf *pf)
9925 {
9926 	struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
9927 	struct i40e_veb *veb = i40e_pf_get_main_veb(pf);
9928 	u8 new_link_speed, old_link_speed;
9929 	bool new_link, old_link;
9930 	int status;
9931 #ifdef CONFIG_I40E_DCB
9932 	int err;
9933 #endif /* CONFIG_I40E_DCB */
9934 
9935 	/* set this to force the get_link_status call to refresh state */
9936 	pf->hw.phy.get_link_info = true;
9937 	old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9938 	status = i40e_get_link_status(&pf->hw, &new_link);
9939 
9940 	/* On success, disable temp link polling */
9941 	if (status == 0) {
9942 		clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9943 	} else {
9944 		/* Enable link polling temporarily until i40e_get_link_status
9945 		 * returns 0
9946 		 */
9947 		set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9948 		dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9949 			status);
9950 		return;
9951 	}
9952 
9953 	old_link_speed = pf->hw.phy.link_info_old.link_speed;
9954 	new_link_speed = pf->hw.phy.link_info.link_speed;
9955 
9956 	if (new_link == old_link &&
9957 	    new_link_speed == old_link_speed &&
9958 	    (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9959 	     new_link == netif_carrier_ok(vsi->netdev)))
9960 		return;
9961 
9962 	i40e_print_link_message(vsi, new_link);
9963 
9964 	/* Notify the base of the switch tree connected to
9965 	 * the link.  Floating VEBs are not notified.
9966 	 */
9967 	if (veb)
9968 		i40e_veb_link_event(veb, new_link);
9969 	else
9970 		i40e_vsi_link_event(vsi, new_link);
9971 
9972 	if (pf->vf)
9973 		i40e_vc_notify_link_state(pf);
9974 
9975 	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
9976 		i40e_ptp_set_increment(pf);
9977 #ifdef CONFIG_I40E_DCB
9978 	if (new_link == old_link)
9979 		return;
9980 	/* Not SW DCB so firmware will take care of default settings */
9981 	if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
9982 		return;
9983 
9984 	/* We cover here only link down, as after link up in case of SW DCB
9985 	 * SW LLDP agent will take care of setting it up
9986 	 */
9987 	if (!new_link) {
9988 		dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
9989 		memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
9990 		err = i40e_dcb_sw_default_config(pf);
9991 		if (err) {
9992 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9993 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
9994 		} else {
9995 			pf->dcbx_cap = DCB_CAP_DCBX_HOST |
9996 				       DCB_CAP_DCBX_VER_IEEE;
9997 			set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9998 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
9999 		}
10000 	}
10001 #endif /* CONFIG_I40E_DCB */
10002 }
10003 
10004 /**
10005  * i40e_watchdog_subtask - periodic checks not using event driven response
10006  * @pf: board private structure
10007  **/
i40e_watchdog_subtask(struct i40e_pf * pf)10008 static void i40e_watchdog_subtask(struct i40e_pf *pf)
10009 {
10010 	struct i40e_vsi *vsi;
10011 	struct i40e_veb *veb;
10012 	int i;
10013 
10014 	/* if interface is down do nothing */
10015 	if (test_bit(__I40E_DOWN, pf->state) ||
10016 	    test_bit(__I40E_CONFIG_BUSY, pf->state))
10017 		return;
10018 
10019 	/* make sure we don't do these things too often */
10020 	if (time_before(jiffies, (pf->service_timer_previous +
10021 				  pf->service_timer_period)))
10022 		return;
10023 	pf->service_timer_previous = jiffies;
10024 
10025 	if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) ||
10026 	    test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
10027 		i40e_link_event(pf);
10028 
10029 	/* Update the stats for active netdevs so the network stack
10030 	 * can look at updated numbers whenever it cares to
10031 	 */
10032 	i40e_pf_for_each_vsi(pf, i, vsi)
10033 		if (vsi->netdev)
10034 			i40e_update_stats(vsi);
10035 
10036 	if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) {
10037 		/* Update the stats for the active switching components */
10038 		i40e_pf_for_each_veb(pf, i, veb)
10039 			i40e_update_veb_stats(veb);
10040 	}
10041 
10042 	i40e_ptp_rx_hang(pf);
10043 	i40e_ptp_tx_hang(pf);
10044 }
10045 
10046 /**
10047  * i40e_reset_subtask - Set up for resetting the device and driver
10048  * @pf: board private structure
10049  **/
i40e_reset_subtask(struct i40e_pf * pf)10050 static void i40e_reset_subtask(struct i40e_pf *pf)
10051 {
10052 	u32 reset_flags = 0;
10053 
10054 	if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
10055 		reset_flags |= BIT(__I40E_REINIT_REQUESTED);
10056 		clear_bit(__I40E_REINIT_REQUESTED, pf->state);
10057 	}
10058 	if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
10059 		reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
10060 		clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
10061 	}
10062 	if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
10063 		reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
10064 		clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
10065 	}
10066 	if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
10067 		reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
10068 		clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
10069 	}
10070 	if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
10071 		reset_flags |= BIT(__I40E_DOWN_REQUESTED);
10072 		clear_bit(__I40E_DOWN_REQUESTED, pf->state);
10073 	}
10074 
10075 	/* If there's a recovery already waiting, it takes
10076 	 * precedence before starting a new reset sequence.
10077 	 */
10078 	if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
10079 		i40e_prep_for_reset(pf);
10080 		i40e_reset(pf);
10081 		i40e_rebuild(pf, false, false);
10082 	}
10083 
10084 	/* If we're already down or resetting, just bail */
10085 	if (reset_flags &&
10086 	    !test_bit(__I40E_DOWN, pf->state) &&
10087 	    !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
10088 		i40e_do_reset(pf, reset_flags, false);
10089 	}
10090 }
10091 
10092 /**
10093  * i40e_handle_link_event - Handle link event
10094  * @pf: board private structure
10095  * @e: event info posted on ARQ
10096  **/
i40e_handle_link_event(struct i40e_pf * pf,struct i40e_arq_event_info * e)10097 static void i40e_handle_link_event(struct i40e_pf *pf,
10098 				   struct i40e_arq_event_info *e)
10099 {
10100 	struct i40e_aqc_get_link_status *status =
10101 		(struct i40e_aqc_get_link_status *)&e->desc.params.raw;
10102 
10103 	/* Do a new status request to re-enable LSE reporting
10104 	 * and load new status information into the hw struct
10105 	 * This completely ignores any state information
10106 	 * in the ARQ event info, instead choosing to always
10107 	 * issue the AQ update link status command.
10108 	 */
10109 	i40e_link_event(pf);
10110 
10111 	/* Check if module meets thermal requirements */
10112 	if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
10113 		dev_err(&pf->pdev->dev,
10114 			"Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
10115 		dev_err(&pf->pdev->dev,
10116 			"Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10117 	} else {
10118 		/* check for unqualified module, if link is down, suppress
10119 		 * the message if link was forced to be down.
10120 		 */
10121 		if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
10122 		    (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
10123 		    (!(status->link_info & I40E_AQ_LINK_UP)) &&
10124 		    (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) {
10125 			dev_err(&pf->pdev->dev,
10126 				"Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
10127 			dev_err(&pf->pdev->dev,
10128 				"Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10129 		}
10130 	}
10131 }
10132 
10133 /**
10134  * i40e_clean_adminq_subtask - Clean the AdminQ rings
10135  * @pf: board private structure
10136  **/
i40e_clean_adminq_subtask(struct i40e_pf * pf)10137 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
10138 {
10139 	struct i40e_arq_event_info event;
10140 	struct i40e_hw *hw = &pf->hw;
10141 	u16 pending, i = 0;
10142 	u16 opcode;
10143 	u32 oldval;
10144 	int ret;
10145 	u32 val;
10146 
10147 	/* Do not run clean AQ when PF reset fails */
10148 	if (test_bit(__I40E_RESET_FAILED, pf->state))
10149 		return;
10150 
10151 	/* check for error indications */
10152 	val = rd32(&pf->hw, I40E_PF_ARQLEN);
10153 	oldval = val;
10154 	if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
10155 		if (hw->debug_mask & I40E_DEBUG_AQ)
10156 			dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
10157 		val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
10158 	}
10159 	if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
10160 		if (hw->debug_mask & I40E_DEBUG_AQ)
10161 			dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
10162 		val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
10163 		pf->arq_overflows++;
10164 	}
10165 	if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
10166 		if (hw->debug_mask & I40E_DEBUG_AQ)
10167 			dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
10168 		val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
10169 	}
10170 	if (oldval != val)
10171 		wr32(&pf->hw, I40E_PF_ARQLEN, val);
10172 
10173 	val = rd32(&pf->hw, I40E_PF_ATQLEN);
10174 	oldval = val;
10175 	if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
10176 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10177 			dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
10178 		val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
10179 	}
10180 	if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
10181 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10182 			dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
10183 		val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
10184 	}
10185 	if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
10186 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10187 			dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
10188 		val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
10189 	}
10190 	if (oldval != val)
10191 		wr32(&pf->hw, I40E_PF_ATQLEN, val);
10192 
10193 	event.buf_len = I40E_MAX_AQ_BUF_SIZE;
10194 	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
10195 	if (!event.msg_buf)
10196 		return;
10197 
10198 	do {
10199 		ret = i40e_clean_arq_element(hw, &event, &pending);
10200 		if (ret == -EALREADY)
10201 			break;
10202 		else if (ret) {
10203 			dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
10204 			break;
10205 		}
10206 
10207 		opcode = le16_to_cpu(event.desc.opcode);
10208 		switch (opcode) {
10209 
10210 		case i40e_aqc_opc_get_link_status:
10211 			rtnl_lock();
10212 			i40e_handle_link_event(pf, &event);
10213 			rtnl_unlock();
10214 			break;
10215 		case i40e_aqc_opc_send_msg_to_pf:
10216 			ret = i40e_vc_process_vf_msg(pf,
10217 					le16_to_cpu(event.desc.retval),
10218 					le32_to_cpu(event.desc.cookie_high),
10219 					le32_to_cpu(event.desc.cookie_low),
10220 					event.msg_buf,
10221 					event.msg_len);
10222 			break;
10223 		case i40e_aqc_opc_lldp_update_mib:
10224 			dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
10225 #ifdef CONFIG_I40E_DCB
10226 			rtnl_lock();
10227 			i40e_handle_lldp_event(pf, &event);
10228 			rtnl_unlock();
10229 #endif /* CONFIG_I40E_DCB */
10230 			break;
10231 		case i40e_aqc_opc_event_lan_overflow:
10232 			dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
10233 			i40e_handle_lan_overflow_event(pf, &event);
10234 			break;
10235 		case i40e_aqc_opc_send_msg_to_peer:
10236 			dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
10237 			break;
10238 		case i40e_aqc_opc_nvm_erase:
10239 		case i40e_aqc_opc_nvm_update:
10240 		case i40e_aqc_opc_oem_post_update:
10241 			i40e_debug(&pf->hw, I40E_DEBUG_NVM,
10242 				   "ARQ NVM operation 0x%04x completed\n",
10243 				   opcode);
10244 			break;
10245 		default:
10246 			dev_info(&pf->pdev->dev,
10247 				 "ARQ: Unknown event 0x%04x ignored\n",
10248 				 opcode);
10249 			break;
10250 		}
10251 	} while (i++ < I40E_AQ_WORK_LIMIT);
10252 
10253 	if (i < I40E_AQ_WORK_LIMIT)
10254 		clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
10255 
10256 	/* re-enable Admin queue interrupt cause */
10257 	val = rd32(hw, I40E_PFINT_ICR0_ENA);
10258 	val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
10259 	wr32(hw, I40E_PFINT_ICR0_ENA, val);
10260 	i40e_flush(hw);
10261 
10262 	kfree(event.msg_buf);
10263 }
10264 
10265 /**
10266  * i40e_verify_eeprom - make sure eeprom is good to use
10267  * @pf: board private structure
10268  **/
i40e_verify_eeprom(struct i40e_pf * pf)10269 static void i40e_verify_eeprom(struct i40e_pf *pf)
10270 {
10271 	int err;
10272 
10273 	err = i40e_diag_eeprom_test(&pf->hw);
10274 	if (err) {
10275 		/* retry in case of garbage read */
10276 		err = i40e_diag_eeprom_test(&pf->hw);
10277 		if (err) {
10278 			dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
10279 				 err);
10280 			set_bit(__I40E_BAD_EEPROM, pf->state);
10281 		}
10282 	}
10283 
10284 	if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
10285 		dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
10286 		clear_bit(__I40E_BAD_EEPROM, pf->state);
10287 	}
10288 }
10289 
10290 /**
10291  * i40e_enable_pf_switch_lb
10292  * @pf: pointer to the PF structure
10293  *
10294  * enable switch loop back or die - no point in a return value
10295  **/
i40e_enable_pf_switch_lb(struct i40e_pf * pf)10296 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
10297 {
10298 	struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
10299 	struct i40e_vsi_context ctxt;
10300 	int ret;
10301 
10302 	ctxt.seid = pf->main_vsi_seid;
10303 	ctxt.pf_num = pf->hw.pf_id;
10304 	ctxt.vf_num = 0;
10305 	ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10306 	if (ret) {
10307 		dev_info(&pf->pdev->dev,
10308 			 "couldn't get PF vsi config, err %pe aq_err %s\n",
10309 			 ERR_PTR(ret),
10310 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10311 		return;
10312 	}
10313 	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10314 	ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10315 	ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10316 
10317 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10318 	if (ret) {
10319 		dev_info(&pf->pdev->dev,
10320 			 "update vsi switch failed, err %pe aq_err %s\n",
10321 			 ERR_PTR(ret),
10322 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10323 	}
10324 }
10325 
10326 /**
10327  * i40e_disable_pf_switch_lb
10328  * @pf: pointer to the PF structure
10329  *
10330  * disable switch loop back or die - no point in a return value
10331  **/
i40e_disable_pf_switch_lb(struct i40e_pf * pf)10332 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
10333 {
10334 	struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
10335 	struct i40e_vsi_context ctxt;
10336 	int ret;
10337 
10338 	ctxt.seid = pf->main_vsi_seid;
10339 	ctxt.pf_num = pf->hw.pf_id;
10340 	ctxt.vf_num = 0;
10341 	ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10342 	if (ret) {
10343 		dev_info(&pf->pdev->dev,
10344 			 "couldn't get PF vsi config, err %pe aq_err %s\n",
10345 			 ERR_PTR(ret),
10346 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10347 		return;
10348 	}
10349 	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10350 	ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10351 	ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10352 
10353 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10354 	if (ret) {
10355 		dev_info(&pf->pdev->dev,
10356 			 "update vsi switch failed, err %pe aq_err %s\n",
10357 			 ERR_PTR(ret),
10358 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10359 	}
10360 }
10361 
10362 /**
10363  * i40e_config_bridge_mode - Configure the HW bridge mode
10364  * @veb: pointer to the bridge instance
10365  *
10366  * Configure the loop back mode for the LAN VSI that is downlink to the
10367  * specified HW bridge instance. It is expected this function is called
10368  * when a new HW bridge is instantiated.
10369  **/
i40e_config_bridge_mode(struct i40e_veb * veb)10370 static void i40e_config_bridge_mode(struct i40e_veb *veb)
10371 {
10372 	struct i40e_pf *pf = veb->pf;
10373 
10374 	if (pf->hw.debug_mask & I40E_DEBUG_LAN)
10375 		dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
10376 			 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10377 	if (veb->bridge_mode & BRIDGE_MODE_VEPA)
10378 		i40e_disable_pf_switch_lb(pf);
10379 	else
10380 		i40e_enable_pf_switch_lb(pf);
10381 }
10382 
10383 /**
10384  * i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it
10385  * @veb: pointer to the VEB instance
10386  *
10387  * This is a function that builds the attached VSIs. We track the connections
10388  * through our own index numbers because the seid's from the HW could change
10389  * across the reset.
10390  **/
i40e_reconstitute_veb(struct i40e_veb * veb)10391 static int i40e_reconstitute_veb(struct i40e_veb *veb)
10392 {
10393 	struct i40e_vsi *ctl_vsi = NULL;
10394 	struct i40e_pf *pf = veb->pf;
10395 	struct i40e_vsi *vsi;
10396 	int v, ret;
10397 
10398 	/* As we do not maintain PV (port virtualizer) switch element then
10399 	 * there can be only one non-floating VEB that have uplink to MAC SEID
10400 	 * and its control VSI is the main one.
10401 	 */
10402 	if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) {
10403 		dev_err(&pf->pdev->dev,
10404 			"Invalid uplink SEID for VEB %d\n", veb->idx);
10405 		return -ENOENT;
10406 	}
10407 
10408 	if (veb->uplink_seid == pf->mac_seid) {
10409 		/* Check that the LAN VSI has VEB owning flag set */
10410 		ctl_vsi = i40e_pf_get_main_vsi(pf);
10411 
10412 		if (WARN_ON(ctl_vsi->veb_idx != veb->idx ||
10413 			    !(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) {
10414 			dev_err(&pf->pdev->dev,
10415 				"Invalid control VSI for VEB %d\n", veb->idx);
10416 			return -ENOENT;
10417 		}
10418 
10419 		/* Add the control VSI to switch */
10420 		ret = i40e_add_vsi(ctl_vsi);
10421 		if (ret) {
10422 			dev_err(&pf->pdev->dev,
10423 				"Rebuild of owner VSI for VEB %d failed: %d\n",
10424 				veb->idx, ret);
10425 			return ret;
10426 		}
10427 
10428 		i40e_vsi_reset_stats(ctl_vsi);
10429 	}
10430 
10431 	/* create the VEB in the switch and move the VSI onto the VEB */
10432 	ret = i40e_add_veb(veb, ctl_vsi);
10433 	if (ret)
10434 		return ret;
10435 
10436 	if (veb->uplink_seid) {
10437 		if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
10438 			veb->bridge_mode = BRIDGE_MODE_VEB;
10439 		else
10440 			veb->bridge_mode = BRIDGE_MODE_VEPA;
10441 		i40e_config_bridge_mode(veb);
10442 	}
10443 
10444 	/* create the remaining VSIs attached to this VEB */
10445 	i40e_pf_for_each_vsi(pf, v, vsi) {
10446 		if (vsi == ctl_vsi)
10447 			continue;
10448 
10449 		if (vsi->veb_idx == veb->idx) {
10450 			vsi->uplink_seid = veb->seid;
10451 			ret = i40e_add_vsi(vsi);
10452 			if (ret) {
10453 				dev_info(&pf->pdev->dev,
10454 					 "rebuild of vsi_idx %d failed: %d\n",
10455 					 v, ret);
10456 				return ret;
10457 			}
10458 			i40e_vsi_reset_stats(vsi);
10459 		}
10460 	}
10461 
10462 	return ret;
10463 }
10464 
10465 /**
10466  * i40e_get_capabilities - get info about the HW
10467  * @pf: the PF struct
10468  * @list_type: AQ capability to be queried
10469  **/
i40e_get_capabilities(struct i40e_pf * pf,enum i40e_admin_queue_opc list_type)10470 static int i40e_get_capabilities(struct i40e_pf *pf,
10471 				 enum i40e_admin_queue_opc list_type)
10472 {
10473 	struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10474 	u16 data_size;
10475 	int buf_len;
10476 	int err;
10477 
10478 	buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10479 	do {
10480 		cap_buf = kzalloc(buf_len, GFP_KERNEL);
10481 		if (!cap_buf)
10482 			return -ENOMEM;
10483 
10484 		/* this loads the data into the hw struct for us */
10485 		err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
10486 						    &data_size, list_type,
10487 						    NULL);
10488 		/* data loaded, buffer no longer needed */
10489 		kfree(cap_buf);
10490 
10491 		if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10492 			/* retry with a larger buffer */
10493 			buf_len = data_size;
10494 		} else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
10495 			dev_info(&pf->pdev->dev,
10496 				 "capability discovery failed, err %pe aq_err %s\n",
10497 				 ERR_PTR(err),
10498 				 i40e_aq_str(&pf->hw,
10499 					     pf->hw.aq.asq_last_status));
10500 			return -ENODEV;
10501 		}
10502 	} while (err);
10503 
10504 	if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10505 		if (list_type == i40e_aqc_opc_list_func_capabilities) {
10506 			dev_info(&pf->pdev->dev,
10507 				 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10508 				 pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10509 				 pf->hw.func_caps.num_msix_vectors,
10510 				 pf->hw.func_caps.num_msix_vectors_vf,
10511 				 pf->hw.func_caps.fd_filters_guaranteed,
10512 				 pf->hw.func_caps.fd_filters_best_effort,
10513 				 pf->hw.func_caps.num_tx_qp,
10514 				 pf->hw.func_caps.num_vsis);
10515 		} else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10516 			dev_info(&pf->pdev->dev,
10517 				 "switch_mode=0x%04x, function_valid=0x%08x\n",
10518 				 pf->hw.dev_caps.switch_mode,
10519 				 pf->hw.dev_caps.valid_functions);
10520 			dev_info(&pf->pdev->dev,
10521 				 "SR-IOV=%d, num_vfs for all function=%u\n",
10522 				 pf->hw.dev_caps.sr_iov_1_1,
10523 				 pf->hw.dev_caps.num_vfs);
10524 			dev_info(&pf->pdev->dev,
10525 				 "num_vsis=%u, num_rx:%u, num_tx=%u\n",
10526 				 pf->hw.dev_caps.num_vsis,
10527 				 pf->hw.dev_caps.num_rx_qp,
10528 				 pf->hw.dev_caps.num_tx_qp);
10529 		}
10530 	}
10531 	if (list_type == i40e_aqc_opc_list_func_capabilities) {
10532 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10533 		       + pf->hw.func_caps.num_vfs)
10534 		if (pf->hw.revision_id == 0 &&
10535 		    pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10536 			dev_info(&pf->pdev->dev,
10537 				 "got num_vsis %d, setting num_vsis to %d\n",
10538 				 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10539 			pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10540 		}
10541 	}
10542 	return 0;
10543 }
10544 
10545 static int i40e_vsi_clear(struct i40e_vsi *vsi);
10546 
10547 /**
10548  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10549  * @pf: board private structure
10550  **/
i40e_fdir_sb_setup(struct i40e_pf * pf)10551 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10552 {
10553 	struct i40e_vsi *main_vsi, *vsi;
10554 
10555 	/* quick workaround for an NVM issue that leaves a critical register
10556 	 * uninitialized
10557 	 */
10558 	if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10559 		static const u32 hkey[] = {
10560 			0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10561 			0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10562 			0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10563 			0x95b3a76d};
10564 		int i;
10565 
10566 		for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10567 			wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10568 	}
10569 
10570 	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
10571 		return;
10572 
10573 	/* find existing VSI and see if it needs configuring */
10574 	vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10575 
10576 	/* create a new VSI if none exists */
10577 	if (!vsi) {
10578 		main_vsi = i40e_pf_get_main_vsi(pf);
10579 		vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, main_vsi->seid, 0);
10580 		if (!vsi) {
10581 			dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10582 			clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
10583 			set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
10584 			return;
10585 		}
10586 	}
10587 
10588 	i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
10589 }
10590 
10591 /**
10592  * i40e_fdir_teardown - release the Flow Director resources
10593  * @pf: board private structure
10594  **/
i40e_fdir_teardown(struct i40e_pf * pf)10595 static void i40e_fdir_teardown(struct i40e_pf *pf)
10596 {
10597 	struct i40e_vsi *vsi;
10598 
10599 	i40e_fdir_filter_exit(pf);
10600 	vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10601 	if (vsi)
10602 		i40e_vsi_release(vsi);
10603 }
10604 
10605 /**
10606  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10607  * @vsi: PF main vsi
10608  * @seid: seid of main or channel VSIs
10609  *
10610  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
10611  * existed before reset
10612  **/
i40e_rebuild_cloud_filters(struct i40e_vsi * vsi,u16 seid)10613 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10614 {
10615 	struct i40e_cloud_filter *cfilter;
10616 	struct i40e_pf *pf = vsi->back;
10617 	struct hlist_node *node;
10618 	int ret;
10619 
10620 	/* Add cloud filters back if they exist */
10621 	hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10622 				  cloud_node) {
10623 		if (cfilter->seid != seid)
10624 			continue;
10625 
10626 		if (cfilter->dst_port)
10627 			ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10628 								true);
10629 		else
10630 			ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10631 
10632 		if (ret) {
10633 			dev_dbg(&pf->pdev->dev,
10634 				"Failed to rebuild cloud filter, err %pe aq_err %s\n",
10635 				ERR_PTR(ret),
10636 				i40e_aq_str(&pf->hw,
10637 					    pf->hw.aq.asq_last_status));
10638 			return ret;
10639 		}
10640 	}
10641 	return 0;
10642 }
10643 
10644 /**
10645  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10646  * @vsi: PF main vsi
10647  *
10648  * Rebuilds channel VSIs if they existed before reset
10649  **/
i40e_rebuild_channels(struct i40e_vsi * vsi)10650 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10651 {
10652 	struct i40e_channel *ch, *ch_tmp;
10653 	int ret;
10654 
10655 	if (list_empty(&vsi->ch_list))
10656 		return 0;
10657 
10658 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10659 		if (!ch->initialized)
10660 			break;
10661 		/* Proceed with creation of channel (VMDq2) VSI */
10662 		ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10663 		if (ret) {
10664 			dev_info(&vsi->back->pdev->dev,
10665 				 "failed to rebuild channels using uplink_seid %u\n",
10666 				 vsi->uplink_seid);
10667 			return ret;
10668 		}
10669 		/* Reconfigure TX queues using QTX_CTL register */
10670 		ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10671 		if (ret) {
10672 			dev_info(&vsi->back->pdev->dev,
10673 				 "failed to configure TX rings for channel %u\n",
10674 				 ch->seid);
10675 			return ret;
10676 		}
10677 		/* update 'next_base_queue' */
10678 		vsi->next_base_queue = vsi->next_base_queue +
10679 							ch->num_queue_pairs;
10680 		if (ch->max_tx_rate) {
10681 			u64 credits = ch->max_tx_rate;
10682 
10683 			if (i40e_set_bw_limit(vsi, ch->seid,
10684 					      ch->max_tx_rate))
10685 				return -EINVAL;
10686 
10687 			do_div(credits, I40E_BW_CREDIT_DIVISOR);
10688 			dev_dbg(&vsi->back->pdev->dev,
10689 				"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10690 				ch->max_tx_rate,
10691 				credits,
10692 				ch->seid);
10693 		}
10694 		ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10695 		if (ret) {
10696 			dev_dbg(&vsi->back->pdev->dev,
10697 				"Failed to rebuild cloud filters for channel VSI %u\n",
10698 				ch->seid);
10699 			return ret;
10700 		}
10701 	}
10702 	return 0;
10703 }
10704 
10705 /**
10706  * i40e_clean_xps_state - clean xps state for every tx_ring
10707  * @vsi: ptr to the VSI
10708  **/
i40e_clean_xps_state(struct i40e_vsi * vsi)10709 static void i40e_clean_xps_state(struct i40e_vsi *vsi)
10710 {
10711 	int i;
10712 
10713 	if (vsi->tx_rings)
10714 		for (i = 0; i < vsi->num_queue_pairs; i++)
10715 			if (vsi->tx_rings[i])
10716 				clear_bit(__I40E_TX_XPS_INIT_DONE,
10717 					  vsi->tx_rings[i]->state);
10718 }
10719 
10720 /**
10721  * i40e_prep_for_reset - prep for the core to reset
10722  * @pf: board private structure
10723  *
10724  * Close up the VFs and other things in prep for PF Reset.
10725   **/
i40e_prep_for_reset(struct i40e_pf * pf)10726 static void i40e_prep_for_reset(struct i40e_pf *pf)
10727 {
10728 	struct i40e_hw *hw = &pf->hw;
10729 	struct i40e_vsi *vsi;
10730 	int ret = 0;
10731 	u32 v;
10732 
10733 	clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10734 	if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10735 		return;
10736 	if (i40e_check_asq_alive(&pf->hw))
10737 		i40e_vc_notify_reset(pf);
10738 
10739 	dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10740 
10741 	/* quiesce the VSIs and their queues that are not already DOWN */
10742 	i40e_pf_quiesce_all_vsi(pf);
10743 
10744 	i40e_pf_for_each_vsi(pf, v, vsi) {
10745 		i40e_clean_xps_state(vsi);
10746 		vsi->seid = 0;
10747 	}
10748 
10749 	i40e_shutdown_adminq(&pf->hw);
10750 
10751 	/* call shutdown HMC */
10752 	if (hw->hmc.hmc_obj) {
10753 		ret = i40e_shutdown_lan_hmc(hw);
10754 		if (ret)
10755 			dev_warn(&pf->pdev->dev,
10756 				 "shutdown_lan_hmc failed: %d\n", ret);
10757 	}
10758 
10759 	/* Save the current PTP time so that we can restore the time after the
10760 	 * reset completes.
10761 	 */
10762 	i40e_ptp_save_hw_time(pf);
10763 }
10764 
10765 /**
10766  * i40e_send_version - update firmware with driver version
10767  * @pf: PF struct
10768  */
i40e_send_version(struct i40e_pf * pf)10769 static void i40e_send_version(struct i40e_pf *pf)
10770 {
10771 	struct i40e_driver_version dv;
10772 
10773 	dv.major_version = 0xff;
10774 	dv.minor_version = 0xff;
10775 	dv.build_version = 0xff;
10776 	dv.subbuild_version = 0;
10777 	strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10778 	i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10779 }
10780 
10781 /**
10782  * i40e_get_oem_version - get OEM specific version information
10783  * @hw: pointer to the hardware structure
10784  **/
i40e_get_oem_version(struct i40e_hw * hw)10785 static void i40e_get_oem_version(struct i40e_hw *hw)
10786 {
10787 	u16 block_offset = 0xffff;
10788 	u16 block_length = 0;
10789 	u16 capabilities = 0;
10790 	u16 gen_snap = 0;
10791 	u16 release = 0;
10792 
10793 #define I40E_SR_NVM_OEM_VERSION_PTR		0x1B
10794 #define I40E_NVM_OEM_LENGTH_OFFSET		0x00
10795 #define I40E_NVM_OEM_CAPABILITIES_OFFSET	0x01
10796 #define I40E_NVM_OEM_GEN_OFFSET			0x02
10797 #define I40E_NVM_OEM_RELEASE_OFFSET		0x03
10798 #define I40E_NVM_OEM_CAPABILITIES_MASK		0x000F
10799 #define I40E_NVM_OEM_LENGTH			3
10800 
10801 	/* Check if pointer to OEM version block is valid. */
10802 	i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10803 	if (block_offset == 0xffff)
10804 		return;
10805 
10806 	/* Check if OEM version block has correct length. */
10807 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10808 			   &block_length);
10809 	if (block_length < I40E_NVM_OEM_LENGTH)
10810 		return;
10811 
10812 	/* Check if OEM version format is as expected. */
10813 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10814 			   &capabilities);
10815 	if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10816 		return;
10817 
10818 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10819 			   &gen_snap);
10820 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10821 			   &release);
10822 	hw->nvm.oem_ver =
10823 		FIELD_PREP(I40E_OEM_GEN_MASK | I40E_OEM_SNAP_MASK, gen_snap) |
10824 		FIELD_PREP(I40E_OEM_RELEASE_MASK, release);
10825 	hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10826 }
10827 
10828 /**
10829  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10830  * @pf: board private structure
10831  **/
i40e_reset(struct i40e_pf * pf)10832 static int i40e_reset(struct i40e_pf *pf)
10833 {
10834 	struct i40e_hw *hw = &pf->hw;
10835 	int ret;
10836 
10837 	ret = i40e_pf_reset(hw);
10838 	if (ret) {
10839 		dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10840 		set_bit(__I40E_RESET_FAILED, pf->state);
10841 		clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10842 	} else {
10843 		pf->pfr_count++;
10844 	}
10845 	return ret;
10846 }
10847 
10848 /**
10849  * i40e_rebuild - rebuild using a saved config
10850  * @pf: board private structure
10851  * @reinit: if the Main VSI needs to re-initialized.
10852  * @lock_acquired: indicates whether or not the lock has been acquired
10853  * before this function was called.
10854  **/
i40e_rebuild(struct i40e_pf * pf,bool reinit,bool lock_acquired)10855 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10856 {
10857 	const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf);
10858 	struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
10859 	struct i40e_hw *hw = &pf->hw;
10860 	struct i40e_veb *veb;
10861 	int ret;
10862 	u32 val;
10863 	int v;
10864 
10865 	if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10866 	    is_recovery_mode_reported)
10867 		i40e_set_ethtool_ops(vsi->netdev);
10868 
10869 	if (test_bit(__I40E_DOWN, pf->state) &&
10870 	    !test_bit(__I40E_RECOVERY_MODE, pf->state))
10871 		goto clear_recovery;
10872 	dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10873 
10874 	/* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10875 	ret = i40e_init_adminq(&pf->hw);
10876 	if (ret) {
10877 		dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n",
10878 			 ERR_PTR(ret),
10879 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10880 		goto clear_recovery;
10881 	}
10882 	i40e_get_oem_version(&pf->hw);
10883 
10884 	if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {
10885 		/* The following delay is necessary for firmware update. */
10886 		mdelay(1000);
10887 	}
10888 
10889 	/* re-verify the eeprom if we just had an EMP reset */
10890 	if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10891 		i40e_verify_eeprom(pf);
10892 
10893 	/* if we are going out of or into recovery mode we have to act
10894 	 * accordingly with regard to resources initialization
10895 	 * and deinitialization
10896 	 */
10897 	if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10898 		if (i40e_get_capabilities(pf,
10899 					  i40e_aqc_opc_list_func_capabilities))
10900 			goto end_unlock;
10901 
10902 		if (is_recovery_mode_reported) {
10903 			/* we're staying in recovery mode so we'll reinitialize
10904 			 * misc vector here
10905 			 */
10906 			if (i40e_setup_misc_vector_for_recovery_mode(pf))
10907 				goto end_unlock;
10908 		} else {
10909 			if (!lock_acquired)
10910 				rtnl_lock();
10911 			/* we're going out of recovery mode so we'll free
10912 			 * the IRQ allocated specifically for recovery mode
10913 			 * and restore the interrupt scheme
10914 			 */
10915 			free_irq(pf->pdev->irq, pf);
10916 			i40e_clear_interrupt_scheme(pf);
10917 			if (i40e_restore_interrupt_scheme(pf))
10918 				goto end_unlock;
10919 		}
10920 
10921 		/* tell the firmware that we're starting */
10922 		i40e_send_version(pf);
10923 
10924 		/* bail out in case recovery mode was detected, as there is
10925 		 * no need for further configuration.
10926 		 */
10927 		goto end_unlock;
10928 	}
10929 
10930 	i40e_clear_pxe_mode(hw);
10931 	ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10932 	if (ret)
10933 		goto end_core_reset;
10934 
10935 	ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10936 				hw->func_caps.num_rx_qp, 0, 0);
10937 	if (ret) {
10938 		dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10939 		goto end_core_reset;
10940 	}
10941 	ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10942 	if (ret) {
10943 		dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10944 		goto end_core_reset;
10945 	}
10946 
10947 #ifdef CONFIG_I40E_DCB
10948 	/* Enable FW to write a default DCB config on link-up
10949 	 * unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10950 	 * is not supported with new link speed
10951 	 */
10952 	if (i40e_is_tc_mqprio_enabled(pf)) {
10953 		i40e_aq_set_dcb_parameters(hw, false, NULL);
10954 	} else {
10955 		if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10956 		    (hw->phy.link_info.link_speed &
10957 		     (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10958 			i40e_aq_set_dcb_parameters(hw, false, NULL);
10959 			dev_warn(&pf->pdev->dev,
10960 				 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
10961 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10962 		} else {
10963 			i40e_aq_set_dcb_parameters(hw, true, NULL);
10964 			ret = i40e_init_pf_dcb(pf);
10965 			if (ret) {
10966 				dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10967 					 ret);
10968 				clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10969 				/* Continue without DCB enabled */
10970 			}
10971 		}
10972 	}
10973 
10974 #endif /* CONFIG_I40E_DCB */
10975 	if (!lock_acquired)
10976 		rtnl_lock();
10977 	ret = i40e_setup_pf_switch(pf, reinit, true);
10978 	if (ret)
10979 		goto end_unlock;
10980 
10981 	/* The driver only wants link up/down and module qualification
10982 	 * reports from firmware.  Note the negative logic.
10983 	 */
10984 	ret = i40e_aq_set_phy_int_mask(&pf->hw,
10985 				       ~(I40E_AQ_EVENT_LINK_UPDOWN |
10986 					 I40E_AQ_EVENT_MEDIA_NA |
10987 					 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10988 	if (ret)
10989 		dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
10990 			 ERR_PTR(ret),
10991 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10992 
10993 	/* Rebuild the VSIs and VEBs that existed before reset.
10994 	 * They are still in our local switch element arrays, so only
10995 	 * need to rebuild the switch model in the HW.
10996 	 *
10997 	 * If there were VEBs but the reconstitution failed, we'll try
10998 	 * to recover minimal use by getting the basic PF VSI working.
10999 	 */
11000 	if (vsi->uplink_seid != pf->mac_seid) {
11001 		dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
11002 
11003 		/* Rebuild VEBs */
11004 		i40e_pf_for_each_veb(pf, v, veb) {
11005 			ret = i40e_reconstitute_veb(veb);
11006 			if (!ret)
11007 				continue;
11008 
11009 			/* If Main VEB failed, we're in deep doodoo,
11010 			 * so give up rebuilding the switch and set up
11011 			 * for minimal rebuild of PF VSI.
11012 			 * If orphan failed, we'll report the error
11013 			 * but try to keep going.
11014 			 */
11015 			if (veb->uplink_seid == pf->mac_seid) {
11016 				dev_info(&pf->pdev->dev,
11017 					 "rebuild of switch failed: %d, will try to set up simple PF connection\n",
11018 					 ret);
11019 				vsi->uplink_seid = pf->mac_seid;
11020 				break;
11021 			} else if (veb->uplink_seid == 0) {
11022 				dev_info(&pf->pdev->dev,
11023 					 "rebuild of orphan VEB failed: %d\n",
11024 					 ret);
11025 			}
11026 		}
11027 	}
11028 
11029 	if (vsi->uplink_seid == pf->mac_seid) {
11030 		dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
11031 		/* no VEB, so rebuild only the Main VSI */
11032 		ret = i40e_add_vsi(vsi);
11033 		if (ret) {
11034 			dev_info(&pf->pdev->dev,
11035 				 "rebuild of Main VSI failed: %d\n", ret);
11036 			goto end_unlock;
11037 		}
11038 	}
11039 
11040 	if (vsi->mqprio_qopt.max_rate[0]) {
11041 		u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
11042 						  vsi->mqprio_qopt.max_rate[0]);
11043 		u64 credits = 0;
11044 
11045 		ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
11046 		if (ret)
11047 			goto end_unlock;
11048 
11049 		credits = max_tx_rate;
11050 		do_div(credits, I40E_BW_CREDIT_DIVISOR);
11051 		dev_dbg(&vsi->back->pdev->dev,
11052 			"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
11053 			max_tx_rate,
11054 			credits,
11055 			vsi->seid);
11056 	}
11057 
11058 	ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
11059 	if (ret)
11060 		goto end_unlock;
11061 
11062 	/* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
11063 	 * for this main VSI if they exist
11064 	 */
11065 	ret = i40e_rebuild_channels(vsi);
11066 	if (ret)
11067 		goto end_unlock;
11068 
11069 	/* Reconfigure hardware for allowing smaller MSS in the case
11070 	 * of TSO, so that we avoid the MDD being fired and causing
11071 	 * a reset in the case of small MSS+TSO.
11072 	 */
11073 #define I40E_REG_MSS          0x000E64DC
11074 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
11075 #define I40E_64BYTE_MSS       0x400000
11076 	val = rd32(hw, I40E_REG_MSS);
11077 	if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11078 		val &= ~I40E_REG_MSS_MIN_MASK;
11079 		val |= I40E_64BYTE_MSS;
11080 		wr32(hw, I40E_REG_MSS, val);
11081 	}
11082 
11083 	if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
11084 		msleep(75);
11085 		ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11086 		if (ret)
11087 			dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
11088 				 ERR_PTR(ret),
11089 				 i40e_aq_str(&pf->hw,
11090 					     pf->hw.aq.asq_last_status));
11091 	}
11092 	/* reinit the misc interrupt */
11093 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
11094 		ret = i40e_setup_misc_vector(pf);
11095 		if (ret)
11096 			goto end_unlock;
11097 	}
11098 
11099 	/* Add a filter to drop all Flow control frames from any VSI from being
11100 	 * transmitted. By doing so we stop a malicious VF from sending out
11101 	 * PAUSE or PFC frames and potentially controlling traffic for other
11102 	 * PF/VF VSIs.
11103 	 * The FW can still send Flow control frames if enabled.
11104 	 */
11105 	i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11106 						       pf->main_vsi_seid);
11107 
11108 	/* restart the VSIs that were rebuilt and running before the reset */
11109 	i40e_pf_unquiesce_all_vsi(pf);
11110 
11111 	/* Release the RTNL lock before we start resetting VFs */
11112 	if (!lock_acquired)
11113 		rtnl_unlock();
11114 
11115 	/* Restore promiscuous settings */
11116 	ret = i40e_set_promiscuous(pf, pf->cur_promisc);
11117 	if (ret)
11118 		dev_warn(&pf->pdev->dev,
11119 			 "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n",
11120 			 pf->cur_promisc ? "on" : "off",
11121 			 ERR_PTR(ret),
11122 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11123 
11124 	i40e_reset_all_vfs(pf, true);
11125 
11126 	/* tell the firmware that we're starting */
11127 	i40e_send_version(pf);
11128 
11129 	/* We've already released the lock, so don't do it again */
11130 	goto end_core_reset;
11131 
11132 end_unlock:
11133 	if (!lock_acquired)
11134 		rtnl_unlock();
11135 end_core_reset:
11136 	clear_bit(__I40E_RESET_FAILED, pf->state);
11137 clear_recovery:
11138 	clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
11139 	clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
11140 }
11141 
11142 /**
11143  * i40e_reset_and_rebuild - reset and rebuild using a saved config
11144  * @pf: board private structure
11145  * @reinit: if the Main VSI needs to re-initialized.
11146  * @lock_acquired: indicates whether or not the lock has been acquired
11147  * before this function was called.
11148  **/
i40e_reset_and_rebuild(struct i40e_pf * pf,bool reinit,bool lock_acquired)11149 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
11150 				   bool lock_acquired)
11151 {
11152 	int ret;
11153 
11154 	if (test_bit(__I40E_IN_REMOVE, pf->state))
11155 		return;
11156 	/* Now we wait for GRST to settle out.
11157 	 * We don't have to delete the VEBs or VSIs from the hw switch
11158 	 * because the reset will make them disappear.
11159 	 */
11160 	ret = i40e_reset(pf);
11161 	if (!ret)
11162 		i40e_rebuild(pf, reinit, lock_acquired);
11163 	else
11164 		dev_err(&pf->pdev->dev, "%s: i40e_reset() FAILED", __func__);
11165 }
11166 
11167 /**
11168  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
11169  * @pf: board private structure
11170  *
11171  * Close up the VFs and other things in prep for a Core Reset,
11172  * then get ready to rebuild the world.
11173  * @lock_acquired: indicates whether or not the lock has been acquired
11174  * before this function was called.
11175  **/
i40e_handle_reset_warning(struct i40e_pf * pf,bool lock_acquired)11176 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
11177 {
11178 	i40e_prep_for_reset(pf);
11179 	i40e_reset_and_rebuild(pf, false, lock_acquired);
11180 }
11181 
11182 /**
11183  * i40e_print_vf_mdd_event - print VF Tx/Rx malicious driver detect event
11184  * @pf: board private structure
11185  * @vf: pointer to the VF structure
11186  * @is_tx: true - for Tx event, false - for  Rx
11187  */
i40e_print_vf_mdd_event(struct i40e_pf * pf,struct i40e_vf * vf,bool is_tx)11188 static void i40e_print_vf_mdd_event(struct i40e_pf *pf, struct i40e_vf *vf,
11189 				    bool is_tx)
11190 {
11191 	dev_err(&pf->pdev->dev, is_tx ?
11192 		"%lld Tx Malicious Driver Detection events detected on PF %d VF %d MAC %pm. mdd-auto-reset-vfs=%s\n" :
11193 		"%lld Rx Malicious Driver Detection events detected on PF %d VF %d MAC %pm. mdd-auto-reset-vfs=%s\n",
11194 		is_tx ? vf->mdd_tx_events.count : vf->mdd_rx_events.count,
11195 		pf->hw.pf_id,
11196 		vf->vf_id,
11197 		vf->default_lan_addr.addr,
11198 		str_on_off(test_bit(I40E_FLAG_MDD_AUTO_RESET_VF, pf->flags)));
11199 }
11200 
11201 /**
11202  * i40e_print_vfs_mdd_events - print VFs malicious driver detect event
11203  * @pf: pointer to the PF structure
11204  *
11205  * Called from i40e_handle_mdd_event to rate limit and print VFs MDD events.
11206  */
i40e_print_vfs_mdd_events(struct i40e_pf * pf)11207 static void i40e_print_vfs_mdd_events(struct i40e_pf *pf)
11208 {
11209 	unsigned int i;
11210 
11211 	/* check that there are pending MDD events to print */
11212 	if (!test_and_clear_bit(__I40E_MDD_VF_PRINT_PENDING, pf->state))
11213 		return;
11214 
11215 	if (!__ratelimit(&pf->mdd_message_rate_limit))
11216 		return;
11217 
11218 	for (i = 0; i < pf->num_alloc_vfs; i++) {
11219 		struct i40e_vf *vf = &pf->vf[i];
11220 		bool is_printed = false;
11221 
11222 		/* only print Rx MDD event message if there are new events */
11223 		if (vf->mdd_rx_events.count != vf->mdd_rx_events.last_printed) {
11224 			vf->mdd_rx_events.last_printed = vf->mdd_rx_events.count;
11225 			i40e_print_vf_mdd_event(pf, vf, false);
11226 			is_printed = true;
11227 		}
11228 
11229 		/* only print Tx MDD event message if there are new events */
11230 		if (vf->mdd_tx_events.count != vf->mdd_tx_events.last_printed) {
11231 			vf->mdd_tx_events.last_printed = vf->mdd_tx_events.count;
11232 			i40e_print_vf_mdd_event(pf, vf, true);
11233 			is_printed = true;
11234 		}
11235 
11236 		if (is_printed && !test_bit(I40E_FLAG_MDD_AUTO_RESET_VF, pf->flags))
11237 			dev_info(&pf->pdev->dev,
11238 				 "Use PF Control I/F to re-enable the VF #%d\n",
11239 				 i);
11240 	}
11241 }
11242 
11243 /**
11244  * i40e_handle_mdd_event
11245  * @pf: pointer to the PF structure
11246  *
11247  * Called from the MDD irq handler to identify possibly malicious vfs
11248  **/
i40e_handle_mdd_event(struct i40e_pf * pf)11249 static void i40e_handle_mdd_event(struct i40e_pf *pf)
11250 {
11251 	struct i40e_hw *hw = &pf->hw;
11252 	bool mdd_detected = false;
11253 	struct i40e_vf *vf;
11254 	u32 reg;
11255 	int i;
11256 
11257 	if (!test_and_clear_bit(__I40E_MDD_EVENT_PENDING, pf->state)) {
11258 		/* Since the VF MDD event logging is rate limited, check if
11259 		 * there are pending MDD events.
11260 		 */
11261 		i40e_print_vfs_mdd_events(pf);
11262 		return;
11263 	}
11264 
11265 	/* find what triggered the MDD event */
11266 	reg = rd32(hw, I40E_GL_MDET_TX);
11267 	if (reg & I40E_GL_MDET_TX_VALID_MASK) {
11268 		u8 pf_num = FIELD_GET(I40E_GL_MDET_TX_PF_NUM_MASK, reg);
11269 		u16 vf_num = FIELD_GET(I40E_GL_MDET_TX_VF_NUM_MASK, reg);
11270 		u8 event = FIELD_GET(I40E_GL_MDET_TX_EVENT_MASK, reg);
11271 		u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) -
11272 				pf->hw.func_caps.base_queue;
11273 		if (netif_msg_tx_err(pf))
11274 			dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
11275 				 event, queue, pf_num, vf_num);
11276 		wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
11277 		mdd_detected = true;
11278 	}
11279 	reg = rd32(hw, I40E_GL_MDET_RX);
11280 	if (reg & I40E_GL_MDET_RX_VALID_MASK) {
11281 		u8 func = FIELD_GET(I40E_GL_MDET_RX_FUNCTION_MASK, reg);
11282 		u8 event = FIELD_GET(I40E_GL_MDET_RX_EVENT_MASK, reg);
11283 		u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) -
11284 				pf->hw.func_caps.base_queue;
11285 		if (netif_msg_rx_err(pf))
11286 			dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
11287 				 event, queue, func);
11288 		wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
11289 		mdd_detected = true;
11290 	}
11291 
11292 	if (mdd_detected) {
11293 		reg = rd32(hw, I40E_PF_MDET_TX);
11294 		if (reg & I40E_PF_MDET_TX_VALID_MASK) {
11295 			wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
11296 			dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
11297 		}
11298 		reg = rd32(hw, I40E_PF_MDET_RX);
11299 		if (reg & I40E_PF_MDET_RX_VALID_MASK) {
11300 			wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
11301 			dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
11302 		}
11303 	}
11304 
11305 	/* see if one of the VFs needs its hand slapped */
11306 	for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
11307 		bool is_mdd_on_tx = false;
11308 		bool is_mdd_on_rx = false;
11309 
11310 		vf = &(pf->vf[i]);
11311 		reg = rd32(hw, I40E_VP_MDET_TX(i));
11312 		if (reg & I40E_VP_MDET_TX_VALID_MASK) {
11313 			set_bit(__I40E_MDD_VF_PRINT_PENDING, pf->state);
11314 			wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
11315 			vf->mdd_tx_events.count++;
11316 			set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11317 			is_mdd_on_tx = true;
11318 		}
11319 
11320 		reg = rd32(hw, I40E_VP_MDET_RX(i));
11321 		if (reg & I40E_VP_MDET_RX_VALID_MASK) {
11322 			set_bit(__I40E_MDD_VF_PRINT_PENDING, pf->state);
11323 			wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
11324 			vf->mdd_rx_events.count++;
11325 			set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11326 			is_mdd_on_rx = true;
11327 		}
11328 
11329 		if ((is_mdd_on_tx || is_mdd_on_rx) &&
11330 		    test_bit(I40E_FLAG_MDD_AUTO_RESET_VF, pf->flags)) {
11331 			/* VF MDD event counters will be cleared by
11332 			 * reset, so print the event prior to reset.
11333 			 */
11334 			if (is_mdd_on_rx)
11335 				i40e_print_vf_mdd_event(pf, vf, false);
11336 			if (is_mdd_on_tx)
11337 				i40e_print_vf_mdd_event(pf, vf, true);
11338 
11339 			i40e_vc_reset_vf(vf, true);
11340 		}
11341 	}
11342 
11343 	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
11344 	reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
11345 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
11346 	i40e_flush(hw);
11347 
11348 	i40e_print_vfs_mdd_events(pf);
11349 }
11350 
11351 /**
11352  * i40e_service_task - Run the driver's async subtasks
11353  * @work: pointer to work_struct containing our data
11354  **/
i40e_service_task(struct work_struct * work)11355 static void i40e_service_task(struct work_struct *work)
11356 {
11357 	struct i40e_pf *pf = container_of(work,
11358 					  struct i40e_pf,
11359 					  service_task);
11360 	unsigned long start_time = jiffies;
11361 
11362 	/* don't bother with service tasks if a reset is in progress */
11363 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
11364 	    test_bit(__I40E_SUSPENDED, pf->state))
11365 		return;
11366 
11367 	if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
11368 		return;
11369 
11370 	if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
11371 		i40e_detect_recover_hung(pf);
11372 		i40e_sync_filters_subtask(pf);
11373 		i40e_reset_subtask(pf);
11374 		i40e_handle_mdd_event(pf);
11375 		i40e_vc_process_vflr_event(pf);
11376 		i40e_watchdog_subtask(pf);
11377 		i40e_fdir_reinit_subtask(pf);
11378 		if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
11379 			/* Client subtask will reopen next time through. */
11380 			i40e_notify_client_of_netdev_close(pf, true);
11381 		} else {
11382 			i40e_client_subtask(pf);
11383 			if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
11384 					       pf->state))
11385 				i40e_notify_client_of_l2_param_changes(pf);
11386 		}
11387 		i40e_sync_filters_subtask(pf);
11388 	} else {
11389 		i40e_reset_subtask(pf);
11390 	}
11391 
11392 	i40e_clean_adminq_subtask(pf);
11393 
11394 	/* flush memory to make sure state is correct before next watchdog */
11395 	smp_mb__before_atomic();
11396 	clear_bit(__I40E_SERVICE_SCHED, pf->state);
11397 
11398 	/* If the tasks have taken longer than one timer cycle or there
11399 	 * is more work to be done, reschedule the service task now
11400 	 * rather than wait for the timer to tick again.
11401 	 */
11402 	if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
11403 	    test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)		 ||
11404 	    test_bit(__I40E_MDD_EVENT_PENDING, pf->state)		 ||
11405 	    test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
11406 		i40e_service_event_schedule(pf);
11407 }
11408 
11409 /**
11410  * i40e_service_timer - timer callback
11411  * @t: timer list pointer
11412  **/
i40e_service_timer(struct timer_list * t)11413 static void i40e_service_timer(struct timer_list *t)
11414 {
11415 	struct i40e_pf *pf = from_timer(pf, t, service_timer);
11416 
11417 	mod_timer(&pf->service_timer,
11418 		  round_jiffies(jiffies + pf->service_timer_period));
11419 	i40e_service_event_schedule(pf);
11420 }
11421 
11422 /**
11423  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11424  * @vsi: the VSI being configured
11425  **/
i40e_set_num_rings_in_vsi(struct i40e_vsi * vsi)11426 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
11427 {
11428 	struct i40e_pf *pf = vsi->back;
11429 
11430 	switch (vsi->type) {
11431 	case I40E_VSI_MAIN:
11432 		vsi->alloc_queue_pairs = pf->num_lan_qps;
11433 		if (!vsi->num_tx_desc)
11434 			vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11435 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11436 		if (!vsi->num_rx_desc)
11437 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11438 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11439 		if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11440 			vsi->num_q_vectors = pf->num_lan_msix;
11441 		else
11442 			vsi->num_q_vectors = 1;
11443 
11444 		break;
11445 
11446 	case I40E_VSI_FDIR:
11447 		vsi->alloc_queue_pairs = 1;
11448 		vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11449 					 I40E_REQ_DESCRIPTOR_MULTIPLE);
11450 		vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11451 					 I40E_REQ_DESCRIPTOR_MULTIPLE);
11452 		vsi->num_q_vectors = pf->num_fdsb_msix;
11453 		break;
11454 
11455 	case I40E_VSI_VMDQ2:
11456 		vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11457 		if (!vsi->num_tx_desc)
11458 			vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11459 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11460 		if (!vsi->num_rx_desc)
11461 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11462 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11463 		vsi->num_q_vectors = pf->num_vmdq_msix;
11464 		break;
11465 
11466 	case I40E_VSI_SRIOV:
11467 		vsi->alloc_queue_pairs = pf->num_vf_qps;
11468 		if (!vsi->num_tx_desc)
11469 			vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11470 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11471 		if (!vsi->num_rx_desc)
11472 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11473 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11474 		break;
11475 
11476 	default:
11477 		WARN_ON(1);
11478 		return -ENODATA;
11479 	}
11480 
11481 	if (is_kdump_kernel()) {
11482 		vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;
11483 		vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;
11484 	}
11485 
11486 	return 0;
11487 }
11488 
11489 /**
11490  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11491  * @vsi: VSI pointer
11492  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11493  *
11494  * On error: returns error code (negative)
11495  * On success: returns 0
11496  **/
i40e_vsi_alloc_arrays(struct i40e_vsi * vsi,bool alloc_qvectors)11497 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11498 {
11499 	struct i40e_ring **next_rings;
11500 	int size;
11501 	int ret = 0;
11502 
11503 	/* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11504 	size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11505 	       (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11506 	vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11507 	if (!vsi->tx_rings)
11508 		return -ENOMEM;
11509 	next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11510 	if (i40e_enabled_xdp_vsi(vsi)) {
11511 		vsi->xdp_rings = next_rings;
11512 		next_rings += vsi->alloc_queue_pairs;
11513 	}
11514 	vsi->rx_rings = next_rings;
11515 
11516 	if (alloc_qvectors) {
11517 		/* allocate memory for q_vector pointers */
11518 		size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11519 		vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11520 		if (!vsi->q_vectors) {
11521 			ret = -ENOMEM;
11522 			goto err_vectors;
11523 		}
11524 	}
11525 	return ret;
11526 
11527 err_vectors:
11528 	kfree(vsi->tx_rings);
11529 	return ret;
11530 }
11531 
11532 /**
11533  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11534  * @pf: board private structure
11535  * @type: type of VSI
11536  *
11537  * On error: returns error code (negative)
11538  * On success: returns vsi index in PF (positive)
11539  **/
i40e_vsi_mem_alloc(struct i40e_pf * pf,enum i40e_vsi_type type)11540 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11541 {
11542 	int ret = -ENODEV;
11543 	struct i40e_vsi *vsi;
11544 	int vsi_idx;
11545 	int i;
11546 
11547 	/* Need to protect the allocation of the VSIs at the PF level */
11548 	mutex_lock(&pf->switch_mutex);
11549 
11550 	/* VSI list may be fragmented if VSI creation/destruction has
11551 	 * been happening.  We can afford to do a quick scan to look
11552 	 * for any free VSIs in the list.
11553 	 *
11554 	 * find next empty vsi slot, looping back around if necessary
11555 	 */
11556 	i = pf->next_vsi;
11557 	while (i < pf->num_alloc_vsi && pf->vsi[i])
11558 		i++;
11559 	if (i >= pf->num_alloc_vsi) {
11560 		i = 0;
11561 		while (i < pf->next_vsi && pf->vsi[i])
11562 			i++;
11563 	}
11564 
11565 	if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11566 		vsi_idx = i;             /* Found one! */
11567 	} else {
11568 		ret = -ENODEV;
11569 		goto unlock_pf;  /* out of VSI slots! */
11570 	}
11571 	pf->next_vsi = ++i;
11572 
11573 	vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
11574 	if (!vsi) {
11575 		ret = -ENOMEM;
11576 		goto unlock_pf;
11577 	}
11578 	vsi->type = type;
11579 	vsi->back = pf;
11580 	set_bit(__I40E_VSI_DOWN, vsi->state);
11581 	vsi->flags = 0;
11582 	vsi->idx = vsi_idx;
11583 	vsi->int_rate_limit = 0;
11584 	vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11585 				pf->rss_table_size : 64;
11586 	vsi->netdev_registered = false;
11587 	vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11588 	hash_init(vsi->mac_filter_hash);
11589 	vsi->irqs_ready = false;
11590 
11591 	if (type == I40E_VSI_MAIN) {
11592 		vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
11593 		if (!vsi->af_xdp_zc_qps)
11594 			goto err_rings;
11595 	}
11596 
11597 	ret = i40e_set_num_rings_in_vsi(vsi);
11598 	if (ret)
11599 		goto err_rings;
11600 
11601 	ret = i40e_vsi_alloc_arrays(vsi, true);
11602 	if (ret)
11603 		goto err_rings;
11604 
11605 	/* Setup default MSIX irq handler for VSI */
11606 	i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
11607 
11608 	/* Initialize VSI lock */
11609 	spin_lock_init(&vsi->mac_filter_hash_lock);
11610 	pf->vsi[vsi_idx] = vsi;
11611 	ret = vsi_idx;
11612 	goto unlock_pf;
11613 
11614 err_rings:
11615 	bitmap_free(vsi->af_xdp_zc_qps);
11616 	pf->next_vsi = i - 1;
11617 	kfree(vsi);
11618 unlock_pf:
11619 	mutex_unlock(&pf->switch_mutex);
11620 	return ret;
11621 }
11622 
11623 /**
11624  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11625  * @vsi: VSI pointer
11626  * @free_qvectors: a bool to specify if q_vectors need to be freed.
11627  *
11628  * On error: returns error code (negative)
11629  * On success: returns 0
11630  **/
i40e_vsi_free_arrays(struct i40e_vsi * vsi,bool free_qvectors)11631 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11632 {
11633 	/* free the ring and vector containers */
11634 	if (free_qvectors) {
11635 		kfree(vsi->q_vectors);
11636 		vsi->q_vectors = NULL;
11637 	}
11638 	kfree(vsi->tx_rings);
11639 	vsi->tx_rings = NULL;
11640 	vsi->rx_rings = NULL;
11641 	vsi->xdp_rings = NULL;
11642 }
11643 
11644 /**
11645  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11646  * and lookup table
11647  * @vsi: Pointer to VSI structure
11648  */
i40e_clear_rss_config_user(struct i40e_vsi * vsi)11649 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11650 {
11651 	if (!vsi)
11652 		return;
11653 
11654 	kfree(vsi->rss_hkey_user);
11655 	vsi->rss_hkey_user = NULL;
11656 
11657 	kfree(vsi->rss_lut_user);
11658 	vsi->rss_lut_user = NULL;
11659 }
11660 
11661 /**
11662  * i40e_vsi_clear - Deallocate the VSI provided
11663  * @vsi: the VSI being un-configured
11664  **/
i40e_vsi_clear(struct i40e_vsi * vsi)11665 static int i40e_vsi_clear(struct i40e_vsi *vsi)
11666 {
11667 	struct i40e_pf *pf;
11668 
11669 	if (!vsi)
11670 		return 0;
11671 
11672 	if (!vsi->back)
11673 		goto free_vsi;
11674 	pf = vsi->back;
11675 
11676 	mutex_lock(&pf->switch_mutex);
11677 	if (!pf->vsi[vsi->idx]) {
11678 		dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11679 			vsi->idx, vsi->idx, vsi->type);
11680 		goto unlock_vsi;
11681 	}
11682 
11683 	if (pf->vsi[vsi->idx] != vsi) {
11684 		dev_err(&pf->pdev->dev,
11685 			"pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11686 			pf->vsi[vsi->idx]->idx,
11687 			pf->vsi[vsi->idx]->type,
11688 			vsi->idx, vsi->type);
11689 		goto unlock_vsi;
11690 	}
11691 
11692 	/* updates the PF for this cleared vsi */
11693 	i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
11694 	i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
11695 
11696 	bitmap_free(vsi->af_xdp_zc_qps);
11697 	i40e_vsi_free_arrays(vsi, true);
11698 	i40e_clear_rss_config_user(vsi);
11699 
11700 	pf->vsi[vsi->idx] = NULL;
11701 	if (vsi->idx < pf->next_vsi)
11702 		pf->next_vsi = vsi->idx;
11703 
11704 unlock_vsi:
11705 	mutex_unlock(&pf->switch_mutex);
11706 free_vsi:
11707 	kfree(vsi);
11708 
11709 	return 0;
11710 }
11711 
11712 /**
11713  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11714  * @vsi: the VSI being cleaned
11715  **/
i40e_vsi_clear_rings(struct i40e_vsi * vsi)11716 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11717 {
11718 	int i;
11719 
11720 	if (vsi->tx_rings && vsi->tx_rings[0]) {
11721 		for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11722 			kfree_rcu(vsi->tx_rings[i], rcu);
11723 			WRITE_ONCE(vsi->tx_rings[i], NULL);
11724 			WRITE_ONCE(vsi->rx_rings[i], NULL);
11725 			if (vsi->xdp_rings)
11726 				WRITE_ONCE(vsi->xdp_rings[i], NULL);
11727 		}
11728 	}
11729 }
11730 
11731 /**
11732  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11733  * @vsi: the VSI being configured
11734  **/
i40e_alloc_rings(struct i40e_vsi * vsi)11735 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11736 {
11737 	int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11738 	struct i40e_pf *pf = vsi->back;
11739 	struct i40e_ring *ring;
11740 
11741 	/* Set basic values in the rings to be used later during open() */
11742 	for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11743 		/* allocate space for both Tx and Rx in one shot */
11744 		ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11745 		if (!ring)
11746 			goto err_out;
11747 
11748 		ring->queue_index = i;
11749 		ring->reg_idx = vsi->base_queue + i;
11750 		ring->ring_active = false;
11751 		ring->vsi = vsi;
11752 		ring->netdev = vsi->netdev;
11753 		ring->dev = &pf->pdev->dev;
11754 		ring->count = vsi->num_tx_desc;
11755 		ring->size = 0;
11756 		ring->dcb_tc = 0;
11757 		if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11758 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11759 		ring->itr_setting = pf->tx_itr_default;
11760 		WRITE_ONCE(vsi->tx_rings[i], ring++);
11761 
11762 		if (!i40e_enabled_xdp_vsi(vsi))
11763 			goto setup_rx;
11764 
11765 		ring->queue_index = vsi->alloc_queue_pairs + i;
11766 		ring->reg_idx = vsi->base_queue + ring->queue_index;
11767 		ring->ring_active = false;
11768 		ring->vsi = vsi;
11769 		ring->netdev = NULL;
11770 		ring->dev = &pf->pdev->dev;
11771 		ring->count = vsi->num_tx_desc;
11772 		ring->size = 0;
11773 		ring->dcb_tc = 0;
11774 		if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11775 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11776 		set_ring_xdp(ring);
11777 		ring->itr_setting = pf->tx_itr_default;
11778 		WRITE_ONCE(vsi->xdp_rings[i], ring++);
11779 
11780 setup_rx:
11781 		ring->queue_index = i;
11782 		ring->reg_idx = vsi->base_queue + i;
11783 		ring->ring_active = false;
11784 		ring->vsi = vsi;
11785 		ring->netdev = vsi->netdev;
11786 		ring->dev = &pf->pdev->dev;
11787 		ring->count = vsi->num_rx_desc;
11788 		ring->size = 0;
11789 		ring->dcb_tc = 0;
11790 		ring->itr_setting = pf->rx_itr_default;
11791 		WRITE_ONCE(vsi->rx_rings[i], ring);
11792 	}
11793 
11794 	return 0;
11795 
11796 err_out:
11797 	i40e_vsi_clear_rings(vsi);
11798 	return -ENOMEM;
11799 }
11800 
11801 /**
11802  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11803  * @pf: board private structure
11804  * @vectors: the number of MSI-X vectors to request
11805  *
11806  * Returns the number of vectors reserved, or error
11807  **/
i40e_reserve_msix_vectors(struct i40e_pf * pf,int vectors)11808 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11809 {
11810 	vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11811 					I40E_MIN_MSIX, vectors);
11812 	if (vectors < 0) {
11813 		dev_info(&pf->pdev->dev,
11814 			 "MSI-X vector reservation failed: %d\n", vectors);
11815 		vectors = 0;
11816 	}
11817 
11818 	return vectors;
11819 }
11820 
11821 /**
11822  * i40e_init_msix - Setup the MSIX capability
11823  * @pf: board private structure
11824  *
11825  * Work with the OS to set up the MSIX vectors needed.
11826  *
11827  * Returns the number of vectors reserved or negative on failure
11828  **/
i40e_init_msix(struct i40e_pf * pf)11829 static int i40e_init_msix(struct i40e_pf *pf)
11830 {
11831 	struct i40e_hw *hw = &pf->hw;
11832 	int cpus, extra_vectors;
11833 	int vectors_left;
11834 	int v_budget, i;
11835 	int v_actual;
11836 	int iwarp_requested = 0;
11837 
11838 	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11839 		return -ENODEV;
11840 
11841 	/* The number of vectors we'll request will be comprised of:
11842 	 *   - Add 1 for "other" cause for Admin Queue events, etc.
11843 	 *   - The number of LAN queue pairs
11844 	 *	- Queues being used for RSS.
11845 	 *		We don't need as many as max_rss_size vectors.
11846 	 *		use rss_size instead in the calculation since that
11847 	 *		is governed by number of cpus in the system.
11848 	 *	- assumes symmetric Tx/Rx pairing
11849 	 *   - The number of VMDq pairs
11850 	 *   - The CPU count within the NUMA node if iWARP is enabled
11851 	 * Once we count this up, try the request.
11852 	 *
11853 	 * If we can't get what we want, we'll simplify to nearly nothing
11854 	 * and try again.  If that still fails, we punt.
11855 	 */
11856 	vectors_left = hw->func_caps.num_msix_vectors;
11857 	v_budget = 0;
11858 
11859 	/* reserve one vector for miscellaneous handler */
11860 	if (vectors_left) {
11861 		v_budget++;
11862 		vectors_left--;
11863 	}
11864 
11865 	/* reserve some vectors for the main PF traffic queues. Initially we
11866 	 * only reserve at most 50% of the available vectors, in the case that
11867 	 * the number of online CPUs is large. This ensures that we can enable
11868 	 * extra features as well. Once we've enabled the other features, we
11869 	 * will use any remaining vectors to reach as close as we can to the
11870 	 * number of online CPUs.
11871 	 */
11872 	cpus = num_online_cpus();
11873 	pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11874 	vectors_left -= pf->num_lan_msix;
11875 
11876 	/* reserve one vector for sideband flow director */
11877 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
11878 		if (vectors_left) {
11879 			pf->num_fdsb_msix = 1;
11880 			v_budget++;
11881 			vectors_left--;
11882 		} else {
11883 			pf->num_fdsb_msix = 0;
11884 		}
11885 	}
11886 
11887 	/* can we reserve enough for iWARP? */
11888 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11889 		iwarp_requested = pf->num_iwarp_msix;
11890 
11891 		if (!vectors_left)
11892 			pf->num_iwarp_msix = 0;
11893 		else if (vectors_left < pf->num_iwarp_msix)
11894 			pf->num_iwarp_msix = 1;
11895 		v_budget += pf->num_iwarp_msix;
11896 		vectors_left -= pf->num_iwarp_msix;
11897 	}
11898 
11899 	/* any vectors left over go for VMDq support */
11900 	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) {
11901 		if (!vectors_left) {
11902 			pf->num_vmdq_msix = 0;
11903 			pf->num_vmdq_qps = 0;
11904 		} else {
11905 			int vmdq_vecs_wanted =
11906 				pf->num_vmdq_vsis * pf->num_vmdq_qps;
11907 			int vmdq_vecs =
11908 				min_t(int, vectors_left, vmdq_vecs_wanted);
11909 
11910 			/* if we're short on vectors for what's desired, we limit
11911 			 * the queues per vmdq.  If this is still more than are
11912 			 * available, the user will need to change the number of
11913 			 * queues/vectors used by the PF later with the ethtool
11914 			 * channels command
11915 			 */
11916 			if (vectors_left < vmdq_vecs_wanted) {
11917 				pf->num_vmdq_qps = 1;
11918 				vmdq_vecs_wanted = pf->num_vmdq_vsis;
11919 				vmdq_vecs = min_t(int,
11920 						  vectors_left,
11921 						  vmdq_vecs_wanted);
11922 			}
11923 			pf->num_vmdq_msix = pf->num_vmdq_qps;
11924 
11925 			v_budget += vmdq_vecs;
11926 			vectors_left -= vmdq_vecs;
11927 		}
11928 	}
11929 
11930 	/* On systems with a large number of SMP cores, we previously limited
11931 	 * the number of vectors for num_lan_msix to be at most 50% of the
11932 	 * available vectors, to allow for other features. Now, we add back
11933 	 * the remaining vectors. However, we ensure that the total
11934 	 * num_lan_msix will not exceed num_online_cpus(). To do this, we
11935 	 * calculate the number of vectors we can add without going over the
11936 	 * cap of CPUs. For systems with a small number of CPUs this will be
11937 	 * zero.
11938 	 */
11939 	extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11940 	pf->num_lan_msix += extra_vectors;
11941 	vectors_left -= extra_vectors;
11942 
11943 	WARN(vectors_left < 0,
11944 	     "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11945 
11946 	v_budget += pf->num_lan_msix;
11947 	pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11948 				   GFP_KERNEL);
11949 	if (!pf->msix_entries)
11950 		return -ENOMEM;
11951 
11952 	for (i = 0; i < v_budget; i++)
11953 		pf->msix_entries[i].entry = i;
11954 	v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11955 
11956 	if (v_actual < I40E_MIN_MSIX) {
11957 		clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
11958 		kfree(pf->msix_entries);
11959 		pf->msix_entries = NULL;
11960 		pci_disable_msix(pf->pdev);
11961 		return -ENODEV;
11962 
11963 	} else if (v_actual == I40E_MIN_MSIX) {
11964 		/* Adjust for minimal MSIX use */
11965 		pf->num_vmdq_vsis = 0;
11966 		pf->num_vmdq_qps = 0;
11967 		pf->num_lan_qps = 1;
11968 		pf->num_lan_msix = 1;
11969 
11970 	} else if (v_actual != v_budget) {
11971 		/* If we have limited resources, we will start with no vectors
11972 		 * for the special features and then allocate vectors to some
11973 		 * of these features based on the policy and at the end disable
11974 		 * the features that did not get any vectors.
11975 		 */
11976 		int vec;
11977 
11978 		dev_info(&pf->pdev->dev,
11979 			 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11980 			 v_actual, v_budget);
11981 		/* reserve the misc vector */
11982 		vec = v_actual - 1;
11983 
11984 		/* Scale vector usage down */
11985 		pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
11986 		pf->num_vmdq_vsis = 1;
11987 		pf->num_vmdq_qps = 1;
11988 
11989 		/* partition out the remaining vectors */
11990 		switch (vec) {
11991 		case 2:
11992 			pf->num_lan_msix = 1;
11993 			break;
11994 		case 3:
11995 			if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11996 				pf->num_lan_msix = 1;
11997 				pf->num_iwarp_msix = 1;
11998 			} else {
11999 				pf->num_lan_msix = 2;
12000 			}
12001 			break;
12002 		default:
12003 			if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
12004 				pf->num_iwarp_msix = min_t(int, (vec / 3),
12005 						 iwarp_requested);
12006 				pf->num_vmdq_vsis = min_t(int, (vec / 3),
12007 						  I40E_DEFAULT_NUM_VMDQ_VSI);
12008 			} else {
12009 				pf->num_vmdq_vsis = min_t(int, (vec / 2),
12010 						  I40E_DEFAULT_NUM_VMDQ_VSI);
12011 			}
12012 			if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
12013 				pf->num_fdsb_msix = 1;
12014 				vec--;
12015 			}
12016 			pf->num_lan_msix = min_t(int,
12017 			       (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
12018 							      pf->num_lan_msix);
12019 			pf->num_lan_qps = pf->num_lan_msix;
12020 			break;
12021 		}
12022 	}
12023 
12024 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) {
12025 		dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
12026 		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12027 		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12028 	}
12029 	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) {
12030 		dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
12031 		clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
12032 	}
12033 
12034 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
12035 	    pf->num_iwarp_msix == 0) {
12036 		dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
12037 		clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
12038 	}
12039 	i40e_debug(&pf->hw, I40E_DEBUG_INIT,
12040 		   "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
12041 		   pf->num_lan_msix,
12042 		   pf->num_vmdq_msix * pf->num_vmdq_vsis,
12043 		   pf->num_fdsb_msix,
12044 		   pf->num_iwarp_msix);
12045 
12046 	return v_actual;
12047 }
12048 
12049 /**
12050  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
12051  * @vsi: the VSI being configured
12052  * @v_idx: index of the vector in the vsi struct
12053  *
12054  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
12055  **/
i40e_vsi_alloc_q_vector(struct i40e_vsi * vsi,int v_idx)12056 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
12057 {
12058 	struct i40e_q_vector *q_vector;
12059 
12060 	/* allocate q_vector */
12061 	q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
12062 	if (!q_vector)
12063 		return -ENOMEM;
12064 
12065 	q_vector->vsi = vsi;
12066 	q_vector->v_idx = v_idx;
12067 	cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
12068 
12069 	if (vsi->netdev)
12070 		netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll);
12071 
12072 	/* tie q_vector and vsi together */
12073 	vsi->q_vectors[v_idx] = q_vector;
12074 
12075 	return 0;
12076 }
12077 
12078 /**
12079  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
12080  * @vsi: the VSI being configured
12081  *
12082  * We allocate one q_vector per queue interrupt.  If allocation fails we
12083  * return -ENOMEM.
12084  **/
i40e_vsi_alloc_q_vectors(struct i40e_vsi * vsi)12085 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
12086 {
12087 	struct i40e_pf *pf = vsi->back;
12088 	int err, v_idx, num_q_vectors;
12089 
12090 	/* if not MSIX, give the one vector only to the LAN VSI */
12091 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
12092 		num_q_vectors = vsi->num_q_vectors;
12093 	else if (vsi->type == I40E_VSI_MAIN)
12094 		num_q_vectors = 1;
12095 	else
12096 		return -EINVAL;
12097 
12098 	for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
12099 		err = i40e_vsi_alloc_q_vector(vsi, v_idx);
12100 		if (err)
12101 			goto err_out;
12102 	}
12103 
12104 	return 0;
12105 
12106 err_out:
12107 	while (v_idx--)
12108 		i40e_free_q_vector(vsi, v_idx);
12109 
12110 	return err;
12111 }
12112 
12113 /**
12114  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
12115  * @pf: board private structure to initialize
12116  **/
i40e_init_interrupt_scheme(struct i40e_pf * pf)12117 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
12118 {
12119 	int vectors = 0;
12120 	ssize_t size;
12121 
12122 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12123 		vectors = i40e_init_msix(pf);
12124 		if (vectors < 0) {
12125 			clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12126 			clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
12127 			clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
12128 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
12129 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
12130 			clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
12131 			clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12132 			clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
12133 			clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
12134 			set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12135 
12136 			/* rework the queue expectations without MSIX */
12137 			i40e_determine_queue_usage(pf);
12138 		}
12139 	}
12140 
12141 	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
12142 	    test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
12143 		dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
12144 		vectors = pci_enable_msi(pf->pdev);
12145 		if (vectors < 0) {
12146 			dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
12147 				 vectors);
12148 			clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
12149 		}
12150 		vectors = 1;  /* one MSI or Legacy vector */
12151 	}
12152 
12153 	if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) &&
12154 	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
12155 		dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
12156 
12157 	/* set up vector assignment tracking */
12158 	size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
12159 	pf->irq_pile = kzalloc(size, GFP_KERNEL);
12160 	if (!pf->irq_pile)
12161 		return -ENOMEM;
12162 
12163 	pf->irq_pile->num_entries = vectors;
12164 
12165 	/* track first vector for misc interrupts, ignore return */
12166 	(void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
12167 
12168 	return 0;
12169 }
12170 
12171 /**
12172  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
12173  * @pf: private board data structure
12174  *
12175  * Restore the interrupt scheme that was cleared when we suspended the
12176  * device. This should be called during resume to re-allocate the q_vectors
12177  * and reacquire IRQs.
12178  */
i40e_restore_interrupt_scheme(struct i40e_pf * pf)12179 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
12180 {
12181 	struct i40e_vsi *vsi;
12182 	int err, i;
12183 
12184 	/* We cleared the MSI and MSI-X flags when disabling the old interrupt
12185 	 * scheme. We need to re-enabled them here in order to attempt to
12186 	 * re-acquire the MSI or MSI-X vectors
12187 	 */
12188 	set_bit(I40E_FLAG_MSI_ENA, pf->flags);
12189 	set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12190 
12191 	err = i40e_init_interrupt_scheme(pf);
12192 	if (err)
12193 		return err;
12194 
12195 	/* Now that we've re-acquired IRQs, we need to remap the vectors and
12196 	 * rings together again.
12197 	 */
12198 	i40e_pf_for_each_vsi(pf, i, vsi) {
12199 		err = i40e_vsi_alloc_q_vectors(vsi);
12200 		if (err)
12201 			goto err_unwind;
12202 
12203 		i40e_vsi_map_rings_to_vectors(vsi);
12204 	}
12205 
12206 	err = i40e_setup_misc_vector(pf);
12207 	if (err)
12208 		goto err_unwind;
12209 
12210 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
12211 		i40e_client_update_msix_info(pf);
12212 
12213 	return 0;
12214 
12215 err_unwind:
12216 	while (i--) {
12217 		if (pf->vsi[i])
12218 			i40e_vsi_free_q_vectors(pf->vsi[i]);
12219 	}
12220 
12221 	return err;
12222 }
12223 
12224 /**
12225  * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
12226  * non queue events in recovery mode
12227  * @pf: board private structure
12228  *
12229  * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
12230  * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
12231  * This is handled differently than in recovery mode since no Tx/Rx resources
12232  * are being allocated.
12233  **/
i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf * pf)12234 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
12235 {
12236 	int err;
12237 
12238 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12239 		err = i40e_setup_misc_vector(pf);
12240 
12241 		if (err) {
12242 			dev_info(&pf->pdev->dev,
12243 				 "MSI-X misc vector request failed, error %d\n",
12244 				 err);
12245 			return err;
12246 		}
12247 	} else {
12248 		u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED;
12249 
12250 		err = request_irq(pf->pdev->irq, i40e_intr, flags,
12251 				  pf->int_name, pf);
12252 
12253 		if (err) {
12254 			dev_info(&pf->pdev->dev,
12255 				 "MSI/legacy misc vector request failed, error %d\n",
12256 				 err);
12257 			return err;
12258 		}
12259 		i40e_enable_misc_int_causes(pf);
12260 		i40e_irq_dynamic_enable_icr0(pf);
12261 	}
12262 
12263 	return 0;
12264 }
12265 
12266 /**
12267  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
12268  * @pf: board private structure
12269  *
12270  * This sets up the handler for MSIX 0, which is used to manage the
12271  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
12272  * when in MSI or Legacy interrupt mode.
12273  **/
i40e_setup_misc_vector(struct i40e_pf * pf)12274 static int i40e_setup_misc_vector(struct i40e_pf *pf)
12275 {
12276 	struct i40e_hw *hw = &pf->hw;
12277 	int err = 0;
12278 
12279 	/* Only request the IRQ once, the first time through. */
12280 	if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
12281 		err = request_irq(pf->msix_entries[0].vector,
12282 				  i40e_intr, 0, pf->int_name, pf);
12283 		if (err) {
12284 			clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
12285 			dev_info(&pf->pdev->dev,
12286 				 "request_irq for %s failed: %d\n",
12287 				 pf->int_name, err);
12288 			return -EFAULT;
12289 		}
12290 	}
12291 
12292 	i40e_enable_misc_int_causes(pf);
12293 
12294 	/* associate no queues to the misc vector */
12295 	wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
12296 	wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
12297 
12298 	i40e_flush(hw);
12299 
12300 	i40e_irq_dynamic_enable_icr0(pf);
12301 
12302 	return err;
12303 }
12304 
12305 /**
12306  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
12307  * @vsi: Pointer to vsi structure
12308  * @seed: Buffter to store the hash keys
12309  * @lut: Buffer to store the lookup table entries
12310  * @lut_size: Size of buffer to store the lookup table entries
12311  *
12312  * Return 0 on success, negative on failure
12313  */
i40e_get_rss_aq(struct i40e_vsi * vsi,const u8 * seed,u8 * lut,u16 lut_size)12314 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
12315 			   u8 *lut, u16 lut_size)
12316 {
12317 	struct i40e_pf *pf = vsi->back;
12318 	struct i40e_hw *hw = &pf->hw;
12319 	int ret = 0;
12320 
12321 	if (seed) {
12322 		ret = i40e_aq_get_rss_key(hw, vsi->id,
12323 			(struct i40e_aqc_get_set_rss_key_data *)seed);
12324 		if (ret) {
12325 			dev_info(&pf->pdev->dev,
12326 				 "Cannot get RSS key, err %pe aq_err %s\n",
12327 				 ERR_PTR(ret),
12328 				 i40e_aq_str(&pf->hw,
12329 					     pf->hw.aq.asq_last_status));
12330 			return ret;
12331 		}
12332 	}
12333 
12334 	if (lut) {
12335 		bool pf_lut = vsi->type == I40E_VSI_MAIN;
12336 
12337 		ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
12338 		if (ret) {
12339 			dev_info(&pf->pdev->dev,
12340 				 "Cannot get RSS lut, err %pe aq_err %s\n",
12341 				 ERR_PTR(ret),
12342 				 i40e_aq_str(&pf->hw,
12343 					     pf->hw.aq.asq_last_status));
12344 			return ret;
12345 		}
12346 	}
12347 
12348 	return ret;
12349 }
12350 
12351 /**
12352  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
12353  * @vsi: Pointer to vsi structure
12354  * @seed: RSS hash seed
12355  * @lut: Lookup table
12356  * @lut_size: Lookup table size
12357  *
12358  * Returns 0 on success, negative on failure
12359  **/
i40e_config_rss_reg(struct i40e_vsi * vsi,const u8 * seed,const u8 * lut,u16 lut_size)12360 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
12361 			       const u8 *lut, u16 lut_size)
12362 {
12363 	struct i40e_pf *pf = vsi->back;
12364 	struct i40e_hw *hw = &pf->hw;
12365 	u16 vf_id = vsi->vf_id;
12366 	u8 i;
12367 
12368 	/* Fill out hash function seed */
12369 	if (seed) {
12370 		u32 *seed_dw = (u32 *)seed;
12371 
12372 		if (vsi->type == I40E_VSI_MAIN) {
12373 			for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12374 				wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
12375 		} else if (vsi->type == I40E_VSI_SRIOV) {
12376 			for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
12377 				wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
12378 		} else {
12379 			dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
12380 		}
12381 	}
12382 
12383 	if (lut) {
12384 		u32 *lut_dw = (u32 *)lut;
12385 
12386 		if (vsi->type == I40E_VSI_MAIN) {
12387 			if (lut_size != I40E_HLUT_ARRAY_SIZE)
12388 				return -EINVAL;
12389 			for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12390 				wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
12391 		} else if (vsi->type == I40E_VSI_SRIOV) {
12392 			if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
12393 				return -EINVAL;
12394 			for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12395 				wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
12396 		} else {
12397 			dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12398 		}
12399 	}
12400 	i40e_flush(hw);
12401 
12402 	return 0;
12403 }
12404 
12405 /**
12406  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
12407  * @vsi: Pointer to VSI structure
12408  * @seed: Buffer to store the keys
12409  * @lut: Buffer to store the lookup table entries
12410  * @lut_size: Size of buffer to store the lookup table entries
12411  *
12412  * Returns 0 on success, negative on failure
12413  */
i40e_get_rss_reg(struct i40e_vsi * vsi,u8 * seed,u8 * lut,u16 lut_size)12414 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
12415 			    u8 *lut, u16 lut_size)
12416 {
12417 	struct i40e_pf *pf = vsi->back;
12418 	struct i40e_hw *hw = &pf->hw;
12419 	u16 i;
12420 
12421 	if (seed) {
12422 		u32 *seed_dw = (u32 *)seed;
12423 
12424 		for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12425 			seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
12426 	}
12427 	if (lut) {
12428 		u32 *lut_dw = (u32 *)lut;
12429 
12430 		if (lut_size != I40E_HLUT_ARRAY_SIZE)
12431 			return -EINVAL;
12432 		for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12433 			lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
12434 	}
12435 
12436 	return 0;
12437 }
12438 
12439 /**
12440  * i40e_config_rss - Configure RSS keys and lut
12441  * @vsi: Pointer to VSI structure
12442  * @seed: RSS hash seed
12443  * @lut: Lookup table
12444  * @lut_size: Lookup table size
12445  *
12446  * Returns 0 on success, negative on failure
12447  */
i40e_config_rss(struct i40e_vsi * vsi,u8 * seed,u8 * lut,u16 lut_size)12448 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12449 {
12450 	struct i40e_pf *pf = vsi->back;
12451 
12452 	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12453 		return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12454 	else
12455 		return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12456 }
12457 
12458 /**
12459  * i40e_get_rss - Get RSS keys and lut
12460  * @vsi: Pointer to VSI structure
12461  * @seed: Buffer to store the keys
12462  * @lut: Buffer to store the lookup table entries
12463  * @lut_size: Size of buffer to store the lookup table entries
12464  *
12465  * Returns 0 on success, negative on failure
12466  */
i40e_get_rss(struct i40e_vsi * vsi,u8 * seed,u8 * lut,u16 lut_size)12467 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12468 {
12469 	struct i40e_pf *pf = vsi->back;
12470 
12471 	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12472 		return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12473 	else
12474 		return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12475 }
12476 
12477 /**
12478  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12479  * @pf: Pointer to board private structure
12480  * @lut: Lookup table
12481  * @rss_table_size: Lookup table size
12482  * @rss_size: Range of queue number for hashing
12483  */
i40e_fill_rss_lut(struct i40e_pf * pf,u8 * lut,u16 rss_table_size,u16 rss_size)12484 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12485 		       u16 rss_table_size, u16 rss_size)
12486 {
12487 	u16 i;
12488 
12489 	for (i = 0; i < rss_table_size; i++)
12490 		lut[i] = i % rss_size;
12491 }
12492 
12493 /**
12494  * i40e_pf_config_rss - Prepare for RSS if used
12495  * @pf: board private structure
12496  **/
i40e_pf_config_rss(struct i40e_pf * pf)12497 static int i40e_pf_config_rss(struct i40e_pf *pf)
12498 {
12499 	struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
12500 	u8 seed[I40E_HKEY_ARRAY_SIZE];
12501 	u8 *lut;
12502 	struct i40e_hw *hw = &pf->hw;
12503 	u32 reg_val;
12504 	u64 hena;
12505 	int ret;
12506 
12507 	/* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12508 	hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12509 		((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12510 	hena |= i40e_pf_get_default_rss_hena(pf);
12511 
12512 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
12513 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
12514 
12515 	/* Determine the RSS table size based on the hardware capabilities */
12516 	reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12517 	reg_val = (pf->rss_table_size == 512) ?
12518 			(reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12519 			(reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12520 	i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12521 
12522 	/* Determine the RSS size of the VSI */
12523 	if (!vsi->rss_size) {
12524 		u16 qcount;
12525 		/* If the firmware does something weird during VSI init, we
12526 		 * could end up with zero TCs. Check for that to avoid
12527 		 * divide-by-zero. It probably won't pass traffic, but it also
12528 		 * won't panic.
12529 		 */
12530 		qcount = vsi->num_queue_pairs /
12531 			 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12532 		vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12533 	}
12534 	if (!vsi->rss_size)
12535 		return -EINVAL;
12536 
12537 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
12538 	if (!lut)
12539 		return -ENOMEM;
12540 
12541 	/* Use user configured lut if there is one, otherwise use default */
12542 	if (vsi->rss_lut_user)
12543 		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12544 	else
12545 		i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
12546 
12547 	/* Use user configured hash key if there is one, otherwise
12548 	 * use default.
12549 	 */
12550 	if (vsi->rss_hkey_user)
12551 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12552 	else
12553 		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
12554 	ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
12555 	kfree(lut);
12556 
12557 	return ret;
12558 }
12559 
12560 /**
12561  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12562  * @pf: board private structure
12563  * @queue_count: the requested queue count for rss.
12564  *
12565  * returns 0 if rss is not enabled, if enabled returns the final rss queue
12566  * count which may be different from the requested queue count.
12567  * Note: expects to be called while under rtnl_lock()
12568  **/
i40e_reconfig_rss_queues(struct i40e_pf * pf,int queue_count)12569 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12570 {
12571 	struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
12572 	int new_rss_size;
12573 
12574 	if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags))
12575 		return 0;
12576 
12577 	queue_count = min_t(int, queue_count, num_online_cpus());
12578 	new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12579 
12580 	if (queue_count != vsi->num_queue_pairs) {
12581 		u16 qcount;
12582 
12583 		vsi->req_queue_pairs = queue_count;
12584 		i40e_prep_for_reset(pf);
12585 		if (test_bit(__I40E_IN_REMOVE, pf->state))
12586 			return pf->alloc_rss_size;
12587 
12588 		pf->alloc_rss_size = new_rss_size;
12589 
12590 		i40e_reset_and_rebuild(pf, true, true);
12591 
12592 		/* Discard the user configured hash keys and lut, if less
12593 		 * queues are enabled.
12594 		 */
12595 		if (queue_count < vsi->rss_size) {
12596 			i40e_clear_rss_config_user(vsi);
12597 			dev_dbg(&pf->pdev->dev,
12598 				"discard user configured hash keys and lut\n");
12599 		}
12600 
12601 		/* Reset vsi->rss_size, as number of enabled queues changed */
12602 		qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12603 		vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12604 
12605 		i40e_pf_config_rss(pf);
12606 	}
12607 	dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
12608 		 vsi->req_queue_pairs, pf->rss_size_max);
12609 	return pf->alloc_rss_size;
12610 }
12611 
12612 /**
12613  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12614  * @pf: board private structure
12615  **/
i40e_get_partition_bw_setting(struct i40e_pf * pf)12616 int i40e_get_partition_bw_setting(struct i40e_pf *pf)
12617 {
12618 	bool min_valid, max_valid;
12619 	u32 max_bw, min_bw;
12620 	int status;
12621 
12622 	status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
12623 					   &min_valid, &max_valid);
12624 
12625 	if (!status) {
12626 		if (min_valid)
12627 			pf->min_bw = min_bw;
12628 		if (max_valid)
12629 			pf->max_bw = max_bw;
12630 	}
12631 
12632 	return status;
12633 }
12634 
12635 /**
12636  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12637  * @pf: board private structure
12638  **/
i40e_set_partition_bw_setting(struct i40e_pf * pf)12639 int i40e_set_partition_bw_setting(struct i40e_pf *pf)
12640 {
12641 	struct i40e_aqc_configure_partition_bw_data bw_data;
12642 	int status;
12643 
12644 	memset(&bw_data, 0, sizeof(bw_data));
12645 
12646 	/* Set the valid bit for this PF */
12647 	bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12648 	bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12649 	bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12650 
12651 	/* Set the new bandwidths */
12652 	status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
12653 
12654 	return status;
12655 }
12656 
12657 /**
12658  * i40e_is_total_port_shutdown_enabled - read NVM and return value
12659  * if total port shutdown feature is enabled for this PF
12660  * @pf: board private structure
12661  **/
i40e_is_total_port_shutdown_enabled(struct i40e_pf * pf)12662 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12663 {
12664 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED	BIT(4)
12665 #define I40E_FEATURES_ENABLE_PTR		0x2A
12666 #define I40E_CURRENT_SETTING_PTR		0x2B
12667 #define I40E_LINK_BEHAVIOR_WORD_OFFSET		0x2D
12668 #define I40E_LINK_BEHAVIOR_WORD_LENGTH		0x1
12669 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED	BIT(0)
12670 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH	4
12671 	u16 sr_emp_sr_settings_ptr = 0;
12672 	u16 features_enable = 0;
12673 	u16 link_behavior = 0;
12674 	int read_status = 0;
12675 	bool ret = false;
12676 
12677 	read_status = i40e_read_nvm_word(&pf->hw,
12678 					 I40E_SR_EMP_SR_SETTINGS_PTR,
12679 					 &sr_emp_sr_settings_ptr);
12680 	if (read_status)
12681 		goto err_nvm;
12682 	read_status = i40e_read_nvm_word(&pf->hw,
12683 					 sr_emp_sr_settings_ptr +
12684 					 I40E_FEATURES_ENABLE_PTR,
12685 					 &features_enable);
12686 	if (read_status)
12687 		goto err_nvm;
12688 	if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12689 		read_status = i40e_read_nvm_module_data(&pf->hw,
12690 							I40E_SR_EMP_SR_SETTINGS_PTR,
12691 							I40E_CURRENT_SETTING_PTR,
12692 							I40E_LINK_BEHAVIOR_WORD_OFFSET,
12693 							I40E_LINK_BEHAVIOR_WORD_LENGTH,
12694 							&link_behavior);
12695 		if (read_status)
12696 			goto err_nvm;
12697 		link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12698 		ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12699 	}
12700 	return ret;
12701 
12702 err_nvm:
12703 	dev_warn(&pf->pdev->dev,
12704 		 "total-port-shutdown feature is off due to read nvm error: %pe\n",
12705 		 ERR_PTR(read_status));
12706 	return ret;
12707 }
12708 
12709 /**
12710  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12711  * @pf: board private structure to initialize
12712  *
12713  * i40e_sw_init initializes the Adapter private data structure.
12714  * Fields are initialized based on PCI device information and
12715  * OS network device settings (MTU size).
12716  **/
i40e_sw_init(struct i40e_pf * pf)12717 static int i40e_sw_init(struct i40e_pf *pf)
12718 {
12719 	int err = 0;
12720 	int size;
12721 	u16 pow;
12722 
12723 	/* Set default capability flags */
12724 	bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS);
12725 	set_bit(I40E_FLAG_MSI_ENA, pf->flags);
12726 	set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12727 
12728 	/* Set default ITR */
12729 	pf->rx_itr_default = I40E_ITR_RX_DEF;
12730 	pf->tx_itr_default = I40E_ITR_TX_DEF;
12731 
12732 	/* Depending on PF configurations, it is possible that the RSS
12733 	 * maximum might end up larger than the available queues
12734 	 */
12735 	pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12736 	pf->alloc_rss_size = 1;
12737 	pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12738 	pf->rss_size_max = min_t(int, pf->rss_size_max,
12739 				 pf->hw.func_caps.num_tx_qp);
12740 
12741 	/* find the next higher power-of-2 of num cpus */
12742 	pow = roundup_pow_of_two(num_online_cpus());
12743 	pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12744 
12745 	if (pf->hw.func_caps.rss) {
12746 		set_bit(I40E_FLAG_RSS_ENA, pf->flags);
12747 		pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12748 					   num_online_cpus());
12749 	}
12750 
12751 	/* MFP mode enabled */
12752 	if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12753 		set_bit(I40E_FLAG_MFP_ENA, pf->flags);
12754 		dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12755 		if (i40e_get_partition_bw_setting(pf)) {
12756 			dev_warn(&pf->pdev->dev,
12757 				 "Could not get partition bw settings\n");
12758 		} else {
12759 			dev_info(&pf->pdev->dev,
12760 				 "Partition BW Min = %8.8x, Max = %8.8x\n",
12761 				 pf->min_bw, pf->max_bw);
12762 
12763 			/* nudge the Tx scheduler */
12764 			i40e_set_partition_bw_setting(pf);
12765 		}
12766 	}
12767 
12768 	if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12769 	    (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12770 		set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
12771 		if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
12772 		    pf->hw.num_partitions > 1)
12773 			dev_info(&pf->pdev->dev,
12774 				 "Flow Director Sideband mode Disabled in MFP mode\n");
12775 		else
12776 			set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12777 		pf->fdir_pf_filter_count =
12778 				 pf->hw.func_caps.fd_filters_guaranteed;
12779 		pf->hw.fdir_shared_filter_count =
12780 				 pf->hw.func_caps.fd_filters_best_effort;
12781 	}
12782 
12783 	/* Enable HW ATR eviction if possible */
12784 	if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps))
12785 		set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags);
12786 
12787 	if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12788 		pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12789 		set_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
12790 		pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12791 	}
12792 
12793 	if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12794 		set_bit(I40E_FLAG_IWARP_ENA, pf->flags);
12795 		/* IWARP needs one extra vector for CQP just like MISC.*/
12796 		pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12797 	}
12798 	/* Stopping FW LLDP engine is supported on XL710 and X722
12799 	 * starting from FW versions determined in i40e_init_adminq.
12800 	 * Stopping the FW LLDP engine is not supported on XL710
12801 	 * if NPAR is functioning so unset this hw flag in this case.
12802 	 */
12803 	if (pf->hw.mac.type == I40E_MAC_XL710 &&
12804 	    pf->hw.func_caps.npar_enable)
12805 		clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps);
12806 
12807 #ifdef CONFIG_PCI_IOV
12808 	if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12809 		pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12810 		set_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
12811 		pf->num_req_vfs = min_t(int,
12812 					pf->hw.func_caps.num_vfs,
12813 					I40E_MAX_VF_COUNT);
12814 	}
12815 #endif /* CONFIG_PCI_IOV */
12816 	pf->lan_veb = I40E_NO_VEB;
12817 	pf->lan_vsi = I40E_NO_VSI;
12818 
12819 	/* By default FW has this off for performance reasons */
12820 	clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
12821 
12822 	/* set up queue assignment tracking */
12823 	size = sizeof(struct i40e_lump_tracking)
12824 		+ (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12825 	pf->qp_pile = kzalloc(size, GFP_KERNEL);
12826 	if (!pf->qp_pile) {
12827 		err = -ENOMEM;
12828 		goto sw_init_done;
12829 	}
12830 	pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12831 
12832 	pf->tx_timeout_recovery_level = 1;
12833 
12834 	if (pf->hw.mac.type != I40E_MAC_X722 &&
12835 	    i40e_is_total_port_shutdown_enabled(pf)) {
12836 		/* Link down on close must be on when total port shutdown
12837 		 * is enabled for a given port
12838 		 */
12839 		set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);
12840 		set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);
12841 		dev_info(&pf->pdev->dev,
12842 			 "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12843 	}
12844 	mutex_init(&pf->switch_mutex);
12845 
12846 sw_init_done:
12847 	return err;
12848 }
12849 
12850 /**
12851  * i40e_set_ntuple - set the ntuple feature flag and take action
12852  * @pf: board private structure to initialize
12853  * @features: the feature set that the stack is suggesting
12854  *
12855  * returns a bool to indicate if reset needs to happen
12856  **/
i40e_set_ntuple(struct i40e_pf * pf,netdev_features_t features)12857 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12858 {
12859 	bool need_reset = false;
12860 
12861 	/* Check if Flow Director n-tuple support was enabled or disabled.  If
12862 	 * the state changed, we need to reset.
12863 	 */
12864 	if (features & NETIF_F_NTUPLE) {
12865 		/* Enable filters and mark for reset */
12866 		if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
12867 			need_reset = true;
12868 		/* enable FD_SB only if there is MSI-X vector and no cloud
12869 		 * filters exist
12870 		 */
12871 		if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12872 			set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12873 			clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12874 		}
12875 	} else {
12876 		/* turn off filters, mark for reset and clear SW filter list */
12877 		if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
12878 			need_reset = true;
12879 			i40e_fdir_filter_exit(pf);
12880 		}
12881 		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12882 		clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12883 		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12884 
12885 		/* reset fd counters */
12886 		pf->fd_add_err = 0;
12887 		pf->fd_atr_cnt = 0;
12888 		/* if ATR was auto disabled it can be re-enabled. */
12889 		if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12890 			if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
12891 			    (I40E_DEBUG_FD & pf->hw.debug_mask))
12892 				dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12893 	}
12894 	return need_reset;
12895 }
12896 
12897 /**
12898  * i40e_clear_rss_lut - clear the rx hash lookup table
12899  * @vsi: the VSI being configured
12900  **/
i40e_clear_rss_lut(struct i40e_vsi * vsi)12901 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12902 {
12903 	struct i40e_pf *pf = vsi->back;
12904 	struct i40e_hw *hw = &pf->hw;
12905 	u16 vf_id = vsi->vf_id;
12906 	u8 i;
12907 
12908 	if (vsi->type == I40E_VSI_MAIN) {
12909 		for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12910 			wr32(hw, I40E_PFQF_HLUT(i), 0);
12911 	} else if (vsi->type == I40E_VSI_SRIOV) {
12912 		for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12913 			i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12914 	} else {
12915 		dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12916 	}
12917 }
12918 
12919 /**
12920  * i40e_set_loopback - turn on/off loopback mode on underlying PF
12921  * @vsi: ptr to VSI
12922  * @ena: flag to indicate the on/off setting
12923  */
i40e_set_loopback(struct i40e_vsi * vsi,bool ena)12924 static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena)
12925 {
12926 	bool if_running = netif_running(vsi->netdev) &&
12927 			  !test_and_set_bit(__I40E_VSI_DOWN, vsi->state);
12928 	int ret;
12929 
12930 	if (if_running)
12931 		i40e_down(vsi);
12932 
12933 	ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);
12934 	if (ret)
12935 		netdev_err(vsi->netdev, "Failed to toggle loopback state\n");
12936 	if (if_running)
12937 		i40e_up(vsi);
12938 
12939 	return ret;
12940 }
12941 
12942 /**
12943  * i40e_set_features - set the netdev feature flags
12944  * @netdev: ptr to the netdev being adjusted
12945  * @features: the feature set that the stack is suggesting
12946  * Note: expects to be called while under rtnl_lock()
12947  **/
i40e_set_features(struct net_device * netdev,netdev_features_t features)12948 static int i40e_set_features(struct net_device *netdev,
12949 			     netdev_features_t features)
12950 {
12951 	struct i40e_netdev_priv *np = netdev_priv(netdev);
12952 	struct i40e_vsi *vsi = np->vsi;
12953 	struct i40e_pf *pf = vsi->back;
12954 	bool need_reset;
12955 
12956 	if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12957 		i40e_pf_config_rss(pf);
12958 	else if (!(features & NETIF_F_RXHASH) &&
12959 		 netdev->features & NETIF_F_RXHASH)
12960 		i40e_clear_rss_lut(vsi);
12961 
12962 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
12963 		i40e_vlan_stripping_enable(vsi);
12964 	else
12965 		i40e_vlan_stripping_disable(vsi);
12966 
12967 	if (!(features & NETIF_F_HW_TC) &&
12968 	    (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12969 		dev_err(&pf->pdev->dev,
12970 			"Offloaded tc filters active, can't turn hw_tc_offload off");
12971 		return -EINVAL;
12972 	}
12973 
12974 	if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12975 		i40e_del_all_macvlans(vsi);
12976 
12977 	need_reset = i40e_set_ntuple(pf, features);
12978 
12979 	if (need_reset)
12980 		i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12981 
12982 	if ((features ^ netdev->features) & NETIF_F_LOOPBACK)
12983 		return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK));
12984 
12985 	return 0;
12986 }
12987 
i40e_udp_tunnel_set_port(struct net_device * netdev,unsigned int table,unsigned int idx,struct udp_tunnel_info * ti)12988 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
12989 				    unsigned int table, unsigned int idx,
12990 				    struct udp_tunnel_info *ti)
12991 {
12992 	struct i40e_netdev_priv *np = netdev_priv(netdev);
12993 	struct i40e_hw *hw = &np->vsi->back->hw;
12994 	u8 type, filter_index;
12995 	int ret;
12996 
12997 	type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
12998 						   I40E_AQC_TUNNEL_TYPE_NGE;
12999 
13000 	ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
13001 				     NULL);
13002 	if (ret) {
13003 		netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n",
13004 			    ERR_PTR(ret),
13005 			    i40e_aq_str(hw, hw->aq.asq_last_status));
13006 		return -EIO;
13007 	}
13008 
13009 	udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
13010 	return 0;
13011 }
13012 
i40e_udp_tunnel_unset_port(struct net_device * netdev,unsigned int table,unsigned int idx,struct udp_tunnel_info * ti)13013 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
13014 				      unsigned int table, unsigned int idx,
13015 				      struct udp_tunnel_info *ti)
13016 {
13017 	struct i40e_netdev_priv *np = netdev_priv(netdev);
13018 	struct i40e_hw *hw = &np->vsi->back->hw;
13019 	int ret;
13020 
13021 	ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
13022 	if (ret) {
13023 		netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n",
13024 			    ERR_PTR(ret),
13025 			    i40e_aq_str(hw, hw->aq.asq_last_status));
13026 		return -EIO;
13027 	}
13028 
13029 	return 0;
13030 }
13031 
i40e_get_phys_port_id(struct net_device * netdev,struct netdev_phys_item_id * ppid)13032 static int i40e_get_phys_port_id(struct net_device *netdev,
13033 				 struct netdev_phys_item_id *ppid)
13034 {
13035 	struct i40e_netdev_priv *np = netdev_priv(netdev);
13036 	struct i40e_pf *pf = np->vsi->back;
13037 	struct i40e_hw *hw = &pf->hw;
13038 
13039 	if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps))
13040 		return -EOPNOTSUPP;
13041 
13042 	ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
13043 	memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
13044 
13045 	return 0;
13046 }
13047 
13048 /**
13049  * i40e_ndo_fdb_add - add an entry to the hardware database
13050  * @ndm: the input from the stack
13051  * @tb: pointer to array of nladdr (unused)
13052  * @dev: the net device pointer
13053  * @addr: the MAC address entry being added
13054  * @vid: VLAN ID
13055  * @flags: instructions from stack about fdb operation
13056  * @notified: whether notification was emitted
13057  * @extack: netlink extended ack, unused currently
13058  */
i40e_ndo_fdb_add(struct ndmsg * ndm,struct nlattr * tb[],struct net_device * dev,const unsigned char * addr,u16 vid,u16 flags,bool * notified,struct netlink_ext_ack * extack)13059 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
13060 			    struct net_device *dev,
13061 			    const unsigned char *addr, u16 vid,
13062 			    u16 flags, bool *notified,
13063 			    struct netlink_ext_ack *extack)
13064 {
13065 	struct i40e_netdev_priv *np = netdev_priv(dev);
13066 	struct i40e_pf *pf = np->vsi->back;
13067 	int err = 0;
13068 
13069 	if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags))
13070 		return -EOPNOTSUPP;
13071 
13072 	if (vid) {
13073 		pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
13074 		return -EINVAL;
13075 	}
13076 
13077 	/* Hardware does not support aging addresses so if a
13078 	 * ndm_state is given only allow permanent addresses
13079 	 */
13080 	if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
13081 		netdev_info(dev, "FDB only supports static addresses\n");
13082 		return -EINVAL;
13083 	}
13084 
13085 	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
13086 		err = dev_uc_add_excl(dev, addr);
13087 	else if (is_multicast_ether_addr(addr))
13088 		err = dev_mc_add_excl(dev, addr);
13089 	else
13090 		err = -EINVAL;
13091 
13092 	/* Only return duplicate errors if NLM_F_EXCL is set */
13093 	if (err == -EEXIST && !(flags & NLM_F_EXCL))
13094 		err = 0;
13095 
13096 	return err;
13097 }
13098 
13099 /**
13100  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
13101  * @dev: the netdev being configured
13102  * @nlh: RTNL message
13103  * @flags: bridge flags
13104  * @extack: netlink extended ack
13105  *
13106  * Inserts a new hardware bridge if not already created and
13107  * enables the bridging mode requested (VEB or VEPA). If the
13108  * hardware bridge has already been inserted and the request
13109  * is to change the mode then that requires a PF reset to
13110  * allow rebuild of the components with required hardware
13111  * bridge mode enabled.
13112  *
13113  * Note: expects to be called while under rtnl_lock()
13114  **/
i40e_ndo_bridge_setlink(struct net_device * dev,struct nlmsghdr * nlh,u16 flags,struct netlink_ext_ack * extack)13115 static int i40e_ndo_bridge_setlink(struct net_device *dev,
13116 				   struct nlmsghdr *nlh,
13117 				   u16 flags,
13118 				   struct netlink_ext_ack *extack)
13119 {
13120 	struct i40e_netdev_priv *np = netdev_priv(dev);
13121 	struct i40e_vsi *vsi = np->vsi;
13122 	struct i40e_pf *pf = vsi->back;
13123 	struct nlattr *attr, *br_spec;
13124 	struct i40e_veb *veb;
13125 	int rem;
13126 
13127 	/* Only for PF VSI for now */
13128 	if (vsi->type != I40E_VSI_MAIN)
13129 		return -EOPNOTSUPP;
13130 
13131 	/* Find the HW bridge for PF VSI */
13132 	veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
13133 
13134 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13135 	if (!br_spec)
13136 		return -EINVAL;
13137 
13138 	nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {
13139 		__u16 mode = nla_get_u16(attr);
13140 
13141 		if ((mode != BRIDGE_MODE_VEPA) &&
13142 		    (mode != BRIDGE_MODE_VEB))
13143 			return -EINVAL;
13144 
13145 		/* Insert a new HW bridge */
13146 		if (!veb) {
13147 			veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,
13148 					     vsi->tc_config.enabled_tc);
13149 			if (veb) {
13150 				veb->bridge_mode = mode;
13151 				i40e_config_bridge_mode(veb);
13152 			} else {
13153 				/* No Bridge HW offload available */
13154 				return -ENOENT;
13155 			}
13156 			break;
13157 		} else if (mode != veb->bridge_mode) {
13158 			/* Existing HW bridge but different mode needs reset */
13159 			veb->bridge_mode = mode;
13160 			/* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
13161 			if (mode == BRIDGE_MODE_VEB)
13162 				set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
13163 			else
13164 				clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
13165 			i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
13166 			break;
13167 		}
13168 	}
13169 
13170 	return 0;
13171 }
13172 
13173 /**
13174  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
13175  * @skb: skb buff
13176  * @pid: process id
13177  * @seq: RTNL message seq #
13178  * @dev: the netdev being configured
13179  * @filter_mask: unused
13180  * @nlflags: netlink flags passed in
13181  *
13182  * Return the mode in which the hardware bridge is operating in
13183  * i.e VEB or VEPA.
13184  **/
i40e_ndo_bridge_getlink(struct sk_buff * skb,u32 pid,u32 seq,struct net_device * dev,u32 __always_unused filter_mask,int nlflags)13185 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13186 				   struct net_device *dev,
13187 				   u32 __always_unused filter_mask,
13188 				   int nlflags)
13189 {
13190 	struct i40e_netdev_priv *np = netdev_priv(dev);
13191 	struct i40e_vsi *vsi = np->vsi;
13192 	struct i40e_pf *pf = vsi->back;
13193 	struct i40e_veb *veb;
13194 
13195 	/* Only for PF VSI for now */
13196 	if (vsi->type != I40E_VSI_MAIN)
13197 		return -EOPNOTSUPP;
13198 
13199 	/* Find the HW bridge for the PF VSI */
13200 	veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
13201 	if (!veb)
13202 		return 0;
13203 
13204 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
13205 				       0, 0, nlflags, filter_mask, NULL);
13206 }
13207 
13208 /**
13209  * i40e_features_check - Validate encapsulated packet conforms to limits
13210  * @skb: skb buff
13211  * @dev: This physical port's netdev
13212  * @features: Offload features that the stack believes apply
13213  **/
i40e_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)13214 static netdev_features_t i40e_features_check(struct sk_buff *skb,
13215 					     struct net_device *dev,
13216 					     netdev_features_t features)
13217 {
13218 	size_t len;
13219 
13220 	/* No point in doing any of this if neither checksum nor GSO are
13221 	 * being requested for this frame.  We can rule out both by just
13222 	 * checking for CHECKSUM_PARTIAL
13223 	 */
13224 	if (skb->ip_summed != CHECKSUM_PARTIAL)
13225 		return features;
13226 
13227 	/* We cannot support GSO if the MSS is going to be less than
13228 	 * 64 bytes.  If it is then we need to drop support for GSO.
13229 	 */
13230 	if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
13231 		features &= ~NETIF_F_GSO_MASK;
13232 
13233 	/* MACLEN can support at most 63 words */
13234 	len = skb_network_offset(skb);
13235 	if (len & ~(63 * 2))
13236 		goto out_err;
13237 
13238 	/* IPLEN and EIPLEN can support at most 127 dwords */
13239 	len = skb_network_header_len(skb);
13240 	if (len & ~(127 * 4))
13241 		goto out_err;
13242 
13243 	if (skb->encapsulation) {
13244 		/* L4TUNLEN can support 127 words */
13245 		len = skb_inner_network_header(skb) - skb_transport_header(skb);
13246 		if (len & ~(127 * 2))
13247 			goto out_err;
13248 
13249 		/* IPLEN can support at most 127 dwords */
13250 		len = skb_inner_transport_header(skb) -
13251 		      skb_inner_network_header(skb);
13252 		if (len & ~(127 * 4))
13253 			goto out_err;
13254 	}
13255 
13256 	/* No need to validate L4LEN as TCP is the only protocol with a
13257 	 * flexible value and we support all possible values supported
13258 	 * by TCP, which is at most 15 dwords
13259 	 */
13260 
13261 	return features;
13262 out_err:
13263 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13264 }
13265 
13266 /**
13267  * i40e_xdp_setup - add/remove an XDP program
13268  * @vsi: VSI to changed
13269  * @prog: XDP program
13270  * @extack: netlink extended ack
13271  **/
i40e_xdp_setup(struct i40e_vsi * vsi,struct bpf_prog * prog,struct netlink_ext_ack * extack)13272 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
13273 			  struct netlink_ext_ack *extack)
13274 {
13275 	int frame_size = i40e_max_vsi_frame_size(vsi, prog);
13276 	struct i40e_pf *pf = vsi->back;
13277 	struct bpf_prog *old_prog;
13278 	bool need_reset;
13279 	int i;
13280 
13281 	/* VSI shall be deleted in a moment, block loading new programs */
13282 	if (prog && test_bit(__I40E_IN_REMOVE, pf->state))
13283 		return -EINVAL;
13284 
13285 	/* Don't allow frames that span over multiple buffers */
13286 	if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) {
13287 		NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags");
13288 		return -EINVAL;
13289 	}
13290 
13291 	/* When turning XDP on->off/off->on we reset and rebuild the rings. */
13292 	need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
13293 	if (need_reset)
13294 		i40e_prep_for_reset(pf);
13295 
13296 	old_prog = xchg(&vsi->xdp_prog, prog);
13297 
13298 	if (need_reset) {
13299 		if (!prog) {
13300 			xdp_features_clear_redirect_target(vsi->netdev);
13301 			/* Wait until ndo_xsk_wakeup completes. */
13302 			synchronize_rcu();
13303 		}
13304 		i40e_reset_and_rebuild(pf, true, true);
13305 	}
13306 
13307 	if (!i40e_enabled_xdp_vsi(vsi) && prog) {
13308 		if (i40e_realloc_rx_bi_zc(vsi, true))
13309 			return -ENOMEM;
13310 	} else if (i40e_enabled_xdp_vsi(vsi) && !prog) {
13311 		if (i40e_realloc_rx_bi_zc(vsi, false))
13312 			return -ENOMEM;
13313 	}
13314 
13315 	for (i = 0; i < vsi->num_queue_pairs; i++)
13316 		WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
13317 
13318 	if (old_prog)
13319 		bpf_prog_put(old_prog);
13320 
13321 	/* Kick start the NAPI context if there is an AF_XDP socket open
13322 	 * on that queue id. This so that receiving will start.
13323 	 */
13324 	if (need_reset && prog) {
13325 		for (i = 0; i < vsi->num_queue_pairs; i++)
13326 			if (vsi->xdp_rings[i]->xsk_pool)
13327 				(void)i40e_xsk_wakeup(vsi->netdev, i,
13328 						      XDP_WAKEUP_RX);
13329 		xdp_features_set_redirect_target(vsi->netdev, true);
13330 	}
13331 
13332 	return 0;
13333 }
13334 
13335 /**
13336  * i40e_enter_busy_conf - Enters busy config state
13337  * @vsi: vsi
13338  *
13339  * Returns 0 on success, <0 for failure.
13340  **/
i40e_enter_busy_conf(struct i40e_vsi * vsi)13341 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
13342 {
13343 	struct i40e_pf *pf = vsi->back;
13344 	int timeout = 50;
13345 
13346 	while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
13347 		timeout--;
13348 		if (!timeout)
13349 			return -EBUSY;
13350 		usleep_range(1000, 2000);
13351 	}
13352 
13353 	return 0;
13354 }
13355 
13356 /**
13357  * i40e_exit_busy_conf - Exits busy config state
13358  * @vsi: vsi
13359  **/
i40e_exit_busy_conf(struct i40e_vsi * vsi)13360 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13361 {
13362 	struct i40e_pf *pf = vsi->back;
13363 
13364 	clear_bit(__I40E_CONFIG_BUSY, pf->state);
13365 }
13366 
13367 /**
13368  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13369  * @vsi: vsi
13370  * @queue_pair: queue pair
13371  **/
i40e_queue_pair_reset_stats(struct i40e_vsi * vsi,int queue_pair)13372 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13373 {
13374 	memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13375 	       sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13376 	memset(&vsi->tx_rings[queue_pair]->stats, 0,
13377 	       sizeof(vsi->tx_rings[queue_pair]->stats));
13378 	if (i40e_enabled_xdp_vsi(vsi)) {
13379 		memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13380 		       sizeof(vsi->xdp_rings[queue_pair]->stats));
13381 	}
13382 }
13383 
13384 /**
13385  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13386  * @vsi: vsi
13387  * @queue_pair: queue pair
13388  **/
i40e_queue_pair_clean_rings(struct i40e_vsi * vsi,int queue_pair)13389 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13390 {
13391 	i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
13392 	if (i40e_enabled_xdp_vsi(vsi)) {
13393 		/* Make sure that in-progress ndo_xdp_xmit calls are
13394 		 * completed.
13395 		 */
13396 		synchronize_rcu();
13397 		i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
13398 	}
13399 	i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13400 }
13401 
13402 /**
13403  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13404  * @vsi: vsi
13405  * @queue_pair: queue pair
13406  * @enable: true for enable, false for disable
13407  **/
i40e_queue_pair_toggle_napi(struct i40e_vsi * vsi,int queue_pair,bool enable)13408 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13409 					bool enable)
13410 {
13411 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13412 	struct i40e_q_vector *q_vector = rxr->q_vector;
13413 
13414 	if (!vsi->netdev)
13415 		return;
13416 
13417 	/* All rings in a qp belong to the same qvector. */
13418 	if (q_vector->rx.ring || q_vector->tx.ring) {
13419 		if (enable)
13420 			napi_enable(&q_vector->napi);
13421 		else
13422 			napi_disable(&q_vector->napi);
13423 	}
13424 }
13425 
13426 /**
13427  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13428  * @vsi: vsi
13429  * @queue_pair: queue pair
13430  * @enable: true for enable, false for disable
13431  *
13432  * Returns 0 on success, <0 on failure.
13433  **/
i40e_queue_pair_toggle_rings(struct i40e_vsi * vsi,int queue_pair,bool enable)13434 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13435 					bool enable)
13436 {
13437 	struct i40e_pf *pf = vsi->back;
13438 	int pf_q, ret = 0;
13439 
13440 	pf_q = vsi->base_queue + queue_pair;
13441 	ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
13442 				     false /*is xdp*/, enable);
13443 	if (ret) {
13444 		dev_info(&pf->pdev->dev,
13445 			 "VSI seid %d Tx ring %d %sable timeout\n",
13446 			 vsi->seid, pf_q, (enable ? "en" : "dis"));
13447 		return ret;
13448 	}
13449 
13450 	i40e_control_rx_q(pf, pf_q, enable);
13451 	ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13452 	if (ret) {
13453 		dev_info(&pf->pdev->dev,
13454 			 "VSI seid %d Rx ring %d %sable timeout\n",
13455 			 vsi->seid, pf_q, (enable ? "en" : "dis"));
13456 		return ret;
13457 	}
13458 
13459 	/* Due to HW errata, on Rx disable only, the register can
13460 	 * indicate done before it really is. Needs 50ms to be sure
13461 	 */
13462 	if (!enable)
13463 		mdelay(50);
13464 
13465 	if (!i40e_enabled_xdp_vsi(vsi))
13466 		return ret;
13467 
13468 	ret = i40e_control_wait_tx_q(vsi->seid, pf,
13469 				     pf_q + vsi->alloc_queue_pairs,
13470 				     true /*is xdp*/, enable);
13471 	if (ret) {
13472 		dev_info(&pf->pdev->dev,
13473 			 "VSI seid %d XDP Tx ring %d %sable timeout\n",
13474 			 vsi->seid, pf_q, (enable ? "en" : "dis"));
13475 	}
13476 
13477 	return ret;
13478 }
13479 
13480 /**
13481  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13482  * @vsi: vsi
13483  * @queue_pair: queue_pair
13484  **/
i40e_queue_pair_enable_irq(struct i40e_vsi * vsi,int queue_pair)13485 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13486 {
13487 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13488 	struct i40e_pf *pf = vsi->back;
13489 	struct i40e_hw *hw = &pf->hw;
13490 
13491 	/* All rings in a qp belong to the same qvector. */
13492 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
13493 		i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
13494 	else
13495 		i40e_irq_dynamic_enable_icr0(pf);
13496 
13497 	i40e_flush(hw);
13498 }
13499 
13500 /**
13501  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13502  * @vsi: vsi
13503  * @queue_pair: queue_pair
13504  **/
i40e_queue_pair_disable_irq(struct i40e_vsi * vsi,int queue_pair)13505 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13506 {
13507 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13508 	struct i40e_pf *pf = vsi->back;
13509 	struct i40e_hw *hw = &pf->hw;
13510 
13511 	/* For simplicity, instead of removing the qp interrupt causes
13512 	 * from the interrupt linked list, we simply disable the interrupt, and
13513 	 * leave the list intact.
13514 	 *
13515 	 * All rings in a qp belong to the same qvector.
13516 	 */
13517 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
13518 		u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13519 
13520 		wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13521 		i40e_flush(hw);
13522 		synchronize_irq(pf->msix_entries[intpf].vector);
13523 	} else {
13524 		/* Legacy and MSI mode - this stops all interrupt handling */
13525 		wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13526 		wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13527 		i40e_flush(hw);
13528 		synchronize_irq(pf->pdev->irq);
13529 	}
13530 }
13531 
13532 /**
13533  * i40e_queue_pair_disable - Disables a queue pair
13534  * @vsi: vsi
13535  * @queue_pair: queue pair
13536  *
13537  * Returns 0 on success, <0 on failure.
13538  **/
i40e_queue_pair_disable(struct i40e_vsi * vsi,int queue_pair)13539 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13540 {
13541 	int err;
13542 
13543 	err = i40e_enter_busy_conf(vsi);
13544 	if (err)
13545 		return err;
13546 
13547 	i40e_queue_pair_disable_irq(vsi, queue_pair);
13548 	i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
13549 	err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
13550 	i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13551 	i40e_queue_pair_clean_rings(vsi, queue_pair);
13552 	i40e_queue_pair_reset_stats(vsi, queue_pair);
13553 
13554 	return err;
13555 }
13556 
13557 /**
13558  * i40e_queue_pair_enable - Enables a queue pair
13559  * @vsi: vsi
13560  * @queue_pair: queue pair
13561  *
13562  * Returns 0 on success, <0 on failure.
13563  **/
i40e_queue_pair_enable(struct i40e_vsi * vsi,int queue_pair)13564 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13565 {
13566 	int err;
13567 
13568 	err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
13569 	if (err)
13570 		return err;
13571 
13572 	if (i40e_enabled_xdp_vsi(vsi)) {
13573 		err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
13574 		if (err)
13575 			return err;
13576 	}
13577 
13578 	err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
13579 	if (err)
13580 		return err;
13581 
13582 	err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
13583 	i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
13584 	i40e_queue_pair_enable_irq(vsi, queue_pair);
13585 
13586 	i40e_exit_busy_conf(vsi);
13587 
13588 	return err;
13589 }
13590 
13591 /**
13592  * i40e_xdp - implements ndo_bpf for i40e
13593  * @dev: netdevice
13594  * @xdp: XDP command
13595  **/
i40e_xdp(struct net_device * dev,struct netdev_bpf * xdp)13596 static int i40e_xdp(struct net_device *dev,
13597 		    struct netdev_bpf *xdp)
13598 {
13599 	struct i40e_netdev_priv *np = netdev_priv(dev);
13600 	struct i40e_vsi *vsi = np->vsi;
13601 
13602 	if (vsi->type != I40E_VSI_MAIN)
13603 		return -EINVAL;
13604 
13605 	switch (xdp->command) {
13606 	case XDP_SETUP_PROG:
13607 		return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);
13608 	case XDP_SETUP_XSK_POOL:
13609 		return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13610 					   xdp->xsk.queue_id);
13611 	default:
13612 		return -EINVAL;
13613 	}
13614 }
13615 
13616 static const struct net_device_ops i40e_netdev_ops = {
13617 	.ndo_open		= i40e_open,
13618 	.ndo_stop		= i40e_close,
13619 	.ndo_start_xmit		= i40e_lan_xmit_frame,
13620 	.ndo_get_stats64	= i40e_get_netdev_stats_struct,
13621 	.ndo_set_rx_mode	= i40e_set_rx_mode,
13622 	.ndo_validate_addr	= eth_validate_addr,
13623 	.ndo_set_mac_address	= i40e_set_mac,
13624 	.ndo_change_mtu		= i40e_change_mtu,
13625 	.ndo_eth_ioctl		= i40e_ioctl,
13626 	.ndo_tx_timeout		= i40e_tx_timeout,
13627 	.ndo_vlan_rx_add_vid	= i40e_vlan_rx_add_vid,
13628 	.ndo_vlan_rx_kill_vid	= i40e_vlan_rx_kill_vid,
13629 #ifdef CONFIG_NET_POLL_CONTROLLER
13630 	.ndo_poll_controller	= i40e_netpoll,
13631 #endif
13632 	.ndo_setup_tc		= __i40e_setup_tc,
13633 	.ndo_select_queue	= i40e_lan_select_queue,
13634 	.ndo_set_features	= i40e_set_features,
13635 	.ndo_set_vf_mac		= i40e_ndo_set_vf_mac,
13636 	.ndo_set_vf_vlan	= i40e_ndo_set_vf_port_vlan,
13637 	.ndo_get_vf_stats	= i40e_get_vf_stats,
13638 	.ndo_set_vf_rate	= i40e_ndo_set_vf_bw,
13639 	.ndo_get_vf_config	= i40e_ndo_get_vf_config,
13640 	.ndo_set_vf_link_state	= i40e_ndo_set_vf_link_state,
13641 	.ndo_set_vf_spoofchk	= i40e_ndo_set_vf_spoofchk,
13642 	.ndo_set_vf_trust	= i40e_ndo_set_vf_trust,
13643 	.ndo_get_phys_port_id	= i40e_get_phys_port_id,
13644 	.ndo_fdb_add		= i40e_ndo_fdb_add,
13645 	.ndo_features_check	= i40e_features_check,
13646 	.ndo_bridge_getlink	= i40e_ndo_bridge_getlink,
13647 	.ndo_bridge_setlink	= i40e_ndo_bridge_setlink,
13648 	.ndo_bpf		= i40e_xdp,
13649 	.ndo_xdp_xmit		= i40e_xdp_xmit,
13650 	.ndo_xsk_wakeup	        = i40e_xsk_wakeup,
13651 	.ndo_dfwd_add_station	= i40e_fwd_add,
13652 	.ndo_dfwd_del_station	= i40e_fwd_del,
13653 };
13654 
13655 /**
13656  * i40e_config_netdev - Setup the netdev flags
13657  * @vsi: the VSI being configured
13658  *
13659  * Returns 0 on success, negative value on failure
13660  **/
i40e_config_netdev(struct i40e_vsi * vsi)13661 static int i40e_config_netdev(struct i40e_vsi *vsi)
13662 {
13663 	struct i40e_pf *pf = vsi->back;
13664 	struct i40e_hw *hw = &pf->hw;
13665 	struct i40e_netdev_priv *np;
13666 	struct net_device *netdev;
13667 	u8 broadcast[ETH_ALEN];
13668 	u8 mac_addr[ETH_ALEN];
13669 	int etherdev_size;
13670 	netdev_features_t hw_enc_features;
13671 	netdev_features_t hw_features;
13672 
13673 	etherdev_size = sizeof(struct i40e_netdev_priv);
13674 	netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13675 	if (!netdev)
13676 		return -ENOMEM;
13677 
13678 	vsi->netdev = netdev;
13679 	np = netdev_priv(netdev);
13680 	np->vsi = vsi;
13681 
13682 	hw_enc_features = NETIF_F_SG			|
13683 			  NETIF_F_HW_CSUM		|
13684 			  NETIF_F_HIGHDMA		|
13685 			  NETIF_F_SOFT_FEATURES		|
13686 			  NETIF_F_TSO			|
13687 			  NETIF_F_TSO_ECN		|
13688 			  NETIF_F_TSO6			|
13689 			  NETIF_F_GSO_GRE		|
13690 			  NETIF_F_GSO_GRE_CSUM		|
13691 			  NETIF_F_GSO_PARTIAL		|
13692 			  NETIF_F_GSO_IPXIP4		|
13693 			  NETIF_F_GSO_IPXIP6		|
13694 			  NETIF_F_GSO_UDP_TUNNEL	|
13695 			  NETIF_F_GSO_UDP_TUNNEL_CSUM	|
13696 			  NETIF_F_GSO_UDP_L4		|
13697 			  NETIF_F_SCTP_CRC		|
13698 			  NETIF_F_RXHASH		|
13699 			  NETIF_F_RXCSUM		|
13700 			  0;
13701 
13702 	if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps))
13703 		netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13704 
13705 	netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13706 
13707 	netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13708 
13709 	netdev->hw_enc_features |= hw_enc_features;
13710 
13711 	/* record features VLANs can make use of */
13712 	netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13713 
13714 #define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE |		\
13715 				   NETIF_F_GSO_GRE_CSUM |	\
13716 				   NETIF_F_GSO_IPXIP4 |		\
13717 				   NETIF_F_GSO_IPXIP6 |		\
13718 				   NETIF_F_GSO_UDP_TUNNEL |	\
13719 				   NETIF_F_GSO_UDP_TUNNEL_CSUM)
13720 
13721 	netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES;
13722 	netdev->features |= NETIF_F_GSO_PARTIAL |
13723 			    I40E_GSO_PARTIAL_FEATURES;
13724 
13725 	netdev->mpls_features |= NETIF_F_SG;
13726 	netdev->mpls_features |= NETIF_F_HW_CSUM;
13727 	netdev->mpls_features |= NETIF_F_TSO;
13728 	netdev->mpls_features |= NETIF_F_TSO6;
13729 	netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES;
13730 
13731 	/* enable macvlan offloads */
13732 	netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13733 
13734 	hw_features = hw_enc_features		|
13735 		      NETIF_F_HW_VLAN_CTAG_TX	|
13736 		      NETIF_F_HW_VLAN_CTAG_RX;
13737 
13738 	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
13739 		hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13740 
13741 	netdev->hw_features |= hw_features | NETIF_F_LOOPBACK;
13742 
13743 	netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13744 	netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13745 
13746 	netdev->features &= ~NETIF_F_HW_TC;
13747 
13748 	if (vsi->type == I40E_VSI_MAIN) {
13749 		SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13750 		ether_addr_copy(mac_addr, hw->mac.perm_addr);
13751 		/* The following steps are necessary for two reasons. First,
13752 		 * some older NVM configurations load a default MAC-VLAN
13753 		 * filter that will accept any tagged packet, and we want to
13754 		 * replace this with a normal filter. Additionally, it is
13755 		 * possible our MAC address was provided by the platform using
13756 		 * Open Firmware or similar.
13757 		 *
13758 		 * Thus, we need to remove the default filter and install one
13759 		 * specific to the MAC address.
13760 		 */
13761 		i40e_rm_default_mac_filter(vsi, mac_addr);
13762 		spin_lock_bh(&vsi->mac_filter_hash_lock);
13763 		i40e_add_mac_filter(vsi, mac_addr);
13764 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
13765 
13766 		netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
13767 				       NETDEV_XDP_ACT_REDIRECT |
13768 				       NETDEV_XDP_ACT_XSK_ZEROCOPY |
13769 				       NETDEV_XDP_ACT_RX_SG;
13770 		netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD;
13771 	} else {
13772 		/* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13773 		 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13774 		 * the end, which is 4 bytes long, so force truncation of the
13775 		 * original name by IFNAMSIZ - 4
13776 		 */
13777 		struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
13778 
13779 		snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", IFNAMSIZ - 4,
13780 			 main_vsi->netdev->name);
13781 		eth_random_addr(mac_addr);
13782 
13783 		spin_lock_bh(&vsi->mac_filter_hash_lock);
13784 		i40e_add_mac_filter(vsi, mac_addr);
13785 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
13786 	}
13787 
13788 	/* Add the broadcast filter so that we initially will receive
13789 	 * broadcast packets. Note that when a new VLAN is first added the
13790 	 * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13791 	 * specific filters as part of transitioning into "vlan" operation.
13792 	 * When more VLANs are added, the driver will copy each existing MAC
13793 	 * filter and add it for the new VLAN.
13794 	 *
13795 	 * Broadcast filters are handled specially by
13796 	 * i40e_sync_filters_subtask, as the driver must to set the broadcast
13797 	 * promiscuous bit instead of adding this directly as a MAC/VLAN
13798 	 * filter. The subtask will update the correct broadcast promiscuous
13799 	 * bits as VLANs become active or inactive.
13800 	 */
13801 	eth_broadcast_addr(broadcast);
13802 	spin_lock_bh(&vsi->mac_filter_hash_lock);
13803 	i40e_add_mac_filter(vsi, broadcast);
13804 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
13805 
13806 	eth_hw_addr_set(netdev, mac_addr);
13807 	ether_addr_copy(netdev->perm_addr, mac_addr);
13808 
13809 	/* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13810 	netdev->neigh_priv_len = sizeof(u32) * 4;
13811 
13812 	netdev->priv_flags |= IFF_UNICAST_FLT;
13813 	netdev->priv_flags |= IFF_SUPP_NOFCS;
13814 	/* Setup netdev TC information */
13815 	i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13816 
13817 	netdev->netdev_ops = &i40e_netdev_ops;
13818 	netdev->watchdog_timeo = 5 * HZ;
13819 	i40e_set_ethtool_ops(netdev);
13820 
13821 	/* MTU range: 68 - 9706 */
13822 	netdev->min_mtu = ETH_MIN_MTU;
13823 	netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13824 
13825 	return 0;
13826 }
13827 
13828 /**
13829  * i40e_vsi_delete - Delete a VSI from the switch
13830  * @vsi: the VSI being removed
13831  *
13832  * Returns 0 on success, negative value on failure
13833  **/
i40e_vsi_delete(struct i40e_vsi * vsi)13834 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13835 {
13836 	/* remove default VSI is not allowed */
13837 	if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13838 		return;
13839 
13840 	i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13841 }
13842 
13843 /**
13844  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13845  * @vsi: the VSI being queried
13846  *
13847  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13848  **/
i40e_is_vsi_uplink_mode_veb(struct i40e_vsi * vsi)13849 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13850 {
13851 	struct i40e_veb *veb;
13852 	struct i40e_pf *pf = vsi->back;
13853 
13854 	/* Uplink is not a bridge so default to VEB */
13855 	if (vsi->veb_idx >= I40E_MAX_VEB)
13856 		return 1;
13857 
13858 	veb = pf->veb[vsi->veb_idx];
13859 	if (!veb) {
13860 		dev_info(&pf->pdev->dev,
13861 			 "There is no veb associated with the bridge\n");
13862 		return -ENOENT;
13863 	}
13864 
13865 	/* Uplink is a bridge in VEPA mode */
13866 	if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13867 		return 0;
13868 	} else {
13869 		/* Uplink is a bridge in VEB mode */
13870 		return 1;
13871 	}
13872 
13873 	/* VEPA is now default bridge, so return 0 */
13874 	return 0;
13875 }
13876 
13877 /**
13878  * i40e_add_vsi - Add a VSI to the switch
13879  * @vsi: the VSI being configured
13880  *
13881  * This initializes a VSI context depending on the VSI type to be added and
13882  * passes it down to the add_vsi aq command.
13883  **/
i40e_add_vsi(struct i40e_vsi * vsi)13884 static int i40e_add_vsi(struct i40e_vsi *vsi)
13885 {
13886 	int ret = -ENODEV;
13887 	struct i40e_pf *pf = vsi->back;
13888 	struct i40e_hw *hw = &pf->hw;
13889 	struct i40e_vsi_context ctxt;
13890 	struct i40e_mac_filter *f;
13891 	struct hlist_node *h;
13892 	int bkt;
13893 
13894 	u8 enabled_tc = 0x1; /* TC0 enabled */
13895 	int f_count = 0;
13896 
13897 	memset(&ctxt, 0, sizeof(ctxt));
13898 	switch (vsi->type) {
13899 	case I40E_VSI_MAIN:
13900 		/* The PF's main VSI is already setup as part of the
13901 		 * device initialization, so we'll not bother with
13902 		 * the add_vsi call, but we will retrieve the current
13903 		 * VSI context.
13904 		 */
13905 		ctxt.seid = pf->main_vsi_seid;
13906 		ctxt.pf_num = pf->hw.pf_id;
13907 		ctxt.vf_num = 0;
13908 		ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13909 		ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13910 		if (ret) {
13911 			dev_info(&pf->pdev->dev,
13912 				 "couldn't get PF vsi config, err %pe aq_err %s\n",
13913 				 ERR_PTR(ret),
13914 				 i40e_aq_str(&pf->hw,
13915 					     pf->hw.aq.asq_last_status));
13916 			return -ENOENT;
13917 		}
13918 		vsi->info = ctxt.info;
13919 		vsi->info.valid_sections = 0;
13920 
13921 		vsi->seid = ctxt.seid;
13922 		vsi->id = ctxt.vsi_number;
13923 
13924 		enabled_tc = i40e_pf_get_tc_map(pf);
13925 
13926 		/* Source pruning is enabled by default, so the flag is
13927 		 * negative logic - if it's set, we need to fiddle with
13928 		 * the VSI to disable source pruning.
13929 		 */
13930 		if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) {
13931 			memset(&ctxt, 0, sizeof(ctxt));
13932 			ctxt.seid = pf->main_vsi_seid;
13933 			ctxt.pf_num = pf->hw.pf_id;
13934 			ctxt.vf_num = 0;
13935 			ctxt.info.valid_sections |=
13936 				     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13937 			ctxt.info.switch_id =
13938 				   cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13939 			ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13940 			if (ret) {
13941 				dev_info(&pf->pdev->dev,
13942 					 "update vsi failed, err %d aq_err %s\n",
13943 					 ret,
13944 					 i40e_aq_str(&pf->hw,
13945 						     pf->hw.aq.asq_last_status));
13946 				ret = -ENOENT;
13947 				goto err;
13948 			}
13949 		}
13950 
13951 		/* MFP mode setup queue map and update VSI */
13952 		if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
13953 		    !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13954 			memset(&ctxt, 0, sizeof(ctxt));
13955 			ctxt.seid = pf->main_vsi_seid;
13956 			ctxt.pf_num = pf->hw.pf_id;
13957 			ctxt.vf_num = 0;
13958 			i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13959 			ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13960 			if (ret) {
13961 				dev_info(&pf->pdev->dev,
13962 					 "update vsi failed, err %pe aq_err %s\n",
13963 					 ERR_PTR(ret),
13964 					 i40e_aq_str(&pf->hw,
13965 						    pf->hw.aq.asq_last_status));
13966 				ret = -ENOENT;
13967 				goto err;
13968 			}
13969 			/* update the local VSI info queue map */
13970 			i40e_vsi_update_queue_map(vsi, &ctxt);
13971 			vsi->info.valid_sections = 0;
13972 		} else {
13973 			/* Default/Main VSI is only enabled for TC0
13974 			 * reconfigure it to enable all TCs that are
13975 			 * available on the port in SFP mode.
13976 			 * For MFP case the iSCSI PF would use this
13977 			 * flow to enable LAN+iSCSI TC.
13978 			 */
13979 			ret = i40e_vsi_config_tc(vsi, enabled_tc);
13980 			if (ret) {
13981 				/* Single TC condition is not fatal,
13982 				 * message and continue
13983 				 */
13984 				dev_info(&pf->pdev->dev,
13985 					 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n",
13986 					 enabled_tc,
13987 					 ERR_PTR(ret),
13988 					 i40e_aq_str(&pf->hw,
13989 						    pf->hw.aq.asq_last_status));
13990 			}
13991 		}
13992 		break;
13993 
13994 	case I40E_VSI_FDIR:
13995 		ctxt.pf_num = hw->pf_id;
13996 		ctxt.vf_num = 0;
13997 		ctxt.uplink_seid = vsi->uplink_seid;
13998 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13999 		ctxt.flags = I40E_AQ_VSI_TYPE_PF;
14000 		if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) &&
14001 		    (i40e_is_vsi_uplink_mode_veb(vsi))) {
14002 			ctxt.info.valid_sections |=
14003 			     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14004 			ctxt.info.switch_id =
14005 			   cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14006 		}
14007 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14008 		break;
14009 
14010 	case I40E_VSI_VMDQ2:
14011 		ctxt.pf_num = hw->pf_id;
14012 		ctxt.vf_num = 0;
14013 		ctxt.uplink_seid = vsi->uplink_seid;
14014 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14015 		ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
14016 
14017 		/* This VSI is connected to VEB so the switch_id
14018 		 * should be set to zero by default.
14019 		 */
14020 		if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14021 			ctxt.info.valid_sections |=
14022 				cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14023 			ctxt.info.switch_id =
14024 				cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14025 		}
14026 
14027 		/* Setup the VSI tx/rx queue map for TC0 only for now */
14028 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14029 		break;
14030 
14031 	case I40E_VSI_SRIOV:
14032 		ctxt.pf_num = hw->pf_id;
14033 		ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
14034 		ctxt.uplink_seid = vsi->uplink_seid;
14035 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14036 		ctxt.flags = I40E_AQ_VSI_TYPE_VF;
14037 
14038 		/* This VSI is connected to VEB so the switch_id
14039 		 * should be set to zero by default.
14040 		 */
14041 		if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14042 			ctxt.info.valid_sections |=
14043 				cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14044 			ctxt.info.switch_id =
14045 				cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14046 		}
14047 
14048 		if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
14049 			ctxt.info.valid_sections |=
14050 				cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
14051 			ctxt.info.queueing_opt_flags |=
14052 				(I40E_AQ_VSI_QUE_OPT_TCP_ENA |
14053 				 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
14054 		}
14055 
14056 		ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
14057 		ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
14058 		if (pf->vf[vsi->vf_id].spoofchk) {
14059 			ctxt.info.valid_sections |=
14060 				cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
14061 			ctxt.info.sec_flags |=
14062 				(I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
14063 				 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
14064 		}
14065 		/* Setup the VSI tx/rx queue map for TC0 only for now */
14066 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14067 		break;
14068 
14069 	case I40E_VSI_IWARP:
14070 		/* send down message to iWARP */
14071 		break;
14072 
14073 	default:
14074 		return -ENODEV;
14075 	}
14076 
14077 	if (vsi->type != I40E_VSI_MAIN) {
14078 		ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
14079 		if (ret) {
14080 			dev_info(&vsi->back->pdev->dev,
14081 				 "add vsi failed, err %pe aq_err %s\n",
14082 				 ERR_PTR(ret),
14083 				 i40e_aq_str(&pf->hw,
14084 					     pf->hw.aq.asq_last_status));
14085 			ret = -ENOENT;
14086 			goto err;
14087 		}
14088 		vsi->info = ctxt.info;
14089 		vsi->info.valid_sections = 0;
14090 		vsi->seid = ctxt.seid;
14091 		vsi->id = ctxt.vsi_number;
14092 	}
14093 
14094 	spin_lock_bh(&vsi->mac_filter_hash_lock);
14095 	vsi->active_filters = 0;
14096 	/* If macvlan filters already exist, force them to get loaded */
14097 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
14098 		f->state = I40E_FILTER_NEW;
14099 		f_count++;
14100 	}
14101 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
14102 	clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
14103 
14104 	if (f_count) {
14105 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
14106 		set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
14107 	}
14108 
14109 	/* Update VSI BW information */
14110 	ret = i40e_vsi_get_bw_info(vsi);
14111 	if (ret) {
14112 		dev_info(&pf->pdev->dev,
14113 			 "couldn't get vsi bw info, err %pe aq_err %s\n",
14114 			 ERR_PTR(ret),
14115 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14116 		/* VSI is already added so not tearing that up */
14117 		ret = 0;
14118 	}
14119 
14120 err:
14121 	return ret;
14122 }
14123 
14124 /**
14125  * i40e_vsi_release - Delete a VSI and free its resources
14126  * @vsi: the VSI being removed
14127  *
14128  * Returns 0 on success or < 0 on error
14129  **/
i40e_vsi_release(struct i40e_vsi * vsi)14130 int i40e_vsi_release(struct i40e_vsi *vsi)
14131 {
14132 	struct i40e_mac_filter *f;
14133 	struct hlist_node *h;
14134 	struct i40e_veb *veb;
14135 	struct i40e_pf *pf;
14136 	u16 uplink_seid;
14137 	int i, n, bkt;
14138 
14139 	pf = vsi->back;
14140 
14141 	/* release of a VEB-owner or last VSI is not allowed */
14142 	if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
14143 		dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
14144 			 vsi->seid, vsi->uplink_seid);
14145 		return -ENODEV;
14146 	}
14147 	if (vsi->type == I40E_VSI_MAIN && !test_bit(__I40E_DOWN, pf->state)) {
14148 		dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
14149 		return -ENODEV;
14150 	}
14151 	set_bit(__I40E_VSI_RELEASING, vsi->state);
14152 	uplink_seid = vsi->uplink_seid;
14153 
14154 	if (vsi->type != I40E_VSI_SRIOV) {
14155 		if (vsi->netdev_registered) {
14156 			vsi->netdev_registered = false;
14157 			if (vsi->netdev) {
14158 				/* results in a call to i40e_close() */
14159 				unregister_netdev(vsi->netdev);
14160 			}
14161 		} else {
14162 			i40e_vsi_close(vsi);
14163 		}
14164 		i40e_vsi_disable_irq(vsi);
14165 	}
14166 
14167 	if (vsi->type == I40E_VSI_MAIN)
14168 		i40e_devlink_destroy_port(pf);
14169 
14170 	spin_lock_bh(&vsi->mac_filter_hash_lock);
14171 
14172 	/* clear the sync flag on all filters */
14173 	if (vsi->netdev) {
14174 		__dev_uc_unsync(vsi->netdev, NULL);
14175 		__dev_mc_unsync(vsi->netdev, NULL);
14176 	}
14177 
14178 	/* make sure any remaining filters are marked for deletion */
14179 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
14180 		__i40e_del_filter(vsi, f);
14181 
14182 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
14183 
14184 	i40e_sync_vsi_filters(vsi);
14185 
14186 	i40e_vsi_delete(vsi);
14187 	i40e_vsi_free_q_vectors(vsi);
14188 	if (vsi->netdev) {
14189 		free_netdev(vsi->netdev);
14190 		vsi->netdev = NULL;
14191 	}
14192 	i40e_vsi_clear_rings(vsi);
14193 	i40e_vsi_clear(vsi);
14194 
14195 	/* If this was the last thing on the VEB, except for the
14196 	 * controlling VSI, remove the VEB, which puts the controlling
14197 	 * VSI onto the uplink port.
14198 	 *
14199 	 * Well, okay, there's one more exception here: don't remove
14200 	 * the floating VEBs yet.  We'll wait for an explicit remove request
14201 	 * from up the network stack.
14202 	 */
14203 	veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);
14204 	if (veb && veb->uplink_seid) {
14205 		n = 0;
14206 
14207 		/* Count non-controlling VSIs present on  the VEB */
14208 		i40e_pf_for_each_vsi(pf, i, vsi)
14209 			if (vsi->uplink_seid == uplink_seid &&
14210 			    (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14211 				n++;
14212 
14213 		/* If there is no VSI except the control one then release
14214 		 * the VEB and put the control VSI onto VEB uplink.
14215 		 */
14216 		if (!n)
14217 			i40e_veb_release(veb);
14218 	}
14219 
14220 	return 0;
14221 }
14222 
14223 /**
14224  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
14225  * @vsi: ptr to the VSI
14226  *
14227  * This should only be called after i40e_vsi_mem_alloc() which allocates the
14228  * corresponding SW VSI structure and initializes num_queue_pairs for the
14229  * newly allocated VSI.
14230  *
14231  * Returns 0 on success or negative on failure
14232  **/
i40e_vsi_setup_vectors(struct i40e_vsi * vsi)14233 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
14234 {
14235 	int ret = -ENOENT;
14236 	struct i40e_pf *pf = vsi->back;
14237 
14238 	if (vsi->q_vectors[0]) {
14239 		dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
14240 			 vsi->seid);
14241 		return -EEXIST;
14242 	}
14243 
14244 	if (vsi->base_vector) {
14245 		dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
14246 			 vsi->seid, vsi->base_vector);
14247 		return -EEXIST;
14248 	}
14249 
14250 	ret = i40e_vsi_alloc_q_vectors(vsi);
14251 	if (ret) {
14252 		dev_info(&pf->pdev->dev,
14253 			 "failed to allocate %d q_vector for VSI %d, ret=%d\n",
14254 			 vsi->num_q_vectors, vsi->seid, ret);
14255 		vsi->num_q_vectors = 0;
14256 		goto vector_setup_out;
14257 	}
14258 
14259 	/* In Legacy mode, we do not have to get any other vector since we
14260 	 * piggyback on the misc/ICR0 for queue interrupts.
14261 	*/
14262 	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
14263 		return ret;
14264 	if (vsi->num_q_vectors)
14265 		vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
14266 						 vsi->num_q_vectors, vsi->idx);
14267 	if (vsi->base_vector < 0) {
14268 		dev_info(&pf->pdev->dev,
14269 			 "failed to get tracking for %d vectors for VSI %d, err=%d\n",
14270 			 vsi->num_q_vectors, vsi->seid, vsi->base_vector);
14271 		i40e_vsi_free_q_vectors(vsi);
14272 		ret = -ENOENT;
14273 		goto vector_setup_out;
14274 	}
14275 
14276 vector_setup_out:
14277 	return ret;
14278 }
14279 
14280 /**
14281  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
14282  * @vsi: pointer to the vsi.
14283  *
14284  * This re-allocates a vsi's queue resources.
14285  *
14286  * Returns pointer to the successfully allocated and configured VSI sw struct
14287  * on success, otherwise returns NULL on failure.
14288  **/
i40e_vsi_reinit_setup(struct i40e_vsi * vsi)14289 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
14290 {
14291 	struct i40e_vsi *main_vsi;
14292 	u16 alloc_queue_pairs;
14293 	struct i40e_pf *pf;
14294 	int ret;
14295 
14296 	if (!vsi)
14297 		return NULL;
14298 
14299 	pf = vsi->back;
14300 
14301 	i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
14302 	i40e_vsi_clear_rings(vsi);
14303 
14304 	i40e_vsi_free_arrays(vsi, false);
14305 	i40e_set_num_rings_in_vsi(vsi);
14306 	ret = i40e_vsi_alloc_arrays(vsi, false);
14307 	if (ret)
14308 		goto err_vsi;
14309 
14310 	alloc_queue_pairs = vsi->alloc_queue_pairs *
14311 			    (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14312 
14313 	ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14314 	if (ret < 0) {
14315 		dev_info(&pf->pdev->dev,
14316 			 "failed to get tracking for %d queues for VSI %d err %d\n",
14317 			 alloc_queue_pairs, vsi->seid, ret);
14318 		goto err_vsi;
14319 	}
14320 	vsi->base_queue = ret;
14321 
14322 	/* Update the FW view of the VSI. Force a reset of TC and queue
14323 	 * layout configurations.
14324 	 */
14325 	main_vsi = i40e_pf_get_main_vsi(pf);
14326 	main_vsi->seid = pf->main_vsi_seid;
14327 	i40e_vsi_reconfig_tc(main_vsi);
14328 
14329 	if (vsi->type == I40E_VSI_MAIN)
14330 		i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
14331 
14332 	/* assign it some queues */
14333 	ret = i40e_alloc_rings(vsi);
14334 	if (ret)
14335 		goto err_rings;
14336 
14337 	/* map all of the rings to the q_vectors */
14338 	i40e_vsi_map_rings_to_vectors(vsi);
14339 	return vsi;
14340 
14341 err_rings:
14342 	i40e_vsi_free_q_vectors(vsi);
14343 	if (vsi->netdev_registered) {
14344 		vsi->netdev_registered = false;
14345 		unregister_netdev(vsi->netdev);
14346 		free_netdev(vsi->netdev);
14347 		vsi->netdev = NULL;
14348 	}
14349 	if (vsi->type == I40E_VSI_MAIN)
14350 		i40e_devlink_destroy_port(pf);
14351 	i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14352 err_vsi:
14353 	i40e_vsi_clear(vsi);
14354 	return NULL;
14355 }
14356 
14357 /**
14358  * i40e_vsi_setup - Set up a VSI by a given type
14359  * @pf: board private structure
14360  * @type: VSI type
14361  * @uplink_seid: the switch element to link to
14362  * @param1: usage depends upon VSI type. For VF types, indicates VF id
14363  *
14364  * This allocates the sw VSI structure and its queue resources, then add a VSI
14365  * to the identified VEB.
14366  *
14367  * Returns pointer to the successfully allocated and configure VSI sw struct on
14368  * success, otherwise returns NULL on failure.
14369  **/
i40e_vsi_setup(struct i40e_pf * pf,u8 type,u16 uplink_seid,u32 param1)14370 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
14371 				u16 uplink_seid, u32 param1)
14372 {
14373 	struct i40e_vsi *vsi = NULL;
14374 	struct i40e_veb *veb = NULL;
14375 	u16 alloc_queue_pairs;
14376 	int v_idx;
14377 	int ret;
14378 
14379 	/* The requested uplink_seid must be either
14380 	 *     - the PF's port seid
14381 	 *              no VEB is needed because this is the PF
14382 	 *              or this is a Flow Director special case VSI
14383 	 *     - seid of an existing VEB
14384 	 *     - seid of a VSI that owns an existing VEB
14385 	 *     - seid of a VSI that doesn't own a VEB
14386 	 *              a new VEB is created and the VSI becomes the owner
14387 	 *     - seid of the PF VSI, which is what creates the first VEB
14388 	 *              this is a special case of the previous
14389 	 *
14390 	 * Find which uplink_seid we were given and create a new VEB if needed
14391 	 */
14392 	veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);
14393 	if (!veb && uplink_seid != pf->mac_seid) {
14394 		vsi = i40e_pf_get_vsi_by_seid(pf, uplink_seid);
14395 		if (!vsi) {
14396 			dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14397 				 uplink_seid);
14398 			return NULL;
14399 		}
14400 
14401 		if (vsi->uplink_seid == pf->mac_seid)
14402 			veb = i40e_veb_setup(pf, pf->mac_seid, vsi->seid,
14403 					     vsi->tc_config.enabled_tc);
14404 		else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14405 			veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,
14406 					     vsi->tc_config.enabled_tc);
14407 		if (veb) {
14408 			if (vsi->type != I40E_VSI_MAIN) {
14409 				dev_info(&vsi->back->pdev->dev,
14410 					 "New VSI creation error, uplink seid of LAN VSI expected.\n");
14411 				return NULL;
14412 			}
14413 			/* We come up by default in VEPA mode if SRIOV is not
14414 			 * already enabled, in which case we can't force VEPA
14415 			 * mode.
14416 			 */
14417 			if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
14418 				veb->bridge_mode = BRIDGE_MODE_VEPA;
14419 				clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
14420 			}
14421 			i40e_config_bridge_mode(veb);
14422 		}
14423 		veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
14424 		if (!veb) {
14425 			dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14426 			return NULL;
14427 		}
14428 
14429 		vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14430 		uplink_seid = veb->seid;
14431 	}
14432 
14433 	/* get vsi sw struct */
14434 	v_idx = i40e_vsi_mem_alloc(pf, type);
14435 	if (v_idx < 0)
14436 		goto err_alloc;
14437 	vsi = pf->vsi[v_idx];
14438 	if (!vsi)
14439 		goto err_alloc;
14440 	vsi->type = type;
14441 	vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14442 
14443 	if (type == I40E_VSI_MAIN)
14444 		pf->lan_vsi = v_idx;
14445 	else if (type == I40E_VSI_SRIOV)
14446 		vsi->vf_id = param1;
14447 	/* assign it some queues */
14448 	alloc_queue_pairs = vsi->alloc_queue_pairs *
14449 			    (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14450 
14451 	ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14452 	if (ret < 0) {
14453 		dev_info(&pf->pdev->dev,
14454 			 "failed to get tracking for %d queues for VSI %d err=%d\n",
14455 			 alloc_queue_pairs, vsi->seid, ret);
14456 		goto err_vsi;
14457 	}
14458 	vsi->base_queue = ret;
14459 
14460 	/* get a VSI from the hardware */
14461 	vsi->uplink_seid = uplink_seid;
14462 	ret = i40e_add_vsi(vsi);
14463 	if (ret)
14464 		goto err_vsi;
14465 
14466 	switch (vsi->type) {
14467 	/* setup the netdev if needed */
14468 	case I40E_VSI_MAIN:
14469 	case I40E_VSI_VMDQ2:
14470 		ret = i40e_config_netdev(vsi);
14471 		if (ret)
14472 			goto err_netdev;
14473 		ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
14474 		if (ret)
14475 			goto err_netdev;
14476 		if (vsi->type == I40E_VSI_MAIN) {
14477 			ret = i40e_devlink_create_port(pf);
14478 			if (ret)
14479 				goto err_netdev;
14480 			SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);
14481 		}
14482 		ret = register_netdev(vsi->netdev);
14483 		if (ret)
14484 			goto err_dl_port;
14485 		vsi->netdev_registered = true;
14486 		netif_carrier_off(vsi->netdev);
14487 #ifdef CONFIG_I40E_DCB
14488 		/* Setup DCB netlink interface */
14489 		i40e_dcbnl_setup(vsi);
14490 #endif /* CONFIG_I40E_DCB */
14491 		fallthrough;
14492 	case I40E_VSI_FDIR:
14493 		/* set up vectors and rings if needed */
14494 		ret = i40e_vsi_setup_vectors(vsi);
14495 		if (ret)
14496 			goto err_msix;
14497 
14498 		ret = i40e_alloc_rings(vsi);
14499 		if (ret)
14500 			goto err_rings;
14501 
14502 		/* map all of the rings to the q_vectors */
14503 		i40e_vsi_map_rings_to_vectors(vsi);
14504 
14505 		i40e_vsi_reset_stats(vsi);
14506 		break;
14507 	default:
14508 		/* no netdev or rings for the other VSI types */
14509 		break;
14510 	}
14511 
14512 	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) &&
14513 	    vsi->type == I40E_VSI_VMDQ2) {
14514 		ret = i40e_vsi_config_rss(vsi);
14515 		if (ret)
14516 			goto err_config;
14517 	}
14518 	return vsi;
14519 
14520 err_config:
14521 	i40e_vsi_clear_rings(vsi);
14522 err_rings:
14523 	i40e_vsi_free_q_vectors(vsi);
14524 err_msix:
14525 	if (vsi->netdev_registered) {
14526 		vsi->netdev_registered = false;
14527 		unregister_netdev(vsi->netdev);
14528 		free_netdev(vsi->netdev);
14529 		vsi->netdev = NULL;
14530 	}
14531 err_dl_port:
14532 	if (vsi->type == I40E_VSI_MAIN)
14533 		i40e_devlink_destroy_port(pf);
14534 err_netdev:
14535 	i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14536 err_vsi:
14537 	i40e_vsi_clear(vsi);
14538 err_alloc:
14539 	return NULL;
14540 }
14541 
14542 /**
14543  * i40e_veb_get_bw_info - Query VEB BW information
14544  * @veb: the veb to query
14545  *
14546  * Query the Tx scheduler BW configuration data for given VEB
14547  **/
i40e_veb_get_bw_info(struct i40e_veb * veb)14548 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14549 {
14550 	struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14551 	struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14552 	struct i40e_pf *pf = veb->pf;
14553 	struct i40e_hw *hw = &pf->hw;
14554 	u32 tc_bw_max;
14555 	int ret = 0;
14556 	int i;
14557 
14558 	ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
14559 						  &bw_data, NULL);
14560 	if (ret) {
14561 		dev_info(&pf->pdev->dev,
14562 			 "query veb bw config failed, err %pe aq_err %s\n",
14563 			 ERR_PTR(ret),
14564 			 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14565 		goto out;
14566 	}
14567 
14568 	ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
14569 						   &ets_data, NULL);
14570 	if (ret) {
14571 		dev_info(&pf->pdev->dev,
14572 			 "query veb bw ets config failed, err %pe aq_err %s\n",
14573 			 ERR_PTR(ret),
14574 			 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14575 		goto out;
14576 	}
14577 
14578 	veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14579 	veb->bw_max_quanta = ets_data.tc_bw_max;
14580 	veb->is_abs_credits = bw_data.absolute_credits_enable;
14581 	veb->enabled_tc = ets_data.tc_valid_bits;
14582 	tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14583 		    (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14584 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14585 		veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14586 		veb->bw_tc_limit_credits[i] =
14587 					le16_to_cpu(bw_data.tc_bw_limits[i]);
14588 		veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14589 	}
14590 
14591 out:
14592 	return ret;
14593 }
14594 
14595 /**
14596  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14597  * @pf: board private structure
14598  *
14599  * On error: returns error code (negative)
14600  * On success: returns vsi index in PF (positive)
14601  **/
i40e_veb_mem_alloc(struct i40e_pf * pf)14602 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14603 {
14604 	int ret = -ENOENT;
14605 	struct i40e_veb *veb;
14606 	int i;
14607 
14608 	/* Need to protect the allocation of switch elements at the PF level */
14609 	mutex_lock(&pf->switch_mutex);
14610 
14611 	/* VEB list may be fragmented if VEB creation/destruction has
14612 	 * been happening.  We can afford to do a quick scan to look
14613 	 * for any free slots in the list.
14614 	 *
14615 	 * find next empty veb slot, looping back around if necessary
14616 	 */
14617 	i = 0;
14618 	while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14619 		i++;
14620 	if (i >= I40E_MAX_VEB) {
14621 		ret = -ENOMEM;
14622 		goto err_alloc_veb;  /* out of VEB slots! */
14623 	}
14624 
14625 	veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14626 	if (!veb) {
14627 		ret = -ENOMEM;
14628 		goto err_alloc_veb;
14629 	}
14630 	veb->pf = pf;
14631 	veb->idx = i;
14632 	veb->enabled_tc = 1;
14633 
14634 	pf->veb[i] = veb;
14635 	ret = i;
14636 err_alloc_veb:
14637 	mutex_unlock(&pf->switch_mutex);
14638 	return ret;
14639 }
14640 
14641 /**
14642  * i40e_switch_branch_release - Delete a branch of the switch tree
14643  * @branch: where to start deleting
14644  *
14645  * This uses recursion to find the tips of the branch to be
14646  * removed, deleting until we get back to and can delete this VEB.
14647  **/
i40e_switch_branch_release(struct i40e_veb * branch)14648 static void i40e_switch_branch_release(struct i40e_veb *branch)
14649 {
14650 	struct i40e_pf *pf = branch->pf;
14651 	u16 branch_seid = branch->seid;
14652 	u16 veb_idx = branch->idx;
14653 	struct i40e_vsi *vsi;
14654 	struct i40e_veb *veb;
14655 	int i;
14656 
14657 	/* release any VEBs on this VEB - RECURSION */
14658 	i40e_pf_for_each_veb(pf, i, veb)
14659 		if (veb->uplink_seid == branch->seid)
14660 			i40e_switch_branch_release(veb);
14661 
14662 	/* Release the VSIs on this VEB, but not the owner VSI.
14663 	 *
14664 	 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14665 	 *       the VEB itself, so don't use (*branch) after this loop.
14666 	 */
14667 	i40e_pf_for_each_vsi(pf, i, vsi)
14668 		if (vsi->uplink_seid == branch_seid &&
14669 		    (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14670 			i40e_vsi_release(vsi);
14671 
14672 	/* There's one corner case where the VEB might not have been
14673 	 * removed, so double check it here and remove it if needed.
14674 	 * This case happens if the veb was created from the debugfs
14675 	 * commands and no VSIs were added to it.
14676 	 */
14677 	if (pf->veb[veb_idx])
14678 		i40e_veb_release(pf->veb[veb_idx]);
14679 }
14680 
14681 /**
14682  * i40e_veb_clear - remove veb struct
14683  * @veb: the veb to remove
14684  **/
i40e_veb_clear(struct i40e_veb * veb)14685 static void i40e_veb_clear(struct i40e_veb *veb)
14686 {
14687 	if (!veb)
14688 		return;
14689 
14690 	if (veb->pf) {
14691 		struct i40e_pf *pf = veb->pf;
14692 
14693 		mutex_lock(&pf->switch_mutex);
14694 		if (pf->veb[veb->idx] == veb)
14695 			pf->veb[veb->idx] = NULL;
14696 		mutex_unlock(&pf->switch_mutex);
14697 	}
14698 
14699 	kfree(veb);
14700 }
14701 
14702 /**
14703  * i40e_veb_release - Delete a VEB and free its resources
14704  * @veb: the VEB being removed
14705  **/
i40e_veb_release(struct i40e_veb * veb)14706 void i40e_veb_release(struct i40e_veb *veb)
14707 {
14708 	struct i40e_vsi *vsi, *vsi_it;
14709 	struct i40e_pf *pf;
14710 	int i, n = 0;
14711 
14712 	pf = veb->pf;
14713 
14714 	/* find the remaining VSI and check for extras */
14715 	i40e_pf_for_each_vsi(pf, i, vsi_it)
14716 		if (vsi_it->uplink_seid == veb->seid) {
14717 			if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER)
14718 				vsi = vsi_it;
14719 			n++;
14720 		}
14721 
14722 	/* Floating VEB has to be empty and regular one must have
14723 	 * single owner VSI.
14724 	 */
14725 	if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) {
14726 		dev_info(&pf->pdev->dev,
14727 			 "can't remove VEB %d with %d VSIs left\n",
14728 			 veb->seid, n);
14729 		return;
14730 	}
14731 
14732 	/* For regular VEB move the owner VSI to uplink port */
14733 	if (veb->uplink_seid) {
14734 		vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14735 		vsi->uplink_seid = veb->uplink_seid;
14736 		vsi->veb_idx = I40E_NO_VEB;
14737 	}
14738 
14739 	i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14740 	i40e_veb_clear(veb);
14741 }
14742 
14743 /**
14744  * i40e_add_veb - create the VEB in the switch
14745  * @veb: the VEB to be instantiated
14746  * @vsi: the controlling VSI
14747  **/
i40e_add_veb(struct i40e_veb * veb,struct i40e_vsi * vsi)14748 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14749 {
14750 	struct i40e_pf *pf = veb->pf;
14751 	bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
14752 	int ret;
14753 
14754 	ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi ? vsi->seid : 0,
14755 			      veb->enabled_tc, vsi ? false : true,
14756 			      &veb->seid, enable_stats, NULL);
14757 
14758 	/* get a VEB from the hardware */
14759 	if (ret) {
14760 		dev_info(&pf->pdev->dev,
14761 			 "couldn't add VEB, err %pe aq_err %s\n",
14762 			 ERR_PTR(ret),
14763 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14764 		return -EPERM;
14765 	}
14766 
14767 	/* get statistics counter */
14768 	ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14769 					 &veb->stats_idx, NULL, NULL, NULL);
14770 	if (ret) {
14771 		dev_info(&pf->pdev->dev,
14772 			 "couldn't get VEB statistics idx, err %pe aq_err %s\n",
14773 			 ERR_PTR(ret),
14774 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14775 		return -EPERM;
14776 	}
14777 	ret = i40e_veb_get_bw_info(veb);
14778 	if (ret) {
14779 		dev_info(&pf->pdev->dev,
14780 			 "couldn't get VEB bw info, err %pe aq_err %s\n",
14781 			 ERR_PTR(ret),
14782 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14783 		i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14784 		return -ENOENT;
14785 	}
14786 
14787 	if (vsi) {
14788 		vsi->uplink_seid = veb->seid;
14789 		vsi->veb_idx = veb->idx;
14790 		vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14791 	}
14792 
14793 	return 0;
14794 }
14795 
14796 /**
14797  * i40e_veb_setup - Set up a VEB
14798  * @pf: board private structure
14799  * @uplink_seid: the switch element to link to
14800  * @vsi_seid: the initial VSI seid
14801  * @enabled_tc: Enabled TC bit-map
14802  *
14803  * This allocates the sw VEB structure and links it into the switch
14804  * It is possible and legal for this to be a duplicate of an already
14805  * existing VEB.  It is also possible for both uplink and vsi seids
14806  * to be zero, in order to create a floating VEB.
14807  *
14808  * Returns pointer to the successfully allocated VEB sw struct on
14809  * success, otherwise returns NULL on failure.
14810  **/
i40e_veb_setup(struct i40e_pf * pf,u16 uplink_seid,u16 vsi_seid,u8 enabled_tc)14811 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 uplink_seid,
14812 				u16 vsi_seid, u8 enabled_tc)
14813 {
14814 	struct i40e_vsi *vsi = NULL;
14815 	struct i40e_veb *veb;
14816 	int veb_idx;
14817 	int ret;
14818 
14819 	/* if one seid is 0, the other must be 0 to create a floating relay */
14820 	if ((uplink_seid == 0 || vsi_seid == 0) &&
14821 	    (uplink_seid + vsi_seid != 0)) {
14822 		dev_info(&pf->pdev->dev,
14823 			 "one, not both seid's are 0: uplink=%d vsi=%d\n",
14824 			 uplink_seid, vsi_seid);
14825 		return NULL;
14826 	}
14827 
14828 	/* make sure there is such a vsi and uplink */
14829 	if (vsi_seid) {
14830 		vsi = i40e_pf_get_vsi_by_seid(pf, vsi_seid);
14831 		if (!vsi) {
14832 			dev_err(&pf->pdev->dev, "vsi seid %d not found\n",
14833 				vsi_seid);
14834 			return NULL;
14835 		}
14836 	}
14837 
14838 	/* get veb sw struct */
14839 	veb_idx = i40e_veb_mem_alloc(pf);
14840 	if (veb_idx < 0)
14841 		goto err_alloc;
14842 	veb = pf->veb[veb_idx];
14843 	veb->uplink_seid = uplink_seid;
14844 	veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14845 
14846 	/* create the VEB in the switch */
14847 	ret = i40e_add_veb(veb, vsi);
14848 	if (ret)
14849 		goto err_veb;
14850 
14851 	if (vsi && vsi->idx == pf->lan_vsi)
14852 		pf->lan_veb = veb->idx;
14853 
14854 	return veb;
14855 
14856 err_veb:
14857 	i40e_veb_clear(veb);
14858 err_alloc:
14859 	return NULL;
14860 }
14861 
14862 /**
14863  * i40e_setup_pf_switch_element - set PF vars based on switch type
14864  * @pf: board private structure
14865  * @ele: element we are building info from
14866  * @num_reported: total number of elements
14867  * @printconfig: should we print the contents
14868  *
14869  * helper function to assist in extracting a few useful SEID values.
14870  **/
i40e_setup_pf_switch_element(struct i40e_pf * pf,struct i40e_aqc_switch_config_element_resp * ele,u16 num_reported,bool printconfig)14871 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14872 				struct i40e_aqc_switch_config_element_resp *ele,
14873 				u16 num_reported, bool printconfig)
14874 {
14875 	u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14876 	u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14877 	u8 element_type = ele->element_type;
14878 	u16 seid = le16_to_cpu(ele->seid);
14879 	struct i40e_veb *veb;
14880 
14881 	if (printconfig)
14882 		dev_info(&pf->pdev->dev,
14883 			 "type=%d seid=%d uplink=%d downlink=%d\n",
14884 			 element_type, seid, uplink_seid, downlink_seid);
14885 
14886 	switch (element_type) {
14887 	case I40E_SWITCH_ELEMENT_TYPE_MAC:
14888 		pf->mac_seid = seid;
14889 		break;
14890 	case I40E_SWITCH_ELEMENT_TYPE_VEB:
14891 		/* Main VEB? */
14892 		if (uplink_seid != pf->mac_seid)
14893 			break;
14894 		veb = i40e_pf_get_main_veb(pf);
14895 		if (!veb) {
14896 			int v;
14897 
14898 			/* find existing or else empty VEB */
14899 			veb = i40e_pf_get_veb_by_seid(pf, seid);
14900 			if (veb) {
14901 				pf->lan_veb = veb->idx;
14902 			} else {
14903 				v = i40e_veb_mem_alloc(pf);
14904 				if (v < 0)
14905 					break;
14906 				pf->lan_veb = v;
14907 			}
14908 		}
14909 
14910 		/* Try to get again main VEB as pf->lan_veb may have changed */
14911 		veb = i40e_pf_get_main_veb(pf);
14912 		if (!veb)
14913 			break;
14914 
14915 		veb->seid = seid;
14916 		veb->uplink_seid = pf->mac_seid;
14917 		veb->pf = pf;
14918 		break;
14919 	case I40E_SWITCH_ELEMENT_TYPE_VSI:
14920 		if (num_reported != 1)
14921 			break;
14922 		/* This is immediately after a reset so we can assume this is
14923 		 * the PF's VSI
14924 		 */
14925 		pf->mac_seid = uplink_seid;
14926 		pf->main_vsi_seid = seid;
14927 		if (printconfig)
14928 			dev_info(&pf->pdev->dev,
14929 				 "pf_seid=%d main_vsi_seid=%d\n",
14930 				 downlink_seid, pf->main_vsi_seid);
14931 		break;
14932 	case I40E_SWITCH_ELEMENT_TYPE_PF:
14933 	case I40E_SWITCH_ELEMENT_TYPE_VF:
14934 	case I40E_SWITCH_ELEMENT_TYPE_EMP:
14935 	case I40E_SWITCH_ELEMENT_TYPE_BMC:
14936 	case I40E_SWITCH_ELEMENT_TYPE_PE:
14937 	case I40E_SWITCH_ELEMENT_TYPE_PA:
14938 		/* ignore these for now */
14939 		break;
14940 	default:
14941 		dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14942 			 element_type, seid);
14943 		break;
14944 	}
14945 }
14946 
14947 /**
14948  * i40e_fetch_switch_configuration - Get switch config from firmware
14949  * @pf: board private structure
14950  * @printconfig: should we print the contents
14951  *
14952  * Get the current switch configuration from the device and
14953  * extract a few useful SEID values.
14954  **/
i40e_fetch_switch_configuration(struct i40e_pf * pf,bool printconfig)14955 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14956 {
14957 	struct i40e_aqc_get_switch_config_resp *sw_config;
14958 	u16 next_seid = 0;
14959 	int ret = 0;
14960 	u8 *aq_buf;
14961 	int i;
14962 
14963 	aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14964 	if (!aq_buf)
14965 		return -ENOMEM;
14966 
14967 	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14968 	do {
14969 		u16 num_reported, num_total;
14970 
14971 		ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14972 						I40E_AQ_LARGE_BUF,
14973 						&next_seid, NULL);
14974 		if (ret) {
14975 			dev_info(&pf->pdev->dev,
14976 				 "get switch config failed err %d aq_err %s\n",
14977 				 ret,
14978 				 i40e_aq_str(&pf->hw,
14979 					     pf->hw.aq.asq_last_status));
14980 			kfree(aq_buf);
14981 			return -ENOENT;
14982 		}
14983 
14984 		num_reported = le16_to_cpu(sw_config->header.num_reported);
14985 		num_total = le16_to_cpu(sw_config->header.num_total);
14986 
14987 		if (printconfig)
14988 			dev_info(&pf->pdev->dev,
14989 				 "header: %d reported %d total\n",
14990 				 num_reported, num_total);
14991 
14992 		for (i = 0; i < num_reported; i++) {
14993 			struct i40e_aqc_switch_config_element_resp *ele =
14994 				&sw_config->element[i];
14995 
14996 			i40e_setup_pf_switch_element(pf, ele, num_reported,
14997 						     printconfig);
14998 		}
14999 	} while (next_seid != 0);
15000 
15001 	kfree(aq_buf);
15002 	return ret;
15003 }
15004 
15005 /**
15006  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
15007  * @pf: board private structure
15008  * @reinit: if the Main VSI needs to re-initialized.
15009  * @lock_acquired: indicates whether or not the lock has been acquired
15010  *
15011  * Returns 0 on success, negative value on failure
15012  **/
i40e_setup_pf_switch(struct i40e_pf * pf,bool reinit,bool lock_acquired)15013 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
15014 {
15015 	struct i40e_vsi *main_vsi;
15016 	u16 flags = 0;
15017 	int ret;
15018 
15019 	/* find out what's out there already */
15020 	ret = i40e_fetch_switch_configuration(pf, false);
15021 	if (ret) {
15022 		dev_info(&pf->pdev->dev,
15023 			 "couldn't fetch switch config, err %pe aq_err %s\n",
15024 			 ERR_PTR(ret),
15025 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15026 		return ret;
15027 	}
15028 	i40e_pf_reset_stats(pf);
15029 
15030 	/* set the switch config bit for the whole device to
15031 	 * support limited promisc or true promisc
15032 	 * when user requests promisc. The default is limited
15033 	 * promisc.
15034 	*/
15035 
15036 	if ((pf->hw.pf_id == 0) &&
15037 	    !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) {
15038 		flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15039 		pf->last_sw_conf_flags = flags;
15040 	}
15041 
15042 	if (pf->hw.pf_id == 0) {
15043 		u16 valid_flags;
15044 
15045 		valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15046 		ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
15047 						NULL);
15048 		if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
15049 			dev_info(&pf->pdev->dev,
15050 				 "couldn't set switch config bits, err %pe aq_err %s\n",
15051 				 ERR_PTR(ret),
15052 				 i40e_aq_str(&pf->hw,
15053 					     pf->hw.aq.asq_last_status));
15054 			/* not a fatal problem, just keep going */
15055 		}
15056 		pf->last_sw_conf_valid_flags = valid_flags;
15057 	}
15058 
15059 	/* first time setup */
15060 	main_vsi = i40e_pf_get_main_vsi(pf);
15061 	if (!main_vsi || reinit) {
15062 		struct i40e_veb *veb;
15063 		u16 uplink_seid;
15064 
15065 		/* Set up the PF VSI associated with the PF's main VSI
15066 		 * that is already in the HW switch
15067 		 */
15068 		veb = i40e_pf_get_main_veb(pf);
15069 		if (veb)
15070 			uplink_seid = veb->seid;
15071 		else
15072 			uplink_seid = pf->mac_seid;
15073 		if (!main_vsi)
15074 			main_vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN,
15075 						  uplink_seid, 0);
15076 		else if (reinit)
15077 			main_vsi = i40e_vsi_reinit_setup(main_vsi);
15078 		if (!main_vsi) {
15079 			dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
15080 			i40e_cloud_filter_exit(pf);
15081 			i40e_fdir_teardown(pf);
15082 			return -EAGAIN;
15083 		}
15084 	} else {
15085 		/* force a reset of TC and queue layout configurations */
15086 		main_vsi->seid = pf->main_vsi_seid;
15087 		i40e_vsi_reconfig_tc(main_vsi);
15088 	}
15089 	i40e_vlan_stripping_disable(main_vsi);
15090 
15091 	i40e_fdir_sb_setup(pf);
15092 
15093 	/* Setup static PF queue filter control settings */
15094 	ret = i40e_setup_pf_filter_control(pf);
15095 	if (ret) {
15096 		dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
15097 			 ret);
15098 		/* Failure here should not stop continuing other steps */
15099 	}
15100 
15101 	/* enable RSS in the HW, even for only one queue, as the stack can use
15102 	 * the hash
15103 	 */
15104 	if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15105 		i40e_pf_config_rss(pf);
15106 
15107 	/* fill in link information and enable LSE reporting */
15108 	i40e_link_event(pf);
15109 
15110 	i40e_ptp_init(pf);
15111 
15112 	if (!lock_acquired)
15113 		rtnl_lock();
15114 
15115 	/* repopulate tunnel port filters */
15116 	udp_tunnel_nic_reset_ntf(main_vsi->netdev);
15117 
15118 	if (!lock_acquired)
15119 		rtnl_unlock();
15120 
15121 	return ret;
15122 }
15123 
15124 /**
15125  * i40e_determine_queue_usage - Work out queue distribution
15126  * @pf: board private structure
15127  **/
i40e_determine_queue_usage(struct i40e_pf * pf)15128 static void i40e_determine_queue_usage(struct i40e_pf *pf)
15129 {
15130 	int queues_left;
15131 	int q_max;
15132 
15133 	pf->num_lan_qps = 0;
15134 
15135 	/* Find the max queues to be put into basic use.  We'll always be
15136 	 * using TC0, whether or not DCB is running, and TC0 will get the
15137 	 * big RSS set.
15138 	 */
15139 	queues_left = pf->hw.func_caps.num_tx_qp;
15140 
15141 	if ((queues_left == 1) ||
15142 	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
15143 		/* one qp for PF, no queues for anything else */
15144 		queues_left = 0;
15145 		pf->alloc_rss_size = pf->num_lan_qps = 1;
15146 
15147 		/* make sure all the fancies are disabled */
15148 		clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
15149 		clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
15150 		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15151 		clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
15152 		clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15153 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15154 		clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
15155 		clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
15156 		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15157 	} else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) &&
15158 		   !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
15159 		   !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
15160 		   !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) {
15161 		/* one qp for PF */
15162 		pf->alloc_rss_size = pf->num_lan_qps = 1;
15163 		queues_left -= pf->num_lan_qps;
15164 
15165 		clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
15166 		clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
15167 		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15168 		clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
15169 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15170 		clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
15171 		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15172 	} else {
15173 		/* Not enough queues for all TCs */
15174 		if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) &&
15175 		    queues_left < I40E_MAX_TRAFFIC_CLASS) {
15176 			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15177 			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15178 			dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
15179 		}
15180 
15181 		/* limit lan qps to the smaller of qps, cpus or msix */
15182 		q_max = max_t(int, pf->rss_size_max, num_online_cpus());
15183 		q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
15184 		q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
15185 		pf->num_lan_qps = q_max;
15186 
15187 		queues_left -= pf->num_lan_qps;
15188 	}
15189 
15190 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15191 		if (queues_left > 1) {
15192 			queues_left -= 1; /* save 1 queue for FD */
15193 		} else {
15194 			clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15195 			set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15196 			dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
15197 		}
15198 	}
15199 
15200 	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15201 	    pf->num_vf_qps && pf->num_req_vfs && queues_left) {
15202 		pf->num_req_vfs = min_t(int, pf->num_req_vfs,
15203 					(queues_left / pf->num_vf_qps));
15204 		queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
15205 	}
15206 
15207 	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) &&
15208 	    pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
15209 		pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
15210 					  (queues_left / pf->num_vmdq_qps));
15211 		queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
15212 	}
15213 
15214 	pf->queues_left = queues_left;
15215 	dev_dbg(&pf->pdev->dev,
15216 		"qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
15217 		pf->hw.func_caps.num_tx_qp,
15218 		!!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags),
15219 		pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
15220 		pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
15221 		queues_left);
15222 }
15223 
15224 /**
15225  * i40e_setup_pf_filter_control - Setup PF static filter control
15226  * @pf: PF to be setup
15227  *
15228  * i40e_setup_pf_filter_control sets up a PF's initial filter control
15229  * settings. If PE/FCoE are enabled then it will also set the per PF
15230  * based filter sizes required for them. It also enables Flow director,
15231  * ethertype and macvlan type filter settings for the pf.
15232  *
15233  * Returns 0 on success, negative on failure
15234  **/
i40e_setup_pf_filter_control(struct i40e_pf * pf)15235 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
15236 {
15237 	struct i40e_filter_control_settings *settings = &pf->filter_settings;
15238 
15239 	settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
15240 
15241 	/* Flow Director is enabled */
15242 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ||
15243 	    test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15244 		settings->enable_fdir = true;
15245 
15246 	/* Ethtype and MACVLAN filters enabled for PF */
15247 	settings->enable_ethtype = true;
15248 	settings->enable_macvlan = true;
15249 
15250 	if (i40e_set_filter_control(&pf->hw, settings))
15251 		return -ENOENT;
15252 
15253 	return 0;
15254 }
15255 
15256 #define INFO_STRING_LEN 255
15257 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
i40e_print_features(struct i40e_pf * pf)15258 static void i40e_print_features(struct i40e_pf *pf)
15259 {
15260 	struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
15261 	struct i40e_hw *hw = &pf->hw;
15262 	char *buf;
15263 	int i;
15264 
15265 	buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
15266 	if (!buf)
15267 		return;
15268 
15269 	i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
15270 #ifdef CONFIG_PCI_IOV
15271 	i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
15272 #endif
15273 	i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
15274 		       pf->hw.func_caps.num_vsis, main_vsi->num_queue_pairs);
15275 	if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15276 		i += scnprintf(&buf[i], REMAIN(i), " RSS");
15277 	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15278 		i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
15279 	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15280 		i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
15281 		i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
15282 	}
15283 	if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
15284 		i += scnprintf(&buf[i], REMAIN(i), " DCB");
15285 	i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
15286 	i += scnprintf(&buf[i], REMAIN(i), " Geneve");
15287 	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
15288 		i += scnprintf(&buf[i], REMAIN(i), " PTP");
15289 	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
15290 		i += scnprintf(&buf[i], REMAIN(i), " VEB");
15291 	else
15292 		i += scnprintf(&buf[i], REMAIN(i), " VEPA");
15293 
15294 	dev_info(&pf->pdev->dev, "%s\n", buf);
15295 	kfree(buf);
15296 	WARN_ON(i > INFO_STRING_LEN);
15297 }
15298 
15299 /**
15300  * i40e_get_platform_mac_addr - get platform-specific MAC address
15301  * @pdev: PCI device information struct
15302  * @pf: board private structure
15303  *
15304  * Look up the MAC address for the device. First we'll try
15305  * eth_platform_get_mac_address, which will check Open Firmware, or arch
15306  * specific fallback. Otherwise, we'll default to the stored value in
15307  * firmware.
15308  **/
i40e_get_platform_mac_addr(struct pci_dev * pdev,struct i40e_pf * pf)15309 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
15310 {
15311 	if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
15312 		i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
15313 }
15314 
15315 /**
15316  * i40e_set_fec_in_flags - helper function for setting FEC options in flags
15317  * @fec_cfg: FEC option to set in flags
15318  * @flags: ptr to flags in which we set FEC option
15319  **/
i40e_set_fec_in_flags(u8 fec_cfg,unsigned long * flags)15320 void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags)
15321 {
15322 	if (fec_cfg & I40E_AQ_SET_FEC_AUTO) {
15323 		set_bit(I40E_FLAG_RS_FEC, flags);
15324 		set_bit(I40E_FLAG_BASE_R_FEC, flags);
15325 	}
15326 	if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
15327 	    (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
15328 		set_bit(I40E_FLAG_RS_FEC, flags);
15329 		clear_bit(I40E_FLAG_BASE_R_FEC, flags);
15330 	}
15331 	if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
15332 	    (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
15333 		set_bit(I40E_FLAG_BASE_R_FEC, flags);
15334 		clear_bit(I40E_FLAG_RS_FEC, flags);
15335 	}
15336 	if (fec_cfg == 0) {
15337 		clear_bit(I40E_FLAG_RS_FEC, flags);
15338 		clear_bit(I40E_FLAG_BASE_R_FEC, flags);
15339 	}
15340 }
15341 
15342 /**
15343  * i40e_check_recovery_mode - check if we are running transition firmware
15344  * @pf: board private structure
15345  *
15346  * Check registers indicating the firmware runs in recovery mode. Sets the
15347  * appropriate driver state.
15348  *
15349  * Returns true if the recovery mode was detected, false otherwise
15350  **/
i40e_check_recovery_mode(struct i40e_pf * pf)15351 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
15352 {
15353 	u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
15354 
15355 	if (val & I40E_GL_FWSTS_FWS1B_MASK) {
15356 		dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
15357 		dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
15358 		set_bit(__I40E_RECOVERY_MODE, pf->state);
15359 
15360 		return true;
15361 	}
15362 	if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15363 		dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
15364 
15365 	return false;
15366 }
15367 
15368 /**
15369  * i40e_pf_loop_reset - perform reset in a loop.
15370  * @pf: board private structure
15371  *
15372  * This function is useful when a NIC is about to enter recovery mode.
15373  * When a NIC's internal data structures are corrupted the NIC's
15374  * firmware is going to enter recovery mode.
15375  * Right after a POR it takes about 7 minutes for firmware to enter
15376  * recovery mode. Until that time a NIC is in some kind of intermediate
15377  * state. After that time period the NIC almost surely enters
15378  * recovery mode. The only way for a driver to detect intermediate
15379  * state is to issue a series of pf-resets and check a return value.
15380  * If a PF reset returns success then the firmware could be in recovery
15381  * mode so the caller of this code needs to check for recovery mode
15382  * if this function returns success. There is a little chance that
15383  * firmware will hang in intermediate state forever.
15384  * Since waiting 7 minutes is quite a lot of time this function waits
15385  * 10 seconds and then gives up by returning an error.
15386  *
15387  * Return 0 on success, negative on failure.
15388  **/
i40e_pf_loop_reset(struct i40e_pf * pf)15389 static int i40e_pf_loop_reset(struct i40e_pf *pf)
15390 {
15391 	/* wait max 10 seconds for PF reset to succeed */
15392 	const unsigned long time_end = jiffies + 10 * HZ;
15393 	struct i40e_hw *hw = &pf->hw;
15394 	int ret;
15395 
15396 	ret = i40e_pf_reset(hw);
15397 	while (ret != 0 && time_before(jiffies, time_end)) {
15398 		usleep_range(10000, 20000);
15399 		ret = i40e_pf_reset(hw);
15400 	}
15401 
15402 	if (ret == 0)
15403 		pf->pfr_count++;
15404 	else
15405 		dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15406 
15407 	return ret;
15408 }
15409 
15410 /**
15411  * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15412  * @pf: board private structure
15413  *
15414  * Check FW registers to determine if FW issued unexpected EMP Reset.
15415  * Every time when unexpected EMP Reset occurs the FW increments
15416  * a counter of unexpected EMP Resets. When the counter reaches 10
15417  * the FW should enter the Recovery mode
15418  *
15419  * Returns true if FW issued unexpected EMP Reset
15420  **/
i40e_check_fw_empr(struct i40e_pf * pf)15421 static bool i40e_check_fw_empr(struct i40e_pf *pf)
15422 {
15423 	const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15424 			   I40E_GL_FWSTS_FWS1B_MASK;
15425 	return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15426 	       (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15427 }
15428 
15429 /**
15430  * i40e_handle_resets - handle EMP resets and PF resets
15431  * @pf: board private structure
15432  *
15433  * Handle both EMP resets and PF resets and conclude whether there are
15434  * any issues regarding these resets. If there are any issues then
15435  * generate log entry.
15436  *
15437  * Return 0 if NIC is healthy or negative value when there are issues
15438  * with resets
15439  **/
i40e_handle_resets(struct i40e_pf * pf)15440 static int i40e_handle_resets(struct i40e_pf *pf)
15441 {
15442 	const int pfr = i40e_pf_loop_reset(pf);
15443 	const bool is_empr = i40e_check_fw_empr(pf);
15444 
15445 	if (is_empr || pfr != 0)
15446 		dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15447 
15448 	return is_empr ? -EIO : pfr;
15449 }
15450 
15451 /**
15452  * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15453  * @pf: board private structure
15454  * @hw: ptr to the hardware info
15455  *
15456  * This function does a minimal setup of all subsystems needed for running
15457  * recovery mode.
15458  *
15459  * Returns 0 on success, negative on failure
15460  **/
i40e_init_recovery_mode(struct i40e_pf * pf,struct i40e_hw * hw)15461 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15462 {
15463 	struct i40e_vsi *vsi;
15464 	int err;
15465 	int v_idx;
15466 
15467 	pci_set_drvdata(pf->pdev, pf);
15468 	pci_save_state(pf->pdev);
15469 
15470 	/* set up periodic task facility */
15471 	timer_setup(&pf->service_timer, i40e_service_timer, 0);
15472 	pf->service_timer_period = HZ;
15473 
15474 	INIT_WORK(&pf->service_task, i40e_service_task);
15475 	clear_bit(__I40E_SERVICE_SCHED, pf->state);
15476 
15477 	err = i40e_init_interrupt_scheme(pf);
15478 	if (err)
15479 		goto err_switch_setup;
15480 
15481 	/* The number of VSIs reported by the FW is the minimum guaranteed
15482 	 * to us; HW supports far more and we share the remaining pool with
15483 	 * the other PFs. We allocate space for more than the guarantee with
15484 	 * the understanding that we might not get them all later.
15485 	 */
15486 	if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15487 		pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15488 	else
15489 		pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15490 
15491 	/* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15492 	pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15493 			  GFP_KERNEL);
15494 	if (!pf->vsi) {
15495 		err = -ENOMEM;
15496 		goto err_switch_setup;
15497 	}
15498 
15499 	/* We allocate one VSI which is needed as absolute minimum
15500 	 * in order to register the netdev
15501 	 */
15502 	v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
15503 	if (v_idx < 0) {
15504 		err = v_idx;
15505 		goto err_switch_setup;
15506 	}
15507 	pf->lan_vsi = v_idx;
15508 	vsi = pf->vsi[v_idx];
15509 	if (!vsi) {
15510 		err = -EFAULT;
15511 		goto err_switch_setup;
15512 	}
15513 	vsi->alloc_queue_pairs = 1;
15514 	err = i40e_config_netdev(vsi);
15515 	if (err)
15516 		goto err_switch_setup;
15517 	err = register_netdev(vsi->netdev);
15518 	if (err)
15519 		goto err_switch_setup;
15520 	vsi->netdev_registered = true;
15521 	i40e_dbg_pf_init(pf);
15522 
15523 	err = i40e_setup_misc_vector_for_recovery_mode(pf);
15524 	if (err)
15525 		goto err_switch_setup;
15526 
15527 	/* tell the firmware that we're starting */
15528 	i40e_send_version(pf);
15529 
15530 	/* since everything's happy, start the service_task timer */
15531 	mod_timer(&pf->service_timer,
15532 		  round_jiffies(jiffies + pf->service_timer_period));
15533 
15534 	return 0;
15535 
15536 err_switch_setup:
15537 	i40e_reset_interrupt_capability(pf);
15538 	timer_shutdown_sync(&pf->service_timer);
15539 	i40e_shutdown_adminq(hw);
15540 	iounmap(hw->hw_addr);
15541 	pci_release_mem_regions(pf->pdev);
15542 	pci_disable_device(pf->pdev);
15543 	i40e_free_pf(pf);
15544 
15545 	return err;
15546 }
15547 
15548 /**
15549  * i40e_set_subsystem_device_id - set subsystem device id
15550  * @hw: pointer to the hardware info
15551  *
15552  * Set PCI subsystem device id either from a pci_dev structure or
15553  * a specific FW register.
15554  **/
i40e_set_subsystem_device_id(struct i40e_hw * hw)15555 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)
15556 {
15557 	struct i40e_pf *pf = i40e_hw_to_pf(hw);
15558 
15559 	hw->subsystem_device_id = pf->pdev->subsystem_device ?
15560 		pf->pdev->subsystem_device :
15561 		(ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);
15562 }
15563 
15564 /**
15565  * i40e_probe - Device initialization routine
15566  * @pdev: PCI device information struct
15567  * @ent: entry in i40e_pci_tbl
15568  *
15569  * i40e_probe initializes a PF identified by a pci_dev structure.
15570  * The OS initialization, configuring of the PF private structure,
15571  * and a hardware reset occur.
15572  *
15573  * Returns 0 on success, negative on failure
15574  **/
i40e_probe(struct pci_dev * pdev,const struct pci_device_id * ent)15575 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15576 {
15577 	struct i40e_aq_get_phy_abilities_resp abilities;
15578 #ifdef CONFIG_I40E_DCB
15579 	enum i40e_get_fw_lldp_status_resp lldp_status;
15580 #endif /* CONFIG_I40E_DCB */
15581 	struct i40e_vsi *vsi;
15582 	struct i40e_pf *pf;
15583 	struct i40e_hw *hw;
15584 	u16 wol_nvm_bits;
15585 	char nvm_ver[32];
15586 	u16 link_status;
15587 #ifdef CONFIG_I40E_DCB
15588 	int status;
15589 #endif /* CONFIG_I40E_DCB */
15590 	int err;
15591 	u32 val;
15592 
15593 	err = pci_enable_device_mem(pdev);
15594 	if (err)
15595 		return err;
15596 
15597 	/* set up for high or low dma */
15598 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
15599 	if (err) {
15600 		dev_err(&pdev->dev,
15601 			"DMA configuration failed: 0x%x\n", err);
15602 		goto err_dma;
15603 	}
15604 
15605 	/* set up pci connections */
15606 	err = pci_request_mem_regions(pdev, i40e_driver_name);
15607 	if (err) {
15608 		dev_info(&pdev->dev,
15609 			 "pci_request_selected_regions failed %d\n", err);
15610 		goto err_pci_reg;
15611 	}
15612 
15613 	pci_set_master(pdev);
15614 
15615 	/* Now that we have a PCI connection, we need to do the
15616 	 * low level device setup.  This is primarily setting up
15617 	 * the Admin Queue structures and then querying for the
15618 	 * device's current profile information.
15619 	 */
15620 	pf = i40e_alloc_pf(&pdev->dev);
15621 	if (!pf) {
15622 		err = -ENOMEM;
15623 		goto err_pf_alloc;
15624 	}
15625 	pf->next_vsi = 0;
15626 	pf->pdev = pdev;
15627 	set_bit(__I40E_DOWN, pf->state);
15628 
15629 	hw = &pf->hw;
15630 
15631 	pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15632 				I40E_MAX_CSR_SPACE);
15633 	/* We believe that the highest register to read is
15634 	 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15635 	 * is not less than that before mapping to prevent a
15636 	 * kernel panic.
15637 	 */
15638 	if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15639 		dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15640 			pf->ioremap_len);
15641 		err = -ENOMEM;
15642 		goto err_ioremap;
15643 	}
15644 	hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15645 	if (!hw->hw_addr) {
15646 		err = -EIO;
15647 		dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15648 			 (unsigned int)pci_resource_start(pdev, 0),
15649 			 pf->ioremap_len, err);
15650 		goto err_ioremap;
15651 	}
15652 	hw->vendor_id = pdev->vendor;
15653 	hw->device_id = pdev->device;
15654 	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15655 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
15656 	i40e_set_subsystem_device_id(hw);
15657 	hw->bus.device = PCI_SLOT(pdev->devfn);
15658 	hw->bus.func = PCI_FUNC(pdev->devfn);
15659 	hw->bus.bus_id = pdev->bus->number;
15660 
15661 	/* Select something other than the 802.1ad ethertype for the
15662 	 * switch to use internally and drop on ingress.
15663 	 */
15664 	hw->switch_tag = 0xffff;
15665 	hw->first_tag = ETH_P_8021AD;
15666 	hw->second_tag = ETH_P_8021Q;
15667 
15668 	INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15669 	INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15670 	INIT_LIST_HEAD(&pf->ddp_old_prof);
15671 
15672 	/* set up the locks for the AQ, do this only once in probe
15673 	 * and destroy them only once in remove
15674 	 */
15675 	mutex_init(&hw->aq.asq_mutex);
15676 	mutex_init(&hw->aq.arq_mutex);
15677 
15678 	pf->msg_enable = netif_msg_init(debug,
15679 					NETIF_MSG_DRV |
15680 					NETIF_MSG_PROBE |
15681 					NETIF_MSG_LINK);
15682 	if (debug < -1)
15683 		pf->hw.debug_mask = debug;
15684 
15685 	/* do a special CORER for clearing PXE mode once at init */
15686 	if (hw->revision_id == 0 &&
15687 	    (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15688 		wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15689 		i40e_flush(hw);
15690 		msleep(200);
15691 		pf->corer_count++;
15692 
15693 		i40e_clear_pxe_mode(hw);
15694 	}
15695 
15696 	/* Reset here to make sure all is clean and to define PF 'n' */
15697 	i40e_clear_hw(hw);
15698 
15699 	err = i40e_set_mac_type(hw);
15700 	if (err) {
15701 		dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15702 			 err);
15703 		goto err_pf_reset;
15704 	}
15705 
15706 	err = i40e_handle_resets(pf);
15707 	if (err)
15708 		goto err_pf_reset;
15709 
15710 	i40e_check_recovery_mode(pf);
15711 
15712 	if (is_kdump_kernel()) {
15713 		hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;
15714 		hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;
15715 	} else {
15716 		hw->aq.num_arq_entries = I40E_AQ_LEN;
15717 		hw->aq.num_asq_entries = I40E_AQ_LEN;
15718 	}
15719 	hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15720 	hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15721 
15722 	snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15723 		 "%s-%s:misc",
15724 		 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15725 
15726 	err = i40e_init_shared_code(hw);
15727 	if (err) {
15728 		dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15729 			 err);
15730 		goto err_pf_reset;
15731 	}
15732 
15733 	/* set up a default setting for link flow control */
15734 	pf->hw.fc.requested_mode = I40E_FC_NONE;
15735 
15736 	err = i40e_init_adminq(hw);
15737 	if (err) {
15738 		if (err == -EIO)
15739 			dev_info(&pdev->dev,
15740 				 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15741 				 hw->aq.api_maj_ver,
15742 				 hw->aq.api_min_ver,
15743 				 I40E_FW_API_VERSION_MAJOR,
15744 				 I40E_FW_MINOR_VERSION(hw));
15745 		else
15746 			dev_info(&pdev->dev,
15747 				 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15748 
15749 		goto err_pf_reset;
15750 	}
15751 	i40e_get_oem_version(hw);
15752 	i40e_get_pba_string(hw);
15753 
15754 	/* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15755 	i40e_nvm_version_str(hw, nvm_ver, sizeof(nvm_ver));
15756 	dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15757 		 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15758 		 hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver,
15759 		 hw->vendor_id, hw->device_id, hw->subsystem_vendor_id,
15760 		 hw->subsystem_device_id);
15761 
15762 	if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR,
15763 				  I40E_FW_MINOR_VERSION(hw) + 1))
15764 		dev_dbg(&pdev->dev,
15765 			"The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
15766 			 hw->aq.api_maj_ver,
15767 			 hw->aq.api_min_ver,
15768 			 I40E_FW_API_VERSION_MAJOR,
15769 			 I40E_FW_MINOR_VERSION(hw));
15770 	else if (i40e_is_aq_api_ver_lt(hw, 1, 4))
15771 		dev_info(&pdev->dev,
15772 			 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15773 			 hw->aq.api_maj_ver,
15774 			 hw->aq.api_min_ver,
15775 			 I40E_FW_API_VERSION_MAJOR,
15776 			 I40E_FW_MINOR_VERSION(hw));
15777 
15778 	i40e_verify_eeprom(pf);
15779 
15780 	/* Rev 0 hardware was never productized */
15781 	if (hw->revision_id < 1)
15782 		dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15783 
15784 	i40e_clear_pxe_mode(hw);
15785 
15786 	err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15787 	if (err)
15788 		goto err_adminq_setup;
15789 
15790 	err = i40e_sw_init(pf);
15791 	if (err) {
15792 		dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15793 		goto err_sw_init;
15794 	}
15795 
15796 	if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15797 		return i40e_init_recovery_mode(pf, hw);
15798 
15799 	err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15800 				hw->func_caps.num_rx_qp, 0, 0);
15801 	if (err) {
15802 		dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15803 		goto err_init_lan_hmc;
15804 	}
15805 
15806 	err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15807 	if (err) {
15808 		dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15809 		err = -ENOENT;
15810 		goto err_configure_lan_hmc;
15811 	}
15812 
15813 	/* Disable LLDP for NICs that have firmware versions lower than v4.3.
15814 	 * Ignore error return codes because if it was already disabled via
15815 	 * hardware settings this will fail
15816 	 */
15817 	if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) {
15818 		dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15819 		i40e_aq_stop_lldp(hw, true, false, NULL);
15820 	}
15821 
15822 	/* allow a platform config to override the HW addr */
15823 	i40e_get_platform_mac_addr(pdev, pf);
15824 
15825 	if (!is_valid_ether_addr(hw->mac.addr)) {
15826 		dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15827 		err = -EIO;
15828 		goto err_mac_addr;
15829 	}
15830 	dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15831 	ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15832 	i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15833 	if (is_valid_ether_addr(hw->mac.port_addr))
15834 		set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps);
15835 
15836 	i40e_ptp_alloc_pins(pf);
15837 	pci_set_drvdata(pdev, pf);
15838 	pci_save_state(pdev);
15839 
15840 #ifdef CONFIG_I40E_DCB
15841 	status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
15842 	(!status &&
15843 	 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15844 		(clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) :
15845 		(set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags));
15846 	dev_info(&pdev->dev,
15847 		 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
15848 			"FW LLDP is disabled\n" :
15849 			"FW LLDP is enabled\n");
15850 
15851 	/* Enable FW to write default DCB config on link-up */
15852 	i40e_aq_set_dcb_parameters(hw, true, NULL);
15853 
15854 	err = i40e_init_pf_dcb(pf);
15855 	if (err) {
15856 		dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15857 		clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15858 		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15859 		/* Continue without DCB enabled */
15860 	}
15861 #endif /* CONFIG_I40E_DCB */
15862 
15863 	/* set up periodic task facility */
15864 	timer_setup(&pf->service_timer, i40e_service_timer, 0);
15865 	pf->service_timer_period = HZ;
15866 
15867 	INIT_WORK(&pf->service_task, i40e_service_task);
15868 	clear_bit(__I40E_SERVICE_SCHED, pf->state);
15869 
15870 	/* NVM bit on means WoL disabled for the port */
15871 	i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15872 	if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15873 		pf->wol_en = false;
15874 	else
15875 		pf->wol_en = true;
15876 	device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15877 
15878 	/* set up the main switch operations */
15879 	i40e_determine_queue_usage(pf);
15880 	err = i40e_init_interrupt_scheme(pf);
15881 	if (err)
15882 		goto err_switch_setup;
15883 
15884 	/* Reduce Tx and Rx pairs for kdump
15885 	 * When MSI-X is enabled, it's not allowed to use more TC queue
15886 	 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus
15887 	 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.
15888 	 */
15889 	if (is_kdump_kernel())
15890 		pf->num_lan_msix = 1;
15891 
15892 	pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15893 	pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15894 	pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15895 	pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15896 	pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15897 	pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15898 						    UDP_TUNNEL_TYPE_GENEVE;
15899 
15900 	/* The number of VSIs reported by the FW is the minimum guaranteed
15901 	 * to us; HW supports far more and we share the remaining pool with
15902 	 * the other PFs. We allocate space for more than the guarantee with
15903 	 * the understanding that we might not get them all later.
15904 	 */
15905 	if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15906 		pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15907 	else
15908 		pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15909 	if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15910 		dev_warn(&pf->pdev->dev,
15911 			 "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15912 			 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15913 		pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15914 	}
15915 
15916 	/* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15917 	pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15918 			  GFP_KERNEL);
15919 	if (!pf->vsi) {
15920 		err = -ENOMEM;
15921 		goto err_switch_setup;
15922 	}
15923 
15924 #ifdef CONFIG_PCI_IOV
15925 	/* prep for VF support */
15926 	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15927 	    test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
15928 	    !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15929 		if (pci_num_vf(pdev))
15930 			set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
15931 	}
15932 #endif
15933 	err = i40e_setup_pf_switch(pf, false, false);
15934 	if (err) {
15935 		dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15936 		goto err_vsis;
15937 	}
15938 
15939 	vsi = i40e_pf_get_main_vsi(pf);
15940 	INIT_LIST_HEAD(&vsi->ch_list);
15941 
15942 	/* if FDIR VSI was set up, start it now */
15943 	vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
15944 	if (vsi)
15945 		i40e_vsi_open(vsi);
15946 
15947 	/* The driver only wants link up/down and module qualification
15948 	 * reports from firmware.  Note the negative logic.
15949 	 */
15950 	err = i40e_aq_set_phy_int_mask(&pf->hw,
15951 				       ~(I40E_AQ_EVENT_LINK_UPDOWN |
15952 					 I40E_AQ_EVENT_MEDIA_NA |
15953 					 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15954 	if (err)
15955 		dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
15956 			 ERR_PTR(err),
15957 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15958 
15959 	/* VF MDD event logs are rate limited to one second intervals */
15960 	ratelimit_state_init(&pf->mdd_message_rate_limit, 1 * HZ, 1);
15961 
15962 	/* Reconfigure hardware for allowing smaller MSS in the case
15963 	 * of TSO, so that we avoid the MDD being fired and causing
15964 	 * a reset in the case of small MSS+TSO.
15965 	 */
15966 	val = rd32(hw, I40E_REG_MSS);
15967 	if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15968 		val &= ~I40E_REG_MSS_MIN_MASK;
15969 		val |= I40E_64BYTE_MSS;
15970 		wr32(hw, I40E_REG_MSS, val);
15971 	}
15972 
15973 	if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
15974 		msleep(75);
15975 		err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15976 		if (err)
15977 			dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
15978 				 ERR_PTR(err),
15979 				 i40e_aq_str(&pf->hw,
15980 					     pf->hw.aq.asq_last_status));
15981 	}
15982 	/* The main driver is (mostly) up and happy. We need to set this state
15983 	 * before setting up the misc vector or we get a race and the vector
15984 	 * ends up disabled forever.
15985 	 */
15986 	clear_bit(__I40E_DOWN, pf->state);
15987 
15988 	/* In case of MSIX we are going to setup the misc vector right here
15989 	 * to handle admin queue events etc. In case of legacy and MSI
15990 	 * the misc functionality and queue processing is combined in
15991 	 * the same vector and that gets setup at open.
15992 	 */
15993 	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
15994 		err = i40e_setup_misc_vector(pf);
15995 		if (err) {
15996 			dev_info(&pdev->dev,
15997 				 "setup of misc vector failed: %d\n", err);
15998 			i40e_cloud_filter_exit(pf);
15999 			i40e_fdir_teardown(pf);
16000 			goto err_vsis;
16001 		}
16002 	}
16003 
16004 #ifdef CONFIG_PCI_IOV
16005 	/* prep for VF support */
16006 	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
16007 	    test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
16008 	    !test_bit(__I40E_BAD_EEPROM, pf->state)) {
16009 		/* disable link interrupts for VFs */
16010 		val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
16011 		val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
16012 		wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
16013 		i40e_flush(hw);
16014 
16015 		if (pci_num_vf(pdev)) {
16016 			dev_info(&pdev->dev,
16017 				 "Active VFs found, allocating resources.\n");
16018 			err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
16019 			if (err)
16020 				dev_info(&pdev->dev,
16021 					 "Error %d allocating resources for existing VFs\n",
16022 					 err);
16023 		}
16024 	}
16025 #endif /* CONFIG_PCI_IOV */
16026 
16027 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16028 		pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
16029 						      pf->num_iwarp_msix,
16030 						      I40E_IWARP_IRQ_PILE_ID);
16031 		if (pf->iwarp_base_vector < 0) {
16032 			dev_info(&pdev->dev,
16033 				 "failed to get tracking for %d vectors for IWARP err=%d\n",
16034 				 pf->num_iwarp_msix, pf->iwarp_base_vector);
16035 			clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
16036 		}
16037 	}
16038 
16039 	i40e_dbg_pf_init(pf);
16040 
16041 	/* tell the firmware that we're starting */
16042 	i40e_send_version(pf);
16043 
16044 	/* since everything's happy, start the service_task timer */
16045 	mod_timer(&pf->service_timer,
16046 		  round_jiffies(jiffies + pf->service_timer_period));
16047 
16048 	/* add this PF to client device list and launch a client service task */
16049 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16050 		err = i40e_lan_add_device(pf);
16051 		if (err)
16052 			dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
16053 				 err);
16054 	}
16055 
16056 #define PCI_SPEED_SIZE 8
16057 #define PCI_WIDTH_SIZE 8
16058 	/* Devices on the IOSF bus do not have this information
16059 	 * and will report PCI Gen 1 x 1 by default so don't bother
16060 	 * checking them.
16061 	 */
16062 	if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) {
16063 		char speed[PCI_SPEED_SIZE] = "Unknown";
16064 		char width[PCI_WIDTH_SIZE] = "Unknown";
16065 
16066 		/* Get the negotiated link width and speed from PCI config
16067 		 * space
16068 		 */
16069 		pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
16070 					  &link_status);
16071 
16072 		i40e_set_pci_config_data(hw, link_status);
16073 
16074 		switch (hw->bus.speed) {
16075 		case i40e_bus_speed_8000:
16076 			strscpy(speed, "8.0", PCI_SPEED_SIZE); break;
16077 		case i40e_bus_speed_5000:
16078 			strscpy(speed, "5.0", PCI_SPEED_SIZE); break;
16079 		case i40e_bus_speed_2500:
16080 			strscpy(speed, "2.5", PCI_SPEED_SIZE); break;
16081 		default:
16082 			break;
16083 		}
16084 		switch (hw->bus.width) {
16085 		case i40e_bus_width_pcie_x8:
16086 			strscpy(width, "8", PCI_WIDTH_SIZE); break;
16087 		case i40e_bus_width_pcie_x4:
16088 			strscpy(width, "4", PCI_WIDTH_SIZE); break;
16089 		case i40e_bus_width_pcie_x2:
16090 			strscpy(width, "2", PCI_WIDTH_SIZE); break;
16091 		case i40e_bus_width_pcie_x1:
16092 			strscpy(width, "1", PCI_WIDTH_SIZE); break;
16093 		default:
16094 			break;
16095 		}
16096 
16097 		dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
16098 			 speed, width);
16099 
16100 		if (hw->bus.width < i40e_bus_width_pcie_x8 ||
16101 		    hw->bus.speed < i40e_bus_speed_8000) {
16102 			dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
16103 			dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
16104 		}
16105 	}
16106 
16107 	/* get the requested speeds from the fw */
16108 	err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
16109 	if (err)
16110 		dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %pe last_status =  %s\n",
16111 			ERR_PTR(err),
16112 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16113 	pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
16114 
16115 	/* set the FEC config due to the board capabilities */
16116 	i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags);
16117 
16118 	/* get the supported phy types from the fw */
16119 	err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
16120 	if (err)
16121 		dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %pe last_status =  %s\n",
16122 			ERR_PTR(err),
16123 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16124 
16125 	/* make sure the MFS hasn't been set lower than the default */
16126 #define MAX_FRAME_SIZE_DEFAULT 0x2600
16127 	val = FIELD_GET(I40E_PRTGL_SAH_MFS_MASK,
16128 			rd32(&pf->hw, I40E_PRTGL_SAH));
16129 	if (val < MAX_FRAME_SIZE_DEFAULT)
16130 		dev_warn(&pdev->dev, "MFS for port %x (%d) has been set below the default (%d)\n",
16131 			 pf->hw.port, val, MAX_FRAME_SIZE_DEFAULT);
16132 
16133 	/* Add a filter to drop all Flow control frames from any VSI from being
16134 	 * transmitted. By doing so we stop a malicious VF from sending out
16135 	 * PAUSE or PFC frames and potentially controlling traffic for other
16136 	 * PF/VF VSIs.
16137 	 * The FW can still send Flow control frames if enabled.
16138 	 */
16139 	i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
16140 						       pf->main_vsi_seid);
16141 
16142 	if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
16143 	    (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
16144 		set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps);
16145 	if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
16146 		set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps);
16147 	/* print a string summarizing features */
16148 	i40e_print_features(pf);
16149 
16150 	i40e_devlink_register(pf);
16151 
16152 	return 0;
16153 
16154 	/* Unwind what we've done if something failed in the setup */
16155 err_vsis:
16156 	set_bit(__I40E_DOWN, pf->state);
16157 	i40e_clear_interrupt_scheme(pf);
16158 	kfree(pf->vsi);
16159 err_switch_setup:
16160 	i40e_reset_interrupt_capability(pf);
16161 	timer_shutdown_sync(&pf->service_timer);
16162 err_mac_addr:
16163 err_configure_lan_hmc:
16164 	(void)i40e_shutdown_lan_hmc(hw);
16165 err_init_lan_hmc:
16166 	kfree(pf->qp_pile);
16167 err_sw_init:
16168 err_adminq_setup:
16169 err_pf_reset:
16170 	iounmap(hw->hw_addr);
16171 err_ioremap:
16172 	i40e_free_pf(pf);
16173 err_pf_alloc:
16174 	pci_release_mem_regions(pdev);
16175 err_pci_reg:
16176 err_dma:
16177 	pci_disable_device(pdev);
16178 	return err;
16179 }
16180 
16181 /**
16182  * i40e_remove - Device removal routine
16183  * @pdev: PCI device information struct
16184  *
16185  * i40e_remove is called by the PCI subsystem to alert the driver
16186  * that is should release a PCI device.  This could be caused by a
16187  * Hot-Plug event, or because the driver is going to be removed from
16188  * memory.
16189  **/
i40e_remove(struct pci_dev * pdev)16190 static void i40e_remove(struct pci_dev *pdev)
16191 {
16192 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16193 	struct i40e_hw *hw = &pf->hw;
16194 	struct i40e_vsi *vsi;
16195 	struct i40e_veb *veb;
16196 	int ret_code;
16197 	int i;
16198 
16199 	i40e_devlink_unregister(pf);
16200 
16201 	i40e_dbg_pf_exit(pf);
16202 
16203 	i40e_ptp_stop(pf);
16204 
16205 	/* Disable RSS in hw */
16206 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
16207 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
16208 
16209 	/* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE
16210 	 * flags, once they are set, i40e_rebuild should not be called as
16211 	 * i40e_prep_for_reset always returns early.
16212 	 */
16213 	while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
16214 		usleep_range(1000, 2000);
16215 	set_bit(__I40E_IN_REMOVE, pf->state);
16216 
16217 	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) {
16218 		set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
16219 		i40e_free_vfs(pf);
16220 		clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
16221 	}
16222 	/* no more scheduling of any task */
16223 	set_bit(__I40E_SUSPENDED, pf->state);
16224 	set_bit(__I40E_DOWN, pf->state);
16225 	if (pf->service_timer.function)
16226 		timer_shutdown_sync(&pf->service_timer);
16227 	if (pf->service_task.func)
16228 		cancel_work_sync(&pf->service_task);
16229 
16230 	if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
16231 		struct i40e_vsi *vsi = pf->vsi[0];
16232 
16233 		/* We know that we have allocated only one vsi for this PF,
16234 		 * it was just for registering netdevice, so the interface
16235 		 * could be visible in the 'ifconfig' output
16236 		 */
16237 		unregister_netdev(vsi->netdev);
16238 		free_netdev(vsi->netdev);
16239 
16240 		goto unmap;
16241 	}
16242 
16243 	/* Client close must be called explicitly here because the timer
16244 	 * has been stopped.
16245 	 */
16246 	i40e_notify_client_of_netdev_close(pf, false);
16247 
16248 	i40e_fdir_teardown(pf);
16249 
16250 	/* If there is a switch structure or any orphans, remove them.
16251 	 * This will leave only the PF's VSI remaining.
16252 	 */
16253 	i40e_pf_for_each_veb(pf, i, veb)
16254 		if (veb->uplink_seid == pf->mac_seid ||
16255 		    veb->uplink_seid == 0)
16256 			i40e_switch_branch_release(veb);
16257 
16258 	/* Now we can shutdown the PF's VSIs, just before we kill
16259 	 * adminq and hmc.
16260 	 */
16261 	i40e_pf_for_each_vsi(pf, i, vsi) {
16262 		i40e_vsi_close(vsi);
16263 		i40e_vsi_release(vsi);
16264 		pf->vsi[i] = NULL;
16265 	}
16266 
16267 	i40e_cloud_filter_exit(pf);
16268 
16269 	/* remove attached clients */
16270 	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16271 		ret_code = i40e_lan_del_device(pf);
16272 		if (ret_code)
16273 			dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
16274 				 ret_code);
16275 	}
16276 
16277 	/* shutdown and destroy the HMC */
16278 	if (hw->hmc.hmc_obj) {
16279 		ret_code = i40e_shutdown_lan_hmc(hw);
16280 		if (ret_code)
16281 			dev_warn(&pdev->dev,
16282 				 "Failed to destroy the HMC resources: %d\n",
16283 				 ret_code);
16284 	}
16285 
16286 unmap:
16287 	/* Free MSI/legacy interrupt 0 when in recovery mode. */
16288 	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16289 	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16290 		free_irq(pf->pdev->irq, pf);
16291 
16292 	/* shutdown the adminq */
16293 	i40e_shutdown_adminq(hw);
16294 
16295 	/* destroy the locks only once, here */
16296 	mutex_destroy(&hw->aq.arq_mutex);
16297 	mutex_destroy(&hw->aq.asq_mutex);
16298 
16299 	/* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
16300 	rtnl_lock();
16301 	i40e_clear_interrupt_scheme(pf);
16302 	i40e_pf_for_each_vsi(pf, i, vsi) {
16303 		if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
16304 			i40e_vsi_clear_rings(vsi);
16305 
16306 		i40e_vsi_clear(vsi);
16307 		pf->vsi[i] = NULL;
16308 	}
16309 	rtnl_unlock();
16310 
16311 	i40e_pf_for_each_veb(pf, i, veb) {
16312 		kfree(veb);
16313 		pf->veb[i] = NULL;
16314 	}
16315 
16316 	kfree(pf->qp_pile);
16317 	kfree(pf->vsi);
16318 
16319 	iounmap(hw->hw_addr);
16320 	i40e_free_pf(pf);
16321 	pci_release_mem_regions(pdev);
16322 
16323 	pci_disable_device(pdev);
16324 }
16325 
16326 /**
16327  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16328  * using the mac_address_write admin q function
16329  * @pf: pointer to i40e_pf struct
16330  **/
i40e_enable_mc_magic_wake(struct i40e_pf * pf)16331 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16332 {
16333 	struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
16334 	struct i40e_hw *hw = &pf->hw;
16335 	u8 mac_addr[6];
16336 	u16 flags = 0;
16337 	int ret;
16338 
16339 	/* Get current MAC address in case it's an LAA */
16340 	if (main_vsi && main_vsi->netdev) {
16341 		ether_addr_copy(mac_addr, main_vsi->netdev->dev_addr);
16342 	} else {
16343 		dev_err(&pf->pdev->dev,
16344 			"Failed to retrieve MAC address; using default\n");
16345 		ether_addr_copy(mac_addr, hw->mac.addr);
16346 	}
16347 
16348 	/* The FW expects the mac address write cmd to first be called with
16349 	 * one of these flags before calling it again with the multicast
16350 	 * enable flags.
16351 	 */
16352 	flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16353 
16354 	if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16355 		flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16356 
16357 	ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16358 	if (ret) {
16359 		dev_err(&pf->pdev->dev,
16360 			"Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16361 		return;
16362 	}
16363 
16364 	flags = I40E_AQC_MC_MAG_EN
16365 			| I40E_AQC_WOL_PRESERVE_ON_PFR
16366 			| I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16367 	ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16368 	if (ret)
16369 		dev_err(&pf->pdev->dev,
16370 			"Failed to enable Multicast Magic Packet wake up\n");
16371 }
16372 
16373 /**
16374  * i40e_io_suspend - suspend all IO operations
16375  * @pf: pointer to i40e_pf struct
16376  *
16377  **/
i40e_io_suspend(struct i40e_pf * pf)16378 static int i40e_io_suspend(struct i40e_pf *pf)
16379 {
16380 	struct i40e_hw *hw = &pf->hw;
16381 
16382 	set_bit(__I40E_DOWN, pf->state);
16383 
16384 	/* Ensure service task will not be running */
16385 	del_timer_sync(&pf->service_timer);
16386 	cancel_work_sync(&pf->service_task);
16387 
16388 	/* Client close must be called explicitly here because the timer
16389 	 * has been stopped.
16390 	 */
16391 	i40e_notify_client_of_netdev_close(pf, false);
16392 
16393 	if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16394 	    pf->wol_en)
16395 		i40e_enable_mc_magic_wake(pf);
16396 
16397 	/* Since we're going to destroy queues during the
16398 	 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16399 	 * whole section
16400 	 */
16401 	rtnl_lock();
16402 
16403 	i40e_prep_for_reset(pf);
16404 
16405 	wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16406 	wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16407 
16408 	/* Clear the interrupt scheme and release our IRQs so that the system
16409 	 * can safely hibernate even when there are a large number of CPUs.
16410 	 * Otherwise hibernation might fail when mapping all the vectors back
16411 	 * to CPU0.
16412 	 */
16413 	i40e_clear_interrupt_scheme(pf);
16414 
16415 	rtnl_unlock();
16416 
16417 	return 0;
16418 }
16419 
16420 /**
16421  * i40e_io_resume - resume IO operations
16422  * @pf: pointer to i40e_pf struct
16423  *
16424  **/
i40e_io_resume(struct i40e_pf * pf)16425 static int i40e_io_resume(struct i40e_pf *pf)
16426 {
16427 	struct device *dev = &pf->pdev->dev;
16428 	int err;
16429 
16430 	/* We need to hold the RTNL lock prior to restoring interrupt schemes,
16431 	 * since we're going to be restoring queues
16432 	 */
16433 	rtnl_lock();
16434 
16435 	/* We cleared the interrupt scheme when we suspended, so we need to
16436 	 * restore it now to resume device functionality.
16437 	 */
16438 	err = i40e_restore_interrupt_scheme(pf);
16439 	if (err) {
16440 		dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16441 			err);
16442 	}
16443 
16444 	clear_bit(__I40E_DOWN, pf->state);
16445 	i40e_reset_and_rebuild(pf, false, true);
16446 
16447 	rtnl_unlock();
16448 
16449 	/* Clear suspended state last after everything is recovered */
16450 	clear_bit(__I40E_SUSPENDED, pf->state);
16451 
16452 	/* Restart the service task */
16453 	mod_timer(&pf->service_timer,
16454 		  round_jiffies(jiffies + pf->service_timer_period));
16455 
16456 	return 0;
16457 }
16458 
16459 /**
16460  * i40e_pci_error_detected - warning that something funky happened in PCI land
16461  * @pdev: PCI device information struct
16462  * @error: the type of PCI error
16463  *
16464  * Called to warn that something happened and the error handling steps
16465  * are in progress.  Allows the driver to quiesce things, be ready for
16466  * remediation.
16467  **/
i40e_pci_error_detected(struct pci_dev * pdev,pci_channel_state_t error)16468 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
16469 						pci_channel_state_t error)
16470 {
16471 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16472 
16473 	dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
16474 
16475 	if (!pf) {
16476 		dev_info(&pdev->dev,
16477 			 "Cannot recover - error happened during device probe\n");
16478 		return PCI_ERS_RESULT_DISCONNECT;
16479 	}
16480 
16481 	/* shutdown all operations */
16482 	if (!test_bit(__I40E_SUSPENDED, pf->state))
16483 		i40e_io_suspend(pf);
16484 
16485 	/* Request a slot reset */
16486 	return PCI_ERS_RESULT_NEED_RESET;
16487 }
16488 
16489 /**
16490  * i40e_pci_error_slot_reset - a PCI slot reset just happened
16491  * @pdev: PCI device information struct
16492  *
16493  * Called to find if the driver can work with the device now that
16494  * the pci slot has been reset.  If a basic connection seems good
16495  * (registers are readable and have sane content) then return a
16496  * happy little PCI_ERS_RESULT_xxx.
16497  **/
i40e_pci_error_slot_reset(struct pci_dev * pdev)16498 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
16499 {
16500 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16501 	pci_ers_result_t result;
16502 	u32 reg;
16503 
16504 	dev_dbg(&pdev->dev, "%s\n", __func__);
16505 	/* enable I/O and memory of the device  */
16506 	if (pci_enable_device(pdev)) {
16507 		dev_info(&pdev->dev,
16508 			 "Cannot re-enable PCI device after reset.\n");
16509 		result = PCI_ERS_RESULT_DISCONNECT;
16510 	} else {
16511 		pci_set_master(pdev);
16512 		pci_restore_state(pdev);
16513 		pci_save_state(pdev);
16514 		pci_wake_from_d3(pdev, false);
16515 
16516 		reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
16517 		if (reg == 0)
16518 			result = PCI_ERS_RESULT_RECOVERED;
16519 		else
16520 			result = PCI_ERS_RESULT_DISCONNECT;
16521 	}
16522 
16523 	return result;
16524 }
16525 
16526 /**
16527  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
16528  * @pdev: PCI device information struct
16529  */
i40e_pci_error_reset_prepare(struct pci_dev * pdev)16530 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
16531 {
16532 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16533 
16534 	i40e_prep_for_reset(pf);
16535 }
16536 
16537 /**
16538  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16539  * @pdev: PCI device information struct
16540  */
i40e_pci_error_reset_done(struct pci_dev * pdev)16541 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
16542 {
16543 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16544 
16545 	if (test_bit(__I40E_IN_REMOVE, pf->state))
16546 		return;
16547 
16548 	i40e_reset_and_rebuild(pf, false, false);
16549 #ifdef CONFIG_PCI_IOV
16550 	i40e_restore_all_vfs_msi_state(pdev);
16551 #endif /* CONFIG_PCI_IOV */
16552 }
16553 
16554 /**
16555  * i40e_pci_error_resume - restart operations after PCI error recovery
16556  * @pdev: PCI device information struct
16557  *
16558  * Called to allow the driver to bring things back up after PCI error
16559  * and/or reset recovery has finished.
16560  **/
i40e_pci_error_resume(struct pci_dev * pdev)16561 static void i40e_pci_error_resume(struct pci_dev *pdev)
16562 {
16563 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16564 
16565 	dev_dbg(&pdev->dev, "%s\n", __func__);
16566 	if (test_bit(__I40E_SUSPENDED, pf->state))
16567 		return;
16568 
16569 	i40e_io_resume(pf);
16570 }
16571 
16572 /**
16573  * i40e_shutdown - PCI callback for shutting down
16574  * @pdev: PCI device information struct
16575  **/
i40e_shutdown(struct pci_dev * pdev)16576 static void i40e_shutdown(struct pci_dev *pdev)
16577 {
16578 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16579 	struct i40e_hw *hw = &pf->hw;
16580 
16581 	set_bit(__I40E_SUSPENDED, pf->state);
16582 	set_bit(__I40E_DOWN, pf->state);
16583 
16584 	del_timer_sync(&pf->service_timer);
16585 	cancel_work_sync(&pf->service_task);
16586 	i40e_cloud_filter_exit(pf);
16587 	i40e_fdir_teardown(pf);
16588 
16589 	/* Client close must be called explicitly here because the timer
16590 	 * has been stopped.
16591 	 */
16592 	i40e_notify_client_of_netdev_close(pf, false);
16593 
16594 	if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16595 	    pf->wol_en)
16596 		i40e_enable_mc_magic_wake(pf);
16597 
16598 	i40e_prep_for_reset(pf);
16599 
16600 	wr32(hw, I40E_PFPM_APM,
16601 	     (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16602 	wr32(hw, I40E_PFPM_WUFC,
16603 	     (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16604 
16605 	/* Free MSI/legacy interrupt 0 when in recovery mode. */
16606 	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16607 	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16608 		free_irq(pf->pdev->irq, pf);
16609 
16610 	/* Since we're going to destroy queues during the
16611 	 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16612 	 * whole section
16613 	 */
16614 	rtnl_lock();
16615 	i40e_clear_interrupt_scheme(pf);
16616 	rtnl_unlock();
16617 
16618 	if (system_state == SYSTEM_POWER_OFF) {
16619 		pci_wake_from_d3(pdev, pf->wol_en);
16620 		pci_set_power_state(pdev, PCI_D3hot);
16621 	}
16622 }
16623 
16624 /**
16625  * i40e_suspend - PM callback for moving to D3
16626  * @dev: generic device information structure
16627  **/
i40e_suspend(struct device * dev)16628 static int i40e_suspend(struct device *dev)
16629 {
16630 	struct i40e_pf *pf = dev_get_drvdata(dev);
16631 
16632 	/* If we're already suspended, then there is nothing to do */
16633 	if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
16634 		return 0;
16635 	return i40e_io_suspend(pf);
16636 }
16637 
16638 /**
16639  * i40e_resume - PM callback for waking up from D3
16640  * @dev: generic device information structure
16641  **/
i40e_resume(struct device * dev)16642 static int i40e_resume(struct device *dev)
16643 {
16644 	struct i40e_pf *pf = dev_get_drvdata(dev);
16645 
16646 	/* If we're not suspended, then there is nothing to do */
16647 	if (!test_bit(__I40E_SUSPENDED, pf->state))
16648 		return 0;
16649 	return i40e_io_resume(pf);
16650 }
16651 
16652 static const struct pci_error_handlers i40e_err_handler = {
16653 	.error_detected = i40e_pci_error_detected,
16654 	.slot_reset = i40e_pci_error_slot_reset,
16655 	.reset_prepare = i40e_pci_error_reset_prepare,
16656 	.reset_done = i40e_pci_error_reset_done,
16657 	.resume = i40e_pci_error_resume,
16658 };
16659 
16660 static DEFINE_SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16661 
16662 static struct pci_driver i40e_driver = {
16663 	.name     = i40e_driver_name,
16664 	.id_table = i40e_pci_tbl,
16665 	.probe    = i40e_probe,
16666 	.remove   = i40e_remove,
16667 	.driver.pm = pm_sleep_ptr(&i40e_pm_ops),
16668 	.shutdown = i40e_shutdown,
16669 	.err_handler = &i40e_err_handler,
16670 	.sriov_configure = i40e_pci_sriov_configure,
16671 };
16672 
16673 /**
16674  * i40e_init_module - Driver registration routine
16675  *
16676  * i40e_init_module is the first routine called when the driver is
16677  * loaded. All it does is register with the PCI subsystem.
16678  **/
i40e_init_module(void)16679 static int __init i40e_init_module(void)
16680 {
16681 	int err;
16682 
16683 	pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16684 	pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16685 
16686 	/* There is no need to throttle the number of active tasks because
16687 	 * each device limits its own task using a state bit for scheduling
16688 	 * the service task, and the device tasks do not interfere with each
16689 	 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16690 	 * since we need to be able to guarantee forward progress even under
16691 	 * memory pressure.
16692 	 */
16693 	i40e_wq = alloc_workqueue("%s", 0, 0, i40e_driver_name);
16694 	if (!i40e_wq) {
16695 		pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16696 		return -ENOMEM;
16697 	}
16698 
16699 	i40e_dbg_init();
16700 	err = pci_register_driver(&i40e_driver);
16701 	if (err) {
16702 		destroy_workqueue(i40e_wq);
16703 		i40e_dbg_exit();
16704 		return err;
16705 	}
16706 
16707 	return 0;
16708 }
16709 module_init(i40e_init_module);
16710 
16711 /**
16712  * i40e_exit_module - Driver exit cleanup routine
16713  *
16714  * i40e_exit_module is called just before the driver is removed
16715  * from memory.
16716  **/
i40e_exit_module(void)16717 static void __exit i40e_exit_module(void)
16718 {
16719 	pci_unregister_driver(&i40e_driver);
16720 	destroy_workqueue(i40e_wq);
16721 	ida_destroy(&i40e_client_ida);
16722 	i40e_dbg_exit();
16723 }
16724 module_exit(i40e_exit_module);
16725