xref: /linux/drivers/net/ethernet/intel/ice/ice_main.c (revision 0a149ab78ee220c75eef797abea7a29f4490e226)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018-2023, Intel Corporation. */
3 
4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <generated/utsrelease.h>
9 #include <linux/crash_dump.h>
10 #include "ice.h"
11 #include "ice_base.h"
12 #include "ice_lib.h"
13 #include "ice_fltr.h"
14 #include "ice_dcb_lib.h"
15 #include "ice_dcb_nl.h"
16 #include "ice_devlink.h"
17 #include "ice_hwmon.h"
18 /* Including ice_trace.h with CREATE_TRACE_POINTS defined will generate the
19  * ice tracepoint functions. This must be done exactly once across the
20  * ice driver.
21  */
22 #define CREATE_TRACE_POINTS
23 #include "ice_trace.h"
24 #include "ice_eswitch.h"
25 #include "ice_tc_lib.h"
26 #include "ice_vsi_vlan_ops.h"
27 #include <net/xdp_sock_drv.h>
28 
29 #define DRV_SUMMARY	"Intel(R) Ethernet Connection E800 Series Linux Driver"
30 static const char ice_driver_string[] = DRV_SUMMARY;
31 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation.";
32 
33 /* DDP Package file located in firmware search paths (e.g. /lib/firmware/) */
34 #define ICE_DDP_PKG_PATH	"intel/ice/ddp/"
35 #define ICE_DDP_PKG_FILE	ICE_DDP_PKG_PATH "ice.pkg"
36 
37 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
38 MODULE_DESCRIPTION(DRV_SUMMARY);
39 MODULE_LICENSE("GPL v2");
40 MODULE_FIRMWARE(ICE_DDP_PKG_FILE);
41 
42 static int debug = -1;
43 module_param(debug, int, 0644);
44 #ifndef CONFIG_DYNAMIC_DEBUG
45 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)");
46 #else
47 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)");
48 #endif /* !CONFIG_DYNAMIC_DEBUG */
49 
50 DEFINE_STATIC_KEY_FALSE(ice_xdp_locking_key);
51 EXPORT_SYMBOL(ice_xdp_locking_key);
52 
53 /**
54  * ice_hw_to_dev - Get device pointer from the hardware structure
55  * @hw: pointer to the device HW structure
56  *
57  * Used to access the device pointer from compilation units which can't easily
58  * include the definition of struct ice_pf without leading to circular header
59  * dependencies.
60  */
61 struct device *ice_hw_to_dev(struct ice_hw *hw)
62 {
63 	struct ice_pf *pf = container_of(hw, struct ice_pf, hw);
64 
65 	return &pf->pdev->dev;
66 }
67 
68 static struct workqueue_struct *ice_wq;
69 struct workqueue_struct *ice_lag_wq;
70 static const struct net_device_ops ice_netdev_safe_mode_ops;
71 static const struct net_device_ops ice_netdev_ops;
72 
73 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type);
74 
75 static void ice_vsi_release_all(struct ice_pf *pf);
76 
77 static int ice_rebuild_channels(struct ice_pf *pf);
78 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_adv_fltr);
79 
80 static int
81 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch,
82 		     void *cb_priv, enum tc_setup_type type, void *type_data,
83 		     void *data,
84 		     void (*cleanup)(struct flow_block_cb *block_cb));
85 
86 bool netif_is_ice(const struct net_device *dev)
87 {
88 	return dev && (dev->netdev_ops == &ice_netdev_ops);
89 }
90 
91 /**
92  * ice_get_tx_pending - returns number of Tx descriptors not processed
93  * @ring: the ring of descriptors
94  */
95 static u16 ice_get_tx_pending(struct ice_tx_ring *ring)
96 {
97 	u16 head, tail;
98 
99 	head = ring->next_to_clean;
100 	tail = ring->next_to_use;
101 
102 	if (head != tail)
103 		return (head < tail) ?
104 			tail - head : (tail + ring->count - head);
105 	return 0;
106 }
107 
108 /**
109  * ice_check_for_hang_subtask - check for and recover hung queues
110  * @pf: pointer to PF struct
111  */
112 static void ice_check_for_hang_subtask(struct ice_pf *pf)
113 {
114 	struct ice_vsi *vsi = NULL;
115 	struct ice_hw *hw;
116 	unsigned int i;
117 	int packets;
118 	u32 v;
119 
120 	ice_for_each_vsi(pf, v)
121 		if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) {
122 			vsi = pf->vsi[v];
123 			break;
124 		}
125 
126 	if (!vsi || test_bit(ICE_VSI_DOWN, vsi->state))
127 		return;
128 
129 	if (!(vsi->netdev && netif_carrier_ok(vsi->netdev)))
130 		return;
131 
132 	hw = &vsi->back->hw;
133 
134 	ice_for_each_txq(vsi, i) {
135 		struct ice_tx_ring *tx_ring = vsi->tx_rings[i];
136 		struct ice_ring_stats *ring_stats;
137 
138 		if (!tx_ring)
139 			continue;
140 		if (ice_ring_ch_enabled(tx_ring))
141 			continue;
142 
143 		ring_stats = tx_ring->ring_stats;
144 		if (!ring_stats)
145 			continue;
146 
147 		if (tx_ring->desc) {
148 			/* If packet counter has not changed the queue is
149 			 * likely stalled, so force an interrupt for this
150 			 * queue.
151 			 *
152 			 * prev_pkt would be negative if there was no
153 			 * pending work.
154 			 */
155 			packets = ring_stats->stats.pkts & INT_MAX;
156 			if (ring_stats->tx_stats.prev_pkt == packets) {
157 				/* Trigger sw interrupt to revive the queue */
158 				ice_trigger_sw_intr(hw, tx_ring->q_vector);
159 				continue;
160 			}
161 
162 			/* Memory barrier between read of packet count and call
163 			 * to ice_get_tx_pending()
164 			 */
165 			smp_rmb();
166 			ring_stats->tx_stats.prev_pkt =
167 			    ice_get_tx_pending(tx_ring) ? packets : -1;
168 		}
169 	}
170 }
171 
172 /**
173  * ice_init_mac_fltr - Set initial MAC filters
174  * @pf: board private structure
175  *
176  * Set initial set of MAC filters for PF VSI; configure filters for permanent
177  * address and broadcast address. If an error is encountered, netdevice will be
178  * unregistered.
179  */
180 static int ice_init_mac_fltr(struct ice_pf *pf)
181 {
182 	struct ice_vsi *vsi;
183 	u8 *perm_addr;
184 
185 	vsi = ice_get_main_vsi(pf);
186 	if (!vsi)
187 		return -EINVAL;
188 
189 	perm_addr = vsi->port_info->mac.perm_addr;
190 	return ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI);
191 }
192 
193 /**
194  * ice_add_mac_to_sync_list - creates list of MAC addresses to be synced
195  * @netdev: the net device on which the sync is happening
196  * @addr: MAC address to sync
197  *
198  * This is a callback function which is called by the in kernel device sync
199  * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only
200  * populates the tmp_sync_list, which is later used by ice_add_mac to add the
201  * MAC filters from the hardware.
202  */
203 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr)
204 {
205 	struct ice_netdev_priv *np = netdev_priv(netdev);
206 	struct ice_vsi *vsi = np->vsi;
207 
208 	if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr,
209 				     ICE_FWD_TO_VSI))
210 		return -EINVAL;
211 
212 	return 0;
213 }
214 
215 /**
216  * ice_add_mac_to_unsync_list - creates list of MAC addresses to be unsynced
217  * @netdev: the net device on which the unsync is happening
218  * @addr: MAC address to unsync
219  *
220  * This is a callback function which is called by the in kernel device unsync
221  * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only
222  * populates the tmp_unsync_list, which is later used by ice_remove_mac to
223  * delete the MAC filters from the hardware.
224  */
225 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr)
226 {
227 	struct ice_netdev_priv *np = netdev_priv(netdev);
228 	struct ice_vsi *vsi = np->vsi;
229 
230 	/* Under some circumstances, we might receive a request to delete our
231 	 * own device address from our uc list. Because we store the device
232 	 * address in the VSI's MAC filter list, we need to ignore such
233 	 * requests and not delete our device address from this list.
234 	 */
235 	if (ether_addr_equal(addr, netdev->dev_addr))
236 		return 0;
237 
238 	if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr,
239 				     ICE_FWD_TO_VSI))
240 		return -EINVAL;
241 
242 	return 0;
243 }
244 
245 /**
246  * ice_vsi_fltr_changed - check if filter state changed
247  * @vsi: VSI to be checked
248  *
249  * returns true if filter state has changed, false otherwise.
250  */
251 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi)
252 {
253 	return test_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state) ||
254 	       test_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
255 }
256 
257 /**
258  * ice_set_promisc - Enable promiscuous mode for a given PF
259  * @vsi: the VSI being configured
260  * @promisc_m: mask of promiscuous config bits
261  *
262  */
263 static int ice_set_promisc(struct ice_vsi *vsi, u8 promisc_m)
264 {
265 	int status;
266 
267 	if (vsi->type != ICE_VSI_PF)
268 		return 0;
269 
270 	if (ice_vsi_has_non_zero_vlans(vsi)) {
271 		promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX);
272 		status = ice_fltr_set_vlan_vsi_promisc(&vsi->back->hw, vsi,
273 						       promisc_m);
274 	} else {
275 		status = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
276 						  promisc_m, 0);
277 	}
278 	if (status && status != -EEXIST)
279 		return status;
280 
281 	netdev_dbg(vsi->netdev, "set promisc filter bits for VSI %i: 0x%x\n",
282 		   vsi->vsi_num, promisc_m);
283 	return 0;
284 }
285 
286 /**
287  * ice_clear_promisc - Disable promiscuous mode for a given PF
288  * @vsi: the VSI being configured
289  * @promisc_m: mask of promiscuous config bits
290  *
291  */
292 static int ice_clear_promisc(struct ice_vsi *vsi, u8 promisc_m)
293 {
294 	int status;
295 
296 	if (vsi->type != ICE_VSI_PF)
297 		return 0;
298 
299 	if (ice_vsi_has_non_zero_vlans(vsi)) {
300 		promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX);
301 		status = ice_fltr_clear_vlan_vsi_promisc(&vsi->back->hw, vsi,
302 							 promisc_m);
303 	} else {
304 		status = ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
305 						    promisc_m, 0);
306 	}
307 
308 	netdev_dbg(vsi->netdev, "clear promisc filter bits for VSI %i: 0x%x\n",
309 		   vsi->vsi_num, promisc_m);
310 	return status;
311 }
312 
313 /**
314  * ice_vsi_sync_fltr - Update the VSI filter list to the HW
315  * @vsi: ptr to the VSI
316  *
317  * Push any outstanding VSI filter changes through the AdminQ.
318  */
319 static int ice_vsi_sync_fltr(struct ice_vsi *vsi)
320 {
321 	struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
322 	struct device *dev = ice_pf_to_dev(vsi->back);
323 	struct net_device *netdev = vsi->netdev;
324 	bool promisc_forced_on = false;
325 	struct ice_pf *pf = vsi->back;
326 	struct ice_hw *hw = &pf->hw;
327 	u32 changed_flags = 0;
328 	int err;
329 
330 	if (!vsi->netdev)
331 		return -EINVAL;
332 
333 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state))
334 		usleep_range(1000, 2000);
335 
336 	changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
337 	vsi->current_netdev_flags = vsi->netdev->flags;
338 
339 	INIT_LIST_HEAD(&vsi->tmp_sync_list);
340 	INIT_LIST_HEAD(&vsi->tmp_unsync_list);
341 
342 	if (ice_vsi_fltr_changed(vsi)) {
343 		clear_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state);
344 		clear_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
345 
346 		/* grab the netdev's addr_list_lock */
347 		netif_addr_lock_bh(netdev);
348 		__dev_uc_sync(netdev, ice_add_mac_to_sync_list,
349 			      ice_add_mac_to_unsync_list);
350 		__dev_mc_sync(netdev, ice_add_mac_to_sync_list,
351 			      ice_add_mac_to_unsync_list);
352 		/* our temp lists are populated. release lock */
353 		netif_addr_unlock_bh(netdev);
354 	}
355 
356 	/* Remove MAC addresses in the unsync list */
357 	err = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list);
358 	ice_fltr_free_list(dev, &vsi->tmp_unsync_list);
359 	if (err) {
360 		netdev_err(netdev, "Failed to delete MAC filters\n");
361 		/* if we failed because of alloc failures, just bail */
362 		if (err == -ENOMEM)
363 			goto out;
364 	}
365 
366 	/* Add MAC addresses in the sync list */
367 	err = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list);
368 	ice_fltr_free_list(dev, &vsi->tmp_sync_list);
369 	/* If filter is added successfully or already exists, do not go into
370 	 * 'if' condition and report it as error. Instead continue processing
371 	 * rest of the function.
372 	 */
373 	if (err && err != -EEXIST) {
374 		netdev_err(netdev, "Failed to add MAC filters\n");
375 		/* If there is no more space for new umac filters, VSI
376 		 * should go into promiscuous mode. There should be some
377 		 * space reserved for promiscuous filters.
378 		 */
379 		if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC &&
380 		    !test_and_set_bit(ICE_FLTR_OVERFLOW_PROMISC,
381 				      vsi->state)) {
382 			promisc_forced_on = true;
383 			netdev_warn(netdev, "Reached MAC filter limit, forcing promisc mode on VSI %d\n",
384 				    vsi->vsi_num);
385 		} else {
386 			goto out;
387 		}
388 	}
389 	err = 0;
390 	/* check for changes in promiscuous modes */
391 	if (changed_flags & IFF_ALLMULTI) {
392 		if (vsi->current_netdev_flags & IFF_ALLMULTI) {
393 			err = ice_set_promisc(vsi, ICE_MCAST_PROMISC_BITS);
394 			if (err) {
395 				vsi->current_netdev_flags &= ~IFF_ALLMULTI;
396 				goto out_promisc;
397 			}
398 		} else {
399 			/* !(vsi->current_netdev_flags & IFF_ALLMULTI) */
400 			err = ice_clear_promisc(vsi, ICE_MCAST_PROMISC_BITS);
401 			if (err) {
402 				vsi->current_netdev_flags |= IFF_ALLMULTI;
403 				goto out_promisc;
404 			}
405 		}
406 	}
407 
408 	if (((changed_flags & IFF_PROMISC) || promisc_forced_on) ||
409 	    test_bit(ICE_VSI_PROMISC_CHANGED, vsi->state)) {
410 		clear_bit(ICE_VSI_PROMISC_CHANGED, vsi->state);
411 		if (vsi->current_netdev_flags & IFF_PROMISC) {
412 			/* Apply Rx filter rule to get traffic from wire */
413 			if (!ice_is_dflt_vsi_in_use(vsi->port_info)) {
414 				err = ice_set_dflt_vsi(vsi);
415 				if (err && err != -EEXIST) {
416 					netdev_err(netdev, "Error %d setting default VSI %i Rx rule\n",
417 						   err, vsi->vsi_num);
418 					vsi->current_netdev_flags &=
419 						~IFF_PROMISC;
420 					goto out_promisc;
421 				}
422 				err = 0;
423 				vlan_ops->dis_rx_filtering(vsi);
424 
425 				/* promiscuous mode implies allmulticast so
426 				 * that VSIs that are in promiscuous mode are
427 				 * subscribed to multicast packets coming to
428 				 * the port
429 				 */
430 				err = ice_set_promisc(vsi,
431 						      ICE_MCAST_PROMISC_BITS);
432 				if (err)
433 					goto out_promisc;
434 			}
435 		} else {
436 			/* Clear Rx filter to remove traffic from wire */
437 			if (ice_is_vsi_dflt_vsi(vsi)) {
438 				err = ice_clear_dflt_vsi(vsi);
439 				if (err) {
440 					netdev_err(netdev, "Error %d clearing default VSI %i Rx rule\n",
441 						   err, vsi->vsi_num);
442 					vsi->current_netdev_flags |=
443 						IFF_PROMISC;
444 					goto out_promisc;
445 				}
446 				if (vsi->netdev->features &
447 				    NETIF_F_HW_VLAN_CTAG_FILTER)
448 					vlan_ops->ena_rx_filtering(vsi);
449 			}
450 
451 			/* disable allmulti here, but only if allmulti is not
452 			 * still enabled for the netdev
453 			 */
454 			if (!(vsi->current_netdev_flags & IFF_ALLMULTI)) {
455 				err = ice_clear_promisc(vsi,
456 							ICE_MCAST_PROMISC_BITS);
457 				if (err) {
458 					netdev_err(netdev, "Error %d clearing multicast promiscuous on VSI %i\n",
459 						   err, vsi->vsi_num);
460 				}
461 			}
462 		}
463 	}
464 	goto exit;
465 
466 out_promisc:
467 	set_bit(ICE_VSI_PROMISC_CHANGED, vsi->state);
468 	goto exit;
469 out:
470 	/* if something went wrong then set the changed flag so we try again */
471 	set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state);
472 	set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
473 exit:
474 	clear_bit(ICE_CFG_BUSY, vsi->state);
475 	return err;
476 }
477 
478 /**
479  * ice_sync_fltr_subtask - Sync the VSI filter list with HW
480  * @pf: board private structure
481  */
482 static void ice_sync_fltr_subtask(struct ice_pf *pf)
483 {
484 	int v;
485 
486 	if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags)))
487 		return;
488 
489 	clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
490 
491 	ice_for_each_vsi(pf, v)
492 		if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) &&
493 		    ice_vsi_sync_fltr(pf->vsi[v])) {
494 			/* come back and try again later */
495 			set_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
496 			break;
497 		}
498 }
499 
500 /**
501  * ice_pf_dis_all_vsi - Pause all VSIs on a PF
502  * @pf: the PF
503  * @locked: is the rtnl_lock already held
504  */
505 static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked)
506 {
507 	int node;
508 	int v;
509 
510 	ice_for_each_vsi(pf, v)
511 		if (pf->vsi[v])
512 			ice_dis_vsi(pf->vsi[v], locked);
513 
514 	for (node = 0; node < ICE_MAX_PF_AGG_NODES; node++)
515 		pf->pf_agg_node[node].num_vsis = 0;
516 
517 	for (node = 0; node < ICE_MAX_VF_AGG_NODES; node++)
518 		pf->vf_agg_node[node].num_vsis = 0;
519 }
520 
521 /**
522  * ice_clear_sw_switch_recipes - clear switch recipes
523  * @pf: board private structure
524  *
525  * Mark switch recipes as not created in sw structures. There are cases where
526  * rules (especially advanced rules) need to be restored, either re-read from
527  * hardware or added again. For example after the reset. 'recp_created' flag
528  * prevents from doing that and need to be cleared upfront.
529  */
530 static void ice_clear_sw_switch_recipes(struct ice_pf *pf)
531 {
532 	struct ice_sw_recipe *recp;
533 	u8 i;
534 
535 	recp = pf->hw.switch_info->recp_list;
536 	for (i = 0; i < ICE_MAX_NUM_RECIPES; i++)
537 		recp[i].recp_created = false;
538 }
539 
540 /**
541  * ice_prepare_for_reset - prep for reset
542  * @pf: board private structure
543  * @reset_type: reset type requested
544  *
545  * Inform or close all dependent features in prep for reset.
546  */
547 static void
548 ice_prepare_for_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
549 {
550 	struct ice_hw *hw = &pf->hw;
551 	struct ice_vsi *vsi;
552 	struct ice_vf *vf;
553 	unsigned int bkt;
554 
555 	dev_dbg(ice_pf_to_dev(pf), "reset_type=%d\n", reset_type);
556 
557 	/* already prepared for reset */
558 	if (test_bit(ICE_PREPARED_FOR_RESET, pf->state))
559 		return;
560 
561 	ice_unplug_aux_dev(pf);
562 
563 	/* Notify VFs of impending reset */
564 	if (ice_check_sq_alive(hw, &hw->mailboxq))
565 		ice_vc_notify_reset(pf);
566 
567 	/* Disable VFs until reset is completed */
568 	mutex_lock(&pf->vfs.table_lock);
569 	ice_for_each_vf(pf, bkt, vf)
570 		ice_set_vf_state_dis(vf);
571 	mutex_unlock(&pf->vfs.table_lock);
572 
573 	if (ice_is_eswitch_mode_switchdev(pf)) {
574 		if (reset_type != ICE_RESET_PFR)
575 			ice_clear_sw_switch_recipes(pf);
576 	}
577 
578 	/* release ADQ specific HW and SW resources */
579 	vsi = ice_get_main_vsi(pf);
580 	if (!vsi)
581 		goto skip;
582 
583 	/* to be on safe side, reset orig_rss_size so that normal flow
584 	 * of deciding rss_size can take precedence
585 	 */
586 	vsi->orig_rss_size = 0;
587 
588 	if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) {
589 		if (reset_type == ICE_RESET_PFR) {
590 			vsi->old_ena_tc = vsi->all_enatc;
591 			vsi->old_numtc = vsi->all_numtc;
592 		} else {
593 			ice_remove_q_channels(vsi, true);
594 
595 			/* for other reset type, do not support channel rebuild
596 			 * hence reset needed info
597 			 */
598 			vsi->old_ena_tc = 0;
599 			vsi->all_enatc = 0;
600 			vsi->old_numtc = 0;
601 			vsi->all_numtc = 0;
602 			vsi->req_txq = 0;
603 			vsi->req_rxq = 0;
604 			clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags);
605 			memset(&vsi->mqprio_qopt, 0, sizeof(vsi->mqprio_qopt));
606 		}
607 	}
608 skip:
609 
610 	/* clear SW filtering DB */
611 	ice_clear_hw_tbls(hw);
612 	/* disable the VSIs and their queues that are not already DOWN */
613 	ice_pf_dis_all_vsi(pf, false);
614 
615 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
616 		ice_ptp_prepare_for_reset(pf);
617 
618 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
619 		ice_gnss_exit(pf);
620 
621 	if (hw->port_info)
622 		ice_sched_clear_port(hw->port_info);
623 
624 	ice_shutdown_all_ctrlq(hw);
625 
626 	set_bit(ICE_PREPARED_FOR_RESET, pf->state);
627 }
628 
629 /**
630  * ice_do_reset - Initiate one of many types of resets
631  * @pf: board private structure
632  * @reset_type: reset type requested before this function was called.
633  */
634 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
635 {
636 	struct device *dev = ice_pf_to_dev(pf);
637 	struct ice_hw *hw = &pf->hw;
638 
639 	dev_dbg(dev, "reset_type 0x%x requested\n", reset_type);
640 
641 	if (pf->lag && pf->lag->bonded && reset_type == ICE_RESET_PFR) {
642 		dev_dbg(dev, "PFR on a bonded interface, promoting to CORER\n");
643 		reset_type = ICE_RESET_CORER;
644 	}
645 
646 	ice_prepare_for_reset(pf, reset_type);
647 
648 	/* trigger the reset */
649 	if (ice_reset(hw, reset_type)) {
650 		dev_err(dev, "reset %d failed\n", reset_type);
651 		set_bit(ICE_RESET_FAILED, pf->state);
652 		clear_bit(ICE_RESET_OICR_RECV, pf->state);
653 		clear_bit(ICE_PREPARED_FOR_RESET, pf->state);
654 		clear_bit(ICE_PFR_REQ, pf->state);
655 		clear_bit(ICE_CORER_REQ, pf->state);
656 		clear_bit(ICE_GLOBR_REQ, pf->state);
657 		wake_up(&pf->reset_wait_queue);
658 		return;
659 	}
660 
661 	/* PFR is a bit of a special case because it doesn't result in an OICR
662 	 * interrupt. So for PFR, rebuild after the reset and clear the reset-
663 	 * associated state bits.
664 	 */
665 	if (reset_type == ICE_RESET_PFR) {
666 		pf->pfr_count++;
667 		ice_rebuild(pf, reset_type);
668 		clear_bit(ICE_PREPARED_FOR_RESET, pf->state);
669 		clear_bit(ICE_PFR_REQ, pf->state);
670 		wake_up(&pf->reset_wait_queue);
671 		ice_reset_all_vfs(pf);
672 	}
673 }
674 
675 /**
676  * ice_reset_subtask - Set up for resetting the device and driver
677  * @pf: board private structure
678  */
679 static void ice_reset_subtask(struct ice_pf *pf)
680 {
681 	enum ice_reset_req reset_type = ICE_RESET_INVAL;
682 
683 	/* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an
684 	 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type
685 	 * of reset is pending and sets bits in pf->state indicating the reset
686 	 * type and ICE_RESET_OICR_RECV. So, if the latter bit is set
687 	 * prepare for pending reset if not already (for PF software-initiated
688 	 * global resets the software should already be prepared for it as
689 	 * indicated by ICE_PREPARED_FOR_RESET; for global resets initiated
690 	 * by firmware or software on other PFs, that bit is not set so prepare
691 	 * for the reset now), poll for reset done, rebuild and return.
692 	 */
693 	if (test_bit(ICE_RESET_OICR_RECV, pf->state)) {
694 		/* Perform the largest reset requested */
695 		if (test_and_clear_bit(ICE_CORER_RECV, pf->state))
696 			reset_type = ICE_RESET_CORER;
697 		if (test_and_clear_bit(ICE_GLOBR_RECV, pf->state))
698 			reset_type = ICE_RESET_GLOBR;
699 		if (test_and_clear_bit(ICE_EMPR_RECV, pf->state))
700 			reset_type = ICE_RESET_EMPR;
701 		/* return if no valid reset type requested */
702 		if (reset_type == ICE_RESET_INVAL)
703 			return;
704 		ice_prepare_for_reset(pf, reset_type);
705 
706 		/* make sure we are ready to rebuild */
707 		if (ice_check_reset(&pf->hw)) {
708 			set_bit(ICE_RESET_FAILED, pf->state);
709 		} else {
710 			/* done with reset. start rebuild */
711 			pf->hw.reset_ongoing = false;
712 			ice_rebuild(pf, reset_type);
713 			/* clear bit to resume normal operations, but
714 			 * ICE_NEEDS_RESTART bit is set in case rebuild failed
715 			 */
716 			clear_bit(ICE_RESET_OICR_RECV, pf->state);
717 			clear_bit(ICE_PREPARED_FOR_RESET, pf->state);
718 			clear_bit(ICE_PFR_REQ, pf->state);
719 			clear_bit(ICE_CORER_REQ, pf->state);
720 			clear_bit(ICE_GLOBR_REQ, pf->state);
721 			wake_up(&pf->reset_wait_queue);
722 			ice_reset_all_vfs(pf);
723 		}
724 
725 		return;
726 	}
727 
728 	/* No pending resets to finish processing. Check for new resets */
729 	if (test_bit(ICE_PFR_REQ, pf->state)) {
730 		reset_type = ICE_RESET_PFR;
731 		if (pf->lag && pf->lag->bonded) {
732 			dev_dbg(ice_pf_to_dev(pf), "PFR on a bonded interface, promoting to CORER\n");
733 			reset_type = ICE_RESET_CORER;
734 		}
735 	}
736 	if (test_bit(ICE_CORER_REQ, pf->state))
737 		reset_type = ICE_RESET_CORER;
738 	if (test_bit(ICE_GLOBR_REQ, pf->state))
739 		reset_type = ICE_RESET_GLOBR;
740 	/* If no valid reset type requested just return */
741 	if (reset_type == ICE_RESET_INVAL)
742 		return;
743 
744 	/* reset if not already down or busy */
745 	if (!test_bit(ICE_DOWN, pf->state) &&
746 	    !test_bit(ICE_CFG_BUSY, pf->state)) {
747 		ice_do_reset(pf, reset_type);
748 	}
749 }
750 
751 /**
752  * ice_print_topo_conflict - print topology conflict message
753  * @vsi: the VSI whose topology status is being checked
754  */
755 static void ice_print_topo_conflict(struct ice_vsi *vsi)
756 {
757 	switch (vsi->port_info->phy.link_info.topo_media_conflict) {
758 	case ICE_AQ_LINK_TOPO_CONFLICT:
759 	case ICE_AQ_LINK_MEDIA_CONFLICT:
760 	case ICE_AQ_LINK_TOPO_UNREACH_PRT:
761 	case ICE_AQ_LINK_TOPO_UNDRUTIL_PRT:
762 	case ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA:
763 		netdev_info(vsi->netdev, "Potential misconfiguration of the Ethernet port detected. If it was not intended, please use the Intel (R) Ethernet Port Configuration Tool to address the issue.\n");
764 		break;
765 	case ICE_AQ_LINK_TOPO_UNSUPP_MEDIA:
766 		if (test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, vsi->back->flags))
767 			netdev_warn(vsi->netdev, "An unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules\n");
768 		else
769 			netdev_err(vsi->netdev, "Rx/Tx is disabled on this device because an unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
770 		break;
771 	default:
772 		break;
773 	}
774 }
775 
776 /**
777  * ice_print_link_msg - print link up or down message
778  * @vsi: the VSI whose link status is being queried
779  * @isup: boolean for if the link is now up or down
780  */
781 void ice_print_link_msg(struct ice_vsi *vsi, bool isup)
782 {
783 	struct ice_aqc_get_phy_caps_data *caps;
784 	const char *an_advertised;
785 	const char *fec_req;
786 	const char *speed;
787 	const char *fec;
788 	const char *fc;
789 	const char *an;
790 	int status;
791 
792 	if (!vsi)
793 		return;
794 
795 	if (vsi->current_isup == isup)
796 		return;
797 
798 	vsi->current_isup = isup;
799 
800 	if (!isup) {
801 		netdev_info(vsi->netdev, "NIC Link is Down\n");
802 		return;
803 	}
804 
805 	switch (vsi->port_info->phy.link_info.link_speed) {
806 	case ICE_AQ_LINK_SPEED_100GB:
807 		speed = "100 G";
808 		break;
809 	case ICE_AQ_LINK_SPEED_50GB:
810 		speed = "50 G";
811 		break;
812 	case ICE_AQ_LINK_SPEED_40GB:
813 		speed = "40 G";
814 		break;
815 	case ICE_AQ_LINK_SPEED_25GB:
816 		speed = "25 G";
817 		break;
818 	case ICE_AQ_LINK_SPEED_20GB:
819 		speed = "20 G";
820 		break;
821 	case ICE_AQ_LINK_SPEED_10GB:
822 		speed = "10 G";
823 		break;
824 	case ICE_AQ_LINK_SPEED_5GB:
825 		speed = "5 G";
826 		break;
827 	case ICE_AQ_LINK_SPEED_2500MB:
828 		speed = "2.5 G";
829 		break;
830 	case ICE_AQ_LINK_SPEED_1000MB:
831 		speed = "1 G";
832 		break;
833 	case ICE_AQ_LINK_SPEED_100MB:
834 		speed = "100 M";
835 		break;
836 	default:
837 		speed = "Unknown ";
838 		break;
839 	}
840 
841 	switch (vsi->port_info->fc.current_mode) {
842 	case ICE_FC_FULL:
843 		fc = "Rx/Tx";
844 		break;
845 	case ICE_FC_TX_PAUSE:
846 		fc = "Tx";
847 		break;
848 	case ICE_FC_RX_PAUSE:
849 		fc = "Rx";
850 		break;
851 	case ICE_FC_NONE:
852 		fc = "None";
853 		break;
854 	default:
855 		fc = "Unknown";
856 		break;
857 	}
858 
859 	/* Get FEC mode based on negotiated link info */
860 	switch (vsi->port_info->phy.link_info.fec_info) {
861 	case ICE_AQ_LINK_25G_RS_528_FEC_EN:
862 	case ICE_AQ_LINK_25G_RS_544_FEC_EN:
863 		fec = "RS-FEC";
864 		break;
865 	case ICE_AQ_LINK_25G_KR_FEC_EN:
866 		fec = "FC-FEC/BASE-R";
867 		break;
868 	default:
869 		fec = "NONE";
870 		break;
871 	}
872 
873 	/* check if autoneg completed, might be false due to not supported */
874 	if (vsi->port_info->phy.link_info.an_info & ICE_AQ_AN_COMPLETED)
875 		an = "True";
876 	else
877 		an = "False";
878 
879 	/* Get FEC mode requested based on PHY caps last SW configuration */
880 	caps = kzalloc(sizeof(*caps), GFP_KERNEL);
881 	if (!caps) {
882 		fec_req = "Unknown";
883 		an_advertised = "Unknown";
884 		goto done;
885 	}
886 
887 	status = ice_aq_get_phy_caps(vsi->port_info, false,
888 				     ICE_AQC_REPORT_ACTIVE_CFG, caps, NULL);
889 	if (status)
890 		netdev_info(vsi->netdev, "Get phy capability failed.\n");
891 
892 	an_advertised = ice_is_phy_caps_an_enabled(caps) ? "On" : "Off";
893 
894 	if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_528_REQ ||
895 	    caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_544_REQ)
896 		fec_req = "RS-FEC";
897 	else if (caps->link_fec_options & ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ ||
898 		 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_KR_REQ)
899 		fec_req = "FC-FEC/BASE-R";
900 	else
901 		fec_req = "NONE";
902 
903 	kfree(caps);
904 
905 done:
906 	netdev_info(vsi->netdev, "NIC Link is up %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg Advertised: %s, Autoneg Negotiated: %s, Flow Control: %s\n",
907 		    speed, fec_req, fec, an_advertised, an, fc);
908 	ice_print_topo_conflict(vsi);
909 }
910 
911 /**
912  * ice_vsi_link_event - update the VSI's netdev
913  * @vsi: the VSI on which the link event occurred
914  * @link_up: whether or not the VSI needs to be set up or down
915  */
916 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up)
917 {
918 	if (!vsi)
919 		return;
920 
921 	if (test_bit(ICE_VSI_DOWN, vsi->state) || !vsi->netdev)
922 		return;
923 
924 	if (vsi->type == ICE_VSI_PF) {
925 		if (link_up == netif_carrier_ok(vsi->netdev))
926 			return;
927 
928 		if (link_up) {
929 			netif_carrier_on(vsi->netdev);
930 			netif_tx_wake_all_queues(vsi->netdev);
931 		} else {
932 			netif_carrier_off(vsi->netdev);
933 			netif_tx_stop_all_queues(vsi->netdev);
934 		}
935 	}
936 }
937 
938 /**
939  * ice_set_dflt_mib - send a default config MIB to the FW
940  * @pf: private PF struct
941  *
942  * This function sends a default configuration MIB to the FW.
943  *
944  * If this function errors out at any point, the driver is still able to
945  * function.  The main impact is that LFC may not operate as expected.
946  * Therefore an error state in this function should be treated with a DBG
947  * message and continue on with driver rebuild/reenable.
948  */
949 static void ice_set_dflt_mib(struct ice_pf *pf)
950 {
951 	struct device *dev = ice_pf_to_dev(pf);
952 	u8 mib_type, *buf, *lldpmib = NULL;
953 	u16 len, typelen, offset = 0;
954 	struct ice_lldp_org_tlv *tlv;
955 	struct ice_hw *hw = &pf->hw;
956 	u32 ouisubtype;
957 
958 	mib_type = SET_LOCAL_MIB_TYPE_LOCAL_MIB;
959 	lldpmib = kzalloc(ICE_LLDPDU_SIZE, GFP_KERNEL);
960 	if (!lldpmib) {
961 		dev_dbg(dev, "%s Failed to allocate MIB memory\n",
962 			__func__);
963 		return;
964 	}
965 
966 	/* Add ETS CFG TLV */
967 	tlv = (struct ice_lldp_org_tlv *)lldpmib;
968 	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
969 		   ICE_IEEE_ETS_TLV_LEN);
970 	tlv->typelen = htons(typelen);
971 	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
972 		      ICE_IEEE_SUBTYPE_ETS_CFG);
973 	tlv->ouisubtype = htonl(ouisubtype);
974 
975 	buf = tlv->tlvinfo;
976 	buf[0] = 0;
977 
978 	/* ETS CFG all UPs map to TC 0. Next 4 (1 - 4) Octets = 0.
979 	 * Octets 5 - 12 are BW values, set octet 5 to 100% BW.
980 	 * Octets 13 - 20 are TSA values - leave as zeros
981 	 */
982 	buf[5] = 0x64;
983 	len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S;
984 	offset += len + 2;
985 	tlv = (struct ice_lldp_org_tlv *)
986 		((char *)tlv + sizeof(tlv->typelen) + len);
987 
988 	/* Add ETS REC TLV */
989 	buf = tlv->tlvinfo;
990 	tlv->typelen = htons(typelen);
991 
992 	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
993 		      ICE_IEEE_SUBTYPE_ETS_REC);
994 	tlv->ouisubtype = htonl(ouisubtype);
995 
996 	/* First octet of buf is reserved
997 	 * Octets 1 - 4 map UP to TC - all UPs map to zero
998 	 * Octets 5 - 12 are BW values - set TC 0 to 100%.
999 	 * Octets 13 - 20 are TSA value - leave as zeros
1000 	 */
1001 	buf[5] = 0x64;
1002 	offset += len + 2;
1003 	tlv = (struct ice_lldp_org_tlv *)
1004 		((char *)tlv + sizeof(tlv->typelen) + len);
1005 
1006 	/* Add PFC CFG TLV */
1007 	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
1008 		   ICE_IEEE_PFC_TLV_LEN);
1009 	tlv->typelen = htons(typelen);
1010 
1011 	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
1012 		      ICE_IEEE_SUBTYPE_PFC_CFG);
1013 	tlv->ouisubtype = htonl(ouisubtype);
1014 
1015 	/* Octet 1 left as all zeros - PFC disabled */
1016 	buf[0] = 0x08;
1017 	len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S;
1018 	offset += len + 2;
1019 
1020 	if (ice_aq_set_lldp_mib(hw, mib_type, (void *)lldpmib, offset, NULL))
1021 		dev_dbg(dev, "%s Failed to set default LLDP MIB\n", __func__);
1022 
1023 	kfree(lldpmib);
1024 }
1025 
1026 /**
1027  * ice_check_phy_fw_load - check if PHY FW load failed
1028  * @pf: pointer to PF struct
1029  * @link_cfg_err: bitmap from the link info structure
1030  *
1031  * check if external PHY FW load failed and print an error message if it did
1032  */
1033 static void ice_check_phy_fw_load(struct ice_pf *pf, u8 link_cfg_err)
1034 {
1035 	if (!(link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE)) {
1036 		clear_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags);
1037 		return;
1038 	}
1039 
1040 	if (test_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags))
1041 		return;
1042 
1043 	if (link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE) {
1044 		dev_err(ice_pf_to_dev(pf), "Device failed to load the FW for the external PHY. Please download and install the latest NVM for your device and try again\n");
1045 		set_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags);
1046 	}
1047 }
1048 
1049 /**
1050  * ice_check_module_power
1051  * @pf: pointer to PF struct
1052  * @link_cfg_err: bitmap from the link info structure
1053  *
1054  * check module power level returned by a previous call to aq_get_link_info
1055  * and print error messages if module power level is not supported
1056  */
1057 static void ice_check_module_power(struct ice_pf *pf, u8 link_cfg_err)
1058 {
1059 	/* if module power level is supported, clear the flag */
1060 	if (!(link_cfg_err & (ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT |
1061 			      ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED))) {
1062 		clear_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags);
1063 		return;
1064 	}
1065 
1066 	/* if ICE_FLAG_MOD_POWER_UNSUPPORTED was previously set and the
1067 	 * above block didn't clear this bit, there's nothing to do
1068 	 */
1069 	if (test_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags))
1070 		return;
1071 
1072 	if (link_cfg_err & ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT) {
1073 		dev_err(ice_pf_to_dev(pf), "The installed module is incompatible with the device's NVM image. Cannot start link\n");
1074 		set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags);
1075 	} else if (link_cfg_err & ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED) {
1076 		dev_err(ice_pf_to_dev(pf), "The module's power requirements exceed the device's power supply. Cannot start link\n");
1077 		set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags);
1078 	}
1079 }
1080 
1081 /**
1082  * ice_check_link_cfg_err - check if link configuration failed
1083  * @pf: pointer to the PF struct
1084  * @link_cfg_err: bitmap from the link info structure
1085  *
1086  * print if any link configuration failure happens due to the value in the
1087  * link_cfg_err parameter in the link info structure
1088  */
1089 static void ice_check_link_cfg_err(struct ice_pf *pf, u8 link_cfg_err)
1090 {
1091 	ice_check_module_power(pf, link_cfg_err);
1092 	ice_check_phy_fw_load(pf, link_cfg_err);
1093 }
1094 
1095 /**
1096  * ice_link_event - process the link event
1097  * @pf: PF that the link event is associated with
1098  * @pi: port_info for the port that the link event is associated with
1099  * @link_up: true if the physical link is up and false if it is down
1100  * @link_speed: current link speed received from the link event
1101  *
1102  * Returns 0 on success and negative on failure
1103  */
1104 static int
1105 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up,
1106 	       u16 link_speed)
1107 {
1108 	struct device *dev = ice_pf_to_dev(pf);
1109 	struct ice_phy_info *phy_info;
1110 	struct ice_vsi *vsi;
1111 	u16 old_link_speed;
1112 	bool old_link;
1113 	int status;
1114 
1115 	phy_info = &pi->phy;
1116 	phy_info->link_info_old = phy_info->link_info;
1117 
1118 	old_link = !!(phy_info->link_info_old.link_info & ICE_AQ_LINK_UP);
1119 	old_link_speed = phy_info->link_info_old.link_speed;
1120 
1121 	/* update the link info structures and re-enable link events,
1122 	 * don't bail on failure due to other book keeping needed
1123 	 */
1124 	status = ice_update_link_info(pi);
1125 	if (status)
1126 		dev_dbg(dev, "Failed to update link status on port %d, err %d aq_err %s\n",
1127 			pi->lport, status,
1128 			ice_aq_str(pi->hw->adminq.sq_last_status));
1129 
1130 	ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err);
1131 
1132 	/* Check if the link state is up after updating link info, and treat
1133 	 * this event as an UP event since the link is actually UP now.
1134 	 */
1135 	if (phy_info->link_info.link_info & ICE_AQ_LINK_UP)
1136 		link_up = true;
1137 
1138 	vsi = ice_get_main_vsi(pf);
1139 	if (!vsi || !vsi->port_info)
1140 		return -EINVAL;
1141 
1142 	/* turn off PHY if media was removed */
1143 	if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags) &&
1144 	    !(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) {
1145 		set_bit(ICE_FLAG_NO_MEDIA, pf->flags);
1146 		ice_set_link(vsi, false);
1147 	}
1148 
1149 	/* if the old link up/down and speed is the same as the new */
1150 	if (link_up == old_link && link_speed == old_link_speed)
1151 		return 0;
1152 
1153 	ice_ptp_link_change(pf, pf->hw.pf_id, link_up);
1154 
1155 	if (ice_is_dcb_active(pf)) {
1156 		if (test_bit(ICE_FLAG_DCB_ENA, pf->flags))
1157 			ice_dcb_rebuild(pf);
1158 	} else {
1159 		if (link_up)
1160 			ice_set_dflt_mib(pf);
1161 	}
1162 	ice_vsi_link_event(vsi, link_up);
1163 	ice_print_link_msg(vsi, link_up);
1164 
1165 	ice_vc_notify_link_state(pf);
1166 
1167 	return 0;
1168 }
1169 
1170 /**
1171  * ice_watchdog_subtask - periodic tasks not using event driven scheduling
1172  * @pf: board private structure
1173  */
1174 static void ice_watchdog_subtask(struct ice_pf *pf)
1175 {
1176 	int i;
1177 
1178 	/* if interface is down do nothing */
1179 	if (test_bit(ICE_DOWN, pf->state) ||
1180 	    test_bit(ICE_CFG_BUSY, pf->state))
1181 		return;
1182 
1183 	/* make sure we don't do these things too often */
1184 	if (time_before(jiffies,
1185 			pf->serv_tmr_prev + pf->serv_tmr_period))
1186 		return;
1187 
1188 	pf->serv_tmr_prev = jiffies;
1189 
1190 	/* Update the stats for active netdevs so the network stack
1191 	 * can look at updated numbers whenever it cares to
1192 	 */
1193 	ice_update_pf_stats(pf);
1194 	ice_for_each_vsi(pf, i)
1195 		if (pf->vsi[i] && pf->vsi[i]->netdev)
1196 			ice_update_vsi_stats(pf->vsi[i]);
1197 }
1198 
1199 /**
1200  * ice_init_link_events - enable/initialize link events
1201  * @pi: pointer to the port_info instance
1202  *
1203  * Returns -EIO on failure, 0 on success
1204  */
1205 static int ice_init_link_events(struct ice_port_info *pi)
1206 {
1207 	u16 mask;
1208 
1209 	mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA |
1210 		       ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL |
1211 		       ICE_AQ_LINK_EVENT_PHY_FW_LOAD_FAIL));
1212 
1213 	if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) {
1214 		dev_dbg(ice_hw_to_dev(pi->hw), "Failed to set link event mask for port %d\n",
1215 			pi->lport);
1216 		return -EIO;
1217 	}
1218 
1219 	if (ice_aq_get_link_info(pi, true, NULL, NULL)) {
1220 		dev_dbg(ice_hw_to_dev(pi->hw), "Failed to enable link events for port %d\n",
1221 			pi->lport);
1222 		return -EIO;
1223 	}
1224 
1225 	return 0;
1226 }
1227 
1228 /**
1229  * ice_handle_link_event - handle link event via ARQ
1230  * @pf: PF that the link event is associated with
1231  * @event: event structure containing link status info
1232  */
1233 static int
1234 ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event)
1235 {
1236 	struct ice_aqc_get_link_status_data *link_data;
1237 	struct ice_port_info *port_info;
1238 	int status;
1239 
1240 	link_data = (struct ice_aqc_get_link_status_data *)event->msg_buf;
1241 	port_info = pf->hw.port_info;
1242 	if (!port_info)
1243 		return -EINVAL;
1244 
1245 	status = ice_link_event(pf, port_info,
1246 				!!(link_data->link_info & ICE_AQ_LINK_UP),
1247 				le16_to_cpu(link_data->link_speed));
1248 	if (status)
1249 		dev_dbg(ice_pf_to_dev(pf), "Could not process link event, error %d\n",
1250 			status);
1251 
1252 	return status;
1253 }
1254 
1255 /**
1256  * ice_aq_prep_for_event - Prepare to wait for an AdminQ event from firmware
1257  * @pf: pointer to the PF private structure
1258  * @task: intermediate helper storage and identifier for waiting
1259  * @opcode: the opcode to wait for
1260  *
1261  * Prepares to wait for a specific AdminQ completion event on the ARQ for
1262  * a given PF. Actual wait would be done by a call to ice_aq_wait_for_event().
1263  *
1264  * Calls are separated to allow caller registering for event before sending
1265  * the command, which mitigates a race between registering and FW responding.
1266  *
1267  * To obtain only the descriptor contents, pass an task->event with null
1268  * msg_buf. If the complete data buffer is desired, allocate the
1269  * task->event.msg_buf with enough space ahead of time.
1270  */
1271 void ice_aq_prep_for_event(struct ice_pf *pf, struct ice_aq_task *task,
1272 			   u16 opcode)
1273 {
1274 	INIT_HLIST_NODE(&task->entry);
1275 	task->opcode = opcode;
1276 	task->state = ICE_AQ_TASK_WAITING;
1277 
1278 	spin_lock_bh(&pf->aq_wait_lock);
1279 	hlist_add_head(&task->entry, &pf->aq_wait_list);
1280 	spin_unlock_bh(&pf->aq_wait_lock);
1281 }
1282 
1283 /**
1284  * ice_aq_wait_for_event - Wait for an AdminQ event from firmware
1285  * @pf: pointer to the PF private structure
1286  * @task: ptr prepared by ice_aq_prep_for_event()
1287  * @timeout: how long to wait, in jiffies
1288  *
1289  * Waits for a specific AdminQ completion event on the ARQ for a given PF. The
1290  * current thread will be put to sleep until the specified event occurs or
1291  * until the given timeout is reached.
1292  *
1293  * Returns: zero on success, or a negative error code on failure.
1294  */
1295 int ice_aq_wait_for_event(struct ice_pf *pf, struct ice_aq_task *task,
1296 			  unsigned long timeout)
1297 {
1298 	enum ice_aq_task_state *state = &task->state;
1299 	struct device *dev = ice_pf_to_dev(pf);
1300 	unsigned long start = jiffies;
1301 	long ret;
1302 	int err;
1303 
1304 	ret = wait_event_interruptible_timeout(pf->aq_wait_queue,
1305 					       *state != ICE_AQ_TASK_WAITING,
1306 					       timeout);
1307 	switch (*state) {
1308 	case ICE_AQ_TASK_NOT_PREPARED:
1309 		WARN(1, "call to %s without ice_aq_prep_for_event()", __func__);
1310 		err = -EINVAL;
1311 		break;
1312 	case ICE_AQ_TASK_WAITING:
1313 		err = ret < 0 ? ret : -ETIMEDOUT;
1314 		break;
1315 	case ICE_AQ_TASK_CANCELED:
1316 		err = ret < 0 ? ret : -ECANCELED;
1317 		break;
1318 	case ICE_AQ_TASK_COMPLETE:
1319 		err = ret < 0 ? ret : 0;
1320 		break;
1321 	default:
1322 		WARN(1, "Unexpected AdminQ wait task state %u", *state);
1323 		err = -EINVAL;
1324 		break;
1325 	}
1326 
1327 	dev_dbg(dev, "Waited %u msecs (max %u msecs) for firmware response to op 0x%04x\n",
1328 		jiffies_to_msecs(jiffies - start),
1329 		jiffies_to_msecs(timeout),
1330 		task->opcode);
1331 
1332 	spin_lock_bh(&pf->aq_wait_lock);
1333 	hlist_del(&task->entry);
1334 	spin_unlock_bh(&pf->aq_wait_lock);
1335 
1336 	return err;
1337 }
1338 
1339 /**
1340  * ice_aq_check_events - Check if any thread is waiting for an AdminQ event
1341  * @pf: pointer to the PF private structure
1342  * @opcode: the opcode of the event
1343  * @event: the event to check
1344  *
1345  * Loops over the current list of pending threads waiting for an AdminQ event.
1346  * For each matching task, copy the contents of the event into the task
1347  * structure and wake up the thread.
1348  *
1349  * If multiple threads wait for the same opcode, they will all be woken up.
1350  *
1351  * Note that event->msg_buf will only be duplicated if the event has a buffer
1352  * with enough space already allocated. Otherwise, only the descriptor and
1353  * message length will be copied.
1354  *
1355  * Returns: true if an event was found, false otherwise
1356  */
1357 static void ice_aq_check_events(struct ice_pf *pf, u16 opcode,
1358 				struct ice_rq_event_info *event)
1359 {
1360 	struct ice_rq_event_info *task_ev;
1361 	struct ice_aq_task *task;
1362 	bool found = false;
1363 
1364 	spin_lock_bh(&pf->aq_wait_lock);
1365 	hlist_for_each_entry(task, &pf->aq_wait_list, entry) {
1366 		if (task->state != ICE_AQ_TASK_WAITING)
1367 			continue;
1368 		if (task->opcode != opcode)
1369 			continue;
1370 
1371 		task_ev = &task->event;
1372 		memcpy(&task_ev->desc, &event->desc, sizeof(event->desc));
1373 		task_ev->msg_len = event->msg_len;
1374 
1375 		/* Only copy the data buffer if a destination was set */
1376 		if (task_ev->msg_buf && task_ev->buf_len >= event->buf_len) {
1377 			memcpy(task_ev->msg_buf, event->msg_buf,
1378 			       event->buf_len);
1379 			task_ev->buf_len = event->buf_len;
1380 		}
1381 
1382 		task->state = ICE_AQ_TASK_COMPLETE;
1383 		found = true;
1384 	}
1385 	spin_unlock_bh(&pf->aq_wait_lock);
1386 
1387 	if (found)
1388 		wake_up(&pf->aq_wait_queue);
1389 }
1390 
1391 /**
1392  * ice_aq_cancel_waiting_tasks - Immediately cancel all waiting tasks
1393  * @pf: the PF private structure
1394  *
1395  * Set all waiting tasks to ICE_AQ_TASK_CANCELED, and wake up their threads.
1396  * This will then cause ice_aq_wait_for_event to exit with -ECANCELED.
1397  */
1398 static void ice_aq_cancel_waiting_tasks(struct ice_pf *pf)
1399 {
1400 	struct ice_aq_task *task;
1401 
1402 	spin_lock_bh(&pf->aq_wait_lock);
1403 	hlist_for_each_entry(task, &pf->aq_wait_list, entry)
1404 		task->state = ICE_AQ_TASK_CANCELED;
1405 	spin_unlock_bh(&pf->aq_wait_lock);
1406 
1407 	wake_up(&pf->aq_wait_queue);
1408 }
1409 
1410 #define ICE_MBX_OVERFLOW_WATERMARK 64
1411 
1412 /**
1413  * __ice_clean_ctrlq - helper function to clean controlq rings
1414  * @pf: ptr to struct ice_pf
1415  * @q_type: specific Control queue type
1416  */
1417 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type)
1418 {
1419 	struct device *dev = ice_pf_to_dev(pf);
1420 	struct ice_rq_event_info event;
1421 	struct ice_hw *hw = &pf->hw;
1422 	struct ice_ctl_q_info *cq;
1423 	u16 pending, i = 0;
1424 	const char *qtype;
1425 	u32 oldval, val;
1426 
1427 	/* Do not clean control queue if/when PF reset fails */
1428 	if (test_bit(ICE_RESET_FAILED, pf->state))
1429 		return 0;
1430 
1431 	switch (q_type) {
1432 	case ICE_CTL_Q_ADMIN:
1433 		cq = &hw->adminq;
1434 		qtype = "Admin";
1435 		break;
1436 	case ICE_CTL_Q_SB:
1437 		cq = &hw->sbq;
1438 		qtype = "Sideband";
1439 		break;
1440 	case ICE_CTL_Q_MAILBOX:
1441 		cq = &hw->mailboxq;
1442 		qtype = "Mailbox";
1443 		/* we are going to try to detect a malicious VF, so set the
1444 		 * state to begin detection
1445 		 */
1446 		hw->mbx_snapshot.mbx_buf.state = ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT;
1447 		break;
1448 	default:
1449 		dev_warn(dev, "Unknown control queue type 0x%x\n", q_type);
1450 		return 0;
1451 	}
1452 
1453 	/* check for error indications - PF_xx_AxQLEN register layout for
1454 	 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN.
1455 	 */
1456 	val = rd32(hw, cq->rq.len);
1457 	if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
1458 		   PF_FW_ARQLEN_ARQCRIT_M)) {
1459 		oldval = val;
1460 		if (val & PF_FW_ARQLEN_ARQVFE_M)
1461 			dev_dbg(dev, "%s Receive Queue VF Error detected\n",
1462 				qtype);
1463 		if (val & PF_FW_ARQLEN_ARQOVFL_M) {
1464 			dev_dbg(dev, "%s Receive Queue Overflow Error detected\n",
1465 				qtype);
1466 		}
1467 		if (val & PF_FW_ARQLEN_ARQCRIT_M)
1468 			dev_dbg(dev, "%s Receive Queue Critical Error detected\n",
1469 				qtype);
1470 		val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
1471 			 PF_FW_ARQLEN_ARQCRIT_M);
1472 		if (oldval != val)
1473 			wr32(hw, cq->rq.len, val);
1474 	}
1475 
1476 	val = rd32(hw, cq->sq.len);
1477 	if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
1478 		   PF_FW_ATQLEN_ATQCRIT_M)) {
1479 		oldval = val;
1480 		if (val & PF_FW_ATQLEN_ATQVFE_M)
1481 			dev_dbg(dev, "%s Send Queue VF Error detected\n",
1482 				qtype);
1483 		if (val & PF_FW_ATQLEN_ATQOVFL_M) {
1484 			dev_dbg(dev, "%s Send Queue Overflow Error detected\n",
1485 				qtype);
1486 		}
1487 		if (val & PF_FW_ATQLEN_ATQCRIT_M)
1488 			dev_dbg(dev, "%s Send Queue Critical Error detected\n",
1489 				qtype);
1490 		val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
1491 			 PF_FW_ATQLEN_ATQCRIT_M);
1492 		if (oldval != val)
1493 			wr32(hw, cq->sq.len, val);
1494 	}
1495 
1496 	event.buf_len = cq->rq_buf_size;
1497 	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
1498 	if (!event.msg_buf)
1499 		return 0;
1500 
1501 	do {
1502 		struct ice_mbx_data data = {};
1503 		u16 opcode;
1504 		int ret;
1505 
1506 		ret = ice_clean_rq_elem(hw, cq, &event, &pending);
1507 		if (ret == -EALREADY)
1508 			break;
1509 		if (ret) {
1510 			dev_err(dev, "%s Receive Queue event error %d\n", qtype,
1511 				ret);
1512 			break;
1513 		}
1514 
1515 		opcode = le16_to_cpu(event.desc.opcode);
1516 
1517 		/* Notify any thread that might be waiting for this event */
1518 		ice_aq_check_events(pf, opcode, &event);
1519 
1520 		switch (opcode) {
1521 		case ice_aqc_opc_get_link_status:
1522 			if (ice_handle_link_event(pf, &event))
1523 				dev_err(dev, "Could not handle link event\n");
1524 			break;
1525 		case ice_aqc_opc_event_lan_overflow:
1526 			ice_vf_lan_overflow_event(pf, &event);
1527 			break;
1528 		case ice_mbx_opc_send_msg_to_pf:
1529 			data.num_msg_proc = i;
1530 			data.num_pending_arq = pending;
1531 			data.max_num_msgs_mbx = hw->mailboxq.num_rq_entries;
1532 			data.async_watermark_val = ICE_MBX_OVERFLOW_WATERMARK;
1533 
1534 			ice_vc_process_vf_msg(pf, &event, &data);
1535 			break;
1536 		case ice_aqc_opc_fw_logging:
1537 			ice_output_fw_log(hw, &event.desc, event.msg_buf);
1538 			break;
1539 		case ice_aqc_opc_lldp_set_mib_change:
1540 			ice_dcb_process_lldp_set_mib_change(pf, &event);
1541 			break;
1542 		default:
1543 			dev_dbg(dev, "%s Receive Queue unknown event 0x%04x ignored\n",
1544 				qtype, opcode);
1545 			break;
1546 		}
1547 	} while (pending && (i++ < ICE_DFLT_IRQ_WORK));
1548 
1549 	kfree(event.msg_buf);
1550 
1551 	return pending && (i == ICE_DFLT_IRQ_WORK);
1552 }
1553 
1554 /**
1555  * ice_ctrlq_pending - check if there is a difference between ntc and ntu
1556  * @hw: pointer to hardware info
1557  * @cq: control queue information
1558  *
1559  * returns true if there are pending messages in a queue, false if there aren't
1560  */
1561 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq)
1562 {
1563 	u16 ntu;
1564 
1565 	ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1566 	return cq->rq.next_to_clean != ntu;
1567 }
1568 
1569 /**
1570  * ice_clean_adminq_subtask - clean the AdminQ rings
1571  * @pf: board private structure
1572  */
1573 static void ice_clean_adminq_subtask(struct ice_pf *pf)
1574 {
1575 	struct ice_hw *hw = &pf->hw;
1576 
1577 	if (!test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state))
1578 		return;
1579 
1580 	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN))
1581 		return;
1582 
1583 	clear_bit(ICE_ADMINQ_EVENT_PENDING, pf->state);
1584 
1585 	/* There might be a situation where new messages arrive to a control
1586 	 * queue between processing the last message and clearing the
1587 	 * EVENT_PENDING bit. So before exiting, check queue head again (using
1588 	 * ice_ctrlq_pending) and process new messages if any.
1589 	 */
1590 	if (ice_ctrlq_pending(hw, &hw->adminq))
1591 		__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN);
1592 
1593 	ice_flush(hw);
1594 }
1595 
1596 /**
1597  * ice_clean_mailboxq_subtask - clean the MailboxQ rings
1598  * @pf: board private structure
1599  */
1600 static void ice_clean_mailboxq_subtask(struct ice_pf *pf)
1601 {
1602 	struct ice_hw *hw = &pf->hw;
1603 
1604 	if (!test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state))
1605 		return;
1606 
1607 	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX))
1608 		return;
1609 
1610 	clear_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state);
1611 
1612 	if (ice_ctrlq_pending(hw, &hw->mailboxq))
1613 		__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX);
1614 
1615 	ice_flush(hw);
1616 }
1617 
1618 /**
1619  * ice_clean_sbq_subtask - clean the Sideband Queue rings
1620  * @pf: board private structure
1621  */
1622 static void ice_clean_sbq_subtask(struct ice_pf *pf)
1623 {
1624 	struct ice_hw *hw = &pf->hw;
1625 
1626 	/* Nothing to do here if sideband queue is not supported */
1627 	if (!ice_is_sbq_supported(hw)) {
1628 		clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state);
1629 		return;
1630 	}
1631 
1632 	if (!test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state))
1633 		return;
1634 
1635 	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_SB))
1636 		return;
1637 
1638 	clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state);
1639 
1640 	if (ice_ctrlq_pending(hw, &hw->sbq))
1641 		__ice_clean_ctrlq(pf, ICE_CTL_Q_SB);
1642 
1643 	ice_flush(hw);
1644 }
1645 
1646 /**
1647  * ice_service_task_schedule - schedule the service task to wake up
1648  * @pf: board private structure
1649  *
1650  * If not already scheduled, this puts the task into the work queue.
1651  */
1652 void ice_service_task_schedule(struct ice_pf *pf)
1653 {
1654 	if (!test_bit(ICE_SERVICE_DIS, pf->state) &&
1655 	    !test_and_set_bit(ICE_SERVICE_SCHED, pf->state) &&
1656 	    !test_bit(ICE_NEEDS_RESTART, pf->state))
1657 		queue_work(ice_wq, &pf->serv_task);
1658 }
1659 
1660 /**
1661  * ice_service_task_complete - finish up the service task
1662  * @pf: board private structure
1663  */
1664 static void ice_service_task_complete(struct ice_pf *pf)
1665 {
1666 	WARN_ON(!test_bit(ICE_SERVICE_SCHED, pf->state));
1667 
1668 	/* force memory (pf->state) to sync before next service task */
1669 	smp_mb__before_atomic();
1670 	clear_bit(ICE_SERVICE_SCHED, pf->state);
1671 }
1672 
1673 /**
1674  * ice_service_task_stop - stop service task and cancel works
1675  * @pf: board private structure
1676  *
1677  * Return 0 if the ICE_SERVICE_DIS bit was not already set,
1678  * 1 otherwise.
1679  */
1680 static int ice_service_task_stop(struct ice_pf *pf)
1681 {
1682 	int ret;
1683 
1684 	ret = test_and_set_bit(ICE_SERVICE_DIS, pf->state);
1685 
1686 	if (pf->serv_tmr.function)
1687 		del_timer_sync(&pf->serv_tmr);
1688 	if (pf->serv_task.func)
1689 		cancel_work_sync(&pf->serv_task);
1690 
1691 	clear_bit(ICE_SERVICE_SCHED, pf->state);
1692 	return ret;
1693 }
1694 
1695 /**
1696  * ice_service_task_restart - restart service task and schedule works
1697  * @pf: board private structure
1698  *
1699  * This function is needed for suspend and resume works (e.g WoL scenario)
1700  */
1701 static void ice_service_task_restart(struct ice_pf *pf)
1702 {
1703 	clear_bit(ICE_SERVICE_DIS, pf->state);
1704 	ice_service_task_schedule(pf);
1705 }
1706 
1707 /**
1708  * ice_service_timer - timer callback to schedule service task
1709  * @t: pointer to timer_list
1710  */
1711 static void ice_service_timer(struct timer_list *t)
1712 {
1713 	struct ice_pf *pf = from_timer(pf, t, serv_tmr);
1714 
1715 	mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies));
1716 	ice_service_task_schedule(pf);
1717 }
1718 
1719 /**
1720  * ice_handle_mdd_event - handle malicious driver detect event
1721  * @pf: pointer to the PF structure
1722  *
1723  * Called from service task. OICR interrupt handler indicates MDD event.
1724  * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log
1725  * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events
1726  * disable the queue, the PF can be configured to reset the VF using ethtool
1727  * private flag mdd-auto-reset-vf.
1728  */
1729 static void ice_handle_mdd_event(struct ice_pf *pf)
1730 {
1731 	struct device *dev = ice_pf_to_dev(pf);
1732 	struct ice_hw *hw = &pf->hw;
1733 	struct ice_vf *vf;
1734 	unsigned int bkt;
1735 	u32 reg;
1736 
1737 	if (!test_and_clear_bit(ICE_MDD_EVENT_PENDING, pf->state)) {
1738 		/* Since the VF MDD event logging is rate limited, check if
1739 		 * there are pending MDD events.
1740 		 */
1741 		ice_print_vfs_mdd_events(pf);
1742 		return;
1743 	}
1744 
1745 	/* find what triggered an MDD event */
1746 	reg = rd32(hw, GL_MDET_TX_PQM);
1747 	if (reg & GL_MDET_TX_PQM_VALID_M) {
1748 		u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >>
1749 				GL_MDET_TX_PQM_PF_NUM_S;
1750 		u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >>
1751 				GL_MDET_TX_PQM_VF_NUM_S;
1752 		u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >>
1753 				GL_MDET_TX_PQM_MAL_TYPE_S;
1754 		u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >>
1755 				GL_MDET_TX_PQM_QNUM_S);
1756 
1757 		if (netif_msg_tx_err(pf))
1758 			dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",
1759 				 event, queue, pf_num, vf_num);
1760 		wr32(hw, GL_MDET_TX_PQM, 0xffffffff);
1761 	}
1762 
1763 	reg = rd32(hw, GL_MDET_TX_TCLAN_BY_MAC(hw));
1764 	if (reg & GL_MDET_TX_TCLAN_VALID_M) {
1765 		u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >>
1766 				GL_MDET_TX_TCLAN_PF_NUM_S;
1767 		u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >>
1768 				GL_MDET_TX_TCLAN_VF_NUM_S;
1769 		u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >>
1770 				GL_MDET_TX_TCLAN_MAL_TYPE_S;
1771 		u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >>
1772 				GL_MDET_TX_TCLAN_QNUM_S);
1773 
1774 		if (netif_msg_tx_err(pf))
1775 			dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",
1776 				 event, queue, pf_num, vf_num);
1777 		wr32(hw, GL_MDET_TX_TCLAN_BY_MAC(hw), U32_MAX);
1778 	}
1779 
1780 	reg = rd32(hw, GL_MDET_RX);
1781 	if (reg & GL_MDET_RX_VALID_M) {
1782 		u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >>
1783 				GL_MDET_RX_PF_NUM_S;
1784 		u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >>
1785 				GL_MDET_RX_VF_NUM_S;
1786 		u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >>
1787 				GL_MDET_RX_MAL_TYPE_S;
1788 		u16 queue = ((reg & GL_MDET_RX_QNUM_M) >>
1789 				GL_MDET_RX_QNUM_S);
1790 
1791 		if (netif_msg_rx_err(pf))
1792 			dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n",
1793 				 event, queue, pf_num, vf_num);
1794 		wr32(hw, GL_MDET_RX, 0xffffffff);
1795 	}
1796 
1797 	/* check to see if this PF caused an MDD event */
1798 	reg = rd32(hw, PF_MDET_TX_PQM);
1799 	if (reg & PF_MDET_TX_PQM_VALID_M) {
1800 		wr32(hw, PF_MDET_TX_PQM, 0xFFFF);
1801 		if (netif_msg_tx_err(pf))
1802 			dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n");
1803 	}
1804 
1805 	reg = rd32(hw, PF_MDET_TX_TCLAN_BY_MAC(hw));
1806 	if (reg & PF_MDET_TX_TCLAN_VALID_M) {
1807 		wr32(hw, PF_MDET_TX_TCLAN_BY_MAC(hw), 0xffff);
1808 		if (netif_msg_tx_err(pf))
1809 			dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n");
1810 	}
1811 
1812 	reg = rd32(hw, PF_MDET_RX);
1813 	if (reg & PF_MDET_RX_VALID_M) {
1814 		wr32(hw, PF_MDET_RX, 0xFFFF);
1815 		if (netif_msg_rx_err(pf))
1816 			dev_info(dev, "Malicious Driver Detection event RX detected on PF\n");
1817 	}
1818 
1819 	/* Check to see if one of the VFs caused an MDD event, and then
1820 	 * increment counters and set print pending
1821 	 */
1822 	mutex_lock(&pf->vfs.table_lock);
1823 	ice_for_each_vf(pf, bkt, vf) {
1824 		reg = rd32(hw, VP_MDET_TX_PQM(vf->vf_id));
1825 		if (reg & VP_MDET_TX_PQM_VALID_M) {
1826 			wr32(hw, VP_MDET_TX_PQM(vf->vf_id), 0xFFFF);
1827 			vf->mdd_tx_events.count++;
1828 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1829 			if (netif_msg_tx_err(pf))
1830 				dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n",
1831 					 vf->vf_id);
1832 		}
1833 
1834 		reg = rd32(hw, VP_MDET_TX_TCLAN(vf->vf_id));
1835 		if (reg & VP_MDET_TX_TCLAN_VALID_M) {
1836 			wr32(hw, VP_MDET_TX_TCLAN(vf->vf_id), 0xFFFF);
1837 			vf->mdd_tx_events.count++;
1838 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1839 			if (netif_msg_tx_err(pf))
1840 				dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n",
1841 					 vf->vf_id);
1842 		}
1843 
1844 		reg = rd32(hw, VP_MDET_TX_TDPU(vf->vf_id));
1845 		if (reg & VP_MDET_TX_TDPU_VALID_M) {
1846 			wr32(hw, VP_MDET_TX_TDPU(vf->vf_id), 0xFFFF);
1847 			vf->mdd_tx_events.count++;
1848 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1849 			if (netif_msg_tx_err(pf))
1850 				dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n",
1851 					 vf->vf_id);
1852 		}
1853 
1854 		reg = rd32(hw, VP_MDET_RX(vf->vf_id));
1855 		if (reg & VP_MDET_RX_VALID_M) {
1856 			wr32(hw, VP_MDET_RX(vf->vf_id), 0xFFFF);
1857 			vf->mdd_rx_events.count++;
1858 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1859 			if (netif_msg_rx_err(pf))
1860 				dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n",
1861 					 vf->vf_id);
1862 
1863 			/* Since the queue is disabled on VF Rx MDD events, the
1864 			 * PF can be configured to reset the VF through ethtool
1865 			 * private flag mdd-auto-reset-vf.
1866 			 */
1867 			if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) {
1868 				/* VF MDD event counters will be cleared by
1869 				 * reset, so print the event prior to reset.
1870 				 */
1871 				ice_print_vf_rx_mdd_event(vf);
1872 				ice_reset_vf(vf, ICE_VF_RESET_LOCK);
1873 			}
1874 		}
1875 	}
1876 	mutex_unlock(&pf->vfs.table_lock);
1877 
1878 	ice_print_vfs_mdd_events(pf);
1879 }
1880 
1881 /**
1882  * ice_force_phys_link_state - Force the physical link state
1883  * @vsi: VSI to force the physical link state to up/down
1884  * @link_up: true/false indicates to set the physical link to up/down
1885  *
1886  * Force the physical link state by getting the current PHY capabilities from
1887  * hardware and setting the PHY config based on the determined capabilities. If
1888  * link changes a link event will be triggered because both the Enable Automatic
1889  * Link Update and LESM Enable bits are set when setting the PHY capabilities.
1890  *
1891  * Returns 0 on success, negative on failure
1892  */
1893 static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up)
1894 {
1895 	struct ice_aqc_get_phy_caps_data *pcaps;
1896 	struct ice_aqc_set_phy_cfg_data *cfg;
1897 	struct ice_port_info *pi;
1898 	struct device *dev;
1899 	int retcode;
1900 
1901 	if (!vsi || !vsi->port_info || !vsi->back)
1902 		return -EINVAL;
1903 	if (vsi->type != ICE_VSI_PF)
1904 		return 0;
1905 
1906 	dev = ice_pf_to_dev(vsi->back);
1907 
1908 	pi = vsi->port_info;
1909 
1910 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
1911 	if (!pcaps)
1912 		return -ENOMEM;
1913 
1914 	retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps,
1915 				      NULL);
1916 	if (retcode) {
1917 		dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n",
1918 			vsi->vsi_num, retcode);
1919 		retcode = -EIO;
1920 		goto out;
1921 	}
1922 
1923 	/* No change in link */
1924 	if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) &&
1925 	    link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP))
1926 		goto out;
1927 
1928 	/* Use the current user PHY configuration. The current user PHY
1929 	 * configuration is initialized during probe from PHY capabilities
1930 	 * software mode, and updated on set PHY configuration.
1931 	 */
1932 	cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL);
1933 	if (!cfg) {
1934 		retcode = -ENOMEM;
1935 		goto out;
1936 	}
1937 
1938 	cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT;
1939 	if (link_up)
1940 		cfg->caps |= ICE_AQ_PHY_ENA_LINK;
1941 	else
1942 		cfg->caps &= ~ICE_AQ_PHY_ENA_LINK;
1943 
1944 	retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL);
1945 	if (retcode) {
1946 		dev_err(dev, "Failed to set phy config, VSI %d error %d\n",
1947 			vsi->vsi_num, retcode);
1948 		retcode = -EIO;
1949 	}
1950 
1951 	kfree(cfg);
1952 out:
1953 	kfree(pcaps);
1954 	return retcode;
1955 }
1956 
1957 /**
1958  * ice_init_nvm_phy_type - Initialize the NVM PHY type
1959  * @pi: port info structure
1960  *
1961  * Initialize nvm_phy_type_[low|high] for link lenient mode support
1962  */
1963 static int ice_init_nvm_phy_type(struct ice_port_info *pi)
1964 {
1965 	struct ice_aqc_get_phy_caps_data *pcaps;
1966 	struct ice_pf *pf = pi->hw->back;
1967 	int err;
1968 
1969 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
1970 	if (!pcaps)
1971 		return -ENOMEM;
1972 
1973 	err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA,
1974 				  pcaps, NULL);
1975 
1976 	if (err) {
1977 		dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n");
1978 		goto out;
1979 	}
1980 
1981 	pf->nvm_phy_type_hi = pcaps->phy_type_high;
1982 	pf->nvm_phy_type_lo = pcaps->phy_type_low;
1983 
1984 out:
1985 	kfree(pcaps);
1986 	return err;
1987 }
1988 
1989 /**
1990  * ice_init_link_dflt_override - Initialize link default override
1991  * @pi: port info structure
1992  *
1993  * Initialize link default override and PHY total port shutdown during probe
1994  */
1995 static void ice_init_link_dflt_override(struct ice_port_info *pi)
1996 {
1997 	struct ice_link_default_override_tlv *ldo;
1998 	struct ice_pf *pf = pi->hw->back;
1999 
2000 	ldo = &pf->link_dflt_override;
2001 	if (ice_get_link_default_override(ldo, pi))
2002 		return;
2003 
2004 	if (!(ldo->options & ICE_LINK_OVERRIDE_PORT_DIS))
2005 		return;
2006 
2007 	/* Enable Total Port Shutdown (override/replace link-down-on-close
2008 	 * ethtool private flag) for ports with Port Disable bit set.
2009 	 */
2010 	set_bit(ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);
2011 	set_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);
2012 }
2013 
2014 /**
2015  * ice_init_phy_cfg_dflt_override - Initialize PHY cfg default override settings
2016  * @pi: port info structure
2017  *
2018  * If default override is enabled, initialize the user PHY cfg speed and FEC
2019  * settings using the default override mask from the NVM.
2020  *
2021  * The PHY should only be configured with the default override settings the
2022  * first time media is available. The ICE_LINK_DEFAULT_OVERRIDE_PENDING state
2023  * is used to indicate that the user PHY cfg default override is initialized
2024  * and the PHY has not been configured with the default override settings. The
2025  * state is set here, and cleared in ice_configure_phy the first time the PHY is
2026  * configured.
2027  *
2028  * This function should be called only if the FW doesn't support default
2029  * configuration mode, as reported by ice_fw_supports_report_dflt_cfg.
2030  */
2031 static void ice_init_phy_cfg_dflt_override(struct ice_port_info *pi)
2032 {
2033 	struct ice_link_default_override_tlv *ldo;
2034 	struct ice_aqc_set_phy_cfg_data *cfg;
2035 	struct ice_phy_info *phy = &pi->phy;
2036 	struct ice_pf *pf = pi->hw->back;
2037 
2038 	ldo = &pf->link_dflt_override;
2039 
2040 	/* If link default override is enabled, use to mask NVM PHY capabilities
2041 	 * for speed and FEC default configuration.
2042 	 */
2043 	cfg = &phy->curr_user_phy_cfg;
2044 
2045 	if (ldo->phy_type_low || ldo->phy_type_high) {
2046 		cfg->phy_type_low = pf->nvm_phy_type_lo &
2047 				    cpu_to_le64(ldo->phy_type_low);
2048 		cfg->phy_type_high = pf->nvm_phy_type_hi &
2049 				     cpu_to_le64(ldo->phy_type_high);
2050 	}
2051 	cfg->link_fec_opt = ldo->fec_options;
2052 	phy->curr_user_fec_req = ICE_FEC_AUTO;
2053 
2054 	set_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING, pf->state);
2055 }
2056 
2057 /**
2058  * ice_init_phy_user_cfg - Initialize the PHY user configuration
2059  * @pi: port info structure
2060  *
2061  * Initialize the current user PHY configuration, speed, FEC, and FC requested
2062  * mode to default. The PHY defaults are from get PHY capabilities topology
2063  * with media so call when media is first available. An error is returned if
2064  * called when media is not available. The PHY initialization completed state is
2065  * set here.
2066  *
2067  * These configurations are used when setting PHY
2068  * configuration. The user PHY configuration is updated on set PHY
2069  * configuration. Returns 0 on success, negative on failure
2070  */
2071 static int ice_init_phy_user_cfg(struct ice_port_info *pi)
2072 {
2073 	struct ice_aqc_get_phy_caps_data *pcaps;
2074 	struct ice_phy_info *phy = &pi->phy;
2075 	struct ice_pf *pf = pi->hw->back;
2076 	int err;
2077 
2078 	if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE))
2079 		return -EIO;
2080 
2081 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
2082 	if (!pcaps)
2083 		return -ENOMEM;
2084 
2085 	if (ice_fw_supports_report_dflt_cfg(pi->hw))
2086 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG,
2087 					  pcaps, NULL);
2088 	else
2089 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA,
2090 					  pcaps, NULL);
2091 	if (err) {
2092 		dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n");
2093 		goto err_out;
2094 	}
2095 
2096 	ice_copy_phy_caps_to_cfg(pi, pcaps, &pi->phy.curr_user_phy_cfg);
2097 
2098 	/* check if lenient mode is supported and enabled */
2099 	if (ice_fw_supports_link_override(pi->hw) &&
2100 	    !(pcaps->module_compliance_enforcement &
2101 	      ICE_AQC_MOD_ENFORCE_STRICT_MODE)) {
2102 		set_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags);
2103 
2104 		/* if the FW supports default PHY configuration mode, then the driver
2105 		 * does not have to apply link override settings. If not,
2106 		 * initialize user PHY configuration with link override values
2107 		 */
2108 		if (!ice_fw_supports_report_dflt_cfg(pi->hw) &&
2109 		    (pf->link_dflt_override.options & ICE_LINK_OVERRIDE_EN)) {
2110 			ice_init_phy_cfg_dflt_override(pi);
2111 			goto out;
2112 		}
2113 	}
2114 
2115 	/* if link default override is not enabled, set user flow control and
2116 	 * FEC settings based on what get_phy_caps returned
2117 	 */
2118 	phy->curr_user_fec_req = ice_caps_to_fec_mode(pcaps->caps,
2119 						      pcaps->link_fec_options);
2120 	phy->curr_user_fc_req = ice_caps_to_fc_mode(pcaps->caps);
2121 
2122 out:
2123 	phy->curr_user_speed_req = ICE_AQ_LINK_SPEED_M;
2124 	set_bit(ICE_PHY_INIT_COMPLETE, pf->state);
2125 err_out:
2126 	kfree(pcaps);
2127 	return err;
2128 }
2129 
2130 /**
2131  * ice_configure_phy - configure PHY
2132  * @vsi: VSI of PHY
2133  *
2134  * Set the PHY configuration. If the current PHY configuration is the same as
2135  * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise
2136  * configure the based get PHY capabilities for topology with media.
2137  */
2138 static int ice_configure_phy(struct ice_vsi *vsi)
2139 {
2140 	struct device *dev = ice_pf_to_dev(vsi->back);
2141 	struct ice_port_info *pi = vsi->port_info;
2142 	struct ice_aqc_get_phy_caps_data *pcaps;
2143 	struct ice_aqc_set_phy_cfg_data *cfg;
2144 	struct ice_phy_info *phy = &pi->phy;
2145 	struct ice_pf *pf = vsi->back;
2146 	int err;
2147 
2148 	/* Ensure we have media as we cannot configure a medialess port */
2149 	if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE))
2150 		return -EPERM;
2151 
2152 	ice_print_topo_conflict(vsi);
2153 
2154 	if (!test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags) &&
2155 	    phy->link_info.topo_media_conflict == ICE_AQ_LINK_TOPO_UNSUPP_MEDIA)
2156 		return -EPERM;
2157 
2158 	if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))
2159 		return ice_force_phys_link_state(vsi, true);
2160 
2161 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
2162 	if (!pcaps)
2163 		return -ENOMEM;
2164 
2165 	/* Get current PHY config */
2166 	err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps,
2167 				  NULL);
2168 	if (err) {
2169 		dev_err(dev, "Failed to get PHY configuration, VSI %d error %d\n",
2170 			vsi->vsi_num, err);
2171 		goto done;
2172 	}
2173 
2174 	/* If PHY enable link is configured and configuration has not changed,
2175 	 * there's nothing to do
2176 	 */
2177 	if (pcaps->caps & ICE_AQC_PHY_EN_LINK &&
2178 	    ice_phy_caps_equals_cfg(pcaps, &phy->curr_user_phy_cfg))
2179 		goto done;
2180 
2181 	/* Use PHY topology as baseline for configuration */
2182 	memset(pcaps, 0, sizeof(*pcaps));
2183 	if (ice_fw_supports_report_dflt_cfg(pi->hw))
2184 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG,
2185 					  pcaps, NULL);
2186 	else
2187 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA,
2188 					  pcaps, NULL);
2189 	if (err) {
2190 		dev_err(dev, "Failed to get PHY caps, VSI %d error %d\n",
2191 			vsi->vsi_num, err);
2192 		goto done;
2193 	}
2194 
2195 	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
2196 	if (!cfg) {
2197 		err = -ENOMEM;
2198 		goto done;
2199 	}
2200 
2201 	ice_copy_phy_caps_to_cfg(pi, pcaps, cfg);
2202 
2203 	/* Speed - If default override pending, use curr_user_phy_cfg set in
2204 	 * ice_init_phy_user_cfg_ldo.
2205 	 */
2206 	if (test_and_clear_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING,
2207 			       vsi->back->state)) {
2208 		cfg->phy_type_low = phy->curr_user_phy_cfg.phy_type_low;
2209 		cfg->phy_type_high = phy->curr_user_phy_cfg.phy_type_high;
2210 	} else {
2211 		u64 phy_low = 0, phy_high = 0;
2212 
2213 		ice_update_phy_type(&phy_low, &phy_high,
2214 				    pi->phy.curr_user_speed_req);
2215 		cfg->phy_type_low = pcaps->phy_type_low & cpu_to_le64(phy_low);
2216 		cfg->phy_type_high = pcaps->phy_type_high &
2217 				     cpu_to_le64(phy_high);
2218 	}
2219 
2220 	/* Can't provide what was requested; use PHY capabilities */
2221 	if (!cfg->phy_type_low && !cfg->phy_type_high) {
2222 		cfg->phy_type_low = pcaps->phy_type_low;
2223 		cfg->phy_type_high = pcaps->phy_type_high;
2224 	}
2225 
2226 	/* FEC */
2227 	ice_cfg_phy_fec(pi, cfg, phy->curr_user_fec_req);
2228 
2229 	/* Can't provide what was requested; use PHY capabilities */
2230 	if (cfg->link_fec_opt !=
2231 	    (cfg->link_fec_opt & pcaps->link_fec_options)) {
2232 		cfg->caps |= pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC;
2233 		cfg->link_fec_opt = pcaps->link_fec_options;
2234 	}
2235 
2236 	/* Flow Control - always supported; no need to check against
2237 	 * capabilities
2238 	 */
2239 	ice_cfg_phy_fc(pi, cfg, phy->curr_user_fc_req);
2240 
2241 	/* Enable link and link update */
2242 	cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK;
2243 
2244 	err = ice_aq_set_phy_cfg(&pf->hw, pi, cfg, NULL);
2245 	if (err)
2246 		dev_err(dev, "Failed to set phy config, VSI %d error %d\n",
2247 			vsi->vsi_num, err);
2248 
2249 	kfree(cfg);
2250 done:
2251 	kfree(pcaps);
2252 	return err;
2253 }
2254 
2255 /**
2256  * ice_check_media_subtask - Check for media
2257  * @pf: pointer to PF struct
2258  *
2259  * If media is available, then initialize PHY user configuration if it is not
2260  * been, and configure the PHY if the interface is up.
2261  */
2262 static void ice_check_media_subtask(struct ice_pf *pf)
2263 {
2264 	struct ice_port_info *pi;
2265 	struct ice_vsi *vsi;
2266 	int err;
2267 
2268 	/* No need to check for media if it's already present */
2269 	if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags))
2270 		return;
2271 
2272 	vsi = ice_get_main_vsi(pf);
2273 	if (!vsi)
2274 		return;
2275 
2276 	/* Refresh link info and check if media is present */
2277 	pi = vsi->port_info;
2278 	err = ice_update_link_info(pi);
2279 	if (err)
2280 		return;
2281 
2282 	ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err);
2283 
2284 	if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
2285 		if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state))
2286 			ice_init_phy_user_cfg(pi);
2287 
2288 		/* PHY settings are reset on media insertion, reconfigure
2289 		 * PHY to preserve settings.
2290 		 */
2291 		if (test_bit(ICE_VSI_DOWN, vsi->state) &&
2292 		    test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags))
2293 			return;
2294 
2295 		err = ice_configure_phy(vsi);
2296 		if (!err)
2297 			clear_bit(ICE_FLAG_NO_MEDIA, pf->flags);
2298 
2299 		/* A Link Status Event will be generated; the event handler
2300 		 * will complete bringing the interface up
2301 		 */
2302 	}
2303 }
2304 
2305 /**
2306  * ice_service_task - manage and run subtasks
2307  * @work: pointer to work_struct contained by the PF struct
2308  */
2309 static void ice_service_task(struct work_struct *work)
2310 {
2311 	struct ice_pf *pf = container_of(work, struct ice_pf, serv_task);
2312 	unsigned long start_time = jiffies;
2313 
2314 	/* subtasks */
2315 
2316 	/* process reset requests first */
2317 	ice_reset_subtask(pf);
2318 
2319 	/* bail if a reset/recovery cycle is pending or rebuild failed */
2320 	if (ice_is_reset_in_progress(pf->state) ||
2321 	    test_bit(ICE_SUSPENDED, pf->state) ||
2322 	    test_bit(ICE_NEEDS_RESTART, pf->state)) {
2323 		ice_service_task_complete(pf);
2324 		return;
2325 	}
2326 
2327 	if (test_and_clear_bit(ICE_AUX_ERR_PENDING, pf->state)) {
2328 		struct iidc_event *event;
2329 
2330 		event = kzalloc(sizeof(*event), GFP_KERNEL);
2331 		if (event) {
2332 			set_bit(IIDC_EVENT_CRIT_ERR, event->type);
2333 			/* report the entire OICR value to AUX driver */
2334 			swap(event->reg, pf->oicr_err_reg);
2335 			ice_send_event_to_aux(pf, event);
2336 			kfree(event);
2337 		}
2338 	}
2339 
2340 	/* unplug aux dev per request, if an unplug request came in
2341 	 * while processing a plug request, this will handle it
2342 	 */
2343 	if (test_and_clear_bit(ICE_FLAG_UNPLUG_AUX_DEV, pf->flags))
2344 		ice_unplug_aux_dev(pf);
2345 
2346 	/* Plug aux device per request */
2347 	if (test_and_clear_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags))
2348 		ice_plug_aux_dev(pf);
2349 
2350 	if (test_and_clear_bit(ICE_FLAG_MTU_CHANGED, pf->flags)) {
2351 		struct iidc_event *event;
2352 
2353 		event = kzalloc(sizeof(*event), GFP_KERNEL);
2354 		if (event) {
2355 			set_bit(IIDC_EVENT_AFTER_MTU_CHANGE, event->type);
2356 			ice_send_event_to_aux(pf, event);
2357 			kfree(event);
2358 		}
2359 	}
2360 
2361 	ice_clean_adminq_subtask(pf);
2362 	ice_check_media_subtask(pf);
2363 	ice_check_for_hang_subtask(pf);
2364 	ice_sync_fltr_subtask(pf);
2365 	ice_handle_mdd_event(pf);
2366 	ice_watchdog_subtask(pf);
2367 
2368 	if (ice_is_safe_mode(pf)) {
2369 		ice_service_task_complete(pf);
2370 		return;
2371 	}
2372 
2373 	ice_process_vflr_event(pf);
2374 	ice_clean_mailboxq_subtask(pf);
2375 	ice_clean_sbq_subtask(pf);
2376 	ice_sync_arfs_fltrs(pf);
2377 	ice_flush_fdir_ctx(pf);
2378 
2379 	/* Clear ICE_SERVICE_SCHED flag to allow scheduling next event */
2380 	ice_service_task_complete(pf);
2381 
2382 	/* If the tasks have taken longer than one service timer period
2383 	 * or there is more work to be done, reset the service timer to
2384 	 * schedule the service task now.
2385 	 */
2386 	if (time_after(jiffies, (start_time + pf->serv_tmr_period)) ||
2387 	    test_bit(ICE_MDD_EVENT_PENDING, pf->state) ||
2388 	    test_bit(ICE_VFLR_EVENT_PENDING, pf->state) ||
2389 	    test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state) ||
2390 	    test_bit(ICE_FD_VF_FLUSH_CTX, pf->state) ||
2391 	    test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state) ||
2392 	    test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state))
2393 		mod_timer(&pf->serv_tmr, jiffies);
2394 }
2395 
2396 /**
2397  * ice_set_ctrlq_len - helper function to set controlq length
2398  * @hw: pointer to the HW instance
2399  */
2400 static void ice_set_ctrlq_len(struct ice_hw *hw)
2401 {
2402 	hw->adminq.num_rq_entries = ICE_AQ_LEN;
2403 	hw->adminq.num_sq_entries = ICE_AQ_LEN;
2404 	hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
2405 	hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;
2406 	hw->mailboxq.num_rq_entries = PF_MBX_ARQLEN_ARQLEN_M;
2407 	hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN;
2408 	hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
2409 	hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
2410 	hw->sbq.num_rq_entries = ICE_SBQ_LEN;
2411 	hw->sbq.num_sq_entries = ICE_SBQ_LEN;
2412 	hw->sbq.rq_buf_size = ICE_SBQ_MAX_BUF_LEN;
2413 	hw->sbq.sq_buf_size = ICE_SBQ_MAX_BUF_LEN;
2414 }
2415 
2416 /**
2417  * ice_schedule_reset - schedule a reset
2418  * @pf: board private structure
2419  * @reset: reset being requested
2420  */
2421 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset)
2422 {
2423 	struct device *dev = ice_pf_to_dev(pf);
2424 
2425 	/* bail out if earlier reset has failed */
2426 	if (test_bit(ICE_RESET_FAILED, pf->state)) {
2427 		dev_dbg(dev, "earlier reset has failed\n");
2428 		return -EIO;
2429 	}
2430 	/* bail if reset/recovery already in progress */
2431 	if (ice_is_reset_in_progress(pf->state)) {
2432 		dev_dbg(dev, "Reset already in progress\n");
2433 		return -EBUSY;
2434 	}
2435 
2436 	switch (reset) {
2437 	case ICE_RESET_PFR:
2438 		set_bit(ICE_PFR_REQ, pf->state);
2439 		break;
2440 	case ICE_RESET_CORER:
2441 		set_bit(ICE_CORER_REQ, pf->state);
2442 		break;
2443 	case ICE_RESET_GLOBR:
2444 		set_bit(ICE_GLOBR_REQ, pf->state);
2445 		break;
2446 	default:
2447 		return -EINVAL;
2448 	}
2449 
2450 	ice_service_task_schedule(pf);
2451 	return 0;
2452 }
2453 
2454 /**
2455  * ice_irq_affinity_notify - Callback for affinity changes
2456  * @notify: context as to what irq was changed
2457  * @mask: the new affinity mask
2458  *
2459  * This is a callback function used by the irq_set_affinity_notifier function
2460  * so that we may register to receive changes to the irq affinity masks.
2461  */
2462 static void
2463 ice_irq_affinity_notify(struct irq_affinity_notify *notify,
2464 			const cpumask_t *mask)
2465 {
2466 	struct ice_q_vector *q_vector =
2467 		container_of(notify, struct ice_q_vector, affinity_notify);
2468 
2469 	cpumask_copy(&q_vector->affinity_mask, mask);
2470 }
2471 
2472 /**
2473  * ice_irq_affinity_release - Callback for affinity notifier release
2474  * @ref: internal core kernel usage
2475  *
2476  * This is a callback function used by the irq_set_affinity_notifier function
2477  * to inform the current notification subscriber that they will no longer
2478  * receive notifications.
2479  */
2480 static void ice_irq_affinity_release(struct kref __always_unused *ref) {}
2481 
2482 /**
2483  * ice_vsi_ena_irq - Enable IRQ for the given VSI
2484  * @vsi: the VSI being configured
2485  */
2486 static int ice_vsi_ena_irq(struct ice_vsi *vsi)
2487 {
2488 	struct ice_hw *hw = &vsi->back->hw;
2489 	int i;
2490 
2491 	ice_for_each_q_vector(vsi, i)
2492 		ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]);
2493 
2494 	ice_flush(hw);
2495 	return 0;
2496 }
2497 
2498 /**
2499  * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI
2500  * @vsi: the VSI being configured
2501  * @basename: name for the vector
2502  */
2503 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename)
2504 {
2505 	int q_vectors = vsi->num_q_vectors;
2506 	struct ice_pf *pf = vsi->back;
2507 	struct device *dev;
2508 	int rx_int_idx = 0;
2509 	int tx_int_idx = 0;
2510 	int vector, err;
2511 	int irq_num;
2512 
2513 	dev = ice_pf_to_dev(pf);
2514 	for (vector = 0; vector < q_vectors; vector++) {
2515 		struct ice_q_vector *q_vector = vsi->q_vectors[vector];
2516 
2517 		irq_num = q_vector->irq.virq;
2518 
2519 		if (q_vector->tx.tx_ring && q_vector->rx.rx_ring) {
2520 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2521 				 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
2522 			tx_int_idx++;
2523 		} else if (q_vector->rx.rx_ring) {
2524 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2525 				 "%s-%s-%d", basename, "rx", rx_int_idx++);
2526 		} else if (q_vector->tx.tx_ring) {
2527 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2528 				 "%s-%s-%d", basename, "tx", tx_int_idx++);
2529 		} else {
2530 			/* skip this unused q_vector */
2531 			continue;
2532 		}
2533 		if (vsi->type == ICE_VSI_CTRL && vsi->vf)
2534 			err = devm_request_irq(dev, irq_num, vsi->irq_handler,
2535 					       IRQF_SHARED, q_vector->name,
2536 					       q_vector);
2537 		else
2538 			err = devm_request_irq(dev, irq_num, vsi->irq_handler,
2539 					       0, q_vector->name, q_vector);
2540 		if (err) {
2541 			netdev_err(vsi->netdev, "MSIX request_irq failed, error: %d\n",
2542 				   err);
2543 			goto free_q_irqs;
2544 		}
2545 
2546 		/* register for affinity change notifications */
2547 		if (!IS_ENABLED(CONFIG_RFS_ACCEL)) {
2548 			struct irq_affinity_notify *affinity_notify;
2549 
2550 			affinity_notify = &q_vector->affinity_notify;
2551 			affinity_notify->notify = ice_irq_affinity_notify;
2552 			affinity_notify->release = ice_irq_affinity_release;
2553 			irq_set_affinity_notifier(irq_num, affinity_notify);
2554 		}
2555 
2556 		/* assign the mask for this irq */
2557 		irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
2558 	}
2559 
2560 	err = ice_set_cpu_rx_rmap(vsi);
2561 	if (err) {
2562 		netdev_err(vsi->netdev, "Failed to setup CPU RMAP on VSI %u: %pe\n",
2563 			   vsi->vsi_num, ERR_PTR(err));
2564 		goto free_q_irqs;
2565 	}
2566 
2567 	vsi->irqs_ready = true;
2568 	return 0;
2569 
2570 free_q_irqs:
2571 	while (vector--) {
2572 		irq_num = vsi->q_vectors[vector]->irq.virq;
2573 		if (!IS_ENABLED(CONFIG_RFS_ACCEL))
2574 			irq_set_affinity_notifier(irq_num, NULL);
2575 		irq_set_affinity_hint(irq_num, NULL);
2576 		devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]);
2577 	}
2578 	return err;
2579 }
2580 
2581 /**
2582  * ice_xdp_alloc_setup_rings - Allocate and setup Tx rings for XDP
2583  * @vsi: VSI to setup Tx rings used by XDP
2584  *
2585  * Return 0 on success and negative value on error
2586  */
2587 static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi)
2588 {
2589 	struct device *dev = ice_pf_to_dev(vsi->back);
2590 	struct ice_tx_desc *tx_desc;
2591 	int i, j;
2592 
2593 	ice_for_each_xdp_txq(vsi, i) {
2594 		u16 xdp_q_idx = vsi->alloc_txq + i;
2595 		struct ice_ring_stats *ring_stats;
2596 		struct ice_tx_ring *xdp_ring;
2597 
2598 		xdp_ring = kzalloc(sizeof(*xdp_ring), GFP_KERNEL);
2599 		if (!xdp_ring)
2600 			goto free_xdp_rings;
2601 
2602 		ring_stats = kzalloc(sizeof(*ring_stats), GFP_KERNEL);
2603 		if (!ring_stats) {
2604 			ice_free_tx_ring(xdp_ring);
2605 			goto free_xdp_rings;
2606 		}
2607 
2608 		xdp_ring->ring_stats = ring_stats;
2609 		xdp_ring->q_index = xdp_q_idx;
2610 		xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx];
2611 		xdp_ring->vsi = vsi;
2612 		xdp_ring->netdev = NULL;
2613 		xdp_ring->dev = dev;
2614 		xdp_ring->count = vsi->num_tx_desc;
2615 		WRITE_ONCE(vsi->xdp_rings[i], xdp_ring);
2616 		if (ice_setup_tx_ring(xdp_ring))
2617 			goto free_xdp_rings;
2618 		ice_set_ring_xdp(xdp_ring);
2619 		spin_lock_init(&xdp_ring->tx_lock);
2620 		for (j = 0; j < xdp_ring->count; j++) {
2621 			tx_desc = ICE_TX_DESC(xdp_ring, j);
2622 			tx_desc->cmd_type_offset_bsz = 0;
2623 		}
2624 	}
2625 
2626 	return 0;
2627 
2628 free_xdp_rings:
2629 	for (; i >= 0; i--) {
2630 		if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) {
2631 			kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu);
2632 			vsi->xdp_rings[i]->ring_stats = NULL;
2633 			ice_free_tx_ring(vsi->xdp_rings[i]);
2634 		}
2635 	}
2636 	return -ENOMEM;
2637 }
2638 
2639 /**
2640  * ice_vsi_assign_bpf_prog - set or clear bpf prog pointer on VSI
2641  * @vsi: VSI to set the bpf prog on
2642  * @prog: the bpf prog pointer
2643  */
2644 static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog)
2645 {
2646 	struct bpf_prog *old_prog;
2647 	int i;
2648 
2649 	old_prog = xchg(&vsi->xdp_prog, prog);
2650 	ice_for_each_rxq(vsi, i)
2651 		WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
2652 
2653 	if (old_prog)
2654 		bpf_prog_put(old_prog);
2655 }
2656 
2657 /**
2658  * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP
2659  * @vsi: VSI to bring up Tx rings used by XDP
2660  * @prog: bpf program that will be assigned to VSI
2661  *
2662  * Return 0 on success and negative value on error
2663  */
2664 int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog)
2665 {
2666 	u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2667 	int xdp_rings_rem = vsi->num_xdp_txq;
2668 	struct ice_pf *pf = vsi->back;
2669 	struct ice_qs_cfg xdp_qs_cfg = {
2670 		.qs_mutex = &pf->avail_q_mutex,
2671 		.pf_map = pf->avail_txqs,
2672 		.pf_map_size = pf->max_pf_txqs,
2673 		.q_count = vsi->num_xdp_txq,
2674 		.scatter_count = ICE_MAX_SCATTER_TXQS,
2675 		.vsi_map = vsi->txq_map,
2676 		.vsi_map_offset = vsi->alloc_txq,
2677 		.mapping_mode = ICE_VSI_MAP_CONTIG
2678 	};
2679 	struct device *dev;
2680 	int i, v_idx;
2681 	int status;
2682 
2683 	dev = ice_pf_to_dev(pf);
2684 	vsi->xdp_rings = devm_kcalloc(dev, vsi->num_xdp_txq,
2685 				      sizeof(*vsi->xdp_rings), GFP_KERNEL);
2686 	if (!vsi->xdp_rings)
2687 		return -ENOMEM;
2688 
2689 	vsi->xdp_mapping_mode = xdp_qs_cfg.mapping_mode;
2690 	if (__ice_vsi_get_qs(&xdp_qs_cfg))
2691 		goto err_map_xdp;
2692 
2693 	if (static_key_enabled(&ice_xdp_locking_key))
2694 		netdev_warn(vsi->netdev,
2695 			    "Could not allocate one XDP Tx ring per CPU, XDP_TX/XDP_REDIRECT actions will be slower\n");
2696 
2697 	if (ice_xdp_alloc_setup_rings(vsi))
2698 		goto clear_xdp_rings;
2699 
2700 	/* follow the logic from ice_vsi_map_rings_to_vectors */
2701 	ice_for_each_q_vector(vsi, v_idx) {
2702 		struct ice_q_vector *q_vector = vsi->q_vectors[v_idx];
2703 		int xdp_rings_per_v, q_id, q_base;
2704 
2705 		xdp_rings_per_v = DIV_ROUND_UP(xdp_rings_rem,
2706 					       vsi->num_q_vectors - v_idx);
2707 		q_base = vsi->num_xdp_txq - xdp_rings_rem;
2708 
2709 		for (q_id = q_base; q_id < (q_base + xdp_rings_per_v); q_id++) {
2710 			struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_id];
2711 
2712 			xdp_ring->q_vector = q_vector;
2713 			xdp_ring->next = q_vector->tx.tx_ring;
2714 			q_vector->tx.tx_ring = xdp_ring;
2715 		}
2716 		xdp_rings_rem -= xdp_rings_per_v;
2717 	}
2718 
2719 	ice_for_each_rxq(vsi, i) {
2720 		if (static_key_enabled(&ice_xdp_locking_key)) {
2721 			vsi->rx_rings[i]->xdp_ring = vsi->xdp_rings[i % vsi->num_xdp_txq];
2722 		} else {
2723 			struct ice_q_vector *q_vector = vsi->rx_rings[i]->q_vector;
2724 			struct ice_tx_ring *ring;
2725 
2726 			ice_for_each_tx_ring(ring, q_vector->tx) {
2727 				if (ice_ring_is_xdp(ring)) {
2728 					vsi->rx_rings[i]->xdp_ring = ring;
2729 					break;
2730 				}
2731 			}
2732 		}
2733 		ice_tx_xsk_pool(vsi, i);
2734 	}
2735 
2736 	/* omit the scheduler update if in reset path; XDP queues will be
2737 	 * taken into account at the end of ice_vsi_rebuild, where
2738 	 * ice_cfg_vsi_lan is being called
2739 	 */
2740 	if (ice_is_reset_in_progress(pf->state))
2741 		return 0;
2742 
2743 	/* tell the Tx scheduler that right now we have
2744 	 * additional queues
2745 	 */
2746 	for (i = 0; i < vsi->tc_cfg.numtc; i++)
2747 		max_txqs[i] = vsi->num_txq + vsi->num_xdp_txq;
2748 
2749 	status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc,
2750 				 max_txqs);
2751 	if (status) {
2752 		dev_err(dev, "Failed VSI LAN queue config for XDP, error: %d\n",
2753 			status);
2754 		goto clear_xdp_rings;
2755 	}
2756 
2757 	/* assign the prog only when it's not already present on VSI;
2758 	 * this flow is a subject of both ethtool -L and ndo_bpf flows;
2759 	 * VSI rebuild that happens under ethtool -L can expose us to
2760 	 * the bpf_prog refcount issues as we would be swapping same
2761 	 * bpf_prog pointers from vsi->xdp_prog and calling bpf_prog_put
2762 	 * on it as it would be treated as an 'old_prog'; for ndo_bpf
2763 	 * this is not harmful as dev_xdp_install bumps the refcount
2764 	 * before calling the op exposed by the driver;
2765 	 */
2766 	if (!ice_is_xdp_ena_vsi(vsi))
2767 		ice_vsi_assign_bpf_prog(vsi, prog);
2768 
2769 	return 0;
2770 clear_xdp_rings:
2771 	ice_for_each_xdp_txq(vsi, i)
2772 		if (vsi->xdp_rings[i]) {
2773 			kfree_rcu(vsi->xdp_rings[i], rcu);
2774 			vsi->xdp_rings[i] = NULL;
2775 		}
2776 
2777 err_map_xdp:
2778 	mutex_lock(&pf->avail_q_mutex);
2779 	ice_for_each_xdp_txq(vsi, i) {
2780 		clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs);
2781 		vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX;
2782 	}
2783 	mutex_unlock(&pf->avail_q_mutex);
2784 
2785 	devm_kfree(dev, vsi->xdp_rings);
2786 	return -ENOMEM;
2787 }
2788 
2789 /**
2790  * ice_destroy_xdp_rings - undo the configuration made by ice_prepare_xdp_rings
2791  * @vsi: VSI to remove XDP rings
2792  *
2793  * Detach XDP rings from irq vectors, clean up the PF bitmap and free
2794  * resources
2795  */
2796 int ice_destroy_xdp_rings(struct ice_vsi *vsi)
2797 {
2798 	u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2799 	struct ice_pf *pf = vsi->back;
2800 	int i, v_idx;
2801 
2802 	/* q_vectors are freed in reset path so there's no point in detaching
2803 	 * rings; in case of rebuild being triggered not from reset bits
2804 	 * in pf->state won't be set, so additionally check first q_vector
2805 	 * against NULL
2806 	 */
2807 	if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0])
2808 		goto free_qmap;
2809 
2810 	ice_for_each_q_vector(vsi, v_idx) {
2811 		struct ice_q_vector *q_vector = vsi->q_vectors[v_idx];
2812 		struct ice_tx_ring *ring;
2813 
2814 		ice_for_each_tx_ring(ring, q_vector->tx)
2815 			if (!ring->tx_buf || !ice_ring_is_xdp(ring))
2816 				break;
2817 
2818 		/* restore the value of last node prior to XDP setup */
2819 		q_vector->tx.tx_ring = ring;
2820 	}
2821 
2822 free_qmap:
2823 	mutex_lock(&pf->avail_q_mutex);
2824 	ice_for_each_xdp_txq(vsi, i) {
2825 		clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs);
2826 		vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX;
2827 	}
2828 	mutex_unlock(&pf->avail_q_mutex);
2829 
2830 	ice_for_each_xdp_txq(vsi, i)
2831 		if (vsi->xdp_rings[i]) {
2832 			if (vsi->xdp_rings[i]->desc) {
2833 				synchronize_rcu();
2834 				ice_free_tx_ring(vsi->xdp_rings[i]);
2835 			}
2836 			kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu);
2837 			vsi->xdp_rings[i]->ring_stats = NULL;
2838 			kfree_rcu(vsi->xdp_rings[i], rcu);
2839 			vsi->xdp_rings[i] = NULL;
2840 		}
2841 
2842 	devm_kfree(ice_pf_to_dev(pf), vsi->xdp_rings);
2843 	vsi->xdp_rings = NULL;
2844 
2845 	if (static_key_enabled(&ice_xdp_locking_key))
2846 		static_branch_dec(&ice_xdp_locking_key);
2847 
2848 	if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0])
2849 		return 0;
2850 
2851 	ice_vsi_assign_bpf_prog(vsi, NULL);
2852 
2853 	/* notify Tx scheduler that we destroyed XDP queues and bring
2854 	 * back the old number of child nodes
2855 	 */
2856 	for (i = 0; i < vsi->tc_cfg.numtc; i++)
2857 		max_txqs[i] = vsi->num_txq;
2858 
2859 	/* change number of XDP Tx queues to 0 */
2860 	vsi->num_xdp_txq = 0;
2861 
2862 	return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc,
2863 			       max_txqs);
2864 }
2865 
2866 /**
2867  * ice_vsi_rx_napi_schedule - Schedule napi on RX queues from VSI
2868  * @vsi: VSI to schedule napi on
2869  */
2870 static void ice_vsi_rx_napi_schedule(struct ice_vsi *vsi)
2871 {
2872 	int i;
2873 
2874 	ice_for_each_rxq(vsi, i) {
2875 		struct ice_rx_ring *rx_ring = vsi->rx_rings[i];
2876 
2877 		if (rx_ring->xsk_pool)
2878 			napi_schedule(&rx_ring->q_vector->napi);
2879 	}
2880 }
2881 
2882 /**
2883  * ice_vsi_determine_xdp_res - figure out how many Tx qs can XDP have
2884  * @vsi: VSI to determine the count of XDP Tx qs
2885  *
2886  * returns 0 if Tx qs count is higher than at least half of CPU count,
2887  * -ENOMEM otherwise
2888  */
2889 int ice_vsi_determine_xdp_res(struct ice_vsi *vsi)
2890 {
2891 	u16 avail = ice_get_avail_txq_count(vsi->back);
2892 	u16 cpus = num_possible_cpus();
2893 
2894 	if (avail < cpus / 2)
2895 		return -ENOMEM;
2896 
2897 	vsi->num_xdp_txq = min_t(u16, avail, cpus);
2898 
2899 	if (vsi->num_xdp_txq < cpus)
2900 		static_branch_inc(&ice_xdp_locking_key);
2901 
2902 	return 0;
2903 }
2904 
2905 /**
2906  * ice_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2907  * @vsi: Pointer to VSI structure
2908  */
2909 static int ice_max_xdp_frame_size(struct ice_vsi *vsi)
2910 {
2911 	if (test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags))
2912 		return ICE_RXBUF_1664;
2913 	else
2914 		return ICE_RXBUF_3072;
2915 }
2916 
2917 /**
2918  * ice_xdp_setup_prog - Add or remove XDP eBPF program
2919  * @vsi: VSI to setup XDP for
2920  * @prog: XDP program
2921  * @extack: netlink extended ack
2922  */
2923 static int
2924 ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog,
2925 		   struct netlink_ext_ack *extack)
2926 {
2927 	unsigned int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD;
2928 	bool if_running = netif_running(vsi->netdev);
2929 	int ret = 0, xdp_ring_err = 0;
2930 
2931 	if (prog && !prog->aux->xdp_has_frags) {
2932 		if (frame_size > ice_max_xdp_frame_size(vsi)) {
2933 			NL_SET_ERR_MSG_MOD(extack,
2934 					   "MTU is too large for linear frames and XDP prog does not support frags");
2935 			return -EOPNOTSUPP;
2936 		}
2937 	}
2938 
2939 	/* hot swap progs and avoid toggling link */
2940 	if (ice_is_xdp_ena_vsi(vsi) == !!prog) {
2941 		ice_vsi_assign_bpf_prog(vsi, prog);
2942 		return 0;
2943 	}
2944 
2945 	/* need to stop netdev while setting up the program for Rx rings */
2946 	if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) {
2947 		ret = ice_down(vsi);
2948 		if (ret) {
2949 			NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed");
2950 			return ret;
2951 		}
2952 	}
2953 
2954 	if (!ice_is_xdp_ena_vsi(vsi) && prog) {
2955 		xdp_ring_err = ice_vsi_determine_xdp_res(vsi);
2956 		if (xdp_ring_err) {
2957 			NL_SET_ERR_MSG_MOD(extack, "Not enough Tx resources for XDP");
2958 		} else {
2959 			xdp_ring_err = ice_prepare_xdp_rings(vsi, prog);
2960 			if (xdp_ring_err)
2961 				NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed");
2962 		}
2963 		xdp_features_set_redirect_target(vsi->netdev, true);
2964 		/* reallocate Rx queues that are used for zero-copy */
2965 		xdp_ring_err = ice_realloc_zc_buf(vsi, true);
2966 		if (xdp_ring_err)
2967 			NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Rx resources failed");
2968 	} else if (ice_is_xdp_ena_vsi(vsi) && !prog) {
2969 		xdp_features_clear_redirect_target(vsi->netdev);
2970 		xdp_ring_err = ice_destroy_xdp_rings(vsi);
2971 		if (xdp_ring_err)
2972 			NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed");
2973 		/* reallocate Rx queues that were used for zero-copy */
2974 		xdp_ring_err = ice_realloc_zc_buf(vsi, false);
2975 		if (xdp_ring_err)
2976 			NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Rx resources failed");
2977 	}
2978 
2979 	if (if_running)
2980 		ret = ice_up(vsi);
2981 
2982 	if (!ret && prog)
2983 		ice_vsi_rx_napi_schedule(vsi);
2984 
2985 	return (ret || xdp_ring_err) ? -ENOMEM : 0;
2986 }
2987 
2988 /**
2989  * ice_xdp_safe_mode - XDP handler for safe mode
2990  * @dev: netdevice
2991  * @xdp: XDP command
2992  */
2993 static int ice_xdp_safe_mode(struct net_device __always_unused *dev,
2994 			     struct netdev_bpf *xdp)
2995 {
2996 	NL_SET_ERR_MSG_MOD(xdp->extack,
2997 			   "Please provide working DDP firmware package in order to use XDP\n"
2998 			   "Refer to Documentation/networking/device_drivers/ethernet/intel/ice.rst");
2999 	return -EOPNOTSUPP;
3000 }
3001 
3002 /**
3003  * ice_xdp - implements XDP handler
3004  * @dev: netdevice
3005  * @xdp: XDP command
3006  */
3007 static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp)
3008 {
3009 	struct ice_netdev_priv *np = netdev_priv(dev);
3010 	struct ice_vsi *vsi = np->vsi;
3011 
3012 	if (vsi->type != ICE_VSI_PF) {
3013 		NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF VSI");
3014 		return -EINVAL;
3015 	}
3016 
3017 	switch (xdp->command) {
3018 	case XDP_SETUP_PROG:
3019 		return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack);
3020 	case XDP_SETUP_XSK_POOL:
3021 		return ice_xsk_pool_setup(vsi, xdp->xsk.pool,
3022 					  xdp->xsk.queue_id);
3023 	default:
3024 		return -EINVAL;
3025 	}
3026 }
3027 
3028 /**
3029  * ice_ena_misc_vector - enable the non-queue interrupts
3030  * @pf: board private structure
3031  */
3032 static void ice_ena_misc_vector(struct ice_pf *pf)
3033 {
3034 	struct ice_hw *hw = &pf->hw;
3035 	u32 val;
3036 
3037 	/* Disable anti-spoof detection interrupt to prevent spurious event
3038 	 * interrupts during a function reset. Anti-spoof functionally is
3039 	 * still supported.
3040 	 */
3041 	val = rd32(hw, GL_MDCK_TX_TDPU);
3042 	val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M;
3043 	wr32(hw, GL_MDCK_TX_TDPU, val);
3044 
3045 	/* clear things first */
3046 	wr32(hw, PFINT_OICR_ENA, 0);	/* disable all */
3047 	rd32(hw, PFINT_OICR);		/* read to clear */
3048 
3049 	val = (PFINT_OICR_ECC_ERR_M |
3050 	       PFINT_OICR_MAL_DETECT_M |
3051 	       PFINT_OICR_GRST_M |
3052 	       PFINT_OICR_PCI_EXCEPTION_M |
3053 	       PFINT_OICR_VFLR_M |
3054 	       PFINT_OICR_HMC_ERR_M |
3055 	       PFINT_OICR_PE_PUSH_M |
3056 	       PFINT_OICR_PE_CRITERR_M);
3057 
3058 	wr32(hw, PFINT_OICR_ENA, val);
3059 
3060 	/* SW_ITR_IDX = 0, but don't change INTENA */
3061 	wr32(hw, GLINT_DYN_CTL(pf->oicr_irq.index),
3062 	     GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M);
3063 }
3064 
3065 /**
3066  * ice_misc_intr - misc interrupt handler
3067  * @irq: interrupt number
3068  * @data: pointer to a q_vector
3069  */
3070 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
3071 {
3072 	struct ice_pf *pf = (struct ice_pf *)data;
3073 	struct ice_hw *hw = &pf->hw;
3074 	struct device *dev;
3075 	u32 oicr, ena_mask;
3076 
3077 	dev = ice_pf_to_dev(pf);
3078 	set_bit(ICE_ADMINQ_EVENT_PENDING, pf->state);
3079 	set_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state);
3080 	set_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state);
3081 
3082 	oicr = rd32(hw, PFINT_OICR);
3083 	ena_mask = rd32(hw, PFINT_OICR_ENA);
3084 
3085 	if (oicr & PFINT_OICR_SWINT_M) {
3086 		ena_mask &= ~PFINT_OICR_SWINT_M;
3087 		pf->sw_int_count++;
3088 	}
3089 
3090 	if (oicr & PFINT_OICR_MAL_DETECT_M) {
3091 		ena_mask &= ~PFINT_OICR_MAL_DETECT_M;
3092 		set_bit(ICE_MDD_EVENT_PENDING, pf->state);
3093 	}
3094 	if (oicr & PFINT_OICR_VFLR_M) {
3095 		/* disable any further VFLR event notifications */
3096 		if (test_bit(ICE_VF_RESETS_DISABLED, pf->state)) {
3097 			u32 reg = rd32(hw, PFINT_OICR_ENA);
3098 
3099 			reg &= ~PFINT_OICR_VFLR_M;
3100 			wr32(hw, PFINT_OICR_ENA, reg);
3101 		} else {
3102 			ena_mask &= ~PFINT_OICR_VFLR_M;
3103 			set_bit(ICE_VFLR_EVENT_PENDING, pf->state);
3104 		}
3105 	}
3106 
3107 	if (oicr & PFINT_OICR_GRST_M) {
3108 		u32 reset;
3109 
3110 		/* we have a reset warning */
3111 		ena_mask &= ~PFINT_OICR_GRST_M;
3112 		reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
3113 			GLGEN_RSTAT_RESET_TYPE_S;
3114 
3115 		if (reset == ICE_RESET_CORER)
3116 			pf->corer_count++;
3117 		else if (reset == ICE_RESET_GLOBR)
3118 			pf->globr_count++;
3119 		else if (reset == ICE_RESET_EMPR)
3120 			pf->empr_count++;
3121 		else
3122 			dev_dbg(dev, "Invalid reset type %d\n", reset);
3123 
3124 		/* If a reset cycle isn't already in progress, we set a bit in
3125 		 * pf->state so that the service task can start a reset/rebuild.
3126 		 */
3127 		if (!test_and_set_bit(ICE_RESET_OICR_RECV, pf->state)) {
3128 			if (reset == ICE_RESET_CORER)
3129 				set_bit(ICE_CORER_RECV, pf->state);
3130 			else if (reset == ICE_RESET_GLOBR)
3131 				set_bit(ICE_GLOBR_RECV, pf->state);
3132 			else
3133 				set_bit(ICE_EMPR_RECV, pf->state);
3134 
3135 			/* There are couple of different bits at play here.
3136 			 * hw->reset_ongoing indicates whether the hardware is
3137 			 * in reset. This is set to true when a reset interrupt
3138 			 * is received and set back to false after the driver
3139 			 * has determined that the hardware is out of reset.
3140 			 *
3141 			 * ICE_RESET_OICR_RECV in pf->state indicates
3142 			 * that a post reset rebuild is required before the
3143 			 * driver is operational again. This is set above.
3144 			 *
3145 			 * As this is the start of the reset/rebuild cycle, set
3146 			 * both to indicate that.
3147 			 */
3148 			hw->reset_ongoing = true;
3149 		}
3150 	}
3151 
3152 	if (oicr & PFINT_OICR_TSYN_TX_M) {
3153 		ena_mask &= ~PFINT_OICR_TSYN_TX_M;
3154 		if (ice_ptp_pf_handles_tx_interrupt(pf))
3155 			set_bit(ICE_MISC_THREAD_TX_TSTAMP, pf->misc_thread);
3156 	}
3157 
3158 	if (oicr & PFINT_OICR_TSYN_EVNT_M) {
3159 		u8 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
3160 		u32 gltsyn_stat = rd32(hw, GLTSYN_STAT(tmr_idx));
3161 
3162 		ena_mask &= ~PFINT_OICR_TSYN_EVNT_M;
3163 
3164 		if (ice_pf_src_tmr_owned(pf)) {
3165 			/* Save EVENTs from GLTSYN register */
3166 			pf->ptp.ext_ts_irq |= gltsyn_stat &
3167 					      (GLTSYN_STAT_EVENT0_M |
3168 					       GLTSYN_STAT_EVENT1_M |
3169 					       GLTSYN_STAT_EVENT2_M);
3170 
3171 			set_bit(ICE_MISC_THREAD_EXTTS_EVENT, pf->misc_thread);
3172 		}
3173 	}
3174 
3175 #define ICE_AUX_CRIT_ERR (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M | PFINT_OICR_PE_PUSH_M)
3176 	if (oicr & ICE_AUX_CRIT_ERR) {
3177 		pf->oicr_err_reg |= oicr;
3178 		set_bit(ICE_AUX_ERR_PENDING, pf->state);
3179 		ena_mask &= ~ICE_AUX_CRIT_ERR;
3180 	}
3181 
3182 	/* Report any remaining unexpected interrupts */
3183 	oicr &= ena_mask;
3184 	if (oicr) {
3185 		dev_dbg(dev, "unhandled interrupt oicr=0x%08x\n", oicr);
3186 		/* If a critical error is pending there is no choice but to
3187 		 * reset the device.
3188 		 */
3189 		if (oicr & (PFINT_OICR_PCI_EXCEPTION_M |
3190 			    PFINT_OICR_ECC_ERR_M)) {
3191 			set_bit(ICE_PFR_REQ, pf->state);
3192 		}
3193 	}
3194 
3195 	return IRQ_WAKE_THREAD;
3196 }
3197 
3198 /**
3199  * ice_misc_intr_thread_fn - misc interrupt thread function
3200  * @irq: interrupt number
3201  * @data: pointer to a q_vector
3202  */
3203 static irqreturn_t ice_misc_intr_thread_fn(int __always_unused irq, void *data)
3204 {
3205 	struct ice_pf *pf = data;
3206 	struct ice_hw *hw;
3207 
3208 	hw = &pf->hw;
3209 
3210 	if (ice_is_reset_in_progress(pf->state))
3211 		return IRQ_HANDLED;
3212 
3213 	ice_service_task_schedule(pf);
3214 
3215 	if (test_and_clear_bit(ICE_MISC_THREAD_EXTTS_EVENT, pf->misc_thread))
3216 		ice_ptp_extts_event(pf);
3217 
3218 	if (test_and_clear_bit(ICE_MISC_THREAD_TX_TSTAMP, pf->misc_thread)) {
3219 		/* Process outstanding Tx timestamps. If there is more work,
3220 		 * re-arm the interrupt to trigger again.
3221 		 */
3222 		if (ice_ptp_process_ts(pf) == ICE_TX_TSTAMP_WORK_PENDING) {
3223 			wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M);
3224 			ice_flush(hw);
3225 		}
3226 	}
3227 
3228 	ice_irq_dynamic_ena(hw, NULL, NULL);
3229 
3230 	return IRQ_HANDLED;
3231 }
3232 
3233 /**
3234  * ice_dis_ctrlq_interrupts - disable control queue interrupts
3235  * @hw: pointer to HW structure
3236  */
3237 static void ice_dis_ctrlq_interrupts(struct ice_hw *hw)
3238 {
3239 	/* disable Admin queue Interrupt causes */
3240 	wr32(hw, PFINT_FW_CTL,
3241 	     rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M);
3242 
3243 	/* disable Mailbox queue Interrupt causes */
3244 	wr32(hw, PFINT_MBX_CTL,
3245 	     rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M);
3246 
3247 	wr32(hw, PFINT_SB_CTL,
3248 	     rd32(hw, PFINT_SB_CTL) & ~PFINT_SB_CTL_CAUSE_ENA_M);
3249 
3250 	/* disable Control queue Interrupt causes */
3251 	wr32(hw, PFINT_OICR_CTL,
3252 	     rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M);
3253 
3254 	ice_flush(hw);
3255 }
3256 
3257 /**
3258  * ice_free_irq_msix_misc - Unroll misc vector setup
3259  * @pf: board private structure
3260  */
3261 static void ice_free_irq_msix_misc(struct ice_pf *pf)
3262 {
3263 	int misc_irq_num = pf->oicr_irq.virq;
3264 	struct ice_hw *hw = &pf->hw;
3265 
3266 	ice_dis_ctrlq_interrupts(hw);
3267 
3268 	/* disable OICR interrupt */
3269 	wr32(hw, PFINT_OICR_ENA, 0);
3270 	ice_flush(hw);
3271 
3272 	synchronize_irq(misc_irq_num);
3273 	devm_free_irq(ice_pf_to_dev(pf), misc_irq_num, pf);
3274 
3275 	ice_free_irq(pf, pf->oicr_irq);
3276 }
3277 
3278 /**
3279  * ice_ena_ctrlq_interrupts - enable control queue interrupts
3280  * @hw: pointer to HW structure
3281  * @reg_idx: HW vector index to associate the control queue interrupts with
3282  */
3283 static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx)
3284 {
3285 	u32 val;
3286 
3287 	val = ((reg_idx & PFINT_OICR_CTL_MSIX_INDX_M) |
3288 	       PFINT_OICR_CTL_CAUSE_ENA_M);
3289 	wr32(hw, PFINT_OICR_CTL, val);
3290 
3291 	/* enable Admin queue Interrupt causes */
3292 	val = ((reg_idx & PFINT_FW_CTL_MSIX_INDX_M) |
3293 	       PFINT_FW_CTL_CAUSE_ENA_M);
3294 	wr32(hw, PFINT_FW_CTL, val);
3295 
3296 	/* enable Mailbox queue Interrupt causes */
3297 	val = ((reg_idx & PFINT_MBX_CTL_MSIX_INDX_M) |
3298 	       PFINT_MBX_CTL_CAUSE_ENA_M);
3299 	wr32(hw, PFINT_MBX_CTL, val);
3300 
3301 	/* This enables Sideband queue Interrupt causes */
3302 	val = ((reg_idx & PFINT_SB_CTL_MSIX_INDX_M) |
3303 	       PFINT_SB_CTL_CAUSE_ENA_M);
3304 	wr32(hw, PFINT_SB_CTL, val);
3305 
3306 	ice_flush(hw);
3307 }
3308 
3309 /**
3310  * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events
3311  * @pf: board private structure
3312  *
3313  * This sets up the handler for MSIX 0, which is used to manage the
3314  * non-queue interrupts, e.g. AdminQ and errors. This is not used
3315  * when in MSI or Legacy interrupt mode.
3316  */
3317 static int ice_req_irq_msix_misc(struct ice_pf *pf)
3318 {
3319 	struct device *dev = ice_pf_to_dev(pf);
3320 	struct ice_hw *hw = &pf->hw;
3321 	struct msi_map oicr_irq;
3322 	int err = 0;
3323 
3324 	if (!pf->int_name[0])
3325 		snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc",
3326 			 dev_driver_string(dev), dev_name(dev));
3327 
3328 	/* Do not request IRQ but do enable OICR interrupt since settings are
3329 	 * lost during reset. Note that this function is called only during
3330 	 * rebuild path and not while reset is in progress.
3331 	 */
3332 	if (ice_is_reset_in_progress(pf->state))
3333 		goto skip_req_irq;
3334 
3335 	/* reserve one vector in irq_tracker for misc interrupts */
3336 	oicr_irq = ice_alloc_irq(pf, false);
3337 	if (oicr_irq.index < 0)
3338 		return oicr_irq.index;
3339 
3340 	pf->oicr_irq = oicr_irq;
3341 	err = devm_request_threaded_irq(dev, pf->oicr_irq.virq, ice_misc_intr,
3342 					ice_misc_intr_thread_fn, 0,
3343 					pf->int_name, pf);
3344 	if (err) {
3345 		dev_err(dev, "devm_request_threaded_irq for %s failed: %d\n",
3346 			pf->int_name, err);
3347 		ice_free_irq(pf, pf->oicr_irq);
3348 		return err;
3349 	}
3350 
3351 skip_req_irq:
3352 	ice_ena_misc_vector(pf);
3353 
3354 	ice_ena_ctrlq_interrupts(hw, pf->oicr_irq.index);
3355 	wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_irq.index),
3356 	     ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S);
3357 
3358 	ice_flush(hw);
3359 	ice_irq_dynamic_ena(hw, NULL, NULL);
3360 
3361 	return 0;
3362 }
3363 
3364 /**
3365  * ice_napi_add - register NAPI handler for the VSI
3366  * @vsi: VSI for which NAPI handler is to be registered
3367  *
3368  * This function is only called in the driver's load path. Registering the NAPI
3369  * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume,
3370  * reset/rebuild, etc.)
3371  */
3372 static void ice_napi_add(struct ice_vsi *vsi)
3373 {
3374 	int v_idx;
3375 
3376 	if (!vsi->netdev)
3377 		return;
3378 
3379 	ice_for_each_q_vector(vsi, v_idx) {
3380 		netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi,
3381 			       ice_napi_poll);
3382 		ice_q_vector_set_napi_queues(vsi->q_vectors[v_idx], false);
3383 	}
3384 }
3385 
3386 /**
3387  * ice_set_ops - set netdev and ethtools ops for the given netdev
3388  * @vsi: the VSI associated with the new netdev
3389  */
3390 static void ice_set_ops(struct ice_vsi *vsi)
3391 {
3392 	struct net_device *netdev = vsi->netdev;
3393 	struct ice_pf *pf = ice_netdev_to_pf(netdev);
3394 
3395 	if (ice_is_safe_mode(pf)) {
3396 		netdev->netdev_ops = &ice_netdev_safe_mode_ops;
3397 		ice_set_ethtool_safe_mode_ops(netdev);
3398 		return;
3399 	}
3400 
3401 	netdev->netdev_ops = &ice_netdev_ops;
3402 	netdev->udp_tunnel_nic_info = &pf->hw.udp_tunnel_nic;
3403 	ice_set_ethtool_ops(netdev);
3404 
3405 	if (vsi->type != ICE_VSI_PF)
3406 		return;
3407 
3408 	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
3409 			       NETDEV_XDP_ACT_XSK_ZEROCOPY |
3410 			       NETDEV_XDP_ACT_RX_SG;
3411 	netdev->xdp_zc_max_segs = ICE_MAX_BUF_TXD;
3412 }
3413 
3414 /**
3415  * ice_set_netdev_features - set features for the given netdev
3416  * @netdev: netdev instance
3417  */
3418 static void ice_set_netdev_features(struct net_device *netdev)
3419 {
3420 	struct ice_pf *pf = ice_netdev_to_pf(netdev);
3421 	bool is_dvm_ena = ice_is_dvm_ena(&pf->hw);
3422 	netdev_features_t csumo_features;
3423 	netdev_features_t vlano_features;
3424 	netdev_features_t dflt_features;
3425 	netdev_features_t tso_features;
3426 
3427 	if (ice_is_safe_mode(pf)) {
3428 		/* safe mode */
3429 		netdev->features = NETIF_F_SG | NETIF_F_HIGHDMA;
3430 		netdev->hw_features = netdev->features;
3431 		return;
3432 	}
3433 
3434 	dflt_features = NETIF_F_SG	|
3435 			NETIF_F_HIGHDMA	|
3436 			NETIF_F_NTUPLE	|
3437 			NETIF_F_RXHASH;
3438 
3439 	csumo_features = NETIF_F_RXCSUM	  |
3440 			 NETIF_F_IP_CSUM  |
3441 			 NETIF_F_SCTP_CRC |
3442 			 NETIF_F_IPV6_CSUM;
3443 
3444 	vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER |
3445 			 NETIF_F_HW_VLAN_CTAG_TX     |
3446 			 NETIF_F_HW_VLAN_CTAG_RX;
3447 
3448 	/* Enable CTAG/STAG filtering by default in Double VLAN Mode (DVM) */
3449 	if (is_dvm_ena)
3450 		vlano_features |= NETIF_F_HW_VLAN_STAG_FILTER;
3451 
3452 	tso_features = NETIF_F_TSO			|
3453 		       NETIF_F_TSO_ECN			|
3454 		       NETIF_F_TSO6			|
3455 		       NETIF_F_GSO_GRE			|
3456 		       NETIF_F_GSO_UDP_TUNNEL		|
3457 		       NETIF_F_GSO_GRE_CSUM		|
3458 		       NETIF_F_GSO_UDP_TUNNEL_CSUM	|
3459 		       NETIF_F_GSO_PARTIAL		|
3460 		       NETIF_F_GSO_IPXIP4		|
3461 		       NETIF_F_GSO_IPXIP6		|
3462 		       NETIF_F_GSO_UDP_L4;
3463 
3464 	netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
3465 					NETIF_F_GSO_GRE_CSUM;
3466 	/* set features that user can change */
3467 	netdev->hw_features = dflt_features | csumo_features |
3468 			      vlano_features | tso_features;
3469 
3470 	/* add support for HW_CSUM on packets with MPLS header */
3471 	netdev->mpls_features =  NETIF_F_HW_CSUM |
3472 				 NETIF_F_TSO     |
3473 				 NETIF_F_TSO6;
3474 
3475 	/* enable features */
3476 	netdev->features |= netdev->hw_features;
3477 
3478 	netdev->hw_features |= NETIF_F_HW_TC;
3479 	netdev->hw_features |= NETIF_F_LOOPBACK;
3480 
3481 	/* encap and VLAN devices inherit default, csumo and tso features */
3482 	netdev->hw_enc_features |= dflt_features | csumo_features |
3483 				   tso_features;
3484 	netdev->vlan_features |= dflt_features | csumo_features |
3485 				 tso_features;
3486 
3487 	/* advertise support but don't enable by default since only one type of
3488 	 * VLAN offload can be enabled at a time (i.e. CTAG or STAG). When one
3489 	 * type turns on the other has to be turned off. This is enforced by the
3490 	 * ice_fix_features() ndo callback.
3491 	 */
3492 	if (is_dvm_ena)
3493 		netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX |
3494 			NETIF_F_HW_VLAN_STAG_TX;
3495 
3496 	/* Leave CRC / FCS stripping enabled by default, but allow the value to
3497 	 * be changed at runtime
3498 	 */
3499 	netdev->hw_features |= NETIF_F_RXFCS;
3500 
3501 	netif_set_tso_max_size(netdev, ICE_MAX_TSO_SIZE);
3502 }
3503 
3504 /**
3505  * ice_fill_rss_lut - Fill the RSS lookup table with default values
3506  * @lut: Lookup table
3507  * @rss_table_size: Lookup table size
3508  * @rss_size: Range of queue number for hashing
3509  */
3510 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size)
3511 {
3512 	u16 i;
3513 
3514 	for (i = 0; i < rss_table_size; i++)
3515 		lut[i] = i % rss_size;
3516 }
3517 
3518 /**
3519  * ice_pf_vsi_setup - Set up a PF VSI
3520  * @pf: board private structure
3521  * @pi: pointer to the port_info instance
3522  *
3523  * Returns pointer to the successfully allocated VSI software struct
3524  * on success, otherwise returns NULL on failure.
3525  */
3526 static struct ice_vsi *
3527 ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
3528 {
3529 	struct ice_vsi_cfg_params params = {};
3530 
3531 	params.type = ICE_VSI_PF;
3532 	params.pi = pi;
3533 	params.flags = ICE_VSI_FLAG_INIT;
3534 
3535 	return ice_vsi_setup(pf, &params);
3536 }
3537 
3538 static struct ice_vsi *
3539 ice_chnl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi,
3540 		   struct ice_channel *ch)
3541 {
3542 	struct ice_vsi_cfg_params params = {};
3543 
3544 	params.type = ICE_VSI_CHNL;
3545 	params.pi = pi;
3546 	params.ch = ch;
3547 	params.flags = ICE_VSI_FLAG_INIT;
3548 
3549 	return ice_vsi_setup(pf, &params);
3550 }
3551 
3552 /**
3553  * ice_ctrl_vsi_setup - Set up a control VSI
3554  * @pf: board private structure
3555  * @pi: pointer to the port_info instance
3556  *
3557  * Returns pointer to the successfully allocated VSI software struct
3558  * on success, otherwise returns NULL on failure.
3559  */
3560 static struct ice_vsi *
3561 ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
3562 {
3563 	struct ice_vsi_cfg_params params = {};
3564 
3565 	params.type = ICE_VSI_CTRL;
3566 	params.pi = pi;
3567 	params.flags = ICE_VSI_FLAG_INIT;
3568 
3569 	return ice_vsi_setup(pf, &params);
3570 }
3571 
3572 /**
3573  * ice_lb_vsi_setup - Set up a loopback VSI
3574  * @pf: board private structure
3575  * @pi: pointer to the port_info instance
3576  *
3577  * Returns pointer to the successfully allocated VSI software struct
3578  * on success, otherwise returns NULL on failure.
3579  */
3580 struct ice_vsi *
3581 ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
3582 {
3583 	struct ice_vsi_cfg_params params = {};
3584 
3585 	params.type = ICE_VSI_LB;
3586 	params.pi = pi;
3587 	params.flags = ICE_VSI_FLAG_INIT;
3588 
3589 	return ice_vsi_setup(pf, &params);
3590 }
3591 
3592 /**
3593  * ice_vlan_rx_add_vid - Add a VLAN ID filter to HW offload
3594  * @netdev: network interface to be adjusted
3595  * @proto: VLAN TPID
3596  * @vid: VLAN ID to be added
3597  *
3598  * net_device_ops implementation for adding VLAN IDs
3599  */
3600 static int
3601 ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
3602 {
3603 	struct ice_netdev_priv *np = netdev_priv(netdev);
3604 	struct ice_vsi_vlan_ops *vlan_ops;
3605 	struct ice_vsi *vsi = np->vsi;
3606 	struct ice_vlan vlan;
3607 	int ret;
3608 
3609 	/* VLAN 0 is added by default during load/reset */
3610 	if (!vid)
3611 		return 0;
3612 
3613 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state))
3614 		usleep_range(1000, 2000);
3615 
3616 	/* Add multicast promisc rule for the VLAN ID to be added if
3617 	 * all-multicast is currently enabled.
3618 	 */
3619 	if (vsi->current_netdev_flags & IFF_ALLMULTI) {
3620 		ret = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
3621 					       ICE_MCAST_VLAN_PROMISC_BITS,
3622 					       vid);
3623 		if (ret)
3624 			goto finish;
3625 	}
3626 
3627 	vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
3628 
3629 	/* Add a switch rule for this VLAN ID so its corresponding VLAN tagged
3630 	 * packets aren't pruned by the device's internal switch on Rx
3631 	 */
3632 	vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0);
3633 	ret = vlan_ops->add_vlan(vsi, &vlan);
3634 	if (ret)
3635 		goto finish;
3636 
3637 	/* If all-multicast is currently enabled and this VLAN ID is only one
3638 	 * besides VLAN-0 we have to update look-up type of multicast promisc
3639 	 * rule for VLAN-0 from ICE_SW_LKUP_PROMISC to ICE_SW_LKUP_PROMISC_VLAN.
3640 	 */
3641 	if ((vsi->current_netdev_flags & IFF_ALLMULTI) &&
3642 	    ice_vsi_num_non_zero_vlans(vsi) == 1) {
3643 		ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3644 					   ICE_MCAST_PROMISC_BITS, 0);
3645 		ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
3646 					 ICE_MCAST_VLAN_PROMISC_BITS, 0);
3647 	}
3648 
3649 finish:
3650 	clear_bit(ICE_CFG_BUSY, vsi->state);
3651 
3652 	return ret;
3653 }
3654 
3655 /**
3656  * ice_vlan_rx_kill_vid - Remove a VLAN ID filter from HW offload
3657  * @netdev: network interface to be adjusted
3658  * @proto: VLAN TPID
3659  * @vid: VLAN ID to be removed
3660  *
3661  * net_device_ops implementation for removing VLAN IDs
3662  */
3663 static int
3664 ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
3665 {
3666 	struct ice_netdev_priv *np = netdev_priv(netdev);
3667 	struct ice_vsi_vlan_ops *vlan_ops;
3668 	struct ice_vsi *vsi = np->vsi;
3669 	struct ice_vlan vlan;
3670 	int ret;
3671 
3672 	/* don't allow removal of VLAN 0 */
3673 	if (!vid)
3674 		return 0;
3675 
3676 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state))
3677 		usleep_range(1000, 2000);
3678 
3679 	ret = ice_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3680 				    ICE_MCAST_VLAN_PROMISC_BITS, vid);
3681 	if (ret) {
3682 		netdev_err(netdev, "Error clearing multicast promiscuous mode on VSI %i\n",
3683 			   vsi->vsi_num);
3684 		vsi->current_netdev_flags |= IFF_ALLMULTI;
3685 	}
3686 
3687 	vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
3688 
3689 	/* Make sure VLAN delete is successful before updating VLAN
3690 	 * information
3691 	 */
3692 	vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0);
3693 	ret = vlan_ops->del_vlan(vsi, &vlan);
3694 	if (ret)
3695 		goto finish;
3696 
3697 	/* Remove multicast promisc rule for the removed VLAN ID if
3698 	 * all-multicast is enabled.
3699 	 */
3700 	if (vsi->current_netdev_flags & IFF_ALLMULTI)
3701 		ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3702 					   ICE_MCAST_VLAN_PROMISC_BITS, vid);
3703 
3704 	if (!ice_vsi_has_non_zero_vlans(vsi)) {
3705 		/* Update look-up type of multicast promisc rule for VLAN 0
3706 		 * from ICE_SW_LKUP_PROMISC_VLAN to ICE_SW_LKUP_PROMISC when
3707 		 * all-multicast is enabled and VLAN 0 is the only VLAN rule.
3708 		 */
3709 		if (vsi->current_netdev_flags & IFF_ALLMULTI) {
3710 			ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3711 						   ICE_MCAST_VLAN_PROMISC_BITS,
3712 						   0);
3713 			ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
3714 						 ICE_MCAST_PROMISC_BITS, 0);
3715 		}
3716 	}
3717 
3718 finish:
3719 	clear_bit(ICE_CFG_BUSY, vsi->state);
3720 
3721 	return ret;
3722 }
3723 
3724 /**
3725  * ice_rep_indr_tc_block_unbind
3726  * @cb_priv: indirection block private data
3727  */
3728 static void ice_rep_indr_tc_block_unbind(void *cb_priv)
3729 {
3730 	struct ice_indr_block_priv *indr_priv = cb_priv;
3731 
3732 	list_del(&indr_priv->list);
3733 	kfree(indr_priv);
3734 }
3735 
3736 /**
3737  * ice_tc_indir_block_unregister - Unregister TC indirect block notifications
3738  * @vsi: VSI struct which has the netdev
3739  */
3740 static void ice_tc_indir_block_unregister(struct ice_vsi *vsi)
3741 {
3742 	struct ice_netdev_priv *np = netdev_priv(vsi->netdev);
3743 
3744 	flow_indr_dev_unregister(ice_indr_setup_tc_cb, np,
3745 				 ice_rep_indr_tc_block_unbind);
3746 }
3747 
3748 /**
3749  * ice_tc_indir_block_register - Register TC indirect block notifications
3750  * @vsi: VSI struct which has the netdev
3751  *
3752  * Returns 0 on success, negative value on failure
3753  */
3754 static int ice_tc_indir_block_register(struct ice_vsi *vsi)
3755 {
3756 	struct ice_netdev_priv *np;
3757 
3758 	if (!vsi || !vsi->netdev)
3759 		return -EINVAL;
3760 
3761 	np = netdev_priv(vsi->netdev);
3762 
3763 	INIT_LIST_HEAD(&np->tc_indr_block_priv_list);
3764 	return flow_indr_dev_register(ice_indr_setup_tc_cb, np);
3765 }
3766 
3767 /**
3768  * ice_get_avail_q_count - Get count of queues in use
3769  * @pf_qmap: bitmap to get queue use count from
3770  * @lock: pointer to a mutex that protects access to pf_qmap
3771  * @size: size of the bitmap
3772  */
3773 static u16
3774 ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size)
3775 {
3776 	unsigned long bit;
3777 	u16 count = 0;
3778 
3779 	mutex_lock(lock);
3780 	for_each_clear_bit(bit, pf_qmap, size)
3781 		count++;
3782 	mutex_unlock(lock);
3783 
3784 	return count;
3785 }
3786 
3787 /**
3788  * ice_get_avail_txq_count - Get count of Tx queues in use
3789  * @pf: pointer to an ice_pf instance
3790  */
3791 u16 ice_get_avail_txq_count(struct ice_pf *pf)
3792 {
3793 	return ice_get_avail_q_count(pf->avail_txqs, &pf->avail_q_mutex,
3794 				     pf->max_pf_txqs);
3795 }
3796 
3797 /**
3798  * ice_get_avail_rxq_count - Get count of Rx queues in use
3799  * @pf: pointer to an ice_pf instance
3800  */
3801 u16 ice_get_avail_rxq_count(struct ice_pf *pf)
3802 {
3803 	return ice_get_avail_q_count(pf->avail_rxqs, &pf->avail_q_mutex,
3804 				     pf->max_pf_rxqs);
3805 }
3806 
3807 /**
3808  * ice_deinit_pf - Unrolls initialziations done by ice_init_pf
3809  * @pf: board private structure to initialize
3810  */
3811 static void ice_deinit_pf(struct ice_pf *pf)
3812 {
3813 	ice_service_task_stop(pf);
3814 	mutex_destroy(&pf->lag_mutex);
3815 	mutex_destroy(&pf->adev_mutex);
3816 	mutex_destroy(&pf->sw_mutex);
3817 	mutex_destroy(&pf->tc_mutex);
3818 	mutex_destroy(&pf->avail_q_mutex);
3819 	mutex_destroy(&pf->vfs.table_lock);
3820 
3821 	if (pf->avail_txqs) {
3822 		bitmap_free(pf->avail_txqs);
3823 		pf->avail_txqs = NULL;
3824 	}
3825 
3826 	if (pf->avail_rxqs) {
3827 		bitmap_free(pf->avail_rxqs);
3828 		pf->avail_rxqs = NULL;
3829 	}
3830 
3831 	if (pf->ptp.clock)
3832 		ptp_clock_unregister(pf->ptp.clock);
3833 }
3834 
3835 /**
3836  * ice_set_pf_caps - set PFs capability flags
3837  * @pf: pointer to the PF instance
3838  */
3839 static void ice_set_pf_caps(struct ice_pf *pf)
3840 {
3841 	struct ice_hw_func_caps *func_caps = &pf->hw.func_caps;
3842 
3843 	clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
3844 	if (func_caps->common_cap.rdma)
3845 		set_bit(ICE_FLAG_RDMA_ENA, pf->flags);
3846 	clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
3847 	if (func_caps->common_cap.dcb)
3848 		set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
3849 	clear_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags);
3850 	if (func_caps->common_cap.sr_iov_1_1) {
3851 		set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags);
3852 		pf->vfs.num_supported = min_t(int, func_caps->num_allocd_vfs,
3853 					      ICE_MAX_SRIOV_VFS);
3854 	}
3855 	clear_bit(ICE_FLAG_RSS_ENA, pf->flags);
3856 	if (func_caps->common_cap.rss_table_size)
3857 		set_bit(ICE_FLAG_RSS_ENA, pf->flags);
3858 
3859 	clear_bit(ICE_FLAG_FD_ENA, pf->flags);
3860 	if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) {
3861 		u16 unused;
3862 
3863 		/* ctrl_vsi_idx will be set to a valid value when flow director
3864 		 * is setup by ice_init_fdir
3865 		 */
3866 		pf->ctrl_vsi_idx = ICE_NO_VSI;
3867 		set_bit(ICE_FLAG_FD_ENA, pf->flags);
3868 		/* force guaranteed filter pool for PF */
3869 		ice_alloc_fd_guar_item(&pf->hw, &unused,
3870 				       func_caps->fd_fltr_guar);
3871 		/* force shared filter pool for PF */
3872 		ice_alloc_fd_shrd_item(&pf->hw, &unused,
3873 				       func_caps->fd_fltr_best_effort);
3874 	}
3875 
3876 	clear_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags);
3877 	if (func_caps->common_cap.ieee_1588 &&
3878 	    !(pf->hw.mac_type == ICE_MAC_E830))
3879 		set_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags);
3880 
3881 	pf->max_pf_txqs = func_caps->common_cap.num_txq;
3882 	pf->max_pf_rxqs = func_caps->common_cap.num_rxq;
3883 }
3884 
3885 /**
3886  * ice_init_pf - Initialize general software structures (struct ice_pf)
3887  * @pf: board private structure to initialize
3888  */
3889 static int ice_init_pf(struct ice_pf *pf)
3890 {
3891 	ice_set_pf_caps(pf);
3892 
3893 	mutex_init(&pf->sw_mutex);
3894 	mutex_init(&pf->tc_mutex);
3895 	mutex_init(&pf->adev_mutex);
3896 	mutex_init(&pf->lag_mutex);
3897 
3898 	INIT_HLIST_HEAD(&pf->aq_wait_list);
3899 	spin_lock_init(&pf->aq_wait_lock);
3900 	init_waitqueue_head(&pf->aq_wait_queue);
3901 
3902 	init_waitqueue_head(&pf->reset_wait_queue);
3903 
3904 	/* setup service timer and periodic service task */
3905 	timer_setup(&pf->serv_tmr, ice_service_timer, 0);
3906 	pf->serv_tmr_period = HZ;
3907 	INIT_WORK(&pf->serv_task, ice_service_task);
3908 	clear_bit(ICE_SERVICE_SCHED, pf->state);
3909 
3910 	mutex_init(&pf->avail_q_mutex);
3911 	pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL);
3912 	if (!pf->avail_txqs)
3913 		return -ENOMEM;
3914 
3915 	pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL);
3916 	if (!pf->avail_rxqs) {
3917 		bitmap_free(pf->avail_txqs);
3918 		pf->avail_txqs = NULL;
3919 		return -ENOMEM;
3920 	}
3921 
3922 	mutex_init(&pf->vfs.table_lock);
3923 	hash_init(pf->vfs.table);
3924 	ice_mbx_init_snapshot(&pf->hw);
3925 
3926 	return 0;
3927 }
3928 
3929 /**
3930  * ice_is_wol_supported - check if WoL is supported
3931  * @hw: pointer to hardware info
3932  *
3933  * Check if WoL is supported based on the HW configuration.
3934  * Returns true if NVM supports and enables WoL for this port, false otherwise
3935  */
3936 bool ice_is_wol_supported(struct ice_hw *hw)
3937 {
3938 	u16 wol_ctrl;
3939 
3940 	/* A bit set to 1 in the NVM Software Reserved Word 2 (WoL control
3941 	 * word) indicates WoL is not supported on the corresponding PF ID.
3942 	 */
3943 	if (ice_read_sr_word(hw, ICE_SR_NVM_WOL_CFG, &wol_ctrl))
3944 		return false;
3945 
3946 	return !(BIT(hw->port_info->lport) & wol_ctrl);
3947 }
3948 
3949 /**
3950  * ice_vsi_recfg_qs - Change the number of queues on a VSI
3951  * @vsi: VSI being changed
3952  * @new_rx: new number of Rx queues
3953  * @new_tx: new number of Tx queues
3954  * @locked: is adev device_lock held
3955  *
3956  * Only change the number of queues if new_tx, or new_rx is non-0.
3957  *
3958  * Returns 0 on success.
3959  */
3960 int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx, bool locked)
3961 {
3962 	struct ice_pf *pf = vsi->back;
3963 	int err = 0, timeout = 50;
3964 
3965 	if (!new_rx && !new_tx)
3966 		return -EINVAL;
3967 
3968 	while (test_and_set_bit(ICE_CFG_BUSY, pf->state)) {
3969 		timeout--;
3970 		if (!timeout)
3971 			return -EBUSY;
3972 		usleep_range(1000, 2000);
3973 	}
3974 
3975 	if (new_tx)
3976 		vsi->req_txq = (u16)new_tx;
3977 	if (new_rx)
3978 		vsi->req_rxq = (u16)new_rx;
3979 
3980 	/* set for the next time the netdev is started */
3981 	if (!netif_running(vsi->netdev)) {
3982 		ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT);
3983 		dev_dbg(ice_pf_to_dev(pf), "Link is down, queue count change happens when link is brought up\n");
3984 		goto done;
3985 	}
3986 
3987 	ice_vsi_close(vsi);
3988 	ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT);
3989 	ice_pf_dcb_recfg(pf, locked);
3990 	ice_vsi_open(vsi);
3991 done:
3992 	clear_bit(ICE_CFG_BUSY, pf->state);
3993 	return err;
3994 }
3995 
3996 /**
3997  * ice_set_safe_mode_vlan_cfg - configure PF VSI to allow all VLANs in safe mode
3998  * @pf: PF to configure
3999  *
4000  * No VLAN offloads/filtering are advertised in safe mode so make sure the PF
4001  * VSI can still Tx/Rx VLAN tagged packets.
4002  */
4003 static void ice_set_safe_mode_vlan_cfg(struct ice_pf *pf)
4004 {
4005 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4006 	struct ice_vsi_ctx *ctxt;
4007 	struct ice_hw *hw;
4008 	int status;
4009 
4010 	if (!vsi)
4011 		return;
4012 
4013 	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
4014 	if (!ctxt)
4015 		return;
4016 
4017 	hw = &pf->hw;
4018 	ctxt->info = vsi->info;
4019 
4020 	ctxt->info.valid_sections =
4021 		cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID |
4022 			    ICE_AQ_VSI_PROP_SECURITY_VALID |
4023 			    ICE_AQ_VSI_PROP_SW_VALID);
4024 
4025 	/* disable VLAN anti-spoof */
4026 	ctxt->info.sec_flags &= ~(ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA <<
4027 				  ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S);
4028 
4029 	/* disable VLAN pruning and keep all other settings */
4030 	ctxt->info.sw_flags2 &= ~ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA;
4031 
4032 	/* allow all VLANs on Tx and don't strip on Rx */
4033 	ctxt->info.inner_vlan_flags = ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL |
4034 		ICE_AQ_VSI_INNER_VLAN_EMODE_NOTHING;
4035 
4036 	status = ice_update_vsi(hw, vsi->idx, ctxt, NULL);
4037 	if (status) {
4038 		dev_err(ice_pf_to_dev(vsi->back), "Failed to update VSI for safe mode VLANs, err %d aq_err %s\n",
4039 			status, ice_aq_str(hw->adminq.sq_last_status));
4040 	} else {
4041 		vsi->info.sec_flags = ctxt->info.sec_flags;
4042 		vsi->info.sw_flags2 = ctxt->info.sw_flags2;
4043 		vsi->info.inner_vlan_flags = ctxt->info.inner_vlan_flags;
4044 	}
4045 
4046 	kfree(ctxt);
4047 }
4048 
4049 /**
4050  * ice_log_pkg_init - log result of DDP package load
4051  * @hw: pointer to hardware info
4052  * @state: state of package load
4053  */
4054 static void ice_log_pkg_init(struct ice_hw *hw, enum ice_ddp_state state)
4055 {
4056 	struct ice_pf *pf = hw->back;
4057 	struct device *dev;
4058 
4059 	dev = ice_pf_to_dev(pf);
4060 
4061 	switch (state) {
4062 	case ICE_DDP_PKG_SUCCESS:
4063 		dev_info(dev, "The DDP package was successfully loaded: %s version %d.%d.%d.%d\n",
4064 			 hw->active_pkg_name,
4065 			 hw->active_pkg_ver.major,
4066 			 hw->active_pkg_ver.minor,
4067 			 hw->active_pkg_ver.update,
4068 			 hw->active_pkg_ver.draft);
4069 		break;
4070 	case ICE_DDP_PKG_SAME_VERSION_ALREADY_LOADED:
4071 		dev_info(dev, "DDP package already present on device: %s version %d.%d.%d.%d\n",
4072 			 hw->active_pkg_name,
4073 			 hw->active_pkg_ver.major,
4074 			 hw->active_pkg_ver.minor,
4075 			 hw->active_pkg_ver.update,
4076 			 hw->active_pkg_ver.draft);
4077 		break;
4078 	case ICE_DDP_PKG_ALREADY_LOADED_NOT_SUPPORTED:
4079 		dev_err(dev, "The device has a DDP package that is not supported by the driver.  The device has package '%s' version %d.%d.x.x.  The driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
4080 			hw->active_pkg_name,
4081 			hw->active_pkg_ver.major,
4082 			hw->active_pkg_ver.minor,
4083 			ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
4084 		break;
4085 	case ICE_DDP_PKG_COMPATIBLE_ALREADY_LOADED:
4086 		dev_info(dev, "The driver could not load the DDP package file because a compatible DDP package is already present on the device.  The device has package '%s' version %d.%d.%d.%d.  The package file found by the driver: '%s' version %d.%d.%d.%d.\n",
4087 			 hw->active_pkg_name,
4088 			 hw->active_pkg_ver.major,
4089 			 hw->active_pkg_ver.minor,
4090 			 hw->active_pkg_ver.update,
4091 			 hw->active_pkg_ver.draft,
4092 			 hw->pkg_name,
4093 			 hw->pkg_ver.major,
4094 			 hw->pkg_ver.minor,
4095 			 hw->pkg_ver.update,
4096 			 hw->pkg_ver.draft);
4097 		break;
4098 	case ICE_DDP_PKG_FW_MISMATCH:
4099 		dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package.  Please update the device's NVM.  Entering safe mode.\n");
4100 		break;
4101 	case ICE_DDP_PKG_INVALID_FILE:
4102 		dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n");
4103 		break;
4104 	case ICE_DDP_PKG_FILE_VERSION_TOO_HIGH:
4105 		dev_err(dev, "The DDP package file version is higher than the driver supports.  Please use an updated driver.  Entering Safe Mode.\n");
4106 		break;
4107 	case ICE_DDP_PKG_FILE_VERSION_TOO_LOW:
4108 		dev_err(dev, "The DDP package file version is lower than the driver supports.  The driver requires version %d.%d.x.x.  Please use an updated DDP Package file.  Entering Safe Mode.\n",
4109 			ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
4110 		break;
4111 	case ICE_DDP_PKG_FILE_SIGNATURE_INVALID:
4112 		dev_err(dev, "The DDP package could not be loaded because its signature is not valid.  Please use a valid DDP Package.  Entering Safe Mode.\n");
4113 		break;
4114 	case ICE_DDP_PKG_FILE_REVISION_TOO_LOW:
4115 		dev_err(dev, "The DDP Package could not be loaded because its security revision is too low.  Please use an updated DDP Package.  Entering Safe Mode.\n");
4116 		break;
4117 	case ICE_DDP_PKG_LOAD_ERROR:
4118 		dev_err(dev, "An error occurred on the device while loading the DDP package.  The device will be reset.\n");
4119 		/* poll for reset to complete */
4120 		if (ice_check_reset(hw))
4121 			dev_err(dev, "Error resetting device. Please reload the driver\n");
4122 		break;
4123 	case ICE_DDP_PKG_ERR:
4124 	default:
4125 		dev_err(dev, "An unknown error occurred when loading the DDP package.  Entering Safe Mode.\n");
4126 		break;
4127 	}
4128 }
4129 
4130 /**
4131  * ice_load_pkg - load/reload the DDP Package file
4132  * @firmware: firmware structure when firmware requested or NULL for reload
4133  * @pf: pointer to the PF instance
4134  *
4135  * Called on probe and post CORER/GLOBR rebuild to load DDP Package and
4136  * initialize HW tables.
4137  */
4138 static void
4139 ice_load_pkg(const struct firmware *firmware, struct ice_pf *pf)
4140 {
4141 	enum ice_ddp_state state = ICE_DDP_PKG_ERR;
4142 	struct device *dev = ice_pf_to_dev(pf);
4143 	struct ice_hw *hw = &pf->hw;
4144 
4145 	/* Load DDP Package */
4146 	if (firmware && !hw->pkg_copy) {
4147 		state = ice_copy_and_init_pkg(hw, firmware->data,
4148 					      firmware->size);
4149 		ice_log_pkg_init(hw, state);
4150 	} else if (!firmware && hw->pkg_copy) {
4151 		/* Reload package during rebuild after CORER/GLOBR reset */
4152 		state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size);
4153 		ice_log_pkg_init(hw, state);
4154 	} else {
4155 		dev_err(dev, "The DDP package file failed to load. Entering Safe Mode.\n");
4156 	}
4157 
4158 	if (!ice_is_init_pkg_successful(state)) {
4159 		/* Safe Mode */
4160 		clear_bit(ICE_FLAG_ADV_FEATURES, pf->flags);
4161 		return;
4162 	}
4163 
4164 	/* Successful download package is the precondition for advanced
4165 	 * features, hence setting the ICE_FLAG_ADV_FEATURES flag
4166 	 */
4167 	set_bit(ICE_FLAG_ADV_FEATURES, pf->flags);
4168 }
4169 
4170 /**
4171  * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines
4172  * @pf: pointer to the PF structure
4173  *
4174  * There is no error returned here because the driver should be able to handle
4175  * 128 Byte cache lines, so we only print a warning in case issues are seen,
4176  * specifically with Tx.
4177  */
4178 static void ice_verify_cacheline_size(struct ice_pf *pf)
4179 {
4180 	if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M)
4181 		dev_warn(ice_pf_to_dev(pf), "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n",
4182 			 ICE_CACHE_LINE_BYTES);
4183 }
4184 
4185 /**
4186  * ice_send_version - update firmware with driver version
4187  * @pf: PF struct
4188  *
4189  * Returns 0 on success, else error code
4190  */
4191 static int ice_send_version(struct ice_pf *pf)
4192 {
4193 	struct ice_driver_ver dv;
4194 
4195 	dv.major_ver = 0xff;
4196 	dv.minor_ver = 0xff;
4197 	dv.build_ver = 0xff;
4198 	dv.subbuild_ver = 0;
4199 	strscpy((char *)dv.driver_string, UTS_RELEASE,
4200 		sizeof(dv.driver_string));
4201 	return ice_aq_send_driver_ver(&pf->hw, &dv, NULL);
4202 }
4203 
4204 /**
4205  * ice_init_fdir - Initialize flow director VSI and configuration
4206  * @pf: pointer to the PF instance
4207  *
4208  * returns 0 on success, negative on error
4209  */
4210 static int ice_init_fdir(struct ice_pf *pf)
4211 {
4212 	struct device *dev = ice_pf_to_dev(pf);
4213 	struct ice_vsi *ctrl_vsi;
4214 	int err;
4215 
4216 	/* Side Band Flow Director needs to have a control VSI.
4217 	 * Allocate it and store it in the PF.
4218 	 */
4219 	ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info);
4220 	if (!ctrl_vsi) {
4221 		dev_dbg(dev, "could not create control VSI\n");
4222 		return -ENOMEM;
4223 	}
4224 
4225 	err = ice_vsi_open_ctrl(ctrl_vsi);
4226 	if (err) {
4227 		dev_dbg(dev, "could not open control VSI\n");
4228 		goto err_vsi_open;
4229 	}
4230 
4231 	mutex_init(&pf->hw.fdir_fltr_lock);
4232 
4233 	err = ice_fdir_create_dflt_rules(pf);
4234 	if (err)
4235 		goto err_fdir_rule;
4236 
4237 	return 0;
4238 
4239 err_fdir_rule:
4240 	ice_fdir_release_flows(&pf->hw);
4241 	ice_vsi_close(ctrl_vsi);
4242 err_vsi_open:
4243 	ice_vsi_release(ctrl_vsi);
4244 	if (pf->ctrl_vsi_idx != ICE_NO_VSI) {
4245 		pf->vsi[pf->ctrl_vsi_idx] = NULL;
4246 		pf->ctrl_vsi_idx = ICE_NO_VSI;
4247 	}
4248 	return err;
4249 }
4250 
4251 static void ice_deinit_fdir(struct ice_pf *pf)
4252 {
4253 	struct ice_vsi *vsi = ice_get_ctrl_vsi(pf);
4254 
4255 	if (!vsi)
4256 		return;
4257 
4258 	ice_vsi_manage_fdir(vsi, false);
4259 	ice_vsi_release(vsi);
4260 	if (pf->ctrl_vsi_idx != ICE_NO_VSI) {
4261 		pf->vsi[pf->ctrl_vsi_idx] = NULL;
4262 		pf->ctrl_vsi_idx = ICE_NO_VSI;
4263 	}
4264 
4265 	mutex_destroy(&(&pf->hw)->fdir_fltr_lock);
4266 }
4267 
4268 /**
4269  * ice_get_opt_fw_name - return optional firmware file name or NULL
4270  * @pf: pointer to the PF instance
4271  */
4272 static char *ice_get_opt_fw_name(struct ice_pf *pf)
4273 {
4274 	/* Optional firmware name same as default with additional dash
4275 	 * followed by a EUI-64 identifier (PCIe Device Serial Number)
4276 	 */
4277 	struct pci_dev *pdev = pf->pdev;
4278 	char *opt_fw_filename;
4279 	u64 dsn;
4280 
4281 	/* Determine the name of the optional file using the DSN (two
4282 	 * dwords following the start of the DSN Capability).
4283 	 */
4284 	dsn = pci_get_dsn(pdev);
4285 	if (!dsn)
4286 		return NULL;
4287 
4288 	opt_fw_filename = kzalloc(NAME_MAX, GFP_KERNEL);
4289 	if (!opt_fw_filename)
4290 		return NULL;
4291 
4292 	snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg",
4293 		 ICE_DDP_PKG_PATH, dsn);
4294 
4295 	return opt_fw_filename;
4296 }
4297 
4298 /**
4299  * ice_request_fw - Device initialization routine
4300  * @pf: pointer to the PF instance
4301  */
4302 static void ice_request_fw(struct ice_pf *pf)
4303 {
4304 	char *opt_fw_filename = ice_get_opt_fw_name(pf);
4305 	const struct firmware *firmware = NULL;
4306 	struct device *dev = ice_pf_to_dev(pf);
4307 	int err = 0;
4308 
4309 	/* optional device-specific DDP (if present) overrides the default DDP
4310 	 * package file. kernel logs a debug message if the file doesn't exist,
4311 	 * and warning messages for other errors.
4312 	 */
4313 	if (opt_fw_filename) {
4314 		err = firmware_request_nowarn(&firmware, opt_fw_filename, dev);
4315 		if (err) {
4316 			kfree(opt_fw_filename);
4317 			goto dflt_pkg_load;
4318 		}
4319 
4320 		/* request for firmware was successful. Download to device */
4321 		ice_load_pkg(firmware, pf);
4322 		kfree(opt_fw_filename);
4323 		release_firmware(firmware);
4324 		return;
4325 	}
4326 
4327 dflt_pkg_load:
4328 	err = request_firmware(&firmware, ICE_DDP_PKG_FILE, dev);
4329 	if (err) {
4330 		dev_err(dev, "The DDP package file was not found or could not be read. Entering Safe Mode\n");
4331 		return;
4332 	}
4333 
4334 	/* request for firmware was successful. Download to device */
4335 	ice_load_pkg(firmware, pf);
4336 	release_firmware(firmware);
4337 }
4338 
4339 /**
4340  * ice_print_wake_reason - show the wake up cause in the log
4341  * @pf: pointer to the PF struct
4342  */
4343 static void ice_print_wake_reason(struct ice_pf *pf)
4344 {
4345 	u32 wus = pf->wakeup_reason;
4346 	const char *wake_str;
4347 
4348 	/* if no wake event, nothing to print */
4349 	if (!wus)
4350 		return;
4351 
4352 	if (wus & PFPM_WUS_LNKC_M)
4353 		wake_str = "Link\n";
4354 	else if (wus & PFPM_WUS_MAG_M)
4355 		wake_str = "Magic Packet\n";
4356 	else if (wus & PFPM_WUS_MNG_M)
4357 		wake_str = "Management\n";
4358 	else if (wus & PFPM_WUS_FW_RST_WK_M)
4359 		wake_str = "Firmware Reset\n";
4360 	else
4361 		wake_str = "Unknown\n";
4362 
4363 	dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str);
4364 }
4365 
4366 /**
4367  * ice_register_netdev - register netdev
4368  * @vsi: pointer to the VSI struct
4369  */
4370 static int ice_register_netdev(struct ice_vsi *vsi)
4371 {
4372 	int err;
4373 
4374 	if (!vsi || !vsi->netdev)
4375 		return -EIO;
4376 
4377 	err = register_netdev(vsi->netdev);
4378 	if (err)
4379 		return err;
4380 
4381 	set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
4382 	netif_carrier_off(vsi->netdev);
4383 	netif_tx_stop_all_queues(vsi->netdev);
4384 
4385 	return 0;
4386 }
4387 
4388 static void ice_unregister_netdev(struct ice_vsi *vsi)
4389 {
4390 	if (!vsi || !vsi->netdev)
4391 		return;
4392 
4393 	unregister_netdev(vsi->netdev);
4394 	clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
4395 }
4396 
4397 /**
4398  * ice_cfg_netdev - Allocate, configure and register a netdev
4399  * @vsi: the VSI associated with the new netdev
4400  *
4401  * Returns 0 on success, negative value on failure
4402  */
4403 static int ice_cfg_netdev(struct ice_vsi *vsi)
4404 {
4405 	struct ice_netdev_priv *np;
4406 	struct net_device *netdev;
4407 	u8 mac_addr[ETH_ALEN];
4408 
4409 	netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq,
4410 				    vsi->alloc_rxq);
4411 	if (!netdev)
4412 		return -ENOMEM;
4413 
4414 	set_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state);
4415 	vsi->netdev = netdev;
4416 	np = netdev_priv(netdev);
4417 	np->vsi = vsi;
4418 
4419 	ice_set_netdev_features(netdev);
4420 	ice_set_ops(vsi);
4421 
4422 	if (vsi->type == ICE_VSI_PF) {
4423 		SET_NETDEV_DEV(netdev, ice_pf_to_dev(vsi->back));
4424 		ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
4425 		eth_hw_addr_set(netdev, mac_addr);
4426 	}
4427 
4428 	netdev->priv_flags |= IFF_UNICAST_FLT;
4429 
4430 	/* Setup netdev TC information */
4431 	ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc);
4432 
4433 	netdev->max_mtu = ICE_MAX_MTU;
4434 
4435 	return 0;
4436 }
4437 
4438 static void ice_decfg_netdev(struct ice_vsi *vsi)
4439 {
4440 	clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state);
4441 	free_netdev(vsi->netdev);
4442 	vsi->netdev = NULL;
4443 }
4444 
4445 static int ice_start_eth(struct ice_vsi *vsi)
4446 {
4447 	int err;
4448 
4449 	err = ice_init_mac_fltr(vsi->back);
4450 	if (err)
4451 		return err;
4452 
4453 	err = ice_vsi_open(vsi);
4454 	if (err)
4455 		ice_fltr_remove_all(vsi);
4456 
4457 	return err;
4458 }
4459 
4460 static void ice_stop_eth(struct ice_vsi *vsi)
4461 {
4462 	ice_fltr_remove_all(vsi);
4463 	ice_vsi_close(vsi);
4464 }
4465 
4466 static int ice_init_eth(struct ice_pf *pf)
4467 {
4468 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4469 	int err;
4470 
4471 	if (!vsi)
4472 		return -EINVAL;
4473 
4474 	/* init channel list */
4475 	INIT_LIST_HEAD(&vsi->ch_list);
4476 
4477 	err = ice_cfg_netdev(vsi);
4478 	if (err)
4479 		return err;
4480 	/* Setup DCB netlink interface */
4481 	ice_dcbnl_setup(vsi);
4482 
4483 	err = ice_init_mac_fltr(pf);
4484 	if (err)
4485 		goto err_init_mac_fltr;
4486 
4487 	err = ice_devlink_create_pf_port(pf);
4488 	if (err)
4489 		goto err_devlink_create_pf_port;
4490 
4491 	SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);
4492 
4493 	err = ice_register_netdev(vsi);
4494 	if (err)
4495 		goto err_register_netdev;
4496 
4497 	err = ice_tc_indir_block_register(vsi);
4498 	if (err)
4499 		goto err_tc_indir_block_register;
4500 
4501 	ice_napi_add(vsi);
4502 
4503 	return 0;
4504 
4505 err_tc_indir_block_register:
4506 	ice_unregister_netdev(vsi);
4507 err_register_netdev:
4508 	ice_devlink_destroy_pf_port(pf);
4509 err_devlink_create_pf_port:
4510 err_init_mac_fltr:
4511 	ice_decfg_netdev(vsi);
4512 	return err;
4513 }
4514 
4515 static void ice_deinit_eth(struct ice_pf *pf)
4516 {
4517 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4518 
4519 	if (!vsi)
4520 		return;
4521 
4522 	ice_vsi_close(vsi);
4523 	ice_unregister_netdev(vsi);
4524 	ice_devlink_destroy_pf_port(pf);
4525 	ice_tc_indir_block_unregister(vsi);
4526 	ice_decfg_netdev(vsi);
4527 }
4528 
4529 /**
4530  * ice_wait_for_fw - wait for full FW readiness
4531  * @hw: pointer to the hardware structure
4532  * @timeout: milliseconds that can elapse before timing out
4533  */
4534 static int ice_wait_for_fw(struct ice_hw *hw, u32 timeout)
4535 {
4536 	int fw_loading;
4537 	u32 elapsed = 0;
4538 
4539 	while (elapsed <= timeout) {
4540 		fw_loading = rd32(hw, GL_MNG_FWSM) & GL_MNG_FWSM_FW_LOADING_M;
4541 
4542 		/* firmware was not yet loaded, we have to wait more */
4543 		if (fw_loading) {
4544 			elapsed += 100;
4545 			msleep(100);
4546 			continue;
4547 		}
4548 		return 0;
4549 	}
4550 
4551 	return -ETIMEDOUT;
4552 }
4553 
4554 static int ice_init_dev(struct ice_pf *pf)
4555 {
4556 	struct device *dev = ice_pf_to_dev(pf);
4557 	struct ice_hw *hw = &pf->hw;
4558 	int err;
4559 
4560 	err = ice_init_hw(hw);
4561 	if (err) {
4562 		dev_err(dev, "ice_init_hw failed: %d\n", err);
4563 		return err;
4564 	}
4565 
4566 	/* Some cards require longer initialization times
4567 	 * due to necessity of loading FW from an external source.
4568 	 * This can take even half a minute.
4569 	 */
4570 	if (ice_is_pf_c827(hw)) {
4571 		err = ice_wait_for_fw(hw, 30000);
4572 		if (err) {
4573 			dev_err(dev, "ice_wait_for_fw timed out");
4574 			return err;
4575 		}
4576 	}
4577 
4578 	ice_init_feature_support(pf);
4579 
4580 	ice_request_fw(pf);
4581 
4582 	/* if ice_request_fw fails, ICE_FLAG_ADV_FEATURES bit won't be
4583 	 * set in pf->state, which will cause ice_is_safe_mode to return
4584 	 * true
4585 	 */
4586 	if (ice_is_safe_mode(pf)) {
4587 		/* we already got function/device capabilities but these don't
4588 		 * reflect what the driver needs to do in safe mode. Instead of
4589 		 * adding conditional logic everywhere to ignore these
4590 		 * device/function capabilities, override them.
4591 		 */
4592 		ice_set_safe_mode_caps(hw);
4593 	}
4594 
4595 	err = ice_init_pf(pf);
4596 	if (err) {
4597 		dev_err(dev, "ice_init_pf failed: %d\n", err);
4598 		goto err_init_pf;
4599 	}
4600 
4601 	pf->hw.udp_tunnel_nic.set_port = ice_udp_tunnel_set_port;
4602 	pf->hw.udp_tunnel_nic.unset_port = ice_udp_tunnel_unset_port;
4603 	pf->hw.udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
4604 	pf->hw.udp_tunnel_nic.shared = &pf->hw.udp_tunnel_shared;
4605 	if (pf->hw.tnl.valid_count[TNL_VXLAN]) {
4606 		pf->hw.udp_tunnel_nic.tables[0].n_entries =
4607 			pf->hw.tnl.valid_count[TNL_VXLAN];
4608 		pf->hw.udp_tunnel_nic.tables[0].tunnel_types =
4609 			UDP_TUNNEL_TYPE_VXLAN;
4610 	}
4611 	if (pf->hw.tnl.valid_count[TNL_GENEVE]) {
4612 		pf->hw.udp_tunnel_nic.tables[1].n_entries =
4613 			pf->hw.tnl.valid_count[TNL_GENEVE];
4614 		pf->hw.udp_tunnel_nic.tables[1].tunnel_types =
4615 			UDP_TUNNEL_TYPE_GENEVE;
4616 	}
4617 
4618 	err = ice_init_interrupt_scheme(pf);
4619 	if (err) {
4620 		dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err);
4621 		err = -EIO;
4622 		goto err_init_interrupt_scheme;
4623 	}
4624 
4625 	/* In case of MSIX we are going to setup the misc vector right here
4626 	 * to handle admin queue events etc. In case of legacy and MSI
4627 	 * the misc functionality and queue processing is combined in
4628 	 * the same vector and that gets setup at open.
4629 	 */
4630 	err = ice_req_irq_msix_misc(pf);
4631 	if (err) {
4632 		dev_err(dev, "setup of misc vector failed: %d\n", err);
4633 		goto err_req_irq_msix_misc;
4634 	}
4635 
4636 	return 0;
4637 
4638 err_req_irq_msix_misc:
4639 	ice_clear_interrupt_scheme(pf);
4640 err_init_interrupt_scheme:
4641 	ice_deinit_pf(pf);
4642 err_init_pf:
4643 	ice_deinit_hw(hw);
4644 	return err;
4645 }
4646 
4647 static void ice_deinit_dev(struct ice_pf *pf)
4648 {
4649 	ice_free_irq_msix_misc(pf);
4650 	ice_deinit_pf(pf);
4651 	ice_deinit_hw(&pf->hw);
4652 
4653 	/* Service task is already stopped, so call reset directly. */
4654 	ice_reset(&pf->hw, ICE_RESET_PFR);
4655 	pci_wait_for_pending_transaction(pf->pdev);
4656 	ice_clear_interrupt_scheme(pf);
4657 }
4658 
4659 static void ice_init_features(struct ice_pf *pf)
4660 {
4661 	struct device *dev = ice_pf_to_dev(pf);
4662 
4663 	if (ice_is_safe_mode(pf))
4664 		return;
4665 
4666 	/* initialize DDP driven features */
4667 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
4668 		ice_ptp_init(pf);
4669 
4670 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
4671 		ice_gnss_init(pf);
4672 
4673 	if (ice_is_feature_supported(pf, ICE_F_CGU) ||
4674 	    ice_is_feature_supported(pf, ICE_F_PHY_RCLK))
4675 		ice_dpll_init(pf);
4676 
4677 	/* Note: Flow director init failure is non-fatal to load */
4678 	if (ice_init_fdir(pf))
4679 		dev_err(dev, "could not initialize flow director\n");
4680 
4681 	/* Note: DCB init failure is non-fatal to load */
4682 	if (ice_init_pf_dcb(pf, false)) {
4683 		clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
4684 		clear_bit(ICE_FLAG_DCB_ENA, pf->flags);
4685 	} else {
4686 		ice_cfg_lldp_mib_change(&pf->hw, true);
4687 	}
4688 
4689 	if (ice_init_lag(pf))
4690 		dev_warn(dev, "Failed to init link aggregation support\n");
4691 
4692 	ice_hwmon_init(pf);
4693 }
4694 
4695 static void ice_deinit_features(struct ice_pf *pf)
4696 {
4697 	if (ice_is_safe_mode(pf))
4698 		return;
4699 
4700 	ice_deinit_lag(pf);
4701 	if (test_bit(ICE_FLAG_DCB_CAPABLE, pf->flags))
4702 		ice_cfg_lldp_mib_change(&pf->hw, false);
4703 	ice_deinit_fdir(pf);
4704 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
4705 		ice_gnss_exit(pf);
4706 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
4707 		ice_ptp_release(pf);
4708 	if (test_bit(ICE_FLAG_DPLL, pf->flags))
4709 		ice_dpll_deinit(pf);
4710 	if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV)
4711 		xa_destroy(&pf->eswitch.reprs);
4712 }
4713 
4714 static void ice_init_wakeup(struct ice_pf *pf)
4715 {
4716 	/* Save wakeup reason register for later use */
4717 	pf->wakeup_reason = rd32(&pf->hw, PFPM_WUS);
4718 
4719 	/* check for a power management event */
4720 	ice_print_wake_reason(pf);
4721 
4722 	/* clear wake status, all bits */
4723 	wr32(&pf->hw, PFPM_WUS, U32_MAX);
4724 
4725 	/* Disable WoL at init, wait for user to enable */
4726 	device_set_wakeup_enable(ice_pf_to_dev(pf), false);
4727 }
4728 
4729 static int ice_init_link(struct ice_pf *pf)
4730 {
4731 	struct device *dev = ice_pf_to_dev(pf);
4732 	int err;
4733 
4734 	err = ice_init_link_events(pf->hw.port_info);
4735 	if (err) {
4736 		dev_err(dev, "ice_init_link_events failed: %d\n", err);
4737 		return err;
4738 	}
4739 
4740 	/* not a fatal error if this fails */
4741 	err = ice_init_nvm_phy_type(pf->hw.port_info);
4742 	if (err)
4743 		dev_err(dev, "ice_init_nvm_phy_type failed: %d\n", err);
4744 
4745 	/* not a fatal error if this fails */
4746 	err = ice_update_link_info(pf->hw.port_info);
4747 	if (err)
4748 		dev_err(dev, "ice_update_link_info failed: %d\n", err);
4749 
4750 	ice_init_link_dflt_override(pf->hw.port_info);
4751 
4752 	ice_check_link_cfg_err(pf,
4753 			       pf->hw.port_info->phy.link_info.link_cfg_err);
4754 
4755 	/* if media available, initialize PHY settings */
4756 	if (pf->hw.port_info->phy.link_info.link_info &
4757 	    ICE_AQ_MEDIA_AVAILABLE) {
4758 		/* not a fatal error if this fails */
4759 		err = ice_init_phy_user_cfg(pf->hw.port_info);
4760 		if (err)
4761 			dev_err(dev, "ice_init_phy_user_cfg failed: %d\n", err);
4762 
4763 		if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) {
4764 			struct ice_vsi *vsi = ice_get_main_vsi(pf);
4765 
4766 			if (vsi)
4767 				ice_configure_phy(vsi);
4768 		}
4769 	} else {
4770 		set_bit(ICE_FLAG_NO_MEDIA, pf->flags);
4771 	}
4772 
4773 	return err;
4774 }
4775 
4776 static int ice_init_pf_sw(struct ice_pf *pf)
4777 {
4778 	bool dvm = ice_is_dvm_ena(&pf->hw);
4779 	struct ice_vsi *vsi;
4780 	int err;
4781 
4782 	/* create switch struct for the switch element created by FW on boot */
4783 	pf->first_sw = kzalloc(sizeof(*pf->first_sw), GFP_KERNEL);
4784 	if (!pf->first_sw)
4785 		return -ENOMEM;
4786 
4787 	if (pf->hw.evb_veb)
4788 		pf->first_sw->bridge_mode = BRIDGE_MODE_VEB;
4789 	else
4790 		pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA;
4791 
4792 	pf->first_sw->pf = pf;
4793 
4794 	/* record the sw_id available for later use */
4795 	pf->first_sw->sw_id = pf->hw.port_info->sw_id;
4796 
4797 	err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL);
4798 	if (err)
4799 		goto err_aq_set_port_params;
4800 
4801 	vsi = ice_pf_vsi_setup(pf, pf->hw.port_info);
4802 	if (!vsi) {
4803 		err = -ENOMEM;
4804 		goto err_pf_vsi_setup;
4805 	}
4806 
4807 	return 0;
4808 
4809 err_pf_vsi_setup:
4810 err_aq_set_port_params:
4811 	kfree(pf->first_sw);
4812 	return err;
4813 }
4814 
4815 static void ice_deinit_pf_sw(struct ice_pf *pf)
4816 {
4817 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4818 
4819 	if (!vsi)
4820 		return;
4821 
4822 	ice_vsi_release(vsi);
4823 	kfree(pf->first_sw);
4824 }
4825 
4826 static int ice_alloc_vsis(struct ice_pf *pf)
4827 {
4828 	struct device *dev = ice_pf_to_dev(pf);
4829 
4830 	pf->num_alloc_vsi = pf->hw.func_caps.guar_num_vsi;
4831 	if (!pf->num_alloc_vsi)
4832 		return -EIO;
4833 
4834 	if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
4835 		dev_warn(dev,
4836 			 "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
4837 			 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
4838 		pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
4839 	}
4840 
4841 	pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi),
4842 			       GFP_KERNEL);
4843 	if (!pf->vsi)
4844 		return -ENOMEM;
4845 
4846 	pf->vsi_stats = devm_kcalloc(dev, pf->num_alloc_vsi,
4847 				     sizeof(*pf->vsi_stats), GFP_KERNEL);
4848 	if (!pf->vsi_stats) {
4849 		devm_kfree(dev, pf->vsi);
4850 		return -ENOMEM;
4851 	}
4852 
4853 	return 0;
4854 }
4855 
4856 static void ice_dealloc_vsis(struct ice_pf *pf)
4857 {
4858 	devm_kfree(ice_pf_to_dev(pf), pf->vsi_stats);
4859 	pf->vsi_stats = NULL;
4860 
4861 	pf->num_alloc_vsi = 0;
4862 	devm_kfree(ice_pf_to_dev(pf), pf->vsi);
4863 	pf->vsi = NULL;
4864 }
4865 
4866 static int ice_init_devlink(struct ice_pf *pf)
4867 {
4868 	int err;
4869 
4870 	err = ice_devlink_register_params(pf);
4871 	if (err)
4872 		return err;
4873 
4874 	ice_devlink_init_regions(pf);
4875 	ice_devlink_register(pf);
4876 
4877 	return 0;
4878 }
4879 
4880 static void ice_deinit_devlink(struct ice_pf *pf)
4881 {
4882 	ice_devlink_unregister(pf);
4883 	ice_devlink_destroy_regions(pf);
4884 	ice_devlink_unregister_params(pf);
4885 }
4886 
4887 static int ice_init(struct ice_pf *pf)
4888 {
4889 	int err;
4890 
4891 	err = ice_init_dev(pf);
4892 	if (err)
4893 		return err;
4894 
4895 	err = ice_alloc_vsis(pf);
4896 	if (err)
4897 		goto err_alloc_vsis;
4898 
4899 	err = ice_init_pf_sw(pf);
4900 	if (err)
4901 		goto err_init_pf_sw;
4902 
4903 	ice_init_wakeup(pf);
4904 
4905 	err = ice_init_link(pf);
4906 	if (err)
4907 		goto err_init_link;
4908 
4909 	err = ice_send_version(pf);
4910 	if (err)
4911 		goto err_init_link;
4912 
4913 	ice_verify_cacheline_size(pf);
4914 
4915 	if (ice_is_safe_mode(pf))
4916 		ice_set_safe_mode_vlan_cfg(pf);
4917 	else
4918 		/* print PCI link speed and width */
4919 		pcie_print_link_status(pf->pdev);
4920 
4921 	/* ready to go, so clear down state bit */
4922 	clear_bit(ICE_DOWN, pf->state);
4923 	clear_bit(ICE_SERVICE_DIS, pf->state);
4924 
4925 	/* since everything is good, start the service timer */
4926 	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
4927 
4928 	return 0;
4929 
4930 err_init_link:
4931 	ice_deinit_pf_sw(pf);
4932 err_init_pf_sw:
4933 	ice_dealloc_vsis(pf);
4934 err_alloc_vsis:
4935 	ice_deinit_dev(pf);
4936 	return err;
4937 }
4938 
4939 static void ice_deinit(struct ice_pf *pf)
4940 {
4941 	set_bit(ICE_SERVICE_DIS, pf->state);
4942 	set_bit(ICE_DOWN, pf->state);
4943 
4944 	ice_deinit_pf_sw(pf);
4945 	ice_dealloc_vsis(pf);
4946 	ice_deinit_dev(pf);
4947 }
4948 
4949 /**
4950  * ice_load - load pf by init hw and starting VSI
4951  * @pf: pointer to the pf instance
4952  */
4953 int ice_load(struct ice_pf *pf)
4954 {
4955 	struct ice_vsi_cfg_params params = {};
4956 	struct ice_vsi *vsi;
4957 	int err;
4958 
4959 	err = ice_init_dev(pf);
4960 	if (err)
4961 		return err;
4962 
4963 	vsi = ice_get_main_vsi(pf);
4964 
4965 	params = ice_vsi_to_params(vsi);
4966 	params.flags = ICE_VSI_FLAG_INIT;
4967 
4968 	rtnl_lock();
4969 	err = ice_vsi_cfg(vsi, &params);
4970 	if (err)
4971 		goto err_vsi_cfg;
4972 
4973 	err = ice_start_eth(ice_get_main_vsi(pf));
4974 	if (err)
4975 		goto err_start_eth;
4976 	rtnl_unlock();
4977 
4978 	err = ice_init_rdma(pf);
4979 	if (err)
4980 		goto err_init_rdma;
4981 
4982 	ice_init_features(pf);
4983 	ice_service_task_restart(pf);
4984 
4985 	clear_bit(ICE_DOWN, pf->state);
4986 
4987 	return 0;
4988 
4989 err_init_rdma:
4990 	ice_vsi_close(ice_get_main_vsi(pf));
4991 	rtnl_lock();
4992 err_start_eth:
4993 	ice_vsi_decfg(ice_get_main_vsi(pf));
4994 err_vsi_cfg:
4995 	rtnl_unlock();
4996 	ice_deinit_dev(pf);
4997 	return err;
4998 }
4999 
5000 /**
5001  * ice_unload - unload pf by stopping VSI and deinit hw
5002  * @pf: pointer to the pf instance
5003  */
5004 void ice_unload(struct ice_pf *pf)
5005 {
5006 	ice_deinit_features(pf);
5007 	ice_deinit_rdma(pf);
5008 	rtnl_lock();
5009 	ice_stop_eth(ice_get_main_vsi(pf));
5010 	ice_vsi_decfg(ice_get_main_vsi(pf));
5011 	rtnl_unlock();
5012 	ice_deinit_dev(pf);
5013 }
5014 
5015 /**
5016  * ice_probe - Device initialization routine
5017  * @pdev: PCI device information struct
5018  * @ent: entry in ice_pci_tbl
5019  *
5020  * Returns 0 on success, negative on failure
5021  */
5022 static int
5023 ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
5024 {
5025 	struct device *dev = &pdev->dev;
5026 	struct ice_pf *pf;
5027 	struct ice_hw *hw;
5028 	int err;
5029 
5030 	if (pdev->is_virtfn) {
5031 		dev_err(dev, "can't probe a virtual function\n");
5032 		return -EINVAL;
5033 	}
5034 
5035 	/* when under a kdump kernel initiate a reset before enabling the
5036 	 * device in order to clear out any pending DMA transactions. These
5037 	 * transactions can cause some systems to machine check when doing
5038 	 * the pcim_enable_device() below.
5039 	 */
5040 	if (is_kdump_kernel()) {
5041 		pci_save_state(pdev);
5042 		pci_clear_master(pdev);
5043 		err = pcie_flr(pdev);
5044 		if (err)
5045 			return err;
5046 		pci_restore_state(pdev);
5047 	}
5048 
5049 	/* this driver uses devres, see
5050 	 * Documentation/driver-api/driver-model/devres.rst
5051 	 */
5052 	err = pcim_enable_device(pdev);
5053 	if (err)
5054 		return err;
5055 
5056 	err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), dev_driver_string(dev));
5057 	if (err) {
5058 		dev_err(dev, "BAR0 I/O map error %d\n", err);
5059 		return err;
5060 	}
5061 
5062 	pf = ice_allocate_pf(dev);
5063 	if (!pf)
5064 		return -ENOMEM;
5065 
5066 	/* initialize Auxiliary index to invalid value */
5067 	pf->aux_idx = -1;
5068 
5069 	/* set up for high or low DMA */
5070 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
5071 	if (err) {
5072 		dev_err(dev, "DMA configuration failed: 0x%x\n", err);
5073 		return err;
5074 	}
5075 
5076 	pci_set_master(pdev);
5077 
5078 	pf->pdev = pdev;
5079 	pci_set_drvdata(pdev, pf);
5080 	set_bit(ICE_DOWN, pf->state);
5081 	/* Disable service task until DOWN bit is cleared */
5082 	set_bit(ICE_SERVICE_DIS, pf->state);
5083 
5084 	hw = &pf->hw;
5085 	hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0];
5086 	pci_save_state(pdev);
5087 
5088 	hw->back = pf;
5089 	hw->port_info = NULL;
5090 	hw->vendor_id = pdev->vendor;
5091 	hw->device_id = pdev->device;
5092 	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
5093 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
5094 	hw->subsystem_device_id = pdev->subsystem_device;
5095 	hw->bus.device = PCI_SLOT(pdev->devfn);
5096 	hw->bus.func = PCI_FUNC(pdev->devfn);
5097 	ice_set_ctrlq_len(hw);
5098 
5099 	pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M);
5100 
5101 #ifndef CONFIG_DYNAMIC_DEBUG
5102 	if (debug < -1)
5103 		hw->debug_mask = debug;
5104 #endif
5105 
5106 	err = ice_init(pf);
5107 	if (err)
5108 		goto err_init;
5109 
5110 	err = ice_init_eth(pf);
5111 	if (err)
5112 		goto err_init_eth;
5113 
5114 	err = ice_init_rdma(pf);
5115 	if (err)
5116 		goto err_init_rdma;
5117 
5118 	err = ice_init_devlink(pf);
5119 	if (err)
5120 		goto err_init_devlink;
5121 
5122 	ice_init_features(pf);
5123 
5124 	return 0;
5125 
5126 err_init_devlink:
5127 	ice_deinit_rdma(pf);
5128 err_init_rdma:
5129 	ice_deinit_eth(pf);
5130 err_init_eth:
5131 	ice_deinit(pf);
5132 err_init:
5133 	pci_disable_device(pdev);
5134 	return err;
5135 }
5136 
5137 /**
5138  * ice_set_wake - enable or disable Wake on LAN
5139  * @pf: pointer to the PF struct
5140  *
5141  * Simple helper for WoL control
5142  */
5143 static void ice_set_wake(struct ice_pf *pf)
5144 {
5145 	struct ice_hw *hw = &pf->hw;
5146 	bool wol = pf->wol_ena;
5147 
5148 	/* clear wake state, otherwise new wake events won't fire */
5149 	wr32(hw, PFPM_WUS, U32_MAX);
5150 
5151 	/* enable / disable APM wake up, no RMW needed */
5152 	wr32(hw, PFPM_APM, wol ? PFPM_APM_APME_M : 0);
5153 
5154 	/* set magic packet filter enabled */
5155 	wr32(hw, PFPM_WUFC, wol ? PFPM_WUFC_MAG_M : 0);
5156 }
5157 
5158 /**
5159  * ice_setup_mc_magic_wake - setup device to wake on multicast magic packet
5160  * @pf: pointer to the PF struct
5161  *
5162  * Issue firmware command to enable multicast magic wake, making
5163  * sure that any locally administered address (LAA) is used for
5164  * wake, and that PF reset doesn't undo the LAA.
5165  */
5166 static void ice_setup_mc_magic_wake(struct ice_pf *pf)
5167 {
5168 	struct device *dev = ice_pf_to_dev(pf);
5169 	struct ice_hw *hw = &pf->hw;
5170 	u8 mac_addr[ETH_ALEN];
5171 	struct ice_vsi *vsi;
5172 	int status;
5173 	u8 flags;
5174 
5175 	if (!pf->wol_ena)
5176 		return;
5177 
5178 	vsi = ice_get_main_vsi(pf);
5179 	if (!vsi)
5180 		return;
5181 
5182 	/* Get current MAC address in case it's an LAA */
5183 	if (vsi->netdev)
5184 		ether_addr_copy(mac_addr, vsi->netdev->dev_addr);
5185 	else
5186 		ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
5187 
5188 	flags = ICE_AQC_MAN_MAC_WR_MC_MAG_EN |
5189 		ICE_AQC_MAN_MAC_UPDATE_LAA_WOL |
5190 		ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP;
5191 
5192 	status = ice_aq_manage_mac_write(hw, mac_addr, flags, NULL);
5193 	if (status)
5194 		dev_err(dev, "Failed to enable Multicast Magic Packet wake, err %d aq_err %s\n",
5195 			status, ice_aq_str(hw->adminq.sq_last_status));
5196 }
5197 
5198 /**
5199  * ice_remove - Device removal routine
5200  * @pdev: PCI device information struct
5201  */
5202 static void ice_remove(struct pci_dev *pdev)
5203 {
5204 	struct ice_pf *pf = pci_get_drvdata(pdev);
5205 	int i;
5206 
5207 	for (i = 0; i < ICE_MAX_RESET_WAIT; i++) {
5208 		if (!ice_is_reset_in_progress(pf->state))
5209 			break;
5210 		msleep(100);
5211 	}
5212 
5213 	if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) {
5214 		set_bit(ICE_VF_RESETS_DISABLED, pf->state);
5215 		ice_free_vfs(pf);
5216 	}
5217 
5218 	ice_hwmon_exit(pf);
5219 
5220 	ice_service_task_stop(pf);
5221 	ice_aq_cancel_waiting_tasks(pf);
5222 	set_bit(ICE_DOWN, pf->state);
5223 
5224 	if (!ice_is_safe_mode(pf))
5225 		ice_remove_arfs(pf);
5226 	ice_deinit_features(pf);
5227 	ice_deinit_devlink(pf);
5228 	ice_deinit_rdma(pf);
5229 	ice_deinit_eth(pf);
5230 	ice_deinit(pf);
5231 
5232 	ice_vsi_release_all(pf);
5233 
5234 	ice_setup_mc_magic_wake(pf);
5235 	ice_set_wake(pf);
5236 
5237 	pci_disable_device(pdev);
5238 }
5239 
5240 /**
5241  * ice_shutdown - PCI callback for shutting down device
5242  * @pdev: PCI device information struct
5243  */
5244 static void ice_shutdown(struct pci_dev *pdev)
5245 {
5246 	struct ice_pf *pf = pci_get_drvdata(pdev);
5247 
5248 	ice_remove(pdev);
5249 
5250 	if (system_state == SYSTEM_POWER_OFF) {
5251 		pci_wake_from_d3(pdev, pf->wol_ena);
5252 		pci_set_power_state(pdev, PCI_D3hot);
5253 	}
5254 }
5255 
5256 #ifdef CONFIG_PM
5257 /**
5258  * ice_prepare_for_shutdown - prep for PCI shutdown
5259  * @pf: board private structure
5260  *
5261  * Inform or close all dependent features in prep for PCI device shutdown
5262  */
5263 static void ice_prepare_for_shutdown(struct ice_pf *pf)
5264 {
5265 	struct ice_hw *hw = &pf->hw;
5266 	u32 v;
5267 
5268 	/* Notify VFs of impending reset */
5269 	if (ice_check_sq_alive(hw, &hw->mailboxq))
5270 		ice_vc_notify_reset(pf);
5271 
5272 	dev_dbg(ice_pf_to_dev(pf), "Tearing down internal switch for shutdown\n");
5273 
5274 	/* disable the VSIs and their queues that are not already DOWN */
5275 	ice_pf_dis_all_vsi(pf, false);
5276 
5277 	ice_for_each_vsi(pf, v)
5278 		if (pf->vsi[v])
5279 			pf->vsi[v]->vsi_num = 0;
5280 
5281 	ice_shutdown_all_ctrlq(hw);
5282 }
5283 
5284 /**
5285  * ice_reinit_interrupt_scheme - Reinitialize interrupt scheme
5286  * @pf: board private structure to reinitialize
5287  *
5288  * This routine reinitialize interrupt scheme that was cleared during
5289  * power management suspend callback.
5290  *
5291  * This should be called during resume routine to re-allocate the q_vectors
5292  * and reacquire interrupts.
5293  */
5294 static int ice_reinit_interrupt_scheme(struct ice_pf *pf)
5295 {
5296 	struct device *dev = ice_pf_to_dev(pf);
5297 	int ret, v;
5298 
5299 	/* Since we clear MSIX flag during suspend, we need to
5300 	 * set it back during resume...
5301 	 */
5302 
5303 	ret = ice_init_interrupt_scheme(pf);
5304 	if (ret) {
5305 		dev_err(dev, "Failed to re-initialize interrupt %d\n", ret);
5306 		return ret;
5307 	}
5308 
5309 	/* Remap vectors and rings, after successful re-init interrupts */
5310 	ice_for_each_vsi(pf, v) {
5311 		if (!pf->vsi[v])
5312 			continue;
5313 
5314 		ret = ice_vsi_alloc_q_vectors(pf->vsi[v]);
5315 		if (ret)
5316 			goto err_reinit;
5317 		ice_vsi_map_rings_to_vectors(pf->vsi[v]);
5318 	}
5319 
5320 	ret = ice_req_irq_msix_misc(pf);
5321 	if (ret) {
5322 		dev_err(dev, "Setting up misc vector failed after device suspend %d\n",
5323 			ret);
5324 		goto err_reinit;
5325 	}
5326 
5327 	return 0;
5328 
5329 err_reinit:
5330 	while (v--)
5331 		if (pf->vsi[v])
5332 			ice_vsi_free_q_vectors(pf->vsi[v]);
5333 
5334 	return ret;
5335 }
5336 
5337 /**
5338  * ice_suspend
5339  * @dev: generic device information structure
5340  *
5341  * Power Management callback to quiesce the device and prepare
5342  * for D3 transition.
5343  */
5344 static int __maybe_unused ice_suspend(struct device *dev)
5345 {
5346 	struct pci_dev *pdev = to_pci_dev(dev);
5347 	struct ice_pf *pf;
5348 	int disabled, v;
5349 
5350 	pf = pci_get_drvdata(pdev);
5351 
5352 	if (!ice_pf_state_is_nominal(pf)) {
5353 		dev_err(dev, "Device is not ready, no need to suspend it\n");
5354 		return -EBUSY;
5355 	}
5356 
5357 	/* Stop watchdog tasks until resume completion.
5358 	 * Even though it is most likely that the service task is
5359 	 * disabled if the device is suspended or down, the service task's
5360 	 * state is controlled by a different state bit, and we should
5361 	 * store and honor whatever state that bit is in at this point.
5362 	 */
5363 	disabled = ice_service_task_stop(pf);
5364 
5365 	ice_unplug_aux_dev(pf);
5366 
5367 	/* Already suspended?, then there is nothing to do */
5368 	if (test_and_set_bit(ICE_SUSPENDED, pf->state)) {
5369 		if (!disabled)
5370 			ice_service_task_restart(pf);
5371 		return 0;
5372 	}
5373 
5374 	if (test_bit(ICE_DOWN, pf->state) ||
5375 	    ice_is_reset_in_progress(pf->state)) {
5376 		dev_err(dev, "can't suspend device in reset or already down\n");
5377 		if (!disabled)
5378 			ice_service_task_restart(pf);
5379 		return 0;
5380 	}
5381 
5382 	ice_setup_mc_magic_wake(pf);
5383 
5384 	ice_prepare_for_shutdown(pf);
5385 
5386 	ice_set_wake(pf);
5387 
5388 	/* Free vectors, clear the interrupt scheme and release IRQs
5389 	 * for proper hibernation, especially with large number of CPUs.
5390 	 * Otherwise hibernation might fail when mapping all the vectors back
5391 	 * to CPU0.
5392 	 */
5393 	ice_free_irq_msix_misc(pf);
5394 	ice_for_each_vsi(pf, v) {
5395 		if (!pf->vsi[v])
5396 			continue;
5397 		ice_vsi_free_q_vectors(pf->vsi[v]);
5398 	}
5399 	ice_clear_interrupt_scheme(pf);
5400 
5401 	pci_save_state(pdev);
5402 	pci_wake_from_d3(pdev, pf->wol_ena);
5403 	pci_set_power_state(pdev, PCI_D3hot);
5404 	return 0;
5405 }
5406 
5407 /**
5408  * ice_resume - PM callback for waking up from D3
5409  * @dev: generic device information structure
5410  */
5411 static int __maybe_unused ice_resume(struct device *dev)
5412 {
5413 	struct pci_dev *pdev = to_pci_dev(dev);
5414 	enum ice_reset_req reset_type;
5415 	struct ice_pf *pf;
5416 	struct ice_hw *hw;
5417 	int ret;
5418 
5419 	pci_set_power_state(pdev, PCI_D0);
5420 	pci_restore_state(pdev);
5421 	pci_save_state(pdev);
5422 
5423 	if (!pci_device_is_present(pdev))
5424 		return -ENODEV;
5425 
5426 	ret = pci_enable_device_mem(pdev);
5427 	if (ret) {
5428 		dev_err(dev, "Cannot enable device after suspend\n");
5429 		return ret;
5430 	}
5431 
5432 	pf = pci_get_drvdata(pdev);
5433 	hw = &pf->hw;
5434 
5435 	pf->wakeup_reason = rd32(hw, PFPM_WUS);
5436 	ice_print_wake_reason(pf);
5437 
5438 	/* We cleared the interrupt scheme when we suspended, so we need to
5439 	 * restore it now to resume device functionality.
5440 	 */
5441 	ret = ice_reinit_interrupt_scheme(pf);
5442 	if (ret)
5443 		dev_err(dev, "Cannot restore interrupt scheme: %d\n", ret);
5444 
5445 	clear_bit(ICE_DOWN, pf->state);
5446 	/* Now perform PF reset and rebuild */
5447 	reset_type = ICE_RESET_PFR;
5448 	/* re-enable service task for reset, but allow reset to schedule it */
5449 	clear_bit(ICE_SERVICE_DIS, pf->state);
5450 
5451 	if (ice_schedule_reset(pf, reset_type))
5452 		dev_err(dev, "Reset during resume failed.\n");
5453 
5454 	clear_bit(ICE_SUSPENDED, pf->state);
5455 	ice_service_task_restart(pf);
5456 
5457 	/* Restart the service task */
5458 	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
5459 
5460 	return 0;
5461 }
5462 #endif /* CONFIG_PM */
5463 
5464 /**
5465  * ice_pci_err_detected - warning that PCI error has been detected
5466  * @pdev: PCI device information struct
5467  * @err: the type of PCI error
5468  *
5469  * Called to warn that something happened on the PCI bus and the error handling
5470  * is in progress.  Allows the driver to gracefully prepare/handle PCI errors.
5471  */
5472 static pci_ers_result_t
5473 ice_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t err)
5474 {
5475 	struct ice_pf *pf = pci_get_drvdata(pdev);
5476 
5477 	if (!pf) {
5478 		dev_err(&pdev->dev, "%s: unrecoverable device error %d\n",
5479 			__func__, err);
5480 		return PCI_ERS_RESULT_DISCONNECT;
5481 	}
5482 
5483 	if (!test_bit(ICE_SUSPENDED, pf->state)) {
5484 		ice_service_task_stop(pf);
5485 
5486 		if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) {
5487 			set_bit(ICE_PFR_REQ, pf->state);
5488 			ice_prepare_for_reset(pf, ICE_RESET_PFR);
5489 		}
5490 	}
5491 
5492 	return PCI_ERS_RESULT_NEED_RESET;
5493 }
5494 
5495 /**
5496  * ice_pci_err_slot_reset - a PCI slot reset has just happened
5497  * @pdev: PCI device information struct
5498  *
5499  * Called to determine if the driver can recover from the PCI slot reset by
5500  * using a register read to determine if the device is recoverable.
5501  */
5502 static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev)
5503 {
5504 	struct ice_pf *pf = pci_get_drvdata(pdev);
5505 	pci_ers_result_t result;
5506 	int err;
5507 	u32 reg;
5508 
5509 	err = pci_enable_device_mem(pdev);
5510 	if (err) {
5511 		dev_err(&pdev->dev, "Cannot re-enable PCI device after reset, error %d\n",
5512 			err);
5513 		result = PCI_ERS_RESULT_DISCONNECT;
5514 	} else {
5515 		pci_set_master(pdev);
5516 		pci_restore_state(pdev);
5517 		pci_save_state(pdev);
5518 		pci_wake_from_d3(pdev, false);
5519 
5520 		/* Check for life */
5521 		reg = rd32(&pf->hw, GLGEN_RTRIG);
5522 		if (!reg)
5523 			result = PCI_ERS_RESULT_RECOVERED;
5524 		else
5525 			result = PCI_ERS_RESULT_DISCONNECT;
5526 	}
5527 
5528 	return result;
5529 }
5530 
5531 /**
5532  * ice_pci_err_resume - restart operations after PCI error recovery
5533  * @pdev: PCI device information struct
5534  *
5535  * Called to allow the driver to bring things back up after PCI error and/or
5536  * reset recovery have finished
5537  */
5538 static void ice_pci_err_resume(struct pci_dev *pdev)
5539 {
5540 	struct ice_pf *pf = pci_get_drvdata(pdev);
5541 
5542 	if (!pf) {
5543 		dev_err(&pdev->dev, "%s failed, device is unrecoverable\n",
5544 			__func__);
5545 		return;
5546 	}
5547 
5548 	if (test_bit(ICE_SUSPENDED, pf->state)) {
5549 		dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n",
5550 			__func__);
5551 		return;
5552 	}
5553 
5554 	ice_restore_all_vfs_msi_state(pf);
5555 
5556 	ice_do_reset(pf, ICE_RESET_PFR);
5557 	ice_service_task_restart(pf);
5558 	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
5559 }
5560 
5561 /**
5562  * ice_pci_err_reset_prepare - prepare device driver for PCI reset
5563  * @pdev: PCI device information struct
5564  */
5565 static void ice_pci_err_reset_prepare(struct pci_dev *pdev)
5566 {
5567 	struct ice_pf *pf = pci_get_drvdata(pdev);
5568 
5569 	if (!test_bit(ICE_SUSPENDED, pf->state)) {
5570 		ice_service_task_stop(pf);
5571 
5572 		if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) {
5573 			set_bit(ICE_PFR_REQ, pf->state);
5574 			ice_prepare_for_reset(pf, ICE_RESET_PFR);
5575 		}
5576 	}
5577 }
5578 
5579 /**
5580  * ice_pci_err_reset_done - PCI reset done, device driver reset can begin
5581  * @pdev: PCI device information struct
5582  */
5583 static void ice_pci_err_reset_done(struct pci_dev *pdev)
5584 {
5585 	ice_pci_err_resume(pdev);
5586 }
5587 
5588 /* ice_pci_tbl - PCI Device ID Table
5589  *
5590  * Wildcard entries (PCI_ANY_ID) should come last
5591  * Last entry must be all 0s
5592  *
5593  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
5594  *   Class, Class Mask, private data (not used) }
5595  */
5596 static const struct pci_device_id ice_pci_tbl[] = {
5597 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE) },
5598 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP) },
5599 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP) },
5600 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_BACKPLANE) },
5601 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_QSFP) },
5602 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_SFP) },
5603 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_BACKPLANE) },
5604 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_QSFP) },
5605 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SFP) },
5606 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_10G_BASE_T) },
5607 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SGMII) },
5608 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_BACKPLANE) },
5609 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_QSFP) },
5610 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SFP) },
5611 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_10G_BASE_T) },
5612 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SGMII) },
5613 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_BACKPLANE) },
5614 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SFP) },
5615 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_10G_BASE_T) },
5616 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SGMII) },
5617 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_BACKPLANE) },
5618 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_SFP) },
5619 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_10G_BASE_T) },
5620 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_1GBE) },
5621 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_QSFP) },
5622 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822_SI_DFLT) },
5623 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E830_BACKPLANE) },
5624 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E830_QSFP56) },
5625 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E830_SFP) },
5626 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E830_SFP_DD) },
5627 	/* required last entry */
5628 	{}
5629 };
5630 MODULE_DEVICE_TABLE(pci, ice_pci_tbl);
5631 
5632 static __maybe_unused SIMPLE_DEV_PM_OPS(ice_pm_ops, ice_suspend, ice_resume);
5633 
5634 static const struct pci_error_handlers ice_pci_err_handler = {
5635 	.error_detected = ice_pci_err_detected,
5636 	.slot_reset = ice_pci_err_slot_reset,
5637 	.reset_prepare = ice_pci_err_reset_prepare,
5638 	.reset_done = ice_pci_err_reset_done,
5639 	.resume = ice_pci_err_resume
5640 };
5641 
5642 static struct pci_driver ice_driver = {
5643 	.name = KBUILD_MODNAME,
5644 	.id_table = ice_pci_tbl,
5645 	.probe = ice_probe,
5646 	.remove = ice_remove,
5647 #ifdef CONFIG_PM
5648 	.driver.pm = &ice_pm_ops,
5649 #endif /* CONFIG_PM */
5650 	.shutdown = ice_shutdown,
5651 	.sriov_configure = ice_sriov_configure,
5652 	.sriov_get_vf_total_msix = ice_sriov_get_vf_total_msix,
5653 	.sriov_set_msix_vec_count = ice_sriov_set_msix_vec_count,
5654 	.err_handler = &ice_pci_err_handler
5655 };
5656 
5657 /**
5658  * ice_module_init - Driver registration routine
5659  *
5660  * ice_module_init is the first routine called when the driver is
5661  * loaded. All it does is register with the PCI subsystem.
5662  */
5663 static int __init ice_module_init(void)
5664 {
5665 	int status = -ENOMEM;
5666 
5667 	pr_info("%s\n", ice_driver_string);
5668 	pr_info("%s\n", ice_copyright);
5669 
5670 	ice_adv_lnk_speed_maps_init();
5671 
5672 	ice_wq = alloc_workqueue("%s", 0, 0, KBUILD_MODNAME);
5673 	if (!ice_wq) {
5674 		pr_err("Failed to create workqueue\n");
5675 		return status;
5676 	}
5677 
5678 	ice_lag_wq = alloc_ordered_workqueue("ice_lag_wq", 0);
5679 	if (!ice_lag_wq) {
5680 		pr_err("Failed to create LAG workqueue\n");
5681 		goto err_dest_wq;
5682 	}
5683 
5684 	status = pci_register_driver(&ice_driver);
5685 	if (status) {
5686 		pr_err("failed to register PCI driver, err %d\n", status);
5687 		goto err_dest_lag_wq;
5688 	}
5689 
5690 	return 0;
5691 
5692 err_dest_lag_wq:
5693 	destroy_workqueue(ice_lag_wq);
5694 err_dest_wq:
5695 	destroy_workqueue(ice_wq);
5696 	return status;
5697 }
5698 module_init(ice_module_init);
5699 
5700 /**
5701  * ice_module_exit - Driver exit cleanup routine
5702  *
5703  * ice_module_exit is called just before the driver is removed
5704  * from memory.
5705  */
5706 static void __exit ice_module_exit(void)
5707 {
5708 	pci_unregister_driver(&ice_driver);
5709 	destroy_workqueue(ice_wq);
5710 	destroy_workqueue(ice_lag_wq);
5711 	pr_info("module unloaded\n");
5712 }
5713 module_exit(ice_module_exit);
5714 
5715 /**
5716  * ice_set_mac_address - NDO callback to set MAC address
5717  * @netdev: network interface device structure
5718  * @pi: pointer to an address structure
5719  *
5720  * Returns 0 on success, negative on failure
5721  */
5722 static int ice_set_mac_address(struct net_device *netdev, void *pi)
5723 {
5724 	struct ice_netdev_priv *np = netdev_priv(netdev);
5725 	struct ice_vsi *vsi = np->vsi;
5726 	struct ice_pf *pf = vsi->back;
5727 	struct ice_hw *hw = &pf->hw;
5728 	struct sockaddr *addr = pi;
5729 	u8 old_mac[ETH_ALEN];
5730 	u8 flags = 0;
5731 	u8 *mac;
5732 	int err;
5733 
5734 	mac = (u8 *)addr->sa_data;
5735 
5736 	if (!is_valid_ether_addr(mac))
5737 		return -EADDRNOTAVAIL;
5738 
5739 	if (test_bit(ICE_DOWN, pf->state) ||
5740 	    ice_is_reset_in_progress(pf->state)) {
5741 		netdev_err(netdev, "can't set mac %pM. device not ready\n",
5742 			   mac);
5743 		return -EBUSY;
5744 	}
5745 
5746 	if (ice_chnl_dmac_fltr_cnt(pf)) {
5747 		netdev_err(netdev, "can't set mac %pM. Device has tc-flower filters, delete all of them and try again\n",
5748 			   mac);
5749 		return -EAGAIN;
5750 	}
5751 
5752 	netif_addr_lock_bh(netdev);
5753 	ether_addr_copy(old_mac, netdev->dev_addr);
5754 	/* change the netdev's MAC address */
5755 	eth_hw_addr_set(netdev, mac);
5756 	netif_addr_unlock_bh(netdev);
5757 
5758 	/* Clean up old MAC filter. Not an error if old filter doesn't exist */
5759 	err = ice_fltr_remove_mac(vsi, old_mac, ICE_FWD_TO_VSI);
5760 	if (err && err != -ENOENT) {
5761 		err = -EADDRNOTAVAIL;
5762 		goto err_update_filters;
5763 	}
5764 
5765 	/* Add filter for new MAC. If filter exists, return success */
5766 	err = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI);
5767 	if (err == -EEXIST) {
5768 		/* Although this MAC filter is already present in hardware it's
5769 		 * possible in some cases (e.g. bonding) that dev_addr was
5770 		 * modified outside of the driver and needs to be restored back
5771 		 * to this value.
5772 		 */
5773 		netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac);
5774 
5775 		return 0;
5776 	} else if (err) {
5777 		/* error if the new filter addition failed */
5778 		err = -EADDRNOTAVAIL;
5779 	}
5780 
5781 err_update_filters:
5782 	if (err) {
5783 		netdev_err(netdev, "can't set MAC %pM. filter update failed\n",
5784 			   mac);
5785 		netif_addr_lock_bh(netdev);
5786 		eth_hw_addr_set(netdev, old_mac);
5787 		netif_addr_unlock_bh(netdev);
5788 		return err;
5789 	}
5790 
5791 	netdev_dbg(vsi->netdev, "updated MAC address to %pM\n",
5792 		   netdev->dev_addr);
5793 
5794 	/* write new MAC address to the firmware */
5795 	flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL;
5796 	err = ice_aq_manage_mac_write(hw, mac, flags, NULL);
5797 	if (err) {
5798 		netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %d\n",
5799 			   mac, err);
5800 	}
5801 	return 0;
5802 }
5803 
5804 /**
5805  * ice_set_rx_mode - NDO callback to set the netdev filters
5806  * @netdev: network interface device structure
5807  */
5808 static void ice_set_rx_mode(struct net_device *netdev)
5809 {
5810 	struct ice_netdev_priv *np = netdev_priv(netdev);
5811 	struct ice_vsi *vsi = np->vsi;
5812 
5813 	if (!vsi || ice_is_switchdev_running(vsi->back))
5814 		return;
5815 
5816 	/* Set the flags to synchronize filters
5817 	 * ndo_set_rx_mode may be triggered even without a change in netdev
5818 	 * flags
5819 	 */
5820 	set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state);
5821 	set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
5822 	set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags);
5823 
5824 	/* schedule our worker thread which will take care of
5825 	 * applying the new filter changes
5826 	 */
5827 	ice_service_task_schedule(vsi->back);
5828 }
5829 
5830 /**
5831  * ice_set_tx_maxrate - NDO callback to set the maximum per-queue bitrate
5832  * @netdev: network interface device structure
5833  * @queue_index: Queue ID
5834  * @maxrate: maximum bandwidth in Mbps
5835  */
5836 static int
5837 ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate)
5838 {
5839 	struct ice_netdev_priv *np = netdev_priv(netdev);
5840 	struct ice_vsi *vsi = np->vsi;
5841 	u16 q_handle;
5842 	int status;
5843 	u8 tc;
5844 
5845 	/* Validate maxrate requested is within permitted range */
5846 	if (maxrate && (maxrate > (ICE_SCHED_MAX_BW / 1000))) {
5847 		netdev_err(netdev, "Invalid max rate %d specified for the queue %d\n",
5848 			   maxrate, queue_index);
5849 		return -EINVAL;
5850 	}
5851 
5852 	q_handle = vsi->tx_rings[queue_index]->q_handle;
5853 	tc = ice_dcb_get_tc(vsi, queue_index);
5854 
5855 	vsi = ice_locate_vsi_using_queue(vsi, queue_index);
5856 	if (!vsi) {
5857 		netdev_err(netdev, "Invalid VSI for given queue %d\n",
5858 			   queue_index);
5859 		return -EINVAL;
5860 	}
5861 
5862 	/* Set BW back to default, when user set maxrate to 0 */
5863 	if (!maxrate)
5864 		status = ice_cfg_q_bw_dflt_lmt(vsi->port_info, vsi->idx, tc,
5865 					       q_handle, ICE_MAX_BW);
5866 	else
5867 		status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc,
5868 					  q_handle, ICE_MAX_BW, maxrate * 1000);
5869 	if (status)
5870 		netdev_err(netdev, "Unable to set Tx max rate, error %d\n",
5871 			   status);
5872 
5873 	return status;
5874 }
5875 
5876 /**
5877  * ice_fdb_add - add an entry to the hardware database
5878  * @ndm: the input from the stack
5879  * @tb: pointer to array of nladdr (unused)
5880  * @dev: the net device pointer
5881  * @addr: the MAC address entry being added
5882  * @vid: VLAN ID
5883  * @flags: instructions from stack about fdb operation
5884  * @extack: netlink extended ack
5885  */
5886 static int
5887 ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[],
5888 	    struct net_device *dev, const unsigned char *addr, u16 vid,
5889 	    u16 flags, struct netlink_ext_ack __always_unused *extack)
5890 {
5891 	int err;
5892 
5893 	if (vid) {
5894 		netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n");
5895 		return -EINVAL;
5896 	}
5897 	if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
5898 		netdev_err(dev, "FDB only supports static addresses\n");
5899 		return -EINVAL;
5900 	}
5901 
5902 	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
5903 		err = dev_uc_add_excl(dev, addr);
5904 	else if (is_multicast_ether_addr(addr))
5905 		err = dev_mc_add_excl(dev, addr);
5906 	else
5907 		err = -EINVAL;
5908 
5909 	/* Only return duplicate errors if NLM_F_EXCL is set */
5910 	if (err == -EEXIST && !(flags & NLM_F_EXCL))
5911 		err = 0;
5912 
5913 	return err;
5914 }
5915 
5916 /**
5917  * ice_fdb_del - delete an entry from the hardware database
5918  * @ndm: the input from the stack
5919  * @tb: pointer to array of nladdr (unused)
5920  * @dev: the net device pointer
5921  * @addr: the MAC address entry being added
5922  * @vid: VLAN ID
5923  * @extack: netlink extended ack
5924  */
5925 static int
5926 ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[],
5927 	    struct net_device *dev, const unsigned char *addr,
5928 	    __always_unused u16 vid, struct netlink_ext_ack *extack)
5929 {
5930 	int err;
5931 
5932 	if (ndm->ndm_state & NUD_PERMANENT) {
5933 		netdev_err(dev, "FDB only supports static addresses\n");
5934 		return -EINVAL;
5935 	}
5936 
5937 	if (is_unicast_ether_addr(addr))
5938 		err = dev_uc_del(dev, addr);
5939 	else if (is_multicast_ether_addr(addr))
5940 		err = dev_mc_del(dev, addr);
5941 	else
5942 		err = -EINVAL;
5943 
5944 	return err;
5945 }
5946 
5947 #define NETIF_VLAN_OFFLOAD_FEATURES	(NETIF_F_HW_VLAN_CTAG_RX | \
5948 					 NETIF_F_HW_VLAN_CTAG_TX | \
5949 					 NETIF_F_HW_VLAN_STAG_RX | \
5950 					 NETIF_F_HW_VLAN_STAG_TX)
5951 
5952 #define NETIF_VLAN_STRIPPING_FEATURES	(NETIF_F_HW_VLAN_CTAG_RX | \
5953 					 NETIF_F_HW_VLAN_STAG_RX)
5954 
5955 #define NETIF_VLAN_FILTERING_FEATURES	(NETIF_F_HW_VLAN_CTAG_FILTER | \
5956 					 NETIF_F_HW_VLAN_STAG_FILTER)
5957 
5958 /**
5959  * ice_fix_features - fix the netdev features flags based on device limitations
5960  * @netdev: ptr to the netdev that flags are being fixed on
5961  * @features: features that need to be checked and possibly fixed
5962  *
5963  * Make sure any fixups are made to features in this callback. This enables the
5964  * driver to not have to check unsupported configurations throughout the driver
5965  * because that's the responsiblity of this callback.
5966  *
5967  * Single VLAN Mode (SVM) Supported Features:
5968  *	NETIF_F_HW_VLAN_CTAG_FILTER
5969  *	NETIF_F_HW_VLAN_CTAG_RX
5970  *	NETIF_F_HW_VLAN_CTAG_TX
5971  *
5972  * Double VLAN Mode (DVM) Supported Features:
5973  *	NETIF_F_HW_VLAN_CTAG_FILTER
5974  *	NETIF_F_HW_VLAN_CTAG_RX
5975  *	NETIF_F_HW_VLAN_CTAG_TX
5976  *
5977  *	NETIF_F_HW_VLAN_STAG_FILTER
5978  *	NETIF_HW_VLAN_STAG_RX
5979  *	NETIF_HW_VLAN_STAG_TX
5980  *
5981  * Features that need fixing:
5982  *	Cannot simultaneously enable CTAG and STAG stripping and/or insertion.
5983  *	These are mutually exlusive as the VSI context cannot support multiple
5984  *	VLAN ethertypes simultaneously for stripping and/or insertion. If this
5985  *	is not done, then default to clearing the requested STAG offload
5986  *	settings.
5987  *
5988  *	All supported filtering has to be enabled or disabled together. For
5989  *	example, in DVM, CTAG and STAG filtering have to be enabled and disabled
5990  *	together. If this is not done, then default to VLAN filtering disabled.
5991  *	These are mutually exclusive as there is currently no way to
5992  *	enable/disable VLAN filtering based on VLAN ethertype when using VLAN
5993  *	prune rules.
5994  */
5995 static netdev_features_t
5996 ice_fix_features(struct net_device *netdev, netdev_features_t features)
5997 {
5998 	struct ice_netdev_priv *np = netdev_priv(netdev);
5999 	netdev_features_t req_vlan_fltr, cur_vlan_fltr;
6000 	bool cur_ctag, cur_stag, req_ctag, req_stag;
6001 
6002 	cur_vlan_fltr = netdev->features & NETIF_VLAN_FILTERING_FEATURES;
6003 	cur_ctag = cur_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER;
6004 	cur_stag = cur_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER;
6005 
6006 	req_vlan_fltr = features & NETIF_VLAN_FILTERING_FEATURES;
6007 	req_ctag = req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER;
6008 	req_stag = req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER;
6009 
6010 	if (req_vlan_fltr != cur_vlan_fltr) {
6011 		if (ice_is_dvm_ena(&np->vsi->back->hw)) {
6012 			if (req_ctag && req_stag) {
6013 				features |= NETIF_VLAN_FILTERING_FEATURES;
6014 			} else if (!req_ctag && !req_stag) {
6015 				features &= ~NETIF_VLAN_FILTERING_FEATURES;
6016 			} else if ((!cur_ctag && req_ctag && !cur_stag) ||
6017 				   (!cur_stag && req_stag && !cur_ctag)) {
6018 				features |= NETIF_VLAN_FILTERING_FEATURES;
6019 				netdev_warn(netdev,  "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been enabled for both types.\n");
6020 			} else if ((cur_ctag && !req_ctag && cur_stag) ||
6021 				   (cur_stag && !req_stag && cur_ctag)) {
6022 				features &= ~NETIF_VLAN_FILTERING_FEATURES;
6023 				netdev_warn(netdev,  "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been disabled for both types.\n");
6024 			}
6025 		} else {
6026 			if (req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER)
6027 				netdev_warn(netdev, "cannot support requested 802.1ad filtering setting in SVM mode\n");
6028 
6029 			if (req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER)
6030 				features |= NETIF_F_HW_VLAN_CTAG_FILTER;
6031 		}
6032 	}
6033 
6034 	if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
6035 	    (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))) {
6036 		netdev_warn(netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
6037 		features &= ~(NETIF_F_HW_VLAN_STAG_RX |
6038 			      NETIF_F_HW_VLAN_STAG_TX);
6039 	}
6040 
6041 	if (!(netdev->features & NETIF_F_RXFCS) &&
6042 	    (features & NETIF_F_RXFCS) &&
6043 	    (features & NETIF_VLAN_STRIPPING_FEATURES) &&
6044 	    !ice_vsi_has_non_zero_vlans(np->vsi)) {
6045 		netdev_warn(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n");
6046 		features &= ~NETIF_VLAN_STRIPPING_FEATURES;
6047 	}
6048 
6049 	return features;
6050 }
6051 
6052 /**
6053  * ice_set_vlan_offload_features - set VLAN offload features for the PF VSI
6054  * @vsi: PF's VSI
6055  * @features: features used to determine VLAN offload settings
6056  *
6057  * First, determine the vlan_ethertype based on the VLAN offload bits in
6058  * features. Then determine if stripping and insertion should be enabled or
6059  * disabled. Finally enable or disable VLAN stripping and insertion.
6060  */
6061 static int
6062 ice_set_vlan_offload_features(struct ice_vsi *vsi, netdev_features_t features)
6063 {
6064 	bool enable_stripping = true, enable_insertion = true;
6065 	struct ice_vsi_vlan_ops *vlan_ops;
6066 	int strip_err = 0, insert_err = 0;
6067 	u16 vlan_ethertype = 0;
6068 
6069 	vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
6070 
6071 	if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
6072 		vlan_ethertype = ETH_P_8021AD;
6073 	else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
6074 		vlan_ethertype = ETH_P_8021Q;
6075 
6076 	if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
6077 		enable_stripping = false;
6078 	if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
6079 		enable_insertion = false;
6080 
6081 	if (enable_stripping)
6082 		strip_err = vlan_ops->ena_stripping(vsi, vlan_ethertype);
6083 	else
6084 		strip_err = vlan_ops->dis_stripping(vsi);
6085 
6086 	if (enable_insertion)
6087 		insert_err = vlan_ops->ena_insertion(vsi, vlan_ethertype);
6088 	else
6089 		insert_err = vlan_ops->dis_insertion(vsi);
6090 
6091 	if (strip_err || insert_err)
6092 		return -EIO;
6093 
6094 	return 0;
6095 }
6096 
6097 /**
6098  * ice_set_vlan_filtering_features - set VLAN filtering features for the PF VSI
6099  * @vsi: PF's VSI
6100  * @features: features used to determine VLAN filtering settings
6101  *
6102  * Enable or disable Rx VLAN filtering based on the VLAN filtering bits in the
6103  * features.
6104  */
6105 static int
6106 ice_set_vlan_filtering_features(struct ice_vsi *vsi, netdev_features_t features)
6107 {
6108 	struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
6109 	int err = 0;
6110 
6111 	/* support Single VLAN Mode (SVM) and Double VLAN Mode (DVM) by checking
6112 	 * if either bit is set
6113 	 */
6114 	if (features &
6115 	    (NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER))
6116 		err = vlan_ops->ena_rx_filtering(vsi);
6117 	else
6118 		err = vlan_ops->dis_rx_filtering(vsi);
6119 
6120 	return err;
6121 }
6122 
6123 /**
6124  * ice_set_vlan_features - set VLAN settings based on suggested feature set
6125  * @netdev: ptr to the netdev being adjusted
6126  * @features: the feature set that the stack is suggesting
6127  *
6128  * Only update VLAN settings if the requested_vlan_features are different than
6129  * the current_vlan_features.
6130  */
6131 static int
6132 ice_set_vlan_features(struct net_device *netdev, netdev_features_t features)
6133 {
6134 	netdev_features_t current_vlan_features, requested_vlan_features;
6135 	struct ice_netdev_priv *np = netdev_priv(netdev);
6136 	struct ice_vsi *vsi = np->vsi;
6137 	int err;
6138 
6139 	current_vlan_features = netdev->features & NETIF_VLAN_OFFLOAD_FEATURES;
6140 	requested_vlan_features = features & NETIF_VLAN_OFFLOAD_FEATURES;
6141 	if (current_vlan_features ^ requested_vlan_features) {
6142 		if ((features & NETIF_F_RXFCS) &&
6143 		    (features & NETIF_VLAN_STRIPPING_FEATURES)) {
6144 			dev_err(ice_pf_to_dev(vsi->back),
6145 				"To enable VLAN stripping, you must first enable FCS/CRC stripping\n");
6146 			return -EIO;
6147 		}
6148 
6149 		err = ice_set_vlan_offload_features(vsi, features);
6150 		if (err)
6151 			return err;
6152 	}
6153 
6154 	current_vlan_features = netdev->features &
6155 		NETIF_VLAN_FILTERING_FEATURES;
6156 	requested_vlan_features = features & NETIF_VLAN_FILTERING_FEATURES;
6157 	if (current_vlan_features ^ requested_vlan_features) {
6158 		err = ice_set_vlan_filtering_features(vsi, features);
6159 		if (err)
6160 			return err;
6161 	}
6162 
6163 	return 0;
6164 }
6165 
6166 /**
6167  * ice_set_loopback - turn on/off loopback mode on underlying PF
6168  * @vsi: ptr to VSI
6169  * @ena: flag to indicate the on/off setting
6170  */
6171 static int ice_set_loopback(struct ice_vsi *vsi, bool ena)
6172 {
6173 	bool if_running = netif_running(vsi->netdev);
6174 	int ret;
6175 
6176 	if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) {
6177 		ret = ice_down(vsi);
6178 		if (ret) {
6179 			netdev_err(vsi->netdev, "Preparing device to toggle loopback failed\n");
6180 			return ret;
6181 		}
6182 	}
6183 	ret = ice_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);
6184 	if (ret)
6185 		netdev_err(vsi->netdev, "Failed to toggle loopback state\n");
6186 	if (if_running)
6187 		ret = ice_up(vsi);
6188 
6189 	return ret;
6190 }
6191 
6192 /**
6193  * ice_set_features - set the netdev feature flags
6194  * @netdev: ptr to the netdev being adjusted
6195  * @features: the feature set that the stack is suggesting
6196  */
6197 static int
6198 ice_set_features(struct net_device *netdev, netdev_features_t features)
6199 {
6200 	netdev_features_t changed = netdev->features ^ features;
6201 	struct ice_netdev_priv *np = netdev_priv(netdev);
6202 	struct ice_vsi *vsi = np->vsi;
6203 	struct ice_pf *pf = vsi->back;
6204 	int ret = 0;
6205 
6206 	/* Don't set any netdev advanced features with device in Safe Mode */
6207 	if (ice_is_safe_mode(pf)) {
6208 		dev_err(ice_pf_to_dev(pf),
6209 			"Device is in Safe Mode - not enabling advanced netdev features\n");
6210 		return ret;
6211 	}
6212 
6213 	/* Do not change setting during reset */
6214 	if (ice_is_reset_in_progress(pf->state)) {
6215 		dev_err(ice_pf_to_dev(pf),
6216 			"Device is resetting, changing advanced netdev features temporarily unavailable.\n");
6217 		return -EBUSY;
6218 	}
6219 
6220 	/* Multiple features can be changed in one call so keep features in
6221 	 * separate if/else statements to guarantee each feature is checked
6222 	 */
6223 	if (changed & NETIF_F_RXHASH)
6224 		ice_vsi_manage_rss_lut(vsi, !!(features & NETIF_F_RXHASH));
6225 
6226 	ret = ice_set_vlan_features(netdev, features);
6227 	if (ret)
6228 		return ret;
6229 
6230 	/* Turn on receive of FCS aka CRC, and after setting this
6231 	 * flag the packet data will have the 4 byte CRC appended
6232 	 */
6233 	if (changed & NETIF_F_RXFCS) {
6234 		if ((features & NETIF_F_RXFCS) &&
6235 		    (features & NETIF_VLAN_STRIPPING_FEATURES)) {
6236 			dev_err(ice_pf_to_dev(vsi->back),
6237 				"To disable FCS/CRC stripping, you must first disable VLAN stripping\n");
6238 			return -EIO;
6239 		}
6240 
6241 		ice_vsi_cfg_crc_strip(vsi, !!(features & NETIF_F_RXFCS));
6242 		ret = ice_down_up(vsi);
6243 		if (ret)
6244 			return ret;
6245 	}
6246 
6247 	if (changed & NETIF_F_NTUPLE) {
6248 		bool ena = !!(features & NETIF_F_NTUPLE);
6249 
6250 		ice_vsi_manage_fdir(vsi, ena);
6251 		ena ? ice_init_arfs(vsi) : ice_clear_arfs(vsi);
6252 	}
6253 
6254 	/* don't turn off hw_tc_offload when ADQ is already enabled */
6255 	if (!(features & NETIF_F_HW_TC) && ice_is_adq_active(pf)) {
6256 		dev_err(ice_pf_to_dev(pf), "ADQ is active, can't turn hw_tc_offload off\n");
6257 		return -EACCES;
6258 	}
6259 
6260 	if (changed & NETIF_F_HW_TC) {
6261 		bool ena = !!(features & NETIF_F_HW_TC);
6262 
6263 		ena ? set_bit(ICE_FLAG_CLS_FLOWER, pf->flags) :
6264 		      clear_bit(ICE_FLAG_CLS_FLOWER, pf->flags);
6265 	}
6266 
6267 	if (changed & NETIF_F_LOOPBACK)
6268 		ret = ice_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK));
6269 
6270 	return ret;
6271 }
6272 
6273 /**
6274  * ice_vsi_vlan_setup - Setup VLAN offload properties on a PF VSI
6275  * @vsi: VSI to setup VLAN properties for
6276  */
6277 static int ice_vsi_vlan_setup(struct ice_vsi *vsi)
6278 {
6279 	int err;
6280 
6281 	err = ice_set_vlan_offload_features(vsi, vsi->netdev->features);
6282 	if (err)
6283 		return err;
6284 
6285 	err = ice_set_vlan_filtering_features(vsi, vsi->netdev->features);
6286 	if (err)
6287 		return err;
6288 
6289 	return ice_vsi_add_vlan_zero(vsi);
6290 }
6291 
6292 /**
6293  * ice_vsi_cfg_lan - Setup the VSI lan related config
6294  * @vsi: the VSI being configured
6295  *
6296  * Return 0 on success and negative value on error
6297  */
6298 int ice_vsi_cfg_lan(struct ice_vsi *vsi)
6299 {
6300 	int err;
6301 
6302 	if (vsi->netdev && vsi->type == ICE_VSI_PF) {
6303 		ice_set_rx_mode(vsi->netdev);
6304 
6305 		err = ice_vsi_vlan_setup(vsi);
6306 		if (err)
6307 			return err;
6308 	}
6309 	ice_vsi_cfg_dcb_rings(vsi);
6310 
6311 	err = ice_vsi_cfg_lan_txqs(vsi);
6312 	if (!err && ice_is_xdp_ena_vsi(vsi))
6313 		err = ice_vsi_cfg_xdp_txqs(vsi);
6314 	if (!err)
6315 		err = ice_vsi_cfg_rxqs(vsi);
6316 
6317 	return err;
6318 }
6319 
6320 /* THEORY OF MODERATION:
6321  * The ice driver hardware works differently than the hardware that DIMLIB was
6322  * originally made for. ice hardware doesn't have packet count limits that
6323  * can trigger an interrupt, but it *does* have interrupt rate limit support,
6324  * which is hard-coded to a limit of 250,000 ints/second.
6325  * If not using dynamic moderation, the INTRL value can be modified
6326  * by ethtool rx-usecs-high.
6327  */
6328 struct ice_dim {
6329 	/* the throttle rate for interrupts, basically worst case delay before
6330 	 * an initial interrupt fires, value is stored in microseconds.
6331 	 */
6332 	u16 itr;
6333 };
6334 
6335 /* Make a different profile for Rx that doesn't allow quite so aggressive
6336  * moderation at the high end (it maxes out at 126us or about 8k interrupts a
6337  * second.
6338  */
6339 static const struct ice_dim rx_profile[] = {
6340 	{2},    /* 500,000 ints/s, capped at 250K by INTRL */
6341 	{8},    /* 125,000 ints/s */
6342 	{16},   /*  62,500 ints/s */
6343 	{62},   /*  16,129 ints/s */
6344 	{126}   /*   7,936 ints/s */
6345 };
6346 
6347 /* The transmit profile, which has the same sorts of values
6348  * as the previous struct
6349  */
6350 static const struct ice_dim tx_profile[] = {
6351 	{2},    /* 500,000 ints/s, capped at 250K by INTRL */
6352 	{8},    /* 125,000 ints/s */
6353 	{40},   /*  16,125 ints/s */
6354 	{128},  /*   7,812 ints/s */
6355 	{256}   /*   3,906 ints/s */
6356 };
6357 
6358 static void ice_tx_dim_work(struct work_struct *work)
6359 {
6360 	struct ice_ring_container *rc;
6361 	struct dim *dim;
6362 	u16 itr;
6363 
6364 	dim = container_of(work, struct dim, work);
6365 	rc = dim->priv;
6366 
6367 	WARN_ON(dim->profile_ix >= ARRAY_SIZE(tx_profile));
6368 
6369 	/* look up the values in our local table */
6370 	itr = tx_profile[dim->profile_ix].itr;
6371 
6372 	ice_trace(tx_dim_work, container_of(rc, struct ice_q_vector, tx), dim);
6373 	ice_write_itr(rc, itr);
6374 
6375 	dim->state = DIM_START_MEASURE;
6376 }
6377 
6378 static void ice_rx_dim_work(struct work_struct *work)
6379 {
6380 	struct ice_ring_container *rc;
6381 	struct dim *dim;
6382 	u16 itr;
6383 
6384 	dim = container_of(work, struct dim, work);
6385 	rc = dim->priv;
6386 
6387 	WARN_ON(dim->profile_ix >= ARRAY_SIZE(rx_profile));
6388 
6389 	/* look up the values in our local table */
6390 	itr = rx_profile[dim->profile_ix].itr;
6391 
6392 	ice_trace(rx_dim_work, container_of(rc, struct ice_q_vector, rx), dim);
6393 	ice_write_itr(rc, itr);
6394 
6395 	dim->state = DIM_START_MEASURE;
6396 }
6397 
6398 #define ICE_DIM_DEFAULT_PROFILE_IX 1
6399 
6400 /**
6401  * ice_init_moderation - set up interrupt moderation
6402  * @q_vector: the vector containing rings to be configured
6403  *
6404  * Set up interrupt moderation registers, with the intent to do the right thing
6405  * when called from reset or from probe, and whether or not dynamic moderation
6406  * is enabled or not. Take special care to write all the registers in both
6407  * dynamic moderation mode or not in order to make sure hardware is in a known
6408  * state.
6409  */
6410 static void ice_init_moderation(struct ice_q_vector *q_vector)
6411 {
6412 	struct ice_ring_container *rc;
6413 	bool tx_dynamic, rx_dynamic;
6414 
6415 	rc = &q_vector->tx;
6416 	INIT_WORK(&rc->dim.work, ice_tx_dim_work);
6417 	rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
6418 	rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX;
6419 	rc->dim.priv = rc;
6420 	tx_dynamic = ITR_IS_DYNAMIC(rc);
6421 
6422 	/* set the initial TX ITR to match the above */
6423 	ice_write_itr(rc, tx_dynamic ?
6424 		      tx_profile[rc->dim.profile_ix].itr : rc->itr_setting);
6425 
6426 	rc = &q_vector->rx;
6427 	INIT_WORK(&rc->dim.work, ice_rx_dim_work);
6428 	rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
6429 	rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX;
6430 	rc->dim.priv = rc;
6431 	rx_dynamic = ITR_IS_DYNAMIC(rc);
6432 
6433 	/* set the initial RX ITR to match the above */
6434 	ice_write_itr(rc, rx_dynamic ? rx_profile[rc->dim.profile_ix].itr :
6435 				       rc->itr_setting);
6436 
6437 	ice_set_q_vector_intrl(q_vector);
6438 }
6439 
6440 /**
6441  * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI
6442  * @vsi: the VSI being configured
6443  */
6444 static void ice_napi_enable_all(struct ice_vsi *vsi)
6445 {
6446 	int q_idx;
6447 
6448 	if (!vsi->netdev)
6449 		return;
6450 
6451 	ice_for_each_q_vector(vsi, q_idx) {
6452 		struct ice_q_vector *q_vector = vsi->q_vectors[q_idx];
6453 
6454 		ice_init_moderation(q_vector);
6455 
6456 		if (q_vector->rx.rx_ring || q_vector->tx.tx_ring)
6457 			napi_enable(&q_vector->napi);
6458 	}
6459 }
6460 
6461 /**
6462  * ice_up_complete - Finish the last steps of bringing up a connection
6463  * @vsi: The VSI being configured
6464  *
6465  * Return 0 on success and negative value on error
6466  */
6467 static int ice_up_complete(struct ice_vsi *vsi)
6468 {
6469 	struct ice_pf *pf = vsi->back;
6470 	int err;
6471 
6472 	ice_vsi_cfg_msix(vsi);
6473 
6474 	/* Enable only Rx rings, Tx rings were enabled by the FW when the
6475 	 * Tx queue group list was configured and the context bits were
6476 	 * programmed using ice_vsi_cfg_txqs
6477 	 */
6478 	err = ice_vsi_start_all_rx_rings(vsi);
6479 	if (err)
6480 		return err;
6481 
6482 	clear_bit(ICE_VSI_DOWN, vsi->state);
6483 	ice_napi_enable_all(vsi);
6484 	ice_vsi_ena_irq(vsi);
6485 
6486 	if (vsi->port_info &&
6487 	    (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) &&
6488 	    vsi->netdev && vsi->type == ICE_VSI_PF) {
6489 		ice_print_link_msg(vsi, true);
6490 		netif_tx_start_all_queues(vsi->netdev);
6491 		netif_carrier_on(vsi->netdev);
6492 		ice_ptp_link_change(pf, pf->hw.pf_id, true);
6493 	}
6494 
6495 	/* Perform an initial read of the statistics registers now to
6496 	 * set the baseline so counters are ready when interface is up
6497 	 */
6498 	ice_update_eth_stats(vsi);
6499 
6500 	if (vsi->type == ICE_VSI_PF)
6501 		ice_service_task_schedule(pf);
6502 
6503 	return 0;
6504 }
6505 
6506 /**
6507  * ice_up - Bring the connection back up after being down
6508  * @vsi: VSI being configured
6509  */
6510 int ice_up(struct ice_vsi *vsi)
6511 {
6512 	int err;
6513 
6514 	err = ice_vsi_cfg_lan(vsi);
6515 	if (!err)
6516 		err = ice_up_complete(vsi);
6517 
6518 	return err;
6519 }
6520 
6521 /**
6522  * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring
6523  * @syncp: pointer to u64_stats_sync
6524  * @stats: stats that pkts and bytes count will be taken from
6525  * @pkts: packets stats counter
6526  * @bytes: bytes stats counter
6527  *
6528  * This function fetches stats from the ring considering the atomic operations
6529  * that needs to be performed to read u64 values in 32 bit machine.
6530  */
6531 void
6532 ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp,
6533 			     struct ice_q_stats stats, u64 *pkts, u64 *bytes)
6534 {
6535 	unsigned int start;
6536 
6537 	do {
6538 		start = u64_stats_fetch_begin(syncp);
6539 		*pkts = stats.pkts;
6540 		*bytes = stats.bytes;
6541 	} while (u64_stats_fetch_retry(syncp, start));
6542 }
6543 
6544 /**
6545  * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters
6546  * @vsi: the VSI to be updated
6547  * @vsi_stats: the stats struct to be updated
6548  * @rings: rings to work on
6549  * @count: number of rings
6550  */
6551 static void
6552 ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi,
6553 			     struct rtnl_link_stats64 *vsi_stats,
6554 			     struct ice_tx_ring **rings, u16 count)
6555 {
6556 	u16 i;
6557 
6558 	for (i = 0; i < count; i++) {
6559 		struct ice_tx_ring *ring;
6560 		u64 pkts = 0, bytes = 0;
6561 
6562 		ring = READ_ONCE(rings[i]);
6563 		if (!ring || !ring->ring_stats)
6564 			continue;
6565 		ice_fetch_u64_stats_per_ring(&ring->ring_stats->syncp,
6566 					     ring->ring_stats->stats, &pkts,
6567 					     &bytes);
6568 		vsi_stats->tx_packets += pkts;
6569 		vsi_stats->tx_bytes += bytes;
6570 		vsi->tx_restart += ring->ring_stats->tx_stats.restart_q;
6571 		vsi->tx_busy += ring->ring_stats->tx_stats.tx_busy;
6572 		vsi->tx_linearize += ring->ring_stats->tx_stats.tx_linearize;
6573 	}
6574 }
6575 
6576 /**
6577  * ice_update_vsi_ring_stats - Update VSI stats counters
6578  * @vsi: the VSI to be updated
6579  */
6580 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi)
6581 {
6582 	struct rtnl_link_stats64 *net_stats, *stats_prev;
6583 	struct rtnl_link_stats64 *vsi_stats;
6584 	u64 pkts, bytes;
6585 	int i;
6586 
6587 	vsi_stats = kzalloc(sizeof(*vsi_stats), GFP_ATOMIC);
6588 	if (!vsi_stats)
6589 		return;
6590 
6591 	/* reset non-netdev (extended) stats */
6592 	vsi->tx_restart = 0;
6593 	vsi->tx_busy = 0;
6594 	vsi->tx_linearize = 0;
6595 	vsi->rx_buf_failed = 0;
6596 	vsi->rx_page_failed = 0;
6597 
6598 	rcu_read_lock();
6599 
6600 	/* update Tx rings counters */
6601 	ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->tx_rings,
6602 				     vsi->num_txq);
6603 
6604 	/* update Rx rings counters */
6605 	ice_for_each_rxq(vsi, i) {
6606 		struct ice_rx_ring *ring = READ_ONCE(vsi->rx_rings[i]);
6607 		struct ice_ring_stats *ring_stats;
6608 
6609 		ring_stats = ring->ring_stats;
6610 		ice_fetch_u64_stats_per_ring(&ring_stats->syncp,
6611 					     ring_stats->stats, &pkts,
6612 					     &bytes);
6613 		vsi_stats->rx_packets += pkts;
6614 		vsi_stats->rx_bytes += bytes;
6615 		vsi->rx_buf_failed += ring_stats->rx_stats.alloc_buf_failed;
6616 		vsi->rx_page_failed += ring_stats->rx_stats.alloc_page_failed;
6617 	}
6618 
6619 	/* update XDP Tx rings counters */
6620 	if (ice_is_xdp_ena_vsi(vsi))
6621 		ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->xdp_rings,
6622 					     vsi->num_xdp_txq);
6623 
6624 	rcu_read_unlock();
6625 
6626 	net_stats = &vsi->net_stats;
6627 	stats_prev = &vsi->net_stats_prev;
6628 
6629 	/* clear prev counters after reset */
6630 	if (vsi_stats->tx_packets < stats_prev->tx_packets ||
6631 	    vsi_stats->rx_packets < stats_prev->rx_packets) {
6632 		stats_prev->tx_packets = 0;
6633 		stats_prev->tx_bytes = 0;
6634 		stats_prev->rx_packets = 0;
6635 		stats_prev->rx_bytes = 0;
6636 	}
6637 
6638 	/* update netdev counters */
6639 	net_stats->tx_packets += vsi_stats->tx_packets - stats_prev->tx_packets;
6640 	net_stats->tx_bytes += vsi_stats->tx_bytes - stats_prev->tx_bytes;
6641 	net_stats->rx_packets += vsi_stats->rx_packets - stats_prev->rx_packets;
6642 	net_stats->rx_bytes += vsi_stats->rx_bytes - stats_prev->rx_bytes;
6643 
6644 	stats_prev->tx_packets = vsi_stats->tx_packets;
6645 	stats_prev->tx_bytes = vsi_stats->tx_bytes;
6646 	stats_prev->rx_packets = vsi_stats->rx_packets;
6647 	stats_prev->rx_bytes = vsi_stats->rx_bytes;
6648 
6649 	kfree(vsi_stats);
6650 }
6651 
6652 /**
6653  * ice_update_vsi_stats - Update VSI stats counters
6654  * @vsi: the VSI to be updated
6655  */
6656 void ice_update_vsi_stats(struct ice_vsi *vsi)
6657 {
6658 	struct rtnl_link_stats64 *cur_ns = &vsi->net_stats;
6659 	struct ice_eth_stats *cur_es = &vsi->eth_stats;
6660 	struct ice_pf *pf = vsi->back;
6661 
6662 	if (test_bit(ICE_VSI_DOWN, vsi->state) ||
6663 	    test_bit(ICE_CFG_BUSY, pf->state))
6664 		return;
6665 
6666 	/* get stats as recorded by Tx/Rx rings */
6667 	ice_update_vsi_ring_stats(vsi);
6668 
6669 	/* get VSI stats as recorded by the hardware */
6670 	ice_update_eth_stats(vsi);
6671 
6672 	cur_ns->tx_errors = cur_es->tx_errors;
6673 	cur_ns->rx_dropped = cur_es->rx_discards;
6674 	cur_ns->tx_dropped = cur_es->tx_discards;
6675 	cur_ns->multicast = cur_es->rx_multicast;
6676 
6677 	/* update some more netdev stats if this is main VSI */
6678 	if (vsi->type == ICE_VSI_PF) {
6679 		cur_ns->rx_crc_errors = pf->stats.crc_errors;
6680 		cur_ns->rx_errors = pf->stats.crc_errors +
6681 				    pf->stats.illegal_bytes +
6682 				    pf->stats.rx_len_errors +
6683 				    pf->stats.rx_undersize +
6684 				    pf->hw_csum_rx_error +
6685 				    pf->stats.rx_jabber +
6686 				    pf->stats.rx_fragments +
6687 				    pf->stats.rx_oversize;
6688 		cur_ns->rx_length_errors = pf->stats.rx_len_errors;
6689 		/* record drops from the port level */
6690 		cur_ns->rx_missed_errors = pf->stats.eth.rx_discards;
6691 	}
6692 }
6693 
6694 /**
6695  * ice_update_pf_stats - Update PF port stats counters
6696  * @pf: PF whose stats needs to be updated
6697  */
6698 void ice_update_pf_stats(struct ice_pf *pf)
6699 {
6700 	struct ice_hw_port_stats *prev_ps, *cur_ps;
6701 	struct ice_hw *hw = &pf->hw;
6702 	u16 fd_ctr_base;
6703 	u8 port;
6704 
6705 	port = hw->port_info->lport;
6706 	prev_ps = &pf->stats_prev;
6707 	cur_ps = &pf->stats;
6708 
6709 	if (ice_is_reset_in_progress(pf->state))
6710 		pf->stat_prev_loaded = false;
6711 
6712 	ice_stat_update40(hw, GLPRT_GORCL(port), pf->stat_prev_loaded,
6713 			  &prev_ps->eth.rx_bytes,
6714 			  &cur_ps->eth.rx_bytes);
6715 
6716 	ice_stat_update40(hw, GLPRT_UPRCL(port), pf->stat_prev_loaded,
6717 			  &prev_ps->eth.rx_unicast,
6718 			  &cur_ps->eth.rx_unicast);
6719 
6720 	ice_stat_update40(hw, GLPRT_MPRCL(port), pf->stat_prev_loaded,
6721 			  &prev_ps->eth.rx_multicast,
6722 			  &cur_ps->eth.rx_multicast);
6723 
6724 	ice_stat_update40(hw, GLPRT_BPRCL(port), pf->stat_prev_loaded,
6725 			  &prev_ps->eth.rx_broadcast,
6726 			  &cur_ps->eth.rx_broadcast);
6727 
6728 	ice_stat_update32(hw, PRTRPB_RDPC, pf->stat_prev_loaded,
6729 			  &prev_ps->eth.rx_discards,
6730 			  &cur_ps->eth.rx_discards);
6731 
6732 	ice_stat_update40(hw, GLPRT_GOTCL(port), pf->stat_prev_loaded,
6733 			  &prev_ps->eth.tx_bytes,
6734 			  &cur_ps->eth.tx_bytes);
6735 
6736 	ice_stat_update40(hw, GLPRT_UPTCL(port), pf->stat_prev_loaded,
6737 			  &prev_ps->eth.tx_unicast,
6738 			  &cur_ps->eth.tx_unicast);
6739 
6740 	ice_stat_update40(hw, GLPRT_MPTCL(port), pf->stat_prev_loaded,
6741 			  &prev_ps->eth.tx_multicast,
6742 			  &cur_ps->eth.tx_multicast);
6743 
6744 	ice_stat_update40(hw, GLPRT_BPTCL(port), pf->stat_prev_loaded,
6745 			  &prev_ps->eth.tx_broadcast,
6746 			  &cur_ps->eth.tx_broadcast);
6747 
6748 	ice_stat_update32(hw, GLPRT_TDOLD(port), pf->stat_prev_loaded,
6749 			  &prev_ps->tx_dropped_link_down,
6750 			  &cur_ps->tx_dropped_link_down);
6751 
6752 	ice_stat_update40(hw, GLPRT_PRC64L(port), pf->stat_prev_loaded,
6753 			  &prev_ps->rx_size_64, &cur_ps->rx_size_64);
6754 
6755 	ice_stat_update40(hw, GLPRT_PRC127L(port), pf->stat_prev_loaded,
6756 			  &prev_ps->rx_size_127, &cur_ps->rx_size_127);
6757 
6758 	ice_stat_update40(hw, GLPRT_PRC255L(port), pf->stat_prev_loaded,
6759 			  &prev_ps->rx_size_255, &cur_ps->rx_size_255);
6760 
6761 	ice_stat_update40(hw, GLPRT_PRC511L(port), pf->stat_prev_loaded,
6762 			  &prev_ps->rx_size_511, &cur_ps->rx_size_511);
6763 
6764 	ice_stat_update40(hw, GLPRT_PRC1023L(port), pf->stat_prev_loaded,
6765 			  &prev_ps->rx_size_1023, &cur_ps->rx_size_1023);
6766 
6767 	ice_stat_update40(hw, GLPRT_PRC1522L(port), pf->stat_prev_loaded,
6768 			  &prev_ps->rx_size_1522, &cur_ps->rx_size_1522);
6769 
6770 	ice_stat_update40(hw, GLPRT_PRC9522L(port), pf->stat_prev_loaded,
6771 			  &prev_ps->rx_size_big, &cur_ps->rx_size_big);
6772 
6773 	ice_stat_update40(hw, GLPRT_PTC64L(port), pf->stat_prev_loaded,
6774 			  &prev_ps->tx_size_64, &cur_ps->tx_size_64);
6775 
6776 	ice_stat_update40(hw, GLPRT_PTC127L(port), pf->stat_prev_loaded,
6777 			  &prev_ps->tx_size_127, &cur_ps->tx_size_127);
6778 
6779 	ice_stat_update40(hw, GLPRT_PTC255L(port), pf->stat_prev_loaded,
6780 			  &prev_ps->tx_size_255, &cur_ps->tx_size_255);
6781 
6782 	ice_stat_update40(hw, GLPRT_PTC511L(port), pf->stat_prev_loaded,
6783 			  &prev_ps->tx_size_511, &cur_ps->tx_size_511);
6784 
6785 	ice_stat_update40(hw, GLPRT_PTC1023L(port), pf->stat_prev_loaded,
6786 			  &prev_ps->tx_size_1023, &cur_ps->tx_size_1023);
6787 
6788 	ice_stat_update40(hw, GLPRT_PTC1522L(port), pf->stat_prev_loaded,
6789 			  &prev_ps->tx_size_1522, &cur_ps->tx_size_1522);
6790 
6791 	ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded,
6792 			  &prev_ps->tx_size_big, &cur_ps->tx_size_big);
6793 
6794 	fd_ctr_base = hw->fd_ctr_base;
6795 
6796 	ice_stat_update40(hw,
6797 			  GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)),
6798 			  pf->stat_prev_loaded, &prev_ps->fd_sb_match,
6799 			  &cur_ps->fd_sb_match);
6800 	ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded,
6801 			  &prev_ps->link_xon_rx, &cur_ps->link_xon_rx);
6802 
6803 	ice_stat_update32(hw, GLPRT_LXOFFRXC(port), pf->stat_prev_loaded,
6804 			  &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx);
6805 
6806 	ice_stat_update32(hw, GLPRT_LXONTXC(port), pf->stat_prev_loaded,
6807 			  &prev_ps->link_xon_tx, &cur_ps->link_xon_tx);
6808 
6809 	ice_stat_update32(hw, GLPRT_LXOFFTXC(port), pf->stat_prev_loaded,
6810 			  &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx);
6811 
6812 	ice_update_dcb_stats(pf);
6813 
6814 	ice_stat_update32(hw, GLPRT_CRCERRS(port), pf->stat_prev_loaded,
6815 			  &prev_ps->crc_errors, &cur_ps->crc_errors);
6816 
6817 	ice_stat_update32(hw, GLPRT_ILLERRC(port), pf->stat_prev_loaded,
6818 			  &prev_ps->illegal_bytes, &cur_ps->illegal_bytes);
6819 
6820 	ice_stat_update32(hw, GLPRT_MLFC(port), pf->stat_prev_loaded,
6821 			  &prev_ps->mac_local_faults,
6822 			  &cur_ps->mac_local_faults);
6823 
6824 	ice_stat_update32(hw, GLPRT_MRFC(port), pf->stat_prev_loaded,
6825 			  &prev_ps->mac_remote_faults,
6826 			  &cur_ps->mac_remote_faults);
6827 
6828 	ice_stat_update32(hw, GLPRT_RLEC(port), pf->stat_prev_loaded,
6829 			  &prev_ps->rx_len_errors, &cur_ps->rx_len_errors);
6830 
6831 	ice_stat_update32(hw, GLPRT_RUC(port), pf->stat_prev_loaded,
6832 			  &prev_ps->rx_undersize, &cur_ps->rx_undersize);
6833 
6834 	ice_stat_update32(hw, GLPRT_RFC(port), pf->stat_prev_loaded,
6835 			  &prev_ps->rx_fragments, &cur_ps->rx_fragments);
6836 
6837 	ice_stat_update32(hw, GLPRT_ROC(port), pf->stat_prev_loaded,
6838 			  &prev_ps->rx_oversize, &cur_ps->rx_oversize);
6839 
6840 	ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded,
6841 			  &prev_ps->rx_jabber, &cur_ps->rx_jabber);
6842 
6843 	cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0;
6844 
6845 	pf->stat_prev_loaded = true;
6846 }
6847 
6848 /**
6849  * ice_get_stats64 - get statistics for network device structure
6850  * @netdev: network interface device structure
6851  * @stats: main device statistics structure
6852  */
6853 static
6854 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
6855 {
6856 	struct ice_netdev_priv *np = netdev_priv(netdev);
6857 	struct rtnl_link_stats64 *vsi_stats;
6858 	struct ice_vsi *vsi = np->vsi;
6859 
6860 	vsi_stats = &vsi->net_stats;
6861 
6862 	if (!vsi->num_txq || !vsi->num_rxq)
6863 		return;
6864 
6865 	/* netdev packet/byte stats come from ring counter. These are obtained
6866 	 * by summing up ring counters (done by ice_update_vsi_ring_stats).
6867 	 * But, only call the update routine and read the registers if VSI is
6868 	 * not down.
6869 	 */
6870 	if (!test_bit(ICE_VSI_DOWN, vsi->state))
6871 		ice_update_vsi_ring_stats(vsi);
6872 	stats->tx_packets = vsi_stats->tx_packets;
6873 	stats->tx_bytes = vsi_stats->tx_bytes;
6874 	stats->rx_packets = vsi_stats->rx_packets;
6875 	stats->rx_bytes = vsi_stats->rx_bytes;
6876 
6877 	/* The rest of the stats can be read from the hardware but instead we
6878 	 * just return values that the watchdog task has already obtained from
6879 	 * the hardware.
6880 	 */
6881 	stats->multicast = vsi_stats->multicast;
6882 	stats->tx_errors = vsi_stats->tx_errors;
6883 	stats->tx_dropped = vsi_stats->tx_dropped;
6884 	stats->rx_errors = vsi_stats->rx_errors;
6885 	stats->rx_dropped = vsi_stats->rx_dropped;
6886 	stats->rx_crc_errors = vsi_stats->rx_crc_errors;
6887 	stats->rx_length_errors = vsi_stats->rx_length_errors;
6888 }
6889 
6890 /**
6891  * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI
6892  * @vsi: VSI having NAPI disabled
6893  */
6894 static void ice_napi_disable_all(struct ice_vsi *vsi)
6895 {
6896 	int q_idx;
6897 
6898 	if (!vsi->netdev)
6899 		return;
6900 
6901 	ice_for_each_q_vector(vsi, q_idx) {
6902 		struct ice_q_vector *q_vector = vsi->q_vectors[q_idx];
6903 
6904 		if (q_vector->rx.rx_ring || q_vector->tx.tx_ring)
6905 			napi_disable(&q_vector->napi);
6906 
6907 		cancel_work_sync(&q_vector->tx.dim.work);
6908 		cancel_work_sync(&q_vector->rx.dim.work);
6909 	}
6910 }
6911 
6912 /**
6913  * ice_down - Shutdown the connection
6914  * @vsi: The VSI being stopped
6915  *
6916  * Caller of this function is expected to set the vsi->state ICE_DOWN bit
6917  */
6918 int ice_down(struct ice_vsi *vsi)
6919 {
6920 	int i, tx_err, rx_err, vlan_err = 0;
6921 
6922 	WARN_ON(!test_bit(ICE_VSI_DOWN, vsi->state));
6923 
6924 	if (vsi->netdev && vsi->type == ICE_VSI_PF) {
6925 		vlan_err = ice_vsi_del_vlan_zero(vsi);
6926 		ice_ptp_link_change(vsi->back, vsi->back->hw.pf_id, false);
6927 		netif_carrier_off(vsi->netdev);
6928 		netif_tx_disable(vsi->netdev);
6929 	} else if (vsi->type == ICE_VSI_SWITCHDEV_CTRL) {
6930 		ice_eswitch_stop_all_tx_queues(vsi->back);
6931 	}
6932 
6933 	ice_vsi_dis_irq(vsi);
6934 
6935 	tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0);
6936 	if (tx_err)
6937 		netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n",
6938 			   vsi->vsi_num, tx_err);
6939 	if (!tx_err && ice_is_xdp_ena_vsi(vsi)) {
6940 		tx_err = ice_vsi_stop_xdp_tx_rings(vsi);
6941 		if (tx_err)
6942 			netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n",
6943 				   vsi->vsi_num, tx_err);
6944 	}
6945 
6946 	rx_err = ice_vsi_stop_all_rx_rings(vsi);
6947 	if (rx_err)
6948 		netdev_err(vsi->netdev, "Failed stop Rx rings, VSI %d error %d\n",
6949 			   vsi->vsi_num, rx_err);
6950 
6951 	ice_napi_disable_all(vsi);
6952 
6953 	ice_for_each_txq(vsi, i)
6954 		ice_clean_tx_ring(vsi->tx_rings[i]);
6955 
6956 	if (ice_is_xdp_ena_vsi(vsi))
6957 		ice_for_each_xdp_txq(vsi, i)
6958 			ice_clean_tx_ring(vsi->xdp_rings[i]);
6959 
6960 	ice_for_each_rxq(vsi, i)
6961 		ice_clean_rx_ring(vsi->rx_rings[i]);
6962 
6963 	if (tx_err || rx_err || vlan_err) {
6964 		netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n",
6965 			   vsi->vsi_num, vsi->vsw->sw_id);
6966 		return -EIO;
6967 	}
6968 
6969 	return 0;
6970 }
6971 
6972 /**
6973  * ice_down_up - shutdown the VSI connection and bring it up
6974  * @vsi: the VSI to be reconnected
6975  */
6976 int ice_down_up(struct ice_vsi *vsi)
6977 {
6978 	int ret;
6979 
6980 	/* if DOWN already set, nothing to do */
6981 	if (test_and_set_bit(ICE_VSI_DOWN, vsi->state))
6982 		return 0;
6983 
6984 	ret = ice_down(vsi);
6985 	if (ret)
6986 		return ret;
6987 
6988 	ret = ice_up(vsi);
6989 	if (ret) {
6990 		netdev_err(vsi->netdev, "reallocating resources failed during netdev features change, may need to reload driver\n");
6991 		return ret;
6992 	}
6993 
6994 	return 0;
6995 }
6996 
6997 /**
6998  * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources
6999  * @vsi: VSI having resources allocated
7000  *
7001  * Return 0 on success, negative on failure
7002  */
7003 int ice_vsi_setup_tx_rings(struct ice_vsi *vsi)
7004 {
7005 	int i, err = 0;
7006 
7007 	if (!vsi->num_txq) {
7008 		dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Tx queues\n",
7009 			vsi->vsi_num);
7010 		return -EINVAL;
7011 	}
7012 
7013 	ice_for_each_txq(vsi, i) {
7014 		struct ice_tx_ring *ring = vsi->tx_rings[i];
7015 
7016 		if (!ring)
7017 			return -EINVAL;
7018 
7019 		if (vsi->netdev)
7020 			ring->netdev = vsi->netdev;
7021 		err = ice_setup_tx_ring(ring);
7022 		if (err)
7023 			break;
7024 	}
7025 
7026 	return err;
7027 }
7028 
7029 /**
7030  * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources
7031  * @vsi: VSI having resources allocated
7032  *
7033  * Return 0 on success, negative on failure
7034  */
7035 int ice_vsi_setup_rx_rings(struct ice_vsi *vsi)
7036 {
7037 	int i, err = 0;
7038 
7039 	if (!vsi->num_rxq) {
7040 		dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Rx queues\n",
7041 			vsi->vsi_num);
7042 		return -EINVAL;
7043 	}
7044 
7045 	ice_for_each_rxq(vsi, i) {
7046 		struct ice_rx_ring *ring = vsi->rx_rings[i];
7047 
7048 		if (!ring)
7049 			return -EINVAL;
7050 
7051 		if (vsi->netdev)
7052 			ring->netdev = vsi->netdev;
7053 		err = ice_setup_rx_ring(ring);
7054 		if (err)
7055 			break;
7056 	}
7057 
7058 	return err;
7059 }
7060 
7061 /**
7062  * ice_vsi_open_ctrl - open control VSI for use
7063  * @vsi: the VSI to open
7064  *
7065  * Initialization of the Control VSI
7066  *
7067  * Returns 0 on success, negative value on error
7068  */
7069 int ice_vsi_open_ctrl(struct ice_vsi *vsi)
7070 {
7071 	char int_name[ICE_INT_NAME_STR_LEN];
7072 	struct ice_pf *pf = vsi->back;
7073 	struct device *dev;
7074 	int err;
7075 
7076 	dev = ice_pf_to_dev(pf);
7077 	/* allocate descriptors */
7078 	err = ice_vsi_setup_tx_rings(vsi);
7079 	if (err)
7080 		goto err_setup_tx;
7081 
7082 	err = ice_vsi_setup_rx_rings(vsi);
7083 	if (err)
7084 		goto err_setup_rx;
7085 
7086 	err = ice_vsi_cfg_lan(vsi);
7087 	if (err)
7088 		goto err_setup_rx;
7089 
7090 	snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl",
7091 		 dev_driver_string(dev), dev_name(dev));
7092 	err = ice_vsi_req_irq_msix(vsi, int_name);
7093 	if (err)
7094 		goto err_setup_rx;
7095 
7096 	ice_vsi_cfg_msix(vsi);
7097 
7098 	err = ice_vsi_start_all_rx_rings(vsi);
7099 	if (err)
7100 		goto err_up_complete;
7101 
7102 	clear_bit(ICE_VSI_DOWN, vsi->state);
7103 	ice_vsi_ena_irq(vsi);
7104 
7105 	return 0;
7106 
7107 err_up_complete:
7108 	ice_down(vsi);
7109 err_setup_rx:
7110 	ice_vsi_free_rx_rings(vsi);
7111 err_setup_tx:
7112 	ice_vsi_free_tx_rings(vsi);
7113 
7114 	return err;
7115 }
7116 
7117 /**
7118  * ice_vsi_open - Called when a network interface is made active
7119  * @vsi: the VSI to open
7120  *
7121  * Initialization of the VSI
7122  *
7123  * Returns 0 on success, negative value on error
7124  */
7125 int ice_vsi_open(struct ice_vsi *vsi)
7126 {
7127 	char int_name[ICE_INT_NAME_STR_LEN];
7128 	struct ice_pf *pf = vsi->back;
7129 	int err;
7130 
7131 	/* allocate descriptors */
7132 	err = ice_vsi_setup_tx_rings(vsi);
7133 	if (err)
7134 		goto err_setup_tx;
7135 
7136 	err = ice_vsi_setup_rx_rings(vsi);
7137 	if (err)
7138 		goto err_setup_rx;
7139 
7140 	err = ice_vsi_cfg_lan(vsi);
7141 	if (err)
7142 		goto err_setup_rx;
7143 
7144 	snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
7145 		 dev_driver_string(ice_pf_to_dev(pf)), vsi->netdev->name);
7146 	err = ice_vsi_req_irq_msix(vsi, int_name);
7147 	if (err)
7148 		goto err_setup_rx;
7149 
7150 	ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc);
7151 
7152 	if (vsi->type == ICE_VSI_PF) {
7153 		/* Notify the stack of the actual queue counts. */
7154 		err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq);
7155 		if (err)
7156 			goto err_set_qs;
7157 
7158 		err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq);
7159 		if (err)
7160 			goto err_set_qs;
7161 	}
7162 
7163 	err = ice_up_complete(vsi);
7164 	if (err)
7165 		goto err_up_complete;
7166 
7167 	return 0;
7168 
7169 err_up_complete:
7170 	ice_down(vsi);
7171 err_set_qs:
7172 	ice_vsi_free_irq(vsi);
7173 err_setup_rx:
7174 	ice_vsi_free_rx_rings(vsi);
7175 err_setup_tx:
7176 	ice_vsi_free_tx_rings(vsi);
7177 
7178 	return err;
7179 }
7180 
7181 /**
7182  * ice_vsi_release_all - Delete all VSIs
7183  * @pf: PF from which all VSIs are being removed
7184  */
7185 static void ice_vsi_release_all(struct ice_pf *pf)
7186 {
7187 	int err, i;
7188 
7189 	if (!pf->vsi)
7190 		return;
7191 
7192 	ice_for_each_vsi(pf, i) {
7193 		if (!pf->vsi[i])
7194 			continue;
7195 
7196 		if (pf->vsi[i]->type == ICE_VSI_CHNL)
7197 			continue;
7198 
7199 		err = ice_vsi_release(pf->vsi[i]);
7200 		if (err)
7201 			dev_dbg(ice_pf_to_dev(pf), "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n",
7202 				i, err, pf->vsi[i]->vsi_num);
7203 	}
7204 }
7205 
7206 /**
7207  * ice_vsi_rebuild_by_type - Rebuild VSI of a given type
7208  * @pf: pointer to the PF instance
7209  * @type: VSI type to rebuild
7210  *
7211  * Iterates through the pf->vsi array and rebuilds VSIs of the requested type
7212  */
7213 static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type)
7214 {
7215 	struct device *dev = ice_pf_to_dev(pf);
7216 	int i, err;
7217 
7218 	ice_for_each_vsi(pf, i) {
7219 		struct ice_vsi *vsi = pf->vsi[i];
7220 
7221 		if (!vsi || vsi->type != type)
7222 			continue;
7223 
7224 		/* rebuild the VSI */
7225 		err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT);
7226 		if (err) {
7227 			dev_err(dev, "rebuild VSI failed, err %d, VSI index %d, type %s\n",
7228 				err, vsi->idx, ice_vsi_type_str(type));
7229 			return err;
7230 		}
7231 
7232 		/* replay filters for the VSI */
7233 		err = ice_replay_vsi(&pf->hw, vsi->idx);
7234 		if (err) {
7235 			dev_err(dev, "replay VSI failed, error %d, VSI index %d, type %s\n",
7236 				err, vsi->idx, ice_vsi_type_str(type));
7237 			return err;
7238 		}
7239 
7240 		/* Re-map HW VSI number, using VSI handle that has been
7241 		 * previously validated in ice_replay_vsi() call above
7242 		 */
7243 		vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx);
7244 
7245 		/* enable the VSI */
7246 		err = ice_ena_vsi(vsi, false);
7247 		if (err) {
7248 			dev_err(dev, "enable VSI failed, err %d, VSI index %d, type %s\n",
7249 				err, vsi->idx, ice_vsi_type_str(type));
7250 			return err;
7251 		}
7252 
7253 		dev_info(dev, "VSI rebuilt. VSI index %d, type %s\n", vsi->idx,
7254 			 ice_vsi_type_str(type));
7255 	}
7256 
7257 	return 0;
7258 }
7259 
7260 /**
7261  * ice_update_pf_netdev_link - Update PF netdev link status
7262  * @pf: pointer to the PF instance
7263  */
7264 static void ice_update_pf_netdev_link(struct ice_pf *pf)
7265 {
7266 	bool link_up;
7267 	int i;
7268 
7269 	ice_for_each_vsi(pf, i) {
7270 		struct ice_vsi *vsi = pf->vsi[i];
7271 
7272 		if (!vsi || vsi->type != ICE_VSI_PF)
7273 			return;
7274 
7275 		ice_get_link_status(pf->vsi[i]->port_info, &link_up);
7276 		if (link_up) {
7277 			netif_carrier_on(pf->vsi[i]->netdev);
7278 			netif_tx_wake_all_queues(pf->vsi[i]->netdev);
7279 		} else {
7280 			netif_carrier_off(pf->vsi[i]->netdev);
7281 			netif_tx_stop_all_queues(pf->vsi[i]->netdev);
7282 		}
7283 	}
7284 }
7285 
7286 /**
7287  * ice_rebuild - rebuild after reset
7288  * @pf: PF to rebuild
7289  * @reset_type: type of reset
7290  *
7291  * Do not rebuild VF VSI in this flow because that is already handled via
7292  * ice_reset_all_vfs(). This is because requirements for resetting a VF after a
7293  * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want
7294  * to reset/rebuild all the VF VSI twice.
7295  */
7296 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type)
7297 {
7298 	struct device *dev = ice_pf_to_dev(pf);
7299 	struct ice_hw *hw = &pf->hw;
7300 	bool dvm;
7301 	int err;
7302 
7303 	if (test_bit(ICE_DOWN, pf->state))
7304 		goto clear_recovery;
7305 
7306 	dev_dbg(dev, "rebuilding PF after reset_type=%d\n", reset_type);
7307 
7308 #define ICE_EMP_RESET_SLEEP_MS 5000
7309 	if (reset_type == ICE_RESET_EMPR) {
7310 		/* If an EMP reset has occurred, any previously pending flash
7311 		 * update will have completed. We no longer know whether or
7312 		 * not the NVM update EMP reset is restricted.
7313 		 */
7314 		pf->fw_emp_reset_disabled = false;
7315 
7316 		msleep(ICE_EMP_RESET_SLEEP_MS);
7317 	}
7318 
7319 	err = ice_init_all_ctrlq(hw);
7320 	if (err) {
7321 		dev_err(dev, "control queues init failed %d\n", err);
7322 		goto err_init_ctrlq;
7323 	}
7324 
7325 	/* if DDP was previously loaded successfully */
7326 	if (!ice_is_safe_mode(pf)) {
7327 		/* reload the SW DB of filter tables */
7328 		if (reset_type == ICE_RESET_PFR)
7329 			ice_fill_blk_tbls(hw);
7330 		else
7331 			/* Reload DDP Package after CORER/GLOBR reset */
7332 			ice_load_pkg(NULL, pf);
7333 	}
7334 
7335 	err = ice_clear_pf_cfg(hw);
7336 	if (err) {
7337 		dev_err(dev, "clear PF configuration failed %d\n", err);
7338 		goto err_init_ctrlq;
7339 	}
7340 
7341 	ice_clear_pxe_mode(hw);
7342 
7343 	err = ice_init_nvm(hw);
7344 	if (err) {
7345 		dev_err(dev, "ice_init_nvm failed %d\n", err);
7346 		goto err_init_ctrlq;
7347 	}
7348 
7349 	err = ice_get_caps(hw);
7350 	if (err) {
7351 		dev_err(dev, "ice_get_caps failed %d\n", err);
7352 		goto err_init_ctrlq;
7353 	}
7354 
7355 	err = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL);
7356 	if (err) {
7357 		dev_err(dev, "set_mac_cfg failed %d\n", err);
7358 		goto err_init_ctrlq;
7359 	}
7360 
7361 	dvm = ice_is_dvm_ena(hw);
7362 
7363 	err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL);
7364 	if (err)
7365 		goto err_init_ctrlq;
7366 
7367 	err = ice_sched_init_port(hw->port_info);
7368 	if (err)
7369 		goto err_sched_init_port;
7370 
7371 	/* start misc vector */
7372 	err = ice_req_irq_msix_misc(pf);
7373 	if (err) {
7374 		dev_err(dev, "misc vector setup failed: %d\n", err);
7375 		goto err_sched_init_port;
7376 	}
7377 
7378 	if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) {
7379 		wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M);
7380 		if (!rd32(hw, PFQF_FD_SIZE)) {
7381 			u16 unused, guar, b_effort;
7382 
7383 			guar = hw->func_caps.fd_fltr_guar;
7384 			b_effort = hw->func_caps.fd_fltr_best_effort;
7385 
7386 			/* force guaranteed filter pool for PF */
7387 			ice_alloc_fd_guar_item(hw, &unused, guar);
7388 			/* force shared filter pool for PF */
7389 			ice_alloc_fd_shrd_item(hw, &unused, b_effort);
7390 		}
7391 	}
7392 
7393 	if (test_bit(ICE_FLAG_DCB_ENA, pf->flags))
7394 		ice_dcb_rebuild(pf);
7395 
7396 	/* If the PF previously had enabled PTP, PTP init needs to happen before
7397 	 * the VSI rebuild. If not, this causes the PTP link status events to
7398 	 * fail.
7399 	 */
7400 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
7401 		ice_ptp_reset(pf);
7402 
7403 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
7404 		ice_gnss_init(pf);
7405 
7406 	/* rebuild PF VSI */
7407 	err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF);
7408 	if (err) {
7409 		dev_err(dev, "PF VSI rebuild failed: %d\n", err);
7410 		goto err_vsi_rebuild;
7411 	}
7412 
7413 	err = ice_eswitch_rebuild(pf);
7414 	if (err) {
7415 		dev_err(dev, "Switchdev rebuild failed: %d\n", err);
7416 		goto err_vsi_rebuild;
7417 	}
7418 
7419 	if (reset_type == ICE_RESET_PFR) {
7420 		err = ice_rebuild_channels(pf);
7421 		if (err) {
7422 			dev_err(dev, "failed to rebuild and replay ADQ VSIs, err %d\n",
7423 				err);
7424 			goto err_vsi_rebuild;
7425 		}
7426 	}
7427 
7428 	/* If Flow Director is active */
7429 	if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) {
7430 		err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL);
7431 		if (err) {
7432 			dev_err(dev, "control VSI rebuild failed: %d\n", err);
7433 			goto err_vsi_rebuild;
7434 		}
7435 
7436 		/* replay HW Flow Director recipes */
7437 		if (hw->fdir_prof)
7438 			ice_fdir_replay_flows(hw);
7439 
7440 		/* replay Flow Director filters */
7441 		ice_fdir_replay_fltrs(pf);
7442 
7443 		ice_rebuild_arfs(pf);
7444 	}
7445 
7446 	ice_update_pf_netdev_link(pf);
7447 
7448 	/* tell the firmware we are up */
7449 	err = ice_send_version(pf);
7450 	if (err) {
7451 		dev_err(dev, "Rebuild failed due to error sending driver version: %d\n",
7452 			err);
7453 		goto err_vsi_rebuild;
7454 	}
7455 
7456 	ice_replay_post(hw);
7457 
7458 	/* if we get here, reset flow is successful */
7459 	clear_bit(ICE_RESET_FAILED, pf->state);
7460 
7461 	ice_plug_aux_dev(pf);
7462 	if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG))
7463 		ice_lag_rebuild(pf);
7464 
7465 	/* Restore timestamp mode settings after VSI rebuild */
7466 	ice_ptp_restore_timestamp_mode(pf);
7467 	return;
7468 
7469 err_vsi_rebuild:
7470 err_sched_init_port:
7471 	ice_sched_cleanup_all(hw);
7472 err_init_ctrlq:
7473 	ice_shutdown_all_ctrlq(hw);
7474 	set_bit(ICE_RESET_FAILED, pf->state);
7475 clear_recovery:
7476 	/* set this bit in PF state to control service task scheduling */
7477 	set_bit(ICE_NEEDS_RESTART, pf->state);
7478 	dev_err(dev, "Rebuild failed, unload and reload driver\n");
7479 }
7480 
7481 /**
7482  * ice_change_mtu - NDO callback to change the MTU
7483  * @netdev: network interface device structure
7484  * @new_mtu: new value for maximum frame size
7485  *
7486  * Returns 0 on success, negative on failure
7487  */
7488 static int ice_change_mtu(struct net_device *netdev, int new_mtu)
7489 {
7490 	struct ice_netdev_priv *np = netdev_priv(netdev);
7491 	struct ice_vsi *vsi = np->vsi;
7492 	struct ice_pf *pf = vsi->back;
7493 	struct bpf_prog *prog;
7494 	u8 count = 0;
7495 	int err = 0;
7496 
7497 	if (new_mtu == (int)netdev->mtu) {
7498 		netdev_warn(netdev, "MTU is already %u\n", netdev->mtu);
7499 		return 0;
7500 	}
7501 
7502 	prog = vsi->xdp_prog;
7503 	if (prog && !prog->aux->xdp_has_frags) {
7504 		int frame_size = ice_max_xdp_frame_size(vsi);
7505 
7506 		if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) {
7507 			netdev_err(netdev, "max MTU for XDP usage is %d\n",
7508 				   frame_size - ICE_ETH_PKT_HDR_PAD);
7509 			return -EINVAL;
7510 		}
7511 	} else if (test_bit(ICE_FLAG_LEGACY_RX, pf->flags)) {
7512 		if (new_mtu + ICE_ETH_PKT_HDR_PAD > ICE_MAX_FRAME_LEGACY_RX) {
7513 			netdev_err(netdev, "Too big MTU for legacy-rx; Max is %d\n",
7514 				   ICE_MAX_FRAME_LEGACY_RX - ICE_ETH_PKT_HDR_PAD);
7515 			return -EINVAL;
7516 		}
7517 	}
7518 
7519 	/* if a reset is in progress, wait for some time for it to complete */
7520 	do {
7521 		if (ice_is_reset_in_progress(pf->state)) {
7522 			count++;
7523 			usleep_range(1000, 2000);
7524 		} else {
7525 			break;
7526 		}
7527 
7528 	} while (count < 100);
7529 
7530 	if (count == 100) {
7531 		netdev_err(netdev, "can't change MTU. Device is busy\n");
7532 		return -EBUSY;
7533 	}
7534 
7535 	netdev->mtu = (unsigned int)new_mtu;
7536 	err = ice_down_up(vsi);
7537 	if (err)
7538 		return err;
7539 
7540 	netdev_dbg(netdev, "changed MTU to %d\n", new_mtu);
7541 	set_bit(ICE_FLAG_MTU_CHANGED, pf->flags);
7542 
7543 	return err;
7544 }
7545 
7546 /**
7547  * ice_eth_ioctl - Access the hwtstamp interface
7548  * @netdev: network interface device structure
7549  * @ifr: interface request data
7550  * @cmd: ioctl command
7551  */
7552 static int ice_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7553 {
7554 	struct ice_netdev_priv *np = netdev_priv(netdev);
7555 	struct ice_pf *pf = np->vsi->back;
7556 
7557 	switch (cmd) {
7558 	case SIOCGHWTSTAMP:
7559 		return ice_ptp_get_ts_config(pf, ifr);
7560 	case SIOCSHWTSTAMP:
7561 		return ice_ptp_set_ts_config(pf, ifr);
7562 	default:
7563 		return -EOPNOTSUPP;
7564 	}
7565 }
7566 
7567 /**
7568  * ice_aq_str - convert AQ err code to a string
7569  * @aq_err: the AQ error code to convert
7570  */
7571 const char *ice_aq_str(enum ice_aq_err aq_err)
7572 {
7573 	switch (aq_err) {
7574 	case ICE_AQ_RC_OK:
7575 		return "OK";
7576 	case ICE_AQ_RC_EPERM:
7577 		return "ICE_AQ_RC_EPERM";
7578 	case ICE_AQ_RC_ENOENT:
7579 		return "ICE_AQ_RC_ENOENT";
7580 	case ICE_AQ_RC_ENOMEM:
7581 		return "ICE_AQ_RC_ENOMEM";
7582 	case ICE_AQ_RC_EBUSY:
7583 		return "ICE_AQ_RC_EBUSY";
7584 	case ICE_AQ_RC_EEXIST:
7585 		return "ICE_AQ_RC_EEXIST";
7586 	case ICE_AQ_RC_EINVAL:
7587 		return "ICE_AQ_RC_EINVAL";
7588 	case ICE_AQ_RC_ENOSPC:
7589 		return "ICE_AQ_RC_ENOSPC";
7590 	case ICE_AQ_RC_ENOSYS:
7591 		return "ICE_AQ_RC_ENOSYS";
7592 	case ICE_AQ_RC_EMODE:
7593 		return "ICE_AQ_RC_EMODE";
7594 	case ICE_AQ_RC_ENOSEC:
7595 		return "ICE_AQ_RC_ENOSEC";
7596 	case ICE_AQ_RC_EBADSIG:
7597 		return "ICE_AQ_RC_EBADSIG";
7598 	case ICE_AQ_RC_ESVN:
7599 		return "ICE_AQ_RC_ESVN";
7600 	case ICE_AQ_RC_EBADMAN:
7601 		return "ICE_AQ_RC_EBADMAN";
7602 	case ICE_AQ_RC_EBADBUF:
7603 		return "ICE_AQ_RC_EBADBUF";
7604 	}
7605 
7606 	return "ICE_AQ_RC_UNKNOWN";
7607 }
7608 
7609 /**
7610  * ice_set_rss_lut - Set RSS LUT
7611  * @vsi: Pointer to VSI structure
7612  * @lut: Lookup table
7613  * @lut_size: Lookup table size
7614  *
7615  * Returns 0 on success, negative on failure
7616  */
7617 int ice_set_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size)
7618 {
7619 	struct ice_aq_get_set_rss_lut_params params = {};
7620 	struct ice_hw *hw = &vsi->back->hw;
7621 	int status;
7622 
7623 	if (!lut)
7624 		return -EINVAL;
7625 
7626 	params.vsi_handle = vsi->idx;
7627 	params.lut_size = lut_size;
7628 	params.lut_type = vsi->rss_lut_type;
7629 	params.lut = lut;
7630 
7631 	status = ice_aq_set_rss_lut(hw, &params);
7632 	if (status)
7633 		dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS lut, err %d aq_err %s\n",
7634 			status, ice_aq_str(hw->adminq.sq_last_status));
7635 
7636 	return status;
7637 }
7638 
7639 /**
7640  * ice_set_rss_key - Set RSS key
7641  * @vsi: Pointer to the VSI structure
7642  * @seed: RSS hash seed
7643  *
7644  * Returns 0 on success, negative on failure
7645  */
7646 int ice_set_rss_key(struct ice_vsi *vsi, u8 *seed)
7647 {
7648 	struct ice_hw *hw = &vsi->back->hw;
7649 	int status;
7650 
7651 	if (!seed)
7652 		return -EINVAL;
7653 
7654 	status = ice_aq_set_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed);
7655 	if (status)
7656 		dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS key, err %d aq_err %s\n",
7657 			status, ice_aq_str(hw->adminq.sq_last_status));
7658 
7659 	return status;
7660 }
7661 
7662 /**
7663  * ice_get_rss_lut - Get RSS LUT
7664  * @vsi: Pointer to VSI structure
7665  * @lut: Buffer to store the lookup table entries
7666  * @lut_size: Size of buffer to store the lookup table entries
7667  *
7668  * Returns 0 on success, negative on failure
7669  */
7670 int ice_get_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size)
7671 {
7672 	struct ice_aq_get_set_rss_lut_params params = {};
7673 	struct ice_hw *hw = &vsi->back->hw;
7674 	int status;
7675 
7676 	if (!lut)
7677 		return -EINVAL;
7678 
7679 	params.vsi_handle = vsi->idx;
7680 	params.lut_size = lut_size;
7681 	params.lut_type = vsi->rss_lut_type;
7682 	params.lut = lut;
7683 
7684 	status = ice_aq_get_rss_lut(hw, &params);
7685 	if (status)
7686 		dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS lut, err %d aq_err %s\n",
7687 			status, ice_aq_str(hw->adminq.sq_last_status));
7688 
7689 	return status;
7690 }
7691 
7692 /**
7693  * ice_get_rss_key - Get RSS key
7694  * @vsi: Pointer to VSI structure
7695  * @seed: Buffer to store the key in
7696  *
7697  * Returns 0 on success, negative on failure
7698  */
7699 int ice_get_rss_key(struct ice_vsi *vsi, u8 *seed)
7700 {
7701 	struct ice_hw *hw = &vsi->back->hw;
7702 	int status;
7703 
7704 	if (!seed)
7705 		return -EINVAL;
7706 
7707 	status = ice_aq_get_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed);
7708 	if (status)
7709 		dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS key, err %d aq_err %s\n",
7710 			status, ice_aq_str(hw->adminq.sq_last_status));
7711 
7712 	return status;
7713 }
7714 
7715 /**
7716  * ice_bridge_getlink - Get the hardware bridge mode
7717  * @skb: skb buff
7718  * @pid: process ID
7719  * @seq: RTNL message seq
7720  * @dev: the netdev being configured
7721  * @filter_mask: filter mask passed in
7722  * @nlflags: netlink flags passed in
7723  *
7724  * Return the bridge mode (VEB/VEPA)
7725  */
7726 static int
7727 ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
7728 		   struct net_device *dev, u32 filter_mask, int nlflags)
7729 {
7730 	struct ice_netdev_priv *np = netdev_priv(dev);
7731 	struct ice_vsi *vsi = np->vsi;
7732 	struct ice_pf *pf = vsi->back;
7733 	u16 bmode;
7734 
7735 	bmode = pf->first_sw->bridge_mode;
7736 
7737 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags,
7738 				       filter_mask, NULL);
7739 }
7740 
7741 /**
7742  * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA)
7743  * @vsi: Pointer to VSI structure
7744  * @bmode: Hardware bridge mode (VEB/VEPA)
7745  *
7746  * Returns 0 on success, negative on failure
7747  */
7748 static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode)
7749 {
7750 	struct ice_aqc_vsi_props *vsi_props;
7751 	struct ice_hw *hw = &vsi->back->hw;
7752 	struct ice_vsi_ctx *ctxt;
7753 	int ret;
7754 
7755 	vsi_props = &vsi->info;
7756 
7757 	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
7758 	if (!ctxt)
7759 		return -ENOMEM;
7760 
7761 	ctxt->info = vsi->info;
7762 
7763 	if (bmode == BRIDGE_MODE_VEB)
7764 		/* change from VEPA to VEB mode */
7765 		ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
7766 	else
7767 		/* change from VEB to VEPA mode */
7768 		ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
7769 	ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID);
7770 
7771 	ret = ice_update_vsi(hw, vsi->idx, ctxt, NULL);
7772 	if (ret) {
7773 		dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %d aq_err %s\n",
7774 			bmode, ret, ice_aq_str(hw->adminq.sq_last_status));
7775 		goto out;
7776 	}
7777 	/* Update sw flags for book keeping */
7778 	vsi_props->sw_flags = ctxt->info.sw_flags;
7779 
7780 out:
7781 	kfree(ctxt);
7782 	return ret;
7783 }
7784 
7785 /**
7786  * ice_bridge_setlink - Set the hardware bridge mode
7787  * @dev: the netdev being configured
7788  * @nlh: RTNL message
7789  * @flags: bridge setlink flags
7790  * @extack: netlink extended ack
7791  *
7792  * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is
7793  * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if
7794  * not already set for all VSIs connected to this switch. And also update the
7795  * unicast switch filter rules for the corresponding switch of the netdev.
7796  */
7797 static int
7798 ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
7799 		   u16 __always_unused flags,
7800 		   struct netlink_ext_ack __always_unused *extack)
7801 {
7802 	struct ice_netdev_priv *np = netdev_priv(dev);
7803 	struct ice_pf *pf = np->vsi->back;
7804 	struct nlattr *attr, *br_spec;
7805 	struct ice_hw *hw = &pf->hw;
7806 	struct ice_sw *pf_sw;
7807 	int rem, v, err = 0;
7808 
7809 	pf_sw = pf->first_sw;
7810 	/* find the attribute in the netlink message */
7811 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
7812 
7813 	nla_for_each_nested(attr, br_spec, rem) {
7814 		__u16 mode;
7815 
7816 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
7817 			continue;
7818 		mode = nla_get_u16(attr);
7819 		if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
7820 			return -EINVAL;
7821 		/* Continue  if bridge mode is not being flipped */
7822 		if (mode == pf_sw->bridge_mode)
7823 			continue;
7824 		/* Iterates through the PF VSI list and update the loopback
7825 		 * mode of the VSI
7826 		 */
7827 		ice_for_each_vsi(pf, v) {
7828 			if (!pf->vsi[v])
7829 				continue;
7830 			err = ice_vsi_update_bridge_mode(pf->vsi[v], mode);
7831 			if (err)
7832 				return err;
7833 		}
7834 
7835 		hw->evb_veb = (mode == BRIDGE_MODE_VEB);
7836 		/* Update the unicast switch filter rules for the corresponding
7837 		 * switch of the netdev
7838 		 */
7839 		err = ice_update_sw_rule_bridge_mode(hw);
7840 		if (err) {
7841 			netdev_err(dev, "switch rule update failed, mode = %d err %d aq_err %s\n",
7842 				   mode, err,
7843 				   ice_aq_str(hw->adminq.sq_last_status));
7844 			/* revert hw->evb_veb */
7845 			hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB);
7846 			return err;
7847 		}
7848 
7849 		pf_sw->bridge_mode = mode;
7850 	}
7851 
7852 	return 0;
7853 }
7854 
7855 /**
7856  * ice_tx_timeout - Respond to a Tx Hang
7857  * @netdev: network interface device structure
7858  * @txqueue: Tx queue
7859  */
7860 static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue)
7861 {
7862 	struct ice_netdev_priv *np = netdev_priv(netdev);
7863 	struct ice_tx_ring *tx_ring = NULL;
7864 	struct ice_vsi *vsi = np->vsi;
7865 	struct ice_pf *pf = vsi->back;
7866 	u32 i;
7867 
7868 	pf->tx_timeout_count++;
7869 
7870 	/* Check if PFC is enabled for the TC to which the queue belongs
7871 	 * to. If yes then Tx timeout is not caused by a hung queue, no
7872 	 * need to reset and rebuild
7873 	 */
7874 	if (ice_is_pfc_causing_hung_q(pf, txqueue)) {
7875 		dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n",
7876 			 txqueue);
7877 		return;
7878 	}
7879 
7880 	/* now that we have an index, find the tx_ring struct */
7881 	ice_for_each_txq(vsi, i)
7882 		if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
7883 			if (txqueue == vsi->tx_rings[i]->q_index) {
7884 				tx_ring = vsi->tx_rings[i];
7885 				break;
7886 			}
7887 
7888 	/* Reset recovery level if enough time has elapsed after last timeout.
7889 	 * Also ensure no new reset action happens before next timeout period.
7890 	 */
7891 	if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20)))
7892 		pf->tx_timeout_recovery_level = 1;
7893 	else if (time_before(jiffies, (pf->tx_timeout_last_recovery +
7894 				       netdev->watchdog_timeo)))
7895 		return;
7896 
7897 	if (tx_ring) {
7898 		struct ice_hw *hw = &pf->hw;
7899 		u32 head, val = 0;
7900 
7901 		head = (rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue])) &
7902 			QTX_COMM_HEAD_HEAD_M) >> QTX_COMM_HEAD_HEAD_S;
7903 		/* Read interrupt register */
7904 		val = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx));
7905 
7906 		netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n",
7907 			    vsi->vsi_num, txqueue, tx_ring->next_to_clean,
7908 			    head, tx_ring->next_to_use, val);
7909 	}
7910 
7911 	pf->tx_timeout_last_recovery = jiffies;
7912 	netdev_info(netdev, "tx_timeout recovery level %d, txqueue %u\n",
7913 		    pf->tx_timeout_recovery_level, txqueue);
7914 
7915 	switch (pf->tx_timeout_recovery_level) {
7916 	case 1:
7917 		set_bit(ICE_PFR_REQ, pf->state);
7918 		break;
7919 	case 2:
7920 		set_bit(ICE_CORER_REQ, pf->state);
7921 		break;
7922 	case 3:
7923 		set_bit(ICE_GLOBR_REQ, pf->state);
7924 		break;
7925 	default:
7926 		netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n");
7927 		set_bit(ICE_DOWN, pf->state);
7928 		set_bit(ICE_VSI_NEEDS_RESTART, vsi->state);
7929 		set_bit(ICE_SERVICE_DIS, pf->state);
7930 		break;
7931 	}
7932 
7933 	ice_service_task_schedule(pf);
7934 	pf->tx_timeout_recovery_level++;
7935 }
7936 
7937 /**
7938  * ice_setup_tc_cls_flower - flower classifier offloads
7939  * @np: net device to configure
7940  * @filter_dev: device on which filter is added
7941  * @cls_flower: offload data
7942  */
7943 static int
7944 ice_setup_tc_cls_flower(struct ice_netdev_priv *np,
7945 			struct net_device *filter_dev,
7946 			struct flow_cls_offload *cls_flower)
7947 {
7948 	struct ice_vsi *vsi = np->vsi;
7949 
7950 	if (cls_flower->common.chain_index)
7951 		return -EOPNOTSUPP;
7952 
7953 	switch (cls_flower->command) {
7954 	case FLOW_CLS_REPLACE:
7955 		return ice_add_cls_flower(filter_dev, vsi, cls_flower);
7956 	case FLOW_CLS_DESTROY:
7957 		return ice_del_cls_flower(vsi, cls_flower);
7958 	default:
7959 		return -EINVAL;
7960 	}
7961 }
7962 
7963 /**
7964  * ice_setup_tc_block_cb - callback handler registered for TC block
7965  * @type: TC SETUP type
7966  * @type_data: TC flower offload data that contains user input
7967  * @cb_priv: netdev private data
7968  */
7969 static int
7970 ice_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
7971 {
7972 	struct ice_netdev_priv *np = cb_priv;
7973 
7974 	switch (type) {
7975 	case TC_SETUP_CLSFLOWER:
7976 		return ice_setup_tc_cls_flower(np, np->vsi->netdev,
7977 					       type_data);
7978 	default:
7979 		return -EOPNOTSUPP;
7980 	}
7981 }
7982 
7983 /**
7984  * ice_validate_mqprio_qopt - Validate TCF input parameters
7985  * @vsi: Pointer to VSI
7986  * @mqprio_qopt: input parameters for mqprio queue configuration
7987  *
7988  * This function validates MQPRIO params, such as qcount (power of 2 wherever
7989  * needed), and make sure user doesn't specify qcount and BW rate limit
7990  * for TCs, which are more than "num_tc"
7991  */
7992 static int
7993 ice_validate_mqprio_qopt(struct ice_vsi *vsi,
7994 			 struct tc_mqprio_qopt_offload *mqprio_qopt)
7995 {
7996 	int non_power_of_2_qcount = 0;
7997 	struct ice_pf *pf = vsi->back;
7998 	int max_rss_q_cnt = 0;
7999 	u64 sum_min_rate = 0;
8000 	struct device *dev;
8001 	int i, speed;
8002 	u8 num_tc;
8003 
8004 	if (vsi->type != ICE_VSI_PF)
8005 		return -EINVAL;
8006 
8007 	if (mqprio_qopt->qopt.offset[0] != 0 ||
8008 	    mqprio_qopt->qopt.num_tc < 1 ||
8009 	    mqprio_qopt->qopt.num_tc > ICE_CHNL_MAX_TC)
8010 		return -EINVAL;
8011 
8012 	dev = ice_pf_to_dev(pf);
8013 	vsi->ch_rss_size = 0;
8014 	num_tc = mqprio_qopt->qopt.num_tc;
8015 	speed = ice_get_link_speed_kbps(vsi);
8016 
8017 	for (i = 0; num_tc; i++) {
8018 		int qcount = mqprio_qopt->qopt.count[i];
8019 		u64 max_rate, min_rate, rem;
8020 
8021 		if (!qcount)
8022 			return -EINVAL;
8023 
8024 		if (is_power_of_2(qcount)) {
8025 			if (non_power_of_2_qcount &&
8026 			    qcount > non_power_of_2_qcount) {
8027 				dev_err(dev, "qcount[%d] cannot be greater than non power of 2 qcount[%d]\n",
8028 					qcount, non_power_of_2_qcount);
8029 				return -EINVAL;
8030 			}
8031 			if (qcount > max_rss_q_cnt)
8032 				max_rss_q_cnt = qcount;
8033 		} else {
8034 			if (non_power_of_2_qcount &&
8035 			    qcount != non_power_of_2_qcount) {
8036 				dev_err(dev, "Only one non power of 2 qcount allowed[%d,%d]\n",
8037 					qcount, non_power_of_2_qcount);
8038 				return -EINVAL;
8039 			}
8040 			if (qcount < max_rss_q_cnt) {
8041 				dev_err(dev, "non power of 2 qcount[%d] cannot be less than other qcount[%d]\n",
8042 					qcount, max_rss_q_cnt);
8043 				return -EINVAL;
8044 			}
8045 			max_rss_q_cnt = qcount;
8046 			non_power_of_2_qcount = qcount;
8047 		}
8048 
8049 		/* TC command takes input in K/N/Gbps or K/M/Gbit etc but
8050 		 * converts the bandwidth rate limit into Bytes/s when
8051 		 * passing it down to the driver. So convert input bandwidth
8052 		 * from Bytes/s to Kbps
8053 		 */
8054 		max_rate = mqprio_qopt->max_rate[i];
8055 		max_rate = div_u64(max_rate, ICE_BW_KBPS_DIVISOR);
8056 
8057 		/* min_rate is minimum guaranteed rate and it can't be zero */
8058 		min_rate = mqprio_qopt->min_rate[i];
8059 		min_rate = div_u64(min_rate, ICE_BW_KBPS_DIVISOR);
8060 		sum_min_rate += min_rate;
8061 
8062 		if (min_rate && min_rate < ICE_MIN_BW_LIMIT) {
8063 			dev_err(dev, "TC%d: min_rate(%llu Kbps) < %u Kbps\n", i,
8064 				min_rate, ICE_MIN_BW_LIMIT);
8065 			return -EINVAL;
8066 		}
8067 
8068 		if (max_rate && max_rate > speed) {
8069 			dev_err(dev, "TC%d: max_rate(%llu Kbps) > link speed of %u Kbps\n",
8070 				i, max_rate, speed);
8071 			return -EINVAL;
8072 		}
8073 
8074 		iter_div_u64_rem(min_rate, ICE_MIN_BW_LIMIT, &rem);
8075 		if (rem) {
8076 			dev_err(dev, "TC%d: Min Rate not multiple of %u Kbps",
8077 				i, ICE_MIN_BW_LIMIT);
8078 			return -EINVAL;
8079 		}
8080 
8081 		iter_div_u64_rem(max_rate, ICE_MIN_BW_LIMIT, &rem);
8082 		if (rem) {
8083 			dev_err(dev, "TC%d: Max Rate not multiple of %u Kbps",
8084 				i, ICE_MIN_BW_LIMIT);
8085 			return -EINVAL;
8086 		}
8087 
8088 		/* min_rate can't be more than max_rate, except when max_rate
8089 		 * is zero (implies max_rate sought is max line rate). In such
8090 		 * a case min_rate can be more than max.
8091 		 */
8092 		if (max_rate && min_rate > max_rate) {
8093 			dev_err(dev, "min_rate %llu Kbps can't be more than max_rate %llu Kbps\n",
8094 				min_rate, max_rate);
8095 			return -EINVAL;
8096 		}
8097 
8098 		if (i >= mqprio_qopt->qopt.num_tc - 1)
8099 			break;
8100 		if (mqprio_qopt->qopt.offset[i + 1] !=
8101 		    (mqprio_qopt->qopt.offset[i] + qcount))
8102 			return -EINVAL;
8103 	}
8104 	if (vsi->num_rxq <
8105 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
8106 		return -EINVAL;
8107 	if (vsi->num_txq <
8108 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
8109 		return -EINVAL;
8110 
8111 	if (sum_min_rate && sum_min_rate > (u64)speed) {
8112 		dev_err(dev, "Invalid min Tx rate(%llu) Kbps > speed (%u) Kbps specified\n",
8113 			sum_min_rate, speed);
8114 		return -EINVAL;
8115 	}
8116 
8117 	/* make sure vsi->ch_rss_size is set correctly based on TC's qcount */
8118 	vsi->ch_rss_size = max_rss_q_cnt;
8119 
8120 	return 0;
8121 }
8122 
8123 /**
8124  * ice_add_vsi_to_fdir - add a VSI to the flow director group for PF
8125  * @pf: ptr to PF device
8126  * @vsi: ptr to VSI
8127  */
8128 static int ice_add_vsi_to_fdir(struct ice_pf *pf, struct ice_vsi *vsi)
8129 {
8130 	struct device *dev = ice_pf_to_dev(pf);
8131 	bool added = false;
8132 	struct ice_hw *hw;
8133 	int flow;
8134 
8135 	if (!(vsi->num_gfltr || vsi->num_bfltr))
8136 		return -EINVAL;
8137 
8138 	hw = &pf->hw;
8139 	for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) {
8140 		struct ice_fd_hw_prof *prof;
8141 		int tun, status;
8142 		u64 entry_h;
8143 
8144 		if (!(hw->fdir_prof && hw->fdir_prof[flow] &&
8145 		      hw->fdir_prof[flow]->cnt))
8146 			continue;
8147 
8148 		for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
8149 			enum ice_flow_priority prio;
8150 			u64 prof_id;
8151 
8152 			/* add this VSI to FDir profile for this flow */
8153 			prio = ICE_FLOW_PRIO_NORMAL;
8154 			prof = hw->fdir_prof[flow];
8155 			prof_id = flow + tun * ICE_FLTR_PTYPE_MAX;
8156 			status = ice_flow_add_entry(hw, ICE_BLK_FD, prof_id,
8157 						    prof->vsi_h[0], vsi->idx,
8158 						    prio, prof->fdir_seg[tun],
8159 						    &entry_h);
8160 			if (status) {
8161 				dev_err(dev, "channel VSI idx %d, not able to add to group %d\n",
8162 					vsi->idx, flow);
8163 				continue;
8164 			}
8165 
8166 			prof->entry_h[prof->cnt][tun] = entry_h;
8167 		}
8168 
8169 		/* store VSI for filter replay and delete */
8170 		prof->vsi_h[prof->cnt] = vsi->idx;
8171 		prof->cnt++;
8172 
8173 		added = true;
8174 		dev_dbg(dev, "VSI idx %d added to fdir group %d\n", vsi->idx,
8175 			flow);
8176 	}
8177 
8178 	if (!added)
8179 		dev_dbg(dev, "VSI idx %d not added to fdir groups\n", vsi->idx);
8180 
8181 	return 0;
8182 }
8183 
8184 /**
8185  * ice_add_channel - add a channel by adding VSI
8186  * @pf: ptr to PF device
8187  * @sw_id: underlying HW switching element ID
8188  * @ch: ptr to channel structure
8189  *
8190  * Add a channel (VSI) using add_vsi and queue_map
8191  */
8192 static int ice_add_channel(struct ice_pf *pf, u16 sw_id, struct ice_channel *ch)
8193 {
8194 	struct device *dev = ice_pf_to_dev(pf);
8195 	struct ice_vsi *vsi;
8196 
8197 	if (ch->type != ICE_VSI_CHNL) {
8198 		dev_err(dev, "add new VSI failed, ch->type %d\n", ch->type);
8199 		return -EINVAL;
8200 	}
8201 
8202 	vsi = ice_chnl_vsi_setup(pf, pf->hw.port_info, ch);
8203 	if (!vsi || vsi->type != ICE_VSI_CHNL) {
8204 		dev_err(dev, "create chnl VSI failure\n");
8205 		return -EINVAL;
8206 	}
8207 
8208 	ice_add_vsi_to_fdir(pf, vsi);
8209 
8210 	ch->sw_id = sw_id;
8211 	ch->vsi_num = vsi->vsi_num;
8212 	ch->info.mapping_flags = vsi->info.mapping_flags;
8213 	ch->ch_vsi = vsi;
8214 	/* set the back pointer of channel for newly created VSI */
8215 	vsi->ch = ch;
8216 
8217 	memcpy(&ch->info.q_mapping, &vsi->info.q_mapping,
8218 	       sizeof(vsi->info.q_mapping));
8219 	memcpy(&ch->info.tc_mapping, vsi->info.tc_mapping,
8220 	       sizeof(vsi->info.tc_mapping));
8221 
8222 	return 0;
8223 }
8224 
8225 /**
8226  * ice_chnl_cfg_res
8227  * @vsi: the VSI being setup
8228  * @ch: ptr to channel structure
8229  *
8230  * Configure channel specific resources such as rings, vector.
8231  */
8232 static void ice_chnl_cfg_res(struct ice_vsi *vsi, struct ice_channel *ch)
8233 {
8234 	int i;
8235 
8236 	for (i = 0; i < ch->num_txq; i++) {
8237 		struct ice_q_vector *tx_q_vector, *rx_q_vector;
8238 		struct ice_ring_container *rc;
8239 		struct ice_tx_ring *tx_ring;
8240 		struct ice_rx_ring *rx_ring;
8241 
8242 		tx_ring = vsi->tx_rings[ch->base_q + i];
8243 		rx_ring = vsi->rx_rings[ch->base_q + i];
8244 		if (!tx_ring || !rx_ring)
8245 			continue;
8246 
8247 		/* setup ring being channel enabled */
8248 		tx_ring->ch = ch;
8249 		rx_ring->ch = ch;
8250 
8251 		/* following code block sets up vector specific attributes */
8252 		tx_q_vector = tx_ring->q_vector;
8253 		rx_q_vector = rx_ring->q_vector;
8254 		if (!tx_q_vector && !rx_q_vector)
8255 			continue;
8256 
8257 		if (tx_q_vector) {
8258 			tx_q_vector->ch = ch;
8259 			/* setup Tx and Rx ITR setting if DIM is off */
8260 			rc = &tx_q_vector->tx;
8261 			if (!ITR_IS_DYNAMIC(rc))
8262 				ice_write_itr(rc, rc->itr_setting);
8263 		}
8264 		if (rx_q_vector) {
8265 			rx_q_vector->ch = ch;
8266 			/* setup Tx and Rx ITR setting if DIM is off */
8267 			rc = &rx_q_vector->rx;
8268 			if (!ITR_IS_DYNAMIC(rc))
8269 				ice_write_itr(rc, rc->itr_setting);
8270 		}
8271 	}
8272 
8273 	/* it is safe to assume that, if channel has non-zero num_t[r]xq, then
8274 	 * GLINT_ITR register would have written to perform in-context
8275 	 * update, hence perform flush
8276 	 */
8277 	if (ch->num_txq || ch->num_rxq)
8278 		ice_flush(&vsi->back->hw);
8279 }
8280 
8281 /**
8282  * ice_cfg_chnl_all_res - configure channel resources
8283  * @vsi: pte to main_vsi
8284  * @ch: ptr to channel structure
8285  *
8286  * This function configures channel specific resources such as flow-director
8287  * counter index, and other resources such as queues, vectors, ITR settings
8288  */
8289 static void
8290 ice_cfg_chnl_all_res(struct ice_vsi *vsi, struct ice_channel *ch)
8291 {
8292 	/* configure channel (aka ADQ) resources such as queues, vectors,
8293 	 * ITR settings for channel specific vectors and anything else
8294 	 */
8295 	ice_chnl_cfg_res(vsi, ch);
8296 }
8297 
8298 /**
8299  * ice_setup_hw_channel - setup new channel
8300  * @pf: ptr to PF device
8301  * @vsi: the VSI being setup
8302  * @ch: ptr to channel structure
8303  * @sw_id: underlying HW switching element ID
8304  * @type: type of channel to be created (VMDq2/VF)
8305  *
8306  * Setup new channel (VSI) based on specified type (VMDq2/VF)
8307  * and configures Tx rings accordingly
8308  */
8309 static int
8310 ice_setup_hw_channel(struct ice_pf *pf, struct ice_vsi *vsi,
8311 		     struct ice_channel *ch, u16 sw_id, u8 type)
8312 {
8313 	struct device *dev = ice_pf_to_dev(pf);
8314 	int ret;
8315 
8316 	ch->base_q = vsi->next_base_q;
8317 	ch->type = type;
8318 
8319 	ret = ice_add_channel(pf, sw_id, ch);
8320 	if (ret) {
8321 		dev_err(dev, "failed to add_channel using sw_id %u\n", sw_id);
8322 		return ret;
8323 	}
8324 
8325 	/* configure/setup ADQ specific resources */
8326 	ice_cfg_chnl_all_res(vsi, ch);
8327 
8328 	/* make sure to update the next_base_q so that subsequent channel's
8329 	 * (aka ADQ) VSI queue map is correct
8330 	 */
8331 	vsi->next_base_q = vsi->next_base_q + ch->num_rxq;
8332 	dev_dbg(dev, "added channel: vsi_num %u, num_rxq %u\n", ch->vsi_num,
8333 		ch->num_rxq);
8334 
8335 	return 0;
8336 }
8337 
8338 /**
8339  * ice_setup_channel - setup new channel using uplink element
8340  * @pf: ptr to PF device
8341  * @vsi: the VSI being setup
8342  * @ch: ptr to channel structure
8343  *
8344  * Setup new channel (VSI) based on specified type (VMDq2/VF)
8345  * and uplink switching element
8346  */
8347 static bool
8348 ice_setup_channel(struct ice_pf *pf, struct ice_vsi *vsi,
8349 		  struct ice_channel *ch)
8350 {
8351 	struct device *dev = ice_pf_to_dev(pf);
8352 	u16 sw_id;
8353 	int ret;
8354 
8355 	if (vsi->type != ICE_VSI_PF) {
8356 		dev_err(dev, "unsupported parent VSI type(%d)\n", vsi->type);
8357 		return false;
8358 	}
8359 
8360 	sw_id = pf->first_sw->sw_id;
8361 
8362 	/* create channel (VSI) */
8363 	ret = ice_setup_hw_channel(pf, vsi, ch, sw_id, ICE_VSI_CHNL);
8364 	if (ret) {
8365 		dev_err(dev, "failed to setup hw_channel\n");
8366 		return false;
8367 	}
8368 	dev_dbg(dev, "successfully created channel()\n");
8369 
8370 	return ch->ch_vsi ? true : false;
8371 }
8372 
8373 /**
8374  * ice_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
8375  * @vsi: VSI to be configured
8376  * @max_tx_rate: max Tx rate in Kbps to be configured as maximum BW limit
8377  * @min_tx_rate: min Tx rate in Kbps to be configured as minimum BW limit
8378  */
8379 static int
8380 ice_set_bw_limit(struct ice_vsi *vsi, u64 max_tx_rate, u64 min_tx_rate)
8381 {
8382 	int err;
8383 
8384 	err = ice_set_min_bw_limit(vsi, min_tx_rate);
8385 	if (err)
8386 		return err;
8387 
8388 	return ice_set_max_bw_limit(vsi, max_tx_rate);
8389 }
8390 
8391 /**
8392  * ice_create_q_channel - function to create channel
8393  * @vsi: VSI to be configured
8394  * @ch: ptr to channel (it contains channel specific params)
8395  *
8396  * This function creates channel (VSI) using num_queues specified by user,
8397  * reconfigs RSS if needed.
8398  */
8399 static int ice_create_q_channel(struct ice_vsi *vsi, struct ice_channel *ch)
8400 {
8401 	struct ice_pf *pf = vsi->back;
8402 	struct device *dev;
8403 
8404 	if (!ch)
8405 		return -EINVAL;
8406 
8407 	dev = ice_pf_to_dev(pf);
8408 	if (!ch->num_txq || !ch->num_rxq) {
8409 		dev_err(dev, "Invalid num_queues requested: %d\n", ch->num_rxq);
8410 		return -EINVAL;
8411 	}
8412 
8413 	if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_txq) {
8414 		dev_err(dev, "cnt_q_avail (%u) less than num_queues %d\n",
8415 			vsi->cnt_q_avail, ch->num_txq);
8416 		return -EINVAL;
8417 	}
8418 
8419 	if (!ice_setup_channel(pf, vsi, ch)) {
8420 		dev_info(dev, "Failed to setup channel\n");
8421 		return -EINVAL;
8422 	}
8423 	/* configure BW rate limit */
8424 	if (ch->ch_vsi && (ch->max_tx_rate || ch->min_tx_rate)) {
8425 		int ret;
8426 
8427 		ret = ice_set_bw_limit(ch->ch_vsi, ch->max_tx_rate,
8428 				       ch->min_tx_rate);
8429 		if (ret)
8430 			dev_err(dev, "failed to set Tx rate of %llu Kbps for VSI(%u)\n",
8431 				ch->max_tx_rate, ch->ch_vsi->vsi_num);
8432 		else
8433 			dev_dbg(dev, "set Tx rate of %llu Kbps for VSI(%u)\n",
8434 				ch->max_tx_rate, ch->ch_vsi->vsi_num);
8435 	}
8436 
8437 	vsi->cnt_q_avail -= ch->num_txq;
8438 
8439 	return 0;
8440 }
8441 
8442 /**
8443  * ice_rem_all_chnl_fltrs - removes all channel filters
8444  * @pf: ptr to PF, TC-flower based filter are tracked at PF level
8445  *
8446  * Remove all advanced switch filters only if they are channel specific
8447  * tc-flower based filter
8448  */
8449 static void ice_rem_all_chnl_fltrs(struct ice_pf *pf)
8450 {
8451 	struct ice_tc_flower_fltr *fltr;
8452 	struct hlist_node *node;
8453 
8454 	/* to remove all channel filters, iterate an ordered list of filters */
8455 	hlist_for_each_entry_safe(fltr, node,
8456 				  &pf->tc_flower_fltr_list,
8457 				  tc_flower_node) {
8458 		struct ice_rule_query_data rule;
8459 		int status;
8460 
8461 		/* for now process only channel specific filters */
8462 		if (!ice_is_chnl_fltr(fltr))
8463 			continue;
8464 
8465 		rule.rid = fltr->rid;
8466 		rule.rule_id = fltr->rule_id;
8467 		rule.vsi_handle = fltr->dest_vsi_handle;
8468 		status = ice_rem_adv_rule_by_id(&pf->hw, &rule);
8469 		if (status) {
8470 			if (status == -ENOENT)
8471 				dev_dbg(ice_pf_to_dev(pf), "TC flower filter (rule_id %u) does not exist\n",
8472 					rule.rule_id);
8473 			else
8474 				dev_err(ice_pf_to_dev(pf), "failed to delete TC flower filter, status %d\n",
8475 					status);
8476 		} else if (fltr->dest_vsi) {
8477 			/* update advanced switch filter count */
8478 			if (fltr->dest_vsi->type == ICE_VSI_CHNL) {
8479 				u32 flags = fltr->flags;
8480 
8481 				fltr->dest_vsi->num_chnl_fltr--;
8482 				if (flags & (ICE_TC_FLWR_FIELD_DST_MAC |
8483 					     ICE_TC_FLWR_FIELD_ENC_DST_MAC))
8484 					pf->num_dmac_chnl_fltrs--;
8485 			}
8486 		}
8487 
8488 		hlist_del(&fltr->tc_flower_node);
8489 		kfree(fltr);
8490 	}
8491 }
8492 
8493 /**
8494  * ice_remove_q_channels - Remove queue channels for the TCs
8495  * @vsi: VSI to be configured
8496  * @rem_fltr: delete advanced switch filter or not
8497  *
8498  * Remove queue channels for the TCs
8499  */
8500 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_fltr)
8501 {
8502 	struct ice_channel *ch, *ch_tmp;
8503 	struct ice_pf *pf = vsi->back;
8504 	int i;
8505 
8506 	/* remove all tc-flower based filter if they are channel filters only */
8507 	if (rem_fltr)
8508 		ice_rem_all_chnl_fltrs(pf);
8509 
8510 	/* remove ntuple filters since queue configuration is being changed */
8511 	if  (vsi->netdev->features & NETIF_F_NTUPLE) {
8512 		struct ice_hw *hw = &pf->hw;
8513 
8514 		mutex_lock(&hw->fdir_fltr_lock);
8515 		ice_fdir_del_all_fltrs(vsi);
8516 		mutex_unlock(&hw->fdir_fltr_lock);
8517 	}
8518 
8519 	/* perform cleanup for channels if they exist */
8520 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
8521 		struct ice_vsi *ch_vsi;
8522 
8523 		list_del(&ch->list);
8524 		ch_vsi = ch->ch_vsi;
8525 		if (!ch_vsi) {
8526 			kfree(ch);
8527 			continue;
8528 		}
8529 
8530 		/* Reset queue contexts */
8531 		for (i = 0; i < ch->num_rxq; i++) {
8532 			struct ice_tx_ring *tx_ring;
8533 			struct ice_rx_ring *rx_ring;
8534 
8535 			tx_ring = vsi->tx_rings[ch->base_q + i];
8536 			rx_ring = vsi->rx_rings[ch->base_q + i];
8537 			if (tx_ring) {
8538 				tx_ring->ch = NULL;
8539 				if (tx_ring->q_vector)
8540 					tx_ring->q_vector->ch = NULL;
8541 			}
8542 			if (rx_ring) {
8543 				rx_ring->ch = NULL;
8544 				if (rx_ring->q_vector)
8545 					rx_ring->q_vector->ch = NULL;
8546 			}
8547 		}
8548 
8549 		/* Release FD resources for the channel VSI */
8550 		ice_fdir_rem_adq_chnl(&pf->hw, ch->ch_vsi->idx);
8551 
8552 		/* clear the VSI from scheduler tree */
8553 		ice_rm_vsi_lan_cfg(ch->ch_vsi->port_info, ch->ch_vsi->idx);
8554 
8555 		/* Delete VSI from FW, PF and HW VSI arrays */
8556 		ice_vsi_delete(ch->ch_vsi);
8557 
8558 		/* free the channel */
8559 		kfree(ch);
8560 	}
8561 
8562 	/* clear the channel VSI map which is stored in main VSI */
8563 	ice_for_each_chnl_tc(i)
8564 		vsi->tc_map_vsi[i] = NULL;
8565 
8566 	/* reset main VSI's all TC information */
8567 	vsi->all_enatc = 0;
8568 	vsi->all_numtc = 0;
8569 }
8570 
8571 /**
8572  * ice_rebuild_channels - rebuild channel
8573  * @pf: ptr to PF
8574  *
8575  * Recreate channel VSIs and replay filters
8576  */
8577 static int ice_rebuild_channels(struct ice_pf *pf)
8578 {
8579 	struct device *dev = ice_pf_to_dev(pf);
8580 	struct ice_vsi *main_vsi;
8581 	bool rem_adv_fltr = true;
8582 	struct ice_channel *ch;
8583 	struct ice_vsi *vsi;
8584 	int tc_idx = 1;
8585 	int i, err;
8586 
8587 	main_vsi = ice_get_main_vsi(pf);
8588 	if (!main_vsi)
8589 		return 0;
8590 
8591 	if (!test_bit(ICE_FLAG_TC_MQPRIO, pf->flags) ||
8592 	    main_vsi->old_numtc == 1)
8593 		return 0; /* nothing to be done */
8594 
8595 	/* reconfigure main VSI based on old value of TC and cached values
8596 	 * for MQPRIO opts
8597 	 */
8598 	err = ice_vsi_cfg_tc(main_vsi, main_vsi->old_ena_tc);
8599 	if (err) {
8600 		dev_err(dev, "failed configuring TC(ena_tc:0x%02x) for HW VSI=%u\n",
8601 			main_vsi->old_ena_tc, main_vsi->vsi_num);
8602 		return err;
8603 	}
8604 
8605 	/* rebuild ADQ VSIs */
8606 	ice_for_each_vsi(pf, i) {
8607 		enum ice_vsi_type type;
8608 
8609 		vsi = pf->vsi[i];
8610 		if (!vsi || vsi->type != ICE_VSI_CHNL)
8611 			continue;
8612 
8613 		type = vsi->type;
8614 
8615 		/* rebuild ADQ VSI */
8616 		err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT);
8617 		if (err) {
8618 			dev_err(dev, "VSI (type:%s) at index %d rebuild failed, err %d\n",
8619 				ice_vsi_type_str(type), vsi->idx, err);
8620 			goto cleanup;
8621 		}
8622 
8623 		/* Re-map HW VSI number, using VSI handle that has been
8624 		 * previously validated in ice_replay_vsi() call above
8625 		 */
8626 		vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx);
8627 
8628 		/* replay filters for the VSI */
8629 		err = ice_replay_vsi(&pf->hw, vsi->idx);
8630 		if (err) {
8631 			dev_err(dev, "VSI (type:%s) replay failed, err %d, VSI index %d\n",
8632 				ice_vsi_type_str(type), err, vsi->idx);
8633 			rem_adv_fltr = false;
8634 			goto cleanup;
8635 		}
8636 		dev_info(dev, "VSI (type:%s) at index %d rebuilt successfully\n",
8637 			 ice_vsi_type_str(type), vsi->idx);
8638 
8639 		/* store ADQ VSI at correct TC index in main VSI's
8640 		 * map of TC to VSI
8641 		 */
8642 		main_vsi->tc_map_vsi[tc_idx++] = vsi;
8643 	}
8644 
8645 	/* ADQ VSI(s) has been rebuilt successfully, so setup
8646 	 * channel for main VSI's Tx and Rx rings
8647 	 */
8648 	list_for_each_entry(ch, &main_vsi->ch_list, list) {
8649 		struct ice_vsi *ch_vsi;
8650 
8651 		ch_vsi = ch->ch_vsi;
8652 		if (!ch_vsi)
8653 			continue;
8654 
8655 		/* reconfig channel resources */
8656 		ice_cfg_chnl_all_res(main_vsi, ch);
8657 
8658 		/* replay BW rate limit if it is non-zero */
8659 		if (!ch->max_tx_rate && !ch->min_tx_rate)
8660 			continue;
8661 
8662 		err = ice_set_bw_limit(ch_vsi, ch->max_tx_rate,
8663 				       ch->min_tx_rate);
8664 		if (err)
8665 			dev_err(dev, "failed (err:%d) to rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n",
8666 				err, ch->max_tx_rate, ch->min_tx_rate,
8667 				ch_vsi->vsi_num);
8668 		else
8669 			dev_dbg(dev, "successfully rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n",
8670 				ch->max_tx_rate, ch->min_tx_rate,
8671 				ch_vsi->vsi_num);
8672 	}
8673 
8674 	/* reconfig RSS for main VSI */
8675 	if (main_vsi->ch_rss_size)
8676 		ice_vsi_cfg_rss_lut_key(main_vsi);
8677 
8678 	return 0;
8679 
8680 cleanup:
8681 	ice_remove_q_channels(main_vsi, rem_adv_fltr);
8682 	return err;
8683 }
8684 
8685 /**
8686  * ice_create_q_channels - Add queue channel for the given TCs
8687  * @vsi: VSI to be configured
8688  *
8689  * Configures queue channel mapping to the given TCs
8690  */
8691 static int ice_create_q_channels(struct ice_vsi *vsi)
8692 {
8693 	struct ice_pf *pf = vsi->back;
8694 	struct ice_channel *ch;
8695 	int ret = 0, i;
8696 
8697 	ice_for_each_chnl_tc(i) {
8698 		if (!(vsi->all_enatc & BIT(i)))
8699 			continue;
8700 
8701 		ch = kzalloc(sizeof(*ch), GFP_KERNEL);
8702 		if (!ch) {
8703 			ret = -ENOMEM;
8704 			goto err_free;
8705 		}
8706 		INIT_LIST_HEAD(&ch->list);
8707 		ch->num_rxq = vsi->mqprio_qopt.qopt.count[i];
8708 		ch->num_txq = vsi->mqprio_qopt.qopt.count[i];
8709 		ch->base_q = vsi->mqprio_qopt.qopt.offset[i];
8710 		ch->max_tx_rate = vsi->mqprio_qopt.max_rate[i];
8711 		ch->min_tx_rate = vsi->mqprio_qopt.min_rate[i];
8712 
8713 		/* convert to Kbits/s */
8714 		if (ch->max_tx_rate)
8715 			ch->max_tx_rate = div_u64(ch->max_tx_rate,
8716 						  ICE_BW_KBPS_DIVISOR);
8717 		if (ch->min_tx_rate)
8718 			ch->min_tx_rate = div_u64(ch->min_tx_rate,
8719 						  ICE_BW_KBPS_DIVISOR);
8720 
8721 		ret = ice_create_q_channel(vsi, ch);
8722 		if (ret) {
8723 			dev_err(ice_pf_to_dev(pf),
8724 				"failed creating channel TC:%d\n", i);
8725 			kfree(ch);
8726 			goto err_free;
8727 		}
8728 		list_add_tail(&ch->list, &vsi->ch_list);
8729 		vsi->tc_map_vsi[i] = ch->ch_vsi;
8730 		dev_dbg(ice_pf_to_dev(pf),
8731 			"successfully created channel: VSI %pK\n", ch->ch_vsi);
8732 	}
8733 	return 0;
8734 
8735 err_free:
8736 	ice_remove_q_channels(vsi, false);
8737 
8738 	return ret;
8739 }
8740 
8741 /**
8742  * ice_setup_tc_mqprio_qdisc - configure multiple traffic classes
8743  * @netdev: net device to configure
8744  * @type_data: TC offload data
8745  */
8746 static int ice_setup_tc_mqprio_qdisc(struct net_device *netdev, void *type_data)
8747 {
8748 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8749 	struct ice_netdev_priv *np = netdev_priv(netdev);
8750 	struct ice_vsi *vsi = np->vsi;
8751 	struct ice_pf *pf = vsi->back;
8752 	u16 mode, ena_tc_qdisc = 0;
8753 	int cur_txq, cur_rxq;
8754 	u8 hw = 0, num_tcf;
8755 	struct device *dev;
8756 	int ret, i;
8757 
8758 	dev = ice_pf_to_dev(pf);
8759 	num_tcf = mqprio_qopt->qopt.num_tc;
8760 	hw = mqprio_qopt->qopt.hw;
8761 	mode = mqprio_qopt->mode;
8762 	if (!hw) {
8763 		clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags);
8764 		vsi->ch_rss_size = 0;
8765 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8766 		goto config_tcf;
8767 	}
8768 
8769 	/* Generate queue region map for number of TCF requested */
8770 	for (i = 0; i < num_tcf; i++)
8771 		ena_tc_qdisc |= BIT(i);
8772 
8773 	switch (mode) {
8774 	case TC_MQPRIO_MODE_CHANNEL:
8775 
8776 		if (pf->hw.port_info->is_custom_tx_enabled) {
8777 			dev_err(dev, "Custom Tx scheduler feature enabled, can't configure ADQ\n");
8778 			return -EBUSY;
8779 		}
8780 		ice_tear_down_devlink_rate_tree(pf);
8781 
8782 		ret = ice_validate_mqprio_qopt(vsi, mqprio_qopt);
8783 		if (ret) {
8784 			netdev_err(netdev, "failed to validate_mqprio_qopt(), ret %d\n",
8785 				   ret);
8786 			return ret;
8787 		}
8788 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8789 		set_bit(ICE_FLAG_TC_MQPRIO, pf->flags);
8790 		/* don't assume state of hw_tc_offload during driver load
8791 		 * and set the flag for TC flower filter if hw_tc_offload
8792 		 * already ON
8793 		 */
8794 		if (vsi->netdev->features & NETIF_F_HW_TC)
8795 			set_bit(ICE_FLAG_CLS_FLOWER, pf->flags);
8796 		break;
8797 	default:
8798 		return -EINVAL;
8799 	}
8800 
8801 config_tcf:
8802 
8803 	/* Requesting same TCF configuration as already enabled */
8804 	if (ena_tc_qdisc == vsi->tc_cfg.ena_tc &&
8805 	    mode != TC_MQPRIO_MODE_CHANNEL)
8806 		return 0;
8807 
8808 	/* Pause VSI queues */
8809 	ice_dis_vsi(vsi, true);
8810 
8811 	if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags))
8812 		ice_remove_q_channels(vsi, true);
8813 
8814 	if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) {
8815 		vsi->req_txq = min_t(int, ice_get_avail_txq_count(pf),
8816 				     num_online_cpus());
8817 		vsi->req_rxq = min_t(int, ice_get_avail_rxq_count(pf),
8818 				     num_online_cpus());
8819 	} else {
8820 		/* logic to rebuild VSI, same like ethtool -L */
8821 		u16 offset = 0, qcount_tx = 0, qcount_rx = 0;
8822 
8823 		for (i = 0; i < num_tcf; i++) {
8824 			if (!(ena_tc_qdisc & BIT(i)))
8825 				continue;
8826 
8827 			offset = vsi->mqprio_qopt.qopt.offset[i];
8828 			qcount_rx = vsi->mqprio_qopt.qopt.count[i];
8829 			qcount_tx = vsi->mqprio_qopt.qopt.count[i];
8830 		}
8831 		vsi->req_txq = offset + qcount_tx;
8832 		vsi->req_rxq = offset + qcount_rx;
8833 
8834 		/* store away original rss_size info, so that it gets reused
8835 		 * form ice_vsi_rebuild during tc-qdisc delete stage - to
8836 		 * determine, what should be the rss_sizefor main VSI
8837 		 */
8838 		vsi->orig_rss_size = vsi->rss_size;
8839 	}
8840 
8841 	/* save current values of Tx and Rx queues before calling VSI rebuild
8842 	 * for fallback option
8843 	 */
8844 	cur_txq = vsi->num_txq;
8845 	cur_rxq = vsi->num_rxq;
8846 
8847 	/* proceed with rebuild main VSI using correct number of queues */
8848 	ret = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT);
8849 	if (ret) {
8850 		/* fallback to current number of queues */
8851 		dev_info(dev, "Rebuild failed with new queues, try with current number of queues\n");
8852 		vsi->req_txq = cur_txq;
8853 		vsi->req_rxq = cur_rxq;
8854 		clear_bit(ICE_RESET_FAILED, pf->state);
8855 		if (ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT)) {
8856 			dev_err(dev, "Rebuild of main VSI failed again\n");
8857 			return ret;
8858 		}
8859 	}
8860 
8861 	vsi->all_numtc = num_tcf;
8862 	vsi->all_enatc = ena_tc_qdisc;
8863 	ret = ice_vsi_cfg_tc(vsi, ena_tc_qdisc);
8864 	if (ret) {
8865 		netdev_err(netdev, "failed configuring TC for VSI id=%d\n",
8866 			   vsi->vsi_num);
8867 		goto exit;
8868 	}
8869 
8870 	if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) {
8871 		u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
8872 		u64 min_tx_rate = vsi->mqprio_qopt.min_rate[0];
8873 
8874 		/* set TC0 rate limit if specified */
8875 		if (max_tx_rate || min_tx_rate) {
8876 			/* convert to Kbits/s */
8877 			if (max_tx_rate)
8878 				max_tx_rate = div_u64(max_tx_rate, ICE_BW_KBPS_DIVISOR);
8879 			if (min_tx_rate)
8880 				min_tx_rate = div_u64(min_tx_rate, ICE_BW_KBPS_DIVISOR);
8881 
8882 			ret = ice_set_bw_limit(vsi, max_tx_rate, min_tx_rate);
8883 			if (!ret) {
8884 				dev_dbg(dev, "set Tx rate max %llu min %llu for VSI(%u)\n",
8885 					max_tx_rate, min_tx_rate, vsi->vsi_num);
8886 			} else {
8887 				dev_err(dev, "failed to set Tx rate max %llu min %llu for VSI(%u)\n",
8888 					max_tx_rate, min_tx_rate, vsi->vsi_num);
8889 				goto exit;
8890 			}
8891 		}
8892 		ret = ice_create_q_channels(vsi);
8893 		if (ret) {
8894 			netdev_err(netdev, "failed configuring queue channels\n");
8895 			goto exit;
8896 		} else {
8897 			netdev_dbg(netdev, "successfully configured channels\n");
8898 		}
8899 	}
8900 
8901 	if (vsi->ch_rss_size)
8902 		ice_vsi_cfg_rss_lut_key(vsi);
8903 
8904 exit:
8905 	/* if error, reset the all_numtc and all_enatc */
8906 	if (ret) {
8907 		vsi->all_numtc = 0;
8908 		vsi->all_enatc = 0;
8909 	}
8910 	/* resume VSI */
8911 	ice_ena_vsi(vsi, true);
8912 
8913 	return ret;
8914 }
8915 
8916 static LIST_HEAD(ice_block_cb_list);
8917 
8918 static int
8919 ice_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8920 	     void *type_data)
8921 {
8922 	struct ice_netdev_priv *np = netdev_priv(netdev);
8923 	struct ice_pf *pf = np->vsi->back;
8924 	bool locked = false;
8925 	int err;
8926 
8927 	switch (type) {
8928 	case TC_SETUP_BLOCK:
8929 		return flow_block_cb_setup_simple(type_data,
8930 						  &ice_block_cb_list,
8931 						  ice_setup_tc_block_cb,
8932 						  np, np, true);
8933 	case TC_SETUP_QDISC_MQPRIO:
8934 		if (ice_is_eswitch_mode_switchdev(pf)) {
8935 			netdev_err(netdev, "TC MQPRIO offload not supported, switchdev is enabled\n");
8936 			return -EOPNOTSUPP;
8937 		}
8938 
8939 		if (pf->adev) {
8940 			mutex_lock(&pf->adev_mutex);
8941 			device_lock(&pf->adev->dev);
8942 			locked = true;
8943 			if (pf->adev->dev.driver) {
8944 				netdev_err(netdev, "Cannot change qdisc when RDMA is active\n");
8945 				err = -EBUSY;
8946 				goto adev_unlock;
8947 			}
8948 		}
8949 
8950 		/* setup traffic classifier for receive side */
8951 		mutex_lock(&pf->tc_mutex);
8952 		err = ice_setup_tc_mqprio_qdisc(netdev, type_data);
8953 		mutex_unlock(&pf->tc_mutex);
8954 
8955 adev_unlock:
8956 		if (locked) {
8957 			device_unlock(&pf->adev->dev);
8958 			mutex_unlock(&pf->adev_mutex);
8959 		}
8960 		return err;
8961 	default:
8962 		return -EOPNOTSUPP;
8963 	}
8964 	return -EOPNOTSUPP;
8965 }
8966 
8967 static struct ice_indr_block_priv *
8968 ice_indr_block_priv_lookup(struct ice_netdev_priv *np,
8969 			   struct net_device *netdev)
8970 {
8971 	struct ice_indr_block_priv *cb_priv;
8972 
8973 	list_for_each_entry(cb_priv, &np->tc_indr_block_priv_list, list) {
8974 		if (!cb_priv->netdev)
8975 			return NULL;
8976 		if (cb_priv->netdev == netdev)
8977 			return cb_priv;
8978 	}
8979 	return NULL;
8980 }
8981 
8982 static int
8983 ice_indr_setup_block_cb(enum tc_setup_type type, void *type_data,
8984 			void *indr_priv)
8985 {
8986 	struct ice_indr_block_priv *priv = indr_priv;
8987 	struct ice_netdev_priv *np = priv->np;
8988 
8989 	switch (type) {
8990 	case TC_SETUP_CLSFLOWER:
8991 		return ice_setup_tc_cls_flower(np, priv->netdev,
8992 					       (struct flow_cls_offload *)
8993 					       type_data);
8994 	default:
8995 		return -EOPNOTSUPP;
8996 	}
8997 }
8998 
8999 static int
9000 ice_indr_setup_tc_block(struct net_device *netdev, struct Qdisc *sch,
9001 			struct ice_netdev_priv *np,
9002 			struct flow_block_offload *f, void *data,
9003 			void (*cleanup)(struct flow_block_cb *block_cb))
9004 {
9005 	struct ice_indr_block_priv *indr_priv;
9006 	struct flow_block_cb *block_cb;
9007 
9008 	if (!ice_is_tunnel_supported(netdev) &&
9009 	    !(is_vlan_dev(netdev) &&
9010 	      vlan_dev_real_dev(netdev) == np->vsi->netdev))
9011 		return -EOPNOTSUPP;
9012 
9013 	if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
9014 		return -EOPNOTSUPP;
9015 
9016 	switch (f->command) {
9017 	case FLOW_BLOCK_BIND:
9018 		indr_priv = ice_indr_block_priv_lookup(np, netdev);
9019 		if (indr_priv)
9020 			return -EEXIST;
9021 
9022 		indr_priv = kzalloc(sizeof(*indr_priv), GFP_KERNEL);
9023 		if (!indr_priv)
9024 			return -ENOMEM;
9025 
9026 		indr_priv->netdev = netdev;
9027 		indr_priv->np = np;
9028 		list_add(&indr_priv->list, &np->tc_indr_block_priv_list);
9029 
9030 		block_cb =
9031 			flow_indr_block_cb_alloc(ice_indr_setup_block_cb,
9032 						 indr_priv, indr_priv,
9033 						 ice_rep_indr_tc_block_unbind,
9034 						 f, netdev, sch, data, np,
9035 						 cleanup);
9036 
9037 		if (IS_ERR(block_cb)) {
9038 			list_del(&indr_priv->list);
9039 			kfree(indr_priv);
9040 			return PTR_ERR(block_cb);
9041 		}
9042 		flow_block_cb_add(block_cb, f);
9043 		list_add_tail(&block_cb->driver_list, &ice_block_cb_list);
9044 		break;
9045 	case FLOW_BLOCK_UNBIND:
9046 		indr_priv = ice_indr_block_priv_lookup(np, netdev);
9047 		if (!indr_priv)
9048 			return -ENOENT;
9049 
9050 		block_cb = flow_block_cb_lookup(f->block,
9051 						ice_indr_setup_block_cb,
9052 						indr_priv);
9053 		if (!block_cb)
9054 			return -ENOENT;
9055 
9056 		flow_indr_block_cb_remove(block_cb, f);
9057 
9058 		list_del(&block_cb->driver_list);
9059 		break;
9060 	default:
9061 		return -EOPNOTSUPP;
9062 	}
9063 	return 0;
9064 }
9065 
9066 static int
9067 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch,
9068 		     void *cb_priv, enum tc_setup_type type, void *type_data,
9069 		     void *data,
9070 		     void (*cleanup)(struct flow_block_cb *block_cb))
9071 {
9072 	switch (type) {
9073 	case TC_SETUP_BLOCK:
9074 		return ice_indr_setup_tc_block(netdev, sch, cb_priv, type_data,
9075 					       data, cleanup);
9076 
9077 	default:
9078 		return -EOPNOTSUPP;
9079 	}
9080 }
9081 
9082 /**
9083  * ice_open - Called when a network interface becomes active
9084  * @netdev: network interface device structure
9085  *
9086  * The open entry point is called when a network interface is made
9087  * active by the system (IFF_UP). At this point all resources needed
9088  * for transmit and receive operations are allocated, the interrupt
9089  * handler is registered with the OS, the netdev watchdog is enabled,
9090  * and the stack is notified that the interface is ready.
9091  *
9092  * Returns 0 on success, negative value on failure
9093  */
9094 int ice_open(struct net_device *netdev)
9095 {
9096 	struct ice_netdev_priv *np = netdev_priv(netdev);
9097 	struct ice_pf *pf = np->vsi->back;
9098 
9099 	if (ice_is_reset_in_progress(pf->state)) {
9100 		netdev_err(netdev, "can't open net device while reset is in progress");
9101 		return -EBUSY;
9102 	}
9103 
9104 	return ice_open_internal(netdev);
9105 }
9106 
9107 /**
9108  * ice_open_internal - Called when a network interface becomes active
9109  * @netdev: network interface device structure
9110  *
9111  * Internal ice_open implementation. Should not be used directly except for ice_open and reset
9112  * handling routine
9113  *
9114  * Returns 0 on success, negative value on failure
9115  */
9116 int ice_open_internal(struct net_device *netdev)
9117 {
9118 	struct ice_netdev_priv *np = netdev_priv(netdev);
9119 	struct ice_vsi *vsi = np->vsi;
9120 	struct ice_pf *pf = vsi->back;
9121 	struct ice_port_info *pi;
9122 	int err;
9123 
9124 	if (test_bit(ICE_NEEDS_RESTART, pf->state)) {
9125 		netdev_err(netdev, "driver needs to be unloaded and reloaded\n");
9126 		return -EIO;
9127 	}
9128 
9129 	netif_carrier_off(netdev);
9130 
9131 	pi = vsi->port_info;
9132 	err = ice_update_link_info(pi);
9133 	if (err) {
9134 		netdev_err(netdev, "Failed to get link info, error %d\n", err);
9135 		return err;
9136 	}
9137 
9138 	ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err);
9139 
9140 	/* Set PHY if there is media, otherwise, turn off PHY */
9141 	if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
9142 		clear_bit(ICE_FLAG_NO_MEDIA, pf->flags);
9143 		if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) {
9144 			err = ice_init_phy_user_cfg(pi);
9145 			if (err) {
9146 				netdev_err(netdev, "Failed to initialize PHY settings, error %d\n",
9147 					   err);
9148 				return err;
9149 			}
9150 		}
9151 
9152 		err = ice_configure_phy(vsi);
9153 		if (err) {
9154 			netdev_err(netdev, "Failed to set physical link up, error %d\n",
9155 				   err);
9156 			return err;
9157 		}
9158 	} else {
9159 		set_bit(ICE_FLAG_NO_MEDIA, pf->flags);
9160 		ice_set_link(vsi, false);
9161 	}
9162 
9163 	err = ice_vsi_open(vsi);
9164 	if (err)
9165 		netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n",
9166 			   vsi->vsi_num, vsi->vsw->sw_id);
9167 
9168 	/* Update existing tunnels information */
9169 	udp_tunnel_get_rx_info(netdev);
9170 
9171 	return err;
9172 }
9173 
9174 /**
9175  * ice_stop - Disables a network interface
9176  * @netdev: network interface device structure
9177  *
9178  * The stop entry point is called when an interface is de-activated by the OS,
9179  * and the netdevice enters the DOWN state. The hardware is still under the
9180  * driver's control, but the netdev interface is disabled.
9181  *
9182  * Returns success only - not allowed to fail
9183  */
9184 int ice_stop(struct net_device *netdev)
9185 {
9186 	struct ice_netdev_priv *np = netdev_priv(netdev);
9187 	struct ice_vsi *vsi = np->vsi;
9188 	struct ice_pf *pf = vsi->back;
9189 
9190 	if (ice_is_reset_in_progress(pf->state)) {
9191 		netdev_err(netdev, "can't stop net device while reset is in progress");
9192 		return -EBUSY;
9193 	}
9194 
9195 	if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) {
9196 		int link_err = ice_force_phys_link_state(vsi, false);
9197 
9198 		if (link_err) {
9199 			netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n",
9200 				   vsi->vsi_num, link_err);
9201 			return -EIO;
9202 		}
9203 	}
9204 
9205 	ice_vsi_close(vsi);
9206 
9207 	return 0;
9208 }
9209 
9210 /**
9211  * ice_features_check - Validate encapsulated packet conforms to limits
9212  * @skb: skb buffer
9213  * @netdev: This port's netdev
9214  * @features: Offload features that the stack believes apply
9215  */
9216 static netdev_features_t
9217 ice_features_check(struct sk_buff *skb,
9218 		   struct net_device __always_unused *netdev,
9219 		   netdev_features_t features)
9220 {
9221 	bool gso = skb_is_gso(skb);
9222 	size_t len;
9223 
9224 	/* No point in doing any of this if neither checksum nor GSO are
9225 	 * being requested for this frame. We can rule out both by just
9226 	 * checking for CHECKSUM_PARTIAL
9227 	 */
9228 	if (skb->ip_summed != CHECKSUM_PARTIAL)
9229 		return features;
9230 
9231 	/* We cannot support GSO if the MSS is going to be less than
9232 	 * 64 bytes. If it is then we need to drop support for GSO.
9233 	 */
9234 	if (gso && (skb_shinfo(skb)->gso_size < ICE_TXD_CTX_MIN_MSS))
9235 		features &= ~NETIF_F_GSO_MASK;
9236 
9237 	len = skb_network_offset(skb);
9238 	if (len > ICE_TXD_MACLEN_MAX || len & 0x1)
9239 		goto out_rm_features;
9240 
9241 	len = skb_network_header_len(skb);
9242 	if (len > ICE_TXD_IPLEN_MAX || len & 0x1)
9243 		goto out_rm_features;
9244 
9245 	if (skb->encapsulation) {
9246 		/* this must work for VXLAN frames AND IPIP/SIT frames, and in
9247 		 * the case of IPIP frames, the transport header pointer is
9248 		 * after the inner header! So check to make sure that this
9249 		 * is a GRE or UDP_TUNNEL frame before doing that math.
9250 		 */
9251 		if (gso && (skb_shinfo(skb)->gso_type &
9252 			    (SKB_GSO_GRE | SKB_GSO_UDP_TUNNEL))) {
9253 			len = skb_inner_network_header(skb) -
9254 			      skb_transport_header(skb);
9255 			if (len > ICE_TXD_L4LEN_MAX || len & 0x1)
9256 				goto out_rm_features;
9257 		}
9258 
9259 		len = skb_inner_network_header_len(skb);
9260 		if (len > ICE_TXD_IPLEN_MAX || len & 0x1)
9261 			goto out_rm_features;
9262 	}
9263 
9264 	return features;
9265 out_rm_features:
9266 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9267 }
9268 
9269 static const struct net_device_ops ice_netdev_safe_mode_ops = {
9270 	.ndo_open = ice_open,
9271 	.ndo_stop = ice_stop,
9272 	.ndo_start_xmit = ice_start_xmit,
9273 	.ndo_set_mac_address = ice_set_mac_address,
9274 	.ndo_validate_addr = eth_validate_addr,
9275 	.ndo_change_mtu = ice_change_mtu,
9276 	.ndo_get_stats64 = ice_get_stats64,
9277 	.ndo_tx_timeout = ice_tx_timeout,
9278 	.ndo_bpf = ice_xdp_safe_mode,
9279 };
9280 
9281 static const struct net_device_ops ice_netdev_ops = {
9282 	.ndo_open = ice_open,
9283 	.ndo_stop = ice_stop,
9284 	.ndo_start_xmit = ice_start_xmit,
9285 	.ndo_select_queue = ice_select_queue,
9286 	.ndo_features_check = ice_features_check,
9287 	.ndo_fix_features = ice_fix_features,
9288 	.ndo_set_rx_mode = ice_set_rx_mode,
9289 	.ndo_set_mac_address = ice_set_mac_address,
9290 	.ndo_validate_addr = eth_validate_addr,
9291 	.ndo_change_mtu = ice_change_mtu,
9292 	.ndo_get_stats64 = ice_get_stats64,
9293 	.ndo_set_tx_maxrate = ice_set_tx_maxrate,
9294 	.ndo_eth_ioctl = ice_eth_ioctl,
9295 	.ndo_set_vf_spoofchk = ice_set_vf_spoofchk,
9296 	.ndo_set_vf_mac = ice_set_vf_mac,
9297 	.ndo_get_vf_config = ice_get_vf_cfg,
9298 	.ndo_set_vf_trust = ice_set_vf_trust,
9299 	.ndo_set_vf_vlan = ice_set_vf_port_vlan,
9300 	.ndo_set_vf_link_state = ice_set_vf_link_state,
9301 	.ndo_get_vf_stats = ice_get_vf_stats,
9302 	.ndo_set_vf_rate = ice_set_vf_bw,
9303 	.ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid,
9304 	.ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid,
9305 	.ndo_setup_tc = ice_setup_tc,
9306 	.ndo_set_features = ice_set_features,
9307 	.ndo_bridge_getlink = ice_bridge_getlink,
9308 	.ndo_bridge_setlink = ice_bridge_setlink,
9309 	.ndo_fdb_add = ice_fdb_add,
9310 	.ndo_fdb_del = ice_fdb_del,
9311 #ifdef CONFIG_RFS_ACCEL
9312 	.ndo_rx_flow_steer = ice_rx_flow_steer,
9313 #endif
9314 	.ndo_tx_timeout = ice_tx_timeout,
9315 	.ndo_bpf = ice_xdp,
9316 	.ndo_xdp_xmit = ice_xdp_xmit,
9317 	.ndo_xsk_wakeup = ice_xsk_wakeup,
9318 };
9319