xref: /freebsd/sys/dev/ice/if_ice_iflib.c (revision e9e8876a4d6afc1ad5315faaa191b25121a813d7)
1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /*  Copyright (c) 2021, Intel Corporation
3  *  All rights reserved.
4  *
5  *  Redistribution and use in source and binary forms, with or without
6  *  modification, are permitted provided that the following conditions are met:
7  *
8  *   1. Redistributions of source code must retain the above copyright notice,
9  *      this list of conditions and the following disclaimer.
10  *
11  *   2. Redistributions in binary form must reproduce the above copyright
12  *      notice, this list of conditions and the following disclaimer in the
13  *      documentation and/or other materials provided with the distribution.
14  *
15  *   3. Neither the name of the Intel Corporation nor the names of its
16  *      contributors may be used to endorse or promote products derived from
17  *      this software without specific prior written permission.
18  *
19  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  *  POSSIBILITY OF SUCH DAMAGE.
30  */
31 /*$FreeBSD$*/
32 
33 /**
34  * @file if_ice_iflib.c
35  * @brief iflib driver implementation
36  *
37  * Contains the main entry point for the iflib driver implementation. It
38  * implements the various ifdi driver methods, and sets up the module and
39  * driver values to load an iflib driver.
40  */
41 
42 #include "ice_iflib.h"
43 #include "ice_drv_info.h"
44 #include "ice_switch.h"
45 #include "ice_sched.h"
46 
47 #include <sys/module.h>
48 #include <sys/sockio.h>
49 #include <sys/smp.h>
50 #include <dev/pci/pcivar.h>
51 #include <dev/pci/pcireg.h>
52 
53 /*
54  * Device method prototypes
55  */
56 
57 static void *ice_register(device_t);
58 static int  ice_if_attach_pre(if_ctx_t);
59 static int  ice_attach_pre_recovery_mode(struct ice_softc *sc);
60 static int  ice_if_attach_post(if_ctx_t);
61 static void ice_attach_post_recovery_mode(struct ice_softc *sc);
62 static int  ice_if_detach(if_ctx_t);
63 static int  ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets);
64 static int  ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
65 static int ice_if_msix_intr_assign(if_ctx_t ctx, int msix);
66 static void ice_if_queues_free(if_ctx_t ctx);
67 static int ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
68 static void ice_if_intr_enable(if_ctx_t ctx);
69 static void ice_if_intr_disable(if_ctx_t ctx);
70 static int ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid);
71 static int ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid);
72 static int ice_if_promisc_set(if_ctx_t ctx, int flags);
73 static void ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr);
74 static int ice_if_media_change(if_ctx_t ctx);
75 static void ice_if_init(if_ctx_t ctx);
76 static void ice_if_timer(if_ctx_t ctx, uint16_t qid);
77 static void ice_if_update_admin_status(if_ctx_t ctx);
78 static void ice_if_multi_set(if_ctx_t ctx);
79 static void ice_if_vlan_register(if_ctx_t ctx, u16 vtag);
80 static void ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag);
81 static void ice_if_stop(if_ctx_t ctx);
82 static uint64_t ice_if_get_counter(if_ctx_t ctx, ift_counter counter);
83 static int ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data);
84 static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req);
85 static int ice_if_suspend(if_ctx_t ctx);
86 static int ice_if_resume(if_ctx_t ctx);
87 
88 static int ice_msix_que(void *arg);
89 static int ice_msix_admin(void *arg);
90 
91 /*
92  * Helper function prototypes
93  */
94 static int ice_pci_mapping(struct ice_softc *sc);
95 static void ice_free_pci_mapping(struct ice_softc *sc);
96 static void ice_update_link_status(struct ice_softc *sc, bool update_media);
97 static void ice_init_device_features(struct ice_softc *sc);
98 static void ice_init_tx_tracking(struct ice_vsi *vsi);
99 static void ice_handle_reset_event(struct ice_softc *sc);
100 static void ice_handle_pf_reset_request(struct ice_softc *sc);
101 static void ice_prepare_for_reset(struct ice_softc *sc);
102 static int ice_rebuild_pf_vsi_qmap(struct ice_softc *sc);
103 static void ice_rebuild(struct ice_softc *sc);
104 static void ice_rebuild_recovery_mode(struct ice_softc *sc);
105 static void ice_free_irqvs(struct ice_softc *sc);
106 static void ice_update_rx_mbuf_sz(struct ice_softc *sc);
107 static void ice_poll_for_media_avail(struct ice_softc *sc);
108 static void ice_setup_scctx(struct ice_softc *sc);
109 static int ice_allocate_msix(struct ice_softc *sc);
110 static void ice_admin_timer(void *arg);
111 static void ice_transition_recovery_mode(struct ice_softc *sc);
112 static void ice_transition_safe_mode(struct ice_softc *sc);
113 
114 /*
115  * Device Interface Declaration
116  */
117 
118 /**
119  * @var ice_methods
120  * @brief ice driver method entry points
121  *
122  * List of device methods implementing the generic device interface used by
123  * the device stack to interact with the ice driver. Since this is an iflib
124  * driver, most of the methods point to the generic iflib implementation.
125  */
126 static device_method_t ice_methods[] = {
127 	/* Device interface */
128 	DEVMETHOD(device_register, ice_register),
129 	DEVMETHOD(device_probe,    iflib_device_probe_vendor),
130 	DEVMETHOD(device_attach,   iflib_device_attach),
131 	DEVMETHOD(device_detach,   iflib_device_detach),
132 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
133 	DEVMETHOD(device_suspend,  iflib_device_suspend),
134 	DEVMETHOD(device_resume,   iflib_device_resume),
135 	DEVMETHOD_END
136 };
137 
138 /**
139  * @var ice_iflib_methods
140  * @brief iflib method entry points
141  *
142  * List of device methods used by the iflib stack to interact with this
143  * driver. These are the real main entry points used to interact with this
144  * driver.
145  */
146 static device_method_t ice_iflib_methods[] = {
147 	DEVMETHOD(ifdi_attach_pre, ice_if_attach_pre),
148 	DEVMETHOD(ifdi_attach_post, ice_if_attach_post),
149 	DEVMETHOD(ifdi_detach, ice_if_detach),
150 	DEVMETHOD(ifdi_tx_queues_alloc, ice_if_tx_queues_alloc),
151 	DEVMETHOD(ifdi_rx_queues_alloc, ice_if_rx_queues_alloc),
152 	DEVMETHOD(ifdi_msix_intr_assign, ice_if_msix_intr_assign),
153 	DEVMETHOD(ifdi_queues_free, ice_if_queues_free),
154 	DEVMETHOD(ifdi_mtu_set, ice_if_mtu_set),
155 	DEVMETHOD(ifdi_intr_enable, ice_if_intr_enable),
156 	DEVMETHOD(ifdi_intr_disable, ice_if_intr_disable),
157 	DEVMETHOD(ifdi_rx_queue_intr_enable, ice_if_rx_queue_intr_enable),
158 	DEVMETHOD(ifdi_tx_queue_intr_enable, ice_if_tx_queue_intr_enable),
159 	DEVMETHOD(ifdi_promisc_set, ice_if_promisc_set),
160 	DEVMETHOD(ifdi_media_status, ice_if_media_status),
161 	DEVMETHOD(ifdi_media_change, ice_if_media_change),
162 	DEVMETHOD(ifdi_init, ice_if_init),
163 	DEVMETHOD(ifdi_stop, ice_if_stop),
164 	DEVMETHOD(ifdi_timer, ice_if_timer),
165 	DEVMETHOD(ifdi_update_admin_status, ice_if_update_admin_status),
166 	DEVMETHOD(ifdi_multi_set, ice_if_multi_set),
167 	DEVMETHOD(ifdi_vlan_register, ice_if_vlan_register),
168 	DEVMETHOD(ifdi_vlan_unregister, ice_if_vlan_unregister),
169 	DEVMETHOD(ifdi_get_counter, ice_if_get_counter),
170 	DEVMETHOD(ifdi_priv_ioctl, ice_if_priv_ioctl),
171 	DEVMETHOD(ifdi_i2c_req, ice_if_i2c_req),
172 	DEVMETHOD(ifdi_suspend, ice_if_suspend),
173 	DEVMETHOD(ifdi_resume, ice_if_resume),
174 	DEVMETHOD_END
175 };
176 
177 /**
178  * @var ice_driver
179  * @brief driver structure for the generic device stack
180  *
181  * driver_t definition used to setup the generic device methods.
182  */
183 static driver_t ice_driver = {
184 	.name = "ice",
185 	.methods = ice_methods,
186 	.size = sizeof(struct ice_softc),
187 };
188 
189 /**
190  * @var ice_iflib_driver
191  * @brief driver structure for the iflib stack
192  *
193  * driver_t definition used to setup the iflib device methods.
194  */
195 static driver_t ice_iflib_driver = {
196 	.name = "ice",
197 	.methods = ice_iflib_methods,
198 	.size = sizeof(struct ice_softc),
199 };
200 
201 extern struct if_txrx ice_txrx;
202 extern struct if_txrx ice_recovery_txrx;
203 
204 /**
205  * @var ice_sctx
206  * @brief ice driver shared context
207  *
208  * Structure defining shared values (context) that is used by all instances of
209  * the device. Primarily used to setup details about how the iflib stack
210  * should treat this driver. Also defines the default, minimum, and maximum
211  * number of descriptors in each ring.
212  */
213 static struct if_shared_ctx ice_sctx = {
214 	.isc_magic = IFLIB_MAGIC,
215 	.isc_q_align = PAGE_SIZE,
216 
217 	.isc_tx_maxsize = ICE_MAX_FRAME_SIZE,
218 	/* We could technically set this as high as ICE_MAX_DMA_SEG_SIZE, but
219 	 * that doesn't make sense since that would be larger than the maximum
220 	 * size of a single packet.
221 	 */
222 	.isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE,
223 
224 	/* XXX: This is only used by iflib to ensure that
225 	 * scctx->isc_tx_tso_size_max + the VLAN header is a valid size.
226 	 */
227 	.isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header),
228 	/* XXX: This is used by iflib to set the number of segments in the TSO
229 	 * DMA tag. However, scctx->isc_tx_tso_segsize_max is used to set the
230 	 * related ifnet parameter.
231 	 */
232 	.isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE,
233 
234 	.isc_rx_maxsize = ICE_MAX_FRAME_SIZE,
235 	.isc_rx_nsegments = ICE_MAX_RX_SEGS,
236 	.isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE,
237 
238 	.isc_nfl = 1,
239 	.isc_ntxqs = 1,
240 	.isc_nrxqs = 1,
241 
242 	.isc_admin_intrcnt = 1,
243 	.isc_vendor_info = ice_vendor_info_array,
244 	.isc_driver_version = __DECONST(char *, ice_driver_version),
245 	.isc_driver = &ice_iflib_driver,
246 
247 	/*
248 	 * IFLIB_NEED_SCRATCH ensures that mbufs have scratch space available
249 	 * for hardware checksum offload
250 	 *
251 	 * IFLIB_TSO_INIT_IP ensures that the TSO packets have zeroed out the
252 	 * IP sum field, required by our hardware to calculate valid TSO
253 	 * checksums.
254 	 *
255 	 * IFLIB_ADMIN_ALWAYS_RUN ensures that the administrative task runs
256 	 * even when the interface is down.
257 	 *
258 	 * IFLIB_SKIP_MSIX allows the driver to handle allocating MSI-X
259 	 * vectors manually instead of relying on iflib code to do this.
260 	 */
261 	.isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP |
262 		IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX,
263 
264 	.isc_nrxd_min = {ICE_MIN_DESC_COUNT},
265 	.isc_ntxd_min = {ICE_MIN_DESC_COUNT},
266 	.isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT},
267 	.isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT},
268 	.isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT},
269 	.isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT},
270 };
271 
272 /**
273  * @var ice_devclass
274  * @brief ice driver device class
275  *
276  * device class used to setup the ice driver module kobject class.
277  */
278 devclass_t ice_devclass;
279 DRIVER_MODULE(ice, pci, ice_driver, ice_devclass, ice_module_event_handler, 0);
280 
281 MODULE_VERSION(ice, 1);
282 MODULE_DEPEND(ice, pci, 1, 1, 1);
283 MODULE_DEPEND(ice, ether, 1, 1, 1);
284 MODULE_DEPEND(ice, iflib, 1, 1, 1);
285 
286 IFLIB_PNP_INFO(pci, ice, ice_vendor_info_array);
287 
288 /* Static driver-wide sysctls */
289 #include "ice_iflib_sysctls.h"
290 
291 /**
292  * ice_pci_mapping - Map PCI BAR memory
293  * @sc: device private softc
294  *
295  * Map PCI BAR 0 for device operation.
296  */
297 static int
298 ice_pci_mapping(struct ice_softc *sc)
299 {
300 	int rc;
301 
302 	/* Map BAR0 */
303 	rc = ice_map_bar(sc->dev, &sc->bar0, 0);
304 	if (rc)
305 		return rc;
306 
307 	return 0;
308 }
309 
310 /**
311  * ice_free_pci_mapping - Release PCI BAR memory
312  * @sc: device private softc
313  *
314  * Release PCI BARs which were previously mapped by ice_pci_mapping().
315  */
316 static void
317 ice_free_pci_mapping(struct ice_softc *sc)
318 {
319 	/* Free BAR0 */
320 	ice_free_bar(sc->dev, &sc->bar0);
321 }
322 
323 /*
324  * Device methods
325  */
326 
327 /**
328  * ice_register - register device method callback
329  * @dev: the device being registered
330  *
331  * Returns a pointer to the shared context structure, which is used by iflib.
332  */
333 static void *
334 ice_register(device_t dev __unused)
335 {
336 	return &ice_sctx;
337 } /* ice_register */
338 
339 /**
340  * ice_setup_scctx - Setup the iflib softc context structure
341  * @sc: the device private structure
342  *
343  * Setup the parameters in if_softc_ctx_t structure used by the iflib stack
344  * when loading.
345  */
346 static void
347 ice_setup_scctx(struct ice_softc *sc)
348 {
349 	if_softc_ctx_t scctx = sc->scctx;
350 	struct ice_hw *hw = &sc->hw;
351 	bool safe_mode, recovery_mode;
352 
353 	safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE);
354 	recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE);
355 
356 	/*
357 	 * If the driver loads in Safe mode or Recovery mode, limit iflib to
358 	 * a single queue pair.
359 	 */
360 	if (safe_mode || recovery_mode) {
361 		scctx->isc_ntxqsets = scctx->isc_nrxqsets = 1;
362 		scctx->isc_ntxqsets_max = 1;
363 		scctx->isc_nrxqsets_max = 1;
364 	} else {
365 		/*
366 		 * iflib initially sets the isc_ntxqsets and isc_nrxqsets to
367 		 * the values of the override sysctls. Cache these initial
368 		 * values so that the driver can be aware of what the iflib
369 		 * sysctl value is when setting up MSI-X vectors.
370 		 */
371 		sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets;
372 		sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets;
373 
374 		if (scctx->isc_ntxqsets == 0)
375 			scctx->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size;
376 		if (scctx->isc_nrxqsets == 0)
377 			scctx->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size;
378 
379 		scctx->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq;
380 		scctx->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq;
381 
382 		/*
383 		 * Sanity check that the iflib sysctl values are within the
384 		 * maximum supported range.
385 		 */
386 		if (sc->ifc_sysctl_ntxqs > scctx->isc_ntxqsets_max)
387 			sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets_max;
388 		if (sc->ifc_sysctl_nrxqs > scctx->isc_nrxqsets_max)
389 			sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets_max;
390 	}
391 
392 	scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]
393 	    * sizeof(struct ice_tx_desc), DBA_ALIGN);
394 	scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0]
395 	    * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN);
396 
397 	scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS;
398 	scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS;
399 	scctx->isc_tx_tso_size_max = ICE_TSO_SIZE;
400 	scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE;
401 
402 	scctx->isc_msix_bar = PCIR_BAR(ICE_MSIX_BAR);
403 	scctx->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size;
404 
405 	/*
406 	 * If the driver loads in recovery mode, disable Tx/Rx functionality
407 	 */
408 	if (recovery_mode)
409 		scctx->isc_txrx = &ice_recovery_txrx;
410 	else
411 		scctx->isc_txrx = &ice_txrx;
412 
413 	/*
414 	 * If the driver loads in Safe mode or Recovery mode, disable
415 	 * advanced features including hardware offloads.
416 	 */
417 	if (safe_mode || recovery_mode) {
418 		scctx->isc_capenable = ICE_SAFE_CAPS;
419 		scctx->isc_tx_csum_flags = 0;
420 	} else {
421 		scctx->isc_capenable = ICE_FULL_CAPS;
422 		scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD;
423 	}
424 
425 	scctx->isc_capabilities = scctx->isc_capenable;
426 } /* ice_setup_scctx */
427 
428 /**
429  * ice_if_attach_pre - Early device attach logic
430  * @ctx: the iflib context structure
431  *
432  * Called by iflib during the attach process. Earliest main driver entry
433  * point which performs necessary hardware and driver initialization. Called
434  * before the Tx and Rx queues are allocated.
435  */
436 static int
437 ice_if_attach_pre(if_ctx_t ctx)
438 {
439 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
440 	enum ice_fw_modes fw_mode;
441 	enum ice_status status;
442 	if_softc_ctx_t scctx;
443 	struct ice_hw *hw;
444 	device_t dev;
445 	int err;
446 
447 	device_printf(iflib_get_dev(ctx), "Loading the iflib ice driver\n");
448 
449 	sc->ctx = ctx;
450 	sc->media = iflib_get_media(ctx);
451 	sc->sctx = iflib_get_sctx(ctx);
452 	sc->iflib_ctx_lock = iflib_ctx_lock_get(ctx);
453 
454 	dev = sc->dev = iflib_get_dev(ctx);
455 	scctx = sc->scctx = iflib_get_softc_ctx(ctx);
456 
457 	hw = &sc->hw;
458 	hw->back = sc;
459 
460 	snprintf(sc->admin_mtx_name, sizeof(sc->admin_mtx_name),
461 		 "%s:admin", device_get_nameunit(dev));
462 	mtx_init(&sc->admin_mtx, sc->admin_mtx_name, NULL, MTX_DEF);
463 	callout_init_mtx(&sc->admin_timer, &sc->admin_mtx, 0);
464 
465 	ASSERT_CTX_LOCKED(sc);
466 
467 	if (ice_pci_mapping(sc)) {
468 		err = (ENXIO);
469 		goto destroy_admin_timer;
470 	}
471 
472 	/* Save off the PCI information */
473 	ice_save_pci_info(hw, dev);
474 
475 	/* create tunables as early as possible */
476 	ice_add_device_tunables(sc);
477 
478 	/* Setup ControlQ lengths */
479 	ice_set_ctrlq_len(hw);
480 
481 	fw_mode = ice_get_fw_mode(hw);
482 	if (fw_mode == ICE_FW_MODE_REC) {
483 		device_printf(dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
484 
485 		err = ice_attach_pre_recovery_mode(sc);
486 		if (err)
487 			goto free_pci_mapping;
488 
489 		return (0);
490 	}
491 
492 	/* Initialize the hw data structure */
493 	status = ice_init_hw(hw);
494 	if (status) {
495 		if (status == ICE_ERR_FW_API_VER) {
496 			/* Enter recovery mode, so that the driver remains
497 			 * loaded. This way, if the system administrator
498 			 * cannot update the driver, they may still attempt to
499 			 * downgrade the NVM.
500 			 */
501 			err = ice_attach_pre_recovery_mode(sc);
502 			if (err)
503 				goto free_pci_mapping;
504 
505 			return (0);
506 		} else {
507 			err = EIO;
508 			device_printf(dev, "Unable to initialize hw, err %s aq_err %s\n",
509 				      ice_status_str(status),
510 				      ice_aq_str(hw->adminq.sq_last_status));
511 		}
512 		goto free_pci_mapping;
513 	}
514 
515 	/* Notify firmware of the device driver version */
516 	err = ice_send_version(sc);
517 	if (err)
518 		goto deinit_hw;
519 
520 	ice_load_pkg_file(sc);
521 
522 	err = ice_init_link_events(sc);
523 	if (err) {
524 		device_printf(dev, "ice_init_link_events failed: %s\n",
525 			      ice_err_str(err));
526 		goto deinit_hw;
527 	}
528 
529 	ice_print_nvm_version(sc);
530 
531 	ice_init_device_features(sc);
532 
533 	/* Setup the MAC address */
534 	iflib_set_mac(ctx, hw->port_info->mac.lan_addr);
535 
536 	/* Setup the iflib softc context structure */
537 	ice_setup_scctx(sc);
538 
539 	/* Initialize the Tx queue manager */
540 	err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq);
541 	if (err) {
542 		device_printf(dev, "Unable to initialize Tx queue manager: %s\n",
543 			      ice_err_str(err));
544 		goto deinit_hw;
545 	}
546 
547 	/* Initialize the Rx queue manager */
548 	err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq);
549 	if (err) {
550 		device_printf(dev, "Unable to initialize Rx queue manager: %s\n",
551 			      ice_err_str(err));
552 		goto free_tx_qmgr;
553 	}
554 
555 	/* Initialize the interrupt resource manager */
556 	err = ice_alloc_intr_tracking(sc);
557 	if (err)
558 		/* Errors are already printed */
559 		goto free_rx_qmgr;
560 
561 	/* Determine maximum number of VSIs we'll prepare for */
562 	sc->num_available_vsi = min(ICE_MAX_VSI_AVAILABLE,
563 				    hw->func_caps.guar_num_vsi);
564 
565 	if (!sc->num_available_vsi) {
566 		err = EIO;
567 		device_printf(dev, "No VSIs allocated to host\n");
568 		goto free_intr_tracking;
569 	}
570 
571 	/* Allocate storage for the VSI pointers */
572 	sc->all_vsi = (struct ice_vsi **)
573 		malloc(sizeof(struct ice_vsi *) * sc->num_available_vsi,
574 		       M_ICE, M_WAITOK | M_ZERO);
575 	if (!sc->all_vsi) {
576 		err = ENOMEM;
577 		device_printf(dev, "Unable to allocate VSI array\n");
578 		goto free_intr_tracking;
579 	}
580 
581 	/*
582 	 * Prepare the statically allocated primary PF VSI in the softc
583 	 * structure. Other VSIs will be dynamically allocated as needed.
584 	 */
585 	ice_setup_pf_vsi(sc);
586 
587 	err = ice_alloc_vsi_qmap(&sc->pf_vsi, scctx->isc_ntxqsets_max,
588 	    scctx->isc_nrxqsets_max);
589 	if (err) {
590 		device_printf(dev, "Unable to allocate VSI Queue maps\n");
591 		goto free_main_vsi;
592 	}
593 
594 	/* Allocate MSI-X vectors (due to isc_flags IFLIB_SKIP_MSIX) */
595 	err = ice_allocate_msix(sc);
596 	if (err)
597 		goto free_main_vsi;
598 
599 	return 0;
600 
601 free_main_vsi:
602 	/* ice_release_vsi will free the queue maps if they were allocated */
603 	ice_release_vsi(&sc->pf_vsi);
604 	free(sc->all_vsi, M_ICE);
605 	sc->all_vsi = NULL;
606 free_intr_tracking:
607 	ice_free_intr_tracking(sc);
608 free_rx_qmgr:
609 	ice_resmgr_destroy(&sc->rx_qmgr);
610 free_tx_qmgr:
611 	ice_resmgr_destroy(&sc->tx_qmgr);
612 deinit_hw:
613 	ice_deinit_hw(hw);
614 free_pci_mapping:
615 	ice_free_pci_mapping(sc);
616 destroy_admin_timer:
617 	mtx_lock(&sc->admin_mtx);
618 	callout_stop(&sc->admin_timer);
619 	mtx_unlock(&sc->admin_mtx);
620 	mtx_destroy(&sc->admin_mtx);
621 	return err;
622 } /* ice_if_attach_pre */
623 
624 /**
625  * ice_attach_pre_recovery_mode - Limited driver attach_pre for FW recovery
626  * @sc: the device private softc
627  *
628  * Loads the device driver in limited Firmware Recovery mode, intended to
629  * allow users to update the firmware to attempt to recover the device.
630  *
631  * @remark We may enter recovery mode in case either (a) the firmware is
632  * detected to be in an invalid state and must be re-programmed, or (b) the
633  * driver detects that the loaded firmware has a non-compatible API version
634  * that the driver cannot operate with.
635  */
636 static int
637 ice_attach_pre_recovery_mode(struct ice_softc *sc)
638 {
639 	ice_set_state(&sc->state, ICE_STATE_RECOVERY_MODE);
640 
641 	/* Setup the iflib softc context */
642 	ice_setup_scctx(sc);
643 
644 	/* Setup the PF VSI back pointer */
645 	sc->pf_vsi.sc = sc;
646 
647 	/*
648 	 * We still need to allocate MSI-X vectors since we need one vector to
649 	 * run the administrative admin interrupt
650 	 */
651 	return ice_allocate_msix(sc);
652 }
653 
654 /**
655  * ice_update_link_status - notify OS of link state change
656  * @sc: device private softc structure
657  * @update_media: true if we should update media even if link didn't change
658  *
659  * Called to notify iflib core of link status changes. Should be called once
660  * during attach_post, and whenever link status changes during runtime.
661  *
662  * This call only updates the currently supported media types if the link
663  * status changed, or if update_media is set to true.
664  */
665 static void
666 ice_update_link_status(struct ice_softc *sc, bool update_media)
667 {
668 	struct ice_hw *hw = &sc->hw;
669 	enum ice_status status;
670 
671 	/* Never report link up when in recovery mode */
672 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
673 		return;
674 
675 	/* Report link status to iflib only once each time it changes */
676 	if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) {
677 		if (sc->link_up) { /* link is up */
678 			uint64_t baudrate = ice_aq_speed_to_rate(sc->hw.port_info);
679 
680 			ice_set_default_local_lldp_mib(sc);
681 
682 			iflib_link_state_change(sc->ctx, LINK_STATE_UP, baudrate);
683 
684 			ice_link_up_msg(sc);
685 
686 			update_media = true;
687 		} else { /* link is down */
688 			iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0);
689 
690 			update_media = true;
691 		}
692 	}
693 
694 	/* Update the supported media types */
695 	if (update_media) {
696 		status = ice_add_media_types(sc, sc->media);
697 		if (status)
698 			device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n",
699 				      ice_status_str(status),
700 				      ice_aq_str(hw->adminq.sq_last_status));
701 	}
702 
703 	/* TODO: notify VFs of link state change */
704 }
705 
706 /**
707  * ice_if_attach_post - Late device attach logic
708  * @ctx: the iflib context structure
709  *
710  * Called by iflib to finish up attaching the device. Performs any attach
711  * logic which must wait until after the Tx and Rx queues have been
712  * allocated.
713  */
714 static int
715 ice_if_attach_post(if_ctx_t ctx)
716 {
717 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
718 	if_t ifp = iflib_get_ifp(ctx);
719 	int err;
720 
721 	ASSERT_CTX_LOCKED(sc);
722 
723 	/* We don't yet support loading if MSI-X is not supported */
724 	if (sc->scctx->isc_intr != IFLIB_INTR_MSIX) {
725 		device_printf(sc->dev, "The ice driver does not support loading without MSI-X\n");
726 		return (ENOTSUP);
727 	}
728 
729 	/* The ifnet structure hasn't yet been initialized when the attach_pre
730 	 * handler is called, so wait until attach_post to setup the
731 	 * isc_max_frame_size.
732 	 */
733 
734 	sc->ifp = ifp;
735 	sc->scctx->isc_max_frame_size = ifp->if_mtu +
736 		ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN;
737 
738 	/*
739 	 * If we are in recovery mode, only perform a limited subset of
740 	 * initialization to support NVM recovery.
741 	 */
742 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
743 		ice_attach_post_recovery_mode(sc);
744 		return (0);
745 	}
746 
747 	sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size;
748 
749 	err = ice_initialize_vsi(&sc->pf_vsi);
750 	if (err) {
751 		device_printf(sc->dev, "Unable to initialize Main VSI: %s\n",
752 			      ice_err_str(err));
753 		return err;
754 	}
755 
756 	/* Enable FW health event reporting */
757 	ice_init_health_events(sc);
758 
759 	/* Configure the main PF VSI for RSS */
760 	err = ice_config_rss(&sc->pf_vsi);
761 	if (err) {
762 		device_printf(sc->dev,
763 			      "Unable to configure RSS for the main VSI, err %s\n",
764 			      ice_err_str(err));
765 		return err;
766 	}
767 
768 	/* Configure switch to drop transmitted LLDP and PAUSE frames */
769 	err = ice_cfg_pf_ethertype_filters(sc);
770 	if (err)
771 		return err;
772 
773 	ice_get_and_print_bus_info(sc);
774 
775 	ice_set_link_management_mode(sc);
776 
777 	ice_init_saved_phy_cfg(sc);
778 
779 	ice_add_device_sysctls(sc);
780 
781 	/* Get DCBX/LLDP state and start DCBX agent */
782 	ice_init_dcb_setup(sc);
783 
784 	/* Setup link configuration parameters */
785 	ice_init_link_configuration(sc);
786 	ice_update_link_status(sc, true);
787 
788 	/* Configure interrupt causes for the administrative interrupt */
789 	ice_configure_misc_interrupts(sc);
790 
791 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
792 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
793 
794 	/* Start the admin timer */
795 	mtx_lock(&sc->admin_mtx);
796 	callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc);
797 	mtx_unlock(&sc->admin_mtx);
798 
799 	return 0;
800 } /* ice_if_attach_post */
801 
802 /**
803  * ice_attach_post_recovery_mode - Limited driver attach_post for FW recovery
804  * @sc: the device private softc
805  *
806  * Performs minimal work to prepare the driver to recover an NVM in case the
807  * firmware is in recovery mode.
808  */
809 static void
810 ice_attach_post_recovery_mode(struct ice_softc *sc)
811 {
812 	/* Configure interrupt causes for the administrative interrupt */
813 	ice_configure_misc_interrupts(sc);
814 
815 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
816 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
817 
818 	/* Start the admin timer */
819 	mtx_lock(&sc->admin_mtx);
820 	callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc);
821 	mtx_unlock(&sc->admin_mtx);
822 }
823 
824 /**
825  * ice_free_irqvs - Free IRQ vector memory
826  * @sc: the device private softc structure
827  *
828  * Free IRQ vector memory allocated during ice_if_msix_intr_assign.
829  */
830 static void
831 ice_free_irqvs(struct ice_softc *sc)
832 {
833 	struct ice_vsi *vsi = &sc->pf_vsi;
834 	if_ctx_t ctx = sc->ctx;
835 	int i;
836 
837 	/* If the irqvs array is NULL, then there are no vectors to free */
838 	if (sc->irqvs == NULL)
839 		return;
840 
841 	/* Free the IRQ vectors */
842 	for (i = 0; i < sc->num_irq_vectors; i++)
843 		iflib_irq_free(ctx, &sc->irqvs[i].irq);
844 
845 	/* Clear the irqv pointers */
846 	for (i = 0; i < vsi->num_rx_queues; i++)
847 		vsi->rx_queues[i].irqv = NULL;
848 
849 	for (i = 0; i < vsi->num_tx_queues; i++)
850 		vsi->tx_queues[i].irqv = NULL;
851 
852 	/* Release the vector array memory */
853 	free(sc->irqvs, M_ICE);
854 	sc->irqvs = NULL;
855 	sc->num_irq_vectors = 0;
856 }
857 
858 /**
859  * ice_if_detach - Device driver detach logic
860  * @ctx: iflib context structure
861  *
862  * Perform device shutdown logic to detach the device driver.
863  *
864  * Note that there is no guarantee of the ordering of ice_if_queues_free() and
865  * ice_if_detach(). It is possible for the functions to be called in either
866  * order, and they must not assume to have a strict ordering.
867  */
868 static int
869 ice_if_detach(if_ctx_t ctx)
870 {
871 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
872 	struct ice_vsi *vsi = &sc->pf_vsi;
873 	int i;
874 
875 	ASSERT_CTX_LOCKED(sc);
876 
877 	/* Indicate that we're detaching */
878 	ice_set_state(&sc->state, ICE_STATE_DETACHING);
879 
880 	/* Stop the admin timer */
881 	mtx_lock(&sc->admin_mtx);
882 	callout_stop(&sc->admin_timer);
883 	mtx_unlock(&sc->admin_mtx);
884 	mtx_destroy(&sc->admin_mtx);
885 
886 	/* Free allocated media types */
887 	ifmedia_removeall(sc->media);
888 
889 	/* Free the Tx and Rx sysctl contexts, and assign NULL to the node
890 	 * pointers. Note, the calls here and those in ice_if_queues_free()
891 	 * are *BOTH* necessary, as we cannot guarantee which path will be
892 	 * run first
893 	 */
894 	ice_vsi_del_txqs_ctx(vsi);
895 	ice_vsi_del_rxqs_ctx(vsi);
896 
897 	/* Release MSI-X resources */
898 	ice_free_irqvs(sc);
899 
900 	for (i = 0; i < sc->num_available_vsi; i++) {
901 		if (sc->all_vsi[i])
902 			ice_release_vsi(sc->all_vsi[i]);
903 	}
904 
905 	if (sc->all_vsi) {
906 		free(sc->all_vsi, M_ICE);
907 		sc->all_vsi = NULL;
908 	}
909 
910 	/* Release MSI-X memory */
911 	pci_release_msi(sc->dev);
912 
913 	if (sc->msix_table != NULL) {
914 		bus_release_resource(sc->dev, SYS_RES_MEMORY,
915 				     rman_get_rid(sc->msix_table),
916 				     sc->msix_table);
917 		sc->msix_table = NULL;
918 	}
919 
920 	ice_free_intr_tracking(sc);
921 
922 	/* Destroy the queue managers */
923 	ice_resmgr_destroy(&sc->tx_qmgr);
924 	ice_resmgr_destroy(&sc->rx_qmgr);
925 
926 	if (!ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
927 		ice_deinit_hw(&sc->hw);
928 
929 	ice_free_pci_mapping(sc);
930 
931 	return 0;
932 } /* ice_if_detach */
933 
934 /**
935  * ice_if_tx_queues_alloc - Allocate Tx queue memory
936  * @ctx: iflib context structure
937  * @vaddrs: virtual addresses for the queue memory
938  * @paddrs: physical addresses for the queue memory
939  * @ntxqs: the number of Tx queues per set (should always be 1)
940  * @ntxqsets: the number of Tx queue sets to allocate
941  *
942  * Called by iflib to allocate Tx queues for the device. Allocates driver
943  * memory to track each queue, the status arrays used for descriptor
944  * status reporting, and Tx queue sysctls.
945  */
946 static int
947 ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
948 		       int __invariant_only ntxqs, int ntxqsets)
949 {
950 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
951 	struct ice_vsi *vsi = &sc->pf_vsi;
952 	struct ice_tx_queue *txq;
953 	int err, i, j;
954 
955 	MPASS(ntxqs == 1);
956 	MPASS(sc->scctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT);
957 	ASSERT_CTX_LOCKED(sc);
958 
959 	/* Do not bother allocating queues if we're in recovery mode */
960 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
961 		return (0);
962 
963 	/* Allocate queue structure memory */
964 	if (!(vsi->tx_queues =
965 	      (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_WAITOK | M_ZERO))) {
966 		device_printf(sc->dev, "Unable to allocate Tx queue memory\n");
967 		return (ENOMEM);
968 	}
969 
970 	/* Allocate report status arrays */
971 	for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
972 		if (!(txq->tx_rsq =
973 		      (uint16_t *) malloc(sizeof(uint16_t) * sc->scctx->isc_ntxd[0], M_ICE, M_WAITOK))) {
974 			device_printf(sc->dev, "Unable to allocate tx_rsq memory\n");
975 			err = ENOMEM;
976 			goto free_tx_queues;
977 		}
978 		/* Initialize report status array */
979 		for (j = 0; j < sc->scctx->isc_ntxd[0]; j++)
980 			txq->tx_rsq[j] = QIDX_INVALID;
981 	}
982 
983 	/* Assign queues from PF space to the main VSI */
984 	err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, ntxqsets);
985 	if (err) {
986 		device_printf(sc->dev, "Unable to assign PF queues: %s\n",
987 			      ice_err_str(err));
988 		goto free_tx_queues;
989 	}
990 	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
991 
992 	/* Add Tx queue sysctls context */
993 	ice_vsi_add_txqs_ctx(vsi);
994 
995 	for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
996 		txq->me = i;
997 		txq->vsi = vsi;
998 
999 		/* store the queue size for easier access */
1000 		txq->desc_count = sc->scctx->isc_ntxd[0];
1001 
1002 		/* get the virtual and physical address of the hardware queues */
1003 		txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);
1004 		txq->tx_base = (struct ice_tx_desc *)vaddrs[i];
1005 		txq->tx_paddr = paddrs[i];
1006 
1007 		ice_add_txq_sysctls(txq);
1008 	}
1009 
1010 	vsi->num_tx_queues = ntxqsets;
1011 
1012 	return (0);
1013 
1014 free_tx_queues:
1015 	for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
1016 		if (txq->tx_rsq != NULL) {
1017 			free(txq->tx_rsq, M_ICE);
1018 			txq->tx_rsq = NULL;
1019 		}
1020 	}
1021 	free(vsi->tx_queues, M_ICE);
1022 	vsi->tx_queues = NULL;
1023 	return err;
1024 }
1025 
1026 /**
1027  * ice_if_rx_queues_alloc - Allocate Rx queue memory
1028  * @ctx: iflib context structure
1029  * @vaddrs: virtual addresses for the queue memory
1030  * @paddrs: physical addresses for the queue memory
1031  * @nrxqs: number of Rx queues per set (should always be 1)
1032  * @nrxqsets: number of Rx queue sets to allocate
1033  *
1034  * Called by iflib to allocate Rx queues for the device. Allocates driver
1035  * memory to track each queue, as well as sets up the Rx queue sysctls.
1036  */
1037 static int
1038 ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
1039 		       int __invariant_only nrxqs, int nrxqsets)
1040 {
1041 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1042 	struct ice_vsi *vsi = &sc->pf_vsi;
1043 	struct ice_rx_queue *rxq;
1044 	int err, i;
1045 
1046 	MPASS(nrxqs == 1);
1047 	MPASS(sc->scctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT);
1048 	ASSERT_CTX_LOCKED(sc);
1049 
1050 	/* Do not bother allocating queues if we're in recovery mode */
1051 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1052 		return (0);
1053 
1054 	/* Allocate queue structure memory */
1055 	if (!(vsi->rx_queues =
1056 	      (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_WAITOK | M_ZERO))) {
1057 		device_printf(sc->dev, "Unable to allocate Rx queue memory\n");
1058 		return (ENOMEM);
1059 	}
1060 
1061 	/* Assign queues from PF space to the main VSI */
1062 	err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, nrxqsets);
1063 	if (err) {
1064 		device_printf(sc->dev, "Unable to assign PF queues: %s\n",
1065 			      ice_err_str(err));
1066 		goto free_rx_queues;
1067 	}
1068 	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
1069 
1070 	/* Add Rx queue sysctls context */
1071 	ice_vsi_add_rxqs_ctx(vsi);
1072 
1073 	for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) {
1074 		rxq->me = i;
1075 		rxq->vsi = vsi;
1076 
1077 		/* store the queue size for easier access */
1078 		rxq->desc_count = sc->scctx->isc_nrxd[0];
1079 
1080 		/* get the virtual and physical address of the hardware queues */
1081 		rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);
1082 		rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i];
1083 		rxq->rx_paddr = paddrs[i];
1084 
1085 		ice_add_rxq_sysctls(rxq);
1086 	}
1087 
1088 	vsi->num_rx_queues = nrxqsets;
1089 
1090 	return (0);
1091 
1092 free_rx_queues:
1093 	free(vsi->rx_queues, M_ICE);
1094 	vsi->rx_queues = NULL;
1095 	return err;
1096 }
1097 
1098 /**
1099  * ice_if_queues_free - Free queue memory
1100  * @ctx: the iflib context structure
1101  *
1102  * Free queue memory allocated by ice_if_tx_queues_alloc() and
1103  * ice_if_rx_queues_alloc().
1104  *
1105  * There is no guarantee that ice_if_queues_free() and ice_if_detach() will be
1106  * called in the same order. It's possible for ice_if_queues_free() to be
1107  * called prior to ice_if_detach(), and vice versa.
1108  *
1109  * For this reason, the main VSI is a static member of the ice_softc, which is
1110  * not free'd until after iflib finishes calling both of these functions.
1111  *
1112  * Thus, care must be taken in how we manage the memory being freed by this
1113  * function, and in what tasks it can and must perform.
1114  */
1115 static void
1116 ice_if_queues_free(if_ctx_t ctx)
1117 {
1118 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1119 	struct ice_vsi *vsi = &sc->pf_vsi;
1120 	struct ice_tx_queue *txq;
1121 	int i;
1122 
1123 	/* Free the Tx and Rx sysctl contexts, and assign NULL to the node
1124 	 * pointers. Note, the calls here and those in ice_if_detach()
1125 	 * are *BOTH* necessary, as we cannot guarantee which path will be
1126 	 * run first
1127 	 */
1128 	ice_vsi_del_txqs_ctx(vsi);
1129 	ice_vsi_del_rxqs_ctx(vsi);
1130 
1131 	/* Release MSI-X IRQ vectors, if not yet released in ice_if_detach */
1132 	ice_free_irqvs(sc);
1133 
1134 	if (vsi->tx_queues != NULL) {
1135 		/* free the tx_rsq arrays */
1136 		for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {
1137 			if (txq->tx_rsq != NULL) {
1138 				free(txq->tx_rsq, M_ICE);
1139 				txq->tx_rsq = NULL;
1140 			}
1141 		}
1142 		free(vsi->tx_queues, M_ICE);
1143 		vsi->tx_queues = NULL;
1144 		vsi->num_tx_queues = 0;
1145 	}
1146 	if (vsi->rx_queues != NULL) {
1147 		free(vsi->rx_queues, M_ICE);
1148 		vsi->rx_queues = NULL;
1149 		vsi->num_rx_queues = 0;
1150 	}
1151 }
1152 
1153 /**
1154  * ice_msix_que - Fast interrupt handler for MSI-X receive queues
1155  * @arg: The Rx queue memory
1156  *
1157  * Interrupt filter function for iflib MSI-X interrupts. Called by iflib when
1158  * an MSI-X interrupt for a given queue is triggered. Currently this just asks
1159  * iflib to schedule the main Rx thread.
1160  */
1161 static int
1162 ice_msix_que(void *arg)
1163 {
1164 	struct ice_rx_queue __unused *rxq = (struct ice_rx_queue *)arg;
1165 
1166 	/* TODO: dynamic ITR algorithm?? */
1167 
1168 	return (FILTER_SCHEDULE_THREAD);
1169 }
1170 
1171 /**
1172  * ice_msix_admin - Fast interrupt handler for MSI-X admin interrupt
1173  * @arg: pointer to device softc memory
1174  *
1175  * Called by iflib when an administrative interrupt occurs. Should perform any
1176  * fast logic for handling the interrupt cause, and then indicate whether the
1177  * admin task needs to be queued.
1178  */
1179 static int
1180 ice_msix_admin(void *arg)
1181 {
1182 	struct ice_softc *sc = (struct ice_softc *)arg;
1183 	struct ice_hw *hw = &sc->hw;
1184 	device_t dev = sc->dev;
1185 	u32 oicr;
1186 
1187 	/* There is no safe way to modify the enabled miscellaneous causes of
1188 	 * the OICR vector at runtime, as doing so would be prone to race
1189 	 * conditions. Reading PFINT_OICR will unmask the associated interrupt
1190 	 * causes and allow future interrupts to occur. The admin interrupt
1191 	 * vector will not be re-enabled until after we exit this function,
1192 	 * but any delayed tasks must be resilient against possible "late
1193 	 * arrival" interrupts that occur while we're already handling the
1194 	 * task. This is done by using state bits and serializing these
1195 	 * delayed tasks via the admin status task function.
1196 	 */
1197 	oicr = rd32(hw, PFINT_OICR);
1198 
1199 	/* Processing multiple controlq interrupts on a single vector does not
1200 	 * provide an indication of which controlq triggered the interrupt.
1201 	 * We might try reading the INTEVENT bit of the respective PFINT_*_CTL
1202 	 * registers. However, the INTEVENT bit is not guaranteed to be set as
1203 	 * it gets automatically cleared when the hardware acknowledges the
1204 	 * interrupt.
1205 	 *
1206 	 * This means we don't really have a good indication of whether or
1207 	 * which controlq triggered this interrupt. We'll just notify the
1208 	 * admin task that it should check all the controlqs.
1209 	 */
1210 	ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
1211 
1212 	if (oicr & PFINT_OICR_VFLR_M) {
1213 		ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING);
1214 	}
1215 
1216 	if (oicr & PFINT_OICR_MAL_DETECT_M) {
1217 		ice_set_state(&sc->state, ICE_STATE_MDD_PENDING);
1218 	}
1219 
1220 	if (oicr & PFINT_OICR_GRST_M) {
1221 		u32 reset;
1222 
1223 		reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
1224 			GLGEN_RSTAT_RESET_TYPE_S;
1225 
1226 		if (reset == ICE_RESET_CORER)
1227 			sc->soft_stats.corer_count++;
1228 		else if (reset == ICE_RESET_GLOBR)
1229 			sc->soft_stats.globr_count++;
1230 		else
1231 			sc->soft_stats.empr_count++;
1232 
1233 		/* There are a couple of bits at play for handling resets.
1234 		 * First, the ICE_STATE_RESET_OICR_RECV bit is used to
1235 		 * indicate that the driver has received an OICR with a reset
1236 		 * bit active, indicating that a CORER/GLOBR/EMPR is about to
1237 		 * happen. Second, we set hw->reset_ongoing to indicate that
1238 		 * the hardware is in reset. We will set this back to false as
1239 		 * soon as the driver has determined that the hardware is out
1240 		 * of reset.
1241 		 *
1242 		 * If the driver wishes to trigger a reqest, it can set one of
1243 		 * the ICE_STATE_RESET_*_REQ bits, which will trigger the
1244 		 * correct type of reset.
1245 		 */
1246 		if (!ice_testandset_state(&sc->state, ICE_STATE_RESET_OICR_RECV))
1247 			hw->reset_ongoing = true;
1248 	}
1249 
1250 	if (oicr & PFINT_OICR_ECC_ERR_M) {
1251 		device_printf(dev, "ECC Error detected!\n");
1252 		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1253 	}
1254 
1255 	if (oicr & PFINT_OICR_PE_CRITERR_M) {
1256 		device_printf(dev, "Critical Protocol Engine Error detected!\n");
1257 		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1258 	}
1259 
1260 	if (oicr & PFINT_OICR_PCI_EXCEPTION_M) {
1261 		device_printf(dev, "PCI Exception detected!\n");
1262 		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1263 	}
1264 
1265 	if (oicr & PFINT_OICR_HMC_ERR_M) {
1266 		/* Log the HMC errors, but don't disable the interrupt cause */
1267 		ice_log_hmc_error(hw, dev);
1268 	}
1269 
1270 	return (FILTER_SCHEDULE_THREAD);
1271 }
1272 
1273 /**
1274  * ice_allocate_msix - Allocate MSI-X vectors for the interface
1275  * @sc: the device private softc
1276  *
1277  * Map the MSI-X bar, and then request MSI-X vectors in a two-stage process.
1278  *
1279  * First, determine a suitable total number of vectors based on the number
1280  * of CPUs, RSS buckets, the administrative vector, and other demands such as
1281  * RDMA.
1282  *
1283  * Request the desired amount of vectors, and see how many we obtain. If we
1284  * don't obtain as many as desired, reduce the demands by lowering the number
1285  * of requested queues or reducing the demand from other features such as
1286  * RDMA.
1287  *
1288  * @remark This function is required because the driver sets the
1289  * IFLIB_SKIP_MSIX flag indicating that the driver will manage MSI-X vectors
1290  * manually.
1291  *
1292  * @remark This driver will only use MSI-X vectors. If this is not possible,
1293  * neither MSI or legacy interrupts will be tried.
1294  *
1295  * @post on success this function must set the following scctx parameters:
1296  * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr.
1297  *
1298  * @returns zero on success or an error code on failure.
1299  */
1300 static int
1301 ice_allocate_msix(struct ice_softc *sc)
1302 {
1303 	bool iflib_override_queue_count = false;
1304 	if_softc_ctx_t scctx = sc->scctx;
1305 	device_t dev = sc->dev;
1306 	cpuset_t cpus;
1307 	int bar, queues, vectors, requested;
1308 	int err = 0;
1309 
1310 	/* Allocate the MSI-X bar */
1311 	bar = scctx->isc_msix_bar;
1312 	sc->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE);
1313 	if (!sc->msix_table) {
1314 		device_printf(dev, "Unable to map MSI-X table\n");
1315 		return (ENOMEM);
1316 	}
1317 
1318 	/* Check if the iflib queue count sysctls have been set */
1319 	if (sc->ifc_sysctl_ntxqs || sc->ifc_sysctl_nrxqs)
1320 		iflib_override_queue_count = true;
1321 
1322 	err = bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus);
1323 	if (err) {
1324 		device_printf(dev, "%s: Unable to fetch the CPU list: %s\n",
1325 			      __func__, ice_err_str(err));
1326 		CPU_COPY(&all_cpus, &cpus);
1327 	}
1328 
1329 	/* Attempt to mimic behavior of iflib_msix_init */
1330 	if (iflib_override_queue_count) {
1331 		/*
1332 		 * If the override sysctls have been set, limit the queues to
1333 		 * the number of logical CPUs.
1334 		 */
1335 		queues = mp_ncpus;
1336 	} else {
1337 		/*
1338 		 * Otherwise, limit the queue count to the CPUs associated
1339 		 * with the NUMA node the device is associated with.
1340 		 */
1341 		queues = CPU_COUNT(&cpus);
1342 	}
1343 
1344 	/* Clamp to the number of RSS buckets */
1345 	queues = imin(queues, rss_getnumbuckets());
1346 
1347 	/*
1348 	 * Clamp the number of queue pairs to the minimum of the requested Tx
1349 	 * and Rx queues.
1350 	 */
1351 	queues = imin(queues, sc->ifc_sysctl_ntxqs ?: scctx->isc_ntxqsets);
1352 	queues = imin(queues, sc->ifc_sysctl_nrxqs ?: scctx->isc_nrxqsets);
1353 
1354 	/*
1355 	 * Determine the number of vectors to request. Note that we also need
1356 	 * to allocate one vector for administrative tasks.
1357 	 */
1358 	requested = queues + 1;
1359 
1360 	vectors = requested;
1361 
1362 	err = pci_alloc_msix(dev, &vectors);
1363 	if (err) {
1364 		device_printf(dev, "Failed to allocate %d MSI-X vectors, err %s\n",
1365 			      vectors, ice_err_str(err));
1366 		goto err_free_msix_table;
1367 	}
1368 
1369 	/* If we don't receive enough vectors, reduce demands */
1370 	if (vectors < requested) {
1371 		int diff = requested - vectors;
1372 
1373 		device_printf(dev, "Requested %d MSI-X vectors, but got only %d\n",
1374 			      requested, vectors);
1375 
1376 		/*
1377 		 * If we still have a difference, we need to reduce the number
1378 		 * of queue pairs.
1379 		 *
1380 		 * However, we still need at least one vector for the admin
1381 		 * interrupt and one queue pair.
1382 		 */
1383 		if (queues <= diff) {
1384 			device_printf(dev, "Unable to allocate sufficient MSI-X vectors\n");
1385 			err = (ERANGE);
1386 			goto err_pci_release_msi;
1387 		}
1388 
1389 		queues -= diff;
1390 	}
1391 
1392 	device_printf(dev, "Using %d Tx and Rx queues\n", queues);
1393 	device_printf(dev, "Using MSI-X interrupts with %d vectors\n",
1394 		      vectors);
1395 
1396 	scctx->isc_vectors = vectors;
1397 	scctx->isc_nrxqsets = queues;
1398 	scctx->isc_ntxqsets = queues;
1399 	scctx->isc_intr = IFLIB_INTR_MSIX;
1400 
1401 	/* Interrupt allocation tracking isn't required in recovery mode,
1402 	 * since neither RDMA nor VFs are enabled.
1403 	 */
1404 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1405 		return (0);
1406 
1407 	/* Keep track of which interrupt indices are being used for what */
1408 	sc->lan_vectors = vectors;
1409 	err = ice_resmgr_assign_contiguous(&sc->imgr, sc->pf_imap, sc->lan_vectors);
1410 	if (err) {
1411 		device_printf(dev, "Unable to assign PF interrupt mapping: %s\n",
1412 			      ice_err_str(err));
1413 		goto err_pci_release_msi;
1414 	}
1415 
1416 	return (0);
1417 
1418 err_pci_release_msi:
1419 	pci_release_msi(dev);
1420 err_free_msix_table:
1421 	if (sc->msix_table != NULL) {
1422 		bus_release_resource(sc->dev, SYS_RES_MEMORY,
1423 				rman_get_rid(sc->msix_table),
1424 				sc->msix_table);
1425 		sc->msix_table = NULL;
1426 	}
1427 
1428 	return (err);
1429 }
1430 
1431 /**
1432  * ice_if_msix_intr_assign - Assign MSI-X interrupt vectors to queues
1433  * @ctx: the iflib context structure
1434  * @msix: the number of vectors we were assigned
1435  *
1436  * Called by iflib to assign MSI-X vectors to queues. Currently requires that
1437  * we get at least the same number of vectors as we have queues, and that we
1438  * always have the same number of Tx and Rx queues.
1439  *
1440  * Tx queues use a softirq instead of using their own hardware interrupt.
1441  */
1442 static int
1443 ice_if_msix_intr_assign(if_ctx_t ctx, int msix)
1444 {
1445 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1446 	struct ice_vsi *vsi = &sc->pf_vsi;
1447 	int err, i, vector;
1448 
1449 	ASSERT_CTX_LOCKED(sc);
1450 
1451 	if (vsi->num_rx_queues != vsi->num_tx_queues) {
1452 		device_printf(sc->dev,
1453 			      "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n",
1454 			      vsi->num_tx_queues, vsi->num_rx_queues);
1455 		return (EOPNOTSUPP);
1456 	}
1457 
1458 	if (msix < (vsi->num_rx_queues + 1)) {
1459 		device_printf(sc->dev,
1460 			      "Not enough MSI-X vectors to assign one vector to each queue pair\n");
1461 		return (EOPNOTSUPP);
1462 	}
1463 
1464 	/* Save the number of vectors for future use */
1465 	sc->num_irq_vectors = vsi->num_rx_queues + 1;
1466 
1467 	/* Allocate space to store the IRQ vector data */
1468 	if (!(sc->irqvs =
1469 	      (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (sc->num_irq_vectors),
1470 					       M_ICE, M_NOWAIT))) {
1471 		device_printf(sc->dev,
1472 			      "Unable to allocate irqv memory\n");
1473 		return (ENOMEM);
1474 	}
1475 
1476 	/* Administrative interrupt events will use vector 0 */
1477 	err = iflib_irq_alloc_generic(ctx, &sc->irqvs[0].irq, 1, IFLIB_INTR_ADMIN,
1478 				      ice_msix_admin, sc, 0, "admin");
1479 	if (err) {
1480 		device_printf(sc->dev,
1481 			      "Failed to register Admin queue handler: %s\n",
1482 			      ice_err_str(err));
1483 		goto free_irqvs;
1484 	}
1485 	sc->irqvs[0].me = 0;
1486 
1487 	/* Do not allocate queue interrupts when in recovery mode */
1488 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1489 		return (0);
1490 
1491 	for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++) {
1492 		struct ice_rx_queue *rxq = &vsi->rx_queues[i];
1493 		struct ice_tx_queue *txq = &vsi->tx_queues[i];
1494 		int rid = vector + 1;
1495 		char irq_name[16];
1496 
1497 		snprintf(irq_name, sizeof(irq_name), "rxq%d", i);
1498 		err = iflib_irq_alloc_generic(ctx, &sc->irqvs[vector].irq, rid,
1499 					      IFLIB_INTR_RXTX, ice_msix_que,
1500 					      rxq, rxq->me, irq_name);
1501 		if (err) {
1502 			device_printf(sc->dev,
1503 				      "Failed to allocate q int %d err: %s\n",
1504 				      i, ice_err_str(err));
1505 			vector--;
1506 			i--;
1507 			goto fail;
1508 		}
1509 		sc->irqvs[vector].me = vector;
1510 		rxq->irqv = &sc->irqvs[vector];
1511 
1512 		bzero(irq_name, sizeof(irq_name));
1513 
1514 		snprintf(irq_name, sizeof(irq_name), "txq%d", i);
1515 		iflib_softirq_alloc_generic(ctx, &sc->irqvs[vector].irq,
1516 					    IFLIB_INTR_TX, txq,
1517 					    txq->me, irq_name);
1518 		txq->irqv = &sc->irqvs[vector];
1519 	}
1520 
1521 	return (0);
1522 fail:
1523 	for (; i >= 0; i--, vector--)
1524 		iflib_irq_free(ctx, &sc->irqvs[vector].irq);
1525 	iflib_irq_free(ctx, &sc->irqvs[0].irq);
1526 free_irqvs:
1527 	free(sc->irqvs, M_ICE);
1528 	sc->irqvs = NULL;
1529 	return err;
1530 }
1531 
1532 /**
1533  * ice_if_mtu_set - Set the device MTU
1534  * @ctx: iflib context structure
1535  * @mtu: the MTU requested
1536  *
1537  * Called by iflib to configure the device's Maximum Transmission Unit (MTU).
1538  *
1539  * @pre assumes the caller holds the iflib CTX lock
1540  */
1541 static int
1542 ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
1543 {
1544 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1545 
1546 	ASSERT_CTX_LOCKED(sc);
1547 
1548 	/* Do not support configuration when in recovery mode */
1549 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1550 		return (ENOSYS);
1551 
1552 	if (mtu < ICE_MIN_MTU || mtu > ICE_MAX_MTU)
1553 		return (EINVAL);
1554 
1555 	sc->scctx->isc_max_frame_size = mtu +
1556 		ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN;
1557 
1558 	sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size;
1559 
1560 	return (0);
1561 }
1562 
1563 /**
1564  * ice_if_intr_enable - Enable device interrupts
1565  * @ctx: iflib context structure
1566  *
1567  * Called by iflib to request enabling device interrupts.
1568  */
1569 static void
1570 ice_if_intr_enable(if_ctx_t ctx)
1571 {
1572 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1573 	struct ice_vsi *vsi = &sc->pf_vsi;
1574 	struct ice_hw *hw = &sc->hw;
1575 
1576 	ASSERT_CTX_LOCKED(sc);
1577 
1578 	/* Enable ITR 0 */
1579 	ice_enable_intr(hw, sc->irqvs[0].me);
1580 
1581 	/* Do not enable queue interrupts in recovery mode */
1582 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1583 		return;
1584 
1585 	/* Enable all queue interrupts */
1586 	for (int i = 0; i < vsi->num_rx_queues; i++)
1587 		ice_enable_intr(hw, vsi->rx_queues[i].irqv->me);
1588 }
1589 
1590 /**
1591  * ice_if_intr_disable - Disable device interrupts
1592  * @ctx: iflib context structure
1593  *
1594  * Called by iflib to request disabling device interrupts.
1595  */
1596 static void
1597 ice_if_intr_disable(if_ctx_t ctx)
1598 {
1599 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1600 	struct ice_hw *hw = &sc->hw;
1601 	unsigned int i;
1602 
1603 	ASSERT_CTX_LOCKED(sc);
1604 
1605 	/* IFDI_INTR_DISABLE may be called prior to interrupts actually being
1606 	 * assigned to queues. Instead of assuming that the interrupt
1607 	 * assignment in the rx_queues structure is valid, just disable all
1608 	 * possible interrupts
1609 	 *
1610 	 * Note that we choose not to disable ITR 0 because this handles the
1611 	 * AdminQ interrupts, and we want to keep processing these even when
1612 	 * the interface is offline.
1613 	 */
1614 	for (i = 1; i < hw->func_caps.common_cap.num_msix_vectors; i++)
1615 		ice_disable_intr(hw, i);
1616 }
1617 
1618 /**
1619  * ice_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt
1620  * @ctx: iflib context structure
1621  * @rxqid: the Rx queue to enable
1622  *
1623  * Enable a specific Rx queue interrupt.
1624  *
1625  * This function is not protected by the iflib CTX lock.
1626  */
1627 static int
1628 ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
1629 {
1630 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1631 	struct ice_vsi *vsi = &sc->pf_vsi;
1632 	struct ice_hw *hw = &sc->hw;
1633 
1634 	/* Do not enable queue interrupts in recovery mode */
1635 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1636 		return (ENOSYS);
1637 
1638 	ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me);
1639 	return (0);
1640 }
1641 
1642 /**
1643  * ice_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt
1644  * @ctx: iflib context structure
1645  * @txqid: the Tx queue to enable
1646  *
1647  * Enable a specific Tx queue interrupt.
1648  *
1649  * This function is not protected by the iflib CTX lock.
1650  */
1651 static int
1652 ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid)
1653 {
1654 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1655 	struct ice_vsi *vsi = &sc->pf_vsi;
1656 	struct ice_hw *hw = &sc->hw;
1657 
1658 	/* Do not enable queue interrupts in recovery mode */
1659 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1660 		return (ENOSYS);
1661 
1662 	ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me);
1663 	return (0);
1664 }
1665 
1666 /**
1667  * ice_if_promisc_set - Set device promiscuous mode
1668  * @ctx: iflib context structure
1669  * @flags: promiscuous flags to configure
1670  *
1671  * Called by iflib to configure device promiscuous mode.
1672  *
1673  * @remark Calls to this function will always overwrite the previous setting
1674  */
1675 static int
1676 ice_if_promisc_set(if_ctx_t ctx, int flags)
1677 {
1678 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1679 	struct ice_hw *hw = &sc->hw;
1680 	device_t dev = sc->dev;
1681 	enum ice_status status;
1682 	bool promisc_enable = flags & IFF_PROMISC;
1683 	bool multi_enable = flags & IFF_ALLMULTI;
1684 
1685 	/* Do not support configuration when in recovery mode */
1686 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1687 		return (ENOSYS);
1688 
1689 	if (multi_enable)
1690 		return (EOPNOTSUPP);
1691 
1692 	if (promisc_enable) {
1693 		status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx,
1694 					     ICE_VSI_PROMISC_MASK, 0);
1695 		if (status && status != ICE_ERR_ALREADY_EXISTS) {
1696 			device_printf(dev,
1697 				      "Failed to enable promiscuous mode for PF VSI, err %s aq_err %s\n",
1698 				      ice_status_str(status),
1699 				      ice_aq_str(hw->adminq.sq_last_status));
1700 			return (EIO);
1701 		}
1702 	} else {
1703 		status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx,
1704 					       ICE_VSI_PROMISC_MASK, 0);
1705 		if (status) {
1706 			device_printf(dev,
1707 				      "Failed to disable promiscuous mode for PF VSI, err %s aq_err %s\n",
1708 				      ice_status_str(status),
1709 				      ice_aq_str(hw->adminq.sq_last_status));
1710 			return (EIO);
1711 		}
1712 	}
1713 
1714 	return (0);
1715 }
1716 
1717 /**
1718  * ice_if_media_change - Change device media
1719  * @ctx: device ctx structure
1720  *
1721  * Called by iflib when a media change is requested. This operation is not
1722  * supported by the hardware, so we just return an error code.
1723  */
1724 static int
1725 ice_if_media_change(if_ctx_t ctx)
1726 {
1727 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1728 
1729 	device_printf(sc->dev, "Media change is not supported.\n");
1730 	return (ENODEV);
1731 }
1732 
1733 /**
1734  * ice_if_media_status - Report current device media
1735  * @ctx: iflib context structure
1736  * @ifmr: ifmedia request structure to update
1737  *
1738  * Updates the provided ifmr with current device media status, including link
1739  * status and media type.
1740  */
1741 static void
1742 ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
1743 {
1744 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1745 	struct ice_link_status *li = &sc->hw.port_info->phy.link_info;
1746 
1747 	ifmr->ifm_status = IFM_AVALID;
1748 	ifmr->ifm_active = IFM_ETHER;
1749 
1750 	/* Never report link up or media types when in recovery mode */
1751 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1752 		return;
1753 
1754 	if (!sc->link_up)
1755 		return;
1756 
1757 	ifmr->ifm_status |= IFM_ACTIVE;
1758 	ifmr->ifm_active |= IFM_FDX;
1759 
1760 	if (li->phy_type_low)
1761 		ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low);
1762 	else if (li->phy_type_high)
1763 		ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high);
1764 	else
1765 		ifmr->ifm_active |= IFM_UNKNOWN;
1766 
1767 	/* Report flow control status as well */
1768 	if (li->an_info & ICE_AQ_LINK_PAUSE_TX)
1769 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1770 	if (li->an_info & ICE_AQ_LINK_PAUSE_RX)
1771 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1772 }
1773 
1774 /**
1775  * ice_init_tx_tracking - Initialize Tx queue software tracking values
1776  * @vsi: the VSI to initialize
1777  *
1778  * Initialize Tx queue software tracking values, including the Report Status
1779  * queue, and related software tracking values.
1780  */
1781 static void
1782 ice_init_tx_tracking(struct ice_vsi *vsi)
1783 {
1784 	struct ice_tx_queue *txq;
1785 	size_t j;
1786 	int i;
1787 
1788 	for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {
1789 
1790 		txq->tx_rs_cidx = txq->tx_rs_pidx = 0;
1791 
1792 		/* Initialize the last processed descriptor to be the end of
1793 		 * the ring, rather than the start, so that we avoid an
1794 		 * off-by-one error in ice_ift_txd_credits_update for the
1795 		 * first packet.
1796 		 */
1797 		txq->tx_cidx_processed = txq->desc_count - 1;
1798 
1799 		for (j = 0; j < txq->desc_count; j++)
1800 			txq->tx_rsq[j] = QIDX_INVALID;
1801 	}
1802 }
1803 
1804 /**
1805  * ice_update_rx_mbuf_sz - Update the Rx buffer size for all queues
1806  * @sc: the device softc
1807  *
1808  * Called to update the Rx queue mbuf_sz parameter for configuring the receive
1809  * buffer sizes when programming hardware.
1810  */
1811 static void
1812 ice_update_rx_mbuf_sz(struct ice_softc *sc)
1813 {
1814 	uint32_t mbuf_sz = iflib_get_rx_mbuf_sz(sc->ctx);
1815 	struct ice_vsi *vsi = &sc->pf_vsi;
1816 
1817 	MPASS(mbuf_sz <= UINT16_MAX);
1818 	vsi->mbuf_sz = mbuf_sz;
1819 }
1820 
1821 /**
1822  * ice_if_init - Initialize the device
1823  * @ctx: iflib ctx structure
1824  *
1825  * Called by iflib to bring the device up, i.e. ifconfig ice0 up. Initializes
1826  * device filters and prepares the Tx and Rx engines.
1827  *
1828  * @pre assumes the caller holds the iflib CTX lock
1829  */
1830 static void
1831 ice_if_init(if_ctx_t ctx)
1832 {
1833 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1834 	device_t dev = sc->dev;
1835 	int err;
1836 
1837 	ASSERT_CTX_LOCKED(sc);
1838 
1839 	/*
1840 	 * We've seen an issue with 11.3/12.1 where sideband routines are
1841 	 * called after detach is called.  This would call routines after
1842 	 * if_stop, causing issues with the teardown process.  This has
1843 	 * seemingly been fixed in STABLE snapshots, but it seems like a
1844 	 * good idea to have this guard here regardless.
1845 	 */
1846 	if (ice_driver_is_detaching(sc))
1847 		return;
1848 
1849 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1850 		return;
1851 
1852 	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) {
1853 		device_printf(sc->dev, "request to start interface cannot be completed as the device failed to reset\n");
1854 		return;
1855 	}
1856 
1857 	if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
1858 		device_printf(sc->dev, "request to start interface while device is prepared for impending reset\n");
1859 		return;
1860 	}
1861 
1862 	ice_update_rx_mbuf_sz(sc);
1863 
1864 	/* Update the MAC address... User might use a LAA */
1865 	err = ice_update_laa_mac(sc);
1866 	if (err) {
1867 		device_printf(dev,
1868 			      "LAA address change failed, err %s\n",
1869 			      ice_err_str(err));
1870 		return;
1871 	}
1872 
1873 	/* Initialize software Tx tracking values */
1874 	ice_init_tx_tracking(&sc->pf_vsi);
1875 
1876 	err = ice_cfg_vsi_for_tx(&sc->pf_vsi);
1877 	if (err) {
1878 		device_printf(dev,
1879 			      "Unable to configure the main VSI for Tx: %s\n",
1880 			      ice_err_str(err));
1881 		return;
1882 	}
1883 
1884 	err = ice_cfg_vsi_for_rx(&sc->pf_vsi);
1885 	if (err) {
1886 		device_printf(dev,
1887 			      "Unable to configure the main VSI for Rx: %s\n",
1888 			      ice_err_str(err));
1889 		goto err_cleanup_tx;
1890 	}
1891 
1892 	err = ice_control_rx_queues(&sc->pf_vsi, true);
1893 	if (err) {
1894 		device_printf(dev,
1895 			      "Unable to enable Rx rings for transmit: %s\n",
1896 			      ice_err_str(err));
1897 		goto err_cleanup_tx;
1898 	}
1899 
1900 	err = ice_cfg_pf_default_mac_filters(sc);
1901 	if (err) {
1902 		device_printf(dev,
1903 			      "Unable to configure default MAC filters: %s\n",
1904 			      ice_err_str(err));
1905 		goto err_stop_rx;
1906 	}
1907 
1908 	/* We use software interrupts for Tx, so we only program the hardware
1909 	 * interrupts for Rx.
1910 	 */
1911 	ice_configure_rxq_interrupts(&sc->pf_vsi);
1912 	ice_configure_rx_itr(&sc->pf_vsi);
1913 
1914 	/* Configure promiscuous mode */
1915 	ice_if_promisc_set(ctx, if_getflags(sc->ifp));
1916 
1917 	ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED);
1918 	return;
1919 
1920 err_stop_rx:
1921 	ice_control_rx_queues(&sc->pf_vsi, false);
1922 err_cleanup_tx:
1923 	ice_vsi_disable_tx(&sc->pf_vsi);
1924 }
1925 
1926 /**
1927  * ice_poll_for_media_avail - Re-enable link if media is detected
1928  * @sc: device private structure
1929  *
1930  * Intended to be called from the driver's timer function, this function
1931  * sends the Get Link Status AQ command and re-enables HW link if the
1932  * command says that media is available.
1933  *
1934  * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing,
1935  * since media removal events are supposed to be sent to the driver through
1936  * a link status event.
1937  */
1938 static void
1939 ice_poll_for_media_avail(struct ice_softc *sc)
1940 {
1941 	struct ice_hw *hw = &sc->hw;
1942 	struct ice_port_info *pi = hw->port_info;
1943 
1944 	if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) {
1945 		pi->phy.get_link_info = true;
1946 		ice_get_link_status(pi, &sc->link_up);
1947 
1948 		if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
1949 			enum ice_status status;
1950 
1951 			/* Re-enable link and re-apply user link settings */
1952 			ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC);
1953 
1954 			/* Update the OS about changes in media capability */
1955 			status = ice_add_media_types(sc, sc->media);
1956 			if (status)
1957 				device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n",
1958 					      ice_status_str(status),
1959 					      ice_aq_str(hw->adminq.sq_last_status));
1960 
1961 			ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA);
1962 		}
1963 	}
1964 }
1965 
1966 /**
1967  * ice_if_timer - called by iflib periodically
1968  * @ctx: iflib ctx structure
1969  * @qid: the queue this timer was called for
1970  *
1971  * This callback is triggered by iflib periodically. We use it to update the
1972  * hw statistics.
1973  *
1974  * @remark this function is not protected by the iflib CTX lock.
1975  */
1976 static void
1977 ice_if_timer(if_ctx_t ctx, uint16_t qid)
1978 {
1979 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1980 	uint64_t prev_link_xoff_rx = sc->stats.cur.link_xoff_rx;
1981 
1982 	if (qid != 0)
1983 		return;
1984 
1985 	/* Do not attempt to update stats when in recovery mode */
1986 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1987 		return;
1988 
1989 	/* Update device statistics */
1990 	ice_update_pf_stats(sc);
1991 
1992 	/*
1993 	 * For proper watchdog management, the iflib stack needs to know if
1994 	 * we've been paused during the last interval. Check if the
1995 	 * link_xoff_rx stat changed, and set the isc_pause_frames, if so.
1996 	 */
1997 	if (sc->stats.cur.link_xoff_rx != prev_link_xoff_rx)
1998 		sc->scctx->isc_pause_frames = 1;
1999 
2000 	/* Update the primary VSI stats */
2001 	ice_update_vsi_hw_stats(&sc->pf_vsi);
2002 }
2003 
2004 /**
2005  * ice_admin_timer - called periodically to trigger the admin task
2006  * @arg: callout(9) argument pointing to the device private softc structure
2007  *
2008  * Timer function used as part of a callout(9) timer that will periodically
2009  * trigger the admin task, even when the interface is down.
2010  *
2011  * @remark this function is not called by iflib and is not protected by the
2012  * iflib CTX lock.
2013  *
2014  * @remark because this is a callout function, it cannot sleep and should not
2015  * attempt taking the iflib CTX lock.
2016  */
2017 static void
2018 ice_admin_timer(void *arg)
2019 {
2020 	struct ice_softc *sc = (struct ice_softc *)arg;
2021 
2022 	/*
2023 	 * There is a point where callout routines are no longer
2024 	 * cancelable.  So there exists a window of time where the
2025 	 * driver enters detach() and tries to cancel the callout, but the
2026 	 * callout routine has passed the cancellation point.  The detach()
2027 	 * routine is unaware of this and tries to free resources that the
2028 	 * callout routine needs.  So we check for the detach state flag to
2029 	 * at least shrink the window of opportunity.
2030 	 */
2031 	if (ice_driver_is_detaching(sc))
2032 		return;
2033 
2034 	/* Fire off the admin task */
2035 	iflib_admin_intr_deferred(sc->ctx);
2036 
2037 	/* Reschedule the admin timer */
2038 	callout_schedule(&sc->admin_timer, hz/2);
2039 }
2040 
2041 /**
2042  * ice_transition_recovery_mode - Transition to recovery mode
2043  * @sc: the device private softc
2044  *
2045  * Called when the driver detects that the firmware has entered recovery mode
2046  * at run time.
2047  */
2048 static void
2049 ice_transition_recovery_mode(struct ice_softc *sc)
2050 {
2051 	struct ice_vsi *vsi = &sc->pf_vsi;
2052 	int i;
2053 
2054 	device_printf(sc->dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
2055 
2056 	/* Tell the stack that the link has gone down */
2057 	iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0);
2058 
2059 	/* Request that the device be re-initialized */
2060 	ice_request_stack_reinit(sc);
2061 
2062 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
2063 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2064 
2065 	ice_vsi_del_txqs_ctx(vsi);
2066 	ice_vsi_del_rxqs_ctx(vsi);
2067 
2068 	for (i = 0; i < sc->num_available_vsi; i++) {
2069 		if (sc->all_vsi[i])
2070 			ice_release_vsi(sc->all_vsi[i]);
2071 	}
2072 	sc->num_available_vsi = 0;
2073 
2074 	if (sc->all_vsi) {
2075 		free(sc->all_vsi, M_ICE);
2076 		sc->all_vsi = NULL;
2077 	}
2078 
2079 	/* Destroy the interrupt manager */
2080 	ice_resmgr_destroy(&sc->imgr);
2081 	/* Destroy the queue managers */
2082 	ice_resmgr_destroy(&sc->tx_qmgr);
2083 	ice_resmgr_destroy(&sc->rx_qmgr);
2084 
2085 	ice_deinit_hw(&sc->hw);
2086 }
2087 
2088 /**
2089  * ice_transition_safe_mode - Transition to safe mode
2090  * @sc: the device private softc
2091  *
2092  * Called when the driver attempts to reload the DDP package during a device
2093  * reset, and the new download fails. If so, we must transition to safe mode
2094  * at run time.
2095  *
2096  * @remark although safe mode normally allocates only a single queue, we can't
2097  * change the number of queues dynamically when using iflib. Due to this, we
2098  * do not attempt to reduce the number of queues.
2099  */
2100 static void
2101 ice_transition_safe_mode(struct ice_softc *sc)
2102 {
2103 	/* Indicate that we are in Safe mode */
2104 	ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap);
2105 	ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en);
2106 
2107 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
2108 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2109 
2110 	ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
2111 	ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en);
2112 }
2113 
2114 /**
2115  * ice_if_update_admin_status - update admin status
2116  * @ctx: iflib ctx structure
2117  *
2118  * Called by iflib to update the admin status. For our purposes, this means
2119  * check the adminq, and update the link status. It's ultimately triggered by
2120  * our admin interrupt, or by the ice_if_timer periodically.
2121  *
2122  * @pre assumes the caller holds the iflib CTX lock
2123  */
2124 static void
2125 ice_if_update_admin_status(if_ctx_t ctx)
2126 {
2127 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2128 	enum ice_fw_modes fw_mode;
2129 	bool reschedule = false;
2130 	u16 pending = 0;
2131 
2132 	ASSERT_CTX_LOCKED(sc);
2133 
2134 	/* Check if the firmware entered recovery mode at run time */
2135 	fw_mode = ice_get_fw_mode(&sc->hw);
2136 	if (fw_mode == ICE_FW_MODE_REC) {
2137 		if (!ice_testandset_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2138 			/* If we just entered recovery mode, log a warning to
2139 			 * the system administrator and deinit driver state
2140 			 * that is no longer functional.
2141 			 */
2142 			ice_transition_recovery_mode(sc);
2143 		}
2144 	} else if (fw_mode == ICE_FW_MODE_ROLLBACK) {
2145 		if (!ice_testandset_state(&sc->state, ICE_STATE_ROLLBACK_MODE)) {
2146 			/* Rollback mode isn't fatal, but we don't want to
2147 			 * repeatedly post a message about it.
2148 			 */
2149 			ice_print_rollback_msg(&sc->hw);
2150 		}
2151 	}
2152 
2153 	/* Handle global reset events */
2154 	ice_handle_reset_event(sc);
2155 
2156 	/* Handle PF reset requests */
2157 	ice_handle_pf_reset_request(sc);
2158 
2159 	/* Handle MDD events */
2160 	ice_handle_mdd_event(sc);
2161 
2162 	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) ||
2163 	    ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) ||
2164 	    ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2165 		/*
2166 		 * If we know the control queues are disabled, skip processing
2167 		 * the control queues entirely.
2168 		 */
2169 		;
2170 	} else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) {
2171 		ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending);
2172 		if (pending > 0)
2173 			reschedule = true;
2174 
2175 		ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending);
2176 		if (pending > 0)
2177 			reschedule = true;
2178 	}
2179 
2180 	/* Poll for link up */
2181 	ice_poll_for_media_avail(sc);
2182 
2183 	/* Check and update link status */
2184 	ice_update_link_status(sc, false);
2185 
2186 	/*
2187 	 * If there are still messages to process, we need to reschedule
2188 	 * ourselves. Otherwise, we can just re-enable the interrupt. We'll be
2189 	 * woken up at the next interrupt or timer event.
2190 	 */
2191 	if (reschedule) {
2192 		ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
2193 		iflib_admin_intr_deferred(ctx);
2194 	} else {
2195 		ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2196 	}
2197 }
2198 
2199 /**
2200  * ice_prepare_for_reset - Prepare device for an impending reset
2201  * @sc: The device private softc
2202  *
2203  * Prepare the driver for an impending reset, shutting down VSIs, clearing the
2204  * scheduler setup, and shutting down controlqs. Uses the
2205  * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the
2206  * driver for reset or not.
2207  */
2208 static void
2209 ice_prepare_for_reset(struct ice_softc *sc)
2210 {
2211 	struct ice_hw *hw = &sc->hw;
2212 
2213 	/* If we're already prepared, there's nothing to do */
2214 	if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET))
2215 		return;
2216 
2217 	log(LOG_INFO, "%s: preparing to reset device logic\n", sc->ifp->if_xname);
2218 
2219 	/* In recovery mode, hardware is not initialized */
2220 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2221 		return;
2222 
2223 	/* Release the main PF VSI queue mappings */
2224 	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2225 				    sc->pf_vsi.num_tx_queues);
2226 	ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
2227 				    sc->pf_vsi.num_rx_queues);
2228 
2229 	ice_clear_hw_tbls(hw);
2230 
2231 	if (hw->port_info)
2232 		ice_sched_clear_port(hw->port_info);
2233 
2234 	ice_shutdown_all_ctrlq(hw);
2235 }
2236 
2237 /**
2238  * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping
2239  * @sc: the device softc pointer
2240  *
2241  * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue
2242  * mapping after a reset occurred.
2243  */
2244 static int
2245 ice_rebuild_pf_vsi_qmap(struct ice_softc *sc)
2246 {
2247 	struct ice_vsi *vsi = &sc->pf_vsi;
2248 	struct ice_tx_queue *txq;
2249 	struct ice_rx_queue *rxq;
2250 	int err, i;
2251 
2252 	/* Re-assign Tx queues from PF space to the main VSI */
2253 	err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap,
2254 					    vsi->num_tx_queues);
2255 	if (err) {
2256 		device_printf(sc->dev, "Unable to re-assign PF Tx queues: %s\n",
2257 			      ice_err_str(err));
2258 		return (err);
2259 	}
2260 
2261 	/* Re-assign Rx queues from PF space to this VSI */
2262 	err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap,
2263 					    vsi->num_rx_queues);
2264 	if (err) {
2265 		device_printf(sc->dev, "Unable to re-assign PF Rx queues: %s\n",
2266 			      ice_err_str(err));
2267 		goto err_release_tx_queues;
2268 	}
2269 
2270 	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
2271 
2272 	/* Re-assign Tx queue tail pointers */
2273 	for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++)
2274 		txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);
2275 
2276 	/* Re-assign Rx queue tail pointers */
2277 	for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++)
2278 		rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);
2279 
2280 	return (0);
2281 
2282 err_release_tx_queues:
2283 	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2284 				   sc->pf_vsi.num_tx_queues);
2285 
2286 	return (err);
2287 }
2288 
2289 /* determine if the iflib context is active */
2290 #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING))
2291 
2292 /**
2293  * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode
2294  * @sc: The device private softc
2295  *
2296  * Handle a driver rebuild while in recovery mode. This will only rebuild the
2297  * limited functionality supported while in recovery mode.
2298  */
2299 static void
2300 ice_rebuild_recovery_mode(struct ice_softc *sc)
2301 {
2302 	device_t dev = sc->dev;
2303 
2304 	/* enable PCIe bus master */
2305 	pci_enable_busmaster(dev);
2306 
2307 	/* Configure interrupt causes for the administrative interrupt */
2308 	ice_configure_misc_interrupts(sc);
2309 
2310 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
2311 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2312 
2313 	/* Now that the rebuild is finished, we're no longer prepared to reset */
2314 	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
2315 
2316 	log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname);
2317 
2318 	/* In order to completely restore device functionality, the iflib core
2319 	 * needs to be reset. We need to request an iflib reset. Additionally,
2320 	 * because the state of IFC_DO_RESET is cached within task_fn_admin in
2321 	 * the iflib core, we also want re-run the admin task so that iflib
2322 	 * resets immediately instead of waiting for the next interrupt.
2323 	 */
2324 	ice_request_stack_reinit(sc);
2325 
2326 	return;
2327 }
2328 
2329 /**
2330  * ice_rebuild - Rebuild driver state post reset
2331  * @sc: The device private softc
2332  *
2333  * Restore driver state after a reset occurred. Restart the controlqs, setup
2334  * the hardware port, and re-enable the VSIs.
2335  */
2336 static void
2337 ice_rebuild(struct ice_softc *sc)
2338 {
2339 	struct ice_hw *hw = &sc->hw;
2340 	device_t dev = sc->dev;
2341 	enum ice_status status;
2342 	int err;
2343 
2344 	sc->rebuild_ticks = ticks;
2345 
2346 	/* If we're rebuilding, then a reset has succeeded. */
2347 	ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED);
2348 
2349 	/*
2350 	 * If the firmware is in recovery mode, only restore the limited
2351 	 * functionality supported by recovery mode.
2352 	 */
2353 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2354 		ice_rebuild_recovery_mode(sc);
2355 		return;
2356 	}
2357 
2358 	/* enable PCIe bus master */
2359 	pci_enable_busmaster(dev);
2360 
2361 	status = ice_init_all_ctrlq(hw);
2362 	if (status) {
2363 		device_printf(dev, "failed to re-init controlqs, err %s\n",
2364 			      ice_status_str(status));
2365 		goto err_shutdown_ctrlq;
2366 	}
2367 
2368 	/* Query the allocated resources for Tx scheduler */
2369 	status = ice_sched_query_res_alloc(hw);
2370 	if (status) {
2371 		device_printf(dev,
2372 			      "Failed to query scheduler resources, err %s aq_err %s\n",
2373 			      ice_status_str(status),
2374 			      ice_aq_str(hw->adminq.sq_last_status));
2375 		goto err_shutdown_ctrlq;
2376 	}
2377 
2378 	err = ice_send_version(sc);
2379 	if (err)
2380 		goto err_shutdown_ctrlq;
2381 
2382 	err = ice_init_link_events(sc);
2383 	if (err) {
2384 		device_printf(dev, "ice_init_link_events failed: %s\n",
2385 			      ice_err_str(err));
2386 		goto err_shutdown_ctrlq;
2387 	}
2388 
2389 	status = ice_clear_pf_cfg(hw);
2390 	if (status) {
2391 		device_printf(dev, "failed to clear PF configuration, err %s\n",
2392 			      ice_status_str(status));
2393 		goto err_shutdown_ctrlq;
2394 	}
2395 
2396 	ice_clear_pxe_mode(hw);
2397 
2398 	status = ice_get_caps(hw);
2399 	if (status) {
2400 		device_printf(dev, "failed to get capabilities, err %s\n",
2401 			      ice_status_str(status));
2402 		goto err_shutdown_ctrlq;
2403 	}
2404 
2405 	status = ice_sched_init_port(hw->port_info);
2406 	if (status) {
2407 		device_printf(dev, "failed to initialize port, err %s\n",
2408 			      ice_status_str(status));
2409 		goto err_sched_cleanup;
2410 	}
2411 
2412 	/* If we previously loaded the package, it needs to be reloaded now */
2413 	if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) {
2414 		status = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size);
2415 		if (status) {
2416 			ice_log_pkg_init(sc, &status);
2417 
2418 			ice_transition_safe_mode(sc);
2419 		}
2420 	}
2421 
2422 	ice_reset_pf_stats(sc);
2423 
2424 	err = ice_rebuild_pf_vsi_qmap(sc);
2425 	if (err) {
2426 		device_printf(sc->dev, "Unable to re-assign main VSI queues, err %s\n",
2427 			      ice_err_str(err));
2428 		goto err_sched_cleanup;
2429 	}
2430 	err = ice_initialize_vsi(&sc->pf_vsi);
2431 	if (err) {
2432 		device_printf(sc->dev, "Unable to re-initialize Main VSI, err %s\n",
2433 			      ice_err_str(err));
2434 		goto err_release_queue_allocations;
2435 	}
2436 
2437 	/* Replay all VSI configuration */
2438 	err = ice_replay_all_vsi_cfg(sc);
2439 	if (err)
2440 		goto err_deinit_pf_vsi;
2441 
2442 	/* Re-enable FW health event reporting */
2443 	ice_init_health_events(sc);
2444 
2445 	/* Reconfigure the main PF VSI for RSS */
2446 	err = ice_config_rss(&sc->pf_vsi);
2447 	if (err) {
2448 		device_printf(sc->dev,
2449 			      "Unable to reconfigure RSS for the main VSI, err %s\n",
2450 			      ice_err_str(err));
2451 		goto err_deinit_pf_vsi;
2452 	}
2453 
2454 	/* Refresh link status */
2455 	ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED);
2456 	sc->hw.port_info->phy.get_link_info = true;
2457 	ice_get_link_status(sc->hw.port_info, &sc->link_up);
2458 	ice_update_link_status(sc, true);
2459 
2460 	/* Configure interrupt causes for the administrative interrupt */
2461 	ice_configure_misc_interrupts(sc);
2462 
2463 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
2464 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2465 
2466 	/* Now that the rebuild is finished, we're no longer prepared to reset */
2467 	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
2468 
2469 	log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname);
2470 
2471 	/* In order to completely restore device functionality, the iflib core
2472 	 * needs to be reset. We need to request an iflib reset. Additionally,
2473 	 * because the state of IFC_DO_RESET is cached within task_fn_admin in
2474 	 * the iflib core, we also want re-run the admin task so that iflib
2475 	 * resets immediately instead of waiting for the next interrupt.
2476 	 */
2477 	ice_request_stack_reinit(sc);
2478 
2479 	return;
2480 
2481 err_deinit_pf_vsi:
2482 	ice_deinit_vsi(&sc->pf_vsi);
2483 err_release_queue_allocations:
2484 	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2485 				    sc->pf_vsi.num_tx_queues);
2486 	ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
2487 				    sc->pf_vsi.num_rx_queues);
2488 err_sched_cleanup:
2489 	ice_sched_cleanup_all(hw);
2490 err_shutdown_ctrlq:
2491 	ice_shutdown_all_ctrlq(hw);
2492 	ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2493 	device_printf(dev, "Driver rebuild failed, please reload the device driver\n");
2494 }
2495 
2496 /**
2497  * ice_handle_reset_event - Handle reset events triggered by OICR
2498  * @sc: The device private softc
2499  *
2500  * Handle reset events triggered by an OICR notification. This includes CORER,
2501  * GLOBR, and EMPR resets triggered by software on this or any other PF or by
2502  * firmware.
2503  *
2504  * @pre assumes the iflib context lock is held, and will unlock it while
2505  * waiting for the hardware to finish reset.
2506  */
2507 static void
2508 ice_handle_reset_event(struct ice_softc *sc)
2509 {
2510 	struct ice_hw *hw = &sc->hw;
2511 	enum ice_status status;
2512 	device_t dev = sc->dev;
2513 
2514 	/* When a CORER, GLOBR, or EMPR is about to happen, the hardware will
2515 	 * trigger an OICR interrupt. Our OICR handler will determine when
2516 	 * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as
2517 	 * appropriate.
2518 	 */
2519 	if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV))
2520 		return;
2521 
2522 	ice_prepare_for_reset(sc);
2523 
2524 	/*
2525 	 * Release the iflib context lock and wait for the device to finish
2526 	 * resetting.
2527 	 */
2528 	IFLIB_CTX_UNLOCK(sc);
2529 	status = ice_check_reset(hw);
2530 	IFLIB_CTX_LOCK(sc);
2531 	if (status) {
2532 		device_printf(dev, "Device never came out of reset, err %s\n",
2533 			      ice_status_str(status));
2534 		ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2535 		return;
2536 	}
2537 
2538 	/* We're done with the reset, so we can rebuild driver state */
2539 	sc->hw.reset_ongoing = false;
2540 	ice_rebuild(sc);
2541 
2542 	/* In the unlikely event that a PF reset request occurs at the same
2543 	 * time as a global reset, clear the request now. This avoids
2544 	 * resetting a second time right after we reset due to a global event.
2545 	 */
2546 	if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
2547 		device_printf(dev, "Ignoring PFR request that occurred while a reset was ongoing\n");
2548 }
2549 
2550 /**
2551  * ice_handle_pf_reset_request - Initiate PF reset requested by software
2552  * @sc: The device private softc
2553  *
2554  * Initiate a PF reset requested by software. We handle this in the admin task
2555  * so that only one thread actually handles driver preparation and cleanup,
2556  * rather than having multiple threads possibly attempt to run this code
2557  * simultaneously.
2558  *
2559  * @pre assumes the iflib context lock is held and will unlock it while
2560  * waiting for the PF reset to complete.
2561  */
2562 static void
2563 ice_handle_pf_reset_request(struct ice_softc *sc)
2564 {
2565 	struct ice_hw *hw = &sc->hw;
2566 	enum ice_status status;
2567 
2568 	/* Check for PF reset requests */
2569 	if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
2570 		return;
2571 
2572 	/* Make sure we're prepared for reset */
2573 	ice_prepare_for_reset(sc);
2574 
2575 	/*
2576 	 * Release the iflib context lock and wait for the device to finish
2577 	 * resetting.
2578 	 */
2579 	IFLIB_CTX_UNLOCK(sc);
2580 	status = ice_reset(hw, ICE_RESET_PFR);
2581 	IFLIB_CTX_LOCK(sc);
2582 	if (status) {
2583 		device_printf(sc->dev, "device PF reset failed, err %s\n",
2584 			      ice_status_str(status));
2585 		ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2586 		return;
2587 	}
2588 
2589 	sc->soft_stats.pfr_count++;
2590 	ice_rebuild(sc);
2591 }
2592 
2593 /**
2594  * ice_init_device_features - Init device driver features
2595  * @sc: driver softc structure
2596  *
2597  * @pre assumes that the function capabilities bits have been set up by
2598  * ice_init_hw().
2599  */
2600 static void
2601 ice_init_device_features(struct ice_softc *sc)
2602 {
2603 	/*
2604 	 * A failed pkg file download triggers safe mode, disabling advanced
2605 	 * device feature support
2606 	 */
2607 	if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE))
2608 		return;
2609 
2610 	/* Set capabilities that all devices support */
2611 	ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2612 	ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap);
2613 	ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap);
2614 	ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_cap);
2615 	ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_cap);
2616 	ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap);
2617 
2618 	/* Disable features due to hardware limitations... */
2619 	if (!sc->hw.func_caps.common_cap.rss_table_size)
2620 		ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
2621 	/* Disable features due to firmware limitations... */
2622 	if (!ice_is_fw_health_report_supported(&sc->hw))
2623 		ice_clear_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap);
2624 
2625 	/* Disable capabilities not supported by the OS */
2626 	ice_disable_unsupported_features(sc->feat_cap);
2627 
2628 	/* RSS is always enabled for iflib */
2629 	if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS))
2630 		ice_set_bit(ICE_FEATURE_RSS, sc->feat_en);
2631 }
2632 
2633 /**
2634  * ice_if_multi_set - Callback to update Multicast filters in HW
2635  * @ctx: iflib ctx structure
2636  *
2637  * Called by iflib in response to SIOCDELMULTI and SIOCADDMULTI. Must search
2638  * the if_multiaddrs list and determine which filters have been added or
2639  * removed from the list, and update HW programming to reflect the new list.
2640  *
2641  * @pre assumes the caller holds the iflib CTX lock
2642  */
2643 static void
2644 ice_if_multi_set(if_ctx_t ctx)
2645 {
2646 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2647 	int err;
2648 
2649 	ASSERT_CTX_LOCKED(sc);
2650 
2651 	/* Do not handle multicast configuration in recovery mode */
2652 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2653 		return;
2654 
2655 	err = ice_sync_multicast_filters(sc);
2656 	if (err) {
2657 		device_printf(sc->dev,
2658 			      "Failed to synchronize multicast filter list: %s\n",
2659 			      ice_err_str(err));
2660 		return;
2661 	}
2662 }
2663 
2664 /**
2665  * ice_if_vlan_register - Register a VLAN with the hardware
2666  * @ctx: iflib ctx pointer
2667  * @vtag: VLAN to add
2668  *
2669  * Programs the main PF VSI with a hardware filter for the given VLAN.
2670  *
2671  * @pre assumes the caller holds the iflib CTX lock
2672  */
2673 static void
2674 ice_if_vlan_register(if_ctx_t ctx, u16 vtag)
2675 {
2676 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2677 	enum ice_status status;
2678 
2679 	ASSERT_CTX_LOCKED(sc);
2680 
2681 	/* Do not handle VLAN configuration in recovery mode */
2682 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2683 		return;
2684 
2685 	status = ice_add_vlan_hw_filter(&sc->pf_vsi, vtag);
2686 	if (status) {
2687 		device_printf(sc->dev,
2688 			      "Failure adding VLAN %d to main VSI, err %s aq_err %s\n",
2689 			      vtag, ice_status_str(status),
2690 			      ice_aq_str(sc->hw.adminq.sq_last_status));
2691 	}
2692 }
2693 
2694 /**
2695  * ice_if_vlan_unregister - Remove a VLAN filter from the hardware
2696  * @ctx: iflib ctx pointer
2697  * @vtag: VLAN to add
2698  *
2699  * Removes the previously programmed VLAN filter from the main PF VSI.
2700  *
2701  * @pre assumes the caller holds the iflib CTX lock
2702  */
2703 static void
2704 ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
2705 {
2706 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2707 	enum ice_status status;
2708 
2709 	ASSERT_CTX_LOCKED(sc);
2710 
2711 	/* Do not handle VLAN configuration in recovery mode */
2712 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2713 		return;
2714 
2715 	status = ice_remove_vlan_hw_filter(&sc->pf_vsi, vtag);
2716 	if (status) {
2717 		device_printf(sc->dev,
2718 			      "Failure removing VLAN %d from main VSI, err %s aq_err %s\n",
2719 			      vtag, ice_status_str(status),
2720 			      ice_aq_str(sc->hw.adminq.sq_last_status));
2721 	}
2722 }
2723 
2724 /**
2725  * ice_if_stop - Stop the device
2726  * @ctx: iflib context structure
2727  *
2728  * Called by iflib to stop the device and bring it down. (i.e. ifconfig ice0
2729  * down)
2730  *
2731  * @pre assumes the caller holds the iflib CTX lock
2732  */
2733 static void
2734 ice_if_stop(if_ctx_t ctx)
2735 {
2736 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2737 
2738 	ASSERT_CTX_LOCKED(sc);
2739 
2740 	/*
2741 	 * The iflib core may call IFDI_STOP prior to the first call to
2742 	 * IFDI_INIT. This will cause us to attempt to remove MAC filters we
2743 	 * don't have, and disable Tx queues which aren't yet configured.
2744 	 * Although it is likely these extra operations are harmless, they do
2745 	 * cause spurious warning messages to be displayed, which may confuse
2746 	 * users.
2747 	 *
2748 	 * To avoid these messages, we use a state bit indicating if we've
2749 	 * been initialized. It will be set when ice_if_init is called, and
2750 	 * cleared here in ice_if_stop.
2751 	 */
2752 	if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED))
2753 		return;
2754 
2755 	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) {
2756 		device_printf(sc->dev, "request to stop interface cannot be completed as the device failed to reset\n");
2757 		return;
2758 	}
2759 
2760 	if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
2761 		device_printf(sc->dev, "request to stop interface while device is prepared for impending reset\n");
2762 		return;
2763 	}
2764 
2765 	/* Remove the MAC filters, stop Tx, and stop Rx. We don't check the
2766 	 * return of these functions because there's nothing we can really do
2767 	 * if they fail, and the functions already print error messages.
2768 	 * Just try to shut down as much as we can.
2769 	 */
2770 	ice_rm_pf_default_mac_filters(sc);
2771 
2772 	/* Dissociate the Tx and Rx queues from the interrupts */
2773 	ice_flush_txq_interrupts(&sc->pf_vsi);
2774 	ice_flush_rxq_interrupts(&sc->pf_vsi);
2775 
2776 	/* Disable the Tx and Rx queues */
2777 	ice_vsi_disable_tx(&sc->pf_vsi);
2778 	ice_control_rx_queues(&sc->pf_vsi, false);
2779 }
2780 
2781 /**
2782  * ice_if_get_counter - Get current value of an ifnet statistic
2783  * @ctx: iflib context pointer
2784  * @counter: ifnet counter to read
2785  *
2786  * Reads the current value of an ifnet counter for the device.
2787  *
2788  * This function is not protected by the iflib CTX lock.
2789  */
2790 static uint64_t
2791 ice_if_get_counter(if_ctx_t ctx, ift_counter counter)
2792 {
2793 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2794 
2795 	/* Return the counter for the main PF VSI */
2796 	return ice_get_ifnet_counter(&sc->pf_vsi, counter);
2797 }
2798 
2799 /**
2800  * ice_request_stack_reinit - Request that iflib re-initialize
2801  * @sc: the device private softc
2802  *
2803  * Request that the device be brought down and up, to re-initialize. For
2804  * example, this may be called when a device reset occurs, or when Tx and Rx
2805  * queues need to be re-initialized.
2806  *
2807  * This is required because the iflib state is outside the driver, and must be
2808  * re-initialized if we need to resart Tx and Rx queues.
2809  */
2810 void
2811 ice_request_stack_reinit(struct ice_softc *sc)
2812 {
2813 	if (CTX_ACTIVE(sc->ctx)) {
2814 		iflib_request_reset(sc->ctx);
2815 		iflib_admin_intr_deferred(sc->ctx);
2816 	}
2817 }
2818 
2819 /**
2820  * ice_driver_is_detaching - Check if the driver is detaching/unloading
2821  * @sc: device private softc
2822  *
2823  * Returns true if the driver is detaching, false otherwise.
2824  *
2825  * @remark on newer kernels, take advantage of iflib_in_detach in order to
2826  * report detachment correctly as early as possible.
2827  *
2828  * @remark this function is used by various code paths that want to avoid
2829  * running if the driver is about to be removed. This includes sysctls and
2830  * other driver access points. Note that it does not fully resolve
2831  * detach-based race conditions as it is possible for a thread to race with
2832  * iflib_in_detach.
2833  */
2834 bool
2835 ice_driver_is_detaching(struct ice_softc *sc)
2836 {
2837 	return (ice_test_state(&sc->state, ICE_STATE_DETACHING) ||
2838 		iflib_in_detach(sc->ctx));
2839 }
2840 
2841 /**
2842  * ice_if_priv_ioctl - Device private ioctl handler
2843  * @ctx: iflib context pointer
2844  * @command: The ioctl command issued
2845  * @data: ioctl specific data
2846  *
2847  * iflib callback for handling custom driver specific ioctls.
2848  *
2849  * @pre Assumes that the iflib context lock is held.
2850  */
2851 static int
2852 ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data)
2853 {
2854 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2855 	struct ifdrv *ifd;
2856 	device_t dev = sc->dev;
2857 
2858 	if (data == NULL)
2859 		return (EINVAL);
2860 
2861 	ASSERT_CTX_LOCKED(sc);
2862 
2863 	/* Make sure the command type is valid */
2864 	switch (command) {
2865 	case SIOCSDRVSPEC:
2866 	case SIOCGDRVSPEC:
2867 		/* Accepted commands */
2868 		break;
2869 	case SIOCGPRIVATE_0:
2870 		/*
2871 		 * Although we do not support this ioctl command, it's
2872 		 * expected that iflib will forward it to the IFDI_PRIV_IOCTL
2873 		 * handler. Do not print a message in this case
2874 		 */
2875 		return (ENOTSUP);
2876 	default:
2877 		/*
2878 		 * If we get a different command for this function, it's
2879 		 * definitely unexpected, so log a message indicating what
2880 		 * command we got for debugging purposes.
2881 		 */
2882 		device_printf(dev, "%s: unexpected ioctl command %08lx\n",
2883 			      __func__, command);
2884 		return (EINVAL);
2885 	}
2886 
2887 	ifd = (struct ifdrv *)data;
2888 
2889 	switch (ifd->ifd_cmd) {
2890 	case ICE_NVM_ACCESS:
2891 		return ice_handle_nvm_access_ioctl(sc, ifd);
2892 	default:
2893 		return EINVAL;
2894 	}
2895 }
2896 
2897 /**
2898  * ice_if_i2c_req - I2C request handler for iflib
2899  * @ctx: iflib context pointer
2900  * @req: The I2C parameters to use
2901  *
2902  * Read from the port's I2C eeprom using the parameters from the ioctl.
2903  *
2904  * @remark The iflib-only part is pretty simple.
2905  */
2906 static int
2907 ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req)
2908 {
2909 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2910 
2911 	return ice_handle_i2c_req(sc, req);
2912 }
2913 
2914 /**
2915  * ice_if_suspend - PCI device suspend handler for iflib
2916  * @ctx: iflib context pointer
2917  *
2918  * Deinitializes the driver and clears HW resources in preparation for
2919  * suspend or an FLR.
2920  *
2921  * @returns 0; this return value is ignored
2922  */
2923 static int
2924 ice_if_suspend(if_ctx_t ctx)
2925 {
2926 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2927 
2928 	/* At least a PFR is always going to happen after this;
2929 	 * either via FLR or during the D3->D0 transition.
2930 	 */
2931 	ice_clear_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
2932 
2933 	ice_prepare_for_reset(sc);
2934 
2935 	return (0);
2936 }
2937 
2938 /**
2939  * ice_if_resume - PCI device resume handler for iflib
2940  * @ctx: iflib context pointer
2941  *
2942  * Reinitializes the driver and the HW after PCI resume or after
2943  * an FLR. An init is performed by iflib after this function is finished.
2944  *
2945  * @returns 0; this return value is ignored
2946  */
2947 static int
2948 ice_if_resume(if_ctx_t ctx)
2949 {
2950 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2951 
2952 	ice_rebuild(sc);
2953 
2954 	return (0);
2955 }
2956 
2957