xref: /freebsd/sys/dev/ice/if_ice_iflib.c (revision 7ef62cebc2f965b0f640263e179276928885e33d)
1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /*  Copyright (c) 2023, Intel Corporation
3  *  All rights reserved.
4  *
5  *  Redistribution and use in source and binary forms, with or without
6  *  modification, are permitted provided that the following conditions are met:
7  *
8  *   1. Redistributions of source code must retain the above copyright notice,
9  *      this list of conditions and the following disclaimer.
10  *
11  *   2. Redistributions in binary form must reproduce the above copyright
12  *      notice, this list of conditions and the following disclaimer in the
13  *      documentation and/or other materials provided with the distribution.
14  *
15  *   3. Neither the name of the Intel Corporation nor the names of its
16  *      contributors may be used to endorse or promote products derived from
17  *      this software without specific prior written permission.
18  *
19  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  *  POSSIBILITY OF SUCH DAMAGE.
30  */
31 /*$FreeBSD$*/
32 
33 /**
34  * @file if_ice_iflib.c
35  * @brief iflib driver implementation
36  *
37  * Contains the main entry point for the iflib driver implementation. It
38  * implements the various ifdi driver methods, and sets up the module and
39  * driver values to load an iflib driver.
40  */
41 
42 #include "ice_iflib.h"
43 #include "ice_drv_info.h"
44 #include "ice_switch.h"
45 #include "ice_sched.h"
46 
47 #include <sys/module.h>
48 #include <sys/sockio.h>
49 #include <sys/smp.h>
50 #include <dev/pci/pcivar.h>
51 #include <dev/pci/pcireg.h>
52 
53 /*
54  * Device method prototypes
55  */
56 
57 static void *ice_register(device_t);
58 static int  ice_if_attach_pre(if_ctx_t);
59 static int  ice_attach_pre_recovery_mode(struct ice_softc *sc);
60 static int  ice_if_attach_post(if_ctx_t);
61 static void ice_attach_post_recovery_mode(struct ice_softc *sc);
62 static int  ice_if_detach(if_ctx_t);
63 static int  ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets);
64 static int  ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
65 static int ice_if_msix_intr_assign(if_ctx_t ctx, int msix);
66 static void ice_if_queues_free(if_ctx_t ctx);
67 static int ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
68 static void ice_if_intr_enable(if_ctx_t ctx);
69 static void ice_if_intr_disable(if_ctx_t ctx);
70 static int ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid);
71 static int ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid);
72 static int ice_if_promisc_set(if_ctx_t ctx, int flags);
73 static void ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr);
74 static int ice_if_media_change(if_ctx_t ctx);
75 static void ice_if_init(if_ctx_t ctx);
76 static void ice_if_timer(if_ctx_t ctx, uint16_t qid);
77 static void ice_if_update_admin_status(if_ctx_t ctx);
78 static void ice_if_multi_set(if_ctx_t ctx);
79 static void ice_if_vlan_register(if_ctx_t ctx, u16 vtag);
80 static void ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag);
81 static void ice_if_stop(if_ctx_t ctx);
82 static uint64_t ice_if_get_counter(if_ctx_t ctx, ift_counter counter);
83 static int ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data);
84 static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req);
85 static int ice_if_suspend(if_ctx_t ctx);
86 static int ice_if_resume(if_ctx_t ctx);
87 
88 static int ice_msix_que(void *arg);
89 static int ice_msix_admin(void *arg);
90 
91 /*
92  * Helper function prototypes
93  */
94 static int ice_pci_mapping(struct ice_softc *sc);
95 static void ice_free_pci_mapping(struct ice_softc *sc);
96 static void ice_update_link_status(struct ice_softc *sc, bool update_media);
97 static void ice_init_device_features(struct ice_softc *sc);
98 static void ice_init_tx_tracking(struct ice_vsi *vsi);
99 static void ice_handle_reset_event(struct ice_softc *sc);
100 static void ice_handle_pf_reset_request(struct ice_softc *sc);
101 static void ice_prepare_for_reset(struct ice_softc *sc);
102 static int ice_rebuild_pf_vsi_qmap(struct ice_softc *sc);
103 static void ice_rebuild(struct ice_softc *sc);
104 static void ice_rebuild_recovery_mode(struct ice_softc *sc);
105 static void ice_free_irqvs(struct ice_softc *sc);
106 static void ice_update_rx_mbuf_sz(struct ice_softc *sc);
107 static void ice_poll_for_media_avail(struct ice_softc *sc);
108 static void ice_setup_scctx(struct ice_softc *sc);
109 static int ice_allocate_msix(struct ice_softc *sc);
110 static void ice_admin_timer(void *arg);
111 static void ice_transition_recovery_mode(struct ice_softc *sc);
112 static void ice_transition_safe_mode(struct ice_softc *sc);
113 
114 /*
115  * Device Interface Declaration
116  */
117 
118 /**
119  * @var ice_methods
120  * @brief ice driver method entry points
121  *
122  * List of device methods implementing the generic device interface used by
123  * the device stack to interact with the ice driver. Since this is an iflib
124  * driver, most of the methods point to the generic iflib implementation.
125  */
126 static device_method_t ice_methods[] = {
127 	/* Device interface */
128 	DEVMETHOD(device_register, ice_register),
129 	DEVMETHOD(device_probe,    iflib_device_probe_vendor),
130 	DEVMETHOD(device_attach,   iflib_device_attach),
131 	DEVMETHOD(device_detach,   iflib_device_detach),
132 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
133 	DEVMETHOD(device_suspend,  iflib_device_suspend),
134 	DEVMETHOD(device_resume,   iflib_device_resume),
135 	DEVMETHOD_END
136 };
137 
138 /**
139  * @var ice_iflib_methods
140  * @brief iflib method entry points
141  *
142  * List of device methods used by the iflib stack to interact with this
143  * driver. These are the real main entry points used to interact with this
144  * driver.
145  */
146 static device_method_t ice_iflib_methods[] = {
147 	DEVMETHOD(ifdi_attach_pre, ice_if_attach_pre),
148 	DEVMETHOD(ifdi_attach_post, ice_if_attach_post),
149 	DEVMETHOD(ifdi_detach, ice_if_detach),
150 	DEVMETHOD(ifdi_tx_queues_alloc, ice_if_tx_queues_alloc),
151 	DEVMETHOD(ifdi_rx_queues_alloc, ice_if_rx_queues_alloc),
152 	DEVMETHOD(ifdi_msix_intr_assign, ice_if_msix_intr_assign),
153 	DEVMETHOD(ifdi_queues_free, ice_if_queues_free),
154 	DEVMETHOD(ifdi_mtu_set, ice_if_mtu_set),
155 	DEVMETHOD(ifdi_intr_enable, ice_if_intr_enable),
156 	DEVMETHOD(ifdi_intr_disable, ice_if_intr_disable),
157 	DEVMETHOD(ifdi_rx_queue_intr_enable, ice_if_rx_queue_intr_enable),
158 	DEVMETHOD(ifdi_tx_queue_intr_enable, ice_if_tx_queue_intr_enable),
159 	DEVMETHOD(ifdi_promisc_set, ice_if_promisc_set),
160 	DEVMETHOD(ifdi_media_status, ice_if_media_status),
161 	DEVMETHOD(ifdi_media_change, ice_if_media_change),
162 	DEVMETHOD(ifdi_init, ice_if_init),
163 	DEVMETHOD(ifdi_stop, ice_if_stop),
164 	DEVMETHOD(ifdi_timer, ice_if_timer),
165 	DEVMETHOD(ifdi_update_admin_status, ice_if_update_admin_status),
166 	DEVMETHOD(ifdi_multi_set, ice_if_multi_set),
167 	DEVMETHOD(ifdi_vlan_register, ice_if_vlan_register),
168 	DEVMETHOD(ifdi_vlan_unregister, ice_if_vlan_unregister),
169 	DEVMETHOD(ifdi_get_counter, ice_if_get_counter),
170 	DEVMETHOD(ifdi_priv_ioctl, ice_if_priv_ioctl),
171 	DEVMETHOD(ifdi_i2c_req, ice_if_i2c_req),
172 	DEVMETHOD(ifdi_suspend, ice_if_suspend),
173 	DEVMETHOD(ifdi_resume, ice_if_resume),
174 	DEVMETHOD_END
175 };
176 
177 /**
178  * @var ice_driver
179  * @brief driver structure for the generic device stack
180  *
181  * driver_t definition used to setup the generic device methods.
182  */
183 static driver_t ice_driver = {
184 	.name = "ice",
185 	.methods = ice_methods,
186 	.size = sizeof(struct ice_softc),
187 };
188 
189 /**
190  * @var ice_iflib_driver
191  * @brief driver structure for the iflib stack
192  *
193  * driver_t definition used to setup the iflib device methods.
194  */
195 static driver_t ice_iflib_driver = {
196 	.name = "ice",
197 	.methods = ice_iflib_methods,
198 	.size = sizeof(struct ice_softc),
199 };
200 
201 extern struct if_txrx ice_txrx;
202 extern struct if_txrx ice_recovery_txrx;
203 
204 /**
205  * @var ice_sctx
206  * @brief ice driver shared context
207  *
208  * Structure defining shared values (context) that is used by all instances of
209  * the device. Primarily used to setup details about how the iflib stack
210  * should treat this driver. Also defines the default, minimum, and maximum
211  * number of descriptors in each ring.
212  */
213 static struct if_shared_ctx ice_sctx = {
214 	.isc_magic = IFLIB_MAGIC,
215 	.isc_q_align = PAGE_SIZE,
216 
217 	.isc_tx_maxsize = ICE_MAX_FRAME_SIZE,
218 	/* We could technically set this as high as ICE_MAX_DMA_SEG_SIZE, but
219 	 * that doesn't make sense since that would be larger than the maximum
220 	 * size of a single packet.
221 	 */
222 	.isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE,
223 
224 	/* XXX: This is only used by iflib to ensure that
225 	 * scctx->isc_tx_tso_size_max + the VLAN header is a valid size.
226 	 */
227 	.isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header),
228 	/* XXX: This is used by iflib to set the number of segments in the TSO
229 	 * DMA tag. However, scctx->isc_tx_tso_segsize_max is used to set the
230 	 * related ifnet parameter.
231 	 */
232 	.isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE,
233 
234 	.isc_rx_maxsize = ICE_MAX_FRAME_SIZE,
235 	.isc_rx_nsegments = ICE_MAX_RX_SEGS,
236 	.isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE,
237 
238 	.isc_nfl = 1,
239 	.isc_ntxqs = 1,
240 	.isc_nrxqs = 1,
241 
242 	.isc_admin_intrcnt = 1,
243 	.isc_vendor_info = ice_vendor_info_array,
244 	.isc_driver_version = __DECONST(char *, ice_driver_version),
245 	.isc_driver = &ice_iflib_driver,
246 
247 	/*
248 	 * IFLIB_NEED_SCRATCH ensures that mbufs have scratch space available
249 	 * for hardware checksum offload
250 	 *
251 	 * IFLIB_TSO_INIT_IP ensures that the TSO packets have zeroed out the
252 	 * IP sum field, required by our hardware to calculate valid TSO
253 	 * checksums.
254 	 *
255 	 * IFLIB_ADMIN_ALWAYS_RUN ensures that the administrative task runs
256 	 * even when the interface is down.
257 	 *
258 	 * IFLIB_SKIP_MSIX allows the driver to handle allocating MSI-X
259 	 * vectors manually instead of relying on iflib code to do this.
260 	 */
261 	.isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP |
262 		IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX,
263 
264 	.isc_nrxd_min = {ICE_MIN_DESC_COUNT},
265 	.isc_ntxd_min = {ICE_MIN_DESC_COUNT},
266 	.isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT},
267 	.isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT},
268 	.isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT},
269 	.isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT},
270 };
271 
272 DRIVER_MODULE(ice, pci, ice_driver, ice_module_event_handler, NULL);
273 
274 MODULE_VERSION(ice, 1);
275 MODULE_DEPEND(ice, pci, 1, 1, 1);
276 MODULE_DEPEND(ice, ether, 1, 1, 1);
277 MODULE_DEPEND(ice, iflib, 1, 1, 1);
278 
279 IFLIB_PNP_INFO(pci, ice, ice_vendor_info_array);
280 
281 /* Static driver-wide sysctls */
282 #include "ice_iflib_sysctls.h"
283 
284 /**
285  * ice_pci_mapping - Map PCI BAR memory
286  * @sc: device private softc
287  *
288  * Map PCI BAR 0 for device operation.
289  */
290 static int
291 ice_pci_mapping(struct ice_softc *sc)
292 {
293 	int rc;
294 
295 	/* Map BAR0 */
296 	rc = ice_map_bar(sc->dev, &sc->bar0, 0);
297 	if (rc)
298 		return rc;
299 
300 	return 0;
301 }
302 
303 /**
304  * ice_free_pci_mapping - Release PCI BAR memory
305  * @sc: device private softc
306  *
307  * Release PCI BARs which were previously mapped by ice_pci_mapping().
308  */
309 static void
310 ice_free_pci_mapping(struct ice_softc *sc)
311 {
312 	/* Free BAR0 */
313 	ice_free_bar(sc->dev, &sc->bar0);
314 }
315 
316 /*
317  * Device methods
318  */
319 
320 /**
321  * ice_register - register device method callback
322  * @dev: the device being registered
323  *
324  * Returns a pointer to the shared context structure, which is used by iflib.
325  */
326 static void *
327 ice_register(device_t dev __unused)
328 {
329 	return &ice_sctx;
330 } /* ice_register */
331 
332 /**
333  * ice_setup_scctx - Setup the iflib softc context structure
334  * @sc: the device private structure
335  *
336  * Setup the parameters in if_softc_ctx_t structure used by the iflib stack
337  * when loading.
338  */
339 static void
340 ice_setup_scctx(struct ice_softc *sc)
341 {
342 	if_softc_ctx_t scctx = sc->scctx;
343 	struct ice_hw *hw = &sc->hw;
344 	bool safe_mode, recovery_mode;
345 
346 	safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE);
347 	recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE);
348 
349 	/*
350 	 * If the driver loads in Safe mode or Recovery mode, limit iflib to
351 	 * a single queue pair.
352 	 */
353 	if (safe_mode || recovery_mode) {
354 		scctx->isc_ntxqsets = scctx->isc_nrxqsets = 1;
355 		scctx->isc_ntxqsets_max = 1;
356 		scctx->isc_nrxqsets_max = 1;
357 	} else {
358 		/*
359 		 * iflib initially sets the isc_ntxqsets and isc_nrxqsets to
360 		 * the values of the override sysctls. Cache these initial
361 		 * values so that the driver can be aware of what the iflib
362 		 * sysctl value is when setting up MSI-X vectors.
363 		 */
364 		sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets;
365 		sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets;
366 
367 		if (scctx->isc_ntxqsets == 0)
368 			scctx->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size;
369 		if (scctx->isc_nrxqsets == 0)
370 			scctx->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size;
371 
372 		scctx->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq;
373 		scctx->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq;
374 
375 		/*
376 		 * Sanity check that the iflib sysctl values are within the
377 		 * maximum supported range.
378 		 */
379 		if (sc->ifc_sysctl_ntxqs > scctx->isc_ntxqsets_max)
380 			sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets_max;
381 		if (sc->ifc_sysctl_nrxqs > scctx->isc_nrxqsets_max)
382 			sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets_max;
383 	}
384 
385 	scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]
386 	    * sizeof(struct ice_tx_desc), DBA_ALIGN);
387 	scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0]
388 	    * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN);
389 
390 	scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS;
391 	scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS;
392 	scctx->isc_tx_tso_size_max = ICE_TSO_SIZE;
393 	scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE;
394 
395 	scctx->isc_msix_bar = PCIR_BAR(ICE_MSIX_BAR);
396 	scctx->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size;
397 
398 	/*
399 	 * If the driver loads in recovery mode, disable Tx/Rx functionality
400 	 */
401 	if (recovery_mode)
402 		scctx->isc_txrx = &ice_recovery_txrx;
403 	else
404 		scctx->isc_txrx = &ice_txrx;
405 
406 	/*
407 	 * If the driver loads in Safe mode or Recovery mode, disable
408 	 * advanced features including hardware offloads.
409 	 */
410 	if (safe_mode || recovery_mode) {
411 		scctx->isc_capenable = ICE_SAFE_CAPS;
412 		scctx->isc_tx_csum_flags = 0;
413 	} else {
414 		scctx->isc_capenable = ICE_FULL_CAPS;
415 		scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD;
416 	}
417 
418 	scctx->isc_capabilities = scctx->isc_capenable;
419 } /* ice_setup_scctx */
420 
421 /**
422  * ice_if_attach_pre - Early device attach logic
423  * @ctx: the iflib context structure
424  *
425  * Called by iflib during the attach process. Earliest main driver entry
426  * point which performs necessary hardware and driver initialization. Called
427  * before the Tx and Rx queues are allocated.
428  */
429 static int
430 ice_if_attach_pre(if_ctx_t ctx)
431 {
432 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
433 	enum ice_fw_modes fw_mode;
434 	enum ice_status status;
435 	if_softc_ctx_t scctx;
436 	struct ice_hw *hw;
437 	device_t dev;
438 	int err;
439 
440 	device_printf(iflib_get_dev(ctx), "Loading the iflib ice driver\n");
441 
442 	ice_set_state(&sc->state, ICE_STATE_ATTACHING);
443 
444 	sc->ctx = ctx;
445 	sc->media = iflib_get_media(ctx);
446 	sc->sctx = iflib_get_sctx(ctx);
447 	sc->iflib_ctx_lock = iflib_ctx_lock_get(ctx);
448 
449 	dev = sc->dev = iflib_get_dev(ctx);
450 	scctx = sc->scctx = iflib_get_softc_ctx(ctx);
451 
452 	hw = &sc->hw;
453 	hw->back = sc;
454 
455 	snprintf(sc->admin_mtx_name, sizeof(sc->admin_mtx_name),
456 		 "%s:admin", device_get_nameunit(dev));
457 	mtx_init(&sc->admin_mtx, sc->admin_mtx_name, NULL, MTX_DEF);
458 	callout_init_mtx(&sc->admin_timer, &sc->admin_mtx, 0);
459 
460 	ASSERT_CTX_LOCKED(sc);
461 
462 	if (ice_pci_mapping(sc)) {
463 		err = (ENXIO);
464 		goto destroy_admin_timer;
465 	}
466 
467 	/* Save off the PCI information */
468 	ice_save_pci_info(hw, dev);
469 
470 	/* create tunables as early as possible */
471 	ice_add_device_tunables(sc);
472 
473 	/* Setup ControlQ lengths */
474 	ice_set_ctrlq_len(hw);
475 
476 reinit_hw:
477 
478 	fw_mode = ice_get_fw_mode(hw);
479 	if (fw_mode == ICE_FW_MODE_REC) {
480 		device_printf(dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
481 
482 		err = ice_attach_pre_recovery_mode(sc);
483 		if (err)
484 			goto free_pci_mapping;
485 
486 		return (0);
487 	}
488 
489 	/* Initialize the hw data structure */
490 	status = ice_init_hw(hw);
491 	if (status) {
492 		if (status == ICE_ERR_FW_API_VER) {
493 			/* Enter recovery mode, so that the driver remains
494 			 * loaded. This way, if the system administrator
495 			 * cannot update the driver, they may still attempt to
496 			 * downgrade the NVM.
497 			 */
498 			err = ice_attach_pre_recovery_mode(sc);
499 			if (err)
500 				goto free_pci_mapping;
501 
502 			return (0);
503 		} else {
504 			err = EIO;
505 			device_printf(dev, "Unable to initialize hw, err %s aq_err %s\n",
506 				      ice_status_str(status),
507 				      ice_aq_str(hw->adminq.sq_last_status));
508 		}
509 		goto free_pci_mapping;
510 	}
511 
512 	ice_init_device_features(sc);
513 
514 	/* Notify firmware of the device driver version */
515 	err = ice_send_version(sc);
516 	if (err)
517 		goto deinit_hw;
518 
519 	/*
520 	 * Success indicates a change was made that requires a reinitialization
521 	 * of the hardware
522 	 */
523 	err = ice_load_pkg_file(sc);
524 	if (err == ICE_SUCCESS) {
525 		ice_deinit_hw(hw);
526 		goto reinit_hw;
527 	}
528 
529 	err = ice_init_link_events(sc);
530 	if (err) {
531 		device_printf(dev, "ice_init_link_events failed: %s\n",
532 			      ice_err_str(err));
533 		goto deinit_hw;
534 	}
535 
536 	/* Initialize VLAN mode in FW; if dual VLAN mode is supported by the package
537 	 * and firmware, this will force them to use single VLAN mode.
538 	 */
539 	status = ice_set_vlan_mode(hw);
540 	if (status) {
541 		err = EIO;
542 		device_printf(dev, "Unable to initialize VLAN mode, err %s aq_err %s\n",
543 			      ice_status_str(status),
544 			      ice_aq_str(hw->adminq.sq_last_status));
545 		goto deinit_hw;
546 	}
547 
548 	ice_print_nvm_version(sc);
549 
550 	/* Setup the MAC address */
551 	iflib_set_mac(ctx, hw->port_info->mac.lan_addr);
552 
553 	/* Setup the iflib softc context structure */
554 	ice_setup_scctx(sc);
555 
556 	/* Initialize the Tx queue manager */
557 	err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq);
558 	if (err) {
559 		device_printf(dev, "Unable to initialize Tx queue manager: %s\n",
560 			      ice_err_str(err));
561 		goto deinit_hw;
562 	}
563 
564 	/* Initialize the Rx queue manager */
565 	err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq);
566 	if (err) {
567 		device_printf(dev, "Unable to initialize Rx queue manager: %s\n",
568 			      ice_err_str(err));
569 		goto free_tx_qmgr;
570 	}
571 
572 	/* Initialize the interrupt resource manager */
573 	err = ice_alloc_intr_tracking(sc);
574 	if (err)
575 		/* Errors are already printed */
576 		goto free_rx_qmgr;
577 
578 	/* Determine maximum number of VSIs we'll prepare for */
579 	sc->num_available_vsi = min(ICE_MAX_VSI_AVAILABLE,
580 				    hw->func_caps.guar_num_vsi);
581 
582 	if (!sc->num_available_vsi) {
583 		err = EIO;
584 		device_printf(dev, "No VSIs allocated to host\n");
585 		goto free_intr_tracking;
586 	}
587 
588 	/* Allocate storage for the VSI pointers */
589 	sc->all_vsi = (struct ice_vsi **)
590 		malloc(sizeof(struct ice_vsi *) * sc->num_available_vsi,
591 		       M_ICE, M_WAITOK | M_ZERO);
592 	if (!sc->all_vsi) {
593 		err = ENOMEM;
594 		device_printf(dev, "Unable to allocate VSI array\n");
595 		goto free_intr_tracking;
596 	}
597 
598 	/*
599 	 * Prepare the statically allocated primary PF VSI in the softc
600 	 * structure. Other VSIs will be dynamically allocated as needed.
601 	 */
602 	ice_setup_pf_vsi(sc);
603 
604 	err = ice_alloc_vsi_qmap(&sc->pf_vsi, scctx->isc_ntxqsets_max,
605 	    scctx->isc_nrxqsets_max);
606 	if (err) {
607 		device_printf(dev, "Unable to allocate VSI Queue maps\n");
608 		goto free_main_vsi;
609 	}
610 
611 	/* Allocate MSI-X vectors (due to isc_flags IFLIB_SKIP_MSIX) */
612 	err = ice_allocate_msix(sc);
613 	if (err)
614 		goto free_main_vsi;
615 
616 	return 0;
617 
618 free_main_vsi:
619 	/* ice_release_vsi will free the queue maps if they were allocated */
620 	ice_release_vsi(&sc->pf_vsi);
621 	free(sc->all_vsi, M_ICE);
622 	sc->all_vsi = NULL;
623 free_intr_tracking:
624 	ice_free_intr_tracking(sc);
625 free_rx_qmgr:
626 	ice_resmgr_destroy(&sc->rx_qmgr);
627 free_tx_qmgr:
628 	ice_resmgr_destroy(&sc->tx_qmgr);
629 deinit_hw:
630 	ice_deinit_hw(hw);
631 free_pci_mapping:
632 	ice_free_pci_mapping(sc);
633 destroy_admin_timer:
634 	mtx_lock(&sc->admin_mtx);
635 	callout_stop(&sc->admin_timer);
636 	mtx_unlock(&sc->admin_mtx);
637 	mtx_destroy(&sc->admin_mtx);
638 	return err;
639 } /* ice_if_attach_pre */
640 
641 /**
642  * ice_attach_pre_recovery_mode - Limited driver attach_pre for FW recovery
643  * @sc: the device private softc
644  *
645  * Loads the device driver in limited Firmware Recovery mode, intended to
646  * allow users to update the firmware to attempt to recover the device.
647  *
648  * @remark We may enter recovery mode in case either (a) the firmware is
649  * detected to be in an invalid state and must be re-programmed, or (b) the
650  * driver detects that the loaded firmware has a non-compatible API version
651  * that the driver cannot operate with.
652  */
653 static int
654 ice_attach_pre_recovery_mode(struct ice_softc *sc)
655 {
656 	ice_set_state(&sc->state, ICE_STATE_RECOVERY_MODE);
657 
658 	/* Setup the iflib softc context */
659 	ice_setup_scctx(sc);
660 
661 	/* Setup the PF VSI back pointer */
662 	sc->pf_vsi.sc = sc;
663 
664 	/*
665 	 * We still need to allocate MSI-X vectors since we need one vector to
666 	 * run the administrative admin interrupt
667 	 */
668 	return ice_allocate_msix(sc);
669 }
670 
671 /**
672  * ice_update_link_status - notify OS of link state change
673  * @sc: device private softc structure
674  * @update_media: true if we should update media even if link didn't change
675  *
676  * Called to notify iflib core of link status changes. Should be called once
677  * during attach_post, and whenever link status changes during runtime.
678  *
679  * This call only updates the currently supported media types if the link
680  * status changed, or if update_media is set to true.
681  */
682 static void
683 ice_update_link_status(struct ice_softc *sc, bool update_media)
684 {
685 	struct ice_hw *hw = &sc->hw;
686 	enum ice_status status;
687 
688 	/* Never report link up when in recovery mode */
689 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
690 		return;
691 
692 	/* Report link status to iflib only once each time it changes */
693 	if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) {
694 		if (sc->link_up) { /* link is up */
695 			uint64_t baudrate = ice_aq_speed_to_rate(sc->hw.port_info);
696 
697 			ice_set_default_local_lldp_mib(sc);
698 
699 			iflib_link_state_change(sc->ctx, LINK_STATE_UP, baudrate);
700 			ice_rdma_link_change(sc, LINK_STATE_UP, baudrate);
701 
702 			ice_link_up_msg(sc);
703 		} else { /* link is down */
704 			iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0);
705 			ice_rdma_link_change(sc, LINK_STATE_DOWN, 0);
706 		}
707 		update_media = true;
708 	}
709 
710 	/* Update the supported media types */
711 	if (update_media) {
712 		status = ice_add_media_types(sc, sc->media);
713 		if (status)
714 			device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n",
715 				      ice_status_str(status),
716 				      ice_aq_str(hw->adminq.sq_last_status));
717 	}
718 }
719 
720 /**
721  * ice_if_attach_post - Late device attach logic
722  * @ctx: the iflib context structure
723  *
724  * Called by iflib to finish up attaching the device. Performs any attach
725  * logic which must wait until after the Tx and Rx queues have been
726  * allocated.
727  */
728 static int
729 ice_if_attach_post(if_ctx_t ctx)
730 {
731 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
732 	if_t ifp = iflib_get_ifp(ctx);
733 	int err;
734 
735 	ASSERT_CTX_LOCKED(sc);
736 
737 	/* We don't yet support loading if MSI-X is not supported */
738 	if (sc->scctx->isc_intr != IFLIB_INTR_MSIX) {
739 		device_printf(sc->dev, "The ice driver does not support loading without MSI-X\n");
740 		return (ENOTSUP);
741 	}
742 
743 	/* The ifnet structure hasn't yet been initialized when the attach_pre
744 	 * handler is called, so wait until attach_post to setup the
745 	 * isc_max_frame_size.
746 	 */
747 
748 	sc->ifp = ifp;
749 	sc->scctx->isc_max_frame_size = if_getmtu(ifp) +
750 		ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN;
751 
752 	/*
753 	 * If we are in recovery mode, only perform a limited subset of
754 	 * initialization to support NVM recovery.
755 	 */
756 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
757 		ice_attach_post_recovery_mode(sc);
758 		return (0);
759 	}
760 
761 	sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size;
762 
763 	err = ice_initialize_vsi(&sc->pf_vsi);
764 	if (err) {
765 		device_printf(sc->dev, "Unable to initialize Main VSI: %s\n",
766 			      ice_err_str(err));
767 		return err;
768 	}
769 
770 	/* Enable FW health event reporting */
771 	ice_init_health_events(sc);
772 
773 	/* Configure the main PF VSI for RSS */
774 	err = ice_config_rss(&sc->pf_vsi);
775 	if (err) {
776 		device_printf(sc->dev,
777 			      "Unable to configure RSS for the main VSI, err %s\n",
778 			      ice_err_str(err));
779 		return err;
780 	}
781 
782 	/* Configure switch to drop transmitted LLDP and PAUSE frames */
783 	err = ice_cfg_pf_ethertype_filters(sc);
784 	if (err)
785 		return err;
786 
787 	ice_get_and_print_bus_info(sc);
788 
789 	ice_set_link_management_mode(sc);
790 
791 	ice_init_saved_phy_cfg(sc);
792 
793 	ice_cfg_pba_num(sc);
794 
795 	ice_add_device_sysctls(sc);
796 
797 	/* Get DCBX/LLDP state and start DCBX agent */
798 	ice_init_dcb_setup(sc);
799 
800 	/* Setup link configuration parameters */
801 	ice_init_link_configuration(sc);
802 	ice_update_link_status(sc, true);
803 
804 	/* Configure interrupt causes for the administrative interrupt */
805 	ice_configure_misc_interrupts(sc);
806 
807 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
808 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
809 
810 	err = ice_rdma_pf_attach(sc);
811 	if (err)
812 		return (err);
813 
814 	/* Start the admin timer */
815 	mtx_lock(&sc->admin_mtx);
816 	callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc);
817 	mtx_unlock(&sc->admin_mtx);
818 
819 	ice_clear_state(&sc->state, ICE_STATE_ATTACHING);
820 
821 	return 0;
822 } /* ice_if_attach_post */
823 
824 /**
825  * ice_attach_post_recovery_mode - Limited driver attach_post for FW recovery
826  * @sc: the device private softc
827  *
828  * Performs minimal work to prepare the driver to recover an NVM in case the
829  * firmware is in recovery mode.
830  */
831 static void
832 ice_attach_post_recovery_mode(struct ice_softc *sc)
833 {
834 	/* Configure interrupt causes for the administrative interrupt */
835 	ice_configure_misc_interrupts(sc);
836 
837 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
838 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
839 
840 	/* Start the admin timer */
841 	mtx_lock(&sc->admin_mtx);
842 	callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc);
843 	mtx_unlock(&sc->admin_mtx);
844 
845 	ice_clear_state(&sc->state, ICE_STATE_ATTACHING);
846 }
847 
848 /**
849  * ice_free_irqvs - Free IRQ vector memory
850  * @sc: the device private softc structure
851  *
852  * Free IRQ vector memory allocated during ice_if_msix_intr_assign.
853  */
854 static void
855 ice_free_irqvs(struct ice_softc *sc)
856 {
857 	struct ice_vsi *vsi = &sc->pf_vsi;
858 	if_ctx_t ctx = sc->ctx;
859 	int i;
860 
861 	/* If the irqvs array is NULL, then there are no vectors to free */
862 	if (sc->irqvs == NULL)
863 		return;
864 
865 	/* Free the IRQ vectors */
866 	for (i = 0; i < sc->num_irq_vectors; i++)
867 		iflib_irq_free(ctx, &sc->irqvs[i].irq);
868 
869 	/* Clear the irqv pointers */
870 	for (i = 0; i < vsi->num_rx_queues; i++)
871 		vsi->rx_queues[i].irqv = NULL;
872 
873 	for (i = 0; i < vsi->num_tx_queues; i++)
874 		vsi->tx_queues[i].irqv = NULL;
875 
876 	/* Release the vector array memory */
877 	free(sc->irqvs, M_ICE);
878 	sc->irqvs = NULL;
879 	sc->num_irq_vectors = 0;
880 }
881 
882 /**
883  * ice_if_detach - Device driver detach logic
884  * @ctx: iflib context structure
885  *
886  * Perform device shutdown logic to detach the device driver.
887  *
888  * Note that there is no guarantee of the ordering of ice_if_queues_free() and
889  * ice_if_detach(). It is possible for the functions to be called in either
890  * order, and they must not assume to have a strict ordering.
891  */
892 static int
893 ice_if_detach(if_ctx_t ctx)
894 {
895 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
896 	struct ice_vsi *vsi = &sc->pf_vsi;
897 	int i;
898 
899 	ASSERT_CTX_LOCKED(sc);
900 
901 	/* Indicate that we're detaching */
902 	ice_set_state(&sc->state, ICE_STATE_DETACHING);
903 
904 	/* Stop the admin timer */
905 	mtx_lock(&sc->admin_mtx);
906 	callout_stop(&sc->admin_timer);
907 	mtx_unlock(&sc->admin_mtx);
908 	mtx_destroy(&sc->admin_mtx);
909 
910 	ice_rdma_pf_detach(sc);
911 
912 	/* Free allocated media types */
913 	ifmedia_removeall(sc->media);
914 
915 	/* Free the Tx and Rx sysctl contexts, and assign NULL to the node
916 	 * pointers. Note, the calls here and those in ice_if_queues_free()
917 	 * are *BOTH* necessary, as we cannot guarantee which path will be
918 	 * run first
919 	 */
920 	ice_vsi_del_txqs_ctx(vsi);
921 	ice_vsi_del_rxqs_ctx(vsi);
922 
923 	/* Release MSI-X resources */
924 	ice_free_irqvs(sc);
925 
926 	for (i = 0; i < sc->num_available_vsi; i++) {
927 		if (sc->all_vsi[i])
928 			ice_release_vsi(sc->all_vsi[i]);
929 	}
930 
931 	if (sc->all_vsi) {
932 		free(sc->all_vsi, M_ICE);
933 		sc->all_vsi = NULL;
934 	}
935 
936 	/* Release MSI-X memory */
937 	pci_release_msi(sc->dev);
938 
939 	if (sc->msix_table != NULL) {
940 		bus_release_resource(sc->dev, SYS_RES_MEMORY,
941 				     rman_get_rid(sc->msix_table),
942 				     sc->msix_table);
943 		sc->msix_table = NULL;
944 	}
945 
946 	ice_free_intr_tracking(sc);
947 
948 	/* Destroy the queue managers */
949 	ice_resmgr_destroy(&sc->tx_qmgr);
950 	ice_resmgr_destroy(&sc->rx_qmgr);
951 
952 	if (!ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
953 		ice_deinit_hw(&sc->hw);
954 
955 	ice_free_pci_mapping(sc);
956 
957 	return 0;
958 } /* ice_if_detach */
959 
960 /**
961  * ice_if_tx_queues_alloc - Allocate Tx queue memory
962  * @ctx: iflib context structure
963  * @vaddrs: virtual addresses for the queue memory
964  * @paddrs: physical addresses for the queue memory
965  * @ntxqs: the number of Tx queues per set (should always be 1)
966  * @ntxqsets: the number of Tx queue sets to allocate
967  *
968  * Called by iflib to allocate Tx queues for the device. Allocates driver
969  * memory to track each queue, the status arrays used for descriptor
970  * status reporting, and Tx queue sysctls.
971  */
972 static int
973 ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
974 		       int __invariant_only ntxqs, int ntxqsets)
975 {
976 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
977 	struct ice_vsi *vsi = &sc->pf_vsi;
978 	struct ice_tx_queue *txq;
979 	int err, i, j;
980 
981 	MPASS(ntxqs == 1);
982 	MPASS(sc->scctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT);
983 	ASSERT_CTX_LOCKED(sc);
984 
985 	/* Do not bother allocating queues if we're in recovery mode */
986 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
987 		return (0);
988 
989 	/* Allocate queue structure memory */
990 	if (!(vsi->tx_queues =
991 	      (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_NOWAIT | M_ZERO))) {
992 		device_printf(sc->dev, "Unable to allocate Tx queue memory\n");
993 		return (ENOMEM);
994 	}
995 
996 	/* Allocate report status arrays */
997 	for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
998 		if (!(txq->tx_rsq =
999 		      (uint16_t *) malloc(sizeof(uint16_t) * sc->scctx->isc_ntxd[0], M_ICE, M_NOWAIT))) {
1000 			device_printf(sc->dev, "Unable to allocate tx_rsq memory\n");
1001 			err = ENOMEM;
1002 			goto free_tx_queues;
1003 		}
1004 		/* Initialize report status array */
1005 		for (j = 0; j < sc->scctx->isc_ntxd[0]; j++)
1006 			txq->tx_rsq[j] = QIDX_INVALID;
1007 	}
1008 
1009 	/* Assign queues from PF space to the main VSI */
1010 	err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, ntxqsets);
1011 	if (err) {
1012 		device_printf(sc->dev, "Unable to assign PF queues: %s\n",
1013 			      ice_err_str(err));
1014 		goto free_tx_queues;
1015 	}
1016 	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
1017 
1018 	/* Add Tx queue sysctls context */
1019 	ice_vsi_add_txqs_ctx(vsi);
1020 
1021 	for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
1022 		/* q_handle == me when only one TC */
1023 		txq->me = txq->q_handle = i;
1024 		txq->vsi = vsi;
1025 
1026 		/* store the queue size for easier access */
1027 		txq->desc_count = sc->scctx->isc_ntxd[0];
1028 
1029 		/* get the virtual and physical address of the hardware queues */
1030 		txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);
1031 		txq->tx_base = (struct ice_tx_desc *)vaddrs[i];
1032 		txq->tx_paddr = paddrs[i];
1033 
1034 		ice_add_txq_sysctls(txq);
1035 	}
1036 
1037 	vsi->num_tx_queues = ntxqsets;
1038 
1039 	return (0);
1040 
1041 free_tx_queues:
1042 	for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) {
1043 		if (txq->tx_rsq != NULL) {
1044 			free(txq->tx_rsq, M_ICE);
1045 			txq->tx_rsq = NULL;
1046 		}
1047 	}
1048 	free(vsi->tx_queues, M_ICE);
1049 	vsi->tx_queues = NULL;
1050 	return err;
1051 }
1052 
1053 /**
1054  * ice_if_rx_queues_alloc - Allocate Rx queue memory
1055  * @ctx: iflib context structure
1056  * @vaddrs: virtual addresses for the queue memory
1057  * @paddrs: physical addresses for the queue memory
1058  * @nrxqs: number of Rx queues per set (should always be 1)
1059  * @nrxqsets: number of Rx queue sets to allocate
1060  *
1061  * Called by iflib to allocate Rx queues for the device. Allocates driver
1062  * memory to track each queue, as well as sets up the Rx queue sysctls.
1063  */
1064 static int
1065 ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
1066 		       int __invariant_only nrxqs, int nrxqsets)
1067 {
1068 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1069 	struct ice_vsi *vsi = &sc->pf_vsi;
1070 	struct ice_rx_queue *rxq;
1071 	int err, i;
1072 
1073 	MPASS(nrxqs == 1);
1074 	MPASS(sc->scctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT);
1075 	ASSERT_CTX_LOCKED(sc);
1076 
1077 	/* Do not bother allocating queues if we're in recovery mode */
1078 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1079 		return (0);
1080 
1081 	/* Allocate queue structure memory */
1082 	if (!(vsi->rx_queues =
1083 	      (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_NOWAIT | M_ZERO))) {
1084 		device_printf(sc->dev, "Unable to allocate Rx queue memory\n");
1085 		return (ENOMEM);
1086 	}
1087 
1088 	/* Assign queues from PF space to the main VSI */
1089 	err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, nrxqsets);
1090 	if (err) {
1091 		device_printf(sc->dev, "Unable to assign PF queues: %s\n",
1092 			      ice_err_str(err));
1093 		goto free_rx_queues;
1094 	}
1095 	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
1096 
1097 	/* Add Rx queue sysctls context */
1098 	ice_vsi_add_rxqs_ctx(vsi);
1099 
1100 	for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) {
1101 		rxq->me = i;
1102 		rxq->vsi = vsi;
1103 
1104 		/* store the queue size for easier access */
1105 		rxq->desc_count = sc->scctx->isc_nrxd[0];
1106 
1107 		/* get the virtual and physical address of the hardware queues */
1108 		rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);
1109 		rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i];
1110 		rxq->rx_paddr = paddrs[i];
1111 
1112 		ice_add_rxq_sysctls(rxq);
1113 	}
1114 
1115 	vsi->num_rx_queues = nrxqsets;
1116 
1117 	return (0);
1118 
1119 free_rx_queues:
1120 	free(vsi->rx_queues, M_ICE);
1121 	vsi->rx_queues = NULL;
1122 	return err;
1123 }
1124 
1125 /**
1126  * ice_if_queues_free - Free queue memory
1127  * @ctx: the iflib context structure
1128  *
1129  * Free queue memory allocated by ice_if_tx_queues_alloc() and
1130  * ice_if_rx_queues_alloc().
1131  *
1132  * There is no guarantee that ice_if_queues_free() and ice_if_detach() will be
1133  * called in the same order. It's possible for ice_if_queues_free() to be
1134  * called prior to ice_if_detach(), and vice versa.
1135  *
1136  * For this reason, the main VSI is a static member of the ice_softc, which is
1137  * not free'd until after iflib finishes calling both of these functions.
1138  *
1139  * Thus, care must be taken in how we manage the memory being freed by this
1140  * function, and in what tasks it can and must perform.
1141  */
1142 static void
1143 ice_if_queues_free(if_ctx_t ctx)
1144 {
1145 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1146 	struct ice_vsi *vsi = &sc->pf_vsi;
1147 	struct ice_tx_queue *txq;
1148 	int i;
1149 
1150 	/* Free the Tx and Rx sysctl contexts, and assign NULL to the node
1151 	 * pointers. Note, the calls here and those in ice_if_detach()
1152 	 * are *BOTH* necessary, as we cannot guarantee which path will be
1153 	 * run first
1154 	 */
1155 	ice_vsi_del_txqs_ctx(vsi);
1156 	ice_vsi_del_rxqs_ctx(vsi);
1157 
1158 	/* Release MSI-X IRQ vectors, if not yet released in ice_if_detach */
1159 	ice_free_irqvs(sc);
1160 
1161 	if (vsi->tx_queues != NULL) {
1162 		/* free the tx_rsq arrays */
1163 		for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {
1164 			if (txq->tx_rsq != NULL) {
1165 				free(txq->tx_rsq, M_ICE);
1166 				txq->tx_rsq = NULL;
1167 			}
1168 		}
1169 		free(vsi->tx_queues, M_ICE);
1170 		vsi->tx_queues = NULL;
1171 		vsi->num_tx_queues = 0;
1172 	}
1173 	if (vsi->rx_queues != NULL) {
1174 		free(vsi->rx_queues, M_ICE);
1175 		vsi->rx_queues = NULL;
1176 		vsi->num_rx_queues = 0;
1177 	}
1178 }
1179 
1180 /**
1181  * ice_msix_que - Fast interrupt handler for MSI-X receive queues
1182  * @arg: The Rx queue memory
1183  *
1184  * Interrupt filter function for iflib MSI-X interrupts. Called by iflib when
1185  * an MSI-X interrupt for a given queue is triggered. Currently this just asks
1186  * iflib to schedule the main Rx thread.
1187  */
1188 static int
1189 ice_msix_que(void *arg)
1190 {
1191 	struct ice_rx_queue __unused *rxq = (struct ice_rx_queue *)arg;
1192 
1193 	/* TODO: dynamic ITR algorithm?? */
1194 
1195 	return (FILTER_SCHEDULE_THREAD);
1196 }
1197 
1198 /**
1199  * ice_msix_admin - Fast interrupt handler for MSI-X admin interrupt
1200  * @arg: pointer to device softc memory
1201  *
1202  * Called by iflib when an administrative interrupt occurs. Should perform any
1203  * fast logic for handling the interrupt cause, and then indicate whether the
1204  * admin task needs to be queued.
1205  */
1206 static int
1207 ice_msix_admin(void *arg)
1208 {
1209 	struct ice_softc *sc = (struct ice_softc *)arg;
1210 	struct ice_hw *hw = &sc->hw;
1211 	device_t dev = sc->dev;
1212 	u32 oicr;
1213 
1214 	/* There is no safe way to modify the enabled miscellaneous causes of
1215 	 * the OICR vector at runtime, as doing so would be prone to race
1216 	 * conditions. Reading PFINT_OICR will unmask the associated interrupt
1217 	 * causes and allow future interrupts to occur. The admin interrupt
1218 	 * vector will not be re-enabled until after we exit this function,
1219 	 * but any delayed tasks must be resilient against possible "late
1220 	 * arrival" interrupts that occur while we're already handling the
1221 	 * task. This is done by using state bits and serializing these
1222 	 * delayed tasks via the admin status task function.
1223 	 */
1224 	oicr = rd32(hw, PFINT_OICR);
1225 
1226 	/* Processing multiple controlq interrupts on a single vector does not
1227 	 * provide an indication of which controlq triggered the interrupt.
1228 	 * We might try reading the INTEVENT bit of the respective PFINT_*_CTL
1229 	 * registers. However, the INTEVENT bit is not guaranteed to be set as
1230 	 * it gets automatically cleared when the hardware acknowledges the
1231 	 * interrupt.
1232 	 *
1233 	 * This means we don't really have a good indication of whether or
1234 	 * which controlq triggered this interrupt. We'll just notify the
1235 	 * admin task that it should check all the controlqs.
1236 	 */
1237 	ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
1238 
1239 	if (oicr & PFINT_OICR_VFLR_M) {
1240 		ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING);
1241 	}
1242 
1243 	if (oicr & PFINT_OICR_MAL_DETECT_M) {
1244 		ice_set_state(&sc->state, ICE_STATE_MDD_PENDING);
1245 	}
1246 
1247 	if (oicr & PFINT_OICR_GRST_M) {
1248 		u32 reset;
1249 
1250 		reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
1251 			GLGEN_RSTAT_RESET_TYPE_S;
1252 
1253 		if (reset == ICE_RESET_CORER)
1254 			sc->soft_stats.corer_count++;
1255 		else if (reset == ICE_RESET_GLOBR)
1256 			sc->soft_stats.globr_count++;
1257 		else
1258 			sc->soft_stats.empr_count++;
1259 
1260 		/* There are a couple of bits at play for handling resets.
1261 		 * First, the ICE_STATE_RESET_OICR_RECV bit is used to
1262 		 * indicate that the driver has received an OICR with a reset
1263 		 * bit active, indicating that a CORER/GLOBR/EMPR is about to
1264 		 * happen. Second, we set hw->reset_ongoing to indicate that
1265 		 * the hardware is in reset. We will set this back to false as
1266 		 * soon as the driver has determined that the hardware is out
1267 		 * of reset.
1268 		 *
1269 		 * If the driver wishes to trigger a request, it can set one of
1270 		 * the ICE_STATE_RESET_*_REQ bits, which will trigger the
1271 		 * correct type of reset.
1272 		 */
1273 		if (!ice_testandset_state(&sc->state, ICE_STATE_RESET_OICR_RECV))
1274 			hw->reset_ongoing = true;
1275 	}
1276 
1277 	if (oicr & PFINT_OICR_ECC_ERR_M) {
1278 		device_printf(dev, "ECC Error detected!\n");
1279 		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1280 	}
1281 
1282 	if (oicr & PFINT_OICR_PE_CRITERR_M) {
1283 		device_printf(dev, "Critical Protocol Engine Error detected!\n");
1284 		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1285 	}
1286 
1287 	if (oicr & PFINT_OICR_PCI_EXCEPTION_M) {
1288 		device_printf(dev, "PCI Exception detected!\n");
1289 		ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
1290 	}
1291 
1292 	if (oicr & PFINT_OICR_HMC_ERR_M) {
1293 		/* Log the HMC errors, but don't disable the interrupt cause */
1294 		ice_log_hmc_error(hw, dev);
1295 	}
1296 
1297 	return (FILTER_SCHEDULE_THREAD);
1298 }
1299 
1300 /**
1301  * ice_allocate_msix - Allocate MSI-X vectors for the interface
1302  * @sc: the device private softc
1303  *
1304  * Map the MSI-X bar, and then request MSI-X vectors in a two-stage process.
1305  *
1306  * First, determine a suitable total number of vectors based on the number
1307  * of CPUs, RSS buckets, the administrative vector, and other demands such as
1308  * RDMA.
1309  *
1310  * Request the desired amount of vectors, and see how many we obtain. If we
1311  * don't obtain as many as desired, reduce the demands by lowering the number
1312  * of requested queues or reducing the demand from other features such as
1313  * RDMA.
1314  *
1315  * @remark This function is required because the driver sets the
1316  * IFLIB_SKIP_MSIX flag indicating that the driver will manage MSI-X vectors
1317  * manually.
1318  *
1319  * @remark This driver will only use MSI-X vectors. If this is not possible,
1320  * neither MSI or legacy interrupts will be tried.
1321  *
1322  * @post on success this function must set the following scctx parameters:
1323  * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr.
1324  *
1325  * @returns zero on success or an error code on failure.
1326  */
1327 static int
1328 ice_allocate_msix(struct ice_softc *sc)
1329 {
1330 	bool iflib_override_queue_count = false;
1331 	if_softc_ctx_t scctx = sc->scctx;
1332 	device_t dev = sc->dev;
1333 	cpuset_t cpus;
1334 	int bar, queues, vectors, requested;
1335 	int err = 0;
1336 	int rdma;
1337 
1338 	/* Allocate the MSI-X bar */
1339 	bar = scctx->isc_msix_bar;
1340 	sc->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE);
1341 	if (!sc->msix_table) {
1342 		device_printf(dev, "Unable to map MSI-X table\n");
1343 		return (ENOMEM);
1344 	}
1345 
1346 	/* Check if the iflib queue count sysctls have been set */
1347 	if (sc->ifc_sysctl_ntxqs || sc->ifc_sysctl_nrxqs)
1348 		iflib_override_queue_count = true;
1349 
1350 	err = bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus);
1351 	if (err) {
1352 		device_printf(dev, "%s: Unable to fetch the CPU list: %s\n",
1353 			      __func__, ice_err_str(err));
1354 		CPU_COPY(&all_cpus, &cpus);
1355 	}
1356 
1357 	/* Attempt to mimic behavior of iflib_msix_init */
1358 	if (iflib_override_queue_count) {
1359 		/*
1360 		 * If the override sysctls have been set, limit the queues to
1361 		 * the number of logical CPUs.
1362 		 */
1363 		queues = mp_ncpus;
1364 	} else {
1365 		/*
1366 		 * Otherwise, limit the queue count to the CPUs associated
1367 		 * with the NUMA node the device is associated with.
1368 		 */
1369 		queues = CPU_COUNT(&cpus);
1370 	}
1371 
1372 	/* Clamp to the number of RSS buckets */
1373 	queues = imin(queues, rss_getnumbuckets());
1374 
1375 	/*
1376 	 * Clamp the number of queue pairs to the minimum of the requested Tx
1377 	 * and Rx queues.
1378 	 */
1379 	queues = imin(queues, sc->ifc_sysctl_ntxqs ?: scctx->isc_ntxqsets);
1380 	queues = imin(queues, sc->ifc_sysctl_nrxqs ?: scctx->isc_nrxqsets);
1381 
1382 	if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RDMA)) {
1383 		/*
1384 		 * Choose a number of RDMA vectors based on the number of CPUs
1385 		 * up to a maximum
1386 		 */
1387 		rdma = min(CPU_COUNT(&cpus), ICE_RDMA_MAX_MSIX);
1388 
1389 		/* Further limit by the user configurable tunable */
1390 		rdma = min(rdma, ice_rdma_max_msix);
1391 	} else {
1392 		rdma = 0;
1393 	}
1394 
1395 	/*
1396 	 * Determine the number of vectors to request. Note that we also need
1397 	 * to allocate one vector for administrative tasks.
1398 	 */
1399 	requested = rdma + queues + 1;
1400 
1401 	vectors = requested;
1402 
1403 	err = pci_alloc_msix(dev, &vectors);
1404 	if (err) {
1405 		device_printf(dev, "Failed to allocate %d MSI-X vectors, err %s\n",
1406 			      vectors, ice_err_str(err));
1407 		goto err_free_msix_table;
1408 	}
1409 
1410 	/* If we don't receive enough vectors, reduce demands */
1411 	if (vectors < requested) {
1412 		int diff = requested - vectors;
1413 
1414 		device_printf(dev, "Requested %d MSI-X vectors, but got only %d\n",
1415 			      requested, vectors);
1416 
1417 		/*
1418 		 * The OS didn't grant us the requested number of vectors.
1419 		 * Check to see if we can reduce demands by limiting the
1420 		 * number of vectors allocated to certain features.
1421 		 */
1422 
1423 		if (rdma >= diff) {
1424 			/* Reduce the number of RDMA vectors we reserve */
1425 			rdma -= diff;
1426 			diff = 0;
1427 		} else {
1428 			/* Disable RDMA and reduce the difference */
1429 			ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap);
1430 			diff -= rdma;
1431 			rdma = 0;
1432 		}
1433 
1434 		/*
1435 		 * If we still have a difference, we need to reduce the number
1436 		 * of queue pairs.
1437 		 *
1438 		 * However, we still need at least one vector for the admin
1439 		 * interrupt and one queue pair.
1440 		 */
1441 		if (queues <= diff) {
1442 			device_printf(dev, "Unable to allocate sufficient MSI-X vectors\n");
1443 			err = (ERANGE);
1444 			goto err_pci_release_msi;
1445 		}
1446 
1447 		queues -= diff;
1448 	}
1449 
1450 	device_printf(dev, "Using %d Tx and Rx queues\n", queues);
1451 	if (rdma)
1452 		device_printf(dev, "Reserving %d MSI-X interrupts for iRDMA\n",
1453 			      rdma);
1454 	device_printf(dev, "Using MSI-X interrupts with %d vectors\n",
1455 		      vectors);
1456 
1457 	scctx->isc_vectors = vectors;
1458 	scctx->isc_nrxqsets = queues;
1459 	scctx->isc_ntxqsets = queues;
1460 	scctx->isc_intr = IFLIB_INTR_MSIX;
1461 
1462 	sc->irdma_vectors = rdma;
1463 
1464 	/* Interrupt allocation tracking isn't required in recovery mode,
1465 	 * since neither RDMA nor VFs are enabled.
1466 	 */
1467 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1468 		return (0);
1469 
1470 	/* Keep track of which interrupt indices are being used for what */
1471 	sc->lan_vectors = vectors - rdma;
1472 	err = ice_resmgr_assign_contiguous(&sc->imgr, sc->pf_imap, sc->lan_vectors);
1473 	if (err) {
1474 		device_printf(dev, "Unable to assign PF interrupt mapping: %s\n",
1475 			      ice_err_str(err));
1476 		goto err_pci_release_msi;
1477 	}
1478 	err = ice_resmgr_assign_contiguous(&sc->imgr, sc->rdma_imap, rdma);
1479 	if (err) {
1480 		device_printf(dev, "Unable to assign PF RDMA interrupt mapping: %s\n",
1481 			      ice_err_str(err));
1482 		ice_resmgr_release_map(&sc->imgr, sc->pf_imap,
1483 					    sc->lan_vectors);
1484 		goto err_pci_release_msi;
1485 	}
1486 
1487 	return (0);
1488 
1489 err_pci_release_msi:
1490 	pci_release_msi(dev);
1491 err_free_msix_table:
1492 	if (sc->msix_table != NULL) {
1493 		bus_release_resource(sc->dev, SYS_RES_MEMORY,
1494 				rman_get_rid(sc->msix_table),
1495 				sc->msix_table);
1496 		sc->msix_table = NULL;
1497 	}
1498 
1499 	return (err);
1500 }
1501 
1502 /**
1503  * ice_if_msix_intr_assign - Assign MSI-X interrupt vectors to queues
1504  * @ctx: the iflib context structure
1505  * @msix: the number of vectors we were assigned
1506  *
1507  * Called by iflib to assign MSI-X vectors to queues. Currently requires that
1508  * we get at least the same number of vectors as we have queues, and that we
1509  * always have the same number of Tx and Rx queues.
1510  *
1511  * Tx queues use a softirq instead of using their own hardware interrupt.
1512  */
1513 static int
1514 ice_if_msix_intr_assign(if_ctx_t ctx, int msix)
1515 {
1516 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1517 	struct ice_vsi *vsi = &sc->pf_vsi;
1518 	int err, i, vector;
1519 
1520 	ASSERT_CTX_LOCKED(sc);
1521 
1522 	if (vsi->num_rx_queues != vsi->num_tx_queues) {
1523 		device_printf(sc->dev,
1524 			      "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n",
1525 			      vsi->num_tx_queues, vsi->num_rx_queues);
1526 		return (EOPNOTSUPP);
1527 	}
1528 
1529 	if (msix < (vsi->num_rx_queues + 1)) {
1530 		device_printf(sc->dev,
1531 			      "Not enough MSI-X vectors to assign one vector to each queue pair\n");
1532 		return (EOPNOTSUPP);
1533 	}
1534 
1535 	/* Save the number of vectors for future use */
1536 	sc->num_irq_vectors = vsi->num_rx_queues + 1;
1537 
1538 	/* Allocate space to store the IRQ vector data */
1539 	if (!(sc->irqvs =
1540 	      (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (sc->num_irq_vectors),
1541 					       M_ICE, M_NOWAIT))) {
1542 		device_printf(sc->dev,
1543 			      "Unable to allocate irqv memory\n");
1544 		return (ENOMEM);
1545 	}
1546 
1547 	/* Administrative interrupt events will use vector 0 */
1548 	err = iflib_irq_alloc_generic(ctx, &sc->irqvs[0].irq, 1, IFLIB_INTR_ADMIN,
1549 				      ice_msix_admin, sc, 0, "admin");
1550 	if (err) {
1551 		device_printf(sc->dev,
1552 			      "Failed to register Admin queue handler: %s\n",
1553 			      ice_err_str(err));
1554 		goto free_irqvs;
1555 	}
1556 	sc->irqvs[0].me = 0;
1557 
1558 	/* Do not allocate queue interrupts when in recovery mode */
1559 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1560 		return (0);
1561 
1562 	for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++) {
1563 		struct ice_rx_queue *rxq = &vsi->rx_queues[i];
1564 		struct ice_tx_queue *txq = &vsi->tx_queues[i];
1565 		int rid = vector + 1;
1566 		char irq_name[16];
1567 
1568 		snprintf(irq_name, sizeof(irq_name), "rxq%d", i);
1569 		err = iflib_irq_alloc_generic(ctx, &sc->irqvs[vector].irq, rid,
1570 					      IFLIB_INTR_RXTX, ice_msix_que,
1571 					      rxq, rxq->me, irq_name);
1572 		if (err) {
1573 			device_printf(sc->dev,
1574 				      "Failed to allocate q int %d err: %s\n",
1575 				      i, ice_err_str(err));
1576 			vector--;
1577 			i--;
1578 			goto fail;
1579 		}
1580 		sc->irqvs[vector].me = vector;
1581 		rxq->irqv = &sc->irqvs[vector];
1582 
1583 		bzero(irq_name, sizeof(irq_name));
1584 
1585 		snprintf(irq_name, sizeof(irq_name), "txq%d", i);
1586 		iflib_softirq_alloc_generic(ctx, &sc->irqvs[vector].irq,
1587 					    IFLIB_INTR_TX, txq,
1588 					    txq->me, irq_name);
1589 		txq->irqv = &sc->irqvs[vector];
1590 	}
1591 
1592 	return (0);
1593 fail:
1594 	for (; i >= 0; i--, vector--)
1595 		iflib_irq_free(ctx, &sc->irqvs[vector].irq);
1596 	iflib_irq_free(ctx, &sc->irqvs[0].irq);
1597 free_irqvs:
1598 	free(sc->irqvs, M_ICE);
1599 	sc->irqvs = NULL;
1600 	return err;
1601 }
1602 
1603 /**
1604  * ice_if_mtu_set - Set the device MTU
1605  * @ctx: iflib context structure
1606  * @mtu: the MTU requested
1607  *
1608  * Called by iflib to configure the device's Maximum Transmission Unit (MTU).
1609  *
1610  * @pre assumes the caller holds the iflib CTX lock
1611  */
1612 static int
1613 ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
1614 {
1615 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1616 
1617 	ASSERT_CTX_LOCKED(sc);
1618 
1619 	/* Do not support configuration when in recovery mode */
1620 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1621 		return (ENOSYS);
1622 
1623 	if (mtu < ICE_MIN_MTU || mtu > ICE_MAX_MTU)
1624 		return (EINVAL);
1625 
1626 	sc->scctx->isc_max_frame_size = mtu +
1627 		ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN;
1628 
1629 	sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size;
1630 
1631 	return (0);
1632 }
1633 
1634 /**
1635  * ice_if_intr_enable - Enable device interrupts
1636  * @ctx: iflib context structure
1637  *
1638  * Called by iflib to request enabling device interrupts.
1639  */
1640 static void
1641 ice_if_intr_enable(if_ctx_t ctx)
1642 {
1643 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1644 	struct ice_vsi *vsi = &sc->pf_vsi;
1645 	struct ice_hw *hw = &sc->hw;
1646 
1647 	ASSERT_CTX_LOCKED(sc);
1648 
1649 	/* Enable ITR 0 */
1650 	ice_enable_intr(hw, sc->irqvs[0].me);
1651 
1652 	/* Do not enable queue interrupts in recovery mode */
1653 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1654 		return;
1655 
1656 	/* Enable all queue interrupts */
1657 	for (int i = 0; i < vsi->num_rx_queues; i++)
1658 		ice_enable_intr(hw, vsi->rx_queues[i].irqv->me);
1659 }
1660 
1661 /**
1662  * ice_if_intr_disable - Disable device interrupts
1663  * @ctx: iflib context structure
1664  *
1665  * Called by iflib to request disabling device interrupts.
1666  */
1667 static void
1668 ice_if_intr_disable(if_ctx_t ctx)
1669 {
1670 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1671 	struct ice_hw *hw = &sc->hw;
1672 	unsigned int i;
1673 
1674 	ASSERT_CTX_LOCKED(sc);
1675 
1676 	/* IFDI_INTR_DISABLE may be called prior to interrupts actually being
1677 	 * assigned to queues. Instead of assuming that the interrupt
1678 	 * assignment in the rx_queues structure is valid, just disable all
1679 	 * possible interrupts
1680 	 *
1681 	 * Note that we choose not to disable ITR 0 because this handles the
1682 	 * AdminQ interrupts, and we want to keep processing these even when
1683 	 * the interface is offline.
1684 	 */
1685 	for (i = 1; i < hw->func_caps.common_cap.num_msix_vectors; i++)
1686 		ice_disable_intr(hw, i);
1687 }
1688 
1689 /**
1690  * ice_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt
1691  * @ctx: iflib context structure
1692  * @rxqid: the Rx queue to enable
1693  *
1694  * Enable a specific Rx queue interrupt.
1695  *
1696  * This function is not protected by the iflib CTX lock.
1697  */
1698 static int
1699 ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
1700 {
1701 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1702 	struct ice_vsi *vsi = &sc->pf_vsi;
1703 	struct ice_hw *hw = &sc->hw;
1704 
1705 	/* Do not enable queue interrupts in recovery mode */
1706 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1707 		return (ENOSYS);
1708 
1709 	ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me);
1710 	return (0);
1711 }
1712 
1713 /**
1714  * ice_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt
1715  * @ctx: iflib context structure
1716  * @txqid: the Tx queue to enable
1717  *
1718  * Enable a specific Tx queue interrupt.
1719  *
1720  * This function is not protected by the iflib CTX lock.
1721  */
1722 static int
1723 ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid)
1724 {
1725 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1726 	struct ice_vsi *vsi = &sc->pf_vsi;
1727 	struct ice_hw *hw = &sc->hw;
1728 
1729 	/* Do not enable queue interrupts in recovery mode */
1730 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1731 		return (ENOSYS);
1732 
1733 	ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me);
1734 	return (0);
1735 }
1736 
1737 /**
1738  * ice_if_promisc_set - Set device promiscuous mode
1739  * @ctx: iflib context structure
1740  * @flags: promiscuous flags to configure
1741  *
1742  * Called by iflib to configure device promiscuous mode.
1743  *
1744  * @remark Calls to this function will always overwrite the previous setting
1745  */
1746 static int
1747 ice_if_promisc_set(if_ctx_t ctx, int flags)
1748 {
1749 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1750 	struct ice_hw *hw = &sc->hw;
1751 	device_t dev = sc->dev;
1752 	enum ice_status status;
1753 	bool promisc_enable = flags & IFF_PROMISC;
1754 	bool multi_enable = flags & IFF_ALLMULTI;
1755 
1756 	/* Do not support configuration when in recovery mode */
1757 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1758 		return (ENOSYS);
1759 
1760 	if (multi_enable)
1761 		return (EOPNOTSUPP);
1762 
1763 	if (promisc_enable) {
1764 		status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx,
1765 					     ICE_VSI_PROMISC_MASK, 0);
1766 		if (status && status != ICE_ERR_ALREADY_EXISTS) {
1767 			device_printf(dev,
1768 				      "Failed to enable promiscuous mode for PF VSI, err %s aq_err %s\n",
1769 				      ice_status_str(status),
1770 				      ice_aq_str(hw->adminq.sq_last_status));
1771 			return (EIO);
1772 		}
1773 	} else {
1774 		status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx,
1775 					       ICE_VSI_PROMISC_MASK, 0);
1776 		if (status) {
1777 			device_printf(dev,
1778 				      "Failed to disable promiscuous mode for PF VSI, err %s aq_err %s\n",
1779 				      ice_status_str(status),
1780 				      ice_aq_str(hw->adminq.sq_last_status));
1781 			return (EIO);
1782 		}
1783 	}
1784 
1785 	return (0);
1786 }
1787 
1788 /**
1789  * ice_if_media_change - Change device media
1790  * @ctx: device ctx structure
1791  *
1792  * Called by iflib when a media change is requested. This operation is not
1793  * supported by the hardware, so we just return an error code.
1794  */
1795 static int
1796 ice_if_media_change(if_ctx_t ctx)
1797 {
1798 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1799 
1800 	device_printf(sc->dev, "Media change is not supported.\n");
1801 	return (ENODEV);
1802 }
1803 
1804 /**
1805  * ice_if_media_status - Report current device media
1806  * @ctx: iflib context structure
1807  * @ifmr: ifmedia request structure to update
1808  *
1809  * Updates the provided ifmr with current device media status, including link
1810  * status and media type.
1811  */
1812 static void
1813 ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
1814 {
1815 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1816 	struct ice_link_status *li = &sc->hw.port_info->phy.link_info;
1817 
1818 	ifmr->ifm_status = IFM_AVALID;
1819 	ifmr->ifm_active = IFM_ETHER;
1820 
1821 	/* Never report link up or media types when in recovery mode */
1822 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1823 		return;
1824 
1825 	if (!sc->link_up)
1826 		return;
1827 
1828 	ifmr->ifm_status |= IFM_ACTIVE;
1829 	ifmr->ifm_active |= IFM_FDX;
1830 
1831 	if (li->phy_type_low)
1832 		ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low);
1833 	else if (li->phy_type_high)
1834 		ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high);
1835 	else
1836 		ifmr->ifm_active |= IFM_UNKNOWN;
1837 
1838 	/* Report flow control status as well */
1839 	if (li->an_info & ICE_AQ_LINK_PAUSE_TX)
1840 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1841 	if (li->an_info & ICE_AQ_LINK_PAUSE_RX)
1842 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1843 }
1844 
1845 /**
1846  * ice_init_tx_tracking - Initialize Tx queue software tracking values
1847  * @vsi: the VSI to initialize
1848  *
1849  * Initialize Tx queue software tracking values, including the Report Status
1850  * queue, and related software tracking values.
1851  */
1852 static void
1853 ice_init_tx_tracking(struct ice_vsi *vsi)
1854 {
1855 	struct ice_tx_queue *txq;
1856 	size_t j;
1857 	int i;
1858 
1859 	for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) {
1860 
1861 		txq->tx_rs_cidx = txq->tx_rs_pidx = 0;
1862 
1863 		/* Initialize the last processed descriptor to be the end of
1864 		 * the ring, rather than the start, so that we avoid an
1865 		 * off-by-one error in ice_ift_txd_credits_update for the
1866 		 * first packet.
1867 		 */
1868 		txq->tx_cidx_processed = txq->desc_count - 1;
1869 
1870 		for (j = 0; j < txq->desc_count; j++)
1871 			txq->tx_rsq[j] = QIDX_INVALID;
1872 	}
1873 }
1874 
1875 /**
1876  * ice_update_rx_mbuf_sz - Update the Rx buffer size for all queues
1877  * @sc: the device softc
1878  *
1879  * Called to update the Rx queue mbuf_sz parameter for configuring the receive
1880  * buffer sizes when programming hardware.
1881  */
1882 static void
1883 ice_update_rx_mbuf_sz(struct ice_softc *sc)
1884 {
1885 	uint32_t mbuf_sz = iflib_get_rx_mbuf_sz(sc->ctx);
1886 	struct ice_vsi *vsi = &sc->pf_vsi;
1887 
1888 	MPASS(mbuf_sz <= UINT16_MAX);
1889 	vsi->mbuf_sz = mbuf_sz;
1890 }
1891 
1892 /**
1893  * ice_if_init - Initialize the device
1894  * @ctx: iflib ctx structure
1895  *
1896  * Called by iflib to bring the device up, i.e. ifconfig ice0 up. Initializes
1897  * device filters and prepares the Tx and Rx engines.
1898  *
1899  * @pre assumes the caller holds the iflib CTX lock
1900  */
1901 static void
1902 ice_if_init(if_ctx_t ctx)
1903 {
1904 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
1905 	device_t dev = sc->dev;
1906 	int err;
1907 
1908 	ASSERT_CTX_LOCKED(sc);
1909 
1910 	/*
1911 	 * We've seen an issue with 11.3/12.1 where sideband routines are
1912 	 * called after detach is called.  This would call routines after
1913 	 * if_stop, causing issues with the teardown process.  This has
1914 	 * seemingly been fixed in STABLE snapshots, but it seems like a
1915 	 * good idea to have this guard here regardless.
1916 	 */
1917 	if (ice_driver_is_detaching(sc))
1918 		return;
1919 
1920 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
1921 		return;
1922 
1923 	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) {
1924 		device_printf(sc->dev, "request to start interface cannot be completed as the device failed to reset\n");
1925 		return;
1926 	}
1927 
1928 	if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
1929 		device_printf(sc->dev, "request to start interface while device is prepared for impending reset\n");
1930 		return;
1931 	}
1932 
1933 	ice_update_rx_mbuf_sz(sc);
1934 
1935 	/* Update the MAC address... User might use a LAA */
1936 	err = ice_update_laa_mac(sc);
1937 	if (err) {
1938 		device_printf(dev,
1939 			      "LAA address change failed, err %s\n",
1940 			      ice_err_str(err));
1941 		return;
1942 	}
1943 
1944 	/* Initialize software Tx tracking values */
1945 	ice_init_tx_tracking(&sc->pf_vsi);
1946 
1947 	err = ice_cfg_vsi_for_tx(&sc->pf_vsi);
1948 	if (err) {
1949 		device_printf(dev,
1950 			      "Unable to configure the main VSI for Tx: %s\n",
1951 			      ice_err_str(err));
1952 		return;
1953 	}
1954 
1955 	err = ice_cfg_vsi_for_rx(&sc->pf_vsi);
1956 	if (err) {
1957 		device_printf(dev,
1958 			      "Unable to configure the main VSI for Rx: %s\n",
1959 			      ice_err_str(err));
1960 		goto err_cleanup_tx;
1961 	}
1962 
1963 	err = ice_control_all_rx_queues(&sc->pf_vsi, true);
1964 	if (err) {
1965 		device_printf(dev,
1966 			      "Unable to enable Rx rings for transmit: %s\n",
1967 			      ice_err_str(err));
1968 		goto err_cleanup_tx;
1969 	}
1970 
1971 	err = ice_cfg_pf_default_mac_filters(sc);
1972 	if (err) {
1973 		device_printf(dev,
1974 			      "Unable to configure default MAC filters: %s\n",
1975 			      ice_err_str(err));
1976 		goto err_stop_rx;
1977 	}
1978 
1979 	/* We use software interrupts for Tx, so we only program the hardware
1980 	 * interrupts for Rx.
1981 	 */
1982 	ice_configure_all_rxq_interrupts(&sc->pf_vsi);
1983 	ice_configure_rx_itr(&sc->pf_vsi);
1984 
1985 	/* Configure promiscuous mode */
1986 	ice_if_promisc_set(ctx, if_getflags(sc->ifp));
1987 
1988 	ice_rdma_pf_init(sc);
1989 
1990 	ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED);
1991 	return;
1992 
1993 err_stop_rx:
1994 	ice_control_all_rx_queues(&sc->pf_vsi, false);
1995 err_cleanup_tx:
1996 	ice_vsi_disable_tx(&sc->pf_vsi);
1997 }
1998 
1999 /**
2000  * ice_poll_for_media_avail - Re-enable link if media is detected
2001  * @sc: device private structure
2002  *
2003  * Intended to be called from the driver's timer function, this function
2004  * sends the Get Link Status AQ command and re-enables HW link if the
2005  * command says that media is available.
2006  *
2007  * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing,
2008  * since media removal events are supposed to be sent to the driver through
2009  * a link status event.
2010  */
2011 static void
2012 ice_poll_for_media_avail(struct ice_softc *sc)
2013 {
2014 	struct ice_hw *hw = &sc->hw;
2015 	struct ice_port_info *pi = hw->port_info;
2016 
2017 	if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) {
2018 		pi->phy.get_link_info = true;
2019 		ice_get_link_status(pi, &sc->link_up);
2020 
2021 		if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
2022 			enum ice_status status;
2023 
2024 			/* Re-enable link and re-apply user link settings */
2025 			ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC);
2026 
2027 			/* Update the OS about changes in media capability */
2028 			status = ice_add_media_types(sc, sc->media);
2029 			if (status)
2030 				device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n",
2031 					      ice_status_str(status),
2032 					      ice_aq_str(hw->adminq.sq_last_status));
2033 
2034 			ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA);
2035 		}
2036 	}
2037 }
2038 
2039 /**
2040  * ice_if_timer - called by iflib periodically
2041  * @ctx: iflib ctx structure
2042  * @qid: the queue this timer was called for
2043  *
2044  * This callback is triggered by iflib periodically. We use it to update the
2045  * hw statistics.
2046  *
2047  * @remark this function is not protected by the iflib CTX lock.
2048  */
2049 static void
2050 ice_if_timer(if_ctx_t ctx, uint16_t qid)
2051 {
2052 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2053 	uint64_t prev_link_xoff_rx = sc->stats.cur.link_xoff_rx;
2054 
2055 	if (qid != 0)
2056 		return;
2057 
2058 	/* Do not attempt to update stats when in recovery mode */
2059 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2060 		return;
2061 
2062 	/* Update device statistics */
2063 	ice_update_pf_stats(sc);
2064 
2065 	/*
2066 	 * For proper watchdog management, the iflib stack needs to know if
2067 	 * we've been paused during the last interval. Check if the
2068 	 * link_xoff_rx stat changed, and set the isc_pause_frames, if so.
2069 	 */
2070 	if (sc->stats.cur.link_xoff_rx != prev_link_xoff_rx)
2071 		sc->scctx->isc_pause_frames = 1;
2072 
2073 	/* Update the primary VSI stats */
2074 	ice_update_vsi_hw_stats(&sc->pf_vsi);
2075 }
2076 
2077 /**
2078  * ice_admin_timer - called periodically to trigger the admin task
2079  * @arg: callout(9) argument pointing to the device private softc structure
2080  *
2081  * Timer function used as part of a callout(9) timer that will periodically
2082  * trigger the admin task, even when the interface is down.
2083  *
2084  * @remark this function is not called by iflib and is not protected by the
2085  * iflib CTX lock.
2086  *
2087  * @remark because this is a callout function, it cannot sleep and should not
2088  * attempt taking the iflib CTX lock.
2089  */
2090 static void
2091 ice_admin_timer(void *arg)
2092 {
2093 	struct ice_softc *sc = (struct ice_softc *)arg;
2094 
2095 	/*
2096 	 * There is a point where callout routines are no longer
2097 	 * cancelable.  So there exists a window of time where the
2098 	 * driver enters detach() and tries to cancel the callout, but the
2099 	 * callout routine has passed the cancellation point.  The detach()
2100 	 * routine is unaware of this and tries to free resources that the
2101 	 * callout routine needs.  So we check for the detach state flag to
2102 	 * at least shrink the window of opportunity.
2103 	 */
2104 	if (ice_driver_is_detaching(sc))
2105 		return;
2106 
2107 	/* Fire off the admin task */
2108 	iflib_admin_intr_deferred(sc->ctx);
2109 
2110 	/* Reschedule the admin timer */
2111 	callout_schedule(&sc->admin_timer, hz/2);
2112 }
2113 
2114 /**
2115  * ice_transition_recovery_mode - Transition to recovery mode
2116  * @sc: the device private softc
2117  *
2118  * Called when the driver detects that the firmware has entered recovery mode
2119  * at run time.
2120  */
2121 static void
2122 ice_transition_recovery_mode(struct ice_softc *sc)
2123 {
2124 	struct ice_vsi *vsi = &sc->pf_vsi;
2125 	int i;
2126 
2127 	device_printf(sc->dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
2128 
2129 	/* Tell the stack that the link has gone down */
2130 	iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0);
2131 
2132 	/* Request that the device be re-initialized */
2133 	ice_request_stack_reinit(sc);
2134 
2135 	ice_rdma_pf_detach(sc);
2136 	ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap);
2137 
2138 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
2139 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2140 
2141 	ice_vsi_del_txqs_ctx(vsi);
2142 	ice_vsi_del_rxqs_ctx(vsi);
2143 
2144 	for (i = 0; i < sc->num_available_vsi; i++) {
2145 		if (sc->all_vsi[i])
2146 			ice_release_vsi(sc->all_vsi[i]);
2147 	}
2148 	sc->num_available_vsi = 0;
2149 
2150 	if (sc->all_vsi) {
2151 		free(sc->all_vsi, M_ICE);
2152 		sc->all_vsi = NULL;
2153 	}
2154 
2155 	/* Destroy the interrupt manager */
2156 	ice_resmgr_destroy(&sc->imgr);
2157 	/* Destroy the queue managers */
2158 	ice_resmgr_destroy(&sc->tx_qmgr);
2159 	ice_resmgr_destroy(&sc->rx_qmgr);
2160 
2161 	ice_deinit_hw(&sc->hw);
2162 }
2163 
2164 /**
2165  * ice_transition_safe_mode - Transition to safe mode
2166  * @sc: the device private softc
2167  *
2168  * Called when the driver attempts to reload the DDP package during a device
2169  * reset, and the new download fails. If so, we must transition to safe mode
2170  * at run time.
2171  *
2172  * @remark although safe mode normally allocates only a single queue, we can't
2173  * change the number of queues dynamically when using iflib. Due to this, we
2174  * do not attempt to reduce the number of queues.
2175  */
2176 static void
2177 ice_transition_safe_mode(struct ice_softc *sc)
2178 {
2179 	/* Indicate that we are in Safe mode */
2180 	ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap);
2181 	ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en);
2182 
2183 	ice_rdma_pf_detach(sc);
2184 	ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap);
2185 
2186 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
2187 	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2188 
2189 	ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
2190 	ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en);
2191 }
2192 
2193 /**
2194  * ice_if_update_admin_status - update admin status
2195  * @ctx: iflib ctx structure
2196  *
2197  * Called by iflib to update the admin status. For our purposes, this means
2198  * check the adminq, and update the link status. It's ultimately triggered by
2199  * our admin interrupt, or by the ice_if_timer periodically.
2200  *
2201  * @pre assumes the caller holds the iflib CTX lock
2202  */
2203 static void
2204 ice_if_update_admin_status(if_ctx_t ctx)
2205 {
2206 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2207 	enum ice_fw_modes fw_mode;
2208 	bool reschedule = false;
2209 	u16 pending = 0;
2210 
2211 	ASSERT_CTX_LOCKED(sc);
2212 
2213 	/* Check if the firmware entered recovery mode at run time */
2214 	fw_mode = ice_get_fw_mode(&sc->hw);
2215 	if (fw_mode == ICE_FW_MODE_REC) {
2216 		if (!ice_testandset_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2217 			/* If we just entered recovery mode, log a warning to
2218 			 * the system administrator and deinit driver state
2219 			 * that is no longer functional.
2220 			 */
2221 			ice_transition_recovery_mode(sc);
2222 		}
2223 	} else if (fw_mode == ICE_FW_MODE_ROLLBACK) {
2224 		if (!ice_testandset_state(&sc->state, ICE_STATE_ROLLBACK_MODE)) {
2225 			/* Rollback mode isn't fatal, but we don't want to
2226 			 * repeatedly post a message about it.
2227 			 */
2228 			ice_print_rollback_msg(&sc->hw);
2229 		}
2230 	}
2231 
2232 	/* Handle global reset events */
2233 	ice_handle_reset_event(sc);
2234 
2235 	/* Handle PF reset requests */
2236 	ice_handle_pf_reset_request(sc);
2237 
2238 	/* Handle MDD events */
2239 	ice_handle_mdd_event(sc);
2240 
2241 	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) ||
2242 	    ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) ||
2243 	    ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2244 		/*
2245 		 * If we know the control queues are disabled, skip processing
2246 		 * the control queues entirely.
2247 		 */
2248 		;
2249 	} else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) {
2250 		ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending);
2251 		if (pending > 0)
2252 			reschedule = true;
2253 
2254 		ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending);
2255 		if (pending > 0)
2256 			reschedule = true;
2257 	}
2258 
2259 	/* Poll for link up */
2260 	ice_poll_for_media_avail(sc);
2261 
2262 	/* Check and update link status */
2263 	ice_update_link_status(sc, false);
2264 
2265 	/*
2266 	 * If there are still messages to process, we need to reschedule
2267 	 * ourselves. Otherwise, we can just re-enable the interrupt. We'll be
2268 	 * woken up at the next interrupt or timer event.
2269 	 */
2270 	if (reschedule) {
2271 		ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
2272 		iflib_admin_intr_deferred(ctx);
2273 	} else {
2274 		ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2275 	}
2276 }
2277 
2278 /**
2279  * ice_prepare_for_reset - Prepare device for an impending reset
2280  * @sc: The device private softc
2281  *
2282  * Prepare the driver for an impending reset, shutting down VSIs, clearing the
2283  * scheduler setup, and shutting down controlqs. Uses the
2284  * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the
2285  * driver for reset or not.
2286  */
2287 static void
2288 ice_prepare_for_reset(struct ice_softc *sc)
2289 {
2290 	struct ice_hw *hw = &sc->hw;
2291 
2292 	/* If we're already prepared, there's nothing to do */
2293 	if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET))
2294 		return;
2295 
2296 	log(LOG_INFO, "%s: preparing to reset device logic\n", if_name(sc->ifp));
2297 
2298 	/* In recovery mode, hardware is not initialized */
2299 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2300 		return;
2301 
2302 	/* stop the RDMA client */
2303 	ice_rdma_pf_stop(sc);
2304 
2305 	/* Release the main PF VSI queue mappings */
2306 	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2307 				    sc->pf_vsi.num_tx_queues);
2308 	ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
2309 				    sc->pf_vsi.num_rx_queues);
2310 
2311 	ice_clear_hw_tbls(hw);
2312 
2313 	if (hw->port_info)
2314 		ice_sched_clear_port(hw->port_info);
2315 
2316 	ice_shutdown_all_ctrlq(hw, false);
2317 }
2318 
2319 /**
2320  * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping
2321  * @sc: the device softc pointer
2322  *
2323  * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue
2324  * mapping after a reset occurred.
2325  */
2326 static int
2327 ice_rebuild_pf_vsi_qmap(struct ice_softc *sc)
2328 {
2329 	struct ice_vsi *vsi = &sc->pf_vsi;
2330 	struct ice_tx_queue *txq;
2331 	struct ice_rx_queue *rxq;
2332 	int err, i;
2333 
2334 	/* Re-assign Tx queues from PF space to the main VSI */
2335 	err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap,
2336 					    vsi->num_tx_queues);
2337 	if (err) {
2338 		device_printf(sc->dev, "Unable to re-assign PF Tx queues: %s\n",
2339 			      ice_err_str(err));
2340 		return (err);
2341 	}
2342 
2343 	/* Re-assign Rx queues from PF space to this VSI */
2344 	err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap,
2345 					    vsi->num_rx_queues);
2346 	if (err) {
2347 		device_printf(sc->dev, "Unable to re-assign PF Rx queues: %s\n",
2348 			      ice_err_str(err));
2349 		goto err_release_tx_queues;
2350 	}
2351 
2352 	vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS;
2353 
2354 	/* Re-assign Tx queue tail pointers */
2355 	for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++)
2356 		txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]);
2357 
2358 	/* Re-assign Rx queue tail pointers */
2359 	for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++)
2360 		rxq->tail = QRX_TAIL(vsi->rx_qmap[i]);
2361 
2362 	return (0);
2363 
2364 err_release_tx_queues:
2365 	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2366 				   sc->pf_vsi.num_tx_queues);
2367 
2368 	return (err);
2369 }
2370 
2371 /* determine if the iflib context is active */
2372 #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING))
2373 
2374 /**
2375  * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode
2376  * @sc: The device private softc
2377  *
2378  * Handle a driver rebuild while in recovery mode. This will only rebuild the
2379  * limited functionality supported while in recovery mode.
2380  */
2381 static void
2382 ice_rebuild_recovery_mode(struct ice_softc *sc)
2383 {
2384 	device_t dev = sc->dev;
2385 
2386 	/* enable PCIe bus master */
2387 	pci_enable_busmaster(dev);
2388 
2389 	/* Configure interrupt causes for the administrative interrupt */
2390 	ice_configure_misc_interrupts(sc);
2391 
2392 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
2393 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2394 
2395 	/* Now that the rebuild is finished, we're no longer prepared to reset */
2396 	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
2397 
2398 	log(LOG_INFO, "%s: device rebuild successful\n", if_name(sc->ifp));
2399 
2400 	/* In order to completely restore device functionality, the iflib core
2401 	 * needs to be reset. We need to request an iflib reset. Additionally,
2402 	 * because the state of IFC_DO_RESET is cached within task_fn_admin in
2403 	 * the iflib core, we also want re-run the admin task so that iflib
2404 	 * resets immediately instead of waiting for the next interrupt.
2405 	 */
2406 	ice_request_stack_reinit(sc);
2407 
2408 	return;
2409 }
2410 
2411 /**
2412  * ice_rebuild - Rebuild driver state post reset
2413  * @sc: The device private softc
2414  *
2415  * Restore driver state after a reset occurred. Restart the controlqs, setup
2416  * the hardware port, and re-enable the VSIs.
2417  */
2418 static void
2419 ice_rebuild(struct ice_softc *sc)
2420 {
2421 	struct ice_hw *hw = &sc->hw;
2422 	device_t dev = sc->dev;
2423 	enum ice_ddp_state pkg_state;
2424 	enum ice_status status;
2425 	int err;
2426 
2427 	sc->rebuild_ticks = ticks;
2428 
2429 	/* If we're rebuilding, then a reset has succeeded. */
2430 	ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED);
2431 
2432 	/*
2433 	 * If the firmware is in recovery mode, only restore the limited
2434 	 * functionality supported by recovery mode.
2435 	 */
2436 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) {
2437 		ice_rebuild_recovery_mode(sc);
2438 		return;
2439 	}
2440 
2441 	/* enable PCIe bus master */
2442 	pci_enable_busmaster(dev);
2443 
2444 	status = ice_init_all_ctrlq(hw);
2445 	if (status) {
2446 		device_printf(dev, "failed to re-init controlqs, err %s\n",
2447 			      ice_status_str(status));
2448 		goto err_shutdown_ctrlq;
2449 	}
2450 
2451 	/* Query the allocated resources for Tx scheduler */
2452 	status = ice_sched_query_res_alloc(hw);
2453 	if (status) {
2454 		device_printf(dev,
2455 			      "Failed to query scheduler resources, err %s aq_err %s\n",
2456 			      ice_status_str(status),
2457 			      ice_aq_str(hw->adminq.sq_last_status));
2458 		goto err_shutdown_ctrlq;
2459 	}
2460 
2461 	/* Re-enable FW logging. Keep going even if this fails */
2462 	status = ice_fwlog_set(hw, &hw->fwlog_cfg);
2463 	if (!status) {
2464 		/*
2465 		 * We should have the most updated cached copy of the
2466 		 * configuration, regardless of whether we're rebuilding
2467 		 * or not.  So we'll simply check to see if logging was
2468 		 * enabled pre-rebuild.
2469 		 */
2470 		if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) {
2471 			status = ice_fwlog_register(hw);
2472 			if (status)
2473 				device_printf(dev, "failed to re-register fw logging, err %s aq_err %s\n",
2474 				   ice_status_str(status),
2475 				   ice_aq_str(hw->adminq.sq_last_status));
2476 		}
2477 	} else
2478 		device_printf(dev, "failed to rebuild fw logging configuration, err %s aq_err %s\n",
2479 		   ice_status_str(status),
2480 		   ice_aq_str(hw->adminq.sq_last_status));
2481 
2482 	err = ice_send_version(sc);
2483 	if (err)
2484 		goto err_shutdown_ctrlq;
2485 
2486 	err = ice_init_link_events(sc);
2487 	if (err) {
2488 		device_printf(dev, "ice_init_link_events failed: %s\n",
2489 			      ice_err_str(err));
2490 		goto err_shutdown_ctrlq;
2491 	}
2492 
2493 	status = ice_clear_pf_cfg(hw);
2494 	if (status) {
2495 		device_printf(dev, "failed to clear PF configuration, err %s\n",
2496 			      ice_status_str(status));
2497 		goto err_shutdown_ctrlq;
2498 	}
2499 
2500 	ice_clear_pxe_mode(hw);
2501 
2502 	status = ice_get_caps(hw);
2503 	if (status) {
2504 		device_printf(dev, "failed to get capabilities, err %s\n",
2505 			      ice_status_str(status));
2506 		goto err_shutdown_ctrlq;
2507 	}
2508 
2509 	status = ice_sched_init_port(hw->port_info);
2510 	if (status) {
2511 		device_printf(dev, "failed to initialize port, err %s\n",
2512 			      ice_status_str(status));
2513 		goto err_sched_cleanup;
2514 	}
2515 
2516 	/* If we previously loaded the package, it needs to be reloaded now */
2517 	if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) {
2518 		pkg_state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size);
2519 		if (!ice_is_init_pkg_successful(pkg_state)) {
2520 			ice_log_pkg_init(sc, pkg_state);
2521 			ice_transition_safe_mode(sc);
2522 		}
2523 	}
2524 
2525 	ice_reset_pf_stats(sc);
2526 
2527 	err = ice_rebuild_pf_vsi_qmap(sc);
2528 	if (err) {
2529 		device_printf(sc->dev, "Unable to re-assign main VSI queues, err %s\n",
2530 			      ice_err_str(err));
2531 		goto err_sched_cleanup;
2532 	}
2533 	err = ice_initialize_vsi(&sc->pf_vsi);
2534 	if (err) {
2535 		device_printf(sc->dev, "Unable to re-initialize Main VSI, err %s\n",
2536 			      ice_err_str(err));
2537 		goto err_release_queue_allocations;
2538 	}
2539 
2540 	/* Replay all VSI configuration */
2541 	err = ice_replay_all_vsi_cfg(sc);
2542 	if (err)
2543 		goto err_deinit_pf_vsi;
2544 
2545 	/* Re-enable FW health event reporting */
2546 	ice_init_health_events(sc);
2547 
2548 	/* Reconfigure the main PF VSI for RSS */
2549 	err = ice_config_rss(&sc->pf_vsi);
2550 	if (err) {
2551 		device_printf(sc->dev,
2552 			      "Unable to reconfigure RSS for the main VSI, err %s\n",
2553 			      ice_err_str(err));
2554 		goto err_deinit_pf_vsi;
2555 	}
2556 
2557 	/* Refresh link status */
2558 	ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED);
2559 	sc->hw.port_info->phy.get_link_info = true;
2560 	ice_get_link_status(sc->hw.port_info, &sc->link_up);
2561 	ice_update_link_status(sc, true);
2562 
2563 	/* RDMA interface will be restarted by the stack re-init */
2564 
2565 	/* Configure interrupt causes for the administrative interrupt */
2566 	ice_configure_misc_interrupts(sc);
2567 
2568 	/* Enable ITR 0 right away, so that we can handle admin interrupts */
2569 	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
2570 
2571 	/* Now that the rebuild is finished, we're no longer prepared to reset */
2572 	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
2573 
2574 	log(LOG_INFO, "%s: device rebuild successful\n", if_name(sc->ifp));
2575 
2576 	/* In order to completely restore device functionality, the iflib core
2577 	 * needs to be reset. We need to request an iflib reset. Additionally,
2578 	 * because the state of IFC_DO_RESET is cached within task_fn_admin in
2579 	 * the iflib core, we also want re-run the admin task so that iflib
2580 	 * resets immediately instead of waiting for the next interrupt.
2581 	 */
2582 	ice_request_stack_reinit(sc);
2583 
2584 	return;
2585 
2586 err_deinit_pf_vsi:
2587 	ice_deinit_vsi(&sc->pf_vsi);
2588 err_release_queue_allocations:
2589 	ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap,
2590 				    sc->pf_vsi.num_tx_queues);
2591 	ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap,
2592 				    sc->pf_vsi.num_rx_queues);
2593 err_sched_cleanup:
2594 	ice_sched_cleanup_all(hw);
2595 err_shutdown_ctrlq:
2596 	ice_shutdown_all_ctrlq(hw, false);
2597 	ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET);
2598 	ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2599 	device_printf(dev, "Driver rebuild failed, please reload the device driver\n");
2600 }
2601 
2602 /**
2603  * ice_handle_reset_event - Handle reset events triggered by OICR
2604  * @sc: The device private softc
2605  *
2606  * Handle reset events triggered by an OICR notification. This includes CORER,
2607  * GLOBR, and EMPR resets triggered by software on this or any other PF or by
2608  * firmware.
2609  *
2610  * @pre assumes the iflib context lock is held, and will unlock it while
2611  * waiting for the hardware to finish reset.
2612  */
2613 static void
2614 ice_handle_reset_event(struct ice_softc *sc)
2615 {
2616 	struct ice_hw *hw = &sc->hw;
2617 	enum ice_status status;
2618 	device_t dev = sc->dev;
2619 
2620 	/* When a CORER, GLOBR, or EMPR is about to happen, the hardware will
2621 	 * trigger an OICR interrupt. Our OICR handler will determine when
2622 	 * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as
2623 	 * appropriate.
2624 	 */
2625 	if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV))
2626 		return;
2627 
2628 	ice_prepare_for_reset(sc);
2629 
2630 	/*
2631 	 * Release the iflib context lock and wait for the device to finish
2632 	 * resetting.
2633 	 */
2634 	IFLIB_CTX_UNLOCK(sc);
2635 	status = ice_check_reset(hw);
2636 	IFLIB_CTX_LOCK(sc);
2637 	if (status) {
2638 		device_printf(dev, "Device never came out of reset, err %s\n",
2639 			      ice_status_str(status));
2640 		ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2641 		return;
2642 	}
2643 
2644 	/* We're done with the reset, so we can rebuild driver state */
2645 	sc->hw.reset_ongoing = false;
2646 	ice_rebuild(sc);
2647 
2648 	/* In the unlikely event that a PF reset request occurs at the same
2649 	 * time as a global reset, clear the request now. This avoids
2650 	 * resetting a second time right after we reset due to a global event.
2651 	 */
2652 	if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
2653 		device_printf(dev, "Ignoring PFR request that occurred while a reset was ongoing\n");
2654 }
2655 
2656 /**
2657  * ice_handle_pf_reset_request - Initiate PF reset requested by software
2658  * @sc: The device private softc
2659  *
2660  * Initiate a PF reset requested by software. We handle this in the admin task
2661  * so that only one thread actually handles driver preparation and cleanup,
2662  * rather than having multiple threads possibly attempt to run this code
2663  * simultaneously.
2664  *
2665  * @pre assumes the iflib context lock is held and will unlock it while
2666  * waiting for the PF reset to complete.
2667  */
2668 static void
2669 ice_handle_pf_reset_request(struct ice_softc *sc)
2670 {
2671 	struct ice_hw *hw = &sc->hw;
2672 	enum ice_status status;
2673 
2674 	/* Check for PF reset requests */
2675 	if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ))
2676 		return;
2677 
2678 	/* Make sure we're prepared for reset */
2679 	ice_prepare_for_reset(sc);
2680 
2681 	/*
2682 	 * Release the iflib context lock and wait for the device to finish
2683 	 * resetting.
2684 	 */
2685 	IFLIB_CTX_UNLOCK(sc);
2686 	status = ice_reset(hw, ICE_RESET_PFR);
2687 	IFLIB_CTX_LOCK(sc);
2688 	if (status) {
2689 		device_printf(sc->dev, "device PF reset failed, err %s\n",
2690 			      ice_status_str(status));
2691 		ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
2692 		return;
2693 	}
2694 
2695 	sc->soft_stats.pfr_count++;
2696 	ice_rebuild(sc);
2697 }
2698 
2699 /**
2700  * ice_init_device_features - Init device driver features
2701  * @sc: driver softc structure
2702  *
2703  * @pre assumes that the function capabilities bits have been set up by
2704  * ice_init_hw().
2705  */
2706 static void
2707 ice_init_device_features(struct ice_softc *sc)
2708 {
2709 	/* Set capabilities that all devices support */
2710 	ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap);
2711 	ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap);
2712 	ice_set_bit(ICE_FEATURE_RDMA, sc->feat_cap);
2713 	ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap);
2714 	ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_cap);
2715 	ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_cap);
2716 	ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap);
2717 	ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap);
2718 	ice_set_bit(ICE_FEATURE_HAS_PBA, sc->feat_cap);
2719 	ice_set_bit(ICE_FEATURE_DCB, sc->feat_cap);
2720 	ice_set_bit(ICE_FEATURE_TX_BALANCE, sc->feat_cap);
2721 
2722 	/* Disable features due to hardware limitations... */
2723 	if (!sc->hw.func_caps.common_cap.rss_table_size)
2724 		ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap);
2725 	if (!sc->hw.func_caps.common_cap.iwarp || !ice_enable_irdma)
2726 		ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap);
2727 	if (!sc->hw.func_caps.common_cap.dcb)
2728 		ice_clear_bit(ICE_FEATURE_DCB, sc->feat_cap);
2729 	/* Disable features due to firmware limitations... */
2730 	if (!ice_is_fw_health_report_supported(&sc->hw))
2731 		ice_clear_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap);
2732 	if (!ice_fwlog_supported(&sc->hw))
2733 		ice_clear_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap);
2734 	if (sc->hw.fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) {
2735 		if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_FW_LOGGING))
2736 			ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_en);
2737 		else
2738 			ice_fwlog_unregister(&sc->hw);
2739 	}
2740 
2741 	/* Disable capabilities not supported by the OS */
2742 	ice_disable_unsupported_features(sc->feat_cap);
2743 
2744 	/* RSS is always enabled for iflib */
2745 	if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS))
2746 		ice_set_bit(ICE_FEATURE_RSS, sc->feat_en);
2747 
2748 	/* Disable features based on sysctl settings */
2749 	if (!ice_tx_balance_en)
2750 		ice_clear_bit(ICE_FEATURE_TX_BALANCE, sc->feat_cap);
2751 }
2752 
2753 /**
2754  * ice_if_multi_set - Callback to update Multicast filters in HW
2755  * @ctx: iflib ctx structure
2756  *
2757  * Called by iflib in response to SIOCDELMULTI and SIOCADDMULTI. Must search
2758  * the if_multiaddrs list and determine which filters have been added or
2759  * removed from the list, and update HW programming to reflect the new list.
2760  *
2761  * @pre assumes the caller holds the iflib CTX lock
2762  */
2763 static void
2764 ice_if_multi_set(if_ctx_t ctx)
2765 {
2766 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2767 	int err;
2768 
2769 	ASSERT_CTX_LOCKED(sc);
2770 
2771 	/* Do not handle multicast configuration in recovery mode */
2772 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2773 		return;
2774 
2775 	err = ice_sync_multicast_filters(sc);
2776 	if (err) {
2777 		device_printf(sc->dev,
2778 			      "Failed to synchronize multicast filter list: %s\n",
2779 			      ice_err_str(err));
2780 		return;
2781 	}
2782 }
2783 
2784 /**
2785  * ice_if_vlan_register - Register a VLAN with the hardware
2786  * @ctx: iflib ctx pointer
2787  * @vtag: VLAN to add
2788  *
2789  * Programs the main PF VSI with a hardware filter for the given VLAN.
2790  *
2791  * @pre assumes the caller holds the iflib CTX lock
2792  */
2793 static void
2794 ice_if_vlan_register(if_ctx_t ctx, u16 vtag)
2795 {
2796 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2797 	enum ice_status status;
2798 
2799 	ASSERT_CTX_LOCKED(sc);
2800 
2801 	/* Do not handle VLAN configuration in recovery mode */
2802 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2803 		return;
2804 
2805 	status = ice_add_vlan_hw_filter(&sc->pf_vsi, vtag);
2806 	if (status) {
2807 		device_printf(sc->dev,
2808 			      "Failure adding VLAN %d to main VSI, err %s aq_err %s\n",
2809 			      vtag, ice_status_str(status),
2810 			      ice_aq_str(sc->hw.adminq.sq_last_status));
2811 	}
2812 }
2813 
2814 /**
2815  * ice_if_vlan_unregister - Remove a VLAN filter from the hardware
2816  * @ctx: iflib ctx pointer
2817  * @vtag: VLAN to add
2818  *
2819  * Removes the previously programmed VLAN filter from the main PF VSI.
2820  *
2821  * @pre assumes the caller holds the iflib CTX lock
2822  */
2823 static void
2824 ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
2825 {
2826 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2827 	enum ice_status status;
2828 
2829 	ASSERT_CTX_LOCKED(sc);
2830 
2831 	/* Do not handle VLAN configuration in recovery mode */
2832 	if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
2833 		return;
2834 
2835 	status = ice_remove_vlan_hw_filter(&sc->pf_vsi, vtag);
2836 	if (status) {
2837 		device_printf(sc->dev,
2838 			      "Failure removing VLAN %d from main VSI, err %s aq_err %s\n",
2839 			      vtag, ice_status_str(status),
2840 			      ice_aq_str(sc->hw.adminq.sq_last_status));
2841 	}
2842 }
2843 
2844 /**
2845  * ice_if_stop - Stop the device
2846  * @ctx: iflib context structure
2847  *
2848  * Called by iflib to stop the device and bring it down. (i.e. ifconfig ice0
2849  * down)
2850  *
2851  * @pre assumes the caller holds the iflib CTX lock
2852  */
2853 static void
2854 ice_if_stop(if_ctx_t ctx)
2855 {
2856 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2857 
2858 	ASSERT_CTX_LOCKED(sc);
2859 
2860 	/*
2861 	 * The iflib core may call IFDI_STOP prior to the first call to
2862 	 * IFDI_INIT. This will cause us to attempt to remove MAC filters we
2863 	 * don't have, and disable Tx queues which aren't yet configured.
2864 	 * Although it is likely these extra operations are harmless, they do
2865 	 * cause spurious warning messages to be displayed, which may confuse
2866 	 * users.
2867 	 *
2868 	 * To avoid these messages, we use a state bit indicating if we've
2869 	 * been initialized. It will be set when ice_if_init is called, and
2870 	 * cleared here in ice_if_stop.
2871 	 */
2872 	if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED))
2873 		return;
2874 
2875 	if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) {
2876 		device_printf(sc->dev, "request to stop interface cannot be completed as the device failed to reset\n");
2877 		return;
2878 	}
2879 
2880 	if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
2881 		device_printf(sc->dev, "request to stop interface while device is prepared for impending reset\n");
2882 		return;
2883 	}
2884 
2885 	ice_rdma_pf_stop(sc);
2886 
2887 	/* Remove the MAC filters, stop Tx, and stop Rx. We don't check the
2888 	 * return of these functions because there's nothing we can really do
2889 	 * if they fail, and the functions already print error messages.
2890 	 * Just try to shut down as much as we can.
2891 	 */
2892 	ice_rm_pf_default_mac_filters(sc);
2893 
2894 	/* Dissociate the Tx and Rx queues from the interrupts */
2895 	ice_flush_txq_interrupts(&sc->pf_vsi);
2896 	ice_flush_rxq_interrupts(&sc->pf_vsi);
2897 
2898 	/* Disable the Tx and Rx queues */
2899 	ice_vsi_disable_tx(&sc->pf_vsi);
2900 	ice_control_all_rx_queues(&sc->pf_vsi, false);
2901 }
2902 
2903 /**
2904  * ice_if_get_counter - Get current value of an ifnet statistic
2905  * @ctx: iflib context pointer
2906  * @counter: ifnet counter to read
2907  *
2908  * Reads the current value of an ifnet counter for the device.
2909  *
2910  * This function is not protected by the iflib CTX lock.
2911  */
2912 static uint64_t
2913 ice_if_get_counter(if_ctx_t ctx, ift_counter counter)
2914 {
2915 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2916 
2917 	/* Return the counter for the main PF VSI */
2918 	return ice_get_ifnet_counter(&sc->pf_vsi, counter);
2919 }
2920 
2921 /**
2922  * ice_request_stack_reinit - Request that iflib re-initialize
2923  * @sc: the device private softc
2924  *
2925  * Request that the device be brought down and up, to re-initialize. For
2926  * example, this may be called when a device reset occurs, or when Tx and Rx
2927  * queues need to be re-initialized.
2928  *
2929  * This is required because the iflib state is outside the driver, and must be
2930  * re-initialized if we need to resart Tx and Rx queues.
2931  */
2932 void
2933 ice_request_stack_reinit(struct ice_softc *sc)
2934 {
2935 	if (CTX_ACTIVE(sc->ctx)) {
2936 		iflib_request_reset(sc->ctx);
2937 		iflib_admin_intr_deferred(sc->ctx);
2938 	}
2939 }
2940 
2941 /**
2942  * ice_driver_is_detaching - Check if the driver is detaching/unloading
2943  * @sc: device private softc
2944  *
2945  * Returns true if the driver is detaching, false otherwise.
2946  *
2947  * @remark on newer kernels, take advantage of iflib_in_detach in order to
2948  * report detachment correctly as early as possible.
2949  *
2950  * @remark this function is used by various code paths that want to avoid
2951  * running if the driver is about to be removed. This includes sysctls and
2952  * other driver access points. Note that it does not fully resolve
2953  * detach-based race conditions as it is possible for a thread to race with
2954  * iflib_in_detach.
2955  */
2956 bool
2957 ice_driver_is_detaching(struct ice_softc *sc)
2958 {
2959 	return (ice_test_state(&sc->state, ICE_STATE_DETACHING) ||
2960 		iflib_in_detach(sc->ctx));
2961 }
2962 
2963 /**
2964  * ice_if_priv_ioctl - Device private ioctl handler
2965  * @ctx: iflib context pointer
2966  * @command: The ioctl command issued
2967  * @data: ioctl specific data
2968  *
2969  * iflib callback for handling custom driver specific ioctls.
2970  *
2971  * @pre Assumes that the iflib context lock is held.
2972  */
2973 static int
2974 ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data)
2975 {
2976 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
2977 	struct ifdrv *ifd;
2978 	device_t dev = sc->dev;
2979 
2980 	if (data == NULL)
2981 		return (EINVAL);
2982 
2983 	ASSERT_CTX_LOCKED(sc);
2984 
2985 	/* Make sure the command type is valid */
2986 	switch (command) {
2987 	case SIOCSDRVSPEC:
2988 	case SIOCGDRVSPEC:
2989 		/* Accepted commands */
2990 		break;
2991 	case SIOCGPRIVATE_0:
2992 		/*
2993 		 * Although we do not support this ioctl command, it's
2994 		 * expected that iflib will forward it to the IFDI_PRIV_IOCTL
2995 		 * handler. Do not print a message in this case
2996 		 */
2997 		return (ENOTSUP);
2998 	default:
2999 		/*
3000 		 * If we get a different command for this function, it's
3001 		 * definitely unexpected, so log a message indicating what
3002 		 * command we got for debugging purposes.
3003 		 */
3004 		device_printf(dev, "%s: unexpected ioctl command %08lx\n",
3005 			      __func__, command);
3006 		return (EINVAL);
3007 	}
3008 
3009 	ifd = (struct ifdrv *)data;
3010 
3011 	switch (ifd->ifd_cmd) {
3012 	case ICE_NVM_ACCESS:
3013 		return ice_handle_nvm_access_ioctl(sc, ifd);
3014 	case ICE_DEBUG_DUMP:
3015 		return ice_handle_debug_dump_ioctl(sc, ifd);
3016 	default:
3017 		return EINVAL;
3018 	}
3019 }
3020 
3021 /**
3022  * ice_if_i2c_req - I2C request handler for iflib
3023  * @ctx: iflib context pointer
3024  * @req: The I2C parameters to use
3025  *
3026  * Read from the port's I2C eeprom using the parameters from the ioctl.
3027  *
3028  * @remark The iflib-only part is pretty simple.
3029  */
3030 static int
3031 ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req)
3032 {
3033 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
3034 
3035 	return ice_handle_i2c_req(sc, req);
3036 }
3037 
3038 /**
3039  * ice_if_suspend - PCI device suspend handler for iflib
3040  * @ctx: iflib context pointer
3041  *
3042  * Deinitializes the driver and clears HW resources in preparation for
3043  * suspend or an FLR.
3044  *
3045  * @returns 0; this return value is ignored
3046  */
3047 static int
3048 ice_if_suspend(if_ctx_t ctx)
3049 {
3050 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
3051 
3052 	/* At least a PFR is always going to happen after this;
3053 	 * either via FLR or during the D3->D0 transition.
3054 	 */
3055 	ice_clear_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
3056 
3057 	ice_prepare_for_reset(sc);
3058 
3059 	return (0);
3060 }
3061 
3062 /**
3063  * ice_if_resume - PCI device resume handler for iflib
3064  * @ctx: iflib context pointer
3065  *
3066  * Reinitializes the driver and the HW after PCI resume or after
3067  * an FLR. An init is performed by iflib after this function is finished.
3068  *
3069  * @returns 0; this return value is ignored
3070  */
3071 static int
3072 ice_if_resume(if_ctx_t ctx)
3073 {
3074 	struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx);
3075 
3076 	ice_rebuild(sc);
3077 
3078 	return (0);
3079 }
3080 
3081