1 /*****************************************************************************
2
3 Copyright (c) 2001-2017, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32 *****************************************************************************/
33
34
35 #include "opt_inet.h"
36 #include "opt_inet6.h"
37 #include "opt_rss.h"
38
39 #include "ixgbe.h"
40 #include "ixgbe_hv_vf.h"
41 #include "ifdi_if.h"
42
43 #include <net/netmap.h>
44 #include <dev/netmap/netmap_kern.h>
45
46 /************************************************************************
47 * Driver version
48 ************************************************************************/
49 static const char ixv_driver_version[] = "2.0.1-k";
50
51 static const sbintime_t ixv_mbx_retry_delay[] = {
52 250 * SBT_1MS,
53 1 * SBT_1S,
54 4 * SBT_1S,
55 8 * SBT_1S,
56 };
57
58 static const struct timeval ixv_mbx_log_interval = { 60, 0 };
59
60 /* Bound stale carrier state without flapping on a busy PF mailbox. */
61 #define IXV_LINK_MBX_FAILURE_LIMIT 3
62 /* Match Intel's two-second VF service timer and spread PF mailbox load. */
63 #define IXV_LINK_POLL_TICKS 4
64
65 /************************************************************************
66 * PCI Device ID Table
67 *
68 * Used by probe to select devices to load on
69 * Last field stores an index into ixv_strings
70 * Last entry must be all 0s
71 *
72 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
73 ************************************************************************/
74 static const pci_vendor_info_t ixv_vendor_info_array[] =
75 {
76 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF,
77 "Intel(R) X520 82599 Virtual Function"),
78 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF_HV,
79 "Intel(R) X520 82599 Hyper-V Virtual Function"),
80 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF,
81 "Intel(R) X540 Virtual Function"),
82 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF_HV,
83 "Intel(R) X540 Hyper-V Virtual Function"),
84 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF,
85 "Intel(R) X550 Virtual Function"),
86 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF_HV,
87 "Intel(R) X550 Hyper-V Virtual Function"),
88 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF,
89 "Intel(R) X552 Virtual Function"),
90 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF_HV,
91 "Intel(R) X552 Hyper-V Virtual Function"),
92 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF,
93 "Intel(R) X553 Virtual Function"),
94 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF_HV,
95 "Intel(R) X553 Hyper-V Virtual Function"),
96 PVID_OEM(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_VF, 0,
97 IXGBE_SUBDEV_ID_E610_VF_HV, 0,
98 "Intel(R) E610 Hyper-V Virtual Function"),
99 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_VF,
100 "Intel(R) E610 Virtual Function"),
101 /* required last entry */
102 PVID_END
103 };
104
105 /************************************************************************
106 * Function prototypes
107 ************************************************************************/
108 static void *ixv_register(device_t);
109 static int ixv_if_attach_pre(if_ctx_t);
110 static int ixv_if_attach_post(if_ctx_t);
111 static int ixv_if_detach(if_ctx_t);
112
113 static int ixv_if_rx_queue_intr_enable(if_ctx_t, uint16_t);
114 static int ixv_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int,
115 int);
116 static int ixv_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int,
117 int);
118 static void ixv_if_queues_free(if_ctx_t);
119 static void ixv_identify_hardware(if_ctx_t);
120 static bool ixv_is_hyperv(const struct ixgbe_hw *);
121 static void ixv_init_device_features(struct ixgbe_softc *);
122 static int ixv_allocate_pci_resources(if_ctx_t);
123 static void ixv_free_pci_resources(if_ctx_t);
124 static int ixv_setup_interface(if_ctx_t);
125 static void ixv_if_media_status(if_ctx_t, struct ifmediareq *);
126 static int ixv_if_media_change(if_ctx_t);
127 static void ixv_if_update_admin_status(if_ctx_t);
128 static int ixv_if_msix_intr_assign(if_ctx_t, int);
129
130 static int ixv_if_mtu_set(if_ctx_t, uint32_t);
131 static int ixv_if_get_rss_key(if_ctx_t, struct ifrsskey *);
132 static int ixv_if_get_rss_hash(if_ctx_t, struct ifrsshash *);
133 static void ixv_reconcile_mac(struct ixgbe_softc *, if_t);
134 static void ixv_if_init(if_ctx_t);
135 static void ixv_if_local_timer(if_ctx_t, uint16_t);
136 static void ixv_if_stop(if_ctx_t);
137 static void ixv_log_negotiate_failure(struct ixgbe_softc *, bool);
138 static void ixv_log_reset_failure(struct ixgbe_softc *, s32, bool);
139 static void ixv_mbx_retry_detach(struct ixgbe_softc *);
140 static void ixv_mbx_retry_failed(if_ctx_t);
141 static void ixv_mbx_retry_prepare(struct ixgbe_softc *);
142 static void ixv_mbx_retry_stop(struct ixgbe_softc *);
143 static void ixv_mbx_retry_succeeded(struct ixgbe_softc *);
144 static int ixv_negotiate_api(struct ixgbe_softc *);
145 static int ixv_queue_limit(struct ixgbe_softc *, bool);
146
147 static void ixv_initialize_transmit_units(if_ctx_t);
148 static void ixv_initialize_receive_units(if_ctx_t);
149 static void ixv_initialize_rss_mapping(struct ixgbe_softc *);
150
151 static void ixv_setup_vlan_support(if_ctx_t);
152 static bool ixv_vlan_error_retryable(s32);
153 static void ixv_vlan_retry_add(struct ixgbe_softc *, u16);
154 static void ixv_vlan_retry_clear(struct ixgbe_softc *, u16);
155 static bool ixv_vlan_retry_pending(const struct ixgbe_softc *);
156 static void ixv_vlan_retry_tick(struct ixgbe_softc *);
157 static void ixv_configure_ivars(struct ixgbe_softc *);
158 static void ixv_if_enable_intr(if_ctx_t);
159 static void ixv_if_disable_intr(if_ctx_t);
160 static void ixv_if_multi_set(if_ctx_t);
161 static int ixv_if_promisc_set(if_ctx_t, int);
162
163 static void ixv_if_register_vlan(if_ctx_t, u16);
164 static void ixv_if_unregister_vlan(if_ctx_t, u16);
165
166 static uint64_t ixv_if_get_counter(if_ctx_t, ift_counter);
167 static bool ixv_if_needs_restart(if_ctx_t, enum iflib_restart_event);
168
169 static void ixv_init_stats(struct ixgbe_softc *);
170 static void ixv_update_stats(struct ixgbe_softc *);
171 static void ixv_add_stats_sysctls(struct ixgbe_softc *);
172
173 static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS);
174 static void ixv_set_ivar(struct ixgbe_softc *, u8, u8, s8);
175
176 static u8 *ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
177
178 /* The MSI-X Interrupt handlers */
179 static int ixv_msix_que(void *);
180 static int ixv_msix_mbx(void *);
181
182 /************************************************************************
183 * FreeBSD Device Interface Entry Points
184 ************************************************************************/
185 static device_method_t ixv_methods[] = {
186 /* Device interface */
187 DEVMETHOD(device_register, ixv_register),
188 DEVMETHOD(device_probe, iflib_device_probe),
189 DEVMETHOD(device_attach, iflib_device_attach),
190 DEVMETHOD(device_detach, iflib_device_detach),
191 DEVMETHOD(device_shutdown, iflib_device_shutdown),
192 DEVMETHOD(device_suspend, iflib_device_suspend),
193 DEVMETHOD(device_resume, iflib_device_resume),
194 DEVMETHOD_END
195 };
196
197 static driver_t ixv_driver = {
198 "ixv", ixv_methods, sizeof(struct ixgbe_softc),
199 };
200
201 DRIVER_MODULE(ixv, pci, ixv_driver, 0, 0);
202 IFLIB_PNP_INFO(pci, ixv_driver, ixv_vendor_info_array);
203 MODULE_DEPEND(ixv, iflib, 1, 1, 1);
204 MODULE_DEPEND(ixv, pci, 1, 1, 1);
205 MODULE_DEPEND(ixv, ether, 1, 1, 1);
206
207 static device_method_t ixv_if_methods[] = {
208 DEVMETHOD(ifdi_attach_pre, ixv_if_attach_pre),
209 DEVMETHOD(ifdi_attach_post, ixv_if_attach_post),
210 DEVMETHOD(ifdi_detach, ixv_if_detach),
211 DEVMETHOD(ifdi_init, ixv_if_init),
212 DEVMETHOD(ifdi_stop, ixv_if_stop),
213 DEVMETHOD(ifdi_msix_intr_assign, ixv_if_msix_intr_assign),
214 DEVMETHOD(ifdi_intr_enable, ixv_if_enable_intr),
215 DEVMETHOD(ifdi_intr_disable, ixv_if_disable_intr),
216 DEVMETHOD(ifdi_tx_queue_intr_enable, ixv_if_rx_queue_intr_enable),
217 DEVMETHOD(ifdi_rx_queue_intr_enable, ixv_if_rx_queue_intr_enable),
218 DEVMETHOD(ifdi_tx_queues_alloc, ixv_if_tx_queues_alloc),
219 DEVMETHOD(ifdi_rx_queues_alloc, ixv_if_rx_queues_alloc),
220 DEVMETHOD(ifdi_queues_free, ixv_if_queues_free),
221 DEVMETHOD(ifdi_update_admin_status, ixv_if_update_admin_status),
222 DEVMETHOD(ifdi_multi_set, ixv_if_multi_set),
223 DEVMETHOD(ifdi_promisc_set, ixv_if_promisc_set),
224 DEVMETHOD(ifdi_mtu_set, ixv_if_mtu_set),
225 DEVMETHOD(ifdi_get_rss_key, ixv_if_get_rss_key),
226 DEVMETHOD(ifdi_get_rss_hash, ixv_if_get_rss_hash),
227 DEVMETHOD(ifdi_media_status, ixv_if_media_status),
228 DEVMETHOD(ifdi_media_change, ixv_if_media_change),
229 DEVMETHOD(ifdi_timer, ixv_if_local_timer),
230 DEVMETHOD(ifdi_vlan_register, ixv_if_register_vlan),
231 DEVMETHOD(ifdi_vlan_unregister, ixv_if_unregister_vlan),
232 DEVMETHOD(ifdi_get_counter, ixv_if_get_counter),
233 DEVMETHOD(ifdi_needs_restart, ixv_if_needs_restart),
234 DEVMETHOD_END
235 };
236
237 static driver_t ixv_if_driver = {
238 "ixv_if", ixv_if_methods, sizeof(struct ixgbe_softc)
239 };
240
241 #define IXV_VLAN_RETRY_BATCH 4
242 #define IXV_VLAN_RETRY_WINDOW (8 * SBT_1S)
243
244 extern struct if_txrx ixgbe_txrx;
245
246 static struct if_shared_ctx ixv_sctx_init = {
247 .isc_magic = IFLIB_MAGIC,
248 .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */
249 .isc_tx_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
250 .isc_tx_maxsegsize = PAGE_SIZE,
251 .isc_tso_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
252 .isc_tso_maxsegsize = PAGE_SIZE,
253 .isc_rx_maxsize = MJUM16BYTES,
254 .isc_rx_nsegments = 1,
255 .isc_rx_maxsegsize = MJUM16BYTES,
256 .isc_nfl = 1,
257 .isc_ntxqs = 1,
258 .isc_nrxqs = 1,
259 .isc_admin_intrcnt = 1,
260 .isc_vendor_info = ixv_vendor_info_array,
261 .isc_driver_version = ixv_driver_version,
262 .isc_driver = &ixv_if_driver,
263 .isc_flags = IFLIB_IS_VF | IFLIB_TSO_INIT_IP,
264
265 .isc_nrxd_min = {MIN_RXD},
266 .isc_ntxd_min = {MIN_TXD},
267 .isc_nrxd_max = {MAX_RXD},
268 .isc_ntxd_max = {MAX_TXD},
269 .isc_nrxd_default = {DEFAULT_RXD},
270 .isc_ntxd_default = {DEFAULT_TXD},
271 };
272
273 static void *
ixv_register(device_t dev)274 ixv_register(device_t dev)
275 {
276 return (&ixv_sctx_init);
277 }
278
279 /************************************************************************
280 * ixv_if_tx_queues_alloc
281 ************************************************************************/
282 static int
ixv_if_tx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int ntxqs,int ntxqsets)283 ixv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
284 int ntxqs, int ntxqsets)
285 {
286 struct ixgbe_softc *sc = iflib_get_softc(ctx);
287 if_softc_ctx_t scctx = sc->shared;
288 struct ix_tx_queue *que;
289 int i, j, error;
290
291 MPASS(sc->num_tx_queues == ntxqsets);
292 MPASS(ntxqs == 1);
293
294 /* Allocate queue structure memory */
295 sc->tx_queues =
296 (struct ix_tx_queue *)malloc(sizeof(struct ix_tx_queue) *
297 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO);
298 if (!sc->tx_queues) {
299 device_printf(iflib_get_dev(ctx),
300 "Unable to allocate TX ring memory\n");
301 return (ENOMEM);
302 }
303
304 for (i = 0, que = sc->tx_queues; i < ntxqsets; i++, que++) {
305 struct tx_ring *txr = &que->txr;
306
307 txr->me = i;
308 txr->sc = que->sc = sc;
309
310 /* Allocate report status array */
311 if (!(txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) *
312 scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) {
313 error = ENOMEM;
314 goto fail;
315 }
316 for (j = 0; j < scctx->isc_ntxd[0]; j++)
317 txr->tx_rsq[j] = QIDX_INVALID;
318 /* get virtual and physical address of the hardware queues */
319 txr->tail = IXGBE_VFTDT(txr->me);
320 txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i*ntxqs];
321 txr->tx_paddr = paddrs[i*ntxqs];
322
323 txr->bytes = 0;
324 txr->total_packets = 0;
325
326 }
327
328 device_printf(iflib_get_dev(ctx), "allocated for %d queues\n",
329 sc->num_tx_queues);
330
331 return (0);
332
333 fail:
334 ixv_if_queues_free(ctx);
335
336 return (error);
337 } /* ixv_if_tx_queues_alloc */
338
339 /************************************************************************
340 * ixv_if_rx_queues_alloc
341 ************************************************************************/
342 static int
ixv_if_rx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int nrxqs,int nrxqsets)343 ixv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
344 int nrxqs, int nrxqsets)
345 {
346 struct ixgbe_softc *sc = iflib_get_softc(ctx);
347 struct ix_rx_queue *que;
348 int i, error;
349
350 MPASS(sc->num_rx_queues == nrxqsets);
351 MPASS(nrxqs == 1);
352
353 /* Allocate queue structure memory */
354 sc->rx_queues =
355 (struct ix_rx_queue *)malloc(sizeof(struct ix_rx_queue) *
356 nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO);
357 if (!sc->rx_queues) {
358 device_printf(iflib_get_dev(ctx),
359 "Unable to allocate TX ring memory\n");
360 error = ENOMEM;
361 goto fail;
362 }
363
364 for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) {
365 struct rx_ring *rxr = &que->rxr;
366 rxr->me = i;
367 rxr->sc = que->sc = sc;
368
369
370 /* get the virtual and physical address of the hw queues */
371 rxr->tail = IXGBE_VFRDT(rxr->me);
372 rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i];
373 rxr->rx_paddr = paddrs[i*nrxqs];
374 rxr->bytes = 0;
375 rxr->que = que;
376 }
377
378 device_printf(iflib_get_dev(ctx), "allocated for %d rx queues\n",
379 sc->num_rx_queues);
380
381 return (0);
382
383 fail:
384 ixv_if_queues_free(ctx);
385
386 return (error);
387 } /* ixv_if_rx_queues_alloc */
388
389 /************************************************************************
390 * ixv_if_queues_free
391 ************************************************************************/
392 static void
ixv_if_queues_free(if_ctx_t ctx)393 ixv_if_queues_free(if_ctx_t ctx)
394 {
395 struct ixgbe_softc *sc = iflib_get_softc(ctx);
396 struct ix_tx_queue *que = sc->tx_queues;
397 int i;
398
399 if (que == NULL)
400 goto free;
401
402 for (i = 0; i < sc->num_tx_queues; i++, que++) {
403 struct tx_ring *txr = &que->txr;
404 if (txr->tx_rsq == NULL)
405 break;
406
407 free(txr->tx_rsq, M_DEVBUF);
408 txr->tx_rsq = NULL;
409 }
410 if (sc->tx_queues != NULL)
411 free(sc->tx_queues, M_DEVBUF);
412 free:
413 if (sc->rx_queues != NULL)
414 free(sc->rx_queues, M_DEVBUF);
415 sc->tx_queues = NULL;
416 sc->rx_queues = NULL;
417 } /* ixv_if_queues_free */
418
419 /************************************************************************
420 * ixv_if_attach_pre - Device initialization routine
421 *
422 * Called when the driver is being loaded.
423 * Identifies the type of hardware, allocates all resources
424 * and initializes the hardware.
425 *
426 * return 0 on success, positive on failure
427 ************************************************************************/
428 static int
ixv_if_attach_pre(if_ctx_t ctx)429 ixv_if_attach_pre(if_ctx_t ctx)
430 {
431 struct ixgbe_softc *sc;
432 device_t dev;
433 if_softc_ctx_t scctx;
434 struct ixgbe_hw *hw;
435 bool mailbox_ready;
436 int error = 0;
437
438 INIT_DEBUGOUT("ixv_attach: begin");
439
440 /* Allocate, clear, and link in our sc structure */
441 dev = iflib_get_dev(ctx);
442 sc = iflib_get_softc(ctx);
443 sc->dev = dev;
444 sc->ctx = ctx;
445 sc->hw.back = sc;
446 scctx = sc->shared = iflib_get_softc_ctx(ctx);
447 sc->media = iflib_get_media(ctx);
448 hw = &sc->hw;
449
450 /* Do base PCI setup - map BAR0 */
451 if (ixv_allocate_pci_resources(ctx)) {
452 device_printf(dev, "ixv_allocate_pci_resources() failed!\n");
453 error = ENXIO;
454 goto err_out;
455 }
456
457 /* SYSCTL APIs */
458 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
459 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
460 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
461 sc, 0, ixv_sysctl_debug, "I", "Debug Info");
462
463 /* Determine hardware revision */
464 ixv_identify_hardware(ctx);
465 ixv_init_device_features(sc);
466 sc->vf_link_poll_tick = device_get_unit(dev) % IXV_LINK_POLL_TICKS;
467
468 /* Initialize the shared code */
469 if (ixv_is_hyperv(hw))
470 error = ixgbevf_hv_init_ops_vf(hw);
471 else
472 error = ixgbe_init_ops_vf(hw);
473 if (error) {
474 device_printf(dev, "VF shared-code initialization failed\n");
475 error = EIO;
476 goto err_out;
477 }
478
479 /* Setup the mailbox */
480 ixgbe_init_mbx_params_vf(hw);
481
482 mailbox_ready = false;
483 error = hw->mac.ops.reset_hw(hw);
484 if (error != IXGBE_SUCCESS) {
485 /*
486 * A PF may be resetting or servicing a slow link event while its
487 * VFs are enumerated. Keep the VF attached so a later if_init can
488 * retry the mailbox handshake.
489 */
490 ixv_log_reset_failure(sc, error, true);
491 } else {
492 error = hw->mac.ops.init_hw(hw);
493 if (error != IXGBE_SUCCESS) {
494 device_printf(dev, "...init_hw() failed with error %d\n",
495 error);
496 error = EIO;
497 goto err_out;
498 }
499
500 /* Negotiate mailbox API version. */
501 error = ixv_negotiate_api(sc);
502 if (error != 0) {
503 ixv_log_negotiate_failure(sc, true);
504 hw->mac.ops.stop_adapter(hw);
505 } else
506 mailbox_ready = true;
507 }
508
509 /* Check if VF was disabled by PF. */
510 if (!mailbox_ready ||
511 hw->mac.ops.get_link_state(hw, &sc->link_enabled) != 0) {
512 /* PF is not capable of controlling VF state. Enable link. */
513 sc->link_enabled = true;
514 }
515
516 /* If no mac address was assigned, make a random one */
517 if (!ixv_check_ether_addr(hw->mac.addr)) {
518 ether_gen_addr(iflib_get_ifp(ctx),
519 (struct ether_addr *)hw->mac.addr);
520 bcopy(hw->mac.addr, hw->mac.perm_addr,
521 sizeof(hw->mac.perm_addr));
522 }
523
524 /* Most of the iflib initialization... */
525
526 iflib_set_mac(ctx, hw->mac.addr);
527 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max =
528 ixv_queue_limit(sc, mailbox_ready);
529 scctx->isc_txqsizes[0] =
530 roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) +
531 sizeof(u32), DBA_ALIGN);
532 scctx->isc_rxqsizes[0] =
533 roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc),
534 DBA_ALIGN);
535 /* XXX */
536 scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |
537 CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO | CSUM_SCTP |
538 CSUM_IP6_SCTP;
539 scctx->isc_tx_nsegments = IXGBE_82599_SCATTER;
540 scctx->isc_msix_bar = pci_msix_table_bar(dev);
541 scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments;
542 scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE;
543 scctx->isc_tx_tso_segsize_max = PAGE_SIZE;
544
545 scctx->isc_txrx = &ixgbe_txrx;
546
547 /* We support everything the PF does; VFs do not do WoL. */
548 scctx->isc_capabilities = IXGBE_CAPS;
549 scctx->isc_capenable = scctx->isc_capabilities;
550 atomic_store_rel_32(&sc->vf_mbx_ready, mailbox_ready);
551 callout_init(&sc->vf_mbx_retry, 1);
552 sc->vf_mbx_retry_initialized = true;
553
554 INIT_DEBUGOUT("ixv_if_attach_pre: end");
555
556 return (0);
557
558 err_out:
559 ixv_free_pci_resources(ctx);
560
561 return (error);
562 } /* ixv_if_attach_pre */
563
564 static int
ixv_if_attach_post(if_ctx_t ctx)565 ixv_if_attach_post(if_ctx_t ctx)
566 {
567 struct ixgbe_softc *sc = iflib_get_softc(ctx);
568 device_t dev = iflib_get_dev(ctx);
569 int error = 0;
570
571 /* Setup OS specific network interface */
572 error = ixv_setup_interface(ctx);
573 if (error) {
574 device_printf(dev, "Interface setup failed: %d\n", error);
575 goto end;
576 }
577
578 /* Do the stats setup */
579 if (atomic_load_acq_32(&sc->vf_mbx_ready) != 0)
580 ixv_init_stats(sc);
581 ixv_add_stats_sysctls(sc);
582
583 end:
584 return error;
585 } /* ixv_if_attach_post */
586
587 /************************************************************************
588 * ixv_detach - Device removal routine
589 *
590 * Called when the driver is being removed.
591 * Stops the adapter and deallocates all the resources
592 * that were allocated for driver operation.
593 *
594 * return 0 on success, positive on failure
595 ************************************************************************/
596 static int
ixv_if_detach(if_ctx_t ctx)597 ixv_if_detach(if_ctx_t ctx)
598 {
599 struct ixgbe_softc *sc;
600
601 INIT_DEBUGOUT("ixv_detach: begin");
602
603 sc = iflib_get_softc(ctx);
604 ixv_mbx_retry_detach(sc);
605 ixv_free_pci_resources(ctx);
606
607 return (0);
608 } /* ixv_if_detach */
609
610 /************************************************************************
611 * ixv_if_mtu_set
612 ************************************************************************/
613 static int
ixv_if_mtu_set(if_ctx_t ctx,uint32_t mtu)614 ixv_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
615 {
616 struct ixgbe_softc *sc = iflib_get_softc(ctx);
617 if_t ifp = iflib_get_ifp(ctx);
618 uint32_t max_mtu;
619 int error = 0;
620
621 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
622 max_mtu = IXGBE_MAX_FRAME_SIZE - IXGBE_MTU_HDR;
623 /* The 82599 VF cannot enforce the X540 RLPML field. */
624 if (ixv_is_hyperv(&sc->hw) &&
625 sc->hw.mac.type == ixgbe_mac_82599_vf)
626 max_mtu = ETHERMTU + ETHER_CRC_LEN;
627 if (mtu > max_mtu) {
628 error = EINVAL;
629 } else {
630 if_setmtu(ifp, mtu);
631 sc->max_frame_size = if_getmtu(ifp) + IXGBE_MTU_HDR;
632 }
633
634 return error;
635 } /* ixv_if_mtu_set */
636
637 static int
ixv_rss_query_status(if_ctx_t ctx)638 ixv_rss_query_status(if_ctx_t ctx)
639 {
640 struct ixgbe_softc *sc = iflib_get_softc(ctx);
641
642 sx_assert(iflib_ctx_lock_get(ctx), SA_XLOCKED);
643 /* Older VFs share PF-controlled RSS settings which we cannot query. */
644 if (sc->hw.mac.type < ixgbe_mac_X550_vf)
645 return (EOPNOTSUPP);
646 if (atomic_load_acq_32(&sc->vf_mbx_ready) == 0 ||
647 sc->vf_rss_mrqc == 0)
648 return (ENXIO);
649 return (0);
650 }
651
652 static int
ixv_if_get_rss_key(if_ctx_t ctx,struct ifrsskey * ifrk)653 ixv_if_get_rss_key(if_ctx_t ctx, struct ifrsskey *ifrk)
654 {
655 struct ixgbe_softc *sc = iflib_get_softc(ctx);
656 int error;
657
658 error = ixv_rss_query_status(ctx);
659 if (error != 0)
660 return (error);
661 ifrk->ifrk_func = RSS_FUNC_TOEPLITZ;
662 ifrk->ifrk_keylen = sizeof(sc->vf_rss_key);
663 _Static_assert(sizeof(ifrk->ifrk_key) >= sizeof(sc->vf_rss_key),
664 "RSS query buffer too small");
665 bzero(ifrk->ifrk_key, sizeof(ifrk->ifrk_key));
666 /* Preserve the byte order of the programmed registers. */
667 for (u_int i = 0; i < nitems(sc->vf_rss_key); i++)
668 le32enc(ifrk->ifrk_key + i * sizeof(u32), sc->vf_rss_key[i]);
669 return (0);
670 }
671
672 static int
ixv_if_get_rss_hash(if_ctx_t ctx,struct ifrsshash * ifrh)673 ixv_if_get_rss_hash(if_ctx_t ctx, struct ifrsshash *ifrh)
674 {
675 struct ixgbe_softc *sc = iflib_get_softc(ctx);
676 u32 mrqc;
677 int error;
678
679 error = ixv_rss_query_status(ctx);
680 if (error != 0)
681 return (error);
682 ifrh->ifrh_func = RSS_FUNC_TOEPLITZ;
683 ifrh->ifrh_types = 0;
684 mrqc = sc->vf_rss_mrqc;
685 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4)
686 ifrh->ifrh_types |= RSS_TYPE_IPV4;
687 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP)
688 ifrh->ifrh_types |= RSS_TYPE_TCP_IPV4;
689 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP)
690 ifrh->ifrh_types |= RSS_TYPE_UDP_IPV4;
691 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6)
692 ifrh->ifrh_types |= RSS_TYPE_IPV6;
693 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP)
694 ifrh->ifrh_types |= RSS_TYPE_TCP_IPV6;
695 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP)
696 ifrh->ifrh_types |= RSS_TYPE_UDP_IPV6;
697 return (0);
698 }
699
700 static void
ixv_reconcile_mac(struct ixgbe_softc * sc,if_t ifp)701 ixv_reconcile_mac(struct ixgbe_softc *sc, if_t ifp)
702 {
703 uint8_t *lladdr;
704
705 if (ixgbe_validate_mac_addr(sc->hw.mac.addr) != IXGBE_SUCCESS)
706 return;
707 lladdr = (uint8_t *)if_getlladdr(ifp);
708 if (bcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0)
709 return;
710
711 device_printf(sc->dev,
712 "PF rejected or replaced the requested MAC; using %6D\n",
713 sc->hw.mac.addr, ":");
714 /*
715 * Initialization holds the context lock; avoid re-entering the driver.
716 */
717 bcopy(sc->hw.mac.addr, lladdr, ETHER_ADDR_LEN);
718 CURVNET_SET_QUIET(if_getvnet(ifp));
719 EVENTHANDLER_INVOKE(iflladdr_event, ifp);
720 CURVNET_RESTORE();
721 } /* ixv_reconcile_mac */
722
723 /************************************************************************
724 * ixv_if_init - Init entry point
725 *
726 * Used in two ways: It is used by the stack as an init entry
727 * point in network interface structure. It is also used
728 * by the driver as a hw/sw initialization routine to get
729 * to a consistent state.
730 *
731 * return 0 on success, positive on failure
732 ************************************************************************/
733 static void
ixv_if_init(if_ctx_t ctx)734 ixv_if_init(if_ctx_t ctx)
735 {
736 struct ixgbe_softc *sc = iflib_get_softc(ctx);
737 if_t ifp = iflib_get_ifp(ctx);
738 device_t dev = iflib_get_dev(ctx);
739 struct ixgbe_hw *hw = &sc->hw;
740 u8 requested_addr[IXGBE_ETH_LENGTH_OF_ADDRESS];
741 int error = 0;
742
743 INIT_DEBUGOUT("ixv_if_init: begin");
744 sc->vf_rss_mrqc = 0;
745 ixv_mbx_retry_prepare(sc);
746 hw->adapter_stopped = false;
747 hw->mac.ops.stop_adapter(hw);
748
749 /* Preserve a requested LAA across the reset handshake. */
750 bcopy(if_getlladdr(ifp), requested_addr, sizeof(requested_addr));
751
752 /* Reset VF and renegotiate mailbox API version. */
753 error = hw->mac.ops.reset_hw(hw);
754 if (error != IXGBE_SUCCESS) {
755 sc->stats.vf.initialized = false;
756 ixv_log_reset_failure(sc, error, false);
757 hw->mac.ops.stop_adapter(hw);
758 ixv_mbx_retry_failed(ctx);
759 return;
760 }
761 hw->mac.ops.start_hw(hw);
762 ixv_init_stats(sc);
763 error = ixv_negotiate_api(sc);
764 if (error) {
765 /*
766 * Leave the adapter stopped until an explicit or deferred retry.
767 * Otherwise the admin-status callback immediately requests another
768 * reset and can keep its taskqueue in a tight loop while the PF is
769 * deliberately withholding mailbox CTS (for example, when the VF is
770 * quarantined).
771 */
772 ixv_log_negotiate_failure(sc, false);
773 hw->mac.ops.stop_adapter(hw);
774 ixv_mbx_retry_failed(ctx);
775 return;
776 }
777 /* Program the address only after the PF mailbox is responsive. */
778 error = hw->mac.ops.set_rar(hw, 0, requested_addr, 0, 1);
779 if (error == IXGBE_SUCCESS)
780 bcopy(requested_addr, hw->mac.addr, sizeof(requested_addr));
781 else
782 hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
783 ixv_reconcile_mac(sc, ifp);
784 ixv_mbx_retry_succeeded(sc);
785
786 ixv_initialize_transmit_units(ctx);
787
788 /* Setup Multicast table */
789 ixv_if_multi_set(ctx);
790
791 sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx);
792
793 /* Configure RX settings */
794 ixv_initialize_receive_units(ctx);
795
796 /* Set up VLAN offload and filter */
797 ixv_setup_vlan_support(ctx);
798
799 /* Set up MSI-X routing */
800 ixv_configure_ivars(sc);
801
802 /* Set up auto-mask */
803 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE);
804
805 /* Set moderation on the Link interrupt */
806 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(sc->vector), IXGBE_LINK_ITR);
807
808 /* Config/Enable Link */
809 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled);
810 if (error) {
811 /* PF is not capable of controlling VF state. Enable the link. */
812 sc->link_enabled = true;
813 } else if (sc->link_enabled == false)
814 device_printf(dev, "VF is disabled by PF\n");
815
816 hw->mac.ops.check_link(hw, &sc->link_speed, &sc->link_up,
817 false);
818
819 /* And now turn on interrupts */
820 ixv_if_enable_intr(ctx);
821
822 return;
823 } /* ixv_if_init */
824
825 static const char *
ixv_reset_error_desc(s32 error)826 ixv_reset_error_desc(s32 error)
827 {
828
829 switch (error) {
830 case IXGBE_ERR_RESET_FAILED:
831 return ("PF reset acknowledgement timed out");
832 case IXGBE_ERR_INVALID_MAC_ADDR:
833 /*
834 * The shared VF reset code historically uses this error for an
835 * unexpected reset reply, before it validates or copies the MAC.
836 */
837 return ("PF returned an invalid VF reset response");
838 case IXGBE_ERR_MBX:
839 case IXGBE_ERR_MBX_NOMSG:
840 case IXGBE_ERR_TIMEOUT:
841 return ("PF mailbox reset exchange failed");
842 default:
843 return ("VF reset handshake failed");
844 }
845 }
846
847 /*
848 * Report each backoff stage, then limit the steady eight-second retry so a
849 * persistent PF outage does not spam the console.
850 */
851 static bool
ixv_mbx_log_allowed(struct ixgbe_softc * sc)852 ixv_mbx_log_allowed(struct ixgbe_softc *sc)
853 {
854
855 if (sc->vf_mbx_retry_stage == nitems(ixv_mbx_retry_delay) - 1 &&
856 !ratecheck(&sc->vf_mbx_last_log, &ixv_mbx_log_interval))
857 return (false);
858 return (true);
859 }
860
861 static void
ixv_log_negotiate_failure(struct ixgbe_softc * sc,bool attaching)862 ixv_log_negotiate_failure(struct ixgbe_softc *sc, bool attaching)
863 {
864
865 if (!ixv_mbx_log_allowed(sc))
866 return;
867 device_printf(sc->dev, "Mailbox API negotiation failed%s\n",
868 attaching ? "; continuing attach" : "");
869 }
870
871 static void
ixv_log_reset_failure(struct ixgbe_softc * sc,s32 error,bool attaching)872 ixv_log_reset_failure(struct ixgbe_softc *sc, s32 error, bool attaching)
873 {
874
875 if (!ixv_mbx_log_allowed(sc))
876 return;
877 device_printf(sc->dev, "%s (%d)%s\n", ixv_reset_error_desc(error),
878 error, attaching ? "; continuing attach" : "");
879 }
880
881 /*
882 * A missing PF can make the posted reset handshake wait for a full mailbox
883 * timeout. Keep that work out of stopped status paths. An administratively
884 * up VF retries complete initialization with an exponential delay capped at
885 * eight seconds, so it recovers without creating a tight mailbox poller.
886 */
887 static void
ixv_mbx_retry_callout(void * arg)888 ixv_mbx_retry_callout(void *arg)
889 {
890 struct ixgbe_softc *sc;
891 if_t ifp;
892
893 sc = arg;
894 if (atomic_readandclear_32(&sc->vf_mbx_retry_pending) == 0 ||
895 atomic_load_acq_32(&sc->vf_mbx_ready) != 0 ||
896 iflib_in_detach(sc->ctx))
897 return;
898 ifp = iflib_get_ifp(sc->ctx);
899 if ((if_getflags(ifp) & IFF_UP) == 0)
900 return;
901
902 iflib_request_reset_if_up(sc->ctx);
903 iflib_admin_intr_deferred(sc->ctx);
904 }
905
906 static void
ixv_mbx_retry_detach(struct ixgbe_softc * sc)907 ixv_mbx_retry_detach(struct ixgbe_softc *sc)
908 {
909
910 if (!sc->vf_mbx_retry_initialized)
911 return;
912 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
913 callout_drain(&sc->vf_mbx_retry);
914 sc->vf_mbx_retry_initialized = false;
915 }
916
917 static void
ixv_mbx_retry_prepare(struct ixgbe_softc * sc)918 ixv_mbx_retry_prepare(struct ixgbe_softc *sc)
919 {
920
921 if (!sc->vf_mbx_retry_initialized)
922 return;
923 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
924 callout_drain(&sc->vf_mbx_retry);
925 }
926
927 static void
ixv_mbx_retry_stop(struct ixgbe_softc * sc)928 ixv_mbx_retry_stop(struct ixgbe_softc *sc)
929 {
930 if_t ifp;
931
932 if (!sc->vf_mbx_retry_initialized)
933 return;
934 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
935 callout_drain(&sc->vf_mbx_retry);
936 ifp = iflib_get_ifp(sc->ctx);
937 if ((if_getflags(ifp) & IFF_UP) == 0)
938 sc->vf_mbx_retry_stage = 0;
939 }
940
941 static void
ixv_mbx_retry_failed(if_ctx_t ctx)942 ixv_mbx_retry_failed(if_ctx_t ctx)
943 {
944 struct ixgbe_softc *sc;
945 if_t ifp;
946 sbintime_t delay;
947 u_int stage;
948
949 sc = iflib_get_softc(ctx);
950 atomic_store_rel_32(&sc->vf_mbx_ready, 0);
951 sc->link_up = false;
952 if (sc->link_active) {
953 sc->link_active = false;
954 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
955 }
956 iflib_init_failed(ctx);
957
958 ifp = iflib_get_ifp(ctx);
959 if (!sc->vf_mbx_retry_initialized ||
960 (if_getflags(ifp) & IFF_UP) == 0)
961 return;
962 stage = sc->vf_mbx_retry_stage;
963 if (stage >= nitems(ixv_mbx_retry_delay))
964 stage = nitems(ixv_mbx_retry_delay) - 1;
965 delay = ixv_mbx_retry_delay[stage];
966 if (sc->vf_mbx_retry_stage + 1 < nitems(ixv_mbx_retry_delay))
967 sc->vf_mbx_retry_stage++;
968 atomic_set_32(&sc->vf_mbx_retry_pending, 1);
969 callout_reset_sbt(&sc->vf_mbx_retry, delay, 0,
970 ixv_mbx_retry_callout, sc, C_PREL(1));
971 }
972
973 static void
ixv_mbx_retry_succeeded(struct ixgbe_softc * sc)974 ixv_mbx_retry_succeeded(struct ixgbe_softc *sc)
975 {
976
977 atomic_store_rel_32(&sc->vf_mbx_ready, 1);
978 atomic_readandclear_32(&sc->vf_mbx_retry_pending);
979 if (sc->vf_mbx_retry_initialized)
980 callout_stop(&sc->vf_mbx_retry);
981 sc->vf_mbx_retry_stage = 0;
982 sc->vf_link_mbx_failures = 0;
983 sc->vf_link_poll_tick =
984 device_get_unit(sc->dev) % IXV_LINK_POLL_TICKS;
985 sc->vf_mbx_last_log.tv_sec = 0;
986 sc->vf_mbx_last_log.tv_usec = 0;
987 }
988
989 /************************************************************************
990 * ixv_enable_queue
991 ************************************************************************/
992 static inline void
ixv_enable_queue(struct ixgbe_softc * sc,u32 vector)993 ixv_enable_queue(struct ixgbe_softc *sc, u32 vector)
994 {
995 struct ixgbe_hw *hw = &sc->hw;
996 u32 queue = 1 << vector;
997 u32 mask;
998
999 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1000 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
1001 } /* ixv_enable_queue */
1002
1003 /************************************************************************
1004 * ixv_disable_queue
1005 ************************************************************************/
1006 static inline void
ixv_disable_queue(struct ixgbe_softc * sc,u32 vector)1007 ixv_disable_queue(struct ixgbe_softc *sc, u32 vector)
1008 {
1009 struct ixgbe_hw *hw = &sc->hw;
1010 u64 queue = (u64)(1 << vector);
1011 u32 mask;
1012
1013 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1014 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask);
1015 } /* ixv_disable_queue */
1016
1017
1018 /************************************************************************
1019 * ixv_msix_que - MSI-X Queue Interrupt Service routine
1020 ************************************************************************/
1021 static int
ixv_msix_que(void * arg)1022 ixv_msix_que(void *arg)
1023 {
1024 struct ix_rx_queue *que = arg;
1025 struct ixgbe_softc *sc = que->sc;
1026
1027 ixv_disable_queue(sc, que->msix);
1028 ++que->irqs;
1029
1030 return (FILTER_SCHEDULE_THREAD);
1031 } /* ixv_msix_que */
1032
1033 /************************************************************************
1034 * ixv_msix_mbx
1035 ************************************************************************/
1036 static int
ixv_msix_mbx(void * arg)1037 ixv_msix_mbx(void *arg)
1038 {
1039 struct ixgbe_softc *sc = arg;
1040 struct ixgbe_hw *hw = &sc->hw;
1041 u32 reg;
1042
1043 ++sc->link_irq;
1044
1045 /* First get the cause */
1046 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS);
1047 /* Clear interrupt with write */
1048 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg);
1049
1050 /* The admin vector also carries PF mailbox notifications. */
1051 iflib_admin_intr_deferred(sc->ctx);
1052
1053 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER);
1054
1055 return (FILTER_HANDLED);
1056 } /* ixv_msix_mbx */
1057
1058 /************************************************************************
1059 * ixv_media_status - Media Ioctl callback
1060 *
1061 * Called whenever the user queries the status of
1062 * the interface using ifconfig.
1063 ************************************************************************/
1064 static void
ixv_if_media_status(if_ctx_t ctx,struct ifmediareq * ifmr)1065 ixv_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr)
1066 {
1067 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1068
1069 INIT_DEBUGOUT("ixv_media_status: begin");
1070
1071 /* E610 link state is refreshed by the phased service timer. */
1072 if (sc->hw.mac.type != ixgbe_mac_E610_vf ||
1073 sc->hw.api_version != ixgbe_mbox_api_16) {
1074 atomic_set_32(&sc->vf_link_update, 1);
1075 iflib_admin_intr_deferred(ctx);
1076 }
1077
1078 ifmr->ifm_status = IFM_AVALID;
1079 ifmr->ifm_active = IFM_ETHER;
1080
1081 if (!sc->link_active)
1082 return;
1083
1084 ifmr->ifm_status |= IFM_ACTIVE;
1085
1086 switch (sc->link_speed) {
1087 case IXGBE_LINK_SPEED_5GB_FULL:
1088 ifmr->ifm_active |= IFM_5000_T | IFM_FDX;
1089 break;
1090 case IXGBE_LINK_SPEED_2_5GB_FULL:
1091 ifmr->ifm_active |= IFM_2500_T | IFM_FDX;
1092 break;
1093 case IXGBE_LINK_SPEED_1GB_FULL:
1094 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1095 break;
1096 case IXGBE_LINK_SPEED_10GB_FULL:
1097 ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
1098 break;
1099 case IXGBE_LINK_SPEED_100_FULL:
1100 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1101 break;
1102 case IXGBE_LINK_SPEED_10_FULL:
1103 ifmr->ifm_active |= IFM_10_T | IFM_FDX;
1104 break;
1105 }
1106 } /* ixv_if_media_status */
1107
1108 /************************************************************************
1109 * ixv_if_media_change - Media Ioctl callback
1110 *
1111 * Called when the user changes speed/duplex using
1112 * media/mediopt option with ifconfig.
1113 ************************************************************************/
1114 static int
ixv_if_media_change(if_ctx_t ctx)1115 ixv_if_media_change(if_ctx_t ctx)
1116 {
1117 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1118 struct ifmedia *ifm = iflib_get_media(ctx);
1119
1120 INIT_DEBUGOUT("ixv_media_change: begin");
1121
1122 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1123 return (EINVAL);
1124
1125 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1126 case IFM_AUTO:
1127 break;
1128 default:
1129 device_printf(sc->dev, "Only auto media type\n");
1130 return (EINVAL);
1131 }
1132
1133 return (0);
1134 } /* ixv_if_media_change */
1135
1136
1137 /************************************************************************
1138 * ixv_negotiate_api
1139 *
1140 * Negotiate the Mailbox API with the PF;
1141 * start with the most featured API first.
1142 ************************************************************************/
1143 static int
ixv_negotiate_api(struct ixgbe_softc * sc)1144 ixv_negotiate_api(struct ixgbe_softc *sc)
1145 {
1146 struct ixgbe_hw *hw = &sc->hw;
1147 int mbx_api[] = {
1148 ixgbe_mbox_api_13,
1149 ixgbe_mbox_api_12,
1150 ixgbe_mbox_api_11,
1151 ixgbe_mbox_api_10,
1152 ixgbe_mbox_api_unknown
1153 };
1154 int i = 0;
1155
1156 if (hw->mac.type == ixgbe_mac_E610_vf && !ixv_is_hyperv(hw) &&
1157 hw->mac.ops.negotiate_api_version(hw, ixgbe_mbox_api_16) == 0)
1158 return (0);
1159
1160 while (mbx_api[i] != ixgbe_mbox_api_unknown) {
1161 if (hw->mac.ops.negotiate_api_version(hw, mbx_api[i]) == 0)
1162 return (0);
1163 i++;
1164 }
1165
1166 return (EINVAL);
1167 } /* ixv_negotiate_api */
1168
1169 /************************************************************************
1170 * ixv_queue_limit
1171 *
1172 * Discover the number of symmetric RSS queue sets available to iflib.
1173 ************************************************************************/
1174 static int
ixv_queue_limit(struct ixgbe_softc * sc,bool mailbox_ready)1175 ixv_queue_limit(struct ixgbe_softc *sc, bool mailbox_ready)
1176 {
1177 struct ixgbe_hw *hw;
1178 unsigned int default_tc, num_tcs;
1179 int admin_vectors, limit, msix_vectors;
1180
1181 hw = &sc->hw;
1182 if (ixv_is_hyperv(hw)) {
1183 limit = ixgbevf_hv_get_queues(hw);
1184 } else {
1185 /* Preserve the current family limit as the mailbox fallback. */
1186 switch (hw->mac.type) {
1187 case ixgbe_mac_82599_vf:
1188 case ixgbe_mac_X540_vf:
1189 limit = 1;
1190 break;
1191 case ixgbe_mac_X550_vf:
1192 case ixgbe_mac_X550EM_x_vf:
1193 case ixgbe_mac_X550EM_a_vf:
1194 limit = 2;
1195 break;
1196 case ixgbe_mac_E610_vf:
1197 limit = 1;
1198 break;
1199 default:
1200 return (1);
1201 }
1202 }
1203
1204 /* Replace the fallback with the queue grant reported by the PF. */
1205 if (mailbox_ready) {
1206 switch (hw->api_version) {
1207 case ixgbe_mbox_api_11:
1208 case ixgbe_mbox_api_12:
1209 case ixgbe_mbox_api_13:
1210 case ixgbe_mbox_api_16:
1211 num_tcs = default_tc = 0;
1212 if (ixgbevf_get_queues(hw, &num_tcs, &default_tc) == 0) {
1213 limit = imin(hw->mac.max_tx_queues,
1214 hw->mac.max_rx_queues);
1215 limit = imin(limit, 2);
1216 }
1217 break;
1218 default:
1219 break;
1220 }
1221 }
1222
1223 /*
1224 * iflib assigns one data vector to each queue set. A VF has at most
1225 * three MSI-X vectors; reserve one of them for the mailbox interrupt.
1226 */
1227 admin_vectors = iflib_get_sctx(sc->ctx)->isc_admin_intrcnt;
1228 msix_vectors = pci_msix_count(sc->dev);
1229 if (msix_vectors <= admin_vectors)
1230 return (1);
1231
1232 return (imax(1, imin(limit, msix_vectors - admin_vectors)));
1233 } /* ixv_queue_limit */
1234
1235 static int
ixv_update_xcast_mode(struct ixgbe_softc * sc,int flags)1236 ixv_update_xcast_mode(struct ixgbe_softc *sc, int flags)
1237 {
1238 if_t ifp;
1239 int mode;
1240
1241 ifp = iflib_get_ifp(sc->ctx);
1242 if (flags & IFF_PROMISC)
1243 mode = IXGBEVF_XCAST_MODE_PROMISC;
1244 else if ((flags & IFF_ALLMULTI) != 0 ||
1245 if_llmaddr_count(ifp) > IXGBE_MAX_VF_MC)
1246 mode = IXGBEVF_XCAST_MODE_ALLMULTI;
1247 else if (if_llmaddr_count(ifp) != 0)
1248 mode = IXGBEVF_XCAST_MODE_MULTI;
1249 else
1250 mode = IXGBEVF_XCAST_MODE_NONE;
1251 return (sc->hw.mac.ops.update_xcast_mode(&sc->hw, mode));
1252 }
1253
1254 static int
ixv_if_promisc_set(if_ctx_t ctx,int flags)1255 ixv_if_promisc_set(if_ctx_t ctx, int flags)
1256 {
1257 struct ixgbe_softc *sc;
1258
1259 sc = iflib_get_softc(ctx);
1260 if (!iflib_is_running(ctx))
1261 return (0);
1262 if (ixv_update_xcast_mode(sc, flags) != IXGBE_SUCCESS)
1263 return (EOPNOTSUPP);
1264 return (0);
1265 } /* ixv_if_promisc_set */
1266
1267
1268 static u_int
ixv_if_multi_set_cb(void * cb_arg,struct sockaddr_dl * addr,u_int cnt)1269 ixv_if_multi_set_cb(void *cb_arg, struct sockaddr_dl *addr, u_int cnt)
1270 {
1271 if (cnt >= MAX_NUM_MULTICAST_ADDRESSES)
1272 return (0);
1273
1274 bcopy(LLADDR(addr),
1275 &((u8 *)cb_arg)[cnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1276 IXGBE_ETH_LENGTH_OF_ADDRESS);
1277
1278 return (1);
1279 }
1280
1281 /************************************************************************
1282 * ixv_if_multi_set - Multicast Update
1283 *
1284 * Called whenever multicast address list is updated.
1285 ************************************************************************/
1286 static void
ixv_if_multi_set(if_ctx_t ctx)1287 ixv_if_multi_set(if_ctx_t ctx)
1288 {
1289 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS];
1290 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1291 u8 *update_ptr;
1292 if_t ifp = iflib_get_ifp(ctx);
1293 int error, mcnt = 0;
1294
1295 IOCTL_DEBUGOUT("ixv_if_multi_set: begin");
1296
1297 mcnt = if_foreach_llmaddr(ifp, ixv_if_multi_set_cb, mta);
1298
1299 update_ptr = mta;
1300
1301 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt,
1302 ixv_mc_array_itr, true);
1303 error = ixv_update_xcast_mode(sc, if_getflags(ifp));
1304 if (mcnt > IXGBE_MAX_VF_MC && error != IXGBE_SUCCESS) {
1305 if (!sc->vf_mcast_overflow_warned)
1306 device_printf(sc->dev,
1307 "PF rejected all-multicast fallback; only %d "
1308 "multicast addresses are active\n",
1309 IXGBE_MAX_VF_MC);
1310 sc->vf_mcast_overflow_warned = true;
1311 } else if (mcnt <= IXGBE_MAX_VF_MC)
1312 sc->vf_mcast_overflow_warned = false;
1313 } /* ixv_if_multi_set */
1314
1315 /************************************************************************
1316 * ixv_mc_array_itr
1317 *
1318 * An iterator function needed by the multicast shared code.
1319 * It feeds the shared code routine the addresses in the
1320 * array of ixv_set_multi() one by one.
1321 ************************************************************************/
1322 static u8 *
ixv_mc_array_itr(struct ixgbe_hw * hw,u8 ** update_ptr,u32 * vmdq)1323 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1324 {
1325 u8 *addr = *update_ptr;
1326 u8 *newptr;
1327
1328 *vmdq = 0;
1329
1330 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1331 *update_ptr = newptr;
1332
1333 return addr;
1334 } /* ixv_mc_array_itr */
1335
1336 /************************************************************************
1337 * ixv_if_local_timer - Timer routine
1338 *
1339 * Checks for link status, updates statistics,
1340 * and runs the watchdog check.
1341 ************************************************************************/
1342 static void
ixv_if_local_timer(if_ctx_t ctx,uint16_t qid)1343 ixv_if_local_timer(if_ctx_t ctx, uint16_t qid)
1344 {
1345 struct ixgbe_softc *sc;
1346
1347 if (qid != 0)
1348 return;
1349
1350 sc = iflib_get_softc(ctx);
1351 atomic_set_32(&sc->vf_vlan_retry_tick, 1);
1352 if (sc->hw.mac.type != ixgbe_mac_E610_vf ||
1353 sc->hw.api_version != ixgbe_mbox_api_16 ||
1354 ++sc->vf_link_poll_tick == IXV_LINK_POLL_TICKS) {
1355 sc->vf_link_poll_tick = 0;
1356 atomic_set_32(&sc->vf_link_update, 1);
1357 }
1358
1359 /* Fire off the adminq task */
1360 iflib_admin_intr_deferred(ctx);
1361 } /* ixv_if_local_timer */
1362
1363 /************************************************************************
1364 * ixv_if_update_admin_status - Update OS on link state
1365 *
1366 * Note: Only updates the OS on the cached link state.
1367 * The real check of the hardware only happens with
1368 * a link interrupt.
1369 ************************************************************************/
1370 static void
ixv_if_update_admin_status(if_ctx_t ctx)1371 ixv_if_update_admin_status(if_ctx_t ctx)
1372 {
1373 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1374 device_t dev = iflib_get_dev(ctx);
1375 bool check_link, reset_seen;
1376 s32 status;
1377 uint64_t baudrate;
1378
1379 if (!iflib_is_running(ctx) ||
1380 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) {
1381 if (sc->link_active) {
1382 sc->link_active = false;
1383 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
1384 }
1385 return;
1386 }
1387
1388 check_link = atomic_readandclear_32(&sc->vf_link_update) != 0;
1389 if (sc->hw.mac.type != ixgbe_mac_E610_vf ||
1390 sc->hw.api_version != ixgbe_mbox_api_16)
1391 check_link = true;
1392 /*
1393 * Hyper-V uses this indication only to invalidate cached VFLINKS
1394 * state; it does not expose the mailbox handshake needed to turn it
1395 * into a driver reset. Its check_link operation samples the bit.
1396 */
1397 reset_seen = !ixv_is_hyperv(&sc->hw) &&
1398 ixgbe_check_for_rst(&sc->hw, 0) == IXGBE_SUCCESS;
1399 if (reset_seen)
1400 sc->hw.mac.get_link_status = true;
1401 if (check_link) {
1402 sc->hw.mac.get_link_status = true;
1403 status = ixgbe_check_link(&sc->hw, &sc->link_speed,
1404 &sc->link_up, false);
1405 } else
1406 status = IXGBE_SUCCESS;
1407 if (sc->hw.mac.type == ixgbe_mac_E610_vf &&
1408 sc->hw.api_version == ixgbe_mbox_api_16 &&
1409 status != IXGBE_SUCCESS && status != IXGBE_ERR_MBX) {
1410 /*
1411 * A busy PF can miss an individual link-state request. Preserve
1412 * the last confirmed state across brief transport failures, but
1413 * bound how long stale carrier can remain visible if the PF is gone.
1414 */
1415 if (sc->vf_link_mbx_failures < IXV_LINK_MBX_FAILURE_LIMIT)
1416 sc->vf_link_mbx_failures++;
1417 if (sc->vf_link_mbx_failures == IXV_LINK_MBX_FAILURE_LIMIT)
1418 sc->link_up = false;
1419 status = IXGBE_SUCCESS;
1420 } else if (status == IXGBE_SUCCESS)
1421 sc->vf_link_mbx_failures = 0;
1422 /* Reinitialize after an unsolicited reset, even while link is down. */
1423 if (reset_seen)
1424 status = IXGBE_ERR_MBX;
1425
1426 if (status != IXGBE_SUCCESS && sc->hw.adapter_stopped == false) {
1427 /* Mailbox's Clear To Send status is lost or timeout occurred.
1428 * We need reinitialization. */
1429 sc->vf_rss_mrqc = 0;
1430 iflib_request_reset(ctx);
1431 iflib_admin_intr_deferred(ctx);
1432 }
1433
1434 if (sc->link_up && sc->link_enabled) {
1435 if (sc->link_active == false) {
1436 if (bootverbose) {
1437 baudrate = ixgbe_link_speed_to_baudrate(
1438 sc->link_speed);
1439 if (baudrate != 0)
1440 device_printf(dev,
1441 "Link is up %ju Mbps Full Duplex\n",
1442 (uintmax_t)(baudrate / IF_Mbps(1)));
1443 else
1444 device_printf(dev,
1445 "Link is up (speed unknown)\n");
1446 }
1447 sc->link_active = true;
1448 iflib_link_state_change(ctx, LINK_STATE_UP,
1449 ixgbe_link_speed_to_baudrate(sc->link_speed));
1450 }
1451 } else { /* Link down */
1452 if (sc->link_active == true) {
1453 if (bootverbose)
1454 device_printf(dev, "Link is Down\n");
1455 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
1456 sc->link_active = false;
1457 }
1458 }
1459
1460 /* iflib closes admission before stop; do not replay after VF reset. */
1461 if (iflib_is_running(ctx) &&
1462 atomic_readandclear_32(&sc->vf_vlan_retry_tick) != 0)
1463 ixv_vlan_retry_tick(sc);
1464
1465 /* Do not treat a PF reset as a hardware-counter wrap. */
1466 if (status == IXGBE_SUCCESS)
1467 ixv_update_stats(sc);
1468 else
1469 sc->stats.vf.initialized = false;
1470 } /* ixv_if_update_admin_status */
1471
1472
1473 /************************************************************************
1474 * ixv_if_stop - Stop the hardware
1475 *
1476 * Disables all traffic on the adapter by issuing a
1477 * global reset on the MAC and deallocates TX/RX buffers.
1478 ************************************************************************/
1479 static void
ixv_if_stop(if_ctx_t ctx)1480 ixv_if_stop(if_ctx_t ctx)
1481 {
1482 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1483 struct ixgbe_hw *hw = &sc->hw;
1484 if_t ifp = iflib_get_ifp(ctx);
1485 bool mailbox_ready, reset_seen;
1486
1487 INIT_DEBUGOUT("ixv_stop: begin\n");
1488 sc->vf_rss_mrqc = 0;
1489
1490 ixv_mbx_retry_stop(sc);
1491 ixv_if_disable_intr(ctx);
1492
1493 mailbox_ready = atomic_load_acq_32(&sc->vf_mbx_ready) != 0;
1494 reset_seen = mailbox_ready &&
1495 ixgbe_check_for_rst(hw, 0) == IXGBE_SUCCESS;
1496 if (reset_seen)
1497 sc->stats.vf.initialized = false;
1498 else if (mailbox_ready && sc->stats.vf.initialized)
1499 ixv_update_stats(sc);
1500 if (mailbox_ready && (if_getflags(ifp) & IFF_UP) == 0) {
1501 if (hw->mac.ops.reset_hw(hw) == IXGBE_SUCCESS)
1502 ixv_init_stats(sc);
1503 else
1504 sc->stats.vf.initialized = false;
1505 }
1506 atomic_store_rel_32(&sc->vf_mbx_ready, 0);
1507 sc->vf_link_mbx_failures = 0;
1508 sc->hw.adapter_stopped = false;
1509 hw->mac.ops.stop_adapter(hw);
1510
1511 /* Publish the stopped state without touching the PF mailbox. */
1512 sc->link_up = false;
1513 if (sc->link_active) {
1514 sc->link_active = false;
1515 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
1516 }
1517 } /* ixv_if_stop */
1518
1519
1520 /************************************************************************
1521 * ixv_identify_hardware - Determine hardware revision.
1522 ************************************************************************/
1523 static void
ixv_identify_hardware(if_ctx_t ctx)1524 ixv_identify_hardware(if_ctx_t ctx)
1525 {
1526 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1527 device_t dev = iflib_get_dev(ctx);
1528 struct ixgbe_hw *hw = &sc->hw;
1529
1530 /* Save off the information about this board */
1531 hw->vendor_id = pci_get_vendor(dev);
1532 hw->device_id = pci_get_device(dev);
1533 hw->revision_id = pci_get_revid(dev);
1534 hw->subsystem_vendor_id = pci_get_subvendor(dev);
1535 hw->subsystem_device_id = pci_get_subdevice(dev);
1536
1537 /* A subset of set_mac_type */
1538 switch (hw->device_id) {
1539 case IXGBE_DEV_ID_82599_VF:
1540 case IXGBE_DEV_ID_82599_VF_HV:
1541 hw->mac.type = ixgbe_mac_82599_vf;
1542 break;
1543 case IXGBE_DEV_ID_X540_VF:
1544 case IXGBE_DEV_ID_X540_VF_HV:
1545 hw->mac.type = ixgbe_mac_X540_vf;
1546 break;
1547 case IXGBE_DEV_ID_X550_VF:
1548 case IXGBE_DEV_ID_X550_VF_HV:
1549 hw->mac.type = ixgbe_mac_X550_vf;
1550 break;
1551 case IXGBE_DEV_ID_X550EM_X_VF:
1552 case IXGBE_DEV_ID_X550EM_X_VF_HV:
1553 hw->mac.type = ixgbe_mac_X550EM_x_vf;
1554 break;
1555 case IXGBE_DEV_ID_X550EM_A_VF:
1556 case IXGBE_DEV_ID_X550EM_A_VF_HV:
1557 hw->mac.type = ixgbe_mac_X550EM_a_vf;
1558 break;
1559 case IXGBE_DEV_ID_E610_VF:
1560 hw->mac.type = ixgbe_mac_E610_vf;
1561 break;
1562 default:
1563 device_printf(dev, "unknown mac type\n");
1564 hw->mac.type = ixgbe_mac_unknown;
1565 break;
1566 }
1567 } /* ixv_identify_hardware */
1568
1569 static bool
ixv_is_hyperv(const struct ixgbe_hw * hw)1570 ixv_is_hyperv(const struct ixgbe_hw *hw)
1571 {
1572 switch (hw->device_id) {
1573 case IXGBE_DEV_ID_82599_VF_HV:
1574 case IXGBE_DEV_ID_X540_VF_HV:
1575 case IXGBE_DEV_ID_X550_VF_HV:
1576 case IXGBE_DEV_ID_X550EM_X_VF_HV:
1577 case IXGBE_DEV_ID_X550EM_A_VF_HV:
1578 return (true);
1579 case IXGBE_DEV_ID_E610_VF:
1580 return (hw->subsystem_device_id ==
1581 IXGBE_SUBDEV_ID_E610_VF_HV);
1582 default:
1583 return (false);
1584 }
1585 }
1586
1587 /************************************************************************
1588 * ixv_if_msix_intr_assign - Setup MSI-X Interrupt resources and handlers
1589 ************************************************************************/
1590 static int
ixv_if_msix_intr_assign(if_ctx_t ctx,int msix)1591 ixv_if_msix_intr_assign(if_ctx_t ctx, int msix)
1592 {
1593 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1594 device_t dev = iflib_get_dev(ctx);
1595 struct ix_rx_queue *rx_que = sc->rx_queues;
1596 struct ix_tx_queue *tx_que;
1597 int error, rid, vector = 0;
1598 char buf[16];
1599
1600 for (int i = 0; i < sc->num_rx_queues; i++, vector++, rx_que++) {
1601 rid = vector + 1;
1602
1603 snprintf(buf, sizeof(buf), "rxq%d", i);
1604 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid,
1605 IFLIB_INTR_RXTX, ixv_msix_que, rx_que, rx_que->rxr.me,
1606 buf);
1607
1608 if (error) {
1609 device_printf(iflib_get_dev(ctx),
1610 "Failed to allocate que int %d err: %d",
1611 i, error);
1612 sc->num_rx_queues = i + 1;
1613 goto fail;
1614 }
1615
1616 rx_que->msix = vector;
1617 }
1618
1619 for (int i = 0; i < sc->num_tx_queues; i++) {
1620 snprintf(buf, sizeof(buf), "txq%d", i);
1621 tx_que = &sc->tx_queues[i];
1622 tx_que->msix = i % sc->num_rx_queues;
1623 iflib_softirq_alloc_generic(ctx,
1624 &sc->rx_queues[tx_que->msix].que_irq,
1625 IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf);
1626 }
1627 rid = vector + 1;
1628 error = iflib_irq_alloc_generic(ctx, &sc->irq, rid,
1629 IFLIB_INTR_ADMIN, ixv_msix_mbx, sc, 0, "aq");
1630 if (error) {
1631 device_printf(iflib_get_dev(ctx),
1632 "Failed to register admin handler");
1633 return (error);
1634 }
1635
1636 sc->vector = vector;
1637 /*
1638 * Due to a broken design QEMU will fail to properly
1639 * enable the guest for MSIX unless the vectors in
1640 * the table are all set up, so we must rewrite the
1641 * ENABLE in the MSIX control register again at this
1642 * point to cause it to successfully initialize us.
1643 */
1644 if (sc->hw.mac.type == ixgbe_mac_82599_vf) {
1645 int msix_ctrl;
1646 if (pci_find_cap(dev, PCIY_MSIX, &rid)) {
1647 device_printf(dev,
1648 "Finding MSIX capability failed\n");
1649 } else {
1650 rid += PCIR_MSIX_CTRL;
1651 msix_ctrl = pci_read_config(dev, rid, 2);
1652 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1653 pci_write_config(dev, rid, msix_ctrl, 2);
1654 }
1655 }
1656
1657 return (0);
1658
1659 fail:
1660 iflib_irq_free(ctx, &sc->irq);
1661 rx_que = sc->rx_queues;
1662 for (int i = 0; i < sc->num_rx_queues; i++, rx_que++)
1663 iflib_irq_free(ctx, &rx_que->que_irq);
1664
1665 return (error);
1666 } /* ixv_if_msix_intr_assign */
1667
1668 /************************************************************************
1669 * ixv_allocate_pci_resources
1670 ************************************************************************/
1671 static int
ixv_allocate_pci_resources(if_ctx_t ctx)1672 ixv_allocate_pci_resources(if_ctx_t ctx)
1673 {
1674 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1675 device_t dev = iflib_get_dev(ctx);
1676 int rid;
1677
1678 rid = PCIR_BAR(0);
1679 sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1680 RF_ACTIVE);
1681
1682 if (!(sc->pci_mem)) {
1683 device_printf(dev,
1684 "Unable to allocate bus resource: memory\n");
1685 return (ENXIO);
1686 }
1687
1688 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->pci_mem);
1689 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->pci_mem);
1690 sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle;
1691
1692 return (0);
1693 } /* ixv_allocate_pci_resources */
1694
1695 /************************************************************************
1696 * ixv_free_pci_resources
1697 ************************************************************************/
1698 static void
ixv_free_pci_resources(if_ctx_t ctx)1699 ixv_free_pci_resources(if_ctx_t ctx)
1700 {
1701 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1702 struct ix_rx_queue *que = sc->rx_queues;
1703 device_t dev = iflib_get_dev(ctx);
1704
1705 /* Release all MSI-X queue resources */
1706 if (sc->intr_type == IFLIB_INTR_MSIX)
1707 iflib_irq_free(ctx, &sc->irq);
1708
1709 if (que != NULL) {
1710 for (int i = 0; i < sc->num_rx_queues; i++, que++) {
1711 iflib_irq_free(ctx, &que->que_irq);
1712 }
1713 }
1714
1715 if (sc->pci_mem != NULL)
1716 bus_release_resource(dev, SYS_RES_MEMORY,
1717 rman_get_rid(sc->pci_mem), sc->pci_mem);
1718 } /* ixv_free_pci_resources */
1719
1720 /************************************************************************
1721 * ixv_setup_interface
1722 *
1723 * Setup networking device structure and register an interface.
1724 ************************************************************************/
1725 static int
ixv_setup_interface(if_ctx_t ctx)1726 ixv_setup_interface(if_ctx_t ctx)
1727 {
1728 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1729 if_softc_ctx_t scctx = sc->shared;
1730 if_t ifp = iflib_get_ifp(ctx);
1731
1732 INIT_DEBUGOUT("ixv_setup_interface: begin");
1733
1734 if_setbaudrate(ifp, IF_Gbps(10));
1735 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 2);
1736
1737
1738 sc->max_frame_size = if_getmtu(ifp) + IXGBE_MTU_HDR;
1739 ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1740 ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO);
1741
1742 return 0;
1743 } /* ixv_setup_interface */
1744
1745 /************************************************************************
1746 * ixv_if_get_counter
1747 ************************************************************************/
1748 static uint64_t
ixv_if_get_counter(if_ctx_t ctx,ift_counter cnt)1749 ixv_if_get_counter(if_ctx_t ctx, ift_counter cnt)
1750 {
1751 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1752 if_t ifp = iflib_get_ifp(ctx);
1753
1754 switch (cnt) {
1755 case IFCOUNTER_IPACKETS:
1756 return (sc->ipackets);
1757 case IFCOUNTER_OPACKETS:
1758 return (sc->opackets);
1759 case IFCOUNTER_IBYTES:
1760 return (sc->ibytes);
1761 case IFCOUNTER_OBYTES:
1762 return (sc->obytes);
1763 case IFCOUNTER_IMCASTS:
1764 return (sc->imcasts);
1765 default:
1766 return (if_get_counter_default(ifp, cnt));
1767 }
1768 } /* ixv_if_get_counter */
1769
1770 /* ixv_if_needs_restart - Tell iflib when the driver needs to be reinitialized
1771 * @ctx: iflib context
1772 * @event: event code to check
1773 *
1774 * Defaults to returning true for every event.
1775 *
1776 * @returns true if iflib needs to reinit the interface
1777 */
1778 static bool
ixv_if_needs_restart(if_ctx_t ctx __unused,enum iflib_restart_event event)1779 ixv_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event)
1780 {
1781 switch (event) {
1782 case IFLIB_RESTART_VLAN_CONFIG:
1783 /* The callbacks update the PF directly and queue failed work. */
1784 return (false);
1785 default:
1786 return (true);
1787 }
1788 }
1789
1790 /************************************************************************
1791 * ixv_initialize_transmit_units - Enable transmit unit.
1792 ************************************************************************/
1793 static void
ixv_initialize_transmit_units(if_ctx_t ctx)1794 ixv_initialize_transmit_units(if_ctx_t ctx)
1795 {
1796 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1797 struct ixgbe_hw *hw = &sc->hw;
1798 if_softc_ctx_t scctx = sc->shared;
1799 struct ix_tx_queue *que = sc->tx_queues;
1800 int i;
1801
1802 for (i = 0; i < sc->num_tx_queues; i++, que++) {
1803 struct tx_ring *txr = &que->txr;
1804 u64 tdba = txr->tx_paddr;
1805 u32 txctrl, txdctl;
1806 int j = txr->me;
1807
1808 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1809 txdctl &= ~IXGBE_TXDCTL_THRESH_MASK;
1810 txdctl |= IXGBE_TXDCTL_THRESH_DEFAULT;
1811 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1812
1813 /* Set the HW Tx Head and Tail indices */
1814 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDH(j), 0);
1815 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDT(j), 0);
1816
1817 /* Set Tx Tail register */
1818 txr->tail = IXGBE_VFTDT(j);
1819
1820 txr->tx_rs_cidx = txr->tx_rs_pidx;
1821 /* Initialize the last processed descriptor to be the end of
1822 * the ring, rather than the start, so that we avoid an
1823 * off-by-one error when calculating how many descriptors are
1824 * done in the credits_update function.
1825 */
1826 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
1827 for (int k = 0; k < scctx->isc_ntxd[0]; k++)
1828 txr->tx_rsq[k] = QIDX_INVALID;
1829
1830 /* Set Ring parameters */
1831 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1832 (tdba & 0x00000000ffffffffULL));
1833 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1834 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j),
1835 scctx->isc_ntxd[0] * sizeof(struct ixgbe_legacy_tx_desc));
1836 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1837 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
1838 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1839
1840 /* Now enable */
1841 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1842 txdctl |= IXGBE_TXDCTL_ENABLE;
1843 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1844 }
1845
1846 return;
1847 } /* ixv_initialize_transmit_units */
1848
1849 /************************************************************************
1850 * ixv_initialize_rss_mapping
1851 ************************************************************************/
1852 static void
ixv_initialize_rss_mapping(struct ixgbe_softc * sc)1853 ixv_initialize_rss_mapping(struct ixgbe_softc *sc)
1854 {
1855 struct ixgbe_hw *hw = &sc->hw;
1856 u32 reta = 0, mrqc;
1857 int queue_id;
1858 int i, j;
1859 u32 rss_hash_config;
1860
1861 rss_getkey((uint8_t *)sc->vf_rss_key);
1862
1863 /* Now fill out hash function seeds */
1864 for (u_int k = 0; k < nitems(sc->vf_rss_key); k++)
1865 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(k), sc->vf_rss_key[k]);
1866
1867 /* Set up the redirection table */
1868 for (i = 0, j = 0; i < 64; i++, j++) {
1869 if (j == sc->num_rx_queues)
1870 j = 0;
1871
1872 if (sc->feat_en & IXGBE_FEATURE_RSS) {
1873 /*
1874 * Fetch the RSS bucket id for the given indirection
1875 * entry. Cap it at the number of configured buckets
1876 * (which is num_rx_queues.)
1877 */
1878 queue_id = rss_get_indirection_to_bucket(i);
1879 queue_id = queue_id % sc->num_rx_queues;
1880 } else
1881 queue_id = j;
1882
1883 /*
1884 * The low 8 bits are for hash value (n+0);
1885 * The next 8 bits are for hash value (n+1), etc.
1886 */
1887 reta >>= 8;
1888 reta |= ((uint32_t)queue_id) << 24;
1889 if ((i & 3) == 3) {
1890 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta);
1891 reta = 0;
1892 }
1893 }
1894
1895 /* Perform hash on these packet types */
1896 if (sc->feat_en & IXGBE_FEATURE_RSS)
1897 rss_hash_config = rss_gethashconfig();
1898 else {
1899 /*
1900 * Disable UDP - IP fragments aren't currently being handled
1901 * and so we end up with a mix of 2-tuple and 4-tuple
1902 * traffic.
1903 */
1904 rss_hash_config = RSS_HASHTYPE_RSS_IPV4
1905 | RSS_HASHTYPE_RSS_TCP_IPV4
1906 | RSS_HASHTYPE_RSS_IPV6
1907 | RSS_HASHTYPE_RSS_TCP_IPV6;
1908 }
1909
1910 mrqc = IXGBE_MRQC_RSSEN;
1911 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
1912 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
1913 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
1914 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
1915 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
1916 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
1917 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
1918 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
1919 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
1920 device_printf(sc->dev,
1921 "%s: RSS_HASHTYPE_RSS_IPV6_EX defined,"
1922 " but not supported\n", __func__);
1923 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
1924 device_printf(sc->dev,
1925 "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined,"
1926 " but not supported\n", __func__);
1927 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
1928 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
1929 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
1930 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
1931 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
1932 device_printf(sc->dev,
1933 "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined,"
1934 " but not supported\n", __func__);
1935 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc);
1936 sc->vf_rss_mrqc = mrqc;
1937 } /* ixv_initialize_rss_mapping */
1938
1939 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
1940 /************************************************************************
1941 * ixv_initialize_receive_units - Setup receive registers and features.
1942 ************************************************************************/
1943 static void
ixv_initialize_receive_units(if_ctx_t ctx)1944 ixv_initialize_receive_units(if_ctx_t ctx)
1945 {
1946 struct ixgbe_softc *sc = iflib_get_softc(ctx);
1947 if_softc_ctx_t scctx;
1948 struct ixgbe_hw *hw = &sc->hw;
1949 #ifdef DEV_NETMAP
1950 if_t ifp = iflib_get_ifp(ctx);
1951 #endif
1952 struct ix_rx_queue *que = sc->rx_queues;
1953 u32 bufsz, psrtype;
1954
1955 bufsz = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >>
1956 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1957
1958 psrtype = IXGBE_PSRTYPE_TCPHDR |
1959 IXGBE_PSRTYPE_UDPHDR |
1960 IXGBE_PSRTYPE_IPV4HDR |
1961 IXGBE_PSRTYPE_IPV6HDR |
1962 IXGBE_PSRTYPE_L2HDR;
1963
1964 if (sc->num_rx_queues > 1)
1965 psrtype |= 1 << 29;
1966
1967 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1968
1969 /* Tell PF our max_frame size */
1970 if (hw->mac.ops.set_rlpml(hw, sc->max_frame_size) != 0) {
1971 device_printf(sc->dev,
1972 "There is a problem with the PF setup. It is likely the"
1973 " receive unit for this VF will not function correctly."
1974 "\n");
1975 }
1976 scctx = sc->shared;
1977
1978 for (int i = 0; i < sc->num_rx_queues; i++, que++) {
1979 struct rx_ring *rxr = &que->rxr;
1980 u64 rdba = rxr->rx_paddr;
1981 u32 reg, rxdctl;
1982 int j = rxr->me;
1983
1984 /* Disable the queue */
1985 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1986 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1987 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1988 for (int k = 0; k < 10; k++) {
1989 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
1990 IXGBE_RXDCTL_ENABLE)
1991 msec_delay(1);
1992 else
1993 break;
1994 }
1995 wmb();
1996 /* Setup the Base and Length of the Rx Descriptor Ring */
1997 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1998 (rdba & 0x00000000ffffffffULL));
1999 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
2000 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j),
2001 scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc));
2002
2003 /* Reset the ring indices */
2004 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0);
2005 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0);
2006
2007 /* Set up the SRRCTL register */
2008 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j));
2009 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2010 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2011 reg |= bufsz;
2012 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2013 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg);
2014
2015 /* Capture Rx Tail index */
2016 rxr->tail = IXGBE_VFRDT(rxr->me);
2017
2018 /* Do the queue enabling last */
2019 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
2020 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
2021 for (int l = 0; l < 10; l++) {
2022 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
2023 IXGBE_RXDCTL_ENABLE)
2024 break;
2025 msec_delay(1);
2026 }
2027 wmb();
2028
2029 /* Set the Tail Pointer */
2030 #ifdef DEV_NETMAP
2031 /*
2032 * In netmap mode, we must preserve the buffers made
2033 * available to userspace before the if_init()
2034 * (this is true by default on the TX side, because
2035 * init makes all buffers available to userspace).
2036 *
2037 * netmap_reset() and the device specific routines
2038 * (e.g. ixgbe_setup_receive_rings()) map these
2039 * buffers at the end of the NIC ring, so here we
2040 * must set the RDT (tail) register to make sure
2041 * they are not overwritten.
2042 *
2043 * In this driver the NIC ring starts at RDH = 0,
2044 * RDT points to the last slot available for reception (?),
2045 * so RDT = num_rx_desc - 1 means the whole ring is available.
2046 */
2047 if (if_getcapenable(ifp) & IFCAP_NETMAP) {
2048 struct netmap_adapter *na = NA(ifp);
2049 struct netmap_kring *kring = na->rx_rings[j];
2050 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
2051
2052 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t);
2053 } else
2054 #endif /* DEV_NETMAP */
2055 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me),
2056 scctx->isc_nrxd[0] - 1);
2057 }
2058
2059 /*
2060 * Do not touch RSS and RETA settings for older hardware
2061 * as those are shared among PF and all VF.
2062 */
2063 if (sc->hw.mac.type >= ixgbe_mac_X550_vf)
2064 ixv_initialize_rss_mapping(sc);
2065 } /* ixv_initialize_receive_units */
2066
2067 /************************************************************************
2068 * VF VLAN mailbox retry helpers
2069 ************************************************************************/
2070 static bool
ixv_vlan_error_retryable(s32 error)2071 ixv_vlan_error_retryable(s32 error)
2072 {
2073 return (error != IXGBE_ERR_FEATURE_NOT_SUPPORTED);
2074 }
2075
2076 static void
ixv_vlan_retry_add(struct ixgbe_softc * sc,u16 vid)2077 ixv_vlan_retry_add(struct ixgbe_softc *sc, u16 vid)
2078 {
2079 bool pending;
2080
2081 pending = ixv_vlan_retry_pending(sc);
2082 sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f);
2083 /* Start a bounded no-progress window when work becomes pending. */
2084 if (!pending || sc->vf_vlan_retry_deadline == 0)
2085 sc->vf_vlan_retry_deadline =
2086 getsbinuptime() + IXV_VLAN_RETRY_WINDOW;
2087 }
2088
2089 static void
ixv_vlan_retry_clear(struct ixgbe_softc * sc,u16 vid)2090 ixv_vlan_retry_clear(struct ixgbe_softc *sc, u16 vid)
2091 {
2092 u32 bit;
2093 bool pending;
2094
2095 bit = 1U << (vid & 0x1f);
2096 pending = (sc->vf_vfta_retry[vid >> 5] & bit) != 0;
2097 sc->vf_vfta_retry[vid >> 5] &= ~bit;
2098 if (!pending) {
2099 /* A successful mailbox operation proves the PF is responsive. */
2100 if (sc->vf_vlan_retry_deadline == 0 &&
2101 ixv_vlan_retry_pending(sc))
2102 sc->vf_vlan_retry_deadline =
2103 getsbinuptime() + IXV_VLAN_RETRY_WINDOW;
2104 return;
2105 }
2106 if (ixv_vlan_retry_pending(sc))
2107 sc->vf_vlan_retry_deadline =
2108 getsbinuptime() + IXV_VLAN_RETRY_WINDOW;
2109 else
2110 sc->vf_vlan_retry_deadline = 0;
2111 }
2112
2113 static bool
ixv_vlan_retry_pending(const struct ixgbe_softc * sc)2114 ixv_vlan_retry_pending(const struct ixgbe_softc *sc)
2115 {
2116 int i;
2117
2118 for (i = 0; i < IXGBE_VFTA_SIZE; i++)
2119 if (sc->vf_vfta_retry[i] != 0)
2120 return (true);
2121 return (false);
2122 }
2123
2124 static void
ixv_vlan_retry_tick(struct ixgbe_softc * sc)2125 ixv_vlan_retry_tick(struct ixgbe_softc *sc)
2126 {
2127 struct ixgbe_hw *hw;
2128 bool enable;
2129 s32 error;
2130 u32 bit;
2131 u16 vid;
2132 int attempts, i, remaining;
2133
2134 if (!ixv_vlan_retry_pending(sc)) {
2135 sc->vf_vlan_retry_deadline = 0;
2136 return;
2137 }
2138 /*
2139 * Exhausted entries remain dormant until reset, a VLAN callback, or
2140 * another successful VLAN mailbox request.
2141 */
2142 if (sc->vf_vlan_retry_deadline == 0)
2143 return;
2144 if (getsbinuptime() >= sc->vf_vlan_retry_deadline) {
2145 remaining = 0;
2146 for (i = 0; i < IXGBE_VFTA_SIZE; i++)
2147 remaining += bitcount32(sc->vf_vfta_retry[i]);
2148 sc->vf_vlan_retry_deadline = 0;
2149 device_printf(sc->dev,
2150 "VF VLAN retries exhausted for %d VIDs\n", remaining);
2151 return;
2152 }
2153
2154 /*
2155 * A mailbox NACK does not distinguish transient PF unavailability
2156 * from a permanent policy rejection or VLVF exhaustion. Reconcile a
2157 * bounded batch per timer tick so none of those cases creates a busy
2158 * mailbox poller. Stop after the first failure so a silent PF can
2159 * consume at most one mailbox timeout per pass, while a responsive PF
2160 * can drain several successful requests.
2161 */
2162 hw = &sc->hw;
2163 for (attempts = 0, i = 0;
2164 attempts < IXV_VLAN_RETRY_BATCH && i < 4096; i++) {
2165 vid = sc->vf_vlan_retry_cursor;
2166 sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff;
2167 bit = 1U << (vid & 0x1f);
2168 if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0)
2169 continue;
2170 attempts++;
2171 enable = (sc->shadow_vfta[vid >> 5] & bit) != 0;
2172 error = hw->mac.ops.set_vfta(hw, vid, 0, enable, false);
2173 if (error != IXGBE_SUCCESS && ixv_vlan_error_retryable(error))
2174 break;
2175 ixv_vlan_retry_clear(sc, vid);
2176 }
2177 }
2178
2179 /************************************************************************
2180 * ixv_setup_vlan_support - Configure and restore VLAN support
2181 ************************************************************************/
2182 static void
ixv_setup_vlan_support(if_ctx_t ctx)2183 ixv_setup_vlan_support(if_ctx_t ctx)
2184 {
2185 if_t ifp = iflib_get_ifp(ctx);
2186 struct ixgbe_softc *sc = iflib_get_softc(ctx);
2187 struct ixgbe_hw *hw = &sc->hw;
2188 s32 error;
2189 u32 ctrl, vfta;
2190 u16 vid;
2191 int restore_failures;
2192
2193 for (int i = 0; i < sc->num_rx_queues; i++) {
2194 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
2195 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
2196 ctrl |= IXGBE_RXDCTL_VME;
2197 sc->rx_queues[i].rxr.vtag_strip = true;
2198 } else {
2199 ctrl &= ~IXGBE_RXDCTL_VME;
2200 sc->rx_queues[i].rxr.vtag_strip = false;
2201 }
2202 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl);
2203 }
2204
2205 /*
2206 * The PF controls the pool membership independently of the VF's local
2207 * HWFILTER capability. A reset removes those memberships, so replay
2208 * every registered VLAN through the mailbox.
2209 *
2210 * Keep failed removal requests pending as well. They are harmless and
2211 * idempotent after a successful reset, and still needed if the reset
2212 * handshake did not reach the PF.
2213 */
2214 sc->vf_vlan_retry_deadline = 0;
2215 sc->vf_vlan_retry_cursor = 0;
2216 restore_failures = 0;
2217 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) {
2218 if (sc->shadow_vfta[i] == 0)
2219 continue;
2220 vfta = sc->shadow_vfta[i];
2221 for (int j = 0; j < 32; j++) {
2222 if ((vfta & (1U << j)) == 0)
2223 continue;
2224 vid = (i * 32) + j;
2225 /* One timeout is enough to declare this replay deferred. */
2226 if (restore_failures == 0)
2227 error = hw->mac.ops.set_vfta(hw, vid, 0, true,
2228 false);
2229 else
2230 error = IXGBE_ERR_MBX;
2231 if (error != IXGBE_SUCCESS &&
2232 ixv_vlan_error_retryable(error)) {
2233 ixv_vlan_retry_add(sc, vid);
2234 restore_failures++;
2235 } else
2236 ixv_vlan_retry_clear(sc, vid);
2237 }
2238 }
2239 if (ixv_vlan_retry_pending(sc))
2240 sc->vf_vlan_retry_deadline =
2241 getsbinuptime() + IXV_VLAN_RETRY_WINDOW;
2242 if (restore_failures != 0)
2243 device_printf(sc->dev,
2244 "VF VLAN restore failed for %d VIDs; retrying\n",
2245 restore_failures);
2246 } /* ixv_setup_vlan_support */
2247
2248 /************************************************************************
2249 * ixv_if_register_vlan
2250 *
2251 * Run via a vlan config EVENT, it enables us to use the
2252 * HW Filter table since we can get the vlan id.
2253 ************************************************************************/
2254 static void
ixv_if_register_vlan(if_ctx_t ctx,u16 vtag)2255 ixv_if_register_vlan(if_ctx_t ctx, u16 vtag)
2256 {
2257 struct ixgbe_softc *sc = iflib_get_softc(ctx);
2258 bool pending, present;
2259 u32 index, mask;
2260 s32 error;
2261
2262 index = (vtag >> 5) & 0x7F;
2263 mask = 1U << (vtag & 0x1F);
2264 present = (sc->shadow_vfta[index] & mask) != 0;
2265 pending = (sc->vf_vfta_retry[index] & mask) != 0;
2266 sc->shadow_vfta[index] |= mask;
2267 if (!present)
2268 ++sc->num_vlans;
2269 if (present && !pending)
2270 return;
2271
2272 error = sc->hw.mac.ops.set_vfta(&sc->hw, vtag, 0, true, false);
2273 if (error != IXGBE_SUCCESS && ixv_vlan_error_retryable(error)) {
2274 ixv_vlan_retry_add(sc, vtag);
2275 if (!pending)
2276 device_printf(sc->dev,
2277 "VF VLAN %u add request failed; retrying\n", vtag);
2278 } else
2279 ixv_vlan_retry_clear(sc, vtag);
2280 } /* ixv_if_register_vlan */
2281
2282 /************************************************************************
2283 * ixv_if_unregister_vlan
2284 *
2285 * Run via a vlan unconfig EVENT, remove our entry
2286 * in the soft vfta.
2287 ************************************************************************/
2288 static void
ixv_if_unregister_vlan(if_ctx_t ctx,u16 vtag)2289 ixv_if_unregister_vlan(if_ctx_t ctx, u16 vtag)
2290 {
2291 struct ixgbe_softc *sc = iflib_get_softc(ctx);
2292 bool pending, present;
2293 u32 index, mask;
2294 s32 error;
2295
2296 index = (vtag >> 5) & 0x7F;
2297 mask = 1U << (vtag & 0x1F);
2298 present = (sc->shadow_vfta[index] & mask) != 0;
2299 pending = (sc->vf_vfta_retry[index] & mask) != 0;
2300 sc->shadow_vfta[index] &= ~mask;
2301 if (present)
2302 --sc->num_vlans;
2303 if (!present && !pending)
2304 return;
2305
2306 error = sc->hw.mac.ops.set_vfta(&sc->hw, vtag, 0, false, false);
2307 if (error != IXGBE_SUCCESS && ixv_vlan_error_retryable(error)) {
2308 ixv_vlan_retry_add(sc, vtag);
2309 if (!pending)
2310 device_printf(sc->dev,
2311 "VF VLAN %u remove request failed; "
2312 "retrying\n", vtag);
2313 } else
2314 ixv_vlan_retry_clear(sc, vtag);
2315 } /* ixv_if_unregister_vlan */
2316
2317 /************************************************************************
2318 * ixv_if_enable_intr
2319 ************************************************************************/
2320 static void
ixv_if_enable_intr(if_ctx_t ctx)2321 ixv_if_enable_intr(if_ctx_t ctx)
2322 {
2323 struct ixgbe_softc *sc = iflib_get_softc(ctx);
2324 struct ixgbe_hw *hw = &sc->hw;
2325 struct ix_rx_queue *que = sc->rx_queues;
2326 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
2327
2328 if (hw->adapter_stopped ||
2329 atomic_load_acq_32(&sc->vf_mbx_ready) == 0)
2330 return;
2331
2332 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
2333
2334 mask = IXGBE_EIMS_ENABLE_MASK;
2335 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
2336 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
2337
2338 for (int i = 0; i < sc->num_rx_queues; i++, que++)
2339 ixv_enable_queue(sc, que->msix);
2340
2341 IXGBE_WRITE_FLUSH(hw);
2342 } /* ixv_if_enable_intr */
2343
2344 /************************************************************************
2345 * ixv_if_disable_intr
2346 ************************************************************************/
2347 static void
ixv_if_disable_intr(if_ctx_t ctx)2348 ixv_if_disable_intr(if_ctx_t ctx)
2349 {
2350 struct ixgbe_softc *sc = iflib_get_softc(ctx);
2351 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIAC, 0);
2352 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIMC, ~0);
2353 IXGBE_WRITE_FLUSH(&sc->hw);
2354 } /* ixv_if_disable_intr */
2355
2356 /************************************************************************
2357 * ixv_if_rx_queue_intr_enable
2358 ************************************************************************/
2359 static int
ixv_if_rx_queue_intr_enable(if_ctx_t ctx,uint16_t rxqid)2360 ixv_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
2361 {
2362 struct ixgbe_softc *sc = iflib_get_softc(ctx);
2363 struct ix_rx_queue *que = &sc->rx_queues[rxqid];
2364
2365 ixv_enable_queue(sc, que->rxr.me);
2366
2367 return (0);
2368 } /* ixv_if_rx_queue_intr_enable */
2369
2370 /************************************************************************
2371 * ixv_set_ivar
2372 *
2373 * Setup the correct IVAR register for a particular MSI-X interrupt
2374 * - entry is the register array entry
2375 * - vector is the MSI-X vector for this queue
2376 * - type is RX/TX/MISC
2377 ************************************************************************/
2378 static void
ixv_set_ivar(struct ixgbe_softc * sc,u8 entry,u8 vector,s8 type)2379 ixv_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type)
2380 {
2381 struct ixgbe_hw *hw = &sc->hw;
2382 u32 ivar, index;
2383
2384 vector |= IXGBE_IVAR_ALLOC_VAL;
2385
2386 if (type == -1) { /* MISC IVAR */
2387 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
2388 ivar &= ~0xFF;
2389 ivar |= vector;
2390 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
2391 } else { /* RX/TX IVARS */
2392 index = (16 * (entry & 1)) + (8 * type);
2393 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1));
2394 ivar &= ~(0xFF << index);
2395 ivar |= (vector << index);
2396 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar);
2397 }
2398 } /* ixv_set_ivar */
2399
2400 /************************************************************************
2401 * ixv_configure_ivars
2402 ************************************************************************/
2403 static void
ixv_configure_ivars(struct ixgbe_softc * sc)2404 ixv_configure_ivars(struct ixgbe_softc *sc)
2405 {
2406 struct ix_rx_queue *que = sc->rx_queues;
2407
2408 MPASS(sc->num_rx_queues == sc->num_tx_queues);
2409
2410 for (int i = 0; i < sc->num_rx_queues; i++, que++) {
2411 /* First the RX queue entry */
2412 ixv_set_ivar(sc, i, que->msix, 0);
2413 /* ... and the TX */
2414 ixv_set_ivar(sc, i, que->msix, 1);
2415 /* Set an initial value in EITR */
2416 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEITR(que->msix),
2417 IXGBE_EITR_DEFAULT);
2418 }
2419
2420 /* For the mailbox interrupt */
2421 ixv_set_ivar(sc, 1, sc->vector, -1);
2422 } /* ixv_configure_ivars */
2423
2424 /************************************************************************
2425 * ixv_init_stats
2426 ************************************************************************/
2427 #define IXV_STAT_36_MASK 0xFFFFFFFFFULL
2428
2429 static void
ixv_init_stats(struct ixgbe_softc * sc)2430 ixv_init_stats(struct ixgbe_softc *sc)
2431 {
2432 struct ixgbe_hw *hw = &sc->hw;
2433
2434 sc->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
2435 sc->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
2436 sc->stats.vf.last_vfgorc |=
2437 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
2438 sc->stats.vf.last_vfgorc &= IXV_STAT_36_MASK;
2439
2440 sc->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
2441 sc->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
2442 sc->stats.vf.last_vfgotc |=
2443 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
2444 sc->stats.vf.last_vfgotc &= IXV_STAT_36_MASK;
2445
2446 sc->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
2447 sc->stats.vf.initialized = true;
2448 } /* ixv_init_stats */
2449
2450 #define UPDATE_STAT_32(reg, last, count) do { \
2451 u32 current = IXGBE_READ_REG(hw, reg); \
2452 count += (u32)(current - (u32)last); \
2453 last = current; \
2454 } while (0)
2455
2456 #define UPDATE_STAT_36(lsb, msb, last, count) do { \
2457 u64 current = IXGBE_READ_REG(hw, lsb); \
2458 current |= (u64)IXGBE_READ_REG(hw, msb) << 32; \
2459 current &= IXV_STAT_36_MASK; \
2460 count += (current - last) & IXV_STAT_36_MASK; \
2461 last = current; \
2462 } while (0)
2463
2464 /************************************************************************
2465 * ixv_update_stats - Update the board statistics counters.
2466 ************************************************************************/
2467 void
ixv_update_stats(struct ixgbe_softc * sc)2468 ixv_update_stats(struct ixgbe_softc *sc)
2469 {
2470 struct ixgbe_hw *hw = &sc->hw;
2471 struct ixgbevf_hw_stats *stats = &sc->stats.vf;
2472
2473 if (!stats->initialized)
2474 return;
2475
2476 UPDATE_STAT_32(IXGBE_VFGPRC, sc->stats.vf.last_vfgprc,
2477 sc->stats.vf.vfgprc);
2478 UPDATE_STAT_32(IXGBE_VFGPTC, sc->stats.vf.last_vfgptc,
2479 sc->stats.vf.vfgptc);
2480 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2481 sc->stats.vf.last_vfgorc, sc->stats.vf.vfgorc);
2482 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2483 sc->stats.vf.last_vfgotc, sc->stats.vf.vfgotc);
2484 UPDATE_STAT_32(IXGBE_VFMPRC, sc->stats.vf.last_vfmprc,
2485 sc->stats.vf.vfmprc);
2486
2487 /* Fill out the OS statistics structure */
2488 IXGBE_SET_IPACKETS(sc, stats->vfgprc);
2489 IXGBE_SET_OPACKETS(sc, stats->vfgptc);
2490 IXGBE_SET_IBYTES(sc, stats->vfgorc);
2491 IXGBE_SET_OBYTES(sc, stats->vfgotc);
2492 IXGBE_SET_IMCASTS(sc, stats->vfmprc);
2493 } /* ixv_update_stats */
2494
2495 /************************************************************************
2496 * ixv_add_stats_sysctls - Add statistic sysctls for the VF.
2497 ************************************************************************/
2498 static void
ixv_add_stats_sysctls(struct ixgbe_softc * sc)2499 ixv_add_stats_sysctls(struct ixgbe_softc *sc)
2500 {
2501 device_t dev = sc->dev;
2502 struct ix_tx_queue *tx_que = sc->tx_queues;
2503 struct ix_rx_queue *rx_que = sc->rx_queues;
2504 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
2505 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
2506 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
2507 struct ixgbevf_hw_stats *stats = &sc->stats.vf;
2508 struct sysctl_oid *stat_node, *queue_node;
2509 struct sysctl_oid_list *stat_list, *queue_list;
2510
2511 #define QUEUE_NAME_LEN 32
2512 char namebuf[QUEUE_NAME_LEN];
2513
2514 /* Driver Statistics */
2515 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
2516 CTLFLAG_RD, &sc->link_irq, "Link MSI-X IRQ Handled");
2517
2518 for (int i = 0; i < sc->num_tx_queues; i++, tx_que++) {
2519 struct tx_ring *txr = &tx_que->txr;
2520 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
2521 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
2522 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name");
2523 queue_list = SYSCTL_CHILDREN(queue_node);
2524
2525 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
2526 CTLFLAG_RD, &(txr->tso_tx), "TSO Packets");
2527 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
2528 CTLFLAG_RD, &(txr->total_packets), "TX Packets");
2529 }
2530
2531 for (int i = 0; i < sc->num_rx_queues; i++, rx_que++) {
2532 struct rx_ring *rxr = &rx_que->rxr;
2533 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
2534 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
2535 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name");
2536 queue_list = SYSCTL_CHILDREN(queue_node);
2537
2538 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
2539 CTLFLAG_RD, &(rx_que->irqs), "IRQs on queue");
2540 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
2541 CTLFLAG_RD, &(rxr->rx_packets), "RX packets");
2542 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
2543 CTLFLAG_RD, &(rxr->rx_bytes), "RX bytes");
2544 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded",
2545 CTLFLAG_RD, &(rxr->rx_discarded), "Discarded RX packets");
2546 }
2547
2548 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
2549 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
2550 "VF Statistics (read from HW registers)");
2551 stat_list = SYSCTL_CHILDREN(stat_node);
2552
2553 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
2554 CTLFLAG_RD, &stats->vfgprc, "Good Packets Received");
2555 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
2556 CTLFLAG_RD, &stats->vfgorc, "Good Octets Received");
2557 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
2558 CTLFLAG_RD, &stats->vfmprc, "Multicast Packets Received");
2559 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
2560 CTLFLAG_RD, &stats->vfgptc, "Good Packets Transmitted");
2561 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
2562 CTLFLAG_RD, &stats->vfgotc, "Good Octets Transmitted");
2563 } /* ixv_add_stats_sysctls */
2564
2565 /************************************************************************
2566 * ixv_print_debug_info
2567 *
2568 * Called only when em_display_debug_stats is enabled.
2569 * Provides a way to take a look at important statistics
2570 * maintained by the driver and hardware.
2571 ************************************************************************/
2572 static void
ixv_print_debug_info(struct ixgbe_softc * sc)2573 ixv_print_debug_info(struct ixgbe_softc *sc)
2574 {
2575 device_t dev = sc->dev;
2576 struct ixgbe_hw *hw = &sc->hw;
2577
2578 device_printf(dev, "Error Byte Count = %u \n",
2579 IXGBE_READ_REG(hw, IXGBE_ERRBC));
2580
2581 device_printf(dev, "MBX IRQ Handled: %lu\n", (long)sc->link_irq);
2582 } /* ixv_print_debug_info */
2583
2584 /************************************************************************
2585 * ixv_sysctl_debug
2586 ************************************************************************/
2587 static int
ixv_sysctl_debug(SYSCTL_HANDLER_ARGS)2588 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS)
2589 {
2590 struct ixgbe_softc *sc;
2591 int error, result;
2592
2593 result = -1;
2594 error = sysctl_handle_int(oidp, &result, 0, req);
2595
2596 if (error || !req->newptr)
2597 return (error);
2598
2599 if (result == 1) {
2600 sc = (struct ixgbe_softc *)arg1;
2601 ixv_print_debug_info(sc);
2602 }
2603
2604 return error;
2605 } /* ixv_sysctl_debug */
2606
2607 /************************************************************************
2608 * ixv_init_device_features
2609 ************************************************************************/
2610 static void
ixv_init_device_features(struct ixgbe_softc * sc)2611 ixv_init_device_features(struct ixgbe_softc *sc)
2612 {
2613 sc->feat_cap = IXGBE_FEATURE_NETMAP |
2614 IXGBE_FEATURE_VF |
2615 IXGBE_FEATURE_LEGACY_TX;
2616
2617 /* A tad short on feature flags for VFs, atm. */
2618 switch (sc->hw.mac.type) {
2619 case ixgbe_mac_82599_vf:
2620 break;
2621 case ixgbe_mac_X540_vf:
2622 break;
2623 case ixgbe_mac_X550_vf:
2624 case ixgbe_mac_X550EM_x_vf:
2625 case ixgbe_mac_X550EM_a_vf:
2626 case ixgbe_mac_E610_vf:
2627 sc->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD;
2628 sc->feat_cap |= IXGBE_FEATURE_RSS;
2629 break;
2630 default:
2631 break;
2632 }
2633
2634 /* Enabled by default... */
2635 /* Is a virtual function (VF) */
2636 if (sc->feat_cap & IXGBE_FEATURE_VF)
2637 sc->feat_en |= IXGBE_FEATURE_VF;
2638 /* Netmap */
2639 if (sc->feat_cap & IXGBE_FEATURE_NETMAP)
2640 sc->feat_en |= IXGBE_FEATURE_NETMAP;
2641 /* Receive-Side Scaling (RSS) */
2642 if (sc->feat_cap & IXGBE_FEATURE_RSS)
2643 sc->feat_en |= IXGBE_FEATURE_RSS;
2644 /* Needs advanced context descriptor regardless of offloads req'd */
2645 if (sc->feat_cap & IXGBE_FEATURE_NEEDS_CTXD)
2646 sc->feat_en |= IXGBE_FEATURE_NEEDS_CTXD;
2647 } /* ixv_init_device_features */
2648