1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
4 */
5
6 #include "opt_rss.h"
7
8 #include <sys/param.h>
9 #include <sys/systm.h>
10 #include <sys/kernel.h>
11 #include <sys/endian.h>
12 #include <sys/sockio.h>
13 #include <sys/mbuf.h>
14 #include <sys/malloc.h>
15 #include <sys/module.h>
16 #include <sys/socket.h>
17 #include <sys/sysctl.h>
18 #include <sys/smp.h>
19 #include <vm/vm.h>
20 #include <vm/pmap.h>
21
22 #include <net/ethernet.h>
23 #include <net/if.h>
24 #include <net/if_var.h>
25 #include <net/if_arp.h>
26 #include <net/if_dl.h>
27 #include <net/if_types.h>
28 #include <net/if_media.h>
29 #include <net/if_vlan_var.h>
30 #include <net/iflib.h>
31 #ifdef RSS
32 #include <net/rss_config.h>
33 #endif
34
35 #include <netinet/in_systm.h>
36 #include <netinet/in.h>
37 #include <netinet/ip.h>
38 #include <netinet/ip6.h>
39 #include <netinet6/ip6_var.h>
40 #include <netinet/udp.h>
41 #include <netinet/tcp.h>
42
43 #include <machine/bus.h>
44 #include <machine/resource.h>
45 #include <sys/bus.h>
46 #include <sys/rman.h>
47
48 #include <dev/pci/pcireg.h>
49 #include <dev/pci/pcivar.h>
50
51 #include "ifdi_if.h"
52 #include "enic.h"
53
54 #include "opt_inet.h"
55 #include "opt_inet6.h"
56
57 static SYSCTL_NODE(_hw, OID_AUTO, enic, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
58 "ENIC");
59
60 static const pci_vendor_info_t enic_vendor_info_array[] =
61 {
62 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET,
63 DRV_DESCRIPTION),
64 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET_VF,
65 DRV_DESCRIPTION " VF"),
66 /* required last entry */
67
68 PVID_END
69 };
70
71 static void *enic_register(device_t);
72 static int enic_attach_pre(if_ctx_t);
73 static int enic_msix_intr_assign(if_ctx_t, int);
74
75 static int enic_attach_post(if_ctx_t);
76 static int enic_detach(if_ctx_t);
77
78 static int enic_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
79 static int enic_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
80 static void enic_queues_free(if_ctx_t);
81 static int enic_rxq_intr(void *);
82 static int enic_event_intr(void *);
83 static int enic_err_intr(void *);
84 static void enic_stop(if_ctx_t);
85 static void enic_init(if_ctx_t);
86 static void enic_multi_set(if_ctx_t);
87 static int enic_mtu_set(if_ctx_t, uint32_t);
88 static void enic_media_status(if_ctx_t, struct ifmediareq *);
89 static int enic_media_change(if_ctx_t);
90 static int enic_promisc_set(if_ctx_t, int);
91 static uint64_t enic_get_counter(if_ctx_t, ift_counter);
92 static void enic_update_admin_status(if_ctx_t);
93 static void enic_txq_timer(if_ctx_t, uint16_t);
94 static int enic_link_is_up(struct enic_softc *);
95 static void enic_link_status(struct enic_softc *);
96 static void enic_set_lladdr(struct enic_softc *);
97 static void enic_setup_txq_sysctl(struct vnic_wq *, int, struct sysctl_ctx_list *,
98 struct sysctl_oid_list *);
99 static void enic_setup_rxq_sysctl(struct vnic_rq *, int, struct sysctl_ctx_list *,
100 struct sysctl_oid_list *);
101 static void enic_setup_sysctl(struct enic_softc *);
102 static int enic_tx_queue_intr_enable(if_ctx_t, uint16_t);
103 static int enic_rx_queue_intr_enable(if_ctx_t, uint16_t);
104 static void enic_enable_intr(struct enic_softc *, int);
105 static void enic_disable_intr(struct enic_softc *, int);
106 static void enic_intr_enable_all(if_ctx_t);
107 static void enic_intr_disable_all(if_ctx_t);
108 static int enic_dev_open(struct enic *);
109 static int enic_dev_init(struct enic *);
110 static void *enic_alloc_consistent(void *, size_t, bus_addr_t *,
111 struct iflib_dma_info *, u8 *);
112 static void enic_free_consistent(void *, size_t, void *, bus_addr_t,
113 struct iflib_dma_info *);
114 static int enic_pci_mapping(struct enic_softc *);
115 static void enic_pci_mapping_free(struct enic_softc *);
116 static int enic_dev_wait(struct vnic_dev *, int (*) (struct vnic_dev *, int),
117 int (*) (struct vnic_dev *, int *), int arg);
118 static int enic_map_bar(struct enic_softc *, struct enic_bar_info *, int, bool);
119 static void enic_update_packet_filter(struct enic *enic);
120 static bool enic_if_needs_restart(if_ctx_t, enum iflib_restart_event);
121
122 typedef enum {
123 ENIC_BARRIER_RD,
124 ENIC_BARRIER_WR,
125 ENIC_BARRIER_RDWR,
126 } enic_barrier_t;
127
128 static device_method_t enic_methods[] = {
129 /* Device interface */
130 DEVMETHOD(device_register, enic_register),
131 DEVMETHOD(device_probe, iflib_device_probe),
132 DEVMETHOD(device_attach, iflib_device_attach),
133 DEVMETHOD(device_detach, iflib_device_detach),
134 DEVMETHOD(device_shutdown, iflib_device_shutdown),
135 DEVMETHOD(device_suspend, iflib_device_suspend),
136 DEVMETHOD(device_resume, iflib_device_resume),
137 DEVMETHOD_END
138 };
139
140 static driver_t enic_driver = {
141 "enic", enic_methods, sizeof(struct enic_softc)
142 };
143
144 DRIVER_MODULE(enic, pci, enic_driver, 0, 0);
145 IFLIB_PNP_INFO(pci, enic, enic_vendor_info_array);
146 MODULE_VERSION(enic, 2);
147
148 MODULE_DEPEND(enic, pci, 1, 1, 1);
149 MODULE_DEPEND(enic, ether, 1, 1, 1);
150 MODULE_DEPEND(enic, iflib, 1, 1, 1);
151
152 static device_method_t enic_iflib_methods[] = {
153 DEVMETHOD(ifdi_tx_queues_alloc, enic_tx_queues_alloc),
154 DEVMETHOD(ifdi_rx_queues_alloc, enic_rx_queues_alloc),
155 DEVMETHOD(ifdi_queues_free, enic_queues_free),
156
157 DEVMETHOD(ifdi_attach_pre, enic_attach_pre),
158 DEVMETHOD(ifdi_attach_post, enic_attach_post),
159 DEVMETHOD(ifdi_detach, enic_detach),
160
161 DEVMETHOD(ifdi_init, enic_init),
162 DEVMETHOD(ifdi_stop, enic_stop),
163 DEVMETHOD(ifdi_multi_set, enic_multi_set),
164 DEVMETHOD(ifdi_mtu_set, enic_mtu_set),
165 DEVMETHOD(ifdi_media_status, enic_media_status),
166 DEVMETHOD(ifdi_media_change, enic_media_change),
167 DEVMETHOD(ifdi_promisc_set, enic_promisc_set),
168 DEVMETHOD(ifdi_get_counter, enic_get_counter),
169 DEVMETHOD(ifdi_update_admin_status, enic_update_admin_status),
170 DEVMETHOD(ifdi_timer, enic_txq_timer),
171
172 DEVMETHOD(ifdi_tx_queue_intr_enable, enic_tx_queue_intr_enable),
173 DEVMETHOD(ifdi_rx_queue_intr_enable, enic_rx_queue_intr_enable),
174 DEVMETHOD(ifdi_intr_enable, enic_intr_enable_all),
175 DEVMETHOD(ifdi_intr_disable, enic_intr_disable_all),
176 DEVMETHOD(ifdi_msix_intr_assign, enic_msix_intr_assign),
177
178 DEVMETHOD(ifdi_needs_restart, enic_if_needs_restart),
179
180 DEVMETHOD_END
181 };
182
183 static driver_t enic_iflib_driver = {
184 "enic", enic_iflib_methods, sizeof(struct enic_softc)
185 };
186
187 extern struct if_txrx enic_txrx;
188
189 static struct if_shared_ctx enic_sctx_init = {
190 .isc_magic = IFLIB_MAGIC,
191 .isc_q_align = 512,
192
193 .isc_tx_maxsize = ENIC_TX_MAX_PKT_SIZE,
194 .isc_tx_maxsegsize = PAGE_SIZE,
195
196 /*
197 * These values are used to configure the busdma tag used for receive
198 * descriptors. Each receive descriptor only points to one buffer.
199 */
200 .isc_rx_maxsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, /* One buf per
201 * descriptor */
202 .isc_rx_nsegments = 1, /* One mapping per descriptor */
203 .isc_rx_maxsegsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE,
204 .isc_admin_intrcnt = 2,
205 .isc_vendor_info = enic_vendor_info_array,
206 .isc_driver_version = "1",
207 .isc_driver = &enic_iflib_driver,
208 .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ | IFLIB_SKIP_MSIX,
209
210 /*
211 * Number of receive queues per receive queue set, with associated
212 * descriptor settings for each.
213 */
214
215 .isc_nrxqs = 2,
216 .isc_nfl = 1, /* one free list for each receive command
217 * queue */
218 .isc_nrxd_min = {16, 16},
219 .isc_nrxd_max = {2048, 2048},
220 .isc_nrxd_default = {64, 64},
221
222 /*
223 * Number of transmit queues per transmit queue set, with associated
224 * descriptor settings for each.
225 */
226 .isc_ntxqs = 2,
227 .isc_ntxd_min = {16, 16},
228 .isc_ntxd_max = {2048, 2048},
229 .isc_ntxd_default = {64, 64},
230 };
231
232 static void *
enic_register(device_t dev)233 enic_register(device_t dev)
234 {
235 return (&enic_sctx_init);
236 }
237
238 static int
enic_allocate_msix(struct enic_softc * softc)239 enic_allocate_msix(struct enic_softc *softc) {
240 if_ctx_t ctx;
241 if_softc_ctx_t scctx;
242 if_shared_ctx_t sctx;
243 device_t dev;
244 cpuset_t cpus;
245 int queues, vectors, requested;
246 int err = 0;
247
248 dev = softc->dev;
249 ctx = softc->ctx;
250 scctx = softc->scctx;
251 sctx = iflib_get_sctx(ctx);
252
253 if (bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus) != 0) {
254 device_printf(dev, "Unable to fetch CPU list\n");
255 CPU_COPY(&all_cpus, &cpus);
256 }
257
258
259 queues = CPU_COUNT(&cpus);
260 queues = imin(queues, scctx->isc_nrxqsets);
261 queues = imin(queues, scctx->isc_ntxqsets);
262 requested = queues * 2 + sctx->isc_admin_intrcnt;
263 scctx->isc_nrxqsets = queues;
264 scctx->isc_ntxqsets = queues;
265
266 vectors = requested;
267 if ((err = pci_alloc_msix(dev, &vectors)) != 0) {
268 device_printf(dev,
269 "failed to allocate %d MSI-X vectors, err: %d\n", requested,
270 err);
271 err = 1;
272 goto enic_allocate_msix_out;
273 } else {
274 if (vectors != requested) {
275 device_printf(dev,
276 "Unable to allocate sufficient MSI-X vectors "
277 "(got %d, need %d)\n", requested, vectors);
278 pci_release_msi(dev);
279 err = 1;
280 goto enic_allocate_msix_out;
281 }
282 }
283
284 device_printf(dev, "Using MSI-X interrupts with %d vectors\n",
285 vectors);
286
287 scctx->isc_intr = IFLIB_INTR_MSIX;
288 scctx->isc_vectors = vectors;
289
290 enic_allocate_msix_out:
291 return (err);
292
293 }
294
295 static struct enic_intr_mod_range mod_range[ENIC_MAX_LINK_SPEEDS] = {
296 {0, 0}, /* 0 - 4 Gbps */
297 {0, 3}, /* 4 - 10 Gbps */
298 {3, 6}, /* 10 - 40 Gbps */
299 };
300
enic_set_rx_coal_setting(struct enic * enic)301 static void enic_set_rx_coal_setting(struct enic *enic)
302 {
303 unsigned int speed;
304 int index = -1;
305 struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting;
306
307 /* 1. Read the link speed from fw
308 * 2. Pick the default range for the speed
309 * 3. Update it in enic->rx_coalesce_setting
310 */
311 speed = vnic_dev_port_speed(enic->vdev);
312 if (ENIC_LINK_SPEED_10G < speed)
313 index = ENIC_LINK_40G_INDEX;
314 else if (ENIC_LINK_SPEED_4G < speed)
315 index = ENIC_LINK_10G_INDEX;
316 else
317 index = ENIC_LINK_4G_INDEX;
318
319 rx_coal->small_pkt_range_start = mod_range[index].small_pkt_range_start;
320 rx_coal->large_pkt_range_start = mod_range[index].large_pkt_range_start;
321 rx_coal->range_end = ENIC_RX_COALESCE_RANGE_END;
322
323 /* Start with the value provided by UCSM */
324 for (index = 0; index < enic->rq_count; index++)
325 enic->cq[index].cur_rx_coal_timeval =
326 enic->config.intr_timer_usec;
327
328 rx_coal->use_adaptive_rx_coalesce = 1;
329 }
330
331 static int
enic_attach_pre(if_ctx_t ctx)332 enic_attach_pre(if_ctx_t ctx)
333 {
334 if_softc_ctx_t scctx;
335 struct enic_softc *softc;
336 struct vnic_dev *vdev;
337 struct enic *enic;
338 device_t dev;
339
340 int err = -1;
341 int rc = 0;
342 int i;
343 u64 a0 = 0, a1 = 0;
344 int wait = 1000;
345 struct vnic_stats *stats;
346 int ret;
347
348 dev = iflib_get_dev(ctx);
349 softc = iflib_get_softc(ctx);
350 softc->dev = dev;
351 softc->ctx = ctx;
352 softc->sctx = iflib_get_sctx(ctx);
353 softc->scctx = iflib_get_softc_ctx(ctx);
354 softc->ifp = iflib_get_ifp(ctx);
355 softc->media = iflib_get_media(ctx);
356 softc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN *
357 ENIC_MAX_MULTICAST_ADDRESSES, M_DEVBUF,
358 M_NOWAIT | M_ZERO);
359 if (softc->mta == NULL)
360 return (ENOMEM);
361 scctx = softc->scctx;
362
363 mtx_init(&softc->enic_lock, "ENIC Lock", NULL, MTX_DEF);
364
365 pci_enable_busmaster(softc->dev);
366 if (enic_pci_mapping(softc))
367 return (ENXIO);
368
369 enic = &softc->enic;
370 enic->softc = softc;
371 vdev = &softc->vdev;
372 vdev->softc = softc;
373 enic->vdev = vdev;
374 vdev->priv = enic;
375
376 ENIC_LOCK(softc);
377 vnic_dev_register(vdev, &softc->mem, 1);
378 enic->vdev = vdev;
379 vnic_dev_cmd_init(enic->vdev);
380
381 vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0);
382
383 vnic_dev_cmd(vdev, CMD_INIT_v1, &a0, &a1, wait);
384 vnic_dev_cmd(vdev, CMD_GET_MAC_ADDR, &a0, &a1, wait);
385
386 bcopy((u_int8_t *) & a0, softc->mac_addr, ETHER_ADDR_LEN);
387 iflib_set_mac(ctx, softc->mac_addr);
388
389 vnic_register_cbacks(enic->vdev, enic_alloc_consistent,
390 enic_free_consistent);
391
392 /*
393 * Allocate the consistent memory for stats and counters upfront so
394 * both primary and secondary processes can access them.
395 */
396 ENIC_UNLOCK(softc);
397 err = vnic_dev_alloc_stats_mem(enic->vdev);
398 ENIC_LOCK(softc);
399 if (err) {
400 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
401 goto err_out_unregister;
402 }
403 vnic_dev_stats_clear(enic->vdev);
404 ret = vnic_dev_stats_dump(enic->vdev, &stats);
405 if (ret) {
406 dev_err(enic, "Error in getting stats\n");
407 goto err_out_unregister;
408 }
409 err = vnic_dev_alloc_counter_mem(enic->vdev);
410 if (err) {
411 dev_err(enic, "Failed to allocate counter memory, aborting\n");
412 goto err_out_unregister;
413 }
414
415 /* Issue device open to get device in known state */
416 err = enic_dev_open(enic);
417 if (err) {
418 dev_err(enic, "vNIC dev open failed, aborting\n");
419 goto err_out_unregister;
420 }
421
422 /* Set ingress vlan rewrite mode before vnic initialization */
423 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN;
424 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN;
425 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
426 enic->ig_vlan_rewrite_mode);
427 if (err) {
428 dev_err(enic,
429 "Failed to set ingress vlan rewrite mode, aborting.\n");
430 goto err_out_dev_close;
431 }
432
433 /*
434 * Issue device init to initialize the vnic-to-switch link. We'll
435 * start with carrier off and wait for link UP notification later to
436 * turn on carrier. We don't need to wait here for the
437 * vnic-to-switch link initialization to complete; link UP
438 * notification is the indication that the process is complete.
439 */
440
441 err = vnic_dev_init(enic->vdev, 0);
442 if (err) {
443 dev_err(enic, "vNIC dev init failed, aborting\n");
444 goto err_out_dev_close;
445 }
446
447 err = enic_dev_init(enic);
448 if (err) {
449 dev_err(enic, "Device initialization failed, aborting\n");
450 goto err_out_dev_close;
451 }
452 /* Queue DMA has not been enabled; the first iflib stop may be skipped. */
453 softc->stopped = 1;
454 ENIC_UNLOCK(softc);
455
456 enic->port_mtu = vnic_dev_mtu(enic->vdev);
457
458 softc->scctx = iflib_get_softc_ctx(ctx);
459 scctx = softc->scctx;
460 scctx->isc_txrx = &enic_txrx;
461 scctx->isc_capabilities = scctx->isc_capenable = \
462 IFCAP_HWCSUM;
463 scctx->isc_tx_csum_flags = 0;
464 if_setmtu(softc->ifp, enic->config.mtu);
465 scctx->isc_max_frame_size = enic->config.mtu + ETHER_HDR_LEN + \
466 ETHER_CRC_LEN;
467 scctx->isc_nrxqsets_max = enic->conf_rq_count;
468 scctx->isc_ntxqsets_max = enic->conf_wq_count;
469 scctx->isc_nrxqsets = enic->conf_rq_count;
470 scctx->isc_ntxqsets = enic->conf_wq_count;
471 for (i = 0; i < enic->conf_wq_count; i++) {
472 scctx->isc_ntxd[i] = enic->config.wq_desc_count;
473 scctx->isc_txqsizes[i] = sizeof(struct cq_enet_wq_desc)
474 * scctx->isc_ntxd[i];
475 scctx->isc_ntxd[i + enic->conf_wq_count] =
476 enic->config.wq_desc_count;
477 scctx->isc_txqsizes[i + enic->conf_wq_count] =
478 sizeof(struct cq_desc) * scctx->isc_ntxd[i +
479 enic->conf_wq_count];
480 }
481 for (i = 0; i < enic->conf_rq_count; i++) {
482 scctx->isc_nrxd[i] = enic->config.rq_desc_count;
483 scctx->isc_rxqsizes[i] = sizeof(struct cq_enet_rq_desc) *
484 scctx->isc_nrxd[i];
485 scctx->isc_nrxd[i + enic->conf_rq_count] =
486 enic->config.rq_desc_count;
487 scctx->isc_rxqsizes[i + enic->conf_rq_count] = sizeof(struct
488 cq_desc) * scctx->isc_nrxd[i + enic->conf_rq_count];
489 }
490 scctx->isc_tx_nsegments = 31;
491
492 scctx->isc_msix_bar = -1;
493
494 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
495 ifmedia_add(softc->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
496 ifmedia_add(softc->media, IFM_ETHER | IFM_10_FL, 0, NULL);
497
498 err = enic_allocate_msix(softc);
499 if (err) {
500 dev_err(enic, "Failed to allocate MSIX, aborting\n");
501 goto err_out_dev_close;
502 }
503
504 return (rc);
505
506 err_out_dev_close:
507 vnic_dev_close(enic->vdev);
508 vnic_dev_deinit_devcmd2(enic->vdev);
509 err_out_unregister:
510 free(softc->vdev.devcmd, M_DEVBUF);
511 free(softc->enic.intr_queues, M_DEVBUF);
512 free(softc->enic.cq, M_DEVBUF);
513 free(softc->mta, M_DEVBUF);
514 rc = -1;
515 pci_disable_busmaster(softc->dev);
516 enic_pci_mapping_free(softc);
517 mtx_destroy(&softc->enic_lock);
518 return (rc);
519 }
520
521 static int
enic_msix_intr_assign(if_ctx_t ctx,int msix)522 enic_msix_intr_assign(if_ctx_t ctx, int msix)
523 {
524 struct enic_softc *softc;
525 struct enic *enic;
526 if_softc_ctx_t scctx;
527
528 int error;
529 int i;
530 char irq_name[16];
531
532 softc = iflib_get_softc(ctx);
533 enic = &softc->enic;
534 scctx = softc->scctx;
535
536 ENIC_LOCK(softc);
537 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSIX);
538 ENIC_UNLOCK(softc);
539
540 if (enic->intr_queues == NULL)
541 enic->intr_queues = malloc(sizeof(*enic->intr_queues) *
542 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO);
543 enic->intr = malloc(sizeof(*enic->intr) * msix, M_DEVBUF, M_NOWAIT
544 | M_ZERO);
545 if (enic->intr_queues == NULL || enic->intr == NULL)
546 return (ENOMEM);
547 for (i = 0; i < scctx->isc_nrxqsets; i++) {
548 snprintf(irq_name, sizeof(irq_name), "erxq%d:%d", i,
549 device_get_unit(softc->dev));
550
551 error = iflib_irq_alloc_generic(ctx,
552 &enic->intr_queues[i].intr_irq, i + 1, IFLIB_INTR_RX,
553 enic_rxq_intr, &enic->rq[i], i, irq_name);
554 if (error) {
555 device_printf(iflib_get_dev(ctx),
556 "Failed to register rxq %d interrupt handler\n", i);
557 return (error);
558 }
559 enic->intr[i].index = i;
560 enic->intr[i].vdev = enic->vdev;
561 ENIC_LOCK(softc);
562 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev,
563 RES_TYPE_INTR_CTRL, i);
564 vnic_intr_mask(&enic->intr[i]);
565 ENIC_UNLOCK(softc);
566 }
567
568 for (i = scctx->isc_nrxqsets; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) {
569 snprintf(irq_name, sizeof(irq_name), "etxq%d:%d", i -
570 scctx->isc_nrxqsets, device_get_unit(softc->dev));
571
572 iflib_softirq_alloc_generic(ctx,
573 &enic->intr_queues[i].intr_irq, IFLIB_INTR_TX,
574 &enic->wq[i - scctx->isc_nrxqsets], i - scctx->isc_nrxqsets,
575 irq_name);
576
577 enic->intr[i].index = i;
578 enic->intr[i].vdev = enic->vdev;
579 ENIC_LOCK(softc);
580 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev,
581 RES_TYPE_INTR_CTRL, i);
582 vnic_intr_mask(&enic->intr[i]);
583 ENIC_UNLOCK(softc);
584 }
585
586 i = scctx->isc_nrxqsets + scctx->isc_ntxqsets;
587 error = iflib_irq_alloc_generic(ctx, &softc->enic_event_intr_irq,
588 i + 1, IFLIB_INTR_ADMIN, enic_event_intr, softc, 0, "event");
589 if (error) {
590 device_printf(iflib_get_dev(ctx),
591 "Failed to register event interrupt handler\n");
592 return (error);
593 }
594
595 enic->intr[i].index = i;
596 enic->intr[i].vdev = enic->vdev;
597 ENIC_LOCK(softc);
598 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL,
599 i);
600 vnic_intr_mask(&enic->intr[i]);
601 ENIC_UNLOCK(softc);
602
603 i++;
604 error = iflib_irq_alloc_generic(ctx, &softc->enic_err_intr_irq,
605 i + 1, IFLIB_INTR_ADMIN, enic_err_intr, softc, 0, "err");
606 if (error) {
607 device_printf(iflib_get_dev(ctx),
608 "Failed to register event interrupt handler\n");
609 return (error);
610 }
611 enic->intr[i].index = i;
612 enic->intr[i].vdev = enic->vdev;
613 ENIC_LOCK(softc);
614 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL,
615 i);
616 vnic_intr_mask(&enic->intr[i]);
617 ENIC_UNLOCK(softc);
618
619 enic->intr_count = msix;
620
621 return (0);
622 }
623
624 static void
enic_free_irqs(struct enic_softc * softc)625 enic_free_irqs(struct enic_softc *softc)
626 {
627 if_softc_ctx_t scctx;
628
629 struct enic *enic;
630 int i;
631
632 scctx = softc->scctx;
633 enic = &softc->enic;
634
635 if (enic->intr_queues != NULL) {
636 for (i = 0;
637 i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++)
638 iflib_irq_free(softc->ctx,
639 &enic->intr_queues[i].intr_irq);
640 }
641
642 iflib_irq_free(softc->ctx, &softc->enic_event_intr_irq);
643 iflib_irq_free(softc->ctx, &softc->enic_err_intr_irq);
644 free(enic->intr_queues, M_DEVBUF);
645 enic->intr_queues = NULL;
646 free(enic->intr, M_DEVBUF);
647 enic->intr = NULL;
648 }
649
650 static int
enic_attach_post(if_ctx_t ctx)651 enic_attach_post(if_ctx_t ctx)
652 {
653 struct enic *enic;
654 struct enic_softc *softc;
655 int error = 0;
656
657 softc = iflib_get_softc(ctx);
658 enic = &softc->enic;
659
660 enic_setup_sysctl(softc);
661
662 enic_init_vnic_resources(enic);
663 enic_set_rx_coal_setting(enic);
664 enic_setup_finish(enic);
665
666 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
667 ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO);
668
669 return (error);
670 }
671
672 static int
enic_detach(if_ctx_t ctx)673 enic_detach(if_ctx_t ctx)
674 {
675 struct enic_softc *softc;
676 struct enic *enic;
677
678 softc = iflib_get_softc(ctx);
679 enic = &softc->enic;
680
681 vnic_dev_notify_unset(enic->vdev);
682
683 enic_free_irqs(softc);
684
685 ENIC_LOCK(softc);
686 vnic_dev_deinit(enic->vdev);
687 vnic_dev_close(enic->vdev);
688 vnic_dev_deinit_devcmd2(enic->vdev);
689 free(softc->vdev.devcmd, M_DEVBUF);
690 softc->vdev.devcmd = NULL;
691 pci_disable_busmaster(softc->dev);
692 enic_pci_mapping_free(softc);
693 ENIC_UNLOCK(softc);
694 if (softc->vdev.stats_res.idi_size != 0) {
695 iflib_dma_free(&softc->vdev.stats_res);
696 softc->vdev.stats = NULL;
697 }
698 if (softc->vdev.flow_counters_res.idi_size != 0) {
699 iflib_dma_free(&softc->vdev.flow_counters_res);
700 softc->vdev.flow_counters = NULL;
701 }
702 free(softc->mta, M_DEVBUF);
703 softc->mta = NULL;
704 mtx_destroy(&softc->enic_lock);
705
706 return 0;
707 }
708
709 static int
enic_tx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int ntxqs,int ntxqsets)710 enic_tx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs,
711 int ntxqs, int ntxqsets)
712 {
713 struct enic_softc *softc;
714 int q;
715
716 softc = iflib_get_softc(ctx);
717 softc->enic.cq = malloc(sizeof(*softc->enic.cq) *
718 (softc->enic.wq_count + softc->enic.rq_count), M_DEVBUF,
719 M_NOWAIT | M_ZERO);
720 if (softc->enic.cq == NULL)
721 return (ENOMEM);
722 /* Allocate the array of transmit queues */
723 softc->enic.wq = malloc(sizeof(struct vnic_wq) *
724 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO);
725 if (softc->enic.wq == NULL) {
726 free(softc->enic.cq, M_DEVBUF);
727 softc->enic.cq = NULL;
728 return (ENOMEM);
729 }
730
731 /* Initialize driver state for each transmit queue */
732
733 /*
734 * Allocate queue state that is shared with the device. This check
735 * and call is performed in both enic_tx_queues_alloc() and
736 * enic_rx_queues_alloc() so that we don't have to care which order
737 * iflib invokes those routines in.
738 */
739
740 /* Record descriptor ring vaddrs and paddrs */
741 ENIC_LOCK(softc);
742 for (q = 0; q < ntxqsets; q++) {
743 struct vnic_wq *wq;
744 struct vnic_cq *cq;
745 unsigned int cq_wq;
746
747 wq = &softc->enic.wq[q];
748 cq_wq = enic_cq_wq(&softc->enic, q);
749 cq = &softc->enic.cq[cq_wq];
750
751 /* Completion ring */
752 wq->vdev = softc->enic.vdev;
753 wq->index = q;
754 wq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_WQ,
755 wq->index);
756 vnic_wq_disable(wq);
757
758 wq->ring.desc_size = sizeof(struct wq_enet_desc);
759 wq->ring.desc_count = softc->scctx->isc_ntxd[q];
760 wq->ring.desc_avail = wq->ring.desc_count - 1;
761 wq->ring.last_count = wq->ring.desc_count;
762 wq->head_idx = 0;
763 wq->tail_idx = 0;
764
765 wq->ring.descs = vaddrs[q * ntxqs + 0];
766 wq->ring.base_addr = paddrs[q * ntxqs + 0];
767
768 /* Command ring */
769 cq->vdev = softc->enic.vdev;
770 cq->index = cq_wq;
771 cq->ctrl = vnic_dev_get_res(softc->enic.vdev,
772 RES_TYPE_CQ, cq->index);
773 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc);
774 cq->ring.desc_count = softc->scctx->isc_ntxd[q];
775 cq->ring.desc_avail = cq->ring.desc_count - 1;
776
777 cq->ring.descs = vaddrs[q * ntxqs + 1];
778 cq->ring.base_addr = paddrs[q * ntxqs + 1];
779
780 }
781
782 ENIC_UNLOCK(softc);
783
784 return (0);
785 }
786
787
788
789 static int
enic_rx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int nrxqs,int nrxqsets)790 enic_rx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs,
791 int nrxqs, int nrxqsets)
792 {
793 struct enic_softc *softc;
794 int q;
795
796 softc = iflib_get_softc(ctx);
797 /* Allocate the array of receive queues */
798 softc->enic.rq = malloc(sizeof(struct vnic_rq) * nrxqsets, M_DEVBUF,
799 M_NOWAIT | M_ZERO);
800 if (softc->enic.rq == NULL) {
801 free(softc->enic.wq, M_DEVBUF);
802 softc->enic.wq = NULL;
803 free(softc->enic.cq, M_DEVBUF);
804 softc->enic.cq = NULL;
805 return (ENOMEM);
806 }
807
808 /* Initialize driver state for each receive queue */
809
810 /*
811 * Allocate queue state that is shared with the device. This check
812 * and call is performed in both enic_tx_queues_alloc() and
813 * enic_rx_queues_alloc() so that we don't have to care which order
814 * iflib invokes those routines in.
815 */
816
817 /* Record descriptor ring vaddrs and paddrs */
818 ENIC_LOCK(softc);
819 for (q = 0; q < nrxqsets; q++) {
820 struct vnic_rq *rq;
821 struct vnic_cq *cq;
822 unsigned int cq_rq;
823
824 rq = &softc->enic.rq[q];
825 cq_rq = enic_cq_rq(&softc->enic, q);
826 cq = &softc->enic.cq[cq_rq];
827
828 /* Completion ring */
829 cq->vdev = softc->enic.vdev;
830 cq->index = cq_rq;
831 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_CQ,
832 cq->index);
833 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc);
834 cq->ring.desc_count = softc->scctx->isc_nrxd[1];
835 cq->ring.desc_avail = cq->ring.desc_count - 1;
836
837 cq->ring.descs = vaddrs[q * nrxqs + 0];
838 cq->ring.base_addr = paddrs[q * nrxqs + 0];
839
840 /* Command ring(s) */
841 rq->vdev = softc->enic.vdev;
842
843 rq->index = q;
844 rq->ctrl = vnic_dev_get_res(softc->enic.vdev,
845 RES_TYPE_RQ, rq->index);
846 vnic_rq_disable(rq);
847
848 rq->ring.desc_size = sizeof(struct rq_enet_desc);
849 rq->ring.desc_count = softc->scctx->isc_nrxd[0];
850 rq->ring.desc_avail = rq->ring.desc_count - 1;
851
852 rq->ring.descs = vaddrs[q * nrxqs + 1];
853 rq->ring.base_addr = paddrs[q * nrxqs + 1];
854 rq->need_initial_post = true;
855 }
856
857 ENIC_UNLOCK(softc);
858
859 return (0);
860 }
861
862 static void
enic_queues_free(if_ctx_t ctx)863 enic_queues_free(if_ctx_t ctx)
864 {
865 struct enic_softc *softc;
866 softc = iflib_get_softc(ctx);
867
868 free(softc->enic.rq, M_DEVBUF);
869 softc->enic.rq = NULL;
870 free(softc->enic.wq, M_DEVBUF);
871 softc->enic.wq = NULL;
872 free(softc->enic.cq, M_DEVBUF);
873 softc->enic.cq = NULL;
874 }
875
876 static int
enic_rxq_intr(void * rxq)877 enic_rxq_intr(void *rxq)
878 {
879 struct vnic_rq *rq;
880
881 rq = (struct vnic_rq *)rxq;
882 if (!iflib_is_running(rq->vdev->softc->ctx))
883 return (FILTER_HANDLED);
884
885 return (FILTER_SCHEDULE_THREAD);
886 }
887
888 static int
enic_event_intr(void * vsc)889 enic_event_intr(void *vsc)
890 {
891 struct enic_softc *softc;
892 struct enic *enic;
893 uint32_t mtu;
894
895 softc = vsc;
896 enic = &softc->enic;
897
898 mtu = vnic_dev_mtu(enic->vdev);
899 if (mtu && mtu != enic->port_mtu) {
900 enic->port_mtu = mtu;
901 }
902
903 enic_link_status(softc);
904
905 return (FILTER_HANDLED);
906 }
907
908 static int
enic_err_intr(void * vsc)909 enic_err_intr(void *vsc)
910 {
911 struct enic_softc *softc;
912
913 softc = vsc;
914
915 iflib_request_reset_if_up(softc->ctx);
916 iflib_admin_intr_deferred(softc->ctx);
917
918 return (FILTER_HANDLED);
919 }
920
921 static void
enic_stop(if_ctx_t ctx)922 enic_stop(if_ctx_t ctx)
923 {
924 struct enic_softc *softc;
925 struct enic *enic;
926 if_softc_ctx_t scctx;
927 unsigned int index;
928 struct vnic_wq *wq;
929 struct vnic_rq *rq;
930 struct vnic_cq *cq;
931 unsigned int cq_wq, cq_rq;
932
933
934 softc = iflib_get_softc(ctx);
935 scctx = softc->scctx;
936 enic = &softc->enic;
937
938 if (softc->stopped)
939 return;
940 softc->link_active = 0;
941 softc->stopped = 1;
942
943 enic_dev_disable(enic);
944
945 for (index = 0; index < scctx->isc_ntxqsets; index++) {
946 enic_stop_wq(enic, index);
947 vnic_wq_clean(&enic->wq[index]);
948 vnic_cq_clean(&enic->cq[enic_cq_wq(enic, index)]);
949
950 wq = &softc->enic.wq[index];
951 wq->ring.desc_avail = wq->ring.desc_count - 1;
952 wq->ring.last_count = wq->ring.desc_count;
953 wq->head_idx = 0;
954 wq->tail_idx = 0;
955
956 cq_wq = enic_cq_wq(&softc->enic, index);
957 cq = &softc->enic.cq[cq_wq];
958 cq->ring.desc_avail = cq->ring.desc_count - 1;
959 }
960
961 for (index = 0; index < scctx->isc_nrxqsets; index++) {
962 enic_stop_rq(enic, index);
963 vnic_rq_clean(&enic->rq[index]);
964 vnic_cq_clean(&enic->cq[enic_cq_rq(enic, index)]);
965
966 rq = &softc->enic.rq[index];
967 cq_rq = enic_cq_rq(&softc->enic, index);
968 cq = &softc->enic.cq[cq_rq];
969
970 cq->ring.desc_avail = cq->ring.desc_count - 1;
971 rq->ring.desc_avail = rq->ring.desc_count - 1;
972 rq->need_initial_post = true;
973 }
974
975 for (index = 0; index < scctx->isc_vectors; index++) {
976 vnic_intr_clean(&enic->intr[index]);
977 }
978 }
979
980 static void
enic_init(if_ctx_t ctx)981 enic_init(if_ctx_t ctx)
982 {
983 struct enic_softc *softc;
984 struct enic *enic;
985 if_softc_ctx_t scctx;
986 unsigned int index;
987 int error;
988
989 softc = iflib_get_softc(ctx);
990 scctx = softc->scctx;
991 enic = &softc->enic;
992
993
994 enic_init_vnic_resources(enic);
995
996 for (index = 0; index < scctx->isc_ntxqsets; index++)
997 enic_prep_wq_for_simple_tx(&softc->enic, index);
998
999 for (index = 0; index < scctx->isc_ntxqsets; index++)
1000 enic_start_wq(enic, index);
1001
1002 for (index = 0; index < scctx->isc_nrxqsets; index++)
1003 enic_start_rq(enic, index);
1004
1005 /* Use the current MAC address. */
1006 bcopy(if_getlladdr(softc->ifp), softc->lladdr, ETHER_ADDR_LEN);
1007 enic_set_lladdr(softc);
1008
1009 /* Queue setup above needs normal stop cleanup even if enable fails. */
1010 softc->stopped = 0;
1011 ENIC_LOCK(softc);
1012 error = vnic_dev_enable_wait(enic->vdev);
1013 ENIC_UNLOCK(softc);
1014 if (error != 0) {
1015 device_printf(softc->dev, "Device enable failed: %d\n", error);
1016 enic_stop(ctx);
1017 iflib_init_failed(ctx);
1018 return;
1019 }
1020
1021 enic_link_status(softc);
1022 }
1023
1024 static void
enic_del_mcast(struct enic_softc * softc)1025 enic_del_mcast(struct enic_softc *softc) {
1026 struct enic *enic;
1027 int i;
1028
1029 enic = &softc->enic;
1030 for (i=0; i < softc->mc_count; i++) {
1031 vnic_dev_del_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]);
1032 }
1033 softc->multicast = 0;
1034 softc->mc_count = 0;
1035 }
1036
1037 static void
enic_add_mcast(struct enic_softc * softc)1038 enic_add_mcast(struct enic_softc *softc) {
1039 struct enic *enic;
1040 int i;
1041
1042 enic = &softc->enic;
1043 for (i=0; i < softc->mc_count; i++) {
1044 vnic_dev_add_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]);
1045 }
1046 softc->multicast = 1;
1047 }
1048
1049 static u_int
enic_copy_maddr(void * arg,struct sockaddr_dl * sdl,u_int idx)1050 enic_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx)
1051 {
1052 uint8_t *mta = arg;
1053
1054 if (idx == ENIC_MAX_MULTICAST_ADDRESSES)
1055 return (0);
1056
1057 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN);
1058 return (1);
1059 }
1060
1061 static void
enic_multi_set(if_ctx_t ctx)1062 enic_multi_set(if_ctx_t ctx)
1063 {
1064 if_t ifp;
1065 struct enic_softc *softc;
1066 u_int count;
1067
1068 softc = iflib_get_softc(ctx);
1069 ifp = iflib_get_ifp(ctx);
1070
1071 ENIC_LOCK(softc);
1072 enic_del_mcast(softc);
1073 count = if_foreach_llmaddr(ifp, enic_copy_maddr, softc->mta);
1074 softc->mc_count = count;
1075 enic_add_mcast(softc);
1076 ENIC_UNLOCK(softc);
1077
1078 if (if_getflags(ifp) & IFF_PROMISC) {
1079 softc->promisc = 1;
1080 } else {
1081 softc->promisc = 0;
1082 }
1083 if (if_getflags(ifp) & IFF_ALLMULTI) {
1084 softc->allmulti = 1;
1085 } else {
1086 softc->allmulti = 0;
1087 }
1088 enic_update_packet_filter(&softc->enic);
1089 }
1090
1091 static int
enic_mtu_set(if_ctx_t ctx,uint32_t mtu)1092 enic_mtu_set(if_ctx_t ctx, uint32_t mtu)
1093 {
1094 struct enic_softc *softc;
1095 struct enic *enic;
1096 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx);
1097
1098 softc = iflib_get_softc(ctx);
1099 enic = &softc->enic;
1100
1101 if (mtu > enic->port_mtu){
1102 return (EINVAL);
1103 }
1104
1105 enic->config.mtu = mtu;
1106 scctx->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1107
1108 return (0);
1109 }
1110
1111 static void
enic_media_status(if_ctx_t ctx,struct ifmediareq * ifmr)1112 enic_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
1113 {
1114 struct enic_softc *softc;
1115 struct ifmedia_entry *next;
1116 uint32_t speed;
1117 uint64_t target_baudrate;
1118
1119 softc = iflib_get_softc(ctx);
1120
1121 ifmr->ifm_status = IFM_AVALID;
1122 ifmr->ifm_active = IFM_ETHER;
1123
1124 if (enic_link_is_up(softc) != 0) {
1125 ENIC_LOCK(softc);
1126 speed = vnic_dev_port_speed(&softc->vdev);
1127 ENIC_UNLOCK(softc);
1128 target_baudrate = 1000ull * speed;
1129 LIST_FOREACH(next, &(iflib_get_media(ctx)->ifm_list), ifm_list) {
1130 if (ifmedia_baudrate(next->ifm_media) == target_baudrate) {
1131 ifmr->ifm_active |= next->ifm_media;
1132 }
1133 }
1134
1135 ifmr->ifm_status |= IFM_ACTIVE;
1136 ifmr->ifm_active |= IFM_AUTO;
1137 } else
1138 ifmr->ifm_active |= IFM_NONE;
1139 }
1140
1141 static int
enic_media_change(if_ctx_t ctx)1142 enic_media_change(if_ctx_t ctx)
1143 {
1144 return (ENODEV);
1145 }
1146
1147 static int
enic_promisc_set(if_ctx_t ctx,int flags)1148 enic_promisc_set(if_ctx_t ctx, int flags)
1149 {
1150 if_t ifp;
1151 struct enic_softc *softc;
1152
1153 softc = iflib_get_softc(ctx);
1154 ifp = iflib_get_ifp(ctx);
1155
1156 if (if_getflags(ifp) & IFF_PROMISC) {
1157 softc->promisc = 1;
1158 } else {
1159 softc->promisc = 0;
1160 }
1161 if (if_getflags(ifp) & IFF_ALLMULTI) {
1162 softc->allmulti = 1;
1163 } else {
1164 softc->allmulti = 0;
1165 }
1166 enic_update_packet_filter(&softc->enic);
1167
1168 return (0);
1169 }
1170
1171 static uint64_t
enic_get_counter(if_ctx_t ctx,ift_counter cnt)1172 enic_get_counter(if_ctx_t ctx, ift_counter cnt) {
1173 if_t ifp = iflib_get_ifp(ctx);
1174
1175 if (cnt < IFCOUNTERS)
1176 return if_get_counter_default(ifp, cnt);
1177
1178 return (0);
1179 }
1180
1181 static void
enic_update_admin_status(if_ctx_t ctx)1182 enic_update_admin_status(if_ctx_t ctx)
1183 {
1184 struct enic_softc *softc;
1185 softc = iflib_get_softc(ctx);
1186
1187 enic_link_status(softc);
1188 }
1189
1190 static void
enic_txq_timer(if_ctx_t ctx,uint16_t qid)1191 enic_txq_timer(if_ctx_t ctx, uint16_t qid)
1192 {
1193
1194 struct enic_softc *softc;
1195 struct enic *enic;
1196 struct vnic_stats *stats;
1197 int ret;
1198
1199 softc = iflib_get_softc(ctx);
1200 enic = &softc->enic;
1201
1202 ENIC_LOCK(softc);
1203 ret = vnic_dev_stats_dump(enic->vdev, &stats);
1204 ENIC_UNLOCK(softc);
1205 if (ret) {
1206 dev_err(enic, "Error in getting stats\n");
1207 }
1208 }
1209
1210 static int
enic_link_is_up(struct enic_softc * softc)1211 enic_link_is_up(struct enic_softc *softc)
1212 {
1213 return (vnic_dev_link_status(&softc->vdev) == 1);
1214 }
1215
1216 static void
enic_link_status(struct enic_softc * softc)1217 enic_link_status(struct enic_softc *softc)
1218 {
1219 if_ctx_t ctx;
1220 uint64_t speed;
1221 int link;
1222
1223 ctx = softc->ctx;
1224 link = enic_link_is_up(softc);
1225 speed = IF_Gbps(10);
1226
1227 ENIC_LOCK(softc);
1228 speed = vnic_dev_port_speed(&softc->vdev);
1229 ENIC_UNLOCK(softc);
1230
1231 if (link != 0 && softc->link_active == 0) {
1232 softc->link_active = 1;
1233 iflib_link_state_change(ctx, LINK_STATE_UP, speed);
1234 } else if (link == 0 && softc->link_active != 0) {
1235 softc->link_active = 0;
1236 iflib_link_state_change(ctx, LINK_STATE_DOWN, speed);
1237 }
1238 }
1239
1240 static void
enic_set_lladdr(struct enic_softc * softc)1241 enic_set_lladdr(struct enic_softc *softc)
1242 {
1243 struct enic *enic;
1244 enic = &softc->enic;
1245
1246 ENIC_LOCK(softc);
1247 vnic_dev_add_addr(enic->vdev, softc->lladdr);
1248 ENIC_UNLOCK(softc);
1249 }
1250
1251
1252 static void
enic_setup_txq_sysctl(struct vnic_wq * wq,int i,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)1253 enic_setup_txq_sysctl(struct vnic_wq *wq, int i, struct sysctl_ctx_list *ctx,
1254 struct sysctl_oid_list *child)
1255 {
1256 struct sysctl_oid *txsnode;
1257 struct sysctl_oid_list *txslist;
1258 struct vnic_stats *stats;
1259
1260 stats = wq[i].vdev->stats;
1261
1262 txsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats",
1263 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
1264 txslist = SYSCTL_CHILDREN(txsnode);
1265
1266 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_frames_ok", CTLFLAG_RD,
1267 &stats->tx.tx_frames_ok, "TX Frames OK");
1268 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_frames_ok", CTLFLAG_RD,
1269 &stats->tx.tx_unicast_frames_ok, "TX unicast frames OK");
1270 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_frames_ok", CTLFLAG_RD,
1271 &stats->tx.tx_multicast_frames_ok, "TX multicast framse OK");
1272 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_frames_ok", CTLFLAG_RD,
1273 &stats->tx.tx_broadcast_frames_ok, "TX Broadcast frames OK");
1274 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_bytes_ok", CTLFLAG_RD,
1275 &stats->tx.tx_bytes_ok, "TX bytes OK ");
1276 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_bytes_ok", CTLFLAG_RD,
1277 &stats->tx.tx_unicast_bytes_ok, "TX unicast bytes OK");
1278 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_bytes_ok", CTLFLAG_RD,
1279 &stats->tx.tx_multicast_bytes_ok, "TX multicast bytes OK");
1280 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_bytes_ok", CTLFLAG_RD,
1281 &stats->tx.tx_broadcast_bytes_ok, "TX broadcast bytes OK");
1282 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_drops", CTLFLAG_RD,
1283 &stats->tx.tx_drops, "TX drops");
1284 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_errors", CTLFLAG_RD,
1285 &stats->tx.tx_errors, "TX errors");
1286 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_tso", CTLFLAG_RD,
1287 &stats->tx.tx_tso, "TX TSO");
1288 }
1289
1290 static void
enic_setup_rxq_sysctl(struct vnic_rq * rq,int i,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)1291 enic_setup_rxq_sysctl(struct vnic_rq *rq, int i, struct sysctl_ctx_list *ctx,
1292 struct sysctl_oid_list *child)
1293 {
1294 struct sysctl_oid *rxsnode;
1295 struct sysctl_oid_list *rxslist;
1296 struct vnic_stats *stats;
1297
1298 stats = rq[i].vdev->stats;
1299
1300 rxsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats",
1301 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
1302 rxslist = SYSCTL_CHILDREN(rxsnode);
1303
1304 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_ok", CTLFLAG_RD,
1305 &stats->rx.rx_frames_ok, "RX Frames OK");
1306 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_total", CTLFLAG_RD,
1307 &stats->rx.rx_frames_total, "RX frames total");
1308 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_frames_ok", CTLFLAG_RD,
1309 &stats->rx.rx_unicast_frames_ok, "RX unicast frames ok");
1310 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_frames_ok", CTLFLAG_RD,
1311 &stats->rx.rx_multicast_frames_ok, "RX multicast Frames ok");
1312 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_frames_ok", CTLFLAG_RD,
1313 &stats->rx.rx_broadcast_frames_ok, "RX broadcast frames ok");
1314 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_bytes_ok", CTLFLAG_RD,
1315 &stats->rx.rx_bytes_ok, "RX bytes ok");
1316 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_bytes_ok", CTLFLAG_RD,
1317 &stats->rx.rx_unicast_bytes_ok, "RX unicast bytes ok");
1318 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_bytes_ok", CTLFLAG_RD,
1319 &stats->rx.rx_multicast_bytes_ok, "RX multicast bytes ok");
1320 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_bytes_ok", CTLFLAG_RD,
1321 &stats->rx.rx_broadcast_bytes_ok, "RX broadcast bytes ok");
1322 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_drop", CTLFLAG_RD,
1323 &stats->rx.rx_drop, "RX drop");
1324 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_errors", CTLFLAG_RD,
1325 &stats->rx.rx_errors, "RX errors");
1326 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_rss", CTLFLAG_RD,
1327 &stats->rx.rx_rss, "RX rss");
1328 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_crc_errors", CTLFLAG_RD,
1329 &stats->rx.rx_crc_errors, "RX crc errors");
1330 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_64", CTLFLAG_RD,
1331 &stats->rx.rx_frames_64, "RX frames 64");
1332 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_127", CTLFLAG_RD,
1333 &stats->rx.rx_frames_127, "RX frames 127");
1334 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_255", CTLFLAG_RD,
1335 &stats->rx.rx_frames_255, "RX frames 255");
1336 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_511", CTLFLAG_RD,
1337 &stats->rx.rx_frames_511, "RX frames 511");
1338 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1023", CTLFLAG_RD,
1339 &stats->rx.rx_frames_1023, "RX frames 1023");
1340 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1518", CTLFLAG_RD,
1341 &stats->rx.rx_frames_1518, "RX frames 1518");
1342 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_to_max", CTLFLAG_RD,
1343 &stats->rx.rx_frames_to_max, "RX frames to max");
1344 }
1345
1346 static void
enic_setup_queue_sysctl(struct enic_softc * softc,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)1347 enic_setup_queue_sysctl(struct enic_softc *softc, struct sysctl_ctx_list *ctx,
1348 struct sysctl_oid_list *child)
1349 {
1350 enic_setup_txq_sysctl(softc->enic.wq, 0, ctx, child);
1351 enic_setup_rxq_sysctl(softc->enic.rq, 0, ctx, child);
1352 }
1353
1354 static void
enic_setup_sysctl(struct enic_softc * softc)1355 enic_setup_sysctl(struct enic_softc *softc)
1356 {
1357 device_t dev;
1358 struct sysctl_ctx_list *ctx;
1359 struct sysctl_oid *tree;
1360 struct sysctl_oid_list *child;
1361
1362 dev = softc->dev;
1363 ctx = device_get_sysctl_ctx(dev);
1364 tree = device_get_sysctl_tree(dev);
1365 child = SYSCTL_CHILDREN(tree);
1366
1367 enic_setup_queue_sysctl(softc, ctx, child);
1368 }
1369
1370 static void
enic_enable_intr(struct enic_softc * softc,int irq)1371 enic_enable_intr(struct enic_softc *softc, int irq)
1372 {
1373 struct enic *enic = &softc->enic;
1374
1375 vnic_intr_unmask(&enic->intr[irq]);
1376 vnic_intr_return_all_credits(&enic->intr[irq]);
1377 }
1378
1379 static void
enic_disable_intr(struct enic_softc * softc,int irq)1380 enic_disable_intr(struct enic_softc *softc, int irq)
1381 {
1382 struct enic *enic = &softc->enic;
1383
1384 vnic_intr_mask(&enic->intr[irq]);
1385 vnic_intr_masked(&enic->intr[irq]); /* flush write */
1386 }
1387
1388 static int
enic_tx_queue_intr_enable(if_ctx_t ctx,uint16_t qid)1389 enic_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
1390 {
1391 struct enic_softc *softc;
1392 if_softc_ctx_t scctx;
1393
1394 softc = iflib_get_softc(ctx);
1395 scctx = softc->scctx;
1396
1397 enic_enable_intr(softc, qid + scctx->isc_nrxqsets);
1398
1399 return 0;
1400 }
1401
1402 static int
enic_rx_queue_intr_enable(if_ctx_t ctx,uint16_t qid)1403 enic_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
1404 {
1405 struct enic_softc *softc;
1406
1407 softc = iflib_get_softc(ctx);
1408 enic_enable_intr(softc, qid);
1409
1410 return 0;
1411 }
1412
1413 static void
enic_intr_enable_all(if_ctx_t ctx)1414 enic_intr_enable_all(if_ctx_t ctx)
1415 {
1416 struct enic_softc *softc;
1417 if_softc_ctx_t scctx;
1418 int i;
1419
1420 softc = iflib_get_softc(ctx);
1421 scctx = softc->scctx;
1422
1423 for (i = 0; i < scctx->isc_vectors; i++) {
1424 enic_enable_intr(softc, i);
1425 }
1426 }
1427
1428 static void
enic_intr_disable_all(if_ctx_t ctx)1429 enic_intr_disable_all(if_ctx_t ctx)
1430 {
1431 struct enic_softc *softc;
1432 if_softc_ctx_t scctx;
1433 int i;
1434
1435 softc = iflib_get_softc(ctx);
1436 scctx = softc->scctx;
1437 /*
1438 * iflib may invoke this routine before enic_attach_post() has run,
1439 * which is before the top level shared data area is initialized and
1440 * the device made aware of it.
1441 */
1442
1443 for (i = 0; i < scctx->isc_vectors; i++) {
1444 enic_disable_intr(softc, i);
1445 }
1446 }
1447
1448 static int
enic_dev_open(struct enic * enic)1449 enic_dev_open(struct enic *enic)
1450 {
1451 int err;
1452 int flags = CMD_OPENF_IG_DESCCACHE;
1453
1454 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1455 vnic_dev_open_done, flags);
1456 if (err)
1457 dev_err(enic_get_dev(enic),
1458 "vNIC device open failed, err %d\n", err);
1459
1460 return err;
1461 }
1462
1463 static int
enic_dev_init(struct enic * enic)1464 enic_dev_init(struct enic *enic)
1465 {
1466 int err;
1467
1468 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1469
1470 /*
1471 * Get vNIC configuration
1472 */
1473 err = enic_get_vnic_config(enic);
1474 if (err) {
1475 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1476 return err;
1477 }
1478
1479 /* Get available resource counts */
1480 enic_get_res_counts(enic);
1481
1482 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1483 enic->intr_queues = malloc(sizeof(*enic->intr_queues) *
1484 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO);
1485
1486 vnic_dev_set_reset_flag(enic->vdev, 0);
1487 enic->max_flow_counter = -1;
1488
1489 /* set up link status checking */
1490 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1491
1492 enic->overlay_offload = false;
1493 if (enic->disable_overlay && enic->vxlan) {
1494 /*
1495 * Explicitly disable overlay offload as the setting is
1496 * sticky, and resetting vNIC does not disable it.
1497 */
1498 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1499 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_DISABLE)) {
1500 dev_err(enic, "failed to disable overlay offload\n");
1501 } else {
1502 dev_info(enic, "Overlay offload is disabled\n");
1503 }
1504 }
1505 if (!enic->disable_overlay && enic->vxlan &&
1506 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1507 vnic_dev_overlay_offload_ctrl(enic->vdev,
1508 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_ENABLE) == 0) {
1509 enic->overlay_offload = true;
1510 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1511 dev_info(enic, "Overlay offload is enabled\n");
1512 /*
1513 * Reset the vxlan port to the default, as the NIC firmware
1514 * does not reset it automatically and keeps the old setting.
1515 */
1516 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1517 OVERLAY_CFG_VXLAN_PORT_UPDATE, ENIC_DEFAULT_VXLAN_PORT)) {
1518 dev_err(enic, "failed to update vxlan port\n");
1519 return (EINVAL);
1520 }
1521 }
1522 return 0;
1523 }
1524
1525 static void *
enic_alloc_consistent(void * priv,size_t size,bus_addr_t * dma_handle,struct iflib_dma_info * res,u8 * name)1526 enic_alloc_consistent(void *priv, size_t size, bus_addr_t * dma_handle,
1527 struct iflib_dma_info *res, u8 * name)
1528 {
1529 void *vaddr;
1530 *dma_handle = 0;
1531 struct enic *enic = (struct enic *)priv;
1532 int rz;
1533
1534 rz = iflib_dma_alloc(enic->softc->ctx, size, res, BUS_DMA_NOWAIT);
1535 if (rz) {
1536 pr_err("%s : Failed to allocate memory requested for %s\n",
1537 __func__, name);
1538 return NULL;
1539 }
1540
1541 vaddr = res->idi_vaddr;
1542 *dma_handle = res->idi_paddr;
1543
1544 return vaddr;
1545 }
1546
1547 static void
enic_free_consistent(void * priv,size_t size,void * vaddr,bus_addr_t dma_handle,struct iflib_dma_info * res)1548 enic_free_consistent(void *priv, size_t size, void *vaddr,
1549 bus_addr_t dma_handle, struct iflib_dma_info *res)
1550 {
1551 iflib_dma_free(res);
1552 }
1553
1554 static int
enic_pci_mapping(struct enic_softc * softc)1555 enic_pci_mapping(struct enic_softc *softc)
1556 {
1557 int rc;
1558
1559 rc = enic_map_bar(softc, &softc->mem, 0, true);
1560 if (rc)
1561 return rc;
1562
1563 rc = enic_map_bar(softc, &softc->io, 2, false);
1564
1565 return rc;
1566 }
1567
1568 static void
enic_pci_mapping_free(struct enic_softc * softc)1569 enic_pci_mapping_free(struct enic_softc *softc)
1570 {
1571 if (softc->mem.res != NULL)
1572 bus_release_resource(softc->dev, SYS_RES_MEMORY,
1573 softc->mem.rid, softc->mem.res);
1574 softc->mem.res = NULL;
1575
1576 if (softc->io.res != NULL)
1577 bus_release_resource(softc->dev, SYS_RES_MEMORY,
1578 softc->io.rid, softc->io.res);
1579 softc->io.res = NULL;
1580 }
1581
1582 static int
enic_dev_wait(struct vnic_dev * vdev,int (* start)(struct vnic_dev *,int),int (* finished)(struct vnic_dev *,int *),int arg)1583 enic_dev_wait(struct vnic_dev *vdev, int (*start) (struct vnic_dev *, int),
1584 int (*finished) (struct vnic_dev *, int *), int arg)
1585 {
1586 int done;
1587 int err;
1588 int i;
1589
1590 err = start(vdev, arg);
1591 if (err)
1592 return err;
1593
1594 /* Wait for func to complete...2 seconds max */
1595 for (i = 0; i < 2000; i++) {
1596 err = finished(vdev, &done);
1597 if (err)
1598 return err;
1599 if (done)
1600 return 0;
1601 usleep(1000);
1602 }
1603 return (ETIMEDOUT);
1604 }
1605
1606 static int
enic_map_bar(struct enic_softc * softc,struct enic_bar_info * bar,int bar_num,bool shareable)1607 enic_map_bar(struct enic_softc *softc, struct enic_bar_info *bar, int bar_num,
1608 bool shareable)
1609 {
1610 uint32_t flag;
1611
1612 if (bar->res != NULL) {
1613 device_printf(softc->dev, "Bar %d already mapped\n", bar_num);
1614 return (EDOOFUS);
1615 }
1616
1617 bar->rid = PCIR_BAR(bar_num);
1618 flag = RF_ACTIVE;
1619 if (shareable)
1620 flag |= RF_SHAREABLE;
1621
1622 if ((bar->res = bus_alloc_resource_any(softc->dev,
1623 SYS_RES_MEMORY, &bar->rid, flag)) == NULL) {
1624 device_printf(softc->dev,
1625 "PCI BAR%d mapping failure\n", bar_num);
1626 return (ENXIO);
1627 }
1628 bar->tag = rman_get_bustag(bar->res);
1629 bar->handle = rman_get_bushandle(bar->res);
1630 bar->size = rman_get_size(bar->res);
1631
1632 return 0;
1633 }
1634
1635 void
enic_init_vnic_resources(struct enic * enic)1636 enic_init_vnic_resources(struct enic *enic)
1637 {
1638 unsigned int error_interrupt_enable = 1;
1639 unsigned int error_interrupt_offset = 0;
1640 unsigned int rxq_interrupt_enable = 0;
1641 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
1642 unsigned int txq_interrupt_enable = 0;
1643 unsigned int txq_interrupt_offset;
1644 unsigned int index = 0;
1645 unsigned int cq_idx;
1646 if_softc_ctx_t scctx;
1647
1648 scctx = enic->softc->scctx;
1649
1650 rxq_interrupt_enable = 1;
1651 txq_interrupt_enable = 0;
1652
1653 rxq_interrupt_offset = 0;
1654 txq_interrupt_offset = scctx->isc_nrxqsets;
1655
1656 for (index = 0; index < enic->intr_count; index++) {
1657 vnic_intr_alloc(enic->vdev, &enic->intr[index], index);
1658 }
1659
1660 for (index = 0; index < scctx->isc_nrxqsets; index++) {
1661 cq_idx = enic_cq_rq(enic, index);
1662
1663 vnic_rq_clean(&enic->rq[index]);
1664 vnic_rq_init(&enic->rq[index], cq_idx, error_interrupt_enable,
1665 error_interrupt_offset);
1666
1667 vnic_cq_clean(&enic->cq[cq_idx]);
1668 vnic_cq_init(&enic->cq[cq_idx],
1669 0 /* flow_control_enable */ ,
1670 1 /* color_enable */ ,
1671 0 /* cq_head */ ,
1672 0 /* cq_tail */ ,
1673 1 /* cq_tail_color */ ,
1674 rxq_interrupt_enable,
1675 1 /* cq_entry_enable */ ,
1676 0 /* cq_message_enable */ ,
1677 rxq_interrupt_offset,
1678 0 /* cq_message_addr */ );
1679 if (rxq_interrupt_enable)
1680 rxq_interrupt_offset++;
1681 }
1682
1683 for (index = 0; index < scctx->isc_ntxqsets; index++) {
1684 cq_idx = enic_cq_wq(enic, index);
1685 vnic_wq_clean(&enic->wq[index]);
1686 vnic_wq_init(&enic->wq[index], cq_idx, error_interrupt_enable,
1687 error_interrupt_offset);
1688 /* Compute unsupported ol flags for enic_prep_pkts() */
1689 enic->wq[index].tx_offload_notsup_mask = 0;
1690
1691 vnic_cq_clean(&enic->cq[cq_idx]);
1692 vnic_cq_init(&enic->cq[cq_idx],
1693 0 /* flow_control_enable */ ,
1694 1 /* color_enable */ ,
1695 0 /* cq_head */ ,
1696 0 /* cq_tail */ ,
1697 1 /* cq_tail_color */ ,
1698 txq_interrupt_enable,
1699 1,
1700 0,
1701 txq_interrupt_offset,
1702 0 /* (u64)enic->wq[index].cqmsg_rz->iova */ );
1703
1704 }
1705
1706 for (index = 0; index < enic->intr_count; index++) {
1707 vnic_intr_init(&enic->intr[index], 125,
1708 enic->config.intr_timer_type, /* mask_on_assertion */ 1);
1709 }
1710 }
1711
1712 static void
enic_update_packet_filter(struct enic * enic)1713 enic_update_packet_filter(struct enic *enic)
1714 {
1715 struct enic_softc *softc = enic->softc;
1716
1717 ENIC_LOCK(softc);
1718 vnic_dev_packet_filter(enic->vdev,
1719 softc->directed,
1720 softc->multicast,
1721 softc->broadcast,
1722 softc->promisc,
1723 softc->allmulti);
1724 ENIC_UNLOCK(softc);
1725 }
1726
1727 static bool
enic_if_needs_restart(if_ctx_t ctx,enum iflib_restart_event event)1728 enic_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event)
1729 {
1730 switch (event) {
1731 case IFLIB_RESTART_VLAN_CONFIG:
1732 default:
1733 return (false);
1734 }
1735 }
1736
1737 int
enic_setup_finish(struct enic * enic)1738 enic_setup_finish(struct enic *enic)
1739 {
1740 struct enic_softc *softc = enic->softc;
1741
1742 /* Default conf */
1743 softc->directed = 1;
1744 softc->multicast = 0;
1745 softc->broadcast = 1;
1746 softc->promisc = 0;
1747 softc->allmulti = 1;
1748 enic_update_packet_filter(enic);
1749
1750 return 0;
1751 }
1752