1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
4 */
5
6 #include "opt_rss.h"
7
8 #include <sys/param.h>
9 #include <sys/systm.h>
10 #include <sys/kernel.h>
11 #include <sys/endian.h>
12 #include <sys/sockio.h>
13 #include <sys/mbuf.h>
14 #include <sys/malloc.h>
15 #include <sys/module.h>
16 #include <sys/socket.h>
17 #include <sys/sysctl.h>
18 #include <sys/smp.h>
19 #include <vm/vm.h>
20 #include <vm/pmap.h>
21
22 #include <net/ethernet.h>
23 #include <net/if.h>
24 #include <net/if_var.h>
25 #include <net/if_arp.h>
26 #include <net/if_dl.h>
27 #include <net/if_types.h>
28 #include <net/if_media.h>
29 #include <net/if_vlan_var.h>
30 #include <net/iflib.h>
31 #ifdef RSS
32 #include <net/rss_config.h>
33 #endif
34
35 #include <netinet/in_systm.h>
36 #include <netinet/in.h>
37 #include <netinet/ip.h>
38 #include <netinet/ip6.h>
39 #include <netinet6/ip6_var.h>
40 #include <netinet/udp.h>
41 #include <netinet/tcp.h>
42
43 #include <machine/bus.h>
44 #include <machine/resource.h>
45 #include <sys/bus.h>
46 #include <sys/rman.h>
47
48 #include <dev/pci/pcireg.h>
49 #include <dev/pci/pcivar.h>
50
51 #include "ifdi_if.h"
52 #include "enic.h"
53
54 #include "opt_inet.h"
55 #include "opt_inet6.h"
56
57 static SYSCTL_NODE(_hw, OID_AUTO, enic, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
58 "ENIC");
59
60 static const pci_vendor_info_t enic_vendor_info_array[] =
61 {
62 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET,
63 DRV_DESCRIPTION),
64 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET_VF,
65 DRV_DESCRIPTION " VF"),
66 /* required last entry */
67
68 PVID_END
69 };
70
71 static void *enic_register(device_t);
72 static int enic_attach_pre(if_ctx_t);
73 static int enic_msix_intr_assign(if_ctx_t, int);
74
75 static int enic_attach_post(if_ctx_t);
76 static int enic_detach(if_ctx_t);
77
78 static int enic_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
79 static int enic_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
80 static void enic_queues_free(if_ctx_t);
81 static int enic_rxq_intr(void *);
82 static int enic_event_intr(void *);
83 static int enic_err_intr(void *);
84 static void enic_stop(if_ctx_t);
85 static void enic_init(if_ctx_t);
86 static void enic_multi_set(if_ctx_t);
87 static int enic_mtu_set(if_ctx_t, uint32_t);
88 static void enic_media_status(if_ctx_t, struct ifmediareq *);
89 static int enic_media_change(if_ctx_t);
90 static int enic_promisc_set(if_ctx_t, int);
91 static uint64_t enic_get_counter(if_ctx_t, ift_counter);
92 static void enic_update_admin_status(if_ctx_t);
93 static void enic_txq_timer(if_ctx_t, uint16_t);
94 static int enic_link_is_up(struct enic_softc *);
95 static void enic_link_status(struct enic_softc *);
96 static void enic_set_lladdr(struct enic_softc *);
97 static void enic_setup_txq_sysctl(struct vnic_wq *, int, struct sysctl_ctx_list *,
98 struct sysctl_oid_list *);
99 static void enic_setup_rxq_sysctl(struct vnic_rq *, int, struct sysctl_ctx_list *,
100 struct sysctl_oid_list *);
101 static void enic_setup_sysctl(struct enic_softc *);
102 static int enic_tx_queue_intr_enable(if_ctx_t, uint16_t);
103 static int enic_rx_queue_intr_enable(if_ctx_t, uint16_t);
104 static void enic_enable_intr(struct enic_softc *, int);
105 static void enic_disable_intr(struct enic_softc *, int);
106 static void enic_intr_enable_all(if_ctx_t);
107 static void enic_intr_disable_all(if_ctx_t);
108 static int enic_dev_open(struct enic *);
109 static int enic_dev_init(struct enic *);
110 static void *enic_alloc_consistent(void *, size_t, bus_addr_t *,
111 struct iflib_dma_info *, u8 *);
112 static void enic_free_consistent(void *, size_t, void *, bus_addr_t,
113 struct iflib_dma_info *);
114 static int enic_pci_mapping(struct enic_softc *);
115 static void enic_pci_mapping_free(struct enic_softc *);
116 static int enic_dev_wait(struct vnic_dev *, int (*) (struct vnic_dev *, int),
117 int (*) (struct vnic_dev *, int *), int arg);
118 static int enic_map_bar(struct enic_softc *, struct enic_bar_info *, int, bool);
119 static void enic_update_packet_filter(struct enic *enic);
120 static bool enic_if_needs_restart(if_ctx_t, enum iflib_restart_event);
121
122 typedef enum {
123 ENIC_BARRIER_RD,
124 ENIC_BARRIER_WR,
125 ENIC_BARRIER_RDWR,
126 } enic_barrier_t;
127
128 static device_method_t enic_methods[] = {
129 /* Device interface */
130 DEVMETHOD(device_register, enic_register),
131 DEVMETHOD(device_probe, iflib_device_probe),
132 DEVMETHOD(device_attach, iflib_device_attach),
133 DEVMETHOD(device_detach, iflib_device_detach),
134 DEVMETHOD(device_shutdown, iflib_device_shutdown),
135 DEVMETHOD(device_suspend, iflib_device_suspend),
136 DEVMETHOD(device_resume, iflib_device_resume),
137 DEVMETHOD_END
138 };
139
140 static driver_t enic_driver = {
141 "enic", enic_methods, sizeof(struct enic_softc)
142 };
143
144 DRIVER_MODULE(enic, pci, enic_driver, 0, 0);
145 IFLIB_PNP_INFO(pci, enic, enic_vendor_info_array);
146 MODULE_VERSION(enic, 2);
147
148 MODULE_DEPEND(enic, pci, 1, 1, 1);
149 MODULE_DEPEND(enic, ether, 1, 1, 1);
150 MODULE_DEPEND(enic, iflib, 1, 1, 1);
151
152 static device_method_t enic_iflib_methods[] = {
153 DEVMETHOD(ifdi_tx_queues_alloc, enic_tx_queues_alloc),
154 DEVMETHOD(ifdi_rx_queues_alloc, enic_rx_queues_alloc),
155 DEVMETHOD(ifdi_queues_free, enic_queues_free),
156
157 DEVMETHOD(ifdi_attach_pre, enic_attach_pre),
158 DEVMETHOD(ifdi_attach_post, enic_attach_post),
159 DEVMETHOD(ifdi_detach, enic_detach),
160
161 DEVMETHOD(ifdi_init, enic_init),
162 DEVMETHOD(ifdi_stop, enic_stop),
163 DEVMETHOD(ifdi_multi_set, enic_multi_set),
164 DEVMETHOD(ifdi_mtu_set, enic_mtu_set),
165 DEVMETHOD(ifdi_media_status, enic_media_status),
166 DEVMETHOD(ifdi_media_change, enic_media_change),
167 DEVMETHOD(ifdi_promisc_set, enic_promisc_set),
168 DEVMETHOD(ifdi_get_counter, enic_get_counter),
169 DEVMETHOD(ifdi_update_admin_status, enic_update_admin_status),
170 DEVMETHOD(ifdi_timer, enic_txq_timer),
171
172 DEVMETHOD(ifdi_tx_queue_intr_enable, enic_tx_queue_intr_enable),
173 DEVMETHOD(ifdi_rx_queue_intr_enable, enic_rx_queue_intr_enable),
174 DEVMETHOD(ifdi_intr_enable, enic_intr_enable_all),
175 DEVMETHOD(ifdi_intr_disable, enic_intr_disable_all),
176 DEVMETHOD(ifdi_msix_intr_assign, enic_msix_intr_assign),
177
178 DEVMETHOD(ifdi_needs_restart, enic_if_needs_restart),
179
180 DEVMETHOD_END
181 };
182
183 static driver_t enic_iflib_driver = {
184 "enic", enic_iflib_methods, sizeof(struct enic_softc)
185 };
186
187 extern struct if_txrx enic_txrx;
188
189 static struct if_shared_ctx enic_sctx_init = {
190 .isc_magic = IFLIB_MAGIC,
191 .isc_q_align = 512,
192
193 .isc_tx_maxsize = ENIC_TX_MAX_PKT_SIZE,
194 .isc_tx_maxsegsize = PAGE_SIZE,
195
196 /*
197 * These values are used to configure the busdma tag used for receive
198 * descriptors. Each receive descriptor only points to one buffer.
199 */
200 .isc_rx_maxsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, /* One buf per
201 * descriptor */
202 .isc_rx_nsegments = 1, /* One mapping per descriptor */
203 .isc_rx_maxsegsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE,
204 .isc_admin_intrcnt = 2,
205 .isc_vendor_info = enic_vendor_info_array,
206 .isc_driver_version = "1",
207 .isc_driver = &enic_iflib_driver,
208 .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ | IFLIB_SKIP_MSIX,
209
210 /*
211 * Number of receive queues per receive queue set, with associated
212 * descriptor settings for each.
213 */
214
215 .isc_nrxqs = 2,
216 .isc_nfl = 1, /* one free list for each receive command
217 * queue */
218 .isc_nrxd_min = {16, 16},
219 .isc_nrxd_max = {2048, 2048},
220 .isc_nrxd_default = {64, 64},
221
222 /*
223 * Number of transmit queues per transmit queue set, with associated
224 * descriptor settings for each.
225 */
226 .isc_ntxqs = 2,
227 .isc_ntxd_min = {16, 16},
228 .isc_ntxd_max = {2048, 2048},
229 .isc_ntxd_default = {64, 64},
230 };
231
232 static void *
enic_register(device_t dev)233 enic_register(device_t dev)
234 {
235 return (&enic_sctx_init);
236 }
237
238 static int
enic_allocate_msix(struct enic_softc * softc)239 enic_allocate_msix(struct enic_softc *softc) {
240 if_ctx_t ctx;
241 if_softc_ctx_t scctx;
242 if_shared_ctx_t sctx;
243 device_t dev;
244 cpuset_t cpus;
245 int queues, vectors, requested;
246 int err = 0;
247
248 dev = softc->dev;
249 ctx = softc->ctx;
250 scctx = softc->scctx;
251 sctx = iflib_get_sctx(ctx);
252
253 if (bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus) != 0) {
254 device_printf(dev, "Unable to fetch CPU list\n");
255 CPU_COPY(&all_cpus, &cpus);
256 }
257
258
259 queues = CPU_COUNT(&cpus);
260 queues = imin(queues, scctx->isc_nrxqsets);
261 queues = imin(queues, scctx->isc_ntxqsets);
262 requested = queues * 2 + sctx->isc_admin_intrcnt;
263 scctx->isc_nrxqsets = queues;
264 scctx->isc_ntxqsets = queues;
265
266 vectors = requested;
267 if ((err = pci_alloc_msix(dev, &vectors)) != 0) {
268 device_printf(dev,
269 "failed to allocate %d MSI-X vectors, err: %d\n", requested,
270 err);
271 err = 1;
272 goto enic_allocate_msix_out;
273 } else {
274 if (vectors != requested) {
275 device_printf(dev,
276 "Unable to allocate sufficient MSI-X vectors "
277 "(got %d, need %d)\n", requested, vectors);
278 pci_release_msi(dev);
279 err = 1;
280 goto enic_allocate_msix_out;
281 }
282 }
283
284 device_printf(dev, "Using MSI-X interrupts with %d vectors\n",
285 vectors);
286
287 scctx->isc_intr = IFLIB_INTR_MSIX;
288 scctx->isc_vectors = vectors;
289
290 enic_allocate_msix_out:
291 return (err);
292
293 }
294
295 static struct enic_intr_mod_range mod_range[ENIC_MAX_LINK_SPEEDS] = {
296 {0, 0}, /* 0 - 4 Gbps */
297 {0, 3}, /* 4 - 10 Gbps */
298 {3, 6}, /* 10 - 40 Gbps */
299 };
300
enic_set_rx_coal_setting(struct enic * enic)301 static void enic_set_rx_coal_setting(struct enic *enic)
302 {
303 unsigned int speed;
304 int index = -1;
305 struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting;
306
307 /* 1. Read the link speed from fw
308 * 2. Pick the default range for the speed
309 * 3. Update it in enic->rx_coalesce_setting
310 */
311 speed = vnic_dev_port_speed(enic->vdev);
312 if (ENIC_LINK_SPEED_10G < speed)
313 index = ENIC_LINK_40G_INDEX;
314 else if (ENIC_LINK_SPEED_4G < speed)
315 index = ENIC_LINK_10G_INDEX;
316 else
317 index = ENIC_LINK_4G_INDEX;
318
319 rx_coal->small_pkt_range_start = mod_range[index].small_pkt_range_start;
320 rx_coal->large_pkt_range_start = mod_range[index].large_pkt_range_start;
321 rx_coal->range_end = ENIC_RX_COALESCE_RANGE_END;
322
323 /* Start with the value provided by UCSM */
324 for (index = 0; index < enic->rq_count; index++)
325 enic->cq[index].cur_rx_coal_timeval =
326 enic->config.intr_timer_usec;
327
328 rx_coal->use_adaptive_rx_coalesce = 1;
329 }
330
331 static int
enic_attach_pre(if_ctx_t ctx)332 enic_attach_pre(if_ctx_t ctx)
333 {
334 if_softc_ctx_t scctx;
335 struct enic_softc *softc;
336 struct vnic_dev *vdev;
337 struct enic *enic;
338 device_t dev;
339
340 int err = -1;
341 int rc = 0;
342 int i;
343 u64 a0 = 0, a1 = 0;
344 int wait = 1000;
345 struct vnic_stats *stats;
346 int ret;
347
348 dev = iflib_get_dev(ctx);
349 softc = iflib_get_softc(ctx);
350 softc->dev = dev;
351 softc->ctx = ctx;
352 softc->sctx = iflib_get_sctx(ctx);
353 softc->scctx = iflib_get_softc_ctx(ctx);
354 softc->ifp = iflib_get_ifp(ctx);
355 softc->media = iflib_get_media(ctx);
356 softc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN *
357 ENIC_MAX_MULTICAST_ADDRESSES, M_DEVBUF,
358 M_NOWAIT | M_ZERO);
359 if (softc->mta == NULL)
360 return (ENOMEM);
361 scctx = softc->scctx;
362
363 mtx_init(&softc->enic_lock, "ENIC Lock", NULL, MTX_DEF);
364
365 pci_enable_busmaster(softc->dev);
366 if (enic_pci_mapping(softc))
367 return (ENXIO);
368
369 enic = &softc->enic;
370 enic->softc = softc;
371 vdev = &softc->vdev;
372 vdev->softc = softc;
373 enic->vdev = vdev;
374 vdev->priv = enic;
375
376 ENIC_LOCK(softc);
377 vnic_dev_register(vdev, &softc->mem, 1);
378 enic->vdev = vdev;
379 vnic_dev_cmd_init(enic->vdev);
380
381 vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0);
382
383 vnic_dev_cmd(vdev, CMD_INIT_v1, &a0, &a1, wait);
384 vnic_dev_cmd(vdev, CMD_GET_MAC_ADDR, &a0, &a1, wait);
385
386 bcopy((u_int8_t *) & a0, softc->mac_addr, ETHER_ADDR_LEN);
387 iflib_set_mac(ctx, softc->mac_addr);
388
389 vnic_register_cbacks(enic->vdev, enic_alloc_consistent,
390 enic_free_consistent);
391
392 /*
393 * Allocate the consistent memory for stats and counters upfront so
394 * both primary and secondary processes can access them.
395 */
396 ENIC_UNLOCK(softc);
397 err = vnic_dev_alloc_stats_mem(enic->vdev);
398 ENIC_LOCK(softc);
399 if (err) {
400 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
401 goto err_out_unregister;
402 }
403 vnic_dev_stats_clear(enic->vdev);
404 ret = vnic_dev_stats_dump(enic->vdev, &stats);
405 if (ret) {
406 dev_err(enic, "Error in getting stats\n");
407 goto err_out_unregister;
408 }
409 err = vnic_dev_alloc_counter_mem(enic->vdev);
410 if (err) {
411 dev_err(enic, "Failed to allocate counter memory, aborting\n");
412 goto err_out_unregister;
413 }
414
415 /* Issue device open to get device in known state */
416 err = enic_dev_open(enic);
417 if (err) {
418 dev_err(enic, "vNIC dev open failed, aborting\n");
419 goto err_out_unregister;
420 }
421
422 /* Set ingress vlan rewrite mode before vnic initialization */
423 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN;
424 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN;
425 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
426 enic->ig_vlan_rewrite_mode);
427 if (err) {
428 dev_err(enic,
429 "Failed to set ingress vlan rewrite mode, aborting.\n");
430 goto err_out_dev_close;
431 }
432
433 /*
434 * Issue device init to initialize the vnic-to-switch link. We'll
435 * start with carrier off and wait for link UP notification later to
436 * turn on carrier. We don't need to wait here for the
437 * vnic-to-switch link initialization to complete; link UP
438 * notification is the indication that the process is complete.
439 */
440
441 err = vnic_dev_init(enic->vdev, 0);
442 if (err) {
443 dev_err(enic, "vNIC dev init failed, aborting\n");
444 goto err_out_dev_close;
445 }
446
447 err = enic_dev_init(enic);
448 if (err) {
449 dev_err(enic, "Device initialization failed, aborting\n");
450 goto err_out_dev_close;
451 }
452 ENIC_UNLOCK(softc);
453
454 enic->port_mtu = vnic_dev_mtu(enic->vdev);
455
456 softc->scctx = iflib_get_softc_ctx(ctx);
457 scctx = softc->scctx;
458 scctx->isc_txrx = &enic_txrx;
459 scctx->isc_capabilities = scctx->isc_capenable = \
460 IFCAP_HWCSUM;
461 scctx->isc_tx_csum_flags = 0;
462 if_setmtu(softc->ifp, enic->config.mtu);
463 scctx->isc_max_frame_size = enic->config.mtu + ETHER_HDR_LEN + \
464 ETHER_CRC_LEN;
465 scctx->isc_nrxqsets_max = enic->conf_rq_count;
466 scctx->isc_ntxqsets_max = enic->conf_wq_count;
467 scctx->isc_nrxqsets = enic->conf_rq_count;
468 scctx->isc_ntxqsets = enic->conf_wq_count;
469 for (i = 0; i < enic->conf_wq_count; i++) {
470 scctx->isc_ntxd[i] = enic->config.wq_desc_count;
471 scctx->isc_txqsizes[i] = sizeof(struct cq_enet_wq_desc)
472 * scctx->isc_ntxd[i];
473 scctx->isc_ntxd[i + enic->conf_wq_count] =
474 enic->config.wq_desc_count;
475 scctx->isc_txqsizes[i + enic->conf_wq_count] =
476 sizeof(struct cq_desc) * scctx->isc_ntxd[i +
477 enic->conf_wq_count];
478 }
479 for (i = 0; i < enic->conf_rq_count; i++) {
480 scctx->isc_nrxd[i] = enic->config.rq_desc_count;
481 scctx->isc_rxqsizes[i] = sizeof(struct cq_enet_rq_desc) *
482 scctx->isc_nrxd[i];
483 scctx->isc_nrxd[i + enic->conf_rq_count] =
484 enic->config.rq_desc_count;
485 scctx->isc_rxqsizes[i + enic->conf_rq_count] = sizeof(struct
486 cq_desc) * scctx->isc_nrxd[i + enic->conf_rq_count];
487 }
488 scctx->isc_tx_nsegments = 31;
489
490 scctx->isc_msix_bar = -1;
491
492 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
493 ifmedia_add(softc->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
494 ifmedia_add(softc->media, IFM_ETHER | IFM_10_FL, 0, NULL);
495
496 err = enic_allocate_msix(softc);
497 if (err) {
498 dev_err(enic, "Failed to allocate MSIX, aborting\n");
499 goto err_out_dev_close;
500 }
501
502 return (rc);
503
504 err_out_dev_close:
505 vnic_dev_close(enic->vdev);
506 vnic_dev_deinit_devcmd2(enic->vdev);
507 err_out_unregister:
508 free(softc->vdev.devcmd, M_DEVBUF);
509 free(softc->enic.intr_queues, M_DEVBUF);
510 free(softc->enic.cq, M_DEVBUF);
511 free(softc->mta, M_DEVBUF);
512 rc = -1;
513 pci_disable_busmaster(softc->dev);
514 enic_pci_mapping_free(softc);
515 mtx_destroy(&softc->enic_lock);
516 return (rc);
517 }
518
519 static int
enic_msix_intr_assign(if_ctx_t ctx,int msix)520 enic_msix_intr_assign(if_ctx_t ctx, int msix)
521 {
522 struct enic_softc *softc;
523 struct enic *enic;
524 if_softc_ctx_t scctx;
525
526 int error;
527 int i;
528 char irq_name[16];
529
530 softc = iflib_get_softc(ctx);
531 enic = &softc->enic;
532 scctx = softc->scctx;
533
534 ENIC_LOCK(softc);
535 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSIX);
536 ENIC_UNLOCK(softc);
537
538 if (enic->intr_queues == NULL)
539 enic->intr_queues = malloc(sizeof(*enic->intr_queues) *
540 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO);
541 enic->intr = malloc(sizeof(*enic->intr) * msix, M_DEVBUF, M_NOWAIT
542 | M_ZERO);
543 if (enic->intr_queues == NULL || enic->intr == NULL)
544 return (ENOMEM);
545 for (i = 0; i < scctx->isc_nrxqsets; i++) {
546 snprintf(irq_name, sizeof(irq_name), "erxq%d:%d", i,
547 device_get_unit(softc->dev));
548
549 error = iflib_irq_alloc_generic(ctx,
550 &enic->intr_queues[i].intr_irq, i + 1, IFLIB_INTR_RX,
551 enic_rxq_intr, &enic->rq[i], i, irq_name);
552 if (error) {
553 device_printf(iflib_get_dev(ctx),
554 "Failed to register rxq %d interrupt handler\n", i);
555 return (error);
556 }
557 enic->intr[i].index = i;
558 enic->intr[i].vdev = enic->vdev;
559 ENIC_LOCK(softc);
560 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev,
561 RES_TYPE_INTR_CTRL, i);
562 vnic_intr_mask(&enic->intr[i]);
563 ENIC_UNLOCK(softc);
564 }
565
566 for (i = scctx->isc_nrxqsets; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) {
567 snprintf(irq_name, sizeof(irq_name), "etxq%d:%d", i -
568 scctx->isc_nrxqsets, device_get_unit(softc->dev));
569
570 iflib_softirq_alloc_generic(ctx,
571 &enic->intr_queues[i].intr_irq, IFLIB_INTR_TX,
572 &enic->wq[i - scctx->isc_nrxqsets], i - scctx->isc_nrxqsets,
573 irq_name);
574
575 enic->intr[i].index = i;
576 enic->intr[i].vdev = enic->vdev;
577 ENIC_LOCK(softc);
578 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev,
579 RES_TYPE_INTR_CTRL, i);
580 vnic_intr_mask(&enic->intr[i]);
581 ENIC_UNLOCK(softc);
582 }
583
584 i = scctx->isc_nrxqsets + scctx->isc_ntxqsets;
585 error = iflib_irq_alloc_generic(ctx, &softc->enic_event_intr_irq,
586 i + 1, IFLIB_INTR_ADMIN, enic_event_intr, softc, 0, "event");
587 if (error) {
588 device_printf(iflib_get_dev(ctx),
589 "Failed to register event interrupt handler\n");
590 return (error);
591 }
592
593 enic->intr[i].index = i;
594 enic->intr[i].vdev = enic->vdev;
595 ENIC_LOCK(softc);
596 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL,
597 i);
598 vnic_intr_mask(&enic->intr[i]);
599 ENIC_UNLOCK(softc);
600
601 i++;
602 error = iflib_irq_alloc_generic(ctx, &softc->enic_err_intr_irq,
603 i + 1, IFLIB_INTR_ADMIN, enic_err_intr, softc, 0, "err");
604 if (error) {
605 device_printf(iflib_get_dev(ctx),
606 "Failed to register event interrupt handler\n");
607 return (error);
608 }
609 enic->intr[i].index = i;
610 enic->intr[i].vdev = enic->vdev;
611 ENIC_LOCK(softc);
612 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL,
613 i);
614 vnic_intr_mask(&enic->intr[i]);
615 ENIC_UNLOCK(softc);
616
617 enic->intr_count = msix;
618
619 return (0);
620 }
621
622 static void
enic_free_irqs(struct enic_softc * softc)623 enic_free_irqs(struct enic_softc *softc)
624 {
625 if_softc_ctx_t scctx;
626
627 struct enic *enic;
628 int i;
629
630 scctx = softc->scctx;
631 enic = &softc->enic;
632
633 if (enic->intr_queues != NULL) {
634 for (i = 0;
635 i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++)
636 iflib_irq_free(softc->ctx,
637 &enic->intr_queues[i].intr_irq);
638 }
639
640 iflib_irq_free(softc->ctx, &softc->enic_event_intr_irq);
641 iflib_irq_free(softc->ctx, &softc->enic_err_intr_irq);
642 free(enic->intr_queues, M_DEVBUF);
643 enic->intr_queues = NULL;
644 free(enic->intr, M_DEVBUF);
645 enic->intr = NULL;
646 }
647
648 static int
enic_attach_post(if_ctx_t ctx)649 enic_attach_post(if_ctx_t ctx)
650 {
651 struct enic *enic;
652 struct enic_softc *softc;
653 int error = 0;
654
655 softc = iflib_get_softc(ctx);
656 enic = &softc->enic;
657
658 enic_setup_sysctl(softc);
659
660 enic_init_vnic_resources(enic);
661 enic_set_rx_coal_setting(enic);
662 enic_setup_finish(enic);
663
664 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
665 ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO);
666
667 return (error);
668 }
669
670 static int
enic_detach(if_ctx_t ctx)671 enic_detach(if_ctx_t ctx)
672 {
673 struct enic_softc *softc;
674 struct enic *enic;
675
676 softc = iflib_get_softc(ctx);
677 enic = &softc->enic;
678
679 vnic_dev_notify_unset(enic->vdev);
680
681 enic_free_irqs(softc);
682
683 ENIC_LOCK(softc);
684 vnic_dev_deinit(enic->vdev);
685 vnic_dev_close(enic->vdev);
686 vnic_dev_deinit_devcmd2(enic->vdev);
687 free(softc->vdev.devcmd, M_DEVBUF);
688 softc->vdev.devcmd = NULL;
689 pci_disable_busmaster(softc->dev);
690 enic_pci_mapping_free(softc);
691 ENIC_UNLOCK(softc);
692 if (softc->vdev.stats_res.idi_size != 0) {
693 iflib_dma_free(&softc->vdev.stats_res);
694 softc->vdev.stats = NULL;
695 }
696 if (softc->vdev.flow_counters_res.idi_size != 0) {
697 iflib_dma_free(&softc->vdev.flow_counters_res);
698 softc->vdev.flow_counters = NULL;
699 }
700 free(softc->mta, M_DEVBUF);
701 softc->mta = NULL;
702 mtx_destroy(&softc->enic_lock);
703
704 return 0;
705 }
706
707 static int
enic_tx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int ntxqs,int ntxqsets)708 enic_tx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs,
709 int ntxqs, int ntxqsets)
710 {
711 struct enic_softc *softc;
712 int q;
713
714 softc = iflib_get_softc(ctx);
715 softc->enic.cq = malloc(sizeof(*softc->enic.cq) *
716 (softc->enic.wq_count + softc->enic.rq_count), M_DEVBUF,
717 M_NOWAIT | M_ZERO);
718 if (softc->enic.cq == NULL)
719 return (ENOMEM);
720 /* Allocate the array of transmit queues */
721 softc->enic.wq = malloc(sizeof(struct vnic_wq) *
722 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO);
723 if (softc->enic.wq == NULL) {
724 free(softc->enic.cq, M_DEVBUF);
725 softc->enic.cq = NULL;
726 return (ENOMEM);
727 }
728
729 /* Initialize driver state for each transmit queue */
730
731 /*
732 * Allocate queue state that is shared with the device. This check
733 * and call is performed in both enic_tx_queues_alloc() and
734 * enic_rx_queues_alloc() so that we don't have to care which order
735 * iflib invokes those routines in.
736 */
737
738 /* Record descriptor ring vaddrs and paddrs */
739 ENIC_LOCK(softc);
740 for (q = 0; q < ntxqsets; q++) {
741 struct vnic_wq *wq;
742 struct vnic_cq *cq;
743 unsigned int cq_wq;
744
745 wq = &softc->enic.wq[q];
746 cq_wq = enic_cq_wq(&softc->enic, q);
747 cq = &softc->enic.cq[cq_wq];
748
749 /* Completion ring */
750 wq->vdev = softc->enic.vdev;
751 wq->index = q;
752 wq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_WQ,
753 wq->index);
754 vnic_wq_disable(wq);
755
756 wq->ring.desc_size = sizeof(struct wq_enet_desc);
757 wq->ring.desc_count = softc->scctx->isc_ntxd[q];
758 wq->ring.desc_avail = wq->ring.desc_count - 1;
759 wq->ring.last_count = wq->ring.desc_count;
760 wq->head_idx = 0;
761 wq->tail_idx = 0;
762
763 wq->ring.descs = vaddrs[q * ntxqs + 0];
764 wq->ring.base_addr = paddrs[q * ntxqs + 0];
765
766 /* Command ring */
767 cq->vdev = softc->enic.vdev;
768 cq->index = cq_wq;
769 cq->ctrl = vnic_dev_get_res(softc->enic.vdev,
770 RES_TYPE_CQ, cq->index);
771 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc);
772 cq->ring.desc_count = softc->scctx->isc_ntxd[q];
773 cq->ring.desc_avail = cq->ring.desc_count - 1;
774
775 cq->ring.descs = vaddrs[q * ntxqs + 1];
776 cq->ring.base_addr = paddrs[q * ntxqs + 1];
777
778 }
779
780 ENIC_UNLOCK(softc);
781
782 return (0);
783 }
784
785
786
787 static int
enic_rx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int nrxqs,int nrxqsets)788 enic_rx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs,
789 int nrxqs, int nrxqsets)
790 {
791 struct enic_softc *softc;
792 int q;
793
794 softc = iflib_get_softc(ctx);
795 /* Allocate the array of receive queues */
796 softc->enic.rq = malloc(sizeof(struct vnic_rq) * nrxqsets, M_DEVBUF,
797 M_NOWAIT | M_ZERO);
798 if (softc->enic.rq == NULL) {
799 free(softc->enic.wq, M_DEVBUF);
800 softc->enic.wq = NULL;
801 free(softc->enic.cq, M_DEVBUF);
802 softc->enic.cq = NULL;
803 return (ENOMEM);
804 }
805
806 /* Initialize driver state for each receive queue */
807
808 /*
809 * Allocate queue state that is shared with the device. This check
810 * and call is performed in both enic_tx_queues_alloc() and
811 * enic_rx_queues_alloc() so that we don't have to care which order
812 * iflib invokes those routines in.
813 */
814
815 /* Record descriptor ring vaddrs and paddrs */
816 ENIC_LOCK(softc);
817 for (q = 0; q < nrxqsets; q++) {
818 struct vnic_rq *rq;
819 struct vnic_cq *cq;
820 unsigned int cq_rq;
821
822 rq = &softc->enic.rq[q];
823 cq_rq = enic_cq_rq(&softc->enic, q);
824 cq = &softc->enic.cq[cq_rq];
825
826 /* Completion ring */
827 cq->vdev = softc->enic.vdev;
828 cq->index = cq_rq;
829 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_CQ,
830 cq->index);
831 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc);
832 cq->ring.desc_count = softc->scctx->isc_nrxd[1];
833 cq->ring.desc_avail = cq->ring.desc_count - 1;
834
835 cq->ring.descs = vaddrs[q * nrxqs + 0];
836 cq->ring.base_addr = paddrs[q * nrxqs + 0];
837
838 /* Command ring(s) */
839 rq->vdev = softc->enic.vdev;
840
841 rq->index = q;
842 rq->ctrl = vnic_dev_get_res(softc->enic.vdev,
843 RES_TYPE_RQ, rq->index);
844 vnic_rq_disable(rq);
845
846 rq->ring.desc_size = sizeof(struct rq_enet_desc);
847 rq->ring.desc_count = softc->scctx->isc_nrxd[0];
848 rq->ring.desc_avail = rq->ring.desc_count - 1;
849
850 rq->ring.descs = vaddrs[q * nrxqs + 1];
851 rq->ring.base_addr = paddrs[q * nrxqs + 1];
852 rq->need_initial_post = true;
853 }
854
855 ENIC_UNLOCK(softc);
856
857 return (0);
858 }
859
860 static void
enic_queues_free(if_ctx_t ctx)861 enic_queues_free(if_ctx_t ctx)
862 {
863 struct enic_softc *softc;
864 softc = iflib_get_softc(ctx);
865
866 free(softc->enic.rq, M_DEVBUF);
867 softc->enic.rq = NULL;
868 free(softc->enic.wq, M_DEVBUF);
869 softc->enic.wq = NULL;
870 free(softc->enic.cq, M_DEVBUF);
871 softc->enic.cq = NULL;
872 }
873
874 static int
enic_rxq_intr(void * rxq)875 enic_rxq_intr(void *rxq)
876 {
877 struct vnic_rq *rq;
878 if_t ifp;
879
880 rq = (struct vnic_rq *)rxq;
881 ifp = iflib_get_ifp(rq->vdev->softc->ctx);
882 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
883 return (FILTER_HANDLED);
884
885 return (FILTER_SCHEDULE_THREAD);
886 }
887
888 static int
enic_event_intr(void * vsc)889 enic_event_intr(void *vsc)
890 {
891 struct enic_softc *softc;
892 struct enic *enic;
893 uint32_t mtu;
894
895 softc = vsc;
896 enic = &softc->enic;
897
898 mtu = vnic_dev_mtu(enic->vdev);
899 if (mtu && mtu != enic->port_mtu) {
900 enic->port_mtu = mtu;
901 }
902
903 enic_link_status(softc);
904
905 return (FILTER_HANDLED);
906 }
907
908 static int
enic_err_intr(void * vsc)909 enic_err_intr(void *vsc)
910 {
911 struct enic_softc *softc;
912
913 softc = vsc;
914
915 enic_stop(softc->ctx);
916 enic_init(softc->ctx);
917
918 return (FILTER_HANDLED);
919 }
920
921 static void
enic_stop(if_ctx_t ctx)922 enic_stop(if_ctx_t ctx)
923 {
924 struct enic_softc *softc;
925 struct enic *enic;
926 if_softc_ctx_t scctx;
927 unsigned int index;
928 struct vnic_wq *wq;
929 struct vnic_rq *rq;
930 struct vnic_cq *cq;
931 unsigned int cq_wq, cq_rq;
932
933
934 softc = iflib_get_softc(ctx);
935 scctx = softc->scctx;
936 enic = &softc->enic;
937
938 if (softc->stopped)
939 return;
940 softc->link_active = 0;
941 softc->stopped = 1;
942
943 enic_dev_disable(enic);
944
945 for (index = 0; index < scctx->isc_ntxqsets; index++) {
946 enic_stop_wq(enic, index);
947 vnic_wq_clean(&enic->wq[index]);
948 vnic_cq_clean(&enic->cq[enic_cq_rq(enic, index)]);
949
950 wq = &softc->enic.wq[index];
951 wq->ring.desc_avail = wq->ring.desc_count - 1;
952 wq->ring.last_count = wq->ring.desc_count;
953 wq->head_idx = 0;
954 wq->tail_idx = 0;
955
956 cq_wq = enic_cq_wq(&softc->enic, index);
957 cq = &softc->enic.cq[cq_wq];
958 cq->ring.desc_avail = cq->ring.desc_count - 1;
959 }
960
961 for (index = 0; index < scctx->isc_nrxqsets; index++) {
962 enic_stop_rq(enic, index);
963 vnic_rq_clean(&enic->rq[index]);
964 vnic_cq_clean(&enic->cq[enic_cq_wq(enic, index)]);
965
966 rq = &softc->enic.rq[index];
967 cq_rq = enic_cq_rq(&softc->enic, index);
968 cq = &softc->enic.cq[cq_rq];
969
970 cq->ring.desc_avail = cq->ring.desc_count - 1;
971 rq->ring.desc_avail = rq->ring.desc_count - 1;
972 rq->need_initial_post = true;
973 }
974
975 for (index = 0; index < scctx->isc_vectors; index++) {
976 vnic_intr_clean(&enic->intr[index]);
977 }
978 }
979
980 static void
enic_init(if_ctx_t ctx)981 enic_init(if_ctx_t ctx)
982 {
983 struct enic_softc *softc;
984 struct enic *enic;
985 if_softc_ctx_t scctx;
986 unsigned int index;
987
988 softc = iflib_get_softc(ctx);
989 scctx = softc->scctx;
990 enic = &softc->enic;
991
992
993 enic_init_vnic_resources(enic);
994
995 for (index = 0; index < scctx->isc_ntxqsets; index++)
996 enic_prep_wq_for_simple_tx(&softc->enic, index);
997
998 for (index = 0; index < scctx->isc_ntxqsets; index++)
999 enic_start_wq(enic, index);
1000
1001 for (index = 0; index < scctx->isc_nrxqsets; index++)
1002 enic_start_rq(enic, index);
1003
1004 /* Use the current MAC address. */
1005 bcopy(if_getlladdr(softc->ifp), softc->lladdr, ETHER_ADDR_LEN);
1006 enic_set_lladdr(softc);
1007
1008 ENIC_LOCK(softc);
1009 vnic_dev_enable_wait(enic->vdev);
1010 ENIC_UNLOCK(softc);
1011
1012 softc->stopped = 0;
1013
1014 enic_link_status(softc);
1015 }
1016
1017 static void
enic_del_mcast(struct enic_softc * softc)1018 enic_del_mcast(struct enic_softc *softc) {
1019 struct enic *enic;
1020 int i;
1021
1022 enic = &softc->enic;
1023 for (i=0; i < softc->mc_count; i++) {
1024 vnic_dev_del_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]);
1025 }
1026 softc->multicast = 0;
1027 softc->mc_count = 0;
1028 }
1029
1030 static void
enic_add_mcast(struct enic_softc * softc)1031 enic_add_mcast(struct enic_softc *softc) {
1032 struct enic *enic;
1033 int i;
1034
1035 enic = &softc->enic;
1036 for (i=0; i < softc->mc_count; i++) {
1037 vnic_dev_add_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]);
1038 }
1039 softc->multicast = 1;
1040 }
1041
1042 static u_int
enic_copy_maddr(void * arg,struct sockaddr_dl * sdl,u_int idx)1043 enic_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx)
1044 {
1045 uint8_t *mta = arg;
1046
1047 if (idx == ENIC_MAX_MULTICAST_ADDRESSES)
1048 return (0);
1049
1050 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN);
1051 return (1);
1052 }
1053
1054 static void
enic_multi_set(if_ctx_t ctx)1055 enic_multi_set(if_ctx_t ctx)
1056 {
1057 if_t ifp;
1058 struct enic_softc *softc;
1059 u_int count;
1060
1061 softc = iflib_get_softc(ctx);
1062 ifp = iflib_get_ifp(ctx);
1063
1064 ENIC_LOCK(softc);
1065 enic_del_mcast(softc);
1066 count = if_foreach_llmaddr(ifp, enic_copy_maddr, softc->mta);
1067 softc->mc_count = count;
1068 enic_add_mcast(softc);
1069 ENIC_UNLOCK(softc);
1070
1071 if (if_getflags(ifp) & IFF_PROMISC) {
1072 softc->promisc = 1;
1073 } else {
1074 softc->promisc = 0;
1075 }
1076 if (if_getflags(ifp) & IFF_ALLMULTI) {
1077 softc->allmulti = 1;
1078 } else {
1079 softc->allmulti = 0;
1080 }
1081 enic_update_packet_filter(&softc->enic);
1082 }
1083
1084 static int
enic_mtu_set(if_ctx_t ctx,uint32_t mtu)1085 enic_mtu_set(if_ctx_t ctx, uint32_t mtu)
1086 {
1087 struct enic_softc *softc;
1088 struct enic *enic;
1089 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx);
1090
1091 softc = iflib_get_softc(ctx);
1092 enic = &softc->enic;
1093
1094 enic_stop(softc->ctx);
1095 if (mtu > enic->port_mtu){
1096 return (EINVAL);
1097 }
1098
1099 enic->config.mtu = mtu;
1100 scctx->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1101 enic_init(softc->ctx);
1102
1103 return (0);
1104 }
1105
1106 static void
enic_media_status(if_ctx_t ctx,struct ifmediareq * ifmr)1107 enic_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
1108 {
1109 struct enic_softc *softc;
1110 struct ifmedia_entry *next;
1111 uint32_t speed;
1112 uint64_t target_baudrate;
1113
1114 softc = iflib_get_softc(ctx);
1115
1116 ifmr->ifm_status = IFM_AVALID;
1117 ifmr->ifm_active = IFM_ETHER;
1118
1119 if (enic_link_is_up(softc) != 0) {
1120 ENIC_LOCK(softc);
1121 speed = vnic_dev_port_speed(&softc->vdev);
1122 ENIC_UNLOCK(softc);
1123 target_baudrate = 1000ull * speed;
1124 LIST_FOREACH(next, &(iflib_get_media(ctx)->ifm_list), ifm_list) {
1125 if (ifmedia_baudrate(next->ifm_media) == target_baudrate) {
1126 ifmr->ifm_active |= next->ifm_media;
1127 }
1128 }
1129
1130 ifmr->ifm_status |= IFM_ACTIVE;
1131 ifmr->ifm_active |= IFM_AUTO;
1132 } else
1133 ifmr->ifm_active |= IFM_NONE;
1134 }
1135
1136 static int
enic_media_change(if_ctx_t ctx)1137 enic_media_change(if_ctx_t ctx)
1138 {
1139 return (ENODEV);
1140 }
1141
1142 static int
enic_promisc_set(if_ctx_t ctx,int flags)1143 enic_promisc_set(if_ctx_t ctx, int flags)
1144 {
1145 if_t ifp;
1146 struct enic_softc *softc;
1147
1148 softc = iflib_get_softc(ctx);
1149 ifp = iflib_get_ifp(ctx);
1150
1151 if (if_getflags(ifp) & IFF_PROMISC) {
1152 softc->promisc = 1;
1153 } else {
1154 softc->promisc = 0;
1155 }
1156 if (if_getflags(ifp) & IFF_ALLMULTI) {
1157 softc->allmulti = 1;
1158 } else {
1159 softc->allmulti = 0;
1160 }
1161 enic_update_packet_filter(&softc->enic);
1162
1163 return (0);
1164 }
1165
1166 static uint64_t
enic_get_counter(if_ctx_t ctx,ift_counter cnt)1167 enic_get_counter(if_ctx_t ctx, ift_counter cnt) {
1168 if_t ifp = iflib_get_ifp(ctx);
1169
1170 if (cnt < IFCOUNTERS)
1171 return if_get_counter_default(ifp, cnt);
1172
1173 return (0);
1174 }
1175
1176 static void
enic_update_admin_status(if_ctx_t ctx)1177 enic_update_admin_status(if_ctx_t ctx)
1178 {
1179 struct enic_softc *softc;
1180 softc = iflib_get_softc(ctx);
1181
1182 enic_link_status(softc);
1183 }
1184
1185 static void
enic_txq_timer(if_ctx_t ctx,uint16_t qid)1186 enic_txq_timer(if_ctx_t ctx, uint16_t qid)
1187 {
1188
1189 struct enic_softc *softc;
1190 struct enic *enic;
1191 struct vnic_stats *stats;
1192 int ret;
1193
1194 softc = iflib_get_softc(ctx);
1195 enic = &softc->enic;
1196
1197 ENIC_LOCK(softc);
1198 ret = vnic_dev_stats_dump(enic->vdev, &stats);
1199 ENIC_UNLOCK(softc);
1200 if (ret) {
1201 dev_err(enic, "Error in getting stats\n");
1202 }
1203 }
1204
1205 static int
enic_link_is_up(struct enic_softc * softc)1206 enic_link_is_up(struct enic_softc *softc)
1207 {
1208 return (vnic_dev_link_status(&softc->vdev) == 1);
1209 }
1210
1211 static void
enic_link_status(struct enic_softc * softc)1212 enic_link_status(struct enic_softc *softc)
1213 {
1214 if_ctx_t ctx;
1215 uint64_t speed;
1216 int link;
1217
1218 ctx = softc->ctx;
1219 link = enic_link_is_up(softc);
1220 speed = IF_Gbps(10);
1221
1222 ENIC_LOCK(softc);
1223 speed = vnic_dev_port_speed(&softc->vdev);
1224 ENIC_UNLOCK(softc);
1225
1226 if (link != 0 && softc->link_active == 0) {
1227 softc->link_active = 1;
1228 iflib_link_state_change(ctx, LINK_STATE_UP, speed);
1229 } else if (link == 0 && softc->link_active != 0) {
1230 softc->link_active = 0;
1231 iflib_link_state_change(ctx, LINK_STATE_DOWN, speed);
1232 }
1233 }
1234
1235 static void
enic_set_lladdr(struct enic_softc * softc)1236 enic_set_lladdr(struct enic_softc *softc)
1237 {
1238 struct enic *enic;
1239 enic = &softc->enic;
1240
1241 ENIC_LOCK(softc);
1242 vnic_dev_add_addr(enic->vdev, softc->lladdr);
1243 ENIC_UNLOCK(softc);
1244 }
1245
1246
1247 static void
enic_setup_txq_sysctl(struct vnic_wq * wq,int i,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)1248 enic_setup_txq_sysctl(struct vnic_wq *wq, int i, struct sysctl_ctx_list *ctx,
1249 struct sysctl_oid_list *child)
1250 {
1251 struct sysctl_oid *txsnode;
1252 struct sysctl_oid_list *txslist;
1253 struct vnic_stats *stats;
1254
1255 stats = wq[i].vdev->stats;
1256
1257 txsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats",
1258 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
1259 txslist = SYSCTL_CHILDREN(txsnode);
1260
1261 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_frames_ok", CTLFLAG_RD,
1262 &stats->tx.tx_frames_ok, "TX Frames OK");
1263 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_frames_ok", CTLFLAG_RD,
1264 &stats->tx.tx_unicast_frames_ok, "TX unicast frames OK");
1265 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_frames_ok", CTLFLAG_RD,
1266 &stats->tx.tx_multicast_frames_ok, "TX multicast framse OK");
1267 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_frames_ok", CTLFLAG_RD,
1268 &stats->tx.tx_broadcast_frames_ok, "TX Broadcast frames OK");
1269 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_bytes_ok", CTLFLAG_RD,
1270 &stats->tx.tx_bytes_ok, "TX bytes OK ");
1271 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_bytes_ok", CTLFLAG_RD,
1272 &stats->tx.tx_unicast_bytes_ok, "TX unicast bytes OK");
1273 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_bytes_ok", CTLFLAG_RD,
1274 &stats->tx.tx_multicast_bytes_ok, "TX multicast bytes OK");
1275 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_bytes_ok", CTLFLAG_RD,
1276 &stats->tx.tx_broadcast_bytes_ok, "TX broadcast bytes OK");
1277 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_drops", CTLFLAG_RD,
1278 &stats->tx.tx_drops, "TX drops");
1279 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_errors", CTLFLAG_RD,
1280 &stats->tx.tx_errors, "TX errors");
1281 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_tso", CTLFLAG_RD,
1282 &stats->tx.tx_tso, "TX TSO");
1283 }
1284
1285 static void
enic_setup_rxq_sysctl(struct vnic_rq * rq,int i,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)1286 enic_setup_rxq_sysctl(struct vnic_rq *rq, int i, struct sysctl_ctx_list *ctx,
1287 struct sysctl_oid_list *child)
1288 {
1289 struct sysctl_oid *rxsnode;
1290 struct sysctl_oid_list *rxslist;
1291 struct vnic_stats *stats;
1292
1293 stats = rq[i].vdev->stats;
1294
1295 rxsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats",
1296 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
1297 rxslist = SYSCTL_CHILDREN(rxsnode);
1298
1299 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_ok", CTLFLAG_RD,
1300 &stats->rx.rx_frames_ok, "RX Frames OK");
1301 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_total", CTLFLAG_RD,
1302 &stats->rx.rx_frames_total, "RX frames total");
1303 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_frames_ok", CTLFLAG_RD,
1304 &stats->rx.rx_unicast_frames_ok, "RX unicast frames ok");
1305 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_frames_ok", CTLFLAG_RD,
1306 &stats->rx.rx_multicast_frames_ok, "RX multicast Frames ok");
1307 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_frames_ok", CTLFLAG_RD,
1308 &stats->rx.rx_broadcast_frames_ok, "RX broadcast frames ok");
1309 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_bytes_ok", CTLFLAG_RD,
1310 &stats->rx.rx_bytes_ok, "RX bytes ok");
1311 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_bytes_ok", CTLFLAG_RD,
1312 &stats->rx.rx_unicast_bytes_ok, "RX unicast bytes ok");
1313 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_bytes_ok", CTLFLAG_RD,
1314 &stats->rx.rx_multicast_bytes_ok, "RX multicast bytes ok");
1315 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_bytes_ok", CTLFLAG_RD,
1316 &stats->rx.rx_broadcast_bytes_ok, "RX broadcast bytes ok");
1317 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_drop", CTLFLAG_RD,
1318 &stats->rx.rx_drop, "RX drop");
1319 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_errors", CTLFLAG_RD,
1320 &stats->rx.rx_errors, "RX errors");
1321 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_rss", CTLFLAG_RD,
1322 &stats->rx.rx_rss, "RX rss");
1323 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_crc_errors", CTLFLAG_RD,
1324 &stats->rx.rx_crc_errors, "RX crc errors");
1325 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_64", CTLFLAG_RD,
1326 &stats->rx.rx_frames_64, "RX frames 64");
1327 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_127", CTLFLAG_RD,
1328 &stats->rx.rx_frames_127, "RX frames 127");
1329 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_255", CTLFLAG_RD,
1330 &stats->rx.rx_frames_255, "RX frames 255");
1331 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_511", CTLFLAG_RD,
1332 &stats->rx.rx_frames_511, "RX frames 511");
1333 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1023", CTLFLAG_RD,
1334 &stats->rx.rx_frames_1023, "RX frames 1023");
1335 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1518", CTLFLAG_RD,
1336 &stats->rx.rx_frames_1518, "RX frames 1518");
1337 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_to_max", CTLFLAG_RD,
1338 &stats->rx.rx_frames_to_max, "RX frames to max");
1339 }
1340
1341 static void
enic_setup_queue_sysctl(struct enic_softc * softc,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)1342 enic_setup_queue_sysctl(struct enic_softc *softc, struct sysctl_ctx_list *ctx,
1343 struct sysctl_oid_list *child)
1344 {
1345 enic_setup_txq_sysctl(softc->enic.wq, 0, ctx, child);
1346 enic_setup_rxq_sysctl(softc->enic.rq, 0, ctx, child);
1347 }
1348
1349 static void
enic_setup_sysctl(struct enic_softc * softc)1350 enic_setup_sysctl(struct enic_softc *softc)
1351 {
1352 device_t dev;
1353 struct sysctl_ctx_list *ctx;
1354 struct sysctl_oid *tree;
1355 struct sysctl_oid_list *child;
1356
1357 dev = softc->dev;
1358 ctx = device_get_sysctl_ctx(dev);
1359 tree = device_get_sysctl_tree(dev);
1360 child = SYSCTL_CHILDREN(tree);
1361
1362 enic_setup_queue_sysctl(softc, ctx, child);
1363 }
1364
1365 static void
enic_enable_intr(struct enic_softc * softc,int irq)1366 enic_enable_intr(struct enic_softc *softc, int irq)
1367 {
1368 struct enic *enic = &softc->enic;
1369
1370 vnic_intr_unmask(&enic->intr[irq]);
1371 vnic_intr_return_all_credits(&enic->intr[irq]);
1372 }
1373
1374 static void
enic_disable_intr(struct enic_softc * softc,int irq)1375 enic_disable_intr(struct enic_softc *softc, int irq)
1376 {
1377 struct enic *enic = &softc->enic;
1378
1379 vnic_intr_mask(&enic->intr[irq]);
1380 vnic_intr_masked(&enic->intr[irq]); /* flush write */
1381 }
1382
1383 static int
enic_tx_queue_intr_enable(if_ctx_t ctx,uint16_t qid)1384 enic_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
1385 {
1386 struct enic_softc *softc;
1387 if_softc_ctx_t scctx;
1388
1389 softc = iflib_get_softc(ctx);
1390 scctx = softc->scctx;
1391
1392 enic_enable_intr(softc, qid + scctx->isc_nrxqsets);
1393
1394 return 0;
1395 }
1396
1397 static int
enic_rx_queue_intr_enable(if_ctx_t ctx,uint16_t qid)1398 enic_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
1399 {
1400 struct enic_softc *softc;
1401
1402 softc = iflib_get_softc(ctx);
1403 enic_enable_intr(softc, qid);
1404
1405 return 0;
1406 }
1407
1408 static void
enic_intr_enable_all(if_ctx_t ctx)1409 enic_intr_enable_all(if_ctx_t ctx)
1410 {
1411 struct enic_softc *softc;
1412 if_softc_ctx_t scctx;
1413 int i;
1414
1415 softc = iflib_get_softc(ctx);
1416 scctx = softc->scctx;
1417
1418 for (i = 0; i < scctx->isc_vectors; i++) {
1419 enic_enable_intr(softc, i);
1420 }
1421 }
1422
1423 static void
enic_intr_disable_all(if_ctx_t ctx)1424 enic_intr_disable_all(if_ctx_t ctx)
1425 {
1426 struct enic_softc *softc;
1427 if_softc_ctx_t scctx;
1428 int i;
1429
1430 softc = iflib_get_softc(ctx);
1431 scctx = softc->scctx;
1432 /*
1433 * iflib may invoke this routine before enic_attach_post() has run,
1434 * which is before the top level shared data area is initialized and
1435 * the device made aware of it.
1436 */
1437
1438 for (i = 0; i < scctx->isc_vectors; i++) {
1439 enic_disable_intr(softc, i);
1440 }
1441 }
1442
1443 static int
enic_dev_open(struct enic * enic)1444 enic_dev_open(struct enic *enic)
1445 {
1446 int err;
1447 int flags = CMD_OPENF_IG_DESCCACHE;
1448
1449 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1450 vnic_dev_open_done, flags);
1451 if (err)
1452 dev_err(enic_get_dev(enic),
1453 "vNIC device open failed, err %d\n", err);
1454
1455 return err;
1456 }
1457
1458 static int
enic_dev_init(struct enic * enic)1459 enic_dev_init(struct enic *enic)
1460 {
1461 int err;
1462
1463 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1464
1465 /*
1466 * Get vNIC configuration
1467 */
1468 err = enic_get_vnic_config(enic);
1469 if (err) {
1470 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1471 return err;
1472 }
1473
1474 /* Get available resource counts */
1475 enic_get_res_counts(enic);
1476
1477 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1478 enic->intr_queues = malloc(sizeof(*enic->intr_queues) *
1479 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO);
1480
1481 vnic_dev_set_reset_flag(enic->vdev, 0);
1482 enic->max_flow_counter = -1;
1483
1484 /* set up link status checking */
1485 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1486
1487 enic->overlay_offload = false;
1488 if (enic->disable_overlay && enic->vxlan) {
1489 /*
1490 * Explicitly disable overlay offload as the setting is
1491 * sticky, and resetting vNIC does not disable it.
1492 */
1493 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1494 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_DISABLE)) {
1495 dev_err(enic, "failed to disable overlay offload\n");
1496 } else {
1497 dev_info(enic, "Overlay offload is disabled\n");
1498 }
1499 }
1500 if (!enic->disable_overlay && enic->vxlan &&
1501 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1502 vnic_dev_overlay_offload_ctrl(enic->vdev,
1503 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_ENABLE) == 0) {
1504 enic->overlay_offload = true;
1505 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1506 dev_info(enic, "Overlay offload is enabled\n");
1507 /*
1508 * Reset the vxlan port to the default, as the NIC firmware
1509 * does not reset it automatically and keeps the old setting.
1510 */
1511 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1512 OVERLAY_CFG_VXLAN_PORT_UPDATE, ENIC_DEFAULT_VXLAN_PORT)) {
1513 dev_err(enic, "failed to update vxlan port\n");
1514 return (EINVAL);
1515 }
1516 }
1517 return 0;
1518 }
1519
1520 static void *
enic_alloc_consistent(void * priv,size_t size,bus_addr_t * dma_handle,struct iflib_dma_info * res,u8 * name)1521 enic_alloc_consistent(void *priv, size_t size, bus_addr_t * dma_handle,
1522 struct iflib_dma_info *res, u8 * name)
1523 {
1524 void *vaddr;
1525 *dma_handle = 0;
1526 struct enic *enic = (struct enic *)priv;
1527 int rz;
1528
1529 rz = iflib_dma_alloc(enic->softc->ctx, size, res, BUS_DMA_NOWAIT);
1530 if (rz) {
1531 pr_err("%s : Failed to allocate memory requested for %s\n",
1532 __func__, name);
1533 return NULL;
1534 }
1535
1536 vaddr = res->idi_vaddr;
1537 *dma_handle = res->idi_paddr;
1538
1539 return vaddr;
1540 }
1541
1542 static void
enic_free_consistent(void * priv,size_t size,void * vaddr,bus_addr_t dma_handle,struct iflib_dma_info * res)1543 enic_free_consistent(void *priv, size_t size, void *vaddr,
1544 bus_addr_t dma_handle, struct iflib_dma_info *res)
1545 {
1546 iflib_dma_free(res);
1547 }
1548
1549 static int
enic_pci_mapping(struct enic_softc * softc)1550 enic_pci_mapping(struct enic_softc *softc)
1551 {
1552 int rc;
1553
1554 rc = enic_map_bar(softc, &softc->mem, 0, true);
1555 if (rc)
1556 return rc;
1557
1558 rc = enic_map_bar(softc, &softc->io, 2, false);
1559
1560 return rc;
1561 }
1562
1563 static void
enic_pci_mapping_free(struct enic_softc * softc)1564 enic_pci_mapping_free(struct enic_softc *softc)
1565 {
1566 if (softc->mem.res != NULL)
1567 bus_release_resource(softc->dev, SYS_RES_MEMORY,
1568 softc->mem.rid, softc->mem.res);
1569 softc->mem.res = NULL;
1570
1571 if (softc->io.res != NULL)
1572 bus_release_resource(softc->dev, SYS_RES_MEMORY,
1573 softc->io.rid, softc->io.res);
1574 softc->io.res = NULL;
1575 }
1576
1577 static int
enic_dev_wait(struct vnic_dev * vdev,int (* start)(struct vnic_dev *,int),int (* finished)(struct vnic_dev *,int *),int arg)1578 enic_dev_wait(struct vnic_dev *vdev, int (*start) (struct vnic_dev *, int),
1579 int (*finished) (struct vnic_dev *, int *), int arg)
1580 {
1581 int done;
1582 int err;
1583 int i;
1584
1585 err = start(vdev, arg);
1586 if (err)
1587 return err;
1588
1589 /* Wait for func to complete...2 seconds max */
1590 for (i = 0; i < 2000; i++) {
1591 err = finished(vdev, &done);
1592 if (err)
1593 return err;
1594 if (done)
1595 return 0;
1596 usleep(1000);
1597 }
1598 return (ETIMEDOUT);
1599 }
1600
1601 static int
enic_map_bar(struct enic_softc * softc,struct enic_bar_info * bar,int bar_num,bool shareable)1602 enic_map_bar(struct enic_softc *softc, struct enic_bar_info *bar, int bar_num,
1603 bool shareable)
1604 {
1605 uint32_t flag;
1606
1607 if (bar->res != NULL) {
1608 device_printf(softc->dev, "Bar %d already mapped\n", bar_num);
1609 return (EDOOFUS);
1610 }
1611
1612 bar->rid = PCIR_BAR(bar_num);
1613 flag = RF_ACTIVE;
1614 if (shareable)
1615 flag |= RF_SHAREABLE;
1616
1617 if ((bar->res = bus_alloc_resource_any(softc->dev,
1618 SYS_RES_MEMORY, &bar->rid, flag)) == NULL) {
1619 device_printf(softc->dev,
1620 "PCI BAR%d mapping failure\n", bar_num);
1621 return (ENXIO);
1622 }
1623 bar->tag = rman_get_bustag(bar->res);
1624 bar->handle = rman_get_bushandle(bar->res);
1625 bar->size = rman_get_size(bar->res);
1626
1627 return 0;
1628 }
1629
1630 void
enic_init_vnic_resources(struct enic * enic)1631 enic_init_vnic_resources(struct enic *enic)
1632 {
1633 unsigned int error_interrupt_enable = 1;
1634 unsigned int error_interrupt_offset = 0;
1635 unsigned int rxq_interrupt_enable = 0;
1636 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
1637 unsigned int txq_interrupt_enable = 0;
1638 unsigned int txq_interrupt_offset;
1639 unsigned int index = 0;
1640 unsigned int cq_idx;
1641 if_softc_ctx_t scctx;
1642
1643 scctx = enic->softc->scctx;
1644
1645 rxq_interrupt_enable = 1;
1646 txq_interrupt_enable = 0;
1647
1648 rxq_interrupt_offset = 0;
1649 txq_interrupt_offset = scctx->isc_nrxqsets;
1650
1651 for (index = 0; index < enic->intr_count; index++) {
1652 vnic_intr_alloc(enic->vdev, &enic->intr[index], index);
1653 }
1654
1655 for (index = 0; index < scctx->isc_nrxqsets; index++) {
1656 cq_idx = enic_cq_rq(enic, index);
1657
1658 vnic_rq_clean(&enic->rq[index]);
1659 vnic_rq_init(&enic->rq[index], cq_idx, error_interrupt_enable,
1660 error_interrupt_offset);
1661
1662 vnic_cq_clean(&enic->cq[cq_idx]);
1663 vnic_cq_init(&enic->cq[cq_idx],
1664 0 /* flow_control_enable */ ,
1665 1 /* color_enable */ ,
1666 0 /* cq_head */ ,
1667 0 /* cq_tail */ ,
1668 1 /* cq_tail_color */ ,
1669 rxq_interrupt_enable,
1670 1 /* cq_entry_enable */ ,
1671 0 /* cq_message_enable */ ,
1672 rxq_interrupt_offset,
1673 0 /* cq_message_addr */ );
1674 if (rxq_interrupt_enable)
1675 rxq_interrupt_offset++;
1676 }
1677
1678 for (index = 0; index < scctx->isc_ntxqsets; index++) {
1679 cq_idx = enic_cq_wq(enic, index);
1680 vnic_wq_clean(&enic->wq[index]);
1681 vnic_wq_init(&enic->wq[index], cq_idx, error_interrupt_enable,
1682 error_interrupt_offset);
1683 /* Compute unsupported ol flags for enic_prep_pkts() */
1684 enic->wq[index].tx_offload_notsup_mask = 0;
1685
1686 vnic_cq_clean(&enic->cq[cq_idx]);
1687 vnic_cq_init(&enic->cq[cq_idx],
1688 0 /* flow_control_enable */ ,
1689 1 /* color_enable */ ,
1690 0 /* cq_head */ ,
1691 0 /* cq_tail */ ,
1692 1 /* cq_tail_color */ ,
1693 txq_interrupt_enable,
1694 1,
1695 0,
1696 txq_interrupt_offset,
1697 0 /* (u64)enic->wq[index].cqmsg_rz->iova */ );
1698
1699 }
1700
1701 for (index = 0; index < enic->intr_count; index++) {
1702 vnic_intr_init(&enic->intr[index], 125,
1703 enic->config.intr_timer_type, /* mask_on_assertion */ 1);
1704 }
1705 }
1706
1707 static void
enic_update_packet_filter(struct enic * enic)1708 enic_update_packet_filter(struct enic *enic)
1709 {
1710 struct enic_softc *softc = enic->softc;
1711
1712 ENIC_LOCK(softc);
1713 vnic_dev_packet_filter(enic->vdev,
1714 softc->directed,
1715 softc->multicast,
1716 softc->broadcast,
1717 softc->promisc,
1718 softc->allmulti);
1719 ENIC_UNLOCK(softc);
1720 }
1721
1722 static bool
enic_if_needs_restart(if_ctx_t ctx,enum iflib_restart_event event)1723 enic_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event)
1724 {
1725 switch (event) {
1726 case IFLIB_RESTART_VLAN_CONFIG:
1727 default:
1728 return (false);
1729 }
1730 }
1731
1732 int
enic_setup_finish(struct enic * enic)1733 enic_setup_finish(struct enic *enic)
1734 {
1735 struct enic_softc *softc = enic->softc;
1736
1737 /* Default conf */
1738 softc->directed = 1;
1739 softc->multicast = 0;
1740 softc->broadcast = 1;
1741 softc->promisc = 0;
1742 softc->allmulti = 1;
1743 enic_update_packet_filter(enic);
1744
1745 return 0;
1746 }
1747