1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
4 */
5
6 #include "opt_rss.h"
7
8 #include <sys/param.h>
9 #include <sys/systm.h>
10 #include <sys/kernel.h>
11 #include <sys/endian.h>
12 #include <sys/sockio.h>
13 #include <sys/mbuf.h>
14 #include <sys/malloc.h>
15 #include <sys/module.h>
16 #include <sys/socket.h>
17 #include <sys/sysctl.h>
18 #include <sys/smp.h>
19 #include <vm/vm.h>
20 #include <vm/pmap.h>
21
22 #include <net/ethernet.h>
23 #include <net/if.h>
24 #include <net/if_var.h>
25 #include <net/if_arp.h>
26 #include <net/if_dl.h>
27 #include <net/if_types.h>
28 #include <net/if_media.h>
29 #include <net/if_vlan_var.h>
30 #include <net/iflib.h>
31 #ifdef RSS
32 #include <net/rss_config.h>
33 #endif
34
35 #include <netinet/in_systm.h>
36 #include <netinet/in.h>
37 #include <netinet/ip.h>
38 #include <netinet/ip6.h>
39 #include <netinet6/ip6_var.h>
40 #include <netinet/udp.h>
41 #include <netinet/tcp.h>
42
43 #include <machine/bus.h>
44 #include <machine/resource.h>
45 #include <sys/bus.h>
46 #include <sys/rman.h>
47
48 #include <dev/pci/pcireg.h>
49 #include <dev/pci/pcivar.h>
50
51 #include "ifdi_if.h"
52 #include "enic.h"
53
54 #include "opt_inet.h"
55 #include "opt_inet6.h"
56
57 static SYSCTL_NODE(_hw, OID_AUTO, enic, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
58 "ENIC");
59
60 static const pci_vendor_info_t enic_vendor_info_array[] =
61 {
62 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET,
63 DRV_DESCRIPTION),
64 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET_VF,
65 DRV_DESCRIPTION " VF"),
66 /* required last entry */
67
68 PVID_END
69 };
70
71 static void *enic_register(device_t);
72 static int enic_attach_pre(if_ctx_t);
73 static int enic_msix_intr_assign(if_ctx_t, int);
74
75 static int enic_attach_post(if_ctx_t);
76 static int enic_detach(if_ctx_t);
77
78 static int enic_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
79 static int enic_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
80 static void enic_queues_free(if_ctx_t);
81 static int enic_rxq_intr(void *);
82 static int enic_event_intr(void *);
83 static int enic_err_intr(void *);
84 static void enic_stop(if_ctx_t);
85 static void enic_init(if_ctx_t);
86 static void enic_multi_set(if_ctx_t);
87 static int enic_mtu_set(if_ctx_t, uint32_t);
88 static void enic_media_status(if_ctx_t, struct ifmediareq *);
89 static int enic_media_change(if_ctx_t);
90 static int enic_promisc_set(if_ctx_t, int);
91 static uint64_t enic_get_counter(if_ctx_t, ift_counter);
92 static void enic_update_admin_status(if_ctx_t);
93 static void enic_txq_timer(if_ctx_t, uint16_t);
94 static int enic_link_is_up(struct enic_softc *);
95 static void enic_link_status(struct enic_softc *);
96 static void enic_set_lladdr(struct enic_softc *);
97 static void enic_setup_txq_sysctl(struct vnic_wq *, int, struct sysctl_ctx_list *,
98 struct sysctl_oid_list *);
99 static void enic_setup_rxq_sysctl(struct vnic_rq *, int, struct sysctl_ctx_list *,
100 struct sysctl_oid_list *);
101 static void enic_setup_sysctl(struct enic_softc *);
102 static int enic_tx_queue_intr_enable(if_ctx_t, uint16_t);
103 static int enic_rx_queue_intr_enable(if_ctx_t, uint16_t);
104 static void enic_enable_intr(struct enic_softc *, int);
105 static void enic_disable_intr(struct enic_softc *, int);
106 static void enic_intr_enable_all(if_ctx_t);
107 static void enic_intr_disable_all(if_ctx_t);
108 static int enic_dev_open(struct enic *);
109 static int enic_dev_init(struct enic *);
110 static void *enic_alloc_consistent(void *, size_t, bus_addr_t *,
111 struct iflib_dma_info *, u8 *);
112 static void enic_free_consistent(void *, size_t, void *, bus_addr_t,
113 struct iflib_dma_info *);
114 static int enic_pci_mapping(struct enic_softc *);
115 static void enic_pci_mapping_free(struct enic_softc *);
116 static int enic_dev_wait(struct vnic_dev *, int (*) (struct vnic_dev *, int),
117 int (*) (struct vnic_dev *, int *), int arg);
118 static int enic_map_bar(struct enic_softc *, struct enic_bar_info *, int, bool);
119 static void enic_update_packet_filter(struct enic *enic);
120 static bool enic_if_needs_restart(if_ctx_t, enum iflib_restart_event);
121
122 typedef enum {
123 ENIC_BARRIER_RD,
124 ENIC_BARRIER_WR,
125 ENIC_BARRIER_RDWR,
126 } enic_barrier_t;
127
128 static device_method_t enic_methods[] = {
129 /* Device interface */
130 DEVMETHOD(device_register, enic_register),
131 DEVMETHOD(device_probe, iflib_device_probe),
132 DEVMETHOD(device_attach, iflib_device_attach),
133 DEVMETHOD(device_detach, iflib_device_detach),
134 DEVMETHOD(device_shutdown, iflib_device_shutdown),
135 DEVMETHOD(device_suspend, iflib_device_suspend),
136 DEVMETHOD(device_resume, iflib_device_resume),
137 DEVMETHOD_END
138 };
139
140 static driver_t enic_driver = {
141 "enic", enic_methods, sizeof(struct enic_softc)
142 };
143
144 DRIVER_MODULE(enic, pci, enic_driver, 0, 0);
145 IFLIB_PNP_INFO(pci, enic, enic_vendor_info_array);
146 MODULE_VERSION(enic, 2);
147
148 MODULE_DEPEND(enic, pci, 1, 1, 1);
149 MODULE_DEPEND(enic, ether, 1, 1, 1);
150 MODULE_DEPEND(enic, iflib, 1, 1, 1);
151
152 static device_method_t enic_iflib_methods[] = {
153 DEVMETHOD(ifdi_tx_queues_alloc, enic_tx_queues_alloc),
154 DEVMETHOD(ifdi_rx_queues_alloc, enic_rx_queues_alloc),
155 DEVMETHOD(ifdi_queues_free, enic_queues_free),
156
157 DEVMETHOD(ifdi_attach_pre, enic_attach_pre),
158 DEVMETHOD(ifdi_attach_post, enic_attach_post),
159 DEVMETHOD(ifdi_detach, enic_detach),
160
161 DEVMETHOD(ifdi_init, enic_init),
162 DEVMETHOD(ifdi_stop, enic_stop),
163 DEVMETHOD(ifdi_multi_set, enic_multi_set),
164 DEVMETHOD(ifdi_mtu_set, enic_mtu_set),
165 DEVMETHOD(ifdi_media_status, enic_media_status),
166 DEVMETHOD(ifdi_media_change, enic_media_change),
167 DEVMETHOD(ifdi_promisc_set, enic_promisc_set),
168 DEVMETHOD(ifdi_get_counter, enic_get_counter),
169 DEVMETHOD(ifdi_update_admin_status, enic_update_admin_status),
170 DEVMETHOD(ifdi_timer, enic_txq_timer),
171
172 DEVMETHOD(ifdi_tx_queue_intr_enable, enic_tx_queue_intr_enable),
173 DEVMETHOD(ifdi_rx_queue_intr_enable, enic_rx_queue_intr_enable),
174 DEVMETHOD(ifdi_intr_enable, enic_intr_enable_all),
175 DEVMETHOD(ifdi_intr_disable, enic_intr_disable_all),
176 DEVMETHOD(ifdi_msix_intr_assign, enic_msix_intr_assign),
177
178 DEVMETHOD(ifdi_needs_restart, enic_if_needs_restart),
179
180 DEVMETHOD_END
181 };
182
183 static driver_t enic_iflib_driver = {
184 "enic", enic_iflib_methods, sizeof(struct enic_softc)
185 };
186
187 extern struct if_txrx enic_txrx;
188
189 static struct if_shared_ctx enic_sctx_init = {
190 .isc_magic = IFLIB_MAGIC,
191 .isc_q_align = 512,
192
193 .isc_tx_maxsize = ENIC_TX_MAX_PKT_SIZE,
194 .isc_tx_maxsegsize = PAGE_SIZE,
195
196 /*
197 * These values are used to configure the busdma tag used for receive
198 * descriptors. Each receive descriptor only points to one buffer.
199 */
200 .isc_rx_maxsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, /* One buf per
201 * descriptor */
202 .isc_rx_nsegments = 1, /* One mapping per descriptor */
203 .isc_rx_maxsegsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE,
204 .isc_admin_intrcnt = 3,
205 .isc_vendor_info = enic_vendor_info_array,
206 .isc_driver_version = "1",
207 .isc_driver = &enic_iflib_driver,
208 .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ,
209
210 /*
211 * Number of receive queues per receive queue set, with associated
212 * descriptor settings for each.
213 */
214
215 .isc_nrxqs = 2,
216 .isc_nfl = 1, /* one free list for each receive command
217 * queue */
218 .isc_nrxd_min = {16, 16},
219 .isc_nrxd_max = {2048, 2048},
220 .isc_nrxd_default = {64, 64},
221
222 /*
223 * Number of transmit queues per transmit queue set, with associated
224 * descriptor settings for each.
225 */
226 .isc_ntxqs = 2,
227 .isc_ntxd_min = {16, 16},
228 .isc_ntxd_max = {2048, 2048},
229 .isc_ntxd_default = {64, 64},
230 };
231
232 static void *
enic_register(device_t dev)233 enic_register(device_t dev)
234 {
235 return (&enic_sctx_init);
236 }
237
238 static int
enic_attach_pre(if_ctx_t ctx)239 enic_attach_pre(if_ctx_t ctx)
240 {
241 if_softc_ctx_t scctx;
242 struct enic_softc *softc;
243 struct vnic_dev *vdev;
244 struct enic *enic;
245 device_t dev;
246
247 int err = -1;
248 int rc = 0;
249 int i;
250 u64 a0 = 0, a1 = 0;
251 int wait = 1000;
252 struct vnic_stats *stats;
253 int ret;
254
255 dev = iflib_get_dev(ctx);
256 softc = iflib_get_softc(ctx);
257 softc->dev = dev;
258 softc->ctx = ctx;
259 softc->sctx = iflib_get_sctx(ctx);
260 softc->scctx = iflib_get_softc_ctx(ctx);
261 softc->ifp = iflib_get_ifp(ctx);
262 softc->media = iflib_get_media(ctx);
263 softc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN *
264 ENIC_MAX_MULTICAST_ADDRESSES, M_DEVBUF,
265 M_NOWAIT | M_ZERO);
266 if (softc->mta == NULL)
267 return (ENOMEM);
268 scctx = softc->scctx;
269
270 mtx_init(&softc->enic_lock, "ENIC Lock", NULL, MTX_DEF);
271
272 pci_enable_busmaster(softc->dev);
273 if (enic_pci_mapping(softc))
274 return (ENXIO);
275
276 enic = &softc->enic;
277 enic->softc = softc;
278 vdev = &softc->vdev;
279 vdev->softc = softc;
280 enic->vdev = vdev;
281 vdev->priv = enic;
282
283 ENIC_LOCK(softc);
284 vnic_dev_register(vdev, &softc->mem, 1);
285 enic->vdev = vdev;
286 vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0);
287
288 vnic_dev_cmd(vdev, CMD_INIT_v1, &a0, &a1, wait);
289 vnic_dev_cmd(vdev, CMD_GET_MAC_ADDR, &a0, &a1, wait);
290
291 bcopy((u_int8_t *) & a0, softc->mac_addr, ETHER_ADDR_LEN);
292 iflib_set_mac(ctx, softc->mac_addr);
293
294 vnic_register_cbacks(enic->vdev, enic_alloc_consistent,
295 enic_free_consistent);
296
297 /*
298 * Allocate the consistent memory for stats and counters upfront so
299 * both primary and secondary processes can access them.
300 */
301 ENIC_UNLOCK(softc);
302 err = vnic_dev_alloc_stats_mem(enic->vdev);
303 ENIC_LOCK(softc);
304 if (err) {
305 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
306 goto err_out_unregister;
307 }
308 vnic_dev_stats_clear(enic->vdev);
309 ret = vnic_dev_stats_dump(enic->vdev, &stats);
310 if (ret) {
311 dev_err(enic, "Error in getting stats\n");
312 goto err_out_unregister;
313 }
314 err = vnic_dev_alloc_counter_mem(enic->vdev);
315 if (err) {
316 dev_err(enic, "Failed to allocate counter memory, aborting\n");
317 goto err_out_unregister;
318 }
319
320 /* Issue device open to get device in known state */
321 err = enic_dev_open(enic);
322 if (err) {
323 dev_err(enic, "vNIC dev open failed, aborting\n");
324 goto err_out_unregister;
325 }
326
327 /* Set ingress vlan rewrite mode before vnic initialization */
328 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN;
329 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
330 enic->ig_vlan_rewrite_mode);
331 if (err) {
332 dev_err(enic,
333 "Failed to set ingress vlan rewrite mode, aborting.\n");
334 goto err_out_dev_close;
335 }
336
337 /*
338 * Issue device init to initialize the vnic-to-switch link. We'll
339 * start with carrier off and wait for link UP notification later to
340 * turn on carrier. We don't need to wait here for the
341 * vnic-to-switch link initialization to complete; link UP
342 * notification is the indication that the process is complete.
343 */
344
345 err = vnic_dev_init(enic->vdev, 0);
346 if (err) {
347 dev_err(enic, "vNIC dev init failed, aborting\n");
348 goto err_out_dev_close;
349 }
350
351 err = enic_dev_init(enic);
352 if (err) {
353 dev_err(enic, "Device initialization failed, aborting\n");
354 goto err_out_dev_close;
355 }
356 ENIC_UNLOCK(softc);
357
358 enic->port_mtu = vnic_dev_mtu(enic->vdev);
359
360 softc->scctx = iflib_get_softc_ctx(ctx);
361 scctx = softc->scctx;
362 scctx->isc_txrx = &enic_txrx;
363 scctx->isc_capabilities = scctx->isc_capenable = 0;
364 scctx->isc_tx_csum_flags = 0;
365 scctx->isc_max_frame_size = enic->config.mtu + ETHER_HDR_LEN + \
366 ETHER_CRC_LEN;
367 scctx->isc_nrxqsets_max = enic->conf_rq_count;
368 scctx->isc_ntxqsets_max = enic->conf_wq_count;
369 scctx->isc_nrxqsets = enic->conf_rq_count;
370 scctx->isc_ntxqsets = enic->conf_wq_count;
371 for (i = 0; i < enic->conf_wq_count; i++) {
372 scctx->isc_ntxd[i] = enic->config.wq_desc_count;
373 scctx->isc_txqsizes[i] = sizeof(struct cq_enet_wq_desc)
374 * scctx->isc_ntxd[i];
375 scctx->isc_ntxd[i + enic->conf_wq_count] =
376 enic->config.wq_desc_count;
377 scctx->isc_txqsizes[i + enic->conf_wq_count] =
378 sizeof(struct cq_desc) * scctx->isc_ntxd[i +
379 enic->conf_wq_count];
380 }
381 for (i = 0; i < enic->conf_rq_count; i++) {
382 scctx->isc_nrxd[i] = enic->config.rq_desc_count;
383 scctx->isc_rxqsizes[i] = sizeof(struct cq_enet_rq_desc) *
384 scctx->isc_nrxd[i];
385 scctx->isc_nrxd[i + enic->conf_rq_count] =
386 enic->config.rq_desc_count;
387 scctx->isc_rxqsizes[i + enic->conf_rq_count] = sizeof(struct
388 cq_desc) * scctx->isc_nrxd[i + enic->conf_rq_count];
389 }
390 scctx->isc_tx_nsegments = 31;
391
392 scctx->isc_vectors = enic->conf_cq_count;
393 scctx->isc_msix_bar = -1;
394
395 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
396 ifmedia_add(softc->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
397 ifmedia_add(softc->media, IFM_ETHER | IFM_10_FL, 0, NULL);
398
399 /*
400 * Allocate the CQ here since TX is called first before RX for now
401 * assume RX and TX are the same
402 */
403 if (softc->enic.cq == NULL)
404 softc->enic.cq = malloc(sizeof(struct vnic_cq) *
405 softc->enic.wq_count + softc->enic.rq_count, M_DEVBUF,
406 M_NOWAIT | M_ZERO);
407 if (softc->enic.cq == NULL)
408 return (ENOMEM);
409
410 softc->enic.cq->ntxqsets = softc->enic.wq_count + softc->enic.rq_count;
411
412 /*
413 * Allocate the consistent memory for stats and counters upfront so
414 * both primary and secondary processes can access them.
415 */
416 err = vnic_dev_alloc_stats_mem(enic->vdev);
417 if (err) {
418 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
419 }
420
421 return (rc);
422
423 err_out_dev_close:
424 vnic_dev_close(enic->vdev);
425 err_out_unregister:
426 free(softc->vdev.devcmd, M_DEVBUF);
427 free(softc->enic.intr_queues, M_DEVBUF);
428 free(softc->enic.cq, M_DEVBUF);
429 free(softc->mta, M_DEVBUF);
430 rc = -1;
431 pci_disable_busmaster(softc->dev);
432 enic_pci_mapping_free(softc);
433 mtx_destroy(&softc->enic_lock);
434 return (rc);
435 }
436
437 static int
enic_msix_intr_assign(if_ctx_t ctx,int msix)438 enic_msix_intr_assign(if_ctx_t ctx, int msix)
439 {
440 struct enic_softc *softc;
441 struct enic *enic;
442 if_softc_ctx_t scctx;
443
444 int error;
445 int i;
446 char irq_name[16];
447
448 softc = iflib_get_softc(ctx);
449 enic = &softc->enic;
450 scctx = softc->scctx;
451
452 ENIC_LOCK(softc);
453 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSIX);
454 ENIC_UNLOCK(softc);
455
456 enic->intr_queues = malloc(sizeof(*enic->intr_queues) *
457 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO);
458 enic->intr = malloc(sizeof(*enic->intr) * msix, M_DEVBUF, M_NOWAIT
459 | M_ZERO);
460 for (i = 0; i < scctx->isc_nrxqsets; i++) {
461 snprintf(irq_name, sizeof(irq_name), "erxq%d:%d", i,
462 device_get_unit(softc->dev));
463
464 error = iflib_irq_alloc_generic(ctx,
465 &enic->intr_queues[i].intr_irq, i + 1, IFLIB_INTR_RX,
466 enic_rxq_intr, &enic->rq[i], i, irq_name);
467 if (error) {
468 device_printf(iflib_get_dev(ctx),
469 "Failed to register rxq %d interrupt handler\n", i);
470 return (error);
471 }
472 enic->intr[i].index = i;
473 enic->intr[i].vdev = enic->vdev;
474 ENIC_LOCK(softc);
475 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev,
476 RES_TYPE_INTR_CTRL, i);
477 vnic_intr_mask(&enic->intr[i]);
478 ENIC_UNLOCK(softc);
479 }
480
481 for (i = scctx->isc_nrxqsets; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) {
482 snprintf(irq_name, sizeof(irq_name), "etxq%d:%d", i -
483 scctx->isc_nrxqsets, device_get_unit(softc->dev));
484
485
486 iflib_softirq_alloc_generic(ctx, &enic->intr_queues[i].intr_irq, IFLIB_INTR_TX, &enic->wq[i - scctx->isc_nrxqsets], i - scctx->isc_nrxqsets, irq_name);
487
488
489 enic->intr[i].index = i;
490 enic->intr[i].vdev = enic->vdev;
491 ENIC_LOCK(softc);
492 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev,
493 RES_TYPE_INTR_CTRL, i);
494 vnic_intr_mask(&enic->intr[i]);
495 ENIC_UNLOCK(softc);
496 }
497
498 i = scctx->isc_nrxqsets + scctx->isc_ntxqsets;
499 error = iflib_irq_alloc_generic(ctx, &softc->enic_event_intr_irq,
500 i + 1, IFLIB_INTR_ADMIN, enic_event_intr, softc, 0, "event");
501 if (error) {
502 device_printf(iflib_get_dev(ctx),
503 "Failed to register event interrupt handler\n");
504 return (error);
505 }
506
507 enic->intr[i].index = i;
508 enic->intr[i].vdev = enic->vdev;
509 ENIC_LOCK(softc);
510 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL,
511 i);
512 vnic_intr_mask(&enic->intr[i]);
513 ENIC_UNLOCK(softc);
514
515 i++;
516 error = iflib_irq_alloc_generic(ctx, &softc->enic_err_intr_irq,
517 i + 1, IFLIB_INTR_ADMIN, enic_err_intr, softc, 0, "err");
518 if (error) {
519 device_printf(iflib_get_dev(ctx),
520 "Failed to register event interrupt handler\n");
521 return (error);
522 }
523 enic->intr[i].index = i;
524 enic->intr[i].vdev = enic->vdev;
525 ENIC_LOCK(softc);
526 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL,
527 i);
528 vnic_intr_mask(&enic->intr[i]);
529 ENIC_UNLOCK(softc);
530
531 enic->intr_count = msix;
532
533 return (0);
534 }
535
536 static void
enic_free_irqs(struct enic_softc * softc)537 enic_free_irqs(struct enic_softc *softc)
538 {
539 if_softc_ctx_t scctx;
540
541 struct enic *enic;
542 int i;
543
544 scctx = softc->scctx;
545 enic = &softc->enic;
546
547 for (i = 0; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) {
548 iflib_irq_free(softc->ctx, &enic->intr_queues[i].intr_irq);
549 }
550
551 iflib_irq_free(softc->ctx, &softc->enic_event_intr_irq);
552 iflib_irq_free(softc->ctx, &softc->enic_err_intr_irq);
553 free(enic->intr_queues, M_DEVBUF);
554 free(enic->intr, M_DEVBUF);
555 }
556
557 static int
enic_attach_post(if_ctx_t ctx)558 enic_attach_post(if_ctx_t ctx)
559 {
560 struct enic *enic;
561 struct enic_softc *softc;
562 int error = 0;
563
564 softc = iflib_get_softc(ctx);
565 enic = &softc->enic;
566
567 enic_setup_sysctl(softc);
568
569 enic_init_vnic_resources(enic);
570 enic_setup_finish(enic);
571
572 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
573 ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO);
574
575 return (error);
576 }
577
578 static int
enic_detach(if_ctx_t ctx)579 enic_detach(if_ctx_t ctx)
580 {
581 struct enic_softc *softc;
582 struct enic *enic;
583
584 softc = iflib_get_softc(ctx);
585 enic = &softc->enic;
586
587 vnic_dev_notify_unset(enic->vdev);
588
589 enic_free_irqs(softc);
590
591 ENIC_LOCK(softc);
592 vnic_dev_close(enic->vdev);
593 free(softc->vdev.devcmd, M_DEVBUF);
594 pci_disable_busmaster(softc->dev);
595 enic_pci_mapping_free(softc);
596 ENIC_UNLOCK(softc);
597
598 return 0;
599 }
600
601 static int
enic_tx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int ntxqs,int ntxqsets)602 enic_tx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs,
603 int ntxqs, int ntxqsets)
604 {
605 struct enic_softc *softc;
606 int q;
607
608 softc = iflib_get_softc(ctx);
609 /* Allocate the array of transmit queues */
610 softc->enic.wq = malloc(sizeof(struct vnic_wq) *
611 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO);
612 if (softc->enic.wq == NULL)
613 return (ENOMEM);
614
615 /* Initialize driver state for each transmit queue */
616
617 /*
618 * Allocate queue state that is shared with the device. This check
619 * and call is performed in both enic_tx_queues_alloc() and
620 * enic_rx_queues_alloc() so that we don't have to care which order
621 * iflib invokes those routines in.
622 */
623
624 /* Record descriptor ring vaddrs and paddrs */
625 ENIC_LOCK(softc);
626 for (q = 0; q < ntxqsets; q++) {
627 struct vnic_wq *wq;
628 struct vnic_cq *cq;
629 unsigned int cq_wq;
630
631 wq = &softc->enic.wq[q];
632 cq_wq = enic_cq_wq(&softc->enic, q);
633 cq = &softc->enic.cq[cq_wq];
634
635 /* Completion ring */
636 wq->vdev = softc->enic.vdev;
637 wq->index = q;
638 wq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_WQ,
639 wq->index);
640 vnic_wq_disable(wq);
641
642 wq->ring.desc_size = sizeof(struct wq_enet_desc);
643 wq->ring.desc_count = softc->scctx->isc_ntxd[q];
644 wq->ring.desc_avail = wq->ring.desc_count - 1;
645 wq->ring.last_count = wq->ring.desc_count;
646 wq->head_idx = 0;
647 wq->tail_idx = 0;
648
649 wq->ring.size = wq->ring.desc_count * wq->ring.desc_size;
650 wq->ring.descs = vaddrs[q * ntxqs + 0];
651 wq->ring.base_addr = paddrs[q * ntxqs + 0];
652
653 /* Command ring */
654 cq->vdev = softc->enic.vdev;
655 cq->index = cq_wq;
656 cq->ctrl = vnic_dev_get_res(softc->enic.vdev,
657 RES_TYPE_CQ, cq->index);
658 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc);
659 cq->ring.desc_count = softc->scctx->isc_ntxd[q];
660 cq->ring.desc_avail = cq->ring.desc_count - 1;
661
662 cq->ring.size = cq->ring.desc_count * cq->ring.desc_size;
663 cq->ring.descs = vaddrs[q * ntxqs + 1];
664 cq->ring.base_addr = paddrs[q * ntxqs + 1];
665
666 }
667
668 ENIC_UNLOCK(softc);
669
670 return (0);
671 }
672
673
674
675 static int
enic_rx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int nrxqs,int nrxqsets)676 enic_rx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs,
677 int nrxqs, int nrxqsets)
678 {
679 struct enic_softc *softc;
680 int q;
681
682 softc = iflib_get_softc(ctx);
683 /* Allocate the array of receive queues */
684 softc->enic.rq = malloc(sizeof(struct vnic_rq) * nrxqsets, M_DEVBUF,
685 M_NOWAIT | M_ZERO);
686 if (softc->enic.rq == NULL)
687 return (ENOMEM);
688
689 /* Initialize driver state for each receive queue */
690
691 /*
692 * Allocate queue state that is shared with the device. This check
693 * and call is performed in both enic_tx_queues_alloc() and
694 * enic_rx_queues_alloc() so that we don't have to care which order
695 * iflib invokes those routines in.
696 */
697
698 /* Record descriptor ring vaddrs and paddrs */
699 ENIC_LOCK(softc);
700 for (q = 0; q < nrxqsets; q++) {
701 struct vnic_rq *rq;
702 struct vnic_cq *cq;
703 unsigned int cq_rq;
704
705 rq = &softc->enic.rq[q];
706 cq_rq = enic_cq_rq(&softc->enic, q);
707 cq = &softc->enic.cq[cq_rq];
708
709 /* Completion ring */
710 cq->vdev = softc->enic.vdev;
711 cq->index = cq_rq;
712 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_CQ,
713 cq->index);
714 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc);
715 cq->ring.desc_count = softc->scctx->isc_nrxd[1];
716 cq->ring.desc_avail = cq->ring.desc_count - 1;
717
718 cq->ring.size = cq->ring.desc_count * cq->ring.desc_size;
719 cq->ring.descs = vaddrs[q * nrxqs + 0];
720 cq->ring.base_addr = paddrs[q * nrxqs + 0];
721
722 /* Command ring(s) */
723 rq->vdev = softc->enic.vdev;
724
725 rq->index = q;
726 rq->ctrl = vnic_dev_get_res(softc->enic.vdev,
727 RES_TYPE_RQ, rq->index);
728 vnic_rq_disable(rq);
729
730 rq->ring.desc_size = sizeof(struct rq_enet_desc);
731 rq->ring.desc_count = softc->scctx->isc_nrxd[0];
732 rq->ring.desc_avail = rq->ring.desc_count - 1;
733
734 rq->ring.size = rq->ring.desc_count * rq->ring.desc_size;
735 rq->ring.descs = vaddrs[q * nrxqs + 1];
736 rq->ring.base_addr = paddrs[q * nrxqs + 1];
737 rq->need_initial_post = true;
738 }
739
740 ENIC_UNLOCK(softc);
741
742 return (0);
743 }
744
745 static void
enic_queues_free(if_ctx_t ctx)746 enic_queues_free(if_ctx_t ctx)
747 {
748 struct enic_softc *softc;
749 softc = iflib_get_softc(ctx);
750
751 free(softc->enic.rq, M_DEVBUF);
752 free(softc->enic.wq, M_DEVBUF);
753 free(softc->enic.cq, M_DEVBUF);
754 }
755
756 static int
enic_rxq_intr(void * rxq)757 enic_rxq_intr(void *rxq)
758 {
759 struct vnic_rq *rq;
760 if_t ifp;
761
762 rq = (struct vnic_rq *)rxq;
763 ifp = iflib_get_ifp(rq->vdev->softc->ctx);
764 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
765 return (FILTER_HANDLED);
766
767 return (FILTER_SCHEDULE_THREAD);
768 }
769
770 static int
enic_event_intr(void * vsc)771 enic_event_intr(void *vsc)
772 {
773 struct enic_softc *softc;
774 struct enic *enic;
775 uint32_t mtu;
776
777 softc = vsc;
778 enic = &softc->enic;
779
780 mtu = vnic_dev_mtu(enic->vdev);
781 if (mtu && mtu != enic->port_mtu) {
782 enic->port_mtu = mtu;
783 }
784
785 enic_link_status(softc);
786
787 return (FILTER_HANDLED);
788 }
789
790 static int
enic_err_intr(void * vsc)791 enic_err_intr(void *vsc)
792 {
793 struct enic_softc *softc;
794
795 softc = vsc;
796
797 enic_stop(softc->ctx);
798 enic_init(softc->ctx);
799
800 return (FILTER_HANDLED);
801 }
802
803 static void
enic_stop(if_ctx_t ctx)804 enic_stop(if_ctx_t ctx)
805 {
806 struct enic_softc *softc;
807 struct enic *enic;
808 if_softc_ctx_t scctx;
809 unsigned int index;
810
811 softc = iflib_get_softc(ctx);
812 scctx = softc->scctx;
813 enic = &softc->enic;
814
815 if (softc->stopped)
816 return;
817 softc->link_active = 0;
818 softc->stopped = 1;
819
820 for (index = 0; index < scctx->isc_ntxqsets; index++) {
821 enic_stop_wq(enic, index);
822 vnic_wq_clean(&enic->wq[index]);
823 vnic_cq_clean(&enic->cq[enic_cq_rq(enic, index)]);
824 }
825
826 for (index = 0; index < scctx->isc_nrxqsets; index++) {
827 vnic_rq_clean(&enic->rq[index]);
828 vnic_cq_clean(&enic->cq[enic_cq_wq(enic, index)]);
829 }
830
831 for (index = 0; index < scctx->isc_vectors; index++) {
832 vnic_intr_clean(&enic->intr[index]);
833 }
834 }
835
836 static void
enic_init(if_ctx_t ctx)837 enic_init(if_ctx_t ctx)
838 {
839 struct enic_softc *softc;
840 struct enic *enic;
841 if_softc_ctx_t scctx;
842 unsigned int index;
843
844 softc = iflib_get_softc(ctx);
845 scctx = softc->scctx;
846 enic = &softc->enic;
847
848 for (index = 0; index < scctx->isc_ntxqsets; index++)
849 enic_prep_wq_for_simple_tx(&softc->enic, index);
850
851 for (index = 0; index < scctx->isc_ntxqsets; index++)
852 enic_start_wq(enic, index);
853
854 for (index = 0; index < scctx->isc_nrxqsets; index++)
855 enic_start_rq(enic, index);
856
857 /* Use the current MAC address. */
858 bcopy(if_getlladdr(softc->ifp), softc->lladdr, ETHER_ADDR_LEN);
859 enic_set_lladdr(softc);
860
861 ENIC_LOCK(softc);
862 vnic_dev_enable_wait(enic->vdev);
863 ENIC_UNLOCK(softc);
864
865 enic_link_status(softc);
866 }
867
868 static void
enic_del_mcast(struct enic_softc * softc)869 enic_del_mcast(struct enic_softc *softc) {
870 struct enic *enic;
871 int i;
872
873 enic = &softc->enic;
874 for (i=0; i < softc->mc_count; i++) {
875 vnic_dev_del_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]);
876 }
877 softc->multicast = 0;
878 softc->mc_count = 0;
879 }
880
881 static void
enic_add_mcast(struct enic_softc * softc)882 enic_add_mcast(struct enic_softc *softc) {
883 struct enic *enic;
884 int i;
885
886 enic = &softc->enic;
887 for (i=0; i < softc->mc_count; i++) {
888 vnic_dev_add_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]);
889 }
890 softc->multicast = 1;
891 }
892
893 static u_int
enic_copy_maddr(void * arg,struct sockaddr_dl * sdl,u_int idx)894 enic_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx)
895 {
896 uint8_t *mta = arg;
897
898 if (idx == ENIC_MAX_MULTICAST_ADDRESSES)
899 return (0);
900
901 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN);
902 return (1);
903 }
904
905 static void
enic_multi_set(if_ctx_t ctx)906 enic_multi_set(if_ctx_t ctx)
907 {
908 if_t ifp;
909 struct enic_softc *softc;
910 u_int count;
911
912 softc = iflib_get_softc(ctx);
913 ifp = iflib_get_ifp(ctx);
914
915 ENIC_LOCK(softc);
916 enic_del_mcast(softc);
917 count = if_foreach_llmaddr(ifp, enic_copy_maddr, softc->mta);
918 softc->mc_count = count;
919 enic_add_mcast(softc);
920 ENIC_UNLOCK(softc);
921
922 if (if_getflags(ifp) & IFF_PROMISC) {
923 softc->promisc = 1;
924 } else {
925 softc->promisc = 0;
926 }
927 if (if_getflags(ifp) & IFF_ALLMULTI) {
928 softc->allmulti = 1;
929 } else {
930 softc->allmulti = 0;
931 }
932 enic_update_packet_filter(&softc->enic);
933 }
934
935 static int
enic_mtu_set(if_ctx_t ctx,uint32_t mtu)936 enic_mtu_set(if_ctx_t ctx, uint32_t mtu)
937 {
938 struct enic_softc *softc;
939 struct enic *enic;
940 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx);
941
942 softc = iflib_get_softc(ctx);
943 enic = &softc->enic;
944
945 if (mtu > enic->port_mtu){
946 return (EINVAL);
947 }
948
949 enic->config.mtu = mtu;
950 scctx->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
951
952 return (0);
953 }
954
955 static void
enic_media_status(if_ctx_t ctx,struct ifmediareq * ifmr)956 enic_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
957 {
958 struct enic_softc *softc;
959 struct ifmedia_entry *next;
960 uint32_t speed;
961 uint64_t target_baudrate;
962
963 softc = iflib_get_softc(ctx);
964
965 ifmr->ifm_status = IFM_AVALID;
966 ifmr->ifm_active = IFM_ETHER;
967
968 if (enic_link_is_up(softc) != 0) {
969 ENIC_LOCK(softc);
970 speed = vnic_dev_port_speed(&softc->vdev);
971 ENIC_UNLOCK(softc);
972 target_baudrate = 1000ull * speed;
973 LIST_FOREACH(next, &(iflib_get_media(ctx)->ifm_list), ifm_list) {
974 if (ifmedia_baudrate(next->ifm_media) == target_baudrate) {
975 ifmr->ifm_active |= next->ifm_media;
976 }
977 }
978
979 ifmr->ifm_status |= IFM_ACTIVE;
980 ifmr->ifm_active |= IFM_AUTO;
981 } else
982 ifmr->ifm_active |= IFM_NONE;
983 }
984
985 static int
enic_media_change(if_ctx_t ctx)986 enic_media_change(if_ctx_t ctx)
987 {
988 return (ENODEV);
989 }
990
991 static int
enic_promisc_set(if_ctx_t ctx,int flags)992 enic_promisc_set(if_ctx_t ctx, int flags)
993 {
994 if_t ifp;
995 struct enic_softc *softc;
996
997 softc = iflib_get_softc(ctx);
998 ifp = iflib_get_ifp(ctx);
999
1000 if (if_getflags(ifp) & IFF_PROMISC) {
1001 softc->promisc = 1;
1002 } else {
1003 softc->promisc = 0;
1004 }
1005 if (if_getflags(ifp) & IFF_ALLMULTI) {
1006 softc->allmulti = 1;
1007 } else {
1008 softc->allmulti = 0;
1009 }
1010 enic_update_packet_filter(&softc->enic);
1011
1012 return (0);
1013 }
1014
1015 static uint64_t
enic_get_counter(if_ctx_t ctx,ift_counter cnt)1016 enic_get_counter(if_ctx_t ctx, ift_counter cnt) {
1017 if_t ifp = iflib_get_ifp(ctx);
1018
1019 if (cnt < IFCOUNTERS)
1020 return if_get_counter_default(ifp, cnt);
1021
1022 return (0);
1023 }
1024
1025 static void
enic_update_admin_status(if_ctx_t ctx)1026 enic_update_admin_status(if_ctx_t ctx)
1027 {
1028 struct enic_softc *softc;
1029
1030 softc = iflib_get_softc(ctx);
1031
1032 enic_link_status(softc);
1033 }
1034
1035 static void
enic_txq_timer(if_ctx_t ctx,uint16_t qid)1036 enic_txq_timer(if_ctx_t ctx, uint16_t qid)
1037 {
1038
1039 struct enic_softc *softc;
1040 struct enic *enic;
1041 struct vnic_stats *stats;
1042 int ret;
1043
1044 softc = iflib_get_softc(ctx);
1045 enic = &softc->enic;
1046
1047 ENIC_LOCK(softc);
1048 ret = vnic_dev_stats_dump(enic->vdev, &stats);
1049 ENIC_UNLOCK(softc);
1050 if (ret) {
1051 dev_err(enic, "Error in getting stats\n");
1052 }
1053 }
1054
1055 static int
enic_link_is_up(struct enic_softc * softc)1056 enic_link_is_up(struct enic_softc *softc)
1057 {
1058 return (vnic_dev_link_status(&softc->vdev) == 1);
1059 }
1060
1061 static void
enic_link_status(struct enic_softc * softc)1062 enic_link_status(struct enic_softc *softc)
1063 {
1064 if_ctx_t ctx;
1065 uint64_t speed;
1066 int link;
1067
1068 ctx = softc->ctx;
1069 link = enic_link_is_up(softc);
1070 speed = IF_Gbps(10);
1071
1072 ENIC_LOCK(softc);
1073 speed = vnic_dev_port_speed(&softc->vdev);
1074 ENIC_UNLOCK(softc);
1075
1076 if (link != 0 && softc->link_active == 0) {
1077 softc->link_active = 1;
1078 iflib_link_state_change(ctx, LINK_STATE_UP, speed);
1079 } else if (link == 0 && softc->link_active != 0) {
1080 softc->link_active = 0;
1081 iflib_link_state_change(ctx, LINK_STATE_DOWN, speed);
1082 }
1083 }
1084
1085 static void
enic_set_lladdr(struct enic_softc * softc)1086 enic_set_lladdr(struct enic_softc *softc)
1087 {
1088 struct enic *enic;
1089 enic = &softc->enic;
1090
1091 ENIC_LOCK(softc);
1092 vnic_dev_add_addr(enic->vdev, softc->lladdr);
1093 ENIC_UNLOCK(softc);
1094 }
1095
1096
1097 static void
enic_setup_txq_sysctl(struct vnic_wq * wq,int i,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)1098 enic_setup_txq_sysctl(struct vnic_wq *wq, int i, struct sysctl_ctx_list *ctx,
1099 struct sysctl_oid_list *child)
1100 {
1101 struct sysctl_oid *txsnode;
1102 struct sysctl_oid_list *txslist;
1103 struct vnic_stats *stats = wq[i].vdev->stats;
1104
1105 txsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats",
1106 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
1107 txslist = SYSCTL_CHILDREN(txsnode);
1108
1109 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_frames_ok", CTLFLAG_RD,
1110 &stats->tx.tx_frames_ok, "TX Frames OK");
1111 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_frames_ok", CTLFLAG_RD,
1112 &stats->tx.tx_unicast_frames_ok, "TX unicast frames OK");
1113 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_frames_ok", CTLFLAG_RD,
1114 &stats->tx.tx_multicast_frames_ok, "TX multicast framse OK");
1115 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_frames_ok", CTLFLAG_RD,
1116 &stats->tx.tx_broadcast_frames_ok, "TX Broadcast frames OK");
1117 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_bytes_ok", CTLFLAG_RD,
1118 &stats->tx.tx_bytes_ok, "TX bytes OK ");
1119 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_bytes_ok", CTLFLAG_RD,
1120 &stats->tx.tx_unicast_bytes_ok, "TX unicast bytes OK");
1121 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_bytes_ok", CTLFLAG_RD,
1122 &stats->tx.tx_multicast_bytes_ok, "TX multicast bytes OK");
1123 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_bytes_ok", CTLFLAG_RD,
1124 &stats->tx.tx_broadcast_bytes_ok, "TX broadcast bytes OK");
1125 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_drops", CTLFLAG_RD,
1126 &stats->tx.tx_drops, "TX drops");
1127 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_errors", CTLFLAG_RD,
1128 &stats->tx.tx_errors, "TX errors");
1129 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_tso", CTLFLAG_RD,
1130 &stats->tx.tx_tso, "TX TSO");
1131 }
1132
1133 static void
enic_setup_rxq_sysctl(struct vnic_rq * rq,int i,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)1134 enic_setup_rxq_sysctl(struct vnic_rq *rq, int i, struct sysctl_ctx_list *ctx,
1135 struct sysctl_oid_list *child)
1136 {
1137 struct sysctl_oid *rxsnode;
1138 struct sysctl_oid_list *rxslist;
1139 struct vnic_stats *stats = rq[i].vdev->stats;
1140
1141 rxsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats",
1142 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
1143 rxslist = SYSCTL_CHILDREN(rxsnode);
1144
1145 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_ok", CTLFLAG_RD,
1146 &stats->rx.rx_frames_ok, "RX Frames OK");
1147 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_total", CTLFLAG_RD,
1148 &stats->rx.rx_frames_total, "RX frames total");
1149 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_frames_ok", CTLFLAG_RD,
1150 &stats->rx.rx_unicast_frames_ok, "RX unicast frames ok");
1151 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_frames_ok", CTLFLAG_RD,
1152 &stats->rx.rx_multicast_frames_ok, "RX multicast Frames ok");
1153 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_frames_ok", CTLFLAG_RD,
1154 &stats->rx.rx_broadcast_frames_ok, "RX broadcast frames ok");
1155 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_bytes_ok", CTLFLAG_RD,
1156 &stats->rx.rx_bytes_ok, "RX bytes ok");
1157 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_bytes_ok", CTLFLAG_RD,
1158 &stats->rx.rx_unicast_bytes_ok, "RX unicast bytes ok");
1159 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_bytes_ok", CTLFLAG_RD,
1160 &stats->rx.rx_multicast_bytes_ok, "RX multicast bytes ok");
1161 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_bytes_ok", CTLFLAG_RD,
1162 &stats->rx.rx_broadcast_bytes_ok, "RX broadcast bytes ok");
1163 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_drop", CTLFLAG_RD,
1164 &stats->rx.rx_drop, "RX drop");
1165 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_errors", CTLFLAG_RD,
1166 &stats->rx.rx_errors, "RX errors");
1167 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_rss", CTLFLAG_RD,
1168 &stats->rx.rx_rss, "RX rss");
1169 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_crc_errors", CTLFLAG_RD,
1170 &stats->rx.rx_crc_errors, "RX crc errors");
1171 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_64", CTLFLAG_RD,
1172 &stats->rx.rx_frames_64, "RX frames 64");
1173 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_127", CTLFLAG_RD,
1174 &stats->rx.rx_frames_127, "RX frames 127");
1175 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_255", CTLFLAG_RD,
1176 &stats->rx.rx_frames_255, "RX frames 255");
1177 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_511", CTLFLAG_RD,
1178 &stats->rx.rx_frames_511, "RX frames 511");
1179 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1023", CTLFLAG_RD,
1180 &stats->rx.rx_frames_1023, "RX frames 1023");
1181 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1518", CTLFLAG_RD,
1182 &stats->rx.rx_frames_1518, "RX frames 1518");
1183 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_to_max", CTLFLAG_RD,
1184 &stats->rx.rx_frames_to_max, "RX frames to max");
1185 }
1186
1187 static void
enic_setup_queue_sysctl(struct enic_softc * softc,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)1188 enic_setup_queue_sysctl(struct enic_softc *softc, struct sysctl_ctx_list *ctx,
1189 struct sysctl_oid_list *child)
1190 {
1191 enic_setup_txq_sysctl(softc->enic.wq, 0, ctx, child);
1192 enic_setup_rxq_sysctl(softc->enic.rq, 0, ctx, child);
1193 }
1194
1195 static void
enic_setup_sysctl(struct enic_softc * softc)1196 enic_setup_sysctl(struct enic_softc *softc)
1197 {
1198 device_t dev;
1199 struct sysctl_ctx_list *ctx;
1200 struct sysctl_oid *tree;
1201 struct sysctl_oid_list *child;
1202
1203 dev = softc->dev;
1204 ctx = device_get_sysctl_ctx(dev);
1205 tree = device_get_sysctl_tree(dev);
1206 child = SYSCTL_CHILDREN(tree);
1207
1208 enic_setup_queue_sysctl(softc, ctx, child);
1209 }
1210
1211 static void
enic_enable_intr(struct enic_softc * softc,int irq)1212 enic_enable_intr(struct enic_softc *softc, int irq)
1213 {
1214 struct enic *enic = &softc->enic;
1215
1216 vnic_intr_unmask(&enic->intr[irq]);
1217 vnic_intr_return_all_credits(&enic->intr[irq]);
1218 }
1219
1220 static void
enic_disable_intr(struct enic_softc * softc,int irq)1221 enic_disable_intr(struct enic_softc *softc, int irq)
1222 {
1223 struct enic *enic = &softc->enic;
1224
1225 vnic_intr_mask(&enic->intr[irq]);
1226 vnic_intr_masked(&enic->intr[irq]); /* flush write */
1227 }
1228
1229 static int
enic_tx_queue_intr_enable(if_ctx_t ctx,uint16_t qid)1230 enic_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
1231 {
1232 struct enic_softc *softc;
1233 if_softc_ctx_t scctx;
1234
1235 softc = iflib_get_softc(ctx);
1236 scctx = softc->scctx;
1237
1238 enic_enable_intr(softc, qid + scctx->isc_nrxqsets);
1239
1240 return 0;
1241 }
1242
1243 static int
enic_rx_queue_intr_enable(if_ctx_t ctx,uint16_t qid)1244 enic_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
1245 {
1246 struct enic_softc *softc;
1247
1248 softc = iflib_get_softc(ctx);
1249 enic_enable_intr(softc, qid);
1250
1251 return 0;
1252 }
1253
1254 static void
enic_intr_enable_all(if_ctx_t ctx)1255 enic_intr_enable_all(if_ctx_t ctx)
1256 {
1257 struct enic_softc *softc;
1258 if_softc_ctx_t scctx;
1259 int i;
1260
1261 softc = iflib_get_softc(ctx);
1262 scctx = softc->scctx;
1263
1264 for (i = 0; i < scctx->isc_vectors; i++) {
1265 enic_enable_intr(softc, i);
1266 }
1267 }
1268
1269 static void
enic_intr_disable_all(if_ctx_t ctx)1270 enic_intr_disable_all(if_ctx_t ctx)
1271 {
1272 struct enic_softc *softc;
1273 if_softc_ctx_t scctx;
1274 int i;
1275
1276 softc = iflib_get_softc(ctx);
1277 scctx = softc->scctx;
1278 /*
1279 * iflib may invoke this routine before enic_attach_post() has run,
1280 * which is before the top level shared data area is initialized and
1281 * the device made aware of it.
1282 */
1283
1284 for (i = 0; i < scctx->isc_vectors; i++) {
1285 enic_disable_intr(softc, i);
1286 }
1287 }
1288
1289 static int
enic_dev_open(struct enic * enic)1290 enic_dev_open(struct enic *enic)
1291 {
1292 int err;
1293 int flags = CMD_OPENF_IG_DESCCACHE;
1294
1295 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1296 vnic_dev_open_done, flags);
1297 if (err)
1298 dev_err(enic_get_dev(enic),
1299 "vNIC device open failed, err %d\n", err);
1300
1301 return err;
1302 }
1303
1304 static int
enic_dev_init(struct enic * enic)1305 enic_dev_init(struct enic *enic)
1306 {
1307 int err;
1308
1309 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1310
1311 /*
1312 * Get vNIC configuration
1313 */
1314 err = enic_get_vnic_config(enic);
1315 if (err) {
1316 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1317 return err;
1318 }
1319
1320 /* Get available resource counts */
1321 enic_get_res_counts(enic);
1322
1323 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1324 enic->intr_queues = malloc(sizeof(*enic->intr_queues) *
1325 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO);
1326
1327 vnic_dev_set_reset_flag(enic->vdev, 0);
1328 enic->max_flow_counter = -1;
1329
1330 /* set up link status checking */
1331 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1332
1333 enic->overlay_offload = false;
1334 if (enic->disable_overlay && enic->vxlan) {
1335 /*
1336 * Explicitly disable overlay offload as the setting is
1337 * sticky, and resetting vNIC does not disable it.
1338 */
1339 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1340 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_DISABLE)) {
1341 dev_err(enic, "failed to disable overlay offload\n");
1342 } else {
1343 dev_info(enic, "Overlay offload is disabled\n");
1344 }
1345 }
1346 if (!enic->disable_overlay && enic->vxlan &&
1347 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1348 vnic_dev_overlay_offload_ctrl(enic->vdev,
1349 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_ENABLE) == 0) {
1350 enic->overlay_offload = true;
1351 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1352 dev_info(enic, "Overlay offload is enabled\n");
1353 /*
1354 * Reset the vxlan port to the default, as the NIC firmware
1355 * does not reset it automatically and keeps the old setting.
1356 */
1357 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1358 OVERLAY_CFG_VXLAN_PORT_UPDATE, ENIC_DEFAULT_VXLAN_PORT)) {
1359 dev_err(enic, "failed to update vxlan port\n");
1360 return -EINVAL;
1361 }
1362 }
1363 return 0;
1364 }
1365
1366 static void *
enic_alloc_consistent(void * priv,size_t size,bus_addr_t * dma_handle,struct iflib_dma_info * res,u8 * name)1367 enic_alloc_consistent(void *priv, size_t size, bus_addr_t * dma_handle,
1368 struct iflib_dma_info *res, u8 * name)
1369 {
1370 void *vaddr;
1371 *dma_handle = 0;
1372 struct enic *enic = (struct enic *)priv;
1373 int rz;
1374
1375 rz = iflib_dma_alloc(enic->softc->ctx, size, res, BUS_DMA_NOWAIT);
1376 if (rz) {
1377 pr_err("%s : Failed to allocate memory requested for %s\n",
1378 __func__, name);
1379 return NULL;
1380 }
1381
1382 vaddr = res->idi_vaddr;
1383 *dma_handle = res->idi_paddr;
1384
1385 return vaddr;
1386 }
1387
1388 static void
enic_free_consistent(void * priv,size_t size,void * vaddr,bus_addr_t dma_handle,struct iflib_dma_info * res)1389 enic_free_consistent(void *priv, size_t size, void *vaddr,
1390 bus_addr_t dma_handle, struct iflib_dma_info *res)
1391 {
1392 iflib_dma_free(res);
1393 }
1394
1395 static int
enic_pci_mapping(struct enic_softc * softc)1396 enic_pci_mapping(struct enic_softc *softc)
1397 {
1398 int rc;
1399
1400 rc = enic_map_bar(softc, &softc->mem, 0, true);
1401 if (rc)
1402 return rc;
1403
1404 rc = enic_map_bar(softc, &softc->io, 2, false);
1405
1406 return rc;
1407 }
1408
1409 static void
enic_pci_mapping_free(struct enic_softc * softc)1410 enic_pci_mapping_free(struct enic_softc *softc)
1411 {
1412 if (softc->mem.res != NULL)
1413 bus_release_resource(softc->dev, SYS_RES_MEMORY,
1414 softc->mem.rid, softc->mem.res);
1415 softc->mem.res = NULL;
1416
1417 if (softc->io.res != NULL)
1418 bus_release_resource(softc->dev, SYS_RES_MEMORY,
1419 softc->io.rid, softc->io.res);
1420 softc->io.res = NULL;
1421 }
1422
1423 static int
enic_dev_wait(struct vnic_dev * vdev,int (* start)(struct vnic_dev *,int),int (* finished)(struct vnic_dev *,int *),int arg)1424 enic_dev_wait(struct vnic_dev *vdev, int (*start) (struct vnic_dev *, int),
1425 int (*finished) (struct vnic_dev *, int *), int arg)
1426 {
1427 int done;
1428 int err;
1429 int i;
1430
1431 err = start(vdev, arg);
1432 if (err)
1433 return err;
1434
1435 /* Wait for func to complete...2 seconds max */
1436 for (i = 0; i < 2000; i++) {
1437 err = finished(vdev, &done);
1438 if (err)
1439 return err;
1440 if (done)
1441 return 0;
1442 usleep(1000);
1443 }
1444 return -ETIMEDOUT;
1445 }
1446
1447 static int
enic_map_bar(struct enic_softc * softc,struct enic_bar_info * bar,int bar_num,bool shareable)1448 enic_map_bar(struct enic_softc *softc, struct enic_bar_info *bar, int bar_num,
1449 bool shareable)
1450 {
1451 uint32_t flag;
1452
1453 if (bar->res != NULL) {
1454 device_printf(softc->dev, "Bar %d already mapped\n", bar_num);
1455 return EDOOFUS;
1456 }
1457
1458 bar->rid = PCIR_BAR(bar_num);
1459 flag = RF_ACTIVE;
1460 if (shareable)
1461 flag |= RF_SHAREABLE;
1462
1463 if ((bar->res = bus_alloc_resource_any(softc->dev,
1464 SYS_RES_MEMORY, &bar->rid, flag)) == NULL) {
1465 device_printf(softc->dev,
1466 "PCI BAR%d mapping failure\n", bar_num);
1467 return (ENXIO);
1468 }
1469 bar->tag = rman_get_bustag(bar->res);
1470 bar->handle = rman_get_bushandle(bar->res);
1471 bar->size = rman_get_size(bar->res);
1472
1473 return 0;
1474 }
1475
1476 void
enic_init_vnic_resources(struct enic * enic)1477 enic_init_vnic_resources(struct enic *enic)
1478 {
1479 unsigned int error_interrupt_enable = 1;
1480 unsigned int error_interrupt_offset = 0;
1481 unsigned int rxq_interrupt_enable = 0;
1482 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
1483 unsigned int txq_interrupt_enable = 0;
1484 unsigned int txq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
1485 unsigned int index = 0;
1486 unsigned int cq_idx;
1487 if_softc_ctx_t scctx;
1488
1489 scctx = enic->softc->scctx;
1490
1491
1492 rxq_interrupt_enable = 1;
1493 txq_interrupt_enable = 1;
1494
1495 rxq_interrupt_offset = 0;
1496 txq_interrupt_offset = enic->intr_count - 2;
1497 txq_interrupt_offset = 1;
1498
1499 for (index = 0; index < enic->intr_count; index++) {
1500 vnic_intr_alloc(enic->vdev, &enic->intr[index], index);
1501 }
1502
1503 for (index = 0; index < scctx->isc_nrxqsets; index++) {
1504 cq_idx = enic_cq_rq(enic, index);
1505
1506 vnic_rq_clean(&enic->rq[index]);
1507 vnic_rq_init(&enic->rq[index], cq_idx, error_interrupt_enable,
1508 error_interrupt_offset);
1509
1510 vnic_cq_clean(&enic->cq[cq_idx]);
1511 vnic_cq_init(&enic->cq[cq_idx],
1512 0 /* flow_control_enable */ ,
1513 1 /* color_enable */ ,
1514 0 /* cq_head */ ,
1515 0 /* cq_tail */ ,
1516 1 /* cq_tail_color */ ,
1517 rxq_interrupt_enable,
1518 1 /* cq_entry_enable */ ,
1519 0 /* cq_message_enable */ ,
1520 rxq_interrupt_offset,
1521 0 /* cq_message_addr */ );
1522 if (rxq_interrupt_enable)
1523 rxq_interrupt_offset++;
1524 }
1525
1526 for (index = 0; index < scctx->isc_ntxqsets; index++) {
1527 cq_idx = enic_cq_wq(enic, index);
1528 vnic_wq_clean(&enic->wq[index]);
1529 vnic_wq_init(&enic->wq[index], cq_idx, error_interrupt_enable,
1530 error_interrupt_offset);
1531 /* Compute unsupported ol flags for enic_prep_pkts() */
1532 enic->wq[index].tx_offload_notsup_mask = 0;
1533
1534 vnic_cq_clean(&enic->cq[cq_idx]);
1535 vnic_cq_init(&enic->cq[cq_idx],
1536 0 /* flow_control_enable */ ,
1537 1 /* color_enable */ ,
1538 0 /* cq_head */ ,
1539 0 /* cq_tail */ ,
1540 1 /* cq_tail_color */ ,
1541 txq_interrupt_enable,
1542 1,
1543 0,
1544 txq_interrupt_offset,
1545 0 /* (u64)enic->wq[index].cqmsg_rz->iova */ );
1546
1547 }
1548
1549 for (index = 0; index < enic->intr_count; index++) {
1550 vnic_intr_init(&enic->intr[index], 125,
1551 enic->config.intr_timer_type, /* mask_on_assertion */ 1);
1552 }
1553 }
1554
1555 static void
enic_update_packet_filter(struct enic * enic)1556 enic_update_packet_filter(struct enic *enic)
1557 {
1558 struct enic_softc *softc = enic->softc;
1559
1560 ENIC_LOCK(softc);
1561 vnic_dev_packet_filter(enic->vdev,
1562 softc->directed,
1563 softc->multicast,
1564 softc->broadcast,
1565 softc->promisc,
1566 softc->allmulti);
1567 ENIC_UNLOCK(softc);
1568 }
1569
1570 static bool
enic_if_needs_restart(if_ctx_t ctx __unused,enum iflib_restart_event event)1571 enic_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event)
1572 {
1573 switch (event) {
1574 case IFLIB_RESTART_VLAN_CONFIG:
1575 default:
1576 return (false);
1577 }
1578 }
1579
1580 int
enic_setup_finish(struct enic * enic)1581 enic_setup_finish(struct enic *enic)
1582 {
1583 struct enic_softc *softc = enic->softc;
1584
1585 /* Default conf */
1586 softc->directed = 1;
1587 softc->multicast = 0;
1588 softc->broadcast = 1;
1589 softc->promisc = 0;
1590 softc->allmulti = 1;
1591 enic_update_packet_filter(enic);
1592
1593 return 0;
1594 }
1595