1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 */ 5 6 #include "opt_rss.h" 7 8 #include <sys/param.h> 9 #include <sys/systm.h> 10 #include <sys/kernel.h> 11 #include <sys/endian.h> 12 #include <sys/sockio.h> 13 #include <sys/mbuf.h> 14 #include <sys/malloc.h> 15 #include <sys/module.h> 16 #include <sys/socket.h> 17 #include <sys/sysctl.h> 18 #include <sys/smp.h> 19 #include <vm/vm.h> 20 #include <vm/pmap.h> 21 22 #include <net/ethernet.h> 23 #include <net/if.h> 24 #include <net/if_var.h> 25 #include <net/if_arp.h> 26 #include <net/if_dl.h> 27 #include <net/if_types.h> 28 #include <net/if_media.h> 29 #include <net/if_vlan_var.h> 30 #include <net/iflib.h> 31 #ifdef RSS 32 #include <net/rss_config.h> 33 #endif 34 35 #include <netinet/in_systm.h> 36 #include <netinet/in.h> 37 #include <netinet/ip.h> 38 #include <netinet/ip6.h> 39 #include <netinet6/ip6_var.h> 40 #include <netinet/udp.h> 41 #include <netinet/tcp.h> 42 43 #include <machine/bus.h> 44 #include <machine/resource.h> 45 #include <sys/bus.h> 46 #include <sys/rman.h> 47 48 #include <dev/pci/pcireg.h> 49 #include <dev/pci/pcivar.h> 50 51 #include "ifdi_if.h" 52 #include "enic.h" 53 54 #include "opt_inet.h" 55 #include "opt_inet6.h" 56 57 static SYSCTL_NODE(_hw, OID_AUTO, enic, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 58 "ENIC"); 59 60 static const pci_vendor_info_t enic_vendor_info_array[] = 61 { 62 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET, 63 DRV_DESCRIPTION), 64 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET_VF, 65 DRV_DESCRIPTION " VF"), 66 /* required last entry */ 67 68 PVID_END 69 }; 70 71 static void *enic_register(device_t); 72 static int enic_attach_pre(if_ctx_t); 73 static int enic_msix_intr_assign(if_ctx_t, int); 74 75 static int enic_attach_post(if_ctx_t); 76 static int enic_detach(if_ctx_t); 77 78 static int enic_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 79 static int enic_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 80 static void enic_queues_free(if_ctx_t); 81 static int enic_rxq_intr(void *); 82 static int enic_event_intr(void *); 83 static int enic_err_intr(void *); 84 static void enic_stop(if_ctx_t); 85 static void enic_init(if_ctx_t); 86 static void enic_multi_set(if_ctx_t); 87 static int enic_mtu_set(if_ctx_t, uint32_t); 88 static void enic_media_status(if_ctx_t, struct ifmediareq *); 89 static int enic_media_change(if_ctx_t); 90 static int enic_promisc_set(if_ctx_t, int); 91 static uint64_t enic_get_counter(if_ctx_t, ift_counter); 92 static void enic_update_admin_status(if_ctx_t); 93 static void enic_txq_timer(if_ctx_t, uint16_t); 94 static int enic_link_is_up(struct enic_softc *); 95 static void enic_link_status(struct enic_softc *); 96 static void enic_set_lladdr(struct enic_softc *); 97 static void enic_setup_txq_sysctl(struct vnic_wq *, int, struct sysctl_ctx_list *, 98 struct sysctl_oid_list *); 99 static void enic_setup_rxq_sysctl(struct vnic_rq *, int, struct sysctl_ctx_list *, 100 struct sysctl_oid_list *); 101 static void enic_setup_sysctl(struct enic_softc *); 102 static int enic_tx_queue_intr_enable(if_ctx_t, uint16_t); 103 static int enic_rx_queue_intr_enable(if_ctx_t, uint16_t); 104 static void enic_enable_intr(struct enic_softc *, int); 105 static void enic_disable_intr(struct enic_softc *, int); 106 static void enic_intr_enable_all(if_ctx_t); 107 static void enic_intr_disable_all(if_ctx_t); 108 static int enic_dev_open(struct enic *); 109 static int enic_dev_init(struct enic *); 110 static void *enic_alloc_consistent(void *, size_t, bus_addr_t *, 111 struct iflib_dma_info *, u8 *); 112 static void enic_free_consistent(void *, size_t, void *, bus_addr_t, 113 struct iflib_dma_info *); 114 static int enic_pci_mapping(struct enic_softc *); 115 static void enic_pci_mapping_free(struct enic_softc *); 116 static int enic_dev_wait(struct vnic_dev *, int (*) (struct vnic_dev *, int), 117 int (*) (struct vnic_dev *, int *), int arg); 118 static int enic_map_bar(struct enic_softc *, struct enic_bar_info *, int, bool); 119 static void enic_update_packet_filter(struct enic *enic); 120 static bool enic_if_needs_restart(if_ctx_t, enum iflib_restart_event); 121 122 typedef enum { 123 ENIC_BARRIER_RD, 124 ENIC_BARRIER_WR, 125 ENIC_BARRIER_RDWR, 126 } enic_barrier_t; 127 128 static device_method_t enic_methods[] = { 129 /* Device interface */ 130 DEVMETHOD(device_register, enic_register), 131 DEVMETHOD(device_probe, iflib_device_probe), 132 DEVMETHOD(device_attach, iflib_device_attach), 133 DEVMETHOD(device_detach, iflib_device_detach), 134 DEVMETHOD(device_shutdown, iflib_device_shutdown), 135 DEVMETHOD(device_suspend, iflib_device_suspend), 136 DEVMETHOD(device_resume, iflib_device_resume), 137 DEVMETHOD_END 138 }; 139 140 static driver_t enic_driver = { 141 "enic", enic_methods, sizeof(struct enic_softc) 142 }; 143 144 DRIVER_MODULE(enic, pci, enic_driver, 0, 0); 145 IFLIB_PNP_INFO(pci, enic, enic_vendor_info_array); 146 MODULE_VERSION(enic, 2); 147 148 MODULE_DEPEND(enic, pci, 1, 1, 1); 149 MODULE_DEPEND(enic, ether, 1, 1, 1); 150 MODULE_DEPEND(enic, iflib, 1, 1, 1); 151 152 static device_method_t enic_iflib_methods[] = { 153 DEVMETHOD(ifdi_tx_queues_alloc, enic_tx_queues_alloc), 154 DEVMETHOD(ifdi_rx_queues_alloc, enic_rx_queues_alloc), 155 DEVMETHOD(ifdi_queues_free, enic_queues_free), 156 157 DEVMETHOD(ifdi_attach_pre, enic_attach_pre), 158 DEVMETHOD(ifdi_attach_post, enic_attach_post), 159 DEVMETHOD(ifdi_detach, enic_detach), 160 161 DEVMETHOD(ifdi_init, enic_init), 162 DEVMETHOD(ifdi_stop, enic_stop), 163 DEVMETHOD(ifdi_multi_set, enic_multi_set), 164 DEVMETHOD(ifdi_mtu_set, enic_mtu_set), 165 DEVMETHOD(ifdi_media_status, enic_media_status), 166 DEVMETHOD(ifdi_media_change, enic_media_change), 167 DEVMETHOD(ifdi_promisc_set, enic_promisc_set), 168 DEVMETHOD(ifdi_get_counter, enic_get_counter), 169 DEVMETHOD(ifdi_update_admin_status, enic_update_admin_status), 170 DEVMETHOD(ifdi_timer, enic_txq_timer), 171 172 DEVMETHOD(ifdi_tx_queue_intr_enable, enic_tx_queue_intr_enable), 173 DEVMETHOD(ifdi_rx_queue_intr_enable, enic_rx_queue_intr_enable), 174 DEVMETHOD(ifdi_intr_enable, enic_intr_enable_all), 175 DEVMETHOD(ifdi_intr_disable, enic_intr_disable_all), 176 DEVMETHOD(ifdi_msix_intr_assign, enic_msix_intr_assign), 177 178 DEVMETHOD(ifdi_needs_restart, enic_if_needs_restart), 179 180 DEVMETHOD_END 181 }; 182 183 static driver_t enic_iflib_driver = { 184 "enic", enic_iflib_methods, sizeof(struct enic_softc) 185 }; 186 187 extern struct if_txrx enic_txrx; 188 189 static struct if_shared_ctx enic_sctx_init = { 190 .isc_magic = IFLIB_MAGIC, 191 .isc_q_align = 512, 192 193 .isc_tx_maxsize = ENIC_TX_MAX_PKT_SIZE, 194 .isc_tx_maxsegsize = PAGE_SIZE, 195 196 /* 197 * These values are used to configure the busdma tag used for receive 198 * descriptors. Each receive descriptor only points to one buffer. 199 */ 200 .isc_rx_maxsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, /* One buf per 201 * descriptor */ 202 .isc_rx_nsegments = 1, /* One mapping per descriptor */ 203 .isc_rx_maxsegsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, 204 .isc_admin_intrcnt = 2, 205 .isc_vendor_info = enic_vendor_info_array, 206 .isc_driver_version = "1", 207 .isc_driver = &enic_iflib_driver, 208 .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ | IFLIB_SKIP_MSIX, 209 210 /* 211 * Number of receive queues per receive queue set, with associated 212 * descriptor settings for each. 213 */ 214 215 .isc_nrxqs = 2, 216 .isc_nfl = 1, /* one free list for each receive command 217 * queue */ 218 .isc_nrxd_min = {16, 16}, 219 .isc_nrxd_max = {2048, 2048}, 220 .isc_nrxd_default = {64, 64}, 221 222 /* 223 * Number of transmit queues per transmit queue set, with associated 224 * descriptor settings for each. 225 */ 226 .isc_ntxqs = 2, 227 .isc_ntxd_min = {16, 16}, 228 .isc_ntxd_max = {2048, 2048}, 229 .isc_ntxd_default = {64, 64}, 230 }; 231 232 static void * 233 enic_register(device_t dev) 234 { 235 return (&enic_sctx_init); 236 } 237 238 static int 239 enic_allocate_msix(struct enic_softc *softc) { 240 if_ctx_t ctx; 241 if_softc_ctx_t scctx; 242 if_shared_ctx_t sctx; 243 device_t dev; 244 cpuset_t cpus; 245 int queues, vectors, requested; 246 int err = 0; 247 248 dev = softc->dev; 249 ctx = softc->ctx; 250 scctx = softc->scctx; 251 sctx = iflib_get_sctx(ctx); 252 253 if (bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus) != 0) { 254 device_printf(dev, "Unable to fetch CPU list\n"); 255 CPU_COPY(&all_cpus, &cpus); 256 } 257 258 259 queues = CPU_COUNT(&cpus); 260 queues = imin(queues, scctx->isc_nrxqsets); 261 queues = imin(queues, scctx->isc_ntxqsets); 262 requested = queues * 2 + sctx->isc_admin_intrcnt; 263 scctx->isc_nrxqsets = queues; 264 scctx->isc_ntxqsets = queues; 265 266 vectors = requested; 267 if ((err = pci_alloc_msix(dev, &vectors)) != 0) { 268 device_printf(dev, 269 "failed to allocate %d MSI-X vectors, err: %d\n", requested, 270 err); 271 err = 1; 272 goto enic_allocate_msix_out; 273 } else { 274 if (vectors != requested) { 275 device_printf(dev, 276 "Unable to allocate sufficient MSI-X vectors " 277 "(got %d, need %d)\n", requested, vectors); 278 pci_release_msi(dev); 279 err = 1; 280 goto enic_allocate_msix_out; 281 } 282 } 283 284 device_printf(dev, "Using MSI-X interrupts with %d vectors\n", 285 vectors); 286 287 scctx->isc_intr = IFLIB_INTR_MSIX; 288 scctx->isc_vectors = vectors; 289 290 enic_allocate_msix_out: 291 return (err); 292 293 } 294 295 static struct enic_intr_mod_range mod_range[ENIC_MAX_LINK_SPEEDS] = { 296 {0, 0}, /* 0 - 4 Gbps */ 297 {0, 3}, /* 4 - 10 Gbps */ 298 {3, 6}, /* 10 - 40 Gbps */ 299 }; 300 301 static void enic_set_rx_coal_setting(struct enic *enic) 302 { 303 unsigned int speed; 304 int index = -1; 305 struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting; 306 307 /* 1. Read the link speed from fw 308 * 2. Pick the default range for the speed 309 * 3. Update it in enic->rx_coalesce_setting 310 */ 311 speed = vnic_dev_port_speed(enic->vdev); 312 if (ENIC_LINK_SPEED_10G < speed) 313 index = ENIC_LINK_40G_INDEX; 314 else if (ENIC_LINK_SPEED_4G < speed) 315 index = ENIC_LINK_10G_INDEX; 316 else 317 index = ENIC_LINK_4G_INDEX; 318 319 rx_coal->small_pkt_range_start = mod_range[index].small_pkt_range_start; 320 rx_coal->large_pkt_range_start = mod_range[index].large_pkt_range_start; 321 rx_coal->range_end = ENIC_RX_COALESCE_RANGE_END; 322 323 /* Start with the value provided by UCSM */ 324 for (index = 0; index < enic->rq_count; index++) 325 enic->cq[index].cur_rx_coal_timeval = 326 enic->config.intr_timer_usec; 327 328 rx_coal->use_adaptive_rx_coalesce = 1; 329 } 330 331 static int 332 enic_attach_pre(if_ctx_t ctx) 333 { 334 if_softc_ctx_t scctx; 335 struct enic_softc *softc; 336 struct vnic_dev *vdev; 337 struct enic *enic; 338 device_t dev; 339 340 int err = -1; 341 int rc = 0; 342 int i; 343 u64 a0 = 0, a1 = 0; 344 int wait = 1000; 345 struct vnic_stats *stats; 346 int ret; 347 348 dev = iflib_get_dev(ctx); 349 softc = iflib_get_softc(ctx); 350 softc->dev = dev; 351 softc->ctx = ctx; 352 softc->sctx = iflib_get_sctx(ctx); 353 softc->scctx = iflib_get_softc_ctx(ctx); 354 softc->ifp = iflib_get_ifp(ctx); 355 softc->media = iflib_get_media(ctx); 356 softc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 357 ENIC_MAX_MULTICAST_ADDRESSES, M_DEVBUF, 358 M_NOWAIT | M_ZERO); 359 if (softc->mta == NULL) 360 return (ENOMEM); 361 scctx = softc->scctx; 362 363 mtx_init(&softc->enic_lock, "ENIC Lock", NULL, MTX_DEF); 364 365 pci_enable_busmaster(softc->dev); 366 if (enic_pci_mapping(softc)) 367 return (ENXIO); 368 369 enic = &softc->enic; 370 enic->softc = softc; 371 vdev = &softc->vdev; 372 vdev->softc = softc; 373 enic->vdev = vdev; 374 vdev->priv = enic; 375 376 ENIC_LOCK(softc); 377 vnic_dev_register(vdev, &softc->mem, 1); 378 enic->vdev = vdev; 379 vnic_dev_cmd_init(enic->vdev); 380 381 vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0); 382 383 vnic_dev_cmd(vdev, CMD_INIT_v1, &a0, &a1, wait); 384 vnic_dev_cmd(vdev, CMD_GET_MAC_ADDR, &a0, &a1, wait); 385 386 bcopy((u_int8_t *) & a0, softc->mac_addr, ETHER_ADDR_LEN); 387 iflib_set_mac(ctx, softc->mac_addr); 388 389 vnic_register_cbacks(enic->vdev, enic_alloc_consistent, 390 enic_free_consistent); 391 392 /* 393 * Allocate the consistent memory for stats and counters upfront so 394 * both primary and secondary processes can access them. 395 */ 396 ENIC_UNLOCK(softc); 397 err = vnic_dev_alloc_stats_mem(enic->vdev); 398 ENIC_LOCK(softc); 399 if (err) { 400 dev_err(enic, "Failed to allocate cmd memory, aborting\n"); 401 goto err_out_unregister; 402 } 403 vnic_dev_stats_clear(enic->vdev); 404 ret = vnic_dev_stats_dump(enic->vdev, &stats); 405 if (ret) { 406 dev_err(enic, "Error in getting stats\n"); 407 goto err_out_unregister; 408 } 409 err = vnic_dev_alloc_counter_mem(enic->vdev); 410 if (err) { 411 dev_err(enic, "Failed to allocate counter memory, aborting\n"); 412 goto err_out_unregister; 413 } 414 415 /* Issue device open to get device in known state */ 416 err = enic_dev_open(enic); 417 if (err) { 418 dev_err(enic, "vNIC dev open failed, aborting\n"); 419 goto err_out_unregister; 420 } 421 422 /* Set ingress vlan rewrite mode before vnic initialization */ 423 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN; 424 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN; 425 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, 426 enic->ig_vlan_rewrite_mode); 427 if (err) { 428 dev_err(enic, 429 "Failed to set ingress vlan rewrite mode, aborting.\n"); 430 goto err_out_dev_close; 431 } 432 433 /* 434 * Issue device init to initialize the vnic-to-switch link. We'll 435 * start with carrier off and wait for link UP notification later to 436 * turn on carrier. We don't need to wait here for the 437 * vnic-to-switch link initialization to complete; link UP 438 * notification is the indication that the process is complete. 439 */ 440 441 err = vnic_dev_init(enic->vdev, 0); 442 if (err) { 443 dev_err(enic, "vNIC dev init failed, aborting\n"); 444 goto err_out_dev_close; 445 } 446 447 err = enic_dev_init(enic); 448 if (err) { 449 dev_err(enic, "Device initialization failed, aborting\n"); 450 goto err_out_dev_close; 451 } 452 /* Queue DMA has not been enabled; the first iflib stop may be skipped. */ 453 softc->stopped = 1; 454 ENIC_UNLOCK(softc); 455 456 enic->port_mtu = vnic_dev_mtu(enic->vdev); 457 458 softc->scctx = iflib_get_softc_ctx(ctx); 459 scctx = softc->scctx; 460 scctx->isc_txrx = &enic_txrx; 461 scctx->isc_capabilities = scctx->isc_capenable = \ 462 IFCAP_HWCSUM; 463 scctx->isc_tx_csum_flags = 0; 464 if_setmtu(softc->ifp, enic->config.mtu); 465 scctx->isc_max_frame_size = enic->config.mtu + ETHER_HDR_LEN + \ 466 ETHER_CRC_LEN; 467 scctx->isc_nrxqsets_max = enic->conf_rq_count; 468 scctx->isc_ntxqsets_max = enic->conf_wq_count; 469 scctx->isc_nrxqsets = enic->conf_rq_count; 470 scctx->isc_ntxqsets = enic->conf_wq_count; 471 for (i = 0; i < enic->conf_wq_count; i++) { 472 scctx->isc_ntxd[i] = enic->config.wq_desc_count; 473 scctx->isc_txqsizes[i] = sizeof(struct cq_enet_wq_desc) 474 * scctx->isc_ntxd[i]; 475 scctx->isc_ntxd[i + enic->conf_wq_count] = 476 enic->config.wq_desc_count; 477 scctx->isc_txqsizes[i + enic->conf_wq_count] = 478 sizeof(struct cq_desc) * scctx->isc_ntxd[i + 479 enic->conf_wq_count]; 480 } 481 for (i = 0; i < enic->conf_rq_count; i++) { 482 scctx->isc_nrxd[i] = enic->config.rq_desc_count; 483 scctx->isc_rxqsizes[i] = sizeof(struct cq_enet_rq_desc) * 484 scctx->isc_nrxd[i]; 485 scctx->isc_nrxd[i + enic->conf_rq_count] = 486 enic->config.rq_desc_count; 487 scctx->isc_rxqsizes[i + enic->conf_rq_count] = sizeof(struct 488 cq_desc) * scctx->isc_nrxd[i + enic->conf_rq_count]; 489 } 490 scctx->isc_tx_nsegments = 31; 491 492 scctx->isc_msix_bar = -1; 493 494 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 495 ifmedia_add(softc->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); 496 ifmedia_add(softc->media, IFM_ETHER | IFM_10_FL, 0, NULL); 497 498 err = enic_allocate_msix(softc); 499 if (err) { 500 dev_err(enic, "Failed to allocate MSIX, aborting\n"); 501 goto err_out_dev_close; 502 } 503 504 return (rc); 505 506 err_out_dev_close: 507 vnic_dev_close(enic->vdev); 508 vnic_dev_deinit_devcmd2(enic->vdev); 509 err_out_unregister: 510 free(softc->vdev.devcmd, M_DEVBUF); 511 free(softc->enic.intr_queues, M_DEVBUF); 512 free(softc->enic.cq, M_DEVBUF); 513 free(softc->mta, M_DEVBUF); 514 rc = -1; 515 pci_disable_busmaster(softc->dev); 516 enic_pci_mapping_free(softc); 517 mtx_destroy(&softc->enic_lock); 518 return (rc); 519 } 520 521 static int 522 enic_msix_intr_assign(if_ctx_t ctx, int msix) 523 { 524 struct enic_softc *softc; 525 struct enic *enic; 526 if_softc_ctx_t scctx; 527 528 int error; 529 int i; 530 char irq_name[16]; 531 532 softc = iflib_get_softc(ctx); 533 enic = &softc->enic; 534 scctx = softc->scctx; 535 536 ENIC_LOCK(softc); 537 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSIX); 538 ENIC_UNLOCK(softc); 539 540 if (enic->intr_queues == NULL) 541 enic->intr_queues = malloc(sizeof(*enic->intr_queues) * 542 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO); 543 enic->intr = malloc(sizeof(*enic->intr) * msix, M_DEVBUF, M_NOWAIT 544 | M_ZERO); 545 if (enic->intr_queues == NULL || enic->intr == NULL) 546 return (ENOMEM); 547 for (i = 0; i < scctx->isc_nrxqsets; i++) { 548 snprintf(irq_name, sizeof(irq_name), "erxq%d:%d", i, 549 device_get_unit(softc->dev)); 550 551 error = iflib_irq_alloc_generic(ctx, 552 &enic->intr_queues[i].intr_irq, i + 1, IFLIB_INTR_RX, 553 enic_rxq_intr, &enic->rq[i], i, irq_name); 554 if (error) { 555 device_printf(iflib_get_dev(ctx), 556 "Failed to register rxq %d interrupt handler\n", i); 557 return (error); 558 } 559 enic->intr[i].index = i; 560 enic->intr[i].vdev = enic->vdev; 561 ENIC_LOCK(softc); 562 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, 563 RES_TYPE_INTR_CTRL, i); 564 vnic_intr_mask(&enic->intr[i]); 565 ENIC_UNLOCK(softc); 566 } 567 568 for (i = scctx->isc_nrxqsets; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) { 569 snprintf(irq_name, sizeof(irq_name), "etxq%d:%d", i - 570 scctx->isc_nrxqsets, device_get_unit(softc->dev)); 571 572 iflib_softirq_alloc_generic(ctx, 573 &enic->intr_queues[i].intr_irq, IFLIB_INTR_TX, 574 &enic->wq[i - scctx->isc_nrxqsets], i - scctx->isc_nrxqsets, 575 irq_name); 576 577 enic->intr[i].index = i; 578 enic->intr[i].vdev = enic->vdev; 579 ENIC_LOCK(softc); 580 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, 581 RES_TYPE_INTR_CTRL, i); 582 vnic_intr_mask(&enic->intr[i]); 583 ENIC_UNLOCK(softc); 584 } 585 586 i = scctx->isc_nrxqsets + scctx->isc_ntxqsets; 587 error = iflib_irq_alloc_generic(ctx, &softc->enic_event_intr_irq, 588 i + 1, IFLIB_INTR_ADMIN, enic_event_intr, softc, 0, "event"); 589 if (error) { 590 device_printf(iflib_get_dev(ctx), 591 "Failed to register event interrupt handler\n"); 592 return (error); 593 } 594 595 enic->intr[i].index = i; 596 enic->intr[i].vdev = enic->vdev; 597 ENIC_LOCK(softc); 598 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL, 599 i); 600 vnic_intr_mask(&enic->intr[i]); 601 ENIC_UNLOCK(softc); 602 603 i++; 604 error = iflib_irq_alloc_generic(ctx, &softc->enic_err_intr_irq, 605 i + 1, IFLIB_INTR_ADMIN, enic_err_intr, softc, 0, "err"); 606 if (error) { 607 device_printf(iflib_get_dev(ctx), 608 "Failed to register event interrupt handler\n"); 609 return (error); 610 } 611 enic->intr[i].index = i; 612 enic->intr[i].vdev = enic->vdev; 613 ENIC_LOCK(softc); 614 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL, 615 i); 616 vnic_intr_mask(&enic->intr[i]); 617 ENIC_UNLOCK(softc); 618 619 enic->intr_count = msix; 620 621 return (0); 622 } 623 624 static void 625 enic_free_irqs(struct enic_softc *softc) 626 { 627 if_softc_ctx_t scctx; 628 629 struct enic *enic; 630 int i; 631 632 scctx = softc->scctx; 633 enic = &softc->enic; 634 635 if (enic->intr_queues != NULL) { 636 for (i = 0; 637 i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) 638 iflib_irq_free(softc->ctx, 639 &enic->intr_queues[i].intr_irq); 640 } 641 642 iflib_irq_free(softc->ctx, &softc->enic_event_intr_irq); 643 iflib_irq_free(softc->ctx, &softc->enic_err_intr_irq); 644 free(enic->intr_queues, M_DEVBUF); 645 enic->intr_queues = NULL; 646 free(enic->intr, M_DEVBUF); 647 enic->intr = NULL; 648 } 649 650 static int 651 enic_attach_post(if_ctx_t ctx) 652 { 653 struct enic *enic; 654 struct enic_softc *softc; 655 int error = 0; 656 657 softc = iflib_get_softc(ctx); 658 enic = &softc->enic; 659 660 enic_setup_sysctl(softc); 661 662 enic_init_vnic_resources(enic); 663 enic_set_rx_coal_setting(enic); 664 enic_setup_finish(enic); 665 666 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 667 ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO); 668 669 return (error); 670 } 671 672 static int 673 enic_detach(if_ctx_t ctx) 674 { 675 struct enic_softc *softc; 676 struct enic *enic; 677 678 softc = iflib_get_softc(ctx); 679 enic = &softc->enic; 680 681 vnic_dev_notify_unset(enic->vdev); 682 683 enic_free_irqs(softc); 684 685 ENIC_LOCK(softc); 686 vnic_dev_deinit(enic->vdev); 687 vnic_dev_close(enic->vdev); 688 vnic_dev_deinit_devcmd2(enic->vdev); 689 free(softc->vdev.devcmd, M_DEVBUF); 690 softc->vdev.devcmd = NULL; 691 pci_disable_busmaster(softc->dev); 692 enic_pci_mapping_free(softc); 693 ENIC_UNLOCK(softc); 694 if (softc->vdev.stats_res.idi_size != 0) { 695 iflib_dma_free(&softc->vdev.stats_res); 696 softc->vdev.stats = NULL; 697 } 698 if (softc->vdev.flow_counters_res.idi_size != 0) { 699 iflib_dma_free(&softc->vdev.flow_counters_res); 700 softc->vdev.flow_counters = NULL; 701 } 702 free(softc->mta, M_DEVBUF); 703 softc->mta = NULL; 704 mtx_destroy(&softc->enic_lock); 705 706 return 0; 707 } 708 709 static int 710 enic_tx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs, 711 int ntxqs, int ntxqsets) 712 { 713 struct enic_softc *softc; 714 int q; 715 716 softc = iflib_get_softc(ctx); 717 softc->enic.cq = malloc(sizeof(*softc->enic.cq) * 718 (softc->enic.wq_count + softc->enic.rq_count), M_DEVBUF, 719 M_NOWAIT | M_ZERO); 720 if (softc->enic.cq == NULL) 721 return (ENOMEM); 722 /* Allocate the array of transmit queues */ 723 softc->enic.wq = malloc(sizeof(struct vnic_wq) * 724 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 725 if (softc->enic.wq == NULL) { 726 free(softc->enic.cq, M_DEVBUF); 727 softc->enic.cq = NULL; 728 return (ENOMEM); 729 } 730 731 /* Initialize driver state for each transmit queue */ 732 733 /* 734 * Allocate queue state that is shared with the device. This check 735 * and call is performed in both enic_tx_queues_alloc() and 736 * enic_rx_queues_alloc() so that we don't have to care which order 737 * iflib invokes those routines in. 738 */ 739 740 /* Record descriptor ring vaddrs and paddrs */ 741 ENIC_LOCK(softc); 742 for (q = 0; q < ntxqsets; q++) { 743 struct vnic_wq *wq; 744 struct vnic_cq *cq; 745 unsigned int cq_wq; 746 747 wq = &softc->enic.wq[q]; 748 cq_wq = enic_cq_wq(&softc->enic, q); 749 cq = &softc->enic.cq[cq_wq]; 750 751 /* Completion ring */ 752 wq->vdev = softc->enic.vdev; 753 wq->index = q; 754 wq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_WQ, 755 wq->index); 756 vnic_wq_disable(wq); 757 758 wq->ring.desc_size = sizeof(struct wq_enet_desc); 759 wq->ring.desc_count = softc->scctx->isc_ntxd[q]; 760 wq->ring.desc_avail = wq->ring.desc_count - 1; 761 wq->ring.last_count = wq->ring.desc_count; 762 wq->head_idx = 0; 763 wq->tail_idx = 0; 764 765 wq->ring.descs = vaddrs[q * ntxqs + 0]; 766 wq->ring.base_addr = paddrs[q * ntxqs + 0]; 767 768 /* Command ring */ 769 cq->vdev = softc->enic.vdev; 770 cq->index = cq_wq; 771 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, 772 RES_TYPE_CQ, cq->index); 773 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc); 774 cq->ring.desc_count = softc->scctx->isc_ntxd[q]; 775 cq->ring.desc_avail = cq->ring.desc_count - 1; 776 777 cq->ring.descs = vaddrs[q * ntxqs + 1]; 778 cq->ring.base_addr = paddrs[q * ntxqs + 1]; 779 780 } 781 782 ENIC_UNLOCK(softc); 783 784 return (0); 785 } 786 787 788 789 static int 790 enic_rx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs, 791 int nrxqs, int nrxqsets) 792 { 793 struct enic_softc *softc; 794 int q; 795 796 softc = iflib_get_softc(ctx); 797 /* Allocate the array of receive queues */ 798 softc->enic.rq = malloc(sizeof(struct vnic_rq) * nrxqsets, M_DEVBUF, 799 M_NOWAIT | M_ZERO); 800 if (softc->enic.rq == NULL) { 801 free(softc->enic.wq, M_DEVBUF); 802 softc->enic.wq = NULL; 803 free(softc->enic.cq, M_DEVBUF); 804 softc->enic.cq = NULL; 805 return (ENOMEM); 806 } 807 808 /* Initialize driver state for each receive queue */ 809 810 /* 811 * Allocate queue state that is shared with the device. This check 812 * and call is performed in both enic_tx_queues_alloc() and 813 * enic_rx_queues_alloc() so that we don't have to care which order 814 * iflib invokes those routines in. 815 */ 816 817 /* Record descriptor ring vaddrs and paddrs */ 818 ENIC_LOCK(softc); 819 for (q = 0; q < nrxqsets; q++) { 820 struct vnic_rq *rq; 821 struct vnic_cq *cq; 822 unsigned int cq_rq; 823 824 rq = &softc->enic.rq[q]; 825 cq_rq = enic_cq_rq(&softc->enic, q); 826 cq = &softc->enic.cq[cq_rq]; 827 828 /* Completion ring */ 829 cq->vdev = softc->enic.vdev; 830 cq->index = cq_rq; 831 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_CQ, 832 cq->index); 833 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc); 834 cq->ring.desc_count = softc->scctx->isc_nrxd[1]; 835 cq->ring.desc_avail = cq->ring.desc_count - 1; 836 837 cq->ring.descs = vaddrs[q * nrxqs + 0]; 838 cq->ring.base_addr = paddrs[q * nrxqs + 0]; 839 840 /* Command ring(s) */ 841 rq->vdev = softc->enic.vdev; 842 843 rq->index = q; 844 rq->ctrl = vnic_dev_get_res(softc->enic.vdev, 845 RES_TYPE_RQ, rq->index); 846 vnic_rq_disable(rq); 847 848 rq->ring.desc_size = sizeof(struct rq_enet_desc); 849 rq->ring.desc_count = softc->scctx->isc_nrxd[0]; 850 rq->ring.desc_avail = rq->ring.desc_count - 1; 851 852 rq->ring.descs = vaddrs[q * nrxqs + 1]; 853 rq->ring.base_addr = paddrs[q * nrxqs + 1]; 854 rq->need_initial_post = true; 855 } 856 857 ENIC_UNLOCK(softc); 858 859 return (0); 860 } 861 862 static void 863 enic_queues_free(if_ctx_t ctx) 864 { 865 struct enic_softc *softc; 866 softc = iflib_get_softc(ctx); 867 868 free(softc->enic.rq, M_DEVBUF); 869 softc->enic.rq = NULL; 870 free(softc->enic.wq, M_DEVBUF); 871 softc->enic.wq = NULL; 872 free(softc->enic.cq, M_DEVBUF); 873 softc->enic.cq = NULL; 874 } 875 876 static int 877 enic_rxq_intr(void *rxq) 878 { 879 struct vnic_rq *rq; 880 if_t ifp; 881 882 rq = (struct vnic_rq *)rxq; 883 ifp = iflib_get_ifp(rq->vdev->softc->ctx); 884 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 885 return (FILTER_HANDLED); 886 887 return (FILTER_SCHEDULE_THREAD); 888 } 889 890 static int 891 enic_event_intr(void *vsc) 892 { 893 struct enic_softc *softc; 894 struct enic *enic; 895 uint32_t mtu; 896 897 softc = vsc; 898 enic = &softc->enic; 899 900 mtu = vnic_dev_mtu(enic->vdev); 901 if (mtu && mtu != enic->port_mtu) { 902 enic->port_mtu = mtu; 903 } 904 905 enic_link_status(softc); 906 907 return (FILTER_HANDLED); 908 } 909 910 static int 911 enic_err_intr(void *vsc) 912 { 913 struct enic_softc *softc; 914 915 softc = vsc; 916 917 iflib_request_reset_if_up(softc->ctx); 918 iflib_admin_intr_deferred(softc->ctx); 919 920 return (FILTER_HANDLED); 921 } 922 923 static void 924 enic_stop(if_ctx_t ctx) 925 { 926 struct enic_softc *softc; 927 struct enic *enic; 928 if_softc_ctx_t scctx; 929 unsigned int index; 930 struct vnic_wq *wq; 931 struct vnic_rq *rq; 932 struct vnic_cq *cq; 933 unsigned int cq_wq, cq_rq; 934 935 936 softc = iflib_get_softc(ctx); 937 scctx = softc->scctx; 938 enic = &softc->enic; 939 940 if (softc->stopped) 941 return; 942 softc->link_active = 0; 943 softc->stopped = 1; 944 945 enic_dev_disable(enic); 946 947 for (index = 0; index < scctx->isc_ntxqsets; index++) { 948 enic_stop_wq(enic, index); 949 vnic_wq_clean(&enic->wq[index]); 950 vnic_cq_clean(&enic->cq[enic_cq_rq(enic, index)]); 951 952 wq = &softc->enic.wq[index]; 953 wq->ring.desc_avail = wq->ring.desc_count - 1; 954 wq->ring.last_count = wq->ring.desc_count; 955 wq->head_idx = 0; 956 wq->tail_idx = 0; 957 958 cq_wq = enic_cq_wq(&softc->enic, index); 959 cq = &softc->enic.cq[cq_wq]; 960 cq->ring.desc_avail = cq->ring.desc_count - 1; 961 } 962 963 for (index = 0; index < scctx->isc_nrxqsets; index++) { 964 enic_stop_rq(enic, index); 965 vnic_rq_clean(&enic->rq[index]); 966 vnic_cq_clean(&enic->cq[enic_cq_wq(enic, index)]); 967 968 rq = &softc->enic.rq[index]; 969 cq_rq = enic_cq_rq(&softc->enic, index); 970 cq = &softc->enic.cq[cq_rq]; 971 972 cq->ring.desc_avail = cq->ring.desc_count - 1; 973 rq->ring.desc_avail = rq->ring.desc_count - 1; 974 rq->need_initial_post = true; 975 } 976 977 for (index = 0; index < scctx->isc_vectors; index++) { 978 vnic_intr_clean(&enic->intr[index]); 979 } 980 } 981 982 static void 983 enic_init(if_ctx_t ctx) 984 { 985 struct enic_softc *softc; 986 struct enic *enic; 987 if_softc_ctx_t scctx; 988 unsigned int index; 989 990 softc = iflib_get_softc(ctx); 991 scctx = softc->scctx; 992 enic = &softc->enic; 993 994 995 enic_init_vnic_resources(enic); 996 997 for (index = 0; index < scctx->isc_ntxqsets; index++) 998 enic_prep_wq_for_simple_tx(&softc->enic, index); 999 1000 for (index = 0; index < scctx->isc_ntxqsets; index++) 1001 enic_start_wq(enic, index); 1002 1003 for (index = 0; index < scctx->isc_nrxqsets; index++) 1004 enic_start_rq(enic, index); 1005 1006 /* Use the current MAC address. */ 1007 bcopy(if_getlladdr(softc->ifp), softc->lladdr, ETHER_ADDR_LEN); 1008 enic_set_lladdr(softc); 1009 1010 ENIC_LOCK(softc); 1011 vnic_dev_enable_wait(enic->vdev); 1012 ENIC_UNLOCK(softc); 1013 1014 softc->stopped = 0; 1015 1016 enic_link_status(softc); 1017 } 1018 1019 static void 1020 enic_del_mcast(struct enic_softc *softc) { 1021 struct enic *enic; 1022 int i; 1023 1024 enic = &softc->enic; 1025 for (i=0; i < softc->mc_count; i++) { 1026 vnic_dev_del_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]); 1027 } 1028 softc->multicast = 0; 1029 softc->mc_count = 0; 1030 } 1031 1032 static void 1033 enic_add_mcast(struct enic_softc *softc) { 1034 struct enic *enic; 1035 int i; 1036 1037 enic = &softc->enic; 1038 for (i=0; i < softc->mc_count; i++) { 1039 vnic_dev_add_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]); 1040 } 1041 softc->multicast = 1; 1042 } 1043 1044 static u_int 1045 enic_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 1046 { 1047 uint8_t *mta = arg; 1048 1049 if (idx == ENIC_MAX_MULTICAST_ADDRESSES) 1050 return (0); 1051 1052 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1053 return (1); 1054 } 1055 1056 static void 1057 enic_multi_set(if_ctx_t ctx) 1058 { 1059 if_t ifp; 1060 struct enic_softc *softc; 1061 u_int count; 1062 1063 softc = iflib_get_softc(ctx); 1064 ifp = iflib_get_ifp(ctx); 1065 1066 ENIC_LOCK(softc); 1067 enic_del_mcast(softc); 1068 count = if_foreach_llmaddr(ifp, enic_copy_maddr, softc->mta); 1069 softc->mc_count = count; 1070 enic_add_mcast(softc); 1071 ENIC_UNLOCK(softc); 1072 1073 if (if_getflags(ifp) & IFF_PROMISC) { 1074 softc->promisc = 1; 1075 } else { 1076 softc->promisc = 0; 1077 } 1078 if (if_getflags(ifp) & IFF_ALLMULTI) { 1079 softc->allmulti = 1; 1080 } else { 1081 softc->allmulti = 0; 1082 } 1083 enic_update_packet_filter(&softc->enic); 1084 } 1085 1086 static int 1087 enic_mtu_set(if_ctx_t ctx, uint32_t mtu) 1088 { 1089 struct enic_softc *softc; 1090 struct enic *enic; 1091 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 1092 1093 softc = iflib_get_softc(ctx); 1094 enic = &softc->enic; 1095 1096 if (mtu > enic->port_mtu){ 1097 return (EINVAL); 1098 } 1099 1100 enic->config.mtu = mtu; 1101 scctx->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1102 1103 return (0); 1104 } 1105 1106 static void 1107 enic_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1108 { 1109 struct enic_softc *softc; 1110 struct ifmedia_entry *next; 1111 uint32_t speed; 1112 uint64_t target_baudrate; 1113 1114 softc = iflib_get_softc(ctx); 1115 1116 ifmr->ifm_status = IFM_AVALID; 1117 ifmr->ifm_active = IFM_ETHER; 1118 1119 if (enic_link_is_up(softc) != 0) { 1120 ENIC_LOCK(softc); 1121 speed = vnic_dev_port_speed(&softc->vdev); 1122 ENIC_UNLOCK(softc); 1123 target_baudrate = 1000ull * speed; 1124 LIST_FOREACH(next, &(iflib_get_media(ctx)->ifm_list), ifm_list) { 1125 if (ifmedia_baudrate(next->ifm_media) == target_baudrate) { 1126 ifmr->ifm_active |= next->ifm_media; 1127 } 1128 } 1129 1130 ifmr->ifm_status |= IFM_ACTIVE; 1131 ifmr->ifm_active |= IFM_AUTO; 1132 } else 1133 ifmr->ifm_active |= IFM_NONE; 1134 } 1135 1136 static int 1137 enic_media_change(if_ctx_t ctx) 1138 { 1139 return (ENODEV); 1140 } 1141 1142 static int 1143 enic_promisc_set(if_ctx_t ctx, int flags) 1144 { 1145 if_t ifp; 1146 struct enic_softc *softc; 1147 1148 softc = iflib_get_softc(ctx); 1149 ifp = iflib_get_ifp(ctx); 1150 1151 if (if_getflags(ifp) & IFF_PROMISC) { 1152 softc->promisc = 1; 1153 } else { 1154 softc->promisc = 0; 1155 } 1156 if (if_getflags(ifp) & IFF_ALLMULTI) { 1157 softc->allmulti = 1; 1158 } else { 1159 softc->allmulti = 0; 1160 } 1161 enic_update_packet_filter(&softc->enic); 1162 1163 return (0); 1164 } 1165 1166 static uint64_t 1167 enic_get_counter(if_ctx_t ctx, ift_counter cnt) { 1168 if_t ifp = iflib_get_ifp(ctx); 1169 1170 if (cnt < IFCOUNTERS) 1171 return if_get_counter_default(ifp, cnt); 1172 1173 return (0); 1174 } 1175 1176 static void 1177 enic_update_admin_status(if_ctx_t ctx) 1178 { 1179 struct enic_softc *softc; 1180 softc = iflib_get_softc(ctx); 1181 1182 enic_link_status(softc); 1183 } 1184 1185 static void 1186 enic_txq_timer(if_ctx_t ctx, uint16_t qid) 1187 { 1188 1189 struct enic_softc *softc; 1190 struct enic *enic; 1191 struct vnic_stats *stats; 1192 int ret; 1193 1194 softc = iflib_get_softc(ctx); 1195 enic = &softc->enic; 1196 1197 ENIC_LOCK(softc); 1198 ret = vnic_dev_stats_dump(enic->vdev, &stats); 1199 ENIC_UNLOCK(softc); 1200 if (ret) { 1201 dev_err(enic, "Error in getting stats\n"); 1202 } 1203 } 1204 1205 static int 1206 enic_link_is_up(struct enic_softc *softc) 1207 { 1208 return (vnic_dev_link_status(&softc->vdev) == 1); 1209 } 1210 1211 static void 1212 enic_link_status(struct enic_softc *softc) 1213 { 1214 if_ctx_t ctx; 1215 uint64_t speed; 1216 int link; 1217 1218 ctx = softc->ctx; 1219 link = enic_link_is_up(softc); 1220 speed = IF_Gbps(10); 1221 1222 ENIC_LOCK(softc); 1223 speed = vnic_dev_port_speed(&softc->vdev); 1224 ENIC_UNLOCK(softc); 1225 1226 if (link != 0 && softc->link_active == 0) { 1227 softc->link_active = 1; 1228 iflib_link_state_change(ctx, LINK_STATE_UP, speed); 1229 } else if (link == 0 && softc->link_active != 0) { 1230 softc->link_active = 0; 1231 iflib_link_state_change(ctx, LINK_STATE_DOWN, speed); 1232 } 1233 } 1234 1235 static void 1236 enic_set_lladdr(struct enic_softc *softc) 1237 { 1238 struct enic *enic; 1239 enic = &softc->enic; 1240 1241 ENIC_LOCK(softc); 1242 vnic_dev_add_addr(enic->vdev, softc->lladdr); 1243 ENIC_UNLOCK(softc); 1244 } 1245 1246 1247 static void 1248 enic_setup_txq_sysctl(struct vnic_wq *wq, int i, struct sysctl_ctx_list *ctx, 1249 struct sysctl_oid_list *child) 1250 { 1251 struct sysctl_oid *txsnode; 1252 struct sysctl_oid_list *txslist; 1253 struct vnic_stats *stats; 1254 1255 stats = wq[i].vdev->stats; 1256 1257 txsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats", 1258 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 1259 txslist = SYSCTL_CHILDREN(txsnode); 1260 1261 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_frames_ok", CTLFLAG_RD, 1262 &stats->tx.tx_frames_ok, "TX Frames OK"); 1263 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_frames_ok", CTLFLAG_RD, 1264 &stats->tx.tx_unicast_frames_ok, "TX unicast frames OK"); 1265 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_frames_ok", CTLFLAG_RD, 1266 &stats->tx.tx_multicast_frames_ok, "TX multicast framse OK"); 1267 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_frames_ok", CTLFLAG_RD, 1268 &stats->tx.tx_broadcast_frames_ok, "TX Broadcast frames OK"); 1269 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_bytes_ok", CTLFLAG_RD, 1270 &stats->tx.tx_bytes_ok, "TX bytes OK "); 1271 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_bytes_ok", CTLFLAG_RD, 1272 &stats->tx.tx_unicast_bytes_ok, "TX unicast bytes OK"); 1273 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_bytes_ok", CTLFLAG_RD, 1274 &stats->tx.tx_multicast_bytes_ok, "TX multicast bytes OK"); 1275 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_bytes_ok", CTLFLAG_RD, 1276 &stats->tx.tx_broadcast_bytes_ok, "TX broadcast bytes OK"); 1277 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_drops", CTLFLAG_RD, 1278 &stats->tx.tx_drops, "TX drops"); 1279 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_errors", CTLFLAG_RD, 1280 &stats->tx.tx_errors, "TX errors"); 1281 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_tso", CTLFLAG_RD, 1282 &stats->tx.tx_tso, "TX TSO"); 1283 } 1284 1285 static void 1286 enic_setup_rxq_sysctl(struct vnic_rq *rq, int i, struct sysctl_ctx_list *ctx, 1287 struct sysctl_oid_list *child) 1288 { 1289 struct sysctl_oid *rxsnode; 1290 struct sysctl_oid_list *rxslist; 1291 struct vnic_stats *stats; 1292 1293 stats = rq[i].vdev->stats; 1294 1295 rxsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats", 1296 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 1297 rxslist = SYSCTL_CHILDREN(rxsnode); 1298 1299 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_ok", CTLFLAG_RD, 1300 &stats->rx.rx_frames_ok, "RX Frames OK"); 1301 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_total", CTLFLAG_RD, 1302 &stats->rx.rx_frames_total, "RX frames total"); 1303 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_frames_ok", CTLFLAG_RD, 1304 &stats->rx.rx_unicast_frames_ok, "RX unicast frames ok"); 1305 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_frames_ok", CTLFLAG_RD, 1306 &stats->rx.rx_multicast_frames_ok, "RX multicast Frames ok"); 1307 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_frames_ok", CTLFLAG_RD, 1308 &stats->rx.rx_broadcast_frames_ok, "RX broadcast frames ok"); 1309 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_bytes_ok", CTLFLAG_RD, 1310 &stats->rx.rx_bytes_ok, "RX bytes ok"); 1311 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_bytes_ok", CTLFLAG_RD, 1312 &stats->rx.rx_unicast_bytes_ok, "RX unicast bytes ok"); 1313 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_bytes_ok", CTLFLAG_RD, 1314 &stats->rx.rx_multicast_bytes_ok, "RX multicast bytes ok"); 1315 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_bytes_ok", CTLFLAG_RD, 1316 &stats->rx.rx_broadcast_bytes_ok, "RX broadcast bytes ok"); 1317 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_drop", CTLFLAG_RD, 1318 &stats->rx.rx_drop, "RX drop"); 1319 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_errors", CTLFLAG_RD, 1320 &stats->rx.rx_errors, "RX errors"); 1321 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_rss", CTLFLAG_RD, 1322 &stats->rx.rx_rss, "RX rss"); 1323 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_crc_errors", CTLFLAG_RD, 1324 &stats->rx.rx_crc_errors, "RX crc errors"); 1325 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_64", CTLFLAG_RD, 1326 &stats->rx.rx_frames_64, "RX frames 64"); 1327 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_127", CTLFLAG_RD, 1328 &stats->rx.rx_frames_127, "RX frames 127"); 1329 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_255", CTLFLAG_RD, 1330 &stats->rx.rx_frames_255, "RX frames 255"); 1331 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_511", CTLFLAG_RD, 1332 &stats->rx.rx_frames_511, "RX frames 511"); 1333 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1023", CTLFLAG_RD, 1334 &stats->rx.rx_frames_1023, "RX frames 1023"); 1335 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1518", CTLFLAG_RD, 1336 &stats->rx.rx_frames_1518, "RX frames 1518"); 1337 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_to_max", CTLFLAG_RD, 1338 &stats->rx.rx_frames_to_max, "RX frames to max"); 1339 } 1340 1341 static void 1342 enic_setup_queue_sysctl(struct enic_softc *softc, struct sysctl_ctx_list *ctx, 1343 struct sysctl_oid_list *child) 1344 { 1345 enic_setup_txq_sysctl(softc->enic.wq, 0, ctx, child); 1346 enic_setup_rxq_sysctl(softc->enic.rq, 0, ctx, child); 1347 } 1348 1349 static void 1350 enic_setup_sysctl(struct enic_softc *softc) 1351 { 1352 device_t dev; 1353 struct sysctl_ctx_list *ctx; 1354 struct sysctl_oid *tree; 1355 struct sysctl_oid_list *child; 1356 1357 dev = softc->dev; 1358 ctx = device_get_sysctl_ctx(dev); 1359 tree = device_get_sysctl_tree(dev); 1360 child = SYSCTL_CHILDREN(tree); 1361 1362 enic_setup_queue_sysctl(softc, ctx, child); 1363 } 1364 1365 static void 1366 enic_enable_intr(struct enic_softc *softc, int irq) 1367 { 1368 struct enic *enic = &softc->enic; 1369 1370 vnic_intr_unmask(&enic->intr[irq]); 1371 vnic_intr_return_all_credits(&enic->intr[irq]); 1372 } 1373 1374 static void 1375 enic_disable_intr(struct enic_softc *softc, int irq) 1376 { 1377 struct enic *enic = &softc->enic; 1378 1379 vnic_intr_mask(&enic->intr[irq]); 1380 vnic_intr_masked(&enic->intr[irq]); /* flush write */ 1381 } 1382 1383 static int 1384 enic_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 1385 { 1386 struct enic_softc *softc; 1387 if_softc_ctx_t scctx; 1388 1389 softc = iflib_get_softc(ctx); 1390 scctx = softc->scctx; 1391 1392 enic_enable_intr(softc, qid + scctx->isc_nrxqsets); 1393 1394 return 0; 1395 } 1396 1397 static int 1398 enic_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 1399 { 1400 struct enic_softc *softc; 1401 1402 softc = iflib_get_softc(ctx); 1403 enic_enable_intr(softc, qid); 1404 1405 return 0; 1406 } 1407 1408 static void 1409 enic_intr_enable_all(if_ctx_t ctx) 1410 { 1411 struct enic_softc *softc; 1412 if_softc_ctx_t scctx; 1413 int i; 1414 1415 softc = iflib_get_softc(ctx); 1416 scctx = softc->scctx; 1417 1418 for (i = 0; i < scctx->isc_vectors; i++) { 1419 enic_enable_intr(softc, i); 1420 } 1421 } 1422 1423 static void 1424 enic_intr_disable_all(if_ctx_t ctx) 1425 { 1426 struct enic_softc *softc; 1427 if_softc_ctx_t scctx; 1428 int i; 1429 1430 softc = iflib_get_softc(ctx); 1431 scctx = softc->scctx; 1432 /* 1433 * iflib may invoke this routine before enic_attach_post() has run, 1434 * which is before the top level shared data area is initialized and 1435 * the device made aware of it. 1436 */ 1437 1438 for (i = 0; i < scctx->isc_vectors; i++) { 1439 enic_disable_intr(softc, i); 1440 } 1441 } 1442 1443 static int 1444 enic_dev_open(struct enic *enic) 1445 { 1446 int err; 1447 int flags = CMD_OPENF_IG_DESCCACHE; 1448 1449 err = enic_dev_wait(enic->vdev, vnic_dev_open, 1450 vnic_dev_open_done, flags); 1451 if (err) 1452 dev_err(enic_get_dev(enic), 1453 "vNIC device open failed, err %d\n", err); 1454 1455 return err; 1456 } 1457 1458 static int 1459 enic_dev_init(struct enic *enic) 1460 { 1461 int err; 1462 1463 vnic_dev_intr_coal_timer_info_default(enic->vdev); 1464 1465 /* 1466 * Get vNIC configuration 1467 */ 1468 err = enic_get_vnic_config(enic); 1469 if (err) { 1470 dev_err(dev, "Get vNIC configuration failed, aborting\n"); 1471 return err; 1472 } 1473 1474 /* Get available resource counts */ 1475 enic_get_res_counts(enic); 1476 1477 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */ 1478 enic->intr_queues = malloc(sizeof(*enic->intr_queues) * 1479 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO); 1480 1481 vnic_dev_set_reset_flag(enic->vdev, 0); 1482 enic->max_flow_counter = -1; 1483 1484 /* set up link status checking */ 1485 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */ 1486 1487 enic->overlay_offload = false; 1488 if (enic->disable_overlay && enic->vxlan) { 1489 /* 1490 * Explicitly disable overlay offload as the setting is 1491 * sticky, and resetting vNIC does not disable it. 1492 */ 1493 if (vnic_dev_overlay_offload_ctrl(enic->vdev, 1494 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_DISABLE)) { 1495 dev_err(enic, "failed to disable overlay offload\n"); 1496 } else { 1497 dev_info(enic, "Overlay offload is disabled\n"); 1498 } 1499 } 1500 if (!enic->disable_overlay && enic->vxlan && 1501 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */ 1502 vnic_dev_overlay_offload_ctrl(enic->vdev, 1503 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_ENABLE) == 0) { 1504 enic->overlay_offload = true; 1505 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT; 1506 dev_info(enic, "Overlay offload is enabled\n"); 1507 /* 1508 * Reset the vxlan port to the default, as the NIC firmware 1509 * does not reset it automatically and keeps the old setting. 1510 */ 1511 if (vnic_dev_overlay_offload_cfg(enic->vdev, 1512 OVERLAY_CFG_VXLAN_PORT_UPDATE, ENIC_DEFAULT_VXLAN_PORT)) { 1513 dev_err(enic, "failed to update vxlan port\n"); 1514 return (EINVAL); 1515 } 1516 } 1517 return 0; 1518 } 1519 1520 static void * 1521 enic_alloc_consistent(void *priv, size_t size, bus_addr_t * dma_handle, 1522 struct iflib_dma_info *res, u8 * name) 1523 { 1524 void *vaddr; 1525 *dma_handle = 0; 1526 struct enic *enic = (struct enic *)priv; 1527 int rz; 1528 1529 rz = iflib_dma_alloc(enic->softc->ctx, size, res, BUS_DMA_NOWAIT); 1530 if (rz) { 1531 pr_err("%s : Failed to allocate memory requested for %s\n", 1532 __func__, name); 1533 return NULL; 1534 } 1535 1536 vaddr = res->idi_vaddr; 1537 *dma_handle = res->idi_paddr; 1538 1539 return vaddr; 1540 } 1541 1542 static void 1543 enic_free_consistent(void *priv, size_t size, void *vaddr, 1544 bus_addr_t dma_handle, struct iflib_dma_info *res) 1545 { 1546 iflib_dma_free(res); 1547 } 1548 1549 static int 1550 enic_pci_mapping(struct enic_softc *softc) 1551 { 1552 int rc; 1553 1554 rc = enic_map_bar(softc, &softc->mem, 0, true); 1555 if (rc) 1556 return rc; 1557 1558 rc = enic_map_bar(softc, &softc->io, 2, false); 1559 1560 return rc; 1561 } 1562 1563 static void 1564 enic_pci_mapping_free(struct enic_softc *softc) 1565 { 1566 if (softc->mem.res != NULL) 1567 bus_release_resource(softc->dev, SYS_RES_MEMORY, 1568 softc->mem.rid, softc->mem.res); 1569 softc->mem.res = NULL; 1570 1571 if (softc->io.res != NULL) 1572 bus_release_resource(softc->dev, SYS_RES_MEMORY, 1573 softc->io.rid, softc->io.res); 1574 softc->io.res = NULL; 1575 } 1576 1577 static int 1578 enic_dev_wait(struct vnic_dev *vdev, int (*start) (struct vnic_dev *, int), 1579 int (*finished) (struct vnic_dev *, int *), int arg) 1580 { 1581 int done; 1582 int err; 1583 int i; 1584 1585 err = start(vdev, arg); 1586 if (err) 1587 return err; 1588 1589 /* Wait for func to complete...2 seconds max */ 1590 for (i = 0; i < 2000; i++) { 1591 err = finished(vdev, &done); 1592 if (err) 1593 return err; 1594 if (done) 1595 return 0; 1596 usleep(1000); 1597 } 1598 return (ETIMEDOUT); 1599 } 1600 1601 static int 1602 enic_map_bar(struct enic_softc *softc, struct enic_bar_info *bar, int bar_num, 1603 bool shareable) 1604 { 1605 uint32_t flag; 1606 1607 if (bar->res != NULL) { 1608 device_printf(softc->dev, "Bar %d already mapped\n", bar_num); 1609 return (EDOOFUS); 1610 } 1611 1612 bar->rid = PCIR_BAR(bar_num); 1613 flag = RF_ACTIVE; 1614 if (shareable) 1615 flag |= RF_SHAREABLE; 1616 1617 if ((bar->res = bus_alloc_resource_any(softc->dev, 1618 SYS_RES_MEMORY, &bar->rid, flag)) == NULL) { 1619 device_printf(softc->dev, 1620 "PCI BAR%d mapping failure\n", bar_num); 1621 return (ENXIO); 1622 } 1623 bar->tag = rman_get_bustag(bar->res); 1624 bar->handle = rman_get_bushandle(bar->res); 1625 bar->size = rman_get_size(bar->res); 1626 1627 return 0; 1628 } 1629 1630 void 1631 enic_init_vnic_resources(struct enic *enic) 1632 { 1633 unsigned int error_interrupt_enable = 1; 1634 unsigned int error_interrupt_offset = 0; 1635 unsigned int rxq_interrupt_enable = 0; 1636 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET; 1637 unsigned int txq_interrupt_enable = 0; 1638 unsigned int txq_interrupt_offset; 1639 unsigned int index = 0; 1640 unsigned int cq_idx; 1641 if_softc_ctx_t scctx; 1642 1643 scctx = enic->softc->scctx; 1644 1645 rxq_interrupt_enable = 1; 1646 txq_interrupt_enable = 0; 1647 1648 rxq_interrupt_offset = 0; 1649 txq_interrupt_offset = scctx->isc_nrxqsets; 1650 1651 for (index = 0; index < enic->intr_count; index++) { 1652 vnic_intr_alloc(enic->vdev, &enic->intr[index], index); 1653 } 1654 1655 for (index = 0; index < scctx->isc_nrxqsets; index++) { 1656 cq_idx = enic_cq_rq(enic, index); 1657 1658 vnic_rq_clean(&enic->rq[index]); 1659 vnic_rq_init(&enic->rq[index], cq_idx, error_interrupt_enable, 1660 error_interrupt_offset); 1661 1662 vnic_cq_clean(&enic->cq[cq_idx]); 1663 vnic_cq_init(&enic->cq[cq_idx], 1664 0 /* flow_control_enable */ , 1665 1 /* color_enable */ , 1666 0 /* cq_head */ , 1667 0 /* cq_tail */ , 1668 1 /* cq_tail_color */ , 1669 rxq_interrupt_enable, 1670 1 /* cq_entry_enable */ , 1671 0 /* cq_message_enable */ , 1672 rxq_interrupt_offset, 1673 0 /* cq_message_addr */ ); 1674 if (rxq_interrupt_enable) 1675 rxq_interrupt_offset++; 1676 } 1677 1678 for (index = 0; index < scctx->isc_ntxqsets; index++) { 1679 cq_idx = enic_cq_wq(enic, index); 1680 vnic_wq_clean(&enic->wq[index]); 1681 vnic_wq_init(&enic->wq[index], cq_idx, error_interrupt_enable, 1682 error_interrupt_offset); 1683 /* Compute unsupported ol flags for enic_prep_pkts() */ 1684 enic->wq[index].tx_offload_notsup_mask = 0; 1685 1686 vnic_cq_clean(&enic->cq[cq_idx]); 1687 vnic_cq_init(&enic->cq[cq_idx], 1688 0 /* flow_control_enable */ , 1689 1 /* color_enable */ , 1690 0 /* cq_head */ , 1691 0 /* cq_tail */ , 1692 1 /* cq_tail_color */ , 1693 txq_interrupt_enable, 1694 1, 1695 0, 1696 txq_interrupt_offset, 1697 0 /* (u64)enic->wq[index].cqmsg_rz->iova */ ); 1698 1699 } 1700 1701 for (index = 0; index < enic->intr_count; index++) { 1702 vnic_intr_init(&enic->intr[index], 125, 1703 enic->config.intr_timer_type, /* mask_on_assertion */ 1); 1704 } 1705 } 1706 1707 static void 1708 enic_update_packet_filter(struct enic *enic) 1709 { 1710 struct enic_softc *softc = enic->softc; 1711 1712 ENIC_LOCK(softc); 1713 vnic_dev_packet_filter(enic->vdev, 1714 softc->directed, 1715 softc->multicast, 1716 softc->broadcast, 1717 softc->promisc, 1718 softc->allmulti); 1719 ENIC_UNLOCK(softc); 1720 } 1721 1722 static bool 1723 enic_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event) 1724 { 1725 switch (event) { 1726 case IFLIB_RESTART_VLAN_CONFIG: 1727 default: 1728 return (false); 1729 } 1730 } 1731 1732 int 1733 enic_setup_finish(struct enic *enic) 1734 { 1735 struct enic_softc *softc = enic->softc; 1736 1737 /* Default conf */ 1738 softc->directed = 1; 1739 softc->multicast = 0; 1740 softc->broadcast = 1; 1741 softc->promisc = 0; 1742 softc->allmulti = 1; 1743 enic_update_packet_filter(enic); 1744 1745 return 0; 1746 } 1747