1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 */ 5 6 #include "opt_rss.h" 7 8 #include <sys/param.h> 9 #include <sys/systm.h> 10 #include <sys/kernel.h> 11 #include <sys/endian.h> 12 #include <sys/sockio.h> 13 #include <sys/mbuf.h> 14 #include <sys/malloc.h> 15 #include <sys/module.h> 16 #include <sys/socket.h> 17 #include <sys/sysctl.h> 18 #include <sys/smp.h> 19 #include <vm/vm.h> 20 #include <vm/pmap.h> 21 22 #include <net/ethernet.h> 23 #include <net/if.h> 24 #include <net/if_var.h> 25 #include <net/if_arp.h> 26 #include <net/if_dl.h> 27 #include <net/if_types.h> 28 #include <net/if_media.h> 29 #include <net/if_vlan_var.h> 30 #include <net/iflib.h> 31 #ifdef RSS 32 #include <net/rss_config.h> 33 #endif 34 35 #include <netinet/in_systm.h> 36 #include <netinet/in.h> 37 #include <netinet/ip.h> 38 #include <netinet/ip6.h> 39 #include <netinet6/ip6_var.h> 40 #include <netinet/udp.h> 41 #include <netinet/tcp.h> 42 43 #include <machine/bus.h> 44 #include <machine/resource.h> 45 #include <sys/bus.h> 46 #include <sys/rman.h> 47 48 #include <dev/pci/pcireg.h> 49 #include <dev/pci/pcivar.h> 50 51 #include "ifdi_if.h" 52 #include "enic.h" 53 54 #include "opt_inet.h" 55 #include "opt_inet6.h" 56 57 static SYSCTL_NODE(_hw, OID_AUTO, enic, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 58 "ENIC"); 59 60 static const pci_vendor_info_t enic_vendor_info_array[] = 61 { 62 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET, 63 DRV_DESCRIPTION), 64 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET_VF, 65 DRV_DESCRIPTION " VF"), 66 /* required last entry */ 67 68 PVID_END 69 }; 70 71 static void *enic_register(device_t); 72 static int enic_attach_pre(if_ctx_t); 73 static int enic_msix_intr_assign(if_ctx_t, int); 74 75 static int enic_attach_post(if_ctx_t); 76 static int enic_detach(if_ctx_t); 77 78 static int enic_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 79 static int enic_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 80 static void enic_queues_free(if_ctx_t); 81 static int enic_rxq_intr(void *); 82 static int enic_event_intr(void *); 83 static int enic_err_intr(void *); 84 static void enic_stop(if_ctx_t); 85 static void enic_init(if_ctx_t); 86 static void enic_multi_set(if_ctx_t); 87 static int enic_mtu_set(if_ctx_t, uint32_t); 88 static void enic_media_status(if_ctx_t, struct ifmediareq *); 89 static int enic_media_change(if_ctx_t); 90 static int enic_promisc_set(if_ctx_t, int); 91 static uint64_t enic_get_counter(if_ctx_t, ift_counter); 92 static void enic_update_admin_status(if_ctx_t); 93 static void enic_txq_timer(if_ctx_t, uint16_t); 94 static int enic_link_is_up(struct enic_softc *); 95 static void enic_link_status(struct enic_softc *); 96 static void enic_set_lladdr(struct enic_softc *); 97 static void enic_setup_txq_sysctl(struct vnic_wq *, int, struct sysctl_ctx_list *, 98 struct sysctl_oid_list *); 99 static void enic_setup_rxq_sysctl(struct vnic_rq *, int, struct sysctl_ctx_list *, 100 struct sysctl_oid_list *); 101 static void enic_setup_sysctl(struct enic_softc *); 102 static int enic_tx_queue_intr_enable(if_ctx_t, uint16_t); 103 static int enic_rx_queue_intr_enable(if_ctx_t, uint16_t); 104 static void enic_enable_intr(struct enic_softc *, int); 105 static void enic_disable_intr(struct enic_softc *, int); 106 static void enic_intr_enable_all(if_ctx_t); 107 static void enic_intr_disable_all(if_ctx_t); 108 static int enic_dev_open(struct enic *); 109 static int enic_dev_init(struct enic *); 110 static void *enic_alloc_consistent(void *, size_t, bus_addr_t *, 111 struct iflib_dma_info *, u8 *); 112 static void enic_free_consistent(void *, size_t, void *, bus_addr_t, 113 struct iflib_dma_info *); 114 static int enic_pci_mapping(struct enic_softc *); 115 static void enic_pci_mapping_free(struct enic_softc *); 116 static int enic_dev_wait(struct vnic_dev *, int (*) (struct vnic_dev *, int), 117 int (*) (struct vnic_dev *, int *), int arg); 118 static int enic_map_bar(struct enic_softc *, struct enic_bar_info *, int, bool); 119 static void enic_update_packet_filter(struct enic *enic); 120 static bool enic_if_needs_restart(if_ctx_t, enum iflib_restart_event); 121 122 typedef enum { 123 ENIC_BARRIER_RD, 124 ENIC_BARRIER_WR, 125 ENIC_BARRIER_RDWR, 126 } enic_barrier_t; 127 128 static device_method_t enic_methods[] = { 129 /* Device interface */ 130 DEVMETHOD(device_register, enic_register), 131 DEVMETHOD(device_probe, iflib_device_probe), 132 DEVMETHOD(device_attach, iflib_device_attach), 133 DEVMETHOD(device_detach, iflib_device_detach), 134 DEVMETHOD(device_shutdown, iflib_device_shutdown), 135 DEVMETHOD(device_suspend, iflib_device_suspend), 136 DEVMETHOD(device_resume, iflib_device_resume), 137 DEVMETHOD_END 138 }; 139 140 static driver_t enic_driver = { 141 "enic", enic_methods, sizeof(struct enic_softc) 142 }; 143 144 DRIVER_MODULE(enic, pci, enic_driver, 0, 0); 145 IFLIB_PNP_INFO(pci, enic, enic_vendor_info_array); 146 MODULE_VERSION(enic, 2); 147 148 MODULE_DEPEND(enic, pci, 1, 1, 1); 149 MODULE_DEPEND(enic, ether, 1, 1, 1); 150 MODULE_DEPEND(enic, iflib, 1, 1, 1); 151 152 static device_method_t enic_iflib_methods[] = { 153 DEVMETHOD(ifdi_tx_queues_alloc, enic_tx_queues_alloc), 154 DEVMETHOD(ifdi_rx_queues_alloc, enic_rx_queues_alloc), 155 DEVMETHOD(ifdi_queues_free, enic_queues_free), 156 157 DEVMETHOD(ifdi_attach_pre, enic_attach_pre), 158 DEVMETHOD(ifdi_attach_post, enic_attach_post), 159 DEVMETHOD(ifdi_detach, enic_detach), 160 161 DEVMETHOD(ifdi_init, enic_init), 162 DEVMETHOD(ifdi_stop, enic_stop), 163 DEVMETHOD(ifdi_multi_set, enic_multi_set), 164 DEVMETHOD(ifdi_mtu_set, enic_mtu_set), 165 DEVMETHOD(ifdi_media_status, enic_media_status), 166 DEVMETHOD(ifdi_media_change, enic_media_change), 167 DEVMETHOD(ifdi_promisc_set, enic_promisc_set), 168 DEVMETHOD(ifdi_get_counter, enic_get_counter), 169 DEVMETHOD(ifdi_update_admin_status, enic_update_admin_status), 170 DEVMETHOD(ifdi_timer, enic_txq_timer), 171 172 DEVMETHOD(ifdi_tx_queue_intr_enable, enic_tx_queue_intr_enable), 173 DEVMETHOD(ifdi_rx_queue_intr_enable, enic_rx_queue_intr_enable), 174 DEVMETHOD(ifdi_intr_enable, enic_intr_enable_all), 175 DEVMETHOD(ifdi_intr_disable, enic_intr_disable_all), 176 DEVMETHOD(ifdi_msix_intr_assign, enic_msix_intr_assign), 177 178 DEVMETHOD(ifdi_needs_restart, enic_if_needs_restart), 179 180 DEVMETHOD_END 181 }; 182 183 static driver_t enic_iflib_driver = { 184 "enic", enic_iflib_methods, sizeof(struct enic_softc) 185 }; 186 187 extern struct if_txrx enic_txrx; 188 189 static struct if_shared_ctx enic_sctx_init = { 190 .isc_magic = IFLIB_MAGIC, 191 .isc_q_align = 512, 192 193 .isc_tx_maxsize = ENIC_TX_MAX_PKT_SIZE, 194 .isc_tx_maxsegsize = PAGE_SIZE, 195 196 /* 197 * These values are used to configure the busdma tag used for receive 198 * descriptors. Each receive descriptor only points to one buffer. 199 */ 200 .isc_rx_maxsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, /* One buf per 201 * descriptor */ 202 .isc_rx_nsegments = 1, /* One mapping per descriptor */ 203 .isc_rx_maxsegsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, 204 .isc_admin_intrcnt = 2, 205 .isc_vendor_info = enic_vendor_info_array, 206 .isc_driver_version = "1", 207 .isc_driver = &enic_iflib_driver, 208 .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ | IFLIB_SKIP_MSIX, 209 210 /* 211 * Number of receive queues per receive queue set, with associated 212 * descriptor settings for each. 213 */ 214 215 .isc_nrxqs = 2, 216 .isc_nfl = 1, /* one free list for each receive command 217 * queue */ 218 .isc_nrxd_min = {16, 16}, 219 .isc_nrxd_max = {2048, 2048}, 220 .isc_nrxd_default = {64, 64}, 221 222 /* 223 * Number of transmit queues per transmit queue set, with associated 224 * descriptor settings for each. 225 */ 226 .isc_ntxqs = 2, 227 .isc_ntxd_min = {16, 16}, 228 .isc_ntxd_max = {2048, 2048}, 229 .isc_ntxd_default = {64, 64}, 230 }; 231 232 static void * 233 enic_register(device_t dev) 234 { 235 return (&enic_sctx_init); 236 } 237 238 static int 239 enic_allocate_msix(struct enic_softc *softc) { 240 if_ctx_t ctx; 241 if_softc_ctx_t scctx; 242 if_shared_ctx_t sctx; 243 device_t dev; 244 cpuset_t cpus; 245 int queues, vectors, requested; 246 int err = 0; 247 248 dev = softc->dev; 249 ctx = softc->ctx; 250 scctx = softc->scctx; 251 sctx = iflib_get_sctx(ctx); 252 253 if (bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus) != 0) { 254 device_printf(dev, "Unable to fetch CPU list\n"); 255 CPU_COPY(&all_cpus, &cpus); 256 } 257 258 259 queues = CPU_COUNT(&cpus); 260 queues = imin(queues, scctx->isc_nrxqsets); 261 queues = imin(queues, scctx->isc_ntxqsets); 262 requested = queues * 2 + sctx->isc_admin_intrcnt; 263 scctx->isc_nrxqsets = queues; 264 scctx->isc_ntxqsets = queues; 265 266 vectors = requested; 267 if ((err = pci_alloc_msix(dev, &vectors)) != 0) { 268 device_printf(dev, 269 "failed to allocate %d MSI-X vectors, err: %d\n", requested, 270 err); 271 err = 1; 272 goto enic_allocate_msix_out; 273 } else { 274 if (vectors != requested) { 275 device_printf(dev, 276 "Unable to allocate sufficient MSI-X vectors " 277 "(got %d, need %d)\n", requested, vectors); 278 pci_release_msi(dev); 279 err = 1; 280 goto enic_allocate_msix_out; 281 } 282 } 283 284 device_printf(dev, "Using MSI-X interrupts with %d vectors\n", 285 vectors); 286 287 scctx->isc_intr = IFLIB_INTR_MSIX; 288 scctx->isc_vectors = vectors; 289 290 enic_allocate_msix_out: 291 return (err); 292 293 } 294 295 static struct enic_intr_mod_range mod_range[ENIC_MAX_LINK_SPEEDS] = { 296 {0, 0}, /* 0 - 4 Gbps */ 297 {0, 3}, /* 4 - 10 Gbps */ 298 {3, 6}, /* 10 - 40 Gbps */ 299 }; 300 301 static void enic_set_rx_coal_setting(struct enic *enic) 302 { 303 unsigned int speed; 304 int index = -1; 305 struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting; 306 307 /* 1. Read the link speed from fw 308 * 2. Pick the default range for the speed 309 * 3. Update it in enic->rx_coalesce_setting 310 */ 311 speed = vnic_dev_port_speed(enic->vdev); 312 if (ENIC_LINK_SPEED_10G < speed) 313 index = ENIC_LINK_40G_INDEX; 314 else if (ENIC_LINK_SPEED_4G < speed) 315 index = ENIC_LINK_10G_INDEX; 316 else 317 index = ENIC_LINK_4G_INDEX; 318 319 rx_coal->small_pkt_range_start = mod_range[index].small_pkt_range_start; 320 rx_coal->large_pkt_range_start = mod_range[index].large_pkt_range_start; 321 rx_coal->range_end = ENIC_RX_COALESCE_RANGE_END; 322 323 /* Start with the value provided by UCSM */ 324 for (index = 0; index < enic->rq_count; index++) 325 enic->cq[index].cur_rx_coal_timeval = 326 enic->config.intr_timer_usec; 327 328 rx_coal->use_adaptive_rx_coalesce = 1; 329 } 330 331 static int 332 enic_attach_pre(if_ctx_t ctx) 333 { 334 if_softc_ctx_t scctx; 335 struct enic_softc *softc; 336 struct vnic_dev *vdev; 337 struct enic *enic; 338 device_t dev; 339 340 int err = -1; 341 int rc = 0; 342 int i; 343 u64 a0 = 0, a1 = 0; 344 int wait = 1000; 345 struct vnic_stats *stats; 346 int ret; 347 348 dev = iflib_get_dev(ctx); 349 softc = iflib_get_softc(ctx); 350 softc->dev = dev; 351 softc->ctx = ctx; 352 softc->sctx = iflib_get_sctx(ctx); 353 softc->scctx = iflib_get_softc_ctx(ctx); 354 softc->ifp = iflib_get_ifp(ctx); 355 softc->media = iflib_get_media(ctx); 356 softc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 357 ENIC_MAX_MULTICAST_ADDRESSES, M_DEVBUF, 358 M_NOWAIT | M_ZERO); 359 if (softc->mta == NULL) 360 return (ENOMEM); 361 scctx = softc->scctx; 362 363 mtx_init(&softc->enic_lock, "ENIC Lock", NULL, MTX_DEF); 364 365 pci_enable_busmaster(softc->dev); 366 if (enic_pci_mapping(softc)) 367 return (ENXIO); 368 369 enic = &softc->enic; 370 enic->softc = softc; 371 vdev = &softc->vdev; 372 vdev->softc = softc; 373 enic->vdev = vdev; 374 vdev->priv = enic; 375 376 ENIC_LOCK(softc); 377 vnic_dev_register(vdev, &softc->mem, 1); 378 enic->vdev = vdev; 379 vnic_dev_cmd_init(enic->vdev); 380 381 vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0); 382 383 vnic_dev_cmd(vdev, CMD_INIT_v1, &a0, &a1, wait); 384 vnic_dev_cmd(vdev, CMD_GET_MAC_ADDR, &a0, &a1, wait); 385 386 bcopy((u_int8_t *) & a0, softc->mac_addr, ETHER_ADDR_LEN); 387 iflib_set_mac(ctx, softc->mac_addr); 388 389 vnic_register_cbacks(enic->vdev, enic_alloc_consistent, 390 enic_free_consistent); 391 392 /* 393 * Allocate the consistent memory for stats and counters upfront so 394 * both primary and secondary processes can access them. 395 */ 396 ENIC_UNLOCK(softc); 397 err = vnic_dev_alloc_stats_mem(enic->vdev); 398 ENIC_LOCK(softc); 399 if (err) { 400 dev_err(enic, "Failed to allocate cmd memory, aborting\n"); 401 goto err_out_unregister; 402 } 403 vnic_dev_stats_clear(enic->vdev); 404 ret = vnic_dev_stats_dump(enic->vdev, &stats); 405 if (ret) { 406 dev_err(enic, "Error in getting stats\n"); 407 goto err_out_unregister; 408 } 409 err = vnic_dev_alloc_counter_mem(enic->vdev); 410 if (err) { 411 dev_err(enic, "Failed to allocate counter memory, aborting\n"); 412 goto err_out_unregister; 413 } 414 415 /* Issue device open to get device in known state */ 416 err = enic_dev_open(enic); 417 if (err) { 418 dev_err(enic, "vNIC dev open failed, aborting\n"); 419 goto err_out_unregister; 420 } 421 422 /* Set ingress vlan rewrite mode before vnic initialization */ 423 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN; 424 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN; 425 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, 426 enic->ig_vlan_rewrite_mode); 427 if (err) { 428 dev_err(enic, 429 "Failed to set ingress vlan rewrite mode, aborting.\n"); 430 goto err_out_dev_close; 431 } 432 433 /* 434 * Issue device init to initialize the vnic-to-switch link. We'll 435 * start with carrier off and wait for link UP notification later to 436 * turn on carrier. We don't need to wait here for the 437 * vnic-to-switch link initialization to complete; link UP 438 * notification is the indication that the process is complete. 439 */ 440 441 err = vnic_dev_init(enic->vdev, 0); 442 if (err) { 443 dev_err(enic, "vNIC dev init failed, aborting\n"); 444 goto err_out_dev_close; 445 } 446 447 err = enic_dev_init(enic); 448 if (err) { 449 dev_err(enic, "Device initialization failed, aborting\n"); 450 goto err_out_dev_close; 451 } 452 /* Queue DMA has not been enabled; the first iflib stop may be skipped. */ 453 softc->stopped = 1; 454 ENIC_UNLOCK(softc); 455 456 enic->port_mtu = vnic_dev_mtu(enic->vdev); 457 458 softc->scctx = iflib_get_softc_ctx(ctx); 459 scctx = softc->scctx; 460 scctx->isc_txrx = &enic_txrx; 461 scctx->isc_capabilities = scctx->isc_capenable = \ 462 IFCAP_HWCSUM; 463 scctx->isc_tx_csum_flags = 0; 464 if_setmtu(softc->ifp, enic->config.mtu); 465 scctx->isc_max_frame_size = enic->config.mtu + ETHER_HDR_LEN + \ 466 ETHER_CRC_LEN; 467 scctx->isc_nrxqsets_max = enic->conf_rq_count; 468 scctx->isc_ntxqsets_max = enic->conf_wq_count; 469 scctx->isc_nrxqsets = enic->conf_rq_count; 470 scctx->isc_ntxqsets = enic->conf_wq_count; 471 for (i = 0; i < enic->conf_wq_count; i++) { 472 scctx->isc_ntxd[i] = enic->config.wq_desc_count; 473 scctx->isc_txqsizes[i] = sizeof(struct cq_enet_wq_desc) 474 * scctx->isc_ntxd[i]; 475 scctx->isc_ntxd[i + enic->conf_wq_count] = 476 enic->config.wq_desc_count; 477 scctx->isc_txqsizes[i + enic->conf_wq_count] = 478 sizeof(struct cq_desc) * scctx->isc_ntxd[i + 479 enic->conf_wq_count]; 480 } 481 for (i = 0; i < enic->conf_rq_count; i++) { 482 scctx->isc_nrxd[i] = enic->config.rq_desc_count; 483 scctx->isc_rxqsizes[i] = sizeof(struct cq_enet_rq_desc) * 484 scctx->isc_nrxd[i]; 485 scctx->isc_nrxd[i + enic->conf_rq_count] = 486 enic->config.rq_desc_count; 487 scctx->isc_rxqsizes[i + enic->conf_rq_count] = sizeof(struct 488 cq_desc) * scctx->isc_nrxd[i + enic->conf_rq_count]; 489 } 490 scctx->isc_tx_nsegments = 31; 491 492 scctx->isc_msix_bar = -1; 493 494 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 495 ifmedia_add(softc->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); 496 ifmedia_add(softc->media, IFM_ETHER | IFM_10_FL, 0, NULL); 497 498 err = enic_allocate_msix(softc); 499 if (err) { 500 dev_err(enic, "Failed to allocate MSIX, aborting\n"); 501 goto err_out_dev_close; 502 } 503 504 return (rc); 505 506 err_out_dev_close: 507 vnic_dev_close(enic->vdev); 508 vnic_dev_deinit_devcmd2(enic->vdev); 509 err_out_unregister: 510 free(softc->vdev.devcmd, M_DEVBUF); 511 free(softc->enic.intr_queues, M_DEVBUF); 512 free(softc->enic.cq, M_DEVBUF); 513 free(softc->mta, M_DEVBUF); 514 rc = -1; 515 pci_disable_busmaster(softc->dev); 516 enic_pci_mapping_free(softc); 517 mtx_destroy(&softc->enic_lock); 518 return (rc); 519 } 520 521 static int 522 enic_msix_intr_assign(if_ctx_t ctx, int msix) 523 { 524 struct enic_softc *softc; 525 struct enic *enic; 526 if_softc_ctx_t scctx; 527 528 int error; 529 int i; 530 char irq_name[16]; 531 532 softc = iflib_get_softc(ctx); 533 enic = &softc->enic; 534 scctx = softc->scctx; 535 536 ENIC_LOCK(softc); 537 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSIX); 538 ENIC_UNLOCK(softc); 539 540 if (enic->intr_queues == NULL) 541 enic->intr_queues = malloc(sizeof(*enic->intr_queues) * 542 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO); 543 enic->intr = malloc(sizeof(*enic->intr) * msix, M_DEVBUF, M_NOWAIT 544 | M_ZERO); 545 if (enic->intr_queues == NULL || enic->intr == NULL) 546 return (ENOMEM); 547 for (i = 0; i < scctx->isc_nrxqsets; i++) { 548 snprintf(irq_name, sizeof(irq_name), "erxq%d:%d", i, 549 device_get_unit(softc->dev)); 550 551 error = iflib_irq_alloc_generic(ctx, 552 &enic->intr_queues[i].intr_irq, i + 1, IFLIB_INTR_RX, 553 enic_rxq_intr, &enic->rq[i], i, irq_name); 554 if (error) { 555 device_printf(iflib_get_dev(ctx), 556 "Failed to register rxq %d interrupt handler\n", i); 557 return (error); 558 } 559 enic->intr[i].index = i; 560 enic->intr[i].vdev = enic->vdev; 561 ENIC_LOCK(softc); 562 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, 563 RES_TYPE_INTR_CTRL, i); 564 vnic_intr_mask(&enic->intr[i]); 565 ENIC_UNLOCK(softc); 566 } 567 568 for (i = scctx->isc_nrxqsets; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) { 569 snprintf(irq_name, sizeof(irq_name), "etxq%d:%d", i - 570 scctx->isc_nrxqsets, device_get_unit(softc->dev)); 571 572 iflib_softirq_alloc_generic(ctx, 573 &enic->intr_queues[i].intr_irq, IFLIB_INTR_TX, 574 &enic->wq[i - scctx->isc_nrxqsets], i - scctx->isc_nrxqsets, 575 irq_name); 576 577 enic->intr[i].index = i; 578 enic->intr[i].vdev = enic->vdev; 579 ENIC_LOCK(softc); 580 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, 581 RES_TYPE_INTR_CTRL, i); 582 vnic_intr_mask(&enic->intr[i]); 583 ENIC_UNLOCK(softc); 584 } 585 586 i = scctx->isc_nrxqsets + scctx->isc_ntxqsets; 587 error = iflib_irq_alloc_generic(ctx, &softc->enic_event_intr_irq, 588 i + 1, IFLIB_INTR_ADMIN, enic_event_intr, softc, 0, "event"); 589 if (error) { 590 device_printf(iflib_get_dev(ctx), 591 "Failed to register event interrupt handler\n"); 592 return (error); 593 } 594 595 enic->intr[i].index = i; 596 enic->intr[i].vdev = enic->vdev; 597 ENIC_LOCK(softc); 598 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL, 599 i); 600 vnic_intr_mask(&enic->intr[i]); 601 ENIC_UNLOCK(softc); 602 603 i++; 604 error = iflib_irq_alloc_generic(ctx, &softc->enic_err_intr_irq, 605 i + 1, IFLIB_INTR_ADMIN, enic_err_intr, softc, 0, "err"); 606 if (error) { 607 device_printf(iflib_get_dev(ctx), 608 "Failed to register event interrupt handler\n"); 609 return (error); 610 } 611 enic->intr[i].index = i; 612 enic->intr[i].vdev = enic->vdev; 613 ENIC_LOCK(softc); 614 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL, 615 i); 616 vnic_intr_mask(&enic->intr[i]); 617 ENIC_UNLOCK(softc); 618 619 enic->intr_count = msix; 620 621 return (0); 622 } 623 624 static void 625 enic_free_irqs(struct enic_softc *softc) 626 { 627 if_softc_ctx_t scctx; 628 629 struct enic *enic; 630 int i; 631 632 scctx = softc->scctx; 633 enic = &softc->enic; 634 635 if (enic->intr_queues != NULL) { 636 for (i = 0; 637 i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) 638 iflib_irq_free(softc->ctx, 639 &enic->intr_queues[i].intr_irq); 640 } 641 642 iflib_irq_free(softc->ctx, &softc->enic_event_intr_irq); 643 iflib_irq_free(softc->ctx, &softc->enic_err_intr_irq); 644 free(enic->intr_queues, M_DEVBUF); 645 enic->intr_queues = NULL; 646 free(enic->intr, M_DEVBUF); 647 enic->intr = NULL; 648 } 649 650 static int 651 enic_attach_post(if_ctx_t ctx) 652 { 653 struct enic *enic; 654 struct enic_softc *softc; 655 int error = 0; 656 657 softc = iflib_get_softc(ctx); 658 enic = &softc->enic; 659 660 enic_setup_sysctl(softc); 661 662 enic_init_vnic_resources(enic); 663 enic_set_rx_coal_setting(enic); 664 enic_setup_finish(enic); 665 666 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 667 ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO); 668 669 return (error); 670 } 671 672 static int 673 enic_detach(if_ctx_t ctx) 674 { 675 struct enic_softc *softc; 676 struct enic *enic; 677 678 softc = iflib_get_softc(ctx); 679 enic = &softc->enic; 680 681 vnic_dev_notify_unset(enic->vdev); 682 683 enic_free_irqs(softc); 684 685 ENIC_LOCK(softc); 686 vnic_dev_deinit(enic->vdev); 687 vnic_dev_close(enic->vdev); 688 vnic_dev_deinit_devcmd2(enic->vdev); 689 free(softc->vdev.devcmd, M_DEVBUF); 690 softc->vdev.devcmd = NULL; 691 pci_disable_busmaster(softc->dev); 692 enic_pci_mapping_free(softc); 693 ENIC_UNLOCK(softc); 694 if (softc->vdev.stats_res.idi_size != 0) { 695 iflib_dma_free(&softc->vdev.stats_res); 696 softc->vdev.stats = NULL; 697 } 698 if (softc->vdev.flow_counters_res.idi_size != 0) { 699 iflib_dma_free(&softc->vdev.flow_counters_res); 700 softc->vdev.flow_counters = NULL; 701 } 702 free(softc->mta, M_DEVBUF); 703 softc->mta = NULL; 704 mtx_destroy(&softc->enic_lock); 705 706 return 0; 707 } 708 709 static int 710 enic_tx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs, 711 int ntxqs, int ntxqsets) 712 { 713 struct enic_softc *softc; 714 int q; 715 716 softc = iflib_get_softc(ctx); 717 softc->enic.cq = malloc(sizeof(*softc->enic.cq) * 718 (softc->enic.wq_count + softc->enic.rq_count), M_DEVBUF, 719 M_NOWAIT | M_ZERO); 720 if (softc->enic.cq == NULL) 721 return (ENOMEM); 722 /* Allocate the array of transmit queues */ 723 softc->enic.wq = malloc(sizeof(struct vnic_wq) * 724 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 725 if (softc->enic.wq == NULL) { 726 free(softc->enic.cq, M_DEVBUF); 727 softc->enic.cq = NULL; 728 return (ENOMEM); 729 } 730 731 /* Initialize driver state for each transmit queue */ 732 733 /* 734 * Allocate queue state that is shared with the device. This check 735 * and call is performed in both enic_tx_queues_alloc() and 736 * enic_rx_queues_alloc() so that we don't have to care which order 737 * iflib invokes those routines in. 738 */ 739 740 /* Record descriptor ring vaddrs and paddrs */ 741 ENIC_LOCK(softc); 742 for (q = 0; q < ntxqsets; q++) { 743 struct vnic_wq *wq; 744 struct vnic_cq *cq; 745 unsigned int cq_wq; 746 747 wq = &softc->enic.wq[q]; 748 cq_wq = enic_cq_wq(&softc->enic, q); 749 cq = &softc->enic.cq[cq_wq]; 750 751 /* Completion ring */ 752 wq->vdev = softc->enic.vdev; 753 wq->index = q; 754 wq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_WQ, 755 wq->index); 756 vnic_wq_disable(wq); 757 758 wq->ring.desc_size = sizeof(struct wq_enet_desc); 759 wq->ring.desc_count = softc->scctx->isc_ntxd[q]; 760 wq->ring.desc_avail = wq->ring.desc_count - 1; 761 wq->ring.last_count = wq->ring.desc_count; 762 wq->head_idx = 0; 763 wq->tail_idx = 0; 764 765 wq->ring.descs = vaddrs[q * ntxqs + 0]; 766 wq->ring.base_addr = paddrs[q * ntxqs + 0]; 767 768 /* Command ring */ 769 cq->vdev = softc->enic.vdev; 770 cq->index = cq_wq; 771 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, 772 RES_TYPE_CQ, cq->index); 773 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc); 774 cq->ring.desc_count = softc->scctx->isc_ntxd[q]; 775 cq->ring.desc_avail = cq->ring.desc_count - 1; 776 777 cq->ring.descs = vaddrs[q * ntxqs + 1]; 778 cq->ring.base_addr = paddrs[q * ntxqs + 1]; 779 780 } 781 782 ENIC_UNLOCK(softc); 783 784 return (0); 785 } 786 787 788 789 static int 790 enic_rx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs, 791 int nrxqs, int nrxqsets) 792 { 793 struct enic_softc *softc; 794 int q; 795 796 softc = iflib_get_softc(ctx); 797 /* Allocate the array of receive queues */ 798 softc->enic.rq = malloc(sizeof(struct vnic_rq) * nrxqsets, M_DEVBUF, 799 M_NOWAIT | M_ZERO); 800 if (softc->enic.rq == NULL) { 801 free(softc->enic.wq, M_DEVBUF); 802 softc->enic.wq = NULL; 803 free(softc->enic.cq, M_DEVBUF); 804 softc->enic.cq = NULL; 805 return (ENOMEM); 806 } 807 808 /* Initialize driver state for each receive queue */ 809 810 /* 811 * Allocate queue state that is shared with the device. This check 812 * and call is performed in both enic_tx_queues_alloc() and 813 * enic_rx_queues_alloc() so that we don't have to care which order 814 * iflib invokes those routines in. 815 */ 816 817 /* Record descriptor ring vaddrs and paddrs */ 818 ENIC_LOCK(softc); 819 for (q = 0; q < nrxqsets; q++) { 820 struct vnic_rq *rq; 821 struct vnic_cq *cq; 822 unsigned int cq_rq; 823 824 rq = &softc->enic.rq[q]; 825 cq_rq = enic_cq_rq(&softc->enic, q); 826 cq = &softc->enic.cq[cq_rq]; 827 828 /* Completion ring */ 829 cq->vdev = softc->enic.vdev; 830 cq->index = cq_rq; 831 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_CQ, 832 cq->index); 833 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc); 834 cq->ring.desc_count = softc->scctx->isc_nrxd[1]; 835 cq->ring.desc_avail = cq->ring.desc_count - 1; 836 837 cq->ring.descs = vaddrs[q * nrxqs + 0]; 838 cq->ring.base_addr = paddrs[q * nrxqs + 0]; 839 840 /* Command ring(s) */ 841 rq->vdev = softc->enic.vdev; 842 843 rq->index = q; 844 rq->ctrl = vnic_dev_get_res(softc->enic.vdev, 845 RES_TYPE_RQ, rq->index); 846 vnic_rq_disable(rq); 847 848 rq->ring.desc_size = sizeof(struct rq_enet_desc); 849 rq->ring.desc_count = softc->scctx->isc_nrxd[0]; 850 rq->ring.desc_avail = rq->ring.desc_count - 1; 851 852 rq->ring.descs = vaddrs[q * nrxqs + 1]; 853 rq->ring.base_addr = paddrs[q * nrxqs + 1]; 854 rq->need_initial_post = true; 855 } 856 857 ENIC_UNLOCK(softc); 858 859 return (0); 860 } 861 862 static void 863 enic_queues_free(if_ctx_t ctx) 864 { 865 struct enic_softc *softc; 866 softc = iflib_get_softc(ctx); 867 868 free(softc->enic.rq, M_DEVBUF); 869 softc->enic.rq = NULL; 870 free(softc->enic.wq, M_DEVBUF); 871 softc->enic.wq = NULL; 872 free(softc->enic.cq, M_DEVBUF); 873 softc->enic.cq = NULL; 874 } 875 876 static int 877 enic_rxq_intr(void *rxq) 878 { 879 struct vnic_rq *rq; 880 881 rq = (struct vnic_rq *)rxq; 882 if (!iflib_is_running(rq->vdev->softc->ctx)) 883 return (FILTER_HANDLED); 884 885 return (FILTER_SCHEDULE_THREAD); 886 } 887 888 static int 889 enic_event_intr(void *vsc) 890 { 891 struct enic_softc *softc; 892 struct enic *enic; 893 uint32_t mtu; 894 895 softc = vsc; 896 enic = &softc->enic; 897 898 mtu = vnic_dev_mtu(enic->vdev); 899 if (mtu && mtu != enic->port_mtu) { 900 enic->port_mtu = mtu; 901 } 902 903 enic_link_status(softc); 904 905 return (FILTER_HANDLED); 906 } 907 908 static int 909 enic_err_intr(void *vsc) 910 { 911 struct enic_softc *softc; 912 913 softc = vsc; 914 915 iflib_request_reset_if_up(softc->ctx); 916 iflib_admin_intr_deferred(softc->ctx); 917 918 return (FILTER_HANDLED); 919 } 920 921 static void 922 enic_stop(if_ctx_t ctx) 923 { 924 struct enic_softc *softc; 925 struct enic *enic; 926 if_softc_ctx_t scctx; 927 unsigned int index; 928 struct vnic_wq *wq; 929 struct vnic_rq *rq; 930 struct vnic_cq *cq; 931 unsigned int cq_wq, cq_rq; 932 933 934 softc = iflib_get_softc(ctx); 935 scctx = softc->scctx; 936 enic = &softc->enic; 937 938 if (softc->stopped) 939 return; 940 softc->link_active = 0; 941 softc->stopped = 1; 942 943 enic_dev_disable(enic); 944 945 for (index = 0; index < scctx->isc_ntxqsets; index++) { 946 enic_stop_wq(enic, index); 947 vnic_wq_clean(&enic->wq[index]); 948 vnic_cq_clean(&enic->cq[enic_cq_wq(enic, index)]); 949 950 wq = &softc->enic.wq[index]; 951 wq->ring.desc_avail = wq->ring.desc_count - 1; 952 wq->ring.last_count = wq->ring.desc_count; 953 wq->head_idx = 0; 954 wq->tail_idx = 0; 955 956 cq_wq = enic_cq_wq(&softc->enic, index); 957 cq = &softc->enic.cq[cq_wq]; 958 cq->ring.desc_avail = cq->ring.desc_count - 1; 959 } 960 961 for (index = 0; index < scctx->isc_nrxqsets; index++) { 962 enic_stop_rq(enic, index); 963 vnic_rq_clean(&enic->rq[index]); 964 vnic_cq_clean(&enic->cq[enic_cq_rq(enic, index)]); 965 966 rq = &softc->enic.rq[index]; 967 cq_rq = enic_cq_rq(&softc->enic, index); 968 cq = &softc->enic.cq[cq_rq]; 969 970 cq->ring.desc_avail = cq->ring.desc_count - 1; 971 rq->ring.desc_avail = rq->ring.desc_count - 1; 972 rq->need_initial_post = true; 973 } 974 975 for (index = 0; index < scctx->isc_vectors; index++) { 976 vnic_intr_clean(&enic->intr[index]); 977 } 978 } 979 980 static void 981 enic_init(if_ctx_t ctx) 982 { 983 struct enic_softc *softc; 984 struct enic *enic; 985 if_softc_ctx_t scctx; 986 unsigned int index; 987 int error; 988 989 softc = iflib_get_softc(ctx); 990 scctx = softc->scctx; 991 enic = &softc->enic; 992 993 994 enic_init_vnic_resources(enic); 995 996 for (index = 0; index < scctx->isc_ntxqsets; index++) 997 enic_prep_wq_for_simple_tx(&softc->enic, index); 998 999 for (index = 0; index < scctx->isc_ntxqsets; index++) 1000 enic_start_wq(enic, index); 1001 1002 for (index = 0; index < scctx->isc_nrxqsets; index++) 1003 enic_start_rq(enic, index); 1004 1005 /* Use the current MAC address. */ 1006 bcopy(if_getlladdr(softc->ifp), softc->lladdr, ETHER_ADDR_LEN); 1007 enic_set_lladdr(softc); 1008 1009 /* Queue setup above needs normal stop cleanup even if enable fails. */ 1010 softc->stopped = 0; 1011 ENIC_LOCK(softc); 1012 error = vnic_dev_enable_wait(enic->vdev); 1013 ENIC_UNLOCK(softc); 1014 if (error != 0) { 1015 device_printf(softc->dev, "Device enable failed: %d\n", error); 1016 enic_stop(ctx); 1017 iflib_init_failed(ctx); 1018 return; 1019 } 1020 1021 enic_link_status(softc); 1022 } 1023 1024 static void 1025 enic_del_mcast(struct enic_softc *softc) { 1026 struct enic *enic; 1027 int i; 1028 1029 enic = &softc->enic; 1030 for (i=0; i < softc->mc_count; i++) { 1031 vnic_dev_del_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]); 1032 } 1033 softc->multicast = 0; 1034 softc->mc_count = 0; 1035 } 1036 1037 static void 1038 enic_add_mcast(struct enic_softc *softc) { 1039 struct enic *enic; 1040 int i; 1041 1042 enic = &softc->enic; 1043 for (i=0; i < softc->mc_count; i++) { 1044 vnic_dev_add_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]); 1045 } 1046 softc->multicast = 1; 1047 } 1048 1049 static u_int 1050 enic_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 1051 { 1052 uint8_t *mta = arg; 1053 1054 if (idx == ENIC_MAX_MULTICAST_ADDRESSES) 1055 return (0); 1056 1057 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1058 return (1); 1059 } 1060 1061 static void 1062 enic_multi_set(if_ctx_t ctx) 1063 { 1064 if_t ifp; 1065 struct enic_softc *softc; 1066 u_int count; 1067 1068 softc = iflib_get_softc(ctx); 1069 ifp = iflib_get_ifp(ctx); 1070 1071 ENIC_LOCK(softc); 1072 enic_del_mcast(softc); 1073 count = if_foreach_llmaddr(ifp, enic_copy_maddr, softc->mta); 1074 softc->mc_count = count; 1075 enic_add_mcast(softc); 1076 ENIC_UNLOCK(softc); 1077 1078 if (if_getflags(ifp) & IFF_PROMISC) { 1079 softc->promisc = 1; 1080 } else { 1081 softc->promisc = 0; 1082 } 1083 if (if_getflags(ifp) & IFF_ALLMULTI) { 1084 softc->allmulti = 1; 1085 } else { 1086 softc->allmulti = 0; 1087 } 1088 enic_update_packet_filter(&softc->enic); 1089 } 1090 1091 static int 1092 enic_mtu_set(if_ctx_t ctx, uint32_t mtu) 1093 { 1094 struct enic_softc *softc; 1095 struct enic *enic; 1096 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 1097 1098 softc = iflib_get_softc(ctx); 1099 enic = &softc->enic; 1100 1101 if (mtu > enic->port_mtu){ 1102 return (EINVAL); 1103 } 1104 1105 enic->config.mtu = mtu; 1106 scctx->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1107 1108 return (0); 1109 } 1110 1111 static void 1112 enic_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1113 { 1114 struct enic_softc *softc; 1115 struct ifmedia_entry *next; 1116 uint32_t speed; 1117 uint64_t target_baudrate; 1118 1119 softc = iflib_get_softc(ctx); 1120 1121 ifmr->ifm_status = IFM_AVALID; 1122 ifmr->ifm_active = IFM_ETHER; 1123 1124 if (enic_link_is_up(softc) != 0) { 1125 ENIC_LOCK(softc); 1126 speed = vnic_dev_port_speed(&softc->vdev); 1127 ENIC_UNLOCK(softc); 1128 target_baudrate = 1000ull * speed; 1129 LIST_FOREACH(next, &(iflib_get_media(ctx)->ifm_list), ifm_list) { 1130 if (ifmedia_baudrate(next->ifm_media) == target_baudrate) { 1131 ifmr->ifm_active |= next->ifm_media; 1132 } 1133 } 1134 1135 ifmr->ifm_status |= IFM_ACTIVE; 1136 ifmr->ifm_active |= IFM_AUTO; 1137 } else 1138 ifmr->ifm_active |= IFM_NONE; 1139 } 1140 1141 static int 1142 enic_media_change(if_ctx_t ctx) 1143 { 1144 return (ENODEV); 1145 } 1146 1147 static int 1148 enic_promisc_set(if_ctx_t ctx, int flags) 1149 { 1150 if_t ifp; 1151 struct enic_softc *softc; 1152 1153 softc = iflib_get_softc(ctx); 1154 ifp = iflib_get_ifp(ctx); 1155 1156 if (if_getflags(ifp) & IFF_PROMISC) { 1157 softc->promisc = 1; 1158 } else { 1159 softc->promisc = 0; 1160 } 1161 if (if_getflags(ifp) & IFF_ALLMULTI) { 1162 softc->allmulti = 1; 1163 } else { 1164 softc->allmulti = 0; 1165 } 1166 enic_update_packet_filter(&softc->enic); 1167 1168 return (0); 1169 } 1170 1171 static uint64_t 1172 enic_get_counter(if_ctx_t ctx, ift_counter cnt) { 1173 if_t ifp = iflib_get_ifp(ctx); 1174 1175 if (cnt < IFCOUNTERS) 1176 return if_get_counter_default(ifp, cnt); 1177 1178 return (0); 1179 } 1180 1181 static void 1182 enic_update_admin_status(if_ctx_t ctx) 1183 { 1184 struct enic_softc *softc; 1185 softc = iflib_get_softc(ctx); 1186 1187 enic_link_status(softc); 1188 } 1189 1190 static void 1191 enic_txq_timer(if_ctx_t ctx, uint16_t qid) 1192 { 1193 1194 struct enic_softc *softc; 1195 struct enic *enic; 1196 struct vnic_stats *stats; 1197 int ret; 1198 1199 softc = iflib_get_softc(ctx); 1200 enic = &softc->enic; 1201 1202 ENIC_LOCK(softc); 1203 ret = vnic_dev_stats_dump(enic->vdev, &stats); 1204 ENIC_UNLOCK(softc); 1205 if (ret) { 1206 dev_err(enic, "Error in getting stats\n"); 1207 } 1208 } 1209 1210 static int 1211 enic_link_is_up(struct enic_softc *softc) 1212 { 1213 return (vnic_dev_link_status(&softc->vdev) == 1); 1214 } 1215 1216 static void 1217 enic_link_status(struct enic_softc *softc) 1218 { 1219 if_ctx_t ctx; 1220 uint64_t speed; 1221 int link; 1222 1223 ctx = softc->ctx; 1224 link = enic_link_is_up(softc); 1225 speed = IF_Gbps(10); 1226 1227 ENIC_LOCK(softc); 1228 speed = vnic_dev_port_speed(&softc->vdev); 1229 ENIC_UNLOCK(softc); 1230 1231 if (link != 0 && softc->link_active == 0) { 1232 softc->link_active = 1; 1233 iflib_link_state_change(ctx, LINK_STATE_UP, speed); 1234 } else if (link == 0 && softc->link_active != 0) { 1235 softc->link_active = 0; 1236 iflib_link_state_change(ctx, LINK_STATE_DOWN, speed); 1237 } 1238 } 1239 1240 static void 1241 enic_set_lladdr(struct enic_softc *softc) 1242 { 1243 struct enic *enic; 1244 enic = &softc->enic; 1245 1246 ENIC_LOCK(softc); 1247 vnic_dev_add_addr(enic->vdev, softc->lladdr); 1248 ENIC_UNLOCK(softc); 1249 } 1250 1251 1252 static void 1253 enic_setup_txq_sysctl(struct vnic_wq *wq, int i, struct sysctl_ctx_list *ctx, 1254 struct sysctl_oid_list *child) 1255 { 1256 struct sysctl_oid *txsnode; 1257 struct sysctl_oid_list *txslist; 1258 struct vnic_stats *stats; 1259 1260 stats = wq[i].vdev->stats; 1261 1262 txsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats", 1263 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 1264 txslist = SYSCTL_CHILDREN(txsnode); 1265 1266 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_frames_ok", CTLFLAG_RD, 1267 &stats->tx.tx_frames_ok, "TX Frames OK"); 1268 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_frames_ok", CTLFLAG_RD, 1269 &stats->tx.tx_unicast_frames_ok, "TX unicast frames OK"); 1270 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_frames_ok", CTLFLAG_RD, 1271 &stats->tx.tx_multicast_frames_ok, "TX multicast framse OK"); 1272 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_frames_ok", CTLFLAG_RD, 1273 &stats->tx.tx_broadcast_frames_ok, "TX Broadcast frames OK"); 1274 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_bytes_ok", CTLFLAG_RD, 1275 &stats->tx.tx_bytes_ok, "TX bytes OK "); 1276 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_bytes_ok", CTLFLAG_RD, 1277 &stats->tx.tx_unicast_bytes_ok, "TX unicast bytes OK"); 1278 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_bytes_ok", CTLFLAG_RD, 1279 &stats->tx.tx_multicast_bytes_ok, "TX multicast bytes OK"); 1280 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_bytes_ok", CTLFLAG_RD, 1281 &stats->tx.tx_broadcast_bytes_ok, "TX broadcast bytes OK"); 1282 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_drops", CTLFLAG_RD, 1283 &stats->tx.tx_drops, "TX drops"); 1284 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_errors", CTLFLAG_RD, 1285 &stats->tx.tx_errors, "TX errors"); 1286 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_tso", CTLFLAG_RD, 1287 &stats->tx.tx_tso, "TX TSO"); 1288 } 1289 1290 static void 1291 enic_setup_rxq_sysctl(struct vnic_rq *rq, int i, struct sysctl_ctx_list *ctx, 1292 struct sysctl_oid_list *child) 1293 { 1294 struct sysctl_oid *rxsnode; 1295 struct sysctl_oid_list *rxslist; 1296 struct vnic_stats *stats; 1297 1298 stats = rq[i].vdev->stats; 1299 1300 rxsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats", 1301 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 1302 rxslist = SYSCTL_CHILDREN(rxsnode); 1303 1304 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_ok", CTLFLAG_RD, 1305 &stats->rx.rx_frames_ok, "RX Frames OK"); 1306 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_total", CTLFLAG_RD, 1307 &stats->rx.rx_frames_total, "RX frames total"); 1308 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_frames_ok", CTLFLAG_RD, 1309 &stats->rx.rx_unicast_frames_ok, "RX unicast frames ok"); 1310 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_frames_ok", CTLFLAG_RD, 1311 &stats->rx.rx_multicast_frames_ok, "RX multicast Frames ok"); 1312 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_frames_ok", CTLFLAG_RD, 1313 &stats->rx.rx_broadcast_frames_ok, "RX broadcast frames ok"); 1314 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_bytes_ok", CTLFLAG_RD, 1315 &stats->rx.rx_bytes_ok, "RX bytes ok"); 1316 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_bytes_ok", CTLFLAG_RD, 1317 &stats->rx.rx_unicast_bytes_ok, "RX unicast bytes ok"); 1318 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_bytes_ok", CTLFLAG_RD, 1319 &stats->rx.rx_multicast_bytes_ok, "RX multicast bytes ok"); 1320 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_bytes_ok", CTLFLAG_RD, 1321 &stats->rx.rx_broadcast_bytes_ok, "RX broadcast bytes ok"); 1322 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_drop", CTLFLAG_RD, 1323 &stats->rx.rx_drop, "RX drop"); 1324 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_errors", CTLFLAG_RD, 1325 &stats->rx.rx_errors, "RX errors"); 1326 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_rss", CTLFLAG_RD, 1327 &stats->rx.rx_rss, "RX rss"); 1328 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_crc_errors", CTLFLAG_RD, 1329 &stats->rx.rx_crc_errors, "RX crc errors"); 1330 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_64", CTLFLAG_RD, 1331 &stats->rx.rx_frames_64, "RX frames 64"); 1332 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_127", CTLFLAG_RD, 1333 &stats->rx.rx_frames_127, "RX frames 127"); 1334 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_255", CTLFLAG_RD, 1335 &stats->rx.rx_frames_255, "RX frames 255"); 1336 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_511", CTLFLAG_RD, 1337 &stats->rx.rx_frames_511, "RX frames 511"); 1338 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1023", CTLFLAG_RD, 1339 &stats->rx.rx_frames_1023, "RX frames 1023"); 1340 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1518", CTLFLAG_RD, 1341 &stats->rx.rx_frames_1518, "RX frames 1518"); 1342 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_to_max", CTLFLAG_RD, 1343 &stats->rx.rx_frames_to_max, "RX frames to max"); 1344 } 1345 1346 static void 1347 enic_setup_queue_sysctl(struct enic_softc *softc, struct sysctl_ctx_list *ctx, 1348 struct sysctl_oid_list *child) 1349 { 1350 enic_setup_txq_sysctl(softc->enic.wq, 0, ctx, child); 1351 enic_setup_rxq_sysctl(softc->enic.rq, 0, ctx, child); 1352 } 1353 1354 static void 1355 enic_setup_sysctl(struct enic_softc *softc) 1356 { 1357 device_t dev; 1358 struct sysctl_ctx_list *ctx; 1359 struct sysctl_oid *tree; 1360 struct sysctl_oid_list *child; 1361 1362 dev = softc->dev; 1363 ctx = device_get_sysctl_ctx(dev); 1364 tree = device_get_sysctl_tree(dev); 1365 child = SYSCTL_CHILDREN(tree); 1366 1367 enic_setup_queue_sysctl(softc, ctx, child); 1368 } 1369 1370 static void 1371 enic_enable_intr(struct enic_softc *softc, int irq) 1372 { 1373 struct enic *enic = &softc->enic; 1374 1375 vnic_intr_unmask(&enic->intr[irq]); 1376 vnic_intr_return_all_credits(&enic->intr[irq]); 1377 } 1378 1379 static void 1380 enic_disable_intr(struct enic_softc *softc, int irq) 1381 { 1382 struct enic *enic = &softc->enic; 1383 1384 vnic_intr_mask(&enic->intr[irq]); 1385 vnic_intr_masked(&enic->intr[irq]); /* flush write */ 1386 } 1387 1388 static int 1389 enic_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 1390 { 1391 struct enic_softc *softc; 1392 if_softc_ctx_t scctx; 1393 1394 softc = iflib_get_softc(ctx); 1395 scctx = softc->scctx; 1396 1397 enic_enable_intr(softc, qid + scctx->isc_nrxqsets); 1398 1399 return 0; 1400 } 1401 1402 static int 1403 enic_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 1404 { 1405 struct enic_softc *softc; 1406 1407 softc = iflib_get_softc(ctx); 1408 enic_enable_intr(softc, qid); 1409 1410 return 0; 1411 } 1412 1413 static void 1414 enic_intr_enable_all(if_ctx_t ctx) 1415 { 1416 struct enic_softc *softc; 1417 if_softc_ctx_t scctx; 1418 int i; 1419 1420 softc = iflib_get_softc(ctx); 1421 scctx = softc->scctx; 1422 1423 for (i = 0; i < scctx->isc_vectors; i++) { 1424 enic_enable_intr(softc, i); 1425 } 1426 } 1427 1428 static void 1429 enic_intr_disable_all(if_ctx_t ctx) 1430 { 1431 struct enic_softc *softc; 1432 if_softc_ctx_t scctx; 1433 int i; 1434 1435 softc = iflib_get_softc(ctx); 1436 scctx = softc->scctx; 1437 /* 1438 * iflib may invoke this routine before enic_attach_post() has run, 1439 * which is before the top level shared data area is initialized and 1440 * the device made aware of it. 1441 */ 1442 1443 for (i = 0; i < scctx->isc_vectors; i++) { 1444 enic_disable_intr(softc, i); 1445 } 1446 } 1447 1448 static int 1449 enic_dev_open(struct enic *enic) 1450 { 1451 int err; 1452 int flags = CMD_OPENF_IG_DESCCACHE; 1453 1454 err = enic_dev_wait(enic->vdev, vnic_dev_open, 1455 vnic_dev_open_done, flags); 1456 if (err) 1457 dev_err(enic_get_dev(enic), 1458 "vNIC device open failed, err %d\n", err); 1459 1460 return err; 1461 } 1462 1463 static int 1464 enic_dev_init(struct enic *enic) 1465 { 1466 int err; 1467 1468 vnic_dev_intr_coal_timer_info_default(enic->vdev); 1469 1470 /* 1471 * Get vNIC configuration 1472 */ 1473 err = enic_get_vnic_config(enic); 1474 if (err) { 1475 dev_err(dev, "Get vNIC configuration failed, aborting\n"); 1476 return err; 1477 } 1478 1479 /* Get available resource counts */ 1480 enic_get_res_counts(enic); 1481 1482 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */ 1483 enic->intr_queues = malloc(sizeof(*enic->intr_queues) * 1484 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO); 1485 1486 vnic_dev_set_reset_flag(enic->vdev, 0); 1487 enic->max_flow_counter = -1; 1488 1489 /* set up link status checking */ 1490 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */ 1491 1492 enic->overlay_offload = false; 1493 if (enic->disable_overlay && enic->vxlan) { 1494 /* 1495 * Explicitly disable overlay offload as the setting is 1496 * sticky, and resetting vNIC does not disable it. 1497 */ 1498 if (vnic_dev_overlay_offload_ctrl(enic->vdev, 1499 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_DISABLE)) { 1500 dev_err(enic, "failed to disable overlay offload\n"); 1501 } else { 1502 dev_info(enic, "Overlay offload is disabled\n"); 1503 } 1504 } 1505 if (!enic->disable_overlay && enic->vxlan && 1506 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */ 1507 vnic_dev_overlay_offload_ctrl(enic->vdev, 1508 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_ENABLE) == 0) { 1509 enic->overlay_offload = true; 1510 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT; 1511 dev_info(enic, "Overlay offload is enabled\n"); 1512 /* 1513 * Reset the vxlan port to the default, as the NIC firmware 1514 * does not reset it automatically and keeps the old setting. 1515 */ 1516 if (vnic_dev_overlay_offload_cfg(enic->vdev, 1517 OVERLAY_CFG_VXLAN_PORT_UPDATE, ENIC_DEFAULT_VXLAN_PORT)) { 1518 dev_err(enic, "failed to update vxlan port\n"); 1519 return (EINVAL); 1520 } 1521 } 1522 return 0; 1523 } 1524 1525 static void * 1526 enic_alloc_consistent(void *priv, size_t size, bus_addr_t * dma_handle, 1527 struct iflib_dma_info *res, u8 * name) 1528 { 1529 void *vaddr; 1530 *dma_handle = 0; 1531 struct enic *enic = (struct enic *)priv; 1532 int rz; 1533 1534 rz = iflib_dma_alloc(enic->softc->ctx, size, res, BUS_DMA_NOWAIT); 1535 if (rz) { 1536 pr_err("%s : Failed to allocate memory requested for %s\n", 1537 __func__, name); 1538 return NULL; 1539 } 1540 1541 vaddr = res->idi_vaddr; 1542 *dma_handle = res->idi_paddr; 1543 1544 return vaddr; 1545 } 1546 1547 static void 1548 enic_free_consistent(void *priv, size_t size, void *vaddr, 1549 bus_addr_t dma_handle, struct iflib_dma_info *res) 1550 { 1551 iflib_dma_free(res); 1552 } 1553 1554 static int 1555 enic_pci_mapping(struct enic_softc *softc) 1556 { 1557 int rc; 1558 1559 rc = enic_map_bar(softc, &softc->mem, 0, true); 1560 if (rc) 1561 return rc; 1562 1563 rc = enic_map_bar(softc, &softc->io, 2, false); 1564 1565 return rc; 1566 } 1567 1568 static void 1569 enic_pci_mapping_free(struct enic_softc *softc) 1570 { 1571 if (softc->mem.res != NULL) 1572 bus_release_resource(softc->dev, SYS_RES_MEMORY, 1573 softc->mem.rid, softc->mem.res); 1574 softc->mem.res = NULL; 1575 1576 if (softc->io.res != NULL) 1577 bus_release_resource(softc->dev, SYS_RES_MEMORY, 1578 softc->io.rid, softc->io.res); 1579 softc->io.res = NULL; 1580 } 1581 1582 static int 1583 enic_dev_wait(struct vnic_dev *vdev, int (*start) (struct vnic_dev *, int), 1584 int (*finished) (struct vnic_dev *, int *), int arg) 1585 { 1586 int done; 1587 int err; 1588 int i; 1589 1590 err = start(vdev, arg); 1591 if (err) 1592 return err; 1593 1594 /* Wait for func to complete...2 seconds max */ 1595 for (i = 0; i < 2000; i++) { 1596 err = finished(vdev, &done); 1597 if (err) 1598 return err; 1599 if (done) 1600 return 0; 1601 usleep(1000); 1602 } 1603 return (ETIMEDOUT); 1604 } 1605 1606 static int 1607 enic_map_bar(struct enic_softc *softc, struct enic_bar_info *bar, int bar_num, 1608 bool shareable) 1609 { 1610 uint32_t flag; 1611 1612 if (bar->res != NULL) { 1613 device_printf(softc->dev, "Bar %d already mapped\n", bar_num); 1614 return (EDOOFUS); 1615 } 1616 1617 bar->rid = PCIR_BAR(bar_num); 1618 flag = RF_ACTIVE; 1619 if (shareable) 1620 flag |= RF_SHAREABLE; 1621 1622 if ((bar->res = bus_alloc_resource_any(softc->dev, 1623 SYS_RES_MEMORY, &bar->rid, flag)) == NULL) { 1624 device_printf(softc->dev, 1625 "PCI BAR%d mapping failure\n", bar_num); 1626 return (ENXIO); 1627 } 1628 bar->tag = rman_get_bustag(bar->res); 1629 bar->handle = rman_get_bushandle(bar->res); 1630 bar->size = rman_get_size(bar->res); 1631 1632 return 0; 1633 } 1634 1635 void 1636 enic_init_vnic_resources(struct enic *enic) 1637 { 1638 unsigned int error_interrupt_enable = 1; 1639 unsigned int error_interrupt_offset = 0; 1640 unsigned int rxq_interrupt_enable = 0; 1641 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET; 1642 unsigned int txq_interrupt_enable = 0; 1643 unsigned int txq_interrupt_offset; 1644 unsigned int index = 0; 1645 unsigned int cq_idx; 1646 if_softc_ctx_t scctx; 1647 1648 scctx = enic->softc->scctx; 1649 1650 rxq_interrupt_enable = 1; 1651 txq_interrupt_enable = 0; 1652 1653 rxq_interrupt_offset = 0; 1654 txq_interrupt_offset = scctx->isc_nrxqsets; 1655 1656 for (index = 0; index < enic->intr_count; index++) { 1657 vnic_intr_alloc(enic->vdev, &enic->intr[index], index); 1658 } 1659 1660 for (index = 0; index < scctx->isc_nrxqsets; index++) { 1661 cq_idx = enic_cq_rq(enic, index); 1662 1663 vnic_rq_clean(&enic->rq[index]); 1664 vnic_rq_init(&enic->rq[index], cq_idx, error_interrupt_enable, 1665 error_interrupt_offset); 1666 1667 vnic_cq_clean(&enic->cq[cq_idx]); 1668 vnic_cq_init(&enic->cq[cq_idx], 1669 0 /* flow_control_enable */ , 1670 1 /* color_enable */ , 1671 0 /* cq_head */ , 1672 0 /* cq_tail */ , 1673 1 /* cq_tail_color */ , 1674 rxq_interrupt_enable, 1675 1 /* cq_entry_enable */ , 1676 0 /* cq_message_enable */ , 1677 rxq_interrupt_offset, 1678 0 /* cq_message_addr */ ); 1679 if (rxq_interrupt_enable) 1680 rxq_interrupt_offset++; 1681 } 1682 1683 for (index = 0; index < scctx->isc_ntxqsets; index++) { 1684 cq_idx = enic_cq_wq(enic, index); 1685 vnic_wq_clean(&enic->wq[index]); 1686 vnic_wq_init(&enic->wq[index], cq_idx, error_interrupt_enable, 1687 error_interrupt_offset); 1688 /* Compute unsupported ol flags for enic_prep_pkts() */ 1689 enic->wq[index].tx_offload_notsup_mask = 0; 1690 1691 vnic_cq_clean(&enic->cq[cq_idx]); 1692 vnic_cq_init(&enic->cq[cq_idx], 1693 0 /* flow_control_enable */ , 1694 1 /* color_enable */ , 1695 0 /* cq_head */ , 1696 0 /* cq_tail */ , 1697 1 /* cq_tail_color */ , 1698 txq_interrupt_enable, 1699 1, 1700 0, 1701 txq_interrupt_offset, 1702 0 /* (u64)enic->wq[index].cqmsg_rz->iova */ ); 1703 1704 } 1705 1706 for (index = 0; index < enic->intr_count; index++) { 1707 vnic_intr_init(&enic->intr[index], 125, 1708 enic->config.intr_timer_type, /* mask_on_assertion */ 1); 1709 } 1710 } 1711 1712 static void 1713 enic_update_packet_filter(struct enic *enic) 1714 { 1715 struct enic_softc *softc = enic->softc; 1716 1717 ENIC_LOCK(softc); 1718 vnic_dev_packet_filter(enic->vdev, 1719 softc->directed, 1720 softc->multicast, 1721 softc->broadcast, 1722 softc->promisc, 1723 softc->allmulti); 1724 ENIC_UNLOCK(softc); 1725 } 1726 1727 static bool 1728 enic_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event) 1729 { 1730 switch (event) { 1731 case IFLIB_RESTART_VLAN_CONFIG: 1732 default: 1733 return (false); 1734 } 1735 } 1736 1737 int 1738 enic_setup_finish(struct enic *enic) 1739 { 1740 struct enic_softc *softc = enic->softc; 1741 1742 /* Default conf */ 1743 softc->directed = 1; 1744 softc->multicast = 0; 1745 softc->broadcast = 1; 1746 softc->promisc = 0; 1747 softc->allmulti = 1; 1748 enic_update_packet_filter(enic); 1749 1750 return 0; 1751 } 1752