1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 */ 5 6 #include "opt_rss.h" 7 8 #include <sys/param.h> 9 #include <sys/systm.h> 10 #include <sys/kernel.h> 11 #include <sys/endian.h> 12 #include <sys/sockio.h> 13 #include <sys/mbuf.h> 14 #include <sys/malloc.h> 15 #include <sys/module.h> 16 #include <sys/socket.h> 17 #include <sys/sysctl.h> 18 #include <sys/smp.h> 19 #include <vm/vm.h> 20 #include <vm/pmap.h> 21 22 #include <net/ethernet.h> 23 #include <net/if.h> 24 #include <net/if_var.h> 25 #include <net/if_arp.h> 26 #include <net/if_dl.h> 27 #include <net/if_types.h> 28 #include <net/if_media.h> 29 #include <net/if_vlan_var.h> 30 #include <net/iflib.h> 31 #ifdef RSS 32 #include <net/rss_config.h> 33 #endif 34 35 #include <netinet/in_systm.h> 36 #include <netinet/in.h> 37 #include <netinet/ip.h> 38 #include <netinet/ip6.h> 39 #include <netinet6/ip6_var.h> 40 #include <netinet/udp.h> 41 #include <netinet/tcp.h> 42 43 #include <machine/bus.h> 44 #include <machine/resource.h> 45 #include <sys/bus.h> 46 #include <sys/rman.h> 47 48 #include <dev/pci/pcireg.h> 49 #include <dev/pci/pcivar.h> 50 51 #include "ifdi_if.h" 52 #include "enic.h" 53 54 #include "opt_inet.h" 55 #include "opt_inet6.h" 56 57 static SYSCTL_NODE(_hw, OID_AUTO, enic, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 58 "ENIC"); 59 60 static const pci_vendor_info_t enic_vendor_info_array[] = 61 { 62 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET, 63 DRV_DESCRIPTION), 64 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET_VF, 65 DRV_DESCRIPTION " VF"), 66 /* required last entry */ 67 68 PVID_END 69 }; 70 71 static void *enic_register(device_t); 72 static int enic_attach_pre(if_ctx_t); 73 static int enic_msix_intr_assign(if_ctx_t, int); 74 75 static int enic_attach_post(if_ctx_t); 76 static int enic_detach(if_ctx_t); 77 78 static int enic_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 79 static int enic_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 80 static void enic_queues_free(if_ctx_t); 81 static int enic_rxq_intr(void *); 82 static int enic_event_intr(void *); 83 static int enic_err_intr(void *); 84 static void enic_stop(if_ctx_t); 85 static void enic_init(if_ctx_t); 86 static void enic_multi_set(if_ctx_t); 87 static int enic_mtu_set(if_ctx_t, uint32_t); 88 static void enic_media_status(if_ctx_t, struct ifmediareq *); 89 static int enic_media_change(if_ctx_t); 90 static int enic_promisc_set(if_ctx_t, int); 91 static uint64_t enic_get_counter(if_ctx_t, ift_counter); 92 static void enic_update_admin_status(if_ctx_t); 93 static void enic_txq_timer(if_ctx_t, uint16_t); 94 static int enic_link_is_up(struct enic_softc *); 95 static void enic_link_status(struct enic_softc *); 96 static void enic_set_lladdr(struct enic_softc *); 97 static void enic_setup_txq_sysctl(struct vnic_wq *, int, struct sysctl_ctx_list *, 98 struct sysctl_oid_list *); 99 static void enic_setup_rxq_sysctl(struct vnic_rq *, int, struct sysctl_ctx_list *, 100 struct sysctl_oid_list *); 101 static void enic_setup_sysctl(struct enic_softc *); 102 static int enic_tx_queue_intr_enable(if_ctx_t, uint16_t); 103 static int enic_rx_queue_intr_enable(if_ctx_t, uint16_t); 104 static void enic_enable_intr(struct enic_softc *, int); 105 static void enic_disable_intr(struct enic_softc *, int); 106 static void enic_intr_enable_all(if_ctx_t); 107 static void enic_intr_disable_all(if_ctx_t); 108 static int enic_dev_open(struct enic *); 109 static int enic_dev_init(struct enic *); 110 static void *enic_alloc_consistent(void *, size_t, bus_addr_t *, 111 struct iflib_dma_info *, u8 *); 112 static void enic_free_consistent(void *, size_t, void *, bus_addr_t, 113 struct iflib_dma_info *); 114 static int enic_pci_mapping(struct enic_softc *); 115 static void enic_pci_mapping_free(struct enic_softc *); 116 static int enic_dev_wait(struct vnic_dev *, int (*) (struct vnic_dev *, int), 117 int (*) (struct vnic_dev *, int *), int arg); 118 static int enic_map_bar(struct enic_softc *, struct enic_bar_info *, int, bool); 119 static void enic_update_packet_filter(struct enic *enic); 120 static bool enic_if_needs_restart(if_ctx_t, enum iflib_restart_event); 121 122 typedef enum { 123 ENIC_BARRIER_RD, 124 ENIC_BARRIER_WR, 125 ENIC_BARRIER_RDWR, 126 } enic_barrier_t; 127 128 static device_method_t enic_methods[] = { 129 /* Device interface */ 130 DEVMETHOD(device_register, enic_register), 131 DEVMETHOD(device_probe, iflib_device_probe), 132 DEVMETHOD(device_attach, iflib_device_attach), 133 DEVMETHOD(device_detach, iflib_device_detach), 134 DEVMETHOD(device_shutdown, iflib_device_shutdown), 135 DEVMETHOD(device_suspend, iflib_device_suspend), 136 DEVMETHOD(device_resume, iflib_device_resume), 137 DEVMETHOD_END 138 }; 139 140 static driver_t enic_driver = { 141 "enic", enic_methods, sizeof(struct enic_softc) 142 }; 143 144 DRIVER_MODULE(enic, pci, enic_driver, 0, 0); 145 IFLIB_PNP_INFO(pci, enic, enic_vendor_info_array); 146 MODULE_VERSION(enic, 2); 147 148 MODULE_DEPEND(enic, pci, 1, 1, 1); 149 MODULE_DEPEND(enic, ether, 1, 1, 1); 150 MODULE_DEPEND(enic, iflib, 1, 1, 1); 151 152 static device_method_t enic_iflib_methods[] = { 153 DEVMETHOD(ifdi_tx_queues_alloc, enic_tx_queues_alloc), 154 DEVMETHOD(ifdi_rx_queues_alloc, enic_rx_queues_alloc), 155 DEVMETHOD(ifdi_queues_free, enic_queues_free), 156 157 DEVMETHOD(ifdi_attach_pre, enic_attach_pre), 158 DEVMETHOD(ifdi_attach_post, enic_attach_post), 159 DEVMETHOD(ifdi_detach, enic_detach), 160 161 DEVMETHOD(ifdi_init, enic_init), 162 DEVMETHOD(ifdi_stop, enic_stop), 163 DEVMETHOD(ifdi_multi_set, enic_multi_set), 164 DEVMETHOD(ifdi_mtu_set, enic_mtu_set), 165 DEVMETHOD(ifdi_media_status, enic_media_status), 166 DEVMETHOD(ifdi_media_change, enic_media_change), 167 DEVMETHOD(ifdi_promisc_set, enic_promisc_set), 168 DEVMETHOD(ifdi_get_counter, enic_get_counter), 169 DEVMETHOD(ifdi_update_admin_status, enic_update_admin_status), 170 DEVMETHOD(ifdi_timer, enic_txq_timer), 171 172 DEVMETHOD(ifdi_tx_queue_intr_enable, enic_tx_queue_intr_enable), 173 DEVMETHOD(ifdi_rx_queue_intr_enable, enic_rx_queue_intr_enable), 174 DEVMETHOD(ifdi_intr_enable, enic_intr_enable_all), 175 DEVMETHOD(ifdi_intr_disable, enic_intr_disable_all), 176 DEVMETHOD(ifdi_msix_intr_assign, enic_msix_intr_assign), 177 178 DEVMETHOD(ifdi_needs_restart, enic_if_needs_restart), 179 180 DEVMETHOD_END 181 }; 182 183 static driver_t enic_iflib_driver = { 184 "enic", enic_iflib_methods, sizeof(struct enic_softc) 185 }; 186 187 extern struct if_txrx enic_txrx; 188 189 static struct if_shared_ctx enic_sctx_init = { 190 .isc_magic = IFLIB_MAGIC, 191 .isc_q_align = 512, 192 193 .isc_tx_maxsize = ENIC_TX_MAX_PKT_SIZE, 194 .isc_tx_maxsegsize = PAGE_SIZE, 195 196 /* 197 * These values are used to configure the busdma tag used for receive 198 * descriptors. Each receive descriptor only points to one buffer. 199 */ 200 .isc_rx_maxsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, /* One buf per 201 * descriptor */ 202 .isc_rx_nsegments = 1, /* One mapping per descriptor */ 203 .isc_rx_maxsegsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, 204 .isc_admin_intrcnt = 2, 205 .isc_vendor_info = enic_vendor_info_array, 206 .isc_driver_version = "1", 207 .isc_driver = &enic_iflib_driver, 208 .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ | IFLIB_SKIP_MSIX, 209 210 /* 211 * Number of receive queues per receive queue set, with associated 212 * descriptor settings for each. 213 */ 214 215 .isc_nrxqs = 2, 216 .isc_nfl = 1, /* one free list for each receive command 217 * queue */ 218 .isc_nrxd_min = {16, 16}, 219 .isc_nrxd_max = {2048, 2048}, 220 .isc_nrxd_default = {64, 64}, 221 222 /* 223 * Number of transmit queues per transmit queue set, with associated 224 * descriptor settings for each. 225 */ 226 .isc_ntxqs = 2, 227 .isc_ntxd_min = {16, 16}, 228 .isc_ntxd_max = {2048, 2048}, 229 .isc_ntxd_default = {64, 64}, 230 }; 231 232 static void * 233 enic_register(device_t dev) 234 { 235 return (&enic_sctx_init); 236 } 237 238 static int 239 enic_allocate_msix(struct enic_softc *softc) { 240 if_ctx_t ctx; 241 if_softc_ctx_t scctx; 242 if_shared_ctx_t sctx; 243 device_t dev; 244 cpuset_t cpus; 245 int queues, vectors, requested; 246 int err = 0; 247 248 dev = softc->dev; 249 ctx = softc->ctx; 250 scctx = softc->scctx; 251 sctx = iflib_get_sctx(ctx); 252 253 if (bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus) != 0) { 254 device_printf(dev, "Unable to fetch CPU list\n"); 255 CPU_COPY(&all_cpus, &cpus); 256 } 257 258 259 queues = CPU_COUNT(&cpus); 260 queues = imin(queues, scctx->isc_nrxqsets); 261 queues = imin(queues, scctx->isc_ntxqsets); 262 requested = queues * 2 + sctx->isc_admin_intrcnt; 263 scctx->isc_nrxqsets = queues; 264 scctx->isc_ntxqsets = queues; 265 266 vectors = requested; 267 if ((err = pci_alloc_msix(dev, &vectors)) != 0) { 268 device_printf(dev, 269 "failed to allocate %d MSI-X vectors, err: %d\n", requested, 270 err); 271 err = 1; 272 goto enic_allocate_msix_out; 273 } else { 274 if (vectors != requested) { 275 device_printf(dev, 276 "Unable to allocate sufficient MSI-X vectors " 277 "(got %d, need %d)\n", requested, vectors); 278 pci_release_msi(dev); 279 err = 1; 280 goto enic_allocate_msix_out; 281 } 282 } 283 284 device_printf(dev, "Using MSI-X interrupts with %d vectors\n", 285 vectors); 286 287 scctx->isc_intr = IFLIB_INTR_MSIX; 288 scctx->isc_vectors = vectors; 289 290 enic_allocate_msix_out: 291 return (err); 292 293 } 294 295 static struct enic_intr_mod_range mod_range[ENIC_MAX_LINK_SPEEDS] = { 296 {0, 0}, /* 0 - 4 Gbps */ 297 {0, 3}, /* 4 - 10 Gbps */ 298 {3, 6}, /* 10 - 40 Gbps */ 299 }; 300 301 static void enic_set_rx_coal_setting(struct enic *enic) 302 { 303 unsigned int speed; 304 int index = -1; 305 struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting; 306 307 /* 1. Read the link speed from fw 308 * 2. Pick the default range for the speed 309 * 3. Update it in enic->rx_coalesce_setting 310 */ 311 speed = vnic_dev_port_speed(enic->vdev); 312 if (ENIC_LINK_SPEED_10G < speed) 313 index = ENIC_LINK_40G_INDEX; 314 else if (ENIC_LINK_SPEED_4G < speed) 315 index = ENIC_LINK_10G_INDEX; 316 else 317 index = ENIC_LINK_4G_INDEX; 318 319 rx_coal->small_pkt_range_start = mod_range[index].small_pkt_range_start; 320 rx_coal->large_pkt_range_start = mod_range[index].large_pkt_range_start; 321 rx_coal->range_end = ENIC_RX_COALESCE_RANGE_END; 322 323 /* Start with the value provided by UCSM */ 324 for (index = 0; index < enic->rq_count; index++) 325 enic->cq[index].cur_rx_coal_timeval = 326 enic->config.intr_timer_usec; 327 328 rx_coal->use_adaptive_rx_coalesce = 1; 329 } 330 331 static int 332 enic_attach_pre(if_ctx_t ctx) 333 { 334 if_softc_ctx_t scctx; 335 struct enic_softc *softc; 336 struct vnic_dev *vdev; 337 struct enic *enic; 338 device_t dev; 339 340 int err = -1; 341 int rc = 0; 342 int i; 343 u64 a0 = 0, a1 = 0; 344 int wait = 1000; 345 struct vnic_stats *stats; 346 int ret; 347 348 dev = iflib_get_dev(ctx); 349 softc = iflib_get_softc(ctx); 350 softc->dev = dev; 351 softc->ctx = ctx; 352 softc->sctx = iflib_get_sctx(ctx); 353 softc->scctx = iflib_get_softc_ctx(ctx); 354 softc->ifp = iflib_get_ifp(ctx); 355 softc->media = iflib_get_media(ctx); 356 softc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 357 ENIC_MAX_MULTICAST_ADDRESSES, M_DEVBUF, 358 M_NOWAIT | M_ZERO); 359 if (softc->mta == NULL) 360 return (ENOMEM); 361 scctx = softc->scctx; 362 363 mtx_init(&softc->enic_lock, "ENIC Lock", NULL, MTX_DEF); 364 365 pci_enable_busmaster(softc->dev); 366 if (enic_pci_mapping(softc)) 367 return (ENXIO); 368 369 enic = &softc->enic; 370 enic->softc = softc; 371 vdev = &softc->vdev; 372 vdev->softc = softc; 373 enic->vdev = vdev; 374 vdev->priv = enic; 375 376 ENIC_LOCK(softc); 377 vnic_dev_register(vdev, &softc->mem, 1); 378 enic->vdev = vdev; 379 vnic_dev_cmd_init(enic->vdev); 380 381 vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0); 382 383 vnic_dev_cmd(vdev, CMD_INIT_v1, &a0, &a1, wait); 384 vnic_dev_cmd(vdev, CMD_GET_MAC_ADDR, &a0, &a1, wait); 385 386 bcopy((u_int8_t *) & a0, softc->mac_addr, ETHER_ADDR_LEN); 387 iflib_set_mac(ctx, softc->mac_addr); 388 389 vnic_register_cbacks(enic->vdev, enic_alloc_consistent, 390 enic_free_consistent); 391 392 /* 393 * Allocate the consistent memory for stats and counters upfront so 394 * both primary and secondary processes can access them. 395 */ 396 ENIC_UNLOCK(softc); 397 err = vnic_dev_alloc_stats_mem(enic->vdev); 398 ENIC_LOCK(softc); 399 if (err) { 400 dev_err(enic, "Failed to allocate cmd memory, aborting\n"); 401 goto err_out_unregister; 402 } 403 vnic_dev_stats_clear(enic->vdev); 404 ret = vnic_dev_stats_dump(enic->vdev, &stats); 405 if (ret) { 406 dev_err(enic, "Error in getting stats\n"); 407 goto err_out_unregister; 408 } 409 err = vnic_dev_alloc_counter_mem(enic->vdev); 410 if (err) { 411 dev_err(enic, "Failed to allocate counter memory, aborting\n"); 412 goto err_out_unregister; 413 } 414 415 /* Issue device open to get device in known state */ 416 err = enic_dev_open(enic); 417 if (err) { 418 dev_err(enic, "vNIC dev open failed, aborting\n"); 419 goto err_out_unregister; 420 } 421 422 /* Set ingress vlan rewrite mode before vnic initialization */ 423 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN; 424 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN; 425 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, 426 enic->ig_vlan_rewrite_mode); 427 if (err) { 428 dev_err(enic, 429 "Failed to set ingress vlan rewrite mode, aborting.\n"); 430 goto err_out_dev_close; 431 } 432 433 /* 434 * Issue device init to initialize the vnic-to-switch link. We'll 435 * start with carrier off and wait for link UP notification later to 436 * turn on carrier. We don't need to wait here for the 437 * vnic-to-switch link initialization to complete; link UP 438 * notification is the indication that the process is complete. 439 */ 440 441 err = vnic_dev_init(enic->vdev, 0); 442 if (err) { 443 dev_err(enic, "vNIC dev init failed, aborting\n"); 444 goto err_out_dev_close; 445 } 446 447 err = enic_dev_init(enic); 448 if (err) { 449 dev_err(enic, "Device initialization failed, aborting\n"); 450 goto err_out_dev_close; 451 } 452 ENIC_UNLOCK(softc); 453 454 enic->port_mtu = vnic_dev_mtu(enic->vdev); 455 456 softc->scctx = iflib_get_softc_ctx(ctx); 457 scctx = softc->scctx; 458 scctx->isc_txrx = &enic_txrx; 459 scctx->isc_capabilities = scctx->isc_capenable = \ 460 IFCAP_HWCSUM; 461 scctx->isc_tx_csum_flags = 0; 462 if_setmtu(softc->ifp, enic->config.mtu); 463 scctx->isc_max_frame_size = enic->config.mtu + ETHER_HDR_LEN + \ 464 ETHER_CRC_LEN; 465 scctx->isc_nrxqsets_max = enic->conf_rq_count; 466 scctx->isc_ntxqsets_max = enic->conf_wq_count; 467 scctx->isc_nrxqsets = enic->conf_rq_count; 468 scctx->isc_ntxqsets = enic->conf_wq_count; 469 for (i = 0; i < enic->conf_wq_count; i++) { 470 scctx->isc_ntxd[i] = enic->config.wq_desc_count; 471 scctx->isc_txqsizes[i] = sizeof(struct cq_enet_wq_desc) 472 * scctx->isc_ntxd[i]; 473 scctx->isc_ntxd[i + enic->conf_wq_count] = 474 enic->config.wq_desc_count; 475 scctx->isc_txqsizes[i + enic->conf_wq_count] = 476 sizeof(struct cq_desc) * scctx->isc_ntxd[i + 477 enic->conf_wq_count]; 478 } 479 for (i = 0; i < enic->conf_rq_count; i++) { 480 scctx->isc_nrxd[i] = enic->config.rq_desc_count; 481 scctx->isc_rxqsizes[i] = sizeof(struct cq_enet_rq_desc) * 482 scctx->isc_nrxd[i]; 483 scctx->isc_nrxd[i + enic->conf_rq_count] = 484 enic->config.rq_desc_count; 485 scctx->isc_rxqsizes[i + enic->conf_rq_count] = sizeof(struct 486 cq_desc) * scctx->isc_nrxd[i + enic->conf_rq_count]; 487 } 488 scctx->isc_tx_nsegments = 31; 489 490 scctx->isc_msix_bar = -1; 491 492 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 493 ifmedia_add(softc->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); 494 ifmedia_add(softc->media, IFM_ETHER | IFM_10_FL, 0, NULL); 495 496 /* 497 * Allocate the CQ here since TX is called first before RX for now 498 * assume RX and TX are the same 499 */ 500 if (softc->enic.cq == NULL) 501 softc->enic.cq = malloc(sizeof(struct vnic_cq) * 502 softc->enic.wq_count + softc->enic.rq_count, M_DEVBUF, 503 M_NOWAIT | M_ZERO); 504 if (softc->enic.cq == NULL) 505 return (ENOMEM); 506 507 softc->enic.cq->ntxqsets = softc->enic.wq_count + softc->enic.rq_count; 508 509 /* 510 * Allocate the consistent memory for stats and counters upfront so 511 * both primary and secondary processes can access them. 512 */ 513 err = vnic_dev_alloc_stats_mem(enic->vdev); 514 if (err) { 515 dev_err(enic, "Failed to allocate cmd memory, aborting\n"); 516 goto err_out_dev_close; 517 } 518 519 err = enic_allocate_msix(softc); 520 if (err) { 521 dev_err(enic, "Failed to allocate MSIX, aborting\n"); 522 goto err_out_dev_close; 523 } 524 525 return (rc); 526 527 err_out_dev_close: 528 vnic_dev_close(enic->vdev); 529 vnic_dev_deinit_devcmd2(enic->vdev); 530 err_out_unregister: 531 free(softc->vdev.devcmd, M_DEVBUF); 532 free(softc->enic.intr_queues, M_DEVBUF); 533 free(softc->enic.cq, M_DEVBUF); 534 free(softc->mta, M_DEVBUF); 535 rc = -1; 536 pci_disable_busmaster(softc->dev); 537 enic_pci_mapping_free(softc); 538 mtx_destroy(&softc->enic_lock); 539 return (rc); 540 } 541 542 static int 543 enic_msix_intr_assign(if_ctx_t ctx, int msix) 544 { 545 struct enic_softc *softc; 546 struct enic *enic; 547 if_softc_ctx_t scctx; 548 549 int error; 550 int i; 551 char irq_name[16]; 552 553 softc = iflib_get_softc(ctx); 554 enic = &softc->enic; 555 scctx = softc->scctx; 556 557 ENIC_LOCK(softc); 558 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSIX); 559 ENIC_UNLOCK(softc); 560 561 enic->intr_queues = malloc(sizeof(*enic->intr_queues) * 562 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO); 563 enic->intr = malloc(sizeof(*enic->intr) * msix, M_DEVBUF, M_NOWAIT 564 | M_ZERO); 565 for (i = 0; i < scctx->isc_nrxqsets; i++) { 566 snprintf(irq_name, sizeof(irq_name), "erxq%d:%d", i, 567 device_get_unit(softc->dev)); 568 569 error = iflib_irq_alloc_generic(ctx, 570 &enic->intr_queues[i].intr_irq, i + 1, IFLIB_INTR_RX, 571 enic_rxq_intr, &enic->rq[i], i, irq_name); 572 if (error) { 573 device_printf(iflib_get_dev(ctx), 574 "Failed to register rxq %d interrupt handler\n", i); 575 return (error); 576 } 577 enic->intr[i].index = i; 578 enic->intr[i].vdev = enic->vdev; 579 ENIC_LOCK(softc); 580 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, 581 RES_TYPE_INTR_CTRL, i); 582 vnic_intr_mask(&enic->intr[i]); 583 ENIC_UNLOCK(softc); 584 } 585 586 for (i = scctx->isc_nrxqsets; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) { 587 snprintf(irq_name, sizeof(irq_name), "etxq%d:%d", i - 588 scctx->isc_nrxqsets, device_get_unit(softc->dev)); 589 590 iflib_softirq_alloc_generic(ctx, 591 &enic->intr_queues[i].intr_irq, IFLIB_INTR_TX, 592 &enic->wq[i - scctx->isc_nrxqsets], i - scctx->isc_nrxqsets, 593 irq_name); 594 595 enic->intr[i].index = i; 596 enic->intr[i].vdev = enic->vdev; 597 ENIC_LOCK(softc); 598 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, 599 RES_TYPE_INTR_CTRL, i); 600 vnic_intr_mask(&enic->intr[i]); 601 ENIC_UNLOCK(softc); 602 } 603 604 i = scctx->isc_nrxqsets + scctx->isc_ntxqsets; 605 error = iflib_irq_alloc_generic(ctx, &softc->enic_event_intr_irq, 606 i + 1, IFLIB_INTR_ADMIN, enic_event_intr, softc, 0, "event"); 607 if (error) { 608 device_printf(iflib_get_dev(ctx), 609 "Failed to register event interrupt handler\n"); 610 return (error); 611 } 612 613 enic->intr[i].index = i; 614 enic->intr[i].vdev = enic->vdev; 615 ENIC_LOCK(softc); 616 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL, 617 i); 618 vnic_intr_mask(&enic->intr[i]); 619 ENIC_UNLOCK(softc); 620 621 i++; 622 error = iflib_irq_alloc_generic(ctx, &softc->enic_err_intr_irq, 623 i + 1, IFLIB_INTR_ADMIN, enic_err_intr, softc, 0, "err"); 624 if (error) { 625 device_printf(iflib_get_dev(ctx), 626 "Failed to register event interrupt handler\n"); 627 return (error); 628 } 629 enic->intr[i].index = i; 630 enic->intr[i].vdev = enic->vdev; 631 ENIC_LOCK(softc); 632 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL, 633 i); 634 vnic_intr_mask(&enic->intr[i]); 635 ENIC_UNLOCK(softc); 636 637 enic->intr_count = msix; 638 639 return (0); 640 } 641 642 static void 643 enic_free_irqs(struct enic_softc *softc) 644 { 645 if_softc_ctx_t scctx; 646 647 struct enic *enic; 648 int i; 649 650 scctx = softc->scctx; 651 enic = &softc->enic; 652 653 for (i = 0; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) { 654 iflib_irq_free(softc->ctx, &enic->intr_queues[i].intr_irq); 655 } 656 657 iflib_irq_free(softc->ctx, &softc->enic_event_intr_irq); 658 iflib_irq_free(softc->ctx, &softc->enic_err_intr_irq); 659 free(enic->intr_queues, M_DEVBUF); 660 free(enic->intr, M_DEVBUF); 661 } 662 663 static int 664 enic_attach_post(if_ctx_t ctx) 665 { 666 struct enic *enic; 667 struct enic_softc *softc; 668 int error = 0; 669 670 softc = iflib_get_softc(ctx); 671 enic = &softc->enic; 672 673 enic_setup_sysctl(softc); 674 675 enic_init_vnic_resources(enic); 676 enic_set_rx_coal_setting(enic); 677 enic_setup_finish(enic); 678 679 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 680 ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO); 681 682 return (error); 683 } 684 685 static int 686 enic_detach(if_ctx_t ctx) 687 { 688 struct enic_softc *softc; 689 struct enic *enic; 690 691 softc = iflib_get_softc(ctx); 692 enic = &softc->enic; 693 694 vnic_dev_notify_unset(enic->vdev); 695 696 enic_free_irqs(softc); 697 698 ENIC_LOCK(softc); 699 vnic_dev_deinit(enic->vdev); 700 vnic_dev_close(enic->vdev); 701 vnic_dev_deinit_devcmd2(enic->vdev); 702 free(softc->vdev.devcmd, M_DEVBUF); 703 pci_disable_busmaster(softc->dev); 704 enic_pci_mapping_free(softc); 705 ENIC_UNLOCK(softc); 706 707 return 0; 708 } 709 710 static int 711 enic_tx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs, 712 int ntxqs, int ntxqsets) 713 { 714 struct enic_softc *softc; 715 int q; 716 717 softc = iflib_get_softc(ctx); 718 /* Allocate the array of transmit queues */ 719 softc->enic.wq = malloc(sizeof(struct vnic_wq) * 720 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 721 if (softc->enic.wq == NULL) 722 return (ENOMEM); 723 724 /* Initialize driver state for each transmit queue */ 725 726 /* 727 * Allocate queue state that is shared with the device. This check 728 * and call is performed in both enic_tx_queues_alloc() and 729 * enic_rx_queues_alloc() so that we don't have to care which order 730 * iflib invokes those routines in. 731 */ 732 733 /* Record descriptor ring vaddrs and paddrs */ 734 ENIC_LOCK(softc); 735 for (q = 0; q < ntxqsets; q++) { 736 struct vnic_wq *wq; 737 struct vnic_cq *cq; 738 unsigned int cq_wq; 739 740 wq = &softc->enic.wq[q]; 741 cq_wq = enic_cq_wq(&softc->enic, q); 742 cq = &softc->enic.cq[cq_wq]; 743 744 /* Completion ring */ 745 wq->vdev = softc->enic.vdev; 746 wq->index = q; 747 wq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_WQ, 748 wq->index); 749 vnic_wq_disable(wq); 750 751 wq->ring.desc_size = sizeof(struct wq_enet_desc); 752 wq->ring.desc_count = softc->scctx->isc_ntxd[q]; 753 wq->ring.desc_avail = wq->ring.desc_count - 1; 754 wq->ring.last_count = wq->ring.desc_count; 755 wq->head_idx = 0; 756 wq->tail_idx = 0; 757 758 wq->ring.size = wq->ring.desc_count * wq->ring.desc_size; 759 wq->ring.descs = vaddrs[q * ntxqs + 0]; 760 wq->ring.base_addr = paddrs[q * ntxqs + 0]; 761 762 /* Command ring */ 763 cq->vdev = softc->enic.vdev; 764 cq->index = cq_wq; 765 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, 766 RES_TYPE_CQ, cq->index); 767 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc); 768 cq->ring.desc_count = softc->scctx->isc_ntxd[q]; 769 cq->ring.desc_avail = cq->ring.desc_count - 1; 770 771 cq->ring.size = cq->ring.desc_count * cq->ring.desc_size; 772 cq->ring.descs = vaddrs[q * ntxqs + 1]; 773 cq->ring.base_addr = paddrs[q * ntxqs + 1]; 774 775 } 776 777 ENIC_UNLOCK(softc); 778 779 return (0); 780 } 781 782 783 784 static int 785 enic_rx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs, 786 int nrxqs, int nrxqsets) 787 { 788 struct enic_softc *softc; 789 int q; 790 791 softc = iflib_get_softc(ctx); 792 /* Allocate the array of receive queues */ 793 softc->enic.rq = malloc(sizeof(struct vnic_rq) * nrxqsets, M_DEVBUF, 794 M_NOWAIT | M_ZERO); 795 if (softc->enic.rq == NULL) 796 return (ENOMEM); 797 798 /* Initialize driver state for each receive queue */ 799 800 /* 801 * Allocate queue state that is shared with the device. This check 802 * and call is performed in both enic_tx_queues_alloc() and 803 * enic_rx_queues_alloc() so that we don't have to care which order 804 * iflib invokes those routines in. 805 */ 806 807 /* Record descriptor ring vaddrs and paddrs */ 808 ENIC_LOCK(softc); 809 for (q = 0; q < nrxqsets; q++) { 810 struct vnic_rq *rq; 811 struct vnic_cq *cq; 812 unsigned int cq_rq; 813 814 rq = &softc->enic.rq[q]; 815 cq_rq = enic_cq_rq(&softc->enic, q); 816 cq = &softc->enic.cq[cq_rq]; 817 818 /* Completion ring */ 819 cq->vdev = softc->enic.vdev; 820 cq->index = cq_rq; 821 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_CQ, 822 cq->index); 823 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc); 824 cq->ring.desc_count = softc->scctx->isc_nrxd[1]; 825 cq->ring.desc_avail = cq->ring.desc_count - 1; 826 827 cq->ring.size = cq->ring.desc_count * cq->ring.desc_size; 828 cq->ring.descs = vaddrs[q * nrxqs + 0]; 829 cq->ring.base_addr = paddrs[q * nrxqs + 0]; 830 831 /* Command ring(s) */ 832 rq->vdev = softc->enic.vdev; 833 834 rq->index = q; 835 rq->ctrl = vnic_dev_get_res(softc->enic.vdev, 836 RES_TYPE_RQ, rq->index); 837 vnic_rq_disable(rq); 838 839 rq->ring.desc_size = sizeof(struct rq_enet_desc); 840 rq->ring.desc_count = softc->scctx->isc_nrxd[0]; 841 rq->ring.desc_avail = rq->ring.desc_count - 1; 842 843 rq->ring.size = rq->ring.desc_count * rq->ring.desc_size; 844 rq->ring.descs = vaddrs[q * nrxqs + 1]; 845 rq->ring.base_addr = paddrs[q * nrxqs + 1]; 846 rq->need_initial_post = true; 847 } 848 849 ENIC_UNLOCK(softc); 850 851 return (0); 852 } 853 854 static void 855 enic_queues_free(if_ctx_t ctx) 856 { 857 struct enic_softc *softc; 858 softc = iflib_get_softc(ctx); 859 860 free(softc->enic.rq, M_DEVBUF); 861 free(softc->enic.wq, M_DEVBUF); 862 free(softc->enic.cq, M_DEVBUF); 863 } 864 865 static int 866 enic_rxq_intr(void *rxq) 867 { 868 struct vnic_rq *rq; 869 if_t ifp; 870 871 rq = (struct vnic_rq *)rxq; 872 ifp = iflib_get_ifp(rq->vdev->softc->ctx); 873 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 874 return (FILTER_HANDLED); 875 876 return (FILTER_SCHEDULE_THREAD); 877 } 878 879 static int 880 enic_event_intr(void *vsc) 881 { 882 struct enic_softc *softc; 883 struct enic *enic; 884 uint32_t mtu; 885 886 softc = vsc; 887 enic = &softc->enic; 888 889 mtu = vnic_dev_mtu(enic->vdev); 890 if (mtu && mtu != enic->port_mtu) { 891 enic->port_mtu = mtu; 892 } 893 894 enic_link_status(softc); 895 896 return (FILTER_HANDLED); 897 } 898 899 static int 900 enic_err_intr(void *vsc) 901 { 902 struct enic_softc *softc; 903 904 softc = vsc; 905 906 enic_stop(softc->ctx); 907 enic_init(softc->ctx); 908 909 return (FILTER_HANDLED); 910 } 911 912 static void 913 enic_stop(if_ctx_t ctx) 914 { 915 struct enic_softc *softc; 916 struct enic *enic; 917 if_softc_ctx_t scctx; 918 unsigned int index; 919 struct vnic_wq *wq; 920 struct vnic_rq *rq; 921 struct vnic_cq *cq; 922 unsigned int cq_wq, cq_rq; 923 924 925 softc = iflib_get_softc(ctx); 926 scctx = softc->scctx; 927 enic = &softc->enic; 928 929 if (softc->stopped) 930 return; 931 softc->link_active = 0; 932 softc->stopped = 1; 933 934 enic_dev_disable(enic); 935 936 for (index = 0; index < scctx->isc_ntxqsets; index++) { 937 enic_stop_wq(enic, index); 938 vnic_wq_clean(&enic->wq[index]); 939 vnic_cq_clean(&enic->cq[enic_cq_rq(enic, index)]); 940 941 wq = &softc->enic.wq[index]; 942 wq->ring.desc_avail = wq->ring.desc_count - 1; 943 wq->ring.last_count = wq->ring.desc_count; 944 wq->head_idx = 0; 945 wq->tail_idx = 0; 946 947 cq_wq = enic_cq_wq(&softc->enic, index); 948 cq = &softc->enic.cq[cq_wq]; 949 cq->ring.desc_avail = cq->ring.desc_count - 1; 950 } 951 952 for (index = 0; index < scctx->isc_nrxqsets; index++) { 953 enic_stop_rq(enic, index); 954 vnic_rq_clean(&enic->rq[index]); 955 vnic_cq_clean(&enic->cq[enic_cq_wq(enic, index)]); 956 957 rq = &softc->enic.rq[index]; 958 cq_rq = enic_cq_rq(&softc->enic, index); 959 cq = &softc->enic.cq[cq_rq]; 960 961 cq->ring.desc_avail = cq->ring.desc_count - 1; 962 rq->ring.desc_avail = rq->ring.desc_count - 1; 963 rq->need_initial_post = true; 964 } 965 966 for (index = 0; index < scctx->isc_vectors; index++) { 967 vnic_intr_clean(&enic->intr[index]); 968 } 969 } 970 971 static void 972 enic_init(if_ctx_t ctx) 973 { 974 struct enic_softc *softc; 975 struct enic *enic; 976 if_softc_ctx_t scctx; 977 unsigned int index; 978 979 softc = iflib_get_softc(ctx); 980 scctx = softc->scctx; 981 enic = &softc->enic; 982 983 984 enic_init_vnic_resources(enic); 985 986 for (index = 0; index < scctx->isc_ntxqsets; index++) 987 enic_prep_wq_for_simple_tx(&softc->enic, index); 988 989 for (index = 0; index < scctx->isc_ntxqsets; index++) 990 enic_start_wq(enic, index); 991 992 for (index = 0; index < scctx->isc_nrxqsets; index++) 993 enic_start_rq(enic, index); 994 995 /* Use the current MAC address. */ 996 bcopy(if_getlladdr(softc->ifp), softc->lladdr, ETHER_ADDR_LEN); 997 enic_set_lladdr(softc); 998 999 ENIC_LOCK(softc); 1000 vnic_dev_enable_wait(enic->vdev); 1001 ENIC_UNLOCK(softc); 1002 1003 softc->stopped = 0; 1004 1005 enic_link_status(softc); 1006 } 1007 1008 static void 1009 enic_del_mcast(struct enic_softc *softc) { 1010 struct enic *enic; 1011 int i; 1012 1013 enic = &softc->enic; 1014 for (i=0; i < softc->mc_count; i++) { 1015 vnic_dev_del_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]); 1016 } 1017 softc->multicast = 0; 1018 softc->mc_count = 0; 1019 } 1020 1021 static void 1022 enic_add_mcast(struct enic_softc *softc) { 1023 struct enic *enic; 1024 int i; 1025 1026 enic = &softc->enic; 1027 for (i=0; i < softc->mc_count; i++) { 1028 vnic_dev_add_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]); 1029 } 1030 softc->multicast = 1; 1031 } 1032 1033 static u_int 1034 enic_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 1035 { 1036 uint8_t *mta = arg; 1037 1038 if (idx == ENIC_MAX_MULTICAST_ADDRESSES) 1039 return (0); 1040 1041 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1042 return (1); 1043 } 1044 1045 static void 1046 enic_multi_set(if_ctx_t ctx) 1047 { 1048 if_t ifp; 1049 struct enic_softc *softc; 1050 u_int count; 1051 1052 softc = iflib_get_softc(ctx); 1053 ifp = iflib_get_ifp(ctx); 1054 1055 ENIC_LOCK(softc); 1056 enic_del_mcast(softc); 1057 count = if_foreach_llmaddr(ifp, enic_copy_maddr, softc->mta); 1058 softc->mc_count = count; 1059 enic_add_mcast(softc); 1060 ENIC_UNLOCK(softc); 1061 1062 if (if_getflags(ifp) & IFF_PROMISC) { 1063 softc->promisc = 1; 1064 } else { 1065 softc->promisc = 0; 1066 } 1067 if (if_getflags(ifp) & IFF_ALLMULTI) { 1068 softc->allmulti = 1; 1069 } else { 1070 softc->allmulti = 0; 1071 } 1072 enic_update_packet_filter(&softc->enic); 1073 } 1074 1075 static int 1076 enic_mtu_set(if_ctx_t ctx, uint32_t mtu) 1077 { 1078 struct enic_softc *softc; 1079 struct enic *enic; 1080 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 1081 1082 softc = iflib_get_softc(ctx); 1083 enic = &softc->enic; 1084 1085 enic_stop(softc->ctx); 1086 if (mtu > enic->port_mtu){ 1087 return (EINVAL); 1088 } 1089 1090 enic->config.mtu = mtu; 1091 scctx->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1092 enic_init(softc->ctx); 1093 1094 return (0); 1095 } 1096 1097 static void 1098 enic_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1099 { 1100 struct enic_softc *softc; 1101 struct ifmedia_entry *next; 1102 uint32_t speed; 1103 uint64_t target_baudrate; 1104 1105 softc = iflib_get_softc(ctx); 1106 1107 ifmr->ifm_status = IFM_AVALID; 1108 ifmr->ifm_active = IFM_ETHER; 1109 1110 if (enic_link_is_up(softc) != 0) { 1111 ENIC_LOCK(softc); 1112 speed = vnic_dev_port_speed(&softc->vdev); 1113 ENIC_UNLOCK(softc); 1114 target_baudrate = 1000ull * speed; 1115 LIST_FOREACH(next, &(iflib_get_media(ctx)->ifm_list), ifm_list) { 1116 if (ifmedia_baudrate(next->ifm_media) == target_baudrate) { 1117 ifmr->ifm_active |= next->ifm_media; 1118 } 1119 } 1120 1121 ifmr->ifm_status |= IFM_ACTIVE; 1122 ifmr->ifm_active |= IFM_AUTO; 1123 } else 1124 ifmr->ifm_active |= IFM_NONE; 1125 } 1126 1127 static int 1128 enic_media_change(if_ctx_t ctx) 1129 { 1130 return (ENODEV); 1131 } 1132 1133 static int 1134 enic_promisc_set(if_ctx_t ctx, int flags) 1135 { 1136 if_t ifp; 1137 struct enic_softc *softc; 1138 1139 softc = iflib_get_softc(ctx); 1140 ifp = iflib_get_ifp(ctx); 1141 1142 if (if_getflags(ifp) & IFF_PROMISC) { 1143 softc->promisc = 1; 1144 } else { 1145 softc->promisc = 0; 1146 } 1147 if (if_getflags(ifp) & IFF_ALLMULTI) { 1148 softc->allmulti = 1; 1149 } else { 1150 softc->allmulti = 0; 1151 } 1152 enic_update_packet_filter(&softc->enic); 1153 1154 return (0); 1155 } 1156 1157 static uint64_t 1158 enic_get_counter(if_ctx_t ctx, ift_counter cnt) { 1159 if_t ifp = iflib_get_ifp(ctx); 1160 1161 if (cnt < IFCOUNTERS) 1162 return if_get_counter_default(ifp, cnt); 1163 1164 return (0); 1165 } 1166 1167 static void 1168 enic_update_admin_status(if_ctx_t ctx) 1169 { 1170 struct enic_softc *softc; 1171 softc = iflib_get_softc(ctx); 1172 1173 enic_link_status(softc); 1174 } 1175 1176 static void 1177 enic_txq_timer(if_ctx_t ctx, uint16_t qid) 1178 { 1179 1180 struct enic_softc *softc; 1181 struct enic *enic; 1182 struct vnic_stats *stats; 1183 int ret; 1184 1185 softc = iflib_get_softc(ctx); 1186 enic = &softc->enic; 1187 1188 ENIC_LOCK(softc); 1189 ret = vnic_dev_stats_dump(enic->vdev, &stats); 1190 ENIC_UNLOCK(softc); 1191 if (ret) { 1192 dev_err(enic, "Error in getting stats\n"); 1193 } 1194 } 1195 1196 static int 1197 enic_link_is_up(struct enic_softc *softc) 1198 { 1199 return (vnic_dev_link_status(&softc->vdev) == 1); 1200 } 1201 1202 static void 1203 enic_link_status(struct enic_softc *softc) 1204 { 1205 if_ctx_t ctx; 1206 uint64_t speed; 1207 int link; 1208 1209 ctx = softc->ctx; 1210 link = enic_link_is_up(softc); 1211 speed = IF_Gbps(10); 1212 1213 ENIC_LOCK(softc); 1214 speed = vnic_dev_port_speed(&softc->vdev); 1215 ENIC_UNLOCK(softc); 1216 1217 if (link != 0 && softc->link_active == 0) { 1218 softc->link_active = 1; 1219 iflib_link_state_change(ctx, LINK_STATE_UP, speed); 1220 } else if (link == 0 && softc->link_active != 0) { 1221 softc->link_active = 0; 1222 iflib_link_state_change(ctx, LINK_STATE_DOWN, speed); 1223 } 1224 } 1225 1226 static void 1227 enic_set_lladdr(struct enic_softc *softc) 1228 { 1229 struct enic *enic; 1230 enic = &softc->enic; 1231 1232 ENIC_LOCK(softc); 1233 vnic_dev_add_addr(enic->vdev, softc->lladdr); 1234 ENIC_UNLOCK(softc); 1235 } 1236 1237 1238 static void 1239 enic_setup_txq_sysctl(struct vnic_wq *wq, int i, struct sysctl_ctx_list *ctx, 1240 struct sysctl_oid_list *child) 1241 { 1242 struct sysctl_oid *txsnode; 1243 struct sysctl_oid_list *txslist; 1244 struct vnic_stats *stats = wq[i].vdev->stats; 1245 1246 txsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats", 1247 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 1248 txslist = SYSCTL_CHILDREN(txsnode); 1249 1250 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_frames_ok", CTLFLAG_RD, 1251 &stats->tx.tx_frames_ok, "TX Frames OK"); 1252 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_frames_ok", CTLFLAG_RD, 1253 &stats->tx.tx_unicast_frames_ok, "TX unicast frames OK"); 1254 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_frames_ok", CTLFLAG_RD, 1255 &stats->tx.tx_multicast_frames_ok, "TX multicast framse OK"); 1256 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_frames_ok", CTLFLAG_RD, 1257 &stats->tx.tx_broadcast_frames_ok, "TX Broadcast frames OK"); 1258 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_bytes_ok", CTLFLAG_RD, 1259 &stats->tx.tx_bytes_ok, "TX bytes OK "); 1260 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_bytes_ok", CTLFLAG_RD, 1261 &stats->tx.tx_unicast_bytes_ok, "TX unicast bytes OK"); 1262 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_bytes_ok", CTLFLAG_RD, 1263 &stats->tx.tx_multicast_bytes_ok, "TX multicast bytes OK"); 1264 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_bytes_ok", CTLFLAG_RD, 1265 &stats->tx.tx_broadcast_bytes_ok, "TX broadcast bytes OK"); 1266 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_drops", CTLFLAG_RD, 1267 &stats->tx.tx_drops, "TX drops"); 1268 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_errors", CTLFLAG_RD, 1269 &stats->tx.tx_errors, "TX errors"); 1270 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_tso", CTLFLAG_RD, 1271 &stats->tx.tx_tso, "TX TSO"); 1272 } 1273 1274 static void 1275 enic_setup_rxq_sysctl(struct vnic_rq *rq, int i, struct sysctl_ctx_list *ctx, 1276 struct sysctl_oid_list *child) 1277 { 1278 struct sysctl_oid *rxsnode; 1279 struct sysctl_oid_list *rxslist; 1280 struct vnic_stats *stats = rq[i].vdev->stats; 1281 1282 rxsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats", 1283 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 1284 rxslist = SYSCTL_CHILDREN(rxsnode); 1285 1286 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_ok", CTLFLAG_RD, 1287 &stats->rx.rx_frames_ok, "RX Frames OK"); 1288 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_total", CTLFLAG_RD, 1289 &stats->rx.rx_frames_total, "RX frames total"); 1290 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_frames_ok", CTLFLAG_RD, 1291 &stats->rx.rx_unicast_frames_ok, "RX unicast frames ok"); 1292 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_frames_ok", CTLFLAG_RD, 1293 &stats->rx.rx_multicast_frames_ok, "RX multicast Frames ok"); 1294 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_frames_ok", CTLFLAG_RD, 1295 &stats->rx.rx_broadcast_frames_ok, "RX broadcast frames ok"); 1296 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_bytes_ok", CTLFLAG_RD, 1297 &stats->rx.rx_bytes_ok, "RX bytes ok"); 1298 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_bytes_ok", CTLFLAG_RD, 1299 &stats->rx.rx_unicast_bytes_ok, "RX unicast bytes ok"); 1300 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_bytes_ok", CTLFLAG_RD, 1301 &stats->rx.rx_multicast_bytes_ok, "RX multicast bytes ok"); 1302 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_bytes_ok", CTLFLAG_RD, 1303 &stats->rx.rx_broadcast_bytes_ok, "RX broadcast bytes ok"); 1304 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_drop", CTLFLAG_RD, 1305 &stats->rx.rx_drop, "RX drop"); 1306 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_errors", CTLFLAG_RD, 1307 &stats->rx.rx_errors, "RX errors"); 1308 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_rss", CTLFLAG_RD, 1309 &stats->rx.rx_rss, "RX rss"); 1310 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_crc_errors", CTLFLAG_RD, 1311 &stats->rx.rx_crc_errors, "RX crc errors"); 1312 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_64", CTLFLAG_RD, 1313 &stats->rx.rx_frames_64, "RX frames 64"); 1314 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_127", CTLFLAG_RD, 1315 &stats->rx.rx_frames_127, "RX frames 127"); 1316 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_255", CTLFLAG_RD, 1317 &stats->rx.rx_frames_255, "RX frames 255"); 1318 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_511", CTLFLAG_RD, 1319 &stats->rx.rx_frames_511, "RX frames 511"); 1320 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1023", CTLFLAG_RD, 1321 &stats->rx.rx_frames_1023, "RX frames 1023"); 1322 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1518", CTLFLAG_RD, 1323 &stats->rx.rx_frames_1518, "RX frames 1518"); 1324 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_to_max", CTLFLAG_RD, 1325 &stats->rx.rx_frames_to_max, "RX frames to max"); 1326 } 1327 1328 static void 1329 enic_setup_queue_sysctl(struct enic_softc *softc, struct sysctl_ctx_list *ctx, 1330 struct sysctl_oid_list *child) 1331 { 1332 enic_setup_txq_sysctl(softc->enic.wq, 0, ctx, child); 1333 enic_setup_rxq_sysctl(softc->enic.rq, 0, ctx, child); 1334 } 1335 1336 static void 1337 enic_setup_sysctl(struct enic_softc *softc) 1338 { 1339 device_t dev; 1340 struct sysctl_ctx_list *ctx; 1341 struct sysctl_oid *tree; 1342 struct sysctl_oid_list *child; 1343 1344 dev = softc->dev; 1345 ctx = device_get_sysctl_ctx(dev); 1346 tree = device_get_sysctl_tree(dev); 1347 child = SYSCTL_CHILDREN(tree); 1348 1349 enic_setup_queue_sysctl(softc, ctx, child); 1350 } 1351 1352 static void 1353 enic_enable_intr(struct enic_softc *softc, int irq) 1354 { 1355 struct enic *enic = &softc->enic; 1356 1357 vnic_intr_unmask(&enic->intr[irq]); 1358 vnic_intr_return_all_credits(&enic->intr[irq]); 1359 } 1360 1361 static void 1362 enic_disable_intr(struct enic_softc *softc, int irq) 1363 { 1364 struct enic *enic = &softc->enic; 1365 1366 vnic_intr_mask(&enic->intr[irq]); 1367 vnic_intr_masked(&enic->intr[irq]); /* flush write */ 1368 } 1369 1370 static int 1371 enic_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 1372 { 1373 struct enic_softc *softc; 1374 if_softc_ctx_t scctx; 1375 1376 softc = iflib_get_softc(ctx); 1377 scctx = softc->scctx; 1378 1379 enic_enable_intr(softc, qid + scctx->isc_nrxqsets); 1380 1381 return 0; 1382 } 1383 1384 static int 1385 enic_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 1386 { 1387 struct enic_softc *softc; 1388 1389 softc = iflib_get_softc(ctx); 1390 enic_enable_intr(softc, qid); 1391 1392 return 0; 1393 } 1394 1395 static void 1396 enic_intr_enable_all(if_ctx_t ctx) 1397 { 1398 struct enic_softc *softc; 1399 if_softc_ctx_t scctx; 1400 int i; 1401 1402 softc = iflib_get_softc(ctx); 1403 scctx = softc->scctx; 1404 1405 for (i = 0; i < scctx->isc_vectors; i++) { 1406 enic_enable_intr(softc, i); 1407 } 1408 } 1409 1410 static void 1411 enic_intr_disable_all(if_ctx_t ctx) 1412 { 1413 struct enic_softc *softc; 1414 if_softc_ctx_t scctx; 1415 int i; 1416 1417 softc = iflib_get_softc(ctx); 1418 scctx = softc->scctx; 1419 /* 1420 * iflib may invoke this routine before enic_attach_post() has run, 1421 * which is before the top level shared data area is initialized and 1422 * the device made aware of it. 1423 */ 1424 1425 for (i = 0; i < scctx->isc_vectors; i++) { 1426 enic_disable_intr(softc, i); 1427 } 1428 } 1429 1430 static int 1431 enic_dev_open(struct enic *enic) 1432 { 1433 int err; 1434 int flags = CMD_OPENF_IG_DESCCACHE; 1435 1436 err = enic_dev_wait(enic->vdev, vnic_dev_open, 1437 vnic_dev_open_done, flags); 1438 if (err) 1439 dev_err(enic_get_dev(enic), 1440 "vNIC device open failed, err %d\n", err); 1441 1442 return err; 1443 } 1444 1445 static int 1446 enic_dev_init(struct enic *enic) 1447 { 1448 int err; 1449 1450 vnic_dev_intr_coal_timer_info_default(enic->vdev); 1451 1452 /* 1453 * Get vNIC configuration 1454 */ 1455 err = enic_get_vnic_config(enic); 1456 if (err) { 1457 dev_err(dev, "Get vNIC configuration failed, aborting\n"); 1458 return err; 1459 } 1460 1461 /* Get available resource counts */ 1462 enic_get_res_counts(enic); 1463 1464 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */ 1465 enic->intr_queues = malloc(sizeof(*enic->intr_queues) * 1466 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO); 1467 1468 vnic_dev_set_reset_flag(enic->vdev, 0); 1469 enic->max_flow_counter = -1; 1470 1471 /* set up link status checking */ 1472 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */ 1473 1474 enic->overlay_offload = false; 1475 if (enic->disable_overlay && enic->vxlan) { 1476 /* 1477 * Explicitly disable overlay offload as the setting is 1478 * sticky, and resetting vNIC does not disable it. 1479 */ 1480 if (vnic_dev_overlay_offload_ctrl(enic->vdev, 1481 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_DISABLE)) { 1482 dev_err(enic, "failed to disable overlay offload\n"); 1483 } else { 1484 dev_info(enic, "Overlay offload is disabled\n"); 1485 } 1486 } 1487 if (!enic->disable_overlay && enic->vxlan && 1488 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */ 1489 vnic_dev_overlay_offload_ctrl(enic->vdev, 1490 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_ENABLE) == 0) { 1491 enic->overlay_offload = true; 1492 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT; 1493 dev_info(enic, "Overlay offload is enabled\n"); 1494 /* 1495 * Reset the vxlan port to the default, as the NIC firmware 1496 * does not reset it automatically and keeps the old setting. 1497 */ 1498 if (vnic_dev_overlay_offload_cfg(enic->vdev, 1499 OVERLAY_CFG_VXLAN_PORT_UPDATE, ENIC_DEFAULT_VXLAN_PORT)) { 1500 dev_err(enic, "failed to update vxlan port\n"); 1501 return (EINVAL); 1502 } 1503 } 1504 return 0; 1505 } 1506 1507 static void * 1508 enic_alloc_consistent(void *priv, size_t size, bus_addr_t * dma_handle, 1509 struct iflib_dma_info *res, u8 * name) 1510 { 1511 void *vaddr; 1512 *dma_handle = 0; 1513 struct enic *enic = (struct enic *)priv; 1514 int rz; 1515 1516 rz = iflib_dma_alloc(enic->softc->ctx, size, res, BUS_DMA_NOWAIT); 1517 if (rz) { 1518 pr_err("%s : Failed to allocate memory requested for %s\n", 1519 __func__, name); 1520 return NULL; 1521 } 1522 1523 vaddr = res->idi_vaddr; 1524 *dma_handle = res->idi_paddr; 1525 1526 return vaddr; 1527 } 1528 1529 static void 1530 enic_free_consistent(void *priv, size_t size, void *vaddr, 1531 bus_addr_t dma_handle, struct iflib_dma_info *res) 1532 { 1533 iflib_dma_free(res); 1534 } 1535 1536 static int 1537 enic_pci_mapping(struct enic_softc *softc) 1538 { 1539 int rc; 1540 1541 rc = enic_map_bar(softc, &softc->mem, 0, true); 1542 if (rc) 1543 return rc; 1544 1545 rc = enic_map_bar(softc, &softc->io, 2, false); 1546 1547 return rc; 1548 } 1549 1550 static void 1551 enic_pci_mapping_free(struct enic_softc *softc) 1552 { 1553 if (softc->mem.res != NULL) 1554 bus_release_resource(softc->dev, SYS_RES_MEMORY, 1555 softc->mem.rid, softc->mem.res); 1556 softc->mem.res = NULL; 1557 1558 if (softc->io.res != NULL) 1559 bus_release_resource(softc->dev, SYS_RES_MEMORY, 1560 softc->io.rid, softc->io.res); 1561 softc->io.res = NULL; 1562 } 1563 1564 static int 1565 enic_dev_wait(struct vnic_dev *vdev, int (*start) (struct vnic_dev *, int), 1566 int (*finished) (struct vnic_dev *, int *), int arg) 1567 { 1568 int done; 1569 int err; 1570 int i; 1571 1572 err = start(vdev, arg); 1573 if (err) 1574 return err; 1575 1576 /* Wait for func to complete...2 seconds max */ 1577 for (i = 0; i < 2000; i++) { 1578 err = finished(vdev, &done); 1579 if (err) 1580 return err; 1581 if (done) 1582 return 0; 1583 usleep(1000); 1584 } 1585 return (ETIMEDOUT); 1586 } 1587 1588 static int 1589 enic_map_bar(struct enic_softc *softc, struct enic_bar_info *bar, int bar_num, 1590 bool shareable) 1591 { 1592 uint32_t flag; 1593 1594 if (bar->res != NULL) { 1595 device_printf(softc->dev, "Bar %d already mapped\n", bar_num); 1596 return (EDOOFUS); 1597 } 1598 1599 bar->rid = PCIR_BAR(bar_num); 1600 flag = RF_ACTIVE; 1601 if (shareable) 1602 flag |= RF_SHAREABLE; 1603 1604 if ((bar->res = bus_alloc_resource_any(softc->dev, 1605 SYS_RES_MEMORY, &bar->rid, flag)) == NULL) { 1606 device_printf(softc->dev, 1607 "PCI BAR%d mapping failure\n", bar_num); 1608 return (ENXIO); 1609 } 1610 bar->tag = rman_get_bustag(bar->res); 1611 bar->handle = rman_get_bushandle(bar->res); 1612 bar->size = rman_get_size(bar->res); 1613 1614 return 0; 1615 } 1616 1617 void 1618 enic_init_vnic_resources(struct enic *enic) 1619 { 1620 unsigned int error_interrupt_enable = 1; 1621 unsigned int error_interrupt_offset = 0; 1622 unsigned int rxq_interrupt_enable = 0; 1623 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET; 1624 unsigned int txq_interrupt_enable = 0; 1625 unsigned int txq_interrupt_offset; 1626 unsigned int index = 0; 1627 unsigned int cq_idx; 1628 if_softc_ctx_t scctx; 1629 1630 scctx = enic->softc->scctx; 1631 1632 rxq_interrupt_enable = 1; 1633 txq_interrupt_enable = 0; 1634 1635 rxq_interrupt_offset = 0; 1636 txq_interrupt_offset = scctx->isc_nrxqsets; 1637 1638 for (index = 0; index < enic->intr_count; index++) { 1639 vnic_intr_alloc(enic->vdev, &enic->intr[index], index); 1640 } 1641 1642 for (index = 0; index < scctx->isc_nrxqsets; index++) { 1643 cq_idx = enic_cq_rq(enic, index); 1644 1645 vnic_rq_clean(&enic->rq[index]); 1646 vnic_rq_init(&enic->rq[index], cq_idx, error_interrupt_enable, 1647 error_interrupt_offset); 1648 1649 vnic_cq_clean(&enic->cq[cq_idx]); 1650 vnic_cq_init(&enic->cq[cq_idx], 1651 0 /* flow_control_enable */ , 1652 1 /* color_enable */ , 1653 0 /* cq_head */ , 1654 0 /* cq_tail */ , 1655 1 /* cq_tail_color */ , 1656 rxq_interrupt_enable, 1657 1 /* cq_entry_enable */ , 1658 0 /* cq_message_enable */ , 1659 rxq_interrupt_offset, 1660 0 /* cq_message_addr */ ); 1661 if (rxq_interrupt_enable) 1662 rxq_interrupt_offset++; 1663 } 1664 1665 for (index = 0; index < scctx->isc_ntxqsets; index++) { 1666 cq_idx = enic_cq_wq(enic, index); 1667 vnic_wq_clean(&enic->wq[index]); 1668 vnic_wq_init(&enic->wq[index], cq_idx, error_interrupt_enable, 1669 error_interrupt_offset); 1670 /* Compute unsupported ol flags for enic_prep_pkts() */ 1671 enic->wq[index].tx_offload_notsup_mask = 0; 1672 1673 vnic_cq_clean(&enic->cq[cq_idx]); 1674 vnic_cq_init(&enic->cq[cq_idx], 1675 0 /* flow_control_enable */ , 1676 1 /* color_enable */ , 1677 0 /* cq_head */ , 1678 0 /* cq_tail */ , 1679 1 /* cq_tail_color */ , 1680 txq_interrupt_enable, 1681 1, 1682 0, 1683 txq_interrupt_offset, 1684 0 /* (u64)enic->wq[index].cqmsg_rz->iova */ ); 1685 1686 } 1687 1688 for (index = 0; index < enic->intr_count; index++) { 1689 vnic_intr_init(&enic->intr[index], 125, 1690 enic->config.intr_timer_type, /* mask_on_assertion */ 1); 1691 } 1692 } 1693 1694 static void 1695 enic_update_packet_filter(struct enic *enic) 1696 { 1697 struct enic_softc *softc = enic->softc; 1698 1699 ENIC_LOCK(softc); 1700 vnic_dev_packet_filter(enic->vdev, 1701 softc->directed, 1702 softc->multicast, 1703 softc->broadcast, 1704 softc->promisc, 1705 softc->allmulti); 1706 ENIC_UNLOCK(softc); 1707 } 1708 1709 static bool 1710 enic_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event) 1711 { 1712 switch (event) { 1713 case IFLIB_RESTART_VLAN_CONFIG: 1714 default: 1715 return (false); 1716 } 1717 } 1718 1719 int 1720 enic_setup_finish(struct enic *enic) 1721 { 1722 struct enic_softc *softc = enic->softc; 1723 1724 /* Default conf */ 1725 softc->directed = 1; 1726 softc->multicast = 0; 1727 softc->broadcast = 1; 1728 softc->promisc = 0; 1729 softc->allmulti = 1; 1730 enic_update_packet_filter(enic); 1731 1732 return 0; 1733 } 1734