1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 */ 5 6 #include "opt_rss.h" 7 8 #include <sys/param.h> 9 #include <sys/systm.h> 10 #include <sys/kernel.h> 11 #include <sys/endian.h> 12 #include <sys/sockio.h> 13 #include <sys/mbuf.h> 14 #include <sys/malloc.h> 15 #include <sys/module.h> 16 #include <sys/socket.h> 17 #include <sys/sysctl.h> 18 #include <sys/smp.h> 19 #include <vm/vm.h> 20 #include <vm/pmap.h> 21 22 #include <net/ethernet.h> 23 #include <net/if.h> 24 #include <net/if_var.h> 25 #include <net/if_arp.h> 26 #include <net/if_dl.h> 27 #include <net/if_types.h> 28 #include <net/if_media.h> 29 #include <net/if_vlan_var.h> 30 #include <net/iflib.h> 31 #ifdef RSS 32 #include <net/rss_config.h> 33 #endif 34 35 #include <netinet/in_systm.h> 36 #include <netinet/in.h> 37 #include <netinet/ip.h> 38 #include <netinet/ip6.h> 39 #include <netinet6/ip6_var.h> 40 #include <netinet/udp.h> 41 #include <netinet/tcp.h> 42 43 #include <machine/bus.h> 44 #include <machine/resource.h> 45 #include <sys/bus.h> 46 #include <sys/rman.h> 47 48 #include <dev/pci/pcireg.h> 49 #include <dev/pci/pcivar.h> 50 51 #include "ifdi_if.h" 52 #include "enic.h" 53 54 #include "opt_inet.h" 55 #include "opt_inet6.h" 56 57 static SYSCTL_NODE(_hw, OID_AUTO, enic, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 58 "ENIC"); 59 60 static pci_vendor_info_t enic_vendor_info_array[] = 61 { 62 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET, 63 DRV_DESCRIPTION), 64 PVID(CISCO_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET_VF, 65 DRV_DESCRIPTION " VF"), 66 /* required last entry */ 67 68 PVID_END 69 }; 70 71 static void *enic_register(device_t); 72 static int enic_attach_pre(if_ctx_t); 73 static int enic_msix_intr_assign(if_ctx_t, int); 74 75 static int enic_attach_post(if_ctx_t); 76 static int enic_detach(if_ctx_t); 77 78 static int enic_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 79 static int enic_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 80 static void enic_queues_free(if_ctx_t); 81 static int enic_rxq_intr(void *); 82 static int enic_event_intr(void *); 83 static int enic_err_intr(void *); 84 static void enic_stop(if_ctx_t); 85 static void enic_init(if_ctx_t); 86 static void enic_multi_set(if_ctx_t); 87 static int enic_mtu_set(if_ctx_t, uint32_t); 88 static void enic_media_status(if_ctx_t, struct ifmediareq *); 89 static int enic_media_change(if_ctx_t); 90 static int enic_promisc_set(if_ctx_t, int); 91 static uint64_t enic_get_counter(if_ctx_t, ift_counter); 92 static void enic_update_admin_status(if_ctx_t); 93 static void enic_txq_timer(if_ctx_t, uint16_t); 94 static int enic_link_is_up(struct enic_softc *); 95 static void enic_link_status(struct enic_softc *); 96 static void enic_set_lladdr(struct enic_softc *); 97 static void enic_setup_txq_sysctl(struct vnic_wq *, int, struct sysctl_ctx_list *, 98 struct sysctl_oid_list *); 99 static void enic_setup_rxq_sysctl(struct vnic_rq *, int, struct sysctl_ctx_list *, 100 struct sysctl_oid_list *); 101 static void enic_setup_sysctl(struct enic_softc *); 102 static int enic_tx_queue_intr_enable(if_ctx_t, uint16_t); 103 static int enic_rx_queue_intr_enable(if_ctx_t, uint16_t); 104 static void enic_enable_intr(struct enic_softc *, int); 105 static void enic_disable_intr(struct enic_softc *, int); 106 static void enic_intr_enable_all(if_ctx_t); 107 static void enic_intr_disable_all(if_ctx_t); 108 static int enic_dev_open(struct enic *); 109 static int enic_dev_init(struct enic *); 110 static void *enic_alloc_consistent(void *, size_t, bus_addr_t *, 111 struct iflib_dma_info *, u8 *); 112 static void enic_free_consistent(void *, size_t, void *, bus_addr_t, 113 struct iflib_dma_info *); 114 static int enic_pci_mapping(struct enic_softc *); 115 static void enic_pci_mapping_free(struct enic_softc *); 116 static int enic_dev_wait(struct vnic_dev *, int (*) (struct vnic_dev *, int), 117 int (*) (struct vnic_dev *, int *), int arg); 118 static int enic_map_bar(struct enic_softc *, struct enic_bar_info *, int, bool); 119 static void enic_update_packet_filter(struct enic *enic); 120 121 typedef enum { 122 ENIC_BARRIER_RD, 123 ENIC_BARRIER_WR, 124 ENIC_BARRIER_RDWR, 125 } enic_barrier_t; 126 127 static device_method_t enic_methods[] = { 128 /* Device interface */ 129 DEVMETHOD(device_register, enic_register), 130 DEVMETHOD(device_probe, iflib_device_probe), 131 DEVMETHOD(device_attach, iflib_device_attach), 132 DEVMETHOD(device_detach, iflib_device_detach), 133 DEVMETHOD(device_shutdown, iflib_device_shutdown), 134 DEVMETHOD(device_suspend, iflib_device_suspend), 135 DEVMETHOD(device_resume, iflib_device_resume), 136 DEVMETHOD_END 137 }; 138 139 static driver_t enic_driver = { 140 "enic", enic_methods, sizeof(struct enic_softc) 141 }; 142 143 DRIVER_MODULE(enic, pci, enic_driver, 0, 0); 144 IFLIB_PNP_INFO(pci, enic, enic_vendor_info_array); 145 MODULE_VERSION(enic, 2); 146 147 MODULE_DEPEND(enic, pci, 1, 1, 1); 148 MODULE_DEPEND(enic, ether, 1, 1, 1); 149 MODULE_DEPEND(enic, iflib, 1, 1, 1); 150 151 static device_method_t enic_iflib_methods[] = { 152 DEVMETHOD(ifdi_tx_queues_alloc, enic_tx_queues_alloc), 153 DEVMETHOD(ifdi_rx_queues_alloc, enic_rx_queues_alloc), 154 DEVMETHOD(ifdi_queues_free, enic_queues_free), 155 156 DEVMETHOD(ifdi_attach_pre, enic_attach_pre), 157 DEVMETHOD(ifdi_attach_post, enic_attach_post), 158 DEVMETHOD(ifdi_detach, enic_detach), 159 160 DEVMETHOD(ifdi_init, enic_init), 161 DEVMETHOD(ifdi_stop, enic_stop), 162 DEVMETHOD(ifdi_multi_set, enic_multi_set), 163 DEVMETHOD(ifdi_mtu_set, enic_mtu_set), 164 DEVMETHOD(ifdi_media_status, enic_media_status), 165 DEVMETHOD(ifdi_media_change, enic_media_change), 166 DEVMETHOD(ifdi_promisc_set, enic_promisc_set), 167 DEVMETHOD(ifdi_get_counter, enic_get_counter), 168 DEVMETHOD(ifdi_update_admin_status, enic_update_admin_status), 169 DEVMETHOD(ifdi_timer, enic_txq_timer), 170 171 DEVMETHOD(ifdi_tx_queue_intr_enable, enic_tx_queue_intr_enable), 172 DEVMETHOD(ifdi_rx_queue_intr_enable, enic_rx_queue_intr_enable), 173 DEVMETHOD(ifdi_intr_enable, enic_intr_enable_all), 174 DEVMETHOD(ifdi_intr_disable, enic_intr_disable_all), 175 DEVMETHOD(ifdi_msix_intr_assign, enic_msix_intr_assign), 176 177 DEVMETHOD_END 178 }; 179 180 static driver_t enic_iflib_driver = { 181 "enic", enic_iflib_methods, sizeof(struct enic_softc) 182 }; 183 184 extern struct if_txrx enic_txrx; 185 186 static struct if_shared_ctx enic_sctx_init = { 187 .isc_magic = IFLIB_MAGIC, 188 .isc_q_align = 512, 189 190 .isc_tx_maxsize = ENIC_TX_MAX_PKT_SIZE, 191 .isc_tx_maxsegsize = PAGE_SIZE, 192 193 /* 194 * These values are used to configure the busdma tag used for receive 195 * descriptors. Each receive descriptor only points to one buffer. 196 */ 197 .isc_rx_maxsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, /* One buf per 198 * descriptor */ 199 .isc_rx_nsegments = 1, /* One mapping per descriptor */ 200 .isc_rx_maxsegsize = ENIC_DEFAULT_RX_MAX_PKT_SIZE, 201 .isc_admin_intrcnt = 3, 202 .isc_vendor_info = enic_vendor_info_array, 203 .isc_driver_version = "1", 204 .isc_driver = &enic_iflib_driver, 205 .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ, 206 207 /* 208 * Number of receive queues per receive queue set, with associated 209 * descriptor settings for each. 210 */ 211 212 .isc_nrxqs = 2, 213 .isc_nfl = 1, /* one free list for each receive command 214 * queue */ 215 .isc_nrxd_min = {16, 16}, 216 .isc_nrxd_max = {2048, 2048}, 217 .isc_nrxd_default = {64, 64}, 218 219 /* 220 * Number of transmit queues per transmit queue set, with associated 221 * descriptor settings for each. 222 */ 223 .isc_ntxqs = 2, 224 .isc_ntxd_min = {16, 16}, 225 .isc_ntxd_max = {2048, 2048}, 226 .isc_ntxd_default = {64, 64}, 227 }; 228 229 static void * 230 enic_register(device_t dev) 231 { 232 return (&enic_sctx_init); 233 } 234 235 static int 236 enic_attach_pre(if_ctx_t ctx) 237 { 238 if_softc_ctx_t scctx; 239 struct enic_softc *softc; 240 struct vnic_dev *vdev; 241 struct enic *enic; 242 device_t dev; 243 244 int err = -1; 245 int rc = 0; 246 int i; 247 u64 a0 = 0, a1 = 0; 248 int wait = 1000; 249 struct vnic_stats *stats; 250 int ret; 251 252 dev = iflib_get_dev(ctx); 253 softc = iflib_get_softc(ctx); 254 softc->dev = dev; 255 softc->ctx = ctx; 256 softc->sctx = iflib_get_sctx(ctx); 257 softc->scctx = iflib_get_softc_ctx(ctx); 258 softc->ifp = iflib_get_ifp(ctx); 259 softc->media = iflib_get_media(ctx); 260 softc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 261 ENIC_MAX_MULTICAST_ADDRESSES, M_DEVBUF, 262 M_NOWAIT | M_ZERO); 263 if (softc->mta == NULL) 264 return (ENOMEM); 265 scctx = softc->scctx; 266 267 mtx_init(&softc->enic_lock, "ENIC Lock", NULL, MTX_DEF); 268 269 pci_enable_busmaster(softc->dev); 270 if (enic_pci_mapping(softc)) 271 return (ENXIO); 272 273 enic = &softc->enic; 274 enic->softc = softc; 275 vdev = &softc->vdev; 276 vdev->softc = softc; 277 enic->vdev = vdev; 278 vdev->priv = enic; 279 280 ENIC_LOCK(softc); 281 vnic_dev_register(vdev, &softc->mem, 1); 282 enic->vdev = vdev; 283 vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0); 284 285 vnic_dev_cmd(vdev, CMD_INIT_v1, &a0, &a1, wait); 286 vnic_dev_cmd(vdev, CMD_GET_MAC_ADDR, &a0, &a1, wait); 287 288 bcopy((u_int8_t *) & a0, softc->mac_addr, ETHER_ADDR_LEN); 289 iflib_set_mac(ctx, softc->mac_addr); 290 291 vnic_register_cbacks(enic->vdev, enic_alloc_consistent, 292 enic_free_consistent); 293 294 /* 295 * Allocate the consistent memory for stats and counters upfront so 296 * both primary and secondary processes can access them. 297 */ 298 ENIC_UNLOCK(softc); 299 err = vnic_dev_alloc_stats_mem(enic->vdev); 300 ENIC_LOCK(softc); 301 if (err) { 302 dev_err(enic, "Failed to allocate cmd memory, aborting\n"); 303 goto err_out_unregister; 304 } 305 vnic_dev_stats_clear(enic->vdev); 306 ret = vnic_dev_stats_dump(enic->vdev, &stats); 307 if (ret) { 308 dev_err(enic, "Error in getting stats\n"); 309 goto err_out_unregister; 310 } 311 err = vnic_dev_alloc_counter_mem(enic->vdev); 312 if (err) { 313 dev_err(enic, "Failed to allocate counter memory, aborting\n"); 314 goto err_out_unregister; 315 } 316 317 /* Issue device open to get device in known state */ 318 err = enic_dev_open(enic); 319 if (err) { 320 dev_err(enic, "vNIC dev open failed, aborting\n"); 321 goto err_out_unregister; 322 } 323 324 /* Set ingress vlan rewrite mode before vnic initialization */ 325 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN; 326 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, 327 enic->ig_vlan_rewrite_mode); 328 if (err) { 329 dev_err(enic, 330 "Failed to set ingress vlan rewrite mode, aborting.\n"); 331 goto err_out_dev_close; 332 } 333 334 /* 335 * Issue device init to initialize the vnic-to-switch link. We'll 336 * start with carrier off and wait for link UP notification later to 337 * turn on carrier. We don't need to wait here for the 338 * vnic-to-switch link initialization to complete; link UP 339 * notification is the indication that the process is complete. 340 */ 341 342 err = vnic_dev_init(enic->vdev, 0); 343 if (err) { 344 dev_err(enic, "vNIC dev init failed, aborting\n"); 345 goto err_out_dev_close; 346 } 347 348 err = enic_dev_init(enic); 349 if (err) { 350 dev_err(enic, "Device initialization failed, aborting\n"); 351 goto err_out_dev_close; 352 } 353 ENIC_UNLOCK(softc); 354 355 enic->port_mtu = vnic_dev_mtu(enic->vdev); 356 357 softc->scctx = iflib_get_softc_ctx(ctx); 358 scctx = softc->scctx; 359 scctx->isc_txrx = &enic_txrx; 360 scctx->isc_capabilities = scctx->isc_capenable = 0; 361 scctx->isc_tx_csum_flags = 0; 362 scctx->isc_max_frame_size = enic->config.mtu + ETHER_HDR_LEN + \ 363 ETHER_CRC_LEN; 364 scctx->isc_nrxqsets_max = enic->conf_rq_count; 365 scctx->isc_ntxqsets_max = enic->conf_wq_count; 366 scctx->isc_nrxqsets = enic->conf_rq_count; 367 scctx->isc_ntxqsets = enic->conf_wq_count; 368 for (i = 0; i < enic->conf_wq_count; i++) { 369 scctx->isc_ntxd[i] = enic->config.wq_desc_count; 370 scctx->isc_txqsizes[i] = sizeof(struct cq_enet_wq_desc) 371 * scctx->isc_ntxd[i]; 372 scctx->isc_ntxd[i + enic->conf_wq_count] = 373 enic->config.wq_desc_count; 374 scctx->isc_txqsizes[i + enic->conf_wq_count] = 375 sizeof(struct cq_desc) * scctx->isc_ntxd[i + 376 enic->conf_wq_count]; 377 } 378 for (i = 0; i < enic->conf_rq_count; i++) { 379 scctx->isc_nrxd[i] = enic->config.rq_desc_count; 380 scctx->isc_rxqsizes[i] = sizeof(struct cq_enet_rq_desc) * 381 scctx->isc_nrxd[i]; 382 scctx->isc_nrxd[i + enic->conf_rq_count] = 383 enic->config.rq_desc_count; 384 scctx->isc_rxqsizes[i + enic->conf_rq_count] = sizeof(struct 385 cq_desc) * scctx->isc_nrxd[i + enic->conf_rq_count]; 386 } 387 scctx->isc_tx_nsegments = 31; 388 389 scctx->isc_vectors = enic->conf_cq_count; 390 scctx->isc_msix_bar = -1; 391 392 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 393 ifmedia_add(softc->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); 394 ifmedia_add(softc->media, IFM_ETHER | IFM_10_FL, 0, NULL); 395 396 /* 397 * Allocate the CQ here since TX is called first before RX for now 398 * assume RX and TX are the same 399 */ 400 if (softc->enic.cq == NULL) 401 softc->enic.cq = malloc(sizeof(struct vnic_cq) * 402 softc->enic.wq_count + softc->enic.rq_count, M_DEVBUF, 403 M_NOWAIT | M_ZERO); 404 if (softc->enic.cq == NULL) 405 return (ENOMEM); 406 407 softc->enic.cq->ntxqsets = softc->enic.wq_count + softc->enic.rq_count; 408 409 /* 410 * Allocate the consistent memory for stats and counters upfront so 411 * both primary and secondary processes can access them. 412 */ 413 err = vnic_dev_alloc_stats_mem(enic->vdev); 414 if (err) { 415 dev_err(enic, "Failed to allocate cmd memory, aborting\n"); 416 } 417 418 return (rc); 419 420 err_out_dev_close: 421 vnic_dev_close(enic->vdev); 422 err_out_unregister: 423 free(softc->vdev.devcmd, M_DEVBUF); 424 free(softc->enic.intr_queues, M_DEVBUF); 425 free(softc->enic.cq, M_DEVBUF); 426 free(softc->mta, M_DEVBUF); 427 rc = -1; 428 pci_disable_busmaster(softc->dev); 429 enic_pci_mapping_free(softc); 430 mtx_destroy(&softc->enic_lock); 431 return (rc); 432 } 433 434 static int 435 enic_msix_intr_assign(if_ctx_t ctx, int msix) 436 { 437 struct enic_softc *softc; 438 struct enic *enic; 439 if_softc_ctx_t scctx; 440 441 int error; 442 int i; 443 char irq_name[16]; 444 445 softc = iflib_get_softc(ctx); 446 enic = &softc->enic; 447 scctx = softc->scctx; 448 449 ENIC_LOCK(softc); 450 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSIX); 451 ENIC_UNLOCK(softc); 452 453 enic->intr_queues = malloc(sizeof(*enic->intr_queues) * 454 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO); 455 enic->intr = malloc(sizeof(*enic->intr) * msix, M_DEVBUF, M_NOWAIT 456 | M_ZERO); 457 for (i = 0; i < scctx->isc_nrxqsets; i++) { 458 snprintf(irq_name, sizeof(irq_name), "erxq%d:%d", i, 459 device_get_unit(softc->dev)); 460 461 error = iflib_irq_alloc_generic(ctx, 462 &enic->intr_queues[i].intr_irq, i + 1, IFLIB_INTR_RX, 463 enic_rxq_intr, &enic->rq[i], i, irq_name); 464 if (error) { 465 device_printf(iflib_get_dev(ctx), 466 "Failed to register rxq %d interrupt handler\n", i); 467 return (error); 468 } 469 enic->intr[i].index = i; 470 enic->intr[i].vdev = enic->vdev; 471 ENIC_LOCK(softc); 472 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, 473 RES_TYPE_INTR_CTRL, i); 474 vnic_intr_mask(&enic->intr[i]); 475 ENIC_UNLOCK(softc); 476 } 477 478 for (i = scctx->isc_nrxqsets; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) { 479 snprintf(irq_name, sizeof(irq_name), "etxq%d:%d", i - 480 scctx->isc_nrxqsets, device_get_unit(softc->dev)); 481 482 483 iflib_softirq_alloc_generic(ctx, &enic->intr_queues[i].intr_irq, IFLIB_INTR_TX, &enic->wq[i - scctx->isc_nrxqsets], i - scctx->isc_nrxqsets, irq_name); 484 485 486 enic->intr[i].index = i; 487 enic->intr[i].vdev = enic->vdev; 488 ENIC_LOCK(softc); 489 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, 490 RES_TYPE_INTR_CTRL, i); 491 vnic_intr_mask(&enic->intr[i]); 492 ENIC_UNLOCK(softc); 493 } 494 495 i = scctx->isc_nrxqsets + scctx->isc_ntxqsets; 496 error = iflib_irq_alloc_generic(ctx, &softc->enic_event_intr_irq, 497 i + 1, IFLIB_INTR_ADMIN, enic_event_intr, softc, 0, "event"); 498 if (error) { 499 device_printf(iflib_get_dev(ctx), 500 "Failed to register event interrupt handler\n"); 501 return (error); 502 } 503 504 enic->intr[i].index = i; 505 enic->intr[i].vdev = enic->vdev; 506 ENIC_LOCK(softc); 507 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL, 508 i); 509 vnic_intr_mask(&enic->intr[i]); 510 ENIC_UNLOCK(softc); 511 512 i++; 513 error = iflib_irq_alloc_generic(ctx, &softc->enic_err_intr_irq, 514 i + 1, IFLIB_INTR_ADMIN, enic_err_intr, softc, 0, "err"); 515 if (error) { 516 device_printf(iflib_get_dev(ctx), 517 "Failed to register event interrupt handler\n"); 518 return (error); 519 } 520 enic->intr[i].index = i; 521 enic->intr[i].vdev = enic->vdev; 522 ENIC_LOCK(softc); 523 enic->intr[i].ctrl = vnic_dev_get_res(enic->vdev, RES_TYPE_INTR_CTRL, 524 i); 525 vnic_intr_mask(&enic->intr[i]); 526 ENIC_UNLOCK(softc); 527 528 enic->intr_count = msix; 529 530 return (0); 531 } 532 533 static void 534 enic_free_irqs(struct enic_softc *softc) 535 { 536 if_softc_ctx_t scctx; 537 538 struct enic *enic; 539 int i; 540 541 scctx = softc->scctx; 542 enic = &softc->enic; 543 544 for (i = 0; i < scctx->isc_nrxqsets + scctx->isc_ntxqsets; i++) { 545 iflib_irq_free(softc->ctx, &enic->intr_queues[i].intr_irq); 546 } 547 548 iflib_irq_free(softc->ctx, &softc->enic_event_intr_irq); 549 iflib_irq_free(softc->ctx, &softc->enic_err_intr_irq); 550 free(enic->intr_queues, M_DEVBUF); 551 free(enic->intr, M_DEVBUF); 552 } 553 554 static int 555 enic_attach_post(if_ctx_t ctx) 556 { 557 struct enic *enic; 558 struct enic_softc *softc; 559 int error = 0; 560 561 softc = iflib_get_softc(ctx); 562 enic = &softc->enic; 563 564 enic_setup_sysctl(softc); 565 566 enic_init_vnic_resources(enic); 567 enic_setup_finish(enic); 568 569 ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 570 ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO); 571 572 return (error); 573 } 574 575 static int 576 enic_detach(if_ctx_t ctx) 577 { 578 struct enic_softc *softc; 579 struct enic *enic; 580 581 softc = iflib_get_softc(ctx); 582 enic = &softc->enic; 583 584 vnic_dev_notify_unset(enic->vdev); 585 586 enic_free_irqs(softc); 587 588 ENIC_LOCK(softc); 589 vnic_dev_close(enic->vdev); 590 free(softc->vdev.devcmd, M_DEVBUF); 591 pci_disable_busmaster(softc->dev); 592 enic_pci_mapping_free(softc); 593 ENIC_UNLOCK(softc); 594 595 return 0; 596 } 597 598 static int 599 enic_tx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs, 600 int ntxqs, int ntxqsets) 601 { 602 struct enic_softc *softc; 603 int q; 604 605 softc = iflib_get_softc(ctx); 606 /* Allocate the array of transmit queues */ 607 softc->enic.wq = malloc(sizeof(struct vnic_wq) * 608 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 609 if (softc->enic.wq == NULL) 610 return (ENOMEM); 611 612 /* Initialize driver state for each transmit queue */ 613 614 /* 615 * Allocate queue state that is shared with the device. This check 616 * and call is performed in both enic_tx_queues_alloc() and 617 * enic_rx_queues_alloc() so that we don't have to care which order 618 * iflib invokes those routines in. 619 */ 620 621 /* Record descriptor ring vaddrs and paddrs */ 622 ENIC_LOCK(softc); 623 for (q = 0; q < ntxqsets; q++) { 624 struct vnic_wq *wq; 625 struct vnic_cq *cq; 626 unsigned int cq_wq; 627 628 wq = &softc->enic.wq[q]; 629 cq_wq = enic_cq_wq(&softc->enic, q); 630 cq = &softc->enic.cq[cq_wq]; 631 632 /* Completion ring */ 633 wq->vdev = softc->enic.vdev; 634 wq->index = q; 635 wq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_WQ, 636 wq->index); 637 vnic_wq_disable(wq); 638 639 wq->ring.desc_size = sizeof(struct wq_enet_desc); 640 wq->ring.desc_count = softc->scctx->isc_ntxd[q]; 641 wq->ring.desc_avail = wq->ring.desc_count - 1; 642 wq->ring.last_count = wq->ring.desc_count; 643 wq->head_idx = 0; 644 wq->tail_idx = 0; 645 646 wq->ring.size = wq->ring.desc_count * wq->ring.desc_size; 647 wq->ring.descs = vaddrs[q * ntxqs + 0]; 648 wq->ring.base_addr = paddrs[q * ntxqs + 0]; 649 650 /* Command ring */ 651 cq->vdev = softc->enic.vdev; 652 cq->index = cq_wq; 653 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, 654 RES_TYPE_CQ, cq->index); 655 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc); 656 cq->ring.desc_count = softc->scctx->isc_ntxd[q]; 657 cq->ring.desc_avail = cq->ring.desc_count - 1; 658 659 cq->ring.size = cq->ring.desc_count * cq->ring.desc_size; 660 cq->ring.descs = vaddrs[q * ntxqs + 1]; 661 cq->ring.base_addr = paddrs[q * ntxqs + 1]; 662 663 } 664 665 ENIC_UNLOCK(softc); 666 667 return (0); 668 } 669 670 671 672 static int 673 enic_rx_queues_alloc(if_ctx_t ctx, caddr_t * vaddrs, uint64_t * paddrs, 674 int nrxqs, int nrxqsets) 675 { 676 struct enic_softc *softc; 677 int q; 678 679 softc = iflib_get_softc(ctx); 680 /* Allocate the array of receive queues */ 681 softc->enic.rq = malloc(sizeof(struct vnic_rq) * nrxqsets, M_DEVBUF, 682 M_NOWAIT | M_ZERO); 683 if (softc->enic.rq == NULL) 684 return (ENOMEM); 685 686 /* Initialize driver state for each receive queue */ 687 688 /* 689 * Allocate queue state that is shared with the device. This check 690 * and call is performed in both enic_tx_queues_alloc() and 691 * enic_rx_queues_alloc() so that we don't have to care which order 692 * iflib invokes those routines in. 693 */ 694 695 /* Record descriptor ring vaddrs and paddrs */ 696 ENIC_LOCK(softc); 697 for (q = 0; q < nrxqsets; q++) { 698 struct vnic_rq *rq; 699 struct vnic_cq *cq; 700 unsigned int cq_rq; 701 702 rq = &softc->enic.rq[q]; 703 cq_rq = enic_cq_rq(&softc->enic, q); 704 cq = &softc->enic.cq[cq_rq]; 705 706 /* Completion ring */ 707 cq->vdev = softc->enic.vdev; 708 cq->index = cq_rq; 709 cq->ctrl = vnic_dev_get_res(softc->enic.vdev, RES_TYPE_CQ, 710 cq->index); 711 cq->ring.desc_size = sizeof(struct cq_enet_wq_desc); 712 cq->ring.desc_count = softc->scctx->isc_nrxd[1]; 713 cq->ring.desc_avail = cq->ring.desc_count - 1; 714 715 cq->ring.size = cq->ring.desc_count * cq->ring.desc_size; 716 cq->ring.descs = vaddrs[q * nrxqs + 0]; 717 cq->ring.base_addr = paddrs[q * nrxqs + 0]; 718 719 /* Command ring(s) */ 720 rq->vdev = softc->enic.vdev; 721 722 rq->index = q; 723 rq->ctrl = vnic_dev_get_res(softc->enic.vdev, 724 RES_TYPE_RQ, rq->index); 725 vnic_rq_disable(rq); 726 727 rq->ring.desc_size = sizeof(struct rq_enet_desc); 728 rq->ring.desc_count = softc->scctx->isc_nrxd[0]; 729 rq->ring.desc_avail = rq->ring.desc_count - 1; 730 731 rq->ring.size = rq->ring.desc_count * rq->ring.desc_size; 732 rq->ring.descs = vaddrs[q * nrxqs + 1]; 733 rq->ring.base_addr = paddrs[q * nrxqs + 1]; 734 rq->need_initial_post = true; 735 } 736 737 ENIC_UNLOCK(softc); 738 739 return (0); 740 } 741 742 static void 743 enic_queues_free(if_ctx_t ctx) 744 { 745 struct enic_softc *softc; 746 softc = iflib_get_softc(ctx); 747 748 free(softc->enic.rq, M_DEVBUF); 749 free(softc->enic.wq, M_DEVBUF); 750 free(softc->enic.cq, M_DEVBUF); 751 } 752 753 static int 754 enic_rxq_intr(void *rxq) 755 { 756 struct vnic_rq *rq; 757 if_t ifp; 758 759 rq = (struct vnic_rq *)rxq; 760 ifp = iflib_get_ifp(rq->vdev->softc->ctx); 761 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 762 return (FILTER_HANDLED); 763 764 return (FILTER_SCHEDULE_THREAD); 765 } 766 767 static int 768 enic_event_intr(void *vsc) 769 { 770 struct enic_softc *softc; 771 struct enic *enic; 772 uint32_t mtu; 773 774 softc = vsc; 775 enic = &softc->enic; 776 777 mtu = vnic_dev_mtu(enic->vdev); 778 if (mtu && mtu != enic->port_mtu) { 779 enic->port_mtu = mtu; 780 } 781 782 enic_link_status(softc); 783 784 return (FILTER_HANDLED); 785 } 786 787 static int 788 enic_err_intr(void *vsc) 789 { 790 struct enic_softc *softc; 791 792 softc = vsc; 793 794 enic_stop(softc->ctx); 795 enic_init(softc->ctx); 796 797 return (FILTER_HANDLED); 798 } 799 800 static void 801 enic_stop(if_ctx_t ctx) 802 { 803 struct enic_softc *softc; 804 struct enic *enic; 805 if_softc_ctx_t scctx; 806 unsigned int index; 807 808 softc = iflib_get_softc(ctx); 809 scctx = softc->scctx; 810 enic = &softc->enic; 811 812 if (softc->stopped) 813 return; 814 softc->link_active = 0; 815 softc->stopped = 1; 816 817 for (index = 0; index < scctx->isc_ntxqsets; index++) { 818 enic_stop_wq(enic, index); 819 vnic_wq_clean(&enic->wq[index]); 820 vnic_cq_clean(&enic->cq[enic_cq_rq(enic, index)]); 821 } 822 823 for (index = 0; index < scctx->isc_nrxqsets; index++) { 824 vnic_rq_clean(&enic->rq[index]); 825 vnic_cq_clean(&enic->cq[enic_cq_wq(enic, index)]); 826 } 827 828 for (index = 0; index < scctx->isc_vectors; index++) { 829 vnic_intr_clean(&enic->intr[index]); 830 } 831 } 832 833 static void 834 enic_init(if_ctx_t ctx) 835 { 836 struct enic_softc *softc; 837 struct enic *enic; 838 if_softc_ctx_t scctx; 839 unsigned int index; 840 841 softc = iflib_get_softc(ctx); 842 scctx = softc->scctx; 843 enic = &softc->enic; 844 845 for (index = 0; index < scctx->isc_ntxqsets; index++) 846 enic_prep_wq_for_simple_tx(&softc->enic, index); 847 848 for (index = 0; index < scctx->isc_ntxqsets; index++) 849 enic_start_wq(enic, index); 850 851 for (index = 0; index < scctx->isc_nrxqsets; index++) 852 enic_start_rq(enic, index); 853 854 /* Use the current MAC address. */ 855 bcopy(if_getlladdr(softc->ifp), softc->lladdr, ETHER_ADDR_LEN); 856 enic_set_lladdr(softc); 857 858 ENIC_LOCK(softc); 859 vnic_dev_enable_wait(enic->vdev); 860 ENIC_UNLOCK(softc); 861 862 enic_link_status(softc); 863 } 864 865 static void 866 enic_del_mcast(struct enic_softc *softc) { 867 struct enic *enic; 868 int i; 869 870 enic = &softc->enic; 871 for (i=0; i < softc->mc_count; i++) { 872 vnic_dev_del_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]); 873 } 874 softc->multicast = 0; 875 softc->mc_count = 0; 876 } 877 878 static void 879 enic_add_mcast(struct enic_softc *softc) { 880 struct enic *enic; 881 int i; 882 883 enic = &softc->enic; 884 for (i=0; i < softc->mc_count; i++) { 885 vnic_dev_add_addr(enic->vdev, &softc->mta[i * ETHER_ADDR_LEN]); 886 } 887 softc->multicast = 1; 888 } 889 890 static u_int 891 enic_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 892 { 893 uint8_t *mta = arg; 894 895 if (idx == ENIC_MAX_MULTICAST_ADDRESSES) 896 return (0); 897 898 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 899 return (1); 900 } 901 902 static void 903 enic_multi_set(if_ctx_t ctx) 904 { 905 if_t ifp; 906 struct enic_softc *softc; 907 u_int count; 908 909 softc = iflib_get_softc(ctx); 910 ifp = iflib_get_ifp(ctx); 911 912 ENIC_LOCK(softc); 913 enic_del_mcast(softc); 914 count = if_foreach_llmaddr(ifp, enic_copy_maddr, softc->mta); 915 softc->mc_count = count; 916 enic_add_mcast(softc); 917 ENIC_UNLOCK(softc); 918 919 if (if_getflags(ifp) & IFF_PROMISC) { 920 softc->promisc = 1; 921 } else { 922 softc->promisc = 0; 923 } 924 if (if_getflags(ifp) & IFF_ALLMULTI) { 925 softc->allmulti = 1; 926 } else { 927 softc->allmulti = 0; 928 } 929 enic_update_packet_filter(&softc->enic); 930 } 931 932 static int 933 enic_mtu_set(if_ctx_t ctx, uint32_t mtu) 934 { 935 struct enic_softc *softc; 936 struct enic *enic; 937 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 938 939 softc = iflib_get_softc(ctx); 940 enic = &softc->enic; 941 942 if (mtu > enic->port_mtu){ 943 return (EINVAL); 944 } 945 946 enic->config.mtu = mtu; 947 scctx->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 948 949 return (0); 950 } 951 952 static void 953 enic_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 954 { 955 struct enic_softc *softc; 956 struct ifmedia_entry *next; 957 uint32_t speed; 958 uint64_t target_baudrate; 959 960 softc = iflib_get_softc(ctx); 961 962 ifmr->ifm_status = IFM_AVALID; 963 ifmr->ifm_active = IFM_ETHER; 964 965 if (enic_link_is_up(softc) != 0) { 966 ENIC_LOCK(softc); 967 speed = vnic_dev_port_speed(&softc->vdev); 968 ENIC_UNLOCK(softc); 969 target_baudrate = 1000ull * speed; 970 LIST_FOREACH(next, &(iflib_get_media(ctx)->ifm_list), ifm_list) { 971 if (ifmedia_baudrate(next->ifm_media) == target_baudrate) { 972 ifmr->ifm_active |= next->ifm_media; 973 } 974 } 975 976 ifmr->ifm_status |= IFM_ACTIVE; 977 ifmr->ifm_active |= IFM_AUTO; 978 } else 979 ifmr->ifm_active |= IFM_NONE; 980 } 981 982 static int 983 enic_media_change(if_ctx_t ctx) 984 { 985 return (ENODEV); 986 } 987 988 static int 989 enic_promisc_set(if_ctx_t ctx, int flags) 990 { 991 if_t ifp; 992 struct enic_softc *softc; 993 994 softc = iflib_get_softc(ctx); 995 ifp = iflib_get_ifp(ctx); 996 997 if (if_getflags(ifp) & IFF_PROMISC) { 998 softc->promisc = 1; 999 } else { 1000 softc->promisc = 0; 1001 } 1002 if (if_getflags(ifp) & IFF_ALLMULTI) { 1003 softc->allmulti = 1; 1004 } else { 1005 softc->allmulti = 0; 1006 } 1007 enic_update_packet_filter(&softc->enic); 1008 1009 return (0); 1010 } 1011 1012 static uint64_t 1013 enic_get_counter(if_ctx_t ctx, ift_counter cnt) { 1014 if_t ifp = iflib_get_ifp(ctx); 1015 1016 if (cnt < IFCOUNTERS) 1017 return if_get_counter_default(ifp, cnt); 1018 1019 return (0); 1020 } 1021 1022 static void 1023 enic_update_admin_status(if_ctx_t ctx) 1024 { 1025 struct enic_softc *softc; 1026 1027 softc = iflib_get_softc(ctx); 1028 1029 enic_link_status(softc); 1030 } 1031 1032 uint32_t iflib_get_flags(if_ctx_t ctx); 1033 1034 static void 1035 enic_txq_timer(if_ctx_t ctx, uint16_t qid) 1036 { 1037 1038 struct enic_softc *softc; 1039 struct enic *enic; 1040 struct vnic_stats *stats; 1041 int ret; 1042 1043 softc = iflib_get_softc(ctx); 1044 enic = &softc->enic; 1045 1046 ENIC_LOCK(softc); 1047 ret = vnic_dev_stats_dump(enic->vdev, &stats); 1048 ENIC_UNLOCK(softc); 1049 if (ret) { 1050 dev_err(enic, "Error in getting stats\n"); 1051 } 1052 } 1053 1054 static int 1055 enic_link_is_up(struct enic_softc *softc) 1056 { 1057 return (vnic_dev_link_status(&softc->vdev) == 1); 1058 } 1059 1060 static void 1061 enic_link_status(struct enic_softc *softc) 1062 { 1063 if_ctx_t ctx; 1064 uint64_t speed; 1065 int link; 1066 1067 ctx = softc->ctx; 1068 link = enic_link_is_up(softc); 1069 speed = IF_Gbps(10); 1070 1071 ENIC_LOCK(softc); 1072 speed = vnic_dev_port_speed(&softc->vdev); 1073 ENIC_UNLOCK(softc); 1074 1075 if (link != 0 && softc->link_active == 0) { 1076 softc->link_active = 1; 1077 iflib_link_state_change(ctx, LINK_STATE_UP, speed); 1078 } else if (link == 0 && softc->link_active != 0) { 1079 softc->link_active = 0; 1080 iflib_link_state_change(ctx, LINK_STATE_DOWN, speed); 1081 } 1082 } 1083 1084 static void 1085 enic_set_lladdr(struct enic_softc *softc) 1086 { 1087 struct enic *enic; 1088 enic = &softc->enic; 1089 1090 ENIC_LOCK(softc); 1091 vnic_dev_add_addr(enic->vdev, softc->lladdr); 1092 ENIC_UNLOCK(softc); 1093 } 1094 1095 1096 static void 1097 enic_setup_txq_sysctl(struct vnic_wq *wq, int i, struct sysctl_ctx_list *ctx, 1098 struct sysctl_oid_list *child) 1099 { 1100 struct sysctl_oid *txsnode; 1101 struct sysctl_oid_list *txslist; 1102 struct vnic_stats *stats = wq[i].vdev->stats; 1103 1104 txsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats", 1105 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 1106 txslist = SYSCTL_CHILDREN(txsnode); 1107 1108 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_frames_ok", CTLFLAG_RD, 1109 &stats->tx.tx_frames_ok, "TX Frames OK"); 1110 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_frames_ok", CTLFLAG_RD, 1111 &stats->tx.tx_unicast_frames_ok, "TX unicast frames OK"); 1112 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_frames_ok", CTLFLAG_RD, 1113 &stats->tx.tx_multicast_frames_ok, "TX multicast framse OK"); 1114 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_frames_ok", CTLFLAG_RD, 1115 &stats->tx.tx_broadcast_frames_ok, "TX Broadcast frames OK"); 1116 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_bytes_ok", CTLFLAG_RD, 1117 &stats->tx.tx_bytes_ok, "TX bytes OK "); 1118 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_unicast_bytes_ok", CTLFLAG_RD, 1119 &stats->tx.tx_unicast_bytes_ok, "TX unicast bytes OK"); 1120 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_multicast_bytes_ok", CTLFLAG_RD, 1121 &stats->tx.tx_multicast_bytes_ok, "TX multicast bytes OK"); 1122 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_broadcast_bytes_ok", CTLFLAG_RD, 1123 &stats->tx.tx_broadcast_bytes_ok, "TX broadcast bytes OK"); 1124 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_drops", CTLFLAG_RD, 1125 &stats->tx.tx_drops, "TX drops"); 1126 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_errors", CTLFLAG_RD, 1127 &stats->tx.tx_errors, "TX errors"); 1128 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tx_tso", CTLFLAG_RD, 1129 &stats->tx.tx_tso, "TX TSO"); 1130 } 1131 1132 static void 1133 enic_setup_rxq_sysctl(struct vnic_rq *rq, int i, struct sysctl_ctx_list *ctx, 1134 struct sysctl_oid_list *child) 1135 { 1136 struct sysctl_oid *rxsnode; 1137 struct sysctl_oid_list *rxslist; 1138 struct vnic_stats *stats = rq[i].vdev->stats; 1139 1140 rxsnode = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "hstats", 1141 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 1142 rxslist = SYSCTL_CHILDREN(rxsnode); 1143 1144 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_ok", CTLFLAG_RD, 1145 &stats->rx.rx_frames_ok, "RX Frames OK"); 1146 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_total", CTLFLAG_RD, 1147 &stats->rx.rx_frames_total, "RX frames total"); 1148 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_frames_ok", CTLFLAG_RD, 1149 &stats->rx.rx_unicast_frames_ok, "RX unicast frames ok"); 1150 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_frames_ok", CTLFLAG_RD, 1151 &stats->rx.rx_multicast_frames_ok, "RX multicast Frames ok"); 1152 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_frames_ok", CTLFLAG_RD, 1153 &stats->rx.rx_broadcast_frames_ok, "RX broadcast frames ok"); 1154 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_bytes_ok", CTLFLAG_RD, 1155 &stats->rx.rx_bytes_ok, "RX bytes ok"); 1156 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_unicast_bytes_ok", CTLFLAG_RD, 1157 &stats->rx.rx_unicast_bytes_ok, "RX unicast bytes ok"); 1158 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_multicast_bytes_ok", CTLFLAG_RD, 1159 &stats->rx.rx_multicast_bytes_ok, "RX multicast bytes ok"); 1160 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_broadcast_bytes_ok", CTLFLAG_RD, 1161 &stats->rx.rx_broadcast_bytes_ok, "RX broadcast bytes ok"); 1162 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_drop", CTLFLAG_RD, 1163 &stats->rx.rx_drop, "RX drop"); 1164 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_errors", CTLFLAG_RD, 1165 &stats->rx.rx_errors, "RX errors"); 1166 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_rss", CTLFLAG_RD, 1167 &stats->rx.rx_rss, "RX rss"); 1168 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_crc_errors", CTLFLAG_RD, 1169 &stats->rx.rx_crc_errors, "RX crc errors"); 1170 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_64", CTLFLAG_RD, 1171 &stats->rx.rx_frames_64, "RX frames 64"); 1172 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_127", CTLFLAG_RD, 1173 &stats->rx.rx_frames_127, "RX frames 127"); 1174 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_255", CTLFLAG_RD, 1175 &stats->rx.rx_frames_255, "RX frames 255"); 1176 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_511", CTLFLAG_RD, 1177 &stats->rx.rx_frames_511, "RX frames 511"); 1178 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1023", CTLFLAG_RD, 1179 &stats->rx.rx_frames_1023, "RX frames 1023"); 1180 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_1518", CTLFLAG_RD, 1181 &stats->rx.rx_frames_1518, "RX frames 1518"); 1182 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "rx_frames_to_max", CTLFLAG_RD, 1183 &stats->rx.rx_frames_to_max, "RX frames to max"); 1184 } 1185 1186 static void 1187 enic_setup_queue_sysctl(struct enic_softc *softc, struct sysctl_ctx_list *ctx, 1188 struct sysctl_oid_list *child) 1189 { 1190 enic_setup_txq_sysctl(softc->enic.wq, 0, ctx, child); 1191 enic_setup_rxq_sysctl(softc->enic.rq, 0, ctx, child); 1192 } 1193 1194 static void 1195 enic_setup_sysctl(struct enic_softc *softc) 1196 { 1197 device_t dev; 1198 struct sysctl_ctx_list *ctx; 1199 struct sysctl_oid *tree; 1200 struct sysctl_oid_list *child; 1201 1202 dev = softc->dev; 1203 ctx = device_get_sysctl_ctx(dev); 1204 tree = device_get_sysctl_tree(dev); 1205 child = SYSCTL_CHILDREN(tree); 1206 1207 enic_setup_queue_sysctl(softc, ctx, child); 1208 } 1209 1210 static void 1211 enic_enable_intr(struct enic_softc *softc, int irq) 1212 { 1213 struct enic *enic = &softc->enic; 1214 1215 vnic_intr_unmask(&enic->intr[irq]); 1216 vnic_intr_return_all_credits(&enic->intr[irq]); 1217 } 1218 1219 static void 1220 enic_disable_intr(struct enic_softc *softc, int irq) 1221 { 1222 struct enic *enic = &softc->enic; 1223 1224 vnic_intr_mask(&enic->intr[irq]); 1225 vnic_intr_masked(&enic->intr[irq]); /* flush write */ 1226 } 1227 1228 static int 1229 enic_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 1230 { 1231 struct enic_softc *softc; 1232 if_softc_ctx_t scctx; 1233 1234 softc = iflib_get_softc(ctx); 1235 scctx = softc->scctx; 1236 1237 enic_enable_intr(softc, qid + scctx->isc_nrxqsets); 1238 1239 return 0; 1240 } 1241 1242 static int 1243 enic_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 1244 { 1245 struct enic_softc *softc; 1246 1247 softc = iflib_get_softc(ctx); 1248 enic_enable_intr(softc, qid); 1249 1250 return 0; 1251 } 1252 1253 static void 1254 enic_intr_enable_all(if_ctx_t ctx) 1255 { 1256 struct enic_softc *softc; 1257 if_softc_ctx_t scctx; 1258 int i; 1259 1260 softc = iflib_get_softc(ctx); 1261 scctx = softc->scctx; 1262 1263 for (i = 0; i < scctx->isc_vectors; i++) { 1264 enic_enable_intr(softc, i); 1265 } 1266 } 1267 1268 static void 1269 enic_intr_disable_all(if_ctx_t ctx) 1270 { 1271 struct enic_softc *softc; 1272 if_softc_ctx_t scctx; 1273 int i; 1274 1275 softc = iflib_get_softc(ctx); 1276 scctx = softc->scctx; 1277 /* 1278 * iflib may invoke this routine before enic_attach_post() has run, 1279 * which is before the top level shared data area is initialized and 1280 * the device made aware of it. 1281 */ 1282 1283 for (i = 0; i < scctx->isc_vectors; i++) { 1284 enic_disable_intr(softc, i); 1285 } 1286 } 1287 1288 static int 1289 enic_dev_open(struct enic *enic) 1290 { 1291 int err; 1292 int flags = CMD_OPENF_IG_DESCCACHE; 1293 1294 err = enic_dev_wait(enic->vdev, vnic_dev_open, 1295 vnic_dev_open_done, flags); 1296 if (err) 1297 dev_err(enic_get_dev(enic), 1298 "vNIC device open failed, err %d\n", err); 1299 1300 return err; 1301 } 1302 1303 static int 1304 enic_dev_init(struct enic *enic) 1305 { 1306 int err; 1307 1308 vnic_dev_intr_coal_timer_info_default(enic->vdev); 1309 1310 /* 1311 * Get vNIC configuration 1312 */ 1313 err = enic_get_vnic_config(enic); 1314 if (err) { 1315 dev_err(dev, "Get vNIC configuration failed, aborting\n"); 1316 return err; 1317 } 1318 1319 /* Get available resource counts */ 1320 enic_get_res_counts(enic); 1321 1322 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */ 1323 enic->intr_queues = malloc(sizeof(*enic->intr_queues) * 1324 enic->conf_intr_count, M_DEVBUF, M_NOWAIT | M_ZERO); 1325 1326 vnic_dev_set_reset_flag(enic->vdev, 0); 1327 enic->max_flow_counter = -1; 1328 1329 /* set up link status checking */ 1330 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */ 1331 1332 enic->overlay_offload = false; 1333 if (enic->disable_overlay && enic->vxlan) { 1334 /* 1335 * Explicitly disable overlay offload as the setting is 1336 * sticky, and resetting vNIC does not disable it. 1337 */ 1338 if (vnic_dev_overlay_offload_ctrl(enic->vdev, 1339 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_DISABLE)) { 1340 dev_err(enic, "failed to disable overlay offload\n"); 1341 } else { 1342 dev_info(enic, "Overlay offload is disabled\n"); 1343 } 1344 } 1345 if (!enic->disable_overlay && enic->vxlan && 1346 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */ 1347 vnic_dev_overlay_offload_ctrl(enic->vdev, 1348 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_ENABLE) == 0) { 1349 enic->overlay_offload = true; 1350 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT; 1351 dev_info(enic, "Overlay offload is enabled\n"); 1352 /* 1353 * Reset the vxlan port to the default, as the NIC firmware 1354 * does not reset it automatically and keeps the old setting. 1355 */ 1356 if (vnic_dev_overlay_offload_cfg(enic->vdev, 1357 OVERLAY_CFG_VXLAN_PORT_UPDATE, ENIC_DEFAULT_VXLAN_PORT)) { 1358 dev_err(enic, "failed to update vxlan port\n"); 1359 return -EINVAL; 1360 } 1361 } 1362 return 0; 1363 } 1364 1365 static void * 1366 enic_alloc_consistent(void *priv, size_t size, bus_addr_t * dma_handle, 1367 struct iflib_dma_info *res, u8 * name) 1368 { 1369 void *vaddr; 1370 *dma_handle = 0; 1371 struct enic *enic = (struct enic *)priv; 1372 int rz; 1373 1374 rz = iflib_dma_alloc(enic->softc->ctx, size, res, BUS_DMA_NOWAIT); 1375 if (rz) { 1376 pr_err("%s : Failed to allocate memory requested for %s\n", 1377 __func__, name); 1378 return NULL; 1379 } 1380 1381 vaddr = res->idi_vaddr; 1382 *dma_handle = res->idi_paddr; 1383 1384 return vaddr; 1385 } 1386 1387 static void 1388 enic_free_consistent(void *priv, size_t size, void *vaddr, 1389 bus_addr_t dma_handle, struct iflib_dma_info *res) 1390 { 1391 iflib_dma_free(res); 1392 } 1393 1394 static int 1395 enic_pci_mapping(struct enic_softc *softc) 1396 { 1397 int rc; 1398 1399 rc = enic_map_bar(softc, &softc->mem, 0, true); 1400 if (rc) 1401 return rc; 1402 1403 rc = enic_map_bar(softc, &softc->io, 2, false); 1404 1405 return rc; 1406 } 1407 1408 static void 1409 enic_pci_mapping_free(struct enic_softc *softc) 1410 { 1411 if (softc->mem.res != NULL) 1412 bus_release_resource(softc->dev, SYS_RES_MEMORY, 1413 softc->mem.rid, softc->mem.res); 1414 softc->mem.res = NULL; 1415 1416 if (softc->io.res != NULL) 1417 bus_release_resource(softc->dev, SYS_RES_MEMORY, 1418 softc->io.rid, softc->io.res); 1419 softc->io.res = NULL; 1420 } 1421 1422 static int 1423 enic_dev_wait(struct vnic_dev *vdev, int (*start) (struct vnic_dev *, int), 1424 int (*finished) (struct vnic_dev *, int *), int arg) 1425 { 1426 int done; 1427 int err; 1428 int i; 1429 1430 err = start(vdev, arg); 1431 if (err) 1432 return err; 1433 1434 /* Wait for func to complete...2 seconds max */ 1435 for (i = 0; i < 2000; i++) { 1436 err = finished(vdev, &done); 1437 if (err) 1438 return err; 1439 if (done) 1440 return 0; 1441 usleep(1000); 1442 } 1443 return -ETIMEDOUT; 1444 } 1445 1446 static int 1447 enic_map_bar(struct enic_softc *softc, struct enic_bar_info *bar, int bar_num, 1448 bool shareable) 1449 { 1450 uint32_t flag; 1451 1452 if (bar->res != NULL) { 1453 device_printf(softc->dev, "Bar %d already mapped\n", bar_num); 1454 return EDOOFUS; 1455 } 1456 1457 bar->rid = PCIR_BAR(bar_num); 1458 flag = RF_ACTIVE; 1459 if (shareable) 1460 flag |= RF_SHAREABLE; 1461 1462 if ((bar->res = bus_alloc_resource_any(softc->dev, 1463 SYS_RES_MEMORY, &bar->rid, flag)) == NULL) { 1464 device_printf(softc->dev, 1465 "PCI BAR%d mapping failure\n", bar_num); 1466 return (ENXIO); 1467 } 1468 bar->tag = rman_get_bustag(bar->res); 1469 bar->handle = rman_get_bushandle(bar->res); 1470 bar->size = rman_get_size(bar->res); 1471 1472 return 0; 1473 } 1474 1475 void 1476 enic_init_vnic_resources(struct enic *enic) 1477 { 1478 unsigned int error_interrupt_enable = 1; 1479 unsigned int error_interrupt_offset = 0; 1480 unsigned int rxq_interrupt_enable = 0; 1481 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET; 1482 unsigned int txq_interrupt_enable = 0; 1483 unsigned int txq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET; 1484 unsigned int index = 0; 1485 unsigned int cq_idx; 1486 if_softc_ctx_t scctx; 1487 1488 scctx = enic->softc->scctx; 1489 1490 1491 rxq_interrupt_enable = 1; 1492 txq_interrupt_enable = 1; 1493 1494 rxq_interrupt_offset = 0; 1495 txq_interrupt_offset = enic->intr_count - 2; 1496 txq_interrupt_offset = 1; 1497 1498 for (index = 0; index < enic->intr_count; index++) { 1499 vnic_intr_alloc(enic->vdev, &enic->intr[index], index); 1500 } 1501 1502 for (index = 0; index < scctx->isc_nrxqsets; index++) { 1503 cq_idx = enic_cq_rq(enic, index); 1504 1505 vnic_rq_clean(&enic->rq[index]); 1506 vnic_rq_init(&enic->rq[index], cq_idx, error_interrupt_enable, 1507 error_interrupt_offset); 1508 1509 vnic_cq_clean(&enic->cq[cq_idx]); 1510 vnic_cq_init(&enic->cq[cq_idx], 1511 0 /* flow_control_enable */ , 1512 1 /* color_enable */ , 1513 0 /* cq_head */ , 1514 0 /* cq_tail */ , 1515 1 /* cq_tail_color */ , 1516 rxq_interrupt_enable, 1517 1 /* cq_entry_enable */ , 1518 0 /* cq_message_enable */ , 1519 rxq_interrupt_offset, 1520 0 /* cq_message_addr */ ); 1521 if (rxq_interrupt_enable) 1522 rxq_interrupt_offset++; 1523 } 1524 1525 for (index = 0; index < scctx->isc_ntxqsets; index++) { 1526 cq_idx = enic_cq_wq(enic, index); 1527 vnic_wq_clean(&enic->wq[index]); 1528 vnic_wq_init(&enic->wq[index], cq_idx, error_interrupt_enable, 1529 error_interrupt_offset); 1530 /* Compute unsupported ol flags for enic_prep_pkts() */ 1531 enic->wq[index].tx_offload_notsup_mask = 0; 1532 1533 vnic_cq_clean(&enic->cq[cq_idx]); 1534 vnic_cq_init(&enic->cq[cq_idx], 1535 0 /* flow_control_enable */ , 1536 1 /* color_enable */ , 1537 0 /* cq_head */ , 1538 0 /* cq_tail */ , 1539 1 /* cq_tail_color */ , 1540 txq_interrupt_enable, 1541 1, 1542 0, 1543 txq_interrupt_offset, 1544 0 /* (u64)enic->wq[index].cqmsg_rz->iova */ ); 1545 1546 } 1547 1548 for (index = 0; index < enic->intr_count; index++) { 1549 vnic_intr_init(&enic->intr[index], 125, 1550 enic->config.intr_timer_type, /* mask_on_assertion */ 1); 1551 } 1552 } 1553 1554 static void 1555 enic_update_packet_filter(struct enic *enic) 1556 { 1557 struct enic_softc *softc = enic->softc; 1558 1559 ENIC_LOCK(softc); 1560 vnic_dev_packet_filter(enic->vdev, 1561 softc->directed, 1562 softc->multicast, 1563 softc->broadcast, 1564 softc->promisc, 1565 softc->allmulti); 1566 ENIC_UNLOCK(softc); 1567 } 1568 1569 int 1570 enic_setup_finish(struct enic *enic) 1571 { 1572 struct enic_softc *softc = enic->softc; 1573 1574 /* Default conf */ 1575 softc->directed = 1; 1576 softc->multicast = 0; 1577 softc->broadcast = 1; 1578 softc->promisc = 0; 1579 softc->allmulti = 1;; 1580 enic_update_packet_filter(enic); 1581 1582 return 0; 1583 } 1584