1 /*- 2 * Copyright (c) 2013 Tsubai Masanari 3 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 * 17 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18 */ 19 20 /* Driver for VMware vmxnet3 virtual ethernet devices. */ 21 22 #include <sys/cdefs.h> 23 __FBSDID("$FreeBSD$"); 24 25 #include <sys/param.h> 26 #include <sys/systm.h> 27 #include <sys/kernel.h> 28 #include <sys/endian.h> 29 #include <sys/sockio.h> 30 #include <sys/mbuf.h> 31 #include <sys/malloc.h> 32 #include <sys/module.h> 33 #include <sys/socket.h> 34 #include <sys/sysctl.h> 35 #include <vm/vm.h> 36 #include <vm/pmap.h> 37 38 #include <net/ethernet.h> 39 #include <net/if.h> 40 #include <net/if_arp.h> 41 #include <net/if_dl.h> 42 #include <net/if_types.h> 43 #include <net/if_media.h> 44 #include <net/if_vlan_var.h> 45 46 #include <net/bpf.h> 47 48 #include <netinet/in_systm.h> 49 #include <netinet/in.h> 50 #include <netinet/ip.h> 51 #include <netinet/ip6.h> 52 #include <netinet6/ip6_var.h> 53 #include <netinet/udp.h> 54 #include <netinet/tcp.h> 55 56 #include <machine/bus.h> 57 #include <machine/resource.h> 58 #include <sys/bus.h> 59 #include <sys/rman.h> 60 61 #include <dev/pci/pcireg.h> 62 #include <dev/pci/pcivar.h> 63 64 #include "if_vmxreg.h" 65 #include "if_vmxvar.h" 66 67 #include "opt_inet.h" 68 #include "opt_inet6.h" 69 70 /* Always enable for now - useful for queue hangs. */ 71 #define VMXNET3_DEBUG_SYSCTL 72 73 #ifdef VMXNET3_FAILPOINTS 74 #include <sys/fail.h> 75 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 76 "vmxnet3 fail points"); 77 #define VMXNET3_FP _debug_fail_point_vmxnet3 78 #endif 79 80 static int vmxnet3_probe(device_t); 81 static int vmxnet3_attach(device_t); 82 static int vmxnet3_detach(device_t); 83 static int vmxnet3_shutdown(device_t); 84 85 static int vmxnet3_alloc_resources(struct vmxnet3_softc *); 86 static void vmxnet3_free_resources(struct vmxnet3_softc *); 87 static int vmxnet3_check_version(struct vmxnet3_softc *); 88 static void vmxnet3_initial_config(struct vmxnet3_softc *); 89 90 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 91 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 92 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 93 static int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 94 struct vmxnet3_interrupt *); 95 static int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 96 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 97 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 98 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 99 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 100 101 static void vmxnet3_free_interrupt(struct vmxnet3_softc *, 102 struct vmxnet3_interrupt *); 103 static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 104 105 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 106 static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 107 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 108 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 109 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 110 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 111 112 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 113 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 114 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 115 static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 116 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 117 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 118 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 119 static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 120 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 121 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 122 static void vmxnet3_reinit_interface(struct vmxnet3_softc *); 123 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 124 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 125 static void vmxnet3_free_data(struct vmxnet3_softc *); 126 static int vmxnet3_setup_interface(struct vmxnet3_softc *); 127 128 static void vmxnet3_evintr(struct vmxnet3_softc *); 129 static void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 130 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 131 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 132 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 133 struct vmxnet3_rxring *, int); 134 static void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 135 static void vmxnet3_legacy_intr(void *); 136 static void vmxnet3_txq_intr(void *); 137 static void vmxnet3_rxq_intr(void *); 138 static void vmxnet3_event_intr(void *); 139 140 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 141 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 142 static void vmxnet3_stop(struct vmxnet3_softc *); 143 144 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 145 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 146 static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 147 static int vmxnet3_enable_device(struct vmxnet3_softc *); 148 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 149 static int vmxnet3_reinit(struct vmxnet3_softc *); 150 static void vmxnet3_init_locked(struct vmxnet3_softc *); 151 static void vmxnet3_init(void *); 152 153 static int vmxnet3_txq_offload_ctx(struct mbuf *, int *, int *, int *); 154 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 155 bus_dmamap_t, bus_dma_segment_t [], int *); 156 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 157 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 158 static void vmxnet3_start_locked(struct ifnet *); 159 static void vmxnet3_start(struct ifnet *); 160 161 static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 162 uint16_t); 163 static void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 164 static void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 165 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 166 static int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 167 static int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 168 169 static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 170 static void vmxnet3_tick(void *); 171 static void vmxnet3_link_status(struct vmxnet3_softc *); 172 static void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 173 static int vmxnet3_media_change(struct ifnet *); 174 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 175 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 176 177 static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 178 struct sysctl_ctx_list *, struct sysctl_oid_list *); 179 static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 180 struct sysctl_ctx_list *, struct sysctl_oid_list *); 181 static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 182 struct sysctl_ctx_list *, struct sysctl_oid_list *); 183 static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 184 185 static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 186 uint32_t); 187 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 188 static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 189 uint32_t); 190 static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 191 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 192 193 static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 194 static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 195 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 196 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 197 198 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 199 bus_size_t, struct vmxnet3_dma_alloc *); 200 static void vmxnet3_dma_free(struct vmxnet3_softc *, 201 struct vmxnet3_dma_alloc *); 202 static int vmxnet3_tunable_int(struct vmxnet3_softc *, 203 const char *, int); 204 205 typedef enum { 206 VMXNET3_BARRIER_RD, 207 VMXNET3_BARRIER_WR, 208 VMXNET3_BARRIER_RDWR, 209 } vmxnet3_barrier_t; 210 211 static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 212 213 /* Tunables. */ 214 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 215 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 216 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 217 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 218 219 static device_method_t vmxnet3_methods[] = { 220 /* Device interface. */ 221 DEVMETHOD(device_probe, vmxnet3_probe), 222 DEVMETHOD(device_attach, vmxnet3_attach), 223 DEVMETHOD(device_detach, vmxnet3_detach), 224 DEVMETHOD(device_shutdown, vmxnet3_shutdown), 225 226 DEVMETHOD_END 227 }; 228 229 static driver_t vmxnet3_driver = { 230 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 231 }; 232 233 static devclass_t vmxnet3_devclass; 234 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 235 236 MODULE_DEPEND(vmx, pci, 1, 1, 1); 237 MODULE_DEPEND(vmx, ether, 1, 1, 1); 238 239 #define VMXNET3_VMWARE_VENDOR_ID 0x15AD 240 #define VMXNET3_VMWARE_DEVICE_ID 0x07B0 241 242 static int 243 vmxnet3_probe(device_t dev) 244 { 245 246 if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 247 pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 248 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 249 return (BUS_PROBE_DEFAULT); 250 } 251 252 return (ENXIO); 253 } 254 255 static int 256 vmxnet3_attach(device_t dev) 257 { 258 struct vmxnet3_softc *sc; 259 int error; 260 261 sc = device_get_softc(dev); 262 sc->vmx_dev = dev; 263 264 pci_enable_busmaster(dev); 265 266 VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 267 callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); 268 269 vmxnet3_initial_config(sc); 270 271 error = vmxnet3_alloc_resources(sc); 272 if (error) 273 goto fail; 274 275 error = vmxnet3_check_version(sc); 276 if (error) 277 goto fail; 278 279 error = vmxnet3_alloc_rxtx_queues(sc); 280 if (error) 281 goto fail; 282 283 error = vmxnet3_alloc_interrupts(sc); 284 if (error) 285 goto fail; 286 287 error = vmxnet3_alloc_data(sc); 288 if (error) 289 goto fail; 290 291 error = vmxnet3_setup_interface(sc); 292 if (error) 293 goto fail; 294 295 error = vmxnet3_setup_interrupts(sc); 296 if (error) { 297 ether_ifdetach(sc->vmx_ifp); 298 device_printf(dev, "could not set up interrupt\n"); 299 goto fail; 300 } 301 302 vmxnet3_setup_sysctl(sc); 303 vmxnet3_link_status(sc); 304 305 fail: 306 if (error) 307 vmxnet3_detach(dev); 308 309 return (error); 310 } 311 312 static int 313 vmxnet3_detach(device_t dev) 314 { 315 struct vmxnet3_softc *sc; 316 struct ifnet *ifp; 317 318 sc = device_get_softc(dev); 319 ifp = sc->vmx_ifp; 320 321 if (device_is_attached(dev)) { 322 ether_ifdetach(ifp); 323 VMXNET3_CORE_LOCK(sc); 324 vmxnet3_stop(sc); 325 VMXNET3_CORE_UNLOCK(sc); 326 callout_drain(&sc->vmx_tick); 327 } 328 329 if (sc->vmx_vlan_attach != NULL) { 330 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 331 sc->vmx_vlan_attach = NULL; 332 } 333 if (sc->vmx_vlan_detach != NULL) { 334 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 335 sc->vmx_vlan_detach = NULL; 336 } 337 338 vmxnet3_free_interrupts(sc); 339 340 if (ifp != NULL) { 341 if_free(ifp); 342 sc->vmx_ifp = NULL; 343 } 344 345 ifmedia_removeall(&sc->vmx_media); 346 347 vmxnet3_free_data(sc); 348 vmxnet3_free_resources(sc); 349 vmxnet3_free_rxtx_queues(sc); 350 351 VMXNET3_CORE_LOCK_DESTROY(sc); 352 353 return (0); 354 } 355 356 static int 357 vmxnet3_shutdown(device_t dev) 358 { 359 360 return (0); 361 } 362 363 static int 364 vmxnet3_alloc_resources(struct vmxnet3_softc *sc) 365 { 366 device_t dev; 367 int rid; 368 369 dev = sc->vmx_dev; 370 371 rid = PCIR_BAR(0); 372 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 373 RF_ACTIVE); 374 if (sc->vmx_res0 == NULL) { 375 device_printf(dev, 376 "could not map BAR0 memory\n"); 377 return (ENXIO); 378 } 379 380 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 381 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 382 383 rid = PCIR_BAR(1); 384 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 385 RF_ACTIVE); 386 if (sc->vmx_res1 == NULL) { 387 device_printf(dev, 388 "could not map BAR1 memory\n"); 389 return (ENXIO); 390 } 391 392 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 393 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 394 395 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 396 rid = PCIR_BAR(2); 397 sc->vmx_msix_res = bus_alloc_resource_any(dev, 398 SYS_RES_MEMORY, &rid, RF_ACTIVE); 399 } 400 401 if (sc->vmx_msix_res == NULL) 402 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 403 404 return (0); 405 } 406 407 static void 408 vmxnet3_free_resources(struct vmxnet3_softc *sc) 409 { 410 device_t dev; 411 int rid; 412 413 dev = sc->vmx_dev; 414 415 if (sc->vmx_res0 != NULL) { 416 rid = PCIR_BAR(0); 417 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 418 sc->vmx_res0 = NULL; 419 } 420 421 if (sc->vmx_res1 != NULL) { 422 rid = PCIR_BAR(1); 423 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 424 sc->vmx_res1 = NULL; 425 } 426 427 if (sc->vmx_msix_res != NULL) { 428 rid = PCIR_BAR(2); 429 bus_release_resource(dev, SYS_RES_MEMORY, rid, 430 sc->vmx_msix_res); 431 sc->vmx_msix_res = NULL; 432 } 433 } 434 435 static int 436 vmxnet3_check_version(struct vmxnet3_softc *sc) 437 { 438 device_t dev; 439 uint32_t version; 440 441 dev = sc->vmx_dev; 442 443 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 444 if ((version & 0x01) == 0) { 445 device_printf(dev, "unsupported hardware version %#x\n", 446 version); 447 return (ENOTSUP); 448 } 449 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 450 451 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 452 if ((version & 0x01) == 0) { 453 device_printf(dev, "unsupported UPT version %#x\n", version); 454 return (ENOTSUP); 455 } 456 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 457 458 return (0); 459 } 460 461 static void 462 vmxnet3_initial_config(struct vmxnet3_softc *sc) 463 { 464 int ndesc; 465 466 /* 467 * BMV Much of the work is already done, but this driver does 468 * not support multiqueue yet. 469 */ 470 sc->vmx_ntxqueues = VMXNET3_TX_QUEUES; 471 sc->vmx_nrxqueues = VMXNET3_RX_QUEUES; 472 473 ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 474 if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 475 ndesc = VMXNET3_DEF_TX_NDESC; 476 if (ndesc & VMXNET3_MASK_TX_NDESC) 477 ndesc &= ~VMXNET3_MASK_TX_NDESC; 478 sc->vmx_ntxdescs = ndesc; 479 480 ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 481 if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 482 ndesc = VMXNET3_DEF_RX_NDESC; 483 if (ndesc & VMXNET3_MASK_RX_NDESC) 484 ndesc &= ~VMXNET3_MASK_RX_NDESC; 485 sc->vmx_nrxdescs = ndesc; 486 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 487 } 488 489 static int 490 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 491 { 492 device_t dev; 493 int nmsix, cnt, required; 494 495 dev = sc->vmx_dev; 496 497 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 498 return (1); 499 500 /* Allocate an additional vector for the events interrupt. */ 501 required = sc->vmx_nrxqueues + sc->vmx_ntxqueues + 1; 502 503 nmsix = pci_msix_count(dev); 504 if (nmsix < required) 505 return (1); 506 507 cnt = required; 508 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 509 sc->vmx_nintrs = required; 510 return (0); 511 } else 512 pci_release_msi(dev); 513 514 return (1); 515 } 516 517 static int 518 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 519 { 520 device_t dev; 521 int nmsi, cnt, required; 522 523 dev = sc->vmx_dev; 524 required = 1; 525 526 nmsi = pci_msi_count(dev); 527 if (nmsi < required) 528 return (1); 529 530 cnt = required; 531 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 532 sc->vmx_nintrs = 1; 533 return (0); 534 } else 535 pci_release_msi(dev); 536 537 return (1); 538 } 539 540 static int 541 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 542 { 543 544 sc->vmx_nintrs = 1; 545 return (0); 546 } 547 548 static int 549 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 550 struct vmxnet3_interrupt *intr) 551 { 552 struct resource *irq; 553 554 irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); 555 if (irq == NULL) 556 return (ENXIO); 557 558 intr->vmxi_irq = irq; 559 intr->vmxi_rid = rid; 560 561 return (0); 562 } 563 564 static int 565 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 566 { 567 int i, rid, flags, error; 568 569 rid = 0; 570 flags = RF_ACTIVE; 571 572 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 573 flags |= RF_SHAREABLE; 574 else 575 rid = 1; 576 577 for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 578 error = vmxnet3_alloc_interrupt(sc, rid, flags, 579 &sc->vmx_intrs[i]); 580 if (error) 581 return (error); 582 } 583 584 return (0); 585 } 586 587 /* 588 * NOTE: We only support the simple case of each Rx and Tx queue on its 589 * own MSIX vector. This is good enough until we support mulitqueue. 590 */ 591 static int 592 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 593 { 594 device_t dev; 595 struct vmxnet3_txqueue *txq; 596 struct vmxnet3_rxqueue *rxq; 597 struct vmxnet3_interrupt *intr; 598 enum intr_type type; 599 int i, error; 600 601 dev = sc->vmx_dev; 602 intr = &sc->vmx_intrs[0]; 603 type = INTR_TYPE_NET | INTR_MPSAFE; 604 605 for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 606 txq = &sc->vmx_txq[i]; 607 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 608 vmxnet3_txq_intr, txq, &intr->vmxi_handler); 609 if (error) 610 return (error); 611 txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 612 } 613 614 for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 615 rxq = &sc->vmx_rxq[i]; 616 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 617 vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); 618 if (error) 619 return (error); 620 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 621 } 622 623 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 624 vmxnet3_event_intr, sc, &intr->vmxi_handler); 625 if (error) 626 return (error); 627 sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 628 629 return (0); 630 } 631 632 static int 633 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 634 { 635 struct vmxnet3_interrupt *intr; 636 int i, error; 637 638 intr = &sc->vmx_intrs[0]; 639 error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 640 INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, 641 &intr->vmxi_handler); 642 643 for (i = 0; i < sc->vmx_ntxqueues; i++) 644 sc->vmx_txq[i].vxtxq_intr_idx = 0; 645 for (i = 0; i < sc->vmx_nrxqueues; i++) 646 sc->vmx_rxq[i].vxrxq_intr_idx = 0; 647 sc->vmx_event_intr_idx = 0; 648 649 return (error); 650 } 651 652 /* 653 * XXX BMV Should probably reorganize the attach and just do 654 * this in vmxnet3_init_shared_data(). 655 */ 656 static void 657 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 658 { 659 struct vmxnet3_txqueue *txq; 660 struct vmxnet3_txq_shared *txs; 661 struct vmxnet3_rxqueue *rxq; 662 struct vmxnet3_rxq_shared *rxs; 663 int i; 664 665 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 666 667 for (i = 0; i < sc->vmx_ntxqueues; i++) { 668 txq = &sc->vmx_txq[i]; 669 txs = txq->vxtxq_ts; 670 txs->intr_idx = txq->vxtxq_intr_idx; 671 } 672 673 for (i = 0; i < sc->vmx_nrxqueues; i++) { 674 rxq = &sc->vmx_rxq[i]; 675 rxs = rxq->vxrxq_rs; 676 rxs->intr_idx = rxq->vxrxq_intr_idx; 677 } 678 } 679 680 static int 681 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 682 { 683 int error; 684 685 error = vmxnet3_alloc_intr_resources(sc); 686 if (error) 687 return (error); 688 689 switch (sc->vmx_intr_type) { 690 case VMXNET3_IT_MSIX: 691 error = vmxnet3_setup_msix_interrupts(sc); 692 break; 693 case VMXNET3_IT_MSI: 694 case VMXNET3_IT_LEGACY: 695 error = vmxnet3_setup_legacy_interrupt(sc); 696 break; 697 default: 698 panic("%s: invalid interrupt type %d", __func__, 699 sc->vmx_intr_type); 700 } 701 702 if (error == 0) 703 vmxnet3_set_interrupt_idx(sc); 704 705 return (error); 706 } 707 708 static int 709 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 710 { 711 device_t dev; 712 uint32_t config; 713 int error; 714 715 dev = sc->vmx_dev; 716 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 717 718 sc->vmx_intr_type = config & 0x03; 719 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 720 721 switch (sc->vmx_intr_type) { 722 case VMXNET3_IT_AUTO: 723 sc->vmx_intr_type = VMXNET3_IT_MSIX; 724 /* FALLTHROUGH */ 725 case VMXNET3_IT_MSIX: 726 error = vmxnet3_alloc_msix_interrupts(sc); 727 if (error == 0) 728 break; 729 sc->vmx_intr_type = VMXNET3_IT_MSI; 730 /* FALLTHROUGH */ 731 case VMXNET3_IT_MSI: 732 error = vmxnet3_alloc_msi_interrupts(sc); 733 if (error == 0) 734 break; 735 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 736 /* FALLTHROUGH */ 737 case VMXNET3_IT_LEGACY: 738 error = vmxnet3_alloc_legacy_interrupts(sc); 739 if (error == 0) 740 break; 741 /* FALLTHROUGH */ 742 default: 743 sc->vmx_intr_type = -1; 744 device_printf(dev, "cannot allocate any interrupt resources\n"); 745 return (ENXIO); 746 } 747 748 return (error); 749 } 750 751 static void 752 vmxnet3_free_interrupt(struct vmxnet3_softc *sc, 753 struct vmxnet3_interrupt *intr) 754 { 755 device_t dev; 756 757 dev = sc->vmx_dev; 758 759 if (intr->vmxi_handler != NULL) { 760 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 761 intr->vmxi_handler = NULL; 762 } 763 764 if (intr->vmxi_irq != NULL) { 765 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 766 intr->vmxi_irq); 767 intr->vmxi_irq = NULL; 768 intr->vmxi_rid = -1; 769 } 770 } 771 772 static void 773 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 774 { 775 int i; 776 777 for (i = 0; i < sc->vmx_nintrs; i++) 778 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 779 780 if (sc->vmx_intr_type == VMXNET3_IT_MSI || 781 sc->vmx_intr_type == VMXNET3_IT_MSIX) 782 pci_release_msi(sc->vmx_dev); 783 } 784 785 static int 786 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 787 { 788 struct vmxnet3_rxqueue *rxq; 789 struct vmxnet3_rxring *rxr; 790 int i; 791 792 rxq = &sc->vmx_rxq[q]; 793 794 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 795 device_get_nameunit(sc->vmx_dev), q); 796 mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); 797 798 rxq->vxrxq_sc = sc; 799 rxq->vxrxq_id = q; 800 801 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 802 rxr = &rxq->vxrxq_cmd_ring[i]; 803 rxr->vxrxr_rid = i; 804 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 805 rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * 806 sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 807 if (rxr->vxrxr_rxbuf == NULL) 808 return (ENOMEM); 809 810 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 811 } 812 813 return (0); 814 } 815 816 static int 817 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 818 { 819 struct vmxnet3_txqueue *txq; 820 struct vmxnet3_txring *txr; 821 822 txq = &sc->vmx_txq[q]; 823 txr = &txq->vxtxq_cmd_ring; 824 825 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 826 device_get_nameunit(sc->vmx_dev), q); 827 mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); 828 829 txq->vxtxq_sc = sc; 830 txq->vxtxq_id = q; 831 832 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 833 txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * 834 sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 835 if (txr->vxtxr_txbuf == NULL) 836 return (ENOMEM); 837 838 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 839 840 return (0); 841 } 842 843 static int 844 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 845 { 846 int i, error; 847 848 sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 849 sc->vmx_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 850 sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 851 sc->vmx_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 852 if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 853 return (ENOMEM); 854 855 for (i = 0; i < sc->vmx_nrxqueues; i++) { 856 error = vmxnet3_init_rxq(sc, i); 857 if (error) 858 return (error); 859 } 860 861 for (i = 0; i < sc->vmx_ntxqueues; i++) { 862 error = vmxnet3_init_txq(sc, i); 863 if (error) 864 return (error); 865 } 866 867 return (0); 868 } 869 870 static void 871 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 872 { 873 struct vmxnet3_rxring *rxr; 874 int i; 875 876 rxq->vxrxq_sc = NULL; 877 rxq->vxrxq_id = -1; 878 879 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 880 rxr = &rxq->vxrxq_cmd_ring[i]; 881 882 if (rxr->vxrxr_rxbuf != NULL) { 883 free(rxr->vxrxr_rxbuf, M_DEVBUF); 884 rxr->vxrxr_rxbuf = NULL; 885 } 886 } 887 888 if (mtx_initialized(&rxq->vxrxq_mtx) != 0) 889 mtx_destroy(&rxq->vxrxq_mtx); 890 } 891 892 static void 893 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 894 { 895 struct vmxnet3_txring *txr; 896 897 txr = &txq->vxtxq_cmd_ring; 898 899 txq->vxtxq_sc = NULL; 900 txq->vxtxq_id = -1; 901 902 if (txr->vxtxr_txbuf != NULL) { 903 free(txr->vxtxr_txbuf, M_DEVBUF); 904 txr->vxtxr_txbuf = NULL; 905 } 906 907 if (mtx_initialized(&txq->vxtxq_mtx) != 0) 908 mtx_destroy(&txq->vxtxq_mtx); 909 } 910 911 static void 912 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 913 { 914 int i; 915 916 if (sc->vmx_rxq != NULL) { 917 for (i = 0; i < sc->vmx_nrxqueues; i++) 918 vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 919 free(sc->vmx_rxq, M_DEVBUF); 920 sc->vmx_rxq = NULL; 921 } 922 923 if (sc->vmx_txq != NULL) { 924 for (i = 0; i < sc->vmx_ntxqueues; i++) 925 vmxnet3_destroy_txq(&sc->vmx_txq[i]); 926 free(sc->vmx_txq, M_DEVBUF); 927 sc->vmx_txq = NULL; 928 } 929 } 930 931 static int 932 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 933 { 934 device_t dev; 935 uint8_t *kva; 936 size_t size; 937 int i, error; 938 939 dev = sc->vmx_dev; 940 941 size = sizeof(struct vmxnet3_driver_shared); 942 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 943 if (error) { 944 device_printf(dev, "cannot alloc shared memory\n"); 945 return (error); 946 } 947 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 948 949 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 950 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 951 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 952 if (error) { 953 device_printf(dev, "cannot alloc queue shared memory\n"); 954 return (error); 955 } 956 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 957 kva = sc->vmx_qs; 958 959 for (i = 0; i < sc->vmx_ntxqueues; i++) { 960 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 961 kva += sizeof(struct vmxnet3_txq_shared); 962 } 963 for (i = 0; i < sc->vmx_nrxqueues; i++) { 964 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 965 kva += sizeof(struct vmxnet3_rxq_shared); 966 } 967 968 return (0); 969 } 970 971 static void 972 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 973 { 974 975 if (sc->vmx_qs != NULL) { 976 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 977 sc->vmx_qs = NULL; 978 } 979 980 if (sc->vmx_ds != NULL) { 981 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 982 sc->vmx_ds = NULL; 983 } 984 } 985 986 static int 987 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 988 { 989 device_t dev; 990 struct vmxnet3_txqueue *txq; 991 struct vmxnet3_txring *txr; 992 struct vmxnet3_comp_ring *txc; 993 size_t descsz, compsz; 994 int i, q, error; 995 996 dev = sc->vmx_dev; 997 998 for (q = 0; q < sc->vmx_ntxqueues; q++) { 999 txq = &sc->vmx_txq[q]; 1000 txr = &txq->vxtxq_cmd_ring; 1001 txc = &txq->vxtxq_comp_ring; 1002 1003 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1004 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1005 1006 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1007 1, 0, /* alignment, boundary */ 1008 BUS_SPACE_MAXADDR, /* lowaddr */ 1009 BUS_SPACE_MAXADDR, /* highaddr */ 1010 NULL, NULL, /* filter, filterarg */ 1011 VMXNET3_TSO_MAXSIZE, /* maxsize */ 1012 VMXNET3_TX_MAXSEGS, /* nsegments */ 1013 VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1014 0, /* flags */ 1015 NULL, NULL, /* lockfunc, lockarg */ 1016 &txr->vxtxr_txtag); 1017 if (error) { 1018 device_printf(dev, 1019 "unable to create Tx buffer tag for queue %d\n", q); 1020 return (error); 1021 } 1022 1023 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1024 if (error) { 1025 device_printf(dev, "cannot alloc Tx descriptors for " 1026 "queue %d error %d\n", q, error); 1027 return (error); 1028 } 1029 txr->vxtxr_txd = 1030 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1031 1032 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1033 if (error) { 1034 device_printf(dev, "cannot alloc Tx comp descriptors " 1035 "for queue %d error %d\n", q, error); 1036 return (error); 1037 } 1038 txc->vxcr_u.txcd = 1039 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1040 1041 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1042 error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1043 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1044 if (error) { 1045 device_printf(dev, "unable to create Tx buf " 1046 "dmamap for queue %d idx %d\n", q, i); 1047 return (error); 1048 } 1049 } 1050 } 1051 1052 return (0); 1053 } 1054 1055 static void 1056 vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1057 { 1058 device_t dev; 1059 struct vmxnet3_txqueue *txq; 1060 struct vmxnet3_txring *txr; 1061 struct vmxnet3_comp_ring *txc; 1062 struct vmxnet3_txbuf *txb; 1063 int i, q; 1064 1065 dev = sc->vmx_dev; 1066 1067 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1068 txq = &sc->vmx_txq[q]; 1069 txr = &txq->vxtxq_cmd_ring; 1070 txc = &txq->vxtxq_comp_ring; 1071 1072 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1073 txb = &txr->vxtxr_txbuf[i]; 1074 if (txb->vtxb_dmamap != NULL) { 1075 bus_dmamap_destroy(txr->vxtxr_txtag, 1076 txb->vtxb_dmamap); 1077 txb->vtxb_dmamap = NULL; 1078 } 1079 } 1080 1081 if (txc->vxcr_u.txcd != NULL) { 1082 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1083 txc->vxcr_u.txcd = NULL; 1084 } 1085 1086 if (txr->vxtxr_txd != NULL) { 1087 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1088 txr->vxtxr_txd = NULL; 1089 } 1090 1091 if (txr->vxtxr_txtag != NULL) { 1092 bus_dma_tag_destroy(txr->vxtxr_txtag); 1093 txr->vxtxr_txtag = NULL; 1094 } 1095 } 1096 } 1097 1098 static int 1099 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1100 { 1101 device_t dev; 1102 struct vmxnet3_rxqueue *rxq; 1103 struct vmxnet3_rxring *rxr; 1104 struct vmxnet3_comp_ring *rxc; 1105 int descsz, compsz; 1106 int i, j, q, error; 1107 1108 dev = sc->vmx_dev; 1109 1110 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1111 rxq = &sc->vmx_rxq[q]; 1112 rxc = &rxq->vxrxq_comp_ring; 1113 compsz = 0; 1114 1115 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1116 rxr = &rxq->vxrxq_cmd_ring[i]; 1117 1118 descsz = rxr->vxrxr_ndesc * 1119 sizeof(struct vmxnet3_rxdesc); 1120 compsz += rxr->vxrxr_ndesc * 1121 sizeof(struct vmxnet3_rxcompdesc); 1122 1123 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1124 1, 0, /* alignment, boundary */ 1125 BUS_SPACE_MAXADDR, /* lowaddr */ 1126 BUS_SPACE_MAXADDR, /* highaddr */ 1127 NULL, NULL, /* filter, filterarg */ 1128 MJUMPAGESIZE, /* maxsize */ 1129 1, /* nsegments */ 1130 MJUMPAGESIZE, /* maxsegsize */ 1131 0, /* flags */ 1132 NULL, NULL, /* lockfunc, lockarg */ 1133 &rxr->vxrxr_rxtag); 1134 if (error) { 1135 device_printf(dev, 1136 "unable to create Rx buffer tag for " 1137 "queue %d\n", q); 1138 return (error); 1139 } 1140 1141 error = vmxnet3_dma_malloc(sc, descsz, 512, 1142 &rxr->vxrxr_dma); 1143 if (error) { 1144 device_printf(dev, "cannot allocate Rx " 1145 "descriptors for queue %d/%d error %d\n", 1146 i, q, error); 1147 return (error); 1148 } 1149 rxr->vxrxr_rxd = 1150 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1151 } 1152 1153 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1154 if (error) { 1155 device_printf(dev, "cannot alloc Rx comp descriptors " 1156 "for queue %d error %d\n", q, error); 1157 return (error); 1158 } 1159 rxc->vxcr_u.rxcd = 1160 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1161 1162 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1163 rxr = &rxq->vxrxq_cmd_ring[i]; 1164 1165 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1166 &rxr->vxrxr_spare_dmap); 1167 if (error) { 1168 device_printf(dev, "unable to create spare " 1169 "dmamap for queue %d/%d error %d\n", 1170 q, i, error); 1171 return (error); 1172 } 1173 1174 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1175 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1176 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1177 if (error) { 1178 device_printf(dev, "unable to create " 1179 "dmamap for queue %d/%d slot %d " 1180 "error %d\n", 1181 q, i, j, error); 1182 return (error); 1183 } 1184 } 1185 } 1186 } 1187 1188 return (0); 1189 } 1190 1191 static void 1192 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1193 { 1194 device_t dev; 1195 struct vmxnet3_rxqueue *rxq; 1196 struct vmxnet3_rxring *rxr; 1197 struct vmxnet3_comp_ring *rxc; 1198 struct vmxnet3_rxbuf *rxb; 1199 int i, j, q; 1200 1201 dev = sc->vmx_dev; 1202 1203 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1204 rxq = &sc->vmx_rxq[q]; 1205 rxc = &rxq->vxrxq_comp_ring; 1206 1207 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1208 rxr = &rxq->vxrxq_cmd_ring[i]; 1209 1210 if (rxr->vxrxr_spare_dmap != NULL) { 1211 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1212 rxr->vxrxr_spare_dmap); 1213 rxr->vxrxr_spare_dmap = NULL; 1214 } 1215 1216 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1217 rxb = &rxr->vxrxr_rxbuf[j]; 1218 if (rxb->vrxb_dmamap != NULL) { 1219 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1220 rxb->vrxb_dmamap); 1221 rxb->vrxb_dmamap = NULL; 1222 } 1223 } 1224 } 1225 1226 if (rxc->vxcr_u.rxcd != NULL) { 1227 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1228 rxc->vxcr_u.rxcd = NULL; 1229 } 1230 1231 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1232 rxr = &rxq->vxrxq_cmd_ring[i]; 1233 1234 if (rxr->vxrxr_rxd != NULL) { 1235 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1236 rxr->vxrxr_rxd = NULL; 1237 } 1238 1239 if (rxr->vxrxr_rxtag != NULL) { 1240 bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1241 rxr->vxrxr_rxtag = NULL; 1242 } 1243 } 1244 } 1245 } 1246 1247 static int 1248 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1249 { 1250 int error; 1251 1252 error = vmxnet3_alloc_txq_data(sc); 1253 if (error) 1254 return (error); 1255 1256 error = vmxnet3_alloc_rxq_data(sc); 1257 if (error) 1258 return (error); 1259 1260 return (0); 1261 } 1262 1263 static void 1264 vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1265 { 1266 1267 if (sc->vmx_rxq != NULL) 1268 vmxnet3_free_rxq_data(sc); 1269 1270 if (sc->vmx_txq != NULL) 1271 vmxnet3_free_txq_data(sc); 1272 } 1273 1274 static int 1275 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1276 { 1277 int error; 1278 1279 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1280 32, &sc->vmx_mcast_dma); 1281 if (error) 1282 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1283 else 1284 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1285 1286 return (error); 1287 } 1288 1289 static void 1290 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1291 { 1292 1293 if (sc->vmx_mcast != NULL) { 1294 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1295 sc->vmx_mcast = NULL; 1296 } 1297 } 1298 1299 static void 1300 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1301 { 1302 struct vmxnet3_driver_shared *ds; 1303 struct vmxnet3_txqueue *txq; 1304 struct vmxnet3_txq_shared *txs; 1305 struct vmxnet3_rxqueue *rxq; 1306 struct vmxnet3_rxq_shared *rxs; 1307 int i; 1308 1309 ds = sc->vmx_ds; 1310 1311 /* 1312 * Initialize fields of the shared data that remains the same across 1313 * reinits. Note the shared data is zero'd when allocated. 1314 */ 1315 1316 ds->magic = VMXNET3_REV1_MAGIC; 1317 1318 /* DriverInfo */ 1319 ds->version = VMXNET3_DRIVER_VERSION; 1320 ds->guest = VMXNET3_GOS_FREEBSD | VMXNET3_GUEST_OS_VERSION | 1321 #ifdef __LP64__ 1322 VMXNET3_GOS_64BIT; 1323 #else 1324 VMXNET3_GOS_32BIT; 1325 #endif 1326 ds->vmxnet3_revision = 1; 1327 ds->upt_version = 1; 1328 1329 /* Misc. conf */ 1330 ds->driver_data = vtophys(sc); 1331 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1332 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1333 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1334 ds->nrxsg_max = sc->vmx_max_rxsegs; 1335 1336 /* Interrupt control. */ 1337 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1338 ds->nintr = sc->vmx_nintrs; 1339 ds->evintr = sc->vmx_event_intr_idx; 1340 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1341 1342 for (i = 0; i < sc->vmx_nintrs; i++) 1343 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1344 1345 /* Receive filter. */ 1346 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1347 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1348 1349 /* Tx queues */ 1350 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1351 txq = &sc->vmx_txq[i]; 1352 txs = txq->vxtxq_ts; 1353 1354 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1355 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1356 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1357 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1358 txs->driver_data = vtophys(txq); 1359 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1360 } 1361 1362 /* Rx queues */ 1363 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1364 rxq = &sc->vmx_rxq[i]; 1365 rxs = rxq->vxrxq_rs; 1366 1367 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1368 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1369 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1370 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1371 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1372 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1373 rxs->driver_data = vtophys(rxq); 1374 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1375 } 1376 } 1377 1378 static void 1379 vmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1380 { 1381 struct ifnet *ifp; 1382 1383 ifp = sc->vmx_ifp; 1384 1385 /* Use the current MAC address. */ 1386 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1387 vmxnet3_set_lladdr(sc); 1388 1389 ifp->if_hwassist = 0; 1390 if (ifp->if_capenable & IFCAP_TXCSUM) 1391 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD; 1392 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1393 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1394 if (ifp->if_capenable & IFCAP_TSO4) 1395 ifp->if_hwassist |= CSUM_TSO; 1396 if (ifp->if_capenable & IFCAP_TSO6) 1397 ifp->if_hwassist |= CSUM_TSO; /* No CSUM_TSO_IPV6. */ 1398 } 1399 1400 static void 1401 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1402 { 1403 struct ifnet *ifp; 1404 struct vmxnet3_driver_shared *ds; 1405 1406 ifp = sc->vmx_ifp; 1407 ds = sc->vmx_ds; 1408 1409 ds->upt_features = 0; 1410 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1411 ds->upt_features |= UPT1_F_CSUM; 1412 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1413 ds->upt_features |= UPT1_F_VLAN; 1414 if (ifp->if_capenable & IFCAP_LRO) 1415 ds->upt_features |= UPT1_F_LRO; 1416 1417 ds->mtu = ifp->if_mtu; 1418 ds->ntxqueue = sc->vmx_ntxqueues; 1419 ds->nrxqueue = sc->vmx_nrxqueues; 1420 1421 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1422 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1423 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1424 } 1425 1426 static int 1427 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1428 { 1429 int error; 1430 1431 error = vmxnet3_alloc_shared_data(sc); 1432 if (error) 1433 return (error); 1434 1435 error = vmxnet3_alloc_queue_data(sc); 1436 if (error) 1437 return (error); 1438 1439 error = vmxnet3_alloc_mcast_table(sc); 1440 if (error) 1441 return (error); 1442 1443 vmxnet3_init_shared_data(sc); 1444 1445 return (0); 1446 } 1447 1448 static void 1449 vmxnet3_free_data(struct vmxnet3_softc *sc) 1450 { 1451 1452 vmxnet3_free_mcast_table(sc); 1453 vmxnet3_free_queue_data(sc); 1454 vmxnet3_free_shared_data(sc); 1455 } 1456 1457 static int 1458 vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1459 { 1460 device_t dev; 1461 struct ifnet *ifp; 1462 1463 dev = sc->vmx_dev; 1464 1465 ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1466 if (ifp == NULL) { 1467 device_printf(dev, "cannot allocate ifnet structure\n"); 1468 return (ENOSPC); 1469 } 1470 1471 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1472 #if __FreeBSD_version < 1000025 1473 ifp->if_baudrate = 1000000000; 1474 #else 1475 if_initbaudrate(ifp, IF_Gbps(10)); 1476 #endif 1477 ifp->if_softc = sc; 1478 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1479 ifp->if_init = vmxnet3_init; 1480 ifp->if_ioctl = vmxnet3_ioctl; 1481 ifp->if_start = vmxnet3_start; 1482 ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; 1483 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1484 IFQ_SET_READY(&ifp->if_snd); 1485 1486 vmxnet3_get_lladdr(sc); 1487 ether_ifattach(ifp, sc->vmx_lladdr); 1488 1489 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1490 ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1491 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1492 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1493 IFCAP_VLAN_HWCSUM; 1494 ifp->if_capenable = ifp->if_capabilities; 1495 1496 /* These capabilities are not enabled by default. */ 1497 ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER; 1498 1499 sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1500 vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1501 sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1502 vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1503 1504 ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1505 vmxnet3_media_status); 1506 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1507 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1508 1509 return (0); 1510 } 1511 1512 static void 1513 vmxnet3_evintr(struct vmxnet3_softc *sc) 1514 { 1515 device_t dev; 1516 struct ifnet *ifp; 1517 struct vmxnet3_txq_shared *ts; 1518 struct vmxnet3_rxq_shared *rs; 1519 uint32_t event; 1520 int reset; 1521 1522 dev = sc->vmx_dev; 1523 ifp = sc->vmx_ifp; 1524 reset = 0; 1525 1526 VMXNET3_CORE_LOCK(sc); 1527 1528 /* Clear events. */ 1529 event = sc->vmx_ds->event; 1530 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1531 1532 if (event & VMXNET3_EVENT_LINK) 1533 vmxnet3_link_status(sc); 1534 1535 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1536 reset = 1; 1537 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1538 ts = sc->vmx_txq[0].vxtxq_ts; 1539 if (ts->stopped != 0) 1540 device_printf(dev, "Tx queue error %#x\n", ts->error); 1541 rs = sc->vmx_rxq[0].vxrxq_rs; 1542 if (rs->stopped != 0) 1543 device_printf(dev, "Rx queue error %#x\n", rs->error); 1544 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1545 } 1546 1547 if (event & VMXNET3_EVENT_DIC) 1548 device_printf(dev, "device implementation change event\n"); 1549 if (event & VMXNET3_EVENT_DEBUG) 1550 device_printf(dev, "debug event\n"); 1551 1552 if (reset != 0) { 1553 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1554 vmxnet3_init_locked(sc); 1555 } 1556 1557 VMXNET3_CORE_UNLOCK(sc); 1558 } 1559 1560 static void 1561 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1562 { 1563 struct vmxnet3_softc *sc; 1564 struct ifnet *ifp; 1565 struct vmxnet3_txring *txr; 1566 struct vmxnet3_comp_ring *txc; 1567 struct vmxnet3_txcompdesc *txcd; 1568 struct vmxnet3_txbuf *txb; 1569 u_int sop; 1570 1571 sc = txq->vxtxq_sc; 1572 ifp = sc->vmx_ifp; 1573 txr = &txq->vxtxq_cmd_ring; 1574 txc = &txq->vxtxq_comp_ring; 1575 1576 VMXNET3_TXQ_LOCK_ASSERT(txq); 1577 1578 for (;;) { 1579 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1580 if (txcd->gen != txc->vxcr_gen) 1581 break; 1582 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1583 1584 if (++txc->vxcr_next == txc->vxcr_ndesc) { 1585 txc->vxcr_next = 0; 1586 txc->vxcr_gen ^= 1; 1587 } 1588 1589 sop = txr->vxtxr_next; 1590 txb = &txr->vxtxr_txbuf[sop]; 1591 1592 if (txb->vtxb_m != NULL) { 1593 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1594 BUS_DMASYNC_POSTWRITE); 1595 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1596 1597 m_freem(txb->vtxb_m); 1598 txb->vtxb_m = NULL; 1599 1600 ifp->if_opackets++; 1601 } 1602 1603 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1604 } 1605 1606 if (txr->vxtxr_head == txr->vxtxr_next) 1607 txq->vxtxq_watchdog = 0; 1608 } 1609 1610 static int 1611 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1612 { 1613 struct ifnet *ifp; 1614 struct mbuf *m; 1615 struct vmxnet3_rxdesc *rxd; 1616 struct vmxnet3_rxbuf *rxb; 1617 bus_dma_tag_t tag; 1618 bus_dmamap_t dmap; 1619 bus_dma_segment_t segs[1]; 1620 int idx, clsize, btype, flags, nsegs, error; 1621 1622 ifp = sc->vmx_ifp; 1623 tag = rxr->vxrxr_rxtag; 1624 dmap = rxr->vxrxr_spare_dmap; 1625 idx = rxr->vxrxr_fill; 1626 rxd = &rxr->vxrxr_rxd[idx]; 1627 rxb = &rxr->vxrxr_rxbuf[idx]; 1628 1629 #ifdef VMXNET3_FAILPOINTS 1630 KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1631 if (rxr->vxrxr_rid != 0) 1632 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1633 #endif 1634 1635 if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1636 flags = M_PKTHDR; 1637 clsize = MCLBYTES; 1638 btype = VMXNET3_BTYPE_HEAD; 1639 } else { 1640 #if __FreeBSD_version < 902001 1641 /* 1642 * These mbufs will never be used for the start of a frame. 1643 * Roughly prior to branching releng/9.2, the load_mbuf_sg() 1644 * required the mbuf to always be a packet header. Avoid 1645 * unnecessary mbuf initialization in newer versions where 1646 * that is not the case. 1647 */ 1648 flags = M_PKTHDR; 1649 #else 1650 flags = 0; 1651 #endif 1652 clsize = MJUMPAGESIZE; 1653 btype = VMXNET3_BTYPE_BODY; 1654 } 1655 1656 m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 1657 if (m == NULL) { 1658 sc->vmx_stats.vmst_mgetcl_failed++; 1659 return (ENOBUFS); 1660 } 1661 1662 if (btype == VMXNET3_BTYPE_HEAD) { 1663 m->m_len = m->m_pkthdr.len = clsize; 1664 m_adj(m, ETHER_ALIGN); 1665 } else 1666 m->m_len = clsize; 1667 1668 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, 1669 BUS_DMA_NOWAIT); 1670 if (error) { 1671 m_freem(m); 1672 sc->vmx_stats.vmst_mbuf_load_failed++; 1673 return (error); 1674 } 1675 KASSERT(nsegs == 1, 1676 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 1677 #if __FreeBSD_version < 902001 1678 if (btype == VMXNET3_BTYPE_BODY) 1679 m->m_flags &= ~M_PKTHDR; 1680 #endif 1681 1682 if (rxb->vrxb_m != NULL) { 1683 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 1684 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 1685 } 1686 1687 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 1688 rxb->vrxb_dmamap = dmap; 1689 rxb->vrxb_m = m; 1690 1691 rxd->addr = segs[0].ds_addr; 1692 rxd->len = segs[0].ds_len; 1693 rxd->btype = btype; 1694 rxd->gen = rxr->vxrxr_gen; 1695 1696 vmxnet3_rxr_increment_fill(rxr); 1697 return (0); 1698 } 1699 1700 static void 1701 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 1702 struct vmxnet3_rxring *rxr, int idx) 1703 { 1704 struct vmxnet3_rxdesc *rxd; 1705 1706 rxd = &rxr->vxrxr_rxd[idx]; 1707 rxd->gen = rxr->vxrxr_gen; 1708 vmxnet3_rxr_increment_fill(rxr); 1709 } 1710 1711 static void 1712 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 1713 { 1714 struct vmxnet3_softc *sc; 1715 struct vmxnet3_rxring *rxr; 1716 struct vmxnet3_comp_ring *rxc; 1717 struct vmxnet3_rxcompdesc *rxcd; 1718 int idx, eof; 1719 1720 sc = rxq->vxrxq_sc; 1721 rxc = &rxq->vxrxq_comp_ring; 1722 1723 do { 1724 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 1725 if (rxcd->gen != rxc->vxcr_gen) 1726 break; /* Not expected. */ 1727 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1728 1729 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 1730 rxc->vxcr_next = 0; 1731 rxc->vxcr_gen ^= 1; 1732 } 1733 1734 idx = rxcd->rxd_idx; 1735 eof = rxcd->eop; 1736 if (rxcd->qid < sc->vmx_nrxqueues) 1737 rxr = &rxq->vxrxq_cmd_ring[0]; 1738 else 1739 rxr = &rxq->vxrxq_cmd_ring[1]; 1740 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1741 } while (!eof); 1742 } 1743 1744 static void 1745 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 1746 { 1747 1748 if (rxcd->ipv4) { 1749 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 1750 if (rxcd->ipcsum_ok) 1751 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 1752 } 1753 1754 if (!rxcd->fragment) { 1755 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 1756 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 1757 CSUM_PSEUDO_HDR; 1758 m->m_pkthdr.csum_data = 0xFFFF; 1759 } 1760 } 1761 } 1762 1763 static void 1764 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 1765 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 1766 { 1767 struct vmxnet3_softc *sc; 1768 struct ifnet *ifp; 1769 1770 sc = rxq->vxrxq_sc; 1771 ifp = sc->vmx_ifp; 1772 1773 if (rxcd->error) { 1774 ifp->if_ierrors++; 1775 m_freem(m); 1776 return; 1777 } 1778 1779 if (!rxcd->no_csum) 1780 vmxnet3_rx_csum(rxcd, m); 1781 if (rxcd->vlan) { 1782 m->m_flags |= M_VLANTAG; 1783 m->m_pkthdr.ether_vtag = rxcd->vtag; 1784 } 1785 1786 ifp->if_ipackets++; 1787 VMXNET3_RXQ_UNLOCK(rxq); 1788 (*ifp->if_input)(ifp, m); 1789 VMXNET3_RXQ_LOCK(rxq); 1790 } 1791 1792 static void 1793 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 1794 { 1795 struct vmxnet3_softc *sc; 1796 struct ifnet *ifp; 1797 struct vmxnet3_rxring *rxr; 1798 struct vmxnet3_comp_ring *rxc; 1799 struct vmxnet3_rxdesc *rxd; 1800 struct vmxnet3_rxcompdesc *rxcd; 1801 struct mbuf *m, *m_head, *m_tail; 1802 int idx, length; 1803 1804 sc = rxq->vxrxq_sc; 1805 ifp = sc->vmx_ifp; 1806 rxc = &rxq->vxrxq_comp_ring; 1807 m_head = m_tail = NULL; 1808 1809 VMXNET3_RXQ_LOCK_ASSERT(rxq); 1810 1811 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1812 return; 1813 1814 for (;;) { 1815 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 1816 if (rxcd->gen != rxc->vxcr_gen) 1817 break; 1818 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1819 1820 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 1821 rxc->vxcr_next = 0; 1822 rxc->vxcr_gen ^= 1; 1823 } 1824 1825 idx = rxcd->rxd_idx; 1826 length = rxcd->len; 1827 if (rxcd->qid < sc->vmx_nrxqueues) 1828 rxr = &rxq->vxrxq_cmd_ring[0]; 1829 else 1830 rxr = &rxq->vxrxq_cmd_ring[1]; 1831 rxd = &rxr->vxrxr_rxd[idx]; 1832 1833 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 1834 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 1835 __func__, rxcd->qid, idx)); 1836 1837 /* 1838 * The host may skip descriptors. We detect this when this 1839 * descriptor does not match the previous fill index. Catch 1840 * up with the host now. 1841 */ 1842 if (__predict_false(rxr->vxrxr_fill != idx)) { 1843 while (rxr->vxrxr_fill != idx) { 1844 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 1845 rxr->vxrxr_gen; 1846 vmxnet3_rxr_increment_fill(rxr); 1847 } 1848 } 1849 1850 if (rxcd->sop) { 1851 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 1852 ("%s: start of frame w/o head buffer", __func__)); 1853 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 1854 ("%s: start of frame not in ring 0", __func__)); 1855 KASSERT((idx % sc->vmx_rx_max_chain) == 0, 1856 ("%s: start of frame at unexcepted index %d (%d)", 1857 __func__, idx, sc->vmx_rx_max_chain)); 1858 KASSERT(m_head == NULL, 1859 ("%s: duplicate start of frame?", __func__)); 1860 1861 if (length == 0) { 1862 /* Just ignore this descriptor. */ 1863 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1864 goto nextp; 1865 } 1866 1867 if (vmxnet3_newbuf(sc, rxr) != 0) { 1868 ifp->if_iqdrops++; 1869 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1870 if (!rxcd->eop) 1871 vmxnet3_rxq_discard_chain(rxq); 1872 goto nextp; 1873 } 1874 1875 m->m_pkthdr.rcvif = ifp; 1876 m->m_pkthdr.len = m->m_len = length; 1877 m->m_pkthdr.csum_flags = 0; 1878 m_head = m_tail = m; 1879 1880 } else { 1881 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 1882 ("%s: non start of frame w/o body buffer", __func__)); 1883 KASSERT(m_head != NULL, 1884 ("%s: frame not started?", __func__)); 1885 1886 if (vmxnet3_newbuf(sc, rxr) != 0) { 1887 ifp->if_iqdrops++; 1888 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1889 if (!rxcd->eop) 1890 vmxnet3_rxq_discard_chain(rxq); 1891 m_freem(m_head); 1892 m_head = m_tail = NULL; 1893 goto nextp; 1894 } 1895 1896 m->m_len = length; 1897 m_head->m_pkthdr.len += length; 1898 m_tail->m_next = m; 1899 m_tail = m; 1900 } 1901 1902 if (rxcd->eop) { 1903 vmxnet3_rxq_input(rxq, rxcd, m_head); 1904 m_head = m_tail = NULL; 1905 1906 /* Must recheck after dropping the Rx lock. */ 1907 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1908 break; 1909 } 1910 1911 nextp: 1912 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 1913 int qid = rxcd->qid; 1914 bus_size_t r; 1915 1916 idx = (idx + 1) % rxr->vxrxr_ndesc; 1917 if (qid >= sc->vmx_nrxqueues) { 1918 qid -= sc->vmx_nrxqueues; 1919 r = VMXNET3_BAR0_RXH2(qid); 1920 } else 1921 r = VMXNET3_BAR0_RXH1(qid); 1922 vmxnet3_write_bar0(sc, r, idx); 1923 } 1924 } 1925 } 1926 1927 static void 1928 vmxnet3_legacy_intr(void *xsc) 1929 { 1930 struct vmxnet3_softc *sc; 1931 struct vmxnet3_rxqueue *rxq; 1932 struct vmxnet3_txqueue *txq; 1933 struct ifnet *ifp; 1934 1935 sc = xsc; 1936 rxq = &sc->vmx_rxq[0]; 1937 txq = &sc->vmx_txq[0]; 1938 ifp = sc->vmx_ifp; 1939 1940 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 1941 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 1942 return; 1943 } 1944 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1945 vmxnet3_disable_all_intrs(sc); 1946 1947 if (sc->vmx_ds->event != 0) 1948 vmxnet3_evintr(sc); 1949 1950 VMXNET3_RXQ_LOCK(rxq); 1951 vmxnet3_rxq_eof(rxq); 1952 VMXNET3_RXQ_UNLOCK(rxq); 1953 1954 VMXNET3_TXQ_LOCK(txq); 1955 vmxnet3_txq_eof(txq); 1956 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1957 vmxnet3_start_locked(ifp); 1958 VMXNET3_TXQ_UNLOCK(txq); 1959 1960 vmxnet3_enable_all_intrs(sc); 1961 } 1962 1963 static void 1964 vmxnet3_txq_intr(void *xtxq) 1965 { 1966 struct vmxnet3_softc *sc; 1967 struct vmxnet3_txqueue *txq; 1968 struct ifnet *ifp; 1969 1970 txq = xtxq; 1971 sc = txq->vxtxq_sc; 1972 ifp = sc->vmx_ifp; 1973 1974 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1975 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 1976 1977 VMXNET3_TXQ_LOCK(txq); 1978 vmxnet3_txq_eof(txq); 1979 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1980 vmxnet3_start_locked(ifp); 1981 VMXNET3_TXQ_UNLOCK(txq); 1982 1983 vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 1984 } 1985 1986 static void 1987 vmxnet3_rxq_intr(void *xrxq) 1988 { 1989 struct vmxnet3_softc *sc; 1990 struct vmxnet3_rxqueue *rxq; 1991 1992 rxq = xrxq; 1993 sc = rxq->vxrxq_sc; 1994 1995 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1996 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 1997 1998 VMXNET3_RXQ_LOCK(rxq); 1999 vmxnet3_rxq_eof(rxq); 2000 VMXNET3_RXQ_UNLOCK(rxq); 2001 2002 vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2003 } 2004 2005 static void 2006 vmxnet3_event_intr(void *xsc) 2007 { 2008 struct vmxnet3_softc *sc; 2009 2010 sc = xsc; 2011 2012 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2013 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2014 2015 if (sc->vmx_ds->event != 0) 2016 vmxnet3_evintr(sc); 2017 2018 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2019 } 2020 2021 static void 2022 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2023 { 2024 struct vmxnet3_txring *txr; 2025 struct vmxnet3_txbuf *txb; 2026 int i; 2027 2028 txr = &txq->vxtxq_cmd_ring; 2029 2030 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2031 txb = &txr->vxtxr_txbuf[i]; 2032 2033 if (txb->vtxb_m == NULL) 2034 continue; 2035 2036 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2037 BUS_DMASYNC_POSTWRITE); 2038 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2039 m_freem(txb->vtxb_m); 2040 txb->vtxb_m = NULL; 2041 } 2042 } 2043 2044 static void 2045 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2046 { 2047 struct vmxnet3_rxring *rxr; 2048 struct vmxnet3_rxbuf *rxb; 2049 int i, j; 2050 2051 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2052 rxr = &rxq->vxrxq_cmd_ring[i]; 2053 2054 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2055 rxb = &rxr->vxrxr_rxbuf[j]; 2056 2057 if (rxb->vrxb_m == NULL) 2058 continue; 2059 bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2060 BUS_DMASYNC_POSTREAD); 2061 bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2062 m_freem(rxb->vrxb_m); 2063 rxb->vrxb_m = NULL; 2064 } 2065 } 2066 } 2067 2068 static void 2069 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2070 { 2071 struct vmxnet3_rxqueue *rxq; 2072 struct vmxnet3_txqueue *txq; 2073 int i; 2074 2075 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2076 rxq = &sc->vmx_rxq[i]; 2077 VMXNET3_RXQ_LOCK(rxq); 2078 VMXNET3_RXQ_UNLOCK(rxq); 2079 } 2080 2081 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2082 txq = &sc->vmx_txq[i]; 2083 VMXNET3_TXQ_LOCK(txq); 2084 VMXNET3_TXQ_UNLOCK(txq); 2085 } 2086 } 2087 2088 static void 2089 vmxnet3_stop(struct vmxnet3_softc *sc) 2090 { 2091 struct ifnet *ifp; 2092 int q; 2093 2094 ifp = sc->vmx_ifp; 2095 VMXNET3_CORE_LOCK_ASSERT(sc); 2096 2097 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2098 sc->vmx_link_active = 0; 2099 callout_stop(&sc->vmx_tick); 2100 2101 /* Disable interrupts. */ 2102 vmxnet3_disable_all_intrs(sc); 2103 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2104 2105 vmxnet3_stop_rendezvous(sc); 2106 2107 for (q = 0; q < sc->vmx_ntxqueues; q++) 2108 vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2109 for (q = 0; q < sc->vmx_nrxqueues; q++) 2110 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2111 2112 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2113 } 2114 2115 static void 2116 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2117 { 2118 struct vmxnet3_txring *txr; 2119 struct vmxnet3_comp_ring *txc; 2120 2121 txr = &txq->vxtxq_cmd_ring; 2122 txr->vxtxr_head = 0; 2123 txr->vxtxr_next = 0; 2124 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2125 bzero(txr->vxtxr_txd, 2126 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2127 2128 txc = &txq->vxtxq_comp_ring; 2129 txc->vxcr_next = 0; 2130 txc->vxcr_gen = VMXNET3_INIT_GEN; 2131 bzero(txc->vxcr_u.txcd, 2132 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2133 } 2134 2135 static int 2136 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2137 { 2138 struct ifnet *ifp; 2139 struct vmxnet3_rxring *rxr; 2140 struct vmxnet3_comp_ring *rxc; 2141 int i, populate, idx, frame_size, error; 2142 2143 ifp = sc->vmx_ifp; 2144 frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2145 ifp->if_mtu; 2146 2147 /* 2148 * If the MTU causes us to exceed what a regular sized cluster can 2149 * handle, we allocate a second MJUMPAGESIZE cluster after it in 2150 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2151 * 2152 * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2153 * our life easier. We do not support changing the ring size after 2154 * the attach. 2155 */ 2156 if (frame_size <= MCLBYTES) 2157 sc->vmx_rx_max_chain = 1; 2158 else 2159 sc->vmx_rx_max_chain = 2; 2160 2161 /* 2162 * Only populate ring 1 if the configuration will take advantage 2163 * of it. That is either when LRO is enabled or the frame size 2164 * exceeds what ring 0 can contain. 2165 */ 2166 if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2167 frame_size <= MCLBYTES + MJUMPAGESIZE) 2168 populate = 1; 2169 else 2170 populate = VMXNET3_RXRINGS_PERQ; 2171 2172 for (i = 0; i < populate; i++) { 2173 rxr = &rxq->vxrxq_cmd_ring[i]; 2174 rxr->vxrxr_fill = 0; 2175 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2176 bzero(rxr->vxrxr_rxd, 2177 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2178 2179 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2180 error = vmxnet3_newbuf(sc, rxr); 2181 if (error) 2182 return (error); 2183 } 2184 } 2185 2186 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2187 rxr = &rxq->vxrxq_cmd_ring[i]; 2188 rxr->vxrxr_fill = 0; 2189 rxr->vxrxr_gen = 0; 2190 bzero(rxr->vxrxr_rxd, 2191 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2192 } 2193 2194 rxc = &rxq->vxrxq_comp_ring; 2195 rxc->vxcr_next = 0; 2196 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2197 bzero(rxc->vxcr_u.rxcd, 2198 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2199 2200 return (0); 2201 } 2202 2203 static int 2204 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2205 { 2206 device_t dev; 2207 int q, error; 2208 2209 dev = sc->vmx_dev; 2210 2211 for (q = 0; q < sc->vmx_ntxqueues; q++) 2212 vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2213 2214 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2215 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2216 if (error) { 2217 device_printf(dev, "cannot populate Rx queue %d\n", q); 2218 return (error); 2219 } 2220 } 2221 2222 return (0); 2223 } 2224 2225 static int 2226 vmxnet3_enable_device(struct vmxnet3_softc *sc) 2227 { 2228 int q; 2229 2230 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2231 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2232 return (1); 2233 } 2234 2235 /* Reset the Rx queue heads. */ 2236 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2237 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2238 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2239 } 2240 2241 return (0); 2242 } 2243 2244 static void 2245 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2246 { 2247 struct ifnet *ifp; 2248 2249 ifp = sc->vmx_ifp; 2250 2251 vmxnet3_set_rxfilter(sc); 2252 2253 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2254 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2255 sizeof(sc->vmx_ds->vlan_filter)); 2256 else 2257 bzero(sc->vmx_ds->vlan_filter, 2258 sizeof(sc->vmx_ds->vlan_filter)); 2259 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2260 } 2261 2262 static int 2263 vmxnet3_reinit(struct vmxnet3_softc *sc) 2264 { 2265 2266 vmxnet3_reinit_interface(sc); 2267 vmxnet3_reinit_shared_data(sc); 2268 2269 if (vmxnet3_reinit_queues(sc) != 0) 2270 return (ENXIO); 2271 2272 if (vmxnet3_enable_device(sc) != 0) 2273 return (ENXIO); 2274 2275 vmxnet3_reinit_rxfilters(sc); 2276 2277 return (0); 2278 } 2279 2280 static void 2281 vmxnet3_init_locked(struct vmxnet3_softc *sc) 2282 { 2283 struct ifnet *ifp; 2284 2285 ifp = sc->vmx_ifp; 2286 2287 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2288 return; 2289 2290 vmxnet3_stop(sc); 2291 2292 if (vmxnet3_reinit(sc) != 0) { 2293 vmxnet3_stop(sc); 2294 return; 2295 } 2296 2297 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2298 vmxnet3_link_status(sc); 2299 2300 vmxnet3_enable_all_intrs(sc); 2301 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2302 } 2303 2304 static void 2305 vmxnet3_init(void *xsc) 2306 { 2307 struct vmxnet3_softc *sc; 2308 2309 sc = xsc; 2310 2311 VMXNET3_CORE_LOCK(sc); 2312 vmxnet3_init_locked(sc); 2313 VMXNET3_CORE_UNLOCK(sc); 2314 } 2315 2316 /* 2317 * BMV: Much of this can go away once we finally have offsets in 2318 * the mbuf packet header. Bug andre@. 2319 */ 2320 static int 2321 vmxnet3_txq_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start) 2322 { 2323 struct ether_vlan_header *evh; 2324 int offset; 2325 2326 evh = mtod(m, struct ether_vlan_header *); 2327 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2328 /* BMV: We should handle nested VLAN tags too. */ 2329 *etype = ntohs(evh->evl_proto); 2330 offset = sizeof(struct ether_vlan_header); 2331 } else { 2332 *etype = ntohs(evh->evl_encap_proto); 2333 offset = sizeof(struct ether_header); 2334 } 2335 2336 switch (*etype) { 2337 #if defined(INET) 2338 case ETHERTYPE_IP: { 2339 struct ip *ip, iphdr; 2340 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2341 m_copydata(m, offset, sizeof(struct ip), 2342 (caddr_t) &iphdr); 2343 ip = &iphdr; 2344 } else 2345 ip = (struct ip *)(m->m_data + offset); 2346 *proto = ip->ip_p; 2347 *start = offset + (ip->ip_hl << 2); 2348 break; 2349 } 2350 #endif 2351 #if defined(INET6) 2352 case ETHERTYPE_IPV6: 2353 *proto = -1; 2354 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2355 /* Assert the network stack sent us a valid packet. */ 2356 KASSERT(*start > offset, 2357 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2358 *start, offset, *proto)); 2359 break; 2360 #endif 2361 default: 2362 return (EINVAL); 2363 } 2364 2365 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2366 struct tcphdr *tcp, tcphdr; 2367 2368 if (__predict_false(*proto != IPPROTO_TCP)) { 2369 /* Likely failed to correctly parse the mbuf. */ 2370 return (EINVAL); 2371 } 2372 2373 if (m->m_len < *start + sizeof(struct tcphdr)) { 2374 m_copydata(m, offset, sizeof(struct tcphdr), 2375 (caddr_t) &tcphdr); 2376 tcp = &tcphdr; 2377 } else 2378 tcp = (struct tcphdr *)(m->m_data + *start); 2379 2380 /* 2381 * For TSO, the size of the protocol header is also 2382 * included in the descriptor header size. 2383 */ 2384 *start += (tcp->th_off << 2); 2385 } 2386 2387 return (0); 2388 } 2389 2390 static int 2391 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2392 bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2393 { 2394 struct vmxnet3_txring *txr; 2395 struct mbuf *m; 2396 bus_dma_tag_t tag; 2397 int maxsegs, error; 2398 2399 txr = &txq->vxtxq_cmd_ring; 2400 m = *m0; 2401 tag = txr->vxtxr_txtag; 2402 maxsegs = VMXNET3_TX_MAXSEGS; 2403 2404 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2405 if (error == 0 || error != EFBIG) 2406 return (error); 2407 2408 m = m_collapse(m, M_NOWAIT, maxsegs); 2409 if (m != NULL) { 2410 *m0 = m; 2411 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2412 } else 2413 error = ENOBUFS; 2414 2415 if (error) { 2416 m_freem(*m0); 2417 *m0 = NULL; 2418 } else 2419 txq->vxtxq_sc->vmx_stats.vmst_collapsed++; 2420 2421 return (error); 2422 } 2423 2424 static void 2425 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2426 { 2427 struct vmxnet3_txring *txr; 2428 2429 txr = &txq->vxtxq_cmd_ring; 2430 bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2431 } 2432 2433 static int 2434 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2435 { 2436 struct vmxnet3_softc *sc; 2437 struct ifnet *ifp; 2438 struct vmxnet3_txring *txr; 2439 struct vmxnet3_txdesc *txd, *sop; 2440 struct mbuf *m; 2441 bus_dmamap_t dmap; 2442 bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2443 int i, gen, nsegs, etype, proto, start, error; 2444 2445 sc = txq->vxtxq_sc; 2446 ifp = sc->vmx_ifp; 2447 start = 0; 2448 txd = NULL; 2449 txr = &txq->vxtxq_cmd_ring; 2450 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2451 2452 error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2453 if (error) 2454 return (error); 2455 2456 m = *m0; 2457 M_ASSERTPKTHDR(m); 2458 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2459 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2460 2461 if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2462 txq->vxtxq_stats.vtxrs_full++; 2463 vmxnet3_txq_unload_mbuf(txq, dmap); 2464 return (ENOSPC); 2465 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2466 error = vmxnet3_txq_offload_ctx(m, &etype, &proto, &start); 2467 if (error) { 2468 txq->vxtxq_stats.vtxrs_offload_failed++; 2469 vmxnet3_txq_unload_mbuf(txq, dmap); 2470 m_freem(m); 2471 *m0 = NULL; 2472 return (error); 2473 } 2474 } 2475 2476 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m = *m0; 2477 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2478 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2479 2480 for (i = 0; i < nsegs; i++) { 2481 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2482 2483 txd->addr = segs[i].ds_addr; 2484 txd->len = segs[i].ds_len; 2485 txd->gen = gen; 2486 txd->dtype = 0; 2487 txd->offload_mode = VMXNET3_OM_NONE; 2488 txd->offload_pos = 0; 2489 txd->hlen = 0; 2490 txd->eop = 0; 2491 txd->compreq = 0; 2492 txd->vtag_mode = 0; 2493 txd->vtag = 0; 2494 2495 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2496 txr->vxtxr_head = 0; 2497 txr->vxtxr_gen ^= 1; 2498 } 2499 gen = txr->vxtxr_gen; 2500 } 2501 txd->eop = 1; 2502 txd->compreq = 1; 2503 2504 if (m->m_flags & M_VLANTAG) { 2505 sop->vtag_mode = 1; 2506 sop->vtag = m->m_pkthdr.ether_vtag; 2507 } 2508 2509 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2510 sop->offload_mode = VMXNET3_OM_TSO; 2511 sop->hlen = start; 2512 sop->offload_pos = m->m_pkthdr.tso_segsz; 2513 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2514 VMXNET3_CSUM_OFFLOAD_IPV6)) { 2515 sop->offload_mode = VMXNET3_OM_CSUM; 2516 sop->hlen = start; 2517 sop->offload_pos = start + m->m_pkthdr.csum_data; 2518 } 2519 2520 /* Finally, change the ownership. */ 2521 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2522 sop->gen ^= 1; 2523 2524 if (++txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2525 txq->vxtxq_ts->npending = 0; 2526 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2527 txr->vxtxr_head); 2528 } 2529 2530 return (0); 2531 } 2532 2533 static void 2534 vmxnet3_start_locked(struct ifnet *ifp) 2535 { 2536 struct vmxnet3_softc *sc; 2537 struct vmxnet3_txqueue *txq; 2538 struct vmxnet3_txring *txr; 2539 struct mbuf *m_head; 2540 int tx, avail; 2541 2542 sc = ifp->if_softc; 2543 txq = &sc->vmx_txq[0]; 2544 txr = &txq->vxtxq_cmd_ring; 2545 tx = 0; 2546 2547 VMXNET3_TXQ_LOCK_ASSERT(txq); 2548 2549 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2550 sc->vmx_link_active == 0) 2551 return; 2552 2553 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2554 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 2555 break; 2556 2557 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 2558 if (m_head == NULL) 2559 break; 2560 2561 /* Assume worse case if this mbuf is the head of a chain. */ 2562 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2563 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2564 break; 2565 } 2566 2567 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 2568 if (m_head != NULL) 2569 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2570 break; 2571 } 2572 2573 tx++; 2574 ETHER_BPF_MTAP(ifp, m_head); 2575 } 2576 2577 if (tx > 0) { 2578 if (txq->vxtxq_ts->npending > 0) { 2579 txq->vxtxq_ts->npending = 0; 2580 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2581 txr->vxtxr_head); 2582 } 2583 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2584 } 2585 } 2586 2587 static void 2588 vmxnet3_start(struct ifnet *ifp) 2589 { 2590 struct vmxnet3_softc *sc; 2591 struct vmxnet3_txqueue *txq; 2592 2593 sc = ifp->if_softc; 2594 txq = &sc->vmx_txq[0]; 2595 2596 VMXNET3_TXQ_LOCK(txq); 2597 vmxnet3_start_locked(ifp); 2598 VMXNET3_TXQ_UNLOCK(txq); 2599 } 2600 2601 static void 2602 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 2603 { 2604 struct ifnet *ifp; 2605 int idx, bit; 2606 2607 ifp = sc->vmx_ifp; 2608 idx = (tag >> 5) & 0x7F; 2609 bit = tag & 0x1F; 2610 2611 if (tag == 0 || tag > 4095) 2612 return; 2613 2614 VMXNET3_CORE_LOCK(sc); 2615 2616 /* Update our private VLAN bitvector. */ 2617 if (add) 2618 sc->vmx_vlan_filter[idx] |= (1 << bit); 2619 else 2620 sc->vmx_vlan_filter[idx] &= ~(1 << bit); 2621 2622 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 2623 if (add) 2624 sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 2625 else 2626 sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 2627 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2628 } 2629 2630 VMXNET3_CORE_UNLOCK(sc); 2631 } 2632 2633 static void 2634 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 2635 { 2636 2637 if (ifp->if_softc == arg) 2638 vmxnet3_update_vlan_filter(arg, 1, tag); 2639 } 2640 2641 static void 2642 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 2643 { 2644 2645 if (ifp->if_softc == arg) 2646 vmxnet3_update_vlan_filter(arg, 0, tag); 2647 } 2648 2649 static void 2650 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 2651 { 2652 struct ifnet *ifp; 2653 struct vmxnet3_driver_shared *ds; 2654 struct ifmultiaddr *ifma; 2655 u_int mode; 2656 2657 ifp = sc->vmx_ifp; 2658 ds = sc->vmx_ds; 2659 2660 mode = VMXNET3_RXMODE_UCAST; 2661 if (ifp->if_flags & IFF_BROADCAST) 2662 mode |= VMXNET3_RXMODE_BCAST; 2663 if (ifp->if_flags & IFF_PROMISC) 2664 mode |= VMXNET3_RXMODE_PROMISC; 2665 if (ifp->if_flags & IFF_ALLMULTI) 2666 mode |= VMXNET3_RXMODE_ALLMULTI; 2667 else { 2668 int cnt = 0, overflow = 0; 2669 2670 if_maddr_rlock(ifp); 2671 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2672 if (ifma->ifma_addr->sa_family != AF_LINK) 2673 continue; 2674 else if (cnt == VMXNET3_MULTICAST_MAX) { 2675 overflow = 1; 2676 break; 2677 } 2678 2679 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2680 &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 2681 cnt++; 2682 } 2683 if_maddr_runlock(ifp); 2684 2685 if (overflow != 0) { 2686 cnt = 0; 2687 mode |= VMXNET3_RXMODE_ALLMULTI; 2688 } else if (cnt > 0) 2689 mode |= VMXNET3_RXMODE_MCAST; 2690 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 2691 } 2692 2693 ds->rxmode = mode; 2694 2695 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 2696 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 2697 } 2698 2699 static int 2700 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 2701 { 2702 struct ifnet *ifp; 2703 2704 ifp = sc->vmx_ifp; 2705 2706 if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 2707 return (EINVAL); 2708 2709 ifp->if_mtu = mtu; 2710 2711 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 2712 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2713 vmxnet3_init_locked(sc); 2714 } 2715 2716 return (0); 2717 } 2718 2719 static int 2720 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 2721 { 2722 struct vmxnet3_softc *sc; 2723 struct ifreq *ifr; 2724 int reinit, mask, error; 2725 2726 sc = ifp->if_softc; 2727 ifr = (struct ifreq *) data; 2728 error = 0; 2729 2730 switch (cmd) { 2731 case SIOCSIFMTU: 2732 if (ifp->if_mtu != ifr->ifr_mtu) { 2733 VMXNET3_CORE_LOCK(sc); 2734 error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 2735 VMXNET3_CORE_UNLOCK(sc); 2736 } 2737 break; 2738 2739 case SIOCSIFFLAGS: 2740 VMXNET3_CORE_LOCK(sc); 2741 if (ifp->if_flags & IFF_UP) { 2742 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2743 if ((ifp->if_flags ^ sc->vmx_if_flags) & 2744 (IFF_PROMISC | IFF_ALLMULTI)) { 2745 vmxnet3_set_rxfilter(sc); 2746 } 2747 } else 2748 vmxnet3_init_locked(sc); 2749 } else { 2750 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2751 vmxnet3_stop(sc); 2752 } 2753 sc->vmx_if_flags = ifp->if_flags; 2754 VMXNET3_CORE_UNLOCK(sc); 2755 break; 2756 2757 case SIOCADDMULTI: 2758 case SIOCDELMULTI: 2759 VMXNET3_CORE_LOCK(sc); 2760 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2761 vmxnet3_set_rxfilter(sc); 2762 VMXNET3_CORE_UNLOCK(sc); 2763 break; 2764 2765 case SIOCSIFMEDIA: 2766 case SIOCGIFMEDIA: 2767 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 2768 break; 2769 2770 case SIOCSIFCAP: 2771 VMXNET3_CORE_LOCK(sc); 2772 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 2773 2774 if (mask & IFCAP_TXCSUM) 2775 ifp->if_capenable ^= IFCAP_TXCSUM; 2776 if (mask & IFCAP_TXCSUM_IPV6) 2777 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 2778 if (mask & IFCAP_TSO4) 2779 ifp->if_capenable ^= IFCAP_TSO4; 2780 if (mask & IFCAP_TSO6) 2781 ifp->if_capenable ^= IFCAP_TSO6; 2782 2783 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 2784 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) { 2785 /* Changing these features requires us to reinit. */ 2786 reinit = 1; 2787 2788 if (mask & IFCAP_RXCSUM) 2789 ifp->if_capenable ^= IFCAP_RXCSUM; 2790 if (mask & IFCAP_RXCSUM_IPV6) 2791 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 2792 if (mask & IFCAP_LRO) 2793 ifp->if_capenable ^= IFCAP_LRO; 2794 if (mask & IFCAP_VLAN_HWTAGGING) 2795 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 2796 if (mask & IFCAP_VLAN_HWFILTER) 2797 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 2798 } else 2799 reinit = 0; 2800 2801 if (mask & IFCAP_VLAN_HWTSO) 2802 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 2803 2804 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2805 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2806 vmxnet3_init_locked(sc); 2807 } 2808 2809 VMXNET3_CORE_UNLOCK(sc); 2810 VLAN_CAPABILITIES(ifp); 2811 break; 2812 2813 default: 2814 error = ether_ioctl(ifp, cmd, data); 2815 break; 2816 } 2817 2818 VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 2819 2820 return (error); 2821 } 2822 2823 static int 2824 vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 2825 { 2826 struct vmxnet3_softc *sc; 2827 2828 sc = txq->vxtxq_sc; 2829 2830 VMXNET3_TXQ_LOCK(txq); 2831 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 2832 VMXNET3_TXQ_UNLOCK(txq); 2833 return (0); 2834 } 2835 VMXNET3_TXQ_UNLOCK(txq); 2836 2837 if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 2838 txq->vxtxq_id); 2839 return (1); 2840 } 2841 2842 static void 2843 vmxnet3_refresh_stats(struct vmxnet3_softc *sc) 2844 { 2845 2846 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 2847 } 2848 2849 static void 2850 vmxnet3_tick(void *xsc) 2851 { 2852 struct vmxnet3_softc *sc; 2853 struct ifnet *ifp; 2854 int i, timedout; 2855 2856 sc = xsc; 2857 ifp = sc->vmx_ifp; 2858 timedout = 0; 2859 2860 VMXNET3_CORE_LOCK_ASSERT(sc); 2861 vmxnet3_refresh_stats(sc); 2862 2863 for (i = 0; i < sc->vmx_ntxqueues; i++) 2864 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 2865 2866 if (timedout != 0) { 2867 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2868 vmxnet3_init_locked(sc); 2869 } else 2870 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2871 } 2872 2873 static int 2874 vmxnet3_link_is_up(struct vmxnet3_softc *sc) 2875 { 2876 uint32_t status; 2877 2878 /* Also update the link speed while here. */ 2879 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 2880 sc->vmx_link_speed = status >> 16; 2881 return !!(status & 0x1); 2882 } 2883 2884 static void 2885 vmxnet3_link_status(struct vmxnet3_softc *sc) 2886 { 2887 struct ifnet *ifp; 2888 int link; 2889 2890 ifp = sc->vmx_ifp; 2891 link = vmxnet3_link_is_up(sc); 2892 2893 if (link != 0 && sc->vmx_link_active == 0) { 2894 sc->vmx_link_active = 1; 2895 if_link_state_change(ifp, LINK_STATE_UP); 2896 } else if (link == 0 && sc->vmx_link_active != 0) { 2897 sc->vmx_link_active = 0; 2898 if_link_state_change(ifp, LINK_STATE_DOWN); 2899 } 2900 } 2901 2902 static void 2903 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2904 { 2905 struct vmxnet3_softc *sc; 2906 2907 sc = ifp->if_softc; 2908 2909 ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 2910 ifmr->ifm_status = IFM_AVALID; 2911 2912 VMXNET3_CORE_LOCK(sc); 2913 if (vmxnet3_link_is_up(sc) != 0) 2914 ifmr->ifm_status |= IFM_ACTIVE; 2915 else 2916 ifmr->ifm_status |= IFM_NONE; 2917 VMXNET3_CORE_UNLOCK(sc); 2918 } 2919 2920 static int 2921 vmxnet3_media_change(struct ifnet *ifp) 2922 { 2923 2924 /* Ignore. */ 2925 return (0); 2926 } 2927 2928 static void 2929 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 2930 { 2931 uint32_t ml, mh; 2932 2933 ml = sc->vmx_lladdr[0]; 2934 ml |= sc->vmx_lladdr[1] << 8; 2935 ml |= sc->vmx_lladdr[2] << 16; 2936 ml |= sc->vmx_lladdr[3] << 24; 2937 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 2938 2939 mh = sc->vmx_lladdr[4]; 2940 mh |= sc->vmx_lladdr[5] << 8; 2941 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 2942 } 2943 2944 static void 2945 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 2946 { 2947 uint32_t ml, mh; 2948 2949 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 2950 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 2951 2952 sc->vmx_lladdr[0] = ml; 2953 sc->vmx_lladdr[1] = ml >> 8; 2954 sc->vmx_lladdr[2] = ml >> 16; 2955 sc->vmx_lladdr[3] = ml >> 24; 2956 sc->vmx_lladdr[4] = mh; 2957 sc->vmx_lladdr[5] = mh >> 8; 2958 } 2959 2960 static void 2961 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 2962 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 2963 { 2964 struct sysctl_oid *node, *txsnode; 2965 struct sysctl_oid_list *list, *txslist; 2966 struct vmxnet3_txq_stats *stats; 2967 struct UPT1_TxStats *txstats; 2968 char namebuf[16]; 2969 2970 stats = &txq->vxtxq_stats; 2971 txstats = &txq->vxtxq_ts->stats; 2972 2973 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 2974 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 2975 NULL, "Transmit Queue"); 2976 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 2977 2978 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 2979 &stats->vtxrs_full, "Tx ring full"); 2980 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 2981 &stats->vtxrs_offload_failed, "Tx checksum offload failed"); 2982 2983 /* 2984 * Add statistics reported by the host. These are updated once 2985 * per second. 2986 */ 2987 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 2988 NULL, "Host Statistics"); 2989 txslist = SYSCTL_CHILDREN(txsnode); 2990 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 2991 &txstats->TSO_packets, "TSO packets"); 2992 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 2993 &txstats->TSO_bytes, "TSO bytes"); 2994 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 2995 &txstats->ucast_packets, "Unicast packets"); 2996 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 2997 &txstats->ucast_bytes, "Unicast bytes"); 2998 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 2999 &txstats->mcast_packets, "Multicast packets"); 3000 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3001 &txstats->mcast_bytes, "Multicast bytes"); 3002 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3003 &txstats->error, "Errors"); 3004 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3005 &txstats->discard, "Discards"); 3006 } 3007 3008 static void 3009 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3010 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3011 { 3012 struct sysctl_oid *node, *rxsnode; 3013 struct sysctl_oid_list *list, *rxslist; 3014 struct vmxnet3_rxq_stats *stats; 3015 struct UPT1_RxStats *rxstats; 3016 char namebuf[16]; 3017 3018 stats = &rxq->vxrxq_stats; 3019 rxstats = &rxq->vxrxq_rs->stats; 3020 3021 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3022 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3023 NULL, "Receive Queue"); 3024 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3025 3026 /* 3027 * Add statistics reported by the host. These are updated once 3028 * per second. 3029 */ 3030 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3031 NULL, "Host Statistics"); 3032 rxslist = SYSCTL_CHILDREN(rxsnode); 3033 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3034 &rxstats->LRO_packets, "LRO packets"); 3035 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3036 &rxstats->LRO_bytes, "LRO bytes"); 3037 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3038 &rxstats->ucast_packets, "Unicast packets"); 3039 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3040 &rxstats->ucast_bytes, "Unicast bytes"); 3041 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3042 &rxstats->mcast_packets, "Multicast packets"); 3043 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3044 &rxstats->mcast_bytes, "Multicast bytes"); 3045 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3046 &rxstats->bcast_packets, "Broadcast packets"); 3047 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3048 &rxstats->bcast_bytes, "Broadcast bytes"); 3049 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3050 &rxstats->nobuffer, "No buffer"); 3051 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3052 &rxstats->error, "Errors"); 3053 } 3054 3055 #ifdef VMXNET3_DEBUG_SYSCTL 3056 static void 3057 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3058 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3059 { 3060 struct sysctl_oid *node; 3061 struct sysctl_oid_list *list; 3062 int i; 3063 3064 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3065 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3066 3067 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3068 "debug", CTLFLAG_RD, NULL, ""); 3069 list = SYSCTL_CHILDREN(node); 3070 3071 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3072 &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3073 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3074 &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3075 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3076 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3077 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3078 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3079 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3080 &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3081 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3082 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3083 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3084 &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3085 } 3086 3087 for (i = 0; i < sc->vmx_nrxqueues; i++) { 3088 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3089 3090 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3091 "debug", CTLFLAG_RD, NULL, ""); 3092 list = SYSCTL_CHILDREN(node); 3093 3094 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3095 &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3096 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3097 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3098 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3099 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3100 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3101 &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3102 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3103 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3104 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3105 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3106 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3107 &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3108 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3109 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3110 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3111 &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3112 } 3113 } 3114 #endif 3115 3116 static void 3117 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3118 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3119 { 3120 int i; 3121 3122 for (i = 0; i < sc->vmx_ntxqueues; i++) 3123 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3124 for (i = 0; i < sc->vmx_nrxqueues; i++) 3125 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3126 3127 #ifdef VMXNET3_DEBUG_SYSCTL 3128 vmxnet3_setup_debug_sysctl(sc, ctx, child); 3129 #endif 3130 } 3131 3132 static void 3133 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3134 { 3135 device_t dev; 3136 struct vmxnet3_statistics *stats; 3137 struct sysctl_ctx_list *ctx; 3138 struct sysctl_oid *tree; 3139 struct sysctl_oid_list *child; 3140 3141 dev = sc->vmx_dev; 3142 ctx = device_get_sysctl_ctx(dev); 3143 tree = device_get_sysctl_tree(dev); 3144 child = SYSCTL_CHILDREN(tree); 3145 3146 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3147 &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3148 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3149 &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3150 3151 stats = &sc->vmx_stats; 3152 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "collapsed", CTLFLAG_RD, 3153 &stats->vmst_collapsed, 0, "Tx mbuf chains collapsed"); 3154 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3155 &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3156 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3157 &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3158 3159 vmxnet3_setup_queue_sysctl(sc, ctx, child); 3160 } 3161 3162 static void 3163 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3164 { 3165 3166 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3167 } 3168 3169 static uint32_t 3170 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3171 { 3172 3173 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3174 } 3175 3176 static void 3177 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3178 { 3179 3180 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3181 } 3182 3183 static void 3184 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3185 { 3186 3187 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3188 } 3189 3190 static uint32_t 3191 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3192 { 3193 3194 vmxnet3_write_cmd(sc, cmd); 3195 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3196 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3197 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3198 } 3199 3200 static void 3201 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3202 { 3203 3204 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3205 } 3206 3207 static void 3208 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3209 { 3210 3211 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3212 } 3213 3214 static void 3215 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3216 { 3217 int i; 3218 3219 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3220 for (i = 0; i < sc->vmx_nintrs; i++) 3221 vmxnet3_enable_intr(sc, i); 3222 } 3223 3224 static void 3225 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3226 { 3227 int i; 3228 3229 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3230 for (i = 0; i < sc->vmx_nintrs; i++) 3231 vmxnet3_disable_intr(sc, i); 3232 } 3233 3234 static void 3235 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3236 { 3237 bus_addr_t *baddr = arg; 3238 3239 if (error == 0) 3240 *baddr = segs->ds_addr; 3241 } 3242 3243 static int 3244 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3245 struct vmxnet3_dma_alloc *dma) 3246 { 3247 device_t dev; 3248 int error; 3249 3250 dev = sc->vmx_dev; 3251 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3252 3253 error = bus_dma_tag_create(bus_get_dma_tag(dev), 3254 align, 0, /* alignment, bounds */ 3255 BUS_SPACE_MAXADDR, /* lowaddr */ 3256 BUS_SPACE_MAXADDR, /* highaddr */ 3257 NULL, NULL, /* filter, filterarg */ 3258 size, /* maxsize */ 3259 1, /* nsegments */ 3260 size, /* maxsegsize */ 3261 BUS_DMA_ALLOCNOW, /* flags */ 3262 NULL, /* lockfunc */ 3263 NULL, /* lockfuncarg */ 3264 &dma->dma_tag); 3265 if (error) { 3266 device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3267 goto fail; 3268 } 3269 3270 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3271 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3272 if (error) { 3273 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3274 goto fail; 3275 } 3276 3277 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3278 size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3279 if (error) { 3280 device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3281 goto fail; 3282 } 3283 3284 dma->dma_size = size; 3285 3286 fail: 3287 if (error) 3288 vmxnet3_dma_free(sc, dma); 3289 3290 return (error); 3291 } 3292 3293 static void 3294 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3295 { 3296 3297 if (dma->dma_tag != NULL) { 3298 if (dma->dma_map != NULL) { 3299 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3300 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3301 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3302 } 3303 3304 if (dma->dma_vaddr != NULL) { 3305 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 3306 dma->dma_map); 3307 } 3308 3309 bus_dma_tag_destroy(dma->dma_tag); 3310 } 3311 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3312 } 3313 3314 static int 3315 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 3316 { 3317 char path[64]; 3318 3319 snprintf(path, sizeof(path), 3320 "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 3321 TUNABLE_INT_FETCH(path, &def); 3322 3323 return (def); 3324 } 3325 3326 /* 3327 * Since this is a purely paravirtualized device, we do not have 3328 * to worry about DMA coherency. But at times, we must make sure 3329 * both the compiler and CPU do not reorder memory operations. 3330 */ 3331 static inline void 3332 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 3333 { 3334 3335 switch (type) { 3336 case VMXNET3_BARRIER_RD: 3337 rmb(); 3338 break; 3339 case VMXNET3_BARRIER_WR: 3340 wmb(); 3341 break; 3342 case VMXNET3_BARRIER_RDWR: 3343 mb(); 3344 break; 3345 default: 3346 panic("%s: bad barrier type %d", __func__, type); 3347 } 3348 } 3349