1 /*- 2 * Copyright (c) 2013 Tsubai Masanari 3 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 * 17 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18 */ 19 20 /* Driver for VMware vmxnet3 virtual ethernet devices. */ 21 22 #include <sys/cdefs.h> 23 __FBSDID("$FreeBSD$"); 24 25 #include <sys/param.h> 26 #include <sys/systm.h> 27 #include <sys/eventhandler.h> 28 #include <sys/kernel.h> 29 #include <sys/endian.h> 30 #include <sys/sockio.h> 31 #include <sys/mbuf.h> 32 #include <sys/malloc.h> 33 #include <sys/module.h> 34 #include <sys/socket.h> 35 #include <sys/sysctl.h> 36 #include <sys/smp.h> 37 #include <sys/taskqueue.h> 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <net/ethernet.h> 42 #include <net/if.h> 43 #include <net/if_var.h> 44 #include <net/if_arp.h> 45 #include <net/if_dl.h> 46 #include <net/if_types.h> 47 #include <net/if_media.h> 48 #include <net/if_vlan_var.h> 49 50 #include <net/bpf.h> 51 52 #include <netinet/in_systm.h> 53 #include <netinet/in.h> 54 #include <netinet/ip.h> 55 #include <netinet/ip6.h> 56 #include <netinet6/ip6_var.h> 57 #include <netinet/udp.h> 58 #include <netinet/tcp.h> 59 60 #include <machine/in_cksum.h> 61 62 #include <machine/bus.h> 63 #include <machine/resource.h> 64 #include <sys/bus.h> 65 #include <sys/rman.h> 66 67 #include <dev/pci/pcireg.h> 68 #include <dev/pci/pcivar.h> 69 70 #include "if_vmxreg.h" 71 #include "if_vmxvar.h" 72 73 #include "opt_inet.h" 74 #include "opt_inet6.h" 75 76 #ifdef VMXNET3_FAILPOINTS 77 #include <sys/fail.h> 78 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 79 "vmxnet3 fail points"); 80 #define VMXNET3_FP _debug_fail_point_vmxnet3 81 #endif 82 83 static int vmxnet3_probe(device_t); 84 static int vmxnet3_attach(device_t); 85 static int vmxnet3_detach(device_t); 86 static int vmxnet3_shutdown(device_t); 87 88 static int vmxnet3_alloc_resources(struct vmxnet3_softc *); 89 static void vmxnet3_free_resources(struct vmxnet3_softc *); 90 static int vmxnet3_check_version(struct vmxnet3_softc *); 91 static void vmxnet3_initial_config(struct vmxnet3_softc *); 92 static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 93 94 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 95 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 96 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 97 static int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 98 struct vmxnet3_interrupt *); 99 static int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 100 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 101 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 102 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 103 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 104 105 static void vmxnet3_free_interrupt(struct vmxnet3_softc *, 106 struct vmxnet3_interrupt *); 107 static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 108 109 #ifndef VMXNET3_LEGACY_TX 110 static int vmxnet3_alloc_taskqueue(struct vmxnet3_softc *); 111 static void vmxnet3_start_taskqueue(struct vmxnet3_softc *); 112 static void vmxnet3_drain_taskqueue(struct vmxnet3_softc *); 113 static void vmxnet3_free_taskqueue(struct vmxnet3_softc *); 114 #endif 115 116 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 117 static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 118 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 119 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 120 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 121 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 122 123 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 124 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 125 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 126 static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 127 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 128 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 129 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 130 static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 131 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 132 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 133 static void vmxnet3_reinit_interface(struct vmxnet3_softc *); 134 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 135 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 136 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 137 static void vmxnet3_free_data(struct vmxnet3_softc *); 138 static int vmxnet3_setup_interface(struct vmxnet3_softc *); 139 140 static void vmxnet3_evintr(struct vmxnet3_softc *); 141 static void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 142 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 143 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 144 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 145 struct vmxnet3_rxring *, int); 146 static void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 147 static void vmxnet3_legacy_intr(void *); 148 static void vmxnet3_txq_intr(void *); 149 static void vmxnet3_rxq_intr(void *); 150 static void vmxnet3_event_intr(void *); 151 152 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 153 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 154 static void vmxnet3_stop(struct vmxnet3_softc *); 155 156 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 157 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 158 static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 159 static int vmxnet3_enable_device(struct vmxnet3_softc *); 160 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 161 static int vmxnet3_reinit(struct vmxnet3_softc *); 162 static void vmxnet3_init_locked(struct vmxnet3_softc *); 163 static void vmxnet3_init(void *); 164 165 static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *, 166 int *, int *, int *); 167 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 168 bus_dmamap_t, bus_dma_segment_t [], int *); 169 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 170 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 171 #ifdef VMXNET3_LEGACY_TX 172 static void vmxnet3_start_locked(struct ifnet *); 173 static void vmxnet3_start(struct ifnet *); 174 #else 175 static int vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *, 176 struct mbuf *); 177 static int vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *); 178 static void vmxnet3_txq_tq_deferred(void *, int); 179 #endif 180 static void vmxnet3_txq_start(struct vmxnet3_txqueue *); 181 static void vmxnet3_tx_start_all(struct vmxnet3_softc *); 182 183 static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 184 uint16_t); 185 static void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 186 static void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 187 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 188 static int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 189 static int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 190 static uint64_t vmxnet3_get_counter(struct ifnet *, ift_counter); 191 192 #ifndef VMXNET3_LEGACY_TX 193 static void vmxnet3_qflush(struct ifnet *); 194 #endif 195 196 static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 197 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 198 static void vmxnet3_tick(void *); 199 static void vmxnet3_link_status(struct vmxnet3_softc *); 200 static void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 201 static int vmxnet3_media_change(struct ifnet *); 202 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 203 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 204 205 static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 206 struct sysctl_ctx_list *, struct sysctl_oid_list *); 207 static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 208 struct sysctl_ctx_list *, struct sysctl_oid_list *); 209 static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 210 struct sysctl_ctx_list *, struct sysctl_oid_list *); 211 static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 212 213 static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 214 uint32_t); 215 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 216 static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 217 uint32_t); 218 static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 219 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 220 221 static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 222 static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 223 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 224 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 225 226 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 227 bus_size_t, struct vmxnet3_dma_alloc *); 228 static void vmxnet3_dma_free(struct vmxnet3_softc *, 229 struct vmxnet3_dma_alloc *); 230 static int vmxnet3_tunable_int(struct vmxnet3_softc *, 231 const char *, int); 232 233 typedef enum { 234 VMXNET3_BARRIER_RD, 235 VMXNET3_BARRIER_WR, 236 VMXNET3_BARRIER_RDWR, 237 } vmxnet3_barrier_t; 238 239 static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 240 241 /* Tunables. */ 242 static int vmxnet3_mq_disable = 0; 243 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable); 244 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES; 245 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue); 246 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES; 247 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue); 248 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 249 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 250 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 251 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 252 253 static device_method_t vmxnet3_methods[] = { 254 /* Device interface. */ 255 DEVMETHOD(device_probe, vmxnet3_probe), 256 DEVMETHOD(device_attach, vmxnet3_attach), 257 DEVMETHOD(device_detach, vmxnet3_detach), 258 DEVMETHOD(device_shutdown, vmxnet3_shutdown), 259 260 DEVMETHOD_END 261 }; 262 263 static driver_t vmxnet3_driver = { 264 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 265 }; 266 267 static devclass_t vmxnet3_devclass; 268 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 269 270 MODULE_DEPEND(vmx, pci, 1, 1, 1); 271 MODULE_DEPEND(vmx, ether, 1, 1, 1); 272 273 #define VMXNET3_VMWARE_VENDOR_ID 0x15AD 274 #define VMXNET3_VMWARE_DEVICE_ID 0x07B0 275 276 static int 277 vmxnet3_probe(device_t dev) 278 { 279 280 if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 281 pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 282 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 283 return (BUS_PROBE_DEFAULT); 284 } 285 286 return (ENXIO); 287 } 288 289 static int 290 vmxnet3_attach(device_t dev) 291 { 292 struct vmxnet3_softc *sc; 293 int error; 294 295 sc = device_get_softc(dev); 296 sc->vmx_dev = dev; 297 298 pci_enable_busmaster(dev); 299 300 VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 301 callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); 302 303 vmxnet3_initial_config(sc); 304 305 error = vmxnet3_alloc_resources(sc); 306 if (error) 307 goto fail; 308 309 error = vmxnet3_check_version(sc); 310 if (error) 311 goto fail; 312 313 error = vmxnet3_alloc_rxtx_queues(sc); 314 if (error) 315 goto fail; 316 317 #ifndef VMXNET3_LEGACY_TX 318 error = vmxnet3_alloc_taskqueue(sc); 319 if (error) 320 goto fail; 321 #endif 322 323 error = vmxnet3_alloc_interrupts(sc); 324 if (error) 325 goto fail; 326 327 vmxnet3_check_multiqueue(sc); 328 329 error = vmxnet3_alloc_data(sc); 330 if (error) 331 goto fail; 332 333 error = vmxnet3_setup_interface(sc); 334 if (error) 335 goto fail; 336 337 error = vmxnet3_setup_interrupts(sc); 338 if (error) { 339 ether_ifdetach(sc->vmx_ifp); 340 device_printf(dev, "could not set up interrupt\n"); 341 goto fail; 342 } 343 344 vmxnet3_setup_sysctl(sc); 345 #ifndef VMXNET3_LEGACY_TX 346 vmxnet3_start_taskqueue(sc); 347 #endif 348 349 fail: 350 if (error) 351 vmxnet3_detach(dev); 352 353 return (error); 354 } 355 356 static int 357 vmxnet3_detach(device_t dev) 358 { 359 struct vmxnet3_softc *sc; 360 struct ifnet *ifp; 361 362 sc = device_get_softc(dev); 363 ifp = sc->vmx_ifp; 364 365 if (device_is_attached(dev)) { 366 VMXNET3_CORE_LOCK(sc); 367 vmxnet3_stop(sc); 368 VMXNET3_CORE_UNLOCK(sc); 369 370 callout_drain(&sc->vmx_tick); 371 #ifndef VMXNET3_LEGACY_TX 372 vmxnet3_drain_taskqueue(sc); 373 #endif 374 375 ether_ifdetach(ifp); 376 } 377 378 if (sc->vmx_vlan_attach != NULL) { 379 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 380 sc->vmx_vlan_attach = NULL; 381 } 382 if (sc->vmx_vlan_detach != NULL) { 383 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 384 sc->vmx_vlan_detach = NULL; 385 } 386 387 #ifndef VMXNET3_LEGACY_TX 388 vmxnet3_free_taskqueue(sc); 389 #endif 390 vmxnet3_free_interrupts(sc); 391 392 if (ifp != NULL) { 393 if_free(ifp); 394 sc->vmx_ifp = NULL; 395 } 396 397 ifmedia_removeall(&sc->vmx_media); 398 399 vmxnet3_free_data(sc); 400 vmxnet3_free_resources(sc); 401 vmxnet3_free_rxtx_queues(sc); 402 403 VMXNET3_CORE_LOCK_DESTROY(sc); 404 405 return (0); 406 } 407 408 static int 409 vmxnet3_shutdown(device_t dev) 410 { 411 412 return (0); 413 } 414 415 static int 416 vmxnet3_alloc_resources(struct vmxnet3_softc *sc) 417 { 418 device_t dev; 419 int rid; 420 421 dev = sc->vmx_dev; 422 423 rid = PCIR_BAR(0); 424 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 425 RF_ACTIVE); 426 if (sc->vmx_res0 == NULL) { 427 device_printf(dev, 428 "could not map BAR0 memory\n"); 429 return (ENXIO); 430 } 431 432 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 433 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 434 435 rid = PCIR_BAR(1); 436 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 437 RF_ACTIVE); 438 if (sc->vmx_res1 == NULL) { 439 device_printf(dev, 440 "could not map BAR1 memory\n"); 441 return (ENXIO); 442 } 443 444 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 445 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 446 447 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 448 rid = PCIR_BAR(2); 449 sc->vmx_msix_res = bus_alloc_resource_any(dev, 450 SYS_RES_MEMORY, &rid, RF_ACTIVE); 451 } 452 453 if (sc->vmx_msix_res == NULL) 454 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 455 456 return (0); 457 } 458 459 static void 460 vmxnet3_free_resources(struct vmxnet3_softc *sc) 461 { 462 device_t dev; 463 int rid; 464 465 dev = sc->vmx_dev; 466 467 if (sc->vmx_res0 != NULL) { 468 rid = PCIR_BAR(0); 469 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 470 sc->vmx_res0 = NULL; 471 } 472 473 if (sc->vmx_res1 != NULL) { 474 rid = PCIR_BAR(1); 475 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 476 sc->vmx_res1 = NULL; 477 } 478 479 if (sc->vmx_msix_res != NULL) { 480 rid = PCIR_BAR(2); 481 bus_release_resource(dev, SYS_RES_MEMORY, rid, 482 sc->vmx_msix_res); 483 sc->vmx_msix_res = NULL; 484 } 485 } 486 487 static int 488 vmxnet3_check_version(struct vmxnet3_softc *sc) 489 { 490 device_t dev; 491 uint32_t version; 492 493 dev = sc->vmx_dev; 494 495 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 496 if ((version & 0x01) == 0) { 497 device_printf(dev, "unsupported hardware version %#x\n", 498 version); 499 return (ENOTSUP); 500 } 501 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 502 503 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 504 if ((version & 0x01) == 0) { 505 device_printf(dev, "unsupported UPT version %#x\n", version); 506 return (ENOTSUP); 507 } 508 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 509 510 return (0); 511 } 512 513 static void 514 vmxnet3_initial_config(struct vmxnet3_softc *sc) 515 { 516 int nqueue, ndesc; 517 518 nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue); 519 if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1) 520 nqueue = VMXNET3_DEF_TX_QUEUES; 521 if (nqueue > mp_ncpus) 522 nqueue = mp_ncpus; 523 sc->vmx_max_ntxqueues = nqueue; 524 525 nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue); 526 if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1) 527 nqueue = VMXNET3_DEF_RX_QUEUES; 528 if (nqueue > mp_ncpus) 529 nqueue = mp_ncpus; 530 sc->vmx_max_nrxqueues = nqueue; 531 532 if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) { 533 sc->vmx_max_nrxqueues = 1; 534 sc->vmx_max_ntxqueues = 1; 535 } 536 537 ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 538 if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 539 ndesc = VMXNET3_DEF_TX_NDESC; 540 if (ndesc & VMXNET3_MASK_TX_NDESC) 541 ndesc &= ~VMXNET3_MASK_TX_NDESC; 542 sc->vmx_ntxdescs = ndesc; 543 544 ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 545 if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 546 ndesc = VMXNET3_DEF_RX_NDESC; 547 if (ndesc & VMXNET3_MASK_RX_NDESC) 548 ndesc &= ~VMXNET3_MASK_RX_NDESC; 549 sc->vmx_nrxdescs = ndesc; 550 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 551 } 552 553 static void 554 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 555 { 556 557 if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 558 goto out; 559 560 /* BMV: Just use the maximum configured for now. */ 561 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 562 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 563 564 if (sc->vmx_nrxqueues > 1) 565 sc->vmx_flags |= VMXNET3_FLAG_RSS; 566 567 return; 568 569 out: 570 sc->vmx_ntxqueues = 1; 571 sc->vmx_nrxqueues = 1; 572 } 573 574 static int 575 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 576 { 577 device_t dev; 578 int nmsix, cnt, required; 579 580 dev = sc->vmx_dev; 581 582 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 583 return (1); 584 585 /* Allocate an additional vector for the events interrupt. */ 586 required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; 587 588 nmsix = pci_msix_count(dev); 589 if (nmsix < required) 590 return (1); 591 592 cnt = required; 593 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 594 sc->vmx_nintrs = required; 595 return (0); 596 } else 597 pci_release_msi(dev); 598 599 /* BMV TODO Fallback to sharing MSIX vectors if possible. */ 600 601 return (1); 602 } 603 604 static int 605 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 606 { 607 device_t dev; 608 int nmsi, cnt, required; 609 610 dev = sc->vmx_dev; 611 required = 1; 612 613 nmsi = pci_msi_count(dev); 614 if (nmsi < required) 615 return (1); 616 617 cnt = required; 618 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 619 sc->vmx_nintrs = 1; 620 return (0); 621 } else 622 pci_release_msi(dev); 623 624 return (1); 625 } 626 627 static int 628 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 629 { 630 631 sc->vmx_nintrs = 1; 632 return (0); 633 } 634 635 static int 636 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 637 struct vmxnet3_interrupt *intr) 638 { 639 struct resource *irq; 640 641 irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); 642 if (irq == NULL) 643 return (ENXIO); 644 645 intr->vmxi_irq = irq; 646 intr->vmxi_rid = rid; 647 648 return (0); 649 } 650 651 static int 652 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 653 { 654 int i, rid, flags, error; 655 656 rid = 0; 657 flags = RF_ACTIVE; 658 659 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 660 flags |= RF_SHAREABLE; 661 else 662 rid = 1; 663 664 for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 665 error = vmxnet3_alloc_interrupt(sc, rid, flags, 666 &sc->vmx_intrs[i]); 667 if (error) 668 return (error); 669 } 670 671 return (0); 672 } 673 674 static int 675 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 676 { 677 device_t dev; 678 struct vmxnet3_txqueue *txq; 679 struct vmxnet3_rxqueue *rxq; 680 struct vmxnet3_interrupt *intr; 681 enum intr_type type; 682 int i, error; 683 684 dev = sc->vmx_dev; 685 intr = &sc->vmx_intrs[0]; 686 type = INTR_TYPE_NET | INTR_MPSAFE; 687 688 for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 689 txq = &sc->vmx_txq[i]; 690 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 691 vmxnet3_txq_intr, txq, &intr->vmxi_handler); 692 if (error) 693 return (error); 694 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 695 "tq%d", i); 696 txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 697 } 698 699 for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 700 rxq = &sc->vmx_rxq[i]; 701 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 702 vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); 703 if (error) 704 return (error); 705 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 706 "rq%d", i); 707 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 708 } 709 710 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 711 vmxnet3_event_intr, sc, &intr->vmxi_handler); 712 if (error) 713 return (error); 714 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event"); 715 sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 716 717 return (0); 718 } 719 720 static int 721 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 722 { 723 struct vmxnet3_interrupt *intr; 724 int i, error; 725 726 intr = &sc->vmx_intrs[0]; 727 error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 728 INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, 729 &intr->vmxi_handler); 730 731 for (i = 0; i < sc->vmx_ntxqueues; i++) 732 sc->vmx_txq[i].vxtxq_intr_idx = 0; 733 for (i = 0; i < sc->vmx_nrxqueues; i++) 734 sc->vmx_rxq[i].vxrxq_intr_idx = 0; 735 sc->vmx_event_intr_idx = 0; 736 737 return (error); 738 } 739 740 static void 741 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 742 { 743 struct vmxnet3_txqueue *txq; 744 struct vmxnet3_txq_shared *txs; 745 struct vmxnet3_rxqueue *rxq; 746 struct vmxnet3_rxq_shared *rxs; 747 int i; 748 749 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 750 751 for (i = 0; i < sc->vmx_ntxqueues; i++) { 752 txq = &sc->vmx_txq[i]; 753 txs = txq->vxtxq_ts; 754 txs->intr_idx = txq->vxtxq_intr_idx; 755 } 756 757 for (i = 0; i < sc->vmx_nrxqueues; i++) { 758 rxq = &sc->vmx_rxq[i]; 759 rxs = rxq->vxrxq_rs; 760 rxs->intr_idx = rxq->vxrxq_intr_idx; 761 } 762 } 763 764 static int 765 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 766 { 767 int error; 768 769 error = vmxnet3_alloc_intr_resources(sc); 770 if (error) 771 return (error); 772 773 switch (sc->vmx_intr_type) { 774 case VMXNET3_IT_MSIX: 775 error = vmxnet3_setup_msix_interrupts(sc); 776 break; 777 case VMXNET3_IT_MSI: 778 case VMXNET3_IT_LEGACY: 779 error = vmxnet3_setup_legacy_interrupt(sc); 780 break; 781 default: 782 panic("%s: invalid interrupt type %d", __func__, 783 sc->vmx_intr_type); 784 } 785 786 if (error == 0) 787 vmxnet3_set_interrupt_idx(sc); 788 789 return (error); 790 } 791 792 static int 793 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 794 { 795 device_t dev; 796 uint32_t config; 797 int error; 798 799 dev = sc->vmx_dev; 800 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 801 802 sc->vmx_intr_type = config & 0x03; 803 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 804 805 switch (sc->vmx_intr_type) { 806 case VMXNET3_IT_AUTO: 807 sc->vmx_intr_type = VMXNET3_IT_MSIX; 808 /* FALLTHROUGH */ 809 case VMXNET3_IT_MSIX: 810 error = vmxnet3_alloc_msix_interrupts(sc); 811 if (error == 0) 812 break; 813 sc->vmx_intr_type = VMXNET3_IT_MSI; 814 /* FALLTHROUGH */ 815 case VMXNET3_IT_MSI: 816 error = vmxnet3_alloc_msi_interrupts(sc); 817 if (error == 0) 818 break; 819 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 820 /* FALLTHROUGH */ 821 case VMXNET3_IT_LEGACY: 822 error = vmxnet3_alloc_legacy_interrupts(sc); 823 if (error == 0) 824 break; 825 /* FALLTHROUGH */ 826 default: 827 sc->vmx_intr_type = -1; 828 device_printf(dev, "cannot allocate any interrupt resources\n"); 829 return (ENXIO); 830 } 831 832 return (error); 833 } 834 835 static void 836 vmxnet3_free_interrupt(struct vmxnet3_softc *sc, 837 struct vmxnet3_interrupt *intr) 838 { 839 device_t dev; 840 841 dev = sc->vmx_dev; 842 843 if (intr->vmxi_handler != NULL) { 844 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 845 intr->vmxi_handler = NULL; 846 } 847 848 if (intr->vmxi_irq != NULL) { 849 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 850 intr->vmxi_irq); 851 intr->vmxi_irq = NULL; 852 intr->vmxi_rid = -1; 853 } 854 } 855 856 static void 857 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 858 { 859 int i; 860 861 for (i = 0; i < sc->vmx_nintrs; i++) 862 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 863 864 if (sc->vmx_intr_type == VMXNET3_IT_MSI || 865 sc->vmx_intr_type == VMXNET3_IT_MSIX) 866 pci_release_msi(sc->vmx_dev); 867 } 868 869 #ifndef VMXNET3_LEGACY_TX 870 static int 871 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc) 872 { 873 device_t dev; 874 875 dev = sc->vmx_dev; 876 877 sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT, 878 taskqueue_thread_enqueue, &sc->vmx_tq); 879 if (sc->vmx_tq == NULL) 880 return (ENOMEM); 881 882 return (0); 883 } 884 885 static void 886 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc) 887 { 888 device_t dev; 889 int nthreads, error; 890 891 dev = sc->vmx_dev; 892 893 /* 894 * The taskqueue is typically not frequently used, so a dedicated 895 * thread for each queue is unnecessary. 896 */ 897 nthreads = MAX(1, sc->vmx_ntxqueues / 2); 898 899 /* 900 * Most drivers just ignore the return value - it only fails 901 * with ENOMEM so an error is not likely. It is hard for us 902 * to recover from an error here. 903 */ 904 error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET, 905 "%s taskq", device_get_nameunit(dev)); 906 if (error) 907 device_printf(dev, "failed to start taskqueue: %d", error); 908 } 909 910 static void 911 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc) 912 { 913 struct vmxnet3_txqueue *txq; 914 int i; 915 916 if (sc->vmx_tq != NULL) { 917 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 918 txq = &sc->vmx_txq[i]; 919 taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask); 920 } 921 } 922 } 923 924 static void 925 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc) 926 { 927 if (sc->vmx_tq != NULL) { 928 taskqueue_free(sc->vmx_tq); 929 sc->vmx_tq = NULL; 930 } 931 } 932 #endif 933 934 static int 935 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 936 { 937 struct vmxnet3_rxqueue *rxq; 938 struct vmxnet3_rxring *rxr; 939 int i; 940 941 rxq = &sc->vmx_rxq[q]; 942 943 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 944 device_get_nameunit(sc->vmx_dev), q); 945 mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); 946 947 rxq->vxrxq_sc = sc; 948 rxq->vxrxq_id = q; 949 950 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 951 rxr = &rxq->vxrxq_cmd_ring[i]; 952 rxr->vxrxr_rid = i; 953 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 954 rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * 955 sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 956 if (rxr->vxrxr_rxbuf == NULL) 957 return (ENOMEM); 958 959 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 960 } 961 962 return (0); 963 } 964 965 static int 966 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 967 { 968 struct vmxnet3_txqueue *txq; 969 struct vmxnet3_txring *txr; 970 971 txq = &sc->vmx_txq[q]; 972 txr = &txq->vxtxq_cmd_ring; 973 974 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 975 device_get_nameunit(sc->vmx_dev), q); 976 mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); 977 978 txq->vxtxq_sc = sc; 979 txq->vxtxq_id = q; 980 981 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 982 txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * 983 sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 984 if (txr->vxtxr_txbuf == NULL) 985 return (ENOMEM); 986 987 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 988 989 #ifndef VMXNET3_LEGACY_TX 990 TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq); 991 992 txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF, 993 M_NOWAIT, &txq->vxtxq_mtx); 994 if (txq->vxtxq_br == NULL) 995 return (ENOMEM); 996 #endif 997 998 return (0); 999 } 1000 1001 static int 1002 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1003 { 1004 int i, error; 1005 1006 /* 1007 * Only attempt to create multiple queues if MSIX is available. MSIX is 1008 * disabled by default because its apparently broken for devices passed 1009 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable 1010 * must be set to zero for MSIX. This check prevents us from allocating 1011 * queue structures that we will not use. 1012 */ 1013 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1014 sc->vmx_max_nrxqueues = 1; 1015 sc->vmx_max_ntxqueues = 1; 1016 } 1017 1018 sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 1019 sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1020 sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 1021 sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1022 if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 1023 return (ENOMEM); 1024 1025 for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1026 error = vmxnet3_init_rxq(sc, i); 1027 if (error) 1028 return (error); 1029 } 1030 1031 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1032 error = vmxnet3_init_txq(sc, i); 1033 if (error) 1034 return (error); 1035 } 1036 1037 return (0); 1038 } 1039 1040 static void 1041 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1042 { 1043 struct vmxnet3_rxring *rxr; 1044 int i; 1045 1046 rxq->vxrxq_sc = NULL; 1047 rxq->vxrxq_id = -1; 1048 1049 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1050 rxr = &rxq->vxrxq_cmd_ring[i]; 1051 1052 if (rxr->vxrxr_rxbuf != NULL) { 1053 free(rxr->vxrxr_rxbuf, M_DEVBUF); 1054 rxr->vxrxr_rxbuf = NULL; 1055 } 1056 } 1057 1058 if (mtx_initialized(&rxq->vxrxq_mtx) != 0) 1059 mtx_destroy(&rxq->vxrxq_mtx); 1060 } 1061 1062 static void 1063 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1064 { 1065 struct vmxnet3_txring *txr; 1066 1067 txr = &txq->vxtxq_cmd_ring; 1068 1069 txq->vxtxq_sc = NULL; 1070 txq->vxtxq_id = -1; 1071 1072 #ifndef VMXNET3_LEGACY_TX 1073 if (txq->vxtxq_br != NULL) { 1074 buf_ring_free(txq->vxtxq_br, M_DEVBUF); 1075 txq->vxtxq_br = NULL; 1076 } 1077 #endif 1078 1079 if (txr->vxtxr_txbuf != NULL) { 1080 free(txr->vxtxr_txbuf, M_DEVBUF); 1081 txr->vxtxr_txbuf = NULL; 1082 } 1083 1084 if (mtx_initialized(&txq->vxtxq_mtx) != 0) 1085 mtx_destroy(&txq->vxtxq_mtx); 1086 } 1087 1088 static void 1089 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1090 { 1091 int i; 1092 1093 if (sc->vmx_rxq != NULL) { 1094 for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1095 vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 1096 free(sc->vmx_rxq, M_DEVBUF); 1097 sc->vmx_rxq = NULL; 1098 } 1099 1100 if (sc->vmx_txq != NULL) { 1101 for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1102 vmxnet3_destroy_txq(&sc->vmx_txq[i]); 1103 free(sc->vmx_txq, M_DEVBUF); 1104 sc->vmx_txq = NULL; 1105 } 1106 } 1107 1108 static int 1109 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1110 { 1111 device_t dev; 1112 uint8_t *kva; 1113 size_t size; 1114 int i, error; 1115 1116 dev = sc->vmx_dev; 1117 1118 size = sizeof(struct vmxnet3_driver_shared); 1119 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1120 if (error) { 1121 device_printf(dev, "cannot alloc shared memory\n"); 1122 return (error); 1123 } 1124 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1125 1126 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1127 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1128 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1129 if (error) { 1130 device_printf(dev, "cannot alloc queue shared memory\n"); 1131 return (error); 1132 } 1133 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1134 kva = sc->vmx_qs; 1135 1136 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1137 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 1138 kva += sizeof(struct vmxnet3_txq_shared); 1139 } 1140 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1141 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 1142 kva += sizeof(struct vmxnet3_rxq_shared); 1143 } 1144 1145 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1146 size = sizeof(struct vmxnet3_rss_shared); 1147 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1148 if (error) { 1149 device_printf(dev, "cannot alloc rss shared memory\n"); 1150 return (error); 1151 } 1152 sc->vmx_rss = 1153 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1154 } 1155 1156 return (0); 1157 } 1158 1159 static void 1160 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1161 { 1162 1163 if (sc->vmx_rss != NULL) { 1164 vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1165 sc->vmx_rss = NULL; 1166 } 1167 1168 if (sc->vmx_qs != NULL) { 1169 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1170 sc->vmx_qs = NULL; 1171 } 1172 1173 if (sc->vmx_ds != NULL) { 1174 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1175 sc->vmx_ds = NULL; 1176 } 1177 } 1178 1179 static int 1180 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1181 { 1182 device_t dev; 1183 struct vmxnet3_txqueue *txq; 1184 struct vmxnet3_txring *txr; 1185 struct vmxnet3_comp_ring *txc; 1186 size_t descsz, compsz; 1187 int i, q, error; 1188 1189 dev = sc->vmx_dev; 1190 1191 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1192 txq = &sc->vmx_txq[q]; 1193 txr = &txq->vxtxq_cmd_ring; 1194 txc = &txq->vxtxq_comp_ring; 1195 1196 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1197 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1198 1199 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1200 1, 0, /* alignment, boundary */ 1201 BUS_SPACE_MAXADDR, /* lowaddr */ 1202 BUS_SPACE_MAXADDR, /* highaddr */ 1203 NULL, NULL, /* filter, filterarg */ 1204 VMXNET3_TX_MAXSIZE, /* maxsize */ 1205 VMXNET3_TX_MAXSEGS, /* nsegments */ 1206 VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1207 0, /* flags */ 1208 NULL, NULL, /* lockfunc, lockarg */ 1209 &txr->vxtxr_txtag); 1210 if (error) { 1211 device_printf(dev, 1212 "unable to create Tx buffer tag for queue %d\n", q); 1213 return (error); 1214 } 1215 1216 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1217 if (error) { 1218 device_printf(dev, "cannot alloc Tx descriptors for " 1219 "queue %d error %d\n", q, error); 1220 return (error); 1221 } 1222 txr->vxtxr_txd = 1223 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1224 1225 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1226 if (error) { 1227 device_printf(dev, "cannot alloc Tx comp descriptors " 1228 "for queue %d error %d\n", q, error); 1229 return (error); 1230 } 1231 txc->vxcr_u.txcd = 1232 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1233 1234 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1235 error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1236 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1237 if (error) { 1238 device_printf(dev, "unable to create Tx buf " 1239 "dmamap for queue %d idx %d\n", q, i); 1240 return (error); 1241 } 1242 } 1243 } 1244 1245 return (0); 1246 } 1247 1248 static void 1249 vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1250 { 1251 device_t dev; 1252 struct vmxnet3_txqueue *txq; 1253 struct vmxnet3_txring *txr; 1254 struct vmxnet3_comp_ring *txc; 1255 struct vmxnet3_txbuf *txb; 1256 int i, q; 1257 1258 dev = sc->vmx_dev; 1259 1260 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1261 txq = &sc->vmx_txq[q]; 1262 txr = &txq->vxtxq_cmd_ring; 1263 txc = &txq->vxtxq_comp_ring; 1264 1265 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1266 txb = &txr->vxtxr_txbuf[i]; 1267 if (txb->vtxb_dmamap != NULL) { 1268 bus_dmamap_destroy(txr->vxtxr_txtag, 1269 txb->vtxb_dmamap); 1270 txb->vtxb_dmamap = NULL; 1271 } 1272 } 1273 1274 if (txc->vxcr_u.txcd != NULL) { 1275 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1276 txc->vxcr_u.txcd = NULL; 1277 } 1278 1279 if (txr->vxtxr_txd != NULL) { 1280 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1281 txr->vxtxr_txd = NULL; 1282 } 1283 1284 if (txr->vxtxr_txtag != NULL) { 1285 bus_dma_tag_destroy(txr->vxtxr_txtag); 1286 txr->vxtxr_txtag = NULL; 1287 } 1288 } 1289 } 1290 1291 static int 1292 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1293 { 1294 device_t dev; 1295 struct vmxnet3_rxqueue *rxq; 1296 struct vmxnet3_rxring *rxr; 1297 struct vmxnet3_comp_ring *rxc; 1298 int descsz, compsz; 1299 int i, j, q, error; 1300 1301 dev = sc->vmx_dev; 1302 1303 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1304 rxq = &sc->vmx_rxq[q]; 1305 rxc = &rxq->vxrxq_comp_ring; 1306 compsz = 0; 1307 1308 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1309 rxr = &rxq->vxrxq_cmd_ring[i]; 1310 1311 descsz = rxr->vxrxr_ndesc * 1312 sizeof(struct vmxnet3_rxdesc); 1313 compsz += rxr->vxrxr_ndesc * 1314 sizeof(struct vmxnet3_rxcompdesc); 1315 1316 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1317 1, 0, /* alignment, boundary */ 1318 BUS_SPACE_MAXADDR, /* lowaddr */ 1319 BUS_SPACE_MAXADDR, /* highaddr */ 1320 NULL, NULL, /* filter, filterarg */ 1321 MJUMPAGESIZE, /* maxsize */ 1322 1, /* nsegments */ 1323 MJUMPAGESIZE, /* maxsegsize */ 1324 0, /* flags */ 1325 NULL, NULL, /* lockfunc, lockarg */ 1326 &rxr->vxrxr_rxtag); 1327 if (error) { 1328 device_printf(dev, 1329 "unable to create Rx buffer tag for " 1330 "queue %d\n", q); 1331 return (error); 1332 } 1333 1334 error = vmxnet3_dma_malloc(sc, descsz, 512, 1335 &rxr->vxrxr_dma); 1336 if (error) { 1337 device_printf(dev, "cannot allocate Rx " 1338 "descriptors for queue %d/%d error %d\n", 1339 i, q, error); 1340 return (error); 1341 } 1342 rxr->vxrxr_rxd = 1343 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1344 } 1345 1346 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1347 if (error) { 1348 device_printf(dev, "cannot alloc Rx comp descriptors " 1349 "for queue %d error %d\n", q, error); 1350 return (error); 1351 } 1352 rxc->vxcr_u.rxcd = 1353 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1354 1355 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1356 rxr = &rxq->vxrxq_cmd_ring[i]; 1357 1358 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1359 &rxr->vxrxr_spare_dmap); 1360 if (error) { 1361 device_printf(dev, "unable to create spare " 1362 "dmamap for queue %d/%d error %d\n", 1363 q, i, error); 1364 return (error); 1365 } 1366 1367 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1368 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1369 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1370 if (error) { 1371 device_printf(dev, "unable to create " 1372 "dmamap for queue %d/%d slot %d " 1373 "error %d\n", 1374 q, i, j, error); 1375 return (error); 1376 } 1377 } 1378 } 1379 } 1380 1381 return (0); 1382 } 1383 1384 static void 1385 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1386 { 1387 device_t dev; 1388 struct vmxnet3_rxqueue *rxq; 1389 struct vmxnet3_rxring *rxr; 1390 struct vmxnet3_comp_ring *rxc; 1391 struct vmxnet3_rxbuf *rxb; 1392 int i, j, q; 1393 1394 dev = sc->vmx_dev; 1395 1396 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1397 rxq = &sc->vmx_rxq[q]; 1398 rxc = &rxq->vxrxq_comp_ring; 1399 1400 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1401 rxr = &rxq->vxrxq_cmd_ring[i]; 1402 1403 if (rxr->vxrxr_spare_dmap != NULL) { 1404 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1405 rxr->vxrxr_spare_dmap); 1406 rxr->vxrxr_spare_dmap = NULL; 1407 } 1408 1409 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1410 rxb = &rxr->vxrxr_rxbuf[j]; 1411 if (rxb->vrxb_dmamap != NULL) { 1412 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1413 rxb->vrxb_dmamap); 1414 rxb->vrxb_dmamap = NULL; 1415 } 1416 } 1417 } 1418 1419 if (rxc->vxcr_u.rxcd != NULL) { 1420 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1421 rxc->vxcr_u.rxcd = NULL; 1422 } 1423 1424 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1425 rxr = &rxq->vxrxq_cmd_ring[i]; 1426 1427 if (rxr->vxrxr_rxd != NULL) { 1428 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1429 rxr->vxrxr_rxd = NULL; 1430 } 1431 1432 if (rxr->vxrxr_rxtag != NULL) { 1433 bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1434 rxr->vxrxr_rxtag = NULL; 1435 } 1436 } 1437 } 1438 } 1439 1440 static int 1441 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1442 { 1443 int error; 1444 1445 error = vmxnet3_alloc_txq_data(sc); 1446 if (error) 1447 return (error); 1448 1449 error = vmxnet3_alloc_rxq_data(sc); 1450 if (error) 1451 return (error); 1452 1453 return (0); 1454 } 1455 1456 static void 1457 vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1458 { 1459 1460 if (sc->vmx_rxq != NULL) 1461 vmxnet3_free_rxq_data(sc); 1462 1463 if (sc->vmx_txq != NULL) 1464 vmxnet3_free_txq_data(sc); 1465 } 1466 1467 static int 1468 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1469 { 1470 int error; 1471 1472 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1473 32, &sc->vmx_mcast_dma); 1474 if (error) 1475 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1476 else 1477 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1478 1479 return (error); 1480 } 1481 1482 static void 1483 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1484 { 1485 1486 if (sc->vmx_mcast != NULL) { 1487 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1488 sc->vmx_mcast = NULL; 1489 } 1490 } 1491 1492 static void 1493 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1494 { 1495 struct vmxnet3_driver_shared *ds; 1496 struct vmxnet3_txqueue *txq; 1497 struct vmxnet3_txq_shared *txs; 1498 struct vmxnet3_rxqueue *rxq; 1499 struct vmxnet3_rxq_shared *rxs; 1500 int i; 1501 1502 ds = sc->vmx_ds; 1503 1504 /* 1505 * Initialize fields of the shared data that remains the same across 1506 * reinits. Note the shared data is zero'd when allocated. 1507 */ 1508 1509 ds->magic = VMXNET3_REV1_MAGIC; 1510 1511 /* DriverInfo */ 1512 ds->version = VMXNET3_DRIVER_VERSION; 1513 ds->guest = VMXNET3_GOS_FREEBSD | 1514 #ifdef __LP64__ 1515 VMXNET3_GOS_64BIT; 1516 #else 1517 VMXNET3_GOS_32BIT; 1518 #endif 1519 ds->vmxnet3_revision = 1; 1520 ds->upt_version = 1; 1521 1522 /* Misc. conf */ 1523 ds->driver_data = vtophys(sc); 1524 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1525 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1526 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1527 ds->nrxsg_max = sc->vmx_max_rxsegs; 1528 1529 /* RSS conf */ 1530 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1531 ds->rss.version = 1; 1532 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1533 ds->rss.len = sc->vmx_rss_dma.dma_size; 1534 } 1535 1536 /* Interrupt control. */ 1537 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1538 ds->nintr = sc->vmx_nintrs; 1539 ds->evintr = sc->vmx_event_intr_idx; 1540 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1541 1542 for (i = 0; i < sc->vmx_nintrs; i++) 1543 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1544 1545 /* Receive filter. */ 1546 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1547 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1548 1549 /* Tx queues */ 1550 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1551 txq = &sc->vmx_txq[i]; 1552 txs = txq->vxtxq_ts; 1553 1554 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1555 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1556 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1557 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1558 txs->driver_data = vtophys(txq); 1559 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1560 } 1561 1562 /* Rx queues */ 1563 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1564 rxq = &sc->vmx_rxq[i]; 1565 rxs = rxq->vxrxq_rs; 1566 1567 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1568 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1569 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1570 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1571 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1572 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1573 rxs->driver_data = vtophys(rxq); 1574 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1575 } 1576 } 1577 1578 static void 1579 vmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1580 { 1581 struct ifnet *ifp; 1582 1583 ifp = sc->vmx_ifp; 1584 1585 /* Use the current MAC address. */ 1586 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1587 vmxnet3_set_lladdr(sc); 1588 1589 ifp->if_hwassist = 0; 1590 if (ifp->if_capenable & IFCAP_TXCSUM) 1591 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD; 1592 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1593 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1594 if (ifp->if_capenable & IFCAP_TSO4) 1595 ifp->if_hwassist |= CSUM_IP_TSO; 1596 if (ifp->if_capenable & IFCAP_TSO6) 1597 ifp->if_hwassist |= CSUM_IP6_TSO; 1598 } 1599 1600 static void 1601 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1602 { 1603 /* 1604 * Use the same key as the Linux driver until FreeBSD can do 1605 * RSS (presumably Toeplitz) in software. 1606 */ 1607 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1608 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1609 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1610 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1611 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1612 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1613 }; 1614 1615 struct vmxnet3_driver_shared *ds; 1616 struct vmxnet3_rss_shared *rss; 1617 int i; 1618 1619 ds = sc->vmx_ds; 1620 rss = sc->vmx_rss; 1621 1622 rss->hash_type = 1623 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1624 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1625 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1626 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1627 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1628 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1629 1630 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1631 rss->ind_table[i] = i % sc->vmx_nrxqueues; 1632 } 1633 1634 static void 1635 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1636 { 1637 struct ifnet *ifp; 1638 struct vmxnet3_driver_shared *ds; 1639 1640 ifp = sc->vmx_ifp; 1641 ds = sc->vmx_ds; 1642 1643 ds->mtu = ifp->if_mtu; 1644 ds->ntxqueue = sc->vmx_ntxqueues; 1645 ds->nrxqueue = sc->vmx_nrxqueues; 1646 1647 ds->upt_features = 0; 1648 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1649 ds->upt_features |= UPT1_F_CSUM; 1650 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1651 ds->upt_features |= UPT1_F_VLAN; 1652 if (ifp->if_capenable & IFCAP_LRO) 1653 ds->upt_features |= UPT1_F_LRO; 1654 1655 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1656 ds->upt_features |= UPT1_F_RSS; 1657 vmxnet3_reinit_rss_shared_data(sc); 1658 } 1659 1660 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1661 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1662 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1663 } 1664 1665 static int 1666 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1667 { 1668 int error; 1669 1670 error = vmxnet3_alloc_shared_data(sc); 1671 if (error) 1672 return (error); 1673 1674 error = vmxnet3_alloc_queue_data(sc); 1675 if (error) 1676 return (error); 1677 1678 error = vmxnet3_alloc_mcast_table(sc); 1679 if (error) 1680 return (error); 1681 1682 vmxnet3_init_shared_data(sc); 1683 1684 return (0); 1685 } 1686 1687 static void 1688 vmxnet3_free_data(struct vmxnet3_softc *sc) 1689 { 1690 1691 vmxnet3_free_mcast_table(sc); 1692 vmxnet3_free_queue_data(sc); 1693 vmxnet3_free_shared_data(sc); 1694 } 1695 1696 static int 1697 vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1698 { 1699 device_t dev; 1700 struct ifnet *ifp; 1701 1702 dev = sc->vmx_dev; 1703 1704 ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1705 if (ifp == NULL) { 1706 device_printf(dev, "cannot allocate ifnet structure\n"); 1707 return (ENOSPC); 1708 } 1709 1710 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1711 #if __FreeBSD_version < 1000025 1712 ifp->if_baudrate = 1000000000; 1713 #elif __FreeBSD_version < 1100011 1714 if_initbaudrate(ifp, IF_Gbps(10)); 1715 #else 1716 ifp->if_baudrate = IF_Gbps(10); 1717 #endif 1718 ifp->if_softc = sc; 1719 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1720 ifp->if_init = vmxnet3_init; 1721 ifp->if_ioctl = vmxnet3_ioctl; 1722 ifp->if_get_counter = vmxnet3_get_counter; 1723 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1724 ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS; 1725 ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE; 1726 1727 #ifdef VMXNET3_LEGACY_TX 1728 ifp->if_start = vmxnet3_start; 1729 ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; 1730 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1731 IFQ_SET_READY(&ifp->if_snd); 1732 #else 1733 ifp->if_transmit = vmxnet3_txq_mq_start; 1734 ifp->if_qflush = vmxnet3_qflush; 1735 #endif 1736 1737 vmxnet3_get_lladdr(sc); 1738 ether_ifattach(ifp, sc->vmx_lladdr); 1739 1740 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1741 ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1742 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1743 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1744 IFCAP_VLAN_HWCSUM; 1745 ifp->if_capenable = ifp->if_capabilities; 1746 1747 /* These capabilities are not enabled by default. */ 1748 ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER; 1749 1750 sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1751 vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1752 sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1753 vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1754 1755 ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1756 vmxnet3_media_status); 1757 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1758 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1759 1760 return (0); 1761 } 1762 1763 static void 1764 vmxnet3_evintr(struct vmxnet3_softc *sc) 1765 { 1766 device_t dev; 1767 struct ifnet *ifp; 1768 struct vmxnet3_txq_shared *ts; 1769 struct vmxnet3_rxq_shared *rs; 1770 uint32_t event; 1771 int reset; 1772 1773 dev = sc->vmx_dev; 1774 ifp = sc->vmx_ifp; 1775 reset = 0; 1776 1777 VMXNET3_CORE_LOCK(sc); 1778 1779 /* Clear events. */ 1780 event = sc->vmx_ds->event; 1781 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1782 1783 if (event & VMXNET3_EVENT_LINK) { 1784 vmxnet3_link_status(sc); 1785 if (sc->vmx_link_active != 0) 1786 vmxnet3_tx_start_all(sc); 1787 } 1788 1789 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1790 reset = 1; 1791 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1792 ts = sc->vmx_txq[0].vxtxq_ts; 1793 if (ts->stopped != 0) 1794 device_printf(dev, "Tx queue error %#x\n", ts->error); 1795 rs = sc->vmx_rxq[0].vxrxq_rs; 1796 if (rs->stopped != 0) 1797 device_printf(dev, "Rx queue error %#x\n", rs->error); 1798 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1799 } 1800 1801 if (event & VMXNET3_EVENT_DIC) 1802 device_printf(dev, "device implementation change event\n"); 1803 if (event & VMXNET3_EVENT_DEBUG) 1804 device_printf(dev, "debug event\n"); 1805 1806 if (reset != 0) { 1807 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1808 vmxnet3_init_locked(sc); 1809 } 1810 1811 VMXNET3_CORE_UNLOCK(sc); 1812 } 1813 1814 static void 1815 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1816 { 1817 struct vmxnet3_softc *sc; 1818 struct ifnet *ifp; 1819 struct vmxnet3_txring *txr; 1820 struct vmxnet3_comp_ring *txc; 1821 struct vmxnet3_txcompdesc *txcd; 1822 struct vmxnet3_txbuf *txb; 1823 struct mbuf *m; 1824 u_int sop; 1825 1826 sc = txq->vxtxq_sc; 1827 ifp = sc->vmx_ifp; 1828 txr = &txq->vxtxq_cmd_ring; 1829 txc = &txq->vxtxq_comp_ring; 1830 1831 VMXNET3_TXQ_LOCK_ASSERT(txq); 1832 1833 for (;;) { 1834 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1835 if (txcd->gen != txc->vxcr_gen) 1836 break; 1837 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1838 1839 if (++txc->vxcr_next == txc->vxcr_ndesc) { 1840 txc->vxcr_next = 0; 1841 txc->vxcr_gen ^= 1; 1842 } 1843 1844 sop = txr->vxtxr_next; 1845 txb = &txr->vxtxr_txbuf[sop]; 1846 1847 if ((m = txb->vtxb_m) != NULL) { 1848 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1849 BUS_DMASYNC_POSTWRITE); 1850 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1851 1852 txq->vxtxq_stats.vmtxs_opackets++; 1853 txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; 1854 if (m->m_flags & M_MCAST) 1855 txq->vxtxq_stats.vmtxs_omcasts++; 1856 1857 m_freem(m); 1858 txb->vtxb_m = NULL; 1859 } 1860 1861 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1862 } 1863 1864 if (txr->vxtxr_head == txr->vxtxr_next) 1865 txq->vxtxq_watchdog = 0; 1866 } 1867 1868 static int 1869 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1870 { 1871 struct ifnet *ifp; 1872 struct mbuf *m; 1873 struct vmxnet3_rxdesc *rxd; 1874 struct vmxnet3_rxbuf *rxb; 1875 bus_dma_tag_t tag; 1876 bus_dmamap_t dmap; 1877 bus_dma_segment_t segs[1]; 1878 int idx, clsize, btype, flags, nsegs, error; 1879 1880 ifp = sc->vmx_ifp; 1881 tag = rxr->vxrxr_rxtag; 1882 dmap = rxr->vxrxr_spare_dmap; 1883 idx = rxr->vxrxr_fill; 1884 rxd = &rxr->vxrxr_rxd[idx]; 1885 rxb = &rxr->vxrxr_rxbuf[idx]; 1886 1887 #ifdef VMXNET3_FAILPOINTS 1888 KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1889 if (rxr->vxrxr_rid != 0) 1890 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1891 #endif 1892 1893 if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1894 flags = M_PKTHDR; 1895 clsize = MCLBYTES; 1896 btype = VMXNET3_BTYPE_HEAD; 1897 } else { 1898 #if __FreeBSD_version < 902001 1899 /* 1900 * These mbufs will never be used for the start of a frame. 1901 * Roughly prior to branching releng/9.2, the load_mbuf_sg() 1902 * required the mbuf to always be a packet header. Avoid 1903 * unnecessary mbuf initialization in newer versions where 1904 * that is not the case. 1905 */ 1906 flags = M_PKTHDR; 1907 #else 1908 flags = 0; 1909 #endif 1910 clsize = MJUMPAGESIZE; 1911 btype = VMXNET3_BTYPE_BODY; 1912 } 1913 1914 m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 1915 if (m == NULL) { 1916 sc->vmx_stats.vmst_mgetcl_failed++; 1917 return (ENOBUFS); 1918 } 1919 1920 if (btype == VMXNET3_BTYPE_HEAD) { 1921 m->m_len = m->m_pkthdr.len = clsize; 1922 m_adj(m, ETHER_ALIGN); 1923 } else 1924 m->m_len = clsize; 1925 1926 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, 1927 BUS_DMA_NOWAIT); 1928 if (error) { 1929 m_freem(m); 1930 sc->vmx_stats.vmst_mbuf_load_failed++; 1931 return (error); 1932 } 1933 KASSERT(nsegs == 1, 1934 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 1935 #if __FreeBSD_version < 902001 1936 if (btype == VMXNET3_BTYPE_BODY) 1937 m->m_flags &= ~M_PKTHDR; 1938 #endif 1939 1940 if (rxb->vrxb_m != NULL) { 1941 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 1942 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 1943 } 1944 1945 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 1946 rxb->vrxb_dmamap = dmap; 1947 rxb->vrxb_m = m; 1948 1949 rxd->addr = segs[0].ds_addr; 1950 rxd->len = segs[0].ds_len; 1951 rxd->btype = btype; 1952 rxd->gen = rxr->vxrxr_gen; 1953 1954 vmxnet3_rxr_increment_fill(rxr); 1955 return (0); 1956 } 1957 1958 static void 1959 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 1960 struct vmxnet3_rxring *rxr, int idx) 1961 { 1962 struct vmxnet3_rxdesc *rxd; 1963 1964 rxd = &rxr->vxrxr_rxd[idx]; 1965 rxd->gen = rxr->vxrxr_gen; 1966 vmxnet3_rxr_increment_fill(rxr); 1967 } 1968 1969 static void 1970 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 1971 { 1972 struct vmxnet3_softc *sc; 1973 struct vmxnet3_rxring *rxr; 1974 struct vmxnet3_comp_ring *rxc; 1975 struct vmxnet3_rxcompdesc *rxcd; 1976 int idx, eof; 1977 1978 sc = rxq->vxrxq_sc; 1979 rxc = &rxq->vxrxq_comp_ring; 1980 1981 do { 1982 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 1983 if (rxcd->gen != rxc->vxcr_gen) 1984 break; /* Not expected. */ 1985 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1986 1987 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 1988 rxc->vxcr_next = 0; 1989 rxc->vxcr_gen ^= 1; 1990 } 1991 1992 idx = rxcd->rxd_idx; 1993 eof = rxcd->eop; 1994 if (rxcd->qid < sc->vmx_nrxqueues) 1995 rxr = &rxq->vxrxq_cmd_ring[0]; 1996 else 1997 rxr = &rxq->vxrxq_cmd_ring[1]; 1998 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 1999 } while (!eof); 2000 } 2001 2002 static void 2003 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2004 { 2005 2006 if (rxcd->ipv4) { 2007 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2008 if (rxcd->ipcsum_ok) 2009 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2010 } 2011 2012 if (!rxcd->fragment) { 2013 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 2014 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2015 CSUM_PSEUDO_HDR; 2016 m->m_pkthdr.csum_data = 0xFFFF; 2017 } 2018 } 2019 } 2020 2021 static void 2022 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2023 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2024 { 2025 struct vmxnet3_softc *sc; 2026 struct ifnet *ifp; 2027 2028 sc = rxq->vxrxq_sc; 2029 ifp = sc->vmx_ifp; 2030 2031 if (rxcd->error) { 2032 rxq->vxrxq_stats.vmrxs_ierrors++; 2033 m_freem(m); 2034 return; 2035 } 2036 2037 #ifdef notyet 2038 switch (rxcd->rss_type) { 2039 case VMXNET3_RCD_RSS_TYPE_IPV4: 2040 m->m_pkthdr.flowid = rxcd->rss_hash; 2041 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4); 2042 break; 2043 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 2044 m->m_pkthdr.flowid = rxcd->rss_hash; 2045 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4); 2046 break; 2047 case VMXNET3_RCD_RSS_TYPE_IPV6: 2048 m->m_pkthdr.flowid = rxcd->rss_hash; 2049 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); 2050 break; 2051 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 2052 m->m_pkthdr.flowid = rxcd->rss_hash; 2053 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6); 2054 break; 2055 default: /* VMXNET3_RCD_RSS_TYPE_NONE */ 2056 m->m_pkthdr.flowid = rxq->vxrxq_id; 2057 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2058 break; 2059 } 2060 #else 2061 m->m_pkthdr.flowid = rxq->vxrxq_id; 2062 m->m_flags |= M_FLOWID; 2063 #endif 2064 2065 if (!rxcd->no_csum) 2066 vmxnet3_rx_csum(rxcd, m); 2067 if (rxcd->vlan) { 2068 m->m_flags |= M_VLANTAG; 2069 m->m_pkthdr.ether_vtag = rxcd->vtag; 2070 } 2071 2072 rxq->vxrxq_stats.vmrxs_ipackets++; 2073 rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; 2074 2075 VMXNET3_RXQ_UNLOCK(rxq); 2076 (*ifp->if_input)(ifp, m); 2077 VMXNET3_RXQ_LOCK(rxq); 2078 } 2079 2080 static void 2081 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 2082 { 2083 struct vmxnet3_softc *sc; 2084 struct ifnet *ifp; 2085 struct vmxnet3_rxring *rxr; 2086 struct vmxnet3_comp_ring *rxc; 2087 struct vmxnet3_rxdesc *rxd; 2088 struct vmxnet3_rxcompdesc *rxcd; 2089 struct mbuf *m, *m_head, *m_tail; 2090 int idx, length; 2091 2092 sc = rxq->vxrxq_sc; 2093 ifp = sc->vmx_ifp; 2094 rxc = &rxq->vxrxq_comp_ring; 2095 2096 VMXNET3_RXQ_LOCK_ASSERT(rxq); 2097 2098 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2099 return; 2100 2101 m_head = rxq->vxrxq_mhead; 2102 rxq->vxrxq_mhead = NULL; 2103 m_tail = rxq->vxrxq_mtail; 2104 rxq->vxrxq_mtail = NULL; 2105 MPASS(m_head == NULL || m_tail != NULL); 2106 2107 for (;;) { 2108 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2109 if (rxcd->gen != rxc->vxcr_gen) { 2110 rxq->vxrxq_mhead = m_head; 2111 rxq->vxrxq_mtail = m_tail; 2112 break; 2113 } 2114 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2115 2116 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2117 rxc->vxcr_next = 0; 2118 rxc->vxcr_gen ^= 1; 2119 } 2120 2121 idx = rxcd->rxd_idx; 2122 length = rxcd->len; 2123 if (rxcd->qid < sc->vmx_nrxqueues) 2124 rxr = &rxq->vxrxq_cmd_ring[0]; 2125 else 2126 rxr = &rxq->vxrxq_cmd_ring[1]; 2127 rxd = &rxr->vxrxr_rxd[idx]; 2128 2129 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2130 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 2131 __func__, rxcd->qid, idx)); 2132 2133 /* 2134 * The host may skip descriptors. We detect this when this 2135 * descriptor does not match the previous fill index. Catch 2136 * up with the host now. 2137 */ 2138 if (__predict_false(rxr->vxrxr_fill != idx)) { 2139 while (rxr->vxrxr_fill != idx) { 2140 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2141 rxr->vxrxr_gen; 2142 vmxnet3_rxr_increment_fill(rxr); 2143 } 2144 } 2145 2146 if (rxcd->sop) { 2147 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 2148 ("%s: start of frame w/o head buffer", __func__)); 2149 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 2150 ("%s: start of frame not in ring 0", __func__)); 2151 KASSERT((idx % sc->vmx_rx_max_chain) == 0, 2152 ("%s: start of frame at unexcepted index %d (%d)", 2153 __func__, idx, sc->vmx_rx_max_chain)); 2154 KASSERT(m_head == NULL, 2155 ("%s: duplicate start of frame?", __func__)); 2156 2157 if (length == 0) { 2158 /* Just ignore this descriptor. */ 2159 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2160 goto nextp; 2161 } 2162 2163 if (vmxnet3_newbuf(sc, rxr) != 0) { 2164 rxq->vxrxq_stats.vmrxs_iqdrops++; 2165 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2166 if (!rxcd->eop) 2167 vmxnet3_rxq_discard_chain(rxq); 2168 goto nextp; 2169 } 2170 2171 m->m_pkthdr.rcvif = ifp; 2172 m->m_pkthdr.len = m->m_len = length; 2173 m->m_pkthdr.csum_flags = 0; 2174 m_head = m_tail = m; 2175 2176 } else { 2177 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 2178 ("%s: non start of frame w/o body buffer", __func__)); 2179 KASSERT(m_head != NULL, 2180 ("%s: frame not started?", __func__)); 2181 2182 if (vmxnet3_newbuf(sc, rxr) != 0) { 2183 rxq->vxrxq_stats.vmrxs_iqdrops++; 2184 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2185 if (!rxcd->eop) 2186 vmxnet3_rxq_discard_chain(rxq); 2187 m_freem(m_head); 2188 m_head = m_tail = NULL; 2189 goto nextp; 2190 } 2191 2192 m->m_len = length; 2193 m_head->m_pkthdr.len += length; 2194 m_tail->m_next = m; 2195 m_tail = m; 2196 } 2197 2198 if (rxcd->eop) { 2199 vmxnet3_rxq_input(rxq, rxcd, m_head); 2200 m_head = m_tail = NULL; 2201 2202 /* Must recheck after dropping the Rx lock. */ 2203 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2204 break; 2205 } 2206 2207 nextp: 2208 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2209 int qid = rxcd->qid; 2210 bus_size_t r; 2211 2212 idx = (idx + 1) % rxr->vxrxr_ndesc; 2213 if (qid >= sc->vmx_nrxqueues) { 2214 qid -= sc->vmx_nrxqueues; 2215 r = VMXNET3_BAR0_RXH2(qid); 2216 } else 2217 r = VMXNET3_BAR0_RXH1(qid); 2218 vmxnet3_write_bar0(sc, r, idx); 2219 } 2220 } 2221 } 2222 2223 static void 2224 vmxnet3_legacy_intr(void *xsc) 2225 { 2226 struct vmxnet3_softc *sc; 2227 struct vmxnet3_rxqueue *rxq; 2228 struct vmxnet3_txqueue *txq; 2229 2230 sc = xsc; 2231 rxq = &sc->vmx_rxq[0]; 2232 txq = &sc->vmx_txq[0]; 2233 2234 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2235 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2236 return; 2237 } 2238 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2239 vmxnet3_disable_all_intrs(sc); 2240 2241 if (sc->vmx_ds->event != 0) 2242 vmxnet3_evintr(sc); 2243 2244 VMXNET3_RXQ_LOCK(rxq); 2245 vmxnet3_rxq_eof(rxq); 2246 VMXNET3_RXQ_UNLOCK(rxq); 2247 2248 VMXNET3_TXQ_LOCK(txq); 2249 vmxnet3_txq_eof(txq); 2250 vmxnet3_txq_start(txq); 2251 VMXNET3_TXQ_UNLOCK(txq); 2252 2253 vmxnet3_enable_all_intrs(sc); 2254 } 2255 2256 static void 2257 vmxnet3_txq_intr(void *xtxq) 2258 { 2259 struct vmxnet3_softc *sc; 2260 struct vmxnet3_txqueue *txq; 2261 2262 txq = xtxq; 2263 sc = txq->vxtxq_sc; 2264 2265 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2266 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 2267 2268 VMXNET3_TXQ_LOCK(txq); 2269 vmxnet3_txq_eof(txq); 2270 vmxnet3_txq_start(txq); 2271 VMXNET3_TXQ_UNLOCK(txq); 2272 2273 vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 2274 } 2275 2276 static void 2277 vmxnet3_rxq_intr(void *xrxq) 2278 { 2279 struct vmxnet3_softc *sc; 2280 struct vmxnet3_rxqueue *rxq; 2281 2282 rxq = xrxq; 2283 sc = rxq->vxrxq_sc; 2284 2285 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2286 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 2287 2288 VMXNET3_RXQ_LOCK(rxq); 2289 vmxnet3_rxq_eof(rxq); 2290 VMXNET3_RXQ_UNLOCK(rxq); 2291 2292 vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2293 } 2294 2295 static void 2296 vmxnet3_event_intr(void *xsc) 2297 { 2298 struct vmxnet3_softc *sc; 2299 2300 sc = xsc; 2301 2302 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2303 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2304 2305 if (sc->vmx_ds->event != 0) 2306 vmxnet3_evintr(sc); 2307 2308 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2309 } 2310 2311 static void 2312 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2313 { 2314 struct vmxnet3_txring *txr; 2315 struct vmxnet3_txbuf *txb; 2316 int i; 2317 2318 txr = &txq->vxtxq_cmd_ring; 2319 2320 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2321 txb = &txr->vxtxr_txbuf[i]; 2322 2323 if (txb->vtxb_m == NULL) 2324 continue; 2325 2326 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2327 BUS_DMASYNC_POSTWRITE); 2328 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2329 m_freem(txb->vtxb_m); 2330 txb->vtxb_m = NULL; 2331 } 2332 } 2333 2334 static void 2335 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2336 { 2337 struct vmxnet3_rxring *rxr; 2338 struct vmxnet3_rxbuf *rxb; 2339 int i, j; 2340 2341 if (rxq->vxrxq_mhead != NULL) { 2342 m_freem(rxq->vxrxq_mhead); 2343 rxq->vxrxq_mhead = NULL; 2344 rxq->vxrxq_mtail = NULL; 2345 } 2346 2347 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2348 rxr = &rxq->vxrxq_cmd_ring[i]; 2349 2350 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2351 rxb = &rxr->vxrxr_rxbuf[j]; 2352 2353 if (rxb->vrxb_m == NULL) 2354 continue; 2355 2356 bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2357 BUS_DMASYNC_POSTREAD); 2358 bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2359 m_freem(rxb->vrxb_m); 2360 rxb->vrxb_m = NULL; 2361 } 2362 } 2363 } 2364 2365 static void 2366 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2367 { 2368 struct vmxnet3_rxqueue *rxq; 2369 struct vmxnet3_txqueue *txq; 2370 int i; 2371 2372 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2373 rxq = &sc->vmx_rxq[i]; 2374 VMXNET3_RXQ_LOCK(rxq); 2375 VMXNET3_RXQ_UNLOCK(rxq); 2376 } 2377 2378 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2379 txq = &sc->vmx_txq[i]; 2380 VMXNET3_TXQ_LOCK(txq); 2381 VMXNET3_TXQ_UNLOCK(txq); 2382 } 2383 } 2384 2385 static void 2386 vmxnet3_stop(struct vmxnet3_softc *sc) 2387 { 2388 struct ifnet *ifp; 2389 int q; 2390 2391 ifp = sc->vmx_ifp; 2392 VMXNET3_CORE_LOCK_ASSERT(sc); 2393 2394 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2395 sc->vmx_link_active = 0; 2396 callout_stop(&sc->vmx_tick); 2397 2398 /* Disable interrupts. */ 2399 vmxnet3_disable_all_intrs(sc); 2400 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2401 2402 vmxnet3_stop_rendezvous(sc); 2403 2404 for (q = 0; q < sc->vmx_ntxqueues; q++) 2405 vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2406 for (q = 0; q < sc->vmx_nrxqueues; q++) 2407 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2408 2409 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2410 } 2411 2412 static void 2413 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2414 { 2415 struct vmxnet3_txring *txr; 2416 struct vmxnet3_comp_ring *txc; 2417 2418 txr = &txq->vxtxq_cmd_ring; 2419 txr->vxtxr_head = 0; 2420 txr->vxtxr_next = 0; 2421 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2422 bzero(txr->vxtxr_txd, 2423 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2424 2425 txc = &txq->vxtxq_comp_ring; 2426 txc->vxcr_next = 0; 2427 txc->vxcr_gen = VMXNET3_INIT_GEN; 2428 bzero(txc->vxcr_u.txcd, 2429 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2430 } 2431 2432 static int 2433 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2434 { 2435 struct ifnet *ifp; 2436 struct vmxnet3_rxring *rxr; 2437 struct vmxnet3_comp_ring *rxc; 2438 int i, populate, idx, frame_size, error; 2439 2440 ifp = sc->vmx_ifp; 2441 frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2442 ifp->if_mtu; 2443 2444 /* 2445 * If the MTU causes us to exceed what a regular sized cluster can 2446 * handle, we allocate a second MJUMPAGESIZE cluster after it in 2447 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2448 * 2449 * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2450 * our life easier. We do not support changing the ring size after 2451 * the attach. 2452 */ 2453 if (frame_size <= MCLBYTES) 2454 sc->vmx_rx_max_chain = 1; 2455 else 2456 sc->vmx_rx_max_chain = 2; 2457 2458 /* 2459 * Only populate ring 1 if the configuration will take advantage 2460 * of it. That is either when LRO is enabled or the frame size 2461 * exceeds what ring 0 can contain. 2462 */ 2463 if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2464 frame_size <= MCLBYTES + MJUMPAGESIZE) 2465 populate = 1; 2466 else 2467 populate = VMXNET3_RXRINGS_PERQ; 2468 2469 for (i = 0; i < populate; i++) { 2470 rxr = &rxq->vxrxq_cmd_ring[i]; 2471 rxr->vxrxr_fill = 0; 2472 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2473 bzero(rxr->vxrxr_rxd, 2474 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2475 2476 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2477 error = vmxnet3_newbuf(sc, rxr); 2478 if (error) 2479 return (error); 2480 } 2481 } 2482 2483 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2484 rxr = &rxq->vxrxq_cmd_ring[i]; 2485 rxr->vxrxr_fill = 0; 2486 rxr->vxrxr_gen = 0; 2487 bzero(rxr->vxrxr_rxd, 2488 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2489 } 2490 2491 rxc = &rxq->vxrxq_comp_ring; 2492 rxc->vxcr_next = 0; 2493 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2494 bzero(rxc->vxcr_u.rxcd, 2495 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2496 2497 return (0); 2498 } 2499 2500 static int 2501 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2502 { 2503 device_t dev; 2504 int q, error; 2505 2506 dev = sc->vmx_dev; 2507 2508 for (q = 0; q < sc->vmx_ntxqueues; q++) 2509 vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2510 2511 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2512 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2513 if (error) { 2514 device_printf(dev, "cannot populate Rx queue %d\n", q); 2515 return (error); 2516 } 2517 } 2518 2519 return (0); 2520 } 2521 2522 static int 2523 vmxnet3_enable_device(struct vmxnet3_softc *sc) 2524 { 2525 int q; 2526 2527 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2528 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2529 return (1); 2530 } 2531 2532 /* Reset the Rx queue heads. */ 2533 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2534 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2535 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2536 } 2537 2538 return (0); 2539 } 2540 2541 static void 2542 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2543 { 2544 struct ifnet *ifp; 2545 2546 ifp = sc->vmx_ifp; 2547 2548 vmxnet3_set_rxfilter(sc); 2549 2550 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2551 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2552 sizeof(sc->vmx_ds->vlan_filter)); 2553 else 2554 bzero(sc->vmx_ds->vlan_filter, 2555 sizeof(sc->vmx_ds->vlan_filter)); 2556 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2557 } 2558 2559 static int 2560 vmxnet3_reinit(struct vmxnet3_softc *sc) 2561 { 2562 2563 vmxnet3_reinit_interface(sc); 2564 vmxnet3_reinit_shared_data(sc); 2565 2566 if (vmxnet3_reinit_queues(sc) != 0) 2567 return (ENXIO); 2568 2569 if (vmxnet3_enable_device(sc) != 0) 2570 return (ENXIO); 2571 2572 vmxnet3_reinit_rxfilters(sc); 2573 2574 return (0); 2575 } 2576 2577 static void 2578 vmxnet3_init_locked(struct vmxnet3_softc *sc) 2579 { 2580 struct ifnet *ifp; 2581 2582 ifp = sc->vmx_ifp; 2583 2584 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2585 return; 2586 2587 vmxnet3_stop(sc); 2588 2589 if (vmxnet3_reinit(sc) != 0) { 2590 vmxnet3_stop(sc); 2591 return; 2592 } 2593 2594 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2595 vmxnet3_link_status(sc); 2596 2597 vmxnet3_enable_all_intrs(sc); 2598 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2599 } 2600 2601 static void 2602 vmxnet3_init(void *xsc) 2603 { 2604 struct vmxnet3_softc *sc; 2605 2606 sc = xsc; 2607 2608 VMXNET3_CORE_LOCK(sc); 2609 vmxnet3_init_locked(sc); 2610 VMXNET3_CORE_UNLOCK(sc); 2611 } 2612 2613 /* 2614 * BMV: Much of this can go away once we finally have offsets in 2615 * the mbuf packet header. Bug andre@. 2616 */ 2617 static int 2618 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2619 int *etype, int *proto, int *start) 2620 { 2621 struct ether_vlan_header *evh; 2622 int offset; 2623 #if defined(INET) 2624 struct ip *ip = NULL; 2625 struct ip iphdr; 2626 #endif 2627 #if defined(INET6) 2628 struct ip6_hdr *ip6 = NULL; 2629 struct ip6_hdr ip6hdr; 2630 #endif 2631 2632 evh = mtod(m, struct ether_vlan_header *); 2633 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2634 /* BMV: We should handle nested VLAN tags too. */ 2635 *etype = ntohs(evh->evl_proto); 2636 offset = sizeof(struct ether_vlan_header); 2637 } else { 2638 *etype = ntohs(evh->evl_encap_proto); 2639 offset = sizeof(struct ether_header); 2640 } 2641 2642 switch (*etype) { 2643 #if defined(INET) 2644 case ETHERTYPE_IP: 2645 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2646 m_copydata(m, offset, sizeof(struct ip), 2647 (caddr_t) &iphdr); 2648 ip = &iphdr; 2649 } else 2650 ip = mtodo(m, offset); 2651 *proto = ip->ip_p; 2652 *start = offset + (ip->ip_hl << 2); 2653 break; 2654 #endif 2655 #if defined(INET6) 2656 case ETHERTYPE_IPV6: 2657 if (__predict_false(m->m_len < 2658 offset + sizeof(struct ip6_hdr))) { 2659 m_copydata(m, offset, sizeof(struct ip6_hdr), 2660 (caddr_t) &ip6hdr); 2661 ip6 = &ip6hdr; 2662 } else 2663 ip6 = mtodo(m, offset); 2664 *proto = -1; 2665 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2666 /* Assert the network stack sent us a valid packet. */ 2667 KASSERT(*start > offset, 2668 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2669 *start, offset, *proto)); 2670 break; 2671 #endif 2672 default: 2673 return (EINVAL); 2674 } 2675 2676 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2677 struct tcphdr *tcp, tcphdr; 2678 uint16_t sum; 2679 2680 if (__predict_false(*proto != IPPROTO_TCP)) { 2681 /* Likely failed to correctly parse the mbuf. */ 2682 return (EINVAL); 2683 } 2684 2685 txq->vxtxq_stats.vmtxs_tso++; 2686 2687 switch (*etype) { 2688 #if defined(INET) 2689 case ETHERTYPE_IP: 2690 sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, 2691 htons(IPPROTO_TCP)); 2692 break; 2693 #endif 2694 #if defined(INET6) 2695 case ETHERTYPE_IPV6: 2696 sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 2697 break; 2698 #endif 2699 default: 2700 sum = 0; 2701 break; 2702 } 2703 2704 if (m->m_len < *start + sizeof(struct tcphdr)) { 2705 m_copyback(m, *start + offsetof(struct tcphdr, th_sum), 2706 sizeof(uint16_t), (caddr_t) &sum); 2707 m_copydata(m, *start, sizeof(struct tcphdr), 2708 (caddr_t) &tcphdr); 2709 tcp = &tcphdr; 2710 } else { 2711 tcp = mtodo(m, *start); 2712 tcp->th_sum = sum; 2713 } 2714 2715 /* 2716 * For TSO, the size of the protocol header is also 2717 * included in the descriptor header size. 2718 */ 2719 *start += (tcp->th_off << 2); 2720 } else 2721 txq->vxtxq_stats.vmtxs_csum++; 2722 2723 return (0); 2724 } 2725 2726 static int 2727 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2728 bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2729 { 2730 struct vmxnet3_txring *txr; 2731 struct mbuf *m; 2732 bus_dma_tag_t tag; 2733 int error; 2734 2735 txr = &txq->vxtxq_cmd_ring; 2736 m = *m0; 2737 tag = txr->vxtxr_txtag; 2738 2739 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2740 if (error == 0 || error != EFBIG) 2741 return (error); 2742 2743 m = m_defrag(m, M_NOWAIT); 2744 if (m != NULL) { 2745 *m0 = m; 2746 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2747 } else 2748 error = ENOBUFS; 2749 2750 if (error) { 2751 m_freem(*m0); 2752 *m0 = NULL; 2753 txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; 2754 } else 2755 txq->vxtxq_sc->vmx_stats.vmst_defragged++; 2756 2757 return (error); 2758 } 2759 2760 static void 2761 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2762 { 2763 struct vmxnet3_txring *txr; 2764 2765 txr = &txq->vxtxq_cmd_ring; 2766 bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2767 } 2768 2769 static int 2770 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2771 { 2772 struct vmxnet3_softc *sc; 2773 struct vmxnet3_txring *txr; 2774 struct vmxnet3_txdesc *txd, *sop; 2775 struct mbuf *m; 2776 bus_dmamap_t dmap; 2777 bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2778 int i, gen, nsegs, etype, proto, start, error; 2779 2780 sc = txq->vxtxq_sc; 2781 start = 0; 2782 txd = NULL; 2783 txr = &txq->vxtxq_cmd_ring; 2784 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2785 2786 error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2787 if (error) 2788 return (error); 2789 2790 m = *m0; 2791 M_ASSERTPKTHDR(m); 2792 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2793 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2794 2795 if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2796 txq->vxtxq_stats.vmtxs_full++; 2797 vmxnet3_txq_unload_mbuf(txq, dmap); 2798 return (ENOSPC); 2799 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2800 error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start); 2801 if (error) { 2802 txq->vxtxq_stats.vmtxs_offload_failed++; 2803 vmxnet3_txq_unload_mbuf(txq, dmap); 2804 m_freem(m); 2805 *m0 = NULL; 2806 return (error); 2807 } 2808 } 2809 2810 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 2811 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2812 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2813 2814 for (i = 0; i < nsegs; i++) { 2815 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2816 2817 txd->addr = segs[i].ds_addr; 2818 txd->len = segs[i].ds_len; 2819 txd->gen = gen; 2820 txd->dtype = 0; 2821 txd->offload_mode = VMXNET3_OM_NONE; 2822 txd->offload_pos = 0; 2823 txd->hlen = 0; 2824 txd->eop = 0; 2825 txd->compreq = 0; 2826 txd->vtag_mode = 0; 2827 txd->vtag = 0; 2828 2829 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2830 txr->vxtxr_head = 0; 2831 txr->vxtxr_gen ^= 1; 2832 } 2833 gen = txr->vxtxr_gen; 2834 } 2835 txd->eop = 1; 2836 txd->compreq = 1; 2837 2838 if (m->m_flags & M_VLANTAG) { 2839 sop->vtag_mode = 1; 2840 sop->vtag = m->m_pkthdr.ether_vtag; 2841 } 2842 2843 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2844 sop->offload_mode = VMXNET3_OM_TSO; 2845 sop->hlen = start; 2846 sop->offload_pos = m->m_pkthdr.tso_segsz; 2847 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2848 VMXNET3_CSUM_OFFLOAD_IPV6)) { 2849 sop->offload_mode = VMXNET3_OM_CSUM; 2850 sop->hlen = start; 2851 sop->offload_pos = start + m->m_pkthdr.csum_data; 2852 } 2853 2854 /* Finally, change the ownership. */ 2855 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2856 sop->gen ^= 1; 2857 2858 txq->vxtxq_ts->npending += nsegs; 2859 if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2860 txq->vxtxq_ts->npending = 0; 2861 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2862 txr->vxtxr_head); 2863 } 2864 2865 return (0); 2866 } 2867 2868 #ifdef VMXNET3_LEGACY_TX 2869 2870 static void 2871 vmxnet3_start_locked(struct ifnet *ifp) 2872 { 2873 struct vmxnet3_softc *sc; 2874 struct vmxnet3_txqueue *txq; 2875 struct vmxnet3_txring *txr; 2876 struct mbuf *m_head; 2877 int tx, avail; 2878 2879 sc = ifp->if_softc; 2880 txq = &sc->vmx_txq[0]; 2881 txr = &txq->vxtxq_cmd_ring; 2882 tx = 0; 2883 2884 VMXNET3_TXQ_LOCK_ASSERT(txq); 2885 2886 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2887 sc->vmx_link_active == 0) 2888 return; 2889 2890 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2891 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 2892 break; 2893 2894 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 2895 if (m_head == NULL) 2896 break; 2897 2898 /* Assume worse case if this mbuf is the head of a chain. */ 2899 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2900 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2901 break; 2902 } 2903 2904 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 2905 if (m_head != NULL) 2906 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2907 break; 2908 } 2909 2910 tx++; 2911 ETHER_BPF_MTAP(ifp, m_head); 2912 } 2913 2914 if (tx > 0) 2915 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2916 } 2917 2918 static void 2919 vmxnet3_start(struct ifnet *ifp) 2920 { 2921 struct vmxnet3_softc *sc; 2922 struct vmxnet3_txqueue *txq; 2923 2924 sc = ifp->if_softc; 2925 txq = &sc->vmx_txq[0]; 2926 2927 VMXNET3_TXQ_LOCK(txq); 2928 vmxnet3_start_locked(ifp); 2929 VMXNET3_TXQ_UNLOCK(txq); 2930 } 2931 2932 #else /* !VMXNET3_LEGACY_TX */ 2933 2934 static int 2935 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m) 2936 { 2937 struct vmxnet3_softc *sc; 2938 struct vmxnet3_txring *txr; 2939 struct buf_ring *br; 2940 struct ifnet *ifp; 2941 int tx, avail, error; 2942 2943 sc = txq->vxtxq_sc; 2944 br = txq->vxtxq_br; 2945 ifp = sc->vmx_ifp; 2946 txr = &txq->vxtxq_cmd_ring; 2947 tx = 0; 2948 error = 0; 2949 2950 VMXNET3_TXQ_LOCK_ASSERT(txq); 2951 2952 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2953 sc->vmx_link_active == 0) { 2954 if (m != NULL) 2955 error = drbr_enqueue(ifp, br, m); 2956 return (error); 2957 } 2958 2959 if (m != NULL) { 2960 error = drbr_enqueue(ifp, br, m); 2961 if (error) 2962 return (error); 2963 } 2964 2965 while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) { 2966 m = drbr_peek(ifp, br); 2967 if (m == NULL) 2968 break; 2969 2970 /* Assume worse case if this mbuf is the head of a chain. */ 2971 if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2972 drbr_putback(ifp, br, m); 2973 break; 2974 } 2975 2976 if (vmxnet3_txq_encap(txq, &m) != 0) { 2977 if (m != NULL) 2978 drbr_putback(ifp, br, m); 2979 else 2980 drbr_advance(ifp, br); 2981 break; 2982 } 2983 drbr_advance(ifp, br); 2984 2985 tx++; 2986 ETHER_BPF_MTAP(ifp, m); 2987 } 2988 2989 if (tx > 0) 2990 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2991 2992 return (0); 2993 } 2994 2995 static int 2996 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 2997 { 2998 struct vmxnet3_softc *sc; 2999 struct vmxnet3_txqueue *txq; 3000 int i, ntxq, error; 3001 3002 sc = ifp->if_softc; 3003 ntxq = sc->vmx_ntxqueues; 3004 3005 if (m->m_flags & M_FLOWID) 3006 i = m->m_pkthdr.flowid % ntxq; 3007 else 3008 i = curcpu % ntxq; 3009 3010 txq = &sc->vmx_txq[i]; 3011 3012 if (VMXNET3_TXQ_TRYLOCK(txq) != 0) { 3013 error = vmxnet3_txq_mq_start_locked(txq, m); 3014 VMXNET3_TXQ_UNLOCK(txq); 3015 } else { 3016 error = drbr_enqueue(ifp, txq->vxtxq_br, m); 3017 taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask); 3018 } 3019 3020 return (error); 3021 } 3022 3023 static void 3024 vmxnet3_txq_tq_deferred(void *xtxq, int pending) 3025 { 3026 struct vmxnet3_softc *sc; 3027 struct vmxnet3_txqueue *txq; 3028 3029 txq = xtxq; 3030 sc = txq->vxtxq_sc; 3031 3032 VMXNET3_TXQ_LOCK(txq); 3033 if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br)) 3034 vmxnet3_txq_mq_start_locked(txq, NULL); 3035 VMXNET3_TXQ_UNLOCK(txq); 3036 } 3037 3038 #endif /* VMXNET3_LEGACY_TX */ 3039 3040 static void 3041 vmxnet3_txq_start(struct vmxnet3_txqueue *txq) 3042 { 3043 struct vmxnet3_softc *sc; 3044 struct ifnet *ifp; 3045 3046 sc = txq->vxtxq_sc; 3047 ifp = sc->vmx_ifp; 3048 3049 #ifdef VMXNET3_LEGACY_TX 3050 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 3051 vmxnet3_start_locked(ifp); 3052 #else 3053 if (!drbr_empty(ifp, txq->vxtxq_br)) 3054 vmxnet3_txq_mq_start_locked(txq, NULL); 3055 #endif 3056 } 3057 3058 static void 3059 vmxnet3_tx_start_all(struct vmxnet3_softc *sc) 3060 { 3061 struct vmxnet3_txqueue *txq; 3062 int i; 3063 3064 VMXNET3_CORE_LOCK_ASSERT(sc); 3065 3066 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3067 txq = &sc->vmx_txq[i]; 3068 3069 VMXNET3_TXQ_LOCK(txq); 3070 vmxnet3_txq_start(txq); 3071 VMXNET3_TXQ_UNLOCK(txq); 3072 } 3073 } 3074 3075 static void 3076 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 3077 { 3078 struct ifnet *ifp; 3079 int idx, bit; 3080 3081 ifp = sc->vmx_ifp; 3082 idx = (tag >> 5) & 0x7F; 3083 bit = tag & 0x1F; 3084 3085 if (tag == 0 || tag > 4095) 3086 return; 3087 3088 VMXNET3_CORE_LOCK(sc); 3089 3090 /* Update our private VLAN bitvector. */ 3091 if (add) 3092 sc->vmx_vlan_filter[idx] |= (1 << bit); 3093 else 3094 sc->vmx_vlan_filter[idx] &= ~(1 << bit); 3095 3096 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3097 if (add) 3098 sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 3099 else 3100 sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 3101 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 3102 } 3103 3104 VMXNET3_CORE_UNLOCK(sc); 3105 } 3106 3107 static void 3108 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3109 { 3110 3111 if (ifp->if_softc == arg) 3112 vmxnet3_update_vlan_filter(arg, 1, tag); 3113 } 3114 3115 static void 3116 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3117 { 3118 3119 if (ifp->if_softc == arg) 3120 vmxnet3_update_vlan_filter(arg, 0, tag); 3121 } 3122 3123 static void 3124 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3125 { 3126 struct ifnet *ifp; 3127 struct vmxnet3_driver_shared *ds; 3128 struct ifmultiaddr *ifma; 3129 u_int mode; 3130 3131 ifp = sc->vmx_ifp; 3132 ds = sc->vmx_ds; 3133 3134 mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 3135 if (ifp->if_flags & IFF_PROMISC) 3136 mode |= VMXNET3_RXMODE_PROMISC; 3137 if (ifp->if_flags & IFF_ALLMULTI) 3138 mode |= VMXNET3_RXMODE_ALLMULTI; 3139 else { 3140 int cnt = 0, overflow = 0; 3141 3142 if_maddr_rlock(ifp); 3143 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3144 if (ifma->ifma_addr->sa_family != AF_LINK) 3145 continue; 3146 else if (cnt == VMXNET3_MULTICAST_MAX) { 3147 overflow = 1; 3148 break; 3149 } 3150 3151 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3152 &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3153 cnt++; 3154 } 3155 if_maddr_runlock(ifp); 3156 3157 if (overflow != 0) { 3158 cnt = 0; 3159 mode |= VMXNET3_RXMODE_ALLMULTI; 3160 } else if (cnt > 0) 3161 mode |= VMXNET3_RXMODE_MCAST; 3162 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 3163 } 3164 3165 ds->rxmode = mode; 3166 3167 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3168 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3169 } 3170 3171 static int 3172 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 3173 { 3174 struct ifnet *ifp; 3175 3176 ifp = sc->vmx_ifp; 3177 3178 if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 3179 return (EINVAL); 3180 3181 ifp->if_mtu = mtu; 3182 3183 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 3184 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3185 vmxnet3_init_locked(sc); 3186 } 3187 3188 return (0); 3189 } 3190 3191 static int 3192 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 3193 { 3194 struct vmxnet3_softc *sc; 3195 struct ifreq *ifr; 3196 int reinit, mask, error; 3197 3198 sc = ifp->if_softc; 3199 ifr = (struct ifreq *) data; 3200 error = 0; 3201 3202 switch (cmd) { 3203 case SIOCSIFMTU: 3204 if (ifp->if_mtu != ifr->ifr_mtu) { 3205 VMXNET3_CORE_LOCK(sc); 3206 error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 3207 VMXNET3_CORE_UNLOCK(sc); 3208 } 3209 break; 3210 3211 case SIOCSIFFLAGS: 3212 VMXNET3_CORE_LOCK(sc); 3213 if (ifp->if_flags & IFF_UP) { 3214 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3215 if ((ifp->if_flags ^ sc->vmx_if_flags) & 3216 (IFF_PROMISC | IFF_ALLMULTI)) { 3217 vmxnet3_set_rxfilter(sc); 3218 } 3219 } else 3220 vmxnet3_init_locked(sc); 3221 } else { 3222 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3223 vmxnet3_stop(sc); 3224 } 3225 sc->vmx_if_flags = ifp->if_flags; 3226 VMXNET3_CORE_UNLOCK(sc); 3227 break; 3228 3229 case SIOCADDMULTI: 3230 case SIOCDELMULTI: 3231 VMXNET3_CORE_LOCK(sc); 3232 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3233 vmxnet3_set_rxfilter(sc); 3234 VMXNET3_CORE_UNLOCK(sc); 3235 break; 3236 3237 case SIOCSIFMEDIA: 3238 case SIOCGIFMEDIA: 3239 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 3240 break; 3241 3242 case SIOCSIFCAP: 3243 VMXNET3_CORE_LOCK(sc); 3244 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3245 3246 if (mask & IFCAP_TXCSUM) 3247 ifp->if_capenable ^= IFCAP_TXCSUM; 3248 if (mask & IFCAP_TXCSUM_IPV6) 3249 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 3250 if (mask & IFCAP_TSO4) 3251 ifp->if_capenable ^= IFCAP_TSO4; 3252 if (mask & IFCAP_TSO6) 3253 ifp->if_capenable ^= IFCAP_TSO6; 3254 3255 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 3256 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) { 3257 /* Changing these features requires us to reinit. */ 3258 reinit = 1; 3259 3260 if (mask & IFCAP_RXCSUM) 3261 ifp->if_capenable ^= IFCAP_RXCSUM; 3262 if (mask & IFCAP_RXCSUM_IPV6) 3263 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 3264 if (mask & IFCAP_LRO) 3265 ifp->if_capenable ^= IFCAP_LRO; 3266 if (mask & IFCAP_VLAN_HWTAGGING) 3267 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 3268 if (mask & IFCAP_VLAN_HWFILTER) 3269 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 3270 } else 3271 reinit = 0; 3272 3273 if (mask & IFCAP_VLAN_HWTSO) 3274 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 3275 3276 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3277 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3278 vmxnet3_init_locked(sc); 3279 } 3280 3281 VMXNET3_CORE_UNLOCK(sc); 3282 VLAN_CAPABILITIES(ifp); 3283 break; 3284 3285 default: 3286 error = ether_ioctl(ifp, cmd, data); 3287 break; 3288 } 3289 3290 VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 3291 3292 return (error); 3293 } 3294 3295 #ifndef VMXNET3_LEGACY_TX 3296 static void 3297 vmxnet3_qflush(struct ifnet *ifp) 3298 { 3299 struct vmxnet3_softc *sc; 3300 struct vmxnet3_txqueue *txq; 3301 struct mbuf *m; 3302 int i; 3303 3304 sc = ifp->if_softc; 3305 3306 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3307 txq = &sc->vmx_txq[i]; 3308 3309 VMXNET3_TXQ_LOCK(txq); 3310 while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL) 3311 m_freem(m); 3312 VMXNET3_TXQ_UNLOCK(txq); 3313 } 3314 3315 if_qflush(ifp); 3316 } 3317 #endif 3318 3319 static int 3320 vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3321 { 3322 struct vmxnet3_softc *sc; 3323 3324 sc = txq->vxtxq_sc; 3325 3326 VMXNET3_TXQ_LOCK(txq); 3327 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3328 VMXNET3_TXQ_UNLOCK(txq); 3329 return (0); 3330 } 3331 VMXNET3_TXQ_UNLOCK(txq); 3332 3333 if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 3334 txq->vxtxq_id); 3335 return (1); 3336 } 3337 3338 static void 3339 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3340 { 3341 3342 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3343 } 3344 3345 static uint64_t 3346 vmxnet3_get_counter(struct ifnet *ifp, ift_counter cnt) 3347 { 3348 struct vmxnet3_softc *sc; 3349 uint64_t rv; 3350 3351 sc = if_getsoftc(ifp); 3352 rv = 0; 3353 3354 /* 3355 * With the exception of if_ierrors, these ifnet statistics are 3356 * only updated in the driver, so just set them to our accumulated 3357 * values. if_ierrors is updated in ether_input() for malformed 3358 * frames that we should have already discarded. 3359 */ 3360 switch (cnt) { 3361 case IFCOUNTER_IPACKETS: 3362 for (int i = 0; i < sc->vmx_nrxqueues; i++) 3363 rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ipackets; 3364 return (rv); 3365 case IFCOUNTER_IQDROPS: 3366 for (int i = 0; i < sc->vmx_nrxqueues; i++) 3367 rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_iqdrops; 3368 return (rv); 3369 case IFCOUNTER_IERRORS: 3370 for (int i = 0; i < sc->vmx_nrxqueues; i++) 3371 rv += sc->vmx_rxq[i].vxrxq_stats.vmrxs_ierrors; 3372 return (rv); 3373 case IFCOUNTER_OPACKETS: 3374 for (int i = 0; i < sc->vmx_ntxqueues; i++) 3375 rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_opackets; 3376 return (rv); 3377 #ifndef VMXNET3_LEGACY_TX 3378 case IFCOUNTER_OBYTES: 3379 for (int i = 0; i < sc->vmx_ntxqueues; i++) 3380 rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_obytes; 3381 return (rv); 3382 case IFCOUNTER_OMCASTS: 3383 for (int i = 0; i < sc->vmx_ntxqueues; i++) 3384 rv += sc->vmx_txq[i].vxtxq_stats.vmtxs_omcasts; 3385 return (rv); 3386 #endif 3387 default: 3388 return (if_get_counter_default(ifp, cnt)); 3389 } 3390 } 3391 3392 static void 3393 vmxnet3_tick(void *xsc) 3394 { 3395 struct vmxnet3_softc *sc; 3396 struct ifnet *ifp; 3397 int i, timedout; 3398 3399 sc = xsc; 3400 ifp = sc->vmx_ifp; 3401 timedout = 0; 3402 3403 VMXNET3_CORE_LOCK_ASSERT(sc); 3404 3405 vmxnet3_refresh_host_stats(sc); 3406 3407 for (i = 0; i < sc->vmx_ntxqueues; i++) 3408 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 3409 3410 if (timedout != 0) { 3411 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3412 vmxnet3_init_locked(sc); 3413 } else 3414 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3415 } 3416 3417 static int 3418 vmxnet3_link_is_up(struct vmxnet3_softc *sc) 3419 { 3420 uint32_t status; 3421 3422 /* Also update the link speed while here. */ 3423 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3424 sc->vmx_link_speed = status >> 16; 3425 return !!(status & 0x1); 3426 } 3427 3428 static void 3429 vmxnet3_link_status(struct vmxnet3_softc *sc) 3430 { 3431 struct ifnet *ifp; 3432 int link; 3433 3434 ifp = sc->vmx_ifp; 3435 link = vmxnet3_link_is_up(sc); 3436 3437 if (link != 0 && sc->vmx_link_active == 0) { 3438 sc->vmx_link_active = 1; 3439 if_link_state_change(ifp, LINK_STATE_UP); 3440 } else if (link == 0 && sc->vmx_link_active != 0) { 3441 sc->vmx_link_active = 0; 3442 if_link_state_change(ifp, LINK_STATE_DOWN); 3443 } 3444 } 3445 3446 static void 3447 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3448 { 3449 struct vmxnet3_softc *sc; 3450 3451 sc = ifp->if_softc; 3452 3453 ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 3454 ifmr->ifm_status = IFM_AVALID; 3455 3456 VMXNET3_CORE_LOCK(sc); 3457 if (vmxnet3_link_is_up(sc) != 0) 3458 ifmr->ifm_status |= IFM_ACTIVE; 3459 else 3460 ifmr->ifm_status |= IFM_NONE; 3461 VMXNET3_CORE_UNLOCK(sc); 3462 } 3463 3464 static int 3465 vmxnet3_media_change(struct ifnet *ifp) 3466 { 3467 3468 /* Ignore. */ 3469 return (0); 3470 } 3471 3472 static void 3473 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3474 { 3475 uint32_t ml, mh; 3476 3477 ml = sc->vmx_lladdr[0]; 3478 ml |= sc->vmx_lladdr[1] << 8; 3479 ml |= sc->vmx_lladdr[2] << 16; 3480 ml |= sc->vmx_lladdr[3] << 24; 3481 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3482 3483 mh = sc->vmx_lladdr[4]; 3484 mh |= sc->vmx_lladdr[5] << 8; 3485 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3486 } 3487 3488 static void 3489 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3490 { 3491 uint32_t ml, mh; 3492 3493 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3494 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3495 3496 sc->vmx_lladdr[0] = ml; 3497 sc->vmx_lladdr[1] = ml >> 8; 3498 sc->vmx_lladdr[2] = ml >> 16; 3499 sc->vmx_lladdr[3] = ml >> 24; 3500 sc->vmx_lladdr[4] = mh; 3501 sc->vmx_lladdr[5] = mh >> 8; 3502 } 3503 3504 static void 3505 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 3506 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3507 { 3508 struct sysctl_oid *node, *txsnode; 3509 struct sysctl_oid_list *list, *txslist; 3510 struct vmxnet3_txq_stats *stats; 3511 struct UPT1_TxStats *txstats; 3512 char namebuf[16]; 3513 3514 stats = &txq->vxtxq_stats; 3515 txstats = &txq->vxtxq_ts->stats; 3516 3517 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 3518 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3519 NULL, "Transmit Queue"); 3520 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 3521 3522 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3523 &stats->vmtxs_opackets, "Transmit packets"); 3524 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3525 &stats->vmtxs_obytes, "Transmit bytes"); 3526 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3527 &stats->vmtxs_omcasts, "Transmit multicasts"); 3528 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3529 &stats->vmtxs_csum, "Transmit checksum offloaded"); 3530 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3531 &stats->vmtxs_tso, "Transmit TCP segmentation offloaded"); 3532 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 3533 &stats->vmtxs_full, "Transmit ring full"); 3534 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 3535 &stats->vmtxs_offload_failed, "Transmit checksum offload failed"); 3536 3537 /* 3538 * Add statistics reported by the host. These are updated once 3539 * per second. 3540 */ 3541 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3542 NULL, "Host Statistics"); 3543 txslist = SYSCTL_CHILDREN(txsnode); 3544 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 3545 &txstats->TSO_packets, "TSO packets"); 3546 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 3547 &txstats->TSO_bytes, "TSO bytes"); 3548 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3549 &txstats->ucast_packets, "Unicast packets"); 3550 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3551 &txstats->ucast_bytes, "Unicast bytes"); 3552 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3553 &txstats->mcast_packets, "Multicast packets"); 3554 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3555 &txstats->mcast_bytes, "Multicast bytes"); 3556 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3557 &txstats->error, "Errors"); 3558 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3559 &txstats->discard, "Discards"); 3560 } 3561 3562 static void 3563 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3564 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3565 { 3566 struct sysctl_oid *node, *rxsnode; 3567 struct sysctl_oid_list *list, *rxslist; 3568 struct vmxnet3_rxq_stats *stats; 3569 struct UPT1_RxStats *rxstats; 3570 char namebuf[16]; 3571 3572 stats = &rxq->vxrxq_stats; 3573 rxstats = &rxq->vxrxq_rs->stats; 3574 3575 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3576 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3577 NULL, "Receive Queue"); 3578 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3579 3580 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3581 &stats->vmrxs_ipackets, "Receive packets"); 3582 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3583 &stats->vmrxs_ibytes, "Receive bytes"); 3584 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3585 &stats->vmrxs_iqdrops, "Receive drops"); 3586 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3587 &stats->vmrxs_ierrors, "Receive errors"); 3588 3589 /* 3590 * Add statistics reported by the host. These are updated once 3591 * per second. 3592 */ 3593 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3594 NULL, "Host Statistics"); 3595 rxslist = SYSCTL_CHILDREN(rxsnode); 3596 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3597 &rxstats->LRO_packets, "LRO packets"); 3598 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3599 &rxstats->LRO_bytes, "LRO bytes"); 3600 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3601 &rxstats->ucast_packets, "Unicast packets"); 3602 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3603 &rxstats->ucast_bytes, "Unicast bytes"); 3604 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3605 &rxstats->mcast_packets, "Multicast packets"); 3606 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3607 &rxstats->mcast_bytes, "Multicast bytes"); 3608 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3609 &rxstats->bcast_packets, "Broadcast packets"); 3610 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3611 &rxstats->bcast_bytes, "Broadcast bytes"); 3612 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3613 &rxstats->nobuffer, "No buffer"); 3614 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3615 &rxstats->error, "Errors"); 3616 } 3617 3618 static void 3619 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3620 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3621 { 3622 struct sysctl_oid *node; 3623 struct sysctl_oid_list *list; 3624 int i; 3625 3626 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3627 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3628 3629 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3630 "debug", CTLFLAG_RD, NULL, ""); 3631 list = SYSCTL_CHILDREN(node); 3632 3633 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3634 &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3635 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3636 &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3637 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3638 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3639 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3640 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3641 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3642 &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3643 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3644 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3645 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3646 &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3647 } 3648 3649 for (i = 0; i < sc->vmx_nrxqueues; i++) { 3650 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3651 3652 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3653 "debug", CTLFLAG_RD, NULL, ""); 3654 list = SYSCTL_CHILDREN(node); 3655 3656 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3657 &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3658 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3659 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3660 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3661 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3662 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3663 &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3664 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3665 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3666 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3667 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3668 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3669 &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3670 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3671 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3672 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3673 &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3674 } 3675 } 3676 3677 static void 3678 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3679 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3680 { 3681 int i; 3682 3683 for (i = 0; i < sc->vmx_ntxqueues; i++) 3684 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3685 for (i = 0; i < sc->vmx_nrxqueues; i++) 3686 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3687 3688 vmxnet3_setup_debug_sysctl(sc, ctx, child); 3689 } 3690 3691 static void 3692 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3693 { 3694 device_t dev; 3695 struct vmxnet3_statistics *stats; 3696 struct sysctl_ctx_list *ctx; 3697 struct sysctl_oid *tree; 3698 struct sysctl_oid_list *child; 3699 3700 dev = sc->vmx_dev; 3701 ctx = device_get_sysctl_ctx(dev); 3702 tree = device_get_sysctl_tree(dev); 3703 child = SYSCTL_CHILDREN(tree); 3704 3705 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD, 3706 &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues"); 3707 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD, 3708 &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues"); 3709 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3710 &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3711 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3712 &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3713 3714 stats = &sc->vmx_stats; 3715 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD, 3716 &stats->vmst_defragged, 0, "Tx mbuf chains defragged"); 3717 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD, 3718 &stats->vmst_defrag_failed, 0, 3719 "Tx mbuf dropped because defrag failed"); 3720 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3721 &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3722 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3723 &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3724 3725 vmxnet3_setup_queue_sysctl(sc, ctx, child); 3726 } 3727 3728 static void 3729 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3730 { 3731 3732 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3733 } 3734 3735 static uint32_t 3736 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3737 { 3738 3739 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3740 } 3741 3742 static void 3743 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3744 { 3745 3746 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3747 } 3748 3749 static void 3750 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3751 { 3752 3753 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3754 } 3755 3756 static uint32_t 3757 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3758 { 3759 3760 vmxnet3_write_cmd(sc, cmd); 3761 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3762 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3763 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3764 } 3765 3766 static void 3767 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3768 { 3769 3770 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3771 } 3772 3773 static void 3774 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3775 { 3776 3777 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3778 } 3779 3780 static void 3781 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3782 { 3783 int i; 3784 3785 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3786 for (i = 0; i < sc->vmx_nintrs; i++) 3787 vmxnet3_enable_intr(sc, i); 3788 } 3789 3790 static void 3791 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3792 { 3793 int i; 3794 3795 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3796 for (i = 0; i < sc->vmx_nintrs; i++) 3797 vmxnet3_disable_intr(sc, i); 3798 } 3799 3800 static void 3801 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3802 { 3803 bus_addr_t *baddr = arg; 3804 3805 if (error == 0) 3806 *baddr = segs->ds_addr; 3807 } 3808 3809 static int 3810 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3811 struct vmxnet3_dma_alloc *dma) 3812 { 3813 device_t dev; 3814 int error; 3815 3816 dev = sc->vmx_dev; 3817 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3818 3819 error = bus_dma_tag_create(bus_get_dma_tag(dev), 3820 align, 0, /* alignment, bounds */ 3821 BUS_SPACE_MAXADDR, /* lowaddr */ 3822 BUS_SPACE_MAXADDR, /* highaddr */ 3823 NULL, NULL, /* filter, filterarg */ 3824 size, /* maxsize */ 3825 1, /* nsegments */ 3826 size, /* maxsegsize */ 3827 BUS_DMA_ALLOCNOW, /* flags */ 3828 NULL, /* lockfunc */ 3829 NULL, /* lockfuncarg */ 3830 &dma->dma_tag); 3831 if (error) { 3832 device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3833 goto fail; 3834 } 3835 3836 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3837 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3838 if (error) { 3839 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3840 goto fail; 3841 } 3842 3843 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3844 size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3845 if (error) { 3846 device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3847 goto fail; 3848 } 3849 3850 dma->dma_size = size; 3851 3852 fail: 3853 if (error) 3854 vmxnet3_dma_free(sc, dma); 3855 3856 return (error); 3857 } 3858 3859 static void 3860 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3861 { 3862 3863 if (dma->dma_tag != NULL) { 3864 if (dma->dma_paddr != 0) { 3865 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3866 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3867 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3868 } 3869 3870 if (dma->dma_vaddr != NULL) { 3871 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 3872 dma->dma_map); 3873 } 3874 3875 bus_dma_tag_destroy(dma->dma_tag); 3876 } 3877 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3878 } 3879 3880 static int 3881 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 3882 { 3883 char path[64]; 3884 3885 snprintf(path, sizeof(path), 3886 "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 3887 TUNABLE_INT_FETCH(path, &def); 3888 3889 return (def); 3890 } 3891 3892 /* 3893 * Since this is a purely paravirtualized device, we do not have 3894 * to worry about DMA coherency. But at times, we must make sure 3895 * both the compiler and CPU do not reorder memory operations. 3896 */ 3897 static inline void 3898 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 3899 { 3900 3901 switch (type) { 3902 case VMXNET3_BARRIER_RD: 3903 rmb(); 3904 break; 3905 case VMXNET3_BARRIER_WR: 3906 wmb(); 3907 break; 3908 case VMXNET3_BARRIER_RDWR: 3909 mb(); 3910 break; 3911 default: 3912 panic("%s: bad barrier type %d", __func__, type); 3913 } 3914 } 3915