1 /*- 2 * Copyright (c) 2013 Tsubai Masanari 3 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 * 17 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 18 */ 19 20 /* Driver for VMware vmxnet3 virtual ethernet devices. */ 21 22 #include <sys/cdefs.h> 23 __FBSDID("$FreeBSD$"); 24 25 #include <sys/param.h> 26 #include <sys/systm.h> 27 #include <sys/eventhandler.h> 28 #include <sys/kernel.h> 29 #include <sys/endian.h> 30 #include <sys/sockio.h> 31 #include <sys/mbuf.h> 32 #include <sys/malloc.h> 33 #include <sys/module.h> 34 #include <sys/socket.h> 35 #include <sys/sysctl.h> 36 #include <sys/smp.h> 37 #include <sys/taskqueue.h> 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <net/ethernet.h> 42 #include <net/if.h> 43 #include <net/if_var.h> 44 #include <net/if_arp.h> 45 #include <net/if_dl.h> 46 #include <net/if_types.h> 47 #include <net/if_media.h> 48 #include <net/if_vlan_var.h> 49 50 #include <net/bpf.h> 51 52 #include <netinet/in_systm.h> 53 #include <netinet/in.h> 54 #include <netinet/ip.h> 55 #include <netinet/ip6.h> 56 #include <netinet6/ip6_var.h> 57 #include <netinet/udp.h> 58 #include <netinet/tcp.h> 59 60 #include <machine/in_cksum.h> 61 62 #include <machine/bus.h> 63 #include <machine/resource.h> 64 #include <sys/bus.h> 65 #include <sys/rman.h> 66 67 #include <dev/pci/pcireg.h> 68 #include <dev/pci/pcivar.h> 69 70 #include "if_vmxreg.h" 71 #include "if_vmxvar.h" 72 73 #include "opt_inet.h" 74 #include "opt_inet6.h" 75 76 #ifdef VMXNET3_FAILPOINTS 77 #include <sys/fail.h> 78 static SYSCTL_NODE(DEBUG_FP, OID_AUTO, vmxnet3, CTLFLAG_RW, 0, 79 "vmxnet3 fail points"); 80 #define VMXNET3_FP _debug_fail_point_vmxnet3 81 #endif 82 83 static int vmxnet3_probe(device_t); 84 static int vmxnet3_attach(device_t); 85 static int vmxnet3_detach(device_t); 86 static int vmxnet3_shutdown(device_t); 87 88 static int vmxnet3_alloc_resources(struct vmxnet3_softc *); 89 static void vmxnet3_free_resources(struct vmxnet3_softc *); 90 static int vmxnet3_check_version(struct vmxnet3_softc *); 91 static void vmxnet3_initial_config(struct vmxnet3_softc *); 92 static void vmxnet3_check_multiqueue(struct vmxnet3_softc *); 93 94 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *); 95 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *); 96 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *); 97 static int vmxnet3_alloc_interrupt(struct vmxnet3_softc *, int, int, 98 struct vmxnet3_interrupt *); 99 static int vmxnet3_alloc_intr_resources(struct vmxnet3_softc *); 100 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *); 101 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *); 102 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *); 103 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *); 104 105 static void vmxnet3_free_interrupt(struct vmxnet3_softc *, 106 struct vmxnet3_interrupt *); 107 static void vmxnet3_free_interrupts(struct vmxnet3_softc *); 108 109 #ifndef VMXNET3_LEGACY_TX 110 static int vmxnet3_alloc_taskqueue(struct vmxnet3_softc *); 111 static void vmxnet3_start_taskqueue(struct vmxnet3_softc *); 112 static void vmxnet3_drain_taskqueue(struct vmxnet3_softc *); 113 static void vmxnet3_free_taskqueue(struct vmxnet3_softc *); 114 #endif 115 116 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int); 117 static int vmxnet3_init_txq(struct vmxnet3_softc *, int); 118 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *); 119 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *); 120 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *); 121 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *); 122 123 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 124 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 125 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *); 126 static void vmxnet3_free_txq_data(struct vmxnet3_softc *); 127 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *); 128 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *); 129 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *); 130 static void vmxnet3_free_queue_data(struct vmxnet3_softc *); 131 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 132 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 133 static void vmxnet3_reinit_interface(struct vmxnet3_softc *); 134 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 135 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 136 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 137 static void vmxnet3_free_data(struct vmxnet3_softc *); 138 static int vmxnet3_setup_interface(struct vmxnet3_softc *); 139 140 static void vmxnet3_evintr(struct vmxnet3_softc *); 141 static void vmxnet3_txq_eof(struct vmxnet3_txqueue *); 142 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *); 143 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *); 144 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *, 145 struct vmxnet3_rxring *, int); 146 static void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *); 147 static void vmxnet3_legacy_intr(void *); 148 static void vmxnet3_txq_intr(void *); 149 static void vmxnet3_rxq_intr(void *); 150 static void vmxnet3_event_intr(void *); 151 152 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 153 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 154 static void vmxnet3_stop(struct vmxnet3_softc *); 155 156 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 157 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 158 static int vmxnet3_reinit_queues(struct vmxnet3_softc *); 159 static int vmxnet3_enable_device(struct vmxnet3_softc *); 160 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 161 static int vmxnet3_reinit(struct vmxnet3_softc *); 162 static void vmxnet3_init_locked(struct vmxnet3_softc *); 163 static void vmxnet3_init(void *); 164 165 static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *,struct mbuf *, 166 int *, int *, int *); 167 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, 168 bus_dmamap_t, bus_dma_segment_t [], int *); 169 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t); 170 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **); 171 #ifdef VMXNET3_LEGACY_TX 172 static void vmxnet3_start_locked(struct ifnet *); 173 static void vmxnet3_start(struct ifnet *); 174 #else 175 static int vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *, 176 struct mbuf *); 177 static int vmxnet3_txq_mq_start(struct ifnet *, struct mbuf *); 178 static void vmxnet3_txq_tq_deferred(void *, int); 179 #endif 180 static void vmxnet3_txq_start(struct vmxnet3_txqueue *); 181 static void vmxnet3_tx_start_all(struct vmxnet3_softc *); 182 183 static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 184 uint16_t); 185 static void vmxnet3_register_vlan(void *, struct ifnet *, uint16_t); 186 static void vmxnet3_unregister_vlan(void *, struct ifnet *, uint16_t); 187 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *); 188 static int vmxnet3_change_mtu(struct vmxnet3_softc *, int); 189 static int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 190 191 #ifndef VMXNET3_LEGACY_TX 192 static void vmxnet3_qflush(struct ifnet *); 193 #endif 194 195 static int vmxnet3_watchdog(struct vmxnet3_txqueue *); 196 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 197 static void vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *, 198 struct vmxnet3_txq_stats *); 199 static void vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *, 200 struct vmxnet3_rxq_stats *); 201 static void vmxnet3_tick(void *); 202 static void vmxnet3_link_status(struct vmxnet3_softc *); 203 static void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 204 static int vmxnet3_media_change(struct ifnet *); 205 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 206 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 207 208 static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 209 struct sysctl_ctx_list *, struct sysctl_oid_list *); 210 static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 211 struct sysctl_ctx_list *, struct sysctl_oid_list *); 212 static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 213 struct sysctl_ctx_list *, struct sysctl_oid_list *); 214 static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 215 216 static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 217 uint32_t); 218 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 219 static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 220 uint32_t); 221 static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 222 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 223 224 static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 225 static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 226 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 227 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 228 229 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, 230 bus_size_t, struct vmxnet3_dma_alloc *); 231 static void vmxnet3_dma_free(struct vmxnet3_softc *, 232 struct vmxnet3_dma_alloc *); 233 static int vmxnet3_tunable_int(struct vmxnet3_softc *, 234 const char *, int); 235 236 typedef enum { 237 VMXNET3_BARRIER_RD, 238 VMXNET3_BARRIER_WR, 239 VMXNET3_BARRIER_RDWR, 240 } vmxnet3_barrier_t; 241 242 static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 243 244 /* Tunables. */ 245 static int vmxnet3_mq_disable = 0; 246 TUNABLE_INT("hw.vmx.mq_disable", &vmxnet3_mq_disable); 247 static int vmxnet3_default_txnqueue = VMXNET3_DEF_TX_QUEUES; 248 TUNABLE_INT("hw.vmx.txnqueue", &vmxnet3_default_txnqueue); 249 static int vmxnet3_default_rxnqueue = VMXNET3_DEF_RX_QUEUES; 250 TUNABLE_INT("hw.vmx.rxnqueue", &vmxnet3_default_rxnqueue); 251 static int vmxnet3_default_txndesc = VMXNET3_DEF_TX_NDESC; 252 TUNABLE_INT("hw.vmx.txndesc", &vmxnet3_default_txndesc); 253 static int vmxnet3_default_rxndesc = VMXNET3_DEF_RX_NDESC; 254 TUNABLE_INT("hw.vmx.rxndesc", &vmxnet3_default_rxndesc); 255 256 static device_method_t vmxnet3_methods[] = { 257 /* Device interface. */ 258 DEVMETHOD(device_probe, vmxnet3_probe), 259 DEVMETHOD(device_attach, vmxnet3_attach), 260 DEVMETHOD(device_detach, vmxnet3_detach), 261 DEVMETHOD(device_shutdown, vmxnet3_shutdown), 262 263 DEVMETHOD_END 264 }; 265 266 static driver_t vmxnet3_driver = { 267 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 268 }; 269 270 static devclass_t vmxnet3_devclass; 271 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 272 273 MODULE_DEPEND(vmx, pci, 1, 1, 1); 274 MODULE_DEPEND(vmx, ether, 1, 1, 1); 275 276 #define VMXNET3_VMWARE_VENDOR_ID 0x15AD 277 #define VMXNET3_VMWARE_DEVICE_ID 0x07B0 278 279 static int 280 vmxnet3_probe(device_t dev) 281 { 282 283 if (pci_get_vendor(dev) == VMXNET3_VMWARE_VENDOR_ID && 284 pci_get_device(dev) == VMXNET3_VMWARE_DEVICE_ID) { 285 device_set_desc(dev, "VMware VMXNET3 Ethernet Adapter"); 286 return (BUS_PROBE_DEFAULT); 287 } 288 289 return (ENXIO); 290 } 291 292 static int 293 vmxnet3_attach(device_t dev) 294 { 295 struct vmxnet3_softc *sc; 296 int error; 297 298 sc = device_get_softc(dev); 299 sc->vmx_dev = dev; 300 301 pci_enable_busmaster(dev); 302 303 VMXNET3_CORE_LOCK_INIT(sc, device_get_nameunit(dev)); 304 callout_init_mtx(&sc->vmx_tick, &sc->vmx_mtx, 0); 305 306 vmxnet3_initial_config(sc); 307 308 error = vmxnet3_alloc_resources(sc); 309 if (error) 310 goto fail; 311 312 error = vmxnet3_check_version(sc); 313 if (error) 314 goto fail; 315 316 error = vmxnet3_alloc_rxtx_queues(sc); 317 if (error) 318 goto fail; 319 320 #ifndef VMXNET3_LEGACY_TX 321 error = vmxnet3_alloc_taskqueue(sc); 322 if (error) 323 goto fail; 324 #endif 325 326 error = vmxnet3_alloc_interrupts(sc); 327 if (error) 328 goto fail; 329 330 vmxnet3_check_multiqueue(sc); 331 332 error = vmxnet3_alloc_data(sc); 333 if (error) 334 goto fail; 335 336 error = vmxnet3_setup_interface(sc); 337 if (error) 338 goto fail; 339 340 error = vmxnet3_setup_interrupts(sc); 341 if (error) { 342 ether_ifdetach(sc->vmx_ifp); 343 device_printf(dev, "could not set up interrupt\n"); 344 goto fail; 345 } 346 347 vmxnet3_setup_sysctl(sc); 348 #ifndef VMXNET3_LEGACY_TX 349 vmxnet3_start_taskqueue(sc); 350 #endif 351 352 fail: 353 if (error) 354 vmxnet3_detach(dev); 355 356 return (error); 357 } 358 359 static int 360 vmxnet3_detach(device_t dev) 361 { 362 struct vmxnet3_softc *sc; 363 struct ifnet *ifp; 364 365 sc = device_get_softc(dev); 366 ifp = sc->vmx_ifp; 367 368 if (device_is_attached(dev)) { 369 VMXNET3_CORE_LOCK(sc); 370 vmxnet3_stop(sc); 371 VMXNET3_CORE_UNLOCK(sc); 372 373 callout_drain(&sc->vmx_tick); 374 #ifndef VMXNET3_LEGACY_TX 375 vmxnet3_drain_taskqueue(sc); 376 #endif 377 378 ether_ifdetach(ifp); 379 } 380 381 if (sc->vmx_vlan_attach != NULL) { 382 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_attach); 383 sc->vmx_vlan_attach = NULL; 384 } 385 if (sc->vmx_vlan_detach != NULL) { 386 EVENTHANDLER_DEREGISTER(vlan_config, sc->vmx_vlan_detach); 387 sc->vmx_vlan_detach = NULL; 388 } 389 390 #ifndef VMXNET3_LEGACY_TX 391 vmxnet3_free_taskqueue(sc); 392 #endif 393 vmxnet3_free_interrupts(sc); 394 395 if (ifp != NULL) { 396 if_free(ifp); 397 sc->vmx_ifp = NULL; 398 } 399 400 ifmedia_removeall(&sc->vmx_media); 401 402 vmxnet3_free_data(sc); 403 vmxnet3_free_resources(sc); 404 vmxnet3_free_rxtx_queues(sc); 405 406 VMXNET3_CORE_LOCK_DESTROY(sc); 407 408 return (0); 409 } 410 411 static int 412 vmxnet3_shutdown(device_t dev) 413 { 414 415 return (0); 416 } 417 418 static int 419 vmxnet3_alloc_resources(struct vmxnet3_softc *sc) 420 { 421 device_t dev; 422 int rid; 423 424 dev = sc->vmx_dev; 425 426 rid = PCIR_BAR(0); 427 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 428 RF_ACTIVE); 429 if (sc->vmx_res0 == NULL) { 430 device_printf(dev, 431 "could not map BAR0 memory\n"); 432 return (ENXIO); 433 } 434 435 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 436 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 437 438 rid = PCIR_BAR(1); 439 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 440 RF_ACTIVE); 441 if (sc->vmx_res1 == NULL) { 442 device_printf(dev, 443 "could not map BAR1 memory\n"); 444 return (ENXIO); 445 } 446 447 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 448 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 449 450 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 451 rid = PCIR_BAR(2); 452 sc->vmx_msix_res = bus_alloc_resource_any(dev, 453 SYS_RES_MEMORY, &rid, RF_ACTIVE); 454 } 455 456 if (sc->vmx_msix_res == NULL) 457 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX; 458 459 return (0); 460 } 461 462 static void 463 vmxnet3_free_resources(struct vmxnet3_softc *sc) 464 { 465 device_t dev; 466 int rid; 467 468 dev = sc->vmx_dev; 469 470 if (sc->vmx_res0 != NULL) { 471 rid = PCIR_BAR(0); 472 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); 473 sc->vmx_res0 = NULL; 474 } 475 476 if (sc->vmx_res1 != NULL) { 477 rid = PCIR_BAR(1); 478 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); 479 sc->vmx_res1 = NULL; 480 } 481 482 if (sc->vmx_msix_res != NULL) { 483 rid = PCIR_BAR(2); 484 bus_release_resource(dev, SYS_RES_MEMORY, rid, 485 sc->vmx_msix_res); 486 sc->vmx_msix_res = NULL; 487 } 488 } 489 490 static int 491 vmxnet3_check_version(struct vmxnet3_softc *sc) 492 { 493 device_t dev; 494 uint32_t version; 495 496 dev = sc->vmx_dev; 497 498 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 499 if ((version & 0x01) == 0) { 500 device_printf(dev, "unsupported hardware version %#x\n", 501 version); 502 return (ENOTSUP); 503 } 504 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 505 506 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 507 if ((version & 0x01) == 0) { 508 device_printf(dev, "unsupported UPT version %#x\n", version); 509 return (ENOTSUP); 510 } 511 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 512 513 return (0); 514 } 515 516 static void 517 vmxnet3_initial_config(struct vmxnet3_softc *sc) 518 { 519 int nqueue, ndesc; 520 521 nqueue = vmxnet3_tunable_int(sc, "txnqueue", vmxnet3_default_txnqueue); 522 if (nqueue > VMXNET3_MAX_TX_QUEUES || nqueue < 1) 523 nqueue = VMXNET3_DEF_TX_QUEUES; 524 if (nqueue > mp_ncpus) 525 nqueue = mp_ncpus; 526 sc->vmx_max_ntxqueues = nqueue; 527 528 nqueue = vmxnet3_tunable_int(sc, "rxnqueue", vmxnet3_default_rxnqueue); 529 if (nqueue > VMXNET3_MAX_RX_QUEUES || nqueue < 1) 530 nqueue = VMXNET3_DEF_RX_QUEUES; 531 if (nqueue > mp_ncpus) 532 nqueue = mp_ncpus; 533 sc->vmx_max_nrxqueues = nqueue; 534 535 if (vmxnet3_tunable_int(sc, "mq_disable", vmxnet3_mq_disable)) { 536 sc->vmx_max_nrxqueues = 1; 537 sc->vmx_max_ntxqueues = 1; 538 } 539 540 ndesc = vmxnet3_tunable_int(sc, "txd", vmxnet3_default_txndesc); 541 if (ndesc > VMXNET3_MAX_TX_NDESC || ndesc < VMXNET3_MIN_TX_NDESC) 542 ndesc = VMXNET3_DEF_TX_NDESC; 543 if (ndesc & VMXNET3_MASK_TX_NDESC) 544 ndesc &= ~VMXNET3_MASK_TX_NDESC; 545 sc->vmx_ntxdescs = ndesc; 546 547 ndesc = vmxnet3_tunable_int(sc, "rxd", vmxnet3_default_rxndesc); 548 if (ndesc > VMXNET3_MAX_RX_NDESC || ndesc < VMXNET3_MIN_RX_NDESC) 549 ndesc = VMXNET3_DEF_RX_NDESC; 550 if (ndesc & VMXNET3_MASK_RX_NDESC) 551 ndesc &= ~VMXNET3_MASK_RX_NDESC; 552 sc->vmx_nrxdescs = ndesc; 553 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS; 554 } 555 556 static void 557 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc) 558 { 559 560 if (sc->vmx_intr_type != VMXNET3_IT_MSIX) 561 goto out; 562 563 /* BMV: Just use the maximum configured for now. */ 564 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues; 565 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues; 566 567 if (sc->vmx_nrxqueues > 1) 568 sc->vmx_flags |= VMXNET3_FLAG_RSS; 569 570 return; 571 572 out: 573 sc->vmx_ntxqueues = 1; 574 sc->vmx_nrxqueues = 1; 575 } 576 577 static int 578 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc) 579 { 580 device_t dev; 581 int nmsix, cnt, required; 582 583 dev = sc->vmx_dev; 584 585 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) 586 return (1); 587 588 /* Allocate an additional vector for the events interrupt. */ 589 required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1; 590 591 nmsix = pci_msix_count(dev); 592 if (nmsix < required) 593 return (1); 594 595 cnt = required; 596 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 597 sc->vmx_nintrs = required; 598 return (0); 599 } else 600 pci_release_msi(dev); 601 602 /* BMV TODO Fallback to sharing MSIX vectors if possible. */ 603 604 return (1); 605 } 606 607 static int 608 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc) 609 { 610 device_t dev; 611 int nmsi, cnt, required; 612 613 dev = sc->vmx_dev; 614 required = 1; 615 616 nmsi = pci_msi_count(dev); 617 if (nmsi < required) 618 return (1); 619 620 cnt = required; 621 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) { 622 sc->vmx_nintrs = 1; 623 return (0); 624 } else 625 pci_release_msi(dev); 626 627 return (1); 628 } 629 630 static int 631 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc) 632 { 633 634 sc->vmx_nintrs = 1; 635 return (0); 636 } 637 638 static int 639 vmxnet3_alloc_interrupt(struct vmxnet3_softc *sc, int rid, int flags, 640 struct vmxnet3_interrupt *intr) 641 { 642 struct resource *irq; 643 644 irq = bus_alloc_resource_any(sc->vmx_dev, SYS_RES_IRQ, &rid, flags); 645 if (irq == NULL) 646 return (ENXIO); 647 648 intr->vmxi_irq = irq; 649 intr->vmxi_rid = rid; 650 651 return (0); 652 } 653 654 static int 655 vmxnet3_alloc_intr_resources(struct vmxnet3_softc *sc) 656 { 657 int i, rid, flags, error; 658 659 rid = 0; 660 flags = RF_ACTIVE; 661 662 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) 663 flags |= RF_SHAREABLE; 664 else 665 rid = 1; 666 667 for (i = 0; i < sc->vmx_nintrs; i++, rid++) { 668 error = vmxnet3_alloc_interrupt(sc, rid, flags, 669 &sc->vmx_intrs[i]); 670 if (error) 671 return (error); 672 } 673 674 return (0); 675 } 676 677 static int 678 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc) 679 { 680 device_t dev; 681 struct vmxnet3_txqueue *txq; 682 struct vmxnet3_rxqueue *rxq; 683 struct vmxnet3_interrupt *intr; 684 enum intr_type type; 685 int i, error; 686 687 dev = sc->vmx_dev; 688 intr = &sc->vmx_intrs[0]; 689 type = INTR_TYPE_NET | INTR_MPSAFE; 690 691 for (i = 0; i < sc->vmx_ntxqueues; i++, intr++) { 692 txq = &sc->vmx_txq[i]; 693 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 694 vmxnet3_txq_intr, txq, &intr->vmxi_handler); 695 if (error) 696 return (error); 697 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 698 "tq%d", i); 699 txq->vxtxq_intr_idx = intr->vmxi_rid - 1; 700 } 701 702 for (i = 0; i < sc->vmx_nrxqueues; i++, intr++) { 703 rxq = &sc->vmx_rxq[i]; 704 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 705 vmxnet3_rxq_intr, rxq, &intr->vmxi_handler); 706 if (error) 707 return (error); 708 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, 709 "rq%d", i); 710 rxq->vxrxq_intr_idx = intr->vmxi_rid - 1; 711 } 712 713 error = bus_setup_intr(dev, intr->vmxi_irq, type, NULL, 714 vmxnet3_event_intr, sc, &intr->vmxi_handler); 715 if (error) 716 return (error); 717 bus_describe_intr(dev, intr->vmxi_irq, intr->vmxi_handler, "event"); 718 sc->vmx_event_intr_idx = intr->vmxi_rid - 1; 719 720 return (0); 721 } 722 723 static int 724 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc) 725 { 726 struct vmxnet3_interrupt *intr; 727 int i, error; 728 729 intr = &sc->vmx_intrs[0]; 730 error = bus_setup_intr(sc->vmx_dev, intr->vmxi_irq, 731 INTR_TYPE_NET | INTR_MPSAFE, NULL, vmxnet3_legacy_intr, sc, 732 &intr->vmxi_handler); 733 734 for (i = 0; i < sc->vmx_ntxqueues; i++) 735 sc->vmx_txq[i].vxtxq_intr_idx = 0; 736 for (i = 0; i < sc->vmx_nrxqueues; i++) 737 sc->vmx_rxq[i].vxrxq_intr_idx = 0; 738 sc->vmx_event_intr_idx = 0; 739 740 return (error); 741 } 742 743 static void 744 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 745 { 746 struct vmxnet3_txqueue *txq; 747 struct vmxnet3_txq_shared *txs; 748 struct vmxnet3_rxqueue *rxq; 749 struct vmxnet3_rxq_shared *rxs; 750 int i; 751 752 sc->vmx_ds->evintr = sc->vmx_event_intr_idx; 753 754 for (i = 0; i < sc->vmx_ntxqueues; i++) { 755 txq = &sc->vmx_txq[i]; 756 txs = txq->vxtxq_ts; 757 txs->intr_idx = txq->vxtxq_intr_idx; 758 } 759 760 for (i = 0; i < sc->vmx_nrxqueues; i++) { 761 rxq = &sc->vmx_rxq[i]; 762 rxs = rxq->vxrxq_rs; 763 rxs->intr_idx = rxq->vxrxq_intr_idx; 764 } 765 } 766 767 static int 768 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc) 769 { 770 int error; 771 772 error = vmxnet3_alloc_intr_resources(sc); 773 if (error) 774 return (error); 775 776 switch (sc->vmx_intr_type) { 777 case VMXNET3_IT_MSIX: 778 error = vmxnet3_setup_msix_interrupts(sc); 779 break; 780 case VMXNET3_IT_MSI: 781 case VMXNET3_IT_LEGACY: 782 error = vmxnet3_setup_legacy_interrupt(sc); 783 break; 784 default: 785 panic("%s: invalid interrupt type %d", __func__, 786 sc->vmx_intr_type); 787 } 788 789 if (error == 0) 790 vmxnet3_set_interrupt_idx(sc); 791 792 return (error); 793 } 794 795 static int 796 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc) 797 { 798 device_t dev; 799 uint32_t config; 800 int error; 801 802 dev = sc->vmx_dev; 803 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 804 805 sc->vmx_intr_type = config & 0x03; 806 sc->vmx_intr_mask_mode = (config >> 2) & 0x03; 807 808 switch (sc->vmx_intr_type) { 809 case VMXNET3_IT_AUTO: 810 sc->vmx_intr_type = VMXNET3_IT_MSIX; 811 /* FALLTHROUGH */ 812 case VMXNET3_IT_MSIX: 813 error = vmxnet3_alloc_msix_interrupts(sc); 814 if (error == 0) 815 break; 816 sc->vmx_intr_type = VMXNET3_IT_MSI; 817 /* FALLTHROUGH */ 818 case VMXNET3_IT_MSI: 819 error = vmxnet3_alloc_msi_interrupts(sc); 820 if (error == 0) 821 break; 822 sc->vmx_intr_type = VMXNET3_IT_LEGACY; 823 /* FALLTHROUGH */ 824 case VMXNET3_IT_LEGACY: 825 error = vmxnet3_alloc_legacy_interrupts(sc); 826 if (error == 0) 827 break; 828 /* FALLTHROUGH */ 829 default: 830 sc->vmx_intr_type = -1; 831 device_printf(dev, "cannot allocate any interrupt resources\n"); 832 return (ENXIO); 833 } 834 835 return (error); 836 } 837 838 static void 839 vmxnet3_free_interrupt(struct vmxnet3_softc *sc, 840 struct vmxnet3_interrupt *intr) 841 { 842 device_t dev; 843 844 dev = sc->vmx_dev; 845 846 if (intr->vmxi_handler != NULL) { 847 bus_teardown_intr(dev, intr->vmxi_irq, intr->vmxi_handler); 848 intr->vmxi_handler = NULL; 849 } 850 851 if (intr->vmxi_irq != NULL) { 852 bus_release_resource(dev, SYS_RES_IRQ, intr->vmxi_rid, 853 intr->vmxi_irq); 854 intr->vmxi_irq = NULL; 855 intr->vmxi_rid = -1; 856 } 857 } 858 859 static void 860 vmxnet3_free_interrupts(struct vmxnet3_softc *sc) 861 { 862 int i; 863 864 for (i = 0; i < sc->vmx_nintrs; i++) 865 vmxnet3_free_interrupt(sc, &sc->vmx_intrs[i]); 866 867 if (sc->vmx_intr_type == VMXNET3_IT_MSI || 868 sc->vmx_intr_type == VMXNET3_IT_MSIX) 869 pci_release_msi(sc->vmx_dev); 870 } 871 872 #ifndef VMXNET3_LEGACY_TX 873 static int 874 vmxnet3_alloc_taskqueue(struct vmxnet3_softc *sc) 875 { 876 device_t dev; 877 878 dev = sc->vmx_dev; 879 880 sc->vmx_tq = taskqueue_create(device_get_nameunit(dev), M_NOWAIT, 881 taskqueue_thread_enqueue, &sc->vmx_tq); 882 if (sc->vmx_tq == NULL) 883 return (ENOMEM); 884 885 return (0); 886 } 887 888 static void 889 vmxnet3_start_taskqueue(struct vmxnet3_softc *sc) 890 { 891 device_t dev; 892 int nthreads, error; 893 894 dev = sc->vmx_dev; 895 896 /* 897 * The taskqueue is typically not frequently used, so a dedicated 898 * thread for each queue is unnecessary. 899 */ 900 nthreads = MAX(1, sc->vmx_ntxqueues / 2); 901 902 /* 903 * Most drivers just ignore the return value - it only fails 904 * with ENOMEM so an error is not likely. It is hard for us 905 * to recover from an error here. 906 */ 907 error = taskqueue_start_threads(&sc->vmx_tq, nthreads, PI_NET, 908 "%s taskq", device_get_nameunit(dev)); 909 if (error) 910 device_printf(dev, "failed to start taskqueue: %d", error); 911 } 912 913 static void 914 vmxnet3_drain_taskqueue(struct vmxnet3_softc *sc) 915 { 916 struct vmxnet3_txqueue *txq; 917 int i; 918 919 if (sc->vmx_tq != NULL) { 920 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 921 txq = &sc->vmx_txq[i]; 922 taskqueue_drain(sc->vmx_tq, &txq->vxtxq_defrtask); 923 } 924 } 925 } 926 927 static void 928 vmxnet3_free_taskqueue(struct vmxnet3_softc *sc) 929 { 930 if (sc->vmx_tq != NULL) { 931 taskqueue_free(sc->vmx_tq); 932 sc->vmx_tq = NULL; 933 } 934 } 935 #endif 936 937 static int 938 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q) 939 { 940 struct vmxnet3_rxqueue *rxq; 941 struct vmxnet3_rxring *rxr; 942 int i; 943 944 rxq = &sc->vmx_rxq[q]; 945 946 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 947 device_get_nameunit(sc->vmx_dev), q); 948 mtx_init(&rxq->vxrxq_mtx, rxq->vxrxq_name, NULL, MTX_DEF); 949 950 rxq->vxrxq_sc = sc; 951 rxq->vxrxq_id = q; 952 953 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 954 rxr = &rxq->vxrxq_cmd_ring[i]; 955 rxr->vxrxr_rid = i; 956 rxr->vxrxr_ndesc = sc->vmx_nrxdescs; 957 rxr->vxrxr_rxbuf = malloc(rxr->vxrxr_ndesc * 958 sizeof(struct vmxnet3_rxbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 959 if (rxr->vxrxr_rxbuf == NULL) 960 return (ENOMEM); 961 962 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs; 963 } 964 965 return (0); 966 } 967 968 static int 969 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 970 { 971 struct vmxnet3_txqueue *txq; 972 struct vmxnet3_txring *txr; 973 974 txq = &sc->vmx_txq[q]; 975 txr = &txq->vxtxq_cmd_ring; 976 977 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 978 device_get_nameunit(sc->vmx_dev), q); 979 mtx_init(&txq->vxtxq_mtx, txq->vxtxq_name, NULL, MTX_DEF); 980 981 txq->vxtxq_sc = sc; 982 txq->vxtxq_id = q; 983 984 txr->vxtxr_ndesc = sc->vmx_ntxdescs; 985 txr->vxtxr_txbuf = malloc(txr->vxtxr_ndesc * 986 sizeof(struct vmxnet3_txbuf), M_DEVBUF, M_NOWAIT | M_ZERO); 987 if (txr->vxtxr_txbuf == NULL) 988 return (ENOMEM); 989 990 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs; 991 992 #ifndef VMXNET3_LEGACY_TX 993 TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq); 994 995 txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF, 996 M_NOWAIT, &txq->vxtxq_mtx); 997 if (txq->vxtxq_br == NULL) 998 return (ENOMEM); 999 #endif 1000 1001 return (0); 1002 } 1003 1004 static int 1005 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc) 1006 { 1007 int i, error; 1008 1009 /* 1010 * Only attempt to create multiple queues if MSIX is available. MSIX is 1011 * disabled by default because its apparently broken for devices passed 1012 * through by at least ESXi 5.1. The hw.pci.honor_msi_blacklist tunable 1013 * must be set to zero for MSIX. This check prevents us from allocating 1014 * queue structures that we will not use. 1015 */ 1016 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) { 1017 sc->vmx_max_nrxqueues = 1; 1018 sc->vmx_max_ntxqueues = 1; 1019 } 1020 1021 sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 1022 sc->vmx_max_nrxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1023 sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 1024 sc->vmx_max_ntxqueues, M_DEVBUF, M_NOWAIT | M_ZERO); 1025 if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL) 1026 return (ENOMEM); 1027 1028 for (i = 0; i < sc->vmx_max_nrxqueues; i++) { 1029 error = vmxnet3_init_rxq(sc, i); 1030 if (error) 1031 return (error); 1032 } 1033 1034 for (i = 0; i < sc->vmx_max_ntxqueues; i++) { 1035 error = vmxnet3_init_txq(sc, i); 1036 if (error) 1037 return (error); 1038 } 1039 1040 return (0); 1041 } 1042 1043 static void 1044 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq) 1045 { 1046 struct vmxnet3_rxring *rxr; 1047 int i; 1048 1049 rxq->vxrxq_sc = NULL; 1050 rxq->vxrxq_id = -1; 1051 1052 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1053 rxr = &rxq->vxrxq_cmd_ring[i]; 1054 1055 if (rxr->vxrxr_rxbuf != NULL) { 1056 free(rxr->vxrxr_rxbuf, M_DEVBUF); 1057 rxr->vxrxr_rxbuf = NULL; 1058 } 1059 } 1060 1061 if (mtx_initialized(&rxq->vxrxq_mtx) != 0) 1062 mtx_destroy(&rxq->vxrxq_mtx); 1063 } 1064 1065 static void 1066 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq) 1067 { 1068 struct vmxnet3_txring *txr; 1069 1070 txr = &txq->vxtxq_cmd_ring; 1071 1072 txq->vxtxq_sc = NULL; 1073 txq->vxtxq_id = -1; 1074 1075 #ifndef VMXNET3_LEGACY_TX 1076 if (txq->vxtxq_br != NULL) { 1077 buf_ring_free(txq->vxtxq_br, M_DEVBUF); 1078 txq->vxtxq_br = NULL; 1079 } 1080 #endif 1081 1082 if (txr->vxtxr_txbuf != NULL) { 1083 free(txr->vxtxr_txbuf, M_DEVBUF); 1084 txr->vxtxr_txbuf = NULL; 1085 } 1086 1087 if (mtx_initialized(&txq->vxtxq_mtx) != 0) 1088 mtx_destroy(&txq->vxtxq_mtx); 1089 } 1090 1091 static void 1092 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc) 1093 { 1094 int i; 1095 1096 if (sc->vmx_rxq != NULL) { 1097 for (i = 0; i < sc->vmx_max_nrxqueues; i++) 1098 vmxnet3_destroy_rxq(&sc->vmx_rxq[i]); 1099 free(sc->vmx_rxq, M_DEVBUF); 1100 sc->vmx_rxq = NULL; 1101 } 1102 1103 if (sc->vmx_txq != NULL) { 1104 for (i = 0; i < sc->vmx_max_ntxqueues; i++) 1105 vmxnet3_destroy_txq(&sc->vmx_txq[i]); 1106 free(sc->vmx_txq, M_DEVBUF); 1107 sc->vmx_txq = NULL; 1108 } 1109 } 1110 1111 static int 1112 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 1113 { 1114 device_t dev; 1115 uint8_t *kva; 1116 size_t size; 1117 int i, error; 1118 1119 dev = sc->vmx_dev; 1120 1121 size = sizeof(struct vmxnet3_driver_shared); 1122 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma); 1123 if (error) { 1124 device_printf(dev, "cannot alloc shared memory\n"); 1125 return (error); 1126 } 1127 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr; 1128 1129 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) + 1130 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared); 1131 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma); 1132 if (error) { 1133 device_printf(dev, "cannot alloc queue shared memory\n"); 1134 return (error); 1135 } 1136 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr; 1137 kva = sc->vmx_qs; 1138 1139 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1140 sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 1141 kva += sizeof(struct vmxnet3_txq_shared); 1142 } 1143 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1144 sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 1145 kva += sizeof(struct vmxnet3_rxq_shared); 1146 } 1147 1148 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1149 size = sizeof(struct vmxnet3_rss_shared); 1150 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma); 1151 if (error) { 1152 device_printf(dev, "cannot alloc rss shared memory\n"); 1153 return (error); 1154 } 1155 sc->vmx_rss = 1156 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr; 1157 } 1158 1159 return (0); 1160 } 1161 1162 static void 1163 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1164 { 1165 1166 if (sc->vmx_rss != NULL) { 1167 vmxnet3_dma_free(sc, &sc->vmx_rss_dma); 1168 sc->vmx_rss = NULL; 1169 } 1170 1171 if (sc->vmx_qs != NULL) { 1172 vmxnet3_dma_free(sc, &sc->vmx_qs_dma); 1173 sc->vmx_qs = NULL; 1174 } 1175 1176 if (sc->vmx_ds != NULL) { 1177 vmxnet3_dma_free(sc, &sc->vmx_ds_dma); 1178 sc->vmx_ds = NULL; 1179 } 1180 } 1181 1182 static int 1183 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc) 1184 { 1185 device_t dev; 1186 struct vmxnet3_txqueue *txq; 1187 struct vmxnet3_txring *txr; 1188 struct vmxnet3_comp_ring *txc; 1189 size_t descsz, compsz; 1190 int i, q, error; 1191 1192 dev = sc->vmx_dev; 1193 1194 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1195 txq = &sc->vmx_txq[q]; 1196 txr = &txq->vxtxq_cmd_ring; 1197 txc = &txq->vxtxq_comp_ring; 1198 1199 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc); 1200 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc); 1201 1202 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1203 1, 0, /* alignment, boundary */ 1204 BUS_SPACE_MAXADDR, /* lowaddr */ 1205 BUS_SPACE_MAXADDR, /* highaddr */ 1206 NULL, NULL, /* filter, filterarg */ 1207 VMXNET3_TX_MAXSIZE, /* maxsize */ 1208 VMXNET3_TX_MAXSEGS, /* nsegments */ 1209 VMXNET3_TX_MAXSEGSIZE, /* maxsegsize */ 1210 0, /* flags */ 1211 NULL, NULL, /* lockfunc, lockarg */ 1212 &txr->vxtxr_txtag); 1213 if (error) { 1214 device_printf(dev, 1215 "unable to create Tx buffer tag for queue %d\n", q); 1216 return (error); 1217 } 1218 1219 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma); 1220 if (error) { 1221 device_printf(dev, "cannot alloc Tx descriptors for " 1222 "queue %d error %d\n", q, error); 1223 return (error); 1224 } 1225 txr->vxtxr_txd = 1226 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr; 1227 1228 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma); 1229 if (error) { 1230 device_printf(dev, "cannot alloc Tx comp descriptors " 1231 "for queue %d error %d\n", q, error); 1232 return (error); 1233 } 1234 txc->vxcr_u.txcd = 1235 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr; 1236 1237 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1238 error = bus_dmamap_create(txr->vxtxr_txtag, 0, 1239 &txr->vxtxr_txbuf[i].vtxb_dmamap); 1240 if (error) { 1241 device_printf(dev, "unable to create Tx buf " 1242 "dmamap for queue %d idx %d\n", q, i); 1243 return (error); 1244 } 1245 } 1246 } 1247 1248 return (0); 1249 } 1250 1251 static void 1252 vmxnet3_free_txq_data(struct vmxnet3_softc *sc) 1253 { 1254 device_t dev; 1255 struct vmxnet3_txqueue *txq; 1256 struct vmxnet3_txring *txr; 1257 struct vmxnet3_comp_ring *txc; 1258 struct vmxnet3_txbuf *txb; 1259 int i, q; 1260 1261 dev = sc->vmx_dev; 1262 1263 for (q = 0; q < sc->vmx_ntxqueues; q++) { 1264 txq = &sc->vmx_txq[q]; 1265 txr = &txq->vxtxq_cmd_ring; 1266 txc = &txq->vxtxq_comp_ring; 1267 1268 for (i = 0; i < txr->vxtxr_ndesc; i++) { 1269 txb = &txr->vxtxr_txbuf[i]; 1270 if (txb->vtxb_dmamap != NULL) { 1271 bus_dmamap_destroy(txr->vxtxr_txtag, 1272 txb->vtxb_dmamap); 1273 txb->vtxb_dmamap = NULL; 1274 } 1275 } 1276 1277 if (txc->vxcr_u.txcd != NULL) { 1278 vmxnet3_dma_free(sc, &txc->vxcr_dma); 1279 txc->vxcr_u.txcd = NULL; 1280 } 1281 1282 if (txr->vxtxr_txd != NULL) { 1283 vmxnet3_dma_free(sc, &txr->vxtxr_dma); 1284 txr->vxtxr_txd = NULL; 1285 } 1286 1287 if (txr->vxtxr_txtag != NULL) { 1288 bus_dma_tag_destroy(txr->vxtxr_txtag); 1289 txr->vxtxr_txtag = NULL; 1290 } 1291 } 1292 } 1293 1294 static int 1295 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc) 1296 { 1297 device_t dev; 1298 struct vmxnet3_rxqueue *rxq; 1299 struct vmxnet3_rxring *rxr; 1300 struct vmxnet3_comp_ring *rxc; 1301 int descsz, compsz; 1302 int i, j, q, error; 1303 1304 dev = sc->vmx_dev; 1305 1306 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1307 rxq = &sc->vmx_rxq[q]; 1308 rxc = &rxq->vxrxq_comp_ring; 1309 compsz = 0; 1310 1311 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1312 rxr = &rxq->vxrxq_cmd_ring[i]; 1313 1314 descsz = rxr->vxrxr_ndesc * 1315 sizeof(struct vmxnet3_rxdesc); 1316 compsz += rxr->vxrxr_ndesc * 1317 sizeof(struct vmxnet3_rxcompdesc); 1318 1319 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1320 1, 0, /* alignment, boundary */ 1321 BUS_SPACE_MAXADDR, /* lowaddr */ 1322 BUS_SPACE_MAXADDR, /* highaddr */ 1323 NULL, NULL, /* filter, filterarg */ 1324 MJUMPAGESIZE, /* maxsize */ 1325 1, /* nsegments */ 1326 MJUMPAGESIZE, /* maxsegsize */ 1327 0, /* flags */ 1328 NULL, NULL, /* lockfunc, lockarg */ 1329 &rxr->vxrxr_rxtag); 1330 if (error) { 1331 device_printf(dev, 1332 "unable to create Rx buffer tag for " 1333 "queue %d\n", q); 1334 return (error); 1335 } 1336 1337 error = vmxnet3_dma_malloc(sc, descsz, 512, 1338 &rxr->vxrxr_dma); 1339 if (error) { 1340 device_printf(dev, "cannot allocate Rx " 1341 "descriptors for queue %d/%d error %d\n", 1342 i, q, error); 1343 return (error); 1344 } 1345 rxr->vxrxr_rxd = 1346 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr; 1347 } 1348 1349 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma); 1350 if (error) { 1351 device_printf(dev, "cannot alloc Rx comp descriptors " 1352 "for queue %d error %d\n", q, error); 1353 return (error); 1354 } 1355 rxc->vxcr_u.rxcd = 1356 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr; 1357 1358 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1359 rxr = &rxq->vxrxq_cmd_ring[i]; 1360 1361 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1362 &rxr->vxrxr_spare_dmap); 1363 if (error) { 1364 device_printf(dev, "unable to create spare " 1365 "dmamap for queue %d/%d error %d\n", 1366 q, i, error); 1367 return (error); 1368 } 1369 1370 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1371 error = bus_dmamap_create(rxr->vxrxr_rxtag, 0, 1372 &rxr->vxrxr_rxbuf[j].vrxb_dmamap); 1373 if (error) { 1374 device_printf(dev, "unable to create " 1375 "dmamap for queue %d/%d slot %d " 1376 "error %d\n", 1377 q, i, j, error); 1378 return (error); 1379 } 1380 } 1381 } 1382 } 1383 1384 return (0); 1385 } 1386 1387 static void 1388 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc) 1389 { 1390 device_t dev; 1391 struct vmxnet3_rxqueue *rxq; 1392 struct vmxnet3_rxring *rxr; 1393 struct vmxnet3_comp_ring *rxc; 1394 struct vmxnet3_rxbuf *rxb; 1395 int i, j, q; 1396 1397 dev = sc->vmx_dev; 1398 1399 for (q = 0; q < sc->vmx_nrxqueues; q++) { 1400 rxq = &sc->vmx_rxq[q]; 1401 rxc = &rxq->vxrxq_comp_ring; 1402 1403 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1404 rxr = &rxq->vxrxq_cmd_ring[i]; 1405 1406 if (rxr->vxrxr_spare_dmap != NULL) { 1407 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1408 rxr->vxrxr_spare_dmap); 1409 rxr->vxrxr_spare_dmap = NULL; 1410 } 1411 1412 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 1413 rxb = &rxr->vxrxr_rxbuf[j]; 1414 if (rxb->vrxb_dmamap != NULL) { 1415 bus_dmamap_destroy(rxr->vxrxr_rxtag, 1416 rxb->vrxb_dmamap); 1417 rxb->vrxb_dmamap = NULL; 1418 } 1419 } 1420 } 1421 1422 if (rxc->vxcr_u.rxcd != NULL) { 1423 vmxnet3_dma_free(sc, &rxc->vxcr_dma); 1424 rxc->vxcr_u.rxcd = NULL; 1425 } 1426 1427 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 1428 rxr = &rxq->vxrxq_cmd_ring[i]; 1429 1430 if (rxr->vxrxr_rxd != NULL) { 1431 vmxnet3_dma_free(sc, &rxr->vxrxr_dma); 1432 rxr->vxrxr_rxd = NULL; 1433 } 1434 1435 if (rxr->vxrxr_rxtag != NULL) { 1436 bus_dma_tag_destroy(rxr->vxrxr_rxtag); 1437 rxr->vxrxr_rxtag = NULL; 1438 } 1439 } 1440 } 1441 } 1442 1443 static int 1444 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc) 1445 { 1446 int error; 1447 1448 error = vmxnet3_alloc_txq_data(sc); 1449 if (error) 1450 return (error); 1451 1452 error = vmxnet3_alloc_rxq_data(sc); 1453 if (error) 1454 return (error); 1455 1456 return (0); 1457 } 1458 1459 static void 1460 vmxnet3_free_queue_data(struct vmxnet3_softc *sc) 1461 { 1462 1463 if (sc->vmx_rxq != NULL) 1464 vmxnet3_free_rxq_data(sc); 1465 1466 if (sc->vmx_txq != NULL) 1467 vmxnet3_free_txq_data(sc); 1468 } 1469 1470 static int 1471 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1472 { 1473 int error; 1474 1475 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 1476 32, &sc->vmx_mcast_dma); 1477 if (error) 1478 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1479 else 1480 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr; 1481 1482 return (error); 1483 } 1484 1485 static void 1486 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1487 { 1488 1489 if (sc->vmx_mcast != NULL) { 1490 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma); 1491 sc->vmx_mcast = NULL; 1492 } 1493 } 1494 1495 static void 1496 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1497 { 1498 struct vmxnet3_driver_shared *ds; 1499 struct vmxnet3_txqueue *txq; 1500 struct vmxnet3_txq_shared *txs; 1501 struct vmxnet3_rxqueue *rxq; 1502 struct vmxnet3_rxq_shared *rxs; 1503 int i; 1504 1505 ds = sc->vmx_ds; 1506 1507 /* 1508 * Initialize fields of the shared data that remains the same across 1509 * reinits. Note the shared data is zero'd when allocated. 1510 */ 1511 1512 ds->magic = VMXNET3_REV1_MAGIC; 1513 1514 /* DriverInfo */ 1515 ds->version = VMXNET3_DRIVER_VERSION; 1516 ds->guest = VMXNET3_GOS_FREEBSD | 1517 #ifdef __LP64__ 1518 VMXNET3_GOS_64BIT; 1519 #else 1520 VMXNET3_GOS_32BIT; 1521 #endif 1522 ds->vmxnet3_revision = 1; 1523 ds->upt_version = 1; 1524 1525 /* Misc. conf */ 1526 ds->driver_data = vtophys(sc); 1527 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1528 ds->queue_shared = sc->vmx_qs_dma.dma_paddr; 1529 ds->queue_shared_len = sc->vmx_qs_dma.dma_size; 1530 ds->nrxsg_max = sc->vmx_max_rxsegs; 1531 1532 /* RSS conf */ 1533 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1534 ds->rss.version = 1; 1535 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr; 1536 ds->rss.len = sc->vmx_rss_dma.dma_size; 1537 } 1538 1539 /* Interrupt control. */ 1540 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1541 ds->nintr = sc->vmx_nintrs; 1542 ds->evintr = sc->vmx_event_intr_idx; 1543 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1544 1545 for (i = 0; i < sc->vmx_nintrs; i++) 1546 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1547 1548 /* Receive filter. */ 1549 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr; 1550 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size; 1551 1552 /* Tx queues */ 1553 for (i = 0; i < sc->vmx_ntxqueues; i++) { 1554 txq = &sc->vmx_txq[i]; 1555 txs = txq->vxtxq_ts; 1556 1557 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr; 1558 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1559 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr; 1560 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1561 txs->driver_data = vtophys(txq); 1562 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1563 } 1564 1565 /* Rx queues */ 1566 for (i = 0; i < sc->vmx_nrxqueues; i++) { 1567 rxq = &sc->vmx_rxq[i]; 1568 rxs = rxq->vxrxq_rs; 1569 1570 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr; 1571 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1572 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr; 1573 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1574 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr; 1575 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1576 rxs->driver_data = vtophys(rxq); 1577 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1578 } 1579 } 1580 1581 static void 1582 vmxnet3_reinit_interface(struct vmxnet3_softc *sc) 1583 { 1584 struct ifnet *ifp; 1585 1586 ifp = sc->vmx_ifp; 1587 1588 /* Use the current MAC address. */ 1589 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1590 vmxnet3_set_lladdr(sc); 1591 1592 ifp->if_hwassist = 0; 1593 if (ifp->if_capenable & IFCAP_TXCSUM) 1594 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD; 1595 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1596 ifp->if_hwassist |= VMXNET3_CSUM_OFFLOAD_IPV6; 1597 if (ifp->if_capenable & IFCAP_TSO4) 1598 ifp->if_hwassist |= CSUM_IP_TSO; 1599 if (ifp->if_capenable & IFCAP_TSO6) 1600 ifp->if_hwassist |= CSUM_IP6_TSO; 1601 } 1602 1603 static void 1604 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1605 { 1606 /* 1607 * Use the same key as the Linux driver until FreeBSD can do 1608 * RSS (presumably Toeplitz) in software. 1609 */ 1610 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1611 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1612 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1613 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1614 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1615 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1616 }; 1617 1618 struct vmxnet3_driver_shared *ds; 1619 struct vmxnet3_rss_shared *rss; 1620 int i; 1621 1622 ds = sc->vmx_ds; 1623 rss = sc->vmx_rss; 1624 1625 rss->hash_type = 1626 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1627 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1628 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1629 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1630 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1631 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1632 1633 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1634 rss->ind_table[i] = i % sc->vmx_nrxqueues; 1635 } 1636 1637 static void 1638 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1639 { 1640 struct ifnet *ifp; 1641 struct vmxnet3_driver_shared *ds; 1642 1643 ifp = sc->vmx_ifp; 1644 ds = sc->vmx_ds; 1645 1646 ds->mtu = ifp->if_mtu; 1647 ds->ntxqueue = sc->vmx_ntxqueues; 1648 ds->nrxqueue = sc->vmx_nrxqueues; 1649 1650 ds->upt_features = 0; 1651 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1652 ds->upt_features |= UPT1_F_CSUM; 1653 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1654 ds->upt_features |= UPT1_F_VLAN; 1655 if (ifp->if_capenable & IFCAP_LRO) 1656 ds->upt_features |= UPT1_F_LRO; 1657 1658 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1659 ds->upt_features |= UPT1_F_RSS; 1660 vmxnet3_reinit_rss_shared_data(sc); 1661 } 1662 1663 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr); 1664 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1665 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32); 1666 } 1667 1668 static int 1669 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1670 { 1671 int error; 1672 1673 error = vmxnet3_alloc_shared_data(sc); 1674 if (error) 1675 return (error); 1676 1677 error = vmxnet3_alloc_queue_data(sc); 1678 if (error) 1679 return (error); 1680 1681 error = vmxnet3_alloc_mcast_table(sc); 1682 if (error) 1683 return (error); 1684 1685 vmxnet3_init_shared_data(sc); 1686 1687 return (0); 1688 } 1689 1690 static void 1691 vmxnet3_free_data(struct vmxnet3_softc *sc) 1692 { 1693 1694 vmxnet3_free_mcast_table(sc); 1695 vmxnet3_free_queue_data(sc); 1696 vmxnet3_free_shared_data(sc); 1697 } 1698 1699 static int 1700 vmxnet3_setup_interface(struct vmxnet3_softc *sc) 1701 { 1702 device_t dev; 1703 struct ifnet *ifp; 1704 1705 dev = sc->vmx_dev; 1706 1707 ifp = sc->vmx_ifp = if_alloc(IFT_ETHER); 1708 if (ifp == NULL) { 1709 device_printf(dev, "cannot allocate ifnet structure\n"); 1710 return (ENOSPC); 1711 } 1712 1713 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1714 #if __FreeBSD_version < 1000025 1715 ifp->if_baudrate = 1000000000; 1716 #elif __FreeBSD_version < 1100011 1717 if_initbaudrate(ifp, IF_Gbps(10)); 1718 #else 1719 ifp->if_baudrate = IF_Gbps(10); 1720 #endif 1721 ifp->if_softc = sc; 1722 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1723 ifp->if_init = vmxnet3_init; 1724 ifp->if_ioctl = vmxnet3_ioctl; 1725 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1726 ifp->if_hw_tsomaxsegcount = VMXNET3_TX_MAXSEGS; 1727 ifp->if_hw_tsomaxsegsize = VMXNET3_TX_MAXSEGSIZE; 1728 1729 #ifdef VMXNET3_LEGACY_TX 1730 ifp->if_start = vmxnet3_start; 1731 ifp->if_snd.ifq_drv_maxlen = sc->vmx_ntxdescs - 1; 1732 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs - 1); 1733 IFQ_SET_READY(&ifp->if_snd); 1734 #else 1735 ifp->if_transmit = vmxnet3_txq_mq_start; 1736 ifp->if_qflush = vmxnet3_qflush; 1737 #endif 1738 1739 vmxnet3_get_lladdr(sc); 1740 ether_ifattach(ifp, sc->vmx_lladdr); 1741 1742 ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; 1743 ifp->if_capabilities |= IFCAP_RXCSUM_IPV6 | IFCAP_TXCSUM_IPV6; 1744 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6; 1745 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 1746 IFCAP_VLAN_HWCSUM; 1747 ifp->if_capenable = ifp->if_capabilities; 1748 1749 /* These capabilities are not enabled by default. */ 1750 ifp->if_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER; 1751 1752 sc->vmx_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1753 vmxnet3_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1754 sc->vmx_vlan_detach = EVENTHANDLER_REGISTER(vlan_config, 1755 vmxnet3_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1756 1757 ifmedia_init(&sc->vmx_media, 0, vmxnet3_media_change, 1758 vmxnet3_media_status); 1759 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1760 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO); 1761 1762 return (0); 1763 } 1764 1765 static void 1766 vmxnet3_evintr(struct vmxnet3_softc *sc) 1767 { 1768 device_t dev; 1769 struct ifnet *ifp; 1770 struct vmxnet3_txq_shared *ts; 1771 struct vmxnet3_rxq_shared *rs; 1772 uint32_t event; 1773 int reset; 1774 1775 dev = sc->vmx_dev; 1776 ifp = sc->vmx_ifp; 1777 reset = 0; 1778 1779 VMXNET3_CORE_LOCK(sc); 1780 1781 /* Clear events. */ 1782 event = sc->vmx_ds->event; 1783 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1784 1785 if (event & VMXNET3_EVENT_LINK) { 1786 vmxnet3_link_status(sc); 1787 if (sc->vmx_link_active != 0) 1788 vmxnet3_tx_start_all(sc); 1789 } 1790 1791 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1792 reset = 1; 1793 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1794 ts = sc->vmx_txq[0].vxtxq_ts; 1795 if (ts->stopped != 0) 1796 device_printf(dev, "Tx queue error %#x\n", ts->error); 1797 rs = sc->vmx_rxq[0].vxrxq_rs; 1798 if (rs->stopped != 0) 1799 device_printf(dev, "Rx queue error %#x\n", rs->error); 1800 device_printf(dev, "Rx/Tx queue error event ... resetting\n"); 1801 } 1802 1803 if (event & VMXNET3_EVENT_DIC) 1804 device_printf(dev, "device implementation change event\n"); 1805 if (event & VMXNET3_EVENT_DEBUG) 1806 device_printf(dev, "debug event\n"); 1807 1808 if (reset != 0) { 1809 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1810 vmxnet3_init_locked(sc); 1811 } 1812 1813 VMXNET3_CORE_UNLOCK(sc); 1814 } 1815 1816 static void 1817 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq) 1818 { 1819 struct vmxnet3_softc *sc; 1820 struct ifnet *ifp; 1821 struct vmxnet3_txring *txr; 1822 struct vmxnet3_comp_ring *txc; 1823 struct vmxnet3_txcompdesc *txcd; 1824 struct vmxnet3_txbuf *txb; 1825 struct mbuf *m; 1826 u_int sop; 1827 1828 sc = txq->vxtxq_sc; 1829 ifp = sc->vmx_ifp; 1830 txr = &txq->vxtxq_cmd_ring; 1831 txc = &txq->vxtxq_comp_ring; 1832 1833 VMXNET3_TXQ_LOCK_ASSERT(txq); 1834 1835 for (;;) { 1836 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1837 if (txcd->gen != txc->vxcr_gen) 1838 break; 1839 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1840 1841 if (++txc->vxcr_next == txc->vxcr_ndesc) { 1842 txc->vxcr_next = 0; 1843 txc->vxcr_gen ^= 1; 1844 } 1845 1846 sop = txr->vxtxr_next; 1847 txb = &txr->vxtxr_txbuf[sop]; 1848 1849 if ((m = txb->vtxb_m) != NULL) { 1850 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 1851 BUS_DMASYNC_POSTWRITE); 1852 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 1853 1854 txq->vxtxq_stats.vmtxs_opackets++; 1855 txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len; 1856 if (m->m_flags & M_MCAST) 1857 txq->vxtxq_stats.vmtxs_omcasts++; 1858 1859 m_freem(m); 1860 txb->vtxb_m = NULL; 1861 } 1862 1863 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1864 } 1865 1866 if (txr->vxtxr_head == txr->vxtxr_next) 1867 txq->vxtxq_watchdog = 0; 1868 } 1869 1870 static int 1871 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr) 1872 { 1873 struct ifnet *ifp; 1874 struct mbuf *m; 1875 struct vmxnet3_rxdesc *rxd; 1876 struct vmxnet3_rxbuf *rxb; 1877 bus_dma_tag_t tag; 1878 bus_dmamap_t dmap; 1879 bus_dma_segment_t segs[1]; 1880 int idx, clsize, btype, flags, nsegs, error; 1881 1882 ifp = sc->vmx_ifp; 1883 tag = rxr->vxrxr_rxtag; 1884 dmap = rxr->vxrxr_spare_dmap; 1885 idx = rxr->vxrxr_fill; 1886 rxd = &rxr->vxrxr_rxd[idx]; 1887 rxb = &rxr->vxrxr_rxbuf[idx]; 1888 1889 #ifdef VMXNET3_FAILPOINTS 1890 KFAIL_POINT_CODE(VMXNET3_FP, newbuf, return ENOBUFS); 1891 if (rxr->vxrxr_rid != 0) 1892 KFAIL_POINT_CODE(VMXNET3_FP, newbuf_body_only, return ENOBUFS); 1893 #endif 1894 1895 if (rxr->vxrxr_rid == 0 && (idx % sc->vmx_rx_max_chain) == 0) { 1896 flags = M_PKTHDR; 1897 clsize = MCLBYTES; 1898 btype = VMXNET3_BTYPE_HEAD; 1899 } else { 1900 #if __FreeBSD_version < 902001 1901 /* 1902 * These mbufs will never be used for the start of a frame. 1903 * Roughly prior to branching releng/9.2, the load_mbuf_sg() 1904 * required the mbuf to always be a packet header. Avoid 1905 * unnecessary mbuf initialization in newer versions where 1906 * that is not the case. 1907 */ 1908 flags = M_PKTHDR; 1909 #else 1910 flags = 0; 1911 #endif 1912 clsize = MJUMPAGESIZE; 1913 btype = VMXNET3_BTYPE_BODY; 1914 } 1915 1916 m = m_getjcl(M_NOWAIT, MT_DATA, flags, clsize); 1917 if (m == NULL) { 1918 sc->vmx_stats.vmst_mgetcl_failed++; 1919 return (ENOBUFS); 1920 } 1921 1922 if (btype == VMXNET3_BTYPE_HEAD) { 1923 m->m_len = m->m_pkthdr.len = clsize; 1924 m_adj(m, ETHER_ALIGN); 1925 } else 1926 m->m_len = clsize; 1927 1928 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, &segs[0], &nsegs, 1929 BUS_DMA_NOWAIT); 1930 if (error) { 1931 m_freem(m); 1932 sc->vmx_stats.vmst_mbuf_load_failed++; 1933 return (error); 1934 } 1935 KASSERT(nsegs == 1, 1936 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 1937 #if __FreeBSD_version < 902001 1938 if (btype == VMXNET3_BTYPE_BODY) 1939 m->m_flags &= ~M_PKTHDR; 1940 #endif 1941 1942 if (rxb->vrxb_m != NULL) { 1943 bus_dmamap_sync(tag, rxb->vrxb_dmamap, BUS_DMASYNC_POSTREAD); 1944 bus_dmamap_unload(tag, rxb->vrxb_dmamap); 1945 } 1946 1947 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap; 1948 rxb->vrxb_dmamap = dmap; 1949 rxb->vrxb_m = m; 1950 1951 rxd->addr = segs[0].ds_addr; 1952 rxd->len = segs[0].ds_len; 1953 rxd->btype = btype; 1954 rxd->gen = rxr->vxrxr_gen; 1955 1956 vmxnet3_rxr_increment_fill(rxr); 1957 return (0); 1958 } 1959 1960 static void 1961 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq, 1962 struct vmxnet3_rxring *rxr, int idx) 1963 { 1964 struct vmxnet3_rxdesc *rxd; 1965 1966 rxd = &rxr->vxrxr_rxd[idx]; 1967 rxd->gen = rxr->vxrxr_gen; 1968 vmxnet3_rxr_increment_fill(rxr); 1969 } 1970 1971 static void 1972 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq) 1973 { 1974 struct vmxnet3_softc *sc; 1975 struct vmxnet3_rxring *rxr; 1976 struct vmxnet3_comp_ring *rxc; 1977 struct vmxnet3_rxcompdesc *rxcd; 1978 int idx, eof; 1979 1980 sc = rxq->vxrxq_sc; 1981 rxc = &rxq->vxrxq_comp_ring; 1982 1983 do { 1984 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 1985 if (rxcd->gen != rxc->vxcr_gen) 1986 break; /* Not expected. */ 1987 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1988 1989 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 1990 rxc->vxcr_next = 0; 1991 rxc->vxcr_gen ^= 1; 1992 } 1993 1994 idx = rxcd->rxd_idx; 1995 eof = rxcd->eop; 1996 if (rxcd->qid < sc->vmx_nrxqueues) 1997 rxr = &rxq->vxrxq_cmd_ring[0]; 1998 else 1999 rxr = &rxq->vxrxq_cmd_ring[1]; 2000 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2001 } while (!eof); 2002 } 2003 2004 static void 2005 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2006 { 2007 2008 if (rxcd->ipv4) { 2009 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 2010 if (rxcd->ipcsum_ok) 2011 m->m_pkthdr.csum_flags |= CSUM_IP_VALID; 2012 } 2013 2014 if (!rxcd->fragment) { 2015 if (rxcd->csum_ok && (rxcd->tcp || rxcd->udp)) { 2016 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 2017 CSUM_PSEUDO_HDR; 2018 m->m_pkthdr.csum_data = 0xFFFF; 2019 } 2020 } 2021 } 2022 2023 static void 2024 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq, 2025 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 2026 { 2027 struct vmxnet3_softc *sc; 2028 struct ifnet *ifp; 2029 2030 sc = rxq->vxrxq_sc; 2031 ifp = sc->vmx_ifp; 2032 2033 if (rxcd->error) { 2034 rxq->vxrxq_stats.vmrxs_ierrors++; 2035 m_freem(m); 2036 return; 2037 } 2038 2039 #ifdef notyet 2040 switch (rxcd->rss_type) { 2041 case VMXNET3_RCD_RSS_TYPE_IPV4: 2042 m->m_pkthdr.flowid = rxcd->rss_hash; 2043 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV4); 2044 break; 2045 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 2046 m->m_pkthdr.flowid = rxcd->rss_hash; 2047 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV4); 2048 break; 2049 case VMXNET3_RCD_RSS_TYPE_IPV6: 2050 m->m_pkthdr.flowid = rxcd->rss_hash; 2051 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_IPV6); 2052 break; 2053 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 2054 m->m_pkthdr.flowid = rxcd->rss_hash; 2055 M_HASHTYPE_SET(m, M_HASHTYPE_RSS_TCP_IPV6); 2056 break; 2057 default: /* VMXNET3_RCD_RSS_TYPE_NONE */ 2058 m->m_pkthdr.flowid = rxq->vxrxq_id; 2059 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2060 break; 2061 } 2062 #else 2063 m->m_pkthdr.flowid = rxq->vxrxq_id; 2064 m->m_flags |= M_FLOWID; 2065 #endif 2066 2067 if (!rxcd->no_csum) 2068 vmxnet3_rx_csum(rxcd, m); 2069 if (rxcd->vlan) { 2070 m->m_flags |= M_VLANTAG; 2071 m->m_pkthdr.ether_vtag = rxcd->vtag; 2072 } 2073 2074 rxq->vxrxq_stats.vmrxs_ipackets++; 2075 rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len; 2076 2077 VMXNET3_RXQ_UNLOCK(rxq); 2078 (*ifp->if_input)(ifp, m); 2079 VMXNET3_RXQ_LOCK(rxq); 2080 } 2081 2082 static void 2083 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq) 2084 { 2085 struct vmxnet3_softc *sc; 2086 struct ifnet *ifp; 2087 struct vmxnet3_rxring *rxr; 2088 struct vmxnet3_comp_ring *rxc; 2089 struct vmxnet3_rxdesc *rxd; 2090 struct vmxnet3_rxcompdesc *rxcd; 2091 struct mbuf *m, *m_head, *m_tail; 2092 int idx, length; 2093 2094 sc = rxq->vxrxq_sc; 2095 ifp = sc->vmx_ifp; 2096 rxc = &rxq->vxrxq_comp_ring; 2097 2098 VMXNET3_RXQ_LOCK_ASSERT(rxq); 2099 2100 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2101 return; 2102 2103 m_head = rxq->vxrxq_mhead; 2104 rxq->vxrxq_mhead = NULL; 2105 m_tail = rxq->vxrxq_mtail; 2106 rxq->vxrxq_mtail = NULL; 2107 MPASS(m_head == NULL || m_tail != NULL); 2108 2109 for (;;) { 2110 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next]; 2111 if (rxcd->gen != rxc->vxcr_gen) { 2112 rxq->vxrxq_mhead = m_head; 2113 rxq->vxrxq_mtail = m_tail; 2114 break; 2115 } 2116 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 2117 2118 if (++rxc->vxcr_next == rxc->vxcr_ndesc) { 2119 rxc->vxcr_next = 0; 2120 rxc->vxcr_gen ^= 1; 2121 } 2122 2123 idx = rxcd->rxd_idx; 2124 length = rxcd->len; 2125 if (rxcd->qid < sc->vmx_nrxqueues) 2126 rxr = &rxq->vxrxq_cmd_ring[0]; 2127 else 2128 rxr = &rxq->vxrxq_cmd_ring[1]; 2129 rxd = &rxr->vxrxr_rxd[idx]; 2130 2131 m = rxr->vxrxr_rxbuf[idx].vrxb_m; 2132 KASSERT(m != NULL, ("%s: queue %d idx %d without mbuf", 2133 __func__, rxcd->qid, idx)); 2134 2135 /* 2136 * The host may skip descriptors. We detect this when this 2137 * descriptor does not match the previous fill index. Catch 2138 * up with the host now. 2139 */ 2140 if (__predict_false(rxr->vxrxr_fill != idx)) { 2141 while (rxr->vxrxr_fill != idx) { 2142 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen = 2143 rxr->vxrxr_gen; 2144 vmxnet3_rxr_increment_fill(rxr); 2145 } 2146 } 2147 2148 if (rxcd->sop) { 2149 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD, 2150 ("%s: start of frame w/o head buffer", __func__)); 2151 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0], 2152 ("%s: start of frame not in ring 0", __func__)); 2153 KASSERT((idx % sc->vmx_rx_max_chain) == 0, 2154 ("%s: start of frame at unexcepted index %d (%d)", 2155 __func__, idx, sc->vmx_rx_max_chain)); 2156 KASSERT(m_head == NULL, 2157 ("%s: duplicate start of frame?", __func__)); 2158 2159 if (length == 0) { 2160 /* Just ignore this descriptor. */ 2161 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2162 goto nextp; 2163 } 2164 2165 if (vmxnet3_newbuf(sc, rxr) != 0) { 2166 rxq->vxrxq_stats.vmrxs_iqdrops++; 2167 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2168 if (!rxcd->eop) 2169 vmxnet3_rxq_discard_chain(rxq); 2170 goto nextp; 2171 } 2172 2173 m->m_pkthdr.rcvif = ifp; 2174 m->m_pkthdr.len = m->m_len = length; 2175 m->m_pkthdr.csum_flags = 0; 2176 m_head = m_tail = m; 2177 2178 } else { 2179 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY, 2180 ("%s: non start of frame w/o body buffer", __func__)); 2181 KASSERT(m_head != NULL, 2182 ("%s: frame not started?", __func__)); 2183 2184 if (vmxnet3_newbuf(sc, rxr) != 0) { 2185 rxq->vxrxq_stats.vmrxs_iqdrops++; 2186 vmxnet3_rxq_eof_discard(rxq, rxr, idx); 2187 if (!rxcd->eop) 2188 vmxnet3_rxq_discard_chain(rxq); 2189 m_freem(m_head); 2190 m_head = m_tail = NULL; 2191 goto nextp; 2192 } 2193 2194 m->m_len = length; 2195 m_head->m_pkthdr.len += length; 2196 m_tail->m_next = m; 2197 m_tail = m; 2198 } 2199 2200 if (rxcd->eop) { 2201 vmxnet3_rxq_input(rxq, rxcd, m_head); 2202 m_head = m_tail = NULL; 2203 2204 /* Must recheck after dropping the Rx lock. */ 2205 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 2206 break; 2207 } 2208 2209 nextp: 2210 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) { 2211 int qid = rxcd->qid; 2212 bus_size_t r; 2213 2214 idx = (idx + 1) % rxr->vxrxr_ndesc; 2215 if (qid >= sc->vmx_nrxqueues) { 2216 qid -= sc->vmx_nrxqueues; 2217 r = VMXNET3_BAR0_RXH2(qid); 2218 } else 2219 r = VMXNET3_BAR0_RXH1(qid); 2220 vmxnet3_write_bar0(sc, r, idx); 2221 } 2222 } 2223 } 2224 2225 static void 2226 vmxnet3_legacy_intr(void *xsc) 2227 { 2228 struct vmxnet3_softc *sc; 2229 struct vmxnet3_rxqueue *rxq; 2230 struct vmxnet3_txqueue *txq; 2231 2232 sc = xsc; 2233 rxq = &sc->vmx_rxq[0]; 2234 txq = &sc->vmx_txq[0]; 2235 2236 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) { 2237 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 2238 return; 2239 } 2240 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2241 vmxnet3_disable_all_intrs(sc); 2242 2243 if (sc->vmx_ds->event != 0) 2244 vmxnet3_evintr(sc); 2245 2246 VMXNET3_RXQ_LOCK(rxq); 2247 vmxnet3_rxq_eof(rxq); 2248 VMXNET3_RXQ_UNLOCK(rxq); 2249 2250 VMXNET3_TXQ_LOCK(txq); 2251 vmxnet3_txq_eof(txq); 2252 vmxnet3_txq_start(txq); 2253 VMXNET3_TXQ_UNLOCK(txq); 2254 2255 vmxnet3_enable_all_intrs(sc); 2256 } 2257 2258 static void 2259 vmxnet3_txq_intr(void *xtxq) 2260 { 2261 struct vmxnet3_softc *sc; 2262 struct vmxnet3_txqueue *txq; 2263 2264 txq = xtxq; 2265 sc = txq->vxtxq_sc; 2266 2267 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2268 vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx); 2269 2270 VMXNET3_TXQ_LOCK(txq); 2271 vmxnet3_txq_eof(txq); 2272 vmxnet3_txq_start(txq); 2273 VMXNET3_TXQ_UNLOCK(txq); 2274 2275 vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx); 2276 } 2277 2278 static void 2279 vmxnet3_rxq_intr(void *xrxq) 2280 { 2281 struct vmxnet3_softc *sc; 2282 struct vmxnet3_rxqueue *rxq; 2283 2284 rxq = xrxq; 2285 sc = rxq->vxrxq_sc; 2286 2287 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2288 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 2289 2290 VMXNET3_RXQ_LOCK(rxq); 2291 vmxnet3_rxq_eof(rxq); 2292 VMXNET3_RXQ_UNLOCK(rxq); 2293 2294 vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx); 2295 } 2296 2297 static void 2298 vmxnet3_event_intr(void *xsc) 2299 { 2300 struct vmxnet3_softc *sc; 2301 2302 sc = xsc; 2303 2304 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 2305 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 2306 2307 if (sc->vmx_ds->event != 0) 2308 vmxnet3_evintr(sc); 2309 2310 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2311 } 2312 2313 static void 2314 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2315 { 2316 struct vmxnet3_txring *txr; 2317 struct vmxnet3_txbuf *txb; 2318 int i; 2319 2320 txr = &txq->vxtxq_cmd_ring; 2321 2322 for (i = 0; i < txr->vxtxr_ndesc; i++) { 2323 txb = &txr->vxtxr_txbuf[i]; 2324 2325 if (txb->vtxb_m == NULL) 2326 continue; 2327 2328 bus_dmamap_sync(txr->vxtxr_txtag, txb->vtxb_dmamap, 2329 BUS_DMASYNC_POSTWRITE); 2330 bus_dmamap_unload(txr->vxtxr_txtag, txb->vtxb_dmamap); 2331 m_freem(txb->vtxb_m); 2332 txb->vtxb_m = NULL; 2333 } 2334 } 2335 2336 static void 2337 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2338 { 2339 struct vmxnet3_rxring *rxr; 2340 struct vmxnet3_rxbuf *rxb; 2341 int i, j; 2342 2343 if (rxq->vxrxq_mhead != NULL) { 2344 m_freem(rxq->vxrxq_mhead); 2345 rxq->vxrxq_mhead = NULL; 2346 rxq->vxrxq_mtail = NULL; 2347 } 2348 2349 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) { 2350 rxr = &rxq->vxrxq_cmd_ring[i]; 2351 2352 for (j = 0; j < rxr->vxrxr_ndesc; j++) { 2353 rxb = &rxr->vxrxr_rxbuf[j]; 2354 2355 if (rxb->vrxb_m == NULL) 2356 continue; 2357 2358 bus_dmamap_sync(rxr->vxrxr_rxtag, rxb->vrxb_dmamap, 2359 BUS_DMASYNC_POSTREAD); 2360 bus_dmamap_unload(rxr->vxrxr_rxtag, rxb->vrxb_dmamap); 2361 m_freem(rxb->vrxb_m); 2362 rxb->vrxb_m = NULL; 2363 } 2364 } 2365 } 2366 2367 static void 2368 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc) 2369 { 2370 struct vmxnet3_rxqueue *rxq; 2371 struct vmxnet3_txqueue *txq; 2372 int i; 2373 2374 for (i = 0; i < sc->vmx_nrxqueues; i++) { 2375 rxq = &sc->vmx_rxq[i]; 2376 VMXNET3_RXQ_LOCK(rxq); 2377 VMXNET3_RXQ_UNLOCK(rxq); 2378 } 2379 2380 for (i = 0; i < sc->vmx_ntxqueues; i++) { 2381 txq = &sc->vmx_txq[i]; 2382 VMXNET3_TXQ_LOCK(txq); 2383 VMXNET3_TXQ_UNLOCK(txq); 2384 } 2385 } 2386 2387 static void 2388 vmxnet3_stop(struct vmxnet3_softc *sc) 2389 { 2390 struct ifnet *ifp; 2391 int q; 2392 2393 ifp = sc->vmx_ifp; 2394 VMXNET3_CORE_LOCK_ASSERT(sc); 2395 2396 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2397 sc->vmx_link_active = 0; 2398 callout_stop(&sc->vmx_tick); 2399 2400 /* Disable interrupts. */ 2401 vmxnet3_disable_all_intrs(sc); 2402 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 2403 2404 vmxnet3_stop_rendezvous(sc); 2405 2406 for (q = 0; q < sc->vmx_ntxqueues; q++) 2407 vmxnet3_txstop(sc, &sc->vmx_txq[q]); 2408 for (q = 0; q < sc->vmx_nrxqueues; q++) 2409 vmxnet3_rxstop(sc, &sc->vmx_rxq[q]); 2410 2411 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 2412 } 2413 2414 static void 2415 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 2416 { 2417 struct vmxnet3_txring *txr; 2418 struct vmxnet3_comp_ring *txc; 2419 2420 txr = &txq->vxtxq_cmd_ring; 2421 txr->vxtxr_head = 0; 2422 txr->vxtxr_next = 0; 2423 txr->vxtxr_gen = VMXNET3_INIT_GEN; 2424 bzero(txr->vxtxr_txd, 2425 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc)); 2426 2427 txc = &txq->vxtxq_comp_ring; 2428 txc->vxcr_next = 0; 2429 txc->vxcr_gen = VMXNET3_INIT_GEN; 2430 bzero(txc->vxcr_u.txcd, 2431 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc)); 2432 } 2433 2434 static int 2435 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 2436 { 2437 struct ifnet *ifp; 2438 struct vmxnet3_rxring *rxr; 2439 struct vmxnet3_comp_ring *rxc; 2440 int i, populate, idx, frame_size, error; 2441 2442 ifp = sc->vmx_ifp; 2443 frame_size = ETHER_ALIGN + sizeof(struct ether_vlan_header) + 2444 ifp->if_mtu; 2445 2446 /* 2447 * If the MTU causes us to exceed what a regular sized cluster can 2448 * handle, we allocate a second MJUMPAGESIZE cluster after it in 2449 * ring 0. If in use, ring 1 always contains MJUMPAGESIZE clusters. 2450 * 2451 * Keep rx_max_chain a divisor of the maximum Rx ring size to make 2452 * our life easier. We do not support changing the ring size after 2453 * the attach. 2454 */ 2455 if (frame_size <= MCLBYTES) 2456 sc->vmx_rx_max_chain = 1; 2457 else 2458 sc->vmx_rx_max_chain = 2; 2459 2460 /* 2461 * Only populate ring 1 if the configuration will take advantage 2462 * of it. That is either when LRO is enabled or the frame size 2463 * exceeds what ring 0 can contain. 2464 */ 2465 if ((ifp->if_capenable & IFCAP_LRO) == 0 && 2466 frame_size <= MCLBYTES + MJUMPAGESIZE) 2467 populate = 1; 2468 else 2469 populate = VMXNET3_RXRINGS_PERQ; 2470 2471 for (i = 0; i < populate; i++) { 2472 rxr = &rxq->vxrxq_cmd_ring[i]; 2473 rxr->vxrxr_fill = 0; 2474 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 2475 bzero(rxr->vxrxr_rxd, 2476 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2477 2478 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) { 2479 error = vmxnet3_newbuf(sc, rxr); 2480 if (error) 2481 return (error); 2482 } 2483 } 2484 2485 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 2486 rxr = &rxq->vxrxq_cmd_ring[i]; 2487 rxr->vxrxr_fill = 0; 2488 rxr->vxrxr_gen = 0; 2489 bzero(rxr->vxrxr_rxd, 2490 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 2491 } 2492 2493 rxc = &rxq->vxrxq_comp_ring; 2494 rxc->vxcr_next = 0; 2495 rxc->vxcr_gen = VMXNET3_INIT_GEN; 2496 bzero(rxc->vxcr_u.rxcd, 2497 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc)); 2498 2499 return (0); 2500 } 2501 2502 static int 2503 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 2504 { 2505 device_t dev; 2506 int q, error; 2507 2508 dev = sc->vmx_dev; 2509 2510 for (q = 0; q < sc->vmx_ntxqueues; q++) 2511 vmxnet3_txinit(sc, &sc->vmx_txq[q]); 2512 2513 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2514 error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 2515 if (error) { 2516 device_printf(dev, "cannot populate Rx queue %d\n", q); 2517 return (error); 2518 } 2519 } 2520 2521 return (0); 2522 } 2523 2524 static int 2525 vmxnet3_enable_device(struct vmxnet3_softc *sc) 2526 { 2527 int q; 2528 2529 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 2530 device_printf(sc->vmx_dev, "device enable command failed!\n"); 2531 return (1); 2532 } 2533 2534 /* Reset the Rx queue heads. */ 2535 for (q = 0; q < sc->vmx_nrxqueues; q++) { 2536 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 2537 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 2538 } 2539 2540 return (0); 2541 } 2542 2543 static void 2544 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 2545 { 2546 struct ifnet *ifp; 2547 2548 ifp = sc->vmx_ifp; 2549 2550 vmxnet3_set_rxfilter(sc); 2551 2552 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2553 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 2554 sizeof(sc->vmx_ds->vlan_filter)); 2555 else 2556 bzero(sc->vmx_ds->vlan_filter, 2557 sizeof(sc->vmx_ds->vlan_filter)); 2558 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 2559 } 2560 2561 static int 2562 vmxnet3_reinit(struct vmxnet3_softc *sc) 2563 { 2564 2565 vmxnet3_reinit_interface(sc); 2566 vmxnet3_reinit_shared_data(sc); 2567 2568 if (vmxnet3_reinit_queues(sc) != 0) 2569 return (ENXIO); 2570 2571 if (vmxnet3_enable_device(sc) != 0) 2572 return (ENXIO); 2573 2574 vmxnet3_reinit_rxfilters(sc); 2575 2576 return (0); 2577 } 2578 2579 static void 2580 vmxnet3_init_locked(struct vmxnet3_softc *sc) 2581 { 2582 struct ifnet *ifp; 2583 2584 ifp = sc->vmx_ifp; 2585 2586 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 2587 return; 2588 2589 vmxnet3_stop(sc); 2590 2591 if (vmxnet3_reinit(sc) != 0) { 2592 vmxnet3_stop(sc); 2593 return; 2594 } 2595 2596 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2597 vmxnet3_link_status(sc); 2598 2599 vmxnet3_enable_all_intrs(sc); 2600 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 2601 } 2602 2603 static void 2604 vmxnet3_init(void *xsc) 2605 { 2606 struct vmxnet3_softc *sc; 2607 2608 sc = xsc; 2609 2610 VMXNET3_CORE_LOCK(sc); 2611 vmxnet3_init_locked(sc); 2612 VMXNET3_CORE_UNLOCK(sc); 2613 } 2614 2615 /* 2616 * BMV: Much of this can go away once we finally have offsets in 2617 * the mbuf packet header. Bug andre@. 2618 */ 2619 static int 2620 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m, 2621 int *etype, int *proto, int *start) 2622 { 2623 struct ether_vlan_header *evh; 2624 int offset; 2625 #if defined(INET) 2626 struct ip *ip = NULL; 2627 struct ip iphdr; 2628 #endif 2629 #if defined(INET6) 2630 struct ip6_hdr *ip6 = NULL; 2631 struct ip6_hdr ip6hdr; 2632 #endif 2633 2634 evh = mtod(m, struct ether_vlan_header *); 2635 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2636 /* BMV: We should handle nested VLAN tags too. */ 2637 *etype = ntohs(evh->evl_proto); 2638 offset = sizeof(struct ether_vlan_header); 2639 } else { 2640 *etype = ntohs(evh->evl_encap_proto); 2641 offset = sizeof(struct ether_header); 2642 } 2643 2644 switch (*etype) { 2645 #if defined(INET) 2646 case ETHERTYPE_IP: 2647 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2648 m_copydata(m, offset, sizeof(struct ip), 2649 (caddr_t) &iphdr); 2650 ip = &iphdr; 2651 } else 2652 ip = mtodo(m, offset); 2653 *proto = ip->ip_p; 2654 *start = offset + (ip->ip_hl << 2); 2655 break; 2656 #endif 2657 #if defined(INET6) 2658 case ETHERTYPE_IPV6: 2659 if (__predict_false(m->m_len < 2660 offset + sizeof(struct ip6_hdr))) { 2661 m_copydata(m, offset, sizeof(struct ip6_hdr), 2662 (caddr_t) &ip6hdr); 2663 ip6 = &ip6hdr; 2664 } else 2665 ip6 = mtodo(m, offset); 2666 *proto = -1; 2667 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2668 /* Assert the network stack sent us a valid packet. */ 2669 KASSERT(*start > offset, 2670 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2671 *start, offset, *proto)); 2672 break; 2673 #endif 2674 default: 2675 return (EINVAL); 2676 } 2677 2678 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2679 struct tcphdr *tcp, tcphdr; 2680 uint16_t sum; 2681 2682 if (__predict_false(*proto != IPPROTO_TCP)) { 2683 /* Likely failed to correctly parse the mbuf. */ 2684 return (EINVAL); 2685 } 2686 2687 txq->vxtxq_stats.vmtxs_tso++; 2688 2689 switch (*etype) { 2690 #if defined(INET) 2691 case ETHERTYPE_IP: 2692 sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, 2693 htons(IPPROTO_TCP)); 2694 break; 2695 #endif 2696 #if defined(INET6) 2697 case ETHERTYPE_IPV6: 2698 sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 2699 break; 2700 #endif 2701 default: 2702 sum = 0; 2703 break; 2704 } 2705 2706 if (m->m_len < *start + sizeof(struct tcphdr)) { 2707 m_copyback(m, *start + offsetof(struct tcphdr, th_sum), 2708 sizeof(uint16_t), (caddr_t) &sum); 2709 m_copydata(m, *start, sizeof(struct tcphdr), 2710 (caddr_t) &tcphdr); 2711 tcp = &tcphdr; 2712 } else { 2713 tcp = mtodo(m, *start); 2714 tcp->th_sum = sum; 2715 } 2716 2717 /* 2718 * For TSO, the size of the protocol header is also 2719 * included in the descriptor header size. 2720 */ 2721 *start += (tcp->th_off << 2); 2722 } else 2723 txq->vxtxq_stats.vmtxs_csum++; 2724 2725 return (0); 2726 } 2727 2728 static int 2729 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0, 2730 bus_dmamap_t dmap, bus_dma_segment_t segs[], int *nsegs) 2731 { 2732 struct vmxnet3_txring *txr; 2733 struct mbuf *m; 2734 bus_dma_tag_t tag; 2735 int error; 2736 2737 txr = &txq->vxtxq_cmd_ring; 2738 m = *m0; 2739 tag = txr->vxtxr_txtag; 2740 2741 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2742 if (error == 0 || error != EFBIG) 2743 return (error); 2744 2745 m = m_defrag(m, M_NOWAIT); 2746 if (m != NULL) { 2747 *m0 = m; 2748 error = bus_dmamap_load_mbuf_sg(tag, dmap, m, segs, nsegs, 0); 2749 } else 2750 error = ENOBUFS; 2751 2752 if (error) { 2753 m_freem(*m0); 2754 *m0 = NULL; 2755 txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++; 2756 } else 2757 txq->vxtxq_sc->vmx_stats.vmst_defragged++; 2758 2759 return (error); 2760 } 2761 2762 static void 2763 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap) 2764 { 2765 struct vmxnet3_txring *txr; 2766 2767 txr = &txq->vxtxq_cmd_ring; 2768 bus_dmamap_unload(txr->vxtxr_txtag, dmap); 2769 } 2770 2771 static int 2772 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0) 2773 { 2774 struct vmxnet3_softc *sc; 2775 struct vmxnet3_txring *txr; 2776 struct vmxnet3_txdesc *txd, *sop; 2777 struct mbuf *m; 2778 bus_dmamap_t dmap; 2779 bus_dma_segment_t segs[VMXNET3_TX_MAXSEGS]; 2780 int i, gen, nsegs, etype, proto, start, error; 2781 2782 sc = txq->vxtxq_sc; 2783 start = 0; 2784 txd = NULL; 2785 txr = &txq->vxtxq_cmd_ring; 2786 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap; 2787 2788 error = vmxnet3_txq_load_mbuf(txq, m0, dmap, segs, &nsegs); 2789 if (error) 2790 return (error); 2791 2792 m = *m0; 2793 M_ASSERTPKTHDR(m); 2794 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 2795 ("%s: mbuf %p with too many segments %d", __func__, m, nsegs)); 2796 2797 if (VMXNET3_TXRING_AVAIL(txr) < nsegs) { 2798 txq->vxtxq_stats.vmtxs_full++; 2799 vmxnet3_txq_unload_mbuf(txq, dmap); 2800 return (ENOSPC); 2801 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) { 2802 error = vmxnet3_txq_offload_ctx(txq, m, &etype, &proto, &start); 2803 if (error) { 2804 txq->vxtxq_stats.vmtxs_offload_failed++; 2805 vmxnet3_txq_unload_mbuf(txq, dmap); 2806 m_freem(m); 2807 *m0 = NULL; 2808 return (error); 2809 } 2810 } 2811 2812 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m; 2813 sop = &txr->vxtxr_txd[txr->vxtxr_head]; 2814 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 2815 2816 for (i = 0; i < nsegs; i++) { 2817 txd = &txr->vxtxr_txd[txr->vxtxr_head]; 2818 2819 txd->addr = segs[i].ds_addr; 2820 txd->len = segs[i].ds_len; 2821 txd->gen = gen; 2822 txd->dtype = 0; 2823 txd->offload_mode = VMXNET3_OM_NONE; 2824 txd->offload_pos = 0; 2825 txd->hlen = 0; 2826 txd->eop = 0; 2827 txd->compreq = 0; 2828 txd->vtag_mode = 0; 2829 txd->vtag = 0; 2830 2831 if (++txr->vxtxr_head == txr->vxtxr_ndesc) { 2832 txr->vxtxr_head = 0; 2833 txr->vxtxr_gen ^= 1; 2834 } 2835 gen = txr->vxtxr_gen; 2836 } 2837 txd->eop = 1; 2838 txd->compreq = 1; 2839 2840 if (m->m_flags & M_VLANTAG) { 2841 sop->vtag_mode = 1; 2842 sop->vtag = m->m_pkthdr.ether_vtag; 2843 } 2844 2845 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 2846 sop->offload_mode = VMXNET3_OM_TSO; 2847 sop->hlen = start; 2848 sop->offload_pos = m->m_pkthdr.tso_segsz; 2849 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD | 2850 VMXNET3_CSUM_OFFLOAD_IPV6)) { 2851 sop->offload_mode = VMXNET3_OM_CSUM; 2852 sop->hlen = start; 2853 sop->offload_pos = start + m->m_pkthdr.csum_data; 2854 } 2855 2856 /* Finally, change the ownership. */ 2857 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 2858 sop->gen ^= 1; 2859 2860 txq->vxtxq_ts->npending += nsegs; 2861 if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) { 2862 txq->vxtxq_ts->npending = 0; 2863 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), 2864 txr->vxtxr_head); 2865 } 2866 2867 return (0); 2868 } 2869 2870 #ifdef VMXNET3_LEGACY_TX 2871 2872 static void 2873 vmxnet3_start_locked(struct ifnet *ifp) 2874 { 2875 struct vmxnet3_softc *sc; 2876 struct vmxnet3_txqueue *txq; 2877 struct vmxnet3_txring *txr; 2878 struct mbuf *m_head; 2879 int tx, avail; 2880 2881 sc = ifp->if_softc; 2882 txq = &sc->vmx_txq[0]; 2883 txr = &txq->vxtxq_cmd_ring; 2884 tx = 0; 2885 2886 VMXNET3_TXQ_LOCK_ASSERT(txq); 2887 2888 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2889 sc->vmx_link_active == 0) 2890 return; 2891 2892 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 2893 if ((avail = VMXNET3_TXRING_AVAIL(txr)) < 2) 2894 break; 2895 2896 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 2897 if (m_head == NULL) 2898 break; 2899 2900 /* Assume worse case if this mbuf is the head of a chain. */ 2901 if (m_head->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2902 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2903 break; 2904 } 2905 2906 if (vmxnet3_txq_encap(txq, &m_head) != 0) { 2907 if (m_head != NULL) 2908 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 2909 break; 2910 } 2911 2912 tx++; 2913 ETHER_BPF_MTAP(ifp, m_head); 2914 } 2915 2916 if (tx > 0) 2917 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2918 } 2919 2920 static void 2921 vmxnet3_start(struct ifnet *ifp) 2922 { 2923 struct vmxnet3_softc *sc; 2924 struct vmxnet3_txqueue *txq; 2925 2926 sc = ifp->if_softc; 2927 txq = &sc->vmx_txq[0]; 2928 2929 VMXNET3_TXQ_LOCK(txq); 2930 vmxnet3_start_locked(ifp); 2931 VMXNET3_TXQ_UNLOCK(txq); 2932 } 2933 2934 #else /* !VMXNET3_LEGACY_TX */ 2935 2936 static int 2937 vmxnet3_txq_mq_start_locked(struct vmxnet3_txqueue *txq, struct mbuf *m) 2938 { 2939 struct vmxnet3_softc *sc; 2940 struct vmxnet3_txring *txr; 2941 struct buf_ring *br; 2942 struct ifnet *ifp; 2943 int tx, avail, error; 2944 2945 sc = txq->vxtxq_sc; 2946 br = txq->vxtxq_br; 2947 ifp = sc->vmx_ifp; 2948 txr = &txq->vxtxq_cmd_ring; 2949 tx = 0; 2950 error = 0; 2951 2952 VMXNET3_TXQ_LOCK_ASSERT(txq); 2953 2954 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 2955 sc->vmx_link_active == 0) { 2956 if (m != NULL) 2957 error = drbr_enqueue(ifp, br, m); 2958 return (error); 2959 } 2960 2961 if (m != NULL) { 2962 error = drbr_enqueue(ifp, br, m); 2963 if (error) 2964 return (error); 2965 } 2966 2967 while ((avail = VMXNET3_TXRING_AVAIL(txr)) >= 2) { 2968 m = drbr_peek(ifp, br); 2969 if (m == NULL) 2970 break; 2971 2972 /* Assume worse case if this mbuf is the head of a chain. */ 2973 if (m->m_next != NULL && avail < VMXNET3_TX_MAXSEGS) { 2974 drbr_putback(ifp, br, m); 2975 break; 2976 } 2977 2978 if (vmxnet3_txq_encap(txq, &m) != 0) { 2979 if (m != NULL) 2980 drbr_putback(ifp, br, m); 2981 else 2982 drbr_advance(ifp, br); 2983 break; 2984 } 2985 drbr_advance(ifp, br); 2986 2987 tx++; 2988 ETHER_BPF_MTAP(ifp, m); 2989 } 2990 2991 if (tx > 0) 2992 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT; 2993 2994 return (0); 2995 } 2996 2997 static int 2998 vmxnet3_txq_mq_start(struct ifnet *ifp, struct mbuf *m) 2999 { 3000 struct vmxnet3_softc *sc; 3001 struct vmxnet3_txqueue *txq; 3002 int i, ntxq, error; 3003 3004 sc = ifp->if_softc; 3005 ntxq = sc->vmx_ntxqueues; 3006 3007 if (m->m_flags & M_FLOWID) 3008 i = m->m_pkthdr.flowid % ntxq; 3009 else 3010 i = curcpu % ntxq; 3011 3012 txq = &sc->vmx_txq[i]; 3013 3014 if (VMXNET3_TXQ_TRYLOCK(txq) != 0) { 3015 error = vmxnet3_txq_mq_start_locked(txq, m); 3016 VMXNET3_TXQ_UNLOCK(txq); 3017 } else { 3018 error = drbr_enqueue(ifp, txq->vxtxq_br, m); 3019 taskqueue_enqueue(sc->vmx_tq, &txq->vxtxq_defrtask); 3020 } 3021 3022 return (error); 3023 } 3024 3025 static void 3026 vmxnet3_txq_tq_deferred(void *xtxq, int pending) 3027 { 3028 struct vmxnet3_softc *sc; 3029 struct vmxnet3_txqueue *txq; 3030 3031 txq = xtxq; 3032 sc = txq->vxtxq_sc; 3033 3034 VMXNET3_TXQ_LOCK(txq); 3035 if (!drbr_empty(sc->vmx_ifp, txq->vxtxq_br)) 3036 vmxnet3_txq_mq_start_locked(txq, NULL); 3037 VMXNET3_TXQ_UNLOCK(txq); 3038 } 3039 3040 #endif /* VMXNET3_LEGACY_TX */ 3041 3042 static void 3043 vmxnet3_txq_start(struct vmxnet3_txqueue *txq) 3044 { 3045 struct vmxnet3_softc *sc; 3046 struct ifnet *ifp; 3047 3048 sc = txq->vxtxq_sc; 3049 ifp = sc->vmx_ifp; 3050 3051 #ifdef VMXNET3_LEGACY_TX 3052 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 3053 vmxnet3_start_locked(ifp); 3054 #else 3055 if (!drbr_empty(ifp, txq->vxtxq_br)) 3056 vmxnet3_txq_mq_start_locked(txq, NULL); 3057 #endif 3058 } 3059 3060 static void 3061 vmxnet3_tx_start_all(struct vmxnet3_softc *sc) 3062 { 3063 struct vmxnet3_txqueue *txq; 3064 int i; 3065 3066 VMXNET3_CORE_LOCK_ASSERT(sc); 3067 3068 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3069 txq = &sc->vmx_txq[i]; 3070 3071 VMXNET3_TXQ_LOCK(txq); 3072 vmxnet3_txq_start(txq); 3073 VMXNET3_TXQ_UNLOCK(txq); 3074 } 3075 } 3076 3077 static void 3078 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 3079 { 3080 struct ifnet *ifp; 3081 int idx, bit; 3082 3083 ifp = sc->vmx_ifp; 3084 idx = (tag >> 5) & 0x7F; 3085 bit = tag & 0x1F; 3086 3087 if (tag == 0 || tag > 4095) 3088 return; 3089 3090 VMXNET3_CORE_LOCK(sc); 3091 3092 /* Update our private VLAN bitvector. */ 3093 if (add) 3094 sc->vmx_vlan_filter[idx] |= (1 << bit); 3095 else 3096 sc->vmx_vlan_filter[idx] &= ~(1 << bit); 3097 3098 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3099 if (add) 3100 sc->vmx_ds->vlan_filter[idx] |= (1 << bit); 3101 else 3102 sc->vmx_ds->vlan_filter[idx] &= ~(1 << bit); 3103 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 3104 } 3105 3106 VMXNET3_CORE_UNLOCK(sc); 3107 } 3108 3109 static void 3110 vmxnet3_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3111 { 3112 3113 if (ifp->if_softc == arg) 3114 vmxnet3_update_vlan_filter(arg, 1, tag); 3115 } 3116 3117 static void 3118 vmxnet3_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 3119 { 3120 3121 if (ifp->if_softc == arg) 3122 vmxnet3_update_vlan_filter(arg, 0, tag); 3123 } 3124 3125 static void 3126 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc) 3127 { 3128 struct ifnet *ifp; 3129 struct vmxnet3_driver_shared *ds; 3130 struct ifmultiaddr *ifma; 3131 u_int mode; 3132 3133 ifp = sc->vmx_ifp; 3134 ds = sc->vmx_ds; 3135 3136 mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 3137 if (ifp->if_flags & IFF_PROMISC) 3138 mode |= VMXNET3_RXMODE_PROMISC; 3139 if (ifp->if_flags & IFF_ALLMULTI) 3140 mode |= VMXNET3_RXMODE_ALLMULTI; 3141 else { 3142 int cnt = 0, overflow = 0; 3143 3144 if_maddr_rlock(ifp); 3145 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3146 if (ifma->ifma_addr->sa_family != AF_LINK) 3147 continue; 3148 else if (cnt == VMXNET3_MULTICAST_MAX) { 3149 overflow = 1; 3150 break; 3151 } 3152 3153 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 3154 &sc->vmx_mcast[cnt*ETHER_ADDR_LEN], ETHER_ADDR_LEN); 3155 cnt++; 3156 } 3157 if_maddr_runlock(ifp); 3158 3159 if (overflow != 0) { 3160 cnt = 0; 3161 mode |= VMXNET3_RXMODE_ALLMULTI; 3162 } else if (cnt > 0) 3163 mode |= VMXNET3_RXMODE_MCAST; 3164 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 3165 } 3166 3167 ds->rxmode = mode; 3168 3169 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 3170 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 3171 } 3172 3173 static int 3174 vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu) 3175 { 3176 struct ifnet *ifp; 3177 3178 ifp = sc->vmx_ifp; 3179 3180 if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU) 3181 return (EINVAL); 3182 3183 ifp->if_mtu = mtu; 3184 3185 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 3186 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3187 vmxnet3_init_locked(sc); 3188 } 3189 3190 return (0); 3191 } 3192 3193 static int 3194 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 3195 { 3196 struct vmxnet3_softc *sc; 3197 struct ifreq *ifr; 3198 int reinit, mask, error; 3199 3200 sc = ifp->if_softc; 3201 ifr = (struct ifreq *) data; 3202 error = 0; 3203 3204 switch (cmd) { 3205 case SIOCSIFMTU: 3206 if (ifp->if_mtu != ifr->ifr_mtu) { 3207 VMXNET3_CORE_LOCK(sc); 3208 error = vmxnet3_change_mtu(sc, ifr->ifr_mtu); 3209 VMXNET3_CORE_UNLOCK(sc); 3210 } 3211 break; 3212 3213 case SIOCSIFFLAGS: 3214 VMXNET3_CORE_LOCK(sc); 3215 if (ifp->if_flags & IFF_UP) { 3216 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3217 if ((ifp->if_flags ^ sc->vmx_if_flags) & 3218 (IFF_PROMISC | IFF_ALLMULTI)) { 3219 vmxnet3_set_rxfilter(sc); 3220 } 3221 } else 3222 vmxnet3_init_locked(sc); 3223 } else { 3224 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3225 vmxnet3_stop(sc); 3226 } 3227 sc->vmx_if_flags = ifp->if_flags; 3228 VMXNET3_CORE_UNLOCK(sc); 3229 break; 3230 3231 case SIOCADDMULTI: 3232 case SIOCDELMULTI: 3233 VMXNET3_CORE_LOCK(sc); 3234 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3235 vmxnet3_set_rxfilter(sc); 3236 VMXNET3_CORE_UNLOCK(sc); 3237 break; 3238 3239 case SIOCSIFMEDIA: 3240 case SIOCGIFMEDIA: 3241 error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd); 3242 break; 3243 3244 case SIOCSIFCAP: 3245 VMXNET3_CORE_LOCK(sc); 3246 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 3247 3248 if (mask & IFCAP_TXCSUM) 3249 ifp->if_capenable ^= IFCAP_TXCSUM; 3250 if (mask & IFCAP_TXCSUM_IPV6) 3251 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 3252 if (mask & IFCAP_TSO4) 3253 ifp->if_capenable ^= IFCAP_TSO4; 3254 if (mask & IFCAP_TSO6) 3255 ifp->if_capenable ^= IFCAP_TSO6; 3256 3257 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO | 3258 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER)) { 3259 /* Changing these features requires us to reinit. */ 3260 reinit = 1; 3261 3262 if (mask & IFCAP_RXCSUM) 3263 ifp->if_capenable ^= IFCAP_RXCSUM; 3264 if (mask & IFCAP_RXCSUM_IPV6) 3265 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 3266 if (mask & IFCAP_LRO) 3267 ifp->if_capenable ^= IFCAP_LRO; 3268 if (mask & IFCAP_VLAN_HWTAGGING) 3269 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 3270 if (mask & IFCAP_VLAN_HWFILTER) 3271 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 3272 } else 3273 reinit = 0; 3274 3275 if (mask & IFCAP_VLAN_HWTSO) 3276 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 3277 3278 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3279 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3280 vmxnet3_init_locked(sc); 3281 } 3282 3283 VMXNET3_CORE_UNLOCK(sc); 3284 VLAN_CAPABILITIES(ifp); 3285 break; 3286 3287 default: 3288 error = ether_ioctl(ifp, cmd, data); 3289 break; 3290 } 3291 3292 VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(sc); 3293 3294 return (error); 3295 } 3296 3297 #ifndef VMXNET3_LEGACY_TX 3298 static void 3299 vmxnet3_qflush(struct ifnet *ifp) 3300 { 3301 struct vmxnet3_softc *sc; 3302 struct vmxnet3_txqueue *txq; 3303 struct mbuf *m; 3304 int i; 3305 3306 sc = ifp->if_softc; 3307 3308 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3309 txq = &sc->vmx_txq[i]; 3310 3311 VMXNET3_TXQ_LOCK(txq); 3312 while ((m = buf_ring_dequeue_sc(txq->vxtxq_br)) != NULL) 3313 m_freem(m); 3314 VMXNET3_TXQ_UNLOCK(txq); 3315 } 3316 3317 if_qflush(ifp); 3318 } 3319 #endif 3320 3321 static int 3322 vmxnet3_watchdog(struct vmxnet3_txqueue *txq) 3323 { 3324 struct vmxnet3_softc *sc; 3325 3326 sc = txq->vxtxq_sc; 3327 3328 VMXNET3_TXQ_LOCK(txq); 3329 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) { 3330 VMXNET3_TXQ_UNLOCK(txq); 3331 return (0); 3332 } 3333 VMXNET3_TXQ_UNLOCK(txq); 3334 3335 if_printf(sc->vmx_ifp, "watchdog timeout on queue %d\n", 3336 txq->vxtxq_id); 3337 return (1); 3338 } 3339 3340 static void 3341 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 3342 { 3343 3344 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 3345 } 3346 3347 static void 3348 vmxnet3_txq_accum_stats(struct vmxnet3_txqueue *txq, 3349 struct vmxnet3_txq_stats *accum) 3350 { 3351 struct vmxnet3_txq_stats *st; 3352 3353 st = &txq->vxtxq_stats; 3354 3355 accum->vmtxs_opackets += st->vmtxs_opackets; 3356 accum->vmtxs_obytes += st->vmtxs_obytes; 3357 accum->vmtxs_omcasts += st->vmtxs_omcasts; 3358 accum->vmtxs_csum += st->vmtxs_csum; 3359 accum->vmtxs_tso += st->vmtxs_tso; 3360 accum->vmtxs_full += st->vmtxs_full; 3361 accum->vmtxs_offload_failed += st->vmtxs_offload_failed; 3362 } 3363 3364 static void 3365 vmxnet3_rxq_accum_stats(struct vmxnet3_rxqueue *rxq, 3366 struct vmxnet3_rxq_stats *accum) 3367 { 3368 struct vmxnet3_rxq_stats *st; 3369 3370 st = &rxq->vxrxq_stats; 3371 3372 accum->vmrxs_ipackets += st->vmrxs_ipackets; 3373 accum->vmrxs_ibytes += st->vmrxs_ibytes; 3374 accum->vmrxs_iqdrops += st->vmrxs_iqdrops; 3375 accum->vmrxs_ierrors += st->vmrxs_ierrors; 3376 } 3377 3378 static void 3379 vmxnet3_accumulate_stats(struct vmxnet3_softc *sc) 3380 { 3381 struct ifnet *ifp; 3382 struct vmxnet3_statistics *st; 3383 struct vmxnet3_txq_stats txaccum; 3384 struct vmxnet3_rxq_stats rxaccum; 3385 int i; 3386 3387 ifp = sc->vmx_ifp; 3388 st = &sc->vmx_stats; 3389 3390 bzero(&txaccum, sizeof(struct vmxnet3_txq_stats)); 3391 bzero(&rxaccum, sizeof(struct vmxnet3_rxq_stats)); 3392 3393 for (i = 0; i < sc->vmx_ntxqueues; i++) 3394 vmxnet3_txq_accum_stats(&sc->vmx_txq[i], &txaccum); 3395 for (i = 0; i < sc->vmx_nrxqueues; i++) 3396 vmxnet3_rxq_accum_stats(&sc->vmx_rxq[i], &rxaccum); 3397 3398 /* 3399 * With the exception of if_ierrors, these ifnet statistics are 3400 * only updated in the driver, so just set them to our accumulated 3401 * values. if_ierrors is updated in ether_input() for malformed 3402 * frames that we should have already discarded. 3403 */ 3404 ifp->if_ipackets = rxaccum.vmrxs_ipackets; 3405 ifp->if_iqdrops = rxaccum.vmrxs_iqdrops; 3406 ifp->if_ierrors = rxaccum.vmrxs_ierrors; 3407 ifp->if_opackets = txaccum.vmtxs_opackets; 3408 #ifndef VMXNET3_LEGACY_TX 3409 ifp->if_obytes = txaccum.vmtxs_obytes; 3410 ifp->if_omcasts = txaccum.vmtxs_omcasts; 3411 #endif 3412 } 3413 3414 static void 3415 vmxnet3_tick(void *xsc) 3416 { 3417 struct vmxnet3_softc *sc; 3418 struct ifnet *ifp; 3419 int i, timedout; 3420 3421 sc = xsc; 3422 ifp = sc->vmx_ifp; 3423 timedout = 0; 3424 3425 VMXNET3_CORE_LOCK_ASSERT(sc); 3426 3427 vmxnet3_accumulate_stats(sc); 3428 vmxnet3_refresh_host_stats(sc); 3429 3430 for (i = 0; i < sc->vmx_ntxqueues; i++) 3431 timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]); 3432 3433 if (timedout != 0) { 3434 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3435 vmxnet3_init_locked(sc); 3436 } else 3437 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc); 3438 } 3439 3440 static int 3441 vmxnet3_link_is_up(struct vmxnet3_softc *sc) 3442 { 3443 uint32_t status; 3444 3445 /* Also update the link speed while here. */ 3446 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 3447 sc->vmx_link_speed = status >> 16; 3448 return !!(status & 0x1); 3449 } 3450 3451 static void 3452 vmxnet3_link_status(struct vmxnet3_softc *sc) 3453 { 3454 struct ifnet *ifp; 3455 int link; 3456 3457 ifp = sc->vmx_ifp; 3458 link = vmxnet3_link_is_up(sc); 3459 3460 if (link != 0 && sc->vmx_link_active == 0) { 3461 sc->vmx_link_active = 1; 3462 if_link_state_change(ifp, LINK_STATE_UP); 3463 } else if (link == 0 && sc->vmx_link_active != 0) { 3464 sc->vmx_link_active = 0; 3465 if_link_state_change(ifp, LINK_STATE_DOWN); 3466 } 3467 } 3468 3469 static void 3470 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 3471 { 3472 struct vmxnet3_softc *sc; 3473 3474 sc = ifp->if_softc; 3475 3476 ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 3477 ifmr->ifm_status = IFM_AVALID; 3478 3479 VMXNET3_CORE_LOCK(sc); 3480 if (vmxnet3_link_is_up(sc) != 0) 3481 ifmr->ifm_status |= IFM_ACTIVE; 3482 else 3483 ifmr->ifm_status |= IFM_NONE; 3484 VMXNET3_CORE_UNLOCK(sc); 3485 } 3486 3487 static int 3488 vmxnet3_media_change(struct ifnet *ifp) 3489 { 3490 3491 /* Ignore. */ 3492 return (0); 3493 } 3494 3495 static void 3496 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 3497 { 3498 uint32_t ml, mh; 3499 3500 ml = sc->vmx_lladdr[0]; 3501 ml |= sc->vmx_lladdr[1] << 8; 3502 ml |= sc->vmx_lladdr[2] << 16; 3503 ml |= sc->vmx_lladdr[3] << 24; 3504 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 3505 3506 mh = sc->vmx_lladdr[4]; 3507 mh |= sc->vmx_lladdr[5] << 8; 3508 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 3509 } 3510 3511 static void 3512 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 3513 { 3514 uint32_t ml, mh; 3515 3516 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 3517 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 3518 3519 sc->vmx_lladdr[0] = ml; 3520 sc->vmx_lladdr[1] = ml >> 8; 3521 sc->vmx_lladdr[2] = ml >> 16; 3522 sc->vmx_lladdr[3] = ml >> 24; 3523 sc->vmx_lladdr[4] = mh; 3524 sc->vmx_lladdr[5] = mh >> 8; 3525 } 3526 3527 static void 3528 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 3529 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3530 { 3531 struct sysctl_oid *node, *txsnode; 3532 struct sysctl_oid_list *list, *txslist; 3533 struct vmxnet3_txq_stats *stats; 3534 struct UPT1_TxStats *txstats; 3535 char namebuf[16]; 3536 3537 stats = &txq->vxtxq_stats; 3538 txstats = &txq->vxtxq_ts->stats; 3539 3540 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 3541 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3542 NULL, "Transmit Queue"); 3543 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 3544 3545 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, 3546 &stats->vmtxs_opackets, "Transmit packets"); 3547 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, 3548 &stats->vmtxs_obytes, "Transmit bytes"); 3549 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, 3550 &stats->vmtxs_omcasts, "Transmit multicasts"); 3551 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, 3552 &stats->vmtxs_csum, "Transmit checksum offloaded"); 3553 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, 3554 &stats->vmtxs_tso, "Transmit TCP segmentation offloaded"); 3555 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ringfull", CTLFLAG_RD, 3556 &stats->vmtxs_full, "Transmit ring full"); 3557 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "offload_failed", CTLFLAG_RD, 3558 &stats->vmtxs_offload_failed, "Transmit checksum offload failed"); 3559 3560 /* 3561 * Add statistics reported by the host. These are updated once 3562 * per second. 3563 */ 3564 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3565 NULL, "Host Statistics"); 3566 txslist = SYSCTL_CHILDREN(txsnode); 3567 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 3568 &txstats->TSO_packets, "TSO packets"); 3569 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 3570 &txstats->TSO_bytes, "TSO bytes"); 3571 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3572 &txstats->ucast_packets, "Unicast packets"); 3573 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3574 &txstats->ucast_bytes, "Unicast bytes"); 3575 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3576 &txstats->mcast_packets, "Multicast packets"); 3577 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3578 &txstats->mcast_bytes, "Multicast bytes"); 3579 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 3580 &txstats->error, "Errors"); 3581 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 3582 &txstats->discard, "Discards"); 3583 } 3584 3585 static void 3586 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 3587 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3588 { 3589 struct sysctl_oid *node, *rxsnode; 3590 struct sysctl_oid_list *list, *rxslist; 3591 struct vmxnet3_rxq_stats *stats; 3592 struct UPT1_RxStats *rxstats; 3593 char namebuf[16]; 3594 3595 stats = &rxq->vxrxq_stats; 3596 rxstats = &rxq->vxrxq_rs->stats; 3597 3598 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 3599 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, 3600 NULL, "Receive Queue"); 3601 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 3602 3603 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, 3604 &stats->vmrxs_ipackets, "Receive packets"); 3605 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, 3606 &stats->vmrxs_ibytes, "Receive bytes"); 3607 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, 3608 &stats->vmrxs_iqdrops, "Receive drops"); 3609 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, 3610 &stats->vmrxs_ierrors, "Receive errors"); 3611 3612 /* 3613 * Add statistics reported by the host. These are updated once 3614 * per second. 3615 */ 3616 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", CTLFLAG_RD, 3617 NULL, "Host Statistics"); 3618 rxslist = SYSCTL_CHILDREN(rxsnode); 3619 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 3620 &rxstats->LRO_packets, "LRO packets"); 3621 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 3622 &rxstats->LRO_bytes, "LRO bytes"); 3623 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 3624 &rxstats->ucast_packets, "Unicast packets"); 3625 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 3626 &rxstats->ucast_bytes, "Unicast bytes"); 3627 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 3628 &rxstats->mcast_packets, "Multicast packets"); 3629 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 3630 &rxstats->mcast_bytes, "Multicast bytes"); 3631 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 3632 &rxstats->bcast_packets, "Broadcast packets"); 3633 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 3634 &rxstats->bcast_bytes, "Broadcast bytes"); 3635 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 3636 &rxstats->nobuffer, "No buffer"); 3637 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 3638 &rxstats->error, "Errors"); 3639 } 3640 3641 static void 3642 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 3643 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3644 { 3645 struct sysctl_oid *node; 3646 struct sysctl_oid_list *list; 3647 int i; 3648 3649 for (i = 0; i < sc->vmx_ntxqueues; i++) { 3650 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 3651 3652 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 3653 "debug", CTLFLAG_RD, NULL, ""); 3654 list = SYSCTL_CHILDREN(node); 3655 3656 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_head", CTLFLAG_RD, 3657 &txq->vxtxq_cmd_ring.vxtxr_head, 0, ""); 3658 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 3659 &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 3660 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 3661 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 3662 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 3663 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 3664 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3665 &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 3666 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3667 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 3668 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3669 &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 3670 } 3671 3672 for (i = 0; i < sc->vmx_nrxqueues; i++) { 3673 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 3674 3675 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 3676 "debug", CTLFLAG_RD, NULL, ""); 3677 list = SYSCTL_CHILDREN(node); 3678 3679 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_fill", CTLFLAG_RD, 3680 &rxq->vxrxq_cmd_ring[0].vxrxr_fill, 0, ""); 3681 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 3682 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 3683 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 3684 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 3685 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_fill", CTLFLAG_RD, 3686 &rxq->vxrxq_cmd_ring[1].vxrxr_fill, 0, ""); 3687 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 3688 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 3689 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 3690 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 3691 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 3692 &rxq->vxrxq_comp_ring.vxcr_next, 0, ""); 3693 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 3694 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 3695 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 3696 &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 3697 } 3698 } 3699 3700 static void 3701 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 3702 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 3703 { 3704 int i; 3705 3706 for (i = 0; i < sc->vmx_ntxqueues; i++) 3707 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 3708 for (i = 0; i < sc->vmx_nrxqueues; i++) 3709 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 3710 3711 vmxnet3_setup_debug_sysctl(sc, ctx, child); 3712 } 3713 3714 static void 3715 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 3716 { 3717 device_t dev; 3718 struct vmxnet3_statistics *stats; 3719 struct sysctl_ctx_list *ctx; 3720 struct sysctl_oid *tree; 3721 struct sysctl_oid_list *child; 3722 3723 dev = sc->vmx_dev; 3724 ctx = device_get_sysctl_ctx(dev); 3725 tree = device_get_sysctl_tree(dev); 3726 child = SYSCTL_CHILDREN(tree); 3727 3728 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_ntxqueues", CTLFLAG_RD, 3729 &sc->vmx_max_ntxqueues, 0, "Maximum number of Tx queues"); 3730 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_nrxqueues", CTLFLAG_RD, 3731 &sc->vmx_max_nrxqueues, 0, "Maximum number of Rx queues"); 3732 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "ntxqueues", CTLFLAG_RD, 3733 &sc->vmx_ntxqueues, 0, "Number of Tx queues"); 3734 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nrxqueues", CTLFLAG_RD, 3735 &sc->vmx_nrxqueues, 0, "Number of Rx queues"); 3736 3737 stats = &sc->vmx_stats; 3738 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defragged", CTLFLAG_RD, 3739 &stats->vmst_defragged, 0, "Tx mbuf chains defragged"); 3740 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "defrag_failed", CTLFLAG_RD, 3741 &stats->vmst_defrag_failed, 0, 3742 "Tx mbuf dropped because defrag failed"); 3743 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mgetcl_failed", CTLFLAG_RD, 3744 &stats->vmst_mgetcl_failed, 0, "mbuf cluster allocation failed"); 3745 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "mbuf_load_failed", CTLFLAG_RD, 3746 &stats->vmst_mbuf_load_failed, 0, "mbuf load segments failed"); 3747 3748 vmxnet3_setup_queue_sysctl(sc, ctx, child); 3749 } 3750 3751 static void 3752 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3753 { 3754 3755 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 3756 } 3757 3758 static uint32_t 3759 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 3760 { 3761 3762 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 3763 } 3764 3765 static void 3766 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 3767 { 3768 3769 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 3770 } 3771 3772 static void 3773 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3774 { 3775 3776 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 3777 } 3778 3779 static uint32_t 3780 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 3781 { 3782 3783 vmxnet3_write_cmd(sc, cmd); 3784 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 3785 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 3786 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 3787 } 3788 3789 static void 3790 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 3791 { 3792 3793 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 3794 } 3795 3796 static void 3797 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 3798 { 3799 3800 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 3801 } 3802 3803 static void 3804 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 3805 { 3806 int i; 3807 3808 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 3809 for (i = 0; i < sc->vmx_nintrs; i++) 3810 vmxnet3_enable_intr(sc, i); 3811 } 3812 3813 static void 3814 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 3815 { 3816 int i; 3817 3818 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 3819 for (i = 0; i < sc->vmx_nintrs; i++) 3820 vmxnet3_disable_intr(sc, i); 3821 } 3822 3823 static void 3824 vmxnet3_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3825 { 3826 bus_addr_t *baddr = arg; 3827 3828 if (error == 0) 3829 *baddr = segs->ds_addr; 3830 } 3831 3832 static int 3833 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align, 3834 struct vmxnet3_dma_alloc *dma) 3835 { 3836 device_t dev; 3837 int error; 3838 3839 dev = sc->vmx_dev; 3840 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3841 3842 error = bus_dma_tag_create(bus_get_dma_tag(dev), 3843 align, 0, /* alignment, bounds */ 3844 BUS_SPACE_MAXADDR, /* lowaddr */ 3845 BUS_SPACE_MAXADDR, /* highaddr */ 3846 NULL, NULL, /* filter, filterarg */ 3847 size, /* maxsize */ 3848 1, /* nsegments */ 3849 size, /* maxsegsize */ 3850 BUS_DMA_ALLOCNOW, /* flags */ 3851 NULL, /* lockfunc */ 3852 NULL, /* lockfuncarg */ 3853 &dma->dma_tag); 3854 if (error) { 3855 device_printf(dev, "bus_dma_tag_create failed: %d\n", error); 3856 goto fail; 3857 } 3858 3859 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 3860 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 3861 if (error) { 3862 device_printf(dev, "bus_dmamem_alloc failed: %d\n", error); 3863 goto fail; 3864 } 3865 3866 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3867 size, vmxnet3_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 3868 if (error) { 3869 device_printf(dev, "bus_dmamap_load failed: %d\n", error); 3870 goto fail; 3871 } 3872 3873 dma->dma_size = size; 3874 3875 fail: 3876 if (error) 3877 vmxnet3_dma_free(sc, dma); 3878 3879 return (error); 3880 } 3881 3882 static void 3883 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma) 3884 { 3885 3886 if (dma->dma_tag != NULL) { 3887 if (dma->dma_paddr != 0) { 3888 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3889 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3890 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3891 } 3892 3893 if (dma->dma_vaddr != NULL) { 3894 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 3895 dma->dma_map); 3896 } 3897 3898 bus_dma_tag_destroy(dma->dma_tag); 3899 } 3900 bzero(dma, sizeof(struct vmxnet3_dma_alloc)); 3901 } 3902 3903 static int 3904 vmxnet3_tunable_int(struct vmxnet3_softc *sc, const char *knob, int def) 3905 { 3906 char path[64]; 3907 3908 snprintf(path, sizeof(path), 3909 "hw.vmx.%d.%s", device_get_unit(sc->vmx_dev), knob); 3910 TUNABLE_INT_FETCH(path, &def); 3911 3912 return (def); 3913 } 3914 3915 /* 3916 * Since this is a purely paravirtualized device, we do not have 3917 * to worry about DMA coherency. But at times, we must make sure 3918 * both the compiler and CPU do not reorder memory operations. 3919 */ 3920 static inline void 3921 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 3922 { 3923 3924 switch (type) { 3925 case VMXNET3_BARRIER_RD: 3926 rmb(); 3927 break; 3928 case VMXNET3_BARRIER_WR: 3929 wmb(); 3930 break; 3931 case VMXNET3_BARRIER_RDWR: 3932 mb(); 3933 break; 3934 default: 3935 panic("%s: bad barrier type %d", __func__, type); 3936 } 3937 } 3938