1 /*- 2 * Copyright (c) 2013 Tsubai Masanari 3 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org> 4 * Copyright (c) 2018 Patrick Kelsey 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 * 18 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $ 19 */ 20 21 /* Driver for VMware vmxnet3 virtual ethernet devices. */ 22 23 #include <sys/cdefs.h> 24 __FBSDID("$FreeBSD$"); 25 26 #include "opt_rss.h" 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/kernel.h> 31 #include <sys/endian.h> 32 #include <sys/sockio.h> 33 #include <sys/mbuf.h> 34 #include <sys/malloc.h> 35 #include <sys/module.h> 36 #include <sys/socket.h> 37 #include <sys/sysctl.h> 38 #include <sys/smp.h> 39 #include <vm/vm.h> 40 #include <vm/pmap.h> 41 42 #include <net/ethernet.h> 43 #include <net/if.h> 44 #include <net/if_var.h> 45 #include <net/if_arp.h> 46 #include <net/if_dl.h> 47 #include <net/if_types.h> 48 #include <net/if_media.h> 49 #include <net/if_vlan_var.h> 50 #include <net/iflib.h> 51 #ifdef RSS 52 #include <net/rss_config.h> 53 #endif 54 55 #include <netinet/in_systm.h> 56 #include <netinet/in.h> 57 #include <netinet/ip.h> 58 #include <netinet/ip6.h> 59 #include <netinet6/ip6_var.h> 60 #include <netinet/udp.h> 61 #include <netinet/tcp.h> 62 63 #include <machine/bus.h> 64 #include <machine/resource.h> 65 #include <sys/bus.h> 66 #include <sys/rman.h> 67 68 #include <dev/pci/pcireg.h> 69 #include <dev/pci/pcivar.h> 70 71 #include "ifdi_if.h" 72 73 #include "if_vmxreg.h" 74 #include "if_vmxvar.h" 75 76 #include "opt_inet.h" 77 #include "opt_inet6.h" 78 79 #define VMXNET3_VMWARE_VENDOR_ID 0x15AD 80 #define VMXNET3_VMWARE_DEVICE_ID 0x07B0 81 82 static pci_vendor_info_t vmxnet3_vendor_info_array[] = 83 { 84 PVID(VMXNET3_VMWARE_VENDOR_ID, VMXNET3_VMWARE_DEVICE_ID, "VMware VMXNET3 Ethernet Adapter"), 85 /* required last entry */ 86 PVID_END 87 }; 88 89 static void *vmxnet3_register(device_t); 90 static int vmxnet3_attach_pre(if_ctx_t); 91 static int vmxnet3_msix_intr_assign(if_ctx_t, int); 92 static void vmxnet3_free_irqs(struct vmxnet3_softc *); 93 static int vmxnet3_attach_post(if_ctx_t); 94 static int vmxnet3_detach(if_ctx_t); 95 static int vmxnet3_shutdown(if_ctx_t); 96 static int vmxnet3_suspend(if_ctx_t); 97 static int vmxnet3_resume(if_ctx_t); 98 99 static int vmxnet3_alloc_resources(struct vmxnet3_softc *); 100 static void vmxnet3_free_resources(struct vmxnet3_softc *); 101 static int vmxnet3_check_version(struct vmxnet3_softc *); 102 static void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *); 103 104 static int vmxnet3_queues_shared_alloc(struct vmxnet3_softc *); 105 static void vmxnet3_init_txq(struct vmxnet3_softc *, int); 106 static int vmxnet3_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 107 static void vmxnet3_init_rxq(struct vmxnet3_softc *, int, int); 108 static int vmxnet3_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); 109 static void vmxnet3_queues_free(if_ctx_t); 110 111 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *); 112 static void vmxnet3_free_shared_data(struct vmxnet3_softc *); 113 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *); 114 static void vmxnet3_free_mcast_table(struct vmxnet3_softc *); 115 static void vmxnet3_init_shared_data(struct vmxnet3_softc *); 116 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *); 117 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *); 118 static int vmxnet3_alloc_data(struct vmxnet3_softc *); 119 static void vmxnet3_free_data(struct vmxnet3_softc *); 120 121 static void vmxnet3_evintr(struct vmxnet3_softc *); 122 static int vmxnet3_isc_txd_encap(void *, if_pkt_info_t); 123 static void vmxnet3_isc_txd_flush(void *, uint16_t, qidx_t); 124 static int vmxnet3_isc_txd_credits_update(void *, uint16_t, bool); 125 static int vmxnet3_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t); 126 static int vmxnet3_isc_rxd_pkt_get(void *, if_rxd_info_t); 127 static void vmxnet3_isc_rxd_refill(void *, if_rxd_update_t); 128 static void vmxnet3_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t); 129 static int vmxnet3_legacy_intr(void *); 130 static int vmxnet3_rxq_intr(void *); 131 static int vmxnet3_event_intr(void *); 132 133 static void vmxnet3_stop(if_ctx_t); 134 135 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 136 static void vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 137 static void vmxnet3_reinit_queues(struct vmxnet3_softc *); 138 static int vmxnet3_enable_device(struct vmxnet3_softc *); 139 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *); 140 static void vmxnet3_init(if_ctx_t); 141 static void vmxnet3_multi_set(if_ctx_t); 142 static int vmxnet3_mtu_set(if_ctx_t, uint32_t); 143 static void vmxnet3_media_status(if_ctx_t, struct ifmediareq *); 144 static int vmxnet3_media_change(if_ctx_t); 145 static int vmxnet3_promisc_set(if_ctx_t, int); 146 static uint64_t vmxnet3_get_counter(if_ctx_t, ift_counter); 147 static void vmxnet3_update_admin_status(if_ctx_t); 148 static void vmxnet3_txq_timer(if_ctx_t, uint16_t); 149 150 static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int, 151 uint16_t); 152 static void vmxnet3_vlan_register(if_ctx_t, uint16_t); 153 static void vmxnet3_vlan_unregister(if_ctx_t, uint16_t); 154 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *, int); 155 156 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *); 157 static int vmxnet3_link_is_up(struct vmxnet3_softc *); 158 static void vmxnet3_link_status(struct vmxnet3_softc *); 159 static void vmxnet3_set_lladdr(struct vmxnet3_softc *); 160 static void vmxnet3_get_lladdr(struct vmxnet3_softc *); 161 162 static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *, 163 struct sysctl_ctx_list *, struct sysctl_oid_list *); 164 static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *, 165 struct sysctl_ctx_list *, struct sysctl_oid_list *); 166 static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *, 167 struct sysctl_ctx_list *, struct sysctl_oid_list *); 168 static void vmxnet3_setup_sysctl(struct vmxnet3_softc *); 169 170 static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t, 171 uint32_t); 172 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t); 173 static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t, 174 uint32_t); 175 static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t); 176 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t); 177 178 static int vmxnet3_tx_queue_intr_enable(if_ctx_t, uint16_t); 179 static int vmxnet3_rx_queue_intr_enable(if_ctx_t, uint16_t); 180 static void vmxnet3_link_intr_enable(if_ctx_t); 181 static void vmxnet3_enable_intr(struct vmxnet3_softc *, int); 182 static void vmxnet3_disable_intr(struct vmxnet3_softc *, int); 183 static void vmxnet3_intr_enable_all(if_ctx_t); 184 static void vmxnet3_intr_disable_all(if_ctx_t); 185 186 typedef enum { 187 VMXNET3_BARRIER_RD, 188 VMXNET3_BARRIER_WR, 189 VMXNET3_BARRIER_RDWR, 190 } vmxnet3_barrier_t; 191 192 static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t); 193 194 static device_method_t vmxnet3_methods[] = { 195 /* Device interface */ 196 DEVMETHOD(device_register, vmxnet3_register), 197 DEVMETHOD(device_probe, iflib_device_probe), 198 DEVMETHOD(device_attach, iflib_device_attach), 199 DEVMETHOD(device_detach, iflib_device_detach), 200 DEVMETHOD(device_shutdown, iflib_device_shutdown), 201 DEVMETHOD(device_suspend, iflib_device_suspend), 202 DEVMETHOD(device_resume, iflib_device_resume), 203 DEVMETHOD_END 204 }; 205 206 static driver_t vmxnet3_driver = { 207 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc) 208 }; 209 210 static devclass_t vmxnet3_devclass; 211 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0); 212 IFLIB_PNP_INFO(pci, vmx, vmxnet3_vendor_info_array); 213 MODULE_VERSION(vmx, 2); 214 215 MODULE_DEPEND(vmx, pci, 1, 1, 1); 216 MODULE_DEPEND(vmx, ether, 1, 1, 1); 217 MODULE_DEPEND(vmx, iflib, 1, 1, 1); 218 219 static device_method_t vmxnet3_iflib_methods[] = { 220 DEVMETHOD(ifdi_tx_queues_alloc, vmxnet3_tx_queues_alloc), 221 DEVMETHOD(ifdi_rx_queues_alloc, vmxnet3_rx_queues_alloc), 222 DEVMETHOD(ifdi_queues_free, vmxnet3_queues_free), 223 224 DEVMETHOD(ifdi_attach_pre, vmxnet3_attach_pre), 225 DEVMETHOD(ifdi_attach_post, vmxnet3_attach_post), 226 DEVMETHOD(ifdi_detach, vmxnet3_detach), 227 228 DEVMETHOD(ifdi_init, vmxnet3_init), 229 DEVMETHOD(ifdi_stop, vmxnet3_stop), 230 DEVMETHOD(ifdi_multi_set, vmxnet3_multi_set), 231 DEVMETHOD(ifdi_mtu_set, vmxnet3_mtu_set), 232 DEVMETHOD(ifdi_media_status, vmxnet3_media_status), 233 DEVMETHOD(ifdi_media_change, vmxnet3_media_change), 234 DEVMETHOD(ifdi_promisc_set, vmxnet3_promisc_set), 235 DEVMETHOD(ifdi_get_counter, vmxnet3_get_counter), 236 DEVMETHOD(ifdi_update_admin_status, vmxnet3_update_admin_status), 237 DEVMETHOD(ifdi_timer, vmxnet3_txq_timer), 238 239 DEVMETHOD(ifdi_tx_queue_intr_enable, vmxnet3_tx_queue_intr_enable), 240 DEVMETHOD(ifdi_rx_queue_intr_enable, vmxnet3_rx_queue_intr_enable), 241 DEVMETHOD(ifdi_link_intr_enable, vmxnet3_link_intr_enable), 242 DEVMETHOD(ifdi_intr_enable, vmxnet3_intr_enable_all), 243 DEVMETHOD(ifdi_intr_disable, vmxnet3_intr_disable_all), 244 DEVMETHOD(ifdi_msix_intr_assign, vmxnet3_msix_intr_assign), 245 246 DEVMETHOD(ifdi_vlan_register, vmxnet3_vlan_register), 247 DEVMETHOD(ifdi_vlan_unregister, vmxnet3_vlan_unregister), 248 249 DEVMETHOD(ifdi_shutdown, vmxnet3_shutdown), 250 DEVMETHOD(ifdi_suspend, vmxnet3_suspend), 251 DEVMETHOD(ifdi_resume, vmxnet3_resume), 252 253 DEVMETHOD_END 254 }; 255 256 static driver_t vmxnet3_iflib_driver = { 257 "vmx", vmxnet3_iflib_methods, sizeof(struct vmxnet3_softc) 258 }; 259 260 struct if_txrx vmxnet3_txrx = { 261 .ift_txd_encap = vmxnet3_isc_txd_encap, 262 .ift_txd_flush = vmxnet3_isc_txd_flush, 263 .ift_txd_credits_update = vmxnet3_isc_txd_credits_update, 264 .ift_rxd_available = vmxnet3_isc_rxd_available, 265 .ift_rxd_pkt_get = vmxnet3_isc_rxd_pkt_get, 266 .ift_rxd_refill = vmxnet3_isc_rxd_refill, 267 .ift_rxd_flush = vmxnet3_isc_rxd_flush, 268 .ift_legacy_intr = vmxnet3_legacy_intr 269 }; 270 271 static struct if_shared_ctx vmxnet3_sctx_init = { 272 .isc_magic = IFLIB_MAGIC, 273 .isc_q_align = 512, 274 275 .isc_tx_maxsize = VMXNET3_TX_MAXSIZE, 276 .isc_tx_maxsegsize = VMXNET3_TX_MAXSEGSIZE, 277 .isc_tso_maxsize = VMXNET3_TSO_MAXSIZE + sizeof(struct ether_vlan_header), 278 .isc_tso_maxsegsize = VMXNET3_TX_MAXSEGSIZE, 279 280 /* 281 * These values are used to configure the busdma tag used for 282 * receive descriptors. Each receive descriptor only points to one 283 * buffer. 284 */ 285 .isc_rx_maxsize = VMXNET3_RX_MAXSEGSIZE, /* One buf per descriptor */ 286 .isc_rx_nsegments = 1, /* One mapping per descriptor */ 287 .isc_rx_maxsegsize = VMXNET3_RX_MAXSEGSIZE, 288 289 .isc_admin_intrcnt = 1, 290 .isc_vendor_info = vmxnet3_vendor_info_array, 291 .isc_driver_version = "2", 292 .isc_driver = &vmxnet3_iflib_driver, 293 .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ | IFLIB_SINGLE_IRQ_RX_ONLY, 294 295 /* 296 * Number of receive queues per receive queue set, with associated 297 * descriptor settings for each. 298 */ 299 .isc_nrxqs = 3, 300 .isc_nfl = 2, /* one free list for each receive command queue */ 301 .isc_nrxd_min = {VMXNET3_MIN_RX_NDESC, VMXNET3_MIN_RX_NDESC, VMXNET3_MIN_RX_NDESC}, 302 .isc_nrxd_max = {VMXNET3_MAX_RX_NDESC, VMXNET3_MAX_RX_NDESC, VMXNET3_MAX_RX_NDESC}, 303 .isc_nrxd_default = {VMXNET3_DEF_RX_NDESC, VMXNET3_DEF_RX_NDESC, VMXNET3_DEF_RX_NDESC}, 304 305 /* 306 * Number of transmit queues per transmit queue set, with associated 307 * descriptor settings for each. 308 */ 309 .isc_ntxqs = 2, 310 .isc_ntxd_min = {VMXNET3_MIN_TX_NDESC, VMXNET3_MIN_TX_NDESC}, 311 .isc_ntxd_max = {VMXNET3_MAX_TX_NDESC, VMXNET3_MAX_TX_NDESC}, 312 .isc_ntxd_default = {VMXNET3_DEF_TX_NDESC, VMXNET3_DEF_TX_NDESC}, 313 }; 314 315 static void * 316 vmxnet3_register(device_t dev) 317 { 318 return (&vmxnet3_sctx_init); 319 } 320 321 static int 322 trunc_powerof2(int val) 323 { 324 325 return (1U << (fls(val) - 1)); 326 } 327 328 static int 329 vmxnet3_attach_pre(if_ctx_t ctx) 330 { 331 device_t dev; 332 if_softc_ctx_t scctx; 333 struct vmxnet3_softc *sc; 334 uint32_t intr_config; 335 int error; 336 337 dev = iflib_get_dev(ctx); 338 sc = iflib_get_softc(ctx); 339 sc->vmx_dev = dev; 340 sc->vmx_ctx = ctx; 341 sc->vmx_sctx = iflib_get_sctx(ctx); 342 sc->vmx_scctx = iflib_get_softc_ctx(ctx); 343 sc->vmx_ifp = iflib_get_ifp(ctx); 344 sc->vmx_media = iflib_get_media(ctx); 345 scctx = sc->vmx_scctx; 346 347 scctx->isc_tx_nsegments = VMXNET3_TX_MAXSEGS; 348 scctx->isc_tx_tso_segments_max = VMXNET3_TX_MAXSEGS; 349 /* isc_tx_tso_size_max doesn't include possible vlan header */ 350 scctx->isc_tx_tso_size_max = VMXNET3_TSO_MAXSIZE; 351 scctx->isc_tx_tso_segsize_max = VMXNET3_TX_MAXSEGSIZE; 352 scctx->isc_txrx = &vmxnet3_txrx; 353 354 /* If 0, the iflib tunable was not set, so set to the default */ 355 if (scctx->isc_nrxqsets == 0) 356 scctx->isc_nrxqsets = VMXNET3_DEF_RX_QUEUES; 357 scctx->isc_nrxqsets = trunc_powerof2(scctx->isc_nrxqsets); 358 scctx->isc_nrxqsets_max = min(VMXNET3_MAX_RX_QUEUES, mp_ncpus); 359 scctx->isc_nrxqsets_max = trunc_powerof2(scctx->isc_nrxqsets_max); 360 361 /* If 0, the iflib tunable was not set, so set to the default */ 362 if (scctx->isc_ntxqsets == 0) 363 scctx->isc_ntxqsets = VMXNET3_DEF_TX_QUEUES; 364 scctx->isc_ntxqsets = trunc_powerof2(scctx->isc_ntxqsets); 365 scctx->isc_ntxqsets_max = min(VMXNET3_MAX_TX_QUEUES, mp_ncpus); 366 scctx->isc_ntxqsets_max = trunc_powerof2(scctx->isc_ntxqsets_max); 367 368 /* 369 * Enforce that the transmit completion queue descriptor count is 370 * the same as the transmit command queue descriptor count. 371 */ 372 scctx->isc_ntxd[0] = scctx->isc_ntxd[1]; 373 scctx->isc_txqsizes[0] = 374 sizeof(struct vmxnet3_txcompdesc) * scctx->isc_ntxd[0]; 375 scctx->isc_txqsizes[1] = 376 sizeof(struct vmxnet3_txdesc) * scctx->isc_ntxd[1]; 377 378 /* 379 * Enforce that the receive completion queue descriptor count is the 380 * sum of the receive command queue descriptor counts, and that the 381 * second receive command queue descriptor count is the same as the 382 * first one. 383 */ 384 scctx->isc_nrxd[2] = scctx->isc_nrxd[1]; 385 scctx->isc_nrxd[0] = scctx->isc_nrxd[1] + scctx->isc_nrxd[2]; 386 scctx->isc_rxqsizes[0] = 387 sizeof(struct vmxnet3_rxcompdesc) * scctx->isc_nrxd[0]; 388 scctx->isc_rxqsizes[1] = 389 sizeof(struct vmxnet3_rxdesc) * scctx->isc_nrxd[1]; 390 scctx->isc_rxqsizes[2] = 391 sizeof(struct vmxnet3_rxdesc) * scctx->isc_nrxd[2]; 392 393 /* 394 * Initialize the max frame size and descriptor queue buffer 395 * sizes. 396 */ 397 vmxnet3_mtu_set(ctx, if_getmtu(sc->vmx_ifp)); 398 399 scctx->isc_rss_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 400 401 /* Map PCI BARs */ 402 error = vmxnet3_alloc_resources(sc); 403 if (error) 404 goto fail; 405 406 /* Check device versions */ 407 error = vmxnet3_check_version(sc); 408 if (error) 409 goto fail; 410 411 /* 412 * The interrupt mode can be set in the hypervisor configuration via 413 * the parameter ethernet<N>.intrMode. 414 */ 415 intr_config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG); 416 sc->vmx_intr_mask_mode = (intr_config >> 2) & 0x03; 417 418 /* 419 * Configure the softc context to attempt to configure the interrupt 420 * mode now indicated by intr_config. iflib will follow the usual 421 * fallback path MSI-X -> MSI -> LEGACY, starting at the configured 422 * starting mode. 423 */ 424 switch (intr_config & 0x03) { 425 case VMXNET3_IT_AUTO: 426 case VMXNET3_IT_MSIX: 427 scctx->isc_msix_bar = pci_msix_table_bar(dev); 428 break; 429 case VMXNET3_IT_MSI: 430 scctx->isc_msix_bar = -1; 431 scctx->isc_disable_msix = 1; 432 break; 433 case VMXNET3_IT_LEGACY: 434 scctx->isc_msix_bar = 0; 435 break; 436 } 437 438 scctx->isc_tx_csum_flags = VMXNET3_CSUM_ALL_OFFLOAD; 439 scctx->isc_capabilities = scctx->isc_capenable = 440 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | 441 IFCAP_TSO4 | IFCAP_TSO6 | 442 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | 443 IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 444 IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | 445 IFCAP_JUMBO_MTU; 446 447 /* These capabilities are not enabled by default. */ 448 scctx->isc_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER; 449 450 vmxnet3_get_lladdr(sc); 451 iflib_set_mac(ctx, sc->vmx_lladdr); 452 453 return (0); 454 fail: 455 /* 456 * We must completely clean up anything allocated above as iflib 457 * will not invoke any other driver entry points as a result of this 458 * failure. 459 */ 460 vmxnet3_free_resources(sc); 461 462 return (error); 463 } 464 465 static int 466 vmxnet3_msix_intr_assign(if_ctx_t ctx, int msix) 467 { 468 struct vmxnet3_softc *sc; 469 if_softc_ctx_t scctx; 470 struct vmxnet3_rxqueue *rxq; 471 int error; 472 int i; 473 char irq_name[16]; 474 475 sc = iflib_get_softc(ctx); 476 scctx = sc->vmx_scctx; 477 478 for (i = 0; i < scctx->isc_nrxqsets; i++) { 479 snprintf(irq_name, sizeof(irq_name), "rxq%d", i); 480 481 rxq = &sc->vmx_rxq[i]; 482 error = iflib_irq_alloc_generic(ctx, &rxq->vxrxq_irq, i + 1, 483 IFLIB_INTR_RXTX, vmxnet3_rxq_intr, rxq, i, irq_name); 484 if (error) { 485 device_printf(iflib_get_dev(ctx), 486 "Failed to register rxq %d interrupt handler\n", i); 487 return (error); 488 } 489 } 490 491 for (i = 0; i < scctx->isc_ntxqsets; i++) { 492 snprintf(irq_name, sizeof(irq_name), "txq%d", i); 493 494 /* 495 * Don't provide the corresponding rxq irq for reference - 496 * we want the transmit task to be attached to a task queue 497 * that is different from the one used by the corresponding 498 * rxq irq. That is because the TX doorbell writes are very 499 * expensive as virtualized MMIO operations, so we want to 500 * be able to defer them to another core when possible so 501 * that they don't steal receive processing cycles during 502 * stack turnarounds like TCP ACK generation. The other 503 * piece to this approach is enabling the iflib abdicate 504 * option (currently via an interface-specific 505 * tunable/sysctl). 506 */ 507 iflib_softirq_alloc_generic(ctx, NULL, IFLIB_INTR_TX, NULL, i, 508 irq_name); 509 } 510 511 error = iflib_irq_alloc_generic(ctx, &sc->vmx_event_intr_irq, 512 scctx->isc_nrxqsets + 1, IFLIB_INTR_ADMIN, vmxnet3_event_intr, sc, 0, 513 "event"); 514 if (error) { 515 device_printf(iflib_get_dev(ctx), 516 "Failed to register event interrupt handler\n"); 517 return (error); 518 } 519 520 return (0); 521 } 522 523 static void 524 vmxnet3_free_irqs(struct vmxnet3_softc *sc) 525 { 526 if_softc_ctx_t scctx; 527 struct vmxnet3_rxqueue *rxq; 528 int i; 529 530 scctx = sc->vmx_scctx; 531 532 for (i = 0; i < scctx->isc_nrxqsets; i++) { 533 rxq = &sc->vmx_rxq[i]; 534 iflib_irq_free(sc->vmx_ctx, &rxq->vxrxq_irq); 535 } 536 537 iflib_irq_free(sc->vmx_ctx, &sc->vmx_event_intr_irq); 538 } 539 540 static int 541 vmxnet3_attach_post(if_ctx_t ctx) 542 { 543 device_t dev; 544 if_softc_ctx_t scctx; 545 struct vmxnet3_softc *sc; 546 int error; 547 548 dev = iflib_get_dev(ctx); 549 scctx = iflib_get_softc_ctx(ctx); 550 sc = iflib_get_softc(ctx); 551 552 if (scctx->isc_nrxqsets > 1) 553 sc->vmx_flags |= VMXNET3_FLAG_RSS; 554 555 error = vmxnet3_alloc_data(sc); 556 if (error) 557 goto fail; 558 559 vmxnet3_set_interrupt_idx(sc); 560 vmxnet3_setup_sysctl(sc); 561 562 ifmedia_add(sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL); 563 ifmedia_set(sc->vmx_media, IFM_ETHER | IFM_AUTO); 564 565 fail: 566 return (error); 567 } 568 569 static int 570 vmxnet3_detach(if_ctx_t ctx) 571 { 572 struct vmxnet3_softc *sc; 573 574 sc = iflib_get_softc(ctx); 575 576 vmxnet3_free_irqs(sc); 577 vmxnet3_free_data(sc); 578 vmxnet3_free_resources(sc); 579 580 return (0); 581 } 582 583 static int 584 vmxnet3_shutdown(if_ctx_t ctx) 585 { 586 587 return (0); 588 } 589 590 static int 591 vmxnet3_suspend(if_ctx_t ctx) 592 { 593 594 return (0); 595 } 596 597 static int 598 vmxnet3_resume(if_ctx_t ctx) 599 { 600 601 return (0); 602 } 603 604 static int 605 vmxnet3_alloc_resources(struct vmxnet3_softc *sc) 606 { 607 device_t dev; 608 int rid; 609 610 dev = sc->vmx_dev; 611 612 rid = PCIR_BAR(0); 613 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 614 RF_ACTIVE); 615 if (sc->vmx_res0 == NULL) { 616 device_printf(dev, 617 "could not map BAR0 memory\n"); 618 return (ENXIO); 619 } 620 621 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0); 622 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0); 623 624 rid = PCIR_BAR(1); 625 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 626 RF_ACTIVE); 627 if (sc->vmx_res1 == NULL) { 628 device_printf(dev, 629 "could not map BAR1 memory\n"); 630 return (ENXIO); 631 } 632 633 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1); 634 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1); 635 636 return (0); 637 } 638 639 static void 640 vmxnet3_free_resources(struct vmxnet3_softc *sc) 641 { 642 device_t dev; 643 644 dev = sc->vmx_dev; 645 646 if (sc->vmx_res0 != NULL) { 647 bus_release_resource(dev, SYS_RES_MEMORY, 648 rman_get_rid(sc->vmx_res0), sc->vmx_res0); 649 sc->vmx_res0 = NULL; 650 } 651 652 if (sc->vmx_res1 != NULL) { 653 bus_release_resource(dev, SYS_RES_MEMORY, 654 rman_get_rid(sc->vmx_res1), sc->vmx_res1); 655 sc->vmx_res1 = NULL; 656 } 657 } 658 659 static int 660 vmxnet3_check_version(struct vmxnet3_softc *sc) 661 { 662 device_t dev; 663 uint32_t version; 664 665 dev = sc->vmx_dev; 666 667 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS); 668 if ((version & 0x01) == 0) { 669 device_printf(dev, "unsupported hardware version %#x\n", 670 version); 671 return (ENOTSUP); 672 } 673 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1); 674 675 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS); 676 if ((version & 0x01) == 0) { 677 device_printf(dev, "unsupported UPT version %#x\n", version); 678 return (ENOTSUP); 679 } 680 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1); 681 682 return (0); 683 } 684 685 static void 686 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc) 687 { 688 if_softc_ctx_t scctx; 689 struct vmxnet3_txqueue *txq; 690 struct vmxnet3_txq_shared *txs; 691 struct vmxnet3_rxqueue *rxq; 692 struct vmxnet3_rxq_shared *rxs; 693 int intr_idx; 694 int i; 695 696 scctx = sc->vmx_scctx; 697 698 /* 699 * There is always one interrupt per receive queue, assigned 700 * starting with the first interrupt. When there is only one 701 * interrupt available, the event interrupt shares the receive queue 702 * interrupt, otherwise it uses the interrupt following the last 703 * receive queue interrupt. Transmit queues are not assigned 704 * interrupts, so they are given indexes beyond the indexes that 705 * correspond to the real interrupts. 706 */ 707 708 /* The event interrupt is always the last vector. */ 709 sc->vmx_event_intr_idx = scctx->isc_vectors - 1; 710 711 intr_idx = 0; 712 for (i = 0; i < scctx->isc_nrxqsets; i++, intr_idx++) { 713 rxq = &sc->vmx_rxq[i]; 714 rxs = rxq->vxrxq_rs; 715 rxq->vxrxq_intr_idx = intr_idx; 716 rxs->intr_idx = rxq->vxrxq_intr_idx; 717 } 718 719 /* 720 * Assign the tx queues interrupt indexes above what we are actually 721 * using. These interrupts will never be enabled. 722 */ 723 intr_idx = scctx->isc_vectors; 724 for (i = 0; i < scctx->isc_ntxqsets; i++, intr_idx++) { 725 txq = &sc->vmx_txq[i]; 726 txs = txq->vxtxq_ts; 727 txq->vxtxq_intr_idx = intr_idx; 728 txs->intr_idx = txq->vxtxq_intr_idx; 729 } 730 } 731 732 static int 733 vmxnet3_queues_shared_alloc(struct vmxnet3_softc *sc) 734 { 735 if_softc_ctx_t scctx; 736 int size; 737 int error; 738 739 scctx = sc->vmx_scctx; 740 741 /* 742 * The txq and rxq shared data areas must be allocated contiguously 743 * as vmxnet3_driver_shared contains only a single address member 744 * for the shared queue data area. 745 */ 746 size = scctx->isc_ntxqsets * sizeof(struct vmxnet3_txq_shared) + 747 scctx->isc_nrxqsets * sizeof(struct vmxnet3_rxq_shared); 748 error = iflib_dma_alloc_align(sc->vmx_ctx, size, 128, &sc->vmx_qs_dma, 0); 749 if (error) { 750 device_printf(sc->vmx_dev, "cannot alloc queue shared memory\n"); 751 return (error); 752 } 753 754 return (0); 755 } 756 757 static void 758 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q) 759 { 760 struct vmxnet3_txqueue *txq; 761 struct vmxnet3_comp_ring *txc; 762 struct vmxnet3_txring *txr; 763 if_softc_ctx_t scctx; 764 765 txq = &sc->vmx_txq[q]; 766 txc = &txq->vxtxq_comp_ring; 767 txr = &txq->vxtxq_cmd_ring; 768 scctx = sc->vmx_scctx; 769 770 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d", 771 device_get_nameunit(sc->vmx_dev), q); 772 773 txq->vxtxq_sc = sc; 774 txq->vxtxq_id = q; 775 txc->vxcr_ndesc = scctx->isc_ntxd[0]; 776 txr->vxtxr_ndesc = scctx->isc_ntxd[1]; 777 } 778 779 static int 780 vmxnet3_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 781 int ntxqs, int ntxqsets) 782 { 783 struct vmxnet3_softc *sc; 784 int q; 785 int error; 786 caddr_t kva; 787 788 sc = iflib_get_softc(ctx); 789 790 /* Allocate the array of transmit queues */ 791 sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) * 792 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 793 if (sc->vmx_txq == NULL) 794 return (ENOMEM); 795 796 /* Initialize driver state for each transmit queue */ 797 for (q = 0; q < ntxqsets; q++) 798 vmxnet3_init_txq(sc, q); 799 800 /* 801 * Allocate queue state that is shared with the device. This check 802 * and call is performed in both vmxnet3_tx_queues_alloc() and 803 * vmxnet3_rx_queues_alloc() so that we don't have to care which 804 * order iflib invokes those routines in. 805 */ 806 if (sc->vmx_qs_dma.idi_size == 0) { 807 error = vmxnet3_queues_shared_alloc(sc); 808 if (error) 809 return (error); 810 } 811 812 kva = sc->vmx_qs_dma.idi_vaddr; 813 for (q = 0; q < ntxqsets; q++) { 814 sc->vmx_txq[q].vxtxq_ts = (struct vmxnet3_txq_shared *) kva; 815 kva += sizeof(struct vmxnet3_txq_shared); 816 } 817 818 /* Record descriptor ring vaddrs and paddrs */ 819 for (q = 0; q < ntxqsets; q++) { 820 struct vmxnet3_txqueue *txq; 821 struct vmxnet3_txring *txr; 822 struct vmxnet3_comp_ring *txc; 823 824 txq = &sc->vmx_txq[q]; 825 txc = &txq->vxtxq_comp_ring; 826 txr = &txq->vxtxq_cmd_ring; 827 828 /* Completion ring */ 829 txc->vxcr_u.txcd = 830 (struct vmxnet3_txcompdesc *) vaddrs[q * ntxqs + 0]; 831 txc->vxcr_paddr = paddrs[q * ntxqs + 0]; 832 833 /* Command ring */ 834 txr->vxtxr_txd = 835 (struct vmxnet3_txdesc *) vaddrs[q * ntxqs + 1]; 836 txr->vxtxr_paddr = paddrs[q * ntxqs + 1]; 837 } 838 839 return (0); 840 } 841 842 static void 843 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q, int nrxqs) 844 { 845 struct vmxnet3_rxqueue *rxq; 846 struct vmxnet3_comp_ring *rxc; 847 struct vmxnet3_rxring *rxr; 848 if_softc_ctx_t scctx; 849 int i; 850 851 rxq = &sc->vmx_rxq[q]; 852 rxc = &rxq->vxrxq_comp_ring; 853 scctx = sc->vmx_scctx; 854 855 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d", 856 device_get_nameunit(sc->vmx_dev), q); 857 858 rxq->vxrxq_sc = sc; 859 rxq->vxrxq_id = q; 860 861 /* 862 * First rxq is the completion queue, so there are nrxqs - 1 command 863 * rings starting at iflib queue id 1. 864 */ 865 rxc->vxcr_ndesc = scctx->isc_nrxd[0]; 866 for (i = 0; i < nrxqs - 1; i++) { 867 rxr = &rxq->vxrxq_cmd_ring[i]; 868 rxr->vxrxr_ndesc = scctx->isc_nrxd[i + 1]; 869 } 870 } 871 872 static int 873 vmxnet3_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 874 int nrxqs, int nrxqsets) 875 { 876 struct vmxnet3_softc *sc; 877 if_softc_ctx_t scctx; 878 int q; 879 int i; 880 int error; 881 caddr_t kva; 882 883 sc = iflib_get_softc(ctx); 884 scctx = sc->vmx_scctx; 885 886 /* Allocate the array of receive queues */ 887 sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) * 888 nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 889 if (sc->vmx_rxq == NULL) 890 return (ENOMEM); 891 892 /* Initialize driver state for each receive queue */ 893 for (q = 0; q < nrxqsets; q++) 894 vmxnet3_init_rxq(sc, q, nrxqs); 895 896 /* 897 * Allocate queue state that is shared with the device. This check 898 * and call is performed in both vmxnet3_tx_queues_alloc() and 899 * vmxnet3_rx_queues_alloc() so that we don't have to care which 900 * order iflib invokes those routines in. 901 */ 902 if (sc->vmx_qs_dma.idi_size == 0) { 903 error = vmxnet3_queues_shared_alloc(sc); 904 if (error) 905 return (error); 906 } 907 908 kva = sc->vmx_qs_dma.idi_vaddr + 909 scctx->isc_ntxqsets * sizeof(struct vmxnet3_txq_shared); 910 for (q = 0; q < nrxqsets; q++) { 911 sc->vmx_rxq[q].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva; 912 kva += sizeof(struct vmxnet3_rxq_shared); 913 } 914 915 /* Record descriptor ring vaddrs and paddrs */ 916 for (q = 0; q < nrxqsets; q++) { 917 struct vmxnet3_rxqueue *rxq; 918 struct vmxnet3_rxring *rxr; 919 struct vmxnet3_comp_ring *rxc; 920 921 rxq = &sc->vmx_rxq[q]; 922 rxc = &rxq->vxrxq_comp_ring; 923 924 /* Completion ring */ 925 rxc->vxcr_u.rxcd = 926 (struct vmxnet3_rxcompdesc *) vaddrs[q * nrxqs + 0]; 927 rxc->vxcr_paddr = paddrs[q * nrxqs + 0]; 928 929 /* Command ring(s) */ 930 for (i = 0; i < nrxqs - 1; i++) { 931 rxr = &rxq->vxrxq_cmd_ring[i]; 932 933 rxr->vxrxr_rxd = 934 (struct vmxnet3_rxdesc *) vaddrs[q * nrxqs + 1 + i]; 935 rxr->vxrxr_paddr = paddrs[q * nrxqs + 1 + i]; 936 } 937 } 938 939 return (0); 940 } 941 942 static void 943 vmxnet3_queues_free(if_ctx_t ctx) 944 { 945 struct vmxnet3_softc *sc; 946 947 sc = iflib_get_softc(ctx); 948 949 /* Free queue state area that is shared with the device */ 950 if (sc->vmx_qs_dma.idi_size != 0) { 951 iflib_dma_free(&sc->vmx_qs_dma); 952 sc->vmx_qs_dma.idi_size = 0; 953 } 954 955 /* Free array of receive queues */ 956 if (sc->vmx_rxq != NULL) { 957 free(sc->vmx_rxq, M_DEVBUF); 958 sc->vmx_rxq = NULL; 959 } 960 961 /* Free array of transmit queues */ 962 if (sc->vmx_txq != NULL) { 963 free(sc->vmx_txq, M_DEVBUF); 964 sc->vmx_txq = NULL; 965 } 966 } 967 968 static int 969 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc) 970 { 971 device_t dev; 972 size_t size; 973 int error; 974 975 dev = sc->vmx_dev; 976 977 /* Top level state structure shared with the device */ 978 size = sizeof(struct vmxnet3_driver_shared); 979 error = iflib_dma_alloc_align(sc->vmx_ctx, size, 1, &sc->vmx_ds_dma, 0); 980 if (error) { 981 device_printf(dev, "cannot alloc shared memory\n"); 982 return (error); 983 } 984 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.idi_vaddr; 985 986 /* RSS table state shared with the device */ 987 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 988 size = sizeof(struct vmxnet3_rss_shared); 989 error = iflib_dma_alloc_align(sc->vmx_ctx, size, 128, 990 &sc->vmx_rss_dma, 0); 991 if (error) { 992 device_printf(dev, "cannot alloc rss shared memory\n"); 993 return (error); 994 } 995 sc->vmx_rss = 996 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.idi_vaddr; 997 } 998 999 return (0); 1000 } 1001 1002 static void 1003 vmxnet3_free_shared_data(struct vmxnet3_softc *sc) 1004 { 1005 1006 /* Free RSS table state shared with the device */ 1007 if (sc->vmx_rss != NULL) { 1008 iflib_dma_free(&sc->vmx_rss_dma); 1009 sc->vmx_rss = NULL; 1010 } 1011 1012 /* Free top level state structure shared with the device */ 1013 if (sc->vmx_ds != NULL) { 1014 iflib_dma_free(&sc->vmx_ds_dma); 1015 sc->vmx_ds = NULL; 1016 } 1017 } 1018 1019 static int 1020 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc) 1021 { 1022 int error; 1023 1024 /* Multicast table state shared with the device */ 1025 error = iflib_dma_alloc_align(sc->vmx_ctx, 1026 VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 32, &sc->vmx_mcast_dma, 0); 1027 if (error) 1028 device_printf(sc->vmx_dev, "unable to alloc multicast table\n"); 1029 else 1030 sc->vmx_mcast = sc->vmx_mcast_dma.idi_vaddr; 1031 1032 return (error); 1033 } 1034 1035 static void 1036 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc) 1037 { 1038 1039 /* Free multicast table state shared with the device */ 1040 if (sc->vmx_mcast != NULL) { 1041 iflib_dma_free(&sc->vmx_mcast_dma); 1042 sc->vmx_mcast = NULL; 1043 } 1044 } 1045 1046 static void 1047 vmxnet3_init_shared_data(struct vmxnet3_softc *sc) 1048 { 1049 struct vmxnet3_driver_shared *ds; 1050 if_shared_ctx_t sctx; 1051 if_softc_ctx_t scctx; 1052 struct vmxnet3_txqueue *txq; 1053 struct vmxnet3_txq_shared *txs; 1054 struct vmxnet3_rxqueue *rxq; 1055 struct vmxnet3_rxq_shared *rxs; 1056 int i; 1057 1058 ds = sc->vmx_ds; 1059 sctx = sc->vmx_sctx; 1060 scctx = sc->vmx_scctx; 1061 1062 /* 1063 * Initialize fields of the shared data that remains the same across 1064 * reinits. Note the shared data is zero'd when allocated. 1065 */ 1066 1067 ds->magic = VMXNET3_REV1_MAGIC; 1068 1069 /* DriverInfo */ 1070 ds->version = VMXNET3_DRIVER_VERSION; 1071 ds->guest = VMXNET3_GOS_FREEBSD | 1072 #ifdef __LP64__ 1073 VMXNET3_GOS_64BIT; 1074 #else 1075 VMXNET3_GOS_32BIT; 1076 #endif 1077 ds->vmxnet3_revision = 1; 1078 ds->upt_version = 1; 1079 1080 /* Misc. conf */ 1081 ds->driver_data = vtophys(sc); 1082 ds->driver_data_len = sizeof(struct vmxnet3_softc); 1083 ds->queue_shared = sc->vmx_qs_dma.idi_paddr; 1084 ds->queue_shared_len = sc->vmx_qs_dma.idi_size; 1085 ds->nrxsg_max = IFLIB_MAX_RX_SEGS; 1086 1087 /* RSS conf */ 1088 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1089 ds->rss.version = 1; 1090 ds->rss.paddr = sc->vmx_rss_dma.idi_paddr; 1091 ds->rss.len = sc->vmx_rss_dma.idi_size; 1092 } 1093 1094 /* Interrupt control. */ 1095 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO; 1096 /* 1097 * Total number of interrupt indexes we are using in the shared 1098 * config data, even though we don't actually allocate interrupt 1099 * resources for the tx queues. Some versions of the device will 1100 * fail to initialize successfully if interrupt indexes are used in 1101 * the shared config that exceed the number of interrupts configured 1102 * here. 1103 */ 1104 ds->nintr = (scctx->isc_vectors == 1) ? 1105 2 : (scctx->isc_nrxqsets + scctx->isc_ntxqsets + 1); 1106 ds->evintr = sc->vmx_event_intr_idx; 1107 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 1108 1109 for (i = 0; i < ds->nintr; i++) 1110 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 1111 1112 /* Receive filter. */ 1113 ds->mcast_table = sc->vmx_mcast_dma.idi_paddr; 1114 ds->mcast_tablelen = sc->vmx_mcast_dma.idi_size; 1115 1116 /* Tx queues */ 1117 for (i = 0; i < scctx->isc_ntxqsets; i++) { 1118 txq = &sc->vmx_txq[i]; 1119 txs = txq->vxtxq_ts; 1120 1121 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_paddr; 1122 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc; 1123 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_paddr; 1124 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc; 1125 txs->driver_data = vtophys(txq); 1126 txs->driver_data_len = sizeof(struct vmxnet3_txqueue); 1127 } 1128 1129 /* Rx queues */ 1130 for (i = 0; i < scctx->isc_nrxqsets; i++) { 1131 rxq = &sc->vmx_rxq[i]; 1132 rxs = rxq->vxrxq_rs; 1133 1134 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_paddr; 1135 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc; 1136 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_paddr; 1137 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc; 1138 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_paddr; 1139 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc; 1140 rxs->driver_data = vtophys(rxq); 1141 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue); 1142 } 1143 } 1144 1145 static void 1146 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc) 1147 { 1148 /* 1149 * Use the same key as the Linux driver until FreeBSD can do 1150 * RSS (presumably Toeplitz) in software. 1151 */ 1152 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = { 1153 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac, 1154 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28, 1155 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70, 1156 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3, 1157 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9, 1158 }; 1159 1160 struct vmxnet3_driver_shared *ds; 1161 if_softc_ctx_t scctx; 1162 struct vmxnet3_rss_shared *rss; 1163 #ifdef RSS 1164 uint8_t rss_algo; 1165 #endif 1166 int i; 1167 1168 ds = sc->vmx_ds; 1169 scctx = sc->vmx_scctx; 1170 rss = sc->vmx_rss; 1171 1172 rss->hash_type = 1173 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 | 1174 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6; 1175 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 1176 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE; 1177 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE; 1178 #ifdef RSS 1179 /* 1180 * If the software RSS is configured to anything else other than 1181 * Toeplitz, then just do Toeplitz in "hardware" for the sake of 1182 * the packet distribution, but report the hash as opaque to 1183 * disengage from the software RSS. 1184 */ 1185 rss_algo = rss_gethashalgo(); 1186 if (rss_algo == RSS_HASH_TOEPLITZ) { 1187 rss_getkey(rss->hash_key); 1188 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) { 1189 rss->ind_table[i] = rss_get_indirection_to_bucket(i) % 1190 scctx->isc_nrxqsets; 1191 } 1192 sc->vmx_flags |= VMXNET3_FLAG_SOFT_RSS; 1193 } else 1194 #endif 1195 { 1196 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE); 1197 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) 1198 rss->ind_table[i] = i % scctx->isc_nrxqsets; 1199 sc->vmx_flags &= ~VMXNET3_FLAG_SOFT_RSS; 1200 } 1201 } 1202 1203 static void 1204 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc) 1205 { 1206 struct ifnet *ifp; 1207 struct vmxnet3_driver_shared *ds; 1208 if_softc_ctx_t scctx; 1209 1210 ifp = sc->vmx_ifp; 1211 ds = sc->vmx_ds; 1212 scctx = sc->vmx_scctx; 1213 1214 ds->mtu = ifp->if_mtu; 1215 ds->ntxqueue = scctx->isc_ntxqsets; 1216 ds->nrxqueue = scctx->isc_nrxqsets; 1217 1218 ds->upt_features = 0; 1219 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 1220 ds->upt_features |= UPT1_F_CSUM; 1221 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1222 ds->upt_features |= UPT1_F_VLAN; 1223 if (ifp->if_capenable & IFCAP_LRO) 1224 ds->upt_features |= UPT1_F_LRO; 1225 1226 if (sc->vmx_flags & VMXNET3_FLAG_RSS) { 1227 ds->upt_features |= UPT1_F_RSS; 1228 vmxnet3_reinit_rss_shared_data(sc); 1229 } 1230 1231 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.idi_paddr); 1232 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH, 1233 (uint64_t) sc->vmx_ds_dma.idi_paddr >> 32); 1234 } 1235 1236 static int 1237 vmxnet3_alloc_data(struct vmxnet3_softc *sc) 1238 { 1239 int error; 1240 1241 error = vmxnet3_alloc_shared_data(sc); 1242 if (error) 1243 return (error); 1244 1245 error = vmxnet3_alloc_mcast_table(sc); 1246 if (error) 1247 return (error); 1248 1249 vmxnet3_init_shared_data(sc); 1250 1251 return (0); 1252 } 1253 1254 static void 1255 vmxnet3_free_data(struct vmxnet3_softc *sc) 1256 { 1257 1258 vmxnet3_free_mcast_table(sc); 1259 vmxnet3_free_shared_data(sc); 1260 } 1261 1262 static void 1263 vmxnet3_evintr(struct vmxnet3_softc *sc) 1264 { 1265 device_t dev; 1266 struct vmxnet3_txq_shared *ts; 1267 struct vmxnet3_rxq_shared *rs; 1268 uint32_t event; 1269 1270 dev = sc->vmx_dev; 1271 1272 /* Clear events. */ 1273 event = sc->vmx_ds->event; 1274 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event); 1275 1276 if (event & VMXNET3_EVENT_LINK) 1277 vmxnet3_link_status(sc); 1278 1279 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1280 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS); 1281 ts = sc->vmx_txq[0].vxtxq_ts; 1282 if (ts->stopped != 0) 1283 device_printf(dev, "Tx queue error %#x\n", ts->error); 1284 rs = sc->vmx_rxq[0].vxrxq_rs; 1285 if (rs->stopped != 0) 1286 device_printf(dev, "Rx queue error %#x\n", rs->error); 1287 1288 /* XXX - rely on liflib watchdog to reset us? */ 1289 device_printf(dev, "Rx/Tx queue error event ... " 1290 "waiting for iflib watchdog reset\n"); 1291 } 1292 1293 if (event & VMXNET3_EVENT_DIC) 1294 device_printf(dev, "device implementation change event\n"); 1295 if (event & VMXNET3_EVENT_DEBUG) 1296 device_printf(dev, "debug event\n"); 1297 } 1298 1299 static int 1300 vmxnet3_isc_txd_encap(void *vsc, if_pkt_info_t pi) 1301 { 1302 struct vmxnet3_softc *sc; 1303 struct vmxnet3_txqueue *txq; 1304 struct vmxnet3_txring *txr; 1305 struct vmxnet3_txdesc *txd, *sop; 1306 bus_dma_segment_t *segs; 1307 int nsegs; 1308 int pidx; 1309 int hdrlen; 1310 int i; 1311 int gen; 1312 1313 sc = vsc; 1314 txq = &sc->vmx_txq[pi->ipi_qsidx]; 1315 txr = &txq->vxtxq_cmd_ring; 1316 segs = pi->ipi_segs; 1317 nsegs = pi->ipi_nsegs; 1318 pidx = pi->ipi_pidx; 1319 1320 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS, 1321 ("%s: packet with too many segments %d", __func__, nsegs)); 1322 1323 sop = &txr->vxtxr_txd[pidx]; 1324 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */ 1325 1326 for (i = 0; i < nsegs; i++) { 1327 txd = &txr->vxtxr_txd[pidx]; 1328 1329 txd->addr = segs[i].ds_addr; 1330 txd->len = segs[i].ds_len; 1331 txd->gen = gen; 1332 txd->dtype = 0; 1333 txd->offload_mode = VMXNET3_OM_NONE; 1334 txd->offload_pos = 0; 1335 txd->hlen = 0; 1336 txd->eop = 0; 1337 txd->compreq = 0; 1338 txd->vtag_mode = 0; 1339 txd->vtag = 0; 1340 1341 if (++pidx == txr->vxtxr_ndesc) { 1342 pidx = 0; 1343 txr->vxtxr_gen ^= 1; 1344 } 1345 gen = txr->vxtxr_gen; 1346 } 1347 txd->eop = 1; 1348 txd->compreq = !!(pi->ipi_flags & IPI_TX_INTR); 1349 pi->ipi_new_pidx = pidx; 1350 1351 /* 1352 * VLAN 1353 */ 1354 if (pi->ipi_mflags & M_VLANTAG) { 1355 sop->vtag_mode = 1; 1356 sop->vtag = pi->ipi_vtag; 1357 } 1358 1359 /* 1360 * TSO and checksum offloads 1361 */ 1362 hdrlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 1363 if (pi->ipi_csum_flags & CSUM_TSO) { 1364 sop->offload_mode = VMXNET3_OM_TSO; 1365 sop->hlen = hdrlen + pi->ipi_tcp_hlen; 1366 sop->offload_pos = pi->ipi_tso_segsz; 1367 } else if (pi->ipi_csum_flags & (VMXNET3_CSUM_OFFLOAD | 1368 VMXNET3_CSUM_OFFLOAD_IPV6)) { 1369 sop->offload_mode = VMXNET3_OM_CSUM; 1370 sop->hlen = hdrlen; 1371 sop->offload_pos = hdrlen + 1372 ((pi->ipi_ipproto == IPPROTO_TCP) ? 1373 offsetof(struct tcphdr, th_sum) : 1374 offsetof(struct udphdr, uh_sum)); 1375 } 1376 1377 /* Finally, change the ownership. */ 1378 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR); 1379 sop->gen ^= 1; 1380 1381 return (0); 1382 } 1383 1384 static void 1385 vmxnet3_isc_txd_flush(void *vsc, uint16_t txqid, qidx_t pidx) 1386 { 1387 struct vmxnet3_softc *sc; 1388 struct vmxnet3_txqueue *txq; 1389 1390 sc = vsc; 1391 txq = &sc->vmx_txq[txqid]; 1392 1393 /* 1394 * pidx is what we last set ipi_new_pidx to in 1395 * vmxnet3_isc_txd_encap() 1396 */ 1397 1398 /* 1399 * Avoid expensive register updates if the flush request is 1400 * redundant. 1401 */ 1402 if (txq->vxtxq_last_flush == pidx) 1403 return; 1404 txq->vxtxq_last_flush = pidx; 1405 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), pidx); 1406 } 1407 1408 static int 1409 vmxnet3_isc_txd_credits_update(void *vsc, uint16_t txqid, bool clear) 1410 { 1411 struct vmxnet3_softc *sc; 1412 struct vmxnet3_txqueue *txq; 1413 struct vmxnet3_comp_ring *txc; 1414 struct vmxnet3_txcompdesc *txcd; 1415 struct vmxnet3_txring *txr; 1416 int processed; 1417 1418 sc = vsc; 1419 txq = &sc->vmx_txq[txqid]; 1420 txc = &txq->vxtxq_comp_ring; 1421 txr = &txq->vxtxq_cmd_ring; 1422 1423 /* 1424 * If clear is true, we need to report the number of TX command ring 1425 * descriptors that have been processed by the device. If clear is 1426 * false, we just need to report whether or not at least one TX 1427 * command ring descriptor has been processed by the device. 1428 */ 1429 processed = 0; 1430 for (;;) { 1431 txcd = &txc->vxcr_u.txcd[txc->vxcr_next]; 1432 if (txcd->gen != txc->vxcr_gen) 1433 break; 1434 else if (!clear) 1435 return (1); 1436 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1437 1438 if (++txc->vxcr_next == txc->vxcr_ndesc) { 1439 txc->vxcr_next = 0; 1440 txc->vxcr_gen ^= 1; 1441 } 1442 1443 if (txcd->eop_idx < txr->vxtxr_next) 1444 processed += txr->vxtxr_ndesc - 1445 (txr->vxtxr_next - txcd->eop_idx) + 1; 1446 else 1447 processed += txcd->eop_idx - txr->vxtxr_next + 1; 1448 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc; 1449 } 1450 1451 return (processed); 1452 } 1453 1454 static int 1455 vmxnet3_isc_rxd_available(void *vsc, uint16_t rxqid, qidx_t idx, qidx_t budget) 1456 { 1457 struct vmxnet3_softc *sc; 1458 struct vmxnet3_rxqueue *rxq; 1459 struct vmxnet3_comp_ring *rxc; 1460 struct vmxnet3_rxcompdesc *rxcd; 1461 int avail; 1462 int completed_gen; 1463 #ifdef INVARIANTS 1464 int expect_sop = 1; 1465 #endif 1466 sc = vsc; 1467 rxq = &sc->vmx_rxq[rxqid]; 1468 rxc = &rxq->vxrxq_comp_ring; 1469 1470 avail = 0; 1471 completed_gen = rxc->vxcr_gen; 1472 for (;;) { 1473 rxcd = &rxc->vxcr_u.rxcd[idx]; 1474 if (rxcd->gen != completed_gen) 1475 break; 1476 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD); 1477 1478 #ifdef INVARIANTS 1479 if (expect_sop) 1480 KASSERT(rxcd->sop, ("%s: expected sop", __func__)); 1481 else 1482 KASSERT(!rxcd->sop, ("%s: unexpected sop", __func__)); 1483 expect_sop = rxcd->eop; 1484 #endif 1485 if (rxcd->eop && (rxcd->len != 0)) 1486 avail++; 1487 if (avail > budget) 1488 break; 1489 if (++idx == rxc->vxcr_ndesc) { 1490 idx = 0; 1491 completed_gen ^= 1; 1492 } 1493 } 1494 1495 return (avail); 1496 } 1497 1498 static int 1499 vmxnet3_isc_rxd_pkt_get(void *vsc, if_rxd_info_t ri) 1500 { 1501 struct vmxnet3_softc *sc; 1502 if_softc_ctx_t scctx; 1503 struct vmxnet3_rxqueue *rxq; 1504 struct vmxnet3_comp_ring *rxc; 1505 struct vmxnet3_rxcompdesc *rxcd; 1506 if_rxd_frag_t frag; 1507 int cqidx; 1508 uint16_t total_len; 1509 uint8_t nfrags; 1510 uint8_t i; 1511 uint8_t flid; 1512 1513 sc = vsc; 1514 scctx = sc->vmx_scctx; 1515 rxq = &sc->vmx_rxq[ri->iri_qsidx]; 1516 rxc = &rxq->vxrxq_comp_ring; 1517 1518 /* 1519 * Get a single packet starting at the given index in the completion 1520 * queue. That we have been called indicates that 1521 * vmxnet3_isc_rxd_available() has already verified that either 1522 * there is a complete packet available starting at the given index, 1523 * or there are one or more zero length packets starting at the 1524 * given index followed by a complete packet, so no verification of 1525 * ownership of the descriptors (and no associated read barrier) is 1526 * required here. 1527 */ 1528 cqidx = ri->iri_cidx; 1529 rxcd = &rxc->vxcr_u.rxcd[cqidx]; 1530 while (rxcd->len == 0) { 1531 KASSERT(rxcd->sop && rxcd->eop, 1532 ("%s: zero-length packet without both sop and eop set", 1533 __func__)); 1534 rxc->vxcr_zero_length++; 1535 if (++cqidx == rxc->vxcr_ndesc) { 1536 cqidx = 0; 1537 rxc->vxcr_gen ^= 1; 1538 } 1539 rxcd = &rxc->vxcr_u.rxcd[cqidx]; 1540 } 1541 KASSERT(rxcd->sop, ("%s: expected sop", __func__)); 1542 1543 /* 1544 * RSS and flow ID. 1545 * Types other than M_HASHTYPE_NONE and M_HASHTYPE_OPAQUE_HASH should 1546 * be used only if the software RSS is enabled and it uses the same 1547 * algorithm and the hash key as the "hardware". If the software RSS 1548 * is not enabled, then it's simply pointless to use those types. 1549 * If it's enabled but with different parameters, then hash values will 1550 * not match. 1551 */ 1552 ri->iri_flowid = rxcd->rss_hash; 1553 #ifdef RSS 1554 if ((sc->vmx_flags & VMXNET3_FLAG_SOFT_RSS) != 0) { 1555 switch (rxcd->rss_type) { 1556 case VMXNET3_RCD_RSS_TYPE_NONE: 1557 ri->iri_flowid = ri->iri_qsidx; 1558 ri->iri_rsstype = M_HASHTYPE_NONE; 1559 break; 1560 case VMXNET3_RCD_RSS_TYPE_IPV4: 1561 ri->iri_rsstype = M_HASHTYPE_RSS_IPV4; 1562 break; 1563 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 1564 ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV4; 1565 break; 1566 case VMXNET3_RCD_RSS_TYPE_IPV6: 1567 ri->iri_rsstype = M_HASHTYPE_RSS_IPV6; 1568 break; 1569 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 1570 ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV6; 1571 break; 1572 default: 1573 ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH; 1574 break; 1575 } 1576 } else 1577 #endif 1578 { 1579 switch (rxcd->rss_type) { 1580 case VMXNET3_RCD_RSS_TYPE_NONE: 1581 ri->iri_flowid = ri->iri_qsidx; 1582 ri->iri_rsstype = M_HASHTYPE_NONE; 1583 break; 1584 default: 1585 ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH; 1586 break; 1587 } 1588 } 1589 1590 /* 1591 * The queue numbering scheme used for rxcd->qid is as follows: 1592 * - All of the command ring 0s are numbered [0, nrxqsets - 1] 1593 * - All of the command ring 1s are numbered [nrxqsets, 2*nrxqsets - 1] 1594 * 1595 * Thus, rxcd->qid less than nrxqsets indicates command ring (and 1596 * flid) 0, and rxcd->qid greater than or equal to nrxqsets 1597 * indicates command ring (and flid) 1. 1598 */ 1599 nfrags = 0; 1600 total_len = 0; 1601 do { 1602 rxcd = &rxc->vxcr_u.rxcd[cqidx]; 1603 KASSERT(rxcd->gen == rxc->vxcr_gen, 1604 ("%s: generation mismatch", __func__)); 1605 KASSERT(nfrags < IFLIB_MAX_RX_SEGS, 1606 ("%s: too many fragments", __func__)); 1607 if (__predict_true(rxcd->len != 0)) { 1608 frag = &ri->iri_frags[nfrags]; 1609 flid = (rxcd->qid >= scctx->isc_nrxqsets) ? 1 : 0; 1610 frag->irf_flid = flid; 1611 frag->irf_idx = rxcd->rxd_idx; 1612 frag->irf_len = rxcd->len; 1613 total_len += rxcd->len; 1614 nfrags++; 1615 } else { 1616 rxc->vcxr_zero_length_frag++; 1617 } 1618 if (++cqidx == rxc->vxcr_ndesc) { 1619 cqidx = 0; 1620 rxc->vxcr_gen ^= 1; 1621 } 1622 } while (!rxcd->eop); 1623 1624 ri->iri_cidx = cqidx; 1625 ri->iri_nfrags = nfrags; 1626 ri->iri_len = total_len; 1627 1628 /* 1629 * If there's an error, the last descriptor in the packet will 1630 * have the error indicator set. In this case, set all 1631 * fragment lengths to zero. This will cause iflib to discard 1632 * the packet, but process all associated descriptors through 1633 * the refill mechanism. 1634 */ 1635 if (__predict_false(rxcd->error)) { 1636 rxc->vxcr_pkt_errors++; 1637 for (i = 0; i < nfrags; i++) { 1638 frag = &ri->iri_frags[i]; 1639 frag->irf_len = 0; 1640 } 1641 } else { 1642 /* Checksum offload information is in the last descriptor. */ 1643 if (!rxcd->no_csum) { 1644 uint32_t csum_flags = 0; 1645 1646 if (rxcd->ipv4) { 1647 csum_flags |= CSUM_IP_CHECKED; 1648 if (rxcd->ipcsum_ok) 1649 csum_flags |= CSUM_IP_VALID; 1650 } 1651 if (!rxcd->fragment && (rxcd->tcp || rxcd->udp)) { 1652 csum_flags |= CSUM_L4_CALC; 1653 if (rxcd->csum_ok) { 1654 csum_flags |= CSUM_L4_VALID; 1655 ri->iri_csum_data = 0xffff; 1656 } 1657 } 1658 ri->iri_csum_flags = csum_flags; 1659 } 1660 1661 /* VLAN information is in the last descriptor. */ 1662 if (rxcd->vlan) { 1663 ri->iri_flags |= M_VLANTAG; 1664 ri->iri_vtag = rxcd->vtag; 1665 } 1666 } 1667 1668 return (0); 1669 } 1670 1671 static void 1672 vmxnet3_isc_rxd_refill(void *vsc, if_rxd_update_t iru) 1673 { 1674 struct vmxnet3_softc *sc; 1675 struct vmxnet3_rxqueue *rxq; 1676 struct vmxnet3_rxring *rxr; 1677 struct vmxnet3_rxdesc *rxd; 1678 uint64_t *paddrs; 1679 int count; 1680 int len; 1681 int idx; 1682 int i; 1683 uint8_t flid; 1684 uint8_t btype; 1685 1686 count = iru->iru_count; 1687 len = iru->iru_buf_size; 1688 flid = iru->iru_flidx; 1689 paddrs = iru->iru_paddrs; 1690 1691 sc = vsc; 1692 rxq = &sc->vmx_rxq[iru->iru_qsidx]; 1693 rxr = &rxq->vxrxq_cmd_ring[flid]; 1694 rxd = rxr->vxrxr_rxd; 1695 1696 /* 1697 * Command ring 0 is filled with BTYPE_HEAD descriptors, and 1698 * command ring 1 is filled with BTYPE_BODY descriptors. 1699 */ 1700 btype = (flid == 0) ? VMXNET3_BTYPE_HEAD : VMXNET3_BTYPE_BODY; 1701 /* 1702 * The refill entries from iflib will advance monotonically, 1703 * but the refilled descriptors may not be contiguous due to 1704 * earlier skipping of descriptors by the device. The refill 1705 * entries from iflib need an entire state update, while the 1706 * descriptors previously skipped by the device only need to 1707 * have their generation numbers updated. 1708 */ 1709 idx = rxr->vxrxr_refill_start; 1710 i = 0; 1711 do { 1712 if (idx == iru->iru_idxs[i]) { 1713 rxd[idx].addr = paddrs[i]; 1714 rxd[idx].len = len; 1715 rxd[idx].btype = btype; 1716 i++; 1717 } else 1718 rxr->vxrxr_desc_skips++; 1719 rxd[idx].gen = rxr->vxrxr_gen; 1720 1721 if (++idx == rxr->vxrxr_ndesc) { 1722 idx = 0; 1723 rxr->vxrxr_gen ^= 1; 1724 } 1725 } while (i != count); 1726 rxr->vxrxr_refill_start = idx; 1727 } 1728 1729 static void 1730 vmxnet3_isc_rxd_flush(void *vsc, uint16_t rxqid, uint8_t flid, qidx_t pidx) 1731 { 1732 struct vmxnet3_softc *sc; 1733 struct vmxnet3_rxqueue *rxq; 1734 struct vmxnet3_rxring *rxr; 1735 bus_size_t r; 1736 1737 sc = vsc; 1738 rxq = &sc->vmx_rxq[rxqid]; 1739 rxr = &rxq->vxrxq_cmd_ring[flid]; 1740 1741 if (flid == 0) 1742 r = VMXNET3_BAR0_RXH1(rxqid); 1743 else 1744 r = VMXNET3_BAR0_RXH2(rxqid); 1745 1746 vmxnet3_write_bar0(sc, r, pidx); 1747 } 1748 1749 static int 1750 vmxnet3_legacy_intr(void *xsc) 1751 { 1752 struct vmxnet3_softc *sc; 1753 if_softc_ctx_t scctx; 1754 if_ctx_t ctx; 1755 1756 sc = xsc; 1757 scctx = sc->vmx_scctx; 1758 ctx = sc->vmx_ctx; 1759 1760 /* 1761 * When there is only a single interrupt configured, this routine 1762 * runs in fast interrupt context, following which the rxq 0 task 1763 * will be enqueued. 1764 */ 1765 if (scctx->isc_intr == IFLIB_INTR_LEGACY) { 1766 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0) 1767 return (FILTER_HANDLED); 1768 } 1769 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1770 vmxnet3_intr_disable_all(ctx); 1771 1772 if (sc->vmx_ds->event != 0) 1773 iflib_admin_intr_deferred(ctx); 1774 1775 /* 1776 * XXX - When there is both rxq and event activity, do we care 1777 * whether the rxq 0 task or the admin task re-enables the interrupt 1778 * first? 1779 */ 1780 return (FILTER_SCHEDULE_THREAD); 1781 } 1782 1783 static int 1784 vmxnet3_rxq_intr(void *vrxq) 1785 { 1786 struct vmxnet3_softc *sc; 1787 struct vmxnet3_rxqueue *rxq; 1788 1789 rxq = vrxq; 1790 sc = rxq->vxrxq_sc; 1791 1792 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1793 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx); 1794 1795 return (FILTER_SCHEDULE_THREAD); 1796 } 1797 1798 static int 1799 vmxnet3_event_intr(void *vsc) 1800 { 1801 struct vmxnet3_softc *sc; 1802 1803 sc = vsc; 1804 1805 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE) 1806 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx); 1807 1808 /* 1809 * The work will be done via vmxnet3_update_admin_status(), and the 1810 * interrupt will be re-enabled in vmxnet3_link_intr_enable(). 1811 * 1812 * The interrupt will be re-enabled by vmxnet3_link_intr_enable(). 1813 */ 1814 return (FILTER_SCHEDULE_THREAD); 1815 } 1816 1817 static void 1818 vmxnet3_stop(if_ctx_t ctx) 1819 { 1820 struct vmxnet3_softc *sc; 1821 1822 sc = iflib_get_softc(ctx); 1823 1824 sc->vmx_link_active = 0; 1825 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE); 1826 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET); 1827 } 1828 1829 static void 1830 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq) 1831 { 1832 struct vmxnet3_txring *txr; 1833 struct vmxnet3_comp_ring *txc; 1834 1835 txq->vxtxq_last_flush = -1; 1836 1837 txr = &txq->vxtxq_cmd_ring; 1838 txr->vxtxr_next = 0; 1839 txr->vxtxr_gen = VMXNET3_INIT_GEN; 1840 /* 1841 * iflib has zeroed out the descriptor array during the prior attach 1842 * or stop 1843 */ 1844 1845 txc = &txq->vxtxq_comp_ring; 1846 txc->vxcr_next = 0; 1847 txc->vxcr_gen = VMXNET3_INIT_GEN; 1848 /* 1849 * iflib has zeroed out the descriptor array during the prior attach 1850 * or stop 1851 */ 1852 } 1853 1854 static void 1855 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq) 1856 { 1857 struct vmxnet3_rxring *rxr; 1858 struct vmxnet3_comp_ring *rxc; 1859 int i; 1860 1861 /* 1862 * The descriptors will be populated with buffers during a 1863 * subsequent invocation of vmxnet3_isc_rxd_refill() 1864 */ 1865 for (i = 0; i < sc->vmx_sctx->isc_nrxqs - 1; i++) { 1866 rxr = &rxq->vxrxq_cmd_ring[i]; 1867 rxr->vxrxr_gen = VMXNET3_INIT_GEN; 1868 rxr->vxrxr_desc_skips = 0; 1869 rxr->vxrxr_refill_start = 0; 1870 /* 1871 * iflib has zeroed out the descriptor array during the 1872 * prior attach or stop 1873 */ 1874 } 1875 1876 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) { 1877 rxr = &rxq->vxrxq_cmd_ring[i]; 1878 rxr->vxrxr_gen = 0; 1879 rxr->vxrxr_desc_skips = 0; 1880 rxr->vxrxr_refill_start = 0; 1881 bzero(rxr->vxrxr_rxd, 1882 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc)); 1883 } 1884 1885 rxc = &rxq->vxrxq_comp_ring; 1886 rxc->vxcr_next = 0; 1887 rxc->vxcr_gen = VMXNET3_INIT_GEN; 1888 rxc->vxcr_zero_length = 0; 1889 rxc->vcxr_zero_length_frag = 0; 1890 rxc->vxcr_pkt_errors = 0; 1891 /* 1892 * iflib has zeroed out the descriptor array during the prior attach 1893 * or stop 1894 */ 1895 } 1896 1897 static void 1898 vmxnet3_reinit_queues(struct vmxnet3_softc *sc) 1899 { 1900 if_softc_ctx_t scctx; 1901 int q; 1902 1903 scctx = sc->vmx_scctx; 1904 1905 for (q = 0; q < scctx->isc_ntxqsets; q++) 1906 vmxnet3_txinit(sc, &sc->vmx_txq[q]); 1907 1908 for (q = 0; q < scctx->isc_nrxqsets; q++) 1909 vmxnet3_rxinit(sc, &sc->vmx_rxq[q]); 1910 } 1911 1912 static int 1913 vmxnet3_enable_device(struct vmxnet3_softc *sc) 1914 { 1915 if_softc_ctx_t scctx; 1916 int q; 1917 1918 scctx = sc->vmx_scctx; 1919 1920 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) { 1921 device_printf(sc->vmx_dev, "device enable command failed!\n"); 1922 return (1); 1923 } 1924 1925 /* Reset the Rx queue heads. */ 1926 for (q = 0; q < scctx->isc_nrxqsets; q++) { 1927 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0); 1928 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0); 1929 } 1930 1931 return (0); 1932 } 1933 1934 static void 1935 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc) 1936 { 1937 struct ifnet *ifp; 1938 1939 ifp = sc->vmx_ifp; 1940 1941 vmxnet3_set_rxfilter(sc, if_getflags(ifp)); 1942 1943 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 1944 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter, 1945 sizeof(sc->vmx_ds->vlan_filter)); 1946 else 1947 bzero(sc->vmx_ds->vlan_filter, 1948 sizeof(sc->vmx_ds->vlan_filter)); 1949 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER); 1950 } 1951 1952 static void 1953 vmxnet3_init(if_ctx_t ctx) 1954 { 1955 struct vmxnet3_softc *sc; 1956 1957 sc = iflib_get_softc(ctx); 1958 1959 /* Use the current MAC address. */ 1960 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN); 1961 vmxnet3_set_lladdr(sc); 1962 1963 vmxnet3_reinit_shared_data(sc); 1964 vmxnet3_reinit_queues(sc); 1965 1966 vmxnet3_enable_device(sc); 1967 1968 vmxnet3_reinit_rxfilters(sc); 1969 vmxnet3_link_status(sc); 1970 } 1971 1972 static void 1973 vmxnet3_multi_set(if_ctx_t ctx) 1974 { 1975 1976 vmxnet3_set_rxfilter(iflib_get_softc(ctx), 1977 if_getflags(iflib_get_ifp(ctx))); 1978 } 1979 1980 static int 1981 vmxnet3_mtu_set(if_ctx_t ctx, uint32_t mtu) 1982 { 1983 struct vmxnet3_softc *sc; 1984 if_softc_ctx_t scctx; 1985 1986 sc = iflib_get_softc(ctx); 1987 scctx = sc->vmx_scctx; 1988 1989 if (mtu > VMXNET3_TX_MAXSIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 1990 ETHER_CRC_LEN)) 1991 return (EINVAL); 1992 1993 /* 1994 * Update the max frame size so that the rx mbuf size is 1995 * chosen based on the new mtu during the interface init that 1996 * will occur after this routine returns. 1997 */ 1998 scctx->isc_max_frame_size = mtu + 1999 ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN; 2000 /* RX completion queue - n/a */ 2001 scctx->isc_rxd_buf_size[0] = 0; 2002 /* 2003 * For header-type descriptors (used for first segment of 2004 * packet), let iflib determine the buffer size based on the 2005 * max frame size. 2006 */ 2007 scctx->isc_rxd_buf_size[1] = 0; 2008 /* 2009 * For body-type descriptors (used for jumbo frames and LRO), 2010 * always use page-sized buffers. 2011 */ 2012 scctx->isc_rxd_buf_size[2] = MJUMPAGESIZE; 2013 2014 return (0); 2015 } 2016 2017 static void 2018 vmxnet3_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) 2019 { 2020 struct vmxnet3_softc *sc; 2021 2022 sc = iflib_get_softc(ctx); 2023 2024 ifmr->ifm_status = IFM_AVALID; 2025 ifmr->ifm_active = IFM_ETHER; 2026 2027 if (vmxnet3_link_is_up(sc) != 0) { 2028 ifmr->ifm_status |= IFM_ACTIVE; 2029 ifmr->ifm_active |= IFM_AUTO; 2030 } else 2031 ifmr->ifm_active |= IFM_NONE; 2032 } 2033 2034 static int 2035 vmxnet3_media_change(if_ctx_t ctx) 2036 { 2037 2038 /* Ignore. */ 2039 return (0); 2040 } 2041 2042 static int 2043 vmxnet3_promisc_set(if_ctx_t ctx, int flags) 2044 { 2045 2046 vmxnet3_set_rxfilter(iflib_get_softc(ctx), flags); 2047 2048 return (0); 2049 } 2050 2051 static uint64_t 2052 vmxnet3_get_counter(if_ctx_t ctx, ift_counter cnt) 2053 { 2054 if_t ifp = iflib_get_ifp(ctx); 2055 2056 if (cnt < IFCOUNTERS) 2057 return if_get_counter_default(ifp, cnt); 2058 2059 return (0); 2060 } 2061 2062 static void 2063 vmxnet3_update_admin_status(if_ctx_t ctx) 2064 { 2065 struct vmxnet3_softc *sc; 2066 2067 sc = iflib_get_softc(ctx); 2068 if (sc->vmx_ds->event != 0) 2069 vmxnet3_evintr(sc); 2070 2071 vmxnet3_refresh_host_stats(sc); 2072 } 2073 2074 static void 2075 vmxnet3_txq_timer(if_ctx_t ctx, uint16_t qid) 2076 { 2077 /* Host stats refresh is global, so just trigger it on txq 0 */ 2078 if (qid == 0) 2079 vmxnet3_refresh_host_stats(iflib_get_softc(ctx)); 2080 } 2081 2082 static void 2083 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag) 2084 { 2085 int idx, bit; 2086 2087 if (tag == 0 || tag > 4095) 2088 return; 2089 2090 idx = (tag >> 5) & 0x7F; 2091 bit = tag & 0x1F; 2092 2093 /* Update our private VLAN bitvector. */ 2094 if (add) 2095 sc->vmx_vlan_filter[idx] |= (1 << bit); 2096 else 2097 sc->vmx_vlan_filter[idx] &= ~(1 << bit); 2098 } 2099 2100 static void 2101 vmxnet3_vlan_register(if_ctx_t ctx, uint16_t tag) 2102 { 2103 2104 vmxnet3_update_vlan_filter(iflib_get_softc(ctx), 1, tag); 2105 } 2106 2107 static void 2108 vmxnet3_vlan_unregister(if_ctx_t ctx, uint16_t tag) 2109 { 2110 2111 vmxnet3_update_vlan_filter(iflib_get_softc(ctx), 0, tag); 2112 } 2113 2114 static u_int 2115 vmxnet3_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int count) 2116 { 2117 struct vmxnet3_softc *sc = arg; 2118 2119 if (count < VMXNET3_MULTICAST_MAX) 2120 bcopy(LLADDR(sdl), &sc->vmx_mcast[count * ETHER_ADDR_LEN], 2121 ETHER_ADDR_LEN); 2122 2123 return (1); 2124 } 2125 2126 static void 2127 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc, int flags) 2128 { 2129 struct ifnet *ifp; 2130 struct vmxnet3_driver_shared *ds; 2131 u_int mode; 2132 2133 ifp = sc->vmx_ifp; 2134 ds = sc->vmx_ds; 2135 2136 mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST; 2137 if (flags & IFF_PROMISC) 2138 mode |= VMXNET3_RXMODE_PROMISC; 2139 if (flags & IFF_ALLMULTI) 2140 mode |= VMXNET3_RXMODE_ALLMULTI; 2141 else { 2142 int cnt; 2143 2144 cnt = if_foreach_llmaddr(ifp, vmxnet3_hash_maddr, sc); 2145 if (cnt >= VMXNET3_MULTICAST_MAX) { 2146 cnt = 0; 2147 mode |= VMXNET3_RXMODE_ALLMULTI; 2148 } else if (cnt > 0) 2149 mode |= VMXNET3_RXMODE_MCAST; 2150 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN; 2151 } 2152 2153 ds->rxmode = mode; 2154 2155 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER); 2156 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE); 2157 } 2158 2159 static void 2160 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc) 2161 { 2162 2163 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS); 2164 } 2165 2166 static int 2167 vmxnet3_link_is_up(struct vmxnet3_softc *sc) 2168 { 2169 uint32_t status; 2170 2171 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK); 2172 return !!(status & 0x1); 2173 } 2174 2175 static void 2176 vmxnet3_link_status(struct vmxnet3_softc *sc) 2177 { 2178 if_ctx_t ctx; 2179 uint64_t speed; 2180 int link; 2181 2182 ctx = sc->vmx_ctx; 2183 link = vmxnet3_link_is_up(sc); 2184 speed = IF_Gbps(10); 2185 2186 if (link != 0 && sc->vmx_link_active == 0) { 2187 sc->vmx_link_active = 1; 2188 iflib_link_state_change(ctx, LINK_STATE_UP, speed); 2189 } else if (link == 0 && sc->vmx_link_active != 0) { 2190 sc->vmx_link_active = 0; 2191 iflib_link_state_change(ctx, LINK_STATE_DOWN, speed); 2192 } 2193 } 2194 2195 static void 2196 vmxnet3_set_lladdr(struct vmxnet3_softc *sc) 2197 { 2198 uint32_t ml, mh; 2199 2200 ml = sc->vmx_lladdr[0]; 2201 ml |= sc->vmx_lladdr[1] << 8; 2202 ml |= sc->vmx_lladdr[2] << 16; 2203 ml |= sc->vmx_lladdr[3] << 24; 2204 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml); 2205 2206 mh = sc->vmx_lladdr[4]; 2207 mh |= sc->vmx_lladdr[5] << 8; 2208 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh); 2209 } 2210 2211 static void 2212 vmxnet3_get_lladdr(struct vmxnet3_softc *sc) 2213 { 2214 uint32_t ml, mh; 2215 2216 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL); 2217 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH); 2218 2219 sc->vmx_lladdr[0] = ml; 2220 sc->vmx_lladdr[1] = ml >> 8; 2221 sc->vmx_lladdr[2] = ml >> 16; 2222 sc->vmx_lladdr[3] = ml >> 24; 2223 sc->vmx_lladdr[4] = mh; 2224 sc->vmx_lladdr[5] = mh >> 8; 2225 } 2226 2227 static void 2228 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq, 2229 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 2230 { 2231 struct sysctl_oid *node, *txsnode; 2232 struct sysctl_oid_list *list, *txslist; 2233 struct UPT1_TxStats *txstats; 2234 char namebuf[16]; 2235 2236 txstats = &txq->vxtxq_ts->stats; 2237 2238 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id); 2239 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 2240 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Transmit Queue"); 2241 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node); 2242 2243 /* 2244 * Add statistics reported by the host. These are updated by the 2245 * iflib txq timer on txq 0. 2246 */ 2247 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", 2248 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 2249 txslist = SYSCTL_CHILDREN(txsnode); 2250 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD, 2251 &txstats->TSO_packets, "TSO packets"); 2252 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD, 2253 &txstats->TSO_bytes, "TSO bytes"); 2254 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 2255 &txstats->ucast_packets, "Unicast packets"); 2256 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 2257 &txstats->ucast_bytes, "Unicast bytes"); 2258 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 2259 &txstats->mcast_packets, "Multicast packets"); 2260 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 2261 &txstats->mcast_bytes, "Multicast bytes"); 2262 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD, 2263 &txstats->error, "Errors"); 2264 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD, 2265 &txstats->discard, "Discards"); 2266 } 2267 2268 static void 2269 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq, 2270 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 2271 { 2272 struct sysctl_oid *node, *rxsnode; 2273 struct sysctl_oid_list *list, *rxslist; 2274 struct UPT1_RxStats *rxstats; 2275 char namebuf[16]; 2276 2277 rxstats = &rxq->vxrxq_rs->stats; 2278 2279 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id); 2280 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 2281 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Receive Queue"); 2282 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node); 2283 2284 /* 2285 * Add statistics reported by the host. These are updated by the 2286 * iflib txq timer on txq 0. 2287 */ 2288 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats", 2289 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics"); 2290 rxslist = SYSCTL_CHILDREN(rxsnode); 2291 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD, 2292 &rxstats->LRO_packets, "LRO packets"); 2293 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD, 2294 &rxstats->LRO_bytes, "LRO bytes"); 2295 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD, 2296 &rxstats->ucast_packets, "Unicast packets"); 2297 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD, 2298 &rxstats->ucast_bytes, "Unicast bytes"); 2299 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD, 2300 &rxstats->mcast_packets, "Multicast packets"); 2301 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD, 2302 &rxstats->mcast_bytes, "Multicast bytes"); 2303 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD, 2304 &rxstats->bcast_packets, "Broadcast packets"); 2305 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD, 2306 &rxstats->bcast_bytes, "Broadcast bytes"); 2307 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD, 2308 &rxstats->nobuffer, "No buffer"); 2309 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD, 2310 &rxstats->error, "Errors"); 2311 } 2312 2313 static void 2314 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc, 2315 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 2316 { 2317 if_softc_ctx_t scctx; 2318 struct sysctl_oid *node; 2319 struct sysctl_oid_list *list; 2320 int i; 2321 2322 scctx = sc->vmx_scctx; 2323 2324 for (i = 0; i < scctx->isc_ntxqsets; i++) { 2325 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i]; 2326 2327 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO, 2328 "debug", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2329 list = SYSCTL_CHILDREN(node); 2330 2331 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD, 2332 &txq->vxtxq_cmd_ring.vxtxr_next, 0, ""); 2333 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD, 2334 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, ""); 2335 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD, 2336 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, ""); 2337 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD, 2338 &txq->vxtxq_comp_ring.vxcr_next, 0, ""); 2339 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 2340 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,""); 2341 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 2342 &txq->vxtxq_comp_ring.vxcr_gen, 0, ""); 2343 } 2344 2345 for (i = 0; i < scctx->isc_nrxqsets; i++) { 2346 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i]; 2347 2348 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO, 2349 "debug", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2350 list = SYSCTL_CHILDREN(node); 2351 2352 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD, 2353 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, ""); 2354 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD, 2355 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, ""); 2356 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "cmd0_desc_skips", CTLFLAG_RD, 2357 &rxq->vxrxq_cmd_ring[0].vxrxr_desc_skips, 0, ""); 2358 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD, 2359 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, ""); 2360 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD, 2361 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, ""); 2362 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "cmd1_desc_skips", CTLFLAG_RD, 2363 &rxq->vxrxq_cmd_ring[1].vxrxr_desc_skips, 0, ""); 2364 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD, 2365 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,""); 2366 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD, 2367 &rxq->vxrxq_comp_ring.vxcr_gen, 0, ""); 2368 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_zero_length", CTLFLAG_RD, 2369 &rxq->vxrxq_comp_ring.vxcr_zero_length, 0, ""); 2370 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_zero_length_frag", 2371 CTLFLAG_RD, &rxq->vxrxq_comp_ring.vcxr_zero_length_frag, 2372 0, ""); 2373 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_pkt_errors", CTLFLAG_RD, 2374 &rxq->vxrxq_comp_ring.vxcr_pkt_errors, 0, ""); 2375 } 2376 } 2377 2378 static void 2379 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc, 2380 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child) 2381 { 2382 if_softc_ctx_t scctx; 2383 int i; 2384 2385 scctx = sc->vmx_scctx; 2386 2387 for (i = 0; i < scctx->isc_ntxqsets; i++) 2388 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child); 2389 for (i = 0; i < scctx->isc_nrxqsets; i++) 2390 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child); 2391 2392 vmxnet3_setup_debug_sysctl(sc, ctx, child); 2393 } 2394 2395 static void 2396 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc) 2397 { 2398 device_t dev; 2399 struct sysctl_ctx_list *ctx; 2400 struct sysctl_oid *tree; 2401 struct sysctl_oid_list *child; 2402 2403 dev = sc->vmx_dev; 2404 ctx = device_get_sysctl_ctx(dev); 2405 tree = device_get_sysctl_tree(dev); 2406 child = SYSCTL_CHILDREN(tree); 2407 2408 vmxnet3_setup_queue_sysctl(sc, ctx, child); 2409 } 2410 2411 static void 2412 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 2413 { 2414 2415 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v); 2416 } 2417 2418 static uint32_t 2419 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r) 2420 { 2421 2422 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r)); 2423 } 2424 2425 static void 2426 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v) 2427 { 2428 2429 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v); 2430 } 2431 2432 static void 2433 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 2434 { 2435 2436 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd); 2437 } 2438 2439 static uint32_t 2440 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd) 2441 { 2442 2443 vmxnet3_write_cmd(sc, cmd); 2444 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0, 2445 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 2446 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD)); 2447 } 2448 2449 static void 2450 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 2451 { 2452 2453 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0); 2454 } 2455 2456 static void 2457 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 2458 { 2459 2460 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1); 2461 } 2462 2463 static int 2464 vmxnet3_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 2465 { 2466 /* Not using interrupts for TX */ 2467 return (0); 2468 } 2469 2470 static int 2471 vmxnet3_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) 2472 { 2473 struct vmxnet3_softc *sc; 2474 2475 sc = iflib_get_softc(ctx); 2476 vmxnet3_enable_intr(sc, sc->vmx_rxq[qid].vxrxq_intr_idx); 2477 return (0); 2478 } 2479 2480 static void 2481 vmxnet3_link_intr_enable(if_ctx_t ctx) 2482 { 2483 struct vmxnet3_softc *sc; 2484 2485 sc = iflib_get_softc(ctx); 2486 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx); 2487 } 2488 2489 static void 2490 vmxnet3_intr_enable_all(if_ctx_t ctx) 2491 { 2492 struct vmxnet3_softc *sc; 2493 if_softc_ctx_t scctx; 2494 int i; 2495 2496 sc = iflib_get_softc(ctx); 2497 scctx = sc->vmx_scctx; 2498 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 2499 for (i = 0; i < scctx->isc_vectors; i++) 2500 vmxnet3_enable_intr(sc, i); 2501 } 2502 2503 static void 2504 vmxnet3_intr_disable_all(if_ctx_t ctx) 2505 { 2506 struct vmxnet3_softc *sc; 2507 int i; 2508 2509 sc = iflib_get_softc(ctx); 2510 /* 2511 * iflib may invoke this routine before vmxnet3_attach_post() has 2512 * run, which is before the top level shared data area is 2513 * initialized and the device made aware of it. 2514 */ 2515 if (sc->vmx_ds != NULL) 2516 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 2517 for (i = 0; i < VMXNET3_MAX_INTRS; i++) 2518 vmxnet3_disable_intr(sc, i); 2519 } 2520 2521 /* 2522 * Since this is a purely paravirtualized device, we do not have 2523 * to worry about DMA coherency. But at times, we must make sure 2524 * both the compiler and CPU do not reorder memory operations. 2525 */ 2526 static inline void 2527 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type) 2528 { 2529 2530 switch (type) { 2531 case VMXNET3_BARRIER_RD: 2532 rmb(); 2533 break; 2534 case VMXNET3_BARRIER_WR: 2535 wmb(); 2536 break; 2537 case VMXNET3_BARRIER_RDWR: 2538 mb(); 2539 break; 2540 default: 2541 panic("%s: bad barrier type %d", __func__, type); 2542 } 2543 } 2544