1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for VirtIO network devices. */ 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/eventhandler.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/sockio.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/msan.h> 43 #include <sys/sbuf.h> 44 #include <sys/socket.h> 45 #include <sys/sysctl.h> 46 #include <sys/random.h> 47 #include <sys/sglist.h> 48 #include <sys/lock.h> 49 #include <sys/mutex.h> 50 #include <sys/taskqueue.h> 51 #include <sys/smp.h> 52 #include <machine/smp.h> 53 54 #include <vm/uma.h> 55 56 #include <net/debugnet.h> 57 #include <net/ethernet.h> 58 #include <net/pfil.h> 59 #include <net/if.h> 60 #include <net/if_var.h> 61 #include <net/if_arp.h> 62 #include <net/if_dl.h> 63 #include <net/if_types.h> 64 #include <net/if_media.h> 65 #include <net/if_vlan_var.h> 66 67 #include <net/bpf.h> 68 69 #include <netinet/in_systm.h> 70 #include <netinet/in.h> 71 #include <netinet/ip.h> 72 #include <netinet/ip6.h> 73 #include <netinet6/ip6_var.h> 74 #include <netinet/udp.h> 75 #include <netinet/tcp.h> 76 #include <netinet/tcp_lro.h> 77 78 #include <machine/bus.h> 79 #include <machine/resource.h> 80 #include <sys/bus.h> 81 #include <sys/rman.h> 82 83 #include <dev/virtio/virtio.h> 84 #include <dev/virtio/virtqueue.h> 85 #include <dev/virtio/network/virtio_net.h> 86 #include <dev/virtio/network/if_vtnetvar.h> 87 #include "virtio_if.h" 88 89 #if defined(INET) || defined(INET6) 90 #include <machine/in_cksum.h> 91 #endif 92 93 #ifdef __NO_STRICT_ALIGNMENT 94 #define VTNET_ETHER_ALIGN 0 95 #else /* Strict alignment */ 96 #define VTNET_ETHER_ALIGN ETHER_ALIGN 97 #endif 98 99 /* 100 * Worst case offset to ensure header doesn't share any cache lines with 101 * payload. 102 */ 103 #define VTNET_RX_BUFFER_HEADER_OFFSET 128 104 105 struct vtnet_rx_buffer_header { 106 bus_addr_t addr; 107 bus_dmamap_t dmap; 108 }; 109 110 static int vtnet_modevent(module_t, int, void *); 111 112 static int vtnet_probe(device_t); 113 static int vtnet_attach(device_t); 114 static int vtnet_detach(device_t); 115 static int vtnet_suspend(device_t); 116 static int vtnet_resume(device_t); 117 static int vtnet_shutdown(device_t); 118 static int vtnet_attach_completed(device_t); 119 static int vtnet_config_change(device_t); 120 121 static int vtnet_negotiate_features(struct vtnet_softc *); 122 static int vtnet_setup_features(struct vtnet_softc *); 123 static int vtnet_init_rxq(struct vtnet_softc *, int); 124 static int vtnet_init_txq(struct vtnet_softc *, int); 125 static int vtnet_alloc_rxtx_queues(struct vtnet_softc *); 126 static void vtnet_free_rxtx_queues(struct vtnet_softc *); 127 static int vtnet_alloc_rx_filters(struct vtnet_softc *); 128 static void vtnet_free_rx_filters(struct vtnet_softc *); 129 static int vtnet_alloc_virtqueues(struct vtnet_softc *); 130 static void vtnet_alloc_interface(struct vtnet_softc *); 131 static int vtnet_setup_interface(struct vtnet_softc *); 132 static int vtnet_ioctl_mtu(struct vtnet_softc *, u_int); 133 static int vtnet_ioctl_ifflags(struct vtnet_softc *); 134 static int vtnet_ioctl_multi(struct vtnet_softc *); 135 static int vtnet_ioctl_ifcap(struct vtnet_softc *, struct ifreq *); 136 static int vtnet_ioctl(if_t, u_long, caddr_t); 137 static uint64_t vtnet_get_counter(if_t, ift_counter); 138 139 static int vtnet_rxq_populate(struct vtnet_rxq *); 140 static void vtnet_rxq_free_mbufs(struct vtnet_rxq *); 141 static struct mbuf * 142 vtnet_rx_alloc_buf(struct vtnet_softc *, int , struct mbuf **); 143 static int vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *, 144 struct mbuf *, int); 145 static int vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int); 146 static int vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *); 147 static int vtnet_rxq_new_buf(struct vtnet_rxq *); 148 #if defined(INET) || defined(INET6) 149 static void vtnet_rxq_csum_needs_csum(struct vtnet_rxq *, struct mbuf *, 150 bool, int, struct virtio_net_hdr *); 151 static void vtnet_rxq_csum_data_valid(struct vtnet_rxq *, struct mbuf *, 152 int); 153 static int vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *, 154 struct virtio_net_hdr *); 155 #endif 156 static void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int); 157 static void vtnet_rxq_discard_buf(struct vtnet_rxq *, struct mbuf *); 158 static int vtnet_rxq_merged_eof(struct vtnet_rxq *, struct mbuf *, int); 159 static void vtnet_rxq_input(struct vtnet_rxq *, struct mbuf *, 160 struct virtio_net_hdr *); 161 static int vtnet_rxq_eof(struct vtnet_rxq *); 162 static void vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries); 163 static void vtnet_rx_vq_intr(void *); 164 static void vtnet_rxq_tq_intr(void *, int); 165 166 static int vtnet_txq_intr_threshold(struct vtnet_txq *); 167 static int vtnet_txq_below_threshold(struct vtnet_txq *); 168 static int vtnet_txq_notify(struct vtnet_txq *); 169 static void vtnet_txq_free_mbufs(struct vtnet_txq *); 170 static int vtnet_txq_offload_ctx(struct vtnet_txq *, struct mbuf *, 171 int *, int *, int *); 172 static int vtnet_txq_offload_tso(struct vtnet_txq *, struct mbuf *, int, 173 int, struct virtio_net_hdr *); 174 static struct mbuf * 175 vtnet_txq_offload(struct vtnet_txq *, struct mbuf *, 176 struct virtio_net_hdr *); 177 static int vtnet_txq_enqueue_buf(struct vtnet_txq *, struct mbuf **, 178 struct vtnet_tx_header *); 179 static int vtnet_txq_encap(struct vtnet_txq *, struct mbuf **, int); 180 181 /* Required for ALTQ */ 182 static void vtnet_start_locked(struct vtnet_txq *, if_t); 183 static void vtnet_start(if_t); 184 185 /* Required for MQ */ 186 static int vtnet_txq_mq_start_locked(struct vtnet_txq *, struct mbuf *); 187 static int vtnet_txq_mq_start(if_t, struct mbuf *); 188 static void vtnet_txq_tq_deferred(void *, int); 189 static void vtnet_qflush(if_t); 190 191 192 static void vtnet_txq_start(struct vtnet_txq *); 193 static void vtnet_txq_tq_intr(void *, int); 194 static int vtnet_txq_eof(struct vtnet_txq *); 195 static void vtnet_tx_vq_intr(void *); 196 static void vtnet_tx_start_all(struct vtnet_softc *); 197 198 static int vtnet_watchdog(struct vtnet_txq *); 199 static void vtnet_accum_stats(struct vtnet_softc *, 200 struct vtnet_rxq_stats *, struct vtnet_txq_stats *); 201 static void vtnet_tick(void *); 202 203 static void vtnet_start_taskqueues(struct vtnet_softc *); 204 static void vtnet_free_taskqueues(struct vtnet_softc *); 205 static void vtnet_drain_taskqueues(struct vtnet_softc *); 206 207 static void vtnet_drain_rxtx_queues(struct vtnet_softc *); 208 static void vtnet_stop_rendezvous(struct vtnet_softc *); 209 static void vtnet_stop(struct vtnet_softc *); 210 static int vtnet_virtio_reinit(struct vtnet_softc *); 211 static void vtnet_init_rx_filters(struct vtnet_softc *); 212 static int vtnet_init_rx_queues(struct vtnet_softc *); 213 static int vtnet_init_tx_queues(struct vtnet_softc *); 214 static int vtnet_init_rxtx_queues(struct vtnet_softc *); 215 static void vtnet_set_active_vq_pairs(struct vtnet_softc *); 216 static void vtnet_update_rx_offloads(struct vtnet_softc *); 217 static int vtnet_reinit(struct vtnet_softc *); 218 static void vtnet_init_locked(struct vtnet_softc *, int); 219 static void vtnet_init(void *); 220 221 static void vtnet_free_ctrl_vq(struct vtnet_softc *); 222 static int vtnet_exec_ctrl_cmd(struct vtnet_softc *, uint8_t *, 223 struct sglist *, int, int); 224 static int vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *); 225 static int vtnet_ctrl_guest_offloads(struct vtnet_softc *, uint64_t); 226 static int vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t); 227 static int vtnet_ctrl_announce_ack_cmd(struct vtnet_softc *); 228 static bool vtnet_announce_pending(struct vtnet_softc *); 229 static void vtnet_announce(void *, int); 230 static int vtnet_ctrl_rx_cmd(struct vtnet_softc *, uint8_t, bool); 231 static int vtnet_set_promisc(struct vtnet_softc *, bool); 232 static int vtnet_set_allmulti(struct vtnet_softc *, bool); 233 static void vtnet_rx_filter(struct vtnet_softc *); 234 static void vtnet_rx_filter_mac(struct vtnet_softc *); 235 static int vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t); 236 static void vtnet_rx_filter_vlan(struct vtnet_softc *); 237 static void vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t); 238 static void vtnet_register_vlan(void *, if_t, uint16_t); 239 static void vtnet_unregister_vlan(void *, if_t, uint16_t); 240 241 static void vtnet_update_speed_duplex(struct vtnet_softc *); 242 static int vtnet_is_link_up(struct vtnet_softc *); 243 static void vtnet_update_link_status(struct vtnet_softc *); 244 static int vtnet_ifmedia_upd(if_t); 245 static void vtnet_ifmedia_sts(if_t, struct ifmediareq *); 246 static void vtnet_get_macaddr(struct vtnet_softc *); 247 static void vtnet_set_macaddr(struct vtnet_softc *); 248 static void vtnet_attached_set_macaddr(struct vtnet_softc *); 249 static void vtnet_vlan_tag_remove(struct mbuf *); 250 static void vtnet_set_rx_process_limit(struct vtnet_softc *); 251 252 static void vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *, 253 struct sysctl_oid_list *, struct vtnet_rxq *); 254 static void vtnet_setup_txq_sysctl(struct sysctl_ctx_list *, 255 struct sysctl_oid_list *, struct vtnet_txq *); 256 static void vtnet_setup_queue_sysctl(struct vtnet_softc *); 257 static void vtnet_load_tunables(struct vtnet_softc *); 258 static void vtnet_setup_sysctl(struct vtnet_softc *); 259 260 static int vtnet_rxq_enable_intr(struct vtnet_rxq *); 261 static void vtnet_rxq_disable_intr(struct vtnet_rxq *); 262 static int vtnet_txq_enable_intr(struct vtnet_txq *); 263 static void vtnet_txq_disable_intr(struct vtnet_txq *); 264 static void vtnet_enable_rx_interrupts(struct vtnet_softc *); 265 static void vtnet_enable_tx_interrupts(struct vtnet_softc *); 266 static void vtnet_enable_interrupts(struct vtnet_softc *); 267 static void vtnet_disable_rx_interrupts(struct vtnet_softc *); 268 static void vtnet_disable_tx_interrupts(struct vtnet_softc *); 269 static void vtnet_disable_interrupts(struct vtnet_softc *); 270 271 static int vtnet_tunable_int(struct vtnet_softc *, const char *, int); 272 273 DEBUGNET_DEFINE(vtnet); 274 275 #define vtnet_htog16(_sc, _val) virtio_htog16(vtnet_modern(_sc), _val) 276 #define vtnet_htog32(_sc, _val) virtio_htog32(vtnet_modern(_sc), _val) 277 #define vtnet_htog64(_sc, _val) virtio_htog64(vtnet_modern(_sc), _val) 278 #define vtnet_gtoh16(_sc, _val) virtio_gtoh16(vtnet_modern(_sc), _val) 279 #define vtnet_gtoh32(_sc, _val) virtio_gtoh32(vtnet_modern(_sc), _val) 280 #define vtnet_gtoh64(_sc, _val) virtio_gtoh64(vtnet_modern(_sc), _val) 281 282 /* Tunables. */ 283 static SYSCTL_NODE(_hw, OID_AUTO, vtnet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 284 "VirtIO Net driver parameters"); 285 286 static int vtnet_csum_disable = 0; 287 SYSCTL_INT(_hw_vtnet, OID_AUTO, csum_disable, CTLFLAG_RDTUN, 288 &vtnet_csum_disable, 0, "Disables receive and send checksum offload"); 289 290 static int vtnet_tso_disable = 0; 291 SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_disable, CTLFLAG_RDTUN, 292 &vtnet_tso_disable, 0, "Disables TSO"); 293 294 static int vtnet_lro_disable = 1; 295 SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_disable, CTLFLAG_RDTUN, 296 &vtnet_lro_disable, 0, "Disables hardware LRO"); 297 298 static int vtnet_mq_disable = 0; 299 SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_disable, CTLFLAG_RDTUN, 300 &vtnet_mq_disable, 0, "Disables multiqueue support"); 301 302 static int vtnet_mq_max_pairs = VTNET_MAX_QUEUE_PAIRS; 303 SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_max_pairs, CTLFLAG_RDTUN, 304 &vtnet_mq_max_pairs, 0, "Maximum number of multiqueue pairs"); 305 306 static int vtnet_tso_maxlen = IP_MAXPACKET; 307 SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN, 308 &vtnet_tso_maxlen, 0, "TSO burst limit"); 309 310 static int vtnet_rx_process_limit = 1024; 311 SYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 312 &vtnet_rx_process_limit, 0, 313 "Number of RX segments processed in one pass"); 314 315 static int vtnet_lro_entry_count = 128; 316 SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN, 317 &vtnet_lro_entry_count, 0, "Software LRO entry count"); 318 319 /* Enable sorted LRO, and the depth of the mbuf queue. */ 320 static int vtnet_lro_mbufq_depth = 0; 321 SYSCTL_UINT(_hw_vtnet, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN, 322 &vtnet_lro_mbufq_depth, 0, "Depth of software LRO mbuf queue"); 323 324 /* Deactivate ALTQ Support */ 325 static int vtnet_altq_disable = 0; 326 SYSCTL_INT(_hw_vtnet, OID_AUTO, altq_disable, CTLFLAG_RDTUN, 327 &vtnet_altq_disable, 0, "Disables ALTQ Support"); 328 329 /* 330 * For the driver to be considered as having altq enabled, 331 * it must be compiled with an ALTQ capable kernel, 332 * and the tunable hw.vtnet.altq_disable must be zero 333 */ 334 #define VTNET_ALTQ_ENABLED (VTNET_ALTQ_CAPABLE && (!vtnet_altq_disable)) 335 336 337 static uma_zone_t vtnet_tx_header_zone; 338 339 static struct virtio_feature_desc vtnet_feature_desc[] = { 340 { VIRTIO_NET_F_CSUM, "TxChecksum" }, 341 { VIRTIO_NET_F_GUEST_CSUM, "RxChecksum" }, 342 { VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, "CtrlRxOffloads" }, 343 { VIRTIO_NET_F_MAC, "MAC" }, 344 { VIRTIO_NET_F_GSO, "TxGSO" }, 345 { VIRTIO_NET_F_GUEST_TSO4, "RxLROv4" }, 346 { VIRTIO_NET_F_GUEST_TSO6, "RxLROv6" }, 347 { VIRTIO_NET_F_GUEST_ECN, "RxLROECN" }, 348 { VIRTIO_NET_F_GUEST_UFO, "RxUFO" }, 349 { VIRTIO_NET_F_HOST_TSO4, "TxTSOv4" }, 350 { VIRTIO_NET_F_HOST_TSO6, "TxTSOv6" }, 351 { VIRTIO_NET_F_HOST_ECN, "TxTSOECN" }, 352 { VIRTIO_NET_F_HOST_UFO, "TxUFO" }, 353 { VIRTIO_NET_F_MRG_RXBUF, "MrgRxBuf" }, 354 { VIRTIO_NET_F_STATUS, "Status" }, 355 { VIRTIO_NET_F_CTRL_VQ, "CtrlVq" }, 356 { VIRTIO_NET_F_CTRL_RX, "CtrlRxMode" }, 357 { VIRTIO_NET_F_CTRL_VLAN, "CtrlVLANFilter" }, 358 { VIRTIO_NET_F_CTRL_RX_EXTRA, "CtrlRxModeExtra" }, 359 { VIRTIO_NET_F_GUEST_ANNOUNCE, "GuestAnnounce" }, 360 { VIRTIO_NET_F_MQ, "Multiqueue" }, 361 { VIRTIO_NET_F_CTRL_MAC_ADDR, "CtrlMacAddr" }, 362 { VIRTIO_NET_F_SPEED_DUPLEX, "SpeedDuplex" }, 363 364 { 0, NULL } 365 }; 366 367 static device_method_t vtnet_methods[] = { 368 /* Device methods. */ 369 DEVMETHOD(device_probe, vtnet_probe), 370 DEVMETHOD(device_attach, vtnet_attach), 371 DEVMETHOD(device_detach, vtnet_detach), 372 DEVMETHOD(device_suspend, vtnet_suspend), 373 DEVMETHOD(device_resume, vtnet_resume), 374 DEVMETHOD(device_shutdown, vtnet_shutdown), 375 376 /* VirtIO methods. */ 377 DEVMETHOD(virtio_attach_completed, vtnet_attach_completed), 378 DEVMETHOD(virtio_config_change, vtnet_config_change), 379 380 DEVMETHOD_END 381 }; 382 383 #ifdef DEV_NETMAP 384 #include <dev/netmap/if_vtnet_netmap.h> 385 #endif 386 387 static driver_t vtnet_driver = { 388 .name = "vtnet", 389 .methods = vtnet_methods, 390 .size = sizeof(struct vtnet_softc) 391 }; 392 VIRTIO_DRIVER_MODULE(vtnet, vtnet_driver, vtnet_modevent, NULL); 393 MODULE_VERSION(vtnet, 1); 394 MODULE_DEPEND(vtnet, virtio, 1, 1, 1); 395 #ifdef DEV_NETMAP 396 MODULE_DEPEND(vtnet, netmap, 1, 1, 1); 397 #endif 398 399 VIRTIO_SIMPLE_PNPINFO(vtnet, VIRTIO_ID_NETWORK, "VirtIO Networking Adapter"); 400 401 static struct vtnet_rx_buffer_header * 402 vtnet_mbuf_to_rx_buffer_header(struct vtnet_softc *sc, struct mbuf *m) 403 { 404 if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0) 405 return (struct vtnet_rx_buffer_header *)((uintptr_t)m->m_data - 406 VTNET_RX_BUFFER_HEADER_OFFSET - VTNET_ETHER_ALIGN); 407 else 408 return (struct vtnet_rx_buffer_header *)((uintptr_t)m->m_data - 409 VTNET_RX_BUFFER_HEADER_OFFSET); 410 } 411 412 static int 413 vtnet_modevent(module_t mod __unused, int type, void *unused __unused) 414 { 415 int error = 0; 416 static int loaded = 0; 417 418 switch (type) { 419 case MOD_LOAD: 420 if (loaded++ == 0) { 421 vtnet_tx_header_zone = uma_zcreate("vtnet_tx_hdr", 422 sizeof(struct vtnet_tx_header), 423 NULL, NULL, NULL, NULL, 0, 0); 424 #ifdef DEBUGNET 425 /* 426 * We need to allocate from this zone in the transmit path, so ensure 427 * that we have at least one item per header available. 428 * XXX add a separate zone like we do for mbufs? otherwise we may alloc 429 * buckets 430 */ 431 uma_zone_reserve(vtnet_tx_header_zone, DEBUGNET_MAX_IN_FLIGHT * 2); 432 uma_prealloc(vtnet_tx_header_zone, DEBUGNET_MAX_IN_FLIGHT * 2); 433 #endif 434 } 435 break; 436 case MOD_QUIESCE: 437 if (uma_zone_get_cur(vtnet_tx_header_zone) > 0) 438 error = EBUSY; 439 break; 440 case MOD_UNLOAD: 441 if (--loaded == 0) { 442 uma_zdestroy(vtnet_tx_header_zone); 443 vtnet_tx_header_zone = NULL; 444 } 445 break; 446 case MOD_SHUTDOWN: 447 break; 448 default: 449 error = EOPNOTSUPP; 450 break; 451 } 452 453 return (error); 454 } 455 456 static int 457 vtnet_probe(device_t dev) 458 { 459 return (VIRTIO_SIMPLE_PROBE(dev, vtnet)); 460 } 461 462 static int 463 vtnet_attach(device_t dev) 464 { 465 struct vtnet_softc *sc; 466 int error; 467 468 sc = device_get_softc(dev); 469 sc->vtnet_dev = dev; 470 virtio_set_feature_desc(dev, vtnet_feature_desc); 471 472 VTNET_CORE_LOCK_INIT(sc); 473 callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0); 474 TASK_INIT(&sc->vtnet_announce_task, 0, vtnet_announce, sc); 475 vtnet_load_tunables(sc); 476 477 vtnet_alloc_interface(sc); 478 vtnet_setup_sysctl(sc); 479 480 error = vtnet_setup_features(sc); 481 if (error) { 482 device_printf(dev, "cannot setup features\n"); 483 goto fail; 484 } 485 486 mtx_init(&sc->vtnet_rx_mtx, device_get_nameunit(dev), 487 "VirtIO Net RX lock", MTX_DEF); 488 489 error = bus_dma_tag_create( 490 bus_get_dma_tag(dev), /* parent */ 491 1, /* alignment */ 492 0, /* boundary */ 493 BUS_SPACE_MAXADDR, /* lowaddr */ 494 BUS_SPACE_MAXADDR, /* highaddr */ 495 NULL, NULL, /* filter, filterarg */ 496 MJUM9BYTES, /* max request size */ 497 1, /* max # segments */ 498 MJUM9BYTES, /* maxsegsize - worst case */ 499 BUS_DMA_COHERENT, /* flags */ 500 busdma_lock_mutex, /* lockfunc */ 501 &sc->vtnet_rx_mtx, /* lockarg */ 502 &sc->vtnet_rx_dmat); 503 if (error) { 504 device_printf(dev, "cannot create bus_dma_tag\n"); 505 goto fail; 506 } 507 508 mtx_init(&sc->vtnet_tx_mtx, device_get_nameunit(dev), 509 "VirtIO Net TX lock", MTX_DEF); 510 511 error = bus_dma_tag_create( 512 bus_get_dma_tag(dev), /* parent */ 513 1, /* alignment */ 514 0, /* boundary */ 515 BUS_SPACE_MAXADDR, /* lowaddr */ 516 BUS_SPACE_MAXADDR, /* highaddr */ 517 NULL, NULL, /* filter, filterarg */ 518 sc->vtnet_tx_nsegs * MJUM9BYTES, /* max request size */ 519 sc->vtnet_tx_nsegs, /* max # segments */ 520 MJUM9BYTES, /* maxsegsize */ 521 BUS_DMA_COHERENT, /* flags */ 522 busdma_lock_mutex, /* lockfunc */ 523 &sc->vtnet_tx_mtx, /* lockarg */ 524 &sc->vtnet_tx_dmat); 525 if (error) { 526 device_printf(dev, "cannot create bus_dma_tag\n"); 527 goto fail; 528 } 529 530 mtx_init(&sc->vtnet_hdr_mtx, device_get_nameunit(dev), 531 "VirtIO Net header lock", MTX_DEF); 532 533 error = bus_dma_tag_create( 534 bus_get_dma_tag(dev), /* parent */ 535 sizeof(uint16_t), /* alignment */ 536 0, /* boundary */ 537 BUS_SPACE_MAXADDR, /* lowaddr */ 538 BUS_SPACE_MAXADDR, /* highaddr */ 539 NULL, NULL, /* filter, filterarg */ 540 PAGE_SIZE, /* max request size */ 541 1, /* max # segments */ 542 PAGE_SIZE, /* maxsegsize */ 543 BUS_DMA_COHERENT, /* flags */ 544 busdma_lock_mutex, /* lockfunc */ 545 &sc->vtnet_hdr_mtx, /* lockarg */ 546 &sc->vtnet_hdr_dmat); 547 if (error) { 548 device_printf(dev, "cannot create bus_dma_tag\n"); 549 goto fail; 550 } 551 552 mtx_init(&sc->vtnet_ack_mtx, device_get_nameunit(dev), 553 "VirtIO Net ACK lock", MTX_DEF); 554 555 error = bus_dma_tag_create( 556 bus_get_dma_tag(dev), /* parent */ 557 sizeof(uint8_t), /* alignment */ 558 0, /* boundary */ 559 BUS_SPACE_MAXADDR, /* lowaddr */ 560 BUS_SPACE_MAXADDR, /* highaddr */ 561 NULL, NULL, /* filter, filterarg */ 562 sizeof(uint8_t), /* max request size */ 563 1, /* max # segments */ 564 sizeof(uint8_t), /* maxsegsize */ 565 BUS_DMA_COHERENT, /* flags */ 566 busdma_lock_mutex, /* lockfunc */ 567 &sc->vtnet_ack_mtx, /* lockarg */ 568 &sc->vtnet_ack_dmat); 569 if (error) { 570 device_printf(dev, "cannot create bus_dma_tag\n"); 571 goto fail; 572 } 573 574 #ifdef __powerpc__ 575 /* 576 * Virtio uses physical addresses rather than bus addresses, so we 577 * need to ask busdma to skip the iommu physical->bus mapping. At 578 * present, this is only a thing on the powerpc architectures. 579 */ 580 bus_dma_tag_set_iommu(sc->vtnet_rx_dmat, NULL, NULL); 581 bus_dma_tag_set_iommu(sc->vtnet_tx_dmat, NULL, NULL); 582 bus_dma_tag_set_iommu(sc->vtnet_hdr_dmat, NULL, NULL); 583 bus_dma_tag_set_iommu(sc->vtnet_ack_dmat, NULL, NULL); 584 #endif 585 586 error = vtnet_alloc_rx_filters(sc); 587 if (error) { 588 device_printf(dev, "cannot allocate Rx filters\n"); 589 goto fail; 590 } 591 592 error = vtnet_alloc_rxtx_queues(sc); 593 if (error) { 594 device_printf(dev, "cannot allocate queues\n"); 595 goto fail; 596 } 597 598 error = vtnet_alloc_virtqueues(sc); 599 if (error) { 600 device_printf(dev, "cannot allocate virtqueues\n"); 601 goto fail; 602 } 603 604 error = vtnet_setup_interface(sc); 605 if (error) { 606 device_printf(dev, "cannot setup interface\n"); 607 goto fail; 608 } 609 610 error = virtio_setup_intr(dev, INTR_TYPE_NET); 611 if (error) { 612 device_printf(dev, "cannot setup interrupts\n"); 613 ether_ifdetach(sc->vtnet_ifp); 614 goto fail; 615 } 616 617 #ifdef DEV_NETMAP 618 vtnet_netmap_attach(sc); 619 #endif 620 vtnet_start_taskqueues(sc); 621 622 fail: 623 if (error) 624 vtnet_detach(dev); 625 626 return (error); 627 } 628 629 static int 630 vtnet_detach(device_t dev) 631 { 632 struct vtnet_softc *sc; 633 if_t ifp; 634 635 sc = device_get_softc(dev); 636 ifp = sc->vtnet_ifp; 637 638 if (device_is_attached(dev)) { 639 VTNET_CORE_LOCK(sc); 640 vtnet_stop(sc); 641 VTNET_CORE_UNLOCK(sc); 642 643 callout_drain(&sc->vtnet_tick_ch); 644 vtnet_drain_taskqueues(sc); 645 646 ether_ifdetach(ifp); 647 } 648 649 taskqueue_drain(taskqueue_thread, &sc->vtnet_announce_task); 650 651 #ifdef DEV_NETMAP 652 netmap_detach(ifp); 653 #endif 654 655 if (sc->vtnet_pfil != NULL) { 656 pfil_head_unregister(sc->vtnet_pfil); 657 sc->vtnet_pfil = NULL; 658 } 659 660 vtnet_free_taskqueues(sc); 661 662 if (sc->vtnet_vlan_attach != NULL) { 663 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach); 664 sc->vtnet_vlan_attach = NULL; 665 } 666 if (sc->vtnet_vlan_detach != NULL) { 667 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vtnet_vlan_detach); 668 sc->vtnet_vlan_detach = NULL; 669 } 670 671 ifmedia_removeall(&sc->vtnet_media); 672 673 if (ifp != NULL) { 674 if_free(ifp); 675 sc->vtnet_ifp = NULL; 676 } 677 678 vtnet_free_rxtx_queues(sc); 679 vtnet_free_rx_filters(sc); 680 681 if (sc->vtnet_ctrl_vq != NULL) 682 vtnet_free_ctrl_vq(sc); 683 684 VTNET_CORE_LOCK_DESTROY(sc); 685 686 return (0); 687 } 688 689 static int 690 vtnet_suspend(device_t dev) 691 { 692 struct vtnet_softc *sc; 693 694 sc = device_get_softc(dev); 695 696 VTNET_CORE_LOCK(sc); 697 vtnet_stop(sc); 698 sc->vtnet_flags |= VTNET_FLAG_SUSPENDED; 699 VTNET_CORE_UNLOCK(sc); 700 701 return (0); 702 } 703 704 static int 705 vtnet_resume(device_t dev) 706 { 707 struct vtnet_softc *sc; 708 if_t ifp; 709 710 sc = device_get_softc(dev); 711 ifp = sc->vtnet_ifp; 712 713 VTNET_CORE_LOCK(sc); 714 if (if_getflags(ifp) & IFF_UP) 715 vtnet_init_locked(sc, 0); 716 sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED; 717 VTNET_CORE_UNLOCK(sc); 718 719 return (0); 720 } 721 722 static int 723 vtnet_shutdown(device_t dev) 724 { 725 /* 726 * Suspend already does all of what we need to 727 * do here; we just never expect to be resumed. 728 */ 729 return (vtnet_suspend(dev)); 730 } 731 732 static int 733 vtnet_attach_completed(device_t dev) 734 { 735 struct vtnet_softc *sc; 736 737 sc = device_get_softc(dev); 738 739 VTNET_CORE_LOCK(sc); 740 vtnet_attached_set_macaddr(sc); 741 VTNET_CORE_UNLOCK(sc); 742 743 return (0); 744 } 745 746 static int 747 vtnet_config_change(device_t dev) 748 { 749 struct vtnet_softc *sc; 750 751 sc = device_get_softc(dev); 752 753 VTNET_CORE_LOCK(sc); 754 vtnet_update_link_status(sc); 755 if (vtnet_announce_pending(sc)) 756 taskqueue_enqueue(taskqueue_thread, &sc->vtnet_announce_task); 757 if (sc->vtnet_link_active != 0) 758 vtnet_tx_start_all(sc); 759 VTNET_CORE_UNLOCK(sc); 760 761 return (0); 762 } 763 764 static int 765 vtnet_negotiate_features(struct vtnet_softc *sc) 766 { 767 device_t dev; 768 uint64_t features, negotiated_features; 769 int error, no_csum; 770 771 dev = sc->vtnet_dev; 772 features = virtio_bus_is_modern(dev) ? VTNET_MODERN_FEATURES : 773 VTNET_LEGACY_FEATURES; 774 775 /* 776 * TSO and LRO are only available when their corresponding checksum 777 * offload feature is also negotiated. 778 */ 779 no_csum = vtnet_tunable_int(sc, "csum_disable", vtnet_csum_disable); 780 if (no_csum) 781 features &= ~(VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM); 782 if (no_csum || vtnet_tunable_int(sc, "tso_disable", vtnet_tso_disable)) 783 features &= ~VTNET_TSO_FEATURES; 784 if (no_csum || vtnet_tunable_int(sc, "lro_disable", vtnet_lro_disable)) 785 features &= ~VTNET_LRO_FEATURES; 786 787 /* Deactivate MQ Feature flag, if driver has ALTQ enabled, or MQ is explicitly disabled */ 788 if (VTNET_ALTQ_ENABLED || vtnet_tunable_int(sc, "mq_disable", vtnet_mq_disable)) 789 features &= ~VIRTIO_NET_F_MQ; 790 791 negotiated_features = virtio_negotiate_features(dev, features); 792 793 if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) { 794 uint16_t mtu; 795 796 mtu = virtio_read_dev_config_2(dev, 797 offsetof(struct virtio_net_config, mtu)); 798 if (mtu < VTNET_MIN_MTU) { 799 device_printf(dev, "Invalid MTU value: %d. " 800 "MTU feature disabled.\n", mtu); 801 features &= ~VIRTIO_NET_F_MTU; 802 negotiated_features = 803 virtio_negotiate_features(dev, features); 804 } 805 } 806 807 if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) { 808 uint16_t npairs; 809 810 npairs = virtio_read_dev_config_2(dev, 811 offsetof(struct virtio_net_config, max_virtqueue_pairs)); 812 if (npairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 813 npairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) { 814 device_printf(dev, "Invalid max_virtqueue_pairs value: " 815 "%d. Multiqueue feature disabled.\n", npairs); 816 features &= ~VIRTIO_NET_F_MQ; 817 negotiated_features = 818 virtio_negotiate_features(dev, features); 819 } 820 } 821 822 if (virtio_with_feature(dev, VTNET_LRO_FEATURES) && 823 virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) { 824 /* 825 * LRO without mergeable buffers requires special care. This 826 * is not ideal because every receive buffer must be large 827 * enough to hold the maximum TCP packet, the Ethernet header, 828 * and the header. This requires up to 34 descriptors with 829 * MCLBYTES clusters. If we do not have indirect descriptors, 830 * LRO is disabled since the virtqueue will not contain very 831 * many receive buffers. 832 */ 833 if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) { 834 device_printf(dev, 835 "Host LRO disabled since both mergeable buffers " 836 "and indirect descriptors were not negotiated\n"); 837 features &= ~VTNET_LRO_FEATURES; 838 negotiated_features = 839 virtio_negotiate_features(dev, features); 840 } else 841 sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG; 842 } 843 844 sc->vtnet_features = negotiated_features; 845 sc->vtnet_negotiated_features = negotiated_features; 846 847 error = virtio_finalize_features(dev); 848 if (error != 0 && (features & VTNET_OFFLOAD_FEATURES) != 0) { 849 device_printf(dev, 850 "retrying feature negotiation without offloads\n"); 851 features &= ~VTNET_OFFLOAD_FEATURES; 852 negotiated_features &= ~VTNET_OFFLOAD_FEATURES; 853 sc->vtnet_flags &= ~VTNET_FLAG_LRO_NOMRG; 854 sc->vtnet_features = negotiated_features; 855 sc->vtnet_negotiated_features = negotiated_features; 856 error = virtio_reinit(dev, features); 857 } 858 859 return (error); 860 } 861 862 static int 863 vtnet_setup_features(struct vtnet_softc *sc) 864 { 865 device_t dev; 866 int error; 867 868 dev = sc->vtnet_dev; 869 870 error = vtnet_negotiate_features(sc); 871 if (error) 872 return (error); 873 874 if (virtio_with_feature(dev, VIRTIO_F_VERSION_1)) 875 sc->vtnet_flags |= VTNET_FLAG_MODERN; 876 if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) 877 sc->vtnet_flags |= VTNET_FLAG_INDIRECT; 878 if (virtio_with_feature(dev, VIRTIO_RING_F_EVENT_IDX)) 879 sc->vtnet_flags |= VTNET_FLAG_EVENT_IDX; 880 881 if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) { 882 /* This feature should always be negotiated. */ 883 sc->vtnet_flags |= VTNET_FLAG_MAC; 884 } 885 886 if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) { 887 sc->vtnet_max_mtu = virtio_read_dev_config_2(dev, 888 offsetof(struct virtio_net_config, mtu)); 889 } else 890 sc->vtnet_max_mtu = VTNET_MAX_MTU; 891 892 if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) { 893 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS; 894 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); 895 } else if (vtnet_modern(sc)) { 896 /* This is identical to the mergeable header. */ 897 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_v1); 898 } else 899 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr); 900 901 if (vtnet_modern(sc) || sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) 902 sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_INLINE; 903 else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) 904 sc->vtnet_rx_nsegs = VTNET_RX_SEGS_LRO_NOMRG; 905 else 906 sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_SEPARATE; 907 908 /* 909 * Favor "hardware" LRO if negotiated, but support software LRO as 910 * a fallback; there is usually little benefit (or worse) with both. 911 */ 912 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) == 0 && 913 virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6) == 0) 914 sc->vtnet_flags |= VTNET_FLAG_SW_LRO; 915 916 if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) || 917 virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) || 918 virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) 919 sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MAX; 920 else 921 sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MIN; 922 923 sc->vtnet_req_vq_pairs = 1; 924 sc->vtnet_max_vq_pairs = 1; 925 926 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) { 927 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ; 928 929 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX)) 930 sc->vtnet_flags |= VTNET_FLAG_CTRL_RX; 931 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN)) 932 sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER; 933 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR)) 934 sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC; 935 936 if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) { 937 sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev, 938 offsetof(struct virtio_net_config, 939 max_virtqueue_pairs)); 940 } 941 } 942 943 if (sc->vtnet_max_vq_pairs > 1) { 944 int req; 945 946 /* 947 * Limit the maximum number of requested queue pairs to the 948 * number of CPUs and the configured maximum. 949 */ 950 req = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs); 951 if (req < 0) 952 req = 1; 953 if (req == 0) 954 req = mp_ncpus; 955 if (req > sc->vtnet_max_vq_pairs) 956 req = sc->vtnet_max_vq_pairs; 957 if (req > mp_ncpus) 958 req = mp_ncpus; 959 if (req > 1) { 960 sc->vtnet_req_vq_pairs = req; 961 sc->vtnet_flags |= VTNET_FLAG_MQ; 962 } 963 } 964 965 return (0); 966 } 967 968 static int 969 vtnet_init_rxq(struct vtnet_softc *sc, int id) 970 { 971 struct vtnet_rxq *rxq; 972 973 rxq = &sc->vtnet_rxqs[id]; 974 975 snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d", 976 device_get_nameunit(sc->vtnet_dev), id); 977 mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF); 978 979 rxq->vtnrx_sc = sc; 980 rxq->vtnrx_id = id; 981 982 rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT); 983 if (rxq->vtnrx_sg == NULL) 984 return (ENOMEM); 985 986 #if defined(INET) || defined(INET6) 987 if (vtnet_software_lro(sc)) { 988 if (tcp_lro_init_args(&rxq->vtnrx_lro, sc->vtnet_ifp, 989 sc->vtnet_lro_entry_count, sc->vtnet_lro_mbufq_depth) != 0) 990 return (ENOMEM); 991 } 992 #endif 993 994 NET_TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq); 995 rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT, 996 taskqueue_thread_enqueue, &rxq->vtnrx_tq); 997 998 return (rxq->vtnrx_tq == NULL ? ENOMEM : 0); 999 } 1000 1001 static int 1002 vtnet_init_txq(struct vtnet_softc *sc, int id) 1003 { 1004 struct vtnet_txq *txq; 1005 1006 txq = &sc->vtnet_txqs[id]; 1007 1008 snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d", 1009 device_get_nameunit(sc->vtnet_dev), id); 1010 mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF); 1011 1012 txq->vtntx_sc = sc; 1013 txq->vtntx_id = id; 1014 1015 txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT); 1016 if (txq->vtntx_sg == NULL) 1017 return (ENOMEM); 1018 1019 if (!VTNET_ALTQ_ENABLED) { 1020 txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF, 1021 M_NOWAIT, &txq->vtntx_mtx); 1022 if (txq->vtntx_br == NULL) 1023 return (ENOMEM); 1024 1025 TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq); 1026 } 1027 TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq); 1028 txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT, 1029 taskqueue_thread_enqueue, &txq->vtntx_tq); 1030 if (txq->vtntx_tq == NULL) 1031 return (ENOMEM); 1032 1033 return (0); 1034 } 1035 1036 static int 1037 vtnet_alloc_rxtx_queues(struct vtnet_softc *sc) 1038 { 1039 int i, npairs, error; 1040 1041 npairs = sc->vtnet_max_vq_pairs; 1042 1043 sc->vtnet_rxqs = malloc(sizeof(struct vtnet_rxq) * npairs, M_DEVBUF, 1044 M_NOWAIT | M_ZERO); 1045 sc->vtnet_txqs = malloc(sizeof(struct vtnet_txq) * npairs, M_DEVBUF, 1046 M_NOWAIT | M_ZERO); 1047 if (sc->vtnet_rxqs == NULL || sc->vtnet_txqs == NULL) 1048 return (ENOMEM); 1049 1050 for (i = 0; i < npairs; i++) { 1051 error = vtnet_init_rxq(sc, i); 1052 if (error) 1053 return (error); 1054 error = vtnet_init_txq(sc, i); 1055 if (error) 1056 return (error); 1057 } 1058 1059 vtnet_set_rx_process_limit(sc); 1060 vtnet_setup_queue_sysctl(sc); 1061 1062 return (0); 1063 } 1064 1065 static void 1066 vtnet_destroy_rxq(struct vtnet_rxq *rxq) 1067 { 1068 1069 rxq->vtnrx_sc = NULL; 1070 rxq->vtnrx_id = -1; 1071 1072 #if defined(INET) || defined(INET6) 1073 tcp_lro_free(&rxq->vtnrx_lro); 1074 #endif 1075 1076 if (rxq->vtnrx_sg != NULL) { 1077 sglist_free(rxq->vtnrx_sg); 1078 rxq->vtnrx_sg = NULL; 1079 } 1080 1081 if (mtx_initialized(&rxq->vtnrx_mtx) != 0) 1082 mtx_destroy(&rxq->vtnrx_mtx); 1083 } 1084 1085 static void 1086 vtnet_destroy_txq(struct vtnet_txq *txq) 1087 { 1088 1089 txq->vtntx_sc = NULL; 1090 txq->vtntx_id = -1; 1091 1092 if (txq->vtntx_sg != NULL) { 1093 sglist_free(txq->vtntx_sg); 1094 txq->vtntx_sg = NULL; 1095 } 1096 1097 if (!VTNET_ALTQ_ENABLED) { 1098 if (txq->vtntx_br != NULL) { 1099 buf_ring_free(txq->vtntx_br, M_DEVBUF); 1100 txq->vtntx_br = NULL; 1101 } 1102 } 1103 1104 if (mtx_initialized(&txq->vtntx_mtx) != 0) 1105 mtx_destroy(&txq->vtntx_mtx); 1106 } 1107 1108 static void 1109 vtnet_free_rxtx_queues(struct vtnet_softc *sc) 1110 { 1111 int i; 1112 1113 if (sc->vtnet_rxqs != NULL) { 1114 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 1115 vtnet_destroy_rxq(&sc->vtnet_rxqs[i]); 1116 free(sc->vtnet_rxqs, M_DEVBUF); 1117 sc->vtnet_rxqs = NULL; 1118 } 1119 1120 if (sc->vtnet_txqs != NULL) { 1121 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 1122 vtnet_destroy_txq(&sc->vtnet_txqs[i]); 1123 free(sc->vtnet_txqs, M_DEVBUF); 1124 sc->vtnet_txqs = NULL; 1125 } 1126 } 1127 1128 static int 1129 vtnet_alloc_rx_filters(struct vtnet_softc *sc) 1130 { 1131 1132 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 1133 sc->vtnet_mac_filter = malloc(sizeof(struct vtnet_mac_filter), 1134 M_DEVBUF, M_NOWAIT | M_ZERO); 1135 if (sc->vtnet_mac_filter == NULL) 1136 return (ENOMEM); 1137 } 1138 1139 if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { 1140 sc->vtnet_vlan_filter = malloc(sizeof(uint32_t) * 1141 VTNET_VLAN_FILTER_NWORDS, M_DEVBUF, M_NOWAIT | M_ZERO); 1142 if (sc->vtnet_vlan_filter == NULL) 1143 return (ENOMEM); 1144 } 1145 1146 return (0); 1147 } 1148 1149 static void 1150 vtnet_free_rx_filters(struct vtnet_softc *sc) 1151 { 1152 1153 if (sc->vtnet_mac_filter != NULL) { 1154 free(sc->vtnet_mac_filter, M_DEVBUF); 1155 sc->vtnet_mac_filter = NULL; 1156 } 1157 1158 if (sc->vtnet_vlan_filter != NULL) { 1159 free(sc->vtnet_vlan_filter, M_DEVBUF); 1160 sc->vtnet_vlan_filter = NULL; 1161 } 1162 } 1163 1164 static int 1165 vtnet_alloc_virtqueues(struct vtnet_softc *sc) 1166 { 1167 device_t dev; 1168 struct vq_alloc_info *info; 1169 struct vtnet_rxq *rxq; 1170 struct vtnet_txq *txq; 1171 int i, idx, nvqs, error; 1172 1173 dev = sc->vtnet_dev; 1174 1175 nvqs = sc->vtnet_max_vq_pairs * 2; 1176 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) 1177 nvqs++; 1178 1179 info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT); 1180 if (info == NULL) 1181 return (ENOMEM); 1182 1183 for (i = 0, idx = 0; i < sc->vtnet_req_vq_pairs; i++, idx += 2) { 1184 rxq = &sc->vtnet_rxqs[i]; 1185 VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs, 1186 vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq, 1187 "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id); 1188 1189 txq = &sc->vtnet_txqs[i]; 1190 VQ_ALLOC_INFO_INIT(&info[idx + 1], sc->vtnet_tx_nsegs, 1191 vtnet_tx_vq_intr, txq, &txq->vtntx_vq, 1192 "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id); 1193 } 1194 1195 /* These queues will not be used so allocate the minimum resources. */ 1196 for (; i < sc->vtnet_max_vq_pairs; i++, idx += 2) { 1197 rxq = &sc->vtnet_rxqs[i]; 1198 VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, rxq, &rxq->vtnrx_vq, 1199 "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id); 1200 1201 txq = &sc->vtnet_txqs[i]; 1202 VQ_ALLOC_INFO_INIT(&info[idx + 1], 0, NULL, txq, &txq->vtntx_vq, 1203 "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id); 1204 } 1205 1206 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { 1207 VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, NULL, 1208 &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev)); 1209 } 1210 1211 error = virtio_alloc_virtqueues(dev, nvqs, info); 1212 free(info, M_TEMP); 1213 1214 return (error); 1215 } 1216 1217 static void 1218 vtnet_alloc_interface(struct vtnet_softc *sc) 1219 { 1220 device_t dev; 1221 if_t ifp; 1222 1223 dev = sc->vtnet_dev; 1224 1225 ifp = if_alloc(IFT_ETHER); 1226 sc->vtnet_ifp = ifp; 1227 if_setsoftc(ifp, sc); 1228 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1229 } 1230 1231 static int 1232 vtnet_setup_interface(struct vtnet_softc *sc) 1233 { 1234 device_t dev; 1235 struct pfil_head_args pa; 1236 if_t ifp; 1237 1238 dev = sc->vtnet_dev; 1239 ifp = sc->vtnet_ifp; 1240 1241 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 1242 if_setbaudrate(ifp, IF_Gbps(10)); 1243 if_setinitfn(ifp, vtnet_init); 1244 if_setioctlfn(ifp, vtnet_ioctl); 1245 if_setgetcounterfn(ifp, vtnet_get_counter); 1246 1247 if (!VTNET_ALTQ_ENABLED) { 1248 if_settransmitfn(ifp, vtnet_txq_mq_start); 1249 if_setqflushfn(ifp, vtnet_qflush); 1250 } else { 1251 struct virtqueue *vq = sc->vtnet_txqs[0].vtntx_vq; 1252 if_setstartfn(ifp, vtnet_start); 1253 if_setsendqlen(ifp, virtqueue_size(vq) - 1); 1254 if_setsendqready(ifp); 1255 } 1256 1257 vtnet_get_macaddr(sc); 1258 1259 if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)) 1260 if_setcapabilitiesbit(ifp, IFCAP_LINKSTATE, 0); 1261 1262 ifmedia_init(&sc->vtnet_media, 0, vtnet_ifmedia_upd, vtnet_ifmedia_sts); 1263 ifmedia_add(&sc->vtnet_media, IFM_ETHER | IFM_AUTO, 0, NULL); 1264 ifmedia_set(&sc->vtnet_media, IFM_ETHER | IFM_AUTO); 1265 1266 if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) { 1267 int gso; 1268 1269 if_setcapabilitiesbit(ifp, IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6, 0); 1270 1271 gso = virtio_with_feature(dev, VIRTIO_NET_F_GSO); 1272 if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4)) 1273 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); 1274 if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) 1275 if_setcapabilitiesbit(ifp, IFCAP_TSO6, 0); 1276 if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN)) 1277 sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; 1278 1279 if (if_getcapabilities(ifp) & (IFCAP_TSO4 | IFCAP_TSO6)) { 1280 int tso_maxlen; 1281 1282 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTSO, 0); 1283 1284 tso_maxlen = vtnet_tunable_int(sc, "tso_maxlen", 1285 vtnet_tso_maxlen); 1286 if_sethwtsomax(ifp, tso_maxlen - 1287 (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)); 1288 if_sethwtsomaxsegcount(ifp, sc->vtnet_tx_nsegs - 1); 1289 if_sethwtsomaxsegsize(ifp, PAGE_SIZE); 1290 } 1291 } 1292 1293 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) { 1294 /* BMV: Rx checksums not distinguished between IPv4 and IPv6. */ 1295 if_setcapabilitiesbit(ifp, IFCAP_RXCSUM, 0); 1296 if_setcapabilitiesbit(ifp, IFCAP_RXCSUM_IPV6, 0); 1297 1298 /* Support either "hardware" or software LRO. */ 1299 if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); 1300 } 1301 1302 if (if_getcapabilities(ifp) & (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6)) { 1303 /* 1304 * VirtIO does not support VLAN tagging, but we can fake 1305 * it by inserting and removing the 802.1Q header during 1306 * transmit and receive. We are then able to do checksum 1307 * offloading of VLAN frames. 1308 */ 1309 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM, 0); 1310 } 1311 1312 if (sc->vtnet_max_mtu >= ETHERMTU_JUMBO) 1313 if_setcapabilitiesbit(ifp, IFCAP_JUMBO_MTU, 0); 1314 if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU, 0); 1315 if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0); 1316 1317 /* 1318 * Capabilities after here are not enabled by default. 1319 */ 1320 if_setcapenable(ifp, if_getcapabilities(ifp)); 1321 1322 if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { 1323 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0); 1324 1325 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 1326 vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 1327 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 1328 vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 1329 } 1330 1331 ether_ifattach(ifp, sc->vtnet_hwaddr); 1332 1333 /* Tell the upper layer(s) we support long frames. */ 1334 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 1335 1336 DEBUGNET_SET(ifp, vtnet); 1337 1338 pa.pa_version = PFIL_VERSION; 1339 pa.pa_flags = PFIL_IN; 1340 pa.pa_type = PFIL_TYPE_ETHERNET; 1341 pa.pa_headname = if_name(ifp); 1342 sc->vtnet_pfil = pfil_head_register(&pa); 1343 1344 return (0); 1345 } 1346 1347 static int 1348 vtnet_rx_cluster_size(struct vtnet_softc *sc, int mtu) 1349 { 1350 int framesz; 1351 1352 if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) 1353 return (MJUMPAGESIZE); 1354 else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) 1355 return (MCLBYTES); 1356 1357 /* 1358 * Try to scale the receive mbuf cluster size from the MTU. We 1359 * could also use the VQ size to influence the selected size, 1360 * but that would only matter for very small queues. 1361 */ 1362 if (vtnet_modern(sc)) { 1363 MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr_v1)); 1364 framesz = sizeof(struct virtio_net_hdr_v1); 1365 } else 1366 framesz = sizeof(struct vtnet_rx_header); 1367 framesz += sizeof(struct ether_vlan_header) + mtu; 1368 /* 1369 * Account for the offsetting we'll do elsewhere so we allocate the 1370 * right size for the mtu. 1371 */ 1372 if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0) { 1373 framesz += VTNET_ETHER_ALIGN; 1374 } 1375 1376 if (framesz <= MCLBYTES) 1377 return (MCLBYTES); 1378 else if (framesz <= MJUMPAGESIZE) 1379 return (MJUMPAGESIZE); 1380 else if (framesz <= MJUM9BYTES) 1381 return (MJUM9BYTES); 1382 1383 /* Sane default; avoid 16KB clusters. */ 1384 return (MCLBYTES); 1385 } 1386 1387 static int 1388 vtnet_ioctl_mtu(struct vtnet_softc *sc, u_int mtu) 1389 { 1390 if_t ifp; 1391 int clustersz; 1392 1393 ifp = sc->vtnet_ifp; 1394 VTNET_CORE_LOCK_ASSERT(sc); 1395 1396 if (if_getmtu(ifp) == mtu) 1397 return (0); 1398 else if (mtu < ETHERMIN || mtu > sc->vtnet_max_mtu) 1399 return (EINVAL); 1400 1401 if_setmtu(ifp, mtu); 1402 clustersz = vtnet_rx_cluster_size(sc, mtu); 1403 1404 if (clustersz != sc->vtnet_rx_clustersz && 1405 if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1406 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 1407 vtnet_init_locked(sc, 0); 1408 } 1409 1410 return (0); 1411 } 1412 1413 static int 1414 vtnet_ioctl_ifflags(struct vtnet_softc *sc) 1415 { 1416 if_t ifp; 1417 int drv_running; 1418 1419 ifp = sc->vtnet_ifp; 1420 drv_running = (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0; 1421 1422 VTNET_CORE_LOCK_ASSERT(sc); 1423 1424 if ((if_getflags(ifp) & IFF_UP) == 0) { 1425 if (drv_running) 1426 vtnet_stop(sc); 1427 goto out; 1428 } 1429 1430 if (!drv_running) { 1431 vtnet_init_locked(sc, 0); 1432 goto out; 1433 } 1434 1435 if ((if_getflags(ifp) ^ sc->vtnet_if_flags) & 1436 (IFF_PROMISC | IFF_ALLMULTI)) { 1437 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) 1438 vtnet_rx_filter(sc); 1439 else { 1440 /* 1441 * We don't support filtering out multicast, so 1442 * ALLMULTI is always set. 1443 */ 1444 if_setflagbits(ifp, IFF_ALLMULTI, 0); 1445 if_setflagbits(ifp, IFF_PROMISC, 0); 1446 } 1447 } 1448 1449 out: 1450 sc->vtnet_if_flags = if_getflags(ifp); 1451 return (0); 1452 } 1453 1454 static int 1455 vtnet_ioctl_multi(struct vtnet_softc *sc) 1456 { 1457 if_t ifp; 1458 1459 ifp = sc->vtnet_ifp; 1460 1461 VTNET_CORE_LOCK_ASSERT(sc); 1462 1463 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX && 1464 if_getdrvflags(ifp) & IFF_DRV_RUNNING) 1465 vtnet_rx_filter_mac(sc); 1466 1467 return (0); 1468 } 1469 1470 static int 1471 vtnet_ioctl_ifcap(struct vtnet_softc *sc, struct ifreq *ifr) 1472 { 1473 if_t ifp; 1474 int mask, reinit, update; 1475 1476 ifp = sc->vtnet_ifp; 1477 mask = (ifr->ifr_reqcap & if_getcapabilities(ifp)) ^ if_getcapenable(ifp); 1478 reinit = update = 0; 1479 1480 VTNET_CORE_LOCK_ASSERT(sc); 1481 1482 if (mask & IFCAP_TXCSUM) { 1483 if (if_getcapenable(ifp) & IFCAP_TXCSUM && 1484 if_getcapenable(ifp) & IFCAP_TSO4) { 1485 /* Disable tso4, because txcsum will be disabled. */ 1486 if_setcapenablebit(ifp, 0, IFCAP_TSO4); 1487 if_sethwassistbits(ifp, 0, CSUM_IP_TSO); 1488 mask &= ~IFCAP_TSO4; 1489 } 1490 if_togglecapenable(ifp, IFCAP_TXCSUM); 1491 if_togglehwassist(ifp, VTNET_CSUM_OFFLOAD); 1492 } 1493 if (mask & IFCAP_TXCSUM_IPV6) { 1494 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6 && 1495 if_getcapenable(ifp) & IFCAP_TSO6) { 1496 /* Disable tso6, because txcsum6 will be disabled. */ 1497 if_setcapenablebit(ifp, 0, IFCAP_TSO6); 1498 if_sethwassistbits(ifp, 0, CSUM_IP6_TSO); 1499 mask &= ~IFCAP_TSO6; 1500 } 1501 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); 1502 if_togglehwassist(ifp, VTNET_CSUM_OFFLOAD_IPV6); 1503 } 1504 if (mask & IFCAP_TSO4) { 1505 if (if_getcapenable(ifp) & (IFCAP_TXCSUM | IFCAP_TSO4)) { 1506 /* tso4 can only be enabled, if txcsum is enabled. */ 1507 if_togglecapenable(ifp, IFCAP_TSO4); 1508 if_togglehwassist(ifp, CSUM_IP_TSO); 1509 } 1510 } 1511 if (mask & IFCAP_TSO6) { 1512 if (if_getcapenable(ifp) & (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6)) { 1513 /* tso6 can only be enabled, if txcsum6 is enabled. */ 1514 if_togglecapenable(ifp, IFCAP_TSO6); 1515 if_togglehwassist(ifp, CSUM_IP6_TSO); 1516 } 1517 } 1518 1519 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) { 1520 /* 1521 * These Rx features require the negotiated features to 1522 * be updated. Avoid a full reinit if possible. 1523 */ 1524 if (sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) 1525 update = 1; 1526 else 1527 reinit = 1; 1528 1529 /* BMV: Avoid needless renegotiation for just software LRO. */ 1530 if ((mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) == 1531 IFCAP_LRO && vtnet_software_lro(sc)) 1532 reinit = update = 0; 1533 /* 1534 * VirtIO does not distinguish between receive checksum offload 1535 * for IPv4 and IPv6 packets, so treat them as a pair. 1536 */ 1537 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { 1538 if_togglecapenable(ifp, IFCAP_RXCSUM); 1539 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); 1540 } 1541 if (mask & IFCAP_LRO) 1542 if_togglecapenable(ifp, IFCAP_LRO); 1543 /* Both SW and HW TCP LRO require receive checksum offload. */ 1544 if ((if_getcapenable(ifp) & 1545 (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) == 0) 1546 if_setcapenablebit(ifp, 0, IFCAP_LRO); 1547 } 1548 1549 if (mask & IFCAP_VLAN_HWFILTER) { 1550 /* These Rx features require renegotiation. */ 1551 reinit = 1; 1552 1553 if (mask & IFCAP_VLAN_HWFILTER) 1554 if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER); 1555 } 1556 1557 if (mask & IFCAP_VLAN_HWTSO) 1558 if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); 1559 if (mask & IFCAP_VLAN_HWTAGGING) 1560 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); 1561 1562 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1563 if (reinit) { 1564 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 1565 vtnet_init_locked(sc, 0); 1566 } else if (update) 1567 vtnet_update_rx_offloads(sc); 1568 } 1569 1570 return (0); 1571 } 1572 1573 static int 1574 vtnet_ioctl(if_t ifp, u_long cmd, caddr_t data) 1575 { 1576 struct vtnet_softc *sc; 1577 struct ifreq *ifr; 1578 int error; 1579 1580 sc = if_getsoftc(ifp); 1581 ifr = (struct ifreq *) data; 1582 error = 0; 1583 1584 switch (cmd) { 1585 case SIOCSIFMTU: 1586 VTNET_CORE_LOCK(sc); 1587 error = vtnet_ioctl_mtu(sc, ifr->ifr_mtu); 1588 VTNET_CORE_UNLOCK(sc); 1589 break; 1590 1591 case SIOCSIFFLAGS: 1592 VTNET_CORE_LOCK(sc); 1593 error = vtnet_ioctl_ifflags(sc); 1594 VTNET_CORE_UNLOCK(sc); 1595 break; 1596 1597 case SIOCADDMULTI: 1598 case SIOCDELMULTI: 1599 VTNET_CORE_LOCK(sc); 1600 error = vtnet_ioctl_multi(sc); 1601 VTNET_CORE_UNLOCK(sc); 1602 break; 1603 1604 case SIOCSIFMEDIA: 1605 case SIOCGIFMEDIA: 1606 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd); 1607 break; 1608 1609 case SIOCSIFCAP: 1610 VTNET_CORE_LOCK(sc); 1611 error = vtnet_ioctl_ifcap(sc, ifr); 1612 VTNET_CORE_UNLOCK(sc); 1613 VLAN_CAPABILITIES(ifp); 1614 break; 1615 1616 default: 1617 error = ether_ioctl(ifp, cmd, data); 1618 break; 1619 } 1620 1621 VTNET_CORE_LOCK_ASSERT_NOTOWNED(sc); 1622 1623 return (error); 1624 } 1625 1626 static int 1627 vtnet_rxq_populate(struct vtnet_rxq *rxq) 1628 { 1629 struct virtqueue *vq; 1630 int nbufs, error; 1631 1632 #ifdef DEV_NETMAP 1633 error = vtnet_netmap_rxq_populate(rxq); 1634 if (error >= 0) 1635 return (error); 1636 #endif /* DEV_NETMAP */ 1637 1638 vq = rxq->vtnrx_vq; 1639 error = ENOSPC; 1640 1641 for (nbufs = 0; !virtqueue_full(vq); nbufs++) { 1642 error = vtnet_rxq_new_buf(rxq); 1643 if (error) 1644 break; 1645 } 1646 1647 if (nbufs > 0) { 1648 virtqueue_notify(vq); 1649 /* 1650 * EMSGSIZE signifies the virtqueue did not have enough 1651 * entries available to hold the last mbuf. This is not 1652 * an error. 1653 */ 1654 if (error == EMSGSIZE) 1655 error = 0; 1656 } 1657 1658 return (error); 1659 } 1660 1661 static void 1662 vtnet_rxq_free_mbufs(struct vtnet_rxq *rxq) 1663 { 1664 struct virtqueue *vq; 1665 struct mbuf *m; 1666 int last; 1667 #ifdef DEV_NETMAP 1668 struct netmap_kring *kring = netmap_kring_on(NA(rxq->vtnrx_sc->vtnet_ifp), 1669 rxq->vtnrx_id, NR_RX); 1670 #else /* !DEV_NETMAP */ 1671 void *kring = NULL; 1672 #endif /* !DEV_NETMAP */ 1673 1674 vq = rxq->vtnrx_vq; 1675 last = 0; 1676 1677 while ((m = virtqueue_drain(vq, &last)) != NULL) { 1678 if (kring == NULL) 1679 m_freem(m); 1680 } 1681 1682 KASSERT(virtqueue_empty(vq), 1683 ("%s: mbufs remaining in rx queue %p", __func__, rxq)); 1684 } 1685 1686 static struct mbuf * 1687 vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp) 1688 { 1689 struct mbuf *m_head, *m_tail, *m; 1690 struct vtnet_rx_buffer_header *vthdr; 1691 bus_dma_segment_t segs[1]; 1692 bus_dmamap_t dmap; 1693 int nsegs; 1694 int err; 1695 int i, size; 1696 1697 m_head = NULL; 1698 size = sc->vtnet_rx_clustersz; 1699 1700 KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, 1701 ("%s: mbuf %d chain requested without LRO_NOMRG", __func__, nbufs)); 1702 1703 for (i = 0; i < nbufs; i++) { 1704 m = m_getjcl(M_NOWAIT, MT_DATA, i == 0 ? M_PKTHDR : 0, size); 1705 if (m == NULL) { 1706 sc->vtnet_stats.mbuf_alloc_failed++; 1707 m_freem(m_head); 1708 return (NULL); 1709 } 1710 1711 m->m_len = size; 1712 vthdr = (struct vtnet_rx_buffer_header *)m->m_data; 1713 1714 /* Reserve space for header */ 1715 m_adj(m, VTNET_RX_BUFFER_HEADER_OFFSET); 1716 1717 /* 1718 * Need to offset the mbuf if the header we're going to add 1719 * will misalign. 1720 */ 1721 if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0) 1722 m_adj(m, VTNET_ETHER_ALIGN); 1723 1724 err = bus_dmamap_create(sc->vtnet_rx_dmat, 0, &dmap); 1725 if (err) { 1726 printf("Failed to create dmamap, err :%d\n", 1727 err); 1728 m_freem(m); 1729 return (NULL); 1730 } 1731 1732 nsegs = 0; 1733 err = bus_dmamap_load_mbuf_sg(sc->vtnet_rx_dmat, dmap, m, segs, 1734 &nsegs, BUS_DMA_NOWAIT); 1735 if (err != 0) { 1736 printf("Failed to map mbuf into DMA visible memory, err: %d\n", 1737 err); 1738 m_freem(m); 1739 bus_dmamap_destroy(sc->vtnet_rx_dmat, dmap); 1740 return (NULL); 1741 } 1742 KASSERT(nsegs == 1, 1743 ("%s: unexpected number of DMA segments for rx buffer: %d", 1744 __func__, nsegs)); 1745 1746 vthdr->addr = segs[0].ds_addr; 1747 vthdr->dmap = dmap; 1748 1749 if (m_head != NULL) { 1750 m_tail->m_next = m; 1751 m_tail = m; 1752 } else 1753 m_head = m_tail = m; 1754 } 1755 1756 if (m_tailp != NULL) 1757 *m_tailp = m_tail; 1758 1759 return (m_head); 1760 } 1761 1762 /* 1763 * Slow path for when LRO without mergeable buffers is negotiated. 1764 */ 1765 static int 1766 vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *rxq, struct mbuf *m0, 1767 int len0) 1768 { 1769 struct vtnet_softc *sc; 1770 struct mbuf *m, *m_prev, *m_new, *m_tail; 1771 int len, clustersz, nreplace, error; 1772 1773 sc = rxq->vtnrx_sc; 1774 clustersz = sc->vtnet_rx_clustersz - VTNET_RX_BUFFER_HEADER_OFFSET; 1775 /* 1776 * Need to offset the mbuf if the header we're going to add will 1777 * misalign, account for that here. 1778 */ 1779 if (VTNET_ETHER_ALIGN != 0 && sc->vtnet_hdr_size % 4 == 0) 1780 clustersz -= VTNET_ETHER_ALIGN; 1781 1782 m_prev = NULL; 1783 m_tail = NULL; 1784 nreplace = 0; 1785 1786 m = m0; 1787 len = len0; 1788 1789 /* 1790 * Since these mbuf chains are so large, avoid allocating a complete 1791 * replacement when the received frame did not consume the entire 1792 * chain. Unused mbufs are moved to the tail of the replacement mbuf. 1793 */ 1794 while (len > 0) { 1795 if (m == NULL) { 1796 sc->vtnet_stats.rx_frame_too_large++; 1797 return (EMSGSIZE); 1798 } 1799 1800 /* 1801 * Every mbuf should have the expected cluster size since that 1802 * is also used to allocate the replacements. 1803 */ 1804 KASSERT(m->m_len == clustersz, 1805 ("%s: mbuf size %d not expected cluster size %d", __func__, 1806 m->m_len, clustersz)); 1807 1808 m->m_len = MIN(m->m_len, len); 1809 len -= m->m_len; 1810 1811 m_prev = m; 1812 m = m->m_next; 1813 nreplace++; 1814 } 1815 1816 KASSERT(nreplace > 0 && nreplace <= sc->vtnet_rx_nmbufs, 1817 ("%s: invalid replacement mbuf count %d max %d", __func__, 1818 nreplace, sc->vtnet_rx_nmbufs)); 1819 1820 m_new = vtnet_rx_alloc_buf(sc, nreplace, &m_tail); 1821 if (m_new == NULL) { 1822 m_prev->m_len = clustersz; 1823 return (ENOBUFS); 1824 } 1825 1826 /* 1827 * Move any unused mbufs from the received mbuf chain onto the 1828 * end of the replacement chain. 1829 */ 1830 if (m_prev->m_next != NULL) { 1831 m_tail->m_next = m_prev->m_next; 1832 m_prev->m_next = NULL; 1833 } 1834 1835 error = vtnet_rxq_enqueue_buf(rxq, m_new); 1836 if (error) { 1837 /* 1838 * The replacement is suppose to be an copy of the one 1839 * dequeued so this is a very unexpected error. 1840 * 1841 * Restore the m0 chain to the original state if it was 1842 * modified so we can then discard it. 1843 */ 1844 if (m_tail->m_next != NULL) { 1845 m_prev->m_next = m_tail->m_next; 1846 m_tail->m_next = NULL; 1847 } 1848 m_prev->m_len = clustersz; 1849 sc->vtnet_stats.rx_enq_replacement_failed++; 1850 m_freem(m_new); 1851 } 1852 1853 return (error); 1854 } 1855 1856 static int 1857 vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len) 1858 { 1859 struct vtnet_softc *sc; 1860 struct mbuf *m_new; 1861 int error; 1862 1863 sc = rxq->vtnrx_sc; 1864 1865 if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) 1866 return (vtnet_rxq_replace_lro_nomrg_buf(rxq, m, len)); 1867 1868 MPASS(m->m_next == NULL); 1869 if (m->m_len < len) 1870 return (EMSGSIZE); 1871 1872 m_new = vtnet_rx_alloc_buf(sc, 1, NULL); 1873 if (m_new == NULL) 1874 return (ENOBUFS); 1875 1876 error = vtnet_rxq_enqueue_buf(rxq, m_new); 1877 if (error) { 1878 sc->vtnet_stats.rx_enq_replacement_failed++; 1879 m_freem(m_new); 1880 } else 1881 m->m_len = len; 1882 1883 return (error); 1884 } 1885 1886 static int 1887 vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m) 1888 { 1889 struct vtnet_rx_buffer_header *hdr; 1890 struct vtnet_softc *sc; 1891 struct sglist *sg; 1892 int header_inlined, error; 1893 bus_addr_t paddr; 1894 struct mbuf *mp; 1895 1896 sc = rxq->vtnrx_sc; 1897 sg = rxq->vtnrx_sg; 1898 1899 KASSERT(m->m_next == NULL || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, 1900 ("%s: mbuf chain without LRO_NOMRG", __func__)); 1901 VTNET_RXQ_LOCK_ASSERT(rxq); 1902 1903 sglist_reset(sg); 1904 header_inlined = vtnet_modern(sc) || 1905 (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) != 0; /* TODO: ANY_LAYOUT */ 1906 1907 hdr = vtnet_mbuf_to_rx_buffer_header(sc, m); 1908 paddr = hdr->addr; 1909 1910 /* 1911 * Note: The mbuf has been already adjusted when we allocate it if we 1912 * have to do strict alignment. 1913 */ 1914 if (header_inlined) { 1915 error = sglist_append_phys(sg, paddr, m->m_len); 1916 } else { 1917 MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr)); 1918 1919 /* Append the header and remaining mbuf data. */ 1920 error = sglist_append_phys(sg, paddr, sc->vtnet_hdr_size); 1921 if (error) 1922 return (error); 1923 error = sglist_append_phys(sg, 1924 paddr + sizeof(struct vtnet_rx_header), 1925 m->m_len - sizeof(struct vtnet_rx_header)); 1926 if (error) 1927 return (error); 1928 1929 mp = m->m_next; 1930 while (mp) { 1931 hdr = vtnet_mbuf_to_rx_buffer_header(sc, mp); 1932 paddr = hdr->addr; 1933 error = sglist_append_phys(sg, paddr, mp->m_len); 1934 if (error) 1935 return (error); 1936 1937 mp = mp->m_next; 1938 } 1939 } 1940 1941 if (error) 1942 return (error); 1943 1944 return (virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg)); 1945 } 1946 1947 static int 1948 vtnet_rxq_new_buf(struct vtnet_rxq *rxq) 1949 { 1950 struct vtnet_softc *sc; 1951 struct mbuf *m; 1952 int error; 1953 1954 sc = rxq->vtnrx_sc; 1955 1956 m = vtnet_rx_alloc_buf(sc, sc->vtnet_rx_nmbufs, NULL); 1957 if (m == NULL) 1958 return (ENOBUFS); 1959 1960 error = vtnet_rxq_enqueue_buf(rxq, m); 1961 if (error) 1962 m_freem(m); 1963 1964 return (error); 1965 } 1966 1967 #if defined(INET) || defined(INET6) 1968 static void 1969 vtnet_rxq_csum_needs_csum(struct vtnet_rxq *rxq, struct mbuf *m, bool isipv6, 1970 int protocol, struct virtio_net_hdr *hdr) 1971 { 1972 /* 1973 * The packet is likely from another VM on the same host or from the 1974 * host that itself performed checksum offloading so Tx/Rx is basically 1975 * a memcpy and the checksum has little value so far. 1976 */ 1977 1978 KASSERT(protocol == IPPROTO_TCP || protocol == IPPROTO_UDP, 1979 ("%s: unsupported IP protocol %d", __func__, protocol)); 1980 1981 /* 1982 * Just forward the order to compute the checksum by setting 1983 * the corresponding mbuf flag (e.g., CSUM_TCP). 1984 */ 1985 switch (protocol) { 1986 case IPPROTO_TCP: 1987 m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_TCP_IPV6 : CSUM_TCP); 1988 break; 1989 case IPPROTO_UDP: 1990 m->m_pkthdr.csum_flags |= (isipv6 ? CSUM_UDP_IPV6 : CSUM_UDP); 1991 break; 1992 } 1993 m->m_pkthdr.csum_data = hdr->csum_offset; 1994 } 1995 1996 static void 1997 vtnet_rxq_csum_data_valid(struct vtnet_rxq *rxq, struct mbuf *m, int protocol) 1998 { 1999 KASSERT(protocol == IPPROTO_TCP || protocol == IPPROTO_UDP, 2000 ("%s: unsupported IP protocol %d", __func__, protocol)); 2001 2002 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 2003 m->m_pkthdr.csum_data = 0xFFFF; 2004 } 2005 2006 static int 2007 vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m, 2008 struct virtio_net_hdr *hdr) 2009 { 2010 const struct ether_header *eh; 2011 struct vtnet_softc *sc; 2012 int hoff, protocol; 2013 uint16_t etype; 2014 bool isipv6; 2015 2016 KASSERT(hdr->flags & 2017 (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID), 2018 ("%s: missing checksum offloading flag %x", __func__, hdr->flags)); 2019 2020 eh = mtod(m, const struct ether_header *); 2021 etype = ntohs(eh->ether_type); 2022 if (etype == ETHERTYPE_VLAN) { 2023 /* TODO BMV: Handle QinQ. */ 2024 const struct ether_vlan_header *evh = 2025 mtod(m, const struct ether_vlan_header *); 2026 etype = ntohs(evh->evl_proto); 2027 hoff = sizeof(struct ether_vlan_header); 2028 } else 2029 hoff = sizeof(struct ether_header); 2030 2031 sc = rxq->vtnrx_sc; 2032 2033 /* Check whether ethernet type is IP or IPv6, and get protocol. */ 2034 switch (etype) { 2035 #if defined(INET) 2036 case ETHERTYPE_IP: 2037 if (__predict_false(m->m_len < hoff + sizeof(struct ip))) { 2038 sc->vtnet_stats.rx_csum_inaccessible_ipproto++; 2039 return (1); 2040 } else { 2041 struct ip *ip = (struct ip *)(m->m_data + hoff); 2042 protocol = ip->ip_p; 2043 } 2044 isipv6 = false; 2045 break; 2046 #endif 2047 #if defined(INET6) 2048 case ETHERTYPE_IPV6: 2049 if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr)) 2050 || ip6_lasthdr(m, hoff, IPPROTO_IPV6, &protocol) < 0) { 2051 sc->vtnet_stats.rx_csum_inaccessible_ipproto++; 2052 return (1); 2053 } 2054 isipv6 = true; 2055 break; 2056 #endif 2057 default: 2058 sc->vtnet_stats.rx_csum_bad_ethtype++; 2059 return (1); 2060 } 2061 2062 /* Check whether protocol is TCP or UDP. */ 2063 switch (protocol) { 2064 case IPPROTO_TCP: 2065 case IPPROTO_UDP: 2066 break; 2067 default: 2068 /* 2069 * FreeBSD does not support checksum offloading of this 2070 * protocol here. 2071 */ 2072 sc->vtnet_stats.rx_csum_bad_ipproto++; 2073 return (1); 2074 } 2075 2076 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 2077 vtnet_rxq_csum_needs_csum(rxq, m, isipv6, protocol, hdr); 2078 else /* VIRTIO_NET_HDR_F_DATA_VALID */ 2079 vtnet_rxq_csum_data_valid(rxq, m, protocol); 2080 2081 return (0); 2082 } 2083 #endif 2084 2085 static void 2086 vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs) 2087 { 2088 struct mbuf *m; 2089 2090 while (--nbufs > 0) { 2091 m = virtqueue_dequeue(rxq->vtnrx_vq, NULL); 2092 if (m == NULL) 2093 break; 2094 vtnet_rxq_discard_buf(rxq, m); 2095 } 2096 } 2097 2098 static void 2099 vtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m) 2100 { 2101 int error __diagused; 2102 2103 /* 2104 * Requeue the discarded mbuf. This should always be successful 2105 * since it was just dequeued. 2106 */ 2107 error = vtnet_rxq_enqueue_buf(rxq, m); 2108 KASSERT(error == 0, 2109 ("%s: cannot requeue discarded mbuf %d", __func__, error)); 2110 } 2111 2112 static int 2113 vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs) 2114 { 2115 struct vtnet_softc *sc; 2116 struct virtqueue *vq; 2117 struct mbuf *m_tail; 2118 2119 sc = rxq->vtnrx_sc; 2120 vq = rxq->vtnrx_vq; 2121 m_tail = m_head; 2122 2123 while (--nbufs > 0) { 2124 struct vtnet_rx_buffer_header *vthdr; 2125 struct mbuf *m; 2126 uint32_t len; 2127 2128 m = virtqueue_dequeue(vq, &len); 2129 if (m == NULL) { 2130 rxq->vtnrx_stats.vrxs_ierrors++; 2131 goto fail; 2132 } 2133 2134 vthdr = vtnet_mbuf_to_rx_buffer_header(sc, m); 2135 bus_dmamap_sync(sc->vtnet_rx_dmat, vthdr->dmap, 2136 BUS_DMASYNC_POSTREAD); 2137 2138 if (vtnet_rxq_new_buf(rxq) != 0) { 2139 rxq->vtnrx_stats.vrxs_iqdrops++; 2140 vtnet_rxq_discard_buf(rxq, m); 2141 if (nbufs > 1) 2142 vtnet_rxq_discard_merged_bufs(rxq, nbufs); 2143 goto fail; 2144 } 2145 2146 bus_dmamap_unload(sc->vtnet_rx_dmat, vthdr->dmap); 2147 bus_dmamap_destroy(sc->vtnet_rx_dmat, vthdr->dmap); 2148 2149 if (m->m_len < len) 2150 len = m->m_len; 2151 2152 m->m_len = len; 2153 m->m_flags &= ~M_PKTHDR; 2154 2155 m_head->m_pkthdr.len += len; 2156 m_tail->m_next = m; 2157 m_tail = m; 2158 } 2159 2160 return (0); 2161 2162 fail: 2163 sc->vtnet_stats.rx_mergeable_failed++; 2164 m_freem(m_head); 2165 2166 return (1); 2167 } 2168 2169 #if defined(INET) || defined(INET6) 2170 static int 2171 vtnet_lro_rx(struct vtnet_rxq *rxq, struct mbuf *m) 2172 { 2173 struct lro_ctrl *lro; 2174 2175 lro = &rxq->vtnrx_lro; 2176 2177 if (lro->lro_mbuf_max != 0) { 2178 tcp_lro_queue_mbuf(lro, m); 2179 return (0); 2180 } 2181 2182 return (tcp_lro_rx(lro, m, 0)); 2183 } 2184 #endif 2185 2186 static void 2187 vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, 2188 struct virtio_net_hdr *hdr) 2189 { 2190 struct vtnet_softc *sc; 2191 if_t ifp; 2192 2193 sc = rxq->vtnrx_sc; 2194 ifp = sc->vtnet_ifp; 2195 2196 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { 2197 struct ether_header *eh = mtod(m, struct ether_header *); 2198 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 2199 vtnet_vlan_tag_remove(m); 2200 /* 2201 * With the 802.1Q header removed, update the 2202 * checksum starting location accordingly. 2203 */ 2204 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 2205 hdr->csum_start -= ETHER_VLAN_ENCAP_LEN; 2206 } 2207 } 2208 2209 if (sc->vtnet_act_vq_pairs == 1) { 2210 /* 2211 * When RSS is not needed (one active rx queue), let the upper 2212 * layer know and react. 2213 */ 2214 M_HASHTYPE_CLEAR(m); 2215 } else { 2216 m->m_pkthdr.flowid = rxq->vtnrx_id; 2217 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 2218 } 2219 2220 if (hdr->flags & 2221 (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID)) { 2222 #if defined(INET) || defined(INET6) 2223 if (vtnet_rxq_csum(rxq, m, hdr) == 0) 2224 rxq->vtnrx_stats.vrxs_csum++; 2225 else 2226 rxq->vtnrx_stats.vrxs_csum_failed++; 2227 #else 2228 sc->vtnet_stats.rx_csum_bad_ethtype++; 2229 rxq->vtnrx_stats.vrxs_csum_failed++; 2230 #endif 2231 } 2232 2233 if (hdr->gso_size != 0) { 2234 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 2235 case VIRTIO_NET_HDR_GSO_TCPV4: 2236 case VIRTIO_NET_HDR_GSO_TCPV6: 2237 m->m_pkthdr.lro_nsegs = 2238 howmany(m->m_pkthdr.len, hdr->gso_size); 2239 rxq->vtnrx_stats.vrxs_host_lro++; 2240 break; 2241 } 2242 } 2243 2244 rxq->vtnrx_stats.vrxs_ipackets++; 2245 rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len; 2246 2247 #if defined(INET) || defined(INET6) 2248 if (vtnet_software_lro(sc) && if_getcapenable(ifp) & IFCAP_LRO) { 2249 if (vtnet_lro_rx(rxq, m) == 0) 2250 return; 2251 } 2252 #endif 2253 2254 if_input(ifp, m); 2255 } 2256 2257 static int 2258 vtnet_rxq_eof(struct vtnet_rxq *rxq) 2259 { 2260 struct virtio_net_hdr lhdr, *hdr; 2261 struct vtnet_rx_buffer_header *vthdr; 2262 struct vtnet_softc *sc; 2263 if_t ifp; 2264 struct virtqueue *vq; 2265 int deq, count; 2266 2267 sc = rxq->vtnrx_sc; 2268 vq = rxq->vtnrx_vq; 2269 ifp = sc->vtnet_ifp; 2270 deq = 0; 2271 count = sc->vtnet_rx_process_limit; 2272 2273 VTNET_RXQ_LOCK_ASSERT(rxq); 2274 2275 CURVNET_SET(if_getvnet(ifp)); 2276 while (count-- > 0) { 2277 struct mbuf *m, *mp; 2278 uint32_t len, nbufs, adjsz; 2279 uint32_t synced; 2280 2281 m = virtqueue_dequeue(vq, &len); 2282 if (m == NULL) 2283 break; 2284 deq++; 2285 2286 mp = m; 2287 2288 /* 2289 * Sync all mbufs in this packet. There will only be a single 2290 * mbuf unless LRO is in use. 2291 */ 2292 synced = 0; 2293 while (mp && synced < len) { 2294 vthdr = vtnet_mbuf_to_rx_buffer_header(sc, mp); 2295 bus_dmamap_sync(sc->vtnet_rx_dmat, vthdr->dmap, 2296 BUS_DMASYNC_POSTREAD); 2297 2298 synced += mp->m_len; 2299 mp = mp->m_next; 2300 } 2301 2302 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) { 2303 rxq->vtnrx_stats.vrxs_ierrors++; 2304 vtnet_rxq_discard_buf(rxq, m); 2305 continue; 2306 } 2307 2308 if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) { 2309 struct virtio_net_hdr_mrg_rxbuf *mhdr = 2310 mtod(m, struct virtio_net_hdr_mrg_rxbuf *); 2311 kmsan_mark(mhdr, sizeof(*mhdr), KMSAN_STATE_INITED); 2312 nbufs = vtnet_htog16(sc, mhdr->num_buffers); 2313 adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf); 2314 } else if (vtnet_modern(sc)) { 2315 nbufs = 1; /* num_buffers is always 1 */ 2316 adjsz = sizeof(struct virtio_net_hdr_v1); 2317 } else { 2318 nbufs = 1; 2319 adjsz = sizeof(struct vtnet_rx_header); 2320 /* 2321 * Account for our gap between the header and start of 2322 * data to keep the segments separated. 2323 */ 2324 len += VTNET_RX_HEADER_PAD; 2325 } 2326 2327 if (vtnet_rxq_replace_buf(rxq, m, len) != 0) { 2328 rxq->vtnrx_stats.vrxs_iqdrops++; 2329 vtnet_rxq_discard_buf(rxq, m); 2330 if (nbufs > 1) 2331 vtnet_rxq_discard_merged_bufs(rxq, nbufs); 2332 continue; 2333 } 2334 2335 mp = m; 2336 synced = 0; 2337 while (mp && synced < len) { 2338 vthdr = vtnet_mbuf_to_rx_buffer_header(sc, mp); 2339 2340 bus_dmamap_unload(sc->vtnet_rx_dmat, vthdr->dmap); 2341 bus_dmamap_destroy(sc->vtnet_rx_dmat, vthdr->dmap); 2342 2343 synced += mp->m_len; 2344 mp = mp->m_next; 2345 } 2346 2347 m->m_pkthdr.len = len; 2348 m->m_pkthdr.rcvif = ifp; 2349 m->m_pkthdr.csum_flags = 0; 2350 2351 if (nbufs > 1) { 2352 /* Dequeue the rest of chain. */ 2353 if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0) 2354 continue; 2355 } 2356 2357 kmsan_mark_mbuf(m, KMSAN_STATE_INITED); 2358 2359 /* 2360 * Save an endian swapped version of the header prior to it 2361 * being stripped. The header is always at the start of the 2362 * mbuf data. num_buffers was already saved (and not needed) 2363 * so use the standard header. 2364 */ 2365 hdr = mtod(m, struct virtio_net_hdr *); 2366 lhdr.flags = hdr->flags; 2367 lhdr.gso_type = hdr->gso_type; 2368 lhdr.hdr_len = vtnet_htog16(sc, hdr->hdr_len); 2369 lhdr.gso_size = vtnet_htog16(sc, hdr->gso_size); 2370 lhdr.csum_start = vtnet_htog16(sc, hdr->csum_start); 2371 lhdr.csum_offset = vtnet_htog16(sc, hdr->csum_offset); 2372 m_adj(m, adjsz); 2373 2374 if (PFIL_HOOKED_IN(sc->vtnet_pfil)) { 2375 pfil_return_t pfil; 2376 2377 pfil = pfil_mbuf_in(sc->vtnet_pfil, &m, ifp, NULL); 2378 switch (pfil) { 2379 case PFIL_DROPPED: 2380 case PFIL_CONSUMED: 2381 continue; 2382 default: 2383 KASSERT(pfil == PFIL_PASS, 2384 ("Filter returned %d!", pfil)); 2385 } 2386 } 2387 2388 vtnet_rxq_input(rxq, m, &lhdr); 2389 } 2390 2391 if (deq > 0) { 2392 #if defined(INET) || defined(INET6) 2393 if (vtnet_software_lro(sc)) 2394 tcp_lro_flush_all(&rxq->vtnrx_lro); 2395 #endif 2396 virtqueue_notify(vq); 2397 } 2398 CURVNET_RESTORE(); 2399 2400 return (count > 0 ? 0 : EAGAIN); 2401 } 2402 2403 static void 2404 vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries) 2405 { 2406 struct vtnet_softc *sc; 2407 if_t ifp; 2408 u_int more; 2409 #ifdef DEV_NETMAP 2410 int nmirq; 2411 #endif /* DEV_NETMAP */ 2412 2413 sc = rxq->vtnrx_sc; 2414 ifp = sc->vtnet_ifp; 2415 2416 if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) { 2417 /* 2418 * Ignore this interrupt. Either this is a spurious interrupt 2419 * or multiqueue without per-VQ MSIX so every queue needs to 2420 * be polled (a brain dead configuration we could try harder 2421 * to avoid). 2422 */ 2423 vtnet_rxq_disable_intr(rxq); 2424 return; 2425 } 2426 2427 VTNET_RXQ_LOCK(rxq); 2428 2429 #ifdef DEV_NETMAP 2430 /* 2431 * We call netmap_rx_irq() under lock to prevent concurrent calls. 2432 * This is not necessary to serialize the access to the RX vq, but 2433 * rather to avoid races that may happen if this interface is 2434 * attached to a VALE switch, which would cause received packets 2435 * to stall in the RX queue (nm_kr_tryget() could find the kring 2436 * busy when called from netmap_bwrap_intr_notify()). 2437 */ 2438 nmirq = netmap_rx_irq(ifp, rxq->vtnrx_id, &more); 2439 if (nmirq != NM_IRQ_PASS) { 2440 VTNET_RXQ_UNLOCK(rxq); 2441 if (nmirq == NM_IRQ_RESCHED) { 2442 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 2443 } 2444 return; 2445 } 2446 #endif /* DEV_NETMAP */ 2447 2448 again: 2449 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 2450 VTNET_RXQ_UNLOCK(rxq); 2451 return; 2452 } 2453 2454 more = vtnet_rxq_eof(rxq); 2455 if (more || vtnet_rxq_enable_intr(rxq) != 0) { 2456 if (!more) 2457 vtnet_rxq_disable_intr(rxq); 2458 /* 2459 * This is an occasional condition or race (when !more), 2460 * so retry a few times before scheduling the taskqueue. 2461 */ 2462 if (tries-- > 0) 2463 goto again; 2464 2465 rxq->vtnrx_stats.vrxs_rescheduled++; 2466 VTNET_RXQ_UNLOCK(rxq); 2467 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 2468 } else 2469 VTNET_RXQ_UNLOCK(rxq); 2470 } 2471 2472 static void 2473 vtnet_rx_vq_intr(void *xrxq) 2474 { 2475 struct vtnet_rxq *rxq; 2476 2477 rxq = xrxq; 2478 vtnet_rx_vq_process(rxq, VTNET_INTR_DISABLE_RETRIES); 2479 } 2480 2481 static void 2482 vtnet_rxq_tq_intr(void *xrxq, int pending __unused) 2483 { 2484 struct vtnet_rxq *rxq; 2485 2486 rxq = xrxq; 2487 vtnet_rx_vq_process(rxq, 0); 2488 } 2489 2490 static int 2491 vtnet_txq_intr_threshold(struct vtnet_txq *txq) 2492 { 2493 struct vtnet_softc *sc; 2494 int threshold; 2495 2496 sc = txq->vtntx_sc; 2497 2498 /* 2499 * The Tx interrupt is disabled until the queue free count falls 2500 * below our threshold. Completed frames are drained from the Tx 2501 * virtqueue before transmitting new frames and in the watchdog 2502 * callout, so the frequency of Tx interrupts is greatly reduced, 2503 * at the cost of not freeing mbufs as quickly as they otherwise 2504 * would be. 2505 */ 2506 threshold = virtqueue_size(txq->vtntx_vq) / 4; 2507 2508 /* 2509 * Without indirect descriptors, leave enough room for the most 2510 * segments we handle. 2511 */ 2512 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) == 0 && 2513 threshold < sc->vtnet_tx_nsegs) 2514 threshold = sc->vtnet_tx_nsegs; 2515 2516 return (threshold); 2517 } 2518 2519 static int 2520 vtnet_txq_below_threshold(struct vtnet_txq *txq) 2521 { 2522 struct virtqueue *vq; 2523 2524 vq = txq->vtntx_vq; 2525 2526 return (virtqueue_nfree(vq) <= txq->vtntx_intr_threshold); 2527 } 2528 2529 static int 2530 vtnet_txq_notify(struct vtnet_txq *txq) 2531 { 2532 struct virtqueue *vq; 2533 2534 vq = txq->vtntx_vq; 2535 2536 txq->vtntx_watchdog = VTNET_TX_TIMEOUT; 2537 virtqueue_notify(vq); 2538 2539 if (vtnet_txq_enable_intr(txq) == 0) 2540 return (0); 2541 2542 /* 2543 * Drain frames that were completed since last checked. If this 2544 * causes the queue to go above the threshold, the caller should 2545 * continue transmitting. 2546 */ 2547 if (vtnet_txq_eof(txq) != 0 && vtnet_txq_below_threshold(txq) == 0) { 2548 virtqueue_disable_intr(vq); 2549 return (1); 2550 } 2551 2552 return (0); 2553 } 2554 2555 static void 2556 vtnet_txq_free_mbufs(struct vtnet_txq *txq) 2557 { 2558 struct virtqueue *vq; 2559 struct vtnet_tx_header *txhdr; 2560 int last; 2561 #ifdef DEV_NETMAP 2562 struct netmap_kring *kring = netmap_kring_on(NA(txq->vtntx_sc->vtnet_ifp), 2563 txq->vtntx_id, NR_TX); 2564 #else /* !DEV_NETMAP */ 2565 void *kring = NULL; 2566 #endif /* !DEV_NETMAP */ 2567 2568 vq = txq->vtntx_vq; 2569 last = 0; 2570 2571 while ((txhdr = virtqueue_drain(vq, &last)) != NULL) { 2572 if (kring == NULL) { 2573 bus_dmamap_unload(txq->vtntx_sc->vtnet_tx_dmat, 2574 txhdr->dmap); 2575 bus_dmamap_destroy(txq->vtntx_sc->vtnet_tx_dmat, 2576 txhdr->dmap); 2577 bus_dmamap_unload(txq->vtntx_sc->vtnet_tx_dmat, 2578 txhdr->hdr_dmap); 2579 bus_dmamap_destroy(txq->vtntx_sc->vtnet_tx_dmat, 2580 txhdr->hdr_dmap); 2581 m_freem(txhdr->vth_mbuf); 2582 uma_zfree(vtnet_tx_header_zone, txhdr); 2583 } 2584 } 2585 2586 KASSERT(virtqueue_empty(vq), 2587 ("%s: mbufs remaining in tx queue %p", __func__, txq)); 2588 } 2589 2590 /* 2591 * BMV: This can go away once we finally have offsets in the mbuf header. 2592 */ 2593 static int 2594 vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m, int *etype, 2595 int *proto, int *start) 2596 { 2597 struct vtnet_softc *sc; 2598 struct ether_vlan_header *evh; 2599 #if defined(INET) || defined(INET6) 2600 int offset; 2601 #endif 2602 2603 sc = txq->vtntx_sc; 2604 2605 evh = mtod(m, struct ether_vlan_header *); 2606 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2607 /* BMV: We should handle nested VLAN tags too. */ 2608 *etype = ntohs(evh->evl_proto); 2609 #if defined(INET) || defined(INET6) 2610 offset = sizeof(struct ether_vlan_header); 2611 #endif 2612 } else { 2613 *etype = ntohs(evh->evl_encap_proto); 2614 #if defined(INET) || defined(INET6) 2615 offset = sizeof(struct ether_header); 2616 #endif 2617 } 2618 2619 switch (*etype) { 2620 #if defined(INET) 2621 case ETHERTYPE_IP: { 2622 struct ip *ip, iphdr; 2623 if (__predict_false(m->m_len < offset + sizeof(struct ip))) { 2624 m_copydata(m, offset, sizeof(struct ip), 2625 (caddr_t) &iphdr); 2626 ip = &iphdr; 2627 } else 2628 ip = (struct ip *)(m->m_data + offset); 2629 *proto = ip->ip_p; 2630 *start = offset + (ip->ip_hl << 2); 2631 break; 2632 } 2633 #endif 2634 #if defined(INET6) 2635 case ETHERTYPE_IPV6: 2636 *proto = -1; 2637 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); 2638 /* Assert the network stack sent us a valid packet. */ 2639 KASSERT(*start > offset, 2640 ("%s: mbuf %p start %d offset %d proto %d", __func__, m, 2641 *start, offset, *proto)); 2642 break; 2643 #endif 2644 default: 2645 sc->vtnet_stats.tx_csum_unknown_ethtype++; 2646 return (EINVAL); 2647 } 2648 2649 return (0); 2650 } 2651 2652 static int 2653 vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type, 2654 int offset, struct virtio_net_hdr *hdr) 2655 { 2656 static struct timeval lastecn; 2657 static int curecn; 2658 struct vtnet_softc *sc; 2659 struct tcphdr *tcp, tcphdr; 2660 2661 sc = txq->vtntx_sc; 2662 2663 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) { 2664 m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr); 2665 tcp = &tcphdr; 2666 } else 2667 tcp = (struct tcphdr *)(m->m_data + offset); 2668 2669 hdr->hdr_len = vtnet_gtoh16(sc, offset + (tcp->th_off << 2)); 2670 hdr->gso_size = vtnet_gtoh16(sc, m->m_pkthdr.tso_segsz); 2671 hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 : 2672 VIRTIO_NET_HDR_GSO_TCPV6; 2673 2674 if (__predict_false(tcp_get_flags(tcp) & TH_CWR)) { 2675 /* 2676 * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In 2677 * FreeBSD, ECN support is not on a per-interface basis, 2678 * but globally via the net.inet.tcp.ecn.enable sysctl 2679 * knob. The default is off. 2680 */ 2681 if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) { 2682 if (ppsratecheck(&lastecn, &curecn, 1)) 2683 if_printf(sc->vtnet_ifp, 2684 "TSO with ECN not negotiated with host\n"); 2685 return (ENOTSUP); 2686 } 2687 hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; 2688 } 2689 2690 txq->vtntx_stats.vtxs_tso++; 2691 2692 return (0); 2693 } 2694 2695 static struct mbuf * 2696 vtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m, 2697 struct virtio_net_hdr *hdr) 2698 { 2699 struct vtnet_softc *sc; 2700 int flags, etype, csum_start, proto, error; 2701 2702 sc = txq->vtntx_sc; 2703 flags = m->m_pkthdr.csum_flags; 2704 2705 error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start); 2706 if (error) 2707 goto drop; 2708 2709 if (flags & (VTNET_CSUM_OFFLOAD | VTNET_CSUM_OFFLOAD_IPV6)) { 2710 /* Sanity check the parsed mbuf matches the offload flags. */ 2711 if (__predict_false((flags & VTNET_CSUM_OFFLOAD && 2712 etype != ETHERTYPE_IP) || (flags & VTNET_CSUM_OFFLOAD_IPV6 2713 && etype != ETHERTYPE_IPV6))) { 2714 sc->vtnet_stats.tx_csum_proto_mismatch++; 2715 goto drop; 2716 } 2717 2718 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM; 2719 hdr->csum_start = vtnet_gtoh16(sc, csum_start); 2720 hdr->csum_offset = vtnet_gtoh16(sc, m->m_pkthdr.csum_data); 2721 txq->vtntx_stats.vtxs_csum++; 2722 } 2723 2724 if (flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) { 2725 /* 2726 * Sanity check the parsed mbuf IP protocol is TCP, and 2727 * VirtIO TSO reqires the checksum offloading above. 2728 */ 2729 if (__predict_false(proto != IPPROTO_TCP)) { 2730 sc->vtnet_stats.tx_tso_not_tcp++; 2731 goto drop; 2732 } else if (__predict_false((hdr->flags & 2733 VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) { 2734 sc->vtnet_stats.tx_tso_without_csum++; 2735 goto drop; 2736 } 2737 2738 error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr); 2739 if (error) 2740 goto drop; 2741 } 2742 2743 return (m); 2744 2745 drop: 2746 m_freem(m); 2747 return (NULL); 2748 } 2749 2750 static void 2751 vtnet_txq_enqueue_callback(void *arg, bus_dma_segment_t *segs, 2752 int nsegs, int error) 2753 { 2754 vm_paddr_t *hdr_paddr; 2755 2756 if (error != 0) 2757 return; 2758 2759 KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs)); 2760 2761 hdr_paddr = (vm_paddr_t *)arg; 2762 *hdr_paddr = segs[0].ds_addr; 2763 } 2764 2765 static int 2766 vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head, 2767 struct vtnet_tx_header *txhdr) 2768 { 2769 bus_dma_segment_t segs[VTNET_TX_SEGS_MAX]; 2770 int nsegs; 2771 struct vtnet_softc *sc; 2772 struct virtqueue *vq; 2773 struct sglist *sg; 2774 struct mbuf *m; 2775 int error; 2776 vm_paddr_t hdr_paddr; 2777 bus_dmamap_t hdr_dmap; 2778 bus_dmamap_t dmap; 2779 int i; 2780 2781 sc = txq->vtntx_sc; 2782 vq = txq->vtntx_vq; 2783 sg = txq->vtntx_sg; 2784 m = *m_head; 2785 2786 sglist_reset(sg); 2787 2788 error = bus_dmamap_create(sc->vtnet_tx_dmat, 0, &hdr_dmap); 2789 if (error) 2790 goto fail; 2791 2792 error = bus_dmamap_load(sc->vtnet_tx_dmat, hdr_dmap, &txhdr->vth_uhdr, 2793 sc->vtnet_hdr_size, vtnet_txq_enqueue_callback, &hdr_paddr, 2794 BUS_DMA_NOWAIT); 2795 if (error) 2796 goto fail_hdr_dmamap_destroy; 2797 2798 error = sglist_append_phys(sg, hdr_paddr, sc->vtnet_hdr_size); 2799 if (error != 0 || sg->sg_nseg != 1) { 2800 KASSERT(0, ("%s: cannot add header to sglist error %d nseg %d", 2801 __func__, error, sg->sg_nseg)); 2802 goto fail_hdr_dmamap_unload; 2803 } 2804 2805 bus_dmamap_sync(sc->vtnet_tx_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE); 2806 2807 error = bus_dmamap_create(sc->vtnet_tx_dmat, 0, &dmap); 2808 if (error) 2809 goto fail_hdr_dmamap_unload; 2810 2811 nsegs = 0; 2812 error = bus_dmamap_load_mbuf_sg(sc->vtnet_tx_dmat, dmap, m, segs, 2813 &nsegs, BUS_DMA_NOWAIT); 2814 if (error != 0) 2815 goto fail_dmamap_destroy; 2816 KASSERT(nsegs <= sc->vtnet_tx_nsegs, 2817 ("%s: unexpected number of DMA segments for tx buffer: %d (max %d)", 2818 __func__, nsegs, sc->vtnet_tx_nsegs)); 2819 2820 bus_dmamap_sync(sc->vtnet_tx_dmat, dmap, BUS_DMASYNC_PREWRITE); 2821 2822 for (i = 0; i < nsegs && !error; i++) 2823 error = sglist_append_phys(sg, segs[i].ds_addr, segs[i].ds_len); 2824 2825 if (error) { 2826 sglist_reset(sg); 2827 bus_dmamap_unload(sc->vtnet_tx_dmat, dmap); 2828 2829 error = sglist_append_phys(sg, hdr_paddr, sc->vtnet_hdr_size); 2830 if (error != 0 || sg->sg_nseg != 1) { 2831 KASSERT(0, ("%s: cannot add header to sglist error %d nseg %d", 2832 __func__, error, sg->sg_nseg)); 2833 goto fail_dmamap_destroy; 2834 } 2835 2836 m = m_defrag(m, M_NOWAIT); 2837 if (m == NULL) { 2838 sc->vtnet_stats.tx_defrag_failed++; 2839 goto fail; 2840 } 2841 2842 *m_head = m; 2843 sc->vtnet_stats.tx_defragged++; 2844 2845 nsegs = 0; 2846 error = bus_dmamap_load_mbuf_sg(sc->vtnet_tx_dmat, dmap, m, 2847 segs, &nsegs, BUS_DMA_NOWAIT); 2848 if (error != 0) 2849 goto fail_dmamap_destroy; 2850 KASSERT(nsegs <= sc->vtnet_tx_nsegs, 2851 ("%s: unexpected number of DMA segments for tx buffer: %d (max %d)", 2852 __func__, nsegs, sc->vtnet_tx_nsegs)); 2853 2854 bus_dmamap_sync(sc->vtnet_tx_dmat, dmap, BUS_DMASYNC_PREWRITE); 2855 2856 for (i = 0; i < nsegs && !error; i++) 2857 error = sglist_append_phys(sg, segs[i].ds_addr, 2858 segs[i].ds_len); 2859 2860 if (error) 2861 goto fail_dmamap_unload; 2862 } 2863 2864 txhdr->vth_mbuf = m; 2865 txhdr->dmap = dmap; 2866 txhdr->hdr_dmap = hdr_dmap; 2867 2868 error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0); 2869 2870 return (error); 2871 2872 fail_dmamap_unload: 2873 bus_dmamap_unload(sc->vtnet_tx_dmat, dmap); 2874 fail_dmamap_destroy: 2875 bus_dmamap_destroy(sc->vtnet_tx_dmat, dmap); 2876 fail_hdr_dmamap_unload: 2877 bus_dmamap_unload(sc->vtnet_tx_dmat, hdr_dmap); 2878 fail_hdr_dmamap_destroy: 2879 bus_dmamap_destroy(sc->vtnet_tx_dmat, hdr_dmap); 2880 fail: 2881 m_freem(*m_head); 2882 *m_head = NULL; 2883 2884 return (ENOBUFS); 2885 } 2886 2887 static int 2888 vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head, int flags) 2889 { 2890 struct vtnet_tx_header *txhdr; 2891 struct virtio_net_hdr *hdr; 2892 struct mbuf *m; 2893 int error; 2894 2895 m = *m_head; 2896 M_ASSERTPKTHDR(m); 2897 2898 txhdr = uma_zalloc(vtnet_tx_header_zone, flags | M_ZERO); 2899 if (txhdr == NULL) { 2900 m_freem(m); 2901 *m_head = NULL; 2902 return (ENOMEM); 2903 } 2904 2905 /* 2906 * Always use the non-mergeable header, regardless if mergable headers 2907 * were negotiated, because for transmit num_buffers is always zero. 2908 * The vtnet_hdr_size is used to enqueue the right header size segment. 2909 */ 2910 hdr = &txhdr->vth_uhdr.hdr; 2911 2912 if (m->m_flags & M_VLANTAG) { 2913 m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); 2914 if ((*m_head = m) == NULL) { 2915 error = ENOBUFS; 2916 goto fail; 2917 } 2918 m->m_flags &= ~M_VLANTAG; 2919 } 2920 2921 if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) { 2922 m = vtnet_txq_offload(txq, m, hdr); 2923 if ((*m_head = m) == NULL) { 2924 error = ENOBUFS; 2925 goto fail; 2926 } 2927 } 2928 2929 error = vtnet_txq_enqueue_buf(txq, m_head, txhdr); 2930 fail: 2931 if (error) 2932 uma_zfree(vtnet_tx_header_zone, txhdr); 2933 2934 return (error); 2935 } 2936 2937 2938 static void 2939 vtnet_start_locked(struct vtnet_txq *txq, if_t ifp) 2940 { 2941 struct vtnet_softc *sc; 2942 struct virtqueue *vq; 2943 struct mbuf *m0; 2944 int tries, enq; 2945 2946 sc = txq->vtntx_sc; 2947 vq = txq->vtntx_vq; 2948 tries = 0; 2949 2950 VTNET_TXQ_LOCK_ASSERT(txq); 2951 2952 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || 2953 sc->vtnet_link_active == 0) 2954 return; 2955 2956 vtnet_txq_eof(txq); 2957 2958 again: 2959 enq = 0; 2960 2961 while (!if_sendq_empty(ifp)) { 2962 if (virtqueue_full(vq)) 2963 break; 2964 2965 m0 = if_dequeue(ifp); 2966 if (m0 == NULL) 2967 break; 2968 2969 if (vtnet_txq_encap(txq, &m0, M_NOWAIT) != 0) { 2970 if (m0 != NULL) 2971 if_sendq_prepend(ifp, m0); 2972 break; 2973 } 2974 2975 enq++; 2976 ETHER_BPF_MTAP(ifp, m0); 2977 } 2978 2979 if (enq > 0 && vtnet_txq_notify(txq) != 0) { 2980 if (tries++ < VTNET_NOTIFY_RETRIES) 2981 goto again; 2982 2983 txq->vtntx_stats.vtxs_rescheduled++; 2984 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); 2985 } 2986 } 2987 2988 static void 2989 vtnet_start(if_t ifp) 2990 { 2991 struct vtnet_softc *sc; 2992 struct vtnet_txq *txq; 2993 2994 sc = if_getsoftc(ifp); 2995 txq = &sc->vtnet_txqs[0]; 2996 2997 VTNET_TXQ_LOCK(txq); 2998 vtnet_start_locked(txq, ifp); 2999 VTNET_TXQ_UNLOCK(txq); 3000 } 3001 3002 3003 static int 3004 vtnet_txq_mq_start_locked(struct vtnet_txq *txq, struct mbuf *m) 3005 { 3006 struct vtnet_softc *sc; 3007 struct virtqueue *vq; 3008 struct buf_ring *br; 3009 if_t ifp; 3010 int enq, tries, error; 3011 3012 sc = txq->vtntx_sc; 3013 vq = txq->vtntx_vq; 3014 br = txq->vtntx_br; 3015 ifp = sc->vtnet_ifp; 3016 tries = 0; 3017 error = 0; 3018 3019 VTNET_TXQ_LOCK_ASSERT(txq); 3020 3021 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || 3022 sc->vtnet_link_active == 0) { 3023 if (m != NULL) 3024 error = drbr_enqueue(ifp, br, m); 3025 return (error); 3026 } 3027 3028 if (m != NULL) { 3029 error = drbr_enqueue(ifp, br, m); 3030 if (error) 3031 return (error); 3032 } 3033 3034 vtnet_txq_eof(txq); 3035 3036 again: 3037 enq = 0; 3038 3039 while ((m = drbr_peek(ifp, br)) != NULL) { 3040 if (virtqueue_full(vq)) { 3041 drbr_putback(ifp, br, m); 3042 break; 3043 } 3044 3045 if (vtnet_txq_encap(txq, &m, M_NOWAIT) != 0) { 3046 if (m != NULL) 3047 drbr_putback(ifp, br, m); 3048 else 3049 drbr_advance(ifp, br); 3050 break; 3051 } 3052 drbr_advance(ifp, br); 3053 3054 enq++; 3055 ETHER_BPF_MTAP(ifp, m); 3056 } 3057 3058 if (enq > 0 && vtnet_txq_notify(txq) != 0) { 3059 if (tries++ < VTNET_NOTIFY_RETRIES) 3060 goto again; 3061 3062 txq->vtntx_stats.vtxs_rescheduled++; 3063 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); 3064 } 3065 3066 return (0); 3067 } 3068 3069 static int 3070 vtnet_txq_mq_start(if_t ifp, struct mbuf *m) 3071 { 3072 struct vtnet_softc *sc; 3073 struct vtnet_txq *txq; 3074 int i, npairs, error; 3075 3076 sc = if_getsoftc(ifp); 3077 npairs = sc->vtnet_act_vq_pairs; 3078 3079 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 3080 i = m->m_pkthdr.flowid % npairs; 3081 else 3082 i = curcpu % npairs; 3083 3084 txq = &sc->vtnet_txqs[i]; 3085 3086 if (VTNET_TXQ_TRYLOCK(txq) != 0) { 3087 error = vtnet_txq_mq_start_locked(txq, m); 3088 VTNET_TXQ_UNLOCK(txq); 3089 } else { 3090 error = drbr_enqueue(ifp, txq->vtntx_br, m); 3091 taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask); 3092 } 3093 3094 return (error); 3095 } 3096 3097 static void 3098 vtnet_txq_tq_deferred(void *xtxq, int pending __unused) 3099 { 3100 struct vtnet_softc *sc; 3101 struct vtnet_txq *txq; 3102 3103 txq = xtxq; 3104 sc = txq->vtntx_sc; 3105 3106 VTNET_TXQ_LOCK(txq); 3107 if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br)) 3108 vtnet_txq_mq_start_locked(txq, NULL); 3109 VTNET_TXQ_UNLOCK(txq); 3110 } 3111 3112 3113 static void 3114 vtnet_txq_start(struct vtnet_txq *txq) 3115 { 3116 struct vtnet_softc *sc; 3117 if_t ifp; 3118 3119 sc = txq->vtntx_sc; 3120 ifp = sc->vtnet_ifp; 3121 3122 if (!VTNET_ALTQ_ENABLED) { 3123 if (!drbr_empty(ifp, txq->vtntx_br)) 3124 vtnet_txq_mq_start_locked(txq, NULL); 3125 } else { 3126 if (!if_sendq_empty(ifp)) 3127 vtnet_start_locked(txq, ifp); 3128 3129 } 3130 } 3131 3132 static void 3133 vtnet_txq_tq_intr(void *xtxq, int pending __unused) 3134 { 3135 struct vtnet_softc *sc; 3136 struct vtnet_txq *txq; 3137 if_t ifp; 3138 3139 txq = xtxq; 3140 sc = txq->vtntx_sc; 3141 ifp = sc->vtnet_ifp; 3142 3143 VTNET_TXQ_LOCK(txq); 3144 3145 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 3146 VTNET_TXQ_UNLOCK(txq); 3147 return; 3148 } 3149 3150 vtnet_txq_eof(txq); 3151 vtnet_txq_start(txq); 3152 3153 VTNET_TXQ_UNLOCK(txq); 3154 } 3155 3156 static int 3157 vtnet_txq_eof(struct vtnet_txq *txq) 3158 { 3159 struct vtnet_softc *sc; 3160 struct virtqueue *vq; 3161 struct vtnet_tx_header *txhdr; 3162 struct mbuf *m; 3163 int deq; 3164 3165 vq = txq->vtntx_vq; 3166 deq = 0; 3167 VTNET_TXQ_LOCK_ASSERT(txq); 3168 3169 sc = txq->vtntx_sc; 3170 3171 while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) { 3172 m = txhdr->vth_mbuf; 3173 deq++; 3174 3175 txq->vtntx_stats.vtxs_opackets++; 3176 txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len; 3177 if (m->m_flags & M_MCAST) 3178 txq->vtntx_stats.vtxs_omcasts++; 3179 3180 bus_dmamap_unload(sc->vtnet_tx_dmat, txhdr->dmap); 3181 bus_dmamap_destroy(sc->vtnet_tx_dmat, txhdr->dmap); 3182 bus_dmamap_unload(sc->vtnet_tx_dmat, txhdr->hdr_dmap); 3183 bus_dmamap_destroy(sc->vtnet_tx_dmat, txhdr->hdr_dmap); 3184 3185 m_freem(m); 3186 uma_zfree(vtnet_tx_header_zone, txhdr); 3187 } 3188 3189 if (virtqueue_empty(vq)) 3190 txq->vtntx_watchdog = 0; 3191 3192 return (deq); 3193 } 3194 3195 static void 3196 vtnet_tx_vq_intr(void *xtxq) 3197 { 3198 struct vtnet_softc *sc; 3199 struct vtnet_txq *txq; 3200 if_t ifp; 3201 3202 txq = xtxq; 3203 sc = txq->vtntx_sc; 3204 ifp = sc->vtnet_ifp; 3205 3206 if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) { 3207 /* 3208 * Ignore this interrupt. Either this is a spurious interrupt 3209 * or multiqueue without per-VQ MSIX so every queue needs to 3210 * be polled (a brain dead configuration we could try harder 3211 * to avoid). 3212 */ 3213 vtnet_txq_disable_intr(txq); 3214 return; 3215 } 3216 3217 #ifdef DEV_NETMAP 3218 if (netmap_tx_irq(ifp, txq->vtntx_id) != NM_IRQ_PASS) 3219 return; 3220 #endif /* DEV_NETMAP */ 3221 3222 VTNET_TXQ_LOCK(txq); 3223 3224 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 3225 VTNET_TXQ_UNLOCK(txq); 3226 return; 3227 } 3228 3229 vtnet_txq_eof(txq); 3230 vtnet_txq_start(txq); 3231 3232 VTNET_TXQ_UNLOCK(txq); 3233 } 3234 3235 static void 3236 vtnet_tx_start_all(struct vtnet_softc *sc) 3237 { 3238 struct vtnet_txq *txq; 3239 int i; 3240 3241 VTNET_CORE_LOCK_ASSERT(sc); 3242 3243 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 3244 txq = &sc->vtnet_txqs[i]; 3245 3246 VTNET_TXQ_LOCK(txq); 3247 vtnet_txq_start(txq); 3248 VTNET_TXQ_UNLOCK(txq); 3249 } 3250 } 3251 3252 static void 3253 vtnet_qflush(if_t ifp) 3254 { 3255 struct vtnet_softc *sc; 3256 struct vtnet_txq *txq; 3257 struct mbuf *m; 3258 int i; 3259 3260 sc = if_getsoftc(ifp); 3261 3262 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 3263 txq = &sc->vtnet_txqs[i]; 3264 3265 VTNET_TXQ_LOCK(txq); 3266 while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL) 3267 m_freem(m); 3268 VTNET_TXQ_UNLOCK(txq); 3269 } 3270 3271 if_qflush(ifp); 3272 } 3273 3274 static int 3275 vtnet_watchdog(struct vtnet_txq *txq) 3276 { 3277 if_t ifp; 3278 3279 ifp = txq->vtntx_sc->vtnet_ifp; 3280 3281 VTNET_TXQ_LOCK(txq); 3282 if (txq->vtntx_watchdog == 1) { 3283 /* 3284 * Only drain completed frames if the watchdog is about to 3285 * expire. If any frames were drained, there may be enough 3286 * free descriptors now available to transmit queued frames. 3287 * In that case, the timer will immediately be decremented 3288 * below, but the timeout is generous enough that should not 3289 * be a problem. 3290 */ 3291 if (vtnet_txq_eof(txq) != 0) 3292 vtnet_txq_start(txq); 3293 } 3294 3295 if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) { 3296 VTNET_TXQ_UNLOCK(txq); 3297 return (0); 3298 } 3299 VTNET_TXQ_UNLOCK(txq); 3300 3301 if_printf(ifp, "watchdog timeout on queue %d\n", txq->vtntx_id); 3302 return (1); 3303 } 3304 3305 static void 3306 vtnet_accum_stats(struct vtnet_softc *sc, struct vtnet_rxq_stats *rxacc, 3307 struct vtnet_txq_stats *txacc) 3308 { 3309 3310 bzero(rxacc, sizeof(struct vtnet_rxq_stats)); 3311 bzero(txacc, sizeof(struct vtnet_txq_stats)); 3312 3313 for (int i = 0; i < sc->vtnet_max_vq_pairs; i++) { 3314 struct vtnet_rxq_stats *rxst; 3315 struct vtnet_txq_stats *txst; 3316 3317 rxst = &sc->vtnet_rxqs[i].vtnrx_stats; 3318 rxacc->vrxs_ipackets += rxst->vrxs_ipackets; 3319 rxacc->vrxs_ibytes += rxst->vrxs_ibytes; 3320 rxacc->vrxs_iqdrops += rxst->vrxs_iqdrops; 3321 rxacc->vrxs_csum += rxst->vrxs_csum; 3322 rxacc->vrxs_csum_failed += rxst->vrxs_csum_failed; 3323 rxacc->vrxs_rescheduled += rxst->vrxs_rescheduled; 3324 3325 txst = &sc->vtnet_txqs[i].vtntx_stats; 3326 txacc->vtxs_opackets += txst->vtxs_opackets; 3327 txacc->vtxs_obytes += txst->vtxs_obytes; 3328 txacc->vtxs_csum += txst->vtxs_csum; 3329 txacc->vtxs_tso += txst->vtxs_tso; 3330 txacc->vtxs_rescheduled += txst->vtxs_rescheduled; 3331 } 3332 } 3333 3334 static uint64_t 3335 vtnet_get_counter(if_t ifp, ift_counter cnt) 3336 { 3337 struct vtnet_softc *sc; 3338 struct vtnet_rxq_stats rxaccum; 3339 struct vtnet_txq_stats txaccum; 3340 3341 sc = if_getsoftc(ifp); 3342 vtnet_accum_stats(sc, &rxaccum, &txaccum); 3343 3344 switch (cnt) { 3345 case IFCOUNTER_IPACKETS: 3346 return (rxaccum.vrxs_ipackets); 3347 case IFCOUNTER_IQDROPS: 3348 return (rxaccum.vrxs_iqdrops); 3349 case IFCOUNTER_IERRORS: 3350 return (rxaccum.vrxs_ierrors); 3351 case IFCOUNTER_IBYTES: 3352 return (rxaccum.vrxs_ibytes); 3353 case IFCOUNTER_OPACKETS: 3354 return (txaccum.vtxs_opackets); 3355 case IFCOUNTER_OBYTES: 3356 return (txaccum.vtxs_obytes); 3357 case IFCOUNTER_OMCASTS: 3358 return (txaccum.vtxs_omcasts); 3359 default: 3360 return (if_get_counter_default(ifp, cnt)); 3361 } 3362 } 3363 3364 static void 3365 vtnet_tick(void *xsc) 3366 { 3367 struct vtnet_softc *sc; 3368 if_t ifp; 3369 int i, timedout; 3370 3371 sc = xsc; 3372 ifp = sc->vtnet_ifp; 3373 timedout = 0; 3374 3375 VTNET_CORE_LOCK_ASSERT(sc); 3376 3377 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 3378 timedout |= vtnet_watchdog(&sc->vtnet_txqs[i]); 3379 3380 if (timedout != 0) { 3381 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 3382 vtnet_init_locked(sc, 0); 3383 } else 3384 callout_schedule(&sc->vtnet_tick_ch, hz); 3385 } 3386 3387 static void 3388 vtnet_start_taskqueues(struct vtnet_softc *sc) 3389 { 3390 device_t dev; 3391 struct vtnet_rxq *rxq; 3392 struct vtnet_txq *txq; 3393 int i, error; 3394 3395 dev = sc->vtnet_dev; 3396 3397 /* 3398 * Errors here are very difficult to recover from - we cannot 3399 * easily fail because, if this is during boot, we will hang 3400 * when freeing any successfully started taskqueues because 3401 * the scheduler isn't up yet. 3402 * 3403 * Most drivers just ignore the return value - it only fails 3404 * with ENOMEM so an error is not likely. 3405 */ 3406 for (i = 0; i < sc->vtnet_req_vq_pairs; i++) { 3407 rxq = &sc->vtnet_rxqs[i]; 3408 error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET, 3409 "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id); 3410 if (error) { 3411 device_printf(dev, "failed to start rx taskq %d\n", 3412 rxq->vtnrx_id); 3413 } 3414 3415 txq = &sc->vtnet_txqs[i]; 3416 error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET, 3417 "%s txq %d", device_get_nameunit(dev), txq->vtntx_id); 3418 if (error) { 3419 device_printf(dev, "failed to start tx taskq %d\n", 3420 txq->vtntx_id); 3421 } 3422 } 3423 } 3424 3425 static void 3426 vtnet_free_taskqueues(struct vtnet_softc *sc) 3427 { 3428 struct vtnet_rxq *rxq; 3429 struct vtnet_txq *txq; 3430 int i; 3431 3432 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 3433 rxq = &sc->vtnet_rxqs[i]; 3434 if (rxq->vtnrx_tq != NULL) { 3435 taskqueue_free(rxq->vtnrx_tq); 3436 rxq->vtnrx_tq = NULL; 3437 } 3438 3439 txq = &sc->vtnet_txqs[i]; 3440 if (txq->vtntx_tq != NULL) { 3441 taskqueue_free(txq->vtntx_tq); 3442 txq->vtntx_tq = NULL; 3443 } 3444 } 3445 } 3446 3447 static void 3448 vtnet_drain_taskqueues(struct vtnet_softc *sc) 3449 { 3450 struct vtnet_rxq *rxq; 3451 struct vtnet_txq *txq; 3452 int i; 3453 3454 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 3455 rxq = &sc->vtnet_rxqs[i]; 3456 if (rxq->vtnrx_tq != NULL) 3457 taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 3458 3459 txq = &sc->vtnet_txqs[i]; 3460 if (txq->vtntx_tq != NULL) { 3461 taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask); 3462 if (!VTNET_ALTQ_ENABLED) 3463 taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask); 3464 } 3465 } 3466 } 3467 3468 static void 3469 vtnet_drain_rxtx_queues(struct vtnet_softc *sc) 3470 { 3471 struct vtnet_rxq *rxq; 3472 struct vtnet_txq *txq; 3473 int i; 3474 3475 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 3476 rxq = &sc->vtnet_rxqs[i]; 3477 vtnet_rxq_free_mbufs(rxq); 3478 3479 txq = &sc->vtnet_txqs[i]; 3480 vtnet_txq_free_mbufs(txq); 3481 } 3482 } 3483 3484 static void 3485 vtnet_stop_rendezvous(struct vtnet_softc *sc) 3486 { 3487 struct vtnet_rxq *rxq; 3488 struct vtnet_txq *txq; 3489 int i; 3490 3491 VTNET_CORE_LOCK_ASSERT(sc); 3492 3493 /* 3494 * Lock and unlock the per-queue mutex so we known the stop 3495 * state is visible. Doing only the active queues should be 3496 * sufficient, but it does not cost much extra to do all the 3497 * queues. 3498 */ 3499 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 3500 rxq = &sc->vtnet_rxqs[i]; 3501 VTNET_RXQ_LOCK(rxq); 3502 VTNET_RXQ_UNLOCK(rxq); 3503 3504 txq = &sc->vtnet_txqs[i]; 3505 VTNET_TXQ_LOCK(txq); 3506 VTNET_TXQ_UNLOCK(txq); 3507 } 3508 } 3509 3510 static void 3511 vtnet_stop(struct vtnet_softc *sc) 3512 { 3513 device_t dev; 3514 if_t ifp; 3515 3516 dev = sc->vtnet_dev; 3517 ifp = sc->vtnet_ifp; 3518 3519 VTNET_CORE_LOCK_ASSERT(sc); 3520 3521 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 3522 sc->vtnet_link_active = 0; 3523 callout_stop(&sc->vtnet_tick_ch); 3524 3525 /* Only advisory. */ 3526 vtnet_disable_interrupts(sc); 3527 3528 #ifdef DEV_NETMAP 3529 /* Stop any pending txsync/rxsync and disable them. */ 3530 netmap_disable_all_rings(ifp); 3531 #endif /* DEV_NETMAP */ 3532 3533 /* 3534 * Stop the host adapter. This resets it to the pre-initialized 3535 * state. It will not generate any interrupts until after it is 3536 * reinitialized. 3537 */ 3538 virtio_stop(dev); 3539 vtnet_stop_rendezvous(sc); 3540 3541 vtnet_drain_rxtx_queues(sc); 3542 sc->vtnet_act_vq_pairs = 1; 3543 } 3544 3545 static int 3546 vtnet_virtio_reinit(struct vtnet_softc *sc) 3547 { 3548 device_t dev; 3549 if_t ifp; 3550 uint64_t features; 3551 int error; 3552 3553 dev = sc->vtnet_dev; 3554 ifp = sc->vtnet_ifp; 3555 features = sc->vtnet_negotiated_features; 3556 3557 /* 3558 * Re-negotiate with the host, removing any disabled receive 3559 * features. Transmit features are disabled only on our side 3560 * via if_capenable and if_hwassist. 3561 */ 3562 3563 if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) == 0) 3564 features &= ~(VIRTIO_NET_F_GUEST_CSUM | VTNET_LRO_FEATURES); 3565 3566 if ((if_getcapenable(ifp) & IFCAP_LRO) == 0) 3567 features &= ~VTNET_LRO_FEATURES; 3568 3569 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) == 0) 3570 features &= ~VIRTIO_NET_F_CTRL_VLAN; 3571 3572 error = virtio_reinit(dev, features); 3573 if (error) { 3574 device_printf(dev, "virtio reinit error %d\n", error); 3575 return (error); 3576 } 3577 3578 sc->vtnet_features = features; 3579 virtio_reinit_complete(dev); 3580 3581 return (0); 3582 } 3583 3584 static void 3585 vtnet_init_rx_filters(struct vtnet_softc *sc) 3586 { 3587 if_t ifp; 3588 3589 ifp = sc->vtnet_ifp; 3590 3591 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 3592 vtnet_rx_filter(sc); 3593 vtnet_rx_filter_mac(sc); 3594 } 3595 3596 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) 3597 vtnet_rx_filter_vlan(sc); 3598 } 3599 3600 static int 3601 vtnet_init_rx_queues(struct vtnet_softc *sc) 3602 { 3603 device_t dev; 3604 if_t ifp; 3605 struct vtnet_rxq *rxq; 3606 int i, clustersz, error; 3607 3608 dev = sc->vtnet_dev; 3609 ifp = sc->vtnet_ifp; 3610 3611 clustersz = vtnet_rx_cluster_size(sc, if_getmtu(ifp)); 3612 sc->vtnet_rx_clustersz = clustersz; 3613 3614 if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) { 3615 sc->vtnet_rx_nmbufs = howmany(sizeof(struct vtnet_rx_header) + 3616 VTNET_MAX_RX_SIZE, clustersz); 3617 KASSERT(sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs, 3618 ("%s: too many rx mbufs %d for %d segments", __func__, 3619 sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs)); 3620 } else 3621 sc->vtnet_rx_nmbufs = 1; 3622 3623 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 3624 rxq = &sc->vtnet_rxqs[i]; 3625 3626 /* Hold the lock to satisfy asserts. */ 3627 VTNET_RXQ_LOCK(rxq); 3628 error = vtnet_rxq_populate(rxq); 3629 VTNET_RXQ_UNLOCK(rxq); 3630 3631 if (error) { 3632 device_printf(dev, "cannot populate Rx queue %d\n", i); 3633 return (error); 3634 } 3635 } 3636 3637 return (0); 3638 } 3639 3640 static int 3641 vtnet_init_tx_queues(struct vtnet_softc *sc) 3642 { 3643 struct vtnet_txq *txq; 3644 int i; 3645 3646 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 3647 txq = &sc->vtnet_txqs[i]; 3648 txq->vtntx_watchdog = 0; 3649 txq->vtntx_intr_threshold = vtnet_txq_intr_threshold(txq); 3650 #ifdef DEV_NETMAP 3651 netmap_reset(NA(sc->vtnet_ifp), NR_TX, i, 0); 3652 #endif /* DEV_NETMAP */ 3653 } 3654 3655 return (0); 3656 } 3657 3658 static int 3659 vtnet_init_rxtx_queues(struct vtnet_softc *sc) 3660 { 3661 int error; 3662 3663 error = vtnet_init_rx_queues(sc); 3664 if (error) 3665 return (error); 3666 3667 error = vtnet_init_tx_queues(sc); 3668 if (error) 3669 return (error); 3670 3671 return (0); 3672 } 3673 3674 static void 3675 vtnet_set_active_vq_pairs(struct vtnet_softc *sc) 3676 { 3677 device_t dev; 3678 int npairs; 3679 3680 dev = sc->vtnet_dev; 3681 3682 if ((sc->vtnet_flags & VTNET_FLAG_MQ) == 0) { 3683 sc->vtnet_act_vq_pairs = 1; 3684 return; 3685 } 3686 3687 npairs = sc->vtnet_req_vq_pairs; 3688 3689 if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) { 3690 device_printf(dev, "cannot set active queue pairs to %d, " 3691 "falling back to 1 queue pair\n", npairs); 3692 npairs = 1; 3693 } 3694 3695 sc->vtnet_act_vq_pairs = npairs; 3696 } 3697 3698 static void 3699 vtnet_update_rx_offloads(struct vtnet_softc *sc) 3700 { 3701 if_t ifp; 3702 uint64_t features; 3703 int error; 3704 3705 ifp = sc->vtnet_ifp; 3706 features = sc->vtnet_features; 3707 3708 VTNET_CORE_LOCK_ASSERT(sc); 3709 3710 if (if_getcapabilities(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { 3711 if (if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) 3712 features |= VIRTIO_NET_F_GUEST_CSUM; 3713 else 3714 features &= ~VIRTIO_NET_F_GUEST_CSUM; 3715 } 3716 3717 if (if_getcapabilities(ifp) & IFCAP_LRO && !vtnet_software_lro(sc)) { 3718 if (if_getcapenable(ifp) & IFCAP_LRO) 3719 features |= VTNET_LRO_FEATURES; 3720 else 3721 features &= ~VTNET_LRO_FEATURES; 3722 } 3723 3724 error = vtnet_ctrl_guest_offloads(sc, 3725 features & (VIRTIO_NET_F_GUEST_CSUM | VIRTIO_NET_F_GUEST_TSO4 | 3726 VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN | 3727 VIRTIO_NET_F_GUEST_UFO)); 3728 if (error) { 3729 device_printf(sc->vtnet_dev, 3730 "%s: cannot update Rx features\n", __func__); 3731 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 3732 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 3733 vtnet_init_locked(sc, 0); 3734 } 3735 } else 3736 sc->vtnet_features = features; 3737 } 3738 3739 static int 3740 vtnet_reinit(struct vtnet_softc *sc) 3741 { 3742 if_t ifp; 3743 int error; 3744 3745 ifp = sc->vtnet_ifp; 3746 3747 bcopy(if_getlladdr(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN); 3748 3749 error = vtnet_virtio_reinit(sc); 3750 if (error) 3751 return (error); 3752 3753 vtnet_set_macaddr(sc); 3754 vtnet_set_active_vq_pairs(sc); 3755 3756 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) 3757 vtnet_init_rx_filters(sc); 3758 3759 if_sethwassist(ifp, 0); 3760 if (if_getcapenable(ifp) & IFCAP_TXCSUM) 3761 if_sethwassistbits(ifp, VTNET_CSUM_OFFLOAD, 0); 3762 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) 3763 if_sethwassistbits(ifp, VTNET_CSUM_OFFLOAD_IPV6, 0); 3764 if (if_getcapenable(ifp) & IFCAP_TSO4) 3765 if_sethwassistbits(ifp, CSUM_IP_TSO, 0); 3766 if (if_getcapenable(ifp) & IFCAP_TSO6) 3767 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); 3768 3769 error = vtnet_init_rxtx_queues(sc); 3770 if (error) 3771 return (error); 3772 3773 return (0); 3774 } 3775 3776 static void 3777 vtnet_init_locked(struct vtnet_softc *sc, int init_mode) 3778 { 3779 if_t ifp; 3780 3781 ifp = sc->vtnet_ifp; 3782 3783 VTNET_CORE_LOCK_ASSERT(sc); 3784 3785 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 3786 return; 3787 3788 vtnet_stop(sc); 3789 3790 #ifdef DEV_NETMAP 3791 /* Once stopped we can update the netmap flags, if necessary. */ 3792 switch (init_mode) { 3793 case VTNET_INIT_NETMAP_ENTER: 3794 nm_set_native_flags(NA(ifp)); 3795 break; 3796 case VTNET_INIT_NETMAP_EXIT: 3797 nm_clear_native_flags(NA(ifp)); 3798 break; 3799 } 3800 #endif /* DEV_NETMAP */ 3801 3802 if (vtnet_reinit(sc) != 0) { 3803 vtnet_stop(sc); 3804 return; 3805 } 3806 3807 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 3808 vtnet_update_link_status(sc); 3809 vtnet_enable_interrupts(sc); 3810 callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc); 3811 3812 #ifdef DEV_NETMAP 3813 /* Re-enable txsync/rxsync. */ 3814 netmap_enable_all_rings(ifp); 3815 #endif /* DEV_NETMAP */ 3816 } 3817 3818 static void 3819 vtnet_init(void *xsc) 3820 { 3821 struct vtnet_softc *sc; 3822 3823 sc = xsc; 3824 3825 VTNET_CORE_LOCK(sc); 3826 vtnet_init_locked(sc, 0); 3827 VTNET_CORE_UNLOCK(sc); 3828 } 3829 3830 static void 3831 vtnet_free_ctrl_vq(struct vtnet_softc *sc) 3832 { 3833 3834 /* 3835 * The control virtqueue is only polled and therefore it should 3836 * already be empty. 3837 */ 3838 KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq), 3839 ("%s: ctrl vq %p not empty", __func__, sc->vtnet_ctrl_vq)); 3840 } 3841 3842 static void 3843 vtnet_load_callback(void *arg, bus_dma_segment_t *segs, int nsegs, 3844 int error) 3845 { 3846 bus_addr_t *paddr; 3847 3848 if (error != 0) 3849 return; 3850 3851 KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs)); 3852 3853 paddr = (bus_addr_t *)arg; 3854 *paddr = segs[0].ds_addr; 3855 } 3856 3857 static int 3858 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, uint8_t *ack, struct sglist *sg, 3859 int readable, int writable) 3860 { 3861 bus_dmamap_t ack_dmap; 3862 bus_addr_t ack_paddr; 3863 struct virtqueue *vq; 3864 int error; 3865 3866 error = bus_dmamap_create(sc->vtnet_ack_dmat, 0, &ack_dmap); 3867 if (error) 3868 goto error_out; 3869 3870 error = bus_dmamap_load(sc->vtnet_ack_dmat, ack_dmap, ack, 3871 sizeof(uint8_t), vtnet_load_callback, &ack_paddr, BUS_DMA_NOWAIT); 3872 if (error) 3873 goto error_destroy; 3874 3875 bus_dmamap_sync(sc->vtnet_ack_dmat, ack_dmap, BUS_DMASYNC_PREWRITE); 3876 3877 error = sglist_append_phys(sg, ack_paddr, sizeof(uint8_t)); 3878 if (error) 3879 goto error_unload; 3880 3881 vq = sc->vtnet_ctrl_vq; 3882 3883 MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ); 3884 VTNET_CORE_LOCK_ASSERT(sc); 3885 3886 if (!virtqueue_empty(vq)) 3887 goto error_unload; 3888 3889 /* 3890 * Poll for the response, but the command is likely completed before 3891 * returning from the notify. 3892 */ 3893 if (virtqueue_enqueue(vq, (void *)ack, sg, readable, writable) == 0) { 3894 virtqueue_notify(vq); 3895 virtqueue_poll(vq, NULL); 3896 } 3897 3898 bus_dmamap_sync(sc->vtnet_ack_dmat, ack_dmap, BUS_DMASYNC_POSTREAD); 3899 3900 error_unload: 3901 bus_dmamap_unload(sc->vtnet_ack_dmat, ack_dmap); 3902 error_destroy: 3903 bus_dmamap_destroy(sc->vtnet_ack_dmat, ack_dmap); 3904 error_out: 3905 return (error); 3906 } 3907 3908 static int 3909 vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr) 3910 { 3911 struct sglist_seg segs[3]; 3912 bus_dmamap_t hdr_dmap; 3913 bus_addr_t hdr_paddr; 3914 struct sglist sg; 3915 struct { 3916 struct virtio_net_ctrl_hdr hdr __aligned(2); 3917 uint8_t pad1; 3918 uint8_t addr[ETHER_ADDR_LEN] __aligned(8); 3919 uint8_t pad2; 3920 } s; 3921 uint8_t ack; 3922 int error; 3923 3924 error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap); 3925 if (error) 3926 goto error_out; 3927 3928 error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s, 3929 sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT); 3930 if (error) 3931 goto error_destroy_hdr; 3932 3933 MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_MAC); 3934 3935 s.hdr.class = VIRTIO_NET_CTRL_MAC; 3936 s.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET; 3937 bcopy(hwaddr, &s.addr[0], ETHER_ADDR_LEN); 3938 ack = VIRTIO_NET_ERR; 3939 bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE); 3940 3941 sglist_init(&sg, nitems(segs), segs); 3942 error |= sglist_append_phys(&sg, hdr_paddr, 3943 sizeof(struct virtio_net_ctrl_hdr)); 3944 error |= sglist_append_phys(&sg, 3945 hdr_paddr + ((uintptr_t)&s.addr - (uintptr_t)&s), 3946 ETHER_ADDR_LEN); 3947 MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1); 3948 3949 if (error == 0) 3950 error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1); 3951 if (error == 0) 3952 error = (ack == VIRTIO_NET_OK ? 0 : EIO); 3953 3954 bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap); 3955 error_destroy_hdr: 3956 bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap); 3957 error_out: 3958 return (error); 3959 } 3960 3961 static int 3962 vtnet_ctrl_guest_offloads(struct vtnet_softc *sc, uint64_t offloads) 3963 { 3964 struct sglist_seg segs[3]; 3965 bus_dmamap_t hdr_dmap; 3966 bus_addr_t hdr_paddr; 3967 struct sglist sg; 3968 struct { 3969 struct virtio_net_ctrl_hdr hdr __aligned(2); 3970 uint8_t pad1; 3971 uint64_t offloads __aligned(8); 3972 uint8_t pad2; 3973 } s; 3974 uint8_t ack; 3975 int error; 3976 3977 error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap); 3978 if (error) 3979 goto error_out; 3980 3981 error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s, 3982 sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT); 3983 if (error) 3984 goto error_destroy_hdr; 3985 3986 MPASS(sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 3987 3988 s.hdr.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; 3989 s.hdr.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; 3990 s.offloads = vtnet_gtoh64(sc, offloads); 3991 ack = VIRTIO_NET_ERR; 3992 bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE); 3993 3994 sglist_init(&sg, nitems(segs), segs); 3995 error |= sglist_append_phys(&sg, hdr_paddr, 3996 sizeof(struct virtio_net_ctrl_hdr)); 3997 error |= sglist_append_phys(&sg, 3998 hdr_paddr + ((uintptr_t)&s.offloads - (uintptr_t)&s), 3999 sizeof(uint64_t)); 4000 MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1); 4001 4002 if (error == 0) 4003 error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1); 4004 if (error == 0) 4005 error = (ack == VIRTIO_NET_OK ? 0 : EIO); 4006 4007 bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap); 4008 error_destroy_hdr: 4009 bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap); 4010 error_out: 4011 return (error); 4012 } 4013 4014 static int 4015 vtnet_ctrl_mq_cmd(struct vtnet_softc *sc, uint16_t npairs) 4016 { 4017 struct sglist_seg segs[3]; 4018 bus_dmamap_t hdr_dmap; 4019 bus_addr_t hdr_paddr; 4020 struct sglist sg; 4021 struct { 4022 struct virtio_net_ctrl_hdr hdr __aligned(2); 4023 uint8_t pad1; 4024 struct virtio_net_ctrl_mq mq __aligned(2); 4025 uint8_t pad2; 4026 } s; 4027 uint8_t ack; 4028 int error; 4029 4030 error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap); 4031 if (error) 4032 goto error_out; 4033 4034 error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s, 4035 sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT); 4036 if (error) 4037 goto error_destroy_hdr; 4038 4039 MPASS(sc->vtnet_flags & VTNET_FLAG_MQ); 4040 4041 s.hdr.class = VIRTIO_NET_CTRL_MQ; 4042 s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; 4043 s.mq.virtqueue_pairs = vtnet_gtoh16(sc, npairs); 4044 ack = VIRTIO_NET_ERR; 4045 bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE); 4046 4047 sglist_init(&sg, nitems(segs), segs); 4048 error |= sglist_append_phys(&sg, hdr_paddr, 4049 sizeof(struct virtio_net_ctrl_hdr)); 4050 error |= sglist_append_phys(&sg, 4051 hdr_paddr + ((uintptr_t)&s.mq - (uintptr_t)&s), 4052 sizeof(struct virtio_net_ctrl_mq)); 4053 MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1); 4054 4055 if (error == 0) 4056 error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1); 4057 if (error == 0) 4058 error = (ack == VIRTIO_NET_OK ? 0 : EIO); 4059 4060 bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap); 4061 error_destroy_hdr: 4062 bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap); 4063 error_out: 4064 return (error); 4065 } 4066 4067 static int 4068 vtnet_ctrl_announce_ack_cmd(struct vtnet_softc *sc) 4069 { 4070 struct sglist_seg segs[2]; 4071 bus_dmamap_t hdr_dmap; 4072 bus_addr_t hdr_paddr; 4073 struct sglist sg; 4074 struct virtio_net_ctrl_hdr hdr __aligned(2); 4075 uint8_t ack; 4076 int error; 4077 4078 error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap); 4079 if (error != 0) 4080 goto error_out; 4081 4082 error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &hdr, 4083 sizeof(hdr), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT); 4084 if (error != 0) 4085 goto error_destroy_hdr; 4086 4087 hdr.class = VIRTIO_NET_CTRL_ANNOUNCE; 4088 hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK; 4089 ack = VIRTIO_NET_ERR; 4090 bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE); 4091 4092 sglist_init(&sg, nitems(segs), segs); 4093 error = sglist_append_phys(&sg, hdr_paddr, sizeof(hdr)); 4094 MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1); 4095 4096 if (error == 0) 4097 error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1); 4098 if (error == 0) 4099 error = (ack == VIRTIO_NET_OK ? 0 : EIO); 4100 4101 bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap); 4102 error_destroy_hdr: 4103 bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap); 4104 error_out: 4105 return (error); 4106 } 4107 4108 static bool 4109 vtnet_announce_pending(struct vtnet_softc *sc) 4110 { 4111 uint16_t status; 4112 4113 if ((sc->vtnet_features & VIRTIO_NET_F_GUEST_ANNOUNCE) == 0 || 4114 (sc->vtnet_features & VIRTIO_NET_F_CTRL_VQ) == 0 || 4115 (sc->vtnet_features & VIRTIO_NET_F_STATUS) == 0) 4116 return (false); 4117 4118 status = virtio_read_dev_config_2(sc->vtnet_dev, 4119 offsetof(struct virtio_net_config, status)); 4120 4121 return ((status & VIRTIO_NET_S_ANNOUNCE) != 0); 4122 } 4123 4124 static void 4125 vtnet_announce(void *xsc, int pending __unused) 4126 { 4127 struct vtnet_softc *sc; 4128 if_t ifp; 4129 4130 sc = xsc; 4131 ifp = sc->vtnet_ifp; 4132 4133 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 4134 return; 4135 4136 CURVNET_SET(if_getvnet(ifp)); 4137 EVENTHANDLER_INVOKE(iflladdr_event, ifp); 4138 CURVNET_RESTORE(); 4139 4140 VTNET_CORE_LOCK(sc); 4141 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0 && 4142 vtnet_ctrl_announce_ack_cmd(sc) != 0) 4143 device_printf(sc->vtnet_dev, "cannot ack announcement\n"); 4144 VTNET_CORE_UNLOCK(sc); 4145 } 4146 4147 static int 4148 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, uint8_t cmd, bool on) 4149 { 4150 struct sglist_seg segs[3]; 4151 bus_dmamap_t hdr_dmap; 4152 bus_addr_t hdr_paddr; 4153 struct sglist sg; 4154 struct { 4155 struct virtio_net_ctrl_hdr hdr __aligned(2); 4156 uint8_t pad1; 4157 uint8_t onoff; 4158 uint8_t pad2; 4159 } s; 4160 uint8_t ack; 4161 int error; 4162 4163 error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap); 4164 if (error) 4165 goto error_out; 4166 4167 error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s, 4168 sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT); 4169 if (error) 4170 goto error_destroy_hdr; 4171 4172 MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX); 4173 4174 s.hdr.class = VIRTIO_NET_CTRL_RX; 4175 s.hdr.cmd = cmd; 4176 s.onoff = on; 4177 ack = VIRTIO_NET_ERR; 4178 bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE); 4179 4180 sglist_init(&sg, nitems(segs), segs); 4181 error |= sglist_append_phys(&sg, hdr_paddr, 4182 sizeof(struct virtio_net_ctrl_hdr)); 4183 error |= sglist_append_phys(&sg, 4184 hdr_paddr + ((uintptr_t)&s.onoff - (uintptr_t)&s), 4185 sizeof(uint8_t)); 4186 MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1); 4187 4188 if (error == 0) 4189 error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1); 4190 if (error == 0) 4191 error = (ack == VIRTIO_NET_OK ? 0 : EIO); 4192 4193 bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap); 4194 error_destroy_hdr: 4195 bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap); 4196 error_out: 4197 return (error); 4198 } 4199 4200 static int 4201 vtnet_set_promisc(struct vtnet_softc *sc, bool on) 4202 { 4203 return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on)); 4204 } 4205 4206 static int 4207 vtnet_set_allmulti(struct vtnet_softc *sc, bool on) 4208 { 4209 return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on)); 4210 } 4211 4212 static void 4213 vtnet_rx_filter(struct vtnet_softc *sc) 4214 { 4215 device_t dev; 4216 if_t ifp; 4217 4218 dev = sc->vtnet_dev; 4219 ifp = sc->vtnet_ifp; 4220 4221 VTNET_CORE_LOCK_ASSERT(sc); 4222 4223 if (vtnet_set_promisc(sc, if_getflags(ifp) & IFF_PROMISC) != 0) { 4224 device_printf(dev, "cannot %s promiscuous mode\n", 4225 if_getflags(ifp) & IFF_PROMISC ? "enable" : "disable"); 4226 } 4227 4228 if (vtnet_set_allmulti(sc, if_getflags(ifp) & IFF_ALLMULTI) != 0) { 4229 device_printf(dev, "cannot %s all-multicast mode\n", 4230 if_getflags(ifp) & IFF_ALLMULTI ? "enable" : "disable"); 4231 } 4232 } 4233 4234 static u_int 4235 vtnet_copy_ifaddr(void *arg, struct sockaddr_dl *sdl, u_int ucnt) 4236 { 4237 struct vtnet_softc *sc = arg; 4238 4239 if (memcmp(LLADDR(sdl), sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0) 4240 return (0); 4241 4242 if (ucnt < VTNET_MAX_MAC_ENTRIES) 4243 bcopy(LLADDR(sdl), 4244 &sc->vtnet_mac_filter->vmf_unicast.macs[ucnt], 4245 ETHER_ADDR_LEN); 4246 4247 return (1); 4248 } 4249 4250 static u_int 4251 vtnet_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int mcnt) 4252 { 4253 struct vtnet_mac_filter *filter = arg; 4254 4255 if (mcnt < VTNET_MAX_MAC_ENTRIES) 4256 bcopy(LLADDR(sdl), &filter->vmf_multicast.macs[mcnt], 4257 ETHER_ADDR_LEN); 4258 4259 return (1); 4260 } 4261 4262 static void 4263 vtnet_rx_filter_mac(struct vtnet_softc *sc) 4264 { 4265 struct virtio_net_ctrl_hdr hdr __aligned(2); 4266 struct vtnet_mac_filter *filter; 4267 struct sglist_seg segs[4]; 4268 bus_dmamap_t filter_dmap; 4269 bus_addr_t filter_paddr; 4270 bus_dmamap_t hdr_dmap; 4271 bus_addr_t hdr_paddr; 4272 struct sglist sg; 4273 if_t ifp; 4274 bool promisc, allmulti; 4275 u_int ucnt, mcnt; 4276 int error; 4277 uint8_t ack; 4278 4279 ifp = sc->vtnet_ifp; 4280 filter = sc->vtnet_mac_filter; 4281 error = 0; 4282 4283 MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX); 4284 VTNET_CORE_LOCK_ASSERT(sc); 4285 4286 /* Unicast MAC addresses: */ 4287 ucnt = if_foreach_lladdr(ifp, vtnet_copy_ifaddr, sc); 4288 promisc = (ucnt > VTNET_MAX_MAC_ENTRIES); 4289 4290 if (promisc) { 4291 ucnt = 0; 4292 if_printf(ifp, "more than %d MAC addresses assigned, " 4293 "falling back to promiscuous mode\n", 4294 VTNET_MAX_MAC_ENTRIES); 4295 } 4296 4297 /* Multicast MAC addresses: */ 4298 mcnt = if_foreach_llmaddr(ifp, vtnet_copy_maddr, filter); 4299 allmulti = (mcnt > VTNET_MAX_MAC_ENTRIES); 4300 4301 if (allmulti) { 4302 mcnt = 0; 4303 if_printf(ifp, "more than %d multicast MAC addresses " 4304 "assigned, falling back to all-multicast mode\n", 4305 VTNET_MAX_MAC_ENTRIES); 4306 } 4307 4308 if (promisc && allmulti) 4309 goto out; 4310 4311 error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap); 4312 if (error) 4313 goto out_error; 4314 4315 error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &hdr, 4316 sizeof(hdr), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT); 4317 if (error) 4318 goto out_destroy_hdr; 4319 4320 error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &filter_dmap); 4321 if (error) 4322 goto out_unload_hdr; 4323 4324 error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, filter, 4325 sizeof(*filter), vtnet_load_callback, &filter_paddr, 4326 BUS_DMA_NOWAIT); 4327 if (error) 4328 goto out_destroy_filter; 4329 4330 filter->vmf_unicast.nentries = vtnet_gtoh32(sc, ucnt); 4331 filter->vmf_multicast.nentries = vtnet_gtoh32(sc, mcnt); 4332 4333 hdr.class = VIRTIO_NET_CTRL_MAC; 4334 hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 4335 ack = VIRTIO_NET_ERR; 4336 4337 sglist_init(&sg, nitems(segs), segs); 4338 error |= sglist_append_phys(&sg, hdr_paddr, 4339 sizeof(struct virtio_net_ctrl_hdr)); 4340 error |= sglist_append_phys(&sg, 4341 filter_paddr + ((uintptr_t)&filter->vmf_unicast - 4342 (uintptr_t)filter), 4343 sizeof(uint32_t) + ucnt * ETHER_ADDR_LEN); 4344 error |= sglist_append_phys(&sg, 4345 filter_paddr + ((uintptr_t)&filter->vmf_multicast - 4346 (uintptr_t)filter), 4347 sizeof(uint32_t) + mcnt * ETHER_ADDR_LEN); 4348 MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1); 4349 4350 if (error == 0) 4351 error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1); 4352 if (error == 0) 4353 error = (ack == VIRTIO_NET_OK ? 0 : EIO); 4354 4355 bus_dmamap_unload(sc->vtnet_hdr_dmat, filter_dmap); 4356 out_destroy_filter: 4357 bus_dmamap_destroy(sc->vtnet_hdr_dmat, filter_dmap); 4358 out_unload_hdr: 4359 bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap); 4360 out_destroy_hdr: 4361 bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap); 4362 out_error: 4363 if (error != 0) 4364 if_printf(ifp, "error setting host MAC filter table\n"); 4365 out: 4366 if (promisc && vtnet_set_promisc(sc, true) != 0) 4367 if_printf(ifp, "cannot enable promiscuous mode\n"); 4368 if (allmulti && vtnet_set_allmulti(sc, true) != 0) 4369 if_printf(ifp, "cannot enable all-multicast mode\n"); 4370 } 4371 4372 static int 4373 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 4374 { 4375 struct sglist_seg segs[3]; 4376 bus_dmamap_t hdr_dmap; 4377 bus_addr_t hdr_paddr; 4378 struct sglist sg; 4379 struct { 4380 struct virtio_net_ctrl_hdr hdr __aligned(2); 4381 uint8_t pad1; 4382 uint16_t tag __aligned(2); 4383 uint8_t pad2; 4384 } s; 4385 uint8_t ack; 4386 int error; 4387 4388 error = bus_dmamap_create(sc->vtnet_hdr_dmat, 0, &hdr_dmap); 4389 if (error) 4390 goto error_out; 4391 4392 error = bus_dmamap_load(sc->vtnet_hdr_dmat, hdr_dmap, &s, 4393 sizeof(s), vtnet_load_callback, &hdr_paddr, BUS_DMA_NOWAIT); 4394 if (error) 4395 goto error_destroy_hdr; 4396 4397 MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER); 4398 4399 s.hdr.class = VIRTIO_NET_CTRL_VLAN; 4400 s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; 4401 s.tag = vtnet_gtoh16(sc, tag); 4402 ack = VIRTIO_NET_ERR; 4403 bus_dmamap_sync(sc->vtnet_hdr_dmat, hdr_dmap, BUS_DMASYNC_PREWRITE); 4404 4405 sglist_init(&sg, nitems(segs), segs); 4406 error |= sglist_append_phys(&sg, hdr_paddr, 4407 sizeof(struct virtio_net_ctrl_hdr)); 4408 error |= sglist_append_phys(&sg, 4409 hdr_paddr + ((uintptr_t)&s.tag - (uintptr_t)&s), 4410 sizeof(uint16_t)); 4411 MPASS(error == 0 && sg.sg_nseg == nitems(segs) - 1); 4412 4413 if (error == 0) 4414 error = vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg, 1); 4415 if (error == 0) 4416 error = (ack == VIRTIO_NET_OK ? 0 : EIO); 4417 4418 bus_dmamap_unload(sc->vtnet_hdr_dmat, hdr_dmap); 4419 error_destroy_hdr: 4420 bus_dmamap_destroy(sc->vtnet_hdr_dmat, hdr_dmap); 4421 error_out: 4422 return (error); 4423 } 4424 4425 static void 4426 vtnet_rx_filter_vlan(struct vtnet_softc *sc) 4427 { 4428 int i, bit; 4429 uint32_t w; 4430 uint16_t tag; 4431 4432 MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER); 4433 VTNET_CORE_LOCK_ASSERT(sc); 4434 4435 /* Enable the filter for each configured VLAN. */ 4436 for (i = 0; i < VTNET_VLAN_FILTER_NWORDS; i++) { 4437 w = sc->vtnet_vlan_filter[i]; 4438 4439 while ((bit = ffs(w) - 1) != -1) { 4440 w &= ~(1 << bit); 4441 tag = sizeof(w) * CHAR_BIT * i + bit; 4442 4443 if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) { 4444 device_printf(sc->vtnet_dev, 4445 "cannot enable VLAN %d filter\n", tag); 4446 } 4447 } 4448 } 4449 } 4450 4451 static void 4452 vtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 4453 { 4454 if_t ifp; 4455 int idx, bit; 4456 4457 ifp = sc->vtnet_ifp; 4458 idx = (tag >> 5) & 0x7F; 4459 bit = tag & 0x1F; 4460 4461 if (tag == 0 || tag > 4095) 4462 return; 4463 4464 VTNET_CORE_LOCK(sc); 4465 4466 if (add) 4467 sc->vtnet_vlan_filter[idx] |= (1 << bit); 4468 else 4469 sc->vtnet_vlan_filter[idx] &= ~(1 << bit); 4470 4471 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER && 4472 if_getdrvflags(ifp) & IFF_DRV_RUNNING && 4473 vtnet_exec_vlan_filter(sc, add, tag) != 0) { 4474 device_printf(sc->vtnet_dev, 4475 "cannot %s VLAN %d %s the host filter table\n", 4476 add ? "add" : "remove", tag, add ? "to" : "from"); 4477 } 4478 4479 VTNET_CORE_UNLOCK(sc); 4480 } 4481 4482 static void 4483 vtnet_register_vlan(void *arg, if_t ifp, uint16_t tag) 4484 { 4485 4486 if (if_getsoftc(ifp) != arg) 4487 return; 4488 4489 vtnet_update_vlan_filter(arg, 1, tag); 4490 } 4491 4492 static void 4493 vtnet_unregister_vlan(void *arg, if_t ifp, uint16_t tag) 4494 { 4495 4496 if (if_getsoftc(ifp) != arg) 4497 return; 4498 4499 vtnet_update_vlan_filter(arg, 0, tag); 4500 } 4501 4502 static void 4503 vtnet_update_speed_duplex(struct vtnet_softc *sc) 4504 { 4505 if_t ifp; 4506 uint32_t speed; 4507 4508 ifp = sc->vtnet_ifp; 4509 4510 if ((sc->vtnet_features & VIRTIO_NET_F_SPEED_DUPLEX) == 0) 4511 return; 4512 4513 /* BMV: Ignore duplex. */ 4514 speed = virtio_read_dev_config_4(sc->vtnet_dev, 4515 offsetof(struct virtio_net_config, speed)); 4516 if (speed != UINT32_MAX) 4517 if_setbaudrate(ifp, IF_Mbps(speed)); 4518 } 4519 4520 static int 4521 vtnet_is_link_up(struct vtnet_softc *sc) 4522 { 4523 uint16_t status; 4524 4525 if ((sc->vtnet_features & VIRTIO_NET_F_STATUS) == 0) 4526 return (1); 4527 4528 status = virtio_read_dev_config_2(sc->vtnet_dev, 4529 offsetof(struct virtio_net_config, status)); 4530 4531 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 4532 } 4533 4534 static void 4535 vtnet_update_link_status(struct vtnet_softc *sc) 4536 { 4537 if_t ifp; 4538 int link; 4539 4540 ifp = sc->vtnet_ifp; 4541 VTNET_CORE_LOCK_ASSERT(sc); 4542 link = vtnet_is_link_up(sc); 4543 4544 /* Notify if the link status has changed. */ 4545 if (link != 0 && sc->vtnet_link_active == 0) { 4546 vtnet_update_speed_duplex(sc); 4547 sc->vtnet_link_active = 1; 4548 if_link_state_change(ifp, LINK_STATE_UP); 4549 } else if (link == 0 && sc->vtnet_link_active != 0) { 4550 sc->vtnet_link_active = 0; 4551 if_link_state_change(ifp, LINK_STATE_DOWN); 4552 } 4553 } 4554 4555 static int 4556 vtnet_ifmedia_upd(if_t ifp __unused) 4557 { 4558 return (EOPNOTSUPP); 4559 } 4560 4561 static void 4562 vtnet_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr) 4563 { 4564 struct vtnet_softc *sc; 4565 4566 sc = if_getsoftc(ifp); 4567 4568 ifmr->ifm_status = IFM_AVALID; 4569 ifmr->ifm_active = IFM_ETHER; 4570 4571 VTNET_CORE_LOCK(sc); 4572 if (vtnet_is_link_up(sc) != 0) { 4573 ifmr->ifm_status |= IFM_ACTIVE; 4574 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 4575 } else 4576 ifmr->ifm_active |= IFM_NONE; 4577 VTNET_CORE_UNLOCK(sc); 4578 } 4579 4580 static void 4581 vtnet_get_macaddr(struct vtnet_softc *sc) 4582 { 4583 4584 if (sc->vtnet_flags & VTNET_FLAG_MAC) { 4585 virtio_read_device_config_array(sc->vtnet_dev, 4586 offsetof(struct virtio_net_config, mac), 4587 &sc->vtnet_hwaddr[0], sizeof(uint8_t), ETHER_ADDR_LEN); 4588 } else { 4589 /* Generate a random locally administered unicast address. */ 4590 sc->vtnet_hwaddr[0] = 0xB2; 4591 arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0); 4592 } 4593 } 4594 4595 static void 4596 vtnet_set_macaddr(struct vtnet_softc *sc) 4597 { 4598 device_t dev; 4599 int error; 4600 4601 dev = sc->vtnet_dev; 4602 4603 if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) { 4604 error = vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr); 4605 if (error) 4606 device_printf(dev, "unable to set MAC address\n"); 4607 return; 4608 } 4609 4610 /* MAC in config is read-only in modern VirtIO. */ 4611 if (!vtnet_modern(sc) && sc->vtnet_flags & VTNET_FLAG_MAC) { 4612 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 4613 virtio_write_dev_config_1(dev, 4614 offsetof(struct virtio_net_config, mac) + i, 4615 sc->vtnet_hwaddr[i]); 4616 } 4617 } 4618 } 4619 4620 static void 4621 vtnet_attached_set_macaddr(struct vtnet_softc *sc) 4622 { 4623 4624 /* Assign MAC address if it was generated. */ 4625 if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) 4626 vtnet_set_macaddr(sc); 4627 } 4628 4629 static void 4630 vtnet_vlan_tag_remove(struct mbuf *m) 4631 { 4632 struct ether_vlan_header *evh; 4633 4634 evh = mtod(m, struct ether_vlan_header *); 4635 m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag); 4636 m->m_flags |= M_VLANTAG; 4637 4638 /* Strip the 802.1Q header. */ 4639 bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN, 4640 ETHER_HDR_LEN - ETHER_TYPE_LEN); 4641 m_adj(m, ETHER_VLAN_ENCAP_LEN); 4642 } 4643 4644 static void 4645 vtnet_set_rx_process_limit(struct vtnet_softc *sc) 4646 { 4647 int limit; 4648 4649 limit = vtnet_tunable_int(sc, "rx_process_limit", 4650 vtnet_rx_process_limit); 4651 if (limit < 0) 4652 limit = INT_MAX; 4653 sc->vtnet_rx_process_limit = limit; 4654 } 4655 4656 static void 4657 vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx, 4658 struct sysctl_oid_list *child, struct vtnet_rxq *rxq) 4659 { 4660 struct sysctl_oid *node; 4661 struct sysctl_oid_list *list; 4662 struct vtnet_rxq_stats *stats; 4663 char namebuf[16]; 4664 4665 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id); 4666 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 4667 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Receive Queue"); 4668 list = SYSCTL_CHILDREN(node); 4669 4670 stats = &rxq->vtnrx_stats; 4671 4672 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", 4673 CTLFLAG_RD | CTLFLAG_STATS, 4674 &stats->vrxs_ipackets, "Receive packets"); 4675 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", 4676 CTLFLAG_RD | CTLFLAG_STATS, 4677 &stats->vrxs_ibytes, "Receive bytes"); 4678 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", 4679 CTLFLAG_RD | CTLFLAG_STATS, 4680 &stats->vrxs_iqdrops, "Receive drops"); 4681 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", 4682 CTLFLAG_RD | CTLFLAG_STATS, 4683 &stats->vrxs_ierrors, "Receive errors"); 4684 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", 4685 CTLFLAG_RD | CTLFLAG_STATS, 4686 &stats->vrxs_csum, "Receive checksum offloaded"); 4687 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", 4688 CTLFLAG_RD | CTLFLAG_STATS, 4689 &stats->vrxs_csum_failed, "Receive checksum offload failed"); 4690 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "host_lro", 4691 CTLFLAG_RD | CTLFLAG_STATS, 4692 &stats->vrxs_host_lro, "Receive host segmentation offloaded"); 4693 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", 4694 CTLFLAG_RD | CTLFLAG_STATS, 4695 &stats->vrxs_rescheduled, 4696 "Receive interrupt handler rescheduled"); 4697 } 4698 4699 static void 4700 vtnet_setup_txq_sysctl(struct sysctl_ctx_list *ctx, 4701 struct sysctl_oid_list *child, struct vtnet_txq *txq) 4702 { 4703 struct sysctl_oid *node; 4704 struct sysctl_oid_list *list; 4705 struct vtnet_txq_stats *stats; 4706 char namebuf[16]; 4707 4708 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id); 4709 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 4710 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Transmit Queue"); 4711 list = SYSCTL_CHILDREN(node); 4712 4713 stats = &txq->vtntx_stats; 4714 4715 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", 4716 CTLFLAG_RD | CTLFLAG_STATS, 4717 &stats->vtxs_opackets, "Transmit packets"); 4718 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", 4719 CTLFLAG_RD | CTLFLAG_STATS, 4720 &stats->vtxs_obytes, "Transmit bytes"); 4721 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", 4722 CTLFLAG_RD | CTLFLAG_STATS, 4723 &stats->vtxs_omcasts, "Transmit multicasts"); 4724 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", 4725 CTLFLAG_RD | CTLFLAG_STATS, 4726 &stats->vtxs_csum, "Transmit checksum offloaded"); 4727 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", 4728 CTLFLAG_RD | CTLFLAG_STATS, 4729 &stats->vtxs_tso, "Transmit TCP segmentation offloaded"); 4730 SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", 4731 CTLFLAG_RD | CTLFLAG_STATS, 4732 &stats->vtxs_rescheduled, 4733 "Transmit interrupt handler rescheduled"); 4734 } 4735 4736 static void 4737 vtnet_setup_queue_sysctl(struct vtnet_softc *sc) 4738 { 4739 device_t dev; 4740 struct sysctl_ctx_list *ctx; 4741 struct sysctl_oid *tree; 4742 struct sysctl_oid_list *child; 4743 int i; 4744 4745 dev = sc->vtnet_dev; 4746 ctx = device_get_sysctl_ctx(dev); 4747 tree = device_get_sysctl_tree(dev); 4748 child = SYSCTL_CHILDREN(tree); 4749 4750 for (i = 0; i < sc->vtnet_req_vq_pairs; i++) { 4751 vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]); 4752 vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]); 4753 } 4754 } 4755 4756 static int 4757 vtnet_sysctl_rx_csum_failed(SYSCTL_HANDLER_ARGS) 4758 { 4759 struct vtnet_softc *sc = (struct vtnet_softc *)arg1; 4760 struct vtnet_statistics *stats = &sc->vtnet_stats; 4761 struct vtnet_rxq_stats *rxst; 4762 int i; 4763 4764 stats->rx_csum_failed = 0; 4765 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 4766 rxst = &sc->vtnet_rxqs[i].vtnrx_stats; 4767 stats->rx_csum_failed += rxst->vrxs_csum_failed; 4768 } 4769 return (sysctl_handle_64(oidp, NULL, stats->rx_csum_failed, req)); 4770 } 4771 4772 static int 4773 vtnet_sysctl_rx_csum_offloaded(SYSCTL_HANDLER_ARGS) 4774 { 4775 struct vtnet_softc *sc = (struct vtnet_softc *)arg1; 4776 struct vtnet_statistics *stats = &sc->vtnet_stats; 4777 struct vtnet_rxq_stats *rxst; 4778 int i; 4779 4780 stats->rx_csum_offloaded = 0; 4781 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 4782 rxst = &sc->vtnet_rxqs[i].vtnrx_stats; 4783 stats->rx_csum_offloaded += rxst->vrxs_csum; 4784 } 4785 return (sysctl_handle_64(oidp, NULL, stats->rx_csum_offloaded, req)); 4786 } 4787 4788 static int 4789 vtnet_sysctl_rx_task_rescheduled(SYSCTL_HANDLER_ARGS) 4790 { 4791 struct vtnet_softc *sc = (struct vtnet_softc *)arg1; 4792 struct vtnet_statistics *stats = &sc->vtnet_stats; 4793 struct vtnet_rxq_stats *rxst; 4794 int i; 4795 4796 stats->rx_task_rescheduled = 0; 4797 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 4798 rxst = &sc->vtnet_rxqs[i].vtnrx_stats; 4799 stats->rx_task_rescheduled += rxst->vrxs_rescheduled; 4800 } 4801 return (sysctl_handle_64(oidp, NULL, stats->rx_task_rescheduled, req)); 4802 } 4803 4804 static int 4805 vtnet_sysctl_tx_csum_offloaded(SYSCTL_HANDLER_ARGS) 4806 { 4807 struct vtnet_softc *sc = (struct vtnet_softc *)arg1; 4808 struct vtnet_statistics *stats = &sc->vtnet_stats; 4809 struct vtnet_txq_stats *txst; 4810 int i; 4811 4812 stats->tx_csum_offloaded = 0; 4813 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 4814 txst = &sc->vtnet_txqs[i].vtntx_stats; 4815 stats->tx_csum_offloaded += txst->vtxs_csum; 4816 } 4817 return (sysctl_handle_64(oidp, NULL, stats->tx_csum_offloaded, req)); 4818 } 4819 4820 static int 4821 vtnet_sysctl_tx_tso_offloaded(SYSCTL_HANDLER_ARGS) 4822 { 4823 struct vtnet_softc *sc = (struct vtnet_softc *)arg1; 4824 struct vtnet_statistics *stats = &sc->vtnet_stats; 4825 struct vtnet_txq_stats *txst; 4826 int i; 4827 4828 stats->tx_tso_offloaded = 0; 4829 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 4830 txst = &sc->vtnet_txqs[i].vtntx_stats; 4831 stats->tx_tso_offloaded += txst->vtxs_tso; 4832 } 4833 return (sysctl_handle_64(oidp, NULL, stats->tx_tso_offloaded, req)); 4834 } 4835 4836 static int 4837 vtnet_sysctl_tx_task_rescheduled(SYSCTL_HANDLER_ARGS) 4838 { 4839 struct vtnet_softc *sc = (struct vtnet_softc *)arg1; 4840 struct vtnet_statistics *stats = &sc->vtnet_stats; 4841 struct vtnet_txq_stats *txst; 4842 int i; 4843 4844 stats->tx_task_rescheduled = 0; 4845 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 4846 txst = &sc->vtnet_txqs[i].vtntx_stats; 4847 stats->tx_task_rescheduled += txst->vtxs_rescheduled; 4848 } 4849 return (sysctl_handle_64(oidp, NULL, stats->tx_task_rescheduled, req)); 4850 } 4851 4852 static void 4853 vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx, 4854 struct sysctl_oid_list *child, struct vtnet_softc *sc) 4855 { 4856 struct vtnet_statistics *stats; 4857 struct vtnet_rxq_stats rxaccum; 4858 struct vtnet_txq_stats txaccum; 4859 4860 vtnet_accum_stats(sc, &rxaccum, &txaccum); 4861 4862 stats = &sc->vtnet_stats; 4863 stats->rx_csum_offloaded = rxaccum.vrxs_csum; 4864 stats->rx_csum_failed = rxaccum.vrxs_csum_failed; 4865 stats->rx_task_rescheduled = rxaccum.vrxs_rescheduled; 4866 stats->tx_csum_offloaded = txaccum.vtxs_csum; 4867 stats->tx_tso_offloaded = txaccum.vtxs_tso; 4868 stats->tx_task_rescheduled = txaccum.vtxs_rescheduled; 4869 4870 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed", 4871 CTLFLAG_RD | CTLFLAG_STATS, &stats->mbuf_alloc_failed, 4872 "Mbuf cluster allocation failures"); 4873 4874 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large", 4875 CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_frame_too_large, 4876 "Received frame larger than the mbuf chain"); 4877 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed", 4878 CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_enq_replacement_failed, 4879 "Enqueuing the replacement receive mbuf failed"); 4880 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed", 4881 CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_mergeable_failed, 4882 "Mergeable buffers receive failures"); 4883 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype", 4884 CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_bad_ethtype, 4885 "Received checksum offloaded buffer with unsupported " 4886 "Ethernet type"); 4887 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto", 4888 CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_bad_ipproto, 4889 "Received checksum offloaded buffer with incorrect IP protocol"); 4890 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_inaccessible_ipproto", 4891 CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_inaccessible_ipproto, 4892 "Received checksum offloaded buffer with inaccessible IP protocol"); 4893 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_csum_failed", 4894 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, 4895 sc, 0, vtnet_sysctl_rx_csum_failed, "QU", 4896 "Received buffer checksum offload failed"); 4897 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_csum_offloaded", 4898 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, 4899 sc, 0, vtnet_sysctl_rx_csum_offloaded, "QU", 4900 "Received buffer checksum offload succeeded"); 4901 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_task_rescheduled", 4902 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, 4903 sc, 0, vtnet_sysctl_rx_task_rescheduled, "QU", 4904 "Times the receive interrupt task rescheduled itself"); 4905 4906 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_unknown_ethtype", 4907 CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_csum_unknown_ethtype, 4908 "Aborted transmit of checksum offloaded buffer with unknown " 4909 "Ethernet type"); 4910 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_proto_mismatch", 4911 CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_csum_proto_mismatch, 4912 "Aborted transmit of checksum offloaded buffer because mismatched " 4913 "protocols"); 4914 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp", 4915 CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_tso_not_tcp, 4916 "Aborted transmit of TSO buffer with non TCP protocol"); 4917 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_without_csum", 4918 CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_tso_without_csum, 4919 "Aborted transmit of TSO buffer without TCP checksum offload"); 4920 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged", 4921 CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_defragged, 4922 "Transmit mbufs defragged"); 4923 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed", 4924 CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_defrag_failed, 4925 "Aborted transmit of buffer because defrag failed"); 4926 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_csum_offloaded", 4927 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, 4928 sc, 0, vtnet_sysctl_tx_csum_offloaded, "QU", 4929 "Offloaded checksum of transmitted buffer"); 4930 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_tso_offloaded", 4931 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, 4932 sc, 0, vtnet_sysctl_tx_tso_offloaded, "QU", 4933 "Segmentation offload of transmitted buffer"); 4934 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_task_rescheduled", 4935 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, 4936 sc, 0, vtnet_sysctl_tx_task_rescheduled, "QU", 4937 "Times the transmit interrupt task rescheduled itself"); 4938 } 4939 4940 static int 4941 vtnet_sysctl_features(SYSCTL_HANDLER_ARGS) 4942 { 4943 struct sbuf sb; 4944 struct vtnet_softc *sc = (struct vtnet_softc *)arg1; 4945 int error; 4946 4947 sbuf_new_for_sysctl(&sb, NULL, 0, req); 4948 sbuf_printf(&sb, "%b", (uint32_t)sc->vtnet_features, 4949 VIRTIO_NET_FEATURE_BITS); 4950 error = sbuf_finish(&sb); 4951 sbuf_delete(&sb); 4952 return (error); 4953 } 4954 4955 static int 4956 vtnet_sysctl_flags(SYSCTL_HANDLER_ARGS) 4957 { 4958 struct sbuf sb; 4959 struct vtnet_softc *sc = (struct vtnet_softc *)arg1; 4960 int error; 4961 4962 sbuf_new_for_sysctl(&sb, NULL, 0, req); 4963 sbuf_printf(&sb, "%b", sc->vtnet_flags, VTNET_FLAGS_BITS); 4964 error = sbuf_finish(&sb); 4965 sbuf_delete(&sb); 4966 return (error); 4967 } 4968 4969 static void 4970 vtnet_setup_sysctl(struct vtnet_softc *sc) 4971 { 4972 device_t dev; 4973 struct sysctl_ctx_list *ctx; 4974 struct sysctl_oid *tree; 4975 struct sysctl_oid_list *child; 4976 4977 dev = sc->vtnet_dev; 4978 ctx = device_get_sysctl_ctx(dev); 4979 tree = device_get_sysctl_tree(dev); 4980 child = SYSCTL_CHILDREN(tree); 4981 4982 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs", 4983 CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0, 4984 "Number of maximum supported virtqueue pairs"); 4985 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "req_vq_pairs", 4986 CTLFLAG_RD, &sc->vtnet_req_vq_pairs, 0, 4987 "Number of requested virtqueue pairs"); 4988 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs", 4989 CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0, 4990 "Number of active virtqueue pairs"); 4991 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "features", 4992 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 4993 vtnet_sysctl_features, "A", "Features"); 4994 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "flags", 4995 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 4996 vtnet_sysctl_flags, "A", "Flags"); 4997 4998 vtnet_setup_stat_sysctl(ctx, child, sc); 4999 } 5000 5001 static void 5002 vtnet_load_tunables(struct vtnet_softc *sc) 5003 { 5004 5005 sc->vtnet_lro_entry_count = vtnet_tunable_int(sc, 5006 "lro_entry_count", vtnet_lro_entry_count); 5007 if (sc->vtnet_lro_entry_count < TCP_LRO_ENTRIES) 5008 sc->vtnet_lro_entry_count = TCP_LRO_ENTRIES; 5009 5010 sc->vtnet_lro_mbufq_depth = vtnet_tunable_int(sc, 5011 "lro_mbufq_depth", vtnet_lro_mbufq_depth); 5012 } 5013 5014 static int 5015 vtnet_rxq_enable_intr(struct vtnet_rxq *rxq) 5016 { 5017 5018 return (virtqueue_enable_intr(rxq->vtnrx_vq)); 5019 } 5020 5021 static void 5022 vtnet_rxq_disable_intr(struct vtnet_rxq *rxq) 5023 { 5024 5025 virtqueue_disable_intr(rxq->vtnrx_vq); 5026 } 5027 5028 static int 5029 vtnet_txq_enable_intr(struct vtnet_txq *txq) 5030 { 5031 struct virtqueue *vq; 5032 5033 vq = txq->vtntx_vq; 5034 5035 if (vtnet_txq_below_threshold(txq) != 0) 5036 return (virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG)); 5037 5038 /* 5039 * The free count is above our threshold. Keep the Tx interrupt 5040 * disabled until the queue is fuller. 5041 */ 5042 return (0); 5043 } 5044 5045 static void 5046 vtnet_txq_disable_intr(struct vtnet_txq *txq) 5047 { 5048 5049 virtqueue_disable_intr(txq->vtntx_vq); 5050 } 5051 5052 static void 5053 vtnet_enable_rx_interrupts(struct vtnet_softc *sc) 5054 { 5055 struct vtnet_rxq *rxq; 5056 int i; 5057 5058 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 5059 rxq = &sc->vtnet_rxqs[i]; 5060 if (vtnet_rxq_enable_intr(rxq) != 0) 5061 taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); 5062 } 5063 } 5064 5065 static void 5066 vtnet_enable_tx_interrupts(struct vtnet_softc *sc) 5067 { 5068 int i; 5069 5070 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 5071 vtnet_txq_enable_intr(&sc->vtnet_txqs[i]); 5072 } 5073 5074 static void 5075 vtnet_enable_interrupts(struct vtnet_softc *sc) 5076 { 5077 5078 vtnet_enable_rx_interrupts(sc); 5079 vtnet_enable_tx_interrupts(sc); 5080 } 5081 5082 static void 5083 vtnet_disable_rx_interrupts(struct vtnet_softc *sc) 5084 { 5085 int i; 5086 5087 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 5088 vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]); 5089 } 5090 5091 static void 5092 vtnet_disable_tx_interrupts(struct vtnet_softc *sc) 5093 { 5094 int i; 5095 5096 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) 5097 vtnet_txq_disable_intr(&sc->vtnet_txqs[i]); 5098 } 5099 5100 static void 5101 vtnet_disable_interrupts(struct vtnet_softc *sc) 5102 { 5103 5104 vtnet_disable_rx_interrupts(sc); 5105 vtnet_disable_tx_interrupts(sc); 5106 } 5107 5108 static int 5109 vtnet_tunable_int(struct vtnet_softc *sc, const char *knob, int def) 5110 { 5111 char path[64]; 5112 5113 snprintf(path, sizeof(path), 5114 "hw.vtnet.%d.%s", device_get_unit(sc->vtnet_dev), knob); 5115 TUNABLE_INT_FETCH(path, &def); 5116 5117 return (def); 5118 } 5119 5120 #ifdef DEBUGNET 5121 static void 5122 vtnet_debugnet_init(if_t ifp, int *nrxr, int *ncl, int *clsize) 5123 { 5124 struct vtnet_softc *sc; 5125 5126 sc = if_getsoftc(ifp); 5127 5128 VTNET_CORE_LOCK(sc); 5129 *nrxr = sc->vtnet_req_vq_pairs; 5130 *ncl = DEBUGNET_MAX_IN_FLIGHT; 5131 *clsize = sc->vtnet_rx_clustersz; 5132 VTNET_CORE_UNLOCK(sc); 5133 } 5134 5135 static void 5136 vtnet_debugnet_event(if_t ifp __unused, enum debugnet_ev event) 5137 { 5138 struct vtnet_softc *sc; 5139 static bool sw_lro_enabled = false; 5140 5141 /* 5142 * Disable software LRO, since it would require entering the network 5143 * epoch when calling vtnet_txq_eof() in vtnet_debugnet_poll(). 5144 */ 5145 sc = if_getsoftc(ifp); 5146 switch (event) { 5147 case DEBUGNET_START: 5148 sw_lro_enabled = (sc->vtnet_flags & VTNET_FLAG_SW_LRO) != 0; 5149 if (sw_lro_enabled) 5150 sc->vtnet_flags &= ~VTNET_FLAG_SW_LRO; 5151 break; 5152 case DEBUGNET_END: 5153 if (sw_lro_enabled) 5154 sc->vtnet_flags |= VTNET_FLAG_SW_LRO; 5155 break; 5156 } 5157 } 5158 5159 static int 5160 vtnet_debugnet_transmit(if_t ifp, struct mbuf *m) 5161 { 5162 struct vtnet_softc *sc; 5163 struct vtnet_txq *txq; 5164 int error; 5165 5166 sc = if_getsoftc(ifp); 5167 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 5168 IFF_DRV_RUNNING) 5169 return (EBUSY); 5170 5171 txq = &sc->vtnet_txqs[0]; 5172 error = vtnet_txq_encap(txq, &m, M_NOWAIT | M_USE_RESERVE); 5173 if (error == 0) 5174 (void)vtnet_txq_notify(txq); 5175 return (error); 5176 } 5177 5178 static int 5179 vtnet_debugnet_poll(if_t ifp, int count) 5180 { 5181 struct vtnet_softc *sc; 5182 int i; 5183 5184 sc = if_getsoftc(ifp); 5185 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 5186 IFF_DRV_RUNNING) 5187 return (EBUSY); 5188 5189 (void)vtnet_txq_eof(&sc->vtnet_txqs[0]); 5190 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) 5191 (void)vtnet_rxq_eof(&sc->vtnet_rxqs[i]); 5192 return (0); 5193 } 5194 #endif /* DEBUGNET */ 5195