1 /*- 2 * Copyright (c) 2010-2012 Citrix Inc. 3 * Copyright (c) 2009-2012,2016-2017 Microsoft Corp. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /*- 30 * Copyright (c) 2004-2006 Kip Macy 31 * All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 42 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 43 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 45 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 46 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 47 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 48 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 49 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 50 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 51 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 52 * SUCH DAMAGE. 53 */ 54 55 #include <sys/cdefs.h> 56 #include "opt_hn.h" 57 #include "opt_inet6.h" 58 #include "opt_inet.h" 59 #include "opt_rss.h" 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/bus.h> 64 #include <sys/counter.h> 65 #include <sys/kernel.h> 66 #include <sys/limits.h> 67 #include <sys/malloc.h> 68 #include <sys/mbuf.h> 69 #include <sys/module.h> 70 #include <sys/queue.h> 71 #include <sys/lock.h> 72 #include <sys/proc.h> 73 #include <sys/rmlock.h> 74 #include <sys/sbuf.h> 75 #include <sys/sched.h> 76 #include <sys/smp.h> 77 #include <sys/socket.h> 78 #include <sys/sockio.h> 79 #include <sys/sx.h> 80 #include <sys/sysctl.h> 81 #include <sys/taskqueue.h> 82 #include <sys/buf_ring.h> 83 #include <sys/eventhandler.h> 84 #include <sys/epoch.h> 85 86 #include <vm/vm.h> 87 #include <vm/vm_extern.h> 88 #include <vm/pmap.h> 89 90 #include <machine/atomic.h> 91 #include <machine/in_cksum.h> 92 93 #include <net/bpf.h> 94 #include <net/ethernet.h> 95 #include <net/if.h> 96 #include <net/if_dl.h> 97 #include <net/if_media.h> 98 #include <net/if_types.h> 99 #include <net/if_var.h> 100 #include <net/if_vlan_var.h> 101 #include <net/rndis.h> 102 #include <net/rss_config.h> 103 104 #include <netinet/in_systm.h> 105 #include <netinet/in.h> 106 #include <netinet/ip.h> 107 #include <netinet/ip6.h> 108 #include <netinet/tcp.h> 109 #include <netinet/tcp_lro.h> 110 #include <netinet/udp.h> 111 112 #include <dev/hyperv/include/hyperv.h> 113 #include <dev/hyperv/include/hyperv_busdma.h> 114 #include <dev/hyperv/include/vmbus.h> 115 #include <dev/hyperv/include/vmbus_xact.h> 116 117 #include <dev/hyperv/netvsc/ndis.h> 118 #include <dev/hyperv/netvsc/if_hnreg.h> 119 #include <dev/hyperv/netvsc/if_hnvar.h> 120 #include <dev/hyperv/netvsc/hn_nvs.h> 121 #include <dev/hyperv/netvsc/hn_rndis.h> 122 123 #include "vmbus_if.h" 124 125 #define HN_IFSTART_SUPPORT 126 127 #define HN_RING_CNT_DEF_MAX 8 128 129 #define HN_VFMAP_SIZE_DEF 8 130 131 #define HN_XPNT_VF_ATTWAIT_MIN 2 /* seconds */ 132 133 /* YYY should get it from the underlying channel */ 134 #define HN_TX_DESC_CNT 512 135 136 #define HN_RNDIS_PKT_LEN \ 137 (sizeof(struct rndis_packet_msg) + \ 138 HN_RNDIS_PKTINFO_SIZE(HN_NDIS_HASH_VALUE_SIZE) + \ 139 HN_RNDIS_PKTINFO_SIZE(NDIS_VLAN_INFO_SIZE) + \ 140 HN_RNDIS_PKTINFO_SIZE(NDIS_LSO2_INFO_SIZE) + \ 141 HN_RNDIS_PKTINFO_SIZE(NDIS_TXCSUM_INFO_SIZE)) 142 #define HN_RNDIS_PKT_BOUNDARY PAGE_SIZE 143 #define HN_RNDIS_PKT_ALIGN CACHE_LINE_SIZE 144 145 #define HN_TX_DATA_BOUNDARY PAGE_SIZE 146 #define HN_TX_DATA_MAXSIZE IP_MAXPACKET 147 #define HN_TX_DATA_SEGSIZE PAGE_SIZE 148 /* -1 for RNDIS packet message */ 149 #define HN_TX_DATA_SEGCNT_MAX (HN_GPACNT_MAX - 1) 150 151 #define HN_DIRECT_TX_SIZE_DEF 128 152 153 #define HN_EARLY_TXEOF_THRESH 8 154 155 #define HN_PKTBUF_LEN_DEF (16 * 1024) 156 157 #define HN_LROENT_CNT_DEF 128 158 159 #define HN_LRO_LENLIM_MULTIRX_DEF (12 * ETHERMTU) 160 #define HN_LRO_LENLIM_DEF (25 * ETHERMTU) 161 /* YYY 2*MTU is a bit rough, but should be good enough. */ 162 #define HN_LRO_LENLIM_MIN(ifp) (2 * if_getmtu(ifp)) 163 164 #define HN_LRO_ACKCNT_DEF 1 165 166 #define HN_LOCK_INIT(sc) \ 167 sx_init(&(sc)->hn_lock, device_get_nameunit((sc)->hn_dev)) 168 #define HN_LOCK_DESTROY(sc) sx_destroy(&(sc)->hn_lock) 169 #define HN_LOCK_ASSERT(sc) sx_assert(&(sc)->hn_lock, SA_XLOCKED) 170 #define HN_LOCK(sc) \ 171 do { \ 172 while (sx_try_xlock(&(sc)->hn_lock) == 0) { \ 173 /* Relinquish cpu to avoid deadlock */ \ 174 sched_relinquish(curthread); \ 175 DELAY(1000); \ 176 } \ 177 } while (0) 178 #define HN_UNLOCK(sc) sx_xunlock(&(sc)->hn_lock) 179 180 #define HN_CSUM_IP_MASK (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP) 181 #define HN_CSUM_IP6_MASK (CSUM_IP6_TCP | CSUM_IP6_UDP) 182 #define HN_CSUM_IP_HWASSIST(sc) \ 183 ((sc)->hn_tx_ring[0].hn_csum_assist & HN_CSUM_IP_MASK) 184 #define HN_CSUM_IP6_HWASSIST(sc) \ 185 ((sc)->hn_tx_ring[0].hn_csum_assist & HN_CSUM_IP6_MASK) 186 187 /* Packet offloads can follow the VF; services requiring hn methods cannot. */ 188 #define HN_XPNT_VF_CAPS (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | \ 189 IFCAP_TSO | IFCAP_LRO | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 190 IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWFILTER | \ 191 IFCAP_JUMBO_MTU | IFCAP_LINKSTATE | IFCAP_HWSTATS | IFCAP_MEXTPG) 192 193 #define HN_PKTSIZE_MIN(align) \ 194 roundup2(ETHER_MIN_LEN + ETHER_VLAN_ENCAP_LEN - ETHER_CRC_LEN + \ 195 HN_RNDIS_PKT_LEN, (align)) 196 #define HN_PKTSIZE(m, align) \ 197 roundup2((m)->m_pkthdr.len + HN_RNDIS_PKT_LEN, (align)) 198 199 #ifdef RSS 200 #define HN_RING_IDX2CPU(sc, idx) rss_getcpu((idx) % rss_getnumbuckets()) 201 #else 202 #define HN_RING_IDX2CPU(sc, idx) (((sc)->hn_cpu + (idx)) % mp_ncpus) 203 #endif 204 205 struct hn_txdesc { 206 #ifndef HN_USE_TXDESC_BUFRING 207 SLIST_ENTRY(hn_txdesc) link; 208 #endif 209 STAILQ_ENTRY(hn_txdesc) agg_link; 210 211 /* Aggregated txdescs, in sending order. */ 212 STAILQ_HEAD(, hn_txdesc) agg_list; 213 214 /* The oldest packet, if transmission aggregation happens. */ 215 struct mbuf *m; 216 struct hn_tx_ring *txr; 217 int refs; 218 uint32_t flags; /* HN_TXD_FLAG_ */ 219 struct hn_nvs_sendctx send_ctx; 220 uint32_t chim_index; 221 int chim_size; 222 223 bus_dmamap_t data_dmap; 224 225 bus_addr_t rndis_pkt_paddr; 226 struct rndis_packet_msg *rndis_pkt; 227 bus_dmamap_t rndis_pkt_dmap; 228 }; 229 230 #define HN_TXD_FLAG_ONLIST 0x0001 231 #define HN_TXD_FLAG_DMAMAP 0x0002 232 #define HN_TXD_FLAG_ONAGG 0x0004 233 234 #define HN_NDIS_PKTINFO_SUBALLOC 0x01 235 #define HN_NDIS_PKTINFO_1ST_FRAG 0x02 236 #define HN_NDIS_PKTINFO_LAST_FRAG 0x04 237 238 struct packet_info_id { 239 uint8_t ver; 240 uint8_t flag; 241 uint16_t pkt_id; 242 }; 243 244 #define NDIS_PKTINFOID_SZ sizeof(struct packet_info_id) 245 246 247 struct hn_rxinfo { 248 const uint32_t *vlan_info; 249 const uint32_t *csum_info; 250 const uint32_t *hash_info; 251 const uint32_t *hash_value; 252 const struct packet_info_id *pktinfo_id; 253 }; 254 255 struct hn_rxvf_setarg { 256 struct hn_rx_ring *rxr; 257 if_t vf_ifp; 258 }; 259 260 #define HN_RXINFO_VLAN 0x0001 261 #define HN_RXINFO_CSUM 0x0002 262 #define HN_RXINFO_HASHINF 0x0004 263 #define HN_RXINFO_HASHVAL 0x0008 264 #define HN_RXINFO_PKTINFO_ID 0x0010 265 #define HN_RXINFO_ALL \ 266 (HN_RXINFO_VLAN | \ 267 HN_RXINFO_CSUM | \ 268 HN_RXINFO_HASHINF | \ 269 HN_RXINFO_HASHVAL | \ 270 HN_RXINFO_PKTINFO_ID) 271 272 static int hn_probe(device_t); 273 static int hn_attach(device_t); 274 static int hn_detach(device_t); 275 static int hn_shutdown(device_t); 276 static void hn_chan_callback(struct vmbus_channel *, 277 void *); 278 279 static void hn_init(void *); 280 static int hn_ioctl(if_t, u_long, caddr_t); 281 #ifdef HN_IFSTART_SUPPORT 282 static void hn_start(if_t); 283 #endif 284 static int hn_transmit(if_t, struct mbuf *); 285 static void hn_xmit_qflush(if_t); 286 static int hn_ifmedia_upd(if_t); 287 static void hn_ifmedia_sts(if_t, 288 struct ifmediareq *); 289 290 static void hn_ifnet_event(void *, if_t, int); 291 static void hn_ifaddr_event(void *, if_t); 292 static void hn_ifnet_attevent(void *, if_t); 293 static void hn_ifnet_detevent(void *, if_t); 294 static void hn_ifnet_lnkevent(void *, if_t, int); 295 296 static bool hn_ismyvf(const struct hn_softc *, 297 const if_t); 298 static void hn_rxvf_change(struct hn_softc *, 299 if_t); 300 static void hn_rxvf_set(struct hn_softc *, if_t); 301 static void hn_rxvf_change_locked(struct hn_softc *, if_t, 302 bool); 303 static void hn_rxvf_set_task(void *, int); 304 static void hn_xpnt_vf_input(if_t, struct mbuf *); 305 static int hn_xpnt_vf_iocsetflags(struct hn_softc *); 306 static int hn_xpnt_vf_iocsetcaps(struct hn_softc *, 307 struct ifreq *); 308 static void hn_xpnt_vf_synccaps(struct hn_softc *); 309 static void hn_xpnt_vf_vlancap_taskfunc(void *, int); 310 static void hn_xpnt_vf_saveifflags(struct hn_softc *); 311 static bool hn_xpnt_vf_isready(struct hn_softc *); 312 static bool hn_xpnt_vf_caninit(struct hn_softc *); 313 static void hn_xpnt_vf_setready(struct hn_softc *); 314 static void hn_xpnt_vf_restore(struct hn_softc *); 315 static void hn_xpnt_vf_deactivate(struct hn_softc *); 316 static void hn_xpnt_vf_init_taskfunc(void *, int); 317 static void hn_xpnt_vf_init(struct hn_softc *); 318 static bool hn_xpnt_vf_setenable(struct hn_softc *); 319 static void hn_xpnt_vf_setdisable(struct hn_softc *, bool); 320 static void hn_vf_rss_fixup(struct hn_softc *, bool); 321 static void hn_vf_rss_restore(struct hn_softc *); 322 323 static int hn_rndis_rxinfo(const void *, int, 324 struct hn_rxinfo *); 325 static void hn_rndis_rx_data(struct hn_rx_ring *, 326 const void *, int); 327 static void hn_rndis_rx_status(struct hn_softc *, 328 const void *, int); 329 static void hn_rndis_init_fixat(struct hn_softc *, int); 330 331 static void hn_nvs_handle_notify(struct hn_softc *, 332 const struct vmbus_chanpkt_hdr *); 333 static void hn_nvs_handle_comp(struct hn_softc *, 334 struct vmbus_channel *, 335 const struct vmbus_chanpkt_hdr *); 336 static void hn_nvs_handle_rxbuf(struct hn_rx_ring *, 337 struct vmbus_channel *, 338 const struct vmbus_chanpkt_hdr *); 339 static void hn_nvs_ack_rxbuf(struct hn_rx_ring *, 340 struct vmbus_channel *, uint64_t); 341 342 static int hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS); 343 static int hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS); 344 static int hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS); 345 static int hn_chim_size_sysctl(SYSCTL_HANDLER_ARGS); 346 static int hn_rx_stat_u64_sysctl(SYSCTL_HANDLER_ARGS); 347 static int hn_rx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS); 348 static int hn_tx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS); 349 static int hn_tx_conf_int_sysctl(SYSCTL_HANDLER_ARGS); 350 static int hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS); 351 static int hn_caps_sysctl(SYSCTL_HANDLER_ARGS); 352 static int hn_hwassist_sysctl(SYSCTL_HANDLER_ARGS); 353 static int hn_rxfilter_sysctl(SYSCTL_HANDLER_ARGS); 354 #ifndef RSS 355 static int hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS); 356 static int hn_rss_ind_sysctl(SYSCTL_HANDLER_ARGS); 357 #endif 358 static int hn_rss_hash_sysctl(SYSCTL_HANDLER_ARGS); 359 static int hn_rss_hcap_sysctl(SYSCTL_HANDLER_ARGS); 360 static int hn_rss_mbuf_sysctl(SYSCTL_HANDLER_ARGS); 361 static int hn_txagg_size_sysctl(SYSCTL_HANDLER_ARGS); 362 static int hn_txagg_pkts_sysctl(SYSCTL_HANDLER_ARGS); 363 static int hn_txagg_pktmax_sysctl(SYSCTL_HANDLER_ARGS); 364 static int hn_txagg_align_sysctl(SYSCTL_HANDLER_ARGS); 365 static int hn_polling_sysctl(SYSCTL_HANDLER_ARGS); 366 static int hn_vf_sysctl(SYSCTL_HANDLER_ARGS); 367 static int hn_rxvf_sysctl(SYSCTL_HANDLER_ARGS); 368 static int hn_vflist_sysctl(SYSCTL_HANDLER_ARGS); 369 static int hn_vfmap_sysctl(SYSCTL_HANDLER_ARGS); 370 static int hn_xpnt_vf_accbpf_sysctl(SYSCTL_HANDLER_ARGS); 371 static int hn_xpnt_vf_enabled_sysctl(SYSCTL_HANDLER_ARGS); 372 373 static void hn_stop(struct hn_softc *, bool); 374 static void hn_init_locked(struct hn_softc *); 375 static int hn_chan_attach(struct hn_softc *, 376 struct vmbus_channel *); 377 static void hn_chan_detach(struct hn_softc *, 378 struct vmbus_channel *); 379 static int hn_attach_subchans(struct hn_softc *); 380 static void hn_detach_allchans(struct hn_softc *); 381 static void hn_chan_rollup(struct hn_rx_ring *, 382 struct hn_tx_ring *); 383 static void hn_set_ring_inuse(struct hn_softc *, int); 384 static int hn_synth_attach(struct hn_softc *, int); 385 static void hn_synth_detach(struct hn_softc *); 386 static int hn_synth_alloc_subchans(struct hn_softc *, 387 int *); 388 static bool hn_synth_attachable(const struct hn_softc *); 389 static void hn_suspend(struct hn_softc *); 390 static void hn_suspend_data(struct hn_softc *); 391 static void hn_suspend_mgmt(struct hn_softc *); 392 static void hn_resume(struct hn_softc *); 393 static void hn_resume_data(struct hn_softc *); 394 static void hn_resume_mgmt(struct hn_softc *); 395 static void hn_suspend_mgmt_taskfunc(void *, int); 396 static void hn_chan_drain(struct hn_softc *, 397 struct vmbus_channel *); 398 static void hn_disable_rx(struct hn_softc *); 399 static void hn_drain_rxtx(struct hn_softc *, int); 400 static void hn_polling(struct hn_softc *, u_int); 401 static void hn_chan_polling(struct vmbus_channel *, u_int); 402 static void hn_mtu_change_fixup(struct hn_softc *); 403 404 static void hn_update_link_status(struct hn_softc *); 405 static void hn_change_network(struct hn_softc *); 406 static void hn_link_taskfunc(void *, int); 407 static void hn_netchg_init_taskfunc(void *, int); 408 static void hn_netchg_status_taskfunc(void *, int); 409 static void hn_link_status(struct hn_softc *); 410 411 static int hn_create_rx_data(struct hn_softc *, int); 412 static void hn_destroy_rx_data(struct hn_softc *); 413 static int hn_check_iplen(const struct mbuf *, int); 414 static void hn_rxpkt_proto(const struct mbuf *, int *, int *); 415 static int hn_set_rxfilter(struct hn_softc *, uint32_t); 416 static int hn_rxfilter_config(struct hn_softc *); 417 static int hn_rss_reconfig(struct hn_softc *); 418 static void hn_rss_ind_fixup(struct hn_softc *); 419 static void hn_rss_mbuf_hash(struct hn_softc *, uint32_t); 420 static int hn_rxpkt(struct hn_rx_ring *); 421 static uint32_t hn_rss_type_fromndis(uint32_t); 422 static uint32_t hn_rss_type_tondis(uint32_t); 423 424 static int hn_tx_ring_create(struct hn_softc *, int); 425 static void hn_tx_ring_destroy(struct hn_tx_ring *); 426 static int hn_create_tx_data(struct hn_softc *, int); 427 static void hn_fixup_tx_data(struct hn_softc *); 428 static void hn_fixup_rx_data(struct hn_softc *); 429 static void hn_destroy_tx_data(struct hn_softc *); 430 static void hn_txdesc_dmamap_destroy(struct hn_txdesc *); 431 static void hn_txdesc_gc(struct hn_tx_ring *, 432 struct hn_txdesc *); 433 static int hn_encap(if_t, struct hn_tx_ring *, 434 struct hn_txdesc *, struct mbuf **); 435 static int hn_txpkt(if_t, struct hn_tx_ring *, 436 struct hn_txdesc *); 437 static void hn_set_chim_size(struct hn_softc *, int); 438 static void hn_set_tso_maxsize(struct hn_softc *, int, int); 439 static bool hn_tx_ring_pending(struct hn_tx_ring *); 440 static void hn_tx_ring_qflush(struct hn_tx_ring *); 441 static void hn_resume_tx(struct hn_softc *, int); 442 static void hn_set_txagg(struct hn_softc *); 443 static void *hn_try_txagg(if_t, 444 struct hn_tx_ring *, struct hn_txdesc *, 445 int); 446 static int hn_get_txswq_depth(const struct hn_tx_ring *); 447 static void hn_txpkt_done(struct hn_nvs_sendctx *, 448 struct hn_softc *, struct vmbus_channel *, 449 const void *, int); 450 static int hn_txpkt_sglist(struct hn_tx_ring *, 451 struct hn_txdesc *); 452 static int hn_txpkt_chim(struct hn_tx_ring *, 453 struct hn_txdesc *); 454 static int hn_xmit(struct hn_tx_ring *, int); 455 static void hn_xmit_taskfunc(void *, int); 456 static void hn_xmit_txeof(struct hn_tx_ring *); 457 static void hn_xmit_txeof_taskfunc(void *, int); 458 #ifdef HN_IFSTART_SUPPORT 459 static int hn_start_locked(struct hn_tx_ring *, int); 460 static void hn_start_taskfunc(void *, int); 461 static void hn_start_txeof(struct hn_tx_ring *); 462 static void hn_start_txeof_taskfunc(void *, int); 463 #endif 464 465 static int hn_rsc_sysctl(SYSCTL_HANDLER_ARGS); 466 467 SYSCTL_NODE(_hw, OID_AUTO, hn, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 468 "Hyper-V network interface"); 469 470 /* Trust tcp segment verification on host side. */ 471 static int hn_trust_hosttcp = 1; 472 SYSCTL_INT(_hw_hn, OID_AUTO, trust_hosttcp, CTLFLAG_RDTUN, 473 &hn_trust_hosttcp, 0, 474 "Trust tcp segment verification on host side, " 475 "when csum info is missing (global setting)"); 476 477 /* Trust udp datagrams verification on host side. */ 478 static int hn_trust_hostudp = 1; 479 SYSCTL_INT(_hw_hn, OID_AUTO, trust_hostudp, CTLFLAG_RDTUN, 480 &hn_trust_hostudp, 0, 481 "Trust udp datagram verification on host side, " 482 "when csum info is missing (global setting)"); 483 484 /* Trust ip packets verification on host side. */ 485 static int hn_trust_hostip = 1; 486 SYSCTL_INT(_hw_hn, OID_AUTO, trust_hostip, CTLFLAG_RDTUN, 487 &hn_trust_hostip, 0, 488 "Trust ip packet verification on host side, " 489 "when csum info is missing (global setting)"); 490 491 /* 492 * Offload UDP/IPv4 checksum. 493 */ 494 static int hn_enable_udp4cs = 1; 495 SYSCTL_INT(_hw_hn, OID_AUTO, enable_udp4cs, CTLFLAG_RDTUN, 496 &hn_enable_udp4cs, 0, "Offload UDP/IPv4 checksum"); 497 498 /* 499 * Offload UDP/IPv6 checksum. 500 */ 501 static int hn_enable_udp6cs = 1; 502 SYSCTL_INT(_hw_hn, OID_AUTO, enable_udp6cs, CTLFLAG_RDTUN, 503 &hn_enable_udp6cs, 0, "Offload UDP/IPv6 checksum"); 504 505 /* Stats. */ 506 static counter_u64_t hn_udpcs_fixup; 507 SYSCTL_COUNTER_U64(_hw_hn, OID_AUTO, udpcs_fixup, CTLFLAG_RW, 508 &hn_udpcs_fixup, "# of UDP checksum fixup"); 509 510 /* 511 * See hn_set_hlen(). 512 * 513 * This value is for Azure. For Hyper-V, set this above 514 * 65536 to disable UDP datagram checksum fixup. 515 */ 516 static int hn_udpcs_fixup_mtu = 1420; 517 SYSCTL_INT(_hw_hn, OID_AUTO, udpcs_fixup_mtu, CTLFLAG_RWTUN, 518 &hn_udpcs_fixup_mtu, 0, "UDP checksum fixup MTU threshold"); 519 520 /* Limit TSO burst size */ 521 static int hn_tso_maxlen = IP_MAXPACKET; 522 SYSCTL_INT(_hw_hn, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN, 523 &hn_tso_maxlen, 0, "TSO burst limit"); 524 525 /* Limit chimney send size */ 526 static int hn_tx_chimney_size = 0; 527 SYSCTL_INT(_hw_hn, OID_AUTO, tx_chimney_size, CTLFLAG_RDTUN, 528 &hn_tx_chimney_size, 0, "Chimney send packet size limit"); 529 530 /* Limit the size of packet for direct transmission */ 531 static int hn_direct_tx_size = HN_DIRECT_TX_SIZE_DEF; 532 SYSCTL_INT(_hw_hn, OID_AUTO, direct_tx_size, CTLFLAG_RDTUN, 533 &hn_direct_tx_size, 0, "Size of the packet for direct transmission"); 534 535 /* # of LRO entries per RX ring */ 536 #if defined(INET) || defined(INET6) 537 static int hn_lro_entry_count = HN_LROENT_CNT_DEF; 538 SYSCTL_INT(_hw_hn, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN, 539 &hn_lro_entry_count, 0, "LRO entry count"); 540 #endif 541 542 static int hn_tx_taskq_cnt = 1; 543 SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_cnt, CTLFLAG_RDTUN, 544 &hn_tx_taskq_cnt, 0, "# of TX taskqueues"); 545 546 #define HN_TX_TASKQ_M_INDEP 0 547 #define HN_TX_TASKQ_M_GLOBAL 1 548 #define HN_TX_TASKQ_M_EVTTQ 2 549 550 static int hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP; 551 SYSCTL_INT(_hw_hn, OID_AUTO, tx_taskq_mode, CTLFLAG_RDTUN, 552 &hn_tx_taskq_mode, 0, "TX taskqueue modes: " 553 "0 - independent, 1 - share global tx taskqs, 2 - share event taskqs"); 554 555 #ifndef HN_USE_TXDESC_BUFRING 556 static int hn_use_txdesc_bufring = 0; 557 #else 558 static int hn_use_txdesc_bufring = 1; 559 #endif 560 SYSCTL_INT(_hw_hn, OID_AUTO, use_txdesc_bufring, CTLFLAG_RD, 561 &hn_use_txdesc_bufring, 0, "Use buf_ring for TX descriptors"); 562 563 #ifdef HN_IFSTART_SUPPORT 564 /* Use ifnet.if_start instead of ifnet.if_transmit */ 565 static int hn_use_if_start = 0; 566 SYSCTL_INT(_hw_hn, OID_AUTO, use_if_start, CTLFLAG_RDTUN, 567 &hn_use_if_start, 0, "Use if_start TX method"); 568 #endif 569 570 /* # of channels to use */ 571 static int hn_chan_cnt = 0; 572 SYSCTL_INT(_hw_hn, OID_AUTO, chan_cnt, CTLFLAG_RDTUN, 573 &hn_chan_cnt, 0, 574 "# of channels to use; each channel has one RX ring and one TX ring"); 575 576 /* # of transmit rings to use */ 577 static int hn_tx_ring_cnt = 0; 578 SYSCTL_INT(_hw_hn, OID_AUTO, tx_ring_cnt, CTLFLAG_RDTUN, 579 &hn_tx_ring_cnt, 0, "# of TX rings to use"); 580 581 /* Software TX ring deptch */ 582 static int hn_tx_swq_depth = 0; 583 SYSCTL_INT(_hw_hn, OID_AUTO, tx_swq_depth, CTLFLAG_RDTUN, 584 &hn_tx_swq_depth, 0, "Depth of IFQ or BUFRING"); 585 586 /* Enable sorted LRO, and the depth of the per-channel mbuf queue */ 587 static u_int hn_lro_mbufq_depth = 0; 588 SYSCTL_UINT(_hw_hn, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN, 589 &hn_lro_mbufq_depth, 0, "Depth of LRO mbuf queue"); 590 591 /* Packet transmission aggregation size limit */ 592 static int hn_tx_agg_size = -1; 593 SYSCTL_INT(_hw_hn, OID_AUTO, tx_agg_size, CTLFLAG_RDTUN, 594 &hn_tx_agg_size, 0, "Packet transmission aggregation size limit"); 595 596 /* Packet transmission aggregation count limit */ 597 static int hn_tx_agg_pkts = -1; 598 SYSCTL_INT(_hw_hn, OID_AUTO, tx_agg_pkts, CTLFLAG_RDTUN, 599 &hn_tx_agg_pkts, 0, "Packet transmission aggregation packet limit"); 600 601 /* VF list */ 602 SYSCTL_PROC(_hw_hn, OID_AUTO, vflist, 603 CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_NEEDGIANT, 0, 0, 604 hn_vflist_sysctl, "A", 605 "VF list"); 606 607 /* VF mapping */ 608 SYSCTL_PROC(_hw_hn, OID_AUTO, vfmap, 609 CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_NEEDGIANT, 0, 0, 610 hn_vfmap_sysctl, "A", 611 "VF mapping"); 612 613 /* Transparent VF */ 614 static int hn_xpnt_vf = 1; 615 SYSCTL_INT(_hw_hn, OID_AUTO, vf_transparent, CTLFLAG_RDTUN, 616 &hn_xpnt_vf, 0, "Transparent VF mod"); 617 618 /* Accurate BPF support for Transparent VF */ 619 static int hn_xpnt_vf_accbpf = 0; 620 SYSCTL_INT(_hw_hn, OID_AUTO, vf_xpnt_accbpf, CTLFLAG_RDTUN, 621 &hn_xpnt_vf_accbpf, 0, "Accurate BPF for transparent VF"); 622 623 /* Extra wait for transparent VF attach routing; unit seconds. */ 624 static int hn_xpnt_vf_attwait = HN_XPNT_VF_ATTWAIT_MIN; 625 SYSCTL_INT(_hw_hn, OID_AUTO, vf_xpnt_attwait, CTLFLAG_RWTUN, 626 &hn_xpnt_vf_attwait, 0, 627 "Extra wait for transparent VF attach routing; unit: seconds"); 628 629 static u_int hn_cpu_index; /* next CPU for channel */ 630 static struct taskqueue **hn_tx_taskque;/* shared TX taskqueues */ 631 632 static struct rmlock hn_vfmap_lock; 633 static int hn_vfmap_size; 634 static if_t *hn_vfmap; 635 636 static const struct hyperv_guid hn_guid = { 637 .hv_guid = { 638 0x63, 0x51, 0x61, 0xf8, 0x3e, 0xdf, 0xc5, 0x46, 639 0x91, 0x3f, 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0x0e } 640 }; 641 642 static device_method_t hn_methods[] = { 643 /* Device interface */ 644 DEVMETHOD(device_probe, hn_probe), 645 DEVMETHOD(device_attach, hn_attach), 646 DEVMETHOD(device_detach, hn_detach), 647 DEVMETHOD(device_shutdown, hn_shutdown), 648 DEVMETHOD_END 649 }; 650 651 static driver_t hn_driver = { 652 "hn", 653 hn_methods, 654 sizeof(struct hn_softc) 655 }; 656 657 DRIVER_MODULE(hn, vmbus, hn_driver, 0, 0); 658 MODULE_VERSION(hn, 1); 659 MODULE_DEPEND(hn, vmbus, 1, 1, 1); 660 661 static void 662 hn_set_lro_lenlim(struct hn_softc *sc, int lenlim) 663 { 664 int i; 665 666 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 667 sc->hn_rx_ring[i].hn_lro.lro_length_lim = lenlim; 668 } 669 670 static int 671 hn_txpkt_sglist(struct hn_tx_ring *txr, struct hn_txdesc *txd) 672 { 673 674 KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID && 675 txd->chim_size == 0, ("invalid rndis sglist txd")); 676 return (hn_nvs_send_rndis_sglist(txr->hn_chan, HN_NVS_RNDIS_MTYPE_DATA, 677 &txd->send_ctx, txr->hn_gpa, txr->hn_gpa_cnt)); 678 } 679 680 static int 681 hn_txpkt_chim(struct hn_tx_ring *txr, struct hn_txdesc *txd) 682 { 683 struct hn_nvs_rndis rndis; 684 685 KASSERT(txd->chim_index != HN_NVS_CHIM_IDX_INVALID && 686 txd->chim_size > 0, ("invalid rndis chim txd")); 687 688 rndis.nvs_type = HN_NVS_TYPE_RNDIS; 689 rndis.nvs_rndis_mtype = HN_NVS_RNDIS_MTYPE_DATA; 690 rndis.nvs_chim_idx = txd->chim_index; 691 rndis.nvs_chim_sz = txd->chim_size; 692 693 return (hn_nvs_send(txr->hn_chan, VMBUS_CHANPKT_FLAG_RC, 694 &rndis, sizeof(rndis), &txd->send_ctx)); 695 } 696 697 static __inline uint32_t 698 hn_chim_alloc(struct hn_softc *sc) 699 { 700 int i, bmap_cnt = sc->hn_chim_bmap_cnt; 701 u_long *bmap = sc->hn_chim_bmap; 702 uint32_t ret = HN_NVS_CHIM_IDX_INVALID; 703 704 for (i = 0; i < bmap_cnt; ++i) { 705 int idx; 706 707 idx = ffsl(~bmap[i]); 708 if (idx == 0) 709 continue; 710 711 --idx; /* ffsl is 1-based */ 712 KASSERT(i * LONG_BIT + idx < sc->hn_chim_cnt, 713 ("invalid i %d and idx %d", i, idx)); 714 715 if (atomic_testandset_long(&bmap[i], idx)) 716 continue; 717 718 ret = i * LONG_BIT + idx; 719 break; 720 } 721 return (ret); 722 } 723 724 static __inline void 725 hn_chim_free(struct hn_softc *sc, uint32_t chim_idx) 726 { 727 u_long mask; 728 uint32_t idx; 729 730 idx = chim_idx / LONG_BIT; 731 KASSERT(idx < sc->hn_chim_bmap_cnt, 732 ("invalid chimney index 0x%x", chim_idx)); 733 734 mask = 1UL << (chim_idx % LONG_BIT); 735 KASSERT(sc->hn_chim_bmap[idx] & mask, 736 ("index bitmap 0x%lx, chimney index %u, " 737 "bitmap idx %d, bitmask 0x%lx", 738 sc->hn_chim_bmap[idx], chim_idx, idx, mask)); 739 740 atomic_clear_long(&sc->hn_chim_bmap[idx], mask); 741 } 742 743 #if defined(INET6) || defined(INET) 744 745 #define PULLUP_HDR(m, len) \ 746 do { \ 747 if (__predict_false((m)->m_len < (len))) { \ 748 (m) = m_pullup((m), (len)); \ 749 if ((m) == NULL) \ 750 return (NULL); \ 751 } \ 752 } while (0) 753 754 /* 755 * NOTE: If this function failed, the m_head would be freed. 756 */ 757 static __inline struct mbuf * 758 hn_tso_fixup(struct mbuf *m_head) 759 { 760 struct ether_vlan_header *evl; 761 struct tcphdr *th; 762 int ehlen; 763 764 KASSERT(M_WRITABLE(m_head), ("TSO mbuf not writable")); 765 766 PULLUP_HDR(m_head, sizeof(*evl)); 767 evl = mtod(m_head, struct ether_vlan_header *); 768 if (evl->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) 769 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 770 else 771 ehlen = ETHER_HDR_LEN; 772 m_head->m_pkthdr.l2hlen = ehlen; 773 774 #ifdef INET 775 if (m_head->m_pkthdr.csum_flags & CSUM_IP_TSO) { 776 struct ip *ip; 777 int iphlen; 778 779 PULLUP_HDR(m_head, ehlen + sizeof(*ip)); 780 ip = mtodo(m_head, ehlen); 781 iphlen = ip->ip_hl << 2; 782 m_head->m_pkthdr.l3hlen = iphlen; 783 784 PULLUP_HDR(m_head, ehlen + iphlen + sizeof(*th)); 785 th = mtodo(m_head, ehlen + iphlen); 786 787 ip->ip_len = 0; 788 ip->ip_sum = 0; 789 th->th_sum = in_pseudo(ip->ip_src.s_addr, 790 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 791 } 792 #endif 793 #if defined(INET6) && defined(INET) 794 else 795 #endif 796 #ifdef INET6 797 { 798 struct ip6_hdr *ip6; 799 800 PULLUP_HDR(m_head, ehlen + sizeof(*ip6)); 801 ip6 = mtodo(m_head, ehlen); 802 if (ip6->ip6_nxt != IPPROTO_TCP) { 803 m_freem(m_head); 804 return (NULL); 805 } 806 m_head->m_pkthdr.l3hlen = sizeof(*ip6); 807 808 PULLUP_HDR(m_head, ehlen + sizeof(*ip6) + sizeof(*th)); 809 th = mtodo(m_head, ehlen + sizeof(*ip6)); 810 811 ip6->ip6_plen = 0; 812 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 813 } 814 #endif 815 return (m_head); 816 } 817 818 /* 819 * NOTE: If this function failed, the m_head would be freed. 820 */ 821 static __inline struct mbuf * 822 hn_set_hlen(struct mbuf *m_head) 823 { 824 const struct ether_vlan_header *evl; 825 int ehlen; 826 827 PULLUP_HDR(m_head, sizeof(*evl)); 828 evl = mtod(m_head, const struct ether_vlan_header *); 829 if (evl->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) 830 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 831 else 832 ehlen = ETHER_HDR_LEN; 833 m_head->m_pkthdr.l2hlen = ehlen; 834 835 #ifdef INET 836 if (m_head->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP_UDP)) { 837 const struct ip *ip; 838 int iphlen; 839 840 PULLUP_HDR(m_head, ehlen + sizeof(*ip)); 841 ip = mtodo(m_head, ehlen); 842 iphlen = ip->ip_hl << 2; 843 m_head->m_pkthdr.l3hlen = iphlen; 844 845 /* 846 * UDP checksum offload does not work in Azure, if the 847 * following conditions meet: 848 * - sizeof(IP hdr + UDP hdr + payload) > 1420. 849 * - IP_DF is not set in the IP hdr. 850 * 851 * Fallback to software checksum for these UDP datagrams. 852 */ 853 if ((m_head->m_pkthdr.csum_flags & CSUM_IP_UDP) && 854 m_head->m_pkthdr.len > hn_udpcs_fixup_mtu + ehlen && 855 (ntohs(ip->ip_off) & IP_DF) == 0) { 856 uint16_t off = ehlen + iphlen; 857 858 counter_u64_add(hn_udpcs_fixup, 1); 859 PULLUP_HDR(m_head, off + sizeof(struct udphdr)); 860 *(uint16_t *)(m_head->m_data + off + 861 m_head->m_pkthdr.csum_data) = in_cksum_skip( 862 m_head, m_head->m_pkthdr.len, off); 863 m_head->m_pkthdr.csum_flags &= ~CSUM_IP_UDP; 864 } 865 } 866 #endif 867 #if defined(INET6) && defined(INET) 868 else 869 #endif 870 #ifdef INET6 871 { 872 const struct ip6_hdr *ip6; 873 874 PULLUP_HDR(m_head, ehlen + sizeof(*ip6)); 875 ip6 = mtodo(m_head, ehlen); 876 if (ip6->ip6_nxt != IPPROTO_TCP && 877 ip6->ip6_nxt != IPPROTO_UDP) { 878 m_freem(m_head); 879 return (NULL); 880 } 881 m_head->m_pkthdr.l3hlen = sizeof(*ip6); 882 } 883 #endif 884 return (m_head); 885 } 886 887 /* 888 * NOTE: If this function failed, the m_head would be freed. 889 */ 890 static __inline struct mbuf * 891 hn_check_tcpsyn(struct mbuf *m_head, int *tcpsyn) 892 { 893 const struct tcphdr *th; 894 int ehlen, iphlen; 895 896 *tcpsyn = 0; 897 ehlen = m_head->m_pkthdr.l2hlen; 898 iphlen = m_head->m_pkthdr.l3hlen; 899 900 PULLUP_HDR(m_head, ehlen + iphlen + sizeof(*th)); 901 th = mtodo(m_head, ehlen + iphlen); 902 if (tcp_get_flags(th) & TH_SYN) 903 *tcpsyn = 1; 904 return (m_head); 905 } 906 907 #undef PULLUP_HDR 908 909 #endif /* INET6 || INET */ 910 911 static int 912 hn_set_rxfilter(struct hn_softc *sc, uint32_t filter) 913 { 914 int error = 0; 915 916 HN_LOCK_ASSERT(sc); 917 918 if (sc->hn_rx_filter != filter) { 919 error = hn_rndis_set_rxfilter(sc, filter); 920 if (!error) 921 sc->hn_rx_filter = filter; 922 } 923 return (error); 924 } 925 926 static int 927 hn_rxfilter_config(struct hn_softc *sc) 928 { 929 if_t ifp = sc->hn_ifp; 930 uint32_t filter; 931 932 HN_LOCK_ASSERT(sc); 933 934 /* 935 * If the non-transparent mode VF is activated, we don't know how 936 * its RX filter is configured, so stick the synthetic device in 937 * the promiscous mode. 938 */ 939 if ((if_getflags(ifp) & IFF_PROMISC) || (sc->hn_flags & HN_FLAG_RXVF)) { 940 filter = NDIS_PACKET_TYPE_PROMISCUOUS; 941 } else { 942 filter = NDIS_PACKET_TYPE_DIRECTED; 943 if (if_getflags(ifp) & IFF_BROADCAST) 944 filter |= NDIS_PACKET_TYPE_BROADCAST; 945 /* TODO: support multicast list */ 946 if ((if_getflags(ifp) & IFF_ALLMULTI) || 947 !if_maddr_empty(ifp)) 948 filter |= NDIS_PACKET_TYPE_ALL_MULTICAST; 949 } 950 return (hn_set_rxfilter(sc, filter)); 951 } 952 953 static void 954 hn_set_txagg(struct hn_softc *sc) 955 { 956 uint32_t size, pkts; 957 int i; 958 959 /* 960 * Setup aggregation size. 961 */ 962 if (sc->hn_agg_size < 0) 963 size = UINT32_MAX; 964 else 965 size = sc->hn_agg_size; 966 967 if (sc->hn_rndis_agg_size < size) 968 size = sc->hn_rndis_agg_size; 969 970 /* NOTE: We only aggregate packets using chimney sending buffers. */ 971 if (size > (uint32_t)sc->hn_chim_szmax) 972 size = sc->hn_chim_szmax; 973 974 if (size <= 2 * HN_PKTSIZE_MIN(sc->hn_rndis_agg_align)) { 975 /* Disable */ 976 size = 0; 977 pkts = 0; 978 goto done; 979 } 980 981 /* NOTE: Type of the per TX ring setting is 'int'. */ 982 if (size > INT_MAX) 983 size = INT_MAX; 984 985 /* 986 * Setup aggregation packet count. 987 */ 988 if (sc->hn_agg_pkts < 0) 989 pkts = UINT32_MAX; 990 else 991 pkts = sc->hn_agg_pkts; 992 993 if (sc->hn_rndis_agg_pkts < pkts) 994 pkts = sc->hn_rndis_agg_pkts; 995 996 if (pkts <= 1) { 997 /* Disable */ 998 size = 0; 999 pkts = 0; 1000 goto done; 1001 } 1002 1003 /* NOTE: Type of the per TX ring setting is 'short'. */ 1004 if (pkts > SHRT_MAX) 1005 pkts = SHRT_MAX; 1006 1007 done: 1008 /* NOTE: Type of the per TX ring setting is 'short'. */ 1009 if (sc->hn_rndis_agg_align > SHRT_MAX) { 1010 /* Disable */ 1011 size = 0; 1012 pkts = 0; 1013 } 1014 1015 if (bootverbose) { 1016 if_printf(sc->hn_ifp, "TX agg size %u, pkts %u, align %u\n", 1017 size, pkts, sc->hn_rndis_agg_align); 1018 } 1019 1020 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 1021 struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; 1022 1023 mtx_lock(&txr->hn_tx_lock); 1024 txr->hn_agg_szmax = size; 1025 txr->hn_agg_pktmax = pkts; 1026 txr->hn_agg_align = sc->hn_rndis_agg_align; 1027 mtx_unlock(&txr->hn_tx_lock); 1028 } 1029 } 1030 1031 static int 1032 hn_get_txswq_depth(const struct hn_tx_ring *txr) 1033 { 1034 1035 KASSERT(txr->hn_txdesc_cnt > 0, ("tx ring is not setup yet")); 1036 if (hn_tx_swq_depth < txr->hn_txdesc_cnt) 1037 return txr->hn_txdesc_cnt; 1038 return hn_tx_swq_depth; 1039 } 1040 1041 static int 1042 hn_rss_reconfig(struct hn_softc *sc) 1043 { 1044 int error; 1045 1046 HN_LOCK_ASSERT(sc); 1047 1048 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) 1049 return (ENXIO); 1050 1051 /* 1052 * Disable RSS first. 1053 * 1054 * NOTE: 1055 * Direct reconfiguration by setting the UNCHG flags does 1056 * _not_ work properly. 1057 */ 1058 if (bootverbose) 1059 if_printf(sc->hn_ifp, "disable RSS\n"); 1060 error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_DISABLE); 1061 if (error) { 1062 if_printf(sc->hn_ifp, "RSS disable failed\n"); 1063 return (error); 1064 } 1065 1066 /* 1067 * Reenable the RSS w/ the updated RSS key or indirect 1068 * table. 1069 */ 1070 if (bootverbose) 1071 if_printf(sc->hn_ifp, "reconfig RSS\n"); 1072 error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_NONE); 1073 if (error) { 1074 if_printf(sc->hn_ifp, "RSS reconfig failed\n"); 1075 return (error); 1076 } 1077 return (0); 1078 } 1079 1080 static void 1081 hn_rss_ind_fixup(struct hn_softc *sc) 1082 { 1083 struct ndis_rssprm_toeplitz *rss = &sc->hn_rss; 1084 int i, nchan; 1085 1086 nchan = sc->hn_rx_ring_inuse; 1087 KASSERT(nchan > 1, ("invalid # of channels %d", nchan)); 1088 1089 /* 1090 * Check indirect table to make sure that all channels in it 1091 * can be used. 1092 */ 1093 for (i = 0; i < NDIS_HASH_INDCNT; ++i) { 1094 if (rss->rss_ind[i] >= nchan) { 1095 if_printf(sc->hn_ifp, 1096 "RSS indirect table %d fixup: %u -> %d\n", 1097 i, rss->rss_ind[i], nchan - 1); 1098 rss->rss_ind[i] = nchan - 1; 1099 } 1100 } 1101 } 1102 1103 static int 1104 hn_ifmedia_upd(if_t ifp __unused) 1105 { 1106 1107 /* Ignore since autoselect is the only defined and valid media */ 1108 return (0); 1109 } 1110 1111 static void 1112 hn_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr) 1113 { 1114 struct hn_softc *sc = if_getsoftc(ifp); 1115 1116 ifmr->ifm_status = IFM_AVALID; 1117 ifmr->ifm_active = IFM_ETHER; 1118 1119 if ((sc->hn_link_flags & HN_LINK_FLAG_LINKUP) == 0) { 1120 ifmr->ifm_active |= IFM_NONE; 1121 return; 1122 } 1123 ifmr->ifm_status |= IFM_ACTIVE; 1124 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1125 } 1126 1127 static void 1128 hn_rxvf_set_task(void *xarg, int pending __unused) 1129 { 1130 struct hn_rxvf_setarg *arg = xarg; 1131 1132 arg->rxr->hn_rxvf_ifp = arg->vf_ifp; 1133 } 1134 1135 static void 1136 hn_rxvf_set(struct hn_softc *sc, if_t vf_ifp) 1137 { 1138 struct hn_rx_ring *rxr; 1139 struct hn_rxvf_setarg arg; 1140 struct task task; 1141 int i; 1142 1143 HN_LOCK_ASSERT(sc); 1144 1145 TASK_INIT(&task, 0, hn_rxvf_set_task, &arg); 1146 1147 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 1148 rxr = &sc->hn_rx_ring[i]; 1149 1150 if (i < sc->hn_rx_ring_inuse) { 1151 arg.rxr = rxr; 1152 arg.vf_ifp = vf_ifp; 1153 vmbus_chan_run_task(rxr->hn_chan, &task); 1154 } else { 1155 rxr->hn_rxvf_ifp = vf_ifp; 1156 } 1157 } 1158 } 1159 1160 static bool 1161 hn_ismyvf(const struct hn_softc *sc, const if_t ifp) 1162 { 1163 if_t hn_ifp; 1164 1165 hn_ifp = sc->hn_ifp; 1166 1167 if (ifp == hn_ifp) 1168 return (false); 1169 1170 if (if_getalloctype(ifp) != IFT_ETHER) 1171 return (false); 1172 1173 /* Ignore lagg/vlan interfaces */ 1174 if (strcmp(if_getdname(ifp), "lagg") == 0 || 1175 strcmp(if_getdname(ifp), "vlan") == 0) 1176 return (false); 1177 1178 /* 1179 * During detach events if_getifaddr(ifp) might be NULL. 1180 * Make sure the bcmp() below doesn't panic on that: 1181 */ 1182 if (if_getifaddr(ifp) == NULL || if_getifaddr(hn_ifp) == NULL) 1183 return (false); 1184 1185 if (bcmp(if_getlladdr(ifp), if_getlladdr(hn_ifp), ETHER_ADDR_LEN) != 0) 1186 return (false); 1187 1188 return (true); 1189 } 1190 1191 static void 1192 hn_rxvf_change(struct hn_softc *sc, if_t ifp) 1193 { 1194 struct rm_priotracker pt; 1195 1196 /* Address events can run on the channel that delivers completions. */ 1197 rm_rlock(&sc->hn_vf_lock, &pt); 1198 if (sc->hn_vf_ifp == ifp) 1199 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 1200 rm_runlock(&sc->hn_vf_lock, &pt); 1201 } 1202 1203 static void 1204 hn_rxvf_change_locked(struct hn_softc *sc, if_t ifp, bool rxvf) 1205 { 1206 if_t hn_ifp; 1207 u_int assoc, old_flags; 1208 int error; 1209 1210 HN_LOCK_ASSERT(sc); 1211 1212 if (!(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED)) 1213 goto out; 1214 1215 if (!hn_ismyvf(sc, ifp)) 1216 goto out; 1217 hn_ifp = sc->hn_ifp; 1218 assoc = atomic_load_acq_int(&sc->hn_vf_assoc); 1219 if (rxvf && !(assoc & HN_VF_ASSOC_ALLOCATED)) 1220 goto out; 1221 old_flags = sc->hn_flags; 1222 1223 if (rxvf) { 1224 if (sc->hn_flags & HN_FLAG_RXVF) 1225 goto out; 1226 1227 sc->hn_flags |= HN_FLAG_RXVF; 1228 hn_rxfilter_config(sc); 1229 } else { 1230 if (!(sc->hn_flags & HN_FLAG_RXVF)) 1231 goto out; 1232 1233 sc->hn_flags &= ~HN_FLAG_RXVF; 1234 if (if_getdrvflags(hn_ifp) & IFF_DRV_RUNNING) 1235 hn_rxfilter_config(sc); 1236 else 1237 hn_set_rxfilter(sc, NDIS_PACKET_TYPE_NONE); 1238 } 1239 1240 /* Prepare receive routing before the host enables the VF path. */ 1241 if (rxvf) 1242 hn_rxvf_set(sc, ifp); 1243 error = hn_nvs_set_datapath(sc, 1244 rxvf ? HN_NVS_DATAPATH_VF : HN_NVS_DATAPATH_SYNTH); 1245 if (error == 0 && rxvf && 1246 assoc != atomic_load_acq_int(&sc->hn_vf_assoc)) { 1247 hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_SYNTH); 1248 error = EAGAIN; 1249 } 1250 if (error) { 1251 sc->hn_flags = old_flags; 1252 hn_rxvf_set(sc, (old_flags & HN_FLAG_RXVF) ? ifp : NULL); 1253 if ((old_flags & HN_FLAG_RXVF) || 1254 (if_getdrvflags(hn_ifp) & IFF_DRV_RUNNING)) 1255 hn_rxfilter_config(sc); 1256 else 1257 hn_set_rxfilter(sc, NDIS_PACKET_TYPE_NONE); 1258 if (error == EAGAIN) 1259 taskqueue_enqueue_timeout(sc->hn_vf_taskq, 1260 &sc->hn_vf_init, hz); 1261 goto out; 1262 } 1263 sc->hn_vf_active_assoc = rxvf ? assoc : 0; 1264 1265 if (!rxvf) 1266 hn_rxvf_set(sc, NULL); 1267 1268 if (rxvf) { 1269 hn_vf_rss_fixup(sc, true); 1270 hn_suspend_mgmt(sc); 1271 sc->hn_link_flags &= 1272 ~(HN_LINK_FLAG_LINKUP | HN_LINK_FLAG_NETCHG); 1273 if_link_state_change(hn_ifp, LINK_STATE_DOWN); 1274 } else { 1275 hn_vf_rss_restore(sc); 1276 hn_resume_mgmt(sc); 1277 } 1278 1279 devctl_notify("HYPERV_NIC_VF", if_name(hn_ifp), 1280 rxvf ? "VF_UP" : "VF_DOWN", NULL); 1281 1282 if (bootverbose) { 1283 if_printf(hn_ifp, "datapath is switched %s %s\n", 1284 rxvf ? "to" : "from", if_name(ifp)); 1285 } 1286 out: 1287 return; 1288 } 1289 1290 static void 1291 hn_ifnet_event(void *arg, if_t ifp, int event) 1292 { 1293 1294 if (event != IFNET_EVENT_UP && event != IFNET_EVENT_DOWN) 1295 return; 1296 hn_rxvf_change(arg, ifp); 1297 } 1298 1299 static void 1300 hn_ifaddr_event(void *arg, if_t ifp) 1301 { 1302 1303 hn_rxvf_change(arg, ifp); 1304 } 1305 1306 static void 1307 hn_xpnt_vf_synccaps(struct hn_softc *sc) 1308 { 1309 if_t ifp, vf_ifp; 1310 int caps; 1311 1312 HN_LOCK_ASSERT(sc); 1313 ifp = sc->hn_ifp; 1314 vf_ifp = sc->hn_vf_ifp; 1315 caps = if_getcapabilities(ifp); 1316 1317 /* Reflect the actual VF state even if an ioctl failed partway through. */ 1318 if_setcapenable(ifp, if_getcapenable(vf_ifp) & caps); 1319 if_sethwassist(ifp, if_gethwassist(vf_ifp) & 1320 (HN_CSUM_IP_MASK | HN_CSUM_IP6_MASK | CSUM_TSO)); 1321 /* The worker refreshes VLAN children without holding hn_lock. */ 1322 if (!sc->hn_detaching) 1323 taskqueue_enqueue(sc->hn_vf_taskq, &sc->hn_vf_vlancap); 1324 } 1325 1326 static void 1327 hn_xpnt_vf_vlancap_taskfunc(void *xsc, int pending __unused) 1328 { 1329 struct hn_softc *sc = xsc; 1330 1331 /* VLAN configuration takes vlan_sx before entering hn ioctls. */ 1332 sx_assert(&sc->hn_lock, SA_UNLOCKED); 1333 if_vlancap(sc->hn_ifp); 1334 } 1335 1336 static int 1337 hn_xpnt_vf_iocsetcaps(struct hn_softc *sc, struct ifreq *ifr) 1338 { 1339 if_t vf_ifp; 1340 u_int assoc; 1341 int caps, error; 1342 1343 HN_LOCK_ASSERT(sc); 1344 if (sc->hn_vf_caps_busy) 1345 return (EBUSY); 1346 if (sc->hn_detaching || sc->hn_vf_detaching || 1347 !hn_xpnt_vf_isready(sc)) 1348 return (ENXIO); 1349 1350 vf_ifp = sc->hn_vf_ifp; 1351 if_ref(vf_ifp); 1352 assoc = sc->hn_vf_active_assoc; 1353 caps = if_getcapabilities(sc->hn_ifp); 1354 /* Leave capabilities which hn does not expose unchanged on the VF. */ 1355 ifr->ifr_reqcap = (ifr->ifr_reqcap & caps) | 1356 (if_getcapenable(vf_ifp) & ~caps); 1357 sc->hn_vf_caps_busy = true; 1358 rm_wlock(&sc->hn_vf_lock); 1359 sc->hn_xvf_flags |= HN_XVFFLAG_SWITCHING; 1360 rm_wunlock(&sc->hn_vf_lock); 1361 1362 /* 1363 * The VF ioctl may acquire vlan_sx, whose callers enter hn ioctls. 1364 * Detach waits for hn_vf_caps_busy with hn_lock released; new VF 1365 * initialization and capability changes cannot overlap this ioctl. 1366 */ 1367 HN_UNLOCK(sc); 1368 error = ifhwioctl(SIOCSIFCAP, vf_ifp, (caddr_t)ifr, curthread); 1369 HN_LOCK(sc); 1370 1371 if (!sc->hn_detaching && !sc->hn_vf_detaching && 1372 sc->hn_vf_ifp == vf_ifp && hn_xpnt_vf_isready(sc) && 1373 sc->hn_vf_active_assoc == assoc) { 1374 hn_xpnt_vf_synccaps(sc); 1375 rm_wlock(&sc->hn_vf_lock); 1376 sc->hn_xvf_flags &= ~HN_XVFFLAG_SWITCHING; 1377 /* A link event during the ioctl was suppressed by SWITCHING. */ 1378 if ((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) && 1379 hn_xpnt_vf_isready(sc)) 1380 if_link_state_change(sc->hn_ifp, 1381 if_getlinkstate(vf_ifp)); 1382 rm_wunlock(&sc->hn_vf_lock); 1383 } else if (error == 0) { 1384 error = sc->hn_detaching || sc->hn_vf_detaching ? ENXIO : EAGAIN; 1385 } 1386 if_rele(vf_ifp); 1387 sc->hn_vf_caps_busy = false; 1388 wakeup(&sc->hn_vf_caps_busy); 1389 /* Retry association work deferred while the ioctl was in flight. */ 1390 if (!sc->hn_detaching && !sc->hn_vf_detaching) 1391 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 1392 return (error); 1393 } 1394 1395 static int 1396 hn_xpnt_vf_iocsetflags(struct hn_softc *sc) 1397 { 1398 if_t vf_ifp; 1399 struct ifreq ifr; 1400 1401 HN_LOCK_ASSERT(sc); 1402 vf_ifp = sc->hn_vf_ifp; 1403 1404 memset(&ifr, 0, sizeof(ifr)); 1405 strlcpy(ifr.ifr_name, if_name(vf_ifp), sizeof(ifr.ifr_name)); 1406 ifr.ifr_flags = if_getflags(vf_ifp) & 0xffff; 1407 ifr.ifr_flagshigh = if_getflags(vf_ifp) >> 16; 1408 return (ifhwioctl(SIOCSIFFLAGS, vf_ifp, (caddr_t)&ifr, curthread)); 1409 } 1410 1411 static void 1412 hn_xpnt_vf_saveifflags(struct hn_softc *sc) 1413 { 1414 if_t ifp = sc->hn_ifp; 1415 int allmulti = 0; 1416 1417 HN_LOCK_ASSERT(sc); 1418 1419 /* XXX vlan(4) style mcast addr maintenance */ 1420 if (!if_maddr_empty(ifp)) 1421 allmulti = IFF_ALLMULTI; 1422 1423 /* Always set the VF's if_flags */ 1424 if_setflags(sc->hn_vf_ifp, if_getflags(ifp) | allmulti); 1425 } 1426 1427 static void 1428 hn_xpnt_vf_input(if_t vf_ifp, struct mbuf *m) 1429 { 1430 struct rm_priotracker pt; 1431 if_t hn_ifp = NULL; 1432 struct mbuf *mn; 1433 1434 /* 1435 * XXX racy, if hn(4) ever detached. 1436 */ 1437 rm_rlock(&hn_vfmap_lock, &pt); 1438 if (if_getindex(vf_ifp) < hn_vfmap_size) 1439 hn_ifp = hn_vfmap[if_getindex(vf_ifp)]; 1440 rm_runlock(&hn_vfmap_lock, &pt); 1441 1442 if (hn_ifp != NULL) { 1443 for (mn = m; mn != NULL; mn = mn->m_nextpkt) { 1444 /* 1445 * Allow tapping on the VF. 1446 */ 1447 ETHER_BPF_MTAP(vf_ifp, mn); 1448 1449 /* 1450 * Update VF stats. 1451 */ 1452 if ((if_getcapenable(vf_ifp) & IFCAP_HWSTATS) == 0) { 1453 if_inc_counter(vf_ifp, IFCOUNTER_IBYTES, 1454 mn->m_pkthdr.len); 1455 } 1456 /* 1457 * XXX IFCOUNTER_IMCAST 1458 * This stat updating is kinda invasive, since it 1459 * requires two checks on the mbuf: the length check 1460 * and the ethernet header check. As of this write, 1461 * all multicast packets go directly to hn(4), which 1462 * makes imcast stat updating in the VF a try in vian. 1463 */ 1464 1465 /* 1466 * Fix up rcvif and increase hn(4)'s ipackets. 1467 */ 1468 mn->m_pkthdr.rcvif = hn_ifp; 1469 if_inc_counter(hn_ifp, IFCOUNTER_IPACKETS, 1); 1470 } 1471 /* 1472 * Go through hn(4)'s if_input. 1473 */ 1474 if_input(hn_ifp, m); 1475 } else { 1476 /* 1477 * In the middle of the transition; free this 1478 * mbuf chain. 1479 */ 1480 while (m != NULL) { 1481 mn = m->m_nextpkt; 1482 m->m_nextpkt = NULL; 1483 m_freem(m); 1484 m = mn; 1485 } 1486 } 1487 } 1488 1489 static void 1490 hn_mtu_change_fixup(struct hn_softc *sc) 1491 { 1492 if_t ifp; 1493 1494 HN_LOCK_ASSERT(sc); 1495 ifp = sc->hn_ifp; 1496 1497 hn_set_tso_maxsize(sc, hn_tso_maxlen, if_getmtu(ifp)); 1498 if (sc->hn_rx_ring[0].hn_lro.lro_length_lim < HN_LRO_LENLIM_MIN(ifp)) 1499 hn_set_lro_lenlim(sc, HN_LRO_LENLIM_MIN(ifp)); 1500 } 1501 1502 static uint32_t 1503 hn_rss_type_fromndis(uint32_t rss_hash) 1504 { 1505 uint32_t types = 0; 1506 1507 if (rss_hash & NDIS_HASH_IPV4) 1508 types |= RSS_TYPE_IPV4; 1509 if (rss_hash & NDIS_HASH_TCP_IPV4) 1510 types |= RSS_TYPE_TCP_IPV4; 1511 if (rss_hash & NDIS_HASH_IPV6) 1512 types |= RSS_TYPE_IPV6; 1513 if (rss_hash & NDIS_HASH_IPV6_EX) 1514 types |= RSS_TYPE_IPV6_EX; 1515 if (rss_hash & NDIS_HASH_TCP_IPV6) 1516 types |= RSS_TYPE_TCP_IPV6; 1517 if (rss_hash & NDIS_HASH_TCP_IPV6_EX) 1518 types |= RSS_TYPE_TCP_IPV6_EX; 1519 if (rss_hash & NDIS_HASH_UDP_IPV4_X) 1520 types |= RSS_TYPE_UDP_IPV4; 1521 return (types); 1522 } 1523 1524 static uint32_t 1525 hn_rss_type_tondis(uint32_t types) 1526 { 1527 uint32_t rss_hash = 0; 1528 1529 KASSERT((types & (RSS_TYPE_UDP_IPV6 | RSS_TYPE_UDP_IPV6_EX)) == 0, 1530 ("UDP6 and UDP6EX are not supported")); 1531 1532 if (types & RSS_TYPE_IPV4) 1533 rss_hash |= NDIS_HASH_IPV4; 1534 if (types & RSS_TYPE_TCP_IPV4) 1535 rss_hash |= NDIS_HASH_TCP_IPV4; 1536 if (types & RSS_TYPE_IPV6) 1537 rss_hash |= NDIS_HASH_IPV6; 1538 if (types & RSS_TYPE_IPV6_EX) 1539 rss_hash |= NDIS_HASH_IPV6_EX; 1540 if (types & RSS_TYPE_TCP_IPV6) 1541 rss_hash |= NDIS_HASH_TCP_IPV6; 1542 if (types & RSS_TYPE_TCP_IPV6_EX) 1543 rss_hash |= NDIS_HASH_TCP_IPV6_EX; 1544 if (types & RSS_TYPE_UDP_IPV4) 1545 rss_hash |= NDIS_HASH_UDP_IPV4_X; 1546 return (rss_hash); 1547 } 1548 1549 static void 1550 hn_rss_mbuf_hash(struct hn_softc *sc, uint32_t mbuf_hash) 1551 { 1552 int i; 1553 1554 HN_LOCK_ASSERT(sc); 1555 1556 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 1557 sc->hn_rx_ring[i].hn_mbuf_hash = mbuf_hash; 1558 } 1559 1560 static void 1561 hn_vf_rss_fixup(struct hn_softc *sc, bool reconf) 1562 { 1563 if_t ifp, vf_ifp; 1564 struct ifrsshash ifrh; 1565 struct ifrsskey ifrk; 1566 int error; 1567 uint32_t my_types, diff_types, mbuf_types = 0; 1568 1569 HN_LOCK_ASSERT(sc); 1570 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 1571 ("%s: synthetic parts are not attached", if_name(sc->hn_ifp))); 1572 1573 if (sc->hn_rx_ring_inuse == 1) { 1574 /* No RSS on synthetic parts; done. */ 1575 return; 1576 } 1577 if ((sc->hn_rss_hcap & NDIS_HASH_FUNCTION_TOEPLITZ) == 0) { 1578 /* Synthetic parts do not support Toeplitz; done. */ 1579 return; 1580 } 1581 1582 ifp = sc->hn_ifp; 1583 vf_ifp = sc->hn_vf_ifp; 1584 1585 /* 1586 * Extract VF's RSS key. Only 40 bytes key for Toeplitz is 1587 * supported. 1588 */ 1589 memset(&ifrk, 0, sizeof(ifrk)); 1590 strlcpy(ifrk.ifrk_name, if_name(vf_ifp), sizeof(ifrk.ifrk_name)); 1591 error = ifhwioctl(SIOCGIFRSSKEY, vf_ifp, (caddr_t)&ifrk, curthread); 1592 if (error) { 1593 if_printf(ifp, "%s SIOCGIFRSSKEY failed: %d\n", 1594 if_name(vf_ifp), error); 1595 goto done; 1596 } 1597 if (ifrk.ifrk_func != RSS_FUNC_TOEPLITZ) { 1598 if_printf(ifp, "%s RSS function %u is not Toeplitz\n", 1599 if_name(vf_ifp), ifrk.ifrk_func); 1600 goto done; 1601 } 1602 if (ifrk.ifrk_keylen != NDIS_HASH_KEYSIZE_TOEPLITZ) { 1603 if_printf(ifp, "%s invalid RSS Toeplitz key length %d\n", 1604 if_name(vf_ifp), ifrk.ifrk_keylen); 1605 goto done; 1606 } 1607 1608 /* 1609 * Extract VF's RSS hash. Only Toeplitz is supported. 1610 */ 1611 memset(&ifrh, 0, sizeof(ifrh)); 1612 strlcpy(ifrh.ifrh_name, if_name(vf_ifp), sizeof(ifrh.ifrh_name)); 1613 error = ifhwioctl(SIOCGIFRSSHASH, vf_ifp, (caddr_t)&ifrh, curthread); 1614 if (error) { 1615 if_printf(ifp, "%s SIOCGRSSHASH failed: %d\n", 1616 if_name(vf_ifp), error); 1617 goto done; 1618 } 1619 if (ifrh.ifrh_func != RSS_FUNC_TOEPLITZ) { 1620 if_printf(ifp, "%s RSS function %u is not Toeplitz\n", 1621 if_name(vf_ifp), ifrh.ifrh_func); 1622 goto done; 1623 } 1624 1625 my_types = hn_rss_type_fromndis(sc->hn_rss_hcap); 1626 if ((ifrh.ifrh_types & my_types) == 0) { 1627 /* This disables RSS; ignore it then */ 1628 if_printf(ifp, "%s intersection of RSS types failed. " 1629 "VF %#x, mine %#x\n", if_name(vf_ifp), 1630 ifrh.ifrh_types, my_types); 1631 goto done; 1632 } 1633 1634 diff_types = my_types ^ ifrh.ifrh_types; 1635 my_types &= ifrh.ifrh_types; 1636 mbuf_types = my_types; 1637 1638 /* 1639 * Detect RSS hash value/type confliction. 1640 * 1641 * NOTE: 1642 * We don't disable the hash type, but stop delivery the hash 1643 * value/type through mbufs on RX path. 1644 * 1645 * XXX If HN_CAP_UDPHASH is set in hn_caps, then UDP 4-tuple 1646 * hash is delivered with type of TCP_IPV4. This means if 1647 * UDP_IPV4 is enabled, then TCP_IPV4 should be forced, at 1648 * least to hn_mbuf_hash. However, given that _all_ of the 1649 * NICs implement TCP_IPV4, this will _not_ impose any issues 1650 * here. 1651 */ 1652 if ((my_types & RSS_TYPE_IPV4) && 1653 (diff_types & ifrh.ifrh_types & 1654 (RSS_TYPE_TCP_IPV4 | RSS_TYPE_UDP_IPV4))) { 1655 /* Conflict; disable IPV4 hash type/value delivery. */ 1656 if_printf(ifp, "disable IPV4 mbuf hash delivery\n"); 1657 mbuf_types &= ~RSS_TYPE_IPV4; 1658 } 1659 if ((my_types & RSS_TYPE_IPV6) && 1660 (diff_types & ifrh.ifrh_types & 1661 (RSS_TYPE_TCP_IPV6 | RSS_TYPE_UDP_IPV6 | 1662 RSS_TYPE_TCP_IPV6_EX | RSS_TYPE_UDP_IPV6_EX | 1663 RSS_TYPE_IPV6_EX))) { 1664 /* Conflict; disable IPV6 hash type/value delivery. */ 1665 if_printf(ifp, "disable IPV6 mbuf hash delivery\n"); 1666 mbuf_types &= ~RSS_TYPE_IPV6; 1667 } 1668 if ((my_types & RSS_TYPE_IPV6_EX) && 1669 (diff_types & ifrh.ifrh_types & 1670 (RSS_TYPE_TCP_IPV6 | RSS_TYPE_UDP_IPV6 | 1671 RSS_TYPE_TCP_IPV6_EX | RSS_TYPE_UDP_IPV6_EX | 1672 RSS_TYPE_IPV6))) { 1673 /* Conflict; disable IPV6_EX hash type/value delivery. */ 1674 if_printf(ifp, "disable IPV6_EX mbuf hash delivery\n"); 1675 mbuf_types &= ~RSS_TYPE_IPV6_EX; 1676 } 1677 if ((my_types & RSS_TYPE_TCP_IPV6) && 1678 (diff_types & ifrh.ifrh_types & RSS_TYPE_TCP_IPV6_EX)) { 1679 /* Conflict; disable TCP_IPV6 hash type/value delivery. */ 1680 if_printf(ifp, "disable TCP_IPV6 mbuf hash delivery\n"); 1681 mbuf_types &= ~RSS_TYPE_TCP_IPV6; 1682 } 1683 if ((my_types & RSS_TYPE_TCP_IPV6_EX) && 1684 (diff_types & ifrh.ifrh_types & RSS_TYPE_TCP_IPV6)) { 1685 /* Conflict; disable TCP_IPV6_EX hash type/value delivery. */ 1686 if_printf(ifp, "disable TCP_IPV6_EX mbuf hash delivery\n"); 1687 mbuf_types &= ~RSS_TYPE_TCP_IPV6_EX; 1688 } 1689 if ((my_types & RSS_TYPE_UDP_IPV6) && 1690 (diff_types & ifrh.ifrh_types & RSS_TYPE_UDP_IPV6_EX)) { 1691 /* Conflict; disable UDP_IPV6 hash type/value delivery. */ 1692 if_printf(ifp, "disable UDP_IPV6 mbuf hash delivery\n"); 1693 mbuf_types &= ~RSS_TYPE_UDP_IPV6; 1694 } 1695 if ((my_types & RSS_TYPE_UDP_IPV6_EX) && 1696 (diff_types & ifrh.ifrh_types & RSS_TYPE_UDP_IPV6)) { 1697 /* Conflict; disable UDP_IPV6_EX hash type/value delivery. */ 1698 if_printf(ifp, "disable UDP_IPV6_EX mbuf hash delivery\n"); 1699 mbuf_types &= ~RSS_TYPE_UDP_IPV6_EX; 1700 } 1701 1702 /* 1703 * Indirect table does not matter. 1704 */ 1705 1706 sc->hn_rss_hash = (sc->hn_rss_hcap & NDIS_HASH_FUNCTION_MASK) | 1707 hn_rss_type_tondis(my_types); 1708 memcpy(sc->hn_rss.rss_key, ifrk.ifrk_key, sizeof(sc->hn_rss.rss_key)); 1709 sc->hn_flags |= HN_FLAG_HAS_RSSKEY; 1710 1711 if (reconf) { 1712 error = hn_rss_reconfig(sc); 1713 if (error) { 1714 /* XXX roll-back? */ 1715 if_printf(ifp, "hn_rss_reconfig failed: %d\n", error); 1716 /* XXX keep going. */ 1717 } 1718 } 1719 done: 1720 /* Hash deliverability for mbufs. */ 1721 hn_rss_mbuf_hash(sc, hn_rss_type_tondis(mbuf_types)); 1722 } 1723 1724 static void 1725 hn_vf_rss_restore(struct hn_softc *sc) 1726 { 1727 1728 HN_LOCK_ASSERT(sc); 1729 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 1730 ("%s: synthetic parts are not attached", if_name(sc->hn_ifp))); 1731 1732 if (sc->hn_rx_ring_inuse == 1) 1733 goto done; 1734 1735 /* 1736 * Restore hash types. Key does _not_ matter. 1737 */ 1738 if (sc->hn_rss_hash != sc->hn_rss_hcap) { 1739 int error; 1740 1741 sc->hn_rss_hash = sc->hn_rss_hcap; 1742 error = hn_rss_reconfig(sc); 1743 if (error) { 1744 if_printf(sc->hn_ifp, "hn_rss_reconfig failed: %d\n", 1745 error); 1746 /* XXX keep going. */ 1747 } 1748 } 1749 done: 1750 /* Hash deliverability for mbufs. */ 1751 hn_rss_mbuf_hash(sc, NDIS_HASH_ALL); 1752 } 1753 1754 static void 1755 hn_xpnt_vf_setready(struct hn_softc *sc) 1756 { 1757 if_t ifp, vf_ifp; 1758 struct ifreq ifr; 1759 1760 HN_LOCK_ASSERT(sc); 1761 ifp = sc->hn_ifp; 1762 vf_ifp = sc->hn_vf_ifp; 1763 1764 /* 1765 * Mark the VF ready. 1766 */ 1767 sc->hn_vf_ready = true; 1768 1769 /* 1770 * Save information for restoration. 1771 */ 1772 sc->hn_saved_caps = if_getcapabilities(ifp); 1773 sc->hn_saved_tsomax = if_gethwtsomax(ifp); 1774 sc->hn_saved_tsosegcnt = if_gethwtsomaxsegcount(ifp); 1775 sc->hn_saved_tsosegsz = if_gethwtsomaxsegsize(ifp); 1776 sc->hn_saved_capenable = if_getcapenable(ifp); 1777 sc->hn_saved_hwassist = if_gethwassist(ifp); 1778 1779 /* 1780 * Expose only capabilities supported by the transparent packet path. 1781 * VF services such as send tags and the extended capability ioctl 1782 * require methods which hn does not implement. 1783 */ 1784 if_setcapabilities(ifp, if_getcapabilities(vf_ifp) & HN_XPNT_VF_CAPS); 1785 1786 /* 1787 * Fix TSO settings. 1788 */ 1789 if (if_gethwtsomax(ifp) > if_gethwtsomax(vf_ifp)) 1790 if_sethwtsomax(ifp, if_gethwtsomax(vf_ifp)); 1791 if (if_gethwtsomaxsegcount(ifp) > if_gethwtsomaxsegcount(vf_ifp)) 1792 if_sethwtsomaxsegcount(ifp, if_gethwtsomaxsegcount(vf_ifp)); 1793 if (if_gethwtsomaxsegsize(ifp) > if_gethwtsomaxsegsize(vf_ifp)) 1794 if_sethwtsomaxsegsize(ifp, if_gethwtsomaxsegsize(vf_ifp)); 1795 1796 /* 1797 * Adopt the VF's enabled capabilities without changing its settings. 1798 */ 1799 hn_xpnt_vf_synccaps(sc); 1800 1801 if (if_getmtu(ifp) != ETHERMTU) { 1802 int error; 1803 1804 /* 1805 * Change VF's MTU. 1806 */ 1807 memset(&ifr, 0, sizeof(ifr)); 1808 strlcpy(ifr.ifr_name, if_name(vf_ifp), sizeof(ifr.ifr_name)); 1809 ifr.ifr_mtu = if_getmtu(ifp); 1810 error = ifhwioctl(SIOCSIFMTU, vf_ifp, (caddr_t)&ifr, curthread); 1811 if (error) { 1812 if_printf(ifp, "%s SIOCSIFMTU %u failed\n", 1813 if_name(vf_ifp), if_getmtu(ifp)); 1814 if (if_getmtu(ifp) > ETHERMTU) { 1815 if_printf(ifp, "change MTU to %d\n", ETHERMTU); 1816 1817 /* 1818 * XXX 1819 * No need to adjust the synthetic parts' MTU; 1820 * failure of the adjustment will cause us 1821 * infinite headache. 1822 */ 1823 if_setmtu(ifp, ETHERMTU); 1824 hn_mtu_change_fixup(sc); 1825 } 1826 } 1827 } 1828 } 1829 1830 static bool 1831 hn_xpnt_vf_isready(struct hn_softc *sc) 1832 { 1833 u_int assoc; 1834 1835 HN_LOCK_ASSERT(sc); 1836 1837 assoc = atomic_load_acq_int(&sc->hn_vf_assoc); 1838 return (hn_xpnt_vf && sc->hn_vf_ifp != NULL && sc->hn_vf_ready && 1839 (assoc & HN_VF_ASSOC_ALLOCATED) && 1840 sc->hn_vf_active_assoc == assoc); 1841 } 1842 1843 static bool 1844 hn_xpnt_vf_caninit(struct hn_softc *sc) 1845 { 1846 1847 HN_LOCK_ASSERT(sc); 1848 return (hn_xpnt_vf && sc->hn_vf_ifp != NULL && 1849 !sc->hn_detaching && !sc->hn_vf_detaching && 1850 !sc->hn_vf_caps_busy && 1851 (atomic_load_acq_int(&sc->hn_vf_assoc) & HN_VF_ASSOC_ALLOCATED) && 1852 (int)(ticks - sc->hn_vf_rdytick) >= 0); 1853 } 1854 1855 static void 1856 hn_xpnt_vf_restore(struct hn_softc *sc) 1857 { 1858 if_t ifp = sc->hn_ifp; 1859 1860 HN_LOCK_ASSERT(sc); 1861 if (!sc->hn_vf_ready) 1862 return; 1863 if_setcapabilities(ifp, sc->hn_saved_caps); 1864 if_sethwtsomax(ifp, sc->hn_saved_tsomax); 1865 if_sethwtsomaxsegcount(ifp, sc->hn_saved_tsosegcnt); 1866 if_sethwtsomaxsegsize(ifp, sc->hn_saved_tsosegsz); 1867 if_setcapenable(ifp, sc->hn_saved_capenable); 1868 if_sethwassist(ifp, sc->hn_saved_hwassist); 1869 if (!sc->hn_detaching) 1870 taskqueue_enqueue(sc->hn_vf_taskq, &sc->hn_vf_vlancap); 1871 sc->hn_vf_ready = false; 1872 } 1873 1874 static void 1875 hn_xpnt_vf_deactivate(struct hn_softc *sc) 1876 { 1877 1878 HN_LOCK_ASSERT(sc); 1879 /* Do not send VF-formatted packets through synthetic during handoff. */ 1880 rm_wlock(&sc->hn_vf_lock); 1881 sc->hn_xvf_flags |= HN_XVFFLAG_SWITCHING; 1882 rm_wunlock(&sc->hn_vf_lock); 1883 hn_xpnt_vf_setdisable(sc, false); 1884 if (sc->hn_vf_active_assoc != 0) { 1885 /* Close local VF transmit even if returning to synthetic fails. */ 1886 hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_SYNTH); 1887 sc->hn_vf_active_assoc = 0; 1888 hn_vf_rss_restore(sc); 1889 } 1890 hn_xpnt_vf_restore(sc); 1891 rm_wlock(&sc->hn_vf_lock); 1892 sc->hn_xvf_flags &= ~HN_XVFFLAG_SWITCHING; 1893 rm_wunlock(&sc->hn_vf_lock); 1894 hn_resume_mgmt(sc); 1895 } 1896 1897 static bool 1898 hn_xpnt_vf_setenable(struct hn_softc *sc) 1899 { 1900 int i; 1901 1902 HN_LOCK_ASSERT(sc); 1903 1904 /* NOTE: hn_vf_lock for hn_transmit()/hn_qflush() */ 1905 rm_wlock(&sc->hn_vf_lock); 1906 if (sc->hn_vf_active_assoc != atomic_load_acq_int(&sc->hn_vf_assoc)) { 1907 rm_wunlock(&sc->hn_vf_lock); 1908 return (false); 1909 } 1910 sc->hn_xvf_flags |= HN_XVFFLAG_ENABLED; 1911 sc->hn_xvf_flags &= ~HN_XVFFLAG_SWITCHING; 1912 rm_wunlock(&sc->hn_vf_lock); 1913 1914 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 1915 sc->hn_rx_ring[i].hn_rx_flags |= HN_RX_FLAG_XPNT_VF; 1916 return (true); 1917 } 1918 1919 static void 1920 hn_xpnt_vf_setdisable(struct hn_softc *sc, bool clear_vf) 1921 { 1922 int i; 1923 1924 HN_LOCK_ASSERT(sc); 1925 1926 /* NOTE: hn_vf_lock for hn_transmit()/hn_qflush() */ 1927 rm_wlock(&sc->hn_vf_lock); 1928 sc->hn_xvf_flags &= ~HN_XVFFLAG_ENABLED; 1929 if (clear_vf) { 1930 sc->hn_xvf_flags &= ~HN_XVFFLAG_SWITCHING; 1931 sc->hn_vf_ifp = NULL; 1932 } 1933 rm_wunlock(&sc->hn_vf_lock); 1934 1935 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 1936 sc->hn_rx_ring[i].hn_rx_flags &= ~HN_RX_FLAG_XPNT_VF; 1937 } 1938 1939 /* 1940 * Do not configure the VF from the VLAN event callback. The worker reads 1941 * and applies the current VLAN topology outside the VLAN configuration 1942 * lock, including VLANs configured before the VF arrives. Only 1943 * transparent mode subscribes. 1944 */ 1945 static void 1946 hn_vlan_event(void *xsc, if_t ifp, uint16_t vid __unused) 1947 { 1948 struct hn_softc *sc = xsc; 1949 1950 if (ifp != sc->hn_ifp) 1951 return; 1952 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 1953 } 1954 1955 /* Apply guest VLAN intent, not the host's administrative access VLAN. */ 1956 static void 1957 hn_xpnt_vf_sync_vlans(struct hn_softc *sc, bool remove) 1958 { 1959 struct epoch_tracker et; 1960 u_int desired[HN_VLAN_WORDS] = { 0 }, changed, mask; 1961 unsigned int i, bit; 1962 uint16_t vid; 1963 1964 HN_LOCK_ASSERT(sc); 1965 KASSERT(sc->hn_vf_ifp != NULL, ("VLAN sync without a VF")); 1966 if (!remove) { 1967 NET_EPOCH_ENTER(et); 1968 for (vid = 1; vid < EVL_VLID_MASK; vid++) { 1969 if (VLAN_DEVAT(sc->hn_ifp, vid) != NULL) 1970 desired[vid / 32] |= 1U << (vid % 32); 1971 } 1972 NET_EPOCH_EXIT(et); 1973 } 1974 /* VF callbacks may sleep; never invoke them inside network epoch. */ 1975 for (i = 0; i < HN_VLAN_WORDS; i++) { 1976 changed = desired[i] ^ sc->hn_vf_vlans[i]; 1977 while (changed != 0) { 1978 bit = ffs(changed) - 1; 1979 mask = 1U << bit; 1980 vid = i * 32 + bit; 1981 if ((desired[i] & mask) != 0) 1982 EVENTHANDLER_INVOKE(vlan_config, sc->hn_vf_ifp, 1983 vid); 1984 else 1985 EVENTHANDLER_INVOKE(vlan_unconfig, sc->hn_vf_ifp, 1986 vid); 1987 changed &= ~mask; 1988 } 1989 sc->hn_vf_vlans[i] = desired[i]; 1990 } 1991 } 1992 1993 static void 1994 hn_xpnt_vf_init(struct hn_softc *sc) 1995 { 1996 int error; 1997 u_int assoc; 1998 1999 HN_LOCK_ASSERT(sc); 2000 2001 KASSERT((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) == 0, 2002 ("%s: transparent VF was enabled", if_name(sc->hn_ifp))); 2003 if (!hn_xpnt_vf_caninit(sc)) 2004 return; 2005 hn_xpnt_vf_sync_vlans(sc, false); 2006 assoc = atomic_load_acq_int(&sc->hn_vf_assoc); 2007 rm_wlock(&sc->hn_vf_lock); 2008 sc->hn_xvf_flags |= HN_XVFFLAG_SWITCHING; 2009 rm_wunlock(&sc->hn_vf_lock); 2010 if (!sc->hn_vf_ready) 2011 hn_xpnt_vf_setready(sc); 2012 2013 if (bootverbose) { 2014 if_printf(sc->hn_ifp, "try bringing up %s\n", 2015 if_name(sc->hn_vf_ifp)); 2016 } 2017 2018 /* 2019 * Bring the VF up. 2020 */ 2021 hn_xpnt_vf_saveifflags(sc); 2022 if_setflagbits(sc->hn_ifp, IFF_UP, 0); 2023 error = hn_xpnt_vf_iocsetflags(sc); 2024 if (error) { 2025 if_printf(sc->hn_ifp, "bringing up %s failed: %d\n", 2026 if_name(sc->hn_vf_ifp), error); 2027 goto failed; 2028 } 2029 2030 /* 2031 * Some VF drivers initialize hwassist only when brought up. Refresh 2032 * the offload state before allowing transmit through the VF. 2033 */ 2034 hn_xpnt_vf_synccaps(sc); 2035 2036 /* 2037 * NOTE: 2038 * Datapath setting must happen _after_ bringing the VF up. 2039 */ 2040 error = hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_VF); 2041 if (error) 2042 goto failed; 2043 sc->hn_vf_active_assoc = assoc; 2044 hn_suspend_mgmt(sc); 2045 2046 /* 2047 * NOTE: 2048 * Fixup RSS related bits _after_ the VF is brought up, since 2049 * many VFs generate RSS key during it's initialization. 2050 */ 2051 hn_vf_rss_fixup(sc, true); 2052 2053 /* Mark transparent mode VF as enabled. */ 2054 if (!hn_xpnt_vf_setenable(sc)) { 2055 error = EAGAIN; 2056 goto failed; 2057 } 2058 if_link_state_change(sc->hn_ifp, if_getlinkstate(sc->hn_vf_ifp)); 2059 return; 2060 2061 failed: 2062 hn_xpnt_vf_deactivate(sc); 2063 if (error == EAGAIN) 2064 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, hz); 2065 } 2066 2067 static void 2068 hn_xpnt_vf_init_taskfunc(void *xsc, int pending __unused) 2069 { 2070 struct hn_softc *sc = xsc; 2071 2072 HN_LOCK(sc); 2073 if (sc->hn_detaching) 2074 goto done; 2075 2076 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) 2077 goto done; 2078 if (sc->hn_vf_ifp == NULL || sc->hn_vf_detaching || 2079 sc->hn_vf_caps_busy) 2080 goto done; 2081 if (!hn_xpnt_vf) { 2082 if ((sc->hn_flags & HN_FLAG_RXVF) && sc->hn_vf_active_assoc != 2083 atomic_load_acq_int(&sc->hn_vf_assoc)) 2084 hn_rxvf_change_locked(sc, sc->hn_vf_ifp, false); 2085 hn_rxvf_change_locked(sc, sc->hn_vf_ifp, 2086 (if_getflags(sc->hn_vf_ifp) & IFF_UP) != 0); 2087 goto rss; 2088 } 2089 if (sc->hn_vf_active_assoc != 0 && sc->hn_vf_active_assoc != 2090 atomic_load_acq_int(&sc->hn_vf_assoc)) 2091 hn_xpnt_vf_deactivate(sc); 2092 if ((int)(ticks - sc->hn_vf_rdytick) < 0) { 2093 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 2094 sc->hn_vf_rdytick - ticks); 2095 goto done; 2096 } 2097 hn_xpnt_vf_sync_vlans(sc, false); 2098 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 2099 goto rss; 2100 2101 if (if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) { 2102 /* 2103 * Delayed VF initialization. 2104 */ 2105 if (bootverbose) { 2106 if_printf(sc->hn_ifp, "delayed initialize %s\n", 2107 if_name(sc->hn_vf_ifp)); 2108 } 2109 hn_xpnt_vf_init(sc); 2110 } 2111 rss: 2112 /* 2113 * A link-up event can follow recovery from a failed handoff RSS query. 2114 * An inactive path can consume this request: its next handoff queries 2115 * RSS again. This is a one-shot refresh, not a readiness poll. 2116 */ 2117 if (atomic_readandclear_int(&sc->hn_vf_rss_refresh) != 0 && 2118 ((sc->hn_flags & HN_FLAG_RXVF) || 2119 (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) && 2120 (sc->hn_vf_active_assoc & HN_VF_ASSOC_ALLOCATED) != 0 && 2121 sc->hn_vf_active_assoc == atomic_load_acq_int(&sc->hn_vf_assoc) && 2122 (if_getflags(sc->hn_vf_ifp) & IFF_UP) != 0 && 2123 if_getlinkstate(sc->hn_vf_ifp) == LINK_STATE_UP) 2124 hn_vf_rss_fixup(sc, true); 2125 done: 2126 HN_UNLOCK(sc); 2127 } 2128 2129 static void 2130 hn_ifnet_attevent(void *xsc, if_t ifp) 2131 { 2132 struct hn_softc *sc = xsc; 2133 2134 HN_LOCK(sc); 2135 2136 if (!(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED)) 2137 goto done; 2138 2139 if (!hn_ismyvf(sc, ifp)) 2140 goto done; 2141 2142 if (sc->hn_vf_ifp != NULL) { 2143 if_printf(sc->hn_ifp, "%s was attached as VF\n", 2144 if_name(sc->hn_vf_ifp)); 2145 goto done; 2146 } 2147 2148 if (hn_xpnt_vf && if_getstartfn(ifp) != NULL) { 2149 /* 2150 * ifnet.if_start is _not_ supported by transparent 2151 * mode VF; mainly due to the IFF_DRV_OACTIVE flag. 2152 */ 2153 if_printf(sc->hn_ifp, "%s uses if_start, which is unsupported " 2154 "in transparent VF mode.\n", if_name(sc->hn_vf_ifp)); 2155 2156 goto done; 2157 } 2158 2159 rm_wlock(&hn_vfmap_lock); 2160 2161 if (if_getindex(ifp) >= hn_vfmap_size) { 2162 if_t *newmap; 2163 int newsize; 2164 2165 newsize = if_getindex(ifp) + HN_VFMAP_SIZE_DEF; 2166 newmap = malloc(sizeof(if_t) * newsize, M_DEVBUF, 2167 M_WAITOK | M_ZERO); 2168 2169 memcpy(newmap, hn_vfmap, 2170 sizeof(if_t) * hn_vfmap_size); 2171 free(hn_vfmap, M_DEVBUF); 2172 hn_vfmap = newmap; 2173 hn_vfmap_size = newsize; 2174 } 2175 KASSERT(hn_vfmap[if_getindex(ifp)] == NULL, 2176 ("%s: ifindex %d was mapped to %s", 2177 if_name(ifp), if_getindex(ifp), if_name(hn_vfmap[if_getindex(ifp)]))); 2178 hn_vfmap[if_getindex(ifp)] = sc->hn_ifp; 2179 2180 rm_wunlock(&hn_vfmap_lock); 2181 2182 /* NOTE: hn_vf_lock for hn_transmit()/hn_qflush() */ 2183 rm_wlock(&sc->hn_vf_lock); 2184 KASSERT((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) == 0, 2185 ("%s: transparent VF was enabled", if_name(sc->hn_ifp))); 2186 sc->hn_vf_ifp = ifp; 2187 rm_wunlock(&sc->hn_vf_lock); 2188 sc->hn_vf_detaching = false; 2189 2190 if (hn_xpnt_vf) { 2191 int wait_ticks; 2192 2193 /* 2194 * Install if_input for vf_ifp, which does vf_ifp -> hn_ifp. 2195 * Save vf_ifp's current if_input for later restoration. 2196 */ 2197 sc->hn_vf_input = if_getinputfn(ifp); 2198 if_setinputfn(ifp, hn_xpnt_vf_input); 2199 2200 /* 2201 * Give VF sometime to complete its attach routing. 2202 */ 2203 wait_ticks = hn_xpnt_vf_attwait * hz; 2204 sc->hn_vf_rdytick = ticks + wait_ticks; 2205 2206 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 2207 wait_ticks); 2208 } else { 2209 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 2210 } 2211 done: 2212 HN_UNLOCK(sc); 2213 } 2214 2215 static void 2216 hn_ifnet_detevent(void *xsc, if_t ifp) 2217 { 2218 struct hn_softc *sc = xsc; 2219 2220 HN_LOCK(sc); 2221 2222 if (sc->hn_vf_ifp == NULL) 2223 goto done; 2224 2225 if (!hn_ismyvf(sc, ifp)) 2226 goto done; 2227 2228 if (hn_xpnt_vf) { 2229 sc->hn_vf_detaching = true; 2230 while (sc->hn_vf_caps_busy) 2231 sx_sleep(&sc->hn_vf_caps_busy, &sc->hn_lock, 0, 2232 "hnvfcap", 0); 2233 /* 2234 * Make sure that the delayed initialization is not running. 2235 * 2236 * NOTE: 2237 * - This lock _must_ be released, since the hn_vf_init task 2238 * will try holding this lock. 2239 * - It is safe to release this lock here, since the 2240 * hn_ifnet_attevent() is interlocked by the hn_vf_ifp. 2241 * 2242 * XXX racy, if hn(4) ever detached. 2243 */ 2244 HN_UNLOCK(sc); 2245 taskqueue_drain_timeout(sc->hn_vf_taskq, &sc->hn_vf_init); 2246 HN_LOCK(sc); 2247 2248 KASSERT(sc->hn_vf_input != NULL, ("%s VF input is not saved", 2249 if_name(sc->hn_ifp))); 2250 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) 2251 hn_xpnt_vf_deactivate(sc); 2252 else 2253 hn_xpnt_vf_restore(sc); 2254 /* A departing VF discards its registration state itself. */ 2255 bzero(sc->hn_vf_vlans, sizeof(sc->hn_vf_vlans)); 2256 if_setinputfn(ifp, sc->hn_vf_input); 2257 sc->hn_vf_input = NULL; 2258 } else if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) { 2259 hn_rxvf_change_locked(sc, ifp, false); 2260 /* Never leave receive routing pointing at a departing VF. */ 2261 if (sc->hn_flags & HN_FLAG_RXVF) { 2262 sc->hn_flags &= ~HN_FLAG_RXVF; 2263 sc->hn_vf_active_assoc = 0; 2264 hn_rxvf_set(sc, NULL); 2265 hn_vf_rss_restore(sc); 2266 hn_resume_mgmt(sc); 2267 } 2268 } 2269 2270 /* Mark transparent mode VF as disabled. */ 2271 hn_xpnt_vf_setdisable(sc, true /* clear hn_vf_ifp */); 2272 2273 rm_wlock(&hn_vfmap_lock); 2274 2275 KASSERT(if_getindex(ifp) < hn_vfmap_size, 2276 ("ifindex %d, vfmapsize %d", if_getindex(ifp), hn_vfmap_size)); 2277 if (hn_vfmap[if_getindex(ifp)] != NULL) { 2278 KASSERT(hn_vfmap[if_getindex(ifp)] == sc->hn_ifp, 2279 ("%s: ifindex %d was mapped to %s", 2280 if_name(ifp), if_getindex(ifp), 2281 if_name(hn_vfmap[if_getindex(ifp)]))); 2282 hn_vfmap[if_getindex(ifp)] = NULL; 2283 } 2284 2285 rm_wunlock(&hn_vfmap_lock); 2286 done: 2287 HN_UNLOCK(sc); 2288 } 2289 2290 static void 2291 hn_ifnet_lnkevent(void *xsc, if_t ifp, int link_state) 2292 { 2293 struct hn_softc *sc = xsc; 2294 struct rm_priotracker pt; 2295 2296 /* Publish before a concurrent handoff can restore synthetic carrier. */ 2297 rm_rlock(&sc->hn_vf_lock, &pt); 2298 if (sc->hn_vf_ifp != ifp) 2299 goto out; 2300 if (link_state == LINK_STATE_UP) { 2301 /* RSS queries and host reconfiguration require sleepable context. */ 2302 atomic_store_rel_int(&sc->hn_vf_rss_refresh, 1); 2303 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 2304 } 2305 if ((sc->hn_xvf_flags & (HN_XVFFLAG_ENABLED | HN_XVFFLAG_SWITCHING)) == 2306 HN_XVFFLAG_ENABLED && sc->hn_vf_active_assoc == 2307 atomic_load_acq_int(&sc->hn_vf_assoc)) 2308 if_link_state_change(sc->hn_ifp, link_state); 2309 out: 2310 rm_runlock(&sc->hn_vf_lock, &pt); 2311 } 2312 2313 static int 2314 hn_tsomax_sysctl(SYSCTL_HANDLER_ARGS) 2315 { 2316 struct hn_softc *sc = arg1; 2317 unsigned int tsomax; 2318 int error; 2319 2320 tsomax = if_gethwtsomax(sc->hn_ifp); 2321 error = sysctl_handle_int(oidp, &tsomax, 0, req); 2322 return error; 2323 } 2324 2325 static int 2326 hn_tsomaxsegcnt_sysctl(SYSCTL_HANDLER_ARGS) 2327 { 2328 struct hn_softc *sc = arg1; 2329 unsigned int tsomaxsegcnt; 2330 int error; 2331 2332 tsomaxsegcnt = if_gethwtsomaxsegcount(sc->hn_ifp); 2333 error = sysctl_handle_int(oidp, &tsomaxsegcnt, 0, req); 2334 return error; 2335 } 2336 2337 static int 2338 hn_tsomaxsegsz_sysctl(SYSCTL_HANDLER_ARGS) 2339 { 2340 struct hn_softc *sc = arg1; 2341 unsigned int tsomaxsegsz; 2342 int error; 2343 2344 tsomaxsegsz = if_gethwtsomaxsegsize(sc->hn_ifp); 2345 error = sysctl_handle_int(oidp, &tsomaxsegsz, 0, req); 2346 return error; 2347 } 2348 2349 static int 2350 hn_probe(device_t dev) 2351 { 2352 2353 if (VMBUS_PROBE_GUID(device_get_parent(dev), dev, &hn_guid) == 0) { 2354 device_set_desc(dev, "Hyper-V Network Interface"); 2355 return BUS_PROBE_DEFAULT; 2356 } 2357 return ENXIO; 2358 } 2359 2360 static int 2361 hn_attach(device_t dev) 2362 { 2363 struct hn_softc *sc = device_get_softc(dev); 2364 struct sysctl_oid_list *child; 2365 struct sysctl_ctx_list *ctx; 2366 uint8_t eaddr[ETHER_ADDR_LEN]; 2367 if_t ifp = NULL; 2368 int error, ring_cnt, tx_ring_cnt; 2369 uint32_t mtu; 2370 2371 sc->hn_dev = dev; 2372 sc->hn_prichan = vmbus_get_channel(dev); 2373 HN_LOCK_INIT(sc); 2374 rm_init(&sc->hn_vf_lock, "hnvf"); 2375 if (hn_xpnt_vf && hn_xpnt_vf_accbpf) 2376 sc->hn_xvf_flags |= HN_XVFFLAG_ACCBPF; 2377 2378 /* 2379 * Initialize these tunables once. 2380 */ 2381 sc->hn_agg_size = hn_tx_agg_size; 2382 sc->hn_agg_pkts = hn_tx_agg_pkts; 2383 2384 /* 2385 * Setup taskqueue for transmission. 2386 */ 2387 if (hn_tx_taskq_mode == HN_TX_TASKQ_M_INDEP) { 2388 int i; 2389 2390 sc->hn_tx_taskqs = 2391 malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *), 2392 M_DEVBUF, M_WAITOK); 2393 for (i = 0; i < hn_tx_taskq_cnt; ++i) { 2394 sc->hn_tx_taskqs[i] = taskqueue_create("hn_tx", 2395 M_WAITOK, taskqueue_thread_enqueue, 2396 &sc->hn_tx_taskqs[i]); 2397 taskqueue_start_threads(&sc->hn_tx_taskqs[i], 1, PI_NET, 2398 "%s tx%d", device_get_nameunit(dev), i); 2399 } 2400 } else if (hn_tx_taskq_mode == HN_TX_TASKQ_M_GLOBAL) { 2401 sc->hn_tx_taskqs = hn_tx_taskque; 2402 } 2403 2404 /* 2405 * Setup taskqueue for mangement tasks, e.g. link status. 2406 */ 2407 sc->hn_mgmt_taskq0 = taskqueue_create("hn_mgmt", M_WAITOK, 2408 taskqueue_thread_enqueue, &sc->hn_mgmt_taskq0); 2409 taskqueue_start_threads(&sc->hn_mgmt_taskq0, 1, PI_NET, "%s mgmt", 2410 device_get_nameunit(dev)); 2411 TASK_INIT(&sc->hn_link_task, 0, hn_link_taskfunc, sc); 2412 TASK_INIT(&sc->hn_netchg_init, 0, hn_netchg_init_taskfunc, sc); 2413 TIMEOUT_TASK_INIT(sc->hn_mgmt_taskq0, &sc->hn_netchg_status, 0, 2414 hn_netchg_status_taskfunc, sc); 2415 2416 /* Association work must not block the channel delivering completions. */ 2417 sc->hn_vf_taskq = taskqueue_create("hn_vf", M_WAITOK, 2418 taskqueue_thread_enqueue, &sc->hn_vf_taskq); 2419 taskqueue_start_threads(&sc->hn_vf_taskq, 1, PI_NET, "%s vf", 2420 device_get_nameunit(dev)); 2421 TIMEOUT_TASK_INIT(sc->hn_vf_taskq, &sc->hn_vf_init, 0, 2422 hn_xpnt_vf_init_taskfunc, sc); 2423 TASK_INIT(&sc->hn_vf_vlancap, 0, hn_xpnt_vf_vlancap_taskfunc, sc); 2424 2425 /* 2426 * Allocate ifnet and setup its name earlier, so that if_printf 2427 * can be used by functions, which will be called after 2428 * ether_ifattach(). 2429 */ 2430 ifp = sc->hn_ifp = if_alloc(IFT_ETHER); 2431 if_setsoftc(ifp, sc); 2432 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2433 2434 /* 2435 * Initialize ifmedia earlier so that it can be unconditionally 2436 * destroyed, if error happened later on. 2437 */ 2438 ifmedia_init(&sc->hn_media, 0, hn_ifmedia_upd, hn_ifmedia_sts); 2439 2440 /* 2441 * Figure out the # of RX rings (ring_cnt) and the # of TX rings 2442 * to use (tx_ring_cnt). 2443 * 2444 * NOTE: 2445 * The # of RX rings to use is same as the # of channels to use. 2446 */ 2447 ring_cnt = hn_chan_cnt; 2448 if (ring_cnt <= 0) { 2449 /* Default */ 2450 ring_cnt = mp_ncpus; 2451 if (ring_cnt > HN_RING_CNT_DEF_MAX) 2452 ring_cnt = HN_RING_CNT_DEF_MAX; 2453 } else if (ring_cnt > mp_ncpus) { 2454 ring_cnt = mp_ncpus; 2455 } 2456 #ifdef RSS 2457 if (ring_cnt > rss_getnumbuckets()) 2458 ring_cnt = rss_getnumbuckets(); 2459 #endif 2460 2461 tx_ring_cnt = hn_tx_ring_cnt; 2462 if (tx_ring_cnt <= 0 || tx_ring_cnt > ring_cnt) 2463 tx_ring_cnt = ring_cnt; 2464 #ifdef HN_IFSTART_SUPPORT 2465 if (hn_use_if_start) { 2466 /* ifnet.if_start only needs one TX ring. */ 2467 tx_ring_cnt = 1; 2468 } 2469 #endif 2470 2471 /* 2472 * Set the leader CPU for channels. 2473 */ 2474 sc->hn_cpu = atomic_fetchadd_int(&hn_cpu_index, ring_cnt) % mp_ncpus; 2475 2476 /* 2477 * Create enough TX/RX rings, even if only limited number of 2478 * channels can be allocated. 2479 */ 2480 error = hn_create_tx_data(sc, tx_ring_cnt); 2481 if (error) 2482 goto failed; 2483 error = hn_create_rx_data(sc, ring_cnt); 2484 if (error) 2485 goto failed; 2486 2487 /* 2488 * Create transaction context for NVS and RNDIS transactions. 2489 */ 2490 sc->hn_xact = vmbus_xact_ctx_create(bus_get_dma_tag(dev), 2491 HN_XACT_REQ_SIZE, HN_XACT_RESP_SIZE, 0); 2492 if (sc->hn_xact == NULL) { 2493 error = ENXIO; 2494 goto failed; 2495 } 2496 2497 /* 2498 * Install orphan handler for the revocation of this device's 2499 * primary channel. 2500 * 2501 * NOTE: 2502 * The processing order is critical here: 2503 * Install the orphan handler, _before_ testing whether this 2504 * device's primary channel has been revoked or not. 2505 */ 2506 vmbus_chan_set_orphan(sc->hn_prichan, sc->hn_xact); 2507 if (vmbus_chan_is_revoked(sc->hn_prichan)) { 2508 error = ENXIO; 2509 goto failed; 2510 } 2511 2512 /* 2513 * Attach the synthetic parts, i.e. NVS and RNDIS. 2514 */ 2515 error = hn_synth_attach(sc, ETHERMTU); 2516 if (error) 2517 goto failed; 2518 2519 error = hn_rndis_get_eaddr(sc, eaddr); 2520 if (error) 2521 goto failed; 2522 2523 error = hn_rndis_get_mtu(sc, &mtu); 2524 if (error) 2525 mtu = ETHERMTU; 2526 else if (bootverbose) 2527 device_printf(dev, "RNDIS mtu %u\n", mtu); 2528 2529 if (sc->hn_rx_ring_inuse > 1) { 2530 /* 2531 * Reduce TCP segment aggregation limit for multiple 2532 * RX rings to increase ACK timeliness. 2533 */ 2534 hn_set_lro_lenlim(sc, HN_LRO_LENLIM_MULTIRX_DEF); 2535 } 2536 2537 /* 2538 * Fixup TX/RX stuffs after synthetic parts are attached. 2539 */ 2540 hn_fixup_tx_data(sc); 2541 hn_fixup_rx_data(sc); 2542 2543 ctx = device_get_sysctl_ctx(dev); 2544 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 2545 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "nvs_version", CTLFLAG_RD, 2546 &sc->hn_nvs_ver, 0, "NVS version"); 2547 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "ndis_version", 2548 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2549 hn_ndis_version_sysctl, "A", "NDIS version"); 2550 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "caps", 2551 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2552 hn_caps_sysctl, "A", "capabilities"); 2553 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "hwassist", 2554 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2555 hn_hwassist_sysctl, "A", "hwassist"); 2556 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tso_max", 2557 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, hn_tsomax_sysctl, 2558 "IU", "max TSO size"); 2559 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tso_maxsegcnt", 2560 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, hn_tsomaxsegcnt_sysctl, 2561 "IU", "max # of TSO segments"); 2562 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tso_maxsegsz", 2563 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, hn_tsomaxsegsz_sysctl, 2564 "IU", "max size of TSO segment"); 2565 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxfilter", 2566 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2567 hn_rxfilter_sysctl, "A", "rxfilter"); 2568 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_hash", 2569 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2570 hn_rss_hash_sysctl, "A", "RSS hash"); 2571 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_hashcap", 2572 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2573 hn_rss_hcap_sysctl, "A", "RSS hash capabilities"); 2574 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "mbuf_hash", 2575 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2576 hn_rss_mbuf_sysctl, "A", "RSS hash for mbufs"); 2577 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rss_ind_size", 2578 CTLFLAG_RD, &sc->hn_rss_ind_size, 0, "RSS indirect entry count"); 2579 #ifndef RSS 2580 /* 2581 * Don't allow RSS key/indirect table changes, if RSS is defined. 2582 */ 2583 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_key", 2584 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2585 hn_rss_key_sysctl, "IU", "RSS key"); 2586 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_ind", 2587 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2588 hn_rss_ind_sysctl, "IU", "RSS indirect table"); 2589 #endif 2590 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rndis_agg_size", 2591 CTLFLAG_RD, &sc->hn_rndis_agg_size, 0, 2592 "RNDIS offered packet transmission aggregation size limit"); 2593 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rndis_agg_pkts", 2594 CTLFLAG_RD, &sc->hn_rndis_agg_pkts, 0, 2595 "RNDIS offered packet transmission aggregation count limit"); 2596 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rndis_agg_align", 2597 CTLFLAG_RD, &sc->hn_rndis_agg_align, 0, 2598 "RNDIS packet transmission aggregation alignment"); 2599 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_size", 2600 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2601 hn_txagg_size_sysctl, "I", 2602 "Packet transmission aggregation size, 0 -- disable, -1 -- auto"); 2603 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_pkts", 2604 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2605 hn_txagg_pkts_sysctl, "I", 2606 "Packet transmission aggregation packets, " 2607 "0 -- disable, -1 -- auto"); 2608 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "polling", 2609 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2610 hn_polling_sysctl, "I", 2611 "Polling frequency: [100,1000000], 0 disable polling"); 2612 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "vf", 2613 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2614 hn_vf_sysctl, "A", "Virtual Function's name"); 2615 if (!hn_xpnt_vf) { 2616 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxvf", 2617 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2618 hn_rxvf_sysctl, "A", "activated Virtual Function's name"); 2619 } else { 2620 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "vf_xpnt_enabled", 2621 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 2622 hn_xpnt_vf_enabled_sysctl, "I", 2623 "Transparent VF enabled"); 2624 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "vf_xpnt_accbpf", 2625 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 2626 hn_xpnt_vf_accbpf_sysctl, "I", 2627 "Accurate BPF for transparent VF"); 2628 } 2629 2630 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rsc_switch", 2631 CTLTYPE_UINT | CTLFLAG_RW, sc, 0, hn_rsc_sysctl, "I", 2632 "switch to rsc"); 2633 2634 /* 2635 * Setup the ifmedia, which has been initialized earlier. 2636 */ 2637 ifmedia_add(&sc->hn_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2638 ifmedia_set(&sc->hn_media, IFM_ETHER | IFM_AUTO); 2639 /* XXX ifmedia_set really should do this for us */ 2640 sc->hn_media.ifm_media = sc->hn_media.ifm_cur->ifm_media; 2641 2642 /* 2643 * Setup the ifnet for this interface. 2644 */ 2645 2646 if_setbaudrate(ifp, IF_Gbps(10)); 2647 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 2648 if_setioctlfn(ifp, hn_ioctl); 2649 if_setinitfn(ifp, hn_init); 2650 #ifdef HN_IFSTART_SUPPORT 2651 if (hn_use_if_start) { 2652 int qdepth = hn_get_txswq_depth(&sc->hn_tx_ring[0]); 2653 2654 if_setstartfn(ifp, hn_start); 2655 if_setsendqlen(ifp, qdepth); 2656 if_setsendqready(ifp); 2657 } else 2658 #endif 2659 { 2660 if_settransmitfn(ifp, hn_transmit); 2661 if_setqflushfn(ifp, hn_xmit_qflush); 2662 } 2663 2664 if_setcapabilitiesbit(ifp, IFCAP_RXCSUM | IFCAP_LRO | IFCAP_LINKSTATE, 0); 2665 #ifdef foo 2666 /* We can't diff IPv6 packets from IPv4 packets on RX path. */ 2667 if_setcapabilitiesbit(ifp, IFCAP_RXCSUM_IPV6, 0); 2668 #endif 2669 if (sc->hn_caps & HN_CAP_VLAN) { 2670 /* XXX not sure about VLAN_MTU. */ 2671 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0); 2672 } 2673 2674 if_sethwassist(ifp, sc->hn_tx_ring[0].hn_csum_assist); 2675 if (if_gethwassist(ifp) & HN_CSUM_IP_MASK) 2676 if_setcapabilitiesbit(ifp, IFCAP_TXCSUM, 0); 2677 if (if_gethwassist(ifp) & HN_CSUM_IP6_MASK) 2678 if_setcapabilitiesbit(ifp, IFCAP_TXCSUM_IPV6, 0); 2679 if (sc->hn_caps & HN_CAP_TSO4) { 2680 if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); 2681 if_sethwassistbits(ifp, CSUM_IP_TSO, 0); 2682 } 2683 if (sc->hn_caps & HN_CAP_TSO6) { 2684 if_setcapabilitiesbit(ifp, IFCAP_TSO6, 0); 2685 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); 2686 } 2687 2688 /* Enable all available capabilities by default. */ 2689 if_setcapenable(ifp, if_getcapabilities(ifp)); 2690 2691 /* 2692 * Disable IPv6 TSO and TXCSUM by default, they still can 2693 * be enabled through SIOCSIFCAP. 2694 */ 2695 if_setcapenablebit(ifp, 0, (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6)); 2696 if_sethwassistbits(ifp, 0, (HN_CSUM_IP6_MASK | CSUM_IP6_TSO)); 2697 2698 if (if_getcapabilities(ifp) & (IFCAP_TSO6 | IFCAP_TSO4)) { 2699 /* 2700 * Lock hn_set_tso_maxsize() to simplify its 2701 * internal logic. 2702 */ 2703 HN_LOCK(sc); 2704 hn_set_tso_maxsize(sc, hn_tso_maxlen, ETHERMTU); 2705 HN_UNLOCK(sc); 2706 if_sethwtsomaxsegcount(ifp, HN_TX_DATA_SEGCNT_MAX); 2707 if_sethwtsomaxsegsize(ifp, PAGE_SIZE); 2708 } 2709 2710 /* Schedule reconciliation when the synthetic interface's VLANs change. */ 2711 if (hn_xpnt_vf) { 2712 sc->hn_vlan_atthand = EVENTHANDLER_REGISTER(vlan_config, 2713 hn_vlan_event, sc, EVENTHANDLER_PRI_ANY); 2714 sc->hn_vlan_dethand = EVENTHANDLER_REGISTER(vlan_unconfig, 2715 hn_vlan_event, sc, EVENTHANDLER_PRI_ANY); 2716 } 2717 ether_ifattach(ifp, eaddr); 2718 2719 if ((if_getcapabilities(ifp) & (IFCAP_TSO6 | IFCAP_TSO4)) && bootverbose) { 2720 if_printf(ifp, "TSO segcnt %u segsz %u\n", 2721 if_gethwtsomaxsegcount(ifp), if_gethwtsomaxsegsize(ifp)); 2722 } 2723 if (mtu < ETHERMTU) { 2724 2725 if_setmtu(ifp, mtu); 2726 } 2727 2728 /* Inform the upper layer about the long frame support. */ 2729 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 2730 2731 /* 2732 * Kick off link status check. 2733 */ 2734 sc->hn_mgmt_taskq = sc->hn_mgmt_taskq0; 2735 hn_update_link_status(sc); 2736 2737 if (!hn_xpnt_vf) { 2738 sc->hn_ifnet_evthand = EVENTHANDLER_REGISTER(ifnet_event, 2739 hn_ifnet_event, sc, EVENTHANDLER_PRI_ANY); 2740 sc->hn_ifaddr_evthand = EVENTHANDLER_REGISTER(ifaddr_event, 2741 hn_ifaddr_event, sc, EVENTHANDLER_PRI_ANY); 2742 } 2743 sc->hn_ifnet_lnkhand = EVENTHANDLER_REGISTER(ifnet_link_event, 2744 hn_ifnet_lnkevent, sc, EVENTHANDLER_PRI_ANY); 2745 2746 /* 2747 * NOTE: 2748 * Subscribe ether_ifattach event, instead of ifnet_arrival event, 2749 * since interface's LLADDR is needed; interface LLADDR is not 2750 * available when ifnet_arrival event is triggered. 2751 */ 2752 sc->hn_ifnet_atthand = EVENTHANDLER_REGISTER(ether_ifattach_event, 2753 hn_ifnet_attevent, sc, EVENTHANDLER_PRI_ANY); 2754 sc->hn_ifnet_dethand = EVENTHANDLER_REGISTER(ifnet_departure_event, 2755 hn_ifnet_detevent, sc, EVENTHANDLER_PRI_ANY); 2756 2757 return (0); 2758 failed: 2759 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) 2760 hn_synth_detach(sc); 2761 hn_detach(dev); 2762 return (error); 2763 } 2764 2765 static int 2766 hn_detach(device_t dev) 2767 { 2768 struct hn_softc *sc = device_get_softc(dev); 2769 if_t ifp = sc->hn_ifp, vf_ifp; 2770 2771 if (sc->hn_xact != NULL && vmbus_chan_is_revoked(sc->hn_prichan)) { 2772 /* 2773 * In case that the vmbus missed the orphan handler 2774 * installation. 2775 */ 2776 vmbus_xact_ctx_orphan(sc->hn_xact); 2777 } 2778 2779 HN_LOCK(sc); 2780 sc->hn_detaching = true; 2781 while (sc->hn_vf_caps_busy) 2782 sx_sleep(&sc->hn_vf_caps_busy, &sc->hn_lock, 0, "hnvfcap", 0); 2783 HN_UNLOCK(sc); 2784 if (sc->hn_vlan_atthand != NULL) 2785 EVENTHANDLER_DEREGISTER(vlan_config, sc->hn_vlan_atthand); 2786 if (sc->hn_vlan_dethand != NULL) 2787 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->hn_vlan_dethand); 2788 taskqueue_drain_timeout(sc->hn_vf_taskq, &sc->hn_vf_init); 2789 taskqueue_drain(sc->hn_vf_taskq, &sc->hn_vf_vlancap); 2790 2791 if (sc->hn_ifaddr_evthand != NULL) 2792 EVENTHANDLER_DEREGISTER(ifaddr_event, sc->hn_ifaddr_evthand); 2793 if (sc->hn_ifnet_evthand != NULL) 2794 EVENTHANDLER_DEREGISTER(ifnet_event, sc->hn_ifnet_evthand); 2795 if (sc->hn_ifnet_atthand != NULL) { 2796 EVENTHANDLER_DEREGISTER(ether_ifattach_event, 2797 sc->hn_ifnet_atthand); 2798 } 2799 if (sc->hn_ifnet_dethand != NULL) { 2800 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 2801 sc->hn_ifnet_dethand); 2802 } 2803 if (sc->hn_ifnet_lnkhand != NULL) 2804 EVENTHANDLER_DEREGISTER(ifnet_link_event, sc->hn_ifnet_lnkhand); 2805 2806 HN_LOCK(sc); 2807 vf_ifp = sc->hn_vf_ifp; 2808 /* hn is leaving; remove its registrations from the live VF. */ 2809 if (vf_ifp != NULL && hn_xpnt_vf) 2810 hn_xpnt_vf_sync_vlans(sc, true); 2811 HN_UNLOCK(sc); 2812 if (vf_ifp != NULL) 2813 hn_ifnet_detevent(sc, vf_ifp); 2814 2815 if (device_is_attached(dev)) { 2816 HN_LOCK(sc); 2817 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) { 2818 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 2819 hn_stop(sc, true); 2820 /* 2821 * NOTE: 2822 * hn_stop() only suspends data, so management 2823 * stuffs have to be suspended manually here. 2824 */ 2825 hn_suspend_mgmt(sc); 2826 hn_synth_detach(sc); 2827 } 2828 HN_UNLOCK(sc); 2829 ether_ifdetach(ifp); 2830 } 2831 2832 ifmedia_removeall(&sc->hn_media); 2833 hn_destroy_rx_data(sc); 2834 hn_destroy_tx_data(sc); 2835 2836 if (sc->hn_tx_taskqs != NULL && sc->hn_tx_taskqs != hn_tx_taskque) { 2837 int i; 2838 2839 for (i = 0; i < hn_tx_taskq_cnt; ++i) 2840 taskqueue_free(sc->hn_tx_taskqs[i]); 2841 free(sc->hn_tx_taskqs, M_DEVBUF); 2842 } 2843 taskqueue_free(sc->hn_mgmt_taskq0); 2844 if (sc->hn_vf_taskq != NULL) 2845 taskqueue_free(sc->hn_vf_taskq); 2846 2847 if (sc->hn_xact != NULL) { 2848 /* 2849 * Uninstall the orphan handler _before_ the xact is 2850 * destructed. 2851 */ 2852 vmbus_chan_unset_orphan(sc->hn_prichan); 2853 vmbus_xact_ctx_destroy(sc->hn_xact); 2854 } 2855 2856 if_free(ifp); 2857 2858 HN_LOCK_DESTROY(sc); 2859 rm_destroy(&sc->hn_vf_lock); 2860 return (0); 2861 } 2862 2863 static int 2864 hn_shutdown(device_t dev) 2865 { 2866 2867 return (0); 2868 } 2869 2870 static void 2871 hn_link_status(struct hn_softc *sc) 2872 { 2873 uint32_t link_status; 2874 int error; 2875 2876 error = hn_rndis_get_linkstatus(sc, &link_status); 2877 if (error) { 2878 /* XXX what to do? */ 2879 return; 2880 } 2881 2882 if (link_status == NDIS_MEDIA_STATE_CONNECTED) 2883 sc->hn_link_flags |= HN_LINK_FLAG_LINKUP; 2884 else 2885 sc->hn_link_flags &= ~HN_LINK_FLAG_LINKUP; 2886 if_link_state_change(sc->hn_ifp, 2887 (sc->hn_link_flags & HN_LINK_FLAG_LINKUP) ? 2888 LINK_STATE_UP : LINK_STATE_DOWN); 2889 } 2890 2891 static void 2892 hn_link_taskfunc(void *xsc, int pending __unused) 2893 { 2894 struct hn_softc *sc = xsc; 2895 2896 if (sc->hn_link_flags & HN_LINK_FLAG_NETCHG) 2897 return; 2898 hn_link_status(sc); 2899 } 2900 2901 static void 2902 hn_netchg_init_taskfunc(void *xsc, int pending __unused) 2903 { 2904 struct hn_softc *sc = xsc; 2905 2906 /* Prevent any link status checks from running. */ 2907 sc->hn_link_flags |= HN_LINK_FLAG_NETCHG; 2908 2909 /* 2910 * Fake up a [link down --> link up] state change; 5 seconds 2911 * delay is used, which closely simulates miibus reaction 2912 * upon link down event. 2913 */ 2914 sc->hn_link_flags &= ~HN_LINK_FLAG_LINKUP; 2915 if_link_state_change(sc->hn_ifp, LINK_STATE_DOWN); 2916 taskqueue_enqueue_timeout(sc->hn_mgmt_taskq0, 2917 &sc->hn_netchg_status, 5 * hz); 2918 } 2919 2920 static void 2921 hn_netchg_status_taskfunc(void *xsc, int pending __unused) 2922 { 2923 struct hn_softc *sc = xsc; 2924 2925 /* Re-allow link status checks. */ 2926 sc->hn_link_flags &= ~HN_LINK_FLAG_NETCHG; 2927 hn_link_status(sc); 2928 } 2929 2930 static void 2931 hn_update_link_status(struct hn_softc *sc) 2932 { 2933 2934 if (sc->hn_mgmt_taskq != NULL) 2935 taskqueue_enqueue(sc->hn_mgmt_taskq, &sc->hn_link_task); 2936 } 2937 2938 static void 2939 hn_change_network(struct hn_softc *sc) 2940 { 2941 2942 if (sc->hn_mgmt_taskq != NULL) 2943 taskqueue_enqueue(sc->hn_mgmt_taskq, &sc->hn_netchg_init); 2944 } 2945 2946 static __inline int 2947 hn_txdesc_dmamap_load(struct hn_tx_ring *txr, struct hn_txdesc *txd, 2948 struct mbuf **m_head, bus_dma_segment_t *segs, int *nsegs) 2949 { 2950 struct mbuf *m = *m_head; 2951 int error; 2952 2953 KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID, ("txd uses chim")); 2954 2955 error = bus_dmamap_load_mbuf_sg(txr->hn_tx_data_dtag, txd->data_dmap, 2956 m, segs, nsegs, BUS_DMA_NOWAIT); 2957 if (error == EFBIG) { 2958 struct mbuf *m_new; 2959 2960 m_new = m_collapse(m, M_NOWAIT, HN_TX_DATA_SEGCNT_MAX); 2961 if (m_new == NULL) 2962 return ENOBUFS; 2963 else 2964 *m_head = m = m_new; 2965 txr->hn_tx_collapsed++; 2966 2967 error = bus_dmamap_load_mbuf_sg(txr->hn_tx_data_dtag, 2968 txd->data_dmap, m, segs, nsegs, BUS_DMA_NOWAIT); 2969 } 2970 if (!error) { 2971 bus_dmamap_sync(txr->hn_tx_data_dtag, txd->data_dmap, 2972 BUS_DMASYNC_PREWRITE); 2973 txd->flags |= HN_TXD_FLAG_DMAMAP; 2974 } 2975 return error; 2976 } 2977 2978 static __inline int 2979 hn_txdesc_put(struct hn_tx_ring *txr, struct hn_txdesc *txd) 2980 { 2981 2982 KASSERT((txd->flags & HN_TXD_FLAG_ONLIST) == 0, 2983 ("put an onlist txd %#x", txd->flags)); 2984 KASSERT((txd->flags & HN_TXD_FLAG_ONAGG) == 0, 2985 ("put an onagg txd %#x", txd->flags)); 2986 2987 KASSERT(txd->refs > 0, ("invalid txd refs %d", txd->refs)); 2988 if (atomic_fetchadd_int(&txd->refs, -1) != 1) 2989 return 0; 2990 2991 if (!STAILQ_EMPTY(&txd->agg_list)) { 2992 struct hn_txdesc *tmp_txd; 2993 2994 while ((tmp_txd = STAILQ_FIRST(&txd->agg_list)) != NULL) { 2995 int freed __diagused; 2996 2997 KASSERT(STAILQ_EMPTY(&tmp_txd->agg_list), 2998 ("resursive aggregation on aggregated txdesc")); 2999 KASSERT((tmp_txd->flags & HN_TXD_FLAG_ONAGG), 3000 ("not aggregated txdesc")); 3001 KASSERT((tmp_txd->flags & HN_TXD_FLAG_DMAMAP) == 0, 3002 ("aggregated txdesc uses dmamap")); 3003 KASSERT(tmp_txd->chim_index == HN_NVS_CHIM_IDX_INVALID, 3004 ("aggregated txdesc consumes " 3005 "chimney sending buffer")); 3006 KASSERT(tmp_txd->chim_size == 0, 3007 ("aggregated txdesc has non-zero " 3008 "chimney sending size")); 3009 3010 STAILQ_REMOVE_HEAD(&txd->agg_list, agg_link); 3011 tmp_txd->flags &= ~HN_TXD_FLAG_ONAGG; 3012 freed = hn_txdesc_put(txr, tmp_txd); 3013 KASSERT(freed, ("failed to free aggregated txdesc")); 3014 } 3015 } 3016 3017 if (txd->chim_index != HN_NVS_CHIM_IDX_INVALID) { 3018 KASSERT((txd->flags & HN_TXD_FLAG_DMAMAP) == 0, 3019 ("chim txd uses dmamap")); 3020 hn_chim_free(txr->hn_sc, txd->chim_index); 3021 txd->chim_index = HN_NVS_CHIM_IDX_INVALID; 3022 txd->chim_size = 0; 3023 } else if (txd->flags & HN_TXD_FLAG_DMAMAP) { 3024 bus_dmamap_sync(txr->hn_tx_data_dtag, 3025 txd->data_dmap, BUS_DMASYNC_POSTWRITE); 3026 bus_dmamap_unload(txr->hn_tx_data_dtag, 3027 txd->data_dmap); 3028 txd->flags &= ~HN_TXD_FLAG_DMAMAP; 3029 } 3030 3031 if (txd->m != NULL) { 3032 m_freem(txd->m); 3033 txd->m = NULL; 3034 } 3035 3036 txd->flags |= HN_TXD_FLAG_ONLIST; 3037 #ifndef HN_USE_TXDESC_BUFRING 3038 mtx_lock_spin(&txr->hn_txlist_spin); 3039 KASSERT(txr->hn_txdesc_avail >= 0 && 3040 txr->hn_txdesc_avail < txr->hn_txdesc_cnt, 3041 ("txdesc_put: invalid txd avail %d", txr->hn_txdesc_avail)); 3042 txr->hn_txdesc_avail++; 3043 SLIST_INSERT_HEAD(&txr->hn_txlist, txd, link); 3044 mtx_unlock_spin(&txr->hn_txlist_spin); 3045 #else /* HN_USE_TXDESC_BUFRING */ 3046 #ifdef HN_DEBUG 3047 atomic_add_int(&txr->hn_txdesc_avail, 1); 3048 #endif 3049 buf_ring_enqueue(txr->hn_txdesc_br, txd); 3050 #endif /* !HN_USE_TXDESC_BUFRING */ 3051 3052 return 1; 3053 } 3054 3055 static __inline struct hn_txdesc * 3056 hn_txdesc_get(struct hn_tx_ring *txr) 3057 { 3058 struct hn_txdesc *txd; 3059 3060 #ifndef HN_USE_TXDESC_BUFRING 3061 mtx_lock_spin(&txr->hn_txlist_spin); 3062 txd = SLIST_FIRST(&txr->hn_txlist); 3063 if (txd != NULL) { 3064 KASSERT(txr->hn_txdesc_avail > 0, 3065 ("txdesc_get: invalid txd avail %d", txr->hn_txdesc_avail)); 3066 txr->hn_txdesc_avail--; 3067 SLIST_REMOVE_HEAD(&txr->hn_txlist, link); 3068 } 3069 mtx_unlock_spin(&txr->hn_txlist_spin); 3070 #else 3071 txd = buf_ring_dequeue_sc(txr->hn_txdesc_br); 3072 #endif 3073 3074 if (txd != NULL) { 3075 #ifdef HN_USE_TXDESC_BUFRING 3076 #ifdef HN_DEBUG 3077 atomic_subtract_int(&txr->hn_txdesc_avail, 1); 3078 #endif 3079 #endif /* HN_USE_TXDESC_BUFRING */ 3080 KASSERT(txd->m == NULL && txd->refs == 0 && 3081 STAILQ_EMPTY(&txd->agg_list) && 3082 txd->chim_index == HN_NVS_CHIM_IDX_INVALID && 3083 txd->chim_size == 0 && 3084 (txd->flags & HN_TXD_FLAG_ONLIST) && 3085 (txd->flags & HN_TXD_FLAG_ONAGG) == 0 && 3086 (txd->flags & HN_TXD_FLAG_DMAMAP) == 0, ("invalid txd")); 3087 txd->flags &= ~HN_TXD_FLAG_ONLIST; 3088 txd->refs = 1; 3089 } 3090 return txd; 3091 } 3092 3093 static __inline void 3094 hn_txdesc_hold(struct hn_txdesc *txd) 3095 { 3096 3097 /* 0->1 transition will never work */ 3098 KASSERT(txd->refs > 0, ("invalid txd refs %d", txd->refs)); 3099 atomic_add_int(&txd->refs, 1); 3100 } 3101 3102 static __inline void 3103 hn_txdesc_agg(struct hn_txdesc *agg_txd, struct hn_txdesc *txd) 3104 { 3105 3106 KASSERT((agg_txd->flags & HN_TXD_FLAG_ONAGG) == 0, 3107 ("recursive aggregation on aggregating txdesc")); 3108 3109 KASSERT((txd->flags & HN_TXD_FLAG_ONAGG) == 0, 3110 ("already aggregated")); 3111 KASSERT(STAILQ_EMPTY(&txd->agg_list), 3112 ("recursive aggregation on to-be-aggregated txdesc")); 3113 3114 txd->flags |= HN_TXD_FLAG_ONAGG; 3115 STAILQ_INSERT_TAIL(&agg_txd->agg_list, txd, agg_link); 3116 } 3117 3118 static bool 3119 hn_tx_ring_pending(struct hn_tx_ring *txr) 3120 { 3121 bool pending = false; 3122 3123 #ifndef HN_USE_TXDESC_BUFRING 3124 mtx_lock_spin(&txr->hn_txlist_spin); 3125 if (txr->hn_txdesc_avail != txr->hn_txdesc_cnt) 3126 pending = true; 3127 mtx_unlock_spin(&txr->hn_txlist_spin); 3128 #else 3129 if (!buf_ring_full(txr->hn_txdesc_br)) 3130 pending = true; 3131 #endif 3132 return (pending); 3133 } 3134 3135 static __inline void 3136 hn_txeof(struct hn_tx_ring *txr) 3137 { 3138 txr->hn_has_txeof = 0; 3139 txr->hn_txeof(txr); 3140 } 3141 3142 static void 3143 hn_txpkt_done(struct hn_nvs_sendctx *sndc, struct hn_softc *sc, 3144 struct vmbus_channel *chan, const void *data __unused, int dlen __unused) 3145 { 3146 struct hn_txdesc *txd = sndc->hn_cbarg; 3147 struct hn_tx_ring *txr; 3148 3149 txr = txd->txr; 3150 KASSERT(txr->hn_chan == chan, 3151 ("channel mismatch, on chan%u, should be chan%u", 3152 vmbus_chan_id(chan), vmbus_chan_id(txr->hn_chan))); 3153 3154 txr->hn_has_txeof = 1; 3155 hn_txdesc_put(txr, txd); 3156 3157 ++txr->hn_txdone_cnt; 3158 if (txr->hn_txdone_cnt >= HN_EARLY_TXEOF_THRESH) { 3159 txr->hn_txdone_cnt = 0; 3160 if (txr->hn_oactive) 3161 hn_txeof(txr); 3162 } 3163 } 3164 3165 static void 3166 hn_chan_rollup(struct hn_rx_ring *rxr, struct hn_tx_ring *txr) 3167 { 3168 #if defined(INET) || defined(INET6) 3169 struct epoch_tracker et; 3170 3171 NET_EPOCH_ENTER(et); 3172 tcp_lro_flush_all(&rxr->hn_lro); 3173 NET_EPOCH_EXIT(et); 3174 #endif 3175 3176 /* 3177 * NOTE: 3178 * 'txr' could be NULL, if multiple channels and 3179 * ifnet.if_start method are enabled. 3180 */ 3181 if (txr == NULL || !txr->hn_has_txeof) 3182 return; 3183 3184 txr->hn_txdone_cnt = 0; 3185 hn_txeof(txr); 3186 } 3187 3188 static __inline uint32_t 3189 hn_rndis_pktmsg_offset(uint32_t ofs) 3190 { 3191 3192 KASSERT(ofs >= sizeof(struct rndis_packet_msg), 3193 ("invalid RNDIS packet msg offset %u", ofs)); 3194 return (ofs - __offsetof(struct rndis_packet_msg, rm_dataoffset)); 3195 } 3196 3197 static __inline void * 3198 hn_rndis_pktinfo_append(struct rndis_packet_msg *pkt, size_t pktsize, 3199 size_t pi_dlen, uint32_t pi_type) 3200 { 3201 const size_t pi_size = HN_RNDIS_PKTINFO_SIZE(pi_dlen); 3202 struct rndis_pktinfo *pi; 3203 3204 KASSERT((pi_size & RNDIS_PACKET_MSG_OFFSET_ALIGNMASK) == 0, 3205 ("unaligned pktinfo size %zu, pktinfo dlen %zu", pi_size, pi_dlen)); 3206 3207 /* 3208 * Per-packet-info does not move; it only grows. 3209 * 3210 * NOTE: 3211 * rm_pktinfooffset in this phase counts from the beginning 3212 * of rndis_packet_msg. 3213 */ 3214 KASSERT(pkt->rm_pktinfooffset + pkt->rm_pktinfolen + pi_size <= pktsize, 3215 ("%u pktinfo overflows RNDIS packet msg", pi_type)); 3216 pi = (struct rndis_pktinfo *)((uint8_t *)pkt + pkt->rm_pktinfooffset + 3217 pkt->rm_pktinfolen); 3218 pkt->rm_pktinfolen += pi_size; 3219 3220 pi->rm_size = pi_size; 3221 pi->rm_type = pi_type; 3222 pi->rm_internal = 0; 3223 pi->rm_pktinfooffset = RNDIS_PKTINFO_OFFSET; 3224 3225 return (pi->rm_data); 3226 } 3227 3228 static __inline int 3229 hn_flush_txagg(if_t ifp, struct hn_tx_ring *txr) 3230 { 3231 struct hn_txdesc *txd; 3232 struct mbuf *m; 3233 int error, pkts; 3234 3235 txd = txr->hn_agg_txd; 3236 KASSERT(txd != NULL, ("no aggregate txdesc")); 3237 3238 /* 3239 * Since hn_txpkt() will reset this temporary stat, save 3240 * it now, so that oerrors can be updated properly, if 3241 * hn_txpkt() ever fails. 3242 */ 3243 pkts = txr->hn_stat_pkts; 3244 3245 /* 3246 * Since txd's mbuf will _not_ be freed upon hn_txpkt() 3247 * failure, save it for later freeing, if hn_txpkt() ever 3248 * fails. 3249 */ 3250 m = txd->m; 3251 error = hn_txpkt(ifp, txr, txd); 3252 if (__predict_false(error)) { 3253 /* txd is freed, but m is not. */ 3254 m_freem(m); 3255 3256 txr->hn_flush_failed++; 3257 if_inc_counter(ifp, IFCOUNTER_OERRORS, pkts); 3258 } 3259 3260 /* Reset all aggregation states. */ 3261 txr->hn_agg_txd = NULL; 3262 txr->hn_agg_szleft = 0; 3263 txr->hn_agg_pktleft = 0; 3264 txr->hn_agg_prevpkt = NULL; 3265 3266 return (error); 3267 } 3268 3269 static void * 3270 hn_try_txagg(if_t ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd, 3271 int pktsize) 3272 { 3273 void *chim; 3274 3275 if (txr->hn_agg_txd != NULL) { 3276 if (txr->hn_agg_pktleft >= 1 && txr->hn_agg_szleft > pktsize) { 3277 struct hn_txdesc *agg_txd = txr->hn_agg_txd; 3278 struct rndis_packet_msg *pkt = txr->hn_agg_prevpkt; 3279 int olen; 3280 3281 /* 3282 * Update the previous RNDIS packet's total length, 3283 * it can be increased due to the mandatory alignment 3284 * padding for this RNDIS packet. And update the 3285 * aggregating txdesc's chimney sending buffer size 3286 * accordingly. 3287 * 3288 * XXX 3289 * Zero-out the padding, as required by the RNDIS spec. 3290 */ 3291 olen = pkt->rm_len; 3292 pkt->rm_len = roundup2(olen, txr->hn_agg_align); 3293 agg_txd->chim_size += pkt->rm_len - olen; 3294 3295 /* Link this txdesc to the parent. */ 3296 hn_txdesc_agg(agg_txd, txd); 3297 3298 chim = (uint8_t *)pkt + pkt->rm_len; 3299 /* Save the current packet for later fixup. */ 3300 txr->hn_agg_prevpkt = chim; 3301 3302 txr->hn_agg_pktleft--; 3303 txr->hn_agg_szleft -= pktsize; 3304 if (txr->hn_agg_szleft <= 3305 HN_PKTSIZE_MIN(txr->hn_agg_align)) { 3306 /* 3307 * Probably can't aggregate more packets, 3308 * flush this aggregating txdesc proactively. 3309 */ 3310 txr->hn_agg_pktleft = 0; 3311 } 3312 /* Done! */ 3313 return (chim); 3314 } 3315 hn_flush_txagg(ifp, txr); 3316 } 3317 KASSERT(txr->hn_agg_txd == NULL, ("lingering aggregating txdesc")); 3318 3319 txr->hn_tx_chimney_tried++; 3320 txd->chim_index = hn_chim_alloc(txr->hn_sc); 3321 if (txd->chim_index == HN_NVS_CHIM_IDX_INVALID) 3322 return (NULL); 3323 txr->hn_tx_chimney++; 3324 3325 chim = txr->hn_sc->hn_chim + 3326 (txd->chim_index * txr->hn_sc->hn_chim_szmax); 3327 3328 if (txr->hn_agg_pktmax > 1 && 3329 txr->hn_agg_szmax > pktsize + HN_PKTSIZE_MIN(txr->hn_agg_align)) { 3330 txr->hn_agg_txd = txd; 3331 txr->hn_agg_pktleft = txr->hn_agg_pktmax - 1; 3332 txr->hn_agg_szleft = txr->hn_agg_szmax - pktsize; 3333 txr->hn_agg_prevpkt = chim; 3334 } 3335 return (chim); 3336 } 3337 3338 /* 3339 * NOTE: 3340 * If this function fails, then both txd and m_head0 will be freed. 3341 */ 3342 static int 3343 hn_encap(if_t ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd, 3344 struct mbuf **m_head0) 3345 { 3346 bus_dma_segment_t segs[HN_TX_DATA_SEGCNT_MAX]; 3347 int error, nsegs, i; 3348 struct mbuf *m_head = *m_head0; 3349 struct rndis_packet_msg *pkt; 3350 uint32_t *pi_data; 3351 void *chim = NULL; 3352 int pkt_hlen, pkt_size; 3353 3354 pkt = txd->rndis_pkt; 3355 pkt_size = HN_PKTSIZE(m_head, txr->hn_agg_align); 3356 if (pkt_size < txr->hn_chim_size) { 3357 chim = hn_try_txagg(ifp, txr, txd, pkt_size); 3358 if (chim != NULL) 3359 pkt = chim; 3360 } else { 3361 if (txr->hn_agg_txd != NULL) 3362 hn_flush_txagg(ifp, txr); 3363 } 3364 3365 pkt->rm_type = REMOTE_NDIS_PACKET_MSG; 3366 pkt->rm_len = m_head->m_pkthdr.len; 3367 pkt->rm_dataoffset = 0; 3368 pkt->rm_datalen = m_head->m_pkthdr.len; 3369 pkt->rm_oobdataoffset = 0; 3370 pkt->rm_oobdatalen = 0; 3371 pkt->rm_oobdataelements = 0; 3372 pkt->rm_pktinfooffset = sizeof(*pkt); 3373 pkt->rm_pktinfolen = 0; 3374 pkt->rm_vchandle = 0; 3375 pkt->rm_reserved = 0; 3376 3377 if (txr->hn_tx_flags & HN_TX_FLAG_HASHVAL) { 3378 /* 3379 * Set the hash value for this packet. 3380 */ 3381 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3382 HN_NDIS_HASH_VALUE_SIZE, HN_NDIS_PKTINFO_TYPE_HASHVAL); 3383 3384 if (M_HASHTYPE_ISHASH(m_head)) 3385 /* 3386 * The flowid field contains the hash value host 3387 * set in the rx queue if it is a ip forwarding pkt. 3388 * Set the same hash value so host can send on the 3389 * cpu it was received. 3390 */ 3391 *pi_data = m_head->m_pkthdr.flowid; 3392 else 3393 /* 3394 * Otherwise just put the tx queue index. 3395 */ 3396 *pi_data = txr->hn_tx_idx; 3397 } 3398 3399 if (m_head->m_flags & M_VLANTAG) { 3400 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3401 NDIS_VLAN_INFO_SIZE, NDIS_PKTINFO_TYPE_VLAN); 3402 *pi_data = NDIS_VLAN_INFO_MAKE( 3403 EVL_VLANOFTAG(m_head->m_pkthdr.ether_vtag), 3404 EVL_PRIOFTAG(m_head->m_pkthdr.ether_vtag), 3405 EVL_CFIOFTAG(m_head->m_pkthdr.ether_vtag)); 3406 } 3407 3408 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 3409 #if defined(INET6) || defined(INET) 3410 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3411 NDIS_LSO2_INFO_SIZE, NDIS_PKTINFO_TYPE_LSO); 3412 #ifdef INET 3413 if (m_head->m_pkthdr.csum_flags & CSUM_IP_TSO) { 3414 *pi_data = NDIS_LSO2_INFO_MAKEIPV4( 3415 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen, 3416 m_head->m_pkthdr.tso_segsz); 3417 } 3418 #endif 3419 #if defined(INET6) && defined(INET) 3420 else 3421 #endif 3422 #ifdef INET6 3423 { 3424 *pi_data = NDIS_LSO2_INFO_MAKEIPV6( 3425 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen, 3426 m_head->m_pkthdr.tso_segsz); 3427 } 3428 #endif 3429 #endif /* INET6 || INET */ 3430 } else if (m_head->m_pkthdr.csum_flags & txr->hn_csum_assist) { 3431 pi_data = hn_rndis_pktinfo_append(pkt, HN_RNDIS_PKT_LEN, 3432 NDIS_TXCSUM_INFO_SIZE, NDIS_PKTINFO_TYPE_CSUM); 3433 if (m_head->m_pkthdr.csum_flags & 3434 (CSUM_IP6_TCP | CSUM_IP6_UDP)) { 3435 *pi_data = NDIS_TXCSUM_INFO_IPV6; 3436 } else { 3437 *pi_data = NDIS_TXCSUM_INFO_IPV4; 3438 if (m_head->m_pkthdr.csum_flags & CSUM_IP) 3439 *pi_data |= NDIS_TXCSUM_INFO_IPCS; 3440 } 3441 3442 if (m_head->m_pkthdr.csum_flags & 3443 (CSUM_IP_TCP | CSUM_IP6_TCP)) { 3444 *pi_data |= NDIS_TXCSUM_INFO_MKTCPCS( 3445 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen); 3446 } else if (m_head->m_pkthdr.csum_flags & 3447 (CSUM_IP_UDP | CSUM_IP6_UDP)) { 3448 *pi_data |= NDIS_TXCSUM_INFO_MKUDPCS( 3449 m_head->m_pkthdr.l2hlen + m_head->m_pkthdr.l3hlen); 3450 } 3451 } 3452 3453 pkt_hlen = pkt->rm_pktinfooffset + pkt->rm_pktinfolen; 3454 /* Fixup RNDIS packet message total length */ 3455 pkt->rm_len += pkt_hlen; 3456 /* Convert RNDIS packet message offsets */ 3457 pkt->rm_dataoffset = hn_rndis_pktmsg_offset(pkt_hlen); 3458 pkt->rm_pktinfooffset = hn_rndis_pktmsg_offset(pkt->rm_pktinfooffset); 3459 3460 /* 3461 * Fast path: Chimney sending. 3462 */ 3463 if (chim != NULL) { 3464 struct hn_txdesc *tgt_txd = txd; 3465 3466 if (txr->hn_agg_txd != NULL) { 3467 tgt_txd = txr->hn_agg_txd; 3468 #ifdef INVARIANTS 3469 *m_head0 = NULL; 3470 #endif 3471 } 3472 3473 KASSERT(pkt == chim, 3474 ("RNDIS pkt not in chimney sending buffer")); 3475 KASSERT(tgt_txd->chim_index != HN_NVS_CHIM_IDX_INVALID, 3476 ("chimney sending buffer is not used")); 3477 tgt_txd->chim_size += pkt->rm_len; 3478 3479 m_copydata(m_head, 0, m_head->m_pkthdr.len, 3480 ((uint8_t *)chim) + pkt_hlen); 3481 3482 txr->hn_gpa_cnt = 0; 3483 txr->hn_sendpkt = hn_txpkt_chim; 3484 goto done; 3485 } 3486 3487 KASSERT(txr->hn_agg_txd == NULL, ("aggregating sglist txdesc")); 3488 KASSERT(txd->chim_index == HN_NVS_CHIM_IDX_INVALID, 3489 ("chimney buffer is used")); 3490 KASSERT(pkt == txd->rndis_pkt, ("RNDIS pkt not in txdesc")); 3491 3492 error = hn_txdesc_dmamap_load(txr, txd, &m_head, segs, &nsegs); 3493 if (__predict_false(error)) { 3494 int freed __diagused; 3495 3496 /* 3497 * This mbuf is not linked w/ the txd yet, so free it now. 3498 */ 3499 m_freem(m_head); 3500 *m_head0 = NULL; 3501 3502 freed = hn_txdesc_put(txr, txd); 3503 KASSERT(freed != 0, 3504 ("fail to free txd upon txdma error")); 3505 3506 txr->hn_txdma_failed++; 3507 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 3508 return error; 3509 } 3510 *m_head0 = m_head; 3511 3512 /* +1 RNDIS packet message */ 3513 txr->hn_gpa_cnt = nsegs + 1; 3514 3515 /* send packet with page buffer */ 3516 txr->hn_gpa[0].gpa_page = atop(txd->rndis_pkt_paddr); 3517 txr->hn_gpa[0].gpa_ofs = txd->rndis_pkt_paddr & PAGE_MASK; 3518 txr->hn_gpa[0].gpa_len = pkt_hlen; 3519 3520 /* 3521 * Fill the page buffers with mbuf info after the page 3522 * buffer for RNDIS packet message. 3523 */ 3524 for (i = 0; i < nsegs; ++i) { 3525 struct vmbus_gpa *gpa = &txr->hn_gpa[i + 1]; 3526 3527 gpa->gpa_page = atop(segs[i].ds_addr); 3528 gpa->gpa_ofs = segs[i].ds_addr & PAGE_MASK; 3529 gpa->gpa_len = segs[i].ds_len; 3530 } 3531 3532 txd->chim_index = HN_NVS_CHIM_IDX_INVALID; 3533 txd->chim_size = 0; 3534 txr->hn_sendpkt = hn_txpkt_sglist; 3535 done: 3536 txd->m = m_head; 3537 3538 /* Set the completion routine */ 3539 hn_nvs_sendctx_init(&txd->send_ctx, hn_txpkt_done, txd); 3540 3541 /* Update temporary stats for later use. */ 3542 txr->hn_stat_pkts++; 3543 txr->hn_stat_size += m_head->m_pkthdr.len; 3544 if (m_head->m_flags & M_MCAST) 3545 txr->hn_stat_mcasts++; 3546 3547 return 0; 3548 } 3549 3550 /* 3551 * NOTE: 3552 * If this function fails, then txd will be freed, but the mbuf 3553 * associated w/ the txd will _not_ be freed. 3554 */ 3555 static int 3556 hn_txpkt(if_t ifp, struct hn_tx_ring *txr, struct hn_txdesc *txd) 3557 { 3558 int error, send_failed = 0, has_bpf; 3559 3560 again: 3561 has_bpf = bpf_peers_present_if(ifp); 3562 if (has_bpf) { 3563 /* 3564 * Make sure that this txd and any aggregated txds are not 3565 * freed before ETHER_BPF_MTAP. 3566 */ 3567 hn_txdesc_hold(txd); 3568 } 3569 error = txr->hn_sendpkt(txr, txd); 3570 if (!error) { 3571 if (has_bpf) { 3572 const struct hn_txdesc *tmp_txd; 3573 3574 ETHER_BPF_MTAP(ifp, txd->m); 3575 STAILQ_FOREACH(tmp_txd, &txd->agg_list, agg_link) 3576 ETHER_BPF_MTAP(ifp, tmp_txd->m); 3577 } 3578 3579 if_inc_counter(ifp, IFCOUNTER_OPACKETS, txr->hn_stat_pkts); 3580 #ifdef HN_IFSTART_SUPPORT 3581 if (!hn_use_if_start) 3582 #endif 3583 { 3584 if_inc_counter(ifp, IFCOUNTER_OBYTES, 3585 txr->hn_stat_size); 3586 if (txr->hn_stat_mcasts != 0) { 3587 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 3588 txr->hn_stat_mcasts); 3589 } 3590 } 3591 txr->hn_pkts += txr->hn_stat_pkts; 3592 txr->hn_sends++; 3593 } 3594 if (has_bpf) 3595 hn_txdesc_put(txr, txd); 3596 3597 if (__predict_false(error)) { 3598 int freed __diagused; 3599 3600 /* 3601 * This should "really rarely" happen. 3602 * 3603 * XXX Too many RX to be acked or too many sideband 3604 * commands to run? Ask netvsc_channel_rollup() 3605 * to kick start later. 3606 */ 3607 txr->hn_has_txeof = 1; 3608 if (!send_failed) { 3609 txr->hn_send_failed++; 3610 send_failed = 1; 3611 /* 3612 * Try sending again after set hn_has_txeof; 3613 * in case that we missed the last 3614 * netvsc_channel_rollup(). 3615 */ 3616 goto again; 3617 } 3618 if_printf(ifp, "send failed\n"); 3619 3620 /* 3621 * Caller will perform further processing on the 3622 * associated mbuf, so don't free it in hn_txdesc_put(); 3623 * only unload it from the DMA map in hn_txdesc_put(), 3624 * if it was loaded. 3625 */ 3626 txd->m = NULL; 3627 freed = hn_txdesc_put(txr, txd); 3628 KASSERT(freed != 0, 3629 ("fail to free txd upon send error")); 3630 3631 txr->hn_send_failed++; 3632 } 3633 3634 /* Reset temporary stats, after this sending is done. */ 3635 txr->hn_stat_size = 0; 3636 txr->hn_stat_pkts = 0; 3637 txr->hn_stat_mcasts = 0; 3638 3639 return (error); 3640 } 3641 3642 /* 3643 * Append the specified data to the indicated mbuf chain, 3644 * Extend the mbuf chain if the new data does not fit in 3645 * existing space. 3646 * 3647 * This is a minor rewrite of m_append() from sys/kern/uipc_mbuf.c. 3648 * There should be an equivalent in the kernel mbuf code, 3649 * but there does not appear to be one yet. 3650 * 3651 * Differs from m_append() in that additional mbufs are 3652 * allocated with cluster size MJUMPAGESIZE, and filled 3653 * accordingly. 3654 * 3655 * Return the last mbuf in the chain or NULL if failed to 3656 * allocate new mbuf. 3657 */ 3658 static struct mbuf * 3659 hv_m_append(struct mbuf *m0, int len, c_caddr_t cp) 3660 { 3661 struct mbuf *m, *n; 3662 int remainder, space; 3663 3664 for (m = m0; m->m_next != NULL; m = m->m_next) 3665 ; 3666 remainder = len; 3667 space = M_TRAILINGSPACE(m); 3668 if (space > 0) { 3669 /* 3670 * Copy into available space. 3671 */ 3672 if (space > remainder) 3673 space = remainder; 3674 bcopy(cp, mtod(m, caddr_t) + m->m_len, space); 3675 m->m_len += space; 3676 cp += space; 3677 remainder -= space; 3678 } 3679 while (remainder > 0) { 3680 /* 3681 * Allocate a new mbuf; could check space 3682 * and allocate a cluster instead. 3683 */ 3684 n = m_getjcl(M_NOWAIT, m->m_type, 0, MJUMPAGESIZE); 3685 if (n == NULL) 3686 return NULL; 3687 n->m_len = min(MJUMPAGESIZE, remainder); 3688 bcopy(cp, mtod(n, caddr_t), n->m_len); 3689 cp += n->m_len; 3690 remainder -= n->m_len; 3691 m->m_next = n; 3692 m = n; 3693 } 3694 3695 return m; 3696 } 3697 3698 #if defined(INET) || defined(INET6) 3699 static __inline int 3700 hn_lro_rx(struct lro_ctrl *lc, struct mbuf *m) 3701 { 3702 if (hn_lro_mbufq_depth) { 3703 tcp_lro_queue_mbuf(lc, m); 3704 return 0; 3705 } 3706 return tcp_lro_rx(lc, m, 0); 3707 } 3708 #endif 3709 3710 static int 3711 hn_rxpkt(struct hn_rx_ring *rxr) 3712 { 3713 if_t ifp, hn_ifp = rxr->hn_ifp; 3714 struct mbuf *m_new, *n; 3715 int size, do_lro = 0, do_csum = 1, is_vf = 0; 3716 int hash_type = M_HASHTYPE_NONE; 3717 int l3proto = ETHERTYPE_MAX, l4proto = IPPROTO_DONE; 3718 int i; 3719 3720 ifp = hn_ifp; 3721 if (rxr->hn_rxvf_ifp != NULL) { 3722 /* 3723 * Non-transparent mode VF; pretend this packet is from 3724 * the VF. 3725 */ 3726 ifp = rxr->hn_rxvf_ifp; 3727 is_vf = 1; 3728 } else if (rxr->hn_rx_flags & HN_RX_FLAG_XPNT_VF) { 3729 /* Transparent mode VF. */ 3730 is_vf = 1; 3731 } 3732 3733 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 3734 /* 3735 * NOTE: 3736 * See the NOTE of hn_rndis_init_fixat(). This 3737 * function can be reached, immediately after the 3738 * RNDIS is initialized but before the ifnet is 3739 * setup on the hn_attach() path; drop the unexpected 3740 * packets. 3741 */ 3742 return (0); 3743 } 3744 3745 if (__predict_false(rxr->rsc.pktlen < ETHER_HDR_LEN)) { 3746 if_inc_counter(hn_ifp, IFCOUNTER_IERRORS, 1); 3747 return (0); 3748 } 3749 3750 if (rxr->rsc.cnt == 1 && rxr->rsc.pktlen <= MHLEN) { 3751 m_new = m_gethdr(M_NOWAIT, MT_DATA); 3752 if (m_new == NULL) { 3753 if_inc_counter(hn_ifp, IFCOUNTER_IQDROPS, 1); 3754 return (0); 3755 } 3756 memcpy(mtod(m_new, void *), rxr->rsc.frag_data[0], 3757 rxr->rsc.frag_len[0]); 3758 m_new->m_pkthdr.len = m_new->m_len = rxr->rsc.frag_len[0]; 3759 } else { 3760 /* 3761 * Get an mbuf with a cluster. For packets 2K or less, 3762 * get a standard 2K cluster. For anything larger, get a 3763 * 4K cluster. Any buffers larger than 4K can cause problems 3764 * if looped around to the Hyper-V TX channel, so avoid them. 3765 */ 3766 size = MCLBYTES; 3767 if (rxr->rsc.pktlen > MCLBYTES) { 3768 /* 4096 */ 3769 size = MJUMPAGESIZE; 3770 } 3771 3772 m_new = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size); 3773 if (m_new == NULL) { 3774 if_inc_counter(hn_ifp, IFCOUNTER_IQDROPS, 1); 3775 return (0); 3776 } 3777 3778 n = m_new; 3779 for (i = 0; i < rxr->rsc.cnt; i++) { 3780 n = hv_m_append(n, rxr->rsc.frag_len[i], 3781 rxr->rsc.frag_data[i]); 3782 if (n == NULL) { 3783 if_inc_counter(hn_ifp, IFCOUNTER_IQDROPS, 1); 3784 return (0); 3785 } else { 3786 m_new->m_pkthdr.len += rxr->rsc.frag_len[i]; 3787 } 3788 } 3789 } 3790 if (rxr->rsc.pktlen <= MHLEN) 3791 rxr->hn_small_pkts++; 3792 3793 m_new->m_pkthdr.rcvif = ifp; 3794 3795 if (__predict_false((if_getcapenable(hn_ifp) & IFCAP_RXCSUM) == 0)) 3796 do_csum = 0; 3797 3798 /* receive side checksum offload */ 3799 if (rxr->rsc.csum_info != NULL) { 3800 /* IP csum offload */ 3801 if ((*(rxr->rsc.csum_info) & NDIS_RXCSUM_INFO_IPCS_OK) && do_csum) { 3802 m_new->m_pkthdr.csum_flags |= 3803 (CSUM_IP_CHECKED | CSUM_IP_VALID); 3804 rxr->hn_csum_ip++; 3805 } 3806 3807 /* TCP/UDP csum offload */ 3808 if ((*(rxr->rsc.csum_info) & (NDIS_RXCSUM_INFO_UDPCS_OK | 3809 NDIS_RXCSUM_INFO_TCPCS_OK)) && do_csum) { 3810 m_new->m_pkthdr.csum_flags |= 3811 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3812 m_new->m_pkthdr.csum_data = 0xffff; 3813 if (*(rxr->rsc.csum_info) & NDIS_RXCSUM_INFO_TCPCS_OK) 3814 rxr->hn_csum_tcp++; 3815 else 3816 rxr->hn_csum_udp++; 3817 } 3818 3819 /* 3820 * XXX 3821 * As of this write (Oct 28th, 2016), host side will turn 3822 * on only TCPCS_OK and IPCS_OK even for UDP datagrams, so 3823 * the do_lro setting here is actually _not_ accurate. We 3824 * depend on the RSS hash type check to reset do_lro. 3825 */ 3826 if ((*(rxr->rsc.csum_info) & 3827 (NDIS_RXCSUM_INFO_TCPCS_OK | NDIS_RXCSUM_INFO_IPCS_OK)) == 3828 (NDIS_RXCSUM_INFO_TCPCS_OK | NDIS_RXCSUM_INFO_IPCS_OK)) 3829 do_lro = 1; 3830 } else { 3831 hn_rxpkt_proto(m_new, &l3proto, &l4proto); 3832 if (l3proto == ETHERTYPE_IP) { 3833 if (l4proto == IPPROTO_TCP) { 3834 if (do_csum && 3835 (rxr->hn_trust_hcsum & 3836 HN_TRUST_HCSUM_TCP)) { 3837 rxr->hn_csum_trusted++; 3838 m_new->m_pkthdr.csum_flags |= 3839 (CSUM_IP_CHECKED | CSUM_IP_VALID | 3840 CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3841 m_new->m_pkthdr.csum_data = 0xffff; 3842 } 3843 do_lro = 1; 3844 } else if (l4proto == IPPROTO_UDP) { 3845 if (do_csum && 3846 (rxr->hn_trust_hcsum & 3847 HN_TRUST_HCSUM_UDP)) { 3848 rxr->hn_csum_trusted++; 3849 m_new->m_pkthdr.csum_flags |= 3850 (CSUM_IP_CHECKED | CSUM_IP_VALID | 3851 CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3852 m_new->m_pkthdr.csum_data = 0xffff; 3853 } 3854 } else if (l4proto != IPPROTO_DONE && do_csum && 3855 (rxr->hn_trust_hcsum & HN_TRUST_HCSUM_IP)) { 3856 rxr->hn_csum_trusted++; 3857 m_new->m_pkthdr.csum_flags |= 3858 (CSUM_IP_CHECKED | CSUM_IP_VALID); 3859 } 3860 } 3861 } 3862 3863 if (rxr->rsc.vlan_info != NULL) { 3864 m_new->m_pkthdr.ether_vtag = EVL_MAKETAG( 3865 NDIS_VLAN_INFO_ID(*(rxr->rsc.vlan_info)), 3866 NDIS_VLAN_INFO_PRI(*(rxr->rsc.vlan_info)), 3867 NDIS_VLAN_INFO_CFI(*(rxr->rsc.vlan_info))); 3868 m_new->m_flags |= M_VLANTAG; 3869 } 3870 3871 /* 3872 * If VF is activated (transparent/non-transparent mode does not 3873 * matter here). 3874 * 3875 * - Disable LRO 3876 * 3877 * hn(4) will only receive broadcast packets, multicast packets, 3878 * TCP SYN and SYN|ACK (in Azure), LRO is useless for these 3879 * packet types. 3880 * 3881 * For non-transparent, we definitely _cannot_ enable LRO at 3882 * all, since the LRO flush will use hn(4) as the receiving 3883 * interface; i.e. hn_ifp->if_input(hn_ifp, m). 3884 */ 3885 if (is_vf) 3886 do_lro = 0; 3887 3888 /* 3889 * If VF is activated (transparent/non-transparent mode does not 3890 * matter here), do _not_ mess with unsupported hash types or 3891 * functions. 3892 */ 3893 if (rxr->rsc.hash_info != NULL) { 3894 rxr->hn_rss_pkts++; 3895 m_new->m_pkthdr.flowid = *(rxr->rsc.hash_value); 3896 if (!is_vf) 3897 hash_type = M_HASHTYPE_OPAQUE_HASH; 3898 if ((*(rxr->rsc.hash_info) & NDIS_HASH_FUNCTION_MASK) == 3899 NDIS_HASH_FUNCTION_TOEPLITZ) { 3900 uint32_t type = (*(rxr->rsc.hash_info) & NDIS_HASH_TYPE_MASK & 3901 rxr->hn_mbuf_hash); 3902 3903 /* 3904 * NOTE: 3905 * do_lro is resetted, if the hash types are not TCP 3906 * related. See the comment in the above csum_flags 3907 * setup section. 3908 */ 3909 switch (type) { 3910 case NDIS_HASH_IPV4: 3911 hash_type = M_HASHTYPE_RSS_IPV4; 3912 do_lro = 0; 3913 break; 3914 3915 case NDIS_HASH_TCP_IPV4: 3916 hash_type = M_HASHTYPE_RSS_TCP_IPV4; 3917 if (rxr->hn_rx_flags & HN_RX_FLAG_UDP_HASH) { 3918 int def_htype = M_HASHTYPE_OPAQUE_HASH; 3919 3920 if (is_vf) 3921 def_htype = M_HASHTYPE_NONE; 3922 3923 /* 3924 * UDP 4-tuple hash is delivered as 3925 * TCP 4-tuple hash. 3926 */ 3927 if (l3proto == ETHERTYPE_MAX) { 3928 hn_rxpkt_proto(m_new, 3929 &l3proto, &l4proto); 3930 } 3931 if (l3proto == ETHERTYPE_IP) { 3932 if (l4proto == IPPROTO_UDP && 3933 (rxr->hn_mbuf_hash & 3934 NDIS_HASH_UDP_IPV4_X)) { 3935 hash_type = 3936 M_HASHTYPE_RSS_UDP_IPV4; 3937 do_lro = 0; 3938 } else if (l4proto != 3939 IPPROTO_TCP) { 3940 hash_type = def_htype; 3941 do_lro = 0; 3942 } 3943 } else { 3944 hash_type = def_htype; 3945 do_lro = 0; 3946 } 3947 } 3948 break; 3949 3950 case NDIS_HASH_IPV6: 3951 hash_type = M_HASHTYPE_RSS_IPV6; 3952 do_lro = 0; 3953 break; 3954 3955 case NDIS_HASH_IPV6_EX: 3956 hash_type = M_HASHTYPE_RSS_IPV6_EX; 3957 do_lro = 0; 3958 break; 3959 3960 case NDIS_HASH_TCP_IPV6: 3961 hash_type = M_HASHTYPE_RSS_TCP_IPV6; 3962 break; 3963 3964 case NDIS_HASH_TCP_IPV6_EX: 3965 hash_type = M_HASHTYPE_RSS_TCP_IPV6_EX; 3966 break; 3967 } 3968 } 3969 } else if (!is_vf) { 3970 m_new->m_pkthdr.flowid = rxr->hn_rx_idx; 3971 hash_type = M_HASHTYPE_OPAQUE; 3972 } 3973 M_HASHTYPE_SET(m_new, hash_type); 3974 3975 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 3976 if (hn_ifp != ifp) { 3977 const struct ether_header *eh; 3978 3979 /* 3980 * Non-transparent mode VF is activated. 3981 */ 3982 3983 /* 3984 * Allow tapping on hn(4). 3985 */ 3986 ETHER_BPF_MTAP(hn_ifp, m_new); 3987 3988 /* 3989 * Update hn(4)'s stats. 3990 */ 3991 if_inc_counter(hn_ifp, IFCOUNTER_IPACKETS, 1); 3992 if_inc_counter(hn_ifp, IFCOUNTER_IBYTES, m_new->m_pkthdr.len); 3993 /* Checked at the beginning of this function. */ 3994 KASSERT(m_new->m_len >= ETHER_HDR_LEN, ("not ethernet frame")); 3995 eh = mtod(m_new, struct ether_header *); 3996 if (ETHER_IS_MULTICAST(eh->ether_dhost)) 3997 if_inc_counter(hn_ifp, IFCOUNTER_IMCASTS, 1); 3998 } 3999 rxr->hn_pkts++; 4000 4001 if ((if_getcapenable(hn_ifp) & IFCAP_LRO) && do_lro) { 4002 #if defined(INET) || defined(INET6) 4003 struct lro_ctrl *lro = &rxr->hn_lro; 4004 4005 if (lro->lro_cnt) { 4006 rxr->hn_lro_tried++; 4007 if (hn_lro_rx(lro, m_new) == 0) { 4008 /* DONE! */ 4009 return 0; 4010 } 4011 } 4012 #endif 4013 } 4014 if_input(ifp, m_new); 4015 4016 return (0); 4017 } 4018 4019 static int 4020 hn_ioctl(if_t ifp, u_long cmd, caddr_t data) 4021 { 4022 struct hn_softc *sc = if_getsoftc(ifp); 4023 struct ifreq *ifr = (struct ifreq *)data, ifr_vf; 4024 if_t vf_ifp; 4025 int mask, error = 0; 4026 struct ifrsskey *ifrk; 4027 struct ifrsshash *ifrh; 4028 uint32_t mtu; 4029 4030 switch (cmd) { 4031 case SIOCSIFMTU: 4032 if (ifr->ifr_mtu > HN_MTU_MAX) { 4033 error = EINVAL; 4034 break; 4035 } 4036 4037 HN_LOCK(sc); 4038 4039 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) { 4040 HN_UNLOCK(sc); 4041 break; 4042 } 4043 4044 if ((sc->hn_caps & HN_CAP_MTU) == 0) { 4045 /* Can't change MTU */ 4046 HN_UNLOCK(sc); 4047 error = EOPNOTSUPP; 4048 break; 4049 } 4050 4051 if (if_getmtu(ifp) == ifr->ifr_mtu) { 4052 HN_UNLOCK(sc); 4053 break; 4054 } 4055 4056 if (hn_xpnt_vf_isready(sc)) { 4057 vf_ifp = sc->hn_vf_ifp; 4058 ifr_vf = *ifr; 4059 strlcpy(ifr_vf.ifr_name, if_name(vf_ifp), 4060 sizeof(ifr_vf.ifr_name)); 4061 error = ifhwioctl(SIOCSIFMTU, vf_ifp, 4062 (caddr_t)&ifr_vf, curthread); 4063 HN_UNLOCK(sc); 4064 if (error) { 4065 if_printf(ifp, "%s SIOCSIFMTU %d failed: %d\n", 4066 if_name(vf_ifp), ifr->ifr_mtu, error); 4067 } else { 4068 if_setmtu(ifp, ifr->ifr_mtu); 4069 } 4070 break; 4071 } 4072 4073 /* 4074 * Suspend this interface before the synthetic parts 4075 * are ripped. 4076 */ 4077 if (sc->hn_vf_ready) 4078 hn_xpnt_vf_deactivate(sc); 4079 if (sc->hn_flags & HN_FLAG_RXVF) { 4080 hn_rxvf_change_locked(sc, sc->hn_vf_ifp, false); 4081 if (sc->hn_flags & HN_FLAG_RXVF) { 4082 error = EIO; 4083 HN_UNLOCK(sc); 4084 break; 4085 } 4086 } 4087 hn_suspend(sc); 4088 4089 /* 4090 * Detach the synthetics parts, i.e. NVS and RNDIS. 4091 */ 4092 hn_synth_detach(sc); 4093 4094 /* 4095 * Reattach the synthetic parts, i.e. NVS and RNDIS, 4096 * with the new MTU setting. 4097 */ 4098 error = hn_synth_attach(sc, ifr->ifr_mtu); 4099 if (error) { 4100 HN_UNLOCK(sc); 4101 break; 4102 } 4103 4104 error = hn_rndis_get_mtu(sc, &mtu); 4105 if (error) 4106 mtu = ifr->ifr_mtu; 4107 else if (bootverbose) 4108 if_printf(ifp, "RNDIS mtu %u\n", mtu); 4109 4110 /* 4111 * Commit the requested MTU, after the synthetic parts 4112 * have been successfully attached. 4113 */ 4114 if (mtu >= ifr->ifr_mtu) { 4115 mtu = ifr->ifr_mtu; 4116 } else { 4117 if_printf(ifp, "fixup mtu %d -> %u\n", 4118 ifr->ifr_mtu, mtu); 4119 } 4120 if_setmtu(ifp, mtu); 4121 4122 /* 4123 * Synthetic parts' reattach may change the chimney 4124 * sending size; update it. 4125 */ 4126 if (sc->hn_tx_ring[0].hn_chim_size > sc->hn_chim_szmax) 4127 hn_set_chim_size(sc, sc->hn_chim_szmax); 4128 4129 /* 4130 * Make sure that various parameters based on MTU are 4131 * still valid, after the MTU change. 4132 */ 4133 hn_mtu_change_fixup(sc); 4134 4135 /* 4136 * All done! Resume the interface now. 4137 */ 4138 hn_resume(sc); 4139 4140 /* Reattach requires a fresh association and acknowledged switch. */ 4141 if (sc->hn_vf_ifp != NULL) 4142 taskqueue_enqueue_timeout(sc->hn_vf_taskq, 4143 &sc->hn_vf_init, 0); 4144 4145 HN_UNLOCK(sc); 4146 break; 4147 4148 case SIOCSIFFLAGS: 4149 HN_LOCK(sc); 4150 4151 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) { 4152 HN_UNLOCK(sc); 4153 break; 4154 } 4155 4156 if (hn_xpnt_vf_isready(sc)) 4157 hn_xpnt_vf_saveifflags(sc); 4158 4159 if (if_getflags(ifp) & IFF_UP) { 4160 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 4161 /* 4162 * Caller meight hold mutex, e.g. 4163 * bpf; use busy-wait for the RNDIS 4164 * reply. 4165 */ 4166 HN_NO_SLEEPING(sc); 4167 hn_rxfilter_config(sc); 4168 HN_SLEEPING_OK(sc); 4169 4170 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 4171 error = hn_xpnt_vf_iocsetflags(sc); 4172 } else { 4173 hn_init_locked(sc); 4174 } 4175 } else { 4176 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 4177 hn_stop(sc, false); 4178 } 4179 sc->hn_if_flags = if_getflags(ifp); 4180 4181 HN_UNLOCK(sc); 4182 break; 4183 4184 case SIOCSIFCAP: 4185 HN_LOCK(sc); 4186 if (sc->hn_detaching || sc->hn_vf_caps_busy) { 4187 error = sc->hn_vf_caps_busy ? EBUSY : ENXIO; 4188 HN_UNLOCK(sc); 4189 break; 4190 } 4191 4192 if (hn_xpnt_vf_isready(sc)) { 4193 ifr_vf = *ifr; 4194 strlcpy(ifr_vf.ifr_name, if_name(sc->hn_vf_ifp), 4195 sizeof(ifr_vf.ifr_name)); 4196 error = hn_xpnt_vf_iocsetcaps(sc, &ifr_vf); 4197 HN_UNLOCK(sc); 4198 break; 4199 } 4200 4201 /* 4202 * Fix up requested capabilities w/ supported capabilities, 4203 * since the supported capabilities could have been changed. 4204 */ 4205 mask = (ifr->ifr_reqcap & if_getcapabilities(ifp)) ^ 4206 if_getcapenable(ifp); 4207 4208 if (mask & IFCAP_TXCSUM) { 4209 if_togglecapenable(ifp, IFCAP_TXCSUM); 4210 if (if_getcapenable(ifp) & IFCAP_TXCSUM) 4211 if_sethwassistbits(ifp, HN_CSUM_IP_HWASSIST(sc), 0); 4212 else 4213 if_sethwassistbits(ifp, 0, HN_CSUM_IP_HWASSIST(sc)); 4214 } 4215 if (mask & IFCAP_TXCSUM_IPV6) { 4216 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); 4217 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) 4218 if_sethwassistbits(ifp, HN_CSUM_IP6_HWASSIST(sc), 0); 4219 else 4220 if_sethwassistbits(ifp, 0, HN_CSUM_IP6_HWASSIST(sc)); 4221 } 4222 4223 /* TODO: flip RNDIS offload parameters for RXCSUM. */ 4224 if (mask & IFCAP_RXCSUM) 4225 if_togglecapenable(ifp, IFCAP_RXCSUM); 4226 #ifdef foo 4227 /* We can't diff IPv6 packets from IPv4 packets on RX path. */ 4228 if (mask & IFCAP_RXCSUM_IPV6) 4229 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); 4230 #endif 4231 4232 if (mask & IFCAP_LRO) 4233 if_togglecapenable(ifp, IFCAP_LRO); 4234 4235 if (mask & IFCAP_TSO4) { 4236 if_togglecapenable(ifp, IFCAP_TSO4); 4237 if (if_getcapenable(ifp) & IFCAP_TSO4) 4238 if_sethwassistbits(ifp, CSUM_IP_TSO, 0); 4239 else 4240 if_sethwassistbits(ifp, 0, CSUM_IP_TSO); 4241 } 4242 if (mask & IFCAP_TSO6) { 4243 if_togglecapenable(ifp, IFCAP_TSO6); 4244 if (if_getcapenable(ifp) & IFCAP_TSO6) 4245 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); 4246 else 4247 if_sethwassistbits(ifp, 0, CSUM_IP6_TSO); 4248 } 4249 4250 HN_UNLOCK(sc); 4251 break; 4252 4253 case SIOCADDMULTI: 4254 case SIOCDELMULTI: 4255 HN_LOCK(sc); 4256 4257 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) { 4258 HN_UNLOCK(sc); 4259 break; 4260 } 4261 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 4262 /* 4263 * Multicast uses mutex; use busy-wait for 4264 * the RNDIS reply. 4265 */ 4266 HN_NO_SLEEPING(sc); 4267 hn_rxfilter_config(sc); 4268 HN_SLEEPING_OK(sc); 4269 } 4270 4271 /* XXX vlan(4) style mcast addr maintenance */ 4272 if (hn_xpnt_vf_isready(sc)) { 4273 int old_if_flags; 4274 4275 old_if_flags = if_getflags(sc->hn_vf_ifp); 4276 hn_xpnt_vf_saveifflags(sc); 4277 4278 if ((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) && 4279 ((old_if_flags ^ if_getflags(sc->hn_vf_ifp)) & 4280 IFF_ALLMULTI)) 4281 error = hn_xpnt_vf_iocsetflags(sc); 4282 } 4283 4284 HN_UNLOCK(sc); 4285 break; 4286 4287 case SIOCSIFMEDIA: 4288 case SIOCGIFMEDIA: 4289 HN_LOCK(sc); 4290 if (hn_xpnt_vf_isready(sc)) { 4291 /* 4292 * SIOCGIFMEDIA expects ifmediareq, so don't 4293 * create and pass ifr_vf to the VF here; just 4294 * replace the ifr_name. 4295 */ 4296 vf_ifp = sc->hn_vf_ifp; 4297 strlcpy(ifr->ifr_name, if_name(vf_ifp), 4298 sizeof(ifr->ifr_name)); 4299 error = ifhwioctl(cmd, vf_ifp, data, curthread); 4300 /* Restore the ifr_name. */ 4301 strlcpy(ifr->ifr_name, if_name(ifp), 4302 sizeof(ifr->ifr_name)); 4303 HN_UNLOCK(sc); 4304 break; 4305 } 4306 HN_UNLOCK(sc); 4307 error = ifmedia_ioctl(ifp, ifr, &sc->hn_media, cmd); 4308 break; 4309 4310 case SIOCGIFRSSHASH: 4311 ifrh = (struct ifrsshash *)data; 4312 HN_LOCK(sc); 4313 if (sc->hn_rx_ring_inuse == 1) { 4314 HN_UNLOCK(sc); 4315 ifrh->ifrh_func = RSS_FUNC_NONE; 4316 ifrh->ifrh_types = 0; 4317 break; 4318 } 4319 4320 if (sc->hn_rss_hash & NDIS_HASH_FUNCTION_TOEPLITZ) 4321 ifrh->ifrh_func = RSS_FUNC_TOEPLITZ; 4322 else 4323 ifrh->ifrh_func = RSS_FUNC_PRIVATE; 4324 ifrh->ifrh_types = hn_rss_type_fromndis(sc->hn_rss_hash); 4325 HN_UNLOCK(sc); 4326 break; 4327 4328 case SIOCGIFRSSKEY: 4329 ifrk = (struct ifrsskey *)data; 4330 HN_LOCK(sc); 4331 if (sc->hn_rx_ring_inuse == 1) { 4332 HN_UNLOCK(sc); 4333 ifrk->ifrk_func = RSS_FUNC_NONE; 4334 ifrk->ifrk_keylen = 0; 4335 break; 4336 } 4337 if (sc->hn_rss_hash & NDIS_HASH_FUNCTION_TOEPLITZ) 4338 ifrk->ifrk_func = RSS_FUNC_TOEPLITZ; 4339 else 4340 ifrk->ifrk_func = RSS_FUNC_PRIVATE; 4341 ifrk->ifrk_keylen = NDIS_HASH_KEYSIZE_TOEPLITZ; 4342 memcpy(ifrk->ifrk_key, sc->hn_rss.rss_key, 4343 NDIS_HASH_KEYSIZE_TOEPLITZ); 4344 HN_UNLOCK(sc); 4345 break; 4346 4347 default: 4348 error = ether_ioctl(ifp, cmd, data); 4349 break; 4350 } 4351 return (error); 4352 } 4353 4354 static void 4355 hn_stop(struct hn_softc *sc, bool detaching) 4356 { 4357 if_t ifp = sc->hn_ifp; 4358 int i; 4359 4360 HN_LOCK_ASSERT(sc); 4361 4362 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 4363 ("synthetic parts were not attached")); 4364 4365 /* Clear RUNNING bit ASAP. */ 4366 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 4367 4368 /* Disable polling. */ 4369 hn_polling(sc, 0); 4370 4371 if (sc->hn_vf_ready || (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) { 4372 KASSERT(sc->hn_vf_ifp != NULL, 4373 ("%s: VF is not attached", if_name(ifp))); 4374 4375 /* 4376 * NOTE: 4377 * Datapath setting must happen _before_ bringing 4378 * the VF down. 4379 */ 4380 hn_xpnt_vf_deactivate(sc); 4381 4382 /* 4383 * Bring the VF down. 4384 */ 4385 hn_xpnt_vf_saveifflags(sc); 4386 if_setflagbits(ifp, 0, IFF_UP); 4387 hn_xpnt_vf_iocsetflags(sc); 4388 } 4389 4390 /* Suspend data transfers. */ 4391 hn_suspend_data(sc); 4392 4393 /* Clear OACTIVE bit. */ 4394 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 4395 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) 4396 sc->hn_tx_ring[i].hn_oactive = 0; 4397 4398 /* 4399 * If the non-transparent mode VF is active, make sure 4400 * that the RX filter still allows packet reception. 4401 */ 4402 if (!detaching && (sc->hn_flags & HN_FLAG_RXVF)) 4403 hn_rxfilter_config(sc); 4404 } 4405 4406 static void 4407 hn_init_locked(struct hn_softc *sc) 4408 { 4409 if_t ifp = sc->hn_ifp; 4410 int i; 4411 4412 HN_LOCK_ASSERT(sc); 4413 4414 if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0) 4415 return; 4416 4417 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 4418 return; 4419 4420 /* Configure RX filter */ 4421 hn_rxfilter_config(sc); 4422 4423 /* Clear OACTIVE bit. */ 4424 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 4425 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) 4426 sc->hn_tx_ring[i].hn_oactive = 0; 4427 4428 /* Clear TX 'suspended' bit. */ 4429 hn_resume_tx(sc, sc->hn_tx_ring_inuse); 4430 4431 if (hn_xpnt_vf_caninit(sc)) { 4432 /* Initialize transparent VF. */ 4433 hn_xpnt_vf_init(sc); 4434 } 4435 4436 /* Everything is ready; unleash! */ 4437 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 4438 4439 /* Re-enable polling if requested. */ 4440 if (sc->hn_pollhz > 0) 4441 hn_polling(sc, sc->hn_pollhz); 4442 } 4443 4444 static void 4445 hn_init(void *xsc) 4446 { 4447 struct hn_softc *sc = xsc; 4448 4449 HN_LOCK(sc); 4450 hn_init_locked(sc); 4451 HN_UNLOCK(sc); 4452 } 4453 4454 static int 4455 hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS) 4456 { 4457 struct hn_softc *sc = arg1; 4458 unsigned int lenlim; 4459 int error; 4460 4461 lenlim = sc->hn_rx_ring[0].hn_lro.lro_length_lim; 4462 error = sysctl_handle_int(oidp, &lenlim, 0, req); 4463 if (error || req->newptr == NULL) 4464 return error; 4465 4466 HN_LOCK(sc); 4467 if (lenlim < HN_LRO_LENLIM_MIN(sc->hn_ifp) || 4468 lenlim > TCP_LRO_LENGTH_MAX) { 4469 HN_UNLOCK(sc); 4470 return EINVAL; 4471 } 4472 hn_set_lro_lenlim(sc, lenlim); 4473 HN_UNLOCK(sc); 4474 4475 return 0; 4476 } 4477 4478 static int 4479 hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS) 4480 { 4481 struct hn_softc *sc = arg1; 4482 int ackcnt, error, i; 4483 4484 /* 4485 * lro_ackcnt_lim is append count limit, 4486 * +1 to turn it into aggregation limit. 4487 */ 4488 ackcnt = sc->hn_rx_ring[0].hn_lro.lro_ackcnt_lim + 1; 4489 error = sysctl_handle_int(oidp, &ackcnt, 0, req); 4490 if (error || req->newptr == NULL) 4491 return error; 4492 4493 if (ackcnt < 2 || ackcnt > (TCP_LRO_ACKCNT_MAX + 1)) 4494 return EINVAL; 4495 4496 /* 4497 * Convert aggregation limit back to append 4498 * count limit. 4499 */ 4500 --ackcnt; 4501 HN_LOCK(sc); 4502 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 4503 sc->hn_rx_ring[i].hn_lro.lro_ackcnt_lim = ackcnt; 4504 HN_UNLOCK(sc); 4505 return 0; 4506 } 4507 4508 static int 4509 hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS) 4510 { 4511 struct hn_softc *sc = arg1; 4512 int hcsum = arg2; 4513 int on, error, i; 4514 4515 on = 0; 4516 if (sc->hn_rx_ring[0].hn_trust_hcsum & hcsum) 4517 on = 1; 4518 4519 error = sysctl_handle_int(oidp, &on, 0, req); 4520 if (error || req->newptr == NULL) 4521 return error; 4522 4523 HN_LOCK(sc); 4524 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4525 struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 4526 4527 if (on) 4528 rxr->hn_trust_hcsum |= hcsum; 4529 else 4530 rxr->hn_trust_hcsum &= ~hcsum; 4531 } 4532 HN_UNLOCK(sc); 4533 return 0; 4534 } 4535 4536 static int 4537 hn_chim_size_sysctl(SYSCTL_HANDLER_ARGS) 4538 { 4539 struct hn_softc *sc = arg1; 4540 int chim_size, error; 4541 4542 chim_size = sc->hn_tx_ring[0].hn_chim_size; 4543 error = sysctl_handle_int(oidp, &chim_size, 0, req); 4544 if (error || req->newptr == NULL) 4545 return error; 4546 4547 if (chim_size > sc->hn_chim_szmax || chim_size <= 0) 4548 return EINVAL; 4549 4550 HN_LOCK(sc); 4551 hn_set_chim_size(sc, chim_size); 4552 HN_UNLOCK(sc); 4553 return 0; 4554 } 4555 4556 static int 4557 hn_rx_stat_u64_sysctl(SYSCTL_HANDLER_ARGS) 4558 { 4559 struct hn_softc *sc = arg1; 4560 int ofs = arg2, i, error; 4561 struct hn_rx_ring *rxr; 4562 uint64_t stat; 4563 4564 stat = 0; 4565 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4566 rxr = &sc->hn_rx_ring[i]; 4567 stat += *((uint64_t *)((uint8_t *)rxr + ofs)); 4568 } 4569 4570 error = sysctl_handle_64(oidp, &stat, 0, req); 4571 if (error || req->newptr == NULL) 4572 return error; 4573 4574 /* Zero out this stat. */ 4575 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4576 rxr = &sc->hn_rx_ring[i]; 4577 *((uint64_t *)((uint8_t *)rxr + ofs)) = 0; 4578 } 4579 return 0; 4580 } 4581 4582 static int 4583 hn_rx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS) 4584 { 4585 struct hn_softc *sc = arg1; 4586 int ofs = arg2, i, error; 4587 struct hn_rx_ring *rxr; 4588 u_long stat; 4589 4590 stat = 0; 4591 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4592 rxr = &sc->hn_rx_ring[i]; 4593 stat += *((u_long *)((uint8_t *)rxr + ofs)); 4594 } 4595 4596 error = sysctl_handle_long(oidp, &stat, 0, req); 4597 if (error || req->newptr == NULL) 4598 return error; 4599 4600 /* Zero out this stat. */ 4601 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 4602 rxr = &sc->hn_rx_ring[i]; 4603 *((u_long *)((uint8_t *)rxr + ofs)) = 0; 4604 } 4605 return 0; 4606 } 4607 4608 static int 4609 hn_tx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS) 4610 { 4611 struct hn_softc *sc = arg1; 4612 int ofs = arg2, i, error; 4613 struct hn_tx_ring *txr; 4614 u_long stat; 4615 4616 stat = 0; 4617 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 4618 txr = &sc->hn_tx_ring[i]; 4619 stat += *((u_long *)((uint8_t *)txr + ofs)); 4620 } 4621 4622 error = sysctl_handle_long(oidp, &stat, 0, req); 4623 if (error || req->newptr == NULL) 4624 return error; 4625 4626 /* Zero out this stat. */ 4627 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 4628 txr = &sc->hn_tx_ring[i]; 4629 *((u_long *)((uint8_t *)txr + ofs)) = 0; 4630 } 4631 return 0; 4632 } 4633 4634 static int 4635 hn_tx_conf_int_sysctl(SYSCTL_HANDLER_ARGS) 4636 { 4637 struct hn_softc *sc = arg1; 4638 int ofs = arg2, i, error, conf; 4639 struct hn_tx_ring *txr; 4640 4641 txr = &sc->hn_tx_ring[0]; 4642 conf = *((int *)((uint8_t *)txr + ofs)); 4643 4644 error = sysctl_handle_int(oidp, &conf, 0, req); 4645 if (error || req->newptr == NULL) 4646 return error; 4647 4648 HN_LOCK(sc); 4649 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 4650 txr = &sc->hn_tx_ring[i]; 4651 *((int *)((uint8_t *)txr + ofs)) = conf; 4652 } 4653 HN_UNLOCK(sc); 4654 4655 return 0; 4656 } 4657 4658 static int 4659 hn_txagg_size_sysctl(SYSCTL_HANDLER_ARGS) 4660 { 4661 struct hn_softc *sc = arg1; 4662 int error, size; 4663 4664 size = sc->hn_agg_size; 4665 error = sysctl_handle_int(oidp, &size, 0, req); 4666 if (error || req->newptr == NULL) 4667 return (error); 4668 4669 HN_LOCK(sc); 4670 sc->hn_agg_size = size; 4671 hn_set_txagg(sc); 4672 HN_UNLOCK(sc); 4673 4674 return (0); 4675 } 4676 4677 static int 4678 hn_txagg_pkts_sysctl(SYSCTL_HANDLER_ARGS) 4679 { 4680 struct hn_softc *sc = arg1; 4681 int error, pkts; 4682 4683 pkts = sc->hn_agg_pkts; 4684 error = sysctl_handle_int(oidp, &pkts, 0, req); 4685 if (error || req->newptr == NULL) 4686 return (error); 4687 4688 HN_LOCK(sc); 4689 sc->hn_agg_pkts = pkts; 4690 hn_set_txagg(sc); 4691 HN_UNLOCK(sc); 4692 4693 return (0); 4694 } 4695 4696 static int 4697 hn_txagg_pktmax_sysctl(SYSCTL_HANDLER_ARGS) 4698 { 4699 struct hn_softc *sc = arg1; 4700 int pkts; 4701 4702 pkts = sc->hn_tx_ring[0].hn_agg_pktmax; 4703 return (sysctl_handle_int(oidp, &pkts, 0, req)); 4704 } 4705 4706 static int 4707 hn_txagg_align_sysctl(SYSCTL_HANDLER_ARGS) 4708 { 4709 struct hn_softc *sc = arg1; 4710 int align; 4711 4712 align = sc->hn_tx_ring[0].hn_agg_align; 4713 return (sysctl_handle_int(oidp, &align, 0, req)); 4714 } 4715 4716 static void 4717 hn_chan_polling(struct vmbus_channel *chan, u_int pollhz) 4718 { 4719 if (pollhz == 0) 4720 vmbus_chan_poll_disable(chan); 4721 else 4722 vmbus_chan_poll_enable(chan, pollhz); 4723 } 4724 4725 static void 4726 hn_polling(struct hn_softc *sc, u_int pollhz) 4727 { 4728 int nsubch = sc->hn_rx_ring_inuse - 1; 4729 4730 HN_LOCK_ASSERT(sc); 4731 4732 if (nsubch > 0) { 4733 struct vmbus_channel **subch; 4734 int i; 4735 4736 subch = vmbus_subchan_get(sc->hn_prichan, nsubch); 4737 for (i = 0; i < nsubch; ++i) 4738 hn_chan_polling(subch[i], pollhz); 4739 vmbus_subchan_rel(subch, nsubch); 4740 } 4741 hn_chan_polling(sc->hn_prichan, pollhz); 4742 } 4743 4744 static int 4745 hn_polling_sysctl(SYSCTL_HANDLER_ARGS) 4746 { 4747 struct hn_softc *sc = arg1; 4748 int pollhz, error; 4749 4750 pollhz = sc->hn_pollhz; 4751 error = sysctl_handle_int(oidp, &pollhz, 0, req); 4752 if (error || req->newptr == NULL) 4753 return (error); 4754 4755 if (pollhz != 0 && 4756 (pollhz < VMBUS_CHAN_POLLHZ_MIN || pollhz > VMBUS_CHAN_POLLHZ_MAX)) 4757 return (EINVAL); 4758 4759 HN_LOCK(sc); 4760 if (sc->hn_pollhz != pollhz) { 4761 sc->hn_pollhz = pollhz; 4762 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) && 4763 (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED)) 4764 hn_polling(sc, sc->hn_pollhz); 4765 } 4766 HN_UNLOCK(sc); 4767 4768 return (0); 4769 } 4770 4771 static int 4772 hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS) 4773 { 4774 struct hn_softc *sc = arg1; 4775 char verstr[16]; 4776 4777 snprintf(verstr, sizeof(verstr), "%u.%u", 4778 HN_NDIS_VERSION_MAJOR(sc->hn_ndis_ver), 4779 HN_NDIS_VERSION_MINOR(sc->hn_ndis_ver)); 4780 return sysctl_handle_string(oidp, verstr, sizeof(verstr), req); 4781 } 4782 4783 static int 4784 hn_caps_sysctl(SYSCTL_HANDLER_ARGS) 4785 { 4786 struct hn_softc *sc = arg1; 4787 char caps_str[128]; 4788 uint32_t caps; 4789 4790 HN_LOCK(sc); 4791 caps = sc->hn_caps; 4792 HN_UNLOCK(sc); 4793 snprintf(caps_str, sizeof(caps_str), "%b", caps, HN_CAP_BITS); 4794 return sysctl_handle_string(oidp, caps_str, sizeof(caps_str), req); 4795 } 4796 4797 static int 4798 hn_hwassist_sysctl(SYSCTL_HANDLER_ARGS) 4799 { 4800 struct hn_softc *sc = arg1; 4801 char assist_str[128]; 4802 uint32_t hwassist; 4803 4804 HN_LOCK(sc); 4805 hwassist = if_gethwassist(sc->hn_ifp); 4806 HN_UNLOCK(sc); 4807 snprintf(assist_str, sizeof(assist_str), "%b", hwassist, CSUM_BITS); 4808 return sysctl_handle_string(oidp, assist_str, sizeof(assist_str), req); 4809 } 4810 4811 static int 4812 hn_rxfilter_sysctl(SYSCTL_HANDLER_ARGS) 4813 { 4814 struct hn_softc *sc = arg1; 4815 char filter_str[128]; 4816 uint32_t filter; 4817 4818 HN_LOCK(sc); 4819 filter = sc->hn_rx_filter; 4820 HN_UNLOCK(sc); 4821 snprintf(filter_str, sizeof(filter_str), "%b", filter, 4822 NDIS_PACKET_TYPES); 4823 return sysctl_handle_string(oidp, filter_str, sizeof(filter_str), req); 4824 } 4825 4826 static int 4827 hn_rsc_sysctl(SYSCTL_HANDLER_ARGS) 4828 { 4829 struct hn_softc *sc = arg1; 4830 int rsc_ctrl, mtu; 4831 int error; 4832 4833 rsc_ctrl = sc->hn_rsc_ctrl; 4834 error = sysctl_handle_int(oidp, &rsc_ctrl, 0, req); 4835 if (error || req->newptr == NULL) 4836 return (error); 4837 4838 if (sc->hn_rsc_ctrl != rsc_ctrl) { 4839 HN_LOCK(sc); 4840 sc->hn_rsc_ctrl = rsc_ctrl; 4841 mtu = if_getmtu(sc->hn_ifp); 4842 error = hn_rndis_reconf_offload(sc, mtu); 4843 HN_UNLOCK(sc); 4844 } 4845 4846 return (error); 4847 } 4848 #ifndef RSS 4849 4850 static int 4851 hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS) 4852 { 4853 struct hn_softc *sc = arg1; 4854 int error; 4855 4856 HN_LOCK(sc); 4857 4858 error = SYSCTL_OUT(req, sc->hn_rss.rss_key, sizeof(sc->hn_rss.rss_key)); 4859 if (error || req->newptr == NULL) 4860 goto back; 4861 4862 if ((sc->hn_flags & HN_FLAG_RXVF) || 4863 (hn_xpnt_vf && sc->hn_vf_ifp != NULL)) { 4864 /* 4865 * RSS key is synchronized w/ VF's, don't allow users 4866 * to change it. 4867 */ 4868 error = EBUSY; 4869 goto back; 4870 } 4871 4872 error = SYSCTL_IN(req, sc->hn_rss.rss_key, sizeof(sc->hn_rss.rss_key)); 4873 if (error) 4874 goto back; 4875 sc->hn_flags |= HN_FLAG_HAS_RSSKEY; 4876 4877 if (sc->hn_rx_ring_inuse > 1) { 4878 error = hn_rss_reconfig(sc); 4879 } else { 4880 /* Not RSS capable, at least for now; just save the RSS key. */ 4881 error = 0; 4882 } 4883 back: 4884 HN_UNLOCK(sc); 4885 return (error); 4886 } 4887 4888 static int 4889 hn_rss_ind_sysctl(SYSCTL_HANDLER_ARGS) 4890 { 4891 struct hn_softc *sc = arg1; 4892 int error; 4893 4894 HN_LOCK(sc); 4895 4896 error = SYSCTL_OUT(req, sc->hn_rss.rss_ind, sizeof(sc->hn_rss.rss_ind)); 4897 if (error || req->newptr == NULL) 4898 goto back; 4899 4900 /* 4901 * Don't allow RSS indirect table change, if this interface is not 4902 * RSS capable currently. 4903 */ 4904 if (sc->hn_rx_ring_inuse == 1) { 4905 error = EOPNOTSUPP; 4906 goto back; 4907 } 4908 4909 error = SYSCTL_IN(req, sc->hn_rss.rss_ind, sizeof(sc->hn_rss.rss_ind)); 4910 if (error) 4911 goto back; 4912 sc->hn_flags |= HN_FLAG_HAS_RSSIND; 4913 4914 hn_rss_ind_fixup(sc); 4915 error = hn_rss_reconfig(sc); 4916 back: 4917 HN_UNLOCK(sc); 4918 return (error); 4919 } 4920 4921 #endif /* !RSS */ 4922 4923 static int 4924 hn_rss_hash_sysctl(SYSCTL_HANDLER_ARGS) 4925 { 4926 struct hn_softc *sc = arg1; 4927 char hash_str[128]; 4928 uint32_t hash; 4929 4930 HN_LOCK(sc); 4931 hash = sc->hn_rss_hash; 4932 HN_UNLOCK(sc); 4933 snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS); 4934 return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req); 4935 } 4936 4937 static int 4938 hn_rss_hcap_sysctl(SYSCTL_HANDLER_ARGS) 4939 { 4940 struct hn_softc *sc = arg1; 4941 char hash_str[128]; 4942 uint32_t hash; 4943 4944 HN_LOCK(sc); 4945 hash = sc->hn_rss_hcap; 4946 HN_UNLOCK(sc); 4947 snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS); 4948 return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req); 4949 } 4950 4951 static int 4952 hn_rss_mbuf_sysctl(SYSCTL_HANDLER_ARGS) 4953 { 4954 struct hn_softc *sc = arg1; 4955 char hash_str[128]; 4956 uint32_t hash; 4957 4958 HN_LOCK(sc); 4959 hash = sc->hn_rx_ring[0].hn_mbuf_hash; 4960 HN_UNLOCK(sc); 4961 snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS); 4962 return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req); 4963 } 4964 4965 static int 4966 hn_vf_sysctl(SYSCTL_HANDLER_ARGS) 4967 { 4968 struct hn_softc *sc = arg1; 4969 char vf_name[IFNAMSIZ + 1]; 4970 if_t vf_ifp; 4971 4972 HN_LOCK(sc); 4973 vf_name[0] = '\0'; 4974 vf_ifp = sc->hn_vf_ifp; 4975 if (vf_ifp != NULL) 4976 snprintf(vf_name, sizeof(vf_name), "%s", if_name(vf_ifp)); 4977 HN_UNLOCK(sc); 4978 return sysctl_handle_string(oidp, vf_name, sizeof(vf_name), req); 4979 } 4980 4981 static int 4982 hn_rxvf_sysctl(SYSCTL_HANDLER_ARGS) 4983 { 4984 struct hn_softc *sc = arg1; 4985 char vf_name[IFNAMSIZ + 1]; 4986 if_t vf_ifp; 4987 4988 HN_LOCK(sc); 4989 vf_name[0] = '\0'; 4990 vf_ifp = sc->hn_rx_ring[0].hn_rxvf_ifp; 4991 if (vf_ifp != NULL) 4992 snprintf(vf_name, sizeof(vf_name), "%s", if_name(vf_ifp)); 4993 HN_UNLOCK(sc); 4994 return sysctl_handle_string(oidp, vf_name, sizeof(vf_name), req); 4995 } 4996 4997 static int 4998 hn_vflist_sysctl(SYSCTL_HANDLER_ARGS) 4999 { 5000 struct rm_priotracker pt; 5001 struct sbuf *sb; 5002 int error, i; 5003 bool first; 5004 5005 error = sysctl_wire_old_buffer(req, 0); 5006 if (error != 0) 5007 return (error); 5008 5009 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5010 if (sb == NULL) 5011 return (ENOMEM); 5012 5013 rm_rlock(&hn_vfmap_lock, &pt); 5014 5015 first = true; 5016 for (i = 0; i < hn_vfmap_size; ++i) { 5017 struct epoch_tracker et; 5018 if_t ifp; 5019 5020 if (hn_vfmap[i] == NULL) 5021 continue; 5022 5023 NET_EPOCH_ENTER(et); 5024 ifp = ifnet_byindex(i); 5025 if (ifp != NULL) { 5026 if (first) 5027 sbuf_printf(sb, "%s", if_name(ifp)); 5028 else 5029 sbuf_printf(sb, " %s", if_name(ifp)); 5030 first = false; 5031 } 5032 NET_EPOCH_EXIT(et); 5033 } 5034 5035 rm_runlock(&hn_vfmap_lock, &pt); 5036 5037 error = sbuf_finish(sb); 5038 sbuf_delete(sb); 5039 return (error); 5040 } 5041 5042 static int 5043 hn_vfmap_sysctl(SYSCTL_HANDLER_ARGS) 5044 { 5045 struct rm_priotracker pt; 5046 struct sbuf *sb; 5047 int error, i; 5048 bool first; 5049 5050 error = sysctl_wire_old_buffer(req, 0); 5051 if (error != 0) 5052 return (error); 5053 5054 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5055 if (sb == NULL) 5056 return (ENOMEM); 5057 5058 rm_rlock(&hn_vfmap_lock, &pt); 5059 5060 first = true; 5061 for (i = 0; i < hn_vfmap_size; ++i) { 5062 struct epoch_tracker et; 5063 if_t ifp, hn_ifp; 5064 5065 hn_ifp = hn_vfmap[i]; 5066 if (hn_ifp == NULL) 5067 continue; 5068 5069 NET_EPOCH_ENTER(et); 5070 ifp = ifnet_byindex(i); 5071 if (ifp != NULL) { 5072 if (first) { 5073 sbuf_printf(sb, "%s:%s", if_name(ifp), 5074 if_name(hn_ifp)); 5075 } else { 5076 sbuf_printf(sb, " %s:%s", if_name(ifp), 5077 if_name(hn_ifp)); 5078 } 5079 first = false; 5080 } 5081 NET_EPOCH_EXIT(et); 5082 } 5083 5084 rm_runlock(&hn_vfmap_lock, &pt); 5085 5086 error = sbuf_finish(sb); 5087 sbuf_delete(sb); 5088 return (error); 5089 } 5090 5091 static int 5092 hn_xpnt_vf_accbpf_sysctl(SYSCTL_HANDLER_ARGS) 5093 { 5094 struct hn_softc *sc = arg1; 5095 int error, onoff = 0; 5096 5097 if (sc->hn_xvf_flags & HN_XVFFLAG_ACCBPF) 5098 onoff = 1; 5099 error = sysctl_handle_int(oidp, &onoff, 0, req); 5100 if (error || req->newptr == NULL) 5101 return (error); 5102 5103 HN_LOCK(sc); 5104 /* NOTE: hn_vf_lock for hn_transmit() */ 5105 rm_wlock(&sc->hn_vf_lock); 5106 if (onoff) 5107 sc->hn_xvf_flags |= HN_XVFFLAG_ACCBPF; 5108 else 5109 sc->hn_xvf_flags &= ~HN_XVFFLAG_ACCBPF; 5110 rm_wunlock(&sc->hn_vf_lock); 5111 HN_UNLOCK(sc); 5112 5113 return (0); 5114 } 5115 5116 static int 5117 hn_xpnt_vf_enabled_sysctl(SYSCTL_HANDLER_ARGS) 5118 { 5119 struct hn_softc *sc = arg1; 5120 int enabled = 0; 5121 5122 if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) 5123 enabled = 1; 5124 return (sysctl_handle_int(oidp, &enabled, 0, req)); 5125 } 5126 5127 static int 5128 hn_check_iplen(const struct mbuf *m, int hoff) 5129 { 5130 const struct ip *ip; 5131 int len, iphlen, iplen; 5132 const struct tcphdr *th; 5133 int thoff; /* TCP data offset */ 5134 5135 len = hoff + sizeof(struct ip); 5136 5137 /* The packet must be at least the size of an IP header. */ 5138 if (m->m_pkthdr.len < len) 5139 return IPPROTO_DONE; 5140 5141 /* The fixed IP header must reside completely in the first mbuf. */ 5142 if (m->m_len < len) 5143 return IPPROTO_DONE; 5144 5145 ip = mtodo(m, hoff); 5146 5147 /* Bound check the packet's stated IP header length. */ 5148 iphlen = ip->ip_hl << 2; 5149 if (iphlen < sizeof(struct ip)) /* minimum header length */ 5150 return IPPROTO_DONE; 5151 5152 /* The full IP header must reside completely in the one mbuf. */ 5153 if (m->m_len < hoff + iphlen) 5154 return IPPROTO_DONE; 5155 5156 iplen = ntohs(ip->ip_len); 5157 5158 /* 5159 * Check that the amount of data in the buffers is as 5160 * at least much as the IP header would have us expect. 5161 */ 5162 if (m->m_pkthdr.len < hoff + iplen) 5163 return IPPROTO_DONE; 5164 5165 /* 5166 * Ignore IP fragments. 5167 */ 5168 if (ntohs(ip->ip_off) & (IP_OFFMASK | IP_MF)) 5169 return IPPROTO_DONE; 5170 5171 /* 5172 * The TCP/IP or UDP/IP header must be entirely contained within 5173 * the first fragment of a packet. 5174 */ 5175 switch (ip->ip_p) { 5176 case IPPROTO_TCP: 5177 if (iplen < iphlen + sizeof(struct tcphdr)) 5178 return IPPROTO_DONE; 5179 if (m->m_len < hoff + iphlen + sizeof(struct tcphdr)) 5180 return IPPROTO_DONE; 5181 th = (const struct tcphdr *)((const uint8_t *)ip + iphlen); 5182 thoff = th->th_off << 2; 5183 if (thoff < sizeof(struct tcphdr) || thoff + iphlen > iplen) 5184 return IPPROTO_DONE; 5185 if (m->m_len < hoff + iphlen + thoff) 5186 return IPPROTO_DONE; 5187 break; 5188 case IPPROTO_UDP: 5189 if (iplen < iphlen + sizeof(struct udphdr)) 5190 return IPPROTO_DONE; 5191 if (m->m_len < hoff + iphlen + sizeof(struct udphdr)) 5192 return IPPROTO_DONE; 5193 break; 5194 default: 5195 if (iplen < iphlen) 5196 return IPPROTO_DONE; 5197 break; 5198 } 5199 return ip->ip_p; 5200 } 5201 5202 static void 5203 hn_rxpkt_proto(const struct mbuf *m_new, int *l3proto, int *l4proto) 5204 { 5205 const struct ether_header *eh; 5206 uint16_t etype; 5207 int hoff; 5208 5209 hoff = sizeof(*eh); 5210 /* Checked at the beginning of this function. */ 5211 KASSERT(m_new->m_len >= hoff, ("not ethernet frame")); 5212 5213 eh = mtod(m_new, const struct ether_header *); 5214 etype = ntohs(eh->ether_type); 5215 if (etype == ETHERTYPE_VLAN) { 5216 const struct ether_vlan_header *evl; 5217 5218 hoff = sizeof(*evl); 5219 if (m_new->m_len < hoff) 5220 return; 5221 evl = mtod(m_new, const struct ether_vlan_header *); 5222 etype = ntohs(evl->evl_proto); 5223 } 5224 *l3proto = etype; 5225 5226 if (etype == ETHERTYPE_IP) 5227 *l4proto = hn_check_iplen(m_new, hoff); 5228 else 5229 *l4proto = IPPROTO_DONE; 5230 } 5231 5232 static int 5233 hn_create_rx_data(struct hn_softc *sc, int ring_cnt) 5234 { 5235 struct sysctl_oid_list *child; 5236 struct sysctl_ctx_list *ctx; 5237 device_t dev = sc->hn_dev; 5238 #if defined(INET) || defined(INET6) 5239 int lroent_cnt; 5240 #endif 5241 int i; 5242 5243 /* 5244 * Create RXBUF for reception. 5245 * 5246 * NOTE: 5247 * - It is shared by all channels. 5248 * - A large enough buffer is allocated, certain version of NVSes 5249 * may further limit the usable space. 5250 */ 5251 sc->hn_rxbuf = contigmalloc(HN_RXBUF_SIZE, M_DEVBUF, M_WAITOK | M_ZERO, 5252 0ul, ~0ul, PAGE_SIZE, 0); 5253 if (sc->hn_rxbuf == NULL) { 5254 device_printf(sc->hn_dev, "allocate rxbuf failed\n"); 5255 return (ENOMEM); 5256 } 5257 5258 sc->hn_rx_ring_cnt = ring_cnt; 5259 sc->hn_rx_ring_inuse = sc->hn_rx_ring_cnt; 5260 5261 sc->hn_rx_ring = malloc(sizeof(struct hn_rx_ring) * sc->hn_rx_ring_cnt, 5262 M_DEVBUF, M_WAITOK | M_ZERO); 5263 5264 #if defined(INET) || defined(INET6) 5265 lroent_cnt = hn_lro_entry_count; 5266 if (lroent_cnt < TCP_LRO_ENTRIES) 5267 lroent_cnt = TCP_LRO_ENTRIES; 5268 if (bootverbose) 5269 device_printf(dev, "LRO: entry count %d\n", lroent_cnt); 5270 #endif /* INET || INET6 */ 5271 5272 ctx = device_get_sysctl_ctx(dev); 5273 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 5274 5275 /* Create dev.hn.UNIT.rx sysctl tree */ 5276 sc->hn_rx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "rx", 5277 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5278 5279 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 5280 struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 5281 5282 rxr->hn_br = contigmalloc(HN_TXBR_SIZE + HN_RXBR_SIZE, M_DEVBUF, 5283 M_WAITOK | M_ZERO, 0ul, ~0ul, PAGE_SIZE, 0); 5284 if (rxr->hn_br == NULL) { 5285 device_printf(dev, "allocate bufring failed\n"); 5286 return (ENOMEM); 5287 } 5288 5289 if (hn_trust_hosttcp) 5290 rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_TCP; 5291 if (hn_trust_hostudp) 5292 rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_UDP; 5293 if (hn_trust_hostip) 5294 rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_IP; 5295 rxr->hn_mbuf_hash = NDIS_HASH_ALL; 5296 rxr->hn_ifp = sc->hn_ifp; 5297 if (i < sc->hn_tx_ring_cnt) 5298 rxr->hn_txr = &sc->hn_tx_ring[i]; 5299 rxr->hn_pktbuf_len = HN_PKTBUF_LEN_DEF; 5300 rxr->hn_pktbuf = malloc(rxr->hn_pktbuf_len, M_DEVBUF, M_WAITOK); 5301 rxr->hn_rx_idx = i; 5302 rxr->hn_rxbuf = sc->hn_rxbuf; 5303 5304 /* 5305 * Initialize LRO. 5306 */ 5307 #if defined(INET) || defined(INET6) 5308 tcp_lro_init_args(&rxr->hn_lro, sc->hn_ifp, lroent_cnt, 5309 hn_lro_mbufq_depth); 5310 rxr->hn_lro.lro_length_lim = HN_LRO_LENLIM_DEF; 5311 rxr->hn_lro.lro_ackcnt_lim = HN_LRO_ACKCNT_DEF; 5312 #endif /* INET || INET6 */ 5313 5314 if (sc->hn_rx_sysctl_tree != NULL) { 5315 char name[16]; 5316 5317 /* 5318 * Create per RX ring sysctl tree: 5319 * dev.hn.UNIT.rx.RINGID 5320 */ 5321 snprintf(name, sizeof(name), "%d", i); 5322 rxr->hn_rx_sysctl_tree = SYSCTL_ADD_NODE(ctx, 5323 SYSCTL_CHILDREN(sc->hn_rx_sysctl_tree), 5324 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5325 5326 if (rxr->hn_rx_sysctl_tree != NULL) { 5327 SYSCTL_ADD_ULONG(ctx, 5328 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5329 OID_AUTO, "packets", 5330 CTLFLAG_RW | CTLFLAG_STATS, &rxr->hn_pkts, 5331 "# of packets received"); 5332 SYSCTL_ADD_ULONG(ctx, 5333 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5334 OID_AUTO, "rss_pkts", 5335 CTLFLAG_RW | CTLFLAG_STATS, 5336 &rxr->hn_rss_pkts, 5337 "# of packets w/ RSS info received"); 5338 SYSCTL_ADD_ULONG(ctx, 5339 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5340 OID_AUTO, "rsc_pkts", 5341 CTLFLAG_RW | CTLFLAG_STATS, 5342 &rxr->hn_rsc_pkts, 5343 "# of RSC packets received"); 5344 SYSCTL_ADD_ULONG(ctx, 5345 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5346 OID_AUTO, "rsc_drop", 5347 CTLFLAG_RW | CTLFLAG_STATS, 5348 &rxr->hn_rsc_drop, 5349 "# of RSC fragments dropped"); 5350 SYSCTL_ADD_INT(ctx, 5351 SYSCTL_CHILDREN(rxr->hn_rx_sysctl_tree), 5352 OID_AUTO, "pktbuf_len", CTLFLAG_RD, 5353 &rxr->hn_pktbuf_len, 0, 5354 "Temporary channel packet buffer length"); 5355 } 5356 } 5357 } 5358 5359 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_queued", 5360 CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5361 __offsetof(struct hn_rx_ring, hn_lro.lro_queued), 5362 hn_rx_stat_u64_sysctl, 5363 "LU", "LRO queued"); 5364 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_flushed", 5365 CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5366 __offsetof(struct hn_rx_ring, hn_lro.lro_flushed), 5367 hn_rx_stat_u64_sysctl, 5368 "LU", "LRO flushed"); 5369 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_tried", 5370 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5371 __offsetof(struct hn_rx_ring, hn_lro_tried), 5372 hn_rx_stat_ulong_sysctl, "LU", "# of LRO tries"); 5373 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_length_lim", 5374 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 5375 hn_lro_lenlim_sysctl, "IU", 5376 "Max # of data bytes to be aggregated by LRO"); 5377 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_ackcnt_lim", 5378 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 5379 hn_lro_ackcnt_sysctl, "I", 5380 "Max # of ACKs to be aggregated by LRO"); 5381 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hosttcp", 5382 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_TCP, 5383 hn_trust_hcsum_sysctl, "I", 5384 "Trust tcp segment verification on host side, " 5385 "when csum info is missing"); 5386 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hostudp", 5387 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_UDP, 5388 hn_trust_hcsum_sysctl, "I", 5389 "Trust udp datagram verification on host side, " 5390 "when csum info is missing"); 5391 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hostip", 5392 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, HN_TRUST_HCSUM_IP, 5393 hn_trust_hcsum_sysctl, "I", 5394 "Trust ip packet verification on host side, " 5395 "when csum info is missing"); 5396 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_ip", 5397 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5398 __offsetof(struct hn_rx_ring, hn_csum_ip), 5399 hn_rx_stat_ulong_sysctl, "LU", "RXCSUM IP"); 5400 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_tcp", 5401 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5402 __offsetof(struct hn_rx_ring, hn_csum_tcp), 5403 hn_rx_stat_ulong_sysctl, "LU", "RXCSUM TCP"); 5404 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_udp", 5405 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5406 __offsetof(struct hn_rx_ring, hn_csum_udp), 5407 hn_rx_stat_ulong_sysctl, "LU", "RXCSUM UDP"); 5408 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "csum_trusted", 5409 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 5410 __offsetof(struct hn_rx_ring, hn_csum_trusted), 5411 hn_rx_stat_ulong_sysctl, "LU", 5412 "# of packets that we trust host's csum verification"); 5413 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "small_pkts", 5414 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5415 __offsetof(struct hn_rx_ring, hn_small_pkts), 5416 hn_rx_stat_ulong_sysctl, "LU", "# of small packets received"); 5417 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_ack_failed", 5418 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS , sc, 5419 __offsetof(struct hn_rx_ring, hn_ack_failed), 5420 hn_rx_stat_ulong_sysctl, "LU", "# of RXBUF ack failures"); 5421 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rx_ring_cnt", 5422 CTLFLAG_RD, &sc->hn_rx_ring_cnt, 0, "# created RX rings"); 5423 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rx_ring_inuse", 5424 CTLFLAG_RD, &sc->hn_rx_ring_inuse, 0, "# used RX rings"); 5425 5426 return (0); 5427 } 5428 5429 static void 5430 hn_destroy_rx_data(struct hn_softc *sc) 5431 { 5432 int i; 5433 5434 if (sc->hn_rxbuf != NULL) { 5435 if ((sc->hn_flags & HN_FLAG_RXBUF_REF) == 0) 5436 free(sc->hn_rxbuf, M_DEVBUF); 5437 else 5438 device_printf(sc->hn_dev, "RXBUF is referenced\n"); 5439 sc->hn_rxbuf = NULL; 5440 } 5441 5442 if (sc->hn_rx_ring_cnt == 0) 5443 return; 5444 5445 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 5446 struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 5447 5448 if (rxr->hn_br == NULL) 5449 continue; 5450 if ((rxr->hn_rx_flags & HN_RX_FLAG_BR_REF) == 0) { 5451 free(rxr->hn_br, M_DEVBUF); 5452 } else { 5453 device_printf(sc->hn_dev, 5454 "%dth channel bufring is referenced", i); 5455 } 5456 rxr->hn_br = NULL; 5457 5458 #if defined(INET) || defined(INET6) 5459 tcp_lro_free(&rxr->hn_lro); 5460 #endif 5461 free(rxr->hn_pktbuf, M_DEVBUF); 5462 } 5463 free(sc->hn_rx_ring, M_DEVBUF); 5464 sc->hn_rx_ring = NULL; 5465 5466 sc->hn_rx_ring_cnt = 0; 5467 sc->hn_rx_ring_inuse = 0; 5468 } 5469 5470 static int 5471 hn_tx_ring_create(struct hn_softc *sc, int id) 5472 { 5473 struct hn_tx_ring *txr = &sc->hn_tx_ring[id]; 5474 device_t dev = sc->hn_dev; 5475 bus_dma_tag_t parent_dtag; 5476 int error, i; 5477 5478 txr->hn_sc = sc; 5479 txr->hn_tx_idx = id; 5480 5481 #ifndef HN_USE_TXDESC_BUFRING 5482 mtx_init(&txr->hn_txlist_spin, "hn txlist", NULL, MTX_SPIN); 5483 #endif 5484 mtx_init(&txr->hn_tx_lock, "hn tx", NULL, MTX_DEF); 5485 5486 txr->hn_txdesc_cnt = HN_TX_DESC_CNT; 5487 txr->hn_txdesc = malloc(sizeof(struct hn_txdesc) * txr->hn_txdesc_cnt, 5488 M_DEVBUF, M_WAITOK | M_ZERO); 5489 #ifndef HN_USE_TXDESC_BUFRING 5490 SLIST_INIT(&txr->hn_txlist); 5491 #else 5492 txr->hn_txdesc_br = buf_ring_alloc(txr->hn_txdesc_cnt, M_DEVBUF, 5493 M_WAITOK, &txr->hn_tx_lock); 5494 #endif 5495 5496 if (hn_tx_taskq_mode == HN_TX_TASKQ_M_EVTTQ) { 5497 txr->hn_tx_taskq = VMBUS_GET_EVENT_TASKQ( 5498 device_get_parent(dev), dev, HN_RING_IDX2CPU(sc, id)); 5499 } else { 5500 txr->hn_tx_taskq = sc->hn_tx_taskqs[id % hn_tx_taskq_cnt]; 5501 } 5502 5503 #ifdef HN_IFSTART_SUPPORT 5504 if (hn_use_if_start) { 5505 txr->hn_txeof = hn_start_txeof; 5506 TASK_INIT(&txr->hn_tx_task, 0, hn_start_taskfunc, txr); 5507 TASK_INIT(&txr->hn_txeof_task, 0, hn_start_txeof_taskfunc, txr); 5508 } else 5509 #endif 5510 { 5511 int br_depth; 5512 5513 txr->hn_txeof = hn_xmit_txeof; 5514 TASK_INIT(&txr->hn_tx_task, 0, hn_xmit_taskfunc, txr); 5515 TASK_INIT(&txr->hn_txeof_task, 0, hn_xmit_txeof_taskfunc, txr); 5516 5517 br_depth = hn_get_txswq_depth(txr); 5518 txr->hn_mbuf_br = buf_ring_alloc(br_depth, M_DEVBUF, 5519 M_WAITOK, &txr->hn_tx_lock); 5520 } 5521 5522 txr->hn_direct_tx_size = hn_direct_tx_size; 5523 5524 /* 5525 * Always schedule transmission instead of trying to do direct 5526 * transmission. This one gives the best performance so far. 5527 */ 5528 txr->hn_sched_tx = 1; 5529 5530 parent_dtag = bus_get_dma_tag(dev); 5531 5532 /* DMA tag for RNDIS packet messages. */ 5533 error = bus_dma_tag_create(parent_dtag, /* parent */ 5534 HN_RNDIS_PKT_ALIGN, /* alignment */ 5535 HN_RNDIS_PKT_BOUNDARY, /* boundary */ 5536 BUS_SPACE_MAXADDR, /* lowaddr */ 5537 BUS_SPACE_MAXADDR, /* highaddr */ 5538 NULL, NULL, /* filter, filterarg */ 5539 HN_RNDIS_PKT_LEN, /* maxsize */ 5540 1, /* nsegments */ 5541 HN_RNDIS_PKT_LEN, /* maxsegsize */ 5542 0, /* flags */ 5543 NULL, /* lockfunc */ 5544 NULL, /* lockfuncarg */ 5545 &txr->hn_tx_rndis_dtag); 5546 if (error) { 5547 device_printf(dev, "failed to create rndis dmatag\n"); 5548 return error; 5549 } 5550 5551 /* DMA tag for data. */ 5552 error = bus_dma_tag_create(parent_dtag, /* parent */ 5553 1, /* alignment */ 5554 HN_TX_DATA_BOUNDARY, /* boundary */ 5555 BUS_SPACE_MAXADDR, /* lowaddr */ 5556 BUS_SPACE_MAXADDR, /* highaddr */ 5557 NULL, NULL, /* filter, filterarg */ 5558 HN_TX_DATA_MAXSIZE, /* maxsize */ 5559 HN_TX_DATA_SEGCNT_MAX, /* nsegments */ 5560 HN_TX_DATA_SEGSIZE, /* maxsegsize */ 5561 0, /* flags */ 5562 NULL, /* lockfunc */ 5563 NULL, /* lockfuncarg */ 5564 &txr->hn_tx_data_dtag); 5565 if (error) { 5566 device_printf(dev, "failed to create data dmatag\n"); 5567 return error; 5568 } 5569 5570 for (i = 0; i < txr->hn_txdesc_cnt; ++i) { 5571 struct hn_txdesc *txd = &txr->hn_txdesc[i]; 5572 5573 txd->txr = txr; 5574 txd->chim_index = HN_NVS_CHIM_IDX_INVALID; 5575 STAILQ_INIT(&txd->agg_list); 5576 5577 /* 5578 * Allocate and load RNDIS packet message. 5579 */ 5580 error = bus_dmamem_alloc(txr->hn_tx_rndis_dtag, 5581 (void **)&txd->rndis_pkt, 5582 BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, 5583 &txd->rndis_pkt_dmap); 5584 if (error) { 5585 device_printf(dev, 5586 "failed to allocate rndis_packet_msg, %d\n", i); 5587 return error; 5588 } 5589 5590 error = bus_dmamap_load(txr->hn_tx_rndis_dtag, 5591 txd->rndis_pkt_dmap, 5592 txd->rndis_pkt, HN_RNDIS_PKT_LEN, 5593 hyperv_dma_map_paddr, &txd->rndis_pkt_paddr, 5594 BUS_DMA_NOWAIT); 5595 if (error) { 5596 device_printf(dev, 5597 "failed to load rndis_packet_msg, %d\n", i); 5598 bus_dmamem_free(txr->hn_tx_rndis_dtag, 5599 txd->rndis_pkt, txd->rndis_pkt_dmap); 5600 return error; 5601 } 5602 5603 /* DMA map for TX data. */ 5604 error = bus_dmamap_create(txr->hn_tx_data_dtag, 0, 5605 &txd->data_dmap); 5606 if (error) { 5607 device_printf(dev, 5608 "failed to allocate tx data dmamap\n"); 5609 bus_dmamap_unload(txr->hn_tx_rndis_dtag, 5610 txd->rndis_pkt_dmap); 5611 bus_dmamem_free(txr->hn_tx_rndis_dtag, 5612 txd->rndis_pkt, txd->rndis_pkt_dmap); 5613 return error; 5614 } 5615 5616 /* All set, put it to list */ 5617 txd->flags |= HN_TXD_FLAG_ONLIST; 5618 #ifndef HN_USE_TXDESC_BUFRING 5619 SLIST_INSERT_HEAD(&txr->hn_txlist, txd, link); 5620 #else 5621 buf_ring_enqueue(txr->hn_txdesc_br, txd); 5622 #endif 5623 } 5624 txr->hn_txdesc_avail = txr->hn_txdesc_cnt; 5625 5626 if (sc->hn_tx_sysctl_tree != NULL) { 5627 struct sysctl_oid_list *child; 5628 struct sysctl_ctx_list *ctx; 5629 char name[16]; 5630 5631 /* 5632 * Create per TX ring sysctl tree: 5633 * dev.hn.UNIT.tx.RINGID 5634 */ 5635 ctx = device_get_sysctl_ctx(dev); 5636 child = SYSCTL_CHILDREN(sc->hn_tx_sysctl_tree); 5637 5638 snprintf(name, sizeof(name), "%d", id); 5639 txr->hn_tx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, 5640 name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5641 5642 if (txr->hn_tx_sysctl_tree != NULL) { 5643 child = SYSCTL_CHILDREN(txr->hn_tx_sysctl_tree); 5644 5645 #ifdef HN_DEBUG 5646 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "txdesc_avail", 5647 CTLFLAG_RD, &txr->hn_txdesc_avail, 0, 5648 "# of available TX descs"); 5649 #endif 5650 #ifdef HN_IFSTART_SUPPORT 5651 if (!hn_use_if_start) 5652 #endif 5653 { 5654 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "oactive", 5655 CTLFLAG_RD, &txr->hn_oactive, 0, 5656 "over active"); 5657 } 5658 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "packets", 5659 CTLFLAG_RW | CTLFLAG_STATS, &txr->hn_pkts, 5660 "# of packets transmitted"); 5661 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "sends", 5662 CTLFLAG_RW | CTLFLAG_STATS, &txr->hn_sends, 5663 "# of sends"); 5664 } 5665 } 5666 5667 return 0; 5668 } 5669 5670 static void 5671 hn_txdesc_dmamap_destroy(struct hn_txdesc *txd) 5672 { 5673 struct hn_tx_ring *txr = txd->txr; 5674 5675 KASSERT(txd->m == NULL, ("still has mbuf installed")); 5676 KASSERT((txd->flags & HN_TXD_FLAG_DMAMAP) == 0, ("still dma mapped")); 5677 5678 bus_dmamap_unload(txr->hn_tx_rndis_dtag, txd->rndis_pkt_dmap); 5679 bus_dmamem_free(txr->hn_tx_rndis_dtag, txd->rndis_pkt, 5680 txd->rndis_pkt_dmap); 5681 bus_dmamap_destroy(txr->hn_tx_data_dtag, txd->data_dmap); 5682 } 5683 5684 static void 5685 hn_txdesc_gc(struct hn_tx_ring *txr, struct hn_txdesc *txd) 5686 { 5687 5688 KASSERT(txd->refs == 0 || txd->refs == 1, 5689 ("invalid txd refs %d", txd->refs)); 5690 5691 /* Aggregated txds will be freed by their aggregating txd. */ 5692 if (txd->refs > 0 && (txd->flags & HN_TXD_FLAG_ONAGG) == 0) { 5693 int freed __diagused; 5694 5695 freed = hn_txdesc_put(txr, txd); 5696 KASSERT(freed, ("can't free txdesc")); 5697 } 5698 } 5699 5700 static void 5701 hn_tx_ring_destroy(struct hn_tx_ring *txr) 5702 { 5703 int i; 5704 5705 if (txr->hn_txdesc == NULL) 5706 return; 5707 5708 /* 5709 * NOTE: 5710 * Because the freeing of aggregated txds will be deferred 5711 * to the aggregating txd, two passes are used here: 5712 * - The first pass GCes any pending txds. This GC is necessary, 5713 * since if the channels are revoked, hypervisor will not 5714 * deliver send-done for all pending txds. 5715 * - The second pass frees the busdma stuffs, i.e. after all txds 5716 * were freed. 5717 */ 5718 for (i = 0; i < txr->hn_txdesc_cnt; ++i) 5719 hn_txdesc_gc(txr, &txr->hn_txdesc[i]); 5720 for (i = 0; i < txr->hn_txdesc_cnt; ++i) 5721 hn_txdesc_dmamap_destroy(&txr->hn_txdesc[i]); 5722 5723 if (txr->hn_tx_data_dtag != NULL) 5724 bus_dma_tag_destroy(txr->hn_tx_data_dtag); 5725 if (txr->hn_tx_rndis_dtag != NULL) 5726 bus_dma_tag_destroy(txr->hn_tx_rndis_dtag); 5727 5728 #ifdef HN_USE_TXDESC_BUFRING 5729 buf_ring_free(txr->hn_txdesc_br, M_DEVBUF); 5730 #endif 5731 5732 free(txr->hn_txdesc, M_DEVBUF); 5733 txr->hn_txdesc = NULL; 5734 5735 if (txr->hn_mbuf_br != NULL) 5736 buf_ring_free(txr->hn_mbuf_br, M_DEVBUF); 5737 5738 #ifndef HN_USE_TXDESC_BUFRING 5739 mtx_destroy(&txr->hn_txlist_spin); 5740 #endif 5741 mtx_destroy(&txr->hn_tx_lock); 5742 } 5743 5744 static int 5745 hn_create_tx_data(struct hn_softc *sc, int ring_cnt) 5746 { 5747 struct sysctl_oid_list *child; 5748 struct sysctl_ctx_list *ctx; 5749 int i; 5750 5751 /* 5752 * Create TXBUF for chimney sending. 5753 * 5754 * NOTE: It is shared by all channels. 5755 */ 5756 sc->hn_chim = contigmalloc(HN_CHIM_SIZE, M_DEVBUF, M_WAITOK | M_ZERO, 5757 0ul, ~0ul, PAGE_SIZE, 0); 5758 if (sc->hn_chim == NULL) { 5759 device_printf(sc->hn_dev, "allocate txbuf failed\n"); 5760 return (ENOMEM); 5761 } 5762 5763 sc->hn_tx_ring_cnt = ring_cnt; 5764 sc->hn_tx_ring_inuse = sc->hn_tx_ring_cnt; 5765 5766 sc->hn_tx_ring = malloc(sizeof(struct hn_tx_ring) * sc->hn_tx_ring_cnt, 5767 M_DEVBUF, M_WAITOK | M_ZERO); 5768 5769 ctx = device_get_sysctl_ctx(sc->hn_dev); 5770 child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->hn_dev)); 5771 5772 /* Create dev.hn.UNIT.tx sysctl tree */ 5773 sc->hn_tx_sysctl_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "tx", 5774 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 5775 5776 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 5777 int error; 5778 5779 error = hn_tx_ring_create(sc, i); 5780 if (error) 5781 return error; 5782 } 5783 5784 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "no_txdescs", 5785 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5786 __offsetof(struct hn_tx_ring, hn_no_txdescs), 5787 hn_tx_stat_ulong_sysctl, "LU", "# of times short of TX descs"); 5788 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "send_failed", 5789 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5790 __offsetof(struct hn_tx_ring, hn_send_failed), 5791 hn_tx_stat_ulong_sysctl, "LU", "# of hyper-v sending failure"); 5792 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txdma_failed", 5793 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5794 __offsetof(struct hn_tx_ring, hn_txdma_failed), 5795 hn_tx_stat_ulong_sysctl, "LU", "# of TX DMA failure"); 5796 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_flush_failed", 5797 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5798 __offsetof(struct hn_tx_ring, hn_flush_failed), 5799 hn_tx_stat_ulong_sysctl, "LU", 5800 "# of packet transmission aggregation flush failure"); 5801 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_collapsed", 5802 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5803 __offsetof(struct hn_tx_ring, hn_tx_collapsed), 5804 hn_tx_stat_ulong_sysctl, "LU", "# of TX mbuf collapsed"); 5805 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney", 5806 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5807 __offsetof(struct hn_tx_ring, hn_tx_chimney), 5808 hn_tx_stat_ulong_sysctl, "LU", "# of chimney send"); 5809 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney_tried", 5810 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_STATS, sc, 5811 __offsetof(struct hn_tx_ring, hn_tx_chimney_tried), 5812 hn_tx_stat_ulong_sysctl, "LU", "# of chimney send tries"); 5813 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "txdesc_cnt", 5814 CTLFLAG_RD, &sc->hn_tx_ring[0].hn_txdesc_cnt, 0, 5815 "# of total TX descs"); 5816 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_chimney_max", 5817 CTLFLAG_RD, &sc->hn_chim_szmax, 0, 5818 "Chimney send packet size upper boundary"); 5819 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_chimney_size", 5820 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 5821 hn_chim_size_sysctl, "I", "Chimney send packet size limit"); 5822 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "direct_tx_size", 5823 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 5824 __offsetof(struct hn_tx_ring, hn_direct_tx_size), 5825 hn_tx_conf_int_sysctl, "I", 5826 "Size of the packet for direct transmission"); 5827 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "sched_tx", 5828 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 5829 __offsetof(struct hn_tx_ring, hn_sched_tx), 5830 hn_tx_conf_int_sysctl, "I", 5831 "Always schedule transmission " 5832 "instead of doing direct transmission"); 5833 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_ring_cnt", 5834 CTLFLAG_RD, &sc->hn_tx_ring_cnt, 0, "# created TX rings"); 5835 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "tx_ring_inuse", 5836 CTLFLAG_RD, &sc->hn_tx_ring_inuse, 0, "# used TX rings"); 5837 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "agg_szmax", 5838 CTLFLAG_RD, &sc->hn_tx_ring[0].hn_agg_szmax, 0, 5839 "Applied packet transmission aggregation size"); 5840 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_pktmax", 5841 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 5842 hn_txagg_pktmax_sysctl, "I", 5843 "Applied packet transmission aggregation packets"); 5844 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "agg_align", 5845 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, 5846 hn_txagg_align_sysctl, "I", 5847 "Applied packet transmission aggregation alignment"); 5848 5849 return 0; 5850 } 5851 5852 static void 5853 hn_set_chim_size(struct hn_softc *sc, int chim_size) 5854 { 5855 int i; 5856 5857 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5858 sc->hn_tx_ring[i].hn_chim_size = chim_size; 5859 } 5860 5861 static void 5862 hn_set_tso_maxsize(struct hn_softc *sc, int tso_maxlen, int mtu) 5863 { 5864 if_t ifp = sc->hn_ifp; 5865 u_int hw_tsomax; 5866 int tso_minlen; 5867 5868 HN_LOCK_ASSERT(sc); 5869 5870 if ((if_getcapabilities(ifp) & (IFCAP_TSO4 | IFCAP_TSO6)) == 0) 5871 return; 5872 5873 KASSERT(sc->hn_ndis_tso_sgmin >= 2, 5874 ("invalid NDIS tso sgmin %d", sc->hn_ndis_tso_sgmin)); 5875 tso_minlen = sc->hn_ndis_tso_sgmin * mtu; 5876 5877 KASSERT(sc->hn_ndis_tso_szmax >= tso_minlen && 5878 sc->hn_ndis_tso_szmax <= IP_MAXPACKET, 5879 ("invalid NDIS tso szmax %d", sc->hn_ndis_tso_szmax)); 5880 5881 if (tso_maxlen < tso_minlen) 5882 tso_maxlen = tso_minlen; 5883 else if (tso_maxlen > IP_MAXPACKET) 5884 tso_maxlen = IP_MAXPACKET; 5885 if (tso_maxlen > sc->hn_ndis_tso_szmax) 5886 tso_maxlen = sc->hn_ndis_tso_szmax; 5887 hw_tsomax = tso_maxlen - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 5888 5889 if (hn_xpnt_vf_isready(sc)) { 5890 if (hw_tsomax > if_gethwtsomax(sc->hn_vf_ifp)) 5891 hw_tsomax = if_gethwtsomax(sc->hn_vf_ifp); 5892 } 5893 if_sethwtsomax(ifp, hw_tsomax); 5894 if (bootverbose) 5895 if_printf(ifp, "TSO size max %u\n", if_gethwtsomax(ifp)); 5896 } 5897 5898 static void 5899 hn_fixup_tx_data(struct hn_softc *sc) 5900 { 5901 uint64_t csum_assist; 5902 int i; 5903 5904 hn_set_chim_size(sc, sc->hn_chim_szmax); 5905 if (hn_tx_chimney_size > 0 && 5906 hn_tx_chimney_size < sc->hn_chim_szmax) 5907 hn_set_chim_size(sc, hn_tx_chimney_size); 5908 5909 csum_assist = 0; 5910 if (sc->hn_caps & HN_CAP_IPCS) 5911 csum_assist |= CSUM_IP; 5912 if (sc->hn_caps & HN_CAP_TCP4CS) 5913 csum_assist |= CSUM_IP_TCP; 5914 if ((sc->hn_caps & HN_CAP_UDP4CS) && hn_enable_udp4cs) 5915 csum_assist |= CSUM_IP_UDP; 5916 if (sc->hn_caps & HN_CAP_TCP6CS) 5917 csum_assist |= CSUM_IP6_TCP; 5918 if ((sc->hn_caps & HN_CAP_UDP6CS) && hn_enable_udp6cs) 5919 csum_assist |= CSUM_IP6_UDP; 5920 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5921 sc->hn_tx_ring[i].hn_csum_assist = csum_assist; 5922 5923 if (sc->hn_caps & HN_CAP_HASHVAL) { 5924 /* 5925 * Support HASHVAL pktinfo on TX path. 5926 */ 5927 if (bootverbose) 5928 if_printf(sc->hn_ifp, "support HASHVAL pktinfo\n"); 5929 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5930 sc->hn_tx_ring[i].hn_tx_flags |= HN_TX_FLAG_HASHVAL; 5931 } 5932 } 5933 5934 static void 5935 hn_fixup_rx_data(struct hn_softc *sc) 5936 { 5937 5938 if (sc->hn_caps & HN_CAP_UDPHASH) { 5939 int i; 5940 5941 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) 5942 sc->hn_rx_ring[i].hn_rx_flags |= HN_RX_FLAG_UDP_HASH; 5943 } 5944 } 5945 5946 static void 5947 hn_destroy_tx_data(struct hn_softc *sc) 5948 { 5949 int i; 5950 5951 if (sc->hn_chim != NULL) { 5952 if ((sc->hn_flags & HN_FLAG_CHIM_REF) == 0) { 5953 free(sc->hn_chim, M_DEVBUF); 5954 } else { 5955 device_printf(sc->hn_dev, 5956 "chimney sending buffer is referenced"); 5957 } 5958 sc->hn_chim = NULL; 5959 } 5960 5961 if (sc->hn_tx_ring_cnt == 0) 5962 return; 5963 5964 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) 5965 hn_tx_ring_destroy(&sc->hn_tx_ring[i]); 5966 5967 free(sc->hn_tx_ring, M_DEVBUF); 5968 sc->hn_tx_ring = NULL; 5969 5970 sc->hn_tx_ring_cnt = 0; 5971 sc->hn_tx_ring_inuse = 0; 5972 } 5973 5974 #ifdef HN_IFSTART_SUPPORT 5975 5976 static void 5977 hn_start_taskfunc(void *xtxr, int pending __unused) 5978 { 5979 struct hn_tx_ring *txr = xtxr; 5980 5981 mtx_lock(&txr->hn_tx_lock); 5982 hn_start_locked(txr, 0); 5983 mtx_unlock(&txr->hn_tx_lock); 5984 } 5985 5986 static int 5987 hn_start_locked(struct hn_tx_ring *txr, int len) 5988 { 5989 struct hn_softc *sc = txr->hn_sc; 5990 if_t ifp = sc->hn_ifp; 5991 int sched = 0; 5992 5993 KASSERT(hn_use_if_start, 5994 ("hn_start_locked is called, when if_start is disabled")); 5995 KASSERT(txr == &sc->hn_tx_ring[0], ("not the first TX ring")); 5996 mtx_assert(&txr->hn_tx_lock, MA_OWNED); 5997 KASSERT(txr->hn_agg_txd == NULL, ("lingering aggregating txdesc")); 5998 5999 if (__predict_false(txr->hn_suspended)) 6000 return (0); 6001 6002 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 6003 IFF_DRV_RUNNING) 6004 return (0); 6005 6006 while (!if_sendq_empty(ifp)) { 6007 struct hn_txdesc *txd; 6008 struct mbuf *m_head; 6009 int error; 6010 6011 m_head = if_dequeue(ifp); 6012 if (m_head == NULL) 6013 break; 6014 6015 if (len > 0 && m_head->m_pkthdr.len > len) { 6016 /* 6017 * This sending could be time consuming; let callers 6018 * dispatch this packet sending (and sending of any 6019 * following up packets) to tx taskqueue. 6020 */ 6021 if_sendq_prepend(ifp, m_head); 6022 sched = 1; 6023 break; 6024 } 6025 6026 #if defined(INET6) || defined(INET) 6027 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 6028 m_head = hn_tso_fixup(m_head); 6029 if (__predict_false(m_head == NULL)) { 6030 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6031 continue; 6032 } 6033 } else if (m_head->m_pkthdr.csum_flags & 6034 (CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP6_UDP | CSUM_IP6_TCP)) { 6035 m_head = hn_set_hlen(m_head); 6036 if (__predict_false(m_head == NULL)) { 6037 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6038 continue; 6039 } 6040 } 6041 #endif 6042 6043 txd = hn_txdesc_get(txr); 6044 if (txd == NULL) { 6045 txr->hn_no_txdescs++; 6046 if_sendq_prepend(ifp, m_head); 6047 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); 6048 break; 6049 } 6050 6051 error = hn_encap(ifp, txr, txd, &m_head); 6052 if (error) { 6053 /* Both txd and m_head are freed */ 6054 KASSERT(txr->hn_agg_txd == NULL, 6055 ("encap failed w/ pending aggregating txdesc")); 6056 continue; 6057 } 6058 6059 if (txr->hn_agg_pktleft == 0) { 6060 if (txr->hn_agg_txd != NULL) { 6061 KASSERT(m_head == NULL, 6062 ("pending mbuf for aggregating txdesc")); 6063 error = hn_flush_txagg(ifp, txr); 6064 if (__predict_false(error)) { 6065 if_setdrvflagbits(ifp, 6066 IFF_DRV_OACTIVE, 0); 6067 break; 6068 } 6069 } else { 6070 KASSERT(m_head != NULL, ("mbuf was freed")); 6071 error = hn_txpkt(ifp, txr, txd); 6072 if (__predict_false(error)) { 6073 /* txd is freed, but m_head is not */ 6074 if_sendq_prepend(ifp, m_head); 6075 if_setdrvflagbits(ifp, 6076 IFF_DRV_OACTIVE, 0); 6077 break; 6078 } 6079 } 6080 } 6081 #ifdef INVARIANTS 6082 else { 6083 KASSERT(txr->hn_agg_txd != NULL, 6084 ("no aggregating txdesc")); 6085 KASSERT(m_head == NULL, 6086 ("pending mbuf for aggregating txdesc")); 6087 } 6088 #endif 6089 } 6090 6091 /* Flush pending aggerated transmission. */ 6092 if (txr->hn_agg_txd != NULL) 6093 hn_flush_txagg(ifp, txr); 6094 return (sched); 6095 } 6096 6097 static void 6098 hn_start(if_t ifp) 6099 { 6100 struct hn_softc *sc = if_getsoftc(ifp); 6101 struct hn_tx_ring *txr = &sc->hn_tx_ring[0]; 6102 6103 if (txr->hn_sched_tx) 6104 goto do_sched; 6105 6106 if (mtx_trylock(&txr->hn_tx_lock)) { 6107 int sched; 6108 6109 sched = hn_start_locked(txr, txr->hn_direct_tx_size); 6110 mtx_unlock(&txr->hn_tx_lock); 6111 if (!sched) 6112 return; 6113 } 6114 do_sched: 6115 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); 6116 } 6117 6118 static void 6119 hn_start_txeof_taskfunc(void *xtxr, int pending __unused) 6120 { 6121 struct hn_tx_ring *txr = xtxr; 6122 6123 mtx_lock(&txr->hn_tx_lock); 6124 if_setdrvflagbits(txr->hn_sc->hn_ifp, 0, IFF_DRV_OACTIVE); 6125 hn_start_locked(txr, 0); 6126 mtx_unlock(&txr->hn_tx_lock); 6127 } 6128 6129 static void 6130 hn_start_txeof(struct hn_tx_ring *txr) 6131 { 6132 struct hn_softc *sc = txr->hn_sc; 6133 if_t ifp = sc->hn_ifp; 6134 6135 KASSERT(txr == &sc->hn_tx_ring[0], ("not the first TX ring")); 6136 6137 if (txr->hn_sched_tx) 6138 goto do_sched; 6139 6140 if (mtx_trylock(&txr->hn_tx_lock)) { 6141 int sched; 6142 6143 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 6144 sched = hn_start_locked(txr, txr->hn_direct_tx_size); 6145 mtx_unlock(&txr->hn_tx_lock); 6146 if (sched) { 6147 taskqueue_enqueue(txr->hn_tx_taskq, 6148 &txr->hn_tx_task); 6149 } 6150 } else { 6151 do_sched: 6152 /* 6153 * Release the OACTIVE earlier, with the hope, that 6154 * others could catch up. The task will clear the 6155 * flag again with the hn_tx_lock to avoid possible 6156 * races. 6157 */ 6158 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 6159 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); 6160 } 6161 } 6162 6163 #endif /* HN_IFSTART_SUPPORT */ 6164 6165 static int 6166 hn_xmit(struct hn_tx_ring *txr, int len) 6167 { 6168 struct hn_softc *sc = txr->hn_sc; 6169 if_t ifp = sc->hn_ifp; 6170 struct mbuf *m_head; 6171 int sched = 0; 6172 6173 mtx_assert(&txr->hn_tx_lock, MA_OWNED); 6174 #ifdef HN_IFSTART_SUPPORT 6175 KASSERT(hn_use_if_start == 0, 6176 ("hn_xmit is called, when if_start is enabled")); 6177 #endif 6178 KASSERT(txr->hn_agg_txd == NULL, ("lingering aggregating txdesc")); 6179 6180 if (__predict_false(txr->hn_suspended)) 6181 return (0); 6182 6183 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || txr->hn_oactive) 6184 return (0); 6185 6186 while ((m_head = drbr_peek(ifp, txr->hn_mbuf_br)) != NULL) { 6187 struct hn_txdesc *txd; 6188 int error; 6189 6190 if (len > 0 && m_head->m_pkthdr.len > len) { 6191 /* 6192 * This sending could be time consuming; let callers 6193 * dispatch this packet sending (and sending of any 6194 * following up packets) to tx taskqueue. 6195 */ 6196 drbr_putback(ifp, txr->hn_mbuf_br, m_head); 6197 sched = 1; 6198 break; 6199 } 6200 6201 txd = hn_txdesc_get(txr); 6202 if (txd == NULL) { 6203 txr->hn_no_txdescs++; 6204 drbr_putback(ifp, txr->hn_mbuf_br, m_head); 6205 txr->hn_oactive = 1; 6206 break; 6207 } 6208 6209 error = hn_encap(ifp, txr, txd, &m_head); 6210 if (error) { 6211 /* Both txd and m_head are freed; discard */ 6212 KASSERT(txr->hn_agg_txd == NULL, 6213 ("encap failed w/ pending aggregating txdesc")); 6214 drbr_advance(ifp, txr->hn_mbuf_br); 6215 continue; 6216 } 6217 6218 if (txr->hn_agg_pktleft == 0) { 6219 if (txr->hn_agg_txd != NULL) { 6220 KASSERT(m_head == NULL, 6221 ("pending mbuf for aggregating txdesc")); 6222 error = hn_flush_txagg(ifp, txr); 6223 if (__predict_false(error)) { 6224 txr->hn_oactive = 1; 6225 break; 6226 } 6227 } else { 6228 KASSERT(m_head != NULL, ("mbuf was freed")); 6229 error = hn_txpkt(ifp, txr, txd); 6230 if (__predict_false(error)) { 6231 /* txd is freed, but m_head is not */ 6232 drbr_putback(ifp, txr->hn_mbuf_br, 6233 m_head); 6234 txr->hn_oactive = 1; 6235 break; 6236 } 6237 } 6238 } 6239 #ifdef INVARIANTS 6240 else { 6241 KASSERT(txr->hn_agg_txd != NULL, 6242 ("no aggregating txdesc")); 6243 KASSERT(m_head == NULL, 6244 ("pending mbuf for aggregating txdesc")); 6245 } 6246 #endif 6247 6248 /* Sent */ 6249 drbr_advance(ifp, txr->hn_mbuf_br); 6250 } 6251 6252 /* Flush pending aggerated transmission. */ 6253 if (txr->hn_agg_txd != NULL) 6254 hn_flush_txagg(ifp, txr); 6255 return (sched); 6256 } 6257 6258 static int 6259 hn_transmit(if_t ifp, struct mbuf *m) 6260 { 6261 struct hn_softc *sc = if_getsoftc(ifp); 6262 struct hn_tx_ring *txr; 6263 int error, idx = 0; 6264 6265 if (sc->hn_xvf_flags & (HN_XVFFLAG_ENABLED | HN_XVFFLAG_SWITCHING)) { 6266 struct rm_priotracker pt; 6267 6268 rm_rlock(&sc->hn_vf_lock, &pt); 6269 if ((sc->hn_xvf_flags & HN_XVFFLAG_SWITCHING) || 6270 ((sc->hn_xvf_flags & HN_XVFFLAG_ENABLED) && 6271 sc->hn_vf_active_assoc != atomic_load_acq_int(&sc->hn_vf_assoc))) { 6272 rm_runlock(&sc->hn_vf_lock, &pt); 6273 m_freem(m); 6274 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 6275 return (ENETDOWN); 6276 } 6277 if (__predict_true(sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) { 6278 struct mbuf *m_bpf = NULL; 6279 int obytes, omcast; 6280 6281 obytes = m->m_pkthdr.len; 6282 omcast = (m->m_flags & M_MCAST) != 0; 6283 6284 if (sc->hn_xvf_flags & HN_XVFFLAG_ACCBPF) { 6285 if (bpf_peers_present_if(ifp)) { 6286 m_bpf = m_copypacket(m, M_NOWAIT); 6287 if (m_bpf == NULL) { 6288 /* 6289 * Failed to grab a shallow 6290 * copy; tap now. 6291 */ 6292 ETHER_BPF_MTAP(ifp, m); 6293 } 6294 } 6295 } else { 6296 ETHER_BPF_MTAP(ifp, m); 6297 } 6298 6299 error = if_transmit(sc->hn_vf_ifp, m); 6300 rm_runlock(&sc->hn_vf_lock, &pt); 6301 6302 if (m_bpf != NULL) { 6303 if (!error) 6304 ETHER_BPF_MTAP(ifp, m_bpf); 6305 m_freem(m_bpf); 6306 } 6307 6308 if (error == ENOBUFS) { 6309 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 6310 } else if (error) { 6311 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6312 } else { 6313 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 6314 if_inc_counter(ifp, IFCOUNTER_OBYTES, obytes); 6315 if (omcast) { 6316 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 6317 omcast); 6318 } 6319 } 6320 return (error); 6321 } 6322 rm_runlock(&sc->hn_vf_lock, &pt); 6323 } 6324 6325 #if defined(INET6) || defined(INET) 6326 /* 6327 * Perform TSO packet header fixup or get l2/l3 header length now, 6328 * since packet headers should be cache-hot. 6329 */ 6330 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 6331 m = hn_tso_fixup(m); 6332 if (__predict_false(m == NULL)) { 6333 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6334 return EIO; 6335 } 6336 } else if (m->m_pkthdr.csum_flags & 6337 (CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP6_UDP | CSUM_IP6_TCP)) { 6338 m = hn_set_hlen(m); 6339 if (__predict_false(m == NULL)) { 6340 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); 6341 return EIO; 6342 } 6343 } 6344 #endif 6345 6346 /* 6347 * Select the TX ring based on flowid 6348 */ 6349 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 6350 #ifdef RSS 6351 uint32_t bid; 6352 6353 if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 6354 &bid) == 0) 6355 idx = bid % sc->hn_tx_ring_inuse; 6356 else 6357 #endif 6358 { 6359 #if defined(INET6) || defined(INET) 6360 int tcpsyn = 0; 6361 6362 if (m->m_pkthdr.len < 128 && 6363 (m->m_pkthdr.csum_flags & 6364 (CSUM_IP_TCP | CSUM_IP6_TCP)) && 6365 (m->m_pkthdr.csum_flags & CSUM_TSO) == 0) { 6366 m = hn_check_tcpsyn(m, &tcpsyn); 6367 if (__predict_false(m == NULL)) { 6368 if_inc_counter(ifp, 6369 IFCOUNTER_OERRORS, 1); 6370 return (EIO); 6371 } 6372 } 6373 #else 6374 const int tcpsyn = 0; 6375 #endif 6376 if (tcpsyn) 6377 idx = 0; 6378 else 6379 idx = m->m_pkthdr.flowid % sc->hn_tx_ring_inuse; 6380 } 6381 } 6382 txr = &sc->hn_tx_ring[idx]; 6383 6384 error = drbr_enqueue(ifp, txr->hn_mbuf_br, m); 6385 if (error) { 6386 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 6387 return error; 6388 } 6389 6390 if (txr->hn_oactive) 6391 return 0; 6392 6393 if (txr->hn_sched_tx) 6394 goto do_sched; 6395 6396 if (mtx_trylock(&txr->hn_tx_lock)) { 6397 int sched; 6398 6399 sched = hn_xmit(txr, txr->hn_direct_tx_size); 6400 mtx_unlock(&txr->hn_tx_lock); 6401 if (!sched) 6402 return 0; 6403 } 6404 do_sched: 6405 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_tx_task); 6406 return 0; 6407 } 6408 6409 static void 6410 hn_tx_ring_qflush(struct hn_tx_ring *txr) 6411 { 6412 struct mbuf *m; 6413 6414 mtx_lock(&txr->hn_tx_lock); 6415 while ((m = buf_ring_dequeue_sc(txr->hn_mbuf_br)) != NULL) 6416 m_freem(m); 6417 mtx_unlock(&txr->hn_tx_lock); 6418 } 6419 6420 static void 6421 hn_xmit_qflush(if_t ifp) 6422 { 6423 struct hn_softc *sc = if_getsoftc(ifp); 6424 struct rm_priotracker pt; 6425 int i; 6426 6427 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) 6428 hn_tx_ring_qflush(&sc->hn_tx_ring[i]); 6429 if_qflush(ifp); 6430 6431 rm_rlock(&sc->hn_vf_lock, &pt); 6432 /* Transparent mode owns the VF's queues even while switching paths. */ 6433 if (hn_xpnt_vf && sc->hn_vf_ifp != NULL) 6434 if_getqflushfn(sc->hn_vf_ifp)(sc->hn_vf_ifp); 6435 rm_runlock(&sc->hn_vf_lock, &pt); 6436 } 6437 6438 static void 6439 hn_xmit_txeof(struct hn_tx_ring *txr) 6440 { 6441 6442 if (txr->hn_sched_tx) 6443 goto do_sched; 6444 6445 if (mtx_trylock(&txr->hn_tx_lock)) { 6446 int sched; 6447 6448 txr->hn_oactive = 0; 6449 sched = hn_xmit(txr, txr->hn_direct_tx_size); 6450 mtx_unlock(&txr->hn_tx_lock); 6451 if (sched) { 6452 taskqueue_enqueue(txr->hn_tx_taskq, 6453 &txr->hn_tx_task); 6454 } 6455 } else { 6456 do_sched: 6457 /* 6458 * Release the oactive earlier, with the hope, that 6459 * others could catch up. The task will clear the 6460 * oactive again with the hn_tx_lock to avoid possible 6461 * races. 6462 */ 6463 txr->hn_oactive = 0; 6464 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); 6465 } 6466 } 6467 6468 static void 6469 hn_xmit_taskfunc(void *xtxr, int pending __unused) 6470 { 6471 struct hn_tx_ring *txr = xtxr; 6472 6473 mtx_lock(&txr->hn_tx_lock); 6474 hn_xmit(txr, 0); 6475 mtx_unlock(&txr->hn_tx_lock); 6476 } 6477 6478 static void 6479 hn_xmit_txeof_taskfunc(void *xtxr, int pending __unused) 6480 { 6481 struct hn_tx_ring *txr = xtxr; 6482 6483 mtx_lock(&txr->hn_tx_lock); 6484 txr->hn_oactive = 0; 6485 hn_xmit(txr, 0); 6486 mtx_unlock(&txr->hn_tx_lock); 6487 } 6488 6489 static int 6490 hn_chan_attach(struct hn_softc *sc, struct vmbus_channel *chan) 6491 { 6492 struct vmbus_chan_br cbr; 6493 struct hn_rx_ring *rxr; 6494 struct hn_tx_ring *txr = NULL; 6495 int idx, error; 6496 6497 idx = vmbus_chan_subidx(chan); 6498 6499 /* 6500 * Link this channel to RX/TX ring. 6501 */ 6502 KASSERT(idx >= 0 && idx < sc->hn_rx_ring_inuse, 6503 ("invalid channel index %d, should > 0 && < %d", 6504 idx, sc->hn_rx_ring_inuse)); 6505 rxr = &sc->hn_rx_ring[idx]; 6506 KASSERT((rxr->hn_rx_flags & HN_RX_FLAG_ATTACHED) == 0, 6507 ("RX ring %d already attached", idx)); 6508 rxr->hn_rx_flags |= HN_RX_FLAG_ATTACHED; 6509 rxr->hn_chan = chan; 6510 6511 if (bootverbose) { 6512 if_printf(sc->hn_ifp, "link RX ring %d to chan%u\n", 6513 idx, vmbus_chan_id(chan)); 6514 } 6515 6516 if (idx < sc->hn_tx_ring_inuse) { 6517 txr = &sc->hn_tx_ring[idx]; 6518 KASSERT((txr->hn_tx_flags & HN_TX_FLAG_ATTACHED) == 0, 6519 ("TX ring %d already attached", idx)); 6520 txr->hn_tx_flags |= HN_TX_FLAG_ATTACHED; 6521 6522 txr->hn_chan = chan; 6523 if (bootverbose) { 6524 if_printf(sc->hn_ifp, "link TX ring %d to chan%u\n", 6525 idx, vmbus_chan_id(chan)); 6526 } 6527 } 6528 6529 /* Bind this channel to a proper CPU. */ 6530 vmbus_chan_cpu_set(chan, HN_RING_IDX2CPU(sc, idx)); 6531 6532 /* 6533 * Open this channel 6534 */ 6535 cbr.cbr = rxr->hn_br; 6536 cbr.cbr_paddr = pmap_kextract((vm_offset_t)rxr->hn_br); 6537 cbr.cbr_txsz = HN_TXBR_SIZE; 6538 cbr.cbr_rxsz = HN_RXBR_SIZE; 6539 error = vmbus_chan_open_br(chan, &cbr, NULL, 0, hn_chan_callback, rxr); 6540 if (error) { 6541 if (error == EISCONN) { 6542 if_printf(sc->hn_ifp, "bufring is connected after " 6543 "chan%u open failure\n", vmbus_chan_id(chan)); 6544 rxr->hn_rx_flags |= HN_RX_FLAG_BR_REF; 6545 } else { 6546 if_printf(sc->hn_ifp, "open chan%u failed: %d\n", 6547 vmbus_chan_id(chan), error); 6548 } 6549 } 6550 return (error); 6551 } 6552 6553 static void 6554 hn_chan_detach(struct hn_softc *sc, struct vmbus_channel *chan) 6555 { 6556 struct hn_rx_ring *rxr; 6557 int idx, error; 6558 6559 idx = vmbus_chan_subidx(chan); 6560 6561 /* 6562 * Link this channel to RX/TX ring. 6563 */ 6564 KASSERT(idx >= 0 && idx < sc->hn_rx_ring_inuse, 6565 ("invalid channel index %d, should > 0 && < %d", 6566 idx, sc->hn_rx_ring_inuse)); 6567 rxr = &sc->hn_rx_ring[idx]; 6568 KASSERT((rxr->hn_rx_flags & HN_RX_FLAG_ATTACHED), 6569 ("RX ring %d is not attached", idx)); 6570 rxr->hn_rx_flags &= ~HN_RX_FLAG_ATTACHED; 6571 6572 if (idx < sc->hn_tx_ring_inuse) { 6573 struct hn_tx_ring *txr = &sc->hn_tx_ring[idx]; 6574 6575 KASSERT((txr->hn_tx_flags & HN_TX_FLAG_ATTACHED), 6576 ("TX ring %d is not attached attached", idx)); 6577 txr->hn_tx_flags &= ~HN_TX_FLAG_ATTACHED; 6578 } 6579 6580 /* 6581 * Close this channel. 6582 * 6583 * NOTE: 6584 * Channel closing does _not_ destroy the target channel. 6585 */ 6586 error = vmbus_chan_close_direct(chan); 6587 if (error == EISCONN) { 6588 if_printf(sc->hn_ifp, "chan%u bufring is connected " 6589 "after being closed\n", vmbus_chan_id(chan)); 6590 rxr->hn_rx_flags |= HN_RX_FLAG_BR_REF; 6591 } else if (error) { 6592 if_printf(sc->hn_ifp, "chan%u close failed: %d\n", 6593 vmbus_chan_id(chan), error); 6594 } 6595 } 6596 6597 static int 6598 hn_attach_subchans(struct hn_softc *sc) 6599 { 6600 struct vmbus_channel **subchans; 6601 int subchan_cnt = sc->hn_rx_ring_inuse - 1; 6602 int i, error = 0; 6603 6604 KASSERT(subchan_cnt > 0, ("no sub-channels")); 6605 6606 /* Attach the sub-channels. */ 6607 subchans = vmbus_subchan_get(sc->hn_prichan, subchan_cnt); 6608 for (i = 0; i < subchan_cnt; ++i) { 6609 int error1; 6610 6611 error1 = hn_chan_attach(sc, subchans[i]); 6612 if (error1) { 6613 error = error1; 6614 /* Move on; all channels will be detached later. */ 6615 } 6616 } 6617 vmbus_subchan_rel(subchans, subchan_cnt); 6618 6619 if (error) { 6620 if_printf(sc->hn_ifp, "sub-channels attach failed: %d\n", error); 6621 } else { 6622 if (bootverbose) { 6623 if_printf(sc->hn_ifp, "%d sub-channels attached\n", 6624 subchan_cnt); 6625 } 6626 } 6627 return (error); 6628 } 6629 6630 static void 6631 hn_detach_allchans(struct hn_softc *sc) 6632 { 6633 struct vmbus_channel **subchans; 6634 int subchan_cnt = sc->hn_rx_ring_inuse - 1; 6635 int i; 6636 6637 if (subchan_cnt == 0) 6638 goto back; 6639 6640 /* Detach the sub-channels. */ 6641 subchans = vmbus_subchan_get(sc->hn_prichan, subchan_cnt); 6642 for (i = 0; i < subchan_cnt; ++i) 6643 hn_chan_detach(sc, subchans[i]); 6644 vmbus_subchan_rel(subchans, subchan_cnt); 6645 6646 back: 6647 /* 6648 * Detach the primary channel, _after_ all sub-channels 6649 * are detached. 6650 */ 6651 hn_chan_detach(sc, sc->hn_prichan); 6652 6653 /* Wait for sub-channels to be destroyed, if any. */ 6654 vmbus_subchan_drain(sc->hn_prichan); 6655 6656 #ifdef INVARIANTS 6657 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 6658 KASSERT((sc->hn_rx_ring[i].hn_rx_flags & 6659 HN_RX_FLAG_ATTACHED) == 0, 6660 ("%dth RX ring is still attached", i)); 6661 } 6662 for (i = 0; i < sc->hn_tx_ring_cnt; ++i) { 6663 KASSERT((sc->hn_tx_ring[i].hn_tx_flags & 6664 HN_TX_FLAG_ATTACHED) == 0, 6665 ("%dth TX ring is still attached", i)); 6666 } 6667 #endif 6668 } 6669 6670 static int 6671 hn_synth_alloc_subchans(struct hn_softc *sc, int *nsubch) 6672 { 6673 struct vmbus_channel **subchans; 6674 int nchan, rxr_cnt, error; 6675 6676 nchan = *nsubch + 1; 6677 if (nchan == 1) { 6678 /* 6679 * Multiple RX/TX rings are not requested. 6680 */ 6681 *nsubch = 0; 6682 return (0); 6683 } 6684 6685 /* 6686 * Query RSS capabilities, e.g. # of RX rings, and # of indirect 6687 * table entries. 6688 */ 6689 error = hn_rndis_query_rsscaps(sc, &rxr_cnt); 6690 if (error) { 6691 /* No RSS; this is benign. */ 6692 *nsubch = 0; 6693 return (0); 6694 } 6695 if (bootverbose) { 6696 if_printf(sc->hn_ifp, "RX rings offered %u, requested %d\n", 6697 rxr_cnt, nchan); 6698 } 6699 6700 if (nchan > rxr_cnt) 6701 nchan = rxr_cnt; 6702 if (nchan == 1) { 6703 if_printf(sc->hn_ifp, "only 1 channel is supported, no vRSS\n"); 6704 *nsubch = 0; 6705 return (0); 6706 } 6707 6708 /* 6709 * Allocate sub-channels from NVS. 6710 */ 6711 *nsubch = nchan - 1; 6712 error = hn_nvs_alloc_subchans(sc, nsubch); 6713 if (error || *nsubch == 0) { 6714 /* Failed to allocate sub-channels. */ 6715 *nsubch = 0; 6716 return (0); 6717 } 6718 6719 /* 6720 * Wait for all sub-channels to become ready before moving on. 6721 */ 6722 subchans = vmbus_subchan_get(sc->hn_prichan, *nsubch); 6723 vmbus_subchan_rel(subchans, *nsubch); 6724 return (0); 6725 } 6726 6727 static bool 6728 hn_synth_attachable(const struct hn_softc *sc) 6729 { 6730 int i; 6731 6732 if (sc->hn_flags & HN_FLAG_ERRORS) 6733 return (false); 6734 6735 for (i = 0; i < sc->hn_rx_ring_cnt; ++i) { 6736 const struct hn_rx_ring *rxr = &sc->hn_rx_ring[i]; 6737 6738 if (rxr->hn_rx_flags & HN_RX_FLAG_BR_REF) 6739 return (false); 6740 } 6741 return (true); 6742 } 6743 6744 /* 6745 * Make sure that the RX filter is zero after the successful 6746 * RNDIS initialization. 6747 * 6748 * NOTE: 6749 * Under certain conditions on certain versions of Hyper-V, 6750 * the RNDIS rxfilter is _not_ zero on the hypervisor side 6751 * after the successful RNDIS initialization, which breaks 6752 * the assumption of any following code (well, it breaks the 6753 * RNDIS API contract actually). Clear the RNDIS rxfilter 6754 * explicitly, drain packets sneaking through, and drain the 6755 * interrupt taskqueues scheduled due to the stealth packets. 6756 */ 6757 static void 6758 hn_rndis_init_fixat(struct hn_softc *sc, int nchan) 6759 { 6760 6761 hn_disable_rx(sc); 6762 hn_drain_rxtx(sc, nchan); 6763 } 6764 6765 static int 6766 hn_synth_attach(struct hn_softc *sc, int mtu) 6767 { 6768 #define ATTACHED_NVS 0x0002 6769 #define ATTACHED_RNDIS 0x0004 6770 6771 struct ndis_rssprm_toeplitz *rss = &sc->hn_rss; 6772 int error, nsubch, nchan = 1, i, rndis_inited; 6773 uint32_t old_caps, attached = 0; 6774 6775 KASSERT((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) == 0, 6776 ("synthetic parts were attached")); 6777 6778 if (!hn_synth_attachable(sc)) 6779 return (ENXIO); 6780 6781 /* Save capabilities for later verification. */ 6782 old_caps = sc->hn_caps; 6783 sc->hn_caps = 0; 6784 6785 /* Clear RSS stuffs. */ 6786 sc->hn_rss_ind_size = 0; 6787 sc->hn_rss_hash = 0; 6788 sc->hn_rss_hcap = 0; 6789 6790 /* 6791 * Attach the primary channel _before_ attaching NVS and RNDIS. 6792 */ 6793 atomic_store_rel_int(&sc->hn_vf_assoc, 6794 (atomic_load_int(&sc->hn_vf_assoc) + HN_VF_ASSOC_GENINC) & 6795 ~HN_VF_ASSOC_ALLOCATED); 6796 error = hn_chan_attach(sc, sc->hn_prichan); 6797 if (error) 6798 goto failed; 6799 6800 /* 6801 * Attach NVS. 6802 */ 6803 error = hn_nvs_attach(sc, mtu); 6804 if (error) 6805 goto failed; 6806 attached |= ATTACHED_NVS; 6807 6808 /* 6809 * Attach RNDIS _after_ NVS is attached. 6810 */ 6811 error = hn_rndis_attach(sc, mtu, &rndis_inited); 6812 if (rndis_inited) 6813 attached |= ATTACHED_RNDIS; 6814 if (error) 6815 goto failed; 6816 6817 /* 6818 * Make sure capabilities are not changed. 6819 */ 6820 if (device_is_attached(sc->hn_dev) && old_caps != sc->hn_caps) { 6821 if_printf(sc->hn_ifp, "caps mismatch old 0x%08x, new 0x%08x\n", 6822 old_caps, sc->hn_caps); 6823 error = ENXIO; 6824 goto failed; 6825 } 6826 6827 /* 6828 * Allocate sub-channels for multi-TX/RX rings. 6829 * 6830 * NOTE: 6831 * The # of RX rings that can be used is equivalent to the # of 6832 * channels to be requested. 6833 */ 6834 nsubch = sc->hn_rx_ring_cnt - 1; 6835 error = hn_synth_alloc_subchans(sc, &nsubch); 6836 if (error) 6837 goto failed; 6838 /* NOTE: _Full_ synthetic parts detach is required now. */ 6839 sc->hn_flags |= HN_FLAG_SYNTH_ATTACHED; 6840 6841 /* 6842 * Set the # of TX/RX rings that could be used according to 6843 * the # of channels that NVS offered. 6844 */ 6845 nchan = nsubch + 1; 6846 hn_set_ring_inuse(sc, nchan); 6847 if (nchan == 1) { 6848 /* Only the primary channel can be used; done */ 6849 goto back; 6850 } 6851 6852 /* 6853 * Attach the sub-channels. 6854 * 6855 * NOTE: hn_set_ring_inuse() _must_ have been called. 6856 */ 6857 error = hn_attach_subchans(sc); 6858 if (error) 6859 goto failed; 6860 6861 /* 6862 * Configure RSS key and indirect table _after_ all sub-channels 6863 * are attached. 6864 */ 6865 if ((sc->hn_flags & HN_FLAG_HAS_RSSKEY) == 0) { 6866 /* 6867 * RSS key is not set yet; set it to the default RSS key. 6868 */ 6869 if (bootverbose) 6870 if_printf(sc->hn_ifp, "setup default RSS key\n"); 6871 rss_getkey(rss->rss_key); 6872 sc->hn_flags |= HN_FLAG_HAS_RSSKEY; 6873 } 6874 6875 if ((sc->hn_flags & HN_FLAG_HAS_RSSIND) == 0) { 6876 /* 6877 * RSS indirect table is not set yet; set it up in round- 6878 * robin fashion. 6879 */ 6880 if (bootverbose) { 6881 if_printf(sc->hn_ifp, "setup default RSS indirect " 6882 "table\n"); 6883 } 6884 for (i = 0; i < NDIS_HASH_INDCNT; ++i) { 6885 uint32_t subidx; 6886 6887 #ifdef RSS 6888 subidx = rss_get_indirection_to_bucket(i); 6889 #else 6890 subidx = i; 6891 #endif 6892 rss->rss_ind[i] = subidx % nchan; 6893 } 6894 sc->hn_flags |= HN_FLAG_HAS_RSSIND; 6895 } else { 6896 /* 6897 * # of usable channels may be changed, so we have to 6898 * make sure that all entries in RSS indirect table 6899 * are valid. 6900 * 6901 * NOTE: hn_set_ring_inuse() _must_ have been called. 6902 */ 6903 hn_rss_ind_fixup(sc); 6904 } 6905 6906 sc->hn_rss_hash = sc->hn_rss_hcap; 6907 if ((sc->hn_flags & HN_FLAG_RXVF) || 6908 (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) { 6909 /* NOTE: Don't reconfigure RSS; will do immediately. */ 6910 hn_vf_rss_fixup(sc, false); 6911 } 6912 error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_NONE); 6913 if (error) 6914 goto failed; 6915 back: 6916 /* 6917 * Fixup transmission aggregation setup. 6918 */ 6919 hn_set_txagg(sc); 6920 hn_rndis_init_fixat(sc, nchan); 6921 return (0); 6922 6923 failed: 6924 if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) { 6925 hn_rndis_init_fixat(sc, nchan); 6926 hn_synth_detach(sc); 6927 } else { 6928 if (attached & ATTACHED_RNDIS) { 6929 hn_rndis_init_fixat(sc, nchan); 6930 hn_rndis_detach(sc); 6931 } 6932 if (attached & ATTACHED_NVS) 6933 hn_nvs_detach(sc); 6934 hn_chan_detach(sc, sc->hn_prichan); 6935 /* Restore old capabilities. */ 6936 sc->hn_caps = old_caps; 6937 } 6938 return (error); 6939 6940 #undef ATTACHED_RNDIS 6941 #undef ATTACHED_NVS 6942 } 6943 6944 /* 6945 * NOTE: 6946 * The interface must have been suspended though hn_suspend(), before 6947 * this function get called. 6948 */ 6949 static void 6950 hn_synth_detach(struct hn_softc *sc) 6951 { 6952 6953 KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED, 6954 ("synthetic parts were not attached")); 6955 6956 /* Detach the RNDIS first. */ 6957 hn_rndis_detach(sc); 6958 6959 /* Detach NVS. */ 6960 hn_nvs_detach(sc); 6961 6962 /* Detach all of the channels. */ 6963 hn_detach_allchans(sc); 6964 6965 if (vmbus_current_version >= VMBUS_VERSION_WIN10 && sc->hn_rxbuf_gpadl != 0) { 6966 /* 6967 * Host is post-Win2016, disconnect RXBUF from primary channel here. 6968 */ 6969 int error; 6970 6971 error = vmbus_chan_gpadl_disconnect(sc->hn_prichan, 6972 sc->hn_rxbuf_gpadl); 6973 if (error) { 6974 if_printf(sc->hn_ifp, 6975 "rxbuf gpadl disconn failed: %d\n", error); 6976 sc->hn_flags |= HN_FLAG_RXBUF_REF; 6977 } 6978 sc->hn_rxbuf_gpadl = 0; 6979 } 6980 6981 if (vmbus_current_version >= VMBUS_VERSION_WIN10 && sc->hn_chim_gpadl != 0) { 6982 /* 6983 * Host is post-Win2016, disconnect chimney sending buffer from 6984 * primary channel here. 6985 */ 6986 int error; 6987 6988 error = vmbus_chan_gpadl_disconnect(sc->hn_prichan, 6989 sc->hn_chim_gpadl); 6990 if (error) { 6991 if_printf(sc->hn_ifp, 6992 "chim gpadl disconn failed: %d\n", error); 6993 sc->hn_flags |= HN_FLAG_CHIM_REF; 6994 } 6995 sc->hn_chim_gpadl = 0; 6996 } 6997 sc->hn_flags &= ~HN_FLAG_SYNTH_ATTACHED; 6998 } 6999 7000 static void 7001 hn_set_ring_inuse(struct hn_softc *sc, int ring_cnt) 7002 { 7003 KASSERT(ring_cnt > 0 && ring_cnt <= sc->hn_rx_ring_cnt, 7004 ("invalid ring count %d", ring_cnt)); 7005 7006 if (sc->hn_tx_ring_cnt > ring_cnt) 7007 sc->hn_tx_ring_inuse = ring_cnt; 7008 else 7009 sc->hn_tx_ring_inuse = sc->hn_tx_ring_cnt; 7010 sc->hn_rx_ring_inuse = ring_cnt; 7011 7012 #ifdef RSS 7013 if (sc->hn_rx_ring_inuse != rss_getnumbuckets()) { 7014 if_printf(sc->hn_ifp, "# of RX rings (%d) does not match " 7015 "# of RSS buckets (%d)\n", sc->hn_rx_ring_inuse, 7016 rss_getnumbuckets()); 7017 } 7018 #endif 7019 7020 if (bootverbose) { 7021 if_printf(sc->hn_ifp, "%d TX ring, %d RX ring\n", 7022 sc->hn_tx_ring_inuse, sc->hn_rx_ring_inuse); 7023 } 7024 } 7025 7026 static void 7027 hn_chan_drain(struct hn_softc *sc, struct vmbus_channel *chan) 7028 { 7029 7030 /* 7031 * NOTE: 7032 * The TX bufring will not be drained by the hypervisor, 7033 * if the primary channel is revoked. 7034 */ 7035 while (!vmbus_chan_rx_empty(chan) || 7036 (!vmbus_chan_is_revoked(sc->hn_prichan) && 7037 !vmbus_chan_tx_empty(chan))) 7038 pause("waitch", 1); 7039 vmbus_chan_intr_drain(chan); 7040 } 7041 7042 static void 7043 hn_disable_rx(struct hn_softc *sc) 7044 { 7045 7046 /* 7047 * Disable RX by clearing RX filter forcefully. 7048 */ 7049 sc->hn_rx_filter = NDIS_PACKET_TYPE_NONE; 7050 hn_rndis_set_rxfilter(sc, sc->hn_rx_filter); /* ignore error */ 7051 7052 /* 7053 * Give RNDIS enough time to flush all pending data packets. 7054 */ 7055 pause("waitrx", (200 * hz) / 1000); 7056 } 7057 7058 /* 7059 * NOTE: 7060 * RX/TX _must_ have been suspended/disabled, before this function 7061 * is called. 7062 */ 7063 static void 7064 hn_drain_rxtx(struct hn_softc *sc, int nchan) 7065 { 7066 struct vmbus_channel **subch = NULL; 7067 int nsubch; 7068 7069 /* 7070 * Drain RX/TX bufrings and interrupts. 7071 */ 7072 nsubch = nchan - 1; 7073 if (nsubch > 0) 7074 subch = vmbus_subchan_get(sc->hn_prichan, nsubch); 7075 7076 if (subch != NULL) { 7077 int i; 7078 7079 for (i = 0; i < nsubch; ++i) 7080 hn_chan_drain(sc, subch[i]); 7081 } 7082 hn_chan_drain(sc, sc->hn_prichan); 7083 7084 if (subch != NULL) 7085 vmbus_subchan_rel(subch, nsubch); 7086 } 7087 7088 static void 7089 hn_suspend_data(struct hn_softc *sc) 7090 { 7091 struct hn_tx_ring *txr; 7092 int i; 7093 7094 HN_LOCK_ASSERT(sc); 7095 7096 /* 7097 * Suspend TX. 7098 */ 7099 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { 7100 txr = &sc->hn_tx_ring[i]; 7101 7102 mtx_lock(&txr->hn_tx_lock); 7103 txr->hn_suspended = 1; 7104 mtx_unlock(&txr->hn_tx_lock); 7105 /* No one is able send more packets now. */ 7106 7107 /* 7108 * Wait for all pending sends to finish. 7109 * 7110 * NOTE: 7111 * We will _not_ receive all pending send-done, if the 7112 * primary channel is revoked. 7113 */ 7114 while (hn_tx_ring_pending(txr) && 7115 !vmbus_chan_is_revoked(sc->hn_prichan)) 7116 pause("hnwtx", 1 /* 1 tick */); 7117 } 7118 7119 /* 7120 * Disable RX. 7121 */ 7122 hn_disable_rx(sc); 7123 7124 /* 7125 * Drain RX/TX. 7126 */ 7127 hn_drain_rxtx(sc, sc->hn_rx_ring_inuse); 7128 7129 /* 7130 * Drain any pending TX tasks. 7131 * 7132 * NOTE: 7133 * The above hn_drain_rxtx() can dispatch TX tasks, so the TX 7134 * tasks will have to be drained _after_ the above hn_drain_rxtx(). 7135 */ 7136 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { 7137 txr = &sc->hn_tx_ring[i]; 7138 7139 taskqueue_drain(txr->hn_tx_taskq, &txr->hn_tx_task); 7140 taskqueue_drain(txr->hn_tx_taskq, &txr->hn_txeof_task); 7141 } 7142 } 7143 7144 static void 7145 hn_suspend_mgmt_taskfunc(void *xsc, int pending __unused) 7146 { 7147 7148 ((struct hn_softc *)xsc)->hn_mgmt_taskq = NULL; 7149 } 7150 7151 static void 7152 hn_suspend_mgmt(struct hn_softc *sc) 7153 { 7154 struct task task; 7155 7156 HN_LOCK_ASSERT(sc); 7157 7158 /* 7159 * Make sure that hn_mgmt_taskq0 can nolonger be accessed 7160 * through hn_mgmt_taskq. 7161 */ 7162 TASK_INIT(&task, 0, hn_suspend_mgmt_taskfunc, sc); 7163 vmbus_chan_run_task(sc->hn_prichan, &task); 7164 7165 /* 7166 * Make sure that all pending management tasks are completed. 7167 */ 7168 taskqueue_drain(sc->hn_mgmt_taskq0, &sc->hn_netchg_init); 7169 taskqueue_drain_timeout(sc->hn_mgmt_taskq0, &sc->hn_netchg_status); 7170 taskqueue_drain_all(sc->hn_mgmt_taskq0); 7171 } 7172 7173 static void 7174 hn_suspend(struct hn_softc *sc) 7175 { 7176 7177 /* Disable polling. */ 7178 hn_polling(sc, 0); 7179 7180 /* 7181 * If the non-transparent mode VF is activated, the synthetic 7182 * device is receiving packets, so the data path of the 7183 * synthetic device must be suspended. 7184 */ 7185 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) || 7186 (sc->hn_flags & HN_FLAG_RXVF)) 7187 hn_suspend_data(sc); 7188 hn_suspend_mgmt(sc); 7189 } 7190 7191 static void 7192 hn_resume_tx(struct hn_softc *sc, int tx_ring_cnt) 7193 { 7194 int i; 7195 7196 KASSERT(tx_ring_cnt <= sc->hn_tx_ring_cnt, 7197 ("invalid TX ring count %d", tx_ring_cnt)); 7198 7199 for (i = 0; i < tx_ring_cnt; ++i) { 7200 struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; 7201 7202 mtx_lock(&txr->hn_tx_lock); 7203 txr->hn_suspended = 0; 7204 mtx_unlock(&txr->hn_tx_lock); 7205 } 7206 } 7207 7208 static void 7209 hn_resume_data(struct hn_softc *sc) 7210 { 7211 int i; 7212 7213 HN_LOCK_ASSERT(sc); 7214 7215 /* 7216 * Re-enable RX. 7217 */ 7218 hn_rxfilter_config(sc); 7219 7220 /* 7221 * Make sure to clear suspend status on "all" TX rings, 7222 * since hn_tx_ring_inuse can be changed after 7223 * hn_suspend_data(). 7224 */ 7225 hn_resume_tx(sc, sc->hn_tx_ring_cnt); 7226 7227 #ifdef HN_IFSTART_SUPPORT 7228 if (!hn_use_if_start) 7229 #endif 7230 { 7231 /* 7232 * Flush unused drbrs, since hn_tx_ring_inuse may be 7233 * reduced. 7234 */ 7235 for (i = sc->hn_tx_ring_inuse; i < sc->hn_tx_ring_cnt; ++i) 7236 hn_tx_ring_qflush(&sc->hn_tx_ring[i]); 7237 } 7238 7239 /* 7240 * Kick start TX. 7241 */ 7242 for (i = 0; i < sc->hn_tx_ring_inuse; ++i) { 7243 struct hn_tx_ring *txr = &sc->hn_tx_ring[i]; 7244 7245 /* 7246 * Use txeof task, so that any pending oactive can be 7247 * cleared properly. 7248 */ 7249 taskqueue_enqueue(txr->hn_tx_taskq, &txr->hn_txeof_task); 7250 } 7251 } 7252 7253 static void 7254 hn_resume_mgmt(struct hn_softc *sc) 7255 { 7256 7257 sc->hn_mgmt_taskq = sc->hn_mgmt_taskq0; 7258 7259 /* 7260 * Kick off network change detection, if it was pending. 7261 * If no network change was pending, start link status 7262 * checks, which is more lightweight than network change 7263 * detection. 7264 */ 7265 if (sc->hn_link_flags & HN_LINK_FLAG_NETCHG) 7266 hn_change_network(sc); 7267 else 7268 hn_update_link_status(sc); 7269 } 7270 7271 static void 7272 hn_resume(struct hn_softc *sc) 7273 { 7274 7275 /* 7276 * If the non-transparent mode VF is activated, the synthetic 7277 * device have to receive packets, so the data path of the 7278 * synthetic device must be resumed. 7279 */ 7280 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) || 7281 (sc->hn_flags & HN_FLAG_RXVF)) 7282 hn_resume_data(sc); 7283 7284 /* 7285 * Don't resume link status change if VF is activated. 7286 * - In the non-transparent VF mode, the synthetic device marks 7287 * link down until the VF is deactivated; i.e. VF is down. 7288 * - In transparent VF mode, VF's media status is used until 7289 * the VF is deactivated. 7290 */ 7291 if ((sc->hn_flags & HN_FLAG_RXVF) == 0 && 7292 !(sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) 7293 hn_resume_mgmt(sc); 7294 7295 /* 7296 * Re-enable polling if this interface is running and 7297 * the polling is requested. 7298 */ 7299 if ((if_getdrvflags(sc->hn_ifp) & IFF_DRV_RUNNING) && sc->hn_pollhz > 0) 7300 hn_polling(sc, sc->hn_pollhz); 7301 } 7302 7303 static void 7304 hn_rndis_rx_status(struct hn_softc *sc, const void *data, int dlen) 7305 { 7306 const struct rndis_status_msg *msg; 7307 int ofs; 7308 7309 if (dlen < sizeof(*msg)) { 7310 if_printf(sc->hn_ifp, "invalid RNDIS status\n"); 7311 return; 7312 } 7313 msg = data; 7314 7315 switch (msg->rm_status) { 7316 case RNDIS_STATUS_MEDIA_CONNECT: 7317 case RNDIS_STATUS_MEDIA_DISCONNECT: 7318 hn_update_link_status(sc); 7319 break; 7320 7321 case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG: 7322 case RNDIS_STATUS_LINK_SPEED_CHANGE: 7323 /* Not really useful; ignore. */ 7324 break; 7325 7326 case RNDIS_STATUS_NETWORK_CHANGE: 7327 ofs = RNDIS_STBUFOFFSET_ABS(msg->rm_stbufoffset); 7328 if (dlen < ofs + msg->rm_stbuflen || 7329 msg->rm_stbuflen < sizeof(uint32_t)) { 7330 if_printf(sc->hn_ifp, "network changed\n"); 7331 } else { 7332 uint32_t change; 7333 7334 memcpy(&change, ((const uint8_t *)msg) + ofs, 7335 sizeof(change)); 7336 if_printf(sc->hn_ifp, "network changed, change %u\n", 7337 change); 7338 } 7339 hn_change_network(sc); 7340 break; 7341 7342 default: 7343 if_printf(sc->hn_ifp, "unknown RNDIS status 0x%08x\n", 7344 msg->rm_status); 7345 break; 7346 } 7347 } 7348 7349 static int 7350 hn_rndis_rxinfo(const void *info_data, int info_dlen, struct hn_rxinfo *info) 7351 { 7352 const struct rndis_pktinfo *pi = info_data; 7353 uint32_t mask = 0; 7354 7355 while (info_dlen != 0) { 7356 const void *data; 7357 uint32_t dlen; 7358 7359 if (__predict_false(info_dlen < sizeof(*pi))) 7360 return (EINVAL); 7361 if (__predict_false(info_dlen < pi->rm_size)) 7362 return (EINVAL); 7363 info_dlen -= pi->rm_size; 7364 7365 if (__predict_false(pi->rm_size & RNDIS_PKTINFO_SIZE_ALIGNMASK)) 7366 return (EINVAL); 7367 if (__predict_false(pi->rm_size < pi->rm_pktinfooffset)) 7368 return (EINVAL); 7369 dlen = pi->rm_size - pi->rm_pktinfooffset; 7370 data = pi->rm_data; 7371 7372 if (pi->rm_internal == 1) { 7373 switch (pi->rm_type) { 7374 case NDIS_PKTINFO_IT_PKTINFO_ID: 7375 if (__predict_false(dlen < NDIS_PKTINFOID_SZ)) 7376 return (EINVAL); 7377 info->pktinfo_id = 7378 (const struct packet_info_id *)data; 7379 mask |= HN_RXINFO_PKTINFO_ID; 7380 break; 7381 7382 default: 7383 goto next; 7384 } 7385 } else { 7386 switch (pi->rm_type) { 7387 case NDIS_PKTINFO_TYPE_VLAN: 7388 if (__predict_false(dlen 7389 < NDIS_VLAN_INFO_SIZE)) 7390 return (EINVAL); 7391 info->vlan_info = (const uint32_t *)data; 7392 mask |= HN_RXINFO_VLAN; 7393 break; 7394 7395 case NDIS_PKTINFO_TYPE_CSUM: 7396 if (__predict_false(dlen 7397 < NDIS_RXCSUM_INFO_SIZE)) 7398 return (EINVAL); 7399 info->csum_info = (const uint32_t *)data; 7400 mask |= HN_RXINFO_CSUM; 7401 break; 7402 7403 case HN_NDIS_PKTINFO_TYPE_HASHVAL: 7404 if (__predict_false(dlen 7405 < HN_NDIS_HASH_VALUE_SIZE)) 7406 return (EINVAL); 7407 info->hash_value = (const uint32_t *)data; 7408 mask |= HN_RXINFO_HASHVAL; 7409 break; 7410 7411 case HN_NDIS_PKTINFO_TYPE_HASHINF: 7412 if (__predict_false(dlen 7413 < HN_NDIS_HASH_INFO_SIZE)) 7414 return (EINVAL); 7415 info->hash_info = (const uint32_t *)data; 7416 mask |= HN_RXINFO_HASHINF; 7417 break; 7418 7419 default: 7420 goto next; 7421 } 7422 } 7423 7424 if (mask == HN_RXINFO_ALL) { 7425 /* All found; done */ 7426 break; 7427 } 7428 next: 7429 pi = (const struct rndis_pktinfo *) 7430 ((const uint8_t *)pi + pi->rm_size); 7431 } 7432 7433 /* 7434 * Final fixup. 7435 * - If there is no hash value, invalidate the hash info. 7436 */ 7437 if ((mask & HN_RXINFO_HASHVAL) == 0) 7438 info->hash_info = NULL; 7439 return (0); 7440 } 7441 7442 static __inline bool 7443 hn_rndis_check_overlap(int off, int len, int check_off, int check_len) 7444 { 7445 7446 if (off < check_off) { 7447 if (__predict_true(off + len <= check_off)) 7448 return (false); 7449 } else if (off > check_off) { 7450 if (__predict_true(check_off + check_len <= off)) 7451 return (false); 7452 } 7453 return (true); 7454 } 7455 7456 static __inline void 7457 hn_rsc_add_data(struct hn_rx_ring *rxr, const void *data, 7458 uint32_t len, struct hn_rxinfo *info) 7459 { 7460 uint32_t cnt = rxr->rsc.cnt; 7461 7462 if (cnt) { 7463 rxr->rsc.pktlen += len; 7464 } else { 7465 rxr->rsc.vlan_info = info->vlan_info; 7466 rxr->rsc.csum_info = info->csum_info; 7467 rxr->rsc.hash_info = info->hash_info; 7468 rxr->rsc.hash_value = info->hash_value; 7469 rxr->rsc.pktlen = len; 7470 } 7471 7472 rxr->rsc.frag_data[cnt] = data; 7473 rxr->rsc.frag_len[cnt] = len; 7474 rxr->rsc.cnt++; 7475 } 7476 7477 static void 7478 hn_rndis_rx_data(struct hn_rx_ring *rxr, const void *data, int dlen) 7479 { 7480 const struct rndis_packet_msg *pkt; 7481 struct hn_rxinfo info; 7482 int data_off, pktinfo_off, data_len, pktinfo_len; 7483 bool rsc_more= false; 7484 7485 /* 7486 * Check length. 7487 */ 7488 if (__predict_false(dlen < sizeof(*pkt))) { 7489 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg\n"); 7490 return; 7491 } 7492 pkt = data; 7493 7494 if (__predict_false(dlen < pkt->rm_len)) { 7495 if_printf(rxr->hn_ifp, "truncated RNDIS packet msg, " 7496 "dlen %d, msglen %u\n", dlen, pkt->rm_len); 7497 return; 7498 } 7499 if (__predict_false(pkt->rm_len < 7500 pkt->rm_datalen + pkt->rm_oobdatalen + pkt->rm_pktinfolen)) { 7501 if_printf(rxr->hn_ifp, "invalid RNDIS packet msglen, " 7502 "msglen %u, data %u, oob %u, pktinfo %u\n", 7503 pkt->rm_len, pkt->rm_datalen, pkt->rm_oobdatalen, 7504 pkt->rm_pktinfolen); 7505 return; 7506 } 7507 if (__predict_false(pkt->rm_datalen == 0)) { 7508 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, no data\n"); 7509 return; 7510 } 7511 7512 /* 7513 * Check offests. 7514 */ 7515 #define IS_OFFSET_INVALID(ofs) \ 7516 ((ofs) < RNDIS_PACKET_MSG_OFFSET_MIN || \ 7517 ((ofs) & RNDIS_PACKET_MSG_OFFSET_ALIGNMASK)) 7518 7519 /* XXX Hyper-V does not meet data offset alignment requirement */ 7520 if (__predict_false(pkt->rm_dataoffset < RNDIS_PACKET_MSG_OFFSET_MIN)) { 7521 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7522 "data offset %u\n", pkt->rm_dataoffset); 7523 return; 7524 } 7525 if (__predict_false(pkt->rm_oobdataoffset > 0 && 7526 IS_OFFSET_INVALID(pkt->rm_oobdataoffset))) { 7527 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7528 "oob offset %u\n", pkt->rm_oobdataoffset); 7529 return; 7530 } 7531 if (__predict_true(pkt->rm_pktinfooffset > 0) && 7532 __predict_false(IS_OFFSET_INVALID(pkt->rm_pktinfooffset))) { 7533 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7534 "pktinfo offset %u\n", pkt->rm_pktinfooffset); 7535 return; 7536 } 7537 7538 #undef IS_OFFSET_INVALID 7539 7540 data_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->rm_dataoffset); 7541 data_len = pkt->rm_datalen; 7542 pktinfo_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->rm_pktinfooffset); 7543 pktinfo_len = pkt->rm_pktinfolen; 7544 7545 /* 7546 * Check OOB coverage. 7547 */ 7548 if (__predict_false(pkt->rm_oobdatalen != 0)) { 7549 int oob_off, oob_len; 7550 7551 if_printf(rxr->hn_ifp, "got oobdata\n"); 7552 oob_off = RNDIS_PACKET_MSG_OFFSET_ABS(pkt->rm_oobdataoffset); 7553 oob_len = pkt->rm_oobdatalen; 7554 7555 if (__predict_false(oob_off + oob_len > pkt->rm_len)) { 7556 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7557 "oob overflow, msglen %u, oob abs %d len %d\n", 7558 pkt->rm_len, oob_off, oob_len); 7559 return; 7560 } 7561 7562 /* 7563 * Check against data. 7564 */ 7565 if (hn_rndis_check_overlap(oob_off, oob_len, 7566 data_off, data_len)) { 7567 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7568 "oob overlaps data, oob abs %d len %d, " 7569 "data abs %d len %d\n", 7570 oob_off, oob_len, data_off, data_len); 7571 return; 7572 } 7573 7574 /* 7575 * Check against pktinfo. 7576 */ 7577 if (pktinfo_len != 0 && 7578 hn_rndis_check_overlap(oob_off, oob_len, 7579 pktinfo_off, pktinfo_len)) { 7580 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7581 "oob overlaps pktinfo, oob abs %d len %d, " 7582 "pktinfo abs %d len %d\n", 7583 oob_off, oob_len, pktinfo_off, pktinfo_len); 7584 return; 7585 } 7586 } 7587 7588 /* 7589 * Check per-packet-info coverage and find useful per-packet-info. 7590 */ 7591 info.vlan_info = NULL; 7592 info.csum_info = NULL; 7593 info.hash_info = NULL; 7594 info.pktinfo_id = NULL; 7595 7596 if (__predict_true(pktinfo_len != 0)) { 7597 bool overlap; 7598 int error; 7599 7600 if (__predict_false(pktinfo_off + pktinfo_len > pkt->rm_len)) { 7601 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7602 "pktinfo overflow, msglen %u, " 7603 "pktinfo abs %d len %d\n", 7604 pkt->rm_len, pktinfo_off, pktinfo_len); 7605 return; 7606 } 7607 7608 /* 7609 * Check packet info coverage. 7610 */ 7611 overlap = hn_rndis_check_overlap(pktinfo_off, pktinfo_len, 7612 data_off, data_len); 7613 if (__predict_false(overlap)) { 7614 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7615 "pktinfo overlap data, pktinfo abs %d len %d, " 7616 "data abs %d len %d\n", 7617 pktinfo_off, pktinfo_len, data_off, data_len); 7618 return; 7619 } 7620 7621 /* 7622 * Find useful per-packet-info. 7623 */ 7624 error = hn_rndis_rxinfo(((const uint8_t *)pkt) + pktinfo_off, 7625 pktinfo_len, &info); 7626 if (__predict_false(error)) { 7627 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg " 7628 "pktinfo\n"); 7629 return; 7630 } 7631 } 7632 7633 if (__predict_false(data_off + data_len > pkt->rm_len)) { 7634 if_printf(rxr->hn_ifp, "invalid RNDIS packet msg, " 7635 "data overflow, msglen %u, data abs %d len %d\n", 7636 pkt->rm_len, data_off, data_len); 7637 return; 7638 } 7639 7640 /* Identify RSC fragments, drop invalid packets */ 7641 if ((info.pktinfo_id != NULL) && 7642 (info.pktinfo_id->flag & HN_NDIS_PKTINFO_SUBALLOC)) { 7643 if (info.pktinfo_id->flag & HN_NDIS_PKTINFO_1ST_FRAG) { 7644 rxr->rsc.cnt = 0; 7645 rxr->hn_rsc_pkts++; 7646 } else if (rxr->rsc.cnt == 0) 7647 goto drop; 7648 7649 rsc_more = true; 7650 7651 if (info.pktinfo_id->flag & HN_NDIS_PKTINFO_LAST_FRAG) 7652 rsc_more = false; 7653 7654 if (rsc_more && rxr->rsc.is_last) 7655 goto drop; 7656 } else { 7657 rxr->rsc.cnt = 0; 7658 } 7659 7660 if (__predict_false(rxr->rsc.cnt >= HN_NVS_RSC_MAX)) 7661 goto drop; 7662 7663 /* Store data in per rx ring structure */ 7664 hn_rsc_add_data(rxr,((const uint8_t *)pkt) + data_off, 7665 data_len, &info); 7666 7667 if (rsc_more) 7668 return; 7669 7670 hn_rxpkt(rxr); 7671 rxr->rsc.cnt = 0; 7672 return; 7673 drop: 7674 rxr->hn_rsc_drop++; 7675 return; 7676 } 7677 7678 static __inline void 7679 hn_rndis_rxpkt(struct hn_rx_ring *rxr, const void *data, int dlen) 7680 { 7681 const struct rndis_msghdr *hdr; 7682 7683 if (__predict_false(dlen < sizeof(*hdr))) { 7684 if_printf(rxr->hn_ifp, "invalid RNDIS msg\n"); 7685 return; 7686 } 7687 hdr = data; 7688 7689 if (__predict_true(hdr->rm_type == REMOTE_NDIS_PACKET_MSG)) { 7690 /* Hot data path. */ 7691 hn_rndis_rx_data(rxr, data, dlen); 7692 /* Done! */ 7693 return; 7694 } 7695 7696 if (hdr->rm_type == REMOTE_NDIS_INDICATE_STATUS_MSG) 7697 hn_rndis_rx_status(if_getsoftc(rxr->hn_ifp), data, dlen); 7698 else 7699 hn_rndis_rx_ctrl(if_getsoftc(rxr->hn_ifp), data, dlen); 7700 } 7701 7702 static void 7703 hn_nvs_handle_notify(struct hn_softc *sc, const struct vmbus_chanpkt_hdr *pkt) 7704 { 7705 const struct hn_nvs_hdr *hdr; 7706 7707 if (VMBUS_CHANPKT_DATALEN(pkt) < sizeof(*hdr)) { 7708 if_printf(sc->hn_ifp, "invalid nvs notify\n"); 7709 return; 7710 } 7711 hdr = VMBUS_CHANPKT_CONST_DATA(pkt); 7712 7713 if (hdr->nvs_type == HN_NVS_TYPE_VFASSOC_NOTE) { 7714 const struct hn_nvs_vfassoc *assoc; 7715 u_int state; 7716 7717 if (VMBUS_CHANPKT_DATALEN(pkt) < sizeof(*assoc)) { 7718 if_printf(sc->hn_ifp, "short VF association notification\n"); 7719 return; 7720 } 7721 assoc = (const struct hn_nvs_vfassoc *)hdr; 7722 if (assoc->nvs_alloc > 1) { 7723 if_printf(sc->hn_ifp, "invalid VF association notification\n"); 7724 return; 7725 } 7726 /* The serial is diagnostic; hn_ismyvf() matches the VF by MAC. */ 7727 /* Preserve withdrawals even when the worker coalesces notices. */ 7728 state = (atomic_load_int(&sc->hn_vf_assoc) + HN_VF_ASSOC_GENINC) & 7729 ~HN_VF_ASSOC_ALLOCATED; 7730 atomic_store_rel_int(&sc->hn_vf_assoc, state | assoc->nvs_alloc); 7731 if (bootverbose) 7732 if_printf(sc->hn_ifp, "VF %u %s\n", assoc->nvs_serial, 7733 assoc->nvs_alloc ? "associated" : "withdrawn"); 7734 taskqueue_enqueue_timeout(sc->hn_vf_taskq, &sc->hn_vf_init, 0); 7735 return; 7736 } 7737 if (hdr->nvs_type == HN_NVS_TYPE_TXTBL_NOTE) { 7738 /* Useless; ignore */ 7739 return; 7740 } 7741 if_printf(sc->hn_ifp, "got notify, nvs type %u\n", hdr->nvs_type); 7742 } 7743 7744 static void 7745 hn_nvs_handle_comp(struct hn_softc *sc, struct vmbus_channel *chan, 7746 const struct vmbus_chanpkt_hdr *pkt) 7747 { 7748 struct hn_nvs_sendctx *sndc; 7749 7750 sndc = (struct hn_nvs_sendctx *)(uintptr_t)pkt->cph_xactid; 7751 sndc->hn_cb(sndc, sc, chan, VMBUS_CHANPKT_CONST_DATA(pkt), 7752 VMBUS_CHANPKT_DATALEN(pkt)); 7753 /* 7754 * NOTE: 7755 * 'sndc' CAN NOT be accessed anymore, since it can be freed by 7756 * its callback. 7757 */ 7758 } 7759 7760 static void 7761 hn_nvs_handle_rxbuf(struct hn_rx_ring *rxr, struct vmbus_channel *chan, 7762 const struct vmbus_chanpkt_hdr *pkthdr) 7763 { 7764 struct epoch_tracker et; 7765 const struct vmbus_chanpkt_rxbuf *pkt; 7766 const struct hn_nvs_hdr *nvs_hdr; 7767 int count, i, hlen; 7768 7769 if (__predict_false(VMBUS_CHANPKT_DATALEN(pkthdr) < sizeof(*nvs_hdr))) { 7770 if_printf(rxr->hn_ifp, "invalid nvs RNDIS\n"); 7771 return; 7772 } 7773 nvs_hdr = VMBUS_CHANPKT_CONST_DATA(pkthdr); 7774 7775 /* Make sure that this is a RNDIS message. */ 7776 if (__predict_false(nvs_hdr->nvs_type != HN_NVS_TYPE_RNDIS)) { 7777 if_printf(rxr->hn_ifp, "nvs type %u, not RNDIS\n", 7778 nvs_hdr->nvs_type); 7779 return; 7780 } 7781 7782 hlen = VMBUS_CHANPKT_GETLEN(pkthdr->cph_hlen); 7783 if (__predict_false(hlen < sizeof(*pkt))) { 7784 if_printf(rxr->hn_ifp, "invalid rxbuf chanpkt\n"); 7785 return; 7786 } 7787 pkt = (const struct vmbus_chanpkt_rxbuf *)pkthdr; 7788 7789 if (__predict_false(pkt->cp_rxbuf_id != HN_NVS_RXBUF_SIG)) { 7790 if_printf(rxr->hn_ifp, "invalid rxbuf_id 0x%08x\n", 7791 pkt->cp_rxbuf_id); 7792 return; 7793 } 7794 7795 count = pkt->cp_rxbuf_cnt; 7796 if (__predict_false(hlen < 7797 __offsetof(struct vmbus_chanpkt_rxbuf, cp_rxbuf[count]))) { 7798 if_printf(rxr->hn_ifp, "invalid rxbuf_cnt %d\n", count); 7799 return; 7800 } 7801 7802 NET_EPOCH_ENTER(et); 7803 /* Each range represents 1 RNDIS pkt that contains 1 Ethernet frame */ 7804 for (i = 0; i < count; ++i) { 7805 int ofs, len; 7806 7807 ofs = pkt->cp_rxbuf[i].rb_ofs; 7808 len = pkt->cp_rxbuf[i].rb_len; 7809 if (__predict_false(ofs + len > HN_RXBUF_SIZE)) { 7810 if_printf(rxr->hn_ifp, "%dth RNDIS msg overflow rxbuf, " 7811 "ofs %d, len %d\n", i, ofs, len); 7812 continue; 7813 } 7814 7815 rxr->rsc.is_last = (i == (count - 1)); 7816 hn_rndis_rxpkt(rxr, rxr->hn_rxbuf + ofs, len); 7817 } 7818 NET_EPOCH_EXIT(et); 7819 7820 /* 7821 * Ack the consumed RXBUF associated w/ this channel packet, 7822 * so that this RXBUF can be recycled by the hypervisor. 7823 */ 7824 hn_nvs_ack_rxbuf(rxr, chan, pkt->cp_hdr.cph_xactid); 7825 } 7826 7827 static void 7828 hn_nvs_ack_rxbuf(struct hn_rx_ring *rxr, struct vmbus_channel *chan, 7829 uint64_t tid) 7830 { 7831 struct hn_nvs_rndis_ack ack; 7832 int retries, error; 7833 7834 ack.nvs_type = HN_NVS_TYPE_RNDIS_ACK; 7835 ack.nvs_status = HN_NVS_STATUS_OK; 7836 7837 retries = 0; 7838 again: 7839 error = vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP, 7840 VMBUS_CHANPKT_FLAG_NONE, &ack, sizeof(ack), tid); 7841 if (__predict_false(error == EAGAIN)) { 7842 /* 7843 * NOTE: 7844 * This should _not_ happen in real world, since the 7845 * consumption of the TX bufring from the TX path is 7846 * controlled. 7847 */ 7848 if (rxr->hn_ack_failed == 0) 7849 if_printf(rxr->hn_ifp, "RXBUF ack retry\n"); 7850 rxr->hn_ack_failed++; 7851 retries++; 7852 if (retries < 10) { 7853 DELAY(100); 7854 goto again; 7855 } 7856 /* RXBUF leaks! */ 7857 if_printf(rxr->hn_ifp, "RXBUF ack failed\n"); 7858 } 7859 } 7860 7861 static void 7862 hn_chan_callback(struct vmbus_channel *chan, void *xrxr) 7863 { 7864 struct hn_rx_ring *rxr = xrxr; 7865 struct hn_softc *sc = if_getsoftc(rxr->hn_ifp); 7866 7867 for (;;) { 7868 struct vmbus_chanpkt_hdr *pkt = rxr->hn_pktbuf; 7869 int error, pktlen; 7870 7871 pktlen = rxr->hn_pktbuf_len; 7872 error = vmbus_chan_recv_pkt(chan, pkt, &pktlen); 7873 if (__predict_false(error == ENOBUFS)) { 7874 void *nbuf; 7875 int nlen; 7876 7877 /* 7878 * Expand channel packet buffer. 7879 * 7880 * XXX 7881 * Use M_WAITOK here, since allocation failure 7882 * is fatal. 7883 */ 7884 nlen = rxr->hn_pktbuf_len * 2; 7885 while (nlen < pktlen) 7886 nlen *= 2; 7887 nbuf = malloc(nlen, M_DEVBUF, M_WAITOK); 7888 7889 if_printf(rxr->hn_ifp, "expand pktbuf %d -> %d\n", 7890 rxr->hn_pktbuf_len, nlen); 7891 7892 free(rxr->hn_pktbuf, M_DEVBUF); 7893 rxr->hn_pktbuf = nbuf; 7894 rxr->hn_pktbuf_len = nlen; 7895 /* Retry! */ 7896 continue; 7897 } else if (__predict_false(error == EAGAIN)) { 7898 /* No more channel packets; done! */ 7899 break; 7900 } 7901 KASSERT(!error, ("vmbus_chan_recv_pkt failed: %d", error)); 7902 7903 switch (pkt->cph_type) { 7904 case VMBUS_CHANPKT_TYPE_COMP: 7905 hn_nvs_handle_comp(sc, chan, pkt); 7906 break; 7907 7908 case VMBUS_CHANPKT_TYPE_RXBUF: 7909 hn_nvs_handle_rxbuf(rxr, chan, pkt); 7910 break; 7911 7912 case VMBUS_CHANPKT_TYPE_INBAND: 7913 if (chan == sc->hn_prichan) 7914 hn_nvs_handle_notify(sc, pkt); 7915 break; 7916 7917 default: 7918 if_printf(rxr->hn_ifp, "unknown chan pkt %u\n", 7919 pkt->cph_type); 7920 break; 7921 } 7922 } 7923 hn_chan_rollup(rxr, rxr->hn_txr); 7924 } 7925 7926 static void 7927 hn_sysinit(void *arg __unused) 7928 { 7929 int i; 7930 7931 hn_udpcs_fixup = counter_u64_alloc(M_WAITOK); 7932 7933 #ifdef HN_IFSTART_SUPPORT 7934 /* 7935 * Don't use ifnet.if_start if transparent VF mode is requested; 7936 * mainly due to the IFF_DRV_OACTIVE flag. 7937 */ 7938 if (hn_xpnt_vf && hn_use_if_start) { 7939 hn_use_if_start = 0; 7940 printf("hn: transparent VF mode, if_transmit will be used, " 7941 "instead of if_start\n"); 7942 } 7943 #endif 7944 if (hn_xpnt_vf_attwait < HN_XPNT_VF_ATTWAIT_MIN) { 7945 printf("hn: invalid transparent VF attach routing " 7946 "wait timeout %d, reset to %d\n", 7947 hn_xpnt_vf_attwait, HN_XPNT_VF_ATTWAIT_MIN); 7948 hn_xpnt_vf_attwait = HN_XPNT_VF_ATTWAIT_MIN; 7949 } 7950 7951 /* 7952 * Initialize VF map. 7953 */ 7954 rm_init_flags(&hn_vfmap_lock, "hn_vfmap", RM_SLEEPABLE); 7955 hn_vfmap_size = HN_VFMAP_SIZE_DEF; 7956 hn_vfmap = malloc(sizeof(if_t) * hn_vfmap_size, M_DEVBUF, 7957 M_WAITOK | M_ZERO); 7958 7959 /* 7960 * Fix the # of TX taskqueues. 7961 */ 7962 if (hn_tx_taskq_cnt <= 0) 7963 hn_tx_taskq_cnt = 1; 7964 else if (hn_tx_taskq_cnt > mp_ncpus) 7965 hn_tx_taskq_cnt = mp_ncpus; 7966 7967 /* 7968 * Fix the TX taskqueue mode. 7969 */ 7970 switch (hn_tx_taskq_mode) { 7971 case HN_TX_TASKQ_M_INDEP: 7972 case HN_TX_TASKQ_M_GLOBAL: 7973 case HN_TX_TASKQ_M_EVTTQ: 7974 break; 7975 default: 7976 hn_tx_taskq_mode = HN_TX_TASKQ_M_INDEP; 7977 break; 7978 } 7979 7980 if (vm_guest != VM_GUEST_HV) 7981 return; 7982 7983 if (hn_tx_taskq_mode != HN_TX_TASKQ_M_GLOBAL) 7984 return; 7985 7986 hn_tx_taskque = malloc(hn_tx_taskq_cnt * sizeof(struct taskqueue *), 7987 M_DEVBUF, M_WAITOK); 7988 for (i = 0; i < hn_tx_taskq_cnt; ++i) { 7989 hn_tx_taskque[i] = taskqueue_create("hn_tx", M_WAITOK, 7990 taskqueue_thread_enqueue, &hn_tx_taskque[i]); 7991 taskqueue_start_threads(&hn_tx_taskque[i], 1, PI_NET, 7992 "hn tx%d", i); 7993 } 7994 } 7995 SYSINIT(hn_sysinit, SI_SUB_DRIVERS, SI_ORDER_SECOND, hn_sysinit, NULL); 7996 7997 static void 7998 hn_sysuninit(void *arg __unused) 7999 { 8000 8001 if (hn_tx_taskque != NULL) { 8002 int i; 8003 8004 for (i = 0; i < hn_tx_taskq_cnt; ++i) 8005 taskqueue_free(hn_tx_taskque[i]); 8006 free(hn_tx_taskque, M_DEVBUF); 8007 } 8008 8009 if (hn_vfmap != NULL) 8010 free(hn_vfmap, M_DEVBUF); 8011 rm_destroy(&hn_vfmap_lock); 8012 8013 counter_u64_free(hn_udpcs_fixup); 8014 } 8015 SYSUNINIT(hn_sysuninit, SI_SUB_DRIVERS, SI_ORDER_SECOND, hn_sysuninit, NULL); 8016